Test/schema coverage

README.md

@@ -403,6 +403,62 @@ end
 
 The default assertion option in 2.* was `validate_success_only=true`, but this becomes `validate_success_only=false` in 3.*. For the smoothest possible upgrade, you should set it to `false` in your test suite before upgrading to 3.*.
 
+**Test schema coverage**  
+You can check how much of your API schema your tests have covered.  
+NOTICE: Currently committee only supports schema coverage for **openapi** schemas, and only checks coverage on responses, via `assert_response_schema_confirm` or `assert_schema_conform` methods.  
+Usage:
+1. Set schema_coverage option of `committee_options`
+2. Use `assert_response_schema_confirm` or `assert_schema_conform`
+3. Then use `SchemaCoverage#report` or `SchemaCoverage#report_flatten` to get coverage report
+
+Example:
+```ruby
+before do
+  schema_coverage = Committee::Test::SchemaCoverage.new(openapi_schema)
+  @committee_options[:schema_coverage] = schema_coverage
+end
+it 'covers /some_api' do
+  get '/some_api'
+  assert_response_schema_confirm # or assert_schema_conform
+  coverage_report = schema_coverage.report
+  # check coverage expectations of /some_api here
+end
+it 'covers /other_api schema' do
+  get '/other_api'
+  assert_response_schema_confirm # or assert_schema_conform
+  coverage_report = schema_coverage.report
+  # check coverage expectations of /other_api here
+end
+after do
+  coverage_report = schema_coverage.report
+  # check coverage expectations of all apis here
+end
+```
+
+Coverage report structure:
+```
+/* using #report */
+{
+  <path> => {
+    <method> => {
+      'responses' => {
+        <status> => <true|false>
+      }
+    }
+  }
+}
+/* using #report_flatten */
+{
+  responses: [
+    { path: <path>, method: <method>, status: <status>, is_covered: <true|false> },
+  ]
+}
+```
+
+Other helper methods:
+* `Committee::Test::SchemaCoverage.merge_report(<Hash>, <Hash>)`: merge 2 coverage reports together
+* `Committee::Test::SchemaCoverage.flatten_report(<Hash>)`: flatten a coverage report Hash into flatten structure
+
 ### Other changes
 
 * `GET` request bodies are ignored in OpenAPI 3 by default. If you want to use them, set the `allow_get_body` option to `true`.

lib/committee.rb

@@ -29,3 +29,4 @@ def self.warn_deprecated(message)
 
 require_relative "committee/bin/committee_stub"
 require_relative "committee/test/methods"
+require_relative "committee/test/schema_coverage"

lib/committee/schema_validator/open_api_3/operation_wrapper.rb

@@ -17,6 +17,10 @@ def original_path
           request_operation.original_path
         end
 
+        def http_method
+          request_operation.http_method
+        end
+
         def coerce_path_parameter(validator_option)
           options = build_openapi_parser_path_option(validator_option)
           return {} unless options.coerce_value

lib/committee/schema_validator/open_api_3/router.rb

@@ -22,8 +22,10 @@ def build_schema_validator(request)
         end
 
         def operation_object(request)
+          return nil unless includes_request?(request)
+
           path = request.path
-          path = path.gsub(@prefix_regexp, '') if prefix_request?(request)
+          path = path.gsub(@prefix_regexp, '') if @prefix_regexp
 
           request_method = request.request_method.downcase
 

lib/committee/test/methods.rb

@@ -10,7 +10,7 @@ def assert_schema_conform
 
       def assert_request_schema_confirm
         unless schema_validator.link_exist?
-          request = "`#{request_object.request_method} #{request_object.path_info}` undefined in schema."
+          request = "`#{request_object.request_method} #{request_object.path_info}` undefined in schema (prefix: #{committee_options[:prefix].inspect})."
           raise Committee::InvalidRequest.new(request)
         end
 
@@ -19,11 +19,17 @@ def assert_request_schema_confirm
 
       def assert_response_schema_confirm
         unless schema_validator.link_exist?
-          response = "`#{request_object.request_method} #{request_object.path_info}` undefined in schema."
+          response = "`#{request_object.request_method} #{request_object.path_info}` undefined in schema (prefix: #{committee_options[:prefix].inspect})."
           raise Committee::InvalidResponse.new(response)
         end
 
         status, headers, body = response_data
+
+        if schema_coverage
+          operation_object = router.operation_object(request_object)
+          schema_coverage&.update_response_coverage!(operation_object.original_path, operation_object.http_method, status)
+        end
+
         schema_validator.response_validate(status, headers, [body], true) if validate_response?(status)
       end
 
@@ -55,6 +61,14 @@ def schema_validator
         @schema_validator ||= router.build_schema_validator(request_object)
       end
 
+      def schema_coverage
+        return nil unless schema.is_a?(Committee::Drivers::OpenAPI3::Schema)
+
+        coverage = committee_options.fetch(:schema_coverage, nil)
+
+        coverage.is_a?(SchemaCoverage) ? coverage : nil
+      end
+
       def old_behavior
         old_assert_behavior = committee_options.fetch(:old_assert_behavior, nil)
         if old_assert_behavior.nil?

lib/committee/test/schema_coverage.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Committee
+  module Test
+    class SchemaCoverage
+      attr_reader :schema
+
+      class << self
+        def merge_report(first, second)
+          report = first.dup
+          second.each do |k, v|
+            if v.is_a?(Hash)
+              if report[k].nil?
+                report[k] = v
+              else
+                report[k] = merge_report(report[k], v)
+              end
+            else
+              report[k] ||= v
+            end
+          end
+          report
+        end
+
+        def flatten_report(report)
+          responses = []
+          report.each do |path_name, path_coverage|
+            path_coverage.each do |method, method_coverage|
+              responses_coverage = method_coverage['responses']
+              responses_coverage.each do |response_status, is_covered|
+                responses << {
+                  path: path_name,
+                  method: method,
+                  status: response_status,
+                  is_covered: is_covered,
+                }
+              end
+            end
+          end
+          {
+            responses: responses,
+          }
+        end
+      end
+
+      def initialize(schema)
+        raise 'Unsupported schema' unless schema.is_a?(Committee::Drivers::OpenAPI3::Schema)
+
+        @schema = schema
+        @covered = {}
+      end
+
+      def update_response_coverage!(path, method, response_status)
+        method = method.to_s.downcase
+        response_status = response_status.to_s
+
+        @covered[path] ||= {}
+        @covered[path][method] ||= {}
+        @covered[path][method]['responses'] ||= {}
+        @covered[path][method]['responses'][response_status] = true
+      end
+
+      def report
+        report = {}
+
+        schema.open_api.paths.path.each do |path_name, path_item|
+          report[path_name] = {}
+          path_item._openapi_all_child_objects.each do |object_name, object|
+            next unless object.is_a?(OpenAPIParser::Schemas::Operation)
+
+            method = object_name.split('/').last&.downcase
+            next unless method
+
+            report[path_name][method] ||= {}
+
+            # TODO: check coverage on request params/body as well?
+
+            report[path_name][method]['responses'] ||= {}
+            object.responses.response.each do |response_status, _|
+              is_covered = @covered.dig(path_name, method, 'responses', response_status) || false
+              report[path_name][method]['responses'][response_status] = is_covered
+            end
+            if object.responses.default
+              is_default_covered = (@covered.dig(path_name, method, 'responses') || {}).any? do |status, is_covered|
+                is_covered && !object.responses.response.key?(status)
+              end
+              report[path_name][method]['responses']['default'] = is_default_covered
+            end
+          end
+        end
+
+        report
+      end
+
+      def report_flatten
+        self.class.flatten_report(report)
+      end
+    end
+  end
+end
+

test/data/openapi3/coverage.yaml

@@ -0,0 +1,74 @@
+openapi: 3.0.0
+info:
+  version: 1.0.0
+  title: OpenAPI3 Coverage Test
+  description: A Sample file for coverage test
+servers:
+- url: https://github.com/interagent/committee/
+paths:
+  /threads/{id}:
+    parameters:
+    - name: id
+      in: path
+      required: true
+      schema:
+        type: string
+    get:
+      description: get a thread
+      responses:
+        '200':
+          description: success
+          content:
+            application/json:
+              schema:
+                type: object
+  /posts:
+    get:
+      description: get a post
+      responses:
+        '200':
+          description: success
+          content:
+            application/json:
+              schema:
+                type: object
+        '404':
+          description: post not found
+          content:
+            application/json:
+              schema:
+                type: object
+        default:
+          description: unknown request
+          content:
+            application/json:
+              schema:
+                type: object
+    post:
+      description: create a new post
+      responses:
+        '200':
+          description: success
+          content:
+            application/json:
+              schema:
+                type: object
+  /likes:
+    post:
+      description: like a post
+      responses:
+        '200':
+          description: success
+          content:
+            application/json:
+              schema:
+                type: object
+    delete:
+      description: unlike a post
+      responses:
+        '200':
+          description: success
+          content:
+            application/json:
+              schema:
+                type: object