Merge pull request #1372 from mbj/expose/test-runner

Add test only runner

.github/workflows/ci.yml

@@ -26,13 +26,16 @@ jobs:
           ruby: ruby-3.1
         - os: macos-latest
           ruby: ruby-3.0
+        execution:
+        - bundle exec rspec spec/unit
+        - bundle exec mutant environment test run spec/unit
     steps:
     - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
         bundler-cache: true
-    - run: bundle exec rspec spec/unit
+    - run: ${{ matrix.execution }}
   ruby-mutant:
     name: Mutation coverage
     runs-on: ${{ matrix.os }}

docs/configuration.md

@@ -182,7 +182,7 @@ mutation:
   ignore_patterns:
   - send{selector=log}
   # Select full mutation operators by default mutant only applies the light set
-  # Only difference between full and light right now is that light does not apply 
+  # Only difference between full and light right now is that light does not apply
   # `#== -> #eql?` mutation
   # At this moment there is no CLI equivalent for this setting.
   operators: full # or `light`

lib/mutant.rb

@@ -205,6 +205,8 @@ module Mutant
     require 'mutant/expression/namespace'
     require 'mutant/expression/parser'
     require 'mutant/test'
+    require 'mutant/test/runner'
+    require 'mutant/test/runner/sink'
     require 'mutant/timer'
     require 'mutant/integration'
     require 'mutant/integration/null'
@@ -244,6 +246,7 @@ module Mutant
     require 'mutant/reporter/cli/printer/mutation_result'
     require 'mutant/reporter/cli/printer/status_progressive'
     require 'mutant/reporter/cli/printer/subject_result'
+    require 'mutant/reporter/cli/printer/test'
     require 'mutant/reporter/cli/format'
     require 'mutant/repository'
     require 'mutant/repository/diff'
@@ -328,7 +331,9 @@ module Mutant
     recorder:              recorder,
     stderr:                $stderr,
     stdout:                $stdout,
+    tempfile:              Tempfile,
     thread:                Thread,
+    time:                  Time,
     timer:                 timer
   )
 

lib/mutant/bootstrap.rb

@@ -47,6 +47,32 @@ def self.call(env)
           selected_subjects.flat_map(&:mutations)
         end
 
+        setup_integration(
+          env:               env,
+          mutations:         mutations,
+          selected_subjects: selected_subjects
+        )
+      end
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    # Run test only bootstrap
+    #
+    # @param [Env] env
+    #
+    # @return [Either<String, Env>]
+    def self.call_test(env)
+      env.record(:bootstrap) do
+        setup_integration(
+          env:               load_hooks(env),
+          mutations:         [],
+          selected_subjects: []
+        )
+      end
+    end
+
+    def self.setup_integration(env:, mutations:, selected_subjects:)
+      env.record(__method__) do
         Integration.setup(env).fmap do |integration|
           env.with(
             integration: integration,
@@ -57,7 +83,7 @@ def self.call(env)
         end
       end
     end
-    # rubocop:enable Metrics/MethodLength
+    private_class_method :setup_integration
 
     def self.load_hooks(env)
       env.record(__method__) do

lib/mutant/cli/command/environment/test.rb

@@ -8,6 +8,21 @@ class Test < self
           NAME              = 'test'
           SHORT_DESCRIPTION = 'test subcommands'
 
+        private
+
+          def parse_remaining_arguments(arguments)
+            arguments.each(&method(:add_integration_argument))
+            Either::Right.new(self)
+          end
+
+          def bootstrap
+            env = Env.empty(world, @config)
+
+            env
+              .record(:config) { Config.load(cli_config: @config, world: world) }
+              .bind { |config| Bootstrap.call_test(env.with(config: config)) }
+          end
+
           class List < self
             NAME              = 'list'
             SHORT_DESCRIPTION = 'List tests detected in the environment'
@@ -28,7 +43,29 @@ def list_tests(env)
             end
           end
 
-          SUBCOMMANDS = [List].freeze
+          class Run < self
+            NAME              = 'run'
+            SHORT_DESCRIPTION = 'Run tests'
+            SUBCOMMANDS       = EMPTY_ARRAY
+
+          private
+
+            def action
+              bootstrap
+                .bind(&Mutant::Test::Runner.public_method(:call))
+                .bind(&method(:from_result))
+            end
+
+            def from_result(result)
+              if result.success?
+                Either::Right.new(nil)
+              else
+                Either::Left.new('Test failures, exiting nonzero!')
+              end
+            end
+          end
+
+          SUBCOMMANDS = [List, Run].freeze
         end # Test
       end # Environment
     end # Command

lib/mutant/env.rb

@@ -66,10 +66,18 @@ def cover_index(mutation_index)
       )
     end
 
+    def run_test_index(test_index)
+      integration.call([integration.all_tests.fetch(test_index)])
+    end
+
     def emit_mutation_worker_process_start(index:)
       hooks.run(:mutation_worker_process_start, index: index)
     end
 
+    def emit_test_worker_process_start(index:)
+      hooks.run(:test_worker_process_start, index: index)
+    end
+
     # The test selections
     #
     # @return Hash{Mutation => Enumerable<Test>}
@@ -175,7 +183,6 @@ def run_mutation_tests(mutation, tests)
     def timer
       world.timer
     end
-
   end # Env
   # rubocop:enable Metrics/ClassLength
 end # Mutant

lib/mutant/hooks.rb

@@ -10,6 +10,7 @@ class Hooks
       mutation_insert_post
       mutation_insert_pre
       mutation_worker_process_start
+      test_worker_process_start
     ].product([EMPTY_ARRAY]).to_h.transform_values(&:freeze).freeze
 
     MESSAGE = 'Unknown hook %s'

lib/mutant/integration.rb

@@ -4,7 +4,11 @@ module Mutant
 
   # Abstract base class mutant test framework integrations
   class Integration
-    include AbstractType, Adamantium, Anima.new(:arguments, :expression_parser, :world)
+    include AbstractType, Adamantium, Anima.new(
+      :arguments,
+      :expression_parser,
+      :world
+    )
 
     LOAD_MESSAGE = <<~'MESSAGE'
       Unable to load integration mutant-%<integration_name>s:

lib/mutant/integration/minitest.rb

@@ -109,6 +109,7 @@ def call(tests)
         reporter.report
 
         Result::Test.new(
+          output:  '',
           passed:  reporter.passed?,
           runtime: timer.now - start
         )

lib/mutant/integration/null.rb

@@ -18,6 +18,7 @@ def all_tests
       # @return [Result::Test]
       def call(_tests)
         Result::Test.new(
+          output:  '',
           passed:  true,
           runtime: 0.0
         )

lib/mutant/integration/rspec.rb

@@ -19,8 +19,9 @@ class Integration
     #   * location
     #   Is NOT enough. It would not be unique. So we add an "example index"
     #   for unique reference.
+    #
+    # rubocop:disable Metrics/ClassLength
     class Rspec < self
-
       ALL_EXPRESSION       = Expression::Namespace::Recursive.new(scope_name: nil)
       EXPRESSION_CANDIDATE = /\A([^ ]+)(?: )?/
       EXIT_SUCCESS         = 0
@@ -29,6 +30,11 @@ class Rspec < self
 
       private_constant(*constants(false))
 
+      def freeze
+        super() if @setup_elapsed
+        self
+      end
+
       # Initialize rspec integration
       #
       # @return [undefined]
@@ -42,10 +48,15 @@ def initialize(*)
       #
       # @return [self]
       def setup
-        @runner.setup($stderr, $stdout)
-        example_group_map
+        @setup_elapsed = timer.elapsed do
+          @runner.setup(world.stderr, world.stdout)
+          fail 'Rspec setup failure' if RSpec.world.respond_to?(:rspec_is_quitting) && RSpec.world.rspec_is_quitting
+          example_group_map
+        end
+        @runner.configuration.force(color_mode: :on)
+        @runner.configuration.reporter
         reset_examples
-        self
+        freeze
       end
       memoize :setup
 
@@ -54,15 +65,24 @@ def setup
       # @param [Enumerable<Mutant::Test>] tests
       #
       # @return [Result::Test]
+      #
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength
       def call(tests)
+        reset_examples
         setup_examples(tests.map(&all_tests_index))
+        @runner.configuration.start_time = world.time.now - @setup_elapsed
         start = timer.now
         passed = @runner.run_specs(@rspec_world.ordered_example_groups).equal?(EXIT_SUCCESS)
+        @runner.configuration.reset_reporter
         Result::Test.new(
+          output:  '',
           passed:  passed,
           runtime: timer.now - start
         )
       end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
 
       # All tests
       #
@@ -149,5 +169,6 @@ def all_examples
         @rspec_world.example_groups.flat_map(&:descendants).flat_map(&:examples)
       end
     end # Rspec
+    # rubocop:enable Metrics/ClassLength
   end # Integration
 end # Mutant

lib/mutant/reporter.rb

@@ -19,20 +19,41 @@ class Reporter
     # @return [self]
     abstract_method :start
 
-    # Report collector state
+    # Report test start
+    #
+    # @param [Env] env
+    #
+    # @return [self]
+    abstract_method :test_start
+
+    # Report final state
     #
     # @param [Runner::Collector] collector
     #
     # @return [self]
     abstract_method :report
 
+    # Report final test state
+    #
+    # @param [Runner::Collector] collector
+    #
+    # @return [self]
+    abstract_method :test_report
+
     # Report progress on object
     #
     # @param [Object] object
     #
     # @return [self]
     abstract_method :progress
 
+    # Report progress on object
+    #
+    # @param [Object] object
+    #
+    # @return [self]
+    abstract_method :test_progress
+
     # The reporter delay
     #
     # @return [Float]

lib/mutant/reporter/cli.rb

@@ -29,6 +29,16 @@ def start(env)
         self
       end
 
+      # Report test start
+      #
+      # @param [Env] env
+      #
+      # @return [self]
+      def test_start(env)
+        write(format.test_start(env))
+        self
+      end
+
       # Report progress object
       #
       # @param [Parallel::Status] status
@@ -39,6 +49,14 @@ def progress(status)
         self
       end
 
+      # Report progress object
+      #
+      # @return [self]
+      def test_progress(status)
+        write(format.test_progress(status))
+        self
+      end
+
       # Report delay in seconds
       #
       # @return [Float]
@@ -66,6 +84,16 @@ def report(env)
         self
       end
 
+      # Report env
+      #
+      # @param [Result::Env] env
+      #
+      # @return [self]
+      def test_report(env)
+        Printer::Test::EnvResult.call(output: output, object: env)
+        self
+      end
+
     private
 
       def write(frame)