diff --git a/ruby/Gemfile.lock b/ruby/Gemfile.lock index 800b4f49..fed01112 100644 --- a/ruby/Gemfile.lock +++ b/ruby/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - ci-queue (0.97.0) + ci-queue (0.98.0) logger GEM diff --git a/ruby/lib/ci/queue/bisect.rb b/ruby/lib/ci/queue/bisect.rb index b73bd94e..4c8ca835 100644 --- a/ruby/lib/ci/queue/bisect.rb +++ b/ruby/lib/ci/queue/bisect.rb @@ -31,6 +31,10 @@ def failing_test_present? @all_tests.find { |t| t.id == config.failing_test } end + def all_candidates + Static.new(@tests + [config.failing_test], config).populate(@all_tests) + end + def candidates Static.new(first_half + [config.failing_test], config).populate(@all_tests) end diff --git a/ruby/lib/ci/queue/version.rb b/ruby/lib/ci/queue/version.rb index e187297a..4ec0e293 100644 --- a/ruby/lib/ci/queue/version.rb +++ b/ruby/lib/ci/queue/version.rb @@ -2,7 +2,7 @@ module CI module Queue - VERSION = '0.97.0' + VERSION = '0.98.0' DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__) RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__) end diff --git a/ruby/lib/minitest/queue/runner.rb b/ruby/lib/minitest/queue/runner.rb index 106c7b5f..c4f456ae 100644 --- a/ruby/lib/minitest/queue/runner.rb +++ b/ruby/lib/minitest/queue/runner.rb @@ -238,6 +238,25 @@ def bisect_command exit! 0 end + if queue.suspects_left == 0 + step(yellow("The failing test was the first test in the test order so there is nothing to bisect.")) + File.write('log/test_order.log', "") + File.write('log/bisect_test_details.log', "") + # Bisect ran successfully; there is simply nothing to bisect against. + # Reserve non-zero exits for cases where the bisect could not run (see failing_test_present? above). + exit! 0 + end + + step("Verifying the leak reproduces against every suspect") + if run_tests_in_fork(queue.all_candidates) + puts reopen_previous_step + puts yellow("The failing test passes after every suspect, so there is no leak to bisect here.") + File.write('log/test_order.log', "") + File.write('log/bisect_test_details.log', "") + exit! 0 + end + puts + run_index = 0 while queue.suspects_left > 1 run_index += 1 @@ -250,15 +269,6 @@ def bisect_command puts end - if queue.suspects_left == 0 - step(yellow("The failing test was the first test in the test order so there is nothing to bisect.")) - File.write('log/test_order.log', "") - File.write('log/bisect_test_details.log', "") - # Bisect ran successfully; there is simply nothing to bisect against. - # Reserve non-zero exits for cases where the bisect could not run (see failing_test_present? above). - exit! 0 - end - failing_order = queue.candidates step("Final validation") if run_tests_in_fork(failing_order) diff --git a/ruby/test/integration/minitest_bisect_test.rb b/ruby/test/integration/minitest_bisect_test.rb index 4dcc94aa..448d7ce9 100644 --- a/ruby/test/integration/minitest_bisect_test.rb +++ b/ruby/test/integration/minitest_bisect_test.rb @@ -21,6 +21,54 @@ def test_bisect expected_output = strip_heredoc <<-EOS --- Testing the failing test in isolation LeakyTest#test_sensible_to_leak PASS + --- Verifying the leak reproduces against every suspect + LeakyTest#test_useless_0 PASS + LeakyTest#test_useless_1 PASS + LeakyTest#test_useless_2 PASS + LeakyTest#test_useless_3 PASS + LeakyTest#test_useless_4 PASS + LeakyTest#test_useless_5 PASS + LeakyTest#test_useless_6 PASS + LeakyTest#test_useless_7 PASS + LeakyTest#test_useless_8 PASS + LeakyTest#test_useless_9 PASS + LeakyTest#test_introduce_leak PASS + LeakyTest#test_useless_10 PASS + LeakyTest#test_useless_11 PASS + LeakyTest#test_useless_12 PASS + LeakyTest#test_useless_13 PASS + LeakyTest#test_useless_14 PASS + LeakyTest#test_useless_15 PASS + LeakyTest#test_useless_16 PASS + LeakyTest#test_useless_17 PASS + LeakyTest#test_useless_18 PASS + LeakyTest#test_useless_19 PASS + LeakyTest#test_useless_20 PASS + LeakyTest#test_useless_21 PASS + LeakyTest#test_useless_22 PASS + LeakyTest#test_useless_23 PASS + LeakyTest#test_useless_24 PASS + LeakyTest#test_useless_25 PASS + LeakyTest#test_useless_26 PASS + LeakyTest#test_useless_27 PASS + LeakyTest#test_useless_28 PASS + LeakyTest#test_useless_29 PASS + LeakyTest#test_useless_30 PASS + LeakyTest#test_useless_31 PASS + LeakyTest#test_useless_32 PASS + LeakyTest#test_useless_33 PASS + LeakyTest#test_useless_34 PASS + LeakyTest#test_useless_35 PASS + LeakyTest#test_useless_36 PASS + LeakyTest#test_useless_37 PASS + LeakyTest#test_useless_38 PASS + LeakyTest#test_useless_39 PASS + LeakyTest#test_useless_40 PASS + LeakyTest#test_useless_41 PASS + LeakyTest#test_useless_42 PASS + LeakyTest#test_useless_43 PASS + LeakyTest#test_sensible_to_leak FAIL + --- Run #1, 45 suspects left LeakyTest#test_useless_0 PASS LeakyTest#test_useless_1 PASS @@ -102,83 +150,26 @@ def test_bisect assert_equal expected_output, normalize(out) end - def test_inconclusive + def test_exits_when_leak_does_not_reproduce result = nil out, err = capture_subprocess_io do result = run_bisect('log/unconclusive_test_order.log', 'LeakyTest#test_sensible_to_leak') end - assert_equal true, result, "inconclusive bisect should exit 0; it ran successfully and reported a valid diagnostic outcome" - assert_empty filter_deprecation_warnings(err) - expected_output = strip_heredoc <<-EOS - --- Testing the failing test in isolation - LeakyTest#test_sensible_to_leak PASS - --- Run #1, 45 suspects left - LeakyTest#test_useless_0 PASS - LeakyTest#test_useless_1 PASS - LeakyTest#test_useless_2 PASS - LeakyTest#test_useless_3 PASS - LeakyTest#test_useless_4 PASS - LeakyTest#test_useless_5 PASS - LeakyTest#test_useless_6 PASS - LeakyTest#test_useless_7 PASS - LeakyTest#test_useless_8 PASS - LeakyTest#test_useless_9 PASS - LeakyTest#test_harmless_test PASS - LeakyTest#test_useless_10 PASS - LeakyTest#test_useless_11 PASS - LeakyTest#test_useless_12 PASS - LeakyTest#test_useless_13 PASS - LeakyTest#test_useless_14 PASS - LeakyTest#test_useless_15 PASS - LeakyTest#test_useless_16 PASS - LeakyTest#test_useless_17 PASS - LeakyTest#test_useless_18 PASS - LeakyTest#test_useless_19 PASS - LeakyTest#test_useless_20 PASS - LeakyTest#test_useless_21 PASS - LeakyTest#test_sensible_to_leak PASS - - --- Run #2, 22 suspects left - LeakyTest#test_useless_22 PASS - LeakyTest#test_useless_23 PASS - LeakyTest#test_useless_24 PASS - LeakyTest#test_useless_25 PASS - LeakyTest#test_useless_26 PASS - LeakyTest#test_useless_27 PASS - LeakyTest#test_useless_28 PASS - LeakyTest#test_useless_29 PASS - LeakyTest#test_useless_30 PASS - LeakyTest#test_useless_31 PASS - LeakyTest#test_useless_32 PASS - LeakyTest#test_sensible_to_leak PASS - - --- Run #3, 11 suspects left - LeakyTest#test_useless_33 PASS - LeakyTest#test_useless_34 PASS - LeakyTest#test_useless_35 PASS - LeakyTest#test_useless_36 PASS - LeakyTest#test_useless_37 PASS - LeakyTest#test_useless_38 PASS - LeakyTest#test_sensible_to_leak PASS - - --- Run #4, 5 suspects left - LeakyTest#test_useless_39 PASS - LeakyTest#test_useless_40 PASS - LeakyTest#test_useless_41 PASS - LeakyTest#test_sensible_to_leak PASS - - --- Run #5, 2 suspects left - LeakyTest#test_useless_42 PASS - LeakyTest#test_sensible_to_leak PASS + normalized_output = normalize(out) + reproduction_step = "--- Verifying the leak reproduces against every suspect" - --- Final validation - LeakyTest#test_useless_43 PASS - LeakyTest#test_sensible_to_leak PASS - --- The bisection was inconclusive, there might not be any leaky test here. - EOS + assert_equal(true, result, "should exit 0 when the leak does not reproduce") + assert_empty(filter_deprecation_warnings(err)) + assert_includes( + normalized_output, + "The failing test passes after every suspect, so there is no leak to bisect here.", + ) + assert_equal(1, normalized_output.scan(reproduction_step).size) + refute_includes(normalized_output, "--- Run #") - assert_equal expected_output, normalize(out) + details_path = File.expand_path('../fixtures/log/bisect_test_details.log', __dir__) + assert_empty(File.read(details_path)) end def test_failing_test_is_the_first_entry_in_the_test_order @@ -198,83 +189,22 @@ def test_failing_test_is_the_first_entry_in_the_test_order assert_equal expected_output, normalize(out) end - def test_broken_tests_which_are_not_evaluated_are_ignored + def test_broken_non_victim_tests_are_ignored_during_reproduction_check result = nil out, err = capture_subprocess_io do result = run_bisect('log/leaky_with_broken_test_order.log', 'LeakyTest#test_sensible_to_leak') end - assert_equal true, result, "inconclusive bisect should exit 0 even when broken tests are skipped along the way" - assert_empty filter_deprecation_warnings(err) - expected_output = strip_heredoc <<-EOS - --- Testing the failing test in isolation - LeakyTest#test_sensible_to_leak PASS - --- Run #1, 45 suspects left - LeakyTest#test_useless_0 PASS - LeakyTest#test_useless_1 PASS - LeakyTest#test_useless_2 PASS - LeakyTest#test_useless_3 PASS - LeakyTest#test_useless_4 PASS - LeakyTest#test_useless_5 PASS - LeakyTest#test_useless_6 PASS - LeakyTest#test_useless_7 PASS - LeakyTest#test_useless_8 PASS - LeakyTest#test_useless_9 PASS - LeakyTest#test_broken_test SKIP - LeakyTest#test_useless_10 PASS - LeakyTest#test_useless_11 PASS - LeakyTest#test_useless_12 PASS - LeakyTest#test_useless_13 PASS - LeakyTest#test_useless_14 PASS - LeakyTest#test_useless_15 PASS - LeakyTest#test_useless_16 PASS - LeakyTest#test_useless_17 PASS - LeakyTest#test_useless_18 PASS - LeakyTest#test_useless_19 PASS - LeakyTest#test_useless_20 PASS - LeakyTest#test_useless_21 PASS - LeakyTest#test_sensible_to_leak PASS - - --- Run #2, 22 suspects left - LeakyTest#test_useless_22 PASS - LeakyTest#test_useless_23 PASS - LeakyTest#test_useless_24 PASS - LeakyTest#test_useless_25 PASS - LeakyTest#test_useless_26 PASS - LeakyTest#test_useless_27 PASS - LeakyTest#test_useless_28 PASS - LeakyTest#test_useless_29 PASS - LeakyTest#test_useless_30 PASS - LeakyTest#test_useless_31 PASS - LeakyTest#test_useless_32 PASS - LeakyTest#test_sensible_to_leak PASS + normalized_output = normalize(out) - --- Run #3, 11 suspects left - LeakyTest#test_useless_33 PASS - LeakyTest#test_useless_34 PASS - LeakyTest#test_useless_35 PASS - LeakyTest#test_useless_36 PASS - LeakyTest#test_useless_37 PASS - LeakyTest#test_useless_38 PASS - LeakyTest#test_sensible_to_leak PASS - - --- Run #4, 5 suspects left - LeakyTest#test_useless_39 PASS - LeakyTest#test_useless_40 PASS - LeakyTest#test_useless_41 PASS - LeakyTest#test_sensible_to_leak PASS - - --- Run #5, 2 suspects left - LeakyTest#test_useless_42 PASS - LeakyTest#test_sensible_to_leak PASS - - --- Final validation - LeakyTest#test_useless_43 PASS - LeakyTest#test_sensible_to_leak PASS - --- The bisection was inconclusive, there might not be any leaky test here. - EOS - - assert_equal expected_output, normalize(out) + assert_equal(true, result, "should exit 0 when the leak does not reproduce") + assert_empty(filter_deprecation_warnings(err)) + assert_includes(normalized_output, "LeakyTest#test_broken_test SKIP") + assert_includes( + normalized_output, + "The failing test passes after every suspect, so there is no leak to bisect here.", + ) + refute_includes(normalized_output, "--- Run #") end def test_broken