Skip to content

Verify leak reproduction before bisecting #421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
acaron wants to merge 1 commit into from
Closed

Verify leak reproduction before bisecting #421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ruby/Gemfile.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
PATH
remote: .
specs:
ci-queue (0.97.0)
ci-queue (0.98.0)
logger

GEM
Expand Down
4 changes: 4 additions & 0 deletions ruby/lib/ci/queue/bisect.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ def failing_test_present?
@all_tests.find { |t| t.id == config.failing_test }
end

def all_candidates
Static.new(@tests + [config.failing_test], config).populate(@all_tests)
end

def candidates
Static.new(first_half + [config.failing_test], config).populate(@all_tests)
end
Expand Down
2 changes: 1 addition & 1 deletion ruby/lib/ci/queue/version.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

module CI
module Queue
VERSION = '0.97.0'
VERSION = '0.98.0'
DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
end
Expand Down
28 changes: 19 additions & 9 deletions ruby/lib/minitest/queue/runner.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,25 @@ def bisect_command
exit! 0
end

if queue.suspects_left == 0
step(yellow("The failing test was the first test in the test order so there is nothing to bisect."))
File.write('log/test_order.log', "")
File.write('log/bisect_test_details.log', "")
# Bisect ran successfully; there is simply nothing to bisect against.
# Reserve non-zero exits for cases where the bisect could not run (see failing_test_present? above).
exit! 0
end

step("Verifying the leak reproduces against every suspect")
if run_tests_in_fork(queue.all_candidates)
puts reopen_previous_step
puts yellow("The failing test passes after every suspect, so there is no leak to bisect here.")
File.write('log/test_order.log', "")
File.write('log/bisect_test_details.log', "")
exit! 0
end
puts

run_index = 0
while queue.suspects_left > 1
run_index += 1
Expand All @@ -250,15 +269,6 @@ def bisect_command
puts
end

if queue.suspects_left == 0
step(yellow("The failing test was the first test in the test order so there is nothing to bisect."))
File.write('log/test_order.log', "")
File.write('log/bisect_test_details.log', "")
# Bisect ran successfully; there is simply nothing to bisect against.
# Reserve non-zero exits for cases where the bisect could not run (see failing_test_present? above).
exit! 0
end

failing_order = queue.candidates
step("Final validation")
if run_tests_in_fork(failing_order)
Expand Down
212 changes: 71 additions & 141 deletions ruby/test/integration/minitest_bisect_test.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,54 @@ def test_bisect
expected_output = strip_heredoc <<-EOS
--- Testing the failing test in isolation
LeakyTest#test_sensible_to_leak PASS
--- Verifying the leak reproduces against every suspect
LeakyTest#test_useless_0 PASS
LeakyTest#test_useless_1 PASS
LeakyTest#test_useless_2 PASS
LeakyTest#test_useless_3 PASS
LeakyTest#test_useless_4 PASS
LeakyTest#test_useless_5 PASS
LeakyTest#test_useless_6 PASS
LeakyTest#test_useless_7 PASS
LeakyTest#test_useless_8 PASS
LeakyTest#test_useless_9 PASS
LeakyTest#test_introduce_leak PASS
LeakyTest#test_useless_10 PASS
LeakyTest#test_useless_11 PASS
LeakyTest#test_useless_12 PASS
LeakyTest#test_useless_13 PASS
LeakyTest#test_useless_14 PASS
LeakyTest#test_useless_15 PASS
LeakyTest#test_useless_16 PASS
LeakyTest#test_useless_17 PASS
LeakyTest#test_useless_18 PASS
LeakyTest#test_useless_19 PASS
LeakyTest#test_useless_20 PASS
LeakyTest#test_useless_21 PASS
LeakyTest#test_useless_22 PASS
LeakyTest#test_useless_23 PASS
LeakyTest#test_useless_24 PASS
LeakyTest#test_useless_25 PASS
LeakyTest#test_useless_26 PASS
LeakyTest#test_useless_27 PASS
LeakyTest#test_useless_28 PASS
LeakyTest#test_useless_29 PASS
LeakyTest#test_useless_30 PASS
LeakyTest#test_useless_31 PASS
LeakyTest#test_useless_32 PASS
LeakyTest#test_useless_33 PASS
LeakyTest#test_useless_34 PASS
LeakyTest#test_useless_35 PASS
LeakyTest#test_useless_36 PASS
LeakyTest#test_useless_37 PASS
LeakyTest#test_useless_38 PASS
LeakyTest#test_useless_39 PASS
LeakyTest#test_useless_40 PASS
LeakyTest#test_useless_41 PASS
LeakyTest#test_useless_42 PASS
LeakyTest#test_useless_43 PASS
LeakyTest#test_sensible_to_leak FAIL

--- Run #1, 45 suspects left
LeakyTest#test_useless_0 PASS
LeakyTest#test_useless_1 PASS
Expand Down Expand Up @@ -102,83 +150,26 @@ def test_bisect
assert_equal expected_output, normalize(out)
end

def test_inconclusive
def test_exits_when_leak_does_not_reproduce
result = nil
out, err = capture_subprocess_io do
result = run_bisect('log/unconclusive_test_order.log', 'LeakyTest#test_sensible_to_leak')
end

assert_equal true, result, "inconclusive bisect should exit 0; it ran successfully and reported a valid diagnostic outcome"
assert_empty filter_deprecation_warnings(err)
expected_output = strip_heredoc <<-EOS
--- Testing the failing test in isolation
LeakyTest#test_sensible_to_leak PASS
--- Run #1, 45 suspects left
LeakyTest#test_useless_0 PASS
LeakyTest#test_useless_1 PASS
LeakyTest#test_useless_2 PASS
LeakyTest#test_useless_3 PASS
LeakyTest#test_useless_4 PASS
LeakyTest#test_useless_5 PASS
LeakyTest#test_useless_6 PASS
LeakyTest#test_useless_7 PASS
LeakyTest#test_useless_8 PASS
LeakyTest#test_useless_9 PASS
LeakyTest#test_harmless_test PASS
LeakyTest#test_useless_10 PASS
LeakyTest#test_useless_11 PASS
LeakyTest#test_useless_12 PASS
LeakyTest#test_useless_13 PASS
LeakyTest#test_useless_14 PASS
LeakyTest#test_useless_15 PASS
LeakyTest#test_useless_16 PASS
LeakyTest#test_useless_17 PASS
LeakyTest#test_useless_18 PASS
LeakyTest#test_useless_19 PASS
LeakyTest#test_useless_20 PASS
LeakyTest#test_useless_21 PASS
LeakyTest#test_sensible_to_leak PASS

--- Run #2, 22 suspects left
LeakyTest#test_useless_22 PASS
LeakyTest#test_useless_23 PASS
LeakyTest#test_useless_24 PASS
LeakyTest#test_useless_25 PASS
LeakyTest#test_useless_26 PASS
LeakyTest#test_useless_27 PASS
LeakyTest#test_useless_28 PASS
LeakyTest#test_useless_29 PASS
LeakyTest#test_useless_30 PASS
LeakyTest#test_useless_31 PASS
LeakyTest#test_useless_32 PASS
LeakyTest#test_sensible_to_leak PASS

--- Run #3, 11 suspects left
LeakyTest#test_useless_33 PASS
LeakyTest#test_useless_34 PASS
LeakyTest#test_useless_35 PASS
LeakyTest#test_useless_36 PASS
LeakyTest#test_useless_37 PASS
LeakyTest#test_useless_38 PASS
LeakyTest#test_sensible_to_leak PASS

--- Run #4, 5 suspects left
LeakyTest#test_useless_39 PASS
LeakyTest#test_useless_40 PASS
LeakyTest#test_useless_41 PASS
LeakyTest#test_sensible_to_leak PASS

--- Run #5, 2 suspects left
LeakyTest#test_useless_42 PASS
LeakyTest#test_sensible_to_leak PASS
normalized_output = normalize(out)
reproduction_step = "--- Verifying the leak reproduces against every suspect"

--- Final validation
LeakyTest#test_useless_43 PASS
LeakyTest#test_sensible_to_leak PASS
--- The bisection was inconclusive, there might not be any leaky test here.
EOS
assert_equal(true, result, "should exit 0 when the leak does not reproduce")
assert_empty(filter_deprecation_warnings(err))
assert_includes(
normalized_output,
"The failing test passes after every suspect, so there is no leak to bisect here.",
)
assert_equal(1, normalized_output.scan(reproduction_step).size)
refute_includes(normalized_output, "--- Run #")

assert_equal expected_output, normalize(out)
details_path = File.expand_path('../fixtures/log/bisect_test_details.log', __dir__)
assert_empty(File.read(details_path))
end

def test_failing_test_is_the_first_entry_in_the_test_order
Expand All @@ -198,83 +189,22 @@ def test_failing_test_is_the_first_entry_in_the_test_order
assert_equal expected_output, normalize(out)
end

def test_broken_tests_which_are_not_evaluated_are_ignored
def test_broken_non_victim_tests_are_ignored_during_reproduction_check
result = nil
out, err = capture_subprocess_io do
result = run_bisect('log/leaky_with_broken_test_order.log', 'LeakyTest#test_sensible_to_leak')
end

assert_equal true, result, "inconclusive bisect should exit 0 even when broken tests are skipped along the way"
assert_empty filter_deprecation_warnings(err)
expected_output = strip_heredoc <<-EOS
--- Testing the failing test in isolation
LeakyTest#test_sensible_to_leak PASS
--- Run #1, 45 suspects left
LeakyTest#test_useless_0 PASS
LeakyTest#test_useless_1 PASS
LeakyTest#test_useless_2 PASS
LeakyTest#test_useless_3 PASS
LeakyTest#test_useless_4 PASS
LeakyTest#test_useless_5 PASS
LeakyTest#test_useless_6 PASS
LeakyTest#test_useless_7 PASS
LeakyTest#test_useless_8 PASS
LeakyTest#test_useless_9 PASS
LeakyTest#test_broken_test SKIP
LeakyTest#test_useless_10 PASS
LeakyTest#test_useless_11 PASS
LeakyTest#test_useless_12 PASS
LeakyTest#test_useless_13 PASS
LeakyTest#test_useless_14 PASS
LeakyTest#test_useless_15 PASS
LeakyTest#test_useless_16 PASS
LeakyTest#test_useless_17 PASS
LeakyTest#test_useless_18 PASS
LeakyTest#test_useless_19 PASS
LeakyTest#test_useless_20 PASS
LeakyTest#test_useless_21 PASS
LeakyTest#test_sensible_to_leak PASS

--- Run #2, 22 suspects left
LeakyTest#test_useless_22 PASS
LeakyTest#test_useless_23 PASS
LeakyTest#test_useless_24 PASS
LeakyTest#test_useless_25 PASS
LeakyTest#test_useless_26 PASS
LeakyTest#test_useless_27 PASS
LeakyTest#test_useless_28 PASS
LeakyTest#test_useless_29 PASS
LeakyTest#test_useless_30 PASS
LeakyTest#test_useless_31 PASS
LeakyTest#test_useless_32 PASS
LeakyTest#test_sensible_to_leak PASS
normalized_output = normalize(out)

--- Run #3, 11 suspects left
LeakyTest#test_useless_33 PASS
LeakyTest#test_useless_34 PASS
LeakyTest#test_useless_35 PASS
LeakyTest#test_useless_36 PASS
LeakyTest#test_useless_37 PASS
LeakyTest#test_useless_38 PASS
LeakyTest#test_sensible_to_leak PASS

--- Run #4, 5 suspects left
LeakyTest#test_useless_39 PASS
LeakyTest#test_useless_40 PASS
LeakyTest#test_useless_41 PASS
LeakyTest#test_sensible_to_leak PASS

--- Run #5, 2 suspects left
LeakyTest#test_useless_42 PASS
LeakyTest#test_sensible_to_leak PASS

--- Final validation
LeakyTest#test_useless_43 PASS
LeakyTest#test_sensible_to_leak PASS
--- The bisection was inconclusive, there might not be any leaky test here.
EOS

assert_equal expected_output, normalize(out)
assert_equal(true, result, "should exit 0 when the leak does not reproduce")
assert_empty(filter_deprecation_warnings(err))
assert_includes(normalized_output, "LeakyTest#test_broken_test SKIP")
assert_includes(
normalized_output,
"The failing test passes after every suspect, so there is no leak to bisect here.",
)
refute_includes(normalized_output, "--- Run #")
end

def test_broken
Expand Down
Loading