fix(iouring): hand a connection to epoll only when no read can still resolve its fd (celeris#657)

.github/workflows/ci.yml

@@ -127,12 +127,15 @@ jobs:
           go test -race -count=1 -timeout=300s $pkgs
       # celeris#657: the step above runs ./engine/iouring WITHOUT -v, so a skip
       # there prints nothing and the package still reports `ok`. That is how the
-      # celeris#656 leak shipped with no cover. Five of the celeris#657 hand-off
-      # loss witness tests can skip: four build a ring through newTestRing, and
-      # one builds io_uring workers. So all eleven witness tests run again here
-      # by name, with the PASS-count interlock the `iouring` job uses
-      # (celeris#664): -v, CELERIS_REQUIRE_IOURING_WORKERS=1 to turn every
-      # environment skip into a failure, and an exact tally.
+      # celeris#656 leak shipped with no cover. Most celeris#657 tests can skip:
+      # they build a ring through newTestRing, build io_uring workers, or start
+      # a whole io_uring engine. That holds for the eleven hand-off loss witness
+      # tests (PR-1, the first list) and the eighteen fd-lifetime tests (PR-2,
+      # the second list; eight of them pin the async-cancel-flags probe, the
+      # reap-failure and dup-failure paths and the reaped recv's cleanup). So
+      # all twenty-nine run again here by name, with the PASS-count interlock
+      # the `iouring` job uses (celeris#664): -v, CELERIS_REQUIRE_IOURING_WORKERS=1
+      # to turn every environment skip into a failure, and an exact tally.
       #
       # The tally counts TOP-LEVEL results: `=== RUN   Name` with nothing after
       # the name, and `--- PASS: Name (`. go test prints the elapsed time after
@@ -145,24 +148,37 @@ jobs:
       # measured (CI and local runner-shape containers), not guaranteed. If a
       # later change makes them not fit, the require-workers env turns the
       # ENOMEM into a red job instead of a skip. That is intended.
-      - name: celeris#657 hand-off loss witness tests (skipping forbidden)
+      #
+      # That 8 MiB is also what makes this the one-worker leg of the
+      # fd-lifetime tests: through the Listen path it funds a single io_uring
+      # worker, the shape in which every hand-off loss was measured (all
+      # connections on one ring). Each engine-level fd-lifetime test logs
+      # "celeris657 engine workers=N"; the step requires at least one such
+      # line and every one to read workers=1, so a runner that funds more
+      # workers fails here instead of silently testing another shape.
+      - name: celeris#657 hand-off witness and fd-lifetime tests, one io_uring worker (skipping forbidden)
         shell: bash
         env:
           CELERIS_REQUIRE_IOURING_WORKERS: "1"
         run: |
           set -o pipefail
           echo "memlock (KiB): $(ulimit -l)"
-          names='TestStaleRecvDataCountsATransplantedConn|TestStaleRecvDataCountsAnAsyncTransplantedConn|TestStaleRecvDataCountsAClosedConn|TestStaleRecvDataCountsAnUnattributedIdentity|TestStaleRecvDataIgnoresCompletionsWithoutData|TestStaleRecvDataCountsEachMultishotCompletion|TestTransplantHandoffInFlightCountsTryTransplant|TestTransplantHandoffInFlightCountsFinishAsyncTransplant|TestMetricsCarriesTheHandoffLossWitnesses|TestWorkersShareTheHandoffLossWitnesses|TestClosedOpsEntryStaysThirtyTwoBytes'
+          pr1='TestStaleRecvDataCountsATransplantedConn|TestStaleRecvDataCountsAnAsyncTransplantedConn|TestStaleRecvDataCountsAClosedConn|TestStaleRecvDataCountsAnUnattributedIdentity|TestStaleRecvDataIgnoresCompletionsWithoutData|TestStaleRecvDataCountsEachMultishotCompletion|TestTransplantHandoffInFlightCountsTryTransplant|TestTransplantHandoffInFlightCountsFinishAsyncTransplant|TestMetricsCarriesTheHandoffLossWitnesses|TestWorkersShareTheHandoffLossWitnesses|TestClosedOpsEntryStaysThirtyTwoBytes'
+          pr2='TestTransplantNeverHandsOffArmedRecv|TestTransplantReapMissIsRetried|TestHoldReleasedWhenDrainStops|TestHoldRescuedByCheckTimeouts|TestOneOwnerPerHandoff|TestNoDrainSQESequenceIsUnchanged|TestHandoffHasNothingInFlight|TestStaleRecvDataCounted|TestHeldRecvIsReArmedWhenTheHandOffDoesNotHappen|TestWorkerParksWithNothingPending|TestTransplantReapFailureIsNotRetried|TestNoReapWithoutAsyncCancelFlags|TestReapSuppressedAfterFailedHandOff|TestReapedRecvLeavesNoLinkOrBuffer|TestAsyncCancelProbeClassifies|TestAsyncCancelProbeOnThisKernel|TestWorkersCarryTheAsyncCancelProbe|TestReapOnTheRunningKernel'
+          names="${pr1}|${pr2}"
           want=$(( $(tr '|' '\n' <<<"$names" | wc -l) ))
           go test -race -count=1 -timeout=300s -v -run "^(${names})\$" \
             ./engine/iouring/ 2>&1 | tee /tmp/witness657.log
           ran=$(grep -cE "^=== RUN   (${names})\$" /tmp/witness657.log || true)
           passed=$(grep -cE "^--- PASS: (${names}) \(" /tmp/witness657.log || true)
           skipped=$(grep -cE '^[[:space:]]*--- SKIP' /tmp/witness657.log || true)
-          echo "celeris#657 witness tests: want $want, ran $ran, passed $passed, SKIP lines $skipped"
-          if [ "$ran" -ne "$want" ] || [ "$passed" -ne "$want" ] || [ "$skipped" -ne 0 ]; then
-            echo "expected exactly $want top-level celeris#657 tests to run and PASS with no SKIP line --"
-            echo "was one renamed or removed, did one skip, or did one fail?"
+          engines=$(grep -cE 'celeris657 engine workers=[0-9]+$' /tmp/witness657.log || true)
+          one=$(grep -cE 'celeris657 engine workers=1$' /tmp/witness657.log || true)
+          echo "celeris#657 witness and fd-lifetime tests: want $want, ran $ran, passed $passed, SKIP lines $skipped, engines $engines at workers=1: $one"
+          if [ "$ran" -ne "$want" ] || [ "$passed" -ne "$want" ] || [ "$skipped" -ne 0 ] || [ "$engines" -eq 0 ] || [ "$one" -ne "$engines" ]; then
+            echo "expected exactly $want top-level celeris#657 tests to run and PASS with no SKIP line, and"
+            echo "every engine at one io_uring worker -- was one renamed or removed, did one skip or fail,"
+            echo "or did this runner fund a different worker count?"
             exit 1
           fi
       - name: middleware/compress
@@ -213,15 +229,89 @@ jobs:
       # connections queued on the paused engine, and on a GitHub runner a
       # 2048-conn promotion loses about a third of them). The other three
       # up-switch tests keep running.
+      #
+      # celeris#657 face 2 has its multi-ring gate here: TestFlapConnsPerRing
+      # (T4) fixes both engines at two workers, so its 256 keep-alive
+      # connections are 128 per io_uring ring whatever the runner funds, and
+      # runs three promote/revert cycles under load. At raised memlock the two
+      # revert tests below spread their 64 connections over several rings and
+      # did not lose on the base; T4 lost requests in 8 of 8 base runs. The
+      # tally makes T4 impossible to lose quietly: exactly one top-level RUN
+      # and PASS, no SKIP line for it, and its own RESULT line must show both
+      # engines at two workers and err=0 (the requests its clients lost).
       - name: Adaptive — race tests (memlock raised, up-switch required)
+        shell: bash
         env:
           CELERIS_REQUIRE_UPSWITCH: "1"
         run: |
+          set -o pipefail
           sudo prlimit --pid "$$" --memlock=unlimited:unlimited
           echo "memlock (KiB): $(ulimit -l)"
           go test -race -count=1 -timeout=600s -v \
             -skip '^(TestBidirectionalFlapAsync|TestRampH1Sync|TestRampH1Async)$' \
-            ./adaptive/...
+            ./adaptive/... 2>&1 | tee /tmp/adaptive-race.log
+          t4='TestFlapConnsPerRing'
+          ran=$(grep -cE "^=== RUN   ${t4}\$" /tmp/adaptive-race.log || true)
+          passed=$(grep -cE "^--- PASS: ${t4} \(" /tmp/adaptive-race.log || true)
+          skipped=$(grep -cE "^[[:space:]]*--- SKIP: ${t4}[ /]" /tmp/adaptive-race.log || true)
+          results=$(grep -cE 'S0T4 RESULT cycles=[0-9]+ conns=[0-9]+ wE=[0-9]+ wI=[0-9]+ ' /tmp/adaptive-race.log || true)
+          clean=$(grep -cE 'S0T4 RESULT cycles=3 conns=256 wE=2 wI=2 conns_per_ring=128 ok=[0-9]+ err=0 ' /tmp/adaptive-race.log || true)
+          echo "celeris#657 T4: want 1, ran $ran, passed $passed, SKIP lines $skipped, RESULT lines $results, at two workers with err=0: $clean"
+          if [ "$ran" -ne 1 ] || [ "$passed" -ne 1 ] || [ "$skipped" -ne 0 ] || [ "$results" -ne 1 ] || [ "$clean" -ne 1 ]; then
+            echo "expected TestFlapConnsPerRing to run once and PASS with no SKIP line, and its RESULT line to show"
+            echo "both engines at two workers (128 connections per io_uring ring) and err=0 -- was it renamed or"
+            echo "removed, did it skip, or did this runner fund a different worker count?"
+            exit 1
+          fi
+      # celeris#657 PR-2: the one-worker leg of the two revert tests. The step
+      # above raises memlock, so io_uring there runs several workers and
+      # TestReverseTransplant / TestBidirectionalFlap never meet the shape in
+      # which the io_uring -> epoll hand-off lost requests: every connection
+      # on ONE io_uring ring. This step does not raise memlock. At the
+      # runner's own 8 MiB io_uring is capped to one worker (12 MiB per
+      # worker, engine/iouring/ring.go minMemlockPerWorker), and the step
+      # requires at least one "io_uring engine listening" line and every one
+      # to read workers=1, so a runner that funds more workers fails here
+      # instead of silently testing another shape.
+      #
+      # Neither test reads a CELERIS_REQUIRE_* variable: they t.Skip when
+      # io_uring is unavailable. The tally is what forbids that: it fails on
+      # ANY `--- SKIP` line, and on any top-level count other than two RUN and
+      # two PASS (same patterns as the unit job's celeris#657 step). It runs
+      # even when the step above failed, so a known flake there cannot hide
+      # this leg's result.
+      #
+      # Their verdicts tolerate up to one lost request per connection (err <=
+      # 64), so a PASS alone does not mean nothing was lost: on the base, the
+      # flap test PASSED once with 56 lost requests. The step therefore also
+      # reads each test's own summary line (the revert test's "before revert"
+      # line and the flap test's "total" line, both for async=false) and
+      # requires err=0 on both.
+      - name: Adaptive — one-worker revert tests (runner memlock, skipping forbidden)
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          set -o pipefail
+          echo "memlock (KiB): $(ulimit -l)"
+          names='TestReverseTransplant|TestBidirectionalFlap'
+          want=$(( $(tr '|' '\n' <<<"$names" | wc -l) ))
+          go test -race -count=1 -timeout=300s -v -run "^(${names})\$" \
+            ./adaptive/ 2>&1 | tee /tmp/revert657.log
+          ran=$(grep -cE "^=== RUN   (${names})\$" /tmp/revert657.log || true)
+          passed=$(grep -cE "^--- PASS: (${names}) \(" /tmp/revert657.log || true)
+          skipped=$(grep -cE '^[[:space:]]*--- SKIP' /tmp/revert657.log || true)
+          engines=$(grep -cE 'io_uring engine listening .*workers=[0-9]+' /tmp/revert657.log || true)
+          one=$(grep -cE 'io_uring engine listening .*workers=1( |$)' /tmp/revert657.log || true)
+          summaries=$(grep -cE '\[async=false\] (before revert: .*\| ok=[0-9]+ err=[0-9]+$|total ok=[0-9]+ err=[0-9]+ \|)' /tmp/revert657.log || true)
+          lossless=$(grep -cE '\[async=false\] (before revert: .*\| ok=[0-9]+ err=0$|total ok=[0-9]+ err=0 \|)' /tmp/revert657.log || true)
+          echo "celeris#657 one-worker revert tests: want $want, ran $ran, passed $passed, SKIP lines $skipped, io_uring engines $engines at workers=1: $one, summaries $summaries with err=0: $lossless"
+          if [ "$ran" -ne "$want" ] || [ "$passed" -ne "$want" ] || [ "$skipped" -ne 0 ] || [ "$engines" -eq 0 ] || [ "$one" -ne "$engines" ] || [ "$summaries" -ne "$want" ] || [ "$lossless" -ne "$summaries" ]; then
+            echo "expected exactly $want top-level revert tests to run and PASS with no SKIP line, every"
+            echo "io_uring engine at one worker, and each test's summary line to report err=0 -- was one renamed"
+            echo "or removed, did one skip or fail, did this runner fund a different worker count, or were"
+            echo "requests lost (a PASS verdict tolerates up to one per connection)?"
+            exit 1
+          fi
 
   iouring:
     name: io_uring init-failure regression (./engine/iouring)

adaptive/engine.go

@@ -1078,6 +1078,21 @@ func (e *Engine) Metrics() engine.EngineMetrics {
 		StaleRecvDataTransplanted: pm.StaleRecvDataTransplanted + sm.StaleRecvDataTransplanted,
 		StaleRecvDataUnattributed: pm.StaleRecvDataUnattributed + sm.StaleRecvDataUnattributed,
 		TransplantHandoffInFlight: pm.TransplantHandoffInFlight + sm.TransplantHandoffInFlight,
+		// The fd-lifetime rule's counters (celeris#657 PR-2), io_uring-only
+		// and cumulative. A hold, a reap and a refused double claim each
+		// happen on the one sub-engine making the hand-off, so the sum
+		// counts each once; like the witnesses above, the standby's half
+		// is where a revert's hand-offs are made.
+		TransplantHeld:        pm.TransplantHeld + sm.TransplantHeld,
+		TransplantReaps:       pm.TransplantReaps + sm.TransplantReaps,
+		TransplantReapMisses:  pm.TransplantReapMisses + sm.TransplantReapMisses,
+		TransplantHoldRescued: pm.TransplantHoldRescued + sm.TransplantHoldRescued,
+		TransplantDoubleClaim: pm.TransplantDoubleClaim + sm.TransplantDoubleClaim,
+		// The same rule for the refusals and fallbacks around them: each
+		// is an event on the one sub-engine attempting the hand-off.
+		TransplantClaimDeferred:   pm.TransplantClaimDeferred + sm.TransplantClaimDeferred,
+		TransplantReapFailed:      pm.TransplantReapFailed + sm.TransplantReapFailed,
+		TransplantReapUnsupported: pm.TransplantReapUnsupported + sm.TransplantReapUnsupported,
 
 		// The celeris#607 recv-stall and linked-recv ledger. io_uring-only,
 		// so the epoll half contributes zero and a switch simply moves which