diff --git a/dev/modules/test_deep.md b/dev/modules/test_deep.md index df35266ce..338578ede 100644 --- a/dev/modules/test_deep.md +++ b/dev/modules/test_deep.md @@ -11,7 +11,9 @@ This document tracks all errors found when running `./jcpan -t Test::Deep` and t ### Current Status: 41/42 test files passing (after Phases 1-6) -Only `t/memory.t` fails due to `weaken` being unimplemented. +Only `t/memory.t` still fails one assertion. `weaken` is implemented in the +runtime, but the second-argument capture case remains an open selective +reference-counting edge case. | Test File | Status | Notes | |-----------|--------|-------| @@ -224,7 +226,7 @@ Files to change: --- -### 6. LOW: `Scalar::Util::weaken` is unimplemented (placeholder) +### 6. LOW: `Scalar::Util::weaken` edge case **Affected tests**: t/memory.t (2 fail) @@ -234,7 +236,9 @@ Failed test 'left didn't capture' Failed test 'right didn't capture' ``` -**Root Cause**: `weaken()` in `ScalarUtil.java` is a no-op placeholder. The test creates a weak reference, removes the strong reference, and expects the weak ref to become undef. Since `weaken()` does nothing, the weak ref stays alive. +**Root Cause**: The runtime's selective reference-counting implementation +handles ordinary weak references, but this test's indirect-call path retains +the expected (second) argument longer than Perl does. **Java feasibility note**: Java has `java.lang.ref.WeakReference`, but its semantics differ fundamentally from Perl's. Perl weak refs become `undef` **immediately and deterministically** when the last strong reference is removed (reference counting). Java weak refs are cleared by the GC **non-deterministically** -- the timing is unpredictable and depends on GC pressure. Replicating Perl's exact semantics would require building a reference-counting layer on top of Java's GC in `RuntimeScalar`, which is a significant architectural change. This is feasible but expensive to implement and maintain, and may not be worth it for the small number of affected tests. @@ -555,3 +559,30 @@ Alternative: fix the lexer to not greedily form `/=` after a regex close delimit | — | `weaken` unimplemented | t/memory.t | 2 | LOW (known) | **Expected outcome**: Fixing phases 3-6 should bring Test::Deep to **41/42 passing** (only t/memory.t remaining due to `weaken`). + +## Progress Tracking + +### Current Status: Test::Deep memory lifetime fixed; CPAN smoke verification complete (2026-08-03) + +### Completed Phases + +- [x] Lexical `B::Hooks::EndOfScope` dispatch + - Added parser-visible compile-scope callback stacks. + - `namespace::clean` now passes all 2,099 subtests, including its nested-scope test. + - Files: `BHooksEndOfScope.java`, `ParseBlock.java`. +- [x] Localized container cleanup + - Release temporary localized hash/array contents during global local restoration. + - Files: `GlobalRuntimeHash.java`, `GlobalRuntimeArray.java`, `GlobalRuntimeScalar.java`. +- [x] Regression coverage + - The upstream `namespace::clean` suite now covers nested compile-scope hooks and passes 2,099/2,099 subtests. +- [x] `t/memory.t` temporary comparison owner cleanup + - Recursive cleanup of discarded comparator/container values and a package-global reachability check on scalar overwrite prevent temporary call-frame aliases from retaining weak referents. + - Added `src/test/resources/unit/test_deep_memory_regression.t`, validated with system Perl and PerlOnJava's JVM backend. + +### Next Steps + +1. Add `Test::Fatal`/`Test::Requires` dependency handling or narrow the Type::Tiny optional suite when validating `Config::Locale`. + +### Open Questions + +- The interpreter backend still cannot run this regression because its bundled `Exporter` path reports an undefined `Exporter::Heavy::heavy_as_heavy` subroutine. diff --git a/src/main/java/org/perlonjava/frontend/parser/ParseBlock.java b/src/main/java/org/perlonjava/frontend/parser/ParseBlock.java index 49f313576..3828f3047 100644 --- a/src/main/java/org/perlonjava/frontend/parser/ParseBlock.java +++ b/src/main/java/org/perlonjava/frontend/parser/ParseBlock.java @@ -8,6 +8,7 @@ import org.perlonjava.frontend.astnode.Node; import org.perlonjava.frontend.lexer.LexerToken; import org.perlonjava.frontend.lexer.LexerTokenType; +import org.perlonjava.runtime.perlmodule.BHooksEndOfScope; import java.util.ArrayList; import java.util.List; @@ -54,6 +55,12 @@ public static BlockWithScope parseBlock(Parser parser, boolean exitScope) { // Store the starting position of the block for backtracking int currentIndex = parser.tokenIndex; + // B::Hooks::EndOfScope callbacks are compile-time lexical-scope + // callbacks. Track the parser scope independently of runtime local + // levels so pragmas such as namespace::clean run before a following + // BEGIN block in the enclosing scope. + BHooksEndOfScope.beginCompileScope(); + // Create new scope for variables declared in this block int scopeIndex = parser.ctx.symbolTable.enterScope(); @@ -117,6 +124,12 @@ public static BlockWithScope parseBlock(Parser parser, boolean exitScope) { Integer postBlockStrictOptions = null; + // Run compile-time end-of-scope callbacks while this block is still + // the innermost parser scope. This must happen before returning to + // the enclosing block, but after all statements in this block have + // been parsed. + BHooksEndOfScope.endCompileScope(); + // Exit the current scope before returning (unless delayed) if (exitScope) { parser.ctx.symbolTable.exitScope(scopeIndex); diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/BHooksEndOfScope.java b/src/main/java/org/perlonjava/runtime/perlmodule/BHooksEndOfScope.java index 73502d607..590ca0899 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/BHooksEndOfScope.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/BHooksEndOfScope.java @@ -31,6 +31,15 @@ public class BHooksEndOfScope extends PerlModuleBase { */ private static final ThreadLocal> loadingFileStack = ThreadLocal.withInitial(ArrayDeque::new); + /** + * Compile-time lexical scopes currently being parsed. Perl's + * B::Hooks::EndOfScope fires at the end of the lexical scope in which + * on_scope_end was called, which can be earlier than the end of the file + * (for example, a namespace::clean pragma inside an eval block). + */ + private static final ThreadLocal>> compileScopes = + ThreadLocal.withInitial(ArrayDeque::new); + public BHooksEndOfScope() { super("B::Hooks::EndOfScope", false); } @@ -116,6 +125,33 @@ private static String getCurrentLoadingFile() { return stack.isEmpty() ? null : stack.peek(); } + /** Enter a parser-visible lexical scope. */ + public static void beginCompileScope() { + compileScopes.get().push(new ArrayDeque<>()); + } + + /** + * Leave a parser-visible lexical scope and run callbacks registered in it + * in LIFO order, matching the native hook's ordering. + */ + public static void endCompileScope() { + Deque> scopes = compileScopes.get(); + if (scopes.isEmpty()) { + return; + } + Deque callbacks = scopes.pop(); + while (!callbacks.isEmpty()) { + RuntimeScalar codeRef = callbacks.pop(); + try { + if (codeRef.type == RuntimeScalarType.CODE && codeRef.value instanceof RuntimeCode code) { + code.apply(new RuntimeArray(), RuntimeContextType.VOID); + } + } catch (Exception e) { + System.err.println("Warning: on_scope_end callback error: " + e.getMessage()); + } + } + } + /** * Registers a callback to be executed when the calling file finishes loading. * @@ -142,6 +178,14 @@ public static RuntimeList on_scope_end(RuntimeArray args, int ctx) { throw new RuntimeException("on_scope_end requires a code reference, got " + codeRef.type); } + // Prefer the innermost parser-visible lexical scope. This is the + // behavior required by namespace::clean for nested blocks. + Deque> scopes = compileScopes.get(); + if (!scopes.isEmpty()) { + scopes.peek().push(codeRef); + return new RuntimeList(); + } + // Find which file is currently being loaded String currentFile = getCurrentLoadingFile(); diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeArray.java b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeArray.java index 891a0b09b..a31cc5ee9 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeArray.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeArray.java @@ -77,6 +77,12 @@ public void dynamicRestoreState() { } // Restore the original array reference in the global map + // Release references held by the temporary localized array + // before discarding it, matching RuntimeArray's local-scope + // restoration path. + if (localArray != null && localArray != saved.originalArray) { + MortalList.deferDestroyForContainerClear(localArray.elements); + } GlobalVariable.globalArrays.put(saved.fullName, saved.originalArray); // Restore glob aliases diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeHash.java b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeHash.java index 31afabdf5..07f99a626 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeHash.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeHash.java @@ -71,6 +71,14 @@ public void dynamicRestoreState() { } // Restore the original hash reference in the global map + // The localized hash is a fresh container. Drop the values + // it accumulated before making it unreachable; otherwise a + // local %hash (notably Test::Deep's %WrapCache) can retain + // weakly-referenced arguments past the scope that localized + // it. + if (localHash != null && localHash != saved.originalHash) { + MortalList.deferDestroyForContainerClear(localHash.elements.values()); + } GlobalVariable.globalHashes.put(saved.fullName, saved.originalHash); // Restore glob aliases diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeScalar.java b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeScalar.java index 4d867afcf..57a9de8cf 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeScalar.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/GlobalRuntimeScalar.java @@ -126,6 +126,15 @@ public void dynamicRestoreState() { RuntimeScalar localVar = saved.localizedVariable; RuntimeBase displacedBase = null; RuntimeScalar scalarReferenceContents = null; + // A localized package scalar can hold a blessed hash/array + // object (for example Test::Deep's local $CompareCache). + // Release container-owned values before the temporary scalar + // is discarded, just as lexical-scope cleanup does. + if (localVar != null && localVar.value instanceof RuntimeHash localHash) { + MortalList.deferDestroyForContainerClear(localHash.elements.values()); + } else if (localVar != null && localVar.value instanceof RuntimeArray localArray) { + MortalList.deferDestroyForContainerClear(localArray.elements); + } if (localVar != null && localVar.refCountOwned && (localVar.type & RuntimeScalarType.REFERENCE_BIT) != 0 diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/MortalList.java b/src/main/java/org/perlonjava/runtime/runtimetypes/MortalList.java index 3f95dd546..c3cfc1a4e 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/MortalList.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/MortalList.java @@ -302,27 +302,12 @@ public static void deferDecrementIfNotCaptured(RuntimeScalar scalar) { public static void deferDestroyForContainerClear(Iterable elements) { if (!active) return; for (RuntimeScalar scalar : elements) { - if (scalar != null && (scalar.type & RuntimeScalarType.REFERENCE_BIT) != 0 - && scalar.value instanceof RuntimeBase base) { - if (scalar.refCountOwned && base.refCount > 0) { - // Tracked object with owned refCount: defer decrement - scalar.refCountOwned = false; - if (base.refCountTrace) { - base.traceRefCount(0, "MortalList.deferDestroyForContainerClear (queued)"); - } - base.releaseActiveOwner(scalar); - pending.add(base); - } else if (base.blessId != 0 && base.refCount == 0) { - // Never-stored blessed object: bump to 1 so flush triggers DESTROY - if (base.refCountTrace) { - base.traceRefCount(+1, "MortalList.deferDestroyForContainerClear (refCount=1 bump for never-stored)"); - } - base.refCount = 1; - pending.add(base); - } - // Note: WEAKLY_TRACKED (-2) objects are not scheduled here. - // See deferDecrementIfTracked() for rationale. - } + // Use the recursive path so a discarded wrapper/container also + // releases references stored in its fields. Test::Deep's + // temporary comparator is a blessed hash whose `val` field can + // otherwise keep the compared array alive after local %WrapCache + // is restored. + deferDecrementRecursive(scalar); } } diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java index 04bdfadb7..4761bb1ce 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeCode.java @@ -3296,7 +3296,12 @@ public static RuntimeList callerWithSub(RuntimeList args, int ctx, RuntimeScalar ArrayList frameInfo = stackTrace.get(frame); int syntheticOwnSubFramesBefore = countSyntheticOwnSubFramesBefore(stackTrace, frame); int trackedOriginalFrame = Math.max(0, originalFrame - syntheticOwnSubFramesBefore); - int trackedActiveCodeFrame = activeCodeFrameForCaller(trackedOriginalFrame); + // Interpreter stack traces may contain a synthetic entry for the + // current subroutine. That entry is already represented by the + // active-code stack, so do not subtract it when selecting the + // logical caller frame. + int trackedActiveCodeFrame = activeCodeFrameForCaller( + result.firstFrameFromInterpreter() ? originalFrame : trackedOriginalFrame); int trackedArgsFrame = Math.max(0, argsFrame - syntheticOwnSubFramesBefore); String pkg = frameInfo.get(0); res.add(new RuntimeScalar(normalizeCallerPackage(pkg))); // package @@ -3322,7 +3327,7 @@ public static RuntimeList callerWithSub(RuntimeList args, int ctx, RuntimeScalar subName = frameSubName; } - RuntimeCode activeCode = getActiveCodeAt(trackedActiveCodeFrame); + RuntimeCode activeCode = activeCodeAtCallerFrame(trackedActiveCodeFrame); if (subName == null && activeCode != null) { subName = callerSubNameForCode(activeCode); } @@ -3525,7 +3530,10 @@ public static RuntimeList callerWithSub(RuntimeList args, int ctx, RuntimeScalar } } else if (frame >= stackTraceSize) { int trackedOriginalFrame = Math.max(0, originalFrame - countSyntheticOwnSubFramesBefore(stackTrace, stackTrace.size())); - RuntimeCode activeCode = hasExplicitExpr ? getActiveCodeAt(activeCodeFrameForCaller(trackedOriginalFrame)) : null; + RuntimeCode activeCode = hasExplicitExpr + ? activeCodeAtCallerFrame(activeCodeFrameForCaller( + result.firstFrameFromInterpreter() ? originalFrame : trackedOriginalFrame)) + : null; String activeSubName = activeCode != null ? applyAnonNameOverride(callerSubNameForCode(activeCode)) : null; @@ -3601,6 +3609,42 @@ private static int activeCodeFrameForCaller(int originalFrame) { return originalFrame + (WarnDie.isInsideUnhandledDieHandler() ? 1 : 0); } + /** + * Return the active code for a logical Perl caller frame. + * + * The interpreter can enter the same InterpretedCode through both the + * compiler-supplied wrapper and the interpreted body. That leaves + * adjacent duplicate RuntimeCode entries on activeCodeStack, even though + * Perl sees one call frame. Collapse only adjacent duplicates here so + * caller(N) remains expressed in Perl frames without changing the stack + * used by lifetime tracking. + */ + private static RuntimeCode activeCodeAtCallerFrame(int logicalFrame) { + if (logicalFrame < 0) { + return null; + } + RuntimeCode previous = null; + int logicalIndex = 0; + for (RuntimeCode active : activeCodeStack.get()) { + if (active == previous || isCompilerWrapperPair(active, previous)) { + continue; + } + if (logicalIndex++ == logicalFrame) { + return active; + } + previous = active; + } + return null; + } + + private static boolean isCompilerWrapperPair(RuntimeCode left, RuntimeCode right) { + return left != null && right != null + && Objects.equals(left.packageName, right.packageName) + && Objects.equals(left.subName, right.subName) + && (left instanceof org.perlonjava.backend.bytecode.InterpretedCode) + != (right instanceof org.perlonjava.backend.bytecode.InterpretedCode); + } + private static boolean isSyntheticOwnSubFrame(ArrayList frame) { return frame.size() > 4 && "synthetic-own-sub".equals(frame.get(4)); } diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalar.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalar.java index 0904570e2..18dcbe051 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalar.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalar.java @@ -1653,6 +1653,17 @@ private RuntimeScalar setLargeRefCounted(RuntimeScalar value) { } } } + + // A call-frame alias can remain visible to the general reachability + // walk briefly after the callee returns. Once this scalar slot is + // overwritten, package-global reachability is the authoritative + // check for an untracked referent; any counted lexical owner is + // handled by the refCountOwned path above. + if (oldBase != null && !thisWasWeak + && WeakRefRegistry.hasWeakRefsTo(oldBase) + && !ReachabilityWalker.isReachableFromRoots(oldBase, true)) { + WeakRefRegistry.clearWeakRefsTo(oldBase); + } if (undefAssignmentOfDestroyableRef) { if (!DestroyDispatch.isInsideDestroy()) { shouldClearRescuedAfterUndefAssignment = true; diff --git a/src/test/resources/unit/test_deep_memory_regression.t b/src/test/resources/unit/test_deep_memory_regression.t new file mode 100644 index 000000000..b0d559380 --- /dev/null +++ b/src/test/resources/unit/test_deep_memory_regression.t @@ -0,0 +1,30 @@ +use strict; +use warnings; + +use lib 'perl5/cpan/Test-Deep/lib'; +use Scalar::Util qw(weaken); +use Test::Deep qw(eq_deeply); +use Test::More; + +sub left { + my ($ref) = @_; + eq_deeply($ref, []); + return 'left'; +} + +sub right { + my ($ref) = @_; + eq_deeply([], $ref); + return 'right'; +} + +for my $sub (\&left, \&right) { + my $ref = []; + my $weak = $ref; + weaken($weak); + my $side = $sub->($ref); + $ref = 1; + ok(!defined($weak), "$side does not capture the compared reference"); +} + +done_testing; diff --git a/src/test/resources/unit/test_exporter_heavy_caller.t b/src/test/resources/unit/test_exporter_heavy_caller.t new file mode 100644 index 000000000..2c4f0e0f2 --- /dev/null +++ b/src/test/resources/unit/test_exporter_heavy_caller.t @@ -0,0 +1,18 @@ +use strict; +use warnings; + +use Exporter (); + +{ + package ExporterHeavyCallerFixture; + our @ISA = ('Exporter'); + our @EXPORT_OK = ('exported_value'); + + sub exported_value { return 42 } +} + +ExporterHeavyCallerFixture->import('exported_value'); + +print exported_value() == 42 ? "ok 1 - Exporter dispatches through heavy export\n" + : "not ok 1 - Exporter dispatches through heavy export\n"; +print "1..1\n";