Handle PowerShell split/index/join projections and pin remaining approval prompts

IMPLEMENTATION_PLAN.md

@@ -181,6 +181,17 @@ priorities.
       and pins the official package. Netclaw policy PR #2177 is also merged;
       its local gates and all required GitHub checks passed. No new Netclaw
       beta was cut.
+- [x] **Harvest post-deployment PowerShell approval prompts.** Review every
+      shell call from the two sessions created after the deployed
+      `0.27.0-beta.4` daemon started. Fourteen of 28 shell calls prompted.
+      Add one sanitized, generator-owned corpus case for each prompt shape and
+      remove product-specific labels from the older Bash corpus. Thirteen
+      prompt shapes already produce complete command projections and therefore
+      remain downstream policy evidence. The remaining split/index/join
+      projection now has a narrow no-command recognizer with positive and
+      adversarial tests; dynamic and executable variants still fail closed.
+      The PII audit now scans every JSON string and filename with exact,
+      synthetic-only placeholder exceptions.
 
 ## Completed v0.3.0 host integration and release acceptance
 

SPEC.POWERSHELL.md

@@ -85,6 +85,15 @@ Pipeline-produced objects and unsupported expressions remain unknown without
 execution. `while`, `if`, `elseif`, `else`, `do`, `switch`, definitions, and
 arbitrary script evaluation stay outside the supported grammar.
 
+The v0.4 grammar also recognizes one bounded projection expression inside a
+cataloged script-block execution region:
+`($_ -split <static-literal>)[<static-integer-or-range>] -join <static-literal>`
+(with `$PSItem` accepted in place of `$_`). Recognition proves only that the
+authored region contains no nested command. It does not evaluate the
+expression, predict its value, or claim that runtime conversion is
+side-effect-free. Every dynamic operand and every other unsupported expression
+still fails closed.
+
 ---
 
 ## 2. Public API Surface
@@ -678,7 +687,12 @@ expression body: doing so could preserve a stale exact or finite binding for a
 later command. Ordinary property reads and comparison/filter expressions
 remain supported empty bodies. That structure proves only that no authored
 simple command was hidden; it does not prove that a runtime property getter is
-side-effect-free.
+side-effect-free. The same empty-body rule applies to the bounded v0.4
+split/index/join projection from §1. Its delimiters must be non-interpolating
+quoted literals without dollar signs, backticks, or newlines; its selector must
+be one static integer or integer range; and the body may contain no statement
+separator, member call, assignment, subexpression, splat, or additional
+operator.
 
 When a balanced increment/decrement expression statement is separated from
 otherwise parsed siblings by a parser-owned semicolon or newline boundary, the
@@ -2063,8 +2077,9 @@ v0.2.0 ships when **all** of these hold:
 - `function`/`filter`/`class`/`enum` definitions,
   `param()`/`begin`/`process`/`end` blocks, `trap`, and `DATA`.
 - `.ps1` script-file parsing.
-- General PowerShell expression evaluation, `$_` / `$PSItem` semantics, .NET
-  method calls, object-to-string prediction, and runtime pipeline evaluation.
+- General PowerShell expression evaluation, `$_` / `$PSItem` semantics beyond
+  the bounded no-command structural recognition in §1, .NET method calls,
+  object-to-string prediction, and runtime pipeline evaluation.
 - Desired State Configuration (DSC).
 - A real `Push-Location` / `Pop-Location` directory-stack model (§9).
 - Per-element path extraction from a comma-separated array

src/ShellSyntaxTree/Internal/Pwsh/Parsing/PwshStructuralCoordinator.cs

@@ -6,6 +6,7 @@
 using System;
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
+using System.Text.RegularExpressions;
 using ShellSyntaxTree.Internal.Parsing;
 using ShellSyntaxTree.Internal.Pwsh.Lexing;
 using ShellSyntaxTree.Internal.Resolving;
@@ -1287,18 +1288,21 @@ private bool TryParseScriptBlockBody(
                 return false;
             }
 
+            var isSupportedProjectionExpression =
+                significant.Count > 0 && IsSupportedProjectionExpression(source);
             if (significant.Count > 0 &&
-                IsUnsupportedSubstitutionBody(source, significant) &&
+                (isSupportedProjectionExpression || IsUnsupportedSubstitutionBody(source, significant)) &&
                 !ContainsIncrementOrDecrementMutation(significant))
             {
                 foreach (var expressionToken in significant)
                 {
-                    if (HasPowerShellSubexpression(expressionToken)
-                        || expressionToken.Kind is PwshTokenKind.Subexpression
-                            or PwshTokenKind.ScriptBlock
-                            or PwshTokenKind.Splat
-                        || expressionToken.IsStatementSeparator
-                        || expressionToken.Kind == PwshTokenKind.Operator)
+                    if (!isSupportedProjectionExpression &&
+                        (HasPowerShellSubexpression(expressionToken)
+                            || expressionToken.Kind is PwshTokenKind.Subexpression
+                                or PwshTokenKind.ScriptBlock
+                                or PwshTokenKind.Splat
+                            || expressionToken.IsStatementSeparator
+                            || expressionToken.Kind == PwshTokenKind.Operator))
                     {
                         body = new ShellBlockSyntax();
                         error = "unsupported execution-bearing PowerShell script-block expression";
@@ -1625,6 +1629,20 @@ private bool TryCollectCommandSubstitutions(
             return true;
         }
 
+        private static readonly Regex SupportedProjectionExpressionPattern =
+            new(
+                @"^[ \t]*\([ \t]*(?:\$_|\$PSItem)[ \t]+-split[ \t]+" +
+                @"(?:'[^'`$\r\n]*'|\x22[^\x22`$\r\n]*\x22)[ \t]*\)" +
+                @"[ \t]*\[[ \t]*[+-]?\d+(?:[ \t]*\.\.[ \t]*[+-]?\d+)?[ \t]*\]" +
+                @"[ \t]+-join[ \t]+(?:'[^'`$\r\n]*'|\x22[^\x22`$\r\n]*\x22)[ \t]*$",
+                RegexOptions.CultureInvariant | RegexOptions.IgnoreCase,
+                TimeSpan.FromMilliseconds(50));
+
+        // This is structural recognition only: it publishes no value and
+        // proves only that this exact script-block body hides no authored command.
+        private static bool IsSupportedProjectionExpression(string source) =>
+            SupportedProjectionExpressionPattern.IsMatch(source);
+
         private static bool IsUnsupportedSubstitutionBody(
             string source,
             IReadOnlyList<PwshToken> tokens)

tests/ShellSyntaxTree.Tests/Corpus/PiiAuditTests.cs

@@ -16,19 +16,16 @@ namespace ShellSyntaxTree.Tests.Corpus;
 
 /// <summary>
 /// PII audit gate per SPEC §14 / SPEC.POWERSHELL.md §14. Scans every JSON
-/// entry under the executable corpus and the pre-implementation design corpus
+/// string under the executable corpus and the pre-implementation design corpus
 /// for the forbidden patterns listed in the sanitization table. The audit
 /// reads from the build-output copies so CI runs against the same bytes a
 /// developer's local <c>dotnet test</c> would.
 /// </summary>
 /// <remarks>
 /// Scanning policy:
 /// <list type="bullet">
-///   <item>Only <c>input</c>, <c>notes</c>, and <c>raw</c> string fields
-///         are scanned. Synthetic resolved paths and the
-///         <c>&lt;dynamic-cwd&gt;</c> sentinel surface in other fields and
-///         would generate noise (e.g. <c>/work/foo</c> from
-///         WorkingDirectory pinning).</item>
+///   <item>Every JSON string is scanned, including expected projections and
+///         metadata, so sensitive text cannot hide in an unscanned field.</item>
 ///   <item>A small allowlist of generic placeholder usernames is honored:
 ///         <c>user, test, foo, dev, runner, gh-actions, ci</c>. Anything
 ///         else under <c>/home/</c> or <c>/Users/</c> trips the audit.</item>
@@ -70,25 +67,25 @@ public class PiiAuditTests
         new(@"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", RegexOptions.Compiled);
 
     private static readonly Regex LongKeyPattern =
-        new(@"[A-Za-z0-9]{32,}", RegexOptions.Compiled);
+        new(@"[A-Za-z0-9]{20,}", RegexOptions.Compiled);
 
     // The home/users patterns capture the username segment; we then allow-list it.
     private static readonly Regex HomePattern =
-        new(@"/home/([a-zA-Z0-9_.-]+)/", RegexOptions.Compiled);
+        new(@"/home/([a-zA-Z0-9_.-]+)(?:/|$)", RegexOptions.Compiled);
 
     private static readonly Regex UsersPattern =
-        new(@"/Users/([a-zA-Z0-9_.-]+)/", RegexOptions.Compiled);
+        new(@"/Users/([a-zA-Z0-9_.-]+)(?:/|$)", RegexOptions.Compiled);
 
     // Repository-path pattern: /home/<user>/repositories/<org>/<repo>/... where
     // <repo> isn't one of the public-corpus placeholders.
     private static readonly Regex RepoPathPattern =
-        new(@"/home/[^/]+/repositories/[^/]+/([a-zA-Z0-9_.-]+)/", RegexOptions.Compiled);
+        new(@"/home/[^/]+/repositories/[^/]+/([a-zA-Z0-9_.-]+)(?:/|$)", RegexOptions.Compiled);
 
     // SPEC.POWERSHELL.md §14: a concrete C:\Users\<username>\ path (mixed
     // slashes allowed). A literal $env:USERNAME / $env:USERPROFILE reference
     // is not PII and is not matched here.
     private static readonly Regex WindowsUserPattern =
-        new(@"[A-Za-z]:[\\/]Users[\\/]([A-Za-z0-9_.-]+)[\\/]", RegexOptions.Compiled);
+        new(@"[A-Za-z]:[\\/]Users[\\/]([A-Za-z0-9_.-]+)(?:[\\/]|$)", RegexOptions.Compiled);
 
     // SPEC.POWERSHELL.md §14: a UNC \\<hostname>\share path.
     private static readonly Regex UncHostPattern =
@@ -97,6 +94,31 @@ public class PiiAuditTests
     private static readonly HashSet<string> AllowedUncHosts =
         new(StringComparer.Ordinal) { "internal-host.example" };
 
+    private static readonly HashSet<string> AllowedSyntheticHomeSegments =
+        new(StringComparer.Ordinal)
+        {
+            "test.json",
+            "test.txt",
+            "user.Length",
+            "user.json",
+            "user.txt",
+        };
+
+    private static bool IsAllowedPlaceholderPathSegment(
+        string segment,
+        HashSet<string> allowlist)
+    {
+        if (allowlist.Contains(segment))
+        {
+            return true;
+        }
+
+        // A few resolver projections append a known synthetic suffix to the
+        // placeholder home. Keep this list exact so a dotted real username
+        // cannot pass merely because its prefix resembles a placeholder.
+        return AllowedSyntheticHomeSegments.Contains(segment);
+    }
+
     [Fact]
     public void Corpus_contains_no_pii_per_spec_section_14()
     {
@@ -115,6 +137,9 @@ public void Corpus_contains_no_pii_per_spec_section_14()
             {
                 var name = Path.GetRelativePath(AppContext.BaseDirectory, file)
                     .Replace('\\', '/');
+                // Filenames are part of the committed corpus surface too;
+                // scan their slug so identifiers cannot be hidden in paths.
+                ScanString(Path.GetFileNameWithoutExtension(file), name, "<filename>", hits);
                 try
                 {
                     using var doc = JsonDocument.Parse(File.ReadAllText(file));
@@ -160,10 +185,7 @@ private static void Walk(JsonElement element, string fileName, string fieldPath,
                 break;
 
             case JsonValueKind.String:
-                if (ShouldScan(fieldPath))
-                {
-                    ScanString(element.GetString() ?? string.Empty, fileName, fieldPath, hits);
-                }
+                ScanString(element.GetString() ?? string.Empty, fileName, fieldPath, hits);
                 break;
         }
     }
@@ -188,22 +210,6 @@ private static bool IsLowEntropyRun(string token)
         return true;
     }
 
-    /// <summary>
-    /// Decide whether a JSON string at <paramref name="fieldPath"/> is in
-    /// scope for the audit. SPEC §14: scan <c>input</c>, <c>notes</c>,
-    /// and any <c>raw</c> nested under args. Skip synthetic fields like
-    /// <c>resolved</c>, <c>target</c>, etc. — those carry parser-produced
-    /// paths (e.g. <c>/home/test/file</c>) that we explicitly want to allow.
-    /// </summary>
-    private static bool ShouldScan(string fieldPath)
-    {
-        if (fieldPath == "input") return true;
-        if (fieldPath == "notes") return true;
-        if (fieldPath.EndsWith(".command", StringComparison.Ordinal)) return true;
-        if (fieldPath.EndsWith(".raw", StringComparison.Ordinal)) return true;
-        return false;
-    }
-
     private static void ScanString(string value, string fileName, string fieldPath, List<string> hits)
     {
         if (string.IsNullOrEmpty(value))
@@ -239,7 +245,9 @@ private static void ScanString(string value, string fileName, string fieldPath,
         {
             foreach (Match m in LongKeyPattern.Matches(value))
             {
-                if (IsLowEntropyRun(m.Value))
+                if (IsLowEntropyRun(m.Value)
+                    || !m.Value.Any(char.IsDigit)
+                    || !m.Value.Any(char.IsLetter))
                 {
                     continue;
                 }
@@ -252,7 +260,7 @@ private static void ScanString(string value, string fileName, string fieldPath,
         foreach (Match m in HomePattern.Matches(value))
         {
             var user = m.Groups[1].Value;
-            if (!AllowedHomeUsernames.Contains(user))
+            if (!IsAllowedPlaceholderPathSegment(user, AllowedHomeUsernames))
             {
                 hits.Add($"{fileName} ({fieldPath}): /home/{user}/ — not in allowed-placeholder list (SPEC §14)");
             }
@@ -262,7 +270,7 @@ private static void ScanString(string value, string fileName, string fieldPath,
         foreach (Match m in UsersPattern.Matches(value))
         {
             var user = m.Groups[1].Value;
-            if (!AllowedUsersUsernames.Contains(user))
+            if (!IsAllowedPlaceholderPathSegment(user, AllowedUsersUsernames))
             {
                 hits.Add($"{fileName} ({fieldPath}): /Users/{user}/ — not in allowed-placeholder list (SPEC §14)");
             }
@@ -282,7 +290,7 @@ private static void ScanString(string value, string fileName, string fieldPath,
         foreach (Match m in WindowsUserPattern.Matches(value))
         {
             var user = m.Groups[1].Value;
-            if (!AllowedUsersUsernames.Contains(user))
+            if (!IsAllowedPlaceholderPathSegment(user, AllowedUsersUsernames))
             {
                 hits.Add($"{fileName} ({fieldPath}): Windows user path 'Users\\{user}\\' — not in allowed-placeholder list (SPEC.POWERSHELL.md §14)");
             }

tests/ShellSyntaxTree.Tests/Corpus/bash/130_netclaw_repro_leading_comment_pipeline.json → tests/ShellSyntaxTree.Tests/Corpus/bash/130_consumer_repro_leading_comment_pipeline.json

@@ -1,5 +1,5 @@
 {
-  "name": "Netclaw repro: leading comment + git worktree list | awk | tr | sort pipeline",
+  "name": "Consumer repro: leading comment + git worktree list | awk | tr | sort pipeline",
   "input": "# Extract all unique branch names from worktrees\ngit -C /home/user/repos/sample-repo worktree list | awk '{print $NF}' | tr -d '[]' | sort -u",
   "expected": {
     "isUnparseable": false,
@@ -48,5 +48,5 @@
       }
     ]
   },
-  "notes": "Issue #25 — original Netclaw repro, paths sanitized per SPEC §14. Without the v0.1.3 comment-skip fix, Clause 0's verb parsed as `[#, Extract]`. Updated for issue #27 / v0.1.4-alpha: the greedy heuristic now walks past the consumed `-C /repo` flag-value pair and captures `worktree list` as part of the verb chain instead of leaving `list` as a stranded positional arg."
+  "notes": "Issue #25 consumer repro, paths sanitized per SPEC §14. Without the v0.1.3 comment-skip fix, Clause 0's verb parsed as `[#, Extract]`. Updated for issue #27 / v0.1.4-alpha: the greedy heuristic now walks past the consumed `-C /repo` flag-value pair and captures `worktree list` as part of the verb chain instead of leaving `list` as a stranded positional arg."
 }

tests/ShellSyntaxTree.Tests/Corpus/bash/131_netclaw_repro_compound_with_comment.json → tests/ShellSyntaxTree.Tests/Corpus/bash/131_consumer_repro_compound_with_comment.json

@@ -1,5 +1,5 @@
 {
-  "name": "Netclaw repro: leading comment + curl|jq pipeline with ||-fallback",
+  "name": "Consumer repro: leading comment + curl|jq pipeline with ||-fallback",
   "input": "# Get open PRs from the upstream repo\ncurl -s \"https://github.com/ghapi/repos/sample-org/sample-repo/pulls?state=open\" | jq -r '.[] | \"PR\"' 2>/dev/null || echo \"API failed, trying alternative...\"",
   "expected": {
     "isUnparseable": false,
@@ -40,5 +40,5 @@
       }
     ]
   },
-  "notes": "Issue #25 follow-up: leading comment + `||`-fallback that surfaced the approval-state desync cascade (verb-chain extracted as `# Get` at persistence time → cache miss at retry-authorization → tool fails after user clicked Approve). The quoted URL wildcard marker is literal. Paths and org names sanitized per SPEC §14."
+  "notes": "Issue #25 follow-up: leading comment + `||`-fallback that surfaced an approval-state desync cascade (verb-chain extracted as `# Get` at persistence time → cache miss at retry-authorization → tool fails after user clicked Approve). The quoted URL wildcard marker is literal. Paths and org names sanitized per SPEC §14."
 }

tests/ShellSyntaxTree.Tests/Corpus/bash/132_supportcli_ticket_list_status_open.json

@@ -0,0 +1,21 @@
+{
+  "name": "Greedy verb chain: supportcli ticket list --status open (synthetic CLI)",
+  "input": "supportcli ticket list --status open",
+  "expected": {
+    "isUnparseable": false,
+    "clauses": [
+      {
+        "operator": "None",
+        "verb": ["supportcli", "ticket", "list"],
+        "args": [
+          { "raw": "--status", "kind": "Literal", "isPath": false, "isFlag": true },
+          { "raw": "open", "kind": "Literal", "isPath": false, "resolved": "__NULL__" }
+        ],
+        "redirects": [],
+        "isSubshell": false,
+        "isCommandStringWrapped": false
+      }
+    ]
+  },
+  "notes": "Issue #27 headline: unknown/private CLIs used to truncate to a 1-token verb chain. The greedy heuristic walks verb-like Word tokens (lowercase[a-z0-9._-]) and stops at the first flag, so the canonical subcommand stack `supportcli ticket list` extracts fully without requiring a curated table entry."
+}