diff --git a/packages/loopover-miner/docs/cross-repo-evaluation.md b/packages/loopover-miner/docs/cross-repo-evaluation.md index cc36444b74..675e79a129 100644 --- a/packages/loopover-miner/docs/cross-repo-evaluation.md +++ b/packages/loopover-miner/docs/cross-repo-evaluation.md @@ -61,18 +61,52 @@ fleet run-manifest). - `--manifest path/to/manifest.json` — alternate benchmark set (e.g. a fixture manifest in tests) - `--require-majority` — exit `1` unless a strict majority of repos pass (for CI-style gating) +## Full-execution mode (#7634) + +`--full-execution` extends the readiness check into the real **discover → plan → code → test** loop, still +**dry-run only**: no live PR submission, no forge API calls, no write access to the third-party repos. Each repo +that passes readiness is copied into a **discardable scratch workspace** (the benchmark clone is never mutated); +the coding agent runs against a synthetic benchmark issue inside that copy, and the repo's own inferred build and +test commands are executed there. The scratch tree is removed afterward in every outcome — execute-locally-and-discard. + +```bash +node packages/loopover-miner/scripts/cross-repo-evaluation.mjs --full-execution +``` + +Prerequisites beyond the readiness mode: a configured coding-agent provider (`MINER_CODING_AGENT_PROVIDER`, see +[`docs/coding-agent-driver.md`](coding-agent-driver.md)). Without one, each repo reports `agent_run_failed` +("No runnable coding-agent driver") rather than silently skipping. + +Execution failures extend — never replace — the readiness taxonomy, in the same report format: + +| Category | Meaning | +| --- | --- | +| `agent_run_failed` | No runnable driver was configured, or the coding agent errored / did not finish | +| `noop_diff` | The agent reported success but the generated diff changed no files | +| `build_failed` | A diff was generated but the stack's inferred build command failed (didn't compile) | +| `test_failed` | The build was fine but the repo's own test suite failed against the diff | + +A repo that passes readiness but has **no inferred test command** fails with the existing `execution_gap` +category — a generated diff that nothing can validate is not a pass. Per-command wall-clock is capped +(`DEFAULT_CROSS_REPO_EXECUTION_COMMAND_TIMEOUT_MS`, 10 minutes each for build and test); the agent's turn budget +defaults to `DEFAULT_CROSS_REPO_EXECUTION_MAX_TURNS` (24). + ## Library API Pure functions live in [`lib/cross-repo-evaluation.js`](../lib/cross-repo-evaluation.js): - `parseCrossRepoEvaluationManifest(content)` - `evaluateRepoReadiness(entry, options)` — inject `existsSync`, `detectRepoStack`, etc. for unit tests -- `runCrossRepoEvaluation(parsed, options)` +- `evaluateRepoExecution(entry, options)` — full-execution mode for one repo (#7634); inject `driver`, + `prepareExecutionWorkspace`, `runCommand`, etc. for unit tests +- `runCrossRepoEvaluation(parsed, options)` / `runCrossRepoFullExecution(parsed, options)` - `summarizeCrossRepoEvaluation(results)` - `formatCrossRepoEvaluationReport(results, summary)` ## Wiring -This harness is **readiness-only**: it does not run the coding agent, open PRs, or call forge APIs. A green report -means the miner’s repo-agnostic stack-detection and coding-task-spec path is prepared for the benchmark repo; a live -attempt still needs credentials, governor policy, and queue state as documented in [`DEPLOYMENT.md`](../DEPLOYMENT.md). +The default mode is **readiness-only**: it does not run the coding agent, open PRs, or call forge APIs. A green +readiness report means the miner’s repo-agnostic stack-detection and coding-task-spec path is prepared for the +benchmark repo. `--full-execution` (#7634) additionally runs the agent and the repo's own test suite locally in a +discarded scratch copy — it still never opens a PR or touches any forge API; a live attempt still needs +credentials, governor policy, and queue state as documented in [`DEPLOYMENT.md`](../DEPLOYMENT.md). diff --git a/packages/loopover-miner/lib/cross-repo-evaluation.d.ts b/packages/loopover-miner/lib/cross-repo-evaluation.d.ts index af8e928558..7e843e635a 100644 --- a/packages/loopover-miner/lib/cross-repo-evaluation.d.ts +++ b/packages/loopover-miner/lib/cross-repo-evaluation.d.ts @@ -65,6 +65,7 @@ type EvaluateRepoReadinessOptions = { ready: boolean; verdict?: string; instructions?: string; + acceptanceCriteriaPath?: string; }; }; /** Canonical `owner/repo` with exactly one slash and safe segments; anything else → null. */ @@ -100,4 +101,87 @@ export declare function summarizeCrossRepoEvaluation(results: CrossRepoEvaluatio * Human-readable pass/fail report for one evaluation run (#4788). */ export declare function formatCrossRepoEvaluationReport(results: CrossRepoEvaluationResult[], summary?: CrossRepoEvaluationSummary): string; +/** Execution-specific failure taxonomy (#7634), extending — not replacing — CROSS_REPO_FAILURE_CATEGORY. */ +export declare const CROSS_REPO_EXECUTION_FAILURE_CATEGORY: Readonly<{ + AGENT_RUN: "agent_run_failed"; + NOOP_DIFF: "noop_diff"; + BUILD: "build_failed"; + TEST: "test_failed"; +}>; +/** A benchmark attempt works a small synthetic issue, so a modest turn cap keeps dry-runs bounded without + * starving a real agent; callers tune via options.maxTurns. */ +export declare const DEFAULT_CROSS_REPO_EXECUTION_MAX_TURNS: number; +/** Per-command (build, then test) wall-clock cap — generous enough for a cold dependency install on the larger + * benchmark repos, small enough that a hung suite cannot wedge the whole run. */ +export declare const DEFAULT_CROSS_REPO_EXECUTION_COMMAND_TIMEOUT_MS: number; +export type CrossRepoExecutionCommandResult = { + code: number | null; + stdout: string; + stderr: string; + timedOut: boolean; +}; +export type CrossRepoExecutionRunCommandFn = (command: string, options: { + cwd: string; + timeoutMs: number; +}) => Promise; +export type CrossRepoExecutionWorkspace = { + path: string; + cleanup: () => void; +}; +/** Structural mirror of the engine's CodingAgentDriver contract — kept local so this module only loads the real + * driver construction (and its engine dependency) lazily, on the one path that actually runs an agent. */ +export type CrossRepoExecutionDriver = { + run(task: { + attemptId: string; + workingDirectory: string; + acceptanceCriteriaPath: string; + instructions: string; + maxTurns: number; + }): Promise<{ + ok: boolean; + changedFiles: readonly string[]; + summary: string; + error?: string | undefined; + }>; +}; +export type CrossRepoExecutionDetails = { + attempted: boolean; + changedFileCount: number | null; + buildRan: boolean; + testRan: boolean; +}; +export type CrossRepoExecutionEvaluationResult = CrossRepoEvaluationResult & { + execution: CrossRepoExecutionDetails | null; +}; +export type EvaluateRepoExecutionOptions = EvaluateRepoReadinessOptions & { + driver?: CrossRepoExecutionDriver; + prepareExecutionWorkspace?: (repoPath: string) => CrossRepoExecutionWorkspace; + runCommand?: CrossRepoExecutionRunCommandFn; + maxTurns?: number; + commandTimeoutMs?: number; +}; +/** Copy the benchmark clone into a discardable temp tree — the agent and the repo's test suite only ever touch + * the copy, so the clone stays pristine and cleanup is a single recursive remove. */ +export declare function defaultPrepareExecutionWorkspace(repoPath: string): CrossRepoExecutionWorkspace; +/** Command runner for the stack's inferred build/test commands. detectRepoStack only ever emits simple + * `tool subcommand` forms ("npm test", "cargo build", "npm run build"), so the command is tokenized on + * whitespace and exec'd DIRECTLY -- deliberately no `shell: true`, so nothing in a benchmark repo's manifest + * can smuggle shell metacharacters into an interpreted shell line. Mirrors coding-agent-construction's + * createRealCliSubprocessSpawn otherwise: capture both streams and RESOLVE (never reject) on timeout or spawn + * error, so partial output stays diagnosable. Promise resolution is idempotent, so a `close` firing after the + * timeout already resolved needs no guard. */ +export declare function createDefaultCrossRepoExecutionRunCommand(): CrossRepoExecutionRunCommandFn; +/** + * Run the full discover -> plan -> code -> test loop for one benchmark repo, dry-run (#7634). Readiness gates + * first (its failures pass through unchanged); execution then happens entirely inside a scratch copy that is + * discarded in every outcome. + */ +export declare function evaluateRepoExecution(entry: CrossRepoEvaluationManifestRepo, options?: EvaluateRepoExecutionOptions): Promise; +/** + * Run full-execution mode across every repo in a parsed manifest (#7634), sequentially — agent runs and test + * suites are heavyweight, so no parallel fan-out. + */ +export declare function runCrossRepoFullExecution(parsed: ParsedCrossRepoEvaluationManifest, options?: { + repoFilter?: string; +} & EvaluateRepoExecutionOptions): Promise; export {}; diff --git a/packages/loopover-miner/lib/cross-repo-evaluation.js b/packages/loopover-miner/lib/cross-repo-evaluation.js index 995ae19eba..0327062a89 100644 --- a/packages/loopover-miner/lib/cross-repo-evaluation.js +++ b/packages/loopover-miner/lib/cross-repo-evaluation.js @@ -3,7 +3,12 @@ // evaluated through the same stack-detection + coding-task-spec path a real attempt uses (detectRepoStack, // resolveMinerGoalSpec, buildCodingTaskSpec) and failures are categorized as stack-detection gaps, execution // readiness gaps, leaked loopover assumptions in agent instructions, clone/setup problems, or other. -import { existsSync } from "node:fs"; +import { spawn as nodeSpawn } from "node:child_process"; +import { cpSync, existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +// Already a transitive dependency via coding-task-spec.js's own engine imports -- this adds no load weight. +import { ACCEPTANCE_CRITERIA_FILENAME } from "@loopover/engine"; import { buildCodingTaskSpec } from "./coding-task-spec.js"; import { resolveMinerGoalSpec } from "./miner-goal-spec.js"; import { isValidRepoSegment, resolveRepoCloneDir } from "./repo-clone.js"; @@ -180,6 +185,10 @@ export function scanPositiveLoopoverAssumptions(text) { } return findings; } +/** One shared thrown-value formatter so every catch in both harness modes carries the same two branches. */ +function describeError(error) { + return error instanceof Error ? error.message : String(error); +} function buildFailure(repoFullName, category, reason, extra = {}) { return { repoFullName, @@ -216,6 +225,24 @@ function resolveEvaluationRepoPath(entry, options = {}) { function defaultClaimLedger(repoFullName) { return { listClaims: () => [] }; } +/** The synthetic-issue spec input both harness modes hand to buildCodingTaskSpec — readiness composes it against + * the benchmark clone, full-execution mode against the scratch workspace copy (#7634), so the acceptance-criteria + * file buildCodingTaskSpec writes lands inside whichever tree the caller is actually working in. */ +function buildHarnessSpecInput(repoFullName, workingDirectory, detectImpl) { + return { + repoFullName, + issue: { + number: 1, + title: "Cross-repo evaluation harness smoke issue", + body: "Synthetic issue used only by the cross-repo evaluation harness.", + labels: ["bug"], + }, + context: { issues: [{ number: 1 }], pullRequests: [] }, + claimLedger: defaultClaimLedger(repoFullName), + workingDirectory, + detectRepoStack: detectImpl, + }; +} /** * Evaluate one benchmark repo's miner readiness without running a live coding agent (#4788). */ @@ -244,23 +271,10 @@ export function evaluateRepoReadiness(entry, options = {}) { } let specResult; try { - specResult = buildSpecImpl({ - repoFullName, - issue: { - number: 1, - title: "Cross-repo evaluation harness smoke issue", - body: "Synthetic issue used only by the cross-repo evaluation harness.", - labels: ["bug"], - }, - context: { issues: [{ number: 1 }], pullRequests: [] }, - claimLedger: defaultClaimLedger(repoFullName), - workingDirectory: repoPath, - detectRepoStack: detectImpl, - }); + specResult = buildSpecImpl(buildHarnessSpecInput(repoFullName, repoPath, detectImpl)); } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return buildFailure(repoFullName, CROSS_REPO_FAILURE_CATEGORY.OTHER, message, { + return buildFailure(repoFullName, CROSS_REPO_FAILURE_CATEGORY.OTHER, describeError(error), { stackDetected: true, usedDefaultGoalSpec, stack, @@ -343,4 +357,205 @@ export function formatCrossRepoEvaluationReport(results, summary = summarizeCros } return lines.join("\n"); } -//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY3Jvc3MtcmVwby1ldmFsdWF0aW9uLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiY3Jvc3MtcmVwby1ldmFsdWF0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGlIQUFpSDtBQUNqSCw4R0FBOEc7QUFDOUcsMkdBQTJHO0FBQzNHLDZHQUE2RztBQUM3RyxxR0FBcUc7QUFFckcsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLFNBQVMsQ0FBQztBQUVyQyxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUM1RCxPQUFPLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUM1RCxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUMxRSxPQUFPLEVBQUUsZUFBZSxFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFHdkQsNkRBQTZEO0FBQzdELE1BQU0sQ0FBQyxNQUFNLDJCQUEyQixHQU1uQyxNQUFNLENBQUMsTUFBTSxDQUFDO0lBQ2pCLGVBQWUsRUFBRSxxQkFBcUI7SUFDdEMsU0FBUyxFQUFFLGVBQWU7SUFDMUIsb0JBQW9CLEVBQUUscUJBQXFCO0lBQzNDLFdBQVcsRUFBRSxhQUFhO0lBQzFCLEtBQUssRUFBRSxPQUFPO0NBQ2YsQ0FBQyxDQUFDO0FBRUg7bUdBQ21HO0FBQ25HLE1BQU0sQ0FBQyxNQUFNLG9DQUFvQyxHQUFtRCxNQUFNLENBQUMsTUFBTSxDQUFDO0lBQ2hILEVBQUUsRUFBRSxFQUFFLGdCQUFnQixFQUFFLE9BQU8sRUFBRSxrQkFBa0IsRUFBRTtJQUNyRCxFQUFFLEVBQUUsRUFBRSxlQUFlLEVBQUUsT0FBTyxFQUFFLGlCQUFpQixFQUFFO0lBQ25ELEVBQUUsRUFBRSxFQUFFLGlCQUFpQixFQUFFLE9BQU8sRUFBRSxxQ0FBcUMsRUFBRTtJQUN6RSxFQUFFLEVBQUUsRUFBRSxlQUFlLEVBQUUsT0FBTyxFQUFFLGdCQUFnQixFQUFFO0NBQ25ELENBQUMsQ0FBQztBQUVILE1BQU0sQ0FBQyxNQUFNLHlDQUF5QyxHQUFXLHFDQUFxQyxDQUFDO0FBQ3ZHLE1BQU0sQ0FBQyxNQUFNLDZCQUE2QixHQUFXLE1BQU0sQ0FBQztBQUM1RCxNQUFNLENBQUMsTUFBTSw2QkFBNkIsR0FBVyxHQUFHLENBQUM7QUFpRHpELGlIQUFpSDtBQUNqSCxnSEFBZ0g7QUFDaEgsbUhBQW1IO0FBQ25ILCtHQUErRztBQUMvRywwQkFBMEI7QUFDMUIsU0FBUyxjQUFjLENBQUMsS0FBYTtJQUNuQyxJQUFJLEtBQUssR0FBRyxDQUFDLENBQUM7SUFDZCxLQUFLLE1BQU0sSUFBSSxJQUFJLEtBQUssRUFBRSxDQUFDO1FBQ3pCLE1BQU0sU0FBUyxHQUFHLElBQUksQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFFLENBQUM7UUFDdkMsSUFBSSxTQUFTLElBQUksSUFBSTtZQUFFLEtBQUssSUFBSSxDQUFDLENBQUM7YUFDN0IsSUFBSSxTQUFTLElBQUksS0FBSztZQUFFLEtBQUssSUFBSSxDQUFDLENBQUM7YUFDbkMsSUFBSSxTQUFTLElBQUksTUFBTTtZQUFFLEtBQUssSUFBSSxDQUFDLENBQUM7O1lBQ3BDLEtBQUssSUFBSSxDQUFDLENBQUM7SUFDbEIsQ0FBQztJQUNELE9BQU8sS0FBSyxDQUFDO0FBQ2YsQ0FBQztBQUVELFNBQVMsa0JBQWtCLENBQUMsV0FBcUIsRUFBRTtJQUNqRCxPQUFPLEVBQUUsT0FBTyxFQUFFLEtBQUssRUFBRSxRQUFRLEVBQUUsRUFBRSxLQUFLLEVBQUUsRUFBRSxFQUFFLEVBQUUsUUFBUSxFQUFFLENBQUM7QUFDL0QsQ0FBQztBQUVELDZGQUE2RjtBQUM3RixNQUFNLFVBQVUsMEJBQTBCLENBQUMsS0FBYztJQUN2RCxJQUFJLE9BQU8sS0FBSyxLQUFLLFFBQVE7UUFBRSxPQUFPLElBQUksQ0FBQztJQUMzQyxNQUFNLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxLQUFLLENBQUMsR0FBRyxLQUFLLENBQUMsSUFBSSxFQUFFLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQ3JELElBQUksQ0FBQyxLQUFLLElBQUksQ0FBQyxJQUFJLElBQUksS0FBSyxLQUFLLFNBQVM7UUFBRSxPQUFPLElBQUksQ0FBQztJQUN4RCxJQUFJLENBQUMsa0JBQWtCLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxJQUFJLENBQUM7UUFBRSxPQUFPLElBQUksQ0FBQztJQUN6RSxPQUFPLEdBQUcsS0FBSyxJQUFJLElBQUksRUFBRSxDQUFDO0FBQzVCLENBQUM7QUFFRCxTQUFTLGdCQUFnQixDQUFDLEtBQWMsRUFBRSxLQUFhLEVBQUUsUUFBaUIsRUFBRSxRQUFrQjtJQUM1RixJQUFJLEtBQUssS0FBSyxTQUFTLElBQUksS0FBSyxLQUFLLElBQUk7UUFBRSxPQUFPLFFBQVEsQ0FBQztJQUMzRCxJQUFJLE9BQU8sS0FBSyxLQUFLLFNBQVM7UUFBRSxPQUFPLEtBQUssQ0FBQztJQUM3QyxRQUFRLENBQUMsSUFBSSxDQUFDLHNDQUFzQyxLQUFLLHdDQUF3QyxRQUFRLEdBQUcsQ0FBQyxDQUFDO0lBQzlHLE9BQU8sUUFBUSxDQUFDO0FBQ2xCLENBQUM7QUFFRCxTQUFTLHVCQUF1QixDQUFDLEtBQWMsRUFBRSxLQUFhLEVBQUUsUUFBa0I7SUFDaEYsSUFBSSxLQUFLLEtBQUssU0FBUyxJQUFJLEtBQUssS0FBSyxJQUFJO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDdkQsSUFBSSxPQUFPLEtBQUssS0FBSyxRQUFRLEVBQUUsQ0FBQztRQUM5QixRQUFRLENBQUMsSUFBSSxDQUFDLHNDQUFzQyxLQUFLLHlDQUF5QyxDQUFDLENBQUM7UUFDcEcsT0FBTyxJQUFJLENBQUM7SUFDZCxDQUFDO0lBQ0QsTUFBTSxPQUFPLEdBQUcsS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDO0lBQzdCLE9BQU8sT0FBTyxJQUFJLElBQUksQ0FBQztBQUN6QixDQUFDO0FBRUQsU0FBUyxpQkFBaUIsQ0FBQyxLQUFjLEVBQUUsUUFBa0I7SUFDM0QsSUFBSSxLQUFLLEtBQUssU0FBUyxJQUFJLEtBQUssS0FBSyxJQUFJO1FBQUUsT0FBTyxFQUFFLENBQUM7SUFDckQsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztRQUMxQixRQUFRLENBQUMsSUFBSSxDQUFDLHdFQUF3RSxPQUFPLEtBQUssU0FBUyxDQUFDLENBQUM7UUFDN0csT0FBTyxFQUFFLENBQUM7SUFDWixDQUFDO0lBQ0QsTUFBTSxNQUFNLEdBQXNDLEVBQUUsQ0FBQztJQUNyRCxNQUFNLElBQUksR0FBRyxJQUFJLEdBQUcsRUFBVSxDQUFDO0lBQy9CLEtBQUssTUFBTSxDQUFDLEtBQUssRUFBRSxLQUFLLENBQUMsSUFBSSxLQUFLLENBQUMsT0FBTyxFQUFFLEVBQUUsQ0FBQztRQUM3QyxJQUFJLEtBQUssSUFBSSw2QkFBNkIsRUFBRSxDQUFDO1lBQzNDLFFBQVEsQ0FBQyxJQUFJLENBQ1gsc0RBQXNELDZCQUE2QixrQ0FBa0MsQ0FDdEgsQ0FBQztZQUNGLE1BQU07UUFDUixDQUFDO1FBQ0QsSUFBSSxZQUFZLEdBQWtCLElBQUksQ0FBQztRQUN2QyxJQUFJLFNBQVMsR0FBa0IsSUFBSSxDQUFDO1FBQ3BDLElBQUksa0JBQWtCLEdBQUcsS0FBSyxDQUFDO1FBQy9CLElBQUksV0FBVyxHQUFrQixJQUFJLENBQUM7UUFDdEMsSUFBSSxPQUFPLEtBQUssS0FBSyxRQUFRLEVBQUUsQ0FBQztZQUM5QixZQUFZLEdBQUcsMEJBQTBCLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDbkQsQ0FBQzthQUFNLElBQUksS0FBSyxJQUFJLE9BQU8sS0FBSyxLQUFLLFFBQVEsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztZQUN2RSxNQUFNLE1BQU0sR0FBRyxLQUFnQyxDQUFDO1lBQ2hELFlBQVksR0FBRywwQkFBMEIsQ0FBQyxNQUFNLENBQUMsWUFBWSxDQUFDLENBQUM7WUFDL0QsU0FBUyxHQUFHLHVCQUF1QixDQUFDLE1BQU0sQ0FBQyxTQUFTLEVBQUUsV0FBVyxFQUFFLFFBQVEsQ0FBQyxDQUFDO1lBQzdFLGtCQUFrQixHQUFHLGdCQUFnQixDQUFDLE1BQU0sQ0FBQyxrQkFBa0IsRUFBRSxvQkFBb0IsRUFBRSxLQUFLLEVBQUUsUUFBUSxDQUFDLENBQUM7WUFDeEcsV0FBVyxHQUFHLHVCQUF1QixDQUFDLE1BQU0sQ0FBQyxXQUFXLEVBQUUsYUFBYSxFQUFFLFFBQVEsQ0FBQyxDQUFDO1FBQ3JGLENBQUM7YUFBTSxDQUFDO1lBQ04sUUFBUSxDQUFDLElBQUksQ0FBQyw4RUFBOEUsQ0FBQyxDQUFDO1lBQzlGLFNBQVM7UUFDWCxDQUFDO1FBQ0QsSUFBSSxZQUFZLEtBQUssSUFBSSxFQUFFLENBQUM7WUFDMUIsUUFBUSxDQUFDLElBQUksQ0FBQyx5RkFBeUYsQ0FBQyxDQUFDO1lBQ3pHLFNBQVM7UUFDWCxDQUFDO1FBQ0QsSUFBSSxJQUFJLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxFQUFFLENBQUM7WUFDM0IsUUFBUSxDQUFDLElBQUksQ0FBQyxxRUFBcUUsWUFBWSxHQUFHLENBQUMsQ0FBQztZQUNwRyxTQUFTO1FBQ1gsQ0FBQztRQUNELElBQUksQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUM7UUFDdkIsTUFBTSxVQUFVLEdBQW9DLEVBQUUsWUFBWSxFQUFFLGtCQUFrQixFQUFFLENBQUM7UUFDekYsSUFBSSxTQUFTO1lBQUUsVUFBVSxDQUFDLFNBQVMsR0FBRyxTQUFTLENBQUM7UUFDaEQsSUFBSSxXQUFXO1lBQUUsVUFBVSxDQUFDLFdBQVcsR0FBRyxXQUFXLENBQUM7UUFDdEQsTUFBTSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztJQUMxQixDQUFDO0lBQ0QsT0FBTyxNQUFNLENBQUM7QUFDaEIsQ0FBQztBQUVEOzs7R0FHRztBQUNILE1BQU0sVUFBVSxnQ0FBZ0MsQ0FDOUMsT0FBa0M7SUFFbEMsSUFBSSxPQUFPLEtBQUssU0FBUyxJQUFJLE9BQU8sS0FBSyxJQUFJO1FBQUUsT0FBTyxrQkFBa0IsRUFBRSxDQUFDO0lBQzNFLElBQUksT0FBTyxPQUFPLEtBQUssUUFBUSxFQUFFLENBQUM7UUFDaEMsT0FBTyxrQkFBa0IsQ0FBQyxDQUFDLDZEQUE2RCxPQUFPLE9BQU8sR0FBRyxDQUFDLENBQUMsQ0FBQztJQUM5RyxDQUFDO0lBQ0QsTUFBTSxPQUFPLEdBQUcsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFDO0lBQy9CLElBQUksQ0FBQyxPQUFPO1FBQUUsT0FBTyxrQkFBa0IsRUFBRSxDQUFDO0lBQzFDLElBQUksY0FBYyxDQUFDLE9BQU8sQ0FBQyxHQUFHLDZCQUE2QixFQUFFLENBQUM7UUFDNUQsT0FBTyxrQkFBa0IsQ0FBQztZQUN4Qix3Q0FBd0MsNkJBQTZCLDRCQUE0QjtTQUNsRyxDQUFDLENBQUM7SUFDTCxDQUFDO0lBQ0QsSUFBSSxHQUFZLENBQUM7SUFDakIsSUFBSSxDQUFDO1FBQ0gsR0FBRyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLENBQUM7SUFDNUIsQ0FBQztJQUFDLE1BQU0sQ0FBQztRQUNQLE9BQU8sa0JBQWtCLENBQUMsQ0FBQyxnREFBZ0QsQ0FBQyxDQUFDLENBQUM7SUFDaEYsQ0FBQztJQUNELElBQUksQ0FBQyxHQUFHLElBQUksT0FBTyxHQUFHLEtBQUssUUFBUSxJQUFJLEtBQUssQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQztRQUMxRCxPQUFPLGtCQUFrQixDQUFDLENBQUMseURBQXlELENBQUMsQ0FBQyxDQUFDO0lBQ3pGLENBQUM7SUFDRCxNQUFNLFFBQVEsR0FBYSxFQUFFLENBQUM7SUFDOUIsTUFBTSxLQUFLLEdBQUcsaUJBQWlCLENBQUUsR0FBMkIsQ0FBQyxLQUFLLEVBQUUsUUFBUSxDQUFDLENBQUM7SUFDOUUsT0FBTyxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsUUFBUSxFQUFFLEVBQUUsS0FBSyxFQUFFLEVBQUUsUUFBUSxFQUFFLENBQUM7QUFDMUQsQ0FBQztBQUVEOzs7R0FHRztBQUNILE1BQU0sVUFBVSwrQkFBK0IsQ0FBQyxJQUFZO0lBQzFELElBQUksT0FBTyxJQUFJLEtBQUssUUFBUTtRQUFFLE9BQU8sRUFBRSxDQUFDO0lBQ3hDLE1BQU0sUUFBUSxHQUF3QyxFQUFFLENBQUM7SUFDekQsS0FBSyxNQUFNLElBQUksSUFBSSxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUM7UUFDcEMsTUFBTSxPQUFPLEdBQUcsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO1FBQzVCLElBQUksQ0FBQyxPQUFPLElBQUksZ0JBQWdCLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQztZQUFFLFNBQVM7UUFDekQsS0FBSyxNQUFNLEtBQUssSUFBSSxvQ0FBb0MsRUFBRSxDQUFDO1lBQ3pELElBQUksS0FBSyxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDO2dCQUFFLFFBQVEsQ0FBQyxJQUFJLENBQUMsRUFBRSxFQUFFLEVBQUUsS0FBSyxDQUFDLEVBQUUsRUFBRSxJQUFJLEVBQUUsT0FBTyxFQUFFLENBQUMsQ0FBQztRQUMvRSxDQUFDO0lBQ0gsQ0FBQztJQUNELE9BQU8sUUFBUSxDQUFDO0FBQ2xCLENBQUM7QUFFRCxTQUFTLFlBQVksQ0FDbkIsWUFBb0IsRUFDcEIsUUFBZ0IsRUFDaEIsTUFBYyxFQUNkLFFBQTRDLEVBQUU7SUFFOUMsT0FBTztRQUNMLFlBQVk7UUFDWixNQUFNLEVBQUUsS0FBSztRQUNiLGVBQWUsRUFBRSxRQUFRO1FBQ3pCLE1BQU07UUFDTixhQUFhLEVBQUUsS0FBSztRQUNwQixtQkFBbUIsRUFBRSxJQUFJO1FBQ3pCLGtCQUFrQixFQUFFLEVBQUU7UUFDdEIsR0FBRyxLQUFLO0tBQ1QsQ0FBQztBQUNKLENBQUM7QUFFRCxTQUFTLFNBQVMsQ0FBQyxZQUFvQixFQUFFLFFBQTRDLEVBQUU7SUFDckYsT0FBTztRQUNMLFlBQVk7UUFDWixNQUFNLEVBQUUsSUFBSTtRQUNaLGVBQWUsRUFBRSxJQUFJO1FBQ3JCLE1BQU0sRUFBRSxJQUFJO1FBQ1osYUFBYSxFQUFFLElBQUk7UUFDbkIsbUJBQW1CLEVBQUUsSUFBSTtRQUN6QixrQkFBa0IsRUFBRSxFQUFFO1FBQ3RCLEdBQUcsS0FBSztLQUNULENBQUM7QUFDSixDQUFDO0FBRUQsU0FBUyx5QkFBeUIsQ0FDaEMsS0FBc0MsRUFDdEMsVUFBd0MsRUFBRTtJQUUxQyxJQUFJLEtBQUssQ0FBQyxXQUFXLElBQUksT0FBTyxLQUFLLENBQUMsV0FBVyxLQUFLLFFBQVE7UUFBRSxPQUFPLEtBQUssQ0FBQyxXQUFXLENBQUM7SUFDekYsSUFBSSxPQUFPLE9BQU8sQ0FBQyxRQUFRLEtBQUssUUFBUSxJQUFJLE9BQU8sQ0FBQyxRQUFRLENBQUMsSUFBSSxFQUFFO1FBQUUsT0FBTyxPQUFPLENBQUMsUUFBUSxDQUFDLElBQUksRUFBRSxDQUFDO0lBQ3BHLElBQUksT0FBTyxPQUFPLENBQUMsZUFBZSxLQUFLLFVBQVU7UUFBRSxPQUFPLE9BQU8sQ0FBQyxlQUFlLENBQUMsS0FBSyxDQUFDLENBQUM7SUFDekYsT0FBTyxtQkFBbUIsQ0FBQyxLQUFLLENBQUMsWUFBWSxFQUFFLE9BQU8sQ0FBQyxHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0FBQzdFLENBQUM7QUFFRCxTQUFTLGtCQUFrQixDQUFDLFlBQW9CO0lBQzlDLE9BQU8sRUFBRSxVQUFVLEVBQUUsR0FBRyxFQUFFLENBQUMsRUFBRSxFQUFFLENBQUM7QUFDbEMsQ0FBQztBQUVEOztHQUVHO0FBQ0gsTUFBTSxVQUFVLHFCQUFxQixDQUNuQyxLQUFzQyxFQUN0QyxVQUF3QyxFQUFFO0lBRTFDLE1BQU0sWUFBWSxHQUFHLEtBQUssRUFBRSxZQUFZLENBQUM7SUFDekMsSUFBSSxPQUFPLFlBQVksS0FBSyxRQUFRLElBQUksQ0FBQywwQkFBMEIsQ0FBQyxZQUFZLENBQUMsRUFBRSxDQUFDO1FBQ2xGLE9BQU8sWUFBWSxDQUNqQixPQUFPLFlBQVksS0FBSyxRQUFRLENBQUMsQ0FBQyxDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUMsV0FBVyxFQUM3RCwyQkFBMkIsQ0FBQyxLQUFLLEVBQ2pDLHFEQUFxRCxDQUN0RCxDQUFDO0lBQ0osQ0FBQztJQUVELE1BQU0sVUFBVSxHQUFHLE9BQU8sQ0FBQyxVQUFVLElBQUksVUFBVSxDQUFDO0lBQ3BELE1BQU0sVUFBVSxHQUFHLE9BQU8sQ0FBQyxlQUFlLElBQUksZUFBZSxDQUFDO0lBQzlELE1BQU0sWUFBWSxHQUFHLE9BQU8sQ0FBQyxvQkFBb0IsSUFBSSxvQkFBb0IsQ0FBQztJQUMxRSxNQUFNLGFBQWEsR0FDakIsT0FBTyxDQUFDLG1CQUFtQjtRQUMxQixtQkFBbUcsQ0FBQztJQUN2RyxNQUFNLFFBQVEsR0FBRyx5QkFBeUIsQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLENBQUM7SUFFM0QsSUFBSSxDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsRUFBRSxDQUFDO1FBQzFCLE9BQU8sWUFBWSxDQUNqQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsV0FBVyxFQUN2QyxtQ0FBbUMsUUFBUSx3REFBd0QsQ0FDcEcsQ0FBQztJQUNKLENBQUM7SUFFRCxNQUFNLFFBQVEsR0FBRyxZQUFZLENBQUMsUUFBUSxDQUFDLENBQUM7SUFDeEMsTUFBTSxtQkFBbUIsR0FBRyxRQUFRLEVBQUUsT0FBTyxLQUFLLElBQUksQ0FBQztJQUV2RCxNQUFNLEtBQUssR0FBRyxVQUFVLENBQUMsUUFBUSxDQUFDLENBQUM7SUFDbkMsSUFBSSxLQUFLLEVBQUUsUUFBUSxLQUFLLElBQUksRUFBRSxDQUFDO1FBQzdCLE9BQU8sWUFBWSxDQUNqQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsZUFBZSxFQUMzQyxLQUFLLEVBQUUsTUFBTSxJQUFJLHlEQUF5RCxFQUMxRSxFQUFFLGFBQWEsRUFBRSxLQUFLLEVBQUUsbUJBQW1CLEVBQUUsQ0FDOUMsQ0FBQztJQUNKLENBQUM7SUFFRCxJQUFJLEtBQUssQ0FBQyxrQkFBa0IsS0FBSyxJQUFJLElBQUksQ0FBQyxLQUFLLENBQUMsV0FBVyxFQUFFLENBQUM7UUFDNUQsT0FBTyxZQUFZLENBQ2pCLFlBQVksRUFDWiwyQkFBMkIsQ0FBQyxTQUFTLEVBQ3JDLDZGQUE2RixFQUM3RixFQUFFLGFBQWEsRUFBRSxJQUFJLEVBQUUsbUJBQW1CLEVBQUUsS0FBSyxFQUFFLENBQ3BELENBQUM7SUFDSixDQUFDO0lBRUQsSUFBSSxVQUFVLENBQUM7SUFDZixJQUFJLENBQUM7UUFDSCxVQUFVLEdBQUcsYUFBYSxDQUFDO1lBQ3pCLFlBQVk7WUFDWixLQUFLLEVBQUU7Z0JBQ0wsTUFBTSxFQUFFLENBQUM7Z0JBQ1QsS0FBSyxFQUFFLDJDQUEyQztnQkFDbEQsSUFBSSxFQUFFLGlFQUFpRTtnQkFDdkUsTUFBTSxFQUFFLENBQUMsS0FBSyxDQUFDO2FBQ2hCO1lBQ0QsT0FBTyxFQUFFLEVBQUUsTUFBTSxFQUFFLENBQUMsRUFBRSxNQUFNLEVBQUUsQ0FBQyxFQUFFLENBQUMsRUFBRSxZQUFZLEVBQUUsRUFBRSxFQUFFO1lBQ3RELFdBQVcsRUFBRSxrQkFBa0IsQ0FBQyxZQUFZLENBQUM7WUFDN0MsZ0JBQWdCLEVBQUUsUUFBUTtZQUMxQixlQUFlLEVBQUUsVUFBVTtTQUM1QixDQUFDLENBQUM7SUFDTCxDQUFDO0lBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztRQUNmLE1BQU0sT0FBTyxHQUFHLEtBQUssWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUN2RSxPQUFPLFlBQVksQ0FBQyxZQUFZLEVBQUUsMkJBQTJCLENBQUMsS0FBSyxFQUFFLE9BQU8sRUFBRTtZQUM1RSxhQUFhLEVBQUUsSUFBSTtZQUNuQixtQkFBbUI7WUFDbkIsS0FBSztTQUNOLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRCxJQUFJLFVBQVUsRUFBRSxLQUFLLEtBQUssSUFBSSxFQUFFLENBQUM7UUFDL0IsT0FBTyxZQUFZLENBQ2pCLFlBQVksRUFDWiwyQkFBMkIsQ0FBQyxTQUFTLEVBQ3JDLDJDQUEyQyxVQUFVLEVBQUUsT0FBTyxJQUFJLFNBQVMsSUFBSSxFQUMvRSxFQUFFLGFBQWEsRUFBRSxJQUFJLEVBQUUsbUJBQW1CLEVBQUUsS0FBSyxFQUFFLENBQ3BELENBQUM7SUFDSixDQUFDO0lBRUQsTUFBTSxrQkFBa0IsR0FBRywrQkFBK0IsQ0FBQyxVQUFVLENBQUMsWUFBWSxJQUFJLEVBQUUsQ0FBQyxDQUFDO0lBQzFGLElBQUksa0JBQWtCLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDO1FBQ2xDLE9BQU8sWUFBWSxDQUNqQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsb0JBQW9CLEVBQ2hELDBEQUEwRCxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksRUFDNUcsRUFBRSxhQUFhLEVBQUUsSUFBSSxFQUFFLG1CQUFtQixFQUFFLEtBQUssRUFBRSxrQkFBa0IsRUFBRSxDQUN4RSxDQUFDO0lBQ0osQ0FBQztJQUVELE9BQU8sU0FBUyxDQUFDLFlBQVksRUFBRSxFQUFFLG1CQUFtQixFQUFFLEtBQUssRUFBRSxDQUFDLENBQUM7QUFDakUsQ0FBQztBQUVEOztHQUVHO0FBQ0gsTUFBTSxVQUFVLHNCQUFzQixDQUNwQyxNQUF5QyxFQUN6QyxVQUFrRSxFQUFFO0lBRXBFLE1BQU0sS0FBSyxHQUFHLE1BQU0sRUFBRSxRQUFRLEVBQUUsS0FBSyxJQUFJLEVBQUUsQ0FBQztJQUM1QyxNQUFNLE9BQU8sR0FBZ0MsRUFBRSxDQUFDO0lBQ2hELEtBQUssTUFBTSxLQUFLLElBQUksS0FBSyxFQUFFLENBQUM7UUFDMUIsSUFBSSxPQUFPLENBQUMsVUFBVSxJQUFJLEtBQUssQ0FBQyxZQUFZLEtBQUssT0FBTyxDQUFDLFVBQVU7WUFBRSxTQUFTO1FBQzlFLE9BQU8sQ0FBQyxJQUFJLENBQUMscUJBQXFCLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFDLENBQUM7SUFDdEQsQ0FBQztJQUNELE9BQU8sT0FBTyxDQUFDO0FBQ2pCLENBQUM7QUFFRDs7R0FFRztBQUNILE1BQU0sVUFBVSw0QkFBNEIsQ0FBQyxPQUFvQztJQUMvRSxNQUFNLElBQUksR0FBRyxLQUFLLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztJQUNuRCxJQUFJLE1BQU0sR0FBRyxDQUFDLENBQUM7SUFDZixJQUFJLE1BQU0sR0FBRyxDQUFDLENBQUM7SUFDZixNQUFNLGtCQUFrQixHQUEyQixFQUFFLENBQUM7SUFDdEQsS0FBSyxNQUFNLE1BQU0sSUFBSSxJQUFJLEVBQUUsQ0FBQztRQUMxQixJQUFJLE1BQU0sRUFBRSxNQUFNLEtBQUssSUFBSSxFQUFFLENBQUM7WUFDNUIsTUFBTSxJQUFJLENBQUMsQ0FBQztZQUNaLFNBQVM7UUFDWCxDQUFDO1FBQ0QsTUFBTSxJQUFJLENBQUMsQ0FBQztRQUNaLE1BQU0sUUFBUSxHQUFHLE1BQU0sRUFBRSxlQUFlLElBQUksMkJBQTJCLENBQUMsS0FBSyxDQUFDO1FBQzlFLGtCQUFrQixDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsa0JBQWtCLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQ3pFLENBQUM7SUFDRCxNQUFNLEtBQUssR0FBRyxNQUFNLEdBQUcsTUFBTSxDQUFDO0lBQzlCLE1BQU0sY0FBYyxHQUFHLEtBQUssR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLE1BQU0sR0FBRyxNQUFNLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQztJQUMzRCxNQUFNLHFCQUFxQixHQUFHLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsRUFBRSxtQkFBbUIsS0FBSyxLQUFLLENBQUMsQ0FBQyxNQUFNLENBQUM7SUFDMUYsT0FBTztRQUNMLEtBQUs7UUFDTCxNQUFNO1FBQ04sTUFBTTtRQUNOLGNBQWM7UUFDZCxxQkFBcUI7UUFDckIsa0JBQWtCO0tBQ25CLENBQUM7QUFDSixDQUFDO0FBRUQ7O0dBRUc7QUFDSCxNQUFNLFVBQVUsK0JBQStCLENBQzdDLE9BQW9DLEVBQ3BDLFVBQXNDLDRCQUE0QixDQUFDLE9BQU8sQ0FBQztJQUUzRSxNQUFNLEtBQUssR0FBRyxDQUFDLHNDQUFzQyxFQUFFLEVBQUUsQ0FBQyxDQUFDO0lBQzNELEtBQUssTUFBTSxNQUFNLElBQUksT0FBTyxFQUFFLENBQUM7UUFDN0IsSUFBSSxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUM7WUFDbEIsS0FBSyxDQUFDLElBQUksQ0FBQyxRQUFRLE1BQU0sQ0FBQyxZQUFZLEVBQUUsQ0FBQyxDQUFDO1lBQzFDLFNBQVM7UUFDWCxDQUFDO1FBQ0QsS0FBSyxDQUFDLElBQUksQ0FBQyxRQUFRLE1BQU0sQ0FBQyxZQUFZLEtBQUssTUFBTSxDQUFDLGVBQWUsS0FBSyxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQztJQUN6RixDQUFDO0lBQ0QsS0FBSyxDQUFDLElBQUksQ0FDUixFQUFFLEVBQ0YsWUFBWSxPQUFPLENBQUMsTUFBTSxJQUFJLE9BQU8sQ0FBQyxLQUFLLFNBQVM7UUFDbEQsQ0FBQyxPQUFPLENBQUMsY0FBYyxDQUFDLENBQUMsQ0FBQyxvQkFBb0IsQ0FBQyxDQUFDLENBQUMsb0JBQW9CLENBQUMsQ0FDekUsQ0FBQztJQUNGLElBQUksT0FBTyxDQUFDLEtBQUssR0FBRyxDQUFDLEVBQUUsQ0FBQztRQUN0QixLQUFLLENBQUMsSUFBSSxDQUFDLDRDQUE0QyxPQUFPLENBQUMscUJBQXFCLElBQUksT0FBTyxDQUFDLEtBQUssRUFBRSxDQUFDLENBQUM7SUFDM0csQ0FBQztJQUNELE1BQU0sVUFBVSxHQUFHLE1BQU0sQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLGtCQUFrQixDQUFDLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsYUFBYSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDckcsSUFBSSxVQUFVLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDO1FBQzFCLEtBQUssQ0FBQyxJQUFJLENBQUMsRUFBRSxFQUFFLHVCQUF1QixDQUFDLENBQUM7UUFDeEMsS0FBSyxNQUFNLENBQUMsUUFBUSxFQUFFLEtBQUssQ0FBQyxJQUFJLFVBQVUsRUFBRSxDQUFDO1lBQzNDLEtBQUssQ0FBQyxJQUFJLENBQUMsS0FBSyxRQUFRLEtBQUssS0FBSyxFQUFFLENBQUMsQ0FBQztRQUN4QyxDQUFDO0lBQ0gsQ0FBQztJQUNELE9BQU8sS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztBQUMxQixDQUFDIn0= \ No newline at end of file +// --------------------------------------------------------------------------------------------------------------- +// Full-execution mode (#7634): past readiness, actually run the discover -> plan -> code -> test loop against a +// benchmark repo — dry-run only. The attempt runs inside a scratch COPY of the local clone that is discarded +// afterward: no live PR submission, no forge API calls, and the benchmark clone itself is never mutated by the +// agent or the test run. Same taxonomy extension point as the readiness categories above — summarize/format are +// data-driven over category strings, so these flow through the existing report unchanged. +/** Execution-specific failure taxonomy (#7634), extending — not replacing — CROSS_REPO_FAILURE_CATEGORY. */ +export const CROSS_REPO_EXECUTION_FAILURE_CATEGORY = Object.freeze({ + AGENT_RUN: "agent_run_failed", + NOOP_DIFF: "noop_diff", + BUILD: "build_failed", + TEST: "test_failed", +}); +/** A benchmark attempt works a small synthetic issue, so a modest turn cap keeps dry-runs bounded without + * starving a real agent; callers tune via options.maxTurns. */ +export const DEFAULT_CROSS_REPO_EXECUTION_MAX_TURNS = 24; +/** Per-command (build, then test) wall-clock cap — generous enough for a cold dependency install on the larger + * benchmark repos, small enough that a hung suite cannot wedge the whole run. */ +export const DEFAULT_CROSS_REPO_EXECUTION_COMMAND_TIMEOUT_MS = 600_000; +/** Copy the benchmark clone into a discardable temp tree — the agent and the repo's test suite only ever touch + * the copy, so the clone stays pristine and cleanup is a single recursive remove. */ +export function defaultPrepareExecutionWorkspace(repoPath) { + const scratchRoot = mkdtempSync(join(tmpdir(), "loopover-cross-repo-exec-")); + const path = join(scratchRoot, "repo"); + cpSync(repoPath, path, { recursive: true }); + return { + path, + cleanup: () => { + rmSync(scratchRoot, { recursive: true, force: true }); + }, + }; +} +/** Command runner for the stack's inferred build/test commands. detectRepoStack only ever emits simple + * `tool subcommand` forms ("npm test", "cargo build", "npm run build"), so the command is tokenized on + * whitespace and exec'd DIRECTLY -- deliberately no `shell: true`, so nothing in a benchmark repo's manifest + * can smuggle shell metacharacters into an interpreted shell line. Mirrors coding-agent-construction's + * createRealCliSubprocessSpawn otherwise: capture both streams and RESOLVE (never reject) on timeout or spawn + * error, so partial output stays diagnosable. Promise resolution is idempotent, so a `close` firing after the + * timeout already resolved needs no guard. */ +export function createDefaultCrossRepoExecutionRunCommand() { + return (command, options) => new Promise((resolve) => { + const [executable, ...args] = command.split(/\s+/).filter(Boolean); + if (!executable) { + resolve({ stdout: "", stderr: "empty_command", code: null, timedOut: false }); + return; + } + const child = nodeSpawn(executable, args, { + cwd: options.cwd, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + resolve({ stdout, stderr, code: null, timedOut: true }); + }, options.timeoutMs); + // stdio is always ["ignore","pipe","pipe"] above, so both streams exist — assert instead of branching. + child.stdout.on("data", (chunk) => { + stdout += chunk.toString("utf8"); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString("utf8"); + }); + child.on("error", (err) => { + clearTimeout(timer); + resolve({ stdout, stderr: err.message, code: null, timedOut: false }); + }); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ stdout, stderr, code, timedOut: false }); + }); + }); +} +function buildExecutionFailure(repoFullName, category, reason, base, execution) { + return { ...buildFailure(repoFullName, category, reason, base), execution }; +} +/** + * Run the full discover -> plan -> code -> test loop for one benchmark repo, dry-run (#7634). Readiness gates + * first (its failures pass through unchanged); execution then happens entirely inside a scratch copy that is + * discarded in every outcome. + */ +export async function evaluateRepoExecution(entry, options = {}) { + const readiness = evaluateRepoReadiness(entry, options); + if (!readiness.passed) + return { ...readiness, execution: null }; + const prepareWorkspace = options.prepareExecutionWorkspace ?? defaultPrepareExecutionWorkspace; + let workspace; + try { + workspace = prepareWorkspace(resolveEvaluationRepoPath(entry, options)); + } + catch (error) { + const reason = `Failed to prepare a scratch execution workspace: ${describeError(error)}`; + // A passed readiness result always carries its detected stack (buildPass is invoked with it). + const base = { stackDetected: true, usedDefaultGoalSpec: readiness.usedDefaultGoalSpec, stack: readiness.stack }; + return buildExecutionFailure(readiness.repoFullName, CROSS_REPO_FAILURE_CATEGORY.CLONE_SETUP, reason, base, null); + } + try { + return await runExecutionPhases(readiness, workspace, options); + } + finally { + try { + workspace.cleanup(); + } + catch { + // Best-effort discard — a cleanup failure must never mask the evaluation outcome itself. + } + } +} +async function runExecutionPhases(readiness, workspace, options) { + const repoFullName = readiness.repoFullName; + // Readiness passed, so the stack is present and detected — narrow once instead of re-branching on it. + const stack = readiness.stack; + const usedDefaultGoalSpec = readiness.usedDefaultGoalSpec; + const failureBase = { stackDetected: true, usedDefaultGoalSpec, stack }; + const detectImpl = options.detectRepoStack ?? detectRepoStack; + const buildSpecImpl = options.buildCodingTaskSpec ?? + buildCodingTaskSpec; + // Re-compose the coding-task spec INSIDE the scratch copy: buildCodingTaskSpec writes the acceptance-criteria + // document into its workingDirectory, and the agent must find it (and work) in the tree it is allowed to touch. + // The readiness pass already wrote that document into the clone (pre-existing #4788 behavior), so the copy + // inherits it -- and writeAcceptanceCriteriaFile opens O_EXCL (never overwrites), so clear the inherited file + // first. The scratch tree is ours to mutate; the clone itself stays untouched. + rmSync(join(workspace.path, ACCEPTANCE_CRITERIA_FILENAME), { force: true }); + let specResult; + try { + specResult = buildSpecImpl(buildHarnessSpecInput(repoFullName, workspace.path, detectImpl)); + } + catch (error) { + return buildExecutionFailure(repoFullName, CROSS_REPO_FAILURE_CATEGORY.OTHER, `Coding task spec failed inside the scratch workspace: ${describeError(error)}`, failureBase, null); + } + if (specResult?.ready !== true) { + return buildExecutionFailure(repoFullName, CROSS_REPO_FAILURE_CATEGORY.EXECUTION, `Coding task spec is not ready inside the scratch workspace (verdict: ${specResult?.verdict ?? "unknown"}).`, failureBase, null); + } + let driver = options.driver; + if (!driver) { + try { + // Lazy so the readiness-only path (and its consumers) never load the engine-backed driver construction. + const { constructProductionCodingAgentDriver } = await import("./coding-agent-construction.js"); + driver = constructProductionCodingAgentDriver(options.env ?? process.env); + } + catch (error) { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_FAILURE_CATEGORY.AGENT_RUN, `No runnable coding-agent driver: ${describeError(error)}`, failureBase, { attempted: false, changedFileCount: null, buildRan: false, testRan: false }); + } + } + const task = { + attemptId: `cross-repo-eval-${repoFullName.replace("/", "-")}`, + workingDirectory: workspace.path, + acceptanceCriteriaPath: specResult.acceptanceCriteriaPath ?? join(workspace.path, "ACCEPTANCE_CRITERIA.md"), + instructions: specResult.instructions ?? "", + maxTurns: options.maxTurns ?? DEFAULT_CROSS_REPO_EXECUTION_MAX_TURNS, + }; + let agentResult; + try { + agentResult = await driver.run(task); + } + catch (error) { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_FAILURE_CATEGORY.AGENT_RUN, `Coding agent run threw: ${describeError(error)}`, failureBase, { attempted: true, changedFileCount: null, buildRan: false, testRan: false }); + } + if (agentResult?.ok !== true) { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_FAILURE_CATEGORY.AGENT_RUN, `Coding agent run did not succeed: ${agentResult?.error ?? agentResult?.summary ?? "no failure detail reported"}`, failureBase, { attempted: true, changedFileCount: null, buildRan: false, testRan: false }); + } + const changedFileCount = Array.isArray(agentResult.changedFiles) ? agentResult.changedFiles.length : 0; + if (changedFileCount === 0) { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_FAILURE_CATEGORY.NOOP_DIFF, "Coding agent reported success but the generated diff is a no-op (no files changed).", failureBase, { attempted: true, changedFileCount: 0, buildRan: false, testRan: false }); + } + const runCommand = options.runCommand ?? createDefaultCrossRepoExecutionRunCommand(); + const commandTimeoutMs = options.commandTimeoutMs ?? DEFAULT_CROSS_REPO_EXECUTION_COMMAND_TIMEOUT_MS; + let buildRan = false; + if (stack.buildCommand) { + buildRan = true; + const buildResult = await runCommand(stack.buildCommand, { cwd: workspace.path, timeoutMs: commandTimeoutMs }); + if (buildResult.timedOut || buildResult.code !== 0) { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_FAILURE_CATEGORY.BUILD, `Diff generated but the build ${buildResult.timedOut ? "timed out" : `failed (exit ${buildResult.code})`}: ${stack.buildCommand}`, failureBase, { attempted: true, changedFileCount, buildRan: true, testRan: false }); + } + } + if (!stack.testCommand) { + return buildExecutionFailure(repoFullName, CROSS_REPO_FAILURE_CATEGORY.EXECUTION, "Stack detection inferred no test command, so the generated diff cannot be validated in execution mode.", failureBase, { attempted: true, changedFileCount, buildRan, testRan: false }); + } + const testResult = await runCommand(stack.testCommand, { cwd: workspace.path, timeoutMs: commandTimeoutMs }); + if (testResult.timedOut || testResult.code !== 0) { + return buildExecutionFailure(repoFullName, CROSS_REPO_EXECUTION_FAILURE_CATEGORY.TEST, `Diff generated but the repo's own test suite ${testResult.timedOut ? "timed out" : `failed (exit ${testResult.code})`}: ${stack.testCommand}`, failureBase, { attempted: true, changedFileCount, buildRan, testRan: true }); + } + return { + ...buildPass(repoFullName, { usedDefaultGoalSpec, stack }), + execution: { attempted: true, changedFileCount, buildRan, testRan: true }, + }; +} +/** + * Run full-execution mode across every repo in a parsed manifest (#7634), sequentially — agent runs and test + * suites are heavyweight, so no parallel fan-out. + */ +export async function runCrossRepoFullExecution(parsed, options = {}) { + const repos = parsed?.manifest?.repos ?? []; + const results = []; + for (const entry of repos) { + if (options.repoFilter && entry.repoFullName !== options.repoFilter) + continue; + results.push(await evaluateRepoExecution(entry, options)); + } + return results; +} +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY3Jvc3MtcmVwby1ldmFsdWF0aW9uLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiY3Jvc3MtcmVwby1ldmFsdWF0aW9uLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGlIQUFpSDtBQUNqSCw4R0FBOEc7QUFDOUcsMkdBQTJHO0FBQzNHLDZHQUE2RztBQUM3RyxxR0FBcUc7QUFFckcsT0FBTyxFQUFFLEtBQUssSUFBSSxTQUFTLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUN4RCxPQUFPLEVBQUUsTUFBTSxFQUFFLFVBQVUsRUFBRSxXQUFXLEVBQUUsTUFBTSxFQUFFLE1BQU0sU0FBUyxDQUFDO0FBQ2xFLE9BQU8sRUFBRSxNQUFNLEVBQUUsTUFBTSxTQUFTLENBQUM7QUFDakMsT0FBTyxFQUFFLElBQUksRUFBRSxNQUFNLFdBQVcsQ0FBQztBQUNqQyw0R0FBNEc7QUFDNUcsT0FBTyxFQUFFLDRCQUE0QixFQUFFLE1BQU0sa0JBQWtCLENBQUM7QUFDaEUsT0FBTyxFQUFFLG1CQUFtQixFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDNUQsT0FBTyxFQUFFLG9CQUFvQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDNUQsT0FBTyxFQUFFLGtCQUFrQixFQUFFLG1CQUFtQixFQUFFLE1BQU0saUJBQWlCLENBQUM7QUFDMUUsT0FBTyxFQUFFLGVBQWUsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBR3ZELDZEQUE2RDtBQUM3RCxNQUFNLENBQUMsTUFBTSwyQkFBMkIsR0FNbkMsTUFBTSxDQUFDLE1BQU0sQ0FBQztJQUNqQixlQUFlLEVBQUUscUJBQXFCO0lBQ3RDLFNBQVMsRUFBRSxlQUFlO0lBQzFCLG9CQUFvQixFQUFFLHFCQUFxQjtJQUMzQyxXQUFXLEVBQUUsYUFBYTtJQUMxQixLQUFLLEVBQUUsT0FBTztDQUNmLENBQUMsQ0FBQztBQUVIO21HQUNtRztBQUNuRyxNQUFNLENBQUMsTUFBTSxvQ0FBb0MsR0FBbUQsTUFBTSxDQUFDLE1BQU0sQ0FBQztJQUNoSCxFQUFFLEVBQUUsRUFBRSxnQkFBZ0IsRUFBRSxPQUFPLEVBQUUsa0JBQWtCLEVBQUU7SUFDckQsRUFBRSxFQUFFLEVBQUUsZUFBZSxFQUFFLE9BQU8sRUFBRSxpQkFBaUIsRUFBRTtJQUNuRCxFQUFFLEVBQUUsRUFBRSxpQkFBaUIsRUFBRSxPQUFPLEVBQUUscUNBQXFDLEVBQUU7SUFDekUsRUFBRSxFQUFFLEVBQUUsZUFBZSxFQUFFLE9BQU8sRUFBRSxnQkFBZ0IsRUFBRTtDQUNuRCxDQUFDLENBQUM7QUFFSCxNQUFNLENBQUMsTUFBTSx5Q0FBeUMsR0FBVyxxQ0FBcUMsQ0FBQztBQUN2RyxNQUFNLENBQUMsTUFBTSw2QkFBNkIsR0FBVyxNQUFNLENBQUM7QUFDNUQsTUFBTSxDQUFDLE1BQU0sNkJBQTZCLEdBQVcsR0FBRyxDQUFDO0FBa0R6RCxpSEFBaUg7QUFDakgsZ0hBQWdIO0FBQ2hILG1IQUFtSDtBQUNuSCwrR0FBK0c7QUFDL0csMEJBQTBCO0FBQzFCLFNBQVMsY0FBYyxDQUFDLEtBQWE7SUFDbkMsSUFBSSxLQUFLLEdBQUcsQ0FBQyxDQUFDO0lBQ2QsS0FBSyxNQUFNLElBQUksSUFBSSxLQUFLLEVBQUUsQ0FBQztRQUN6QixNQUFNLFNBQVMsR0FBRyxJQUFJLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBRSxDQUFDO1FBQ3ZDLElBQUksU0FBUyxJQUFJLElBQUk7WUFBRSxLQUFLLElBQUksQ0FBQyxDQUFDO2FBQzdCLElBQUksU0FBUyxJQUFJLEtBQUs7WUFBRSxLQUFLLElBQUksQ0FBQyxDQUFDO2FBQ25DLElBQUksU0FBUyxJQUFJLE1BQU07WUFBRSxLQUFLLElBQUksQ0FBQyxDQUFDOztZQUNwQyxLQUFLLElBQUksQ0FBQyxDQUFDO0lBQ2xCLENBQUM7SUFDRCxPQUFPLEtBQUssQ0FBQztBQUNmLENBQUM7QUFFRCxTQUFTLGtCQUFrQixDQUFDLFdBQXFCLEVBQUU7SUFDakQsT0FBTyxFQUFFLE9BQU8sRUFBRSxLQUFLLEVBQUUsUUFBUSxFQUFFLEVBQUUsS0FBSyxFQUFFLEVBQUUsRUFBRSxFQUFFLFFBQVEsRUFBRSxDQUFDO0FBQy9ELENBQUM7QUFFRCw2RkFBNkY7QUFDN0YsTUFBTSxVQUFVLDBCQUEwQixDQUFDLEtBQWM7SUFDdkQsSUFBSSxPQUFPLEtBQUssS0FBSyxRQUFRO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDM0MsTUFBTSxDQUFDLEtBQUssRUFBRSxJQUFJLEVBQUUsS0FBSyxDQUFDLEdBQUcsS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDLEtBQUssQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUNyRCxJQUFJLENBQUMsS0FBSyxJQUFJLENBQUMsSUFBSSxJQUFJLEtBQUssS0FBSyxTQUFTO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDeEQsSUFBSSxDQUFDLGtCQUFrQixDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsa0JBQWtCLENBQUMsSUFBSSxDQUFDO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDekUsT0FBTyxHQUFHLEtBQUssSUFBSSxJQUFJLEVBQUUsQ0FBQztBQUM1QixDQUFDO0FBRUQsU0FBUyxnQkFBZ0IsQ0FBQyxLQUFjLEVBQUUsS0FBYSxFQUFFLFFBQWlCLEVBQUUsUUFBa0I7SUFDNUYsSUFBSSxLQUFLLEtBQUssU0FBUyxJQUFJLEtBQUssS0FBSyxJQUFJO1FBQUUsT0FBTyxRQUFRLENBQUM7SUFDM0QsSUFBSSxPQUFPLEtBQUssS0FBSyxTQUFTO1FBQUUsT0FBTyxLQUFLLENBQUM7SUFDN0MsUUFBUSxDQUFDLElBQUksQ0FBQyxzQ0FBc0MsS0FBSyx3Q0FBd0MsUUFBUSxHQUFHLENBQUMsQ0FBQztJQUM5RyxPQUFPLFFBQVEsQ0FBQztBQUNsQixDQUFDO0FBRUQsU0FBUyx1QkFBdUIsQ0FBQyxLQUFjLEVBQUUsS0FBYSxFQUFFLFFBQWtCO0lBQ2hGLElBQUksS0FBSyxLQUFLLFNBQVMsSUFBSSxLQUFLLEtBQUssSUFBSTtRQUFFLE9BQU8sSUFBSSxDQUFDO0lBQ3ZELElBQUksT0FBTyxLQUFLLEtBQUssUUFBUSxFQUFFLENBQUM7UUFDOUIsUUFBUSxDQUFDLElBQUksQ0FBQyxzQ0FBc0MsS0FBSyx5Q0FBeUMsQ0FBQyxDQUFDO1FBQ3BHLE9BQU8sSUFBSSxDQUFDO0lBQ2QsQ0FBQztJQUNELE1BQU0sT0FBTyxHQUFHLEtBQUssQ0FBQyxJQUFJLEVBQUUsQ0FBQztJQUM3QixPQUFPLE9BQU8sSUFBSSxJQUFJLENBQUM7QUFDekIsQ0FBQztBQUVELFNBQVMsaUJBQWlCLENBQUMsS0FBYyxFQUFFLFFBQWtCO0lBQzNELElBQUksS0FBSyxLQUFLLFNBQVMsSUFBSSxLQUFLLEtBQUssSUFBSTtRQUFFLE9BQU8sRUFBRSxDQUFDO0lBQ3JELElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDMUIsUUFBUSxDQUFDLElBQUksQ0FBQyx3RUFBd0UsT0FBTyxLQUFLLFNBQVMsQ0FBQyxDQUFDO1FBQzdHLE9BQU8sRUFBRSxDQUFDO0lBQ1osQ0FBQztJQUNELE1BQU0sTUFBTSxHQUFzQyxFQUFFLENBQUM7SUFDckQsTUFBTSxJQUFJLEdBQUcsSUFBSSxHQUFHLEVBQVUsQ0FBQztJQUMvQixLQUFLLE1BQU0sQ0FBQyxLQUFLLEVBQUUsS0FBSyxDQUFDLElBQUksS0FBSyxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUM7UUFDN0MsSUFBSSxLQUFLLElBQUksNkJBQTZCLEVBQUUsQ0FBQztZQUMzQyxRQUFRLENBQUMsSUFBSSxDQUNYLHNEQUFzRCw2QkFBNkIsa0NBQWtDLENBQ3RILENBQUM7WUFDRixNQUFNO1FBQ1IsQ0FBQztRQUNELElBQUksWUFBWSxHQUFrQixJQUFJLENBQUM7UUFDdkMsSUFBSSxTQUFTLEdBQWtCLElBQUksQ0FBQztRQUNwQyxJQUFJLGtCQUFrQixHQUFHLEtBQUssQ0FBQztRQUMvQixJQUFJLFdBQVcsR0FBa0IsSUFBSSxDQUFDO1FBQ3RDLElBQUksT0FBTyxLQUFLLEtBQUssUUFBUSxFQUFFLENBQUM7WUFDOUIsWUFBWSxHQUFHLDBCQUEwQixDQUFDLEtBQUssQ0FBQyxDQUFDO1FBQ25ELENBQUM7YUFBTSxJQUFJLEtBQUssSUFBSSxPQUFPLEtBQUssS0FBSyxRQUFRLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7WUFDdkUsTUFBTSxNQUFNLEdBQUcsS0FBZ0MsQ0FBQztZQUNoRCxZQUFZLEdBQUcsMEJBQTBCLENBQUMsTUFBTSxDQUFDLFlBQVksQ0FBQyxDQUFDO1lBQy9ELFNBQVMsR0FBRyx1QkFBdUIsQ0FBQyxNQUFNLENBQUMsU0FBUyxFQUFFLFdBQVcsRUFBRSxRQUFRLENBQUMsQ0FBQztZQUM3RSxrQkFBa0IsR0FBRyxnQkFBZ0IsQ0FBQyxNQUFNLENBQUMsa0JBQWtCLEVBQUUsb0JBQW9CLEVBQUUsS0FBSyxFQUFFLFFBQVEsQ0FBQyxDQUFDO1lBQ3hHLFdBQVcsR0FBRyx1QkFBdUIsQ0FBQyxNQUFNLENBQUMsV0FBVyxFQUFFLGFBQWEsRUFBRSxRQUFRLENBQUMsQ0FBQztRQUNyRixDQUFDO2FBQU0sQ0FBQztZQUNOLFFBQVEsQ0FBQyxJQUFJLENBQUMsOEVBQThFLENBQUMsQ0FBQztZQUM5RixTQUFTO1FBQ1gsQ0FBQztRQUNELElBQUksWUFBWSxLQUFLLElBQUksRUFBRSxDQUFDO1lBQzFCLFFBQVEsQ0FBQyxJQUFJLENBQUMseUZBQXlGLENBQUMsQ0FBQztZQUN6RyxTQUFTO1FBQ1gsQ0FBQztRQUNELElBQUksSUFBSSxDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsRUFBRSxDQUFDO1lBQzNCLFFBQVEsQ0FBQyxJQUFJLENBQUMscUVBQXFFLFlBQVksR0FBRyxDQUFDLENBQUM7WUFDcEcsU0FBUztRQUNYLENBQUM7UUFDRCxJQUFJLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDO1FBQ3ZCLE1BQU0sVUFBVSxHQUFvQyxFQUFFLFlBQVksRUFBRSxrQkFBa0IsRUFBRSxDQUFDO1FBQ3pGLElBQUksU0FBUztZQUFFLFVBQVUsQ0FBQyxTQUFTLEdBQUcsU0FBUyxDQUFDO1FBQ2hELElBQUksV0FBVztZQUFFLFVBQVUsQ0FBQyxXQUFXLEdBQUcsV0FBVyxDQUFDO1FBQ3RELE1BQU0sQ0FBQyxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUM7SUFDMUIsQ0FBQztJQUNELE9BQU8sTUFBTSxDQUFDO0FBQ2hCLENBQUM7QUFFRDs7O0dBR0c7QUFDSCxNQUFNLFVBQVUsZ0NBQWdDLENBQzlDLE9BQWtDO0lBRWxDLElBQUksT0FBTyxLQUFLLFNBQVMsSUFBSSxPQUFPLEtBQUssSUFBSTtRQUFFLE9BQU8sa0JBQWtCLEVBQUUsQ0FBQztJQUMzRSxJQUFJLE9BQU8sT0FBTyxLQUFLLFFBQVEsRUFBRSxDQUFDO1FBQ2hDLE9BQU8sa0JBQWtCLENBQUMsQ0FBQyw2REFBNkQsT0FBTyxPQUFPLEdBQUcsQ0FBQyxDQUFDLENBQUM7SUFDOUcsQ0FBQztJQUNELE1BQU0sT0FBTyxHQUFHLE9BQU8sQ0FBQyxJQUFJLEVBQUUsQ0FBQztJQUMvQixJQUFJLENBQUMsT0FBTztRQUFFLE9BQU8sa0JBQWtCLEVBQUUsQ0FBQztJQUMxQyxJQUFJLGNBQWMsQ0FBQyxPQUFPLENBQUMsR0FBRyw2QkFBNkIsRUFBRSxDQUFDO1FBQzVELE9BQU8sa0JBQWtCLENBQUM7WUFDeEIsd0NBQXdDLDZCQUE2Qiw0QkFBNEI7U0FDbEcsQ0FBQyxDQUFDO0lBQ0wsQ0FBQztJQUNELElBQUksR0FBWSxDQUFDO0lBQ2pCLElBQUksQ0FBQztRQUNILEdBQUcsR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDO0lBQzVCLENBQUM7SUFBQyxNQUFNLENBQUM7UUFDUCxPQUFPLGtCQUFrQixDQUFDLENBQUMsZ0RBQWdELENBQUMsQ0FBQyxDQUFDO0lBQ2hGLENBQUM7SUFDRCxJQUFJLENBQUMsR0FBRyxJQUFJLE9BQU8sR0FBRyxLQUFLLFFBQVEsSUFBSSxLQUFLLENBQUMsT0FBTyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUM7UUFDMUQsT0FBTyxrQkFBa0IsQ0FBQyxDQUFDLHlEQUF5RCxDQUFDLENBQUMsQ0FBQztJQUN6RixDQUFDO0lBQ0QsTUFBTSxRQUFRLEdBQWEsRUFBRSxDQUFDO0lBQzlCLE1BQU0sS0FBSyxHQUFHLGlCQUFpQixDQUFFLEdBQTJCLENBQUMsS0FBSyxFQUFFLFFBQVEsQ0FBQyxDQUFDO0lBQzlFLE9BQU8sRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLFFBQVEsRUFBRSxFQUFFLEtBQUssRUFBRSxFQUFFLFFBQVEsRUFBRSxDQUFDO0FBQzFELENBQUM7QUFFRDs7O0dBR0c7QUFDSCxNQUFNLFVBQVUsK0JBQStCLENBQUMsSUFBWTtJQUMxRCxJQUFJLE9BQU8sSUFBSSxLQUFLLFFBQVE7UUFBRSxPQUFPLEVBQUUsQ0FBQztJQUN4QyxNQUFNLFFBQVEsR0FBd0MsRUFBRSxDQUFDO0lBQ3pELEtBQUssTUFBTSxJQUFJLElBQUksSUFBSSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDO1FBQ3BDLE1BQU0sT0FBTyxHQUFHLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztRQUM1QixJQUFJLENBQUMsT0FBTyxJQUFJLGdCQUFnQixDQUFDLElBQUksQ0FBQyxPQUFPLENBQUM7WUFBRSxTQUFTO1FBQ3pELEtBQUssTUFBTSxLQUFLLElBQUksb0NBQW9DLEVBQUUsQ0FBQztZQUN6RCxJQUFJLEtBQUssQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQztnQkFBRSxRQUFRLENBQUMsSUFBSSxDQUFDLEVBQUUsRUFBRSxFQUFFLEtBQUssQ0FBQyxFQUFFLEVBQUUsSUFBSSxFQUFFLE9BQU8sRUFBRSxDQUFDLENBQUM7UUFDL0UsQ0FBQztJQUNILENBQUM7SUFDRCxPQUFPLFFBQVEsQ0FBQztBQUNsQixDQUFDO0FBRUQsNEdBQTRHO0FBQzVHLFNBQVMsYUFBYSxDQUFDLEtBQWM7SUFDbkMsT0FBTyxLQUFLLFlBQVksS0FBSyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLENBQUM7QUFDaEUsQ0FBQztBQUVELFNBQVMsWUFBWSxDQUNuQixZQUFvQixFQUNwQixRQUFnQixFQUNoQixNQUFjLEVBQ2QsUUFBNEMsRUFBRTtJQUU5QyxPQUFPO1FBQ0wsWUFBWTtRQUNaLE1BQU0sRUFBRSxLQUFLO1FBQ2IsZUFBZSxFQUFFLFFBQVE7UUFDekIsTUFBTTtRQUNOLGFBQWEsRUFBRSxLQUFLO1FBQ3BCLG1CQUFtQixFQUFFLElBQUk7UUFDekIsa0JBQWtCLEVBQUUsRUFBRTtRQUN0QixHQUFHLEtBQUs7S0FDVCxDQUFDO0FBQ0osQ0FBQztBQUVELFNBQVMsU0FBUyxDQUFDLFlBQW9CLEVBQUUsUUFBNEMsRUFBRTtJQUNyRixPQUFPO1FBQ0wsWUFBWTtRQUNaLE1BQU0sRUFBRSxJQUFJO1FBQ1osZUFBZSxFQUFFLElBQUk7UUFDckIsTUFBTSxFQUFFLElBQUk7UUFDWixhQUFhLEVBQUUsSUFBSTtRQUNuQixtQkFBbUIsRUFBRSxJQUFJO1FBQ3pCLGtCQUFrQixFQUFFLEVBQUU7UUFDdEIsR0FBRyxLQUFLO0tBQ1QsQ0FBQztBQUNKLENBQUM7QUFFRCxTQUFTLHlCQUF5QixDQUNoQyxLQUFzQyxFQUN0QyxVQUF3QyxFQUFFO0lBRTFDLElBQUksS0FBSyxDQUFDLFdBQVcsSUFBSSxPQUFPLEtBQUssQ0FBQyxXQUFXLEtBQUssUUFBUTtRQUFFLE9BQU8sS0FBSyxDQUFDLFdBQVcsQ0FBQztJQUN6RixJQUFJLE9BQU8sT0FBTyxDQUFDLFFBQVEsS0FBSyxRQUFRLElBQUksT0FBTyxDQUFDLFFBQVEsQ0FBQyxJQUFJLEVBQUU7UUFBRSxPQUFPLE9BQU8sQ0FBQyxRQUFRLENBQUMsSUFBSSxFQUFFLENBQUM7SUFDcEcsSUFBSSxPQUFPLE9BQU8sQ0FBQyxlQUFlLEtBQUssVUFBVTtRQUFFLE9BQU8sT0FBTyxDQUFDLGVBQWUsQ0FBQyxLQUFLLENBQUMsQ0FBQztJQUN6RixPQUFPLG1CQUFtQixDQUFDLEtBQUssQ0FBQyxZQUFZLEVBQUUsT0FBTyxDQUFDLEdBQUcsSUFBSSxPQUFPLENBQUMsR0FBRyxDQUFDLENBQUM7QUFDN0UsQ0FBQztBQUVELFNBQVMsa0JBQWtCLENBQUMsWUFBb0I7SUFDOUMsT0FBTyxFQUFFLFVBQVUsRUFBRSxHQUFHLEVBQUUsQ0FBQyxFQUFFLEVBQUUsQ0FBQztBQUNsQyxDQUFDO0FBRUQ7O3FHQUVxRztBQUNyRyxTQUFTLHFCQUFxQixDQUM1QixZQUFvQixFQUNwQixnQkFBd0IsRUFDeEIsVUFBaUQ7SUFFakQsT0FBTztRQUNMLFlBQVk7UUFDWixLQUFLLEVBQUU7WUFDTCxNQUFNLEVBQUUsQ0FBQztZQUNULEtBQUssRUFBRSwyQ0FBMkM7WUFDbEQsSUFBSSxFQUFFLGlFQUFpRTtZQUN2RSxNQUFNLEVBQUUsQ0FBQyxLQUFLLENBQUM7U0FDaEI7UUFDRCxPQUFPLEVBQUUsRUFBRSxNQUFNLEVBQUUsQ0FBQyxFQUFFLE1BQU0sRUFBRSxDQUFDLEVBQUUsQ0FBQyxFQUFFLFlBQVksRUFBRSxFQUFFLEVBQUU7UUFDdEQsV0FBVyxFQUFFLGtCQUFrQixDQUFDLFlBQVksQ0FBQztRQUM3QyxnQkFBZ0I7UUFDaEIsZUFBZSxFQUFFLFVBQVU7S0FDNUIsQ0FBQztBQUNKLENBQUM7QUFFRDs7R0FFRztBQUNILE1BQU0sVUFBVSxxQkFBcUIsQ0FDbkMsS0FBc0MsRUFDdEMsVUFBd0MsRUFBRTtJQUUxQyxNQUFNLFlBQVksR0FBRyxLQUFLLEVBQUUsWUFBWSxDQUFDO0lBQ3pDLElBQUksT0FBTyxZQUFZLEtBQUssUUFBUSxJQUFJLENBQUMsMEJBQTBCLENBQUMsWUFBWSxDQUFDLEVBQUUsQ0FBQztRQUNsRixPQUFPLFlBQVksQ0FDakIsT0FBTyxZQUFZLEtBQUssUUFBUSxDQUFDLENBQUMsQ0FBQyxZQUFZLENBQUMsQ0FBQyxDQUFDLFdBQVcsRUFDN0QsMkJBQTJCLENBQUMsS0FBSyxFQUNqQyxxREFBcUQsQ0FDdEQsQ0FBQztJQUNKLENBQUM7SUFFRCxNQUFNLFVBQVUsR0FBRyxPQUFPLENBQUMsVUFBVSxJQUFJLFVBQVUsQ0FBQztJQUNwRCxNQUFNLFVBQVUsR0FBRyxPQUFPLENBQUMsZUFBZSxJQUFJLGVBQWUsQ0FBQztJQUM5RCxNQUFNLFlBQVksR0FBRyxPQUFPLENBQUMsb0JBQW9CLElBQUksb0JBQW9CLENBQUM7SUFDMUUsTUFBTSxhQUFhLEdBQ2pCLE9BQU8sQ0FBQyxtQkFBbUI7UUFDMUIsbUJBQW1HLENBQUM7SUFDdkcsTUFBTSxRQUFRLEdBQUcseUJBQXlCLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFDO0lBRTNELElBQUksQ0FBQyxVQUFVLENBQUMsUUFBUSxDQUFDLEVBQUUsQ0FBQztRQUMxQixPQUFPLFlBQVksQ0FDakIsWUFBWSxFQUNaLDJCQUEyQixDQUFDLFdBQVcsRUFDdkMsbUNBQW1DLFFBQVEsd0RBQXdELENBQ3BHLENBQUM7SUFDSixDQUFDO0lBRUQsTUFBTSxRQUFRLEdBQUcsWUFBWSxDQUFDLFFBQVEsQ0FBQyxDQUFDO0lBQ3hDLE1BQU0sbUJBQW1CLEdBQUcsUUFBUSxFQUFFLE9BQU8sS0FBSyxJQUFJLENBQUM7SUFFdkQsTUFBTSxLQUFLLEdBQUcsVUFBVSxDQUFDLFFBQVEsQ0FBQyxDQUFDO0lBQ25DLElBQUksS0FBSyxFQUFFLFFBQVEsS0FBSyxJQUFJLEVBQUUsQ0FBQztRQUM3QixPQUFPLFlBQVksQ0FDakIsWUFBWSxFQUNaLDJCQUEyQixDQUFDLGVBQWUsRUFDM0MsS0FBSyxFQUFFLE1BQU0sSUFBSSx5REFBeUQsRUFDMUUsRUFBRSxhQUFhLEVBQUUsS0FBSyxFQUFFLG1CQUFtQixFQUFFLENBQzlDLENBQUM7SUFDSixDQUFDO0lBRUQsSUFBSSxLQUFLLENBQUMsa0JBQWtCLEtBQUssSUFBSSxJQUFJLENBQUMsS0FBSyxDQUFDLFdBQVcsRUFBRSxDQUFDO1FBQzVELE9BQU8sWUFBWSxDQUNqQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsU0FBUyxFQUNyQyw2RkFBNkYsRUFDN0YsRUFBRSxhQUFhLEVBQUUsSUFBSSxFQUFFLG1CQUFtQixFQUFFLEtBQUssRUFBRSxDQUNwRCxDQUFDO0lBQ0osQ0FBQztJQUVELElBQUksVUFBVSxDQUFDO0lBQ2YsSUFBSSxDQUFDO1FBQ0gsVUFBVSxHQUFHLGFBQWEsQ0FBQyxxQkFBcUIsQ0FBQyxZQUFZLEVBQUUsUUFBUSxFQUFFLFVBQVUsQ0FBQyxDQUFDLENBQUM7SUFDeEYsQ0FBQztJQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7UUFDZixPQUFPLFlBQVksQ0FBQyxZQUFZLEVBQUUsMkJBQTJCLENBQUMsS0FBSyxFQUFFLGFBQWEsQ0FBQyxLQUFLLENBQUMsRUFBRTtZQUN6RixhQUFhLEVBQUUsSUFBSTtZQUNuQixtQkFBbUI7WUFDbkIsS0FBSztTQUNOLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRCxJQUFJLFVBQVUsRUFBRSxLQUFLLEtBQUssSUFBSSxFQUFFLENBQUM7UUFDL0IsT0FBTyxZQUFZLENBQ2pCLFlBQVksRUFDWiwyQkFBMkIsQ0FBQyxTQUFTLEVBQ3JDLDJDQUEyQyxVQUFVLEVBQUUsT0FBTyxJQUFJLFNBQVMsSUFBSSxFQUMvRSxFQUFFLGFBQWEsRUFBRSxJQUFJLEVBQUUsbUJBQW1CLEVBQUUsS0FBSyxFQUFFLENBQ3BELENBQUM7SUFDSixDQUFDO0lBRUQsTUFBTSxrQkFBa0IsR0FBRywrQkFBK0IsQ0FBQyxVQUFVLENBQUMsWUFBWSxJQUFJLEVBQUUsQ0FBQyxDQUFDO0lBQzFGLElBQUksa0JBQWtCLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDO1FBQ2xDLE9BQU8sWUFBWSxDQUNqQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsb0JBQW9CLEVBQ2hELDBEQUEwRCxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksRUFDNUcsRUFBRSxhQUFhLEVBQUUsSUFBSSxFQUFFLG1CQUFtQixFQUFFLEtBQUssRUFBRSxrQkFBa0IsRUFBRSxDQUN4RSxDQUFDO0lBQ0osQ0FBQztJQUVELE9BQU8sU0FBUyxDQUFDLFlBQVksRUFBRSxFQUFFLG1CQUFtQixFQUFFLEtBQUssRUFBRSxDQUFDLENBQUM7QUFDakUsQ0FBQztBQUVEOztHQUVHO0FBQ0gsTUFBTSxVQUFVLHNCQUFzQixDQUNwQyxNQUF5QyxFQUN6QyxVQUFrRSxFQUFFO0lBRXBFLE1BQU0sS0FBSyxHQUFHLE1BQU0sRUFBRSxRQUFRLEVBQUUsS0FBSyxJQUFJLEVBQUUsQ0FBQztJQUM1QyxNQUFNLE9BQU8sR0FBZ0MsRUFBRSxDQUFDO0lBQ2hELEtBQUssTUFBTSxLQUFLLElBQUksS0FBSyxFQUFFLENBQUM7UUFDMUIsSUFBSSxPQUFPLENBQUMsVUFBVSxJQUFJLEtBQUssQ0FBQyxZQUFZLEtBQUssT0FBTyxDQUFDLFVBQVU7WUFBRSxTQUFTO1FBQzlFLE9BQU8sQ0FBQyxJQUFJLENBQUMscUJBQXFCLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFDLENBQUM7SUFDdEQsQ0FBQztJQUNELE9BQU8sT0FBTyxDQUFDO0FBQ2pCLENBQUM7QUFFRDs7R0FFRztBQUNILE1BQU0sVUFBVSw0QkFBNEIsQ0FBQyxPQUFvQztJQUMvRSxNQUFNLElBQUksR0FBRyxLQUFLLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztJQUNuRCxJQUFJLE1BQU0sR0FBRyxDQUFDLENBQUM7SUFDZixJQUFJLE1BQU0sR0FBRyxDQUFDLENBQUM7SUFDZixNQUFNLGtCQUFrQixHQUEyQixFQUFFLENBQUM7SUFDdEQsS0FBSyxNQUFNLE1BQU0sSUFBSSxJQUFJLEVBQUUsQ0FBQztRQUMxQixJQUFJLE1BQU0sRUFBRSxNQUFNLEtBQUssSUFBSSxFQUFFLENBQUM7WUFDNUIsTUFBTSxJQUFJLENBQUMsQ0FBQztZQUNaLFNBQVM7UUFDWCxDQUFDO1FBQ0QsTUFBTSxJQUFJLENBQUMsQ0FBQztRQUNaLE1BQU0sUUFBUSxHQUFHLE1BQU0sRUFBRSxlQUFlLElBQUksMkJBQTJCLENBQUMsS0FBSyxDQUFDO1FBQzlFLGtCQUFrQixDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsa0JBQWtCLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQ3pFLENBQUM7SUFDRCxNQUFNLEtBQUssR0FBRyxNQUFNLEdBQUcsTUFBTSxDQUFDO0lBQzlCLE1BQU0sY0FBYyxHQUFHLEtBQUssR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLE1BQU0sR0FBRyxNQUFNLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQztJQUMzRCxNQUFNLHFCQUFxQixHQUFHLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsRUFBRSxtQkFBbUIsS0FBSyxLQUFLLENBQUMsQ0FBQyxNQUFNLENBQUM7SUFDMUYsT0FBTztRQUNMLEtBQUs7UUFDTCxNQUFNO1FBQ04sTUFBTTtRQUNOLGNBQWM7UUFDZCxxQkFBcUI7UUFDckIsa0JBQWtCO0tBQ25CLENBQUM7QUFDSixDQUFDO0FBRUQ7O0dBRUc7QUFDSCxNQUFNLFVBQVUsK0JBQStCLENBQzdDLE9BQW9DLEVBQ3BDLFVBQXNDLDRCQUE0QixDQUFDLE9BQU8sQ0FBQztJQUUzRSxNQUFNLEtBQUssR0FBRyxDQUFDLHNDQUFzQyxFQUFFLEVBQUUsQ0FBQyxDQUFDO0lBQzNELEtBQUssTUFBTSxNQUFNLElBQUksT0FBTyxFQUFFLENBQUM7UUFDN0IsSUFBSSxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUM7WUFDbEIsS0FBSyxDQUFDLElBQUksQ0FBQyxRQUFRLE1BQU0sQ0FBQyxZQUFZLEVBQUUsQ0FBQyxDQUFDO1lBQzFDLFNBQVM7UUFDWCxDQUFDO1FBQ0QsS0FBSyxDQUFDLElBQUksQ0FBQyxRQUFRLE1BQU0sQ0FBQyxZQUFZLEtBQUssTUFBTSxDQUFDLGVBQWUsS0FBSyxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQztJQUN6RixDQUFDO0lBQ0QsS0FBSyxDQUFDLElBQUksQ0FDUixFQUFFLEVBQ0YsWUFBWSxPQUFPLENBQUMsTUFBTSxJQUFJLE9BQU8sQ0FBQyxLQUFLLFNBQVM7UUFDbEQsQ0FBQyxPQUFPLENBQUMsY0FBYyxDQUFDLENBQUMsQ0FBQyxvQkFBb0IsQ0FBQyxDQUFDLENBQUMsb0JBQW9CLENBQUMsQ0FDekUsQ0FBQztJQUNGLElBQUksT0FBTyxDQUFDLEtBQUssR0FBRyxDQUFDLEVBQUUsQ0FBQztRQUN0QixLQUFLLENBQUMsSUFBSSxDQUFDLDRDQUE0QyxPQUFPLENBQUMscUJBQXFCLElBQUksT0FBTyxDQUFDLEtBQUssRUFBRSxDQUFDLENBQUM7SUFDM0csQ0FBQztJQUNELE1BQU0sVUFBVSxHQUFHLE1BQU0sQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLGtCQUFrQixDQUFDLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsYUFBYSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDckcsSUFBSSxVQUFVLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDO1FBQzFCLEtBQUssQ0FBQyxJQUFJLENBQUMsRUFBRSxFQUFFLHVCQUF1QixDQUFDLENBQUM7UUFDeEMsS0FBSyxNQUFNLENBQUMsUUFBUSxFQUFFLEtBQUssQ0FBQyxJQUFJLFVBQVUsRUFBRSxDQUFDO1lBQzNDLEtBQUssQ0FBQyxJQUFJLENBQUMsS0FBSyxRQUFRLEtBQUssS0FBSyxFQUFFLENBQUMsQ0FBQztRQUN4QyxDQUFDO0lBQ0gsQ0FBQztJQUNELE9BQU8sS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztBQUMxQixDQUFDO0FBRUQsa0hBQWtIO0FBQ2xILGdIQUFnSDtBQUNoSCw2R0FBNkc7QUFDN0csK0dBQStHO0FBQy9HLGdIQUFnSDtBQUNoSCwwRkFBMEY7QUFFMUYsNEdBQTRHO0FBQzVHLE1BQU0sQ0FBQyxNQUFNLHFDQUFxQyxHQUs3QyxNQUFNLENBQUMsTUFBTSxDQUFDO0lBQ2pCLFNBQVMsRUFBRSxrQkFBa0I7SUFDN0IsU0FBUyxFQUFFLFdBQVc7SUFDdEIsS0FBSyxFQUFFLGNBQWM7SUFDckIsSUFBSSxFQUFFLGFBQWE7Q0FDcEIsQ0FBQyxDQUFDO0FBRUg7Z0VBQ2dFO0FBQ2hFLE1BQU0sQ0FBQyxNQUFNLHNDQUFzQyxHQUFXLEVBQUUsQ0FBQztBQUNqRTtrRkFDa0Y7QUFDbEYsTUFBTSxDQUFDLE1BQU0sK0NBQStDLEdBQVcsT0FBTyxDQUFDO0FBb0QvRTtzRkFDc0Y7QUFDdEYsTUFBTSxVQUFVLGdDQUFnQyxDQUFDLFFBQWdCO0lBQy9ELE1BQU0sV0FBVyxHQUFHLFdBQVcsQ0FBQyxJQUFJLENBQUMsTUFBTSxFQUFFLEVBQUUsMkJBQTJCLENBQUMsQ0FBQyxDQUFDO0lBQzdFLE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxXQUFXLEVBQUUsTUFBTSxDQUFDLENBQUM7SUFDdkMsTUFBTSxDQUFDLFFBQVEsRUFBRSxJQUFJLEVBQUUsRUFBRSxTQUFTLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQztJQUM1QyxPQUFPO1FBQ0wsSUFBSTtRQUNKLE9BQU8sRUFBRSxHQUFHLEVBQUU7WUFDWixNQUFNLENBQUMsV0FBVyxFQUFFLEVBQUUsU0FBUyxFQUFFLElBQUksRUFBRSxLQUFLLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQztRQUN4RCxDQUFDO0tBQ0YsQ0FBQztBQUNKLENBQUM7QUFFRDs7Ozs7OytDQU0rQztBQUMvQyxNQUFNLFVBQVUseUNBQXlDO0lBQ3ZELE9BQU8sQ0FBQyxPQUFPLEVBQUUsT0FBTyxFQUFFLEVBQUUsQ0FDMUIsSUFBSSxPQUFPLENBQUMsQ0FBQyxPQUFPLEVBQUUsRUFBRTtRQUN0QixNQUFNLENBQUMsVUFBVSxFQUFFLEdBQUcsSUFBSSxDQUFDLEdBQUcsT0FBTyxDQUFDLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQyxNQUFNLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDbkUsSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDO1lBQ2hCLE9BQU8sQ0FBQyxFQUFFLE1BQU0sRUFBRSxFQUFFLEVBQUUsTUFBTSxFQUFFLGVBQWUsRUFBRSxJQUFJLEVBQUUsSUFBSSxFQUFFLFFBQVEsRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFDO1lBQzlFLE9BQU87UUFDVCxDQUFDO1FBQ0QsTUFBTSxLQUFLLEdBQUcsU0FBUyxDQUFDLFVBQVUsRUFBRSxJQUFJLEVBQUU7WUFDeEMsR0FBRyxFQUFFLE9BQU8sQ0FBQyxHQUFHO1lBQ2hCLEtBQUssRUFBRSxDQUFDLFFBQVEsRUFBRSxNQUFNLEVBQUUsTUFBTSxDQUFDO1NBQ2xDLENBQUMsQ0FBQztRQUNILElBQUksTUFBTSxHQUFHLEVBQUUsQ0FBQztRQUNoQixJQUFJLE1BQU0sR0FBRyxFQUFFLENBQUM7UUFDaEIsTUFBTSxLQUFLLEdBQUcsVUFBVSxDQUFDLEdBQUcsRUFBRTtZQUM1QixLQUFLLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxDQUFDO1lBQ3RCLE9BQU8sQ0FBQyxFQUFFLE1BQU0sRUFBRSxNQUFNLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRSxRQUFRLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQztRQUMxRCxDQUFDLEVBQUUsT0FBTyxDQUFDLFNBQVMsQ0FBQyxDQUFDO1FBQ3RCLHVHQUF1RztRQUN2RyxLQUFLLENBQUMsTUFBTyxDQUFDLEVBQUUsQ0FBQyxNQUFNLEVBQUUsQ0FBQyxLQUFzQixFQUFFLEVBQUU7WUFDbEQsTUFBTSxJQUFJLEtBQUssQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLENBQUM7UUFDbkMsQ0FBQyxDQUFDLENBQUM7UUFDSCxLQUFLLENBQUMsTUFBTyxDQUFDLEVBQUUsQ0FBQyxNQUFNLEVBQUUsQ0FBQyxLQUFzQixFQUFFLEVBQUU7WUFDbEQsTUFBTSxJQUFJLEtBQUssQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLENBQUM7UUFDbkMsQ0FBQyxDQUFDLENBQUM7UUFDSCxLQUFLLENBQUMsRUFBRSxDQUFDLE9BQU8sRUFBRSxDQUFDLEdBQVUsRUFBRSxFQUFFO1lBQy9CLFlBQVksQ0FBQyxLQUFLLENBQUMsQ0FBQztZQUNwQixPQUFPLENBQUMsRUFBRSxNQUFNLEVBQUUsTUFBTSxFQUFFLEdBQUcsQ0FBQyxPQUFPLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRSxRQUFRLEVBQUUsS0FBSyxFQUFFLENBQUMsQ0FBQztRQUN4RSxDQUFDLENBQUMsQ0FBQztRQUNILEtBQUssQ0FBQyxFQUFFLENBQUMsT0FBTyxFQUFFLENBQUMsSUFBbUIsRUFBRSxFQUFFO1lBQ3hDLFlBQVksQ0FBQyxLQUFLLENBQUMsQ0FBQztZQUNwQixPQUFPLENBQUMsRUFBRSxNQUFNLEVBQUUsTUFBTSxFQUFFLElBQUksRUFBRSxRQUFRLEVBQUUsS0FBSyxFQUFFLENBQUMsQ0FBQztRQUNyRCxDQUFDLENBQUMsQ0FBQztJQUNMLENBQUMsQ0FBQyxDQUFDO0FBQ1AsQ0FBQztBQUVELFNBQVMscUJBQXFCLENBQzVCLFlBQW9CLEVBQ3BCLFFBQWdCLEVBQ2hCLE1BQWMsRUFDZCxJQUF3QyxFQUN4QyxTQUEyQztJQUUzQyxPQUFPLEVBQUUsR0FBRyxZQUFZLENBQUMsWUFBWSxFQUFFLFFBQVEsRUFBRSxNQUFNLEVBQUUsSUFBSSxDQUFDLEVBQUUsU0FBUyxFQUFFLENBQUM7QUFDOUUsQ0FBQztBQUVEOzs7O0dBSUc7QUFDSCxNQUFNLENBQUMsS0FBSyxVQUFVLHFCQUFxQixDQUN6QyxLQUFzQyxFQUN0QyxVQUF3QyxFQUFFO0lBRTFDLE1BQU0sU0FBUyxHQUFHLHFCQUFxQixDQUFDLEtBQUssRUFBRSxPQUFPLENBQUMsQ0FBQztJQUN4RCxJQUFJLENBQUMsU0FBUyxDQUFDLE1BQU07UUFBRSxPQUFPLEVBQUUsR0FBRyxTQUFTLEVBQUUsU0FBUyxFQUFFLElBQUksRUFBRSxDQUFDO0lBRWhFLE1BQU0sZ0JBQWdCLEdBQUcsT0FBTyxDQUFDLHlCQUF5QixJQUFJLGdDQUFnQyxDQUFDO0lBQy9GLElBQUksU0FBc0MsQ0FBQztJQUMzQyxJQUFJLENBQUM7UUFDSCxTQUFTLEdBQUcsZ0JBQWdCLENBQUMseUJBQXlCLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFDLENBQUM7SUFDMUUsQ0FBQztJQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7UUFDZixNQUFNLE1BQU0sR0FBRyxvREFBb0QsYUFBYSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDMUYsOEZBQThGO1FBQzlGLE1BQU0sSUFBSSxHQUFHLEVBQUUsYUFBYSxFQUFFLElBQUksRUFBRSxtQkFBbUIsRUFBRSxTQUFTLENBQUMsbUJBQW1CLEVBQUUsS0FBSyxFQUFFLFNBQVMsQ0FBQyxLQUFNLEVBQUUsQ0FBQztRQUNsSCxPQUFPLHFCQUFxQixDQUFDLFNBQVMsQ0FBQyxZQUFZLEVBQUUsMkJBQTJCLENBQUMsV0FBVyxFQUFFLE1BQU0sRUFBRSxJQUFJLEVBQUUsSUFBSSxDQUFDLENBQUM7SUFDcEgsQ0FBQztJQUNELElBQUksQ0FBQztRQUNILE9BQU8sTUFBTSxrQkFBa0IsQ0FBQyxTQUFTLEVBQUUsU0FBUyxFQUFFLE9BQU8sQ0FBQyxDQUFDO0lBQ2pFLENBQUM7WUFBUyxDQUFDO1FBQ1QsSUFBSSxDQUFDO1lBQ0gsU0FBUyxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ3RCLENBQUM7UUFBQyxNQUFNLENBQUM7WUFDUCx5RkFBeUY7UUFDM0YsQ0FBQztJQUNILENBQUM7QUFDSCxDQUFDO0FBRUQsS0FBSyxVQUFVLGtCQUFrQixDQUMvQixTQUFvQyxFQUNwQyxTQUFzQyxFQUN0QyxPQUFxQztJQUVyQyxNQUFNLFlBQVksR0FBRyxTQUFTLENBQUMsWUFBWSxDQUFDO0lBQzVDLHNHQUFzRztJQUN0RyxNQUFNLEtBQUssR0FBRyxTQUFTLENBQUMsS0FBMEIsQ0FBQztJQUNuRCxNQUFNLG1CQUFtQixHQUFHLFNBQVMsQ0FBQyxtQkFBbUIsQ0FBQztJQUMxRCxNQUFNLFdBQVcsR0FBRyxFQUFFLGFBQWEsRUFBRSxJQUFJLEVBQUUsbUJBQW1CLEVBQUUsS0FBSyxFQUFFLENBQUM7SUFFeEUsTUFBTSxVQUFVLEdBQUcsT0FBTyxDQUFDLGVBQWUsSUFBSSxlQUFlLENBQUM7SUFDOUQsTUFBTSxhQUFhLEdBQ2pCLE9BQU8sQ0FBQyxtQkFBbUI7UUFDMUIsbUJBQW1HLENBQUM7SUFFdkcsOEdBQThHO0lBQzlHLGdIQUFnSDtJQUNoSCwyR0FBMkc7SUFDM0csOEdBQThHO0lBQzlHLCtFQUErRTtJQUMvRSxNQUFNLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxJQUFJLEVBQUUsNEJBQTRCLENBQUMsRUFBRSxFQUFFLEtBQUssRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDO0lBQzVFLElBQUksVUFBVSxDQUFDO0lBQ2YsSUFBSSxDQUFDO1FBQ0gsVUFBVSxHQUFHLGFBQWEsQ0FBQyxxQkFBcUIsQ0FBQyxZQUFZLEVBQUUsU0FBUyxDQUFDLElBQUksRUFBRSxVQUFVLENBQUMsQ0FBQyxDQUFDO0lBQzlGLENBQUM7SUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1FBQ2YsT0FBTyxxQkFBcUIsQ0FDMUIsWUFBWSxFQUNaLDJCQUEyQixDQUFDLEtBQUssRUFDakMseURBQXlELGFBQWEsQ0FBQyxLQUFLLENBQUMsRUFBRSxFQUMvRSxXQUFXLEVBQ1gsSUFBSSxDQUNMLENBQUM7SUFDSixDQUFDO0lBQ0QsSUFBSSxVQUFVLEVBQUUsS0FBSyxLQUFLLElBQUksRUFBRSxDQUFDO1FBQy9CLE9BQU8scUJBQXFCLENBQzFCLFlBQVksRUFDWiwyQkFBMkIsQ0FBQyxTQUFTLEVBQ3JDLHdFQUF3RSxVQUFVLEVBQUUsT0FBTyxJQUFJLFNBQVMsSUFBSSxFQUM1RyxXQUFXLEVBQ1gsSUFBSSxDQUNMLENBQUM7SUFDSixDQUFDO0lBRUQsSUFBSSxNQUFNLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQztJQUM1QixJQUFJLENBQUMsTUFBTSxFQUFFLENBQUM7UUFDWixJQUFJLENBQUM7WUFDSCx3R0FBd0c7WUFDeEcsTUFBTSxFQUFFLG9DQUFvQyxFQUFFLEdBQUcsTUFBTSxNQUFNLENBQUMsZ0NBQWdDLENBQUMsQ0FBQztZQUNoRyxNQUFNLEdBQUcsb0NBQW9DLENBQUMsT0FBTyxDQUFDLEdBQUcsSUFBSSxPQUFPLENBQUMsR0FBRyxDQUFDLENBQUM7UUFDNUUsQ0FBQztRQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7WUFDZixPQUFPLHFCQUFxQixDQUMxQixZQUFZLEVBQ1oscUNBQXFDLENBQUMsU0FBUyxFQUMvQyxvQ0FBb0MsYUFBYSxDQUFDLEtBQUssQ0FBQyxFQUFFLEVBQzFELFdBQVcsRUFDWCxFQUFFLFNBQVMsRUFBRSxLQUFLLEVBQUUsZ0JBQWdCLEVBQUUsSUFBSSxFQUFFLFFBQVEsRUFBRSxLQUFLLEVBQUUsT0FBTyxFQUFFLEtBQUssRUFBRSxDQUM5RSxDQUFDO1FBQ0osQ0FBQztJQUNILENBQUM7SUFFRCxNQUFNLElBQUksR0FBRztRQUNYLFNBQVMsRUFBRSxtQkFBbUIsWUFBWSxDQUFDLE9BQU8sQ0FBQyxHQUFHLEVBQUUsR0FBRyxDQUFDLEVBQUU7UUFDOUQsZ0JBQWdCLEVBQUUsU0FBUyxDQUFDLElBQUk7UUFDaEMsc0JBQXNCLEVBQUUsVUFBVSxDQUFDLHNCQUFzQixJQUFJLElBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxFQUFFLHdCQUF3QixDQUFDO1FBQzNHLFlBQVksRUFBRSxVQUFVLENBQUMsWUFBWSxJQUFJLEVBQUU7UUFDM0MsUUFBUSxFQUFFLE9BQU8sQ0FBQyxRQUFRLElBQUksc0NBQXNDO0tBQ3JFLENBQUM7SUFDRixJQUFJLFdBQVcsQ0FBQztJQUNoQixJQUFJLENBQUM7UUFDSCxXQUFXLEdBQUcsTUFBTSxNQUFNLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBQ3ZDLENBQUM7SUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1FBQ2YsT0FBTyxxQkFBcUIsQ0FDMUIsWUFBWSxFQUNaLHFDQUFxQyxDQUFDLFNBQVMsRUFDL0MsMkJBQTJCLGFBQWEsQ0FBQyxLQUFLLENBQUMsRUFBRSxFQUNqRCxXQUFXLEVBQ1gsRUFBRSxTQUFTLEVBQUUsSUFBSSxFQUFFLGdCQUFnQixFQUFFLElBQUksRUFBRSxRQUFRLEVBQUUsS0FBSyxFQUFFLE9BQU8sRUFBRSxLQUFLLEVBQUUsQ0FDN0UsQ0FBQztJQUNKLENBQUM7SUFDRCxJQUFJLFdBQVcsRUFBRSxFQUFFLEtBQUssSUFBSSxFQUFFLENBQUM7UUFDN0IsT0FBTyxxQkFBcUIsQ0FDMUIsWUFBWSxFQUNaLHFDQUFxQyxDQUFDLFNBQVMsRUFDL0MscUNBQXFDLFdBQVcsRUFBRSxLQUFLLElBQUksV0FBVyxFQUFFLE9BQU8sSUFBSSw0QkFBNEIsRUFBRSxFQUNqSCxXQUFXLEVBQ1gsRUFBRSxTQUFTLEVBQUUsSUFBSSxFQUFFLGdCQUFnQixFQUFFLElBQUksRUFBRSxRQUFRLEVBQUUsS0FBSyxFQUFFLE9BQU8sRUFBRSxLQUFLLEVBQUUsQ0FDN0UsQ0FBQztJQUNKLENBQUM7SUFFRCxNQUFNLGdCQUFnQixHQUFHLEtBQUssQ0FBQyxPQUFPLENBQUMsV0FBVyxDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUMsQ0FBQyxXQUFXLENBQUMsWUFBWSxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDO0lBQ3ZHLElBQUksZ0JBQWdCLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDM0IsT0FBTyxxQkFBcUIsQ0FDMUIsWUFBWSxFQUNaLHFDQUFxQyxDQUFDLFNBQVMsRUFDL0MscUZBQXFGLEVBQ3JGLFdBQVcsRUFDWCxFQUFFLFNBQVMsRUFBRSxJQUFJLEVBQUUsZ0JBQWdCLEVBQUUsQ0FBQyxFQUFFLFFBQVEsRUFBRSxLQUFLLEVBQUUsT0FBTyxFQUFFLEtBQUssRUFBRSxDQUMxRSxDQUFDO0lBQ0osQ0FBQztJQUVELE1BQU0sVUFBVSxHQUFHLE9BQU8sQ0FBQyxVQUFVLElBQUkseUNBQXlDLEVBQUUsQ0FBQztJQUNyRixNQUFNLGdCQUFnQixHQUFHLE9BQU8sQ0FBQyxnQkFBZ0IsSUFBSSwrQ0FBK0MsQ0FBQztJQUVyRyxJQUFJLFFBQVEsR0FBRyxLQUFLLENBQUM7SUFDckIsSUFBSSxLQUFLLENBQUMsWUFBWSxFQUFFLENBQUM7UUFDdkIsUUFBUSxHQUFHLElBQUksQ0FBQztRQUNoQixNQUFNLFdBQVcsR0FBRyxNQUFNLFVBQVUsQ0FBQyxLQUFLLENBQUMsWUFBWSxFQUFFLEVBQUUsR0FBRyxFQUFFLFNBQVMsQ0FBQyxJQUFJLEVBQUUsU0FBUyxFQUFFLGdCQUFnQixFQUFFLENBQUMsQ0FBQztRQUMvRyxJQUFJLFdBQVcsQ0FBQyxRQUFRLElBQUksV0FBVyxDQUFDLElBQUksS0FBSyxDQUFDLEVBQUUsQ0FBQztZQUNuRCxPQUFPLHFCQUFxQixDQUMxQixZQUFZLEVBQ1oscUNBQXFDLENBQUMsS0FBSyxFQUMzQyxnQ0FBZ0MsV0FBVyxDQUFDLFFBQVEsQ0FBQyxDQUFDLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxnQkFBZ0IsV0FBVyxDQUFDLElBQUksR0FBRyxLQUFLLEtBQUssQ0FBQyxZQUFZLEVBQUUsRUFDakksV0FBVyxFQUNYLEVBQUUsU0FBUyxFQUFFLElBQUksRUFBRSxnQkFBZ0IsRUFBRSxRQUFRLEVBQUUsSUFBSSxFQUFFLE9BQU8sRUFBRSxLQUFLLEVBQUUsQ0FDdEUsQ0FBQztRQUNKLENBQUM7SUFDSCxDQUFDO0lBRUQsSUFBSSxDQUFDLEtBQUssQ0FBQyxXQUFXLEVBQUUsQ0FBQztRQUN2QixPQUFPLHFCQUFxQixDQUMxQixZQUFZLEVBQ1osMkJBQTJCLENBQUMsU0FBUyxFQUNyQyx3R0FBd0csRUFDeEcsV0FBVyxFQUNYLEVBQUUsU0FBUyxFQUFFLElBQUksRUFBRSxnQkFBZ0IsRUFBRSxRQUFRLEVBQUUsT0FBTyxFQUFFLEtBQUssRUFBRSxDQUNoRSxDQUFDO0lBQ0osQ0FBQztJQUNELE1BQU0sVUFBVSxHQUFHLE1BQU0sVUFBVSxDQUFDLEtBQUssQ0FBQyxXQUFXLEVBQUUsRUFBRSxHQUFHLEVBQUUsU0FBUyxDQUFDLElBQUksRUFBRSxTQUFTLEVBQUUsZ0JBQWdCLEVBQUUsQ0FBQyxDQUFDO0lBQzdHLElBQUksVUFBVSxDQUFDLFFBQVEsSUFBSSxVQUFVLENBQUMsSUFBSSxLQUFLLENBQUMsRUFBRSxDQUFDO1FBQ2pELE9BQU8scUJBQXFCLENBQzFCLFlBQVksRUFDWixxQ0FBcUMsQ0FBQyxJQUFJLEVBQzFDLGdEQUFnRCxVQUFVLENBQUMsUUFBUSxDQUFDLENBQUMsQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLGdCQUFnQixVQUFVLENBQUMsSUFBSSxHQUFHLEtBQUssS0FBSyxDQUFDLFdBQVcsRUFBRSxFQUM5SSxXQUFXLEVBQ1gsRUFBRSxTQUFTLEVBQUUsSUFBSSxFQUFFLGdCQUFnQixFQUFFLFFBQVEsRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLENBQy9ELENBQUM7SUFDSixDQUFDO0lBRUQsT0FBTztRQUNMLEdBQUcsU0FBUyxDQUFDLFlBQVksRUFBRSxFQUFFLG1CQUFtQixFQUFFLEtBQUssRUFBRSxDQUFDO1FBQzFELFNBQVMsRUFBRSxFQUFFLFNBQVMsRUFBRSxJQUFJLEVBQUUsZ0JBQWdCLEVBQUUsUUFBUSxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUU7S0FDMUUsQ0FBQztBQUNKLENBQUM7QUFFRDs7O0dBR0c7QUFDSCxNQUFNLENBQUMsS0FBSyxVQUFVLHlCQUF5QixDQUM3QyxNQUF5QyxFQUN6QyxVQUFrRSxFQUFFO0lBRXBFLE1BQU0sS0FBSyxHQUFHLE1BQU0sRUFBRSxRQUFRLEVBQUUsS0FBSyxJQUFJLEVBQUUsQ0FBQztJQUM1QyxNQUFNLE9BQU8sR0FBeUMsRUFBRSxDQUFDO0lBQ3pELEtBQUssTUFBTSxLQUFLLElBQUksS0FBSyxFQUFFLENBQUM7UUFDMUIsSUFBSSxPQUFPLENBQUMsVUFBVSxJQUFJLEtBQUssQ0FBQyxZQUFZLEtBQUssT0FBTyxDQUFDLFVBQVU7WUFBRSxTQUFTO1FBQzlFLE9BQU8sQ0FBQyxJQUFJLENBQUMsTUFBTSxxQkFBcUIsQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLENBQUMsQ0FBQztJQUM1RCxDQUFDO0lBQ0QsT0FBTyxPQUFPLENBQUM7QUFDakIsQ0FBQyJ9 \ No newline at end of file diff --git a/packages/loopover-miner/lib/cross-repo-evaluation.ts b/packages/loopover-miner/lib/cross-repo-evaluation.ts index f579b18aef..61fdf68c6d 100644 --- a/packages/loopover-miner/lib/cross-repo-evaluation.ts +++ b/packages/loopover-miner/lib/cross-repo-evaluation.ts @@ -4,13 +4,17 @@ // resolveMinerGoalSpec, buildCodingTaskSpec) and failures are categorized as stack-detection gaps, execution // readiness gaps, leaked loopover assumptions in agent instructions, clone/setup problems, or other. -import { existsSync } from "node:fs"; +import { spawn as nodeSpawn } from "node:child_process"; +import { cpSync, existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; +// Already a transitive dependency via coding-task-spec.js's own engine imports -- this adds no load weight. +import { ACCEPTANCE_CRITERIA_FILENAME } from "@loopover/engine"; import { buildCodingTaskSpec } from "./coding-task-spec.js"; import { resolveMinerGoalSpec } from "./miner-goal-spec.js"; import { isValidRepoSegment, resolveRepoCloneDir } from "./repo-clone.js"; import { detectRepoStack } from "./stack-detection.js"; -import type { RepoStackResult } from "./stack-detection.js"; +import type { DetectedRepoStack, RepoStackResult } from "./stack-detection.js"; /** Failure taxonomy surfaced in per-repo reports (#4788). */ export const CROSS_REPO_FAILURE_CATEGORY: Readonly<{ @@ -84,6 +88,7 @@ type EvaluateRepoReadinessOptions = { ready: boolean; verdict?: string; instructions?: string; + acceptanceCriteriaPath?: string; }; }; @@ -231,6 +236,11 @@ export function scanPositiveLoopoverAssumptions(text: string): Array<{ id: strin return findings; } +/** One shared thrown-value formatter so every catch in both harness modes carries the same two branches. */ +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + function buildFailure( repoFullName: string, category: string, @@ -276,6 +286,29 @@ function defaultClaimLedger(repoFullName: string): { listClaims: () => never[] } return { listClaims: () => [] }; } +/** The synthetic-issue spec input both harness modes hand to buildCodingTaskSpec — readiness composes it against + * the benchmark clone, full-execution mode against the scratch workspace copy (#7634), so the acceptance-criteria + * file buildCodingTaskSpec writes lands inside whichever tree the caller is actually working in. */ +function buildHarnessSpecInput( + repoFullName: string, + workingDirectory: string, + detectImpl: (repoPath: string) => RepoStackResult, +): Record { + return { + repoFullName, + issue: { + number: 1, + title: "Cross-repo evaluation harness smoke issue", + body: "Synthetic issue used only by the cross-repo evaluation harness.", + labels: ["bug"], + }, + context: { issues: [{ number: 1 }], pullRequests: [] }, + claimLedger: defaultClaimLedger(repoFullName), + workingDirectory, + detectRepoStack: detectImpl, + }; +} + /** * Evaluate one benchmark repo's miner readiness without running a live coding agent (#4788). */ @@ -332,22 +365,9 @@ export function evaluateRepoReadiness( let specResult; try { - specResult = buildSpecImpl({ - repoFullName, - issue: { - number: 1, - title: "Cross-repo evaluation harness smoke issue", - body: "Synthetic issue used only by the cross-repo evaluation harness.", - labels: ["bug"], - }, - context: { issues: [{ number: 1 }], pullRequests: [] }, - claimLedger: defaultClaimLedger(repoFullName), - workingDirectory: repoPath, - detectRepoStack: detectImpl, - }); + specResult = buildSpecImpl(buildHarnessSpecInput(repoFullName, repoPath, detectImpl)); } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return buildFailure(repoFullName, CROSS_REPO_FAILURE_CATEGORY.OTHER, message, { + return buildFailure(repoFullName, CROSS_REPO_FAILURE_CATEGORY.OTHER, describeError(error), { stackDetected: true, usedDefaultGoalSpec, stack, @@ -454,3 +474,342 @@ export function formatCrossRepoEvaluationReport( } return lines.join("\n"); } + +// --------------------------------------------------------------------------------------------------------------- +// Full-execution mode (#7634): past readiness, actually run the discover -> plan -> code -> test loop against a +// benchmark repo — dry-run only. The attempt runs inside a scratch COPY of the local clone that is discarded +// afterward: no live PR submission, no forge API calls, and the benchmark clone itself is never mutated by the +// agent or the test run. Same taxonomy extension point as the readiness categories above — summarize/format are +// data-driven over category strings, so these flow through the existing report unchanged. + +/** Execution-specific failure taxonomy (#7634), extending — not replacing — CROSS_REPO_FAILURE_CATEGORY. */ +export const CROSS_REPO_EXECUTION_FAILURE_CATEGORY: Readonly<{ + AGENT_RUN: "agent_run_failed"; + NOOP_DIFF: "noop_diff"; + BUILD: "build_failed"; + TEST: "test_failed"; +}> = Object.freeze({ + AGENT_RUN: "agent_run_failed", + NOOP_DIFF: "noop_diff", + BUILD: "build_failed", + TEST: "test_failed", +}); + +/** A benchmark attempt works a small synthetic issue, so a modest turn cap keeps dry-runs bounded without + * starving a real agent; callers tune via options.maxTurns. */ +export const DEFAULT_CROSS_REPO_EXECUTION_MAX_TURNS: number = 24; +/** Per-command (build, then test) wall-clock cap — generous enough for a cold dependency install on the larger + * benchmark repos, small enough that a hung suite cannot wedge the whole run. */ +export const DEFAULT_CROSS_REPO_EXECUTION_COMMAND_TIMEOUT_MS: number = 600_000; + +export type CrossRepoExecutionCommandResult = { + code: number | null; + stdout: string; + stderr: string; + timedOut: boolean; +}; + +export type CrossRepoExecutionRunCommandFn = ( + command: string, + options: { cwd: string; timeoutMs: number }, +) => Promise; + +export type CrossRepoExecutionWorkspace = { path: string; cleanup: () => void }; + +/** Structural mirror of the engine's CodingAgentDriver contract — kept local so this module only loads the real + * driver construction (and its engine dependency) lazily, on the one path that actually runs an agent. */ +export type CrossRepoExecutionDriver = { + run(task: { + attemptId: string; + workingDirectory: string; + acceptanceCriteriaPath: string; + instructions: string; + maxTurns: number; + }): Promise<{ + ok: boolean; + changedFiles: readonly string[]; + summary: string; + error?: string | undefined; + }>; +}; + +export type CrossRepoExecutionDetails = { + attempted: boolean; + changedFileCount: number | null; + buildRan: boolean; + testRan: boolean; +}; + +export type CrossRepoExecutionEvaluationResult = CrossRepoEvaluationResult & { + execution: CrossRepoExecutionDetails | null; +}; + +export type EvaluateRepoExecutionOptions = EvaluateRepoReadinessOptions & { + driver?: CrossRepoExecutionDriver; + prepareExecutionWorkspace?: (repoPath: string) => CrossRepoExecutionWorkspace; + runCommand?: CrossRepoExecutionRunCommandFn; + maxTurns?: number; + commandTimeoutMs?: number; +}; + +/** Copy the benchmark clone into a discardable temp tree — the agent and the repo's test suite only ever touch + * the copy, so the clone stays pristine and cleanup is a single recursive remove. */ +export function defaultPrepareExecutionWorkspace(repoPath: string): CrossRepoExecutionWorkspace { + const scratchRoot = mkdtempSync(join(tmpdir(), "loopover-cross-repo-exec-")); + const path = join(scratchRoot, "repo"); + cpSync(repoPath, path, { recursive: true }); + return { + path, + cleanup: () => { + rmSync(scratchRoot, { recursive: true, force: true }); + }, + }; +} + +/** Command runner for the stack's inferred build/test commands. detectRepoStack only ever emits simple + * `tool subcommand` forms ("npm test", "cargo build", "npm run build"), so the command is tokenized on + * whitespace and exec'd DIRECTLY -- deliberately no `shell: true`, so nothing in a benchmark repo's manifest + * can smuggle shell metacharacters into an interpreted shell line. Mirrors coding-agent-construction's + * createRealCliSubprocessSpawn otherwise: capture both streams and RESOLVE (never reject) on timeout or spawn + * error, so partial output stays diagnosable. Promise resolution is idempotent, so a `close` firing after the + * timeout already resolved needs no guard. */ +export function createDefaultCrossRepoExecutionRunCommand(): CrossRepoExecutionRunCommandFn { + return (command, options) => + new Promise((resolve) => { + const [executable, ...args] = command.split(/\s+/).filter(Boolean); + if (!executable) { + resolve({ stdout: "", stderr: "empty_command", code: null, timedOut: false }); + return; + } + const child = nodeSpawn(executable, args, { + cwd: options.cwd, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + resolve({ stdout, stderr, code: null, timedOut: true }); + }, options.timeoutMs); + // stdio is always ["ignore","pipe","pipe"] above, so both streams exist — assert instead of branching. + child.stdout!.on("data", (chunk: Buffer | string) => { + stdout += chunk.toString("utf8"); + }); + child.stderr!.on("data", (chunk: Buffer | string) => { + stderr += chunk.toString("utf8"); + }); + child.on("error", (err: Error) => { + clearTimeout(timer); + resolve({ stdout, stderr: err.message, code: null, timedOut: false }); + }); + child.on("close", (code: number | null) => { + clearTimeout(timer); + resolve({ stdout, stderr, code, timedOut: false }); + }); + }); +} + +function buildExecutionFailure( + repoFullName: string, + category: string, + reason: string, + base: Partial, + execution: CrossRepoExecutionDetails | null, +): CrossRepoExecutionEvaluationResult { + return { ...buildFailure(repoFullName, category, reason, base), execution }; +} + +/** + * Run the full discover -> plan -> code -> test loop for one benchmark repo, dry-run (#7634). Readiness gates + * first (its failures pass through unchanged); execution then happens entirely inside a scratch copy that is + * discarded in every outcome. + */ +export async function evaluateRepoExecution( + entry: CrossRepoEvaluationManifestRepo, + options: EvaluateRepoExecutionOptions = {}, +): Promise { + const readiness = evaluateRepoReadiness(entry, options); + if (!readiness.passed) return { ...readiness, execution: null }; + + const prepareWorkspace = options.prepareExecutionWorkspace ?? defaultPrepareExecutionWorkspace; + let workspace: CrossRepoExecutionWorkspace; + try { + workspace = prepareWorkspace(resolveEvaluationRepoPath(entry, options)); + } catch (error) { + const reason = `Failed to prepare a scratch execution workspace: ${describeError(error)}`; + // A passed readiness result always carries its detected stack (buildPass is invoked with it). + const base = { stackDetected: true, usedDefaultGoalSpec: readiness.usedDefaultGoalSpec, stack: readiness.stack! }; + return buildExecutionFailure(readiness.repoFullName, CROSS_REPO_FAILURE_CATEGORY.CLONE_SETUP, reason, base, null); + } + try { + return await runExecutionPhases(readiness, workspace, options); + } finally { + try { + workspace.cleanup(); + } catch { + // Best-effort discard — a cleanup failure must never mask the evaluation outcome itself. + } + } +} + +async function runExecutionPhases( + readiness: CrossRepoEvaluationResult, + workspace: CrossRepoExecutionWorkspace, + options: EvaluateRepoExecutionOptions, +): Promise { + const repoFullName = readiness.repoFullName; + // Readiness passed, so the stack is present and detected — narrow once instead of re-branching on it. + const stack = readiness.stack as DetectedRepoStack; + const usedDefaultGoalSpec = readiness.usedDefaultGoalSpec; + const failureBase = { stackDetected: true, usedDefaultGoalSpec, stack }; + + const detectImpl = options.detectRepoStack ?? detectRepoStack; + const buildSpecImpl: NonNullable = + options.buildCodingTaskSpec ?? + (buildCodingTaskSpec as unknown as NonNullable); + + // Re-compose the coding-task spec INSIDE the scratch copy: buildCodingTaskSpec writes the acceptance-criteria + // document into its workingDirectory, and the agent must find it (and work) in the tree it is allowed to touch. + // The readiness pass already wrote that document into the clone (pre-existing #4788 behavior), so the copy + // inherits it -- and writeAcceptanceCriteriaFile opens O_EXCL (never overwrites), so clear the inherited file + // first. The scratch tree is ours to mutate; the clone itself stays untouched. + rmSync(join(workspace.path, ACCEPTANCE_CRITERIA_FILENAME), { force: true }); + let specResult; + try { + specResult = buildSpecImpl(buildHarnessSpecInput(repoFullName, workspace.path, detectImpl)); + } catch (error) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_FAILURE_CATEGORY.OTHER, + `Coding task spec failed inside the scratch workspace: ${describeError(error)}`, + failureBase, + null, + ); + } + if (specResult?.ready !== true) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_FAILURE_CATEGORY.EXECUTION, + `Coding task spec is not ready inside the scratch workspace (verdict: ${specResult?.verdict ?? "unknown"}).`, + failureBase, + null, + ); + } + + let driver = options.driver; + if (!driver) { + try { + // Lazy so the readiness-only path (and its consumers) never load the engine-backed driver construction. + const { constructProductionCodingAgentDriver } = await import("./coding-agent-construction.js"); + driver = constructProductionCodingAgentDriver(options.env ?? process.env); + } catch (error) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_FAILURE_CATEGORY.AGENT_RUN, + `No runnable coding-agent driver: ${describeError(error)}`, + failureBase, + { attempted: false, changedFileCount: null, buildRan: false, testRan: false }, + ); + } + } + + const task = { + attemptId: `cross-repo-eval-${repoFullName.replace("/", "-")}`, + workingDirectory: workspace.path, + acceptanceCriteriaPath: specResult.acceptanceCriteriaPath ?? join(workspace.path, "ACCEPTANCE_CRITERIA.md"), + instructions: specResult.instructions ?? "", + maxTurns: options.maxTurns ?? DEFAULT_CROSS_REPO_EXECUTION_MAX_TURNS, + }; + let agentResult; + try { + agentResult = await driver.run(task); + } catch (error) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_FAILURE_CATEGORY.AGENT_RUN, + `Coding agent run threw: ${describeError(error)}`, + failureBase, + { attempted: true, changedFileCount: null, buildRan: false, testRan: false }, + ); + } + if (agentResult?.ok !== true) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_FAILURE_CATEGORY.AGENT_RUN, + `Coding agent run did not succeed: ${agentResult?.error ?? agentResult?.summary ?? "no failure detail reported"}`, + failureBase, + { attempted: true, changedFileCount: null, buildRan: false, testRan: false }, + ); + } + + const changedFileCount = Array.isArray(agentResult.changedFiles) ? agentResult.changedFiles.length : 0; + if (changedFileCount === 0) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_FAILURE_CATEGORY.NOOP_DIFF, + "Coding agent reported success but the generated diff is a no-op (no files changed).", + failureBase, + { attempted: true, changedFileCount: 0, buildRan: false, testRan: false }, + ); + } + + const runCommand = options.runCommand ?? createDefaultCrossRepoExecutionRunCommand(); + const commandTimeoutMs = options.commandTimeoutMs ?? DEFAULT_CROSS_REPO_EXECUTION_COMMAND_TIMEOUT_MS; + + let buildRan = false; + if (stack.buildCommand) { + buildRan = true; + const buildResult = await runCommand(stack.buildCommand, { cwd: workspace.path, timeoutMs: commandTimeoutMs }); + if (buildResult.timedOut || buildResult.code !== 0) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_FAILURE_CATEGORY.BUILD, + `Diff generated but the build ${buildResult.timedOut ? "timed out" : `failed (exit ${buildResult.code})`}: ${stack.buildCommand}`, + failureBase, + { attempted: true, changedFileCount, buildRan: true, testRan: false }, + ); + } + } + + if (!stack.testCommand) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_FAILURE_CATEGORY.EXECUTION, + "Stack detection inferred no test command, so the generated diff cannot be validated in execution mode.", + failureBase, + { attempted: true, changedFileCount, buildRan, testRan: false }, + ); + } + const testResult = await runCommand(stack.testCommand, { cwd: workspace.path, timeoutMs: commandTimeoutMs }); + if (testResult.timedOut || testResult.code !== 0) { + return buildExecutionFailure( + repoFullName, + CROSS_REPO_EXECUTION_FAILURE_CATEGORY.TEST, + `Diff generated but the repo's own test suite ${testResult.timedOut ? "timed out" : `failed (exit ${testResult.code})`}: ${stack.testCommand}`, + failureBase, + { attempted: true, changedFileCount, buildRan, testRan: true }, + ); + } + + return { + ...buildPass(repoFullName, { usedDefaultGoalSpec, stack }), + execution: { attempted: true, changedFileCount, buildRan, testRan: true }, + }; +} + +/** + * Run full-execution mode across every repo in a parsed manifest (#7634), sequentially — agent runs and test + * suites are heavyweight, so no parallel fan-out. + */ +export async function runCrossRepoFullExecution( + parsed: ParsedCrossRepoEvaluationManifest, + options: { repoFilter?: string } & EvaluateRepoExecutionOptions = {}, +): Promise { + const repos = parsed?.manifest?.repos ?? []; + const results: CrossRepoExecutionEvaluationResult[] = []; + for (const entry of repos) { + if (options.repoFilter && entry.repoFullName !== options.repoFilter) continue; + results.push(await evaluateRepoExecution(entry, options)); + } + return results; +} diff --git a/packages/loopover-miner/scripts/cross-repo-evaluation.d.mts b/packages/loopover-miner/scripts/cross-repo-evaluation.d.mts index c263aa2347..263810db73 100644 --- a/packages/loopover-miner/scripts/cross-repo-evaluation.d.mts +++ b/packages/loopover-miner/scripts/cross-repo-evaluation.d.mts @@ -1,11 +1,12 @@ import type { CrossRepoEvaluationResult, CrossRepoEvaluationSummary, + CrossRepoExecutionEvaluationResult, ParsedCrossRepoEvaluationManifest, } from "../lib/cross-repo-evaluation.js"; export type CrossRepoEvaluationCliArgs = - | { manifestPath: string; json: boolean; repoFilter: string | null; requireMajority: boolean } + | { manifestPath: string; json: boolean; repoFilter: string | null; requireMajority: boolean; fullExecution: boolean } | { error: string } | { help: true }; @@ -26,3 +27,9 @@ export declare function runCrossRepoEvaluationCli(options?: CrossRepoEvaluationC results: CrossRepoEvaluationResult[]; summary: CrossRepoEvaluationSummary; }; + +export declare function runCrossRepoFullExecutionCli(options?: CrossRepoEvaluationCliOptions): Promise<{ + parsed: ParsedCrossRepoEvaluationManifest; + results: CrossRepoExecutionEvaluationResult[]; + summary: CrossRepoEvaluationSummary; +}>; diff --git a/packages/loopover-miner/scripts/cross-repo-evaluation.mjs b/packages/loopover-miner/scripts/cross-repo-evaluation.mjs index 04de1d9fa4..bfe8e4c4d5 100644 --- a/packages/loopover-miner/scripts/cross-repo-evaluation.mjs +++ b/packages/loopover-miner/scripts/cross-repo-evaluation.mjs @@ -7,6 +7,7 @@ import { formatCrossRepoEvaluationReport, parseCrossRepoEvaluationManifest, runCrossRepoEvaluation, + runCrossRepoFullExecution, summarizeCrossRepoEvaluation, } from "../lib/cross-repo-evaluation.js"; @@ -22,6 +23,7 @@ export function parseCrossRepoEvaluationArgs(argv) { let json = false; let repoFilter = null; let requireMajority = false; + let fullExecution = false; for (let i = 0; i < args.length; i += 1) { const token = args[i]; if (token === "--json") { @@ -32,6 +34,10 @@ export function parseCrossRepoEvaluationArgs(argv) { requireMajority = true; continue; } + if (token === "--full-execution") { + fullExecution = true; + continue; + } if (token === "--manifest") { const value = args[i + 1]; if (!value) return { error: "Missing value for --manifest." }; @@ -51,7 +57,7 @@ export function parseCrossRepoEvaluationArgs(argv) { } return { error: `Unknown argument: ${token}` }; } - return { manifestPath, json, repoFilter, requireMajority }; + return { manifestPath, json, repoFilter, requireMajority, fullExecution }; } export function loadCrossRepoEvaluationManifest(manifestPath) { @@ -66,6 +72,15 @@ export function runCrossRepoEvaluationCli(options = {}) { return { parsed, results, summary }; } +/** Full-execution counterpart of runCrossRepoEvaluationCli (#7634) — same shape, async because agent runs and + * the benchmark repos' own test suites are. Dry-run: see runCrossRepoFullExecution. */ +export async function runCrossRepoFullExecutionCli(options = {}) { + const parsed = options.parsed ?? loadCrossRepoEvaluationManifest(options.manifestPath ?? resolveDefaultManifestPath()); + const results = await runCrossRepoFullExecution(parsed, { repoFilter: options.repoFilter ?? null }); + const summary = summarizeCrossRepoEvaluation(results); + return { parsed, results, summary }; +} + function printHelp() { console.log( [ @@ -77,6 +92,9 @@ function printHelp() { "Options:", " --manifest Benchmark manifest (default: benchmarks/cross-repo/manifest.json)", " --repo Evaluate a single benchmark entry", + " --full-execution Past readiness, run the coding agent + the repo's own test suite in a", + " discardable scratch copy (dry-run: no PRs, no writes to the clone).", + " Requires MINER_CODING_AGENT_PROVIDER to be configured.", " --json Emit machine-readable JSON on stdout", " --require-majority Exit 1 unless a strict majority of repos pass", " -h, --help Show this help", @@ -86,7 +104,7 @@ function printHelp() { ); } -function main() { +async function main() { const parsedArgs = parseCrossRepoEvaluationArgs(); if (parsedArgs.help) { printHelp(); @@ -97,7 +115,9 @@ function main() { return 2; } - const { parsed, results, summary } = runCrossRepoEvaluationCli(parsedArgs); + const { parsed, results, summary } = parsedArgs.fullExecution + ? await runCrossRepoFullExecutionCli(parsedArgs) + : runCrossRepoEvaluationCli(parsedArgs); if (parsedArgs.json) { console.log(JSON.stringify({ warnings: parsed.warnings, results, summary }, null, 2)); } else { @@ -112,5 +132,5 @@ function main() { } if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { - process.exitCode = main(); + process.exitCode = await main(); } diff --git a/test/unit/miner-cross-repo-evaluation.test.ts b/test/unit/miner-cross-repo-evaluation.test.ts index 1f1de6a21d..91f1ab8825 100644 --- a/test/unit/miner-cross-repo-evaluation.test.ts +++ b/test/unit/miner-cross-repo-evaluation.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -8,23 +8,35 @@ vi.mock("@loopover/engine", async () => { }); import { + CROSS_REPO_EXECUTION_FAILURE_CATEGORY, CROSS_REPO_FAILURE_CATEGORY, + DEFAULT_CROSS_REPO_EXECUTION_COMMAND_TIMEOUT_MS, + DEFAULT_CROSS_REPO_EXECUTION_MAX_TURNS, DEFAULT_CROSS_REPO_MANIFEST_RELATIVE_PATH, MAX_CROSS_REPO_MANIFEST_BYTES, + createDefaultCrossRepoExecutionRunCommand, + defaultPrepareExecutionWorkspace, + evaluateRepoExecution, formatCrossRepoEvaluationReport, evaluateRepoReadiness, normalizeCrossRepoFullName, parseCrossRepoEvaluationManifest, runCrossRepoEvaluation, + runCrossRepoFullExecution, scanPositiveLoopoverAssumptions, summarizeCrossRepoEvaluation, } from "../../packages/loopover-miner/lib/cross-repo-evaluation.js"; +import type { + CrossRepoExecutionCommandResult, + CrossRepoExecutionDriver, +} from "../../packages/loopover-miner/lib/cross-repo-evaluation.js"; import type { RepoStackResult } from "../../packages/loopover-miner/lib/stack-detection.js"; import { loadCrossRepoEvaluationManifest, parseCrossRepoEvaluationArgs, resolveDefaultManifestPath, runCrossRepoEvaluationCli, + runCrossRepoFullExecutionCli, } from "../../packages/loopover-miner/scripts/cross-repo-evaluation.mjs"; const roots: string[] = []; @@ -495,7 +507,9 @@ describe("cross-repo evaluation harness (#4788)", () => { json: true, repoFilter: "acme/widgets", requireMajority: true, + fullExecution: false, }); + expect(parseCrossRepoEvaluationArgs(["--full-execution"])).toMatchObject({ fullExecution: true }); expect(parseCrossRepoEvaluationArgs(["--manifest"])).toEqual({ error: "Missing value for --manifest." }); expect(parseCrossRepoEvaluationArgs(["--nope"])).toEqual({ error: "Unknown argument: --nope" }); expect(parseCrossRepoEvaluationArgs(["--help"])).toEqual({ help: true }); @@ -528,11 +542,440 @@ describe("cross-repo evaluation harness (#4788)", () => { }); }); + describe("full-execution mode (#7634)", () => { + /** A minimal Node fixture the real detector/spec-builder both handle. */ + function nodeFixture(scripts: Record = { test: "node --test" }) { + return tempRepo({ "package.json": pkg({ name: "exec-fixture", version: "0.0.0", scripts }) }); + } + + function entryFor(repoPath: string, name = "acme/exec") { + return { repoFullName: name, fixturePath: repoPath, requireTestCommand: false }; + } + + /** Recording driver — returns `result` (or the queue of results) and captures every task it was handed. */ + function recordingDriver(result: unknown = { ok: true, changedFiles: ["src/x.js"], summary: "done" }) { + const tasks: Array> = []; + const driver = { + run: async (task: Record) => { + tasks.push(task); + return result; + }, + } as unknown as CrossRepoExecutionDriver; + return { tasks, driver }; + } + + /** Sequential fake runCommand — shifts scripted results, recording each (command, options) call. */ + function scriptedRunCommand(script: Array>) { + const calls: Array<{ command: string; cwd: string; timeoutMs: number }> = []; + const runCommand = async (command: string, options: { cwd: string; timeoutMs: number }) => { + calls.push({ command, cwd: options.cwd, timeoutMs: options.timeoutMs }); + return { code: 0, stdout: "", stderr: "", timedOut: false, ...script.shift() }; + }; + return { calls, runCommand }; + } + + /** Reuse the fixture itself as the "scratch" tree so real detect/spec impls work inside it. */ + function inPlaceWorkspace(cleanup: () => void = () => {}) { + return (repoPath: string) => ({ path: repoPath, cleanup }); + } + + it("passes readiness failures through with execution:null and never touches the agent", async () => { + const { tasks, driver } = recordingDriver(); + const result = await evaluateRepoExecution({ repoFullName: "not-a-repo", requireTestCommand: false }, { driver }); + expect(result.passed).toBe(false); + expect(result.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.OTHER); + expect(result.execution).toBeNull(); + expect(tasks).toHaveLength(0); + }); + + it("fails clone_setup when the scratch workspace cannot be prepared (Error and non-Error throws)", async () => { + // Fresh fixture per call: the real spec builder writes ACCEPTANCE_CRITERIA.md (O_EXCL) during readiness, + // so re-evaluating the same tree would fail readiness instead of reaching the phase under test. + const errorRepo = nodeFixture(); + const viaError = await evaluateRepoExecution(entryFor(errorRepo), { + repoPath: errorRepo, + existsSync: () => true, + driver: recordingDriver().driver, + prepareExecutionWorkspace: () => { + throw new Error("no space left"); + }, + }); + expect(viaError.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.CLONE_SETUP); + expect(viaError.reason).toContain("scratch execution workspace"); + expect(viaError.reason).toContain("no space left"); + expect(viaError.execution).toBeNull(); + + const stringRepo = nodeFixture(); + const viaString = await evaluateRepoExecution(entryFor(stringRepo), { + repoPath: stringRepo, + existsSync: () => true, + driver: recordingDriver().driver, + prepareExecutionWorkspace: () => { + throw "disk-full-string"; + }, + }); + expect(viaString.reason).toContain("disk-full-string"); + }); + + it("always discards the workspace, even when cleanup itself throws, without masking the outcome", async () => { + const repoPath = nodeFixture(); + const cleanup = vi.fn(() => { + throw new Error("cleanup-fail"); + }); + const { runCommand } = scriptedRunCommand([{ code: 0 }]); + const result = await evaluateRepoExecution(entryFor(repoPath), { + repoPath, + existsSync: () => true, + prepareExecutionWorkspace: inPlaceWorkspace(cleanup), + driver: recordingDriver().driver, + runCommand, + }); + expect(result.passed).toBe(true); + expect(result.execution).toEqual({ attempted: true, changedFileCount: 1, buildRan: false, testRan: true }); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it("categorizes a spec failure inside the scratch workspace as other / execution_gap with verdict fallback", async () => { + const repoPath = nodeFixture(); + const optionsFor = (secondCall: () => unknown) => { + let calls = 0; + return { + repoPath, + existsSync: () => true, + prepareExecutionWorkspace: inPlaceWorkspace(), + driver: recordingDriver().driver, + buildCodingTaskSpec: () => { + calls += 1; + if (calls === 1) return { ready: true, instructions: "fine" }; + return secondCall() as never; + }, + }; + }; + const thrown = await evaluateRepoExecution( + entryFor(repoPath), + optionsFor(() => { + throw new Error("spec-workspace-boom"); + }), + ); + expect(thrown.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.OTHER); + expect(thrown.reason).toContain("scratch workspace"); + expect(thrown.reason).toContain("spec-workspace-boom"); + expect(thrown.execution).toBeNull(); + + const nullish = await evaluateRepoExecution( + entryFor(repoPath), + optionsFor(() => null), + ); + expect(nullish.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.EXECUTION); + expect(nullish.reason).toContain("unknown"); + + const unready = await evaluateRepoExecution( + entryFor(repoPath), + optionsFor(() => ({ ready: false, verdict: "held" })), + ); + expect(unready.reason).toContain("held"); + }); + + it("fails agent_run_failed via the default driver construction when no provider is configured", async () => { + const injectedEnvRepo = nodeFixture(); + const viaInjectedEnv = await evaluateRepoExecution(entryFor(injectedEnvRepo), { + repoPath: injectedEnvRepo, + existsSync: () => true, + prepareExecutionWorkspace: inPlaceWorkspace(), + env: {} as NodeJS.ProcessEnv, + }); + expect(viaInjectedEnv.failureCategory).toBe(CROSS_REPO_EXECUTION_FAILURE_CATEGORY.AGENT_RUN); + expect(viaInjectedEnv.reason).toContain("No runnable coding-agent driver"); + expect(viaInjectedEnv.reason).toContain("unconfigured_coding_agent_driver"); + expect(viaInjectedEnv.execution).toEqual({ + attempted: false, + changedFileCount: null, + buildRan: false, + testRan: false, + }); + + // Same failure through the `?? process.env` default — pinned empty so the test never launches a real agent. + const processEnvRepo = nodeFixture(); + vi.stubEnv("MINER_CODING_AGENT_PROVIDER", ""); + try { + const viaProcessEnv = await evaluateRepoExecution(entryFor(processEnvRepo), { + repoPath: processEnvRepo, + existsSync: () => true, + prepareExecutionWorkspace: inPlaceWorkspace(), + }); + expect(viaProcessEnv.failureCategory).toBe(CROSS_REPO_EXECUTION_FAILURE_CATEGORY.AGENT_RUN); + expect(viaProcessEnv.reason).toContain("unconfigured_coding_agent_driver"); + } finally { + vi.unstubAllEnvs(); + } + }); + + /** Fresh fixture + options per driver scenario — see the clone_setup test's O_EXCL note. */ + async function evaluateWithDriver(driver: CrossRepoExecutionDriver) { + const repoPath = nodeFixture(); + return evaluateRepoExecution(entryFor(repoPath), { + repoPath, + existsSync: () => true, + prepareExecutionWorkspace: inPlaceWorkspace(), + driver, + }); + } + + it("fails agent_run_failed when the driver throws, with detail fallbacks when it reports badly", async () => { + const threw = await evaluateWithDriver({ + run: async () => { + throw new Error("agent exploded"); + }, + } as never); + expect(threw.failureCategory).toBe(CROSS_REPO_EXECUTION_FAILURE_CATEGORY.AGENT_RUN); + expect(threw.reason).toContain("Coding agent run threw"); + expect(threw.reason).toContain("agent exploded"); + expect(threw.execution).toEqual({ attempted: true, changedFileCount: null, buildRan: false, testRan: false }); + + const nullResult = await evaluateWithDriver(recordingDriver(null).driver); + expect(nullResult.reason).toContain("no failure detail reported"); + + const withError = await evaluateWithDriver( + recordingDriver({ ok: false, changedFiles: [], summary: "s", error: "exploded mid-run" }).driver, + ); + expect(withError.reason).toContain("exploded mid-run"); + + const withSummaryOnly = await evaluateWithDriver( + recordingDriver({ ok: false, changedFiles: [], summary: "gave up early" }).driver, + ); + expect(withSummaryOnly.reason).toContain("gave up early"); + }); + + it("fails noop_diff when the agent succeeds without changing any file (empty or missing changedFiles)", async () => { + const emptyDiff = await evaluateWithDriver( + recordingDriver({ ok: true, changedFiles: [], summary: "did nothing" }).driver, + ); + expect(emptyDiff.failureCategory).toBe(CROSS_REPO_EXECUTION_FAILURE_CATEGORY.NOOP_DIFF); + expect(emptyDiff.execution).toEqual({ attempted: true, changedFileCount: 0, buildRan: false, testRan: false }); + + const missingDiff = await evaluateWithDriver(recordingDriver({ ok: true, summary: "malformed result" }).driver); + expect(missingDiff.failureCategory).toBe(CROSS_REPO_EXECUTION_FAILURE_CATEGORY.NOOP_DIFF); + }); + + /** Fresh build+test fixture per command scenario — see the clone_setup test's O_EXCL note. */ + async function evaluateWithRunCommand( + runCommand: (command: string, options: { cwd: string; timeoutMs: number }) => Promise, + scripts: Record = { test: "node --test" }, + ) { + const repoPath = nodeFixture(scripts); + return evaluateRepoExecution(entryFor(repoPath), { + repoPath, + existsSync: () => true, + prepareExecutionWorkspace: inPlaceWorkspace(), + driver: recordingDriver().driver, + runCommand, + }); + } + + it("runs the inferred build command first and fails build_failed on a non-zero exit or a timeout", async () => { + const buildScripts = { build: "node -e ok", test: "node --test" }; + const exited = scriptedRunCommand([{ code: 2 }]); + const buildFailed = await evaluateWithRunCommand(exited.runCommand, buildScripts); + expect(buildFailed.failureCategory).toBe(CROSS_REPO_EXECUTION_FAILURE_CATEGORY.BUILD); + expect(buildFailed.reason).toContain("failed (exit 2)"); + expect(buildFailed.reason).toContain("npm run build"); + expect(buildFailed.execution).toEqual({ attempted: true, changedFileCount: 1, buildRan: true, testRan: false }); + expect(exited.calls[0]?.command).toBe("npm run build"); + + const timedOut = scriptedRunCommand([{ code: null, timedOut: true }]); + const buildTimeout = await evaluateWithRunCommand(timedOut.runCommand, buildScripts); + expect(buildTimeout.failureCategory).toBe(CROSS_REPO_EXECUTION_FAILURE_CATEGORY.BUILD); + expect(buildTimeout.reason).toContain("timed out"); + }); + + it("passes end-to-end with build + test green, using the default command timeout", async () => { + const repoPath = nodeFixture({ build: "node -e ok", test: "node --test" }); + const { calls, runCommand } = scriptedRunCommand([{ code: 0 }, { code: 0 }]); + const result = await evaluateRepoExecution(entryFor(repoPath), { + repoPath, + existsSync: () => true, + prepareExecutionWorkspace: inPlaceWorkspace(), + driver: recordingDriver().driver, + runCommand, + }); + expect(result.passed).toBe(true); + expect(result.execution).toEqual({ attempted: true, changedFileCount: 1, buildRan: true, testRan: true }); + expect(calls.map((call) => call.command)).toEqual(["npm run build", "npm test"]); + expect(calls[0]?.timeoutMs).toBe(DEFAULT_CROSS_REPO_EXECUTION_COMMAND_TIMEOUT_MS); + expect(calls[0]?.cwd).toBe(repoPath); + }); + + it("honors options.commandTimeoutMs and options.maxTurns", async () => { + const repoPath = nodeFixture(); + const { calls, runCommand } = scriptedRunCommand([{ code: 0 }]); + const { tasks, driver } = recordingDriver(); + const result = await evaluateRepoExecution(entryFor(repoPath), { + repoPath, + existsSync: () => true, + prepareExecutionWorkspace: inPlaceWorkspace(), + driver, + runCommand, + commandTimeoutMs: 12_345, + maxTurns: 7, + }); + expect(result.passed).toBe(true); + expect(calls[0]?.timeoutMs).toBe(12_345); + expect(tasks[0]?.maxTurns).toBe(7); + }); + + it("fails execution_gap when no test command was inferred to validate the diff", async () => { + const repoPath = nodeFixture({}); + const result = await evaluateRepoExecution(entryFor(repoPath), { + repoPath, + existsSync: () => true, + prepareExecutionWorkspace: inPlaceWorkspace(), + driver: recordingDriver().driver, + buildCodingTaskSpec: () => ({ ready: true, instructions: "fine" }), + }); + expect(result.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.EXECUTION); + expect(result.reason).toContain("cannot be validated"); + expect(result.execution).toEqual({ attempted: true, changedFileCount: 1, buildRan: false, testRan: false }); + }); + + it("fails test_failed when the repo's own test suite exits non-zero or times out", async () => { + const exited = await evaluateWithRunCommand(scriptedRunCommand([{ code: 3 }]).runCommand); + expect(exited.failureCategory).toBe(CROSS_REPO_EXECUTION_FAILURE_CATEGORY.TEST); + expect(exited.reason).toContain("failed (exit 3)"); + expect(exited.reason).toContain("npm test"); + expect(exited.execution).toEqual({ attempted: true, changedFileCount: 1, buildRan: false, testRan: true }); + + const timedOut = await evaluateWithRunCommand(scriptedRunCommand([{ code: null, timedOut: true }]).runCommand); + expect(timedOut.failureCategory).toBe(CROSS_REPO_EXECUTION_FAILURE_CATEGORY.TEST); + expect(timedOut.reason).toContain("timed out"); + }); + + it("hands the agent a task rooted in a REAL discardable scratch copy, then removes it (dry-run posture)", async () => { + const repoPath = nodeFixture(); + const { tasks, driver } = recordingDriver(); + const { runCommand } = scriptedRunCommand([{ code: 0 }]); + const result = await evaluateRepoExecution(entryFor(repoPath), { + repoPath, + existsSync, + driver, + runCommand, + }); + expect(result.passed).toBe(true); + const task = tasks[0]!; + // The agent worked in the scratch copy, not the benchmark clone… + expect(task.workingDirectory).not.toBe(repoPath); + expect(String(task.workingDirectory)).toContain("loopover-cross-repo-exec-"); + // …the acceptance-criteria document was written INSIDE that copy… + expect(String(task.acceptanceCriteriaPath).startsWith(String(task.workingDirectory))).toBe(true); + expect(String(task.instructions)).not.toBe(""); + expect(task.maxTurns).toBe(DEFAULT_CROSS_REPO_EXECUTION_MAX_TURNS); + // …and the whole scratch tree is discarded afterward while the clone survives untouched. + expect(existsSync(String(task.workingDirectory))).toBe(false); + expect(existsSync(join(repoPath, "package.json"))).toBe(true); + }); + + it("falls back to empty instructions and a workspace-local acceptance path when a spec omits them", async () => { + const repoPath = nodeFixture(); + let calls = 0; + const { tasks, driver } = recordingDriver(); + const result = await evaluateRepoExecution(entryFor(repoPath), { + repoPath, + existsSync: () => true, + prepareExecutionWorkspace: inPlaceWorkspace(), + driver, + runCommand: scriptedRunCommand([{ code: 0 }]).runCommand, + buildCodingTaskSpec: () => { + calls += 1; + return calls === 1 ? { ready: true, instructions: "readiness pass" } : { ready: true }; + }, + }); + expect(result.passed).toBe(true); + expect(tasks[0]?.instructions).toBe(""); + expect(String(tasks[0]?.acceptanceCriteriaPath).endsWith("ACCEPTANCE_CRITERIA.md")).toBe(true); + }); + + it("runCrossRepoFullExecution mirrors the readiness runner's manifest and filter handling", async () => { + expect(await runCrossRepoFullExecution(undefined as never)).toEqual([]); + + const parsed = parseCrossRepoEvaluationManifest(JSON.stringify({ repos: ["acme/a", "acme/b"] })); + const all = await runCrossRepoFullExecution(parsed, { existsSync: () => false }); + expect(all).toHaveLength(2); + expect(all[0]?.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.CLONE_SETUP); + expect(all[0]?.execution).toBeNull(); + + const filtered = await runCrossRepoFullExecution(parsed, { repoFilter: "acme/b", existsSync: () => false }); + expect(filtered).toHaveLength(1); + expect(filtered[0]?.repoFullName).toBe("acme/b"); + }); + + it("defaultPrepareExecutionWorkspace copies the repo and cleanup removes the whole scratch tree", () => { + const repoPath = nodeFixture(); + const workspace = defaultPrepareExecutionWorkspace(repoPath); + expect(workspace.path).not.toBe(repoPath); + expect(existsSync(join(workspace.path, "package.json"))).toBe(true); + workspace.cleanup(); + expect(existsSync(workspace.path)).toBe(false); + }); + + it("createDefaultCrossRepoExecutionRunCommand reports exit codes, output, timeouts, and spawn errors", async () => { + // The runner tokenizes on whitespace and execs directly (no shell), so every command here is a plain + // `tool arg...` form — the only shape detectRepoStack ever emits. + const runCommand = createDefaultCrossRepoExecutionRunCommand(); + const cwd = tempRepo(); + + const ok = await runCommand("node -p 40+2", { cwd, timeoutMs: 30_000 }); + expect(ok).toMatchObject({ code: 0, timedOut: false }); + expect(ok.stdout).toContain("42"); + + const failing = await runCommand("node --definitely-not-a-real-flag", { cwd, timeoutMs: 30_000 }); + expect(failing.code).not.toBe(0); + expect(failing.code).not.toBeNull(); + expect(failing.stderr).toContain("bad option"); + + const hung = await runCommand("node -e setTimeout(function(){},10000)", { cwd, timeoutMs: 300 }); + expect(hung.timedOut).toBe(true); + expect(hung.code).toBeNull(); + + const spawnError = await runCommand("node -v", { cwd: join(cwd, "definitely-missing-subdir"), timeoutMs: 5_000 }); + expect(spawnError.code).toBeNull(); + expect(spawnError.timedOut).toBe(false); + expect(spawnError.stderr).not.toBe(""); + + const empty = await runCommand(" ", { cwd, timeoutMs: 5_000 }); + expect(empty).toEqual({ stdout: "", stderr: "empty_command", code: null, timedOut: false }); + }); + + it("runCrossRepoFullExecutionCli reports the new categories through the existing summary + report format", async () => { + const repoPath = nodeFixture(); + const manifestDir = tempRepo(); + writeFileSync( + join(manifestDir, "manifest.json"), + JSON.stringify({ repos: [{ repoFullName: "acme/exec-cli", fixturePath: repoPath }] }), + "utf8", + ); + // Pin the provider empty so the CLI's default driver path fails closed instead of launching a real agent. + vi.stubEnv("MINER_CODING_AGENT_PROVIDER", ""); + try { + const { results, summary } = await runCrossRepoFullExecutionCli({ + manifestPath: join(manifestDir, "manifest.json"), + }); + expect(results[0]?.failureCategory).toBe(CROSS_REPO_EXECUTION_FAILURE_CATEGORY.AGENT_RUN); + expect(summary.failuresByCategory[CROSS_REPO_EXECUTION_FAILURE_CATEGORY.AGENT_RUN]).toBe(1); + expect(formatCrossRepoEvaluationReport(results, summary)).toContain("- agent_run_failed: 1"); + } finally { + vi.unstubAllEnvs(); + } + }); + }); + it("documents the harness in packages/loopover-miner/docs/cross-repo-evaluation.md", () => { const doc = readFileSync(join(process.cwd(), "packages/loopover-miner/docs/cross-repo-evaluation.md"), "utf8"); expect(doc).toContain("#4788"); expect(doc).toContain("stack_detection_gap"); expect(doc).toContain("cross-repo-evaluation.mjs"); expect(doc).toContain("benchmarks/cross-repo/manifest.json"); + // #7634: the full-execution mode and its execution-specific failure categories are documented too. + expect(doc).toContain("--full-execution"); + expect(doc).toContain("noop_diff"); + expect(doc).toContain("dry-run"); }); });