diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 27942f694d..f8d888e362 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -2900,6 +2900,29 @@ export async function hasAuditEventForDelivery(env: Env, actor: string, eventTyp return (row?.count ?? 0) > 0; } +/** Whether `eventType` has ALREADY been recorded for this `targetKey` at this EXACT `headSha` -- unlike + * `hasAuditEventForDelivery` above (which guards a single redelivered webhook within a short window), this has + * no time bound: a head SHA is a stable, permanent identity, so a match at any point in the past is still a + * match. Used by the `manifest_missing_tests` auto-trigger (#4196) to guard against re-spending an LLM call on + * every re-review/sweep pass over an UNCHANGED commit -- a genuinely new push (a new head SHA) is always a + * fresh miss regardless of how many prior SHAs already fired. json_extract mirrors hasAuditEventForDelivery's + * own metadata-predicate pattern rather than a fragile LIKE match on the raw JSON string. */ +export async function hasAuditEventForHeadSha(env: Env, eventType: string, targetKey: string, headSha: string): Promise { + const db = getDb(env.DB); + const [row] = await db + .select({ count: sql`count(*)` }) + .from(auditEvents) + .where( + and( + eq(auditEvents.eventType, eventType), + eq(auditEvents.targetKey, targetKey), + sql`json_extract(${auditEvents.metadataJson}, '$.headSha') = ${headSha}`, + ), + ); + /* v8 ignore next -- count(*) always returns exactly one row; the empty-array guard only satisfies the destructure type. */ + return (row?.count ?? 0) > 0; +} + /** Observability for the queue dead-letter rate (#1276): how many jobs (across BOTH the maintenance and webhook * lanes) were dead-lettered since `sinceIso`. Reads the `github_app.dlq_dead_lettered` audit events written by * processDlqBatch — NOT gated behind any review-ops flag, so the infra drop rate is always visible. */ diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 4fac0fb7b6..a3065e0502 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -65,6 +65,7 @@ import { countRecentAuditEventsForActorInRepo, countRecentAuditEventsForActorInRepoWithTargetSuffix, hasAuditEventForDelivery, + hasAuditEventForHeadSha, recordGateBlockOutcome, getGateBlockOutcome, hasActiveReviewForHeadSha, @@ -9102,6 +9103,48 @@ async function maybePublishPrPublicSurface( if (!policyCodes.has(finding.code)) continue; advisory.findings.push(publicSafeManifestPolicyFinding(finding)); } + // E2E test-generation auto-trigger (#4196, part of the #4189 epic): promotes the deterministic + // manifest_missing_tests finding above from advisory-only text into an actual trigger for #4192/#4194's + // generation-and-render path -- additive to, never a replacement for, the explicit `@gittensory + // generate-tests` command (#4195), which stays available regardless of whether this signal fired. + // Filters the SAME guidance.findings just computed above rather than re-deriving "PR probably needs + // tests" from scratch, per the issue's own requirement -- this is why the auto-trigger lives inside this + // exact manifestPolicyGateMode-gated block instead of a parallel code path: that is the only place this + // finding is computed at all today. + if (pr.headSha && guidance.findings.some((finding) => finding.code === "manifest_missing_tests") && resolveConvergedFeature(env, manifest, "e2eTests", repoFullName)) { + const e2eTargetKey = `${repoFullName}#${pr.number}`; + // Double-generation guard: an unchanged head SHA re-entering this pass (a re-review/sweep tick, not a + // new push) must never re-spend an LLM call or repost a duplicate suggestion. A genuinely NEW push + // (a new head SHA) is always a fresh miss here regardless of how many prior SHAs already fired. The + // explicit command deliberately does NOT consult this guard -- a maintainer typing the command always + // gets a fresh generation, even on a SHA the auto-trigger already covered (simplicity over a cache that + // would need its own invalidation rules; the daily neuron budget shared by both paths already bounds + // the cost of a maintainer choosing to ask twice). + const alreadyTriggered = await hasAuditEventForHeadSha(env, "github_app.e2e_tests_generation", e2eTargetKey, pr.headSha); + if (!alreadyTriggered) { + const e2eMode = resolveAgentActionMode({ globalPaused: isGlobalAgentPause(env) || (await isGlobalAgentFrozen(env)), agentPaused: settings.agentPaused, agentDryRun: settings.agentDryRun }); + if (e2eMode === "live") { + await runE2eTestGenerationAndDeliver(env, { + repoFullName, + installationId, + pr, + settings, + manifest, + files: manifestFiles, + // No comment-invoker exists for an automated trigger -- the PR's own author is the closest + // analogue to "who this generated test is for" (unlike the explicit command, where `actor` is + // whoever typed the command). + actor: author ?? "the PR author", + mode: e2eMode, + deliveryId: webhook.deliveryId, + targetKey: e2eTargetKey, + trigger: "auto", + }); + } else { + await recordGenerateTestsSkip(env, webhook.deliveryId, repoFullName, e2eTargetKey, author, e2eMode === "dry_run" ? "dry_run" : "agent_paused"); + } + } + } } // Pre-merge checks (#review-pre-merge-checks, opt-in via .gittensory.yml review.pre_merge_checks). DETERMINISTIC // content assertions (title/description must contain a phrase, a label must be present), optionally path-gated. @@ -11468,22 +11511,62 @@ async function maybeProcessGenerateTestsCommand(env: Env, deliveryId: string, pa return true; } const files = await listPullRequestFiles(env, req.repoFullName, req.pr.number); - const changedPaths = files.map((file) => file.path); + await runE2eTestGenerationAndDeliver(env, { + repoFullName: req.repoFullName, + installationId: req.installationId, + pr, + settings, + manifest, + files, + actor: req.actor, + mode, + deliveryId, + targetKey, + trigger: "command", + }); + return true; +} + +/** + * The shared generation-and-delivery core behind BOTH `@gittensory generate-tests` (#4195, the explicit + * command) and the `manifest_missing_tests` auto-trigger (#4196) — one code path, so the two triggers can + * never silently drift apart. Everything the caller must have already resolved BEFORE this runs: the feature + * is enabled (#4192's `resolveConvergedFeature` gate), the repo is not paused/dry-run (`mode === "live"`), and + * (for the auto-trigger specifically) the per-head-SHA double-generation guard has already passed — this + * function itself has no opinion on any of that, it only generates, delivers, and audits. + */ +async function runE2eTestGenerationAndDeliver( + env: Env, + args: { + repoFullName: string; + installationId: number; + pr: PullRequestRecord; + settings: RepositorySettings; + manifest: FocusManifest | null; + files: Awaited>; + actor: string; + mode: ReturnType; + deliveryId: string; + targetKey: string; + trigger: "command" | "auto"; + }, +): Promise { + const changedPaths = args.files.map((file) => file.path); // BYOK resolution mirrors runAiReviewForAdvisory's own (re-resolved per-caller is this codebase's // established convention for this exact 3-line block — see e.g. the vision-capture caller above). - const storedKey = settings.aiReviewByok ? await getDecryptedRepositoryAiKey(env, req.repoFullName) : null; + const storedKey = args.settings.aiReviewByok ? await getDecryptedRepositoryAiKey(env, args.repoFullName) : null; const providerKey = - storedKey && (!settings.aiReviewProvider || settings.aiReviewProvider === storedKey.provider) - ? { provider: storedKey.provider, key: storedKey.key, model: settings.aiReviewModel ?? storedKey.model } + storedKey && (!args.settings.aiReviewProvider || args.settings.aiReviewProvider === storedKey.provider) + ? { provider: storedKey.provider, key: storedKey.key, model: args.settings.aiReviewModel ?? storedKey.model } : null; const result = await runGittensoryE2eTestGeneration(env, { - repoFullName: req.repoFullName, - prNumber: req.pr.number, - title: pr.title, - body: pr.body, - files: files.map((file) => ({ path: file.path, patch: typeof file.payload?.patch === "string" ? file.payload.patch : undefined })), - instructions: resolveE2eTestGenInstructions(manifest?.review, changedPaths), - actor: req.actor, + repoFullName: args.repoFullName, + prNumber: args.pr.number, + title: args.pr.title, + body: args.pr.body, + files: args.files.map((file) => ({ path: file.path, patch: typeof file.payload?.patch === "string" ? file.payload.patch : undefined })), + instructions: resolveE2eTestGenInstructions(args.manifest?.review, changedPaths), + actor: args.actor, providerKey, }); const testSource = result.status === "ok" ? result.testSource : null; @@ -11493,24 +11576,24 @@ async function maybeProcessGenerateTestsCommand(env: Env, deliveryId: string, pa // scoring-integrity safeguard) — that check runs regardless of this repo's own delivery config, since the // external, upstream-computed score must never be able to include a maintainer-authored line a miner didn't // write themselves. - const deliveryMode = resolveReviewPromptOverrides(manifest).e2eTestDelivery ?? "comment"; + const deliveryMode = resolveReviewPromptOverrides(args.manifest).e2eTestDelivery ?? "comment"; let commitOutcome: E2eTestGenCommitOutcome | undefined; if (testSource && deliveryMode === "commit") { - const minerDetection = pr.authorLogin - ? await getCachedOfficialMinerDetection(env, pr.authorLogin, { targetKey, deliveryId }) + const minerDetection = args.pr.authorLogin + ? await getCachedOfficialMinerDetection(env, args.pr.authorLogin, { targetKey: args.targetKey, deliveryId: args.deliveryId }) : ({ status: "not_found" } as const); if (minerDetection.status === "confirmed") { commitOutcome = { status: "blocked" }; - } else if (pr.headSha && pr.headRef) { + } else if (args.pr.headSha && args.pr.headRef) { const attempt = await commitE2eTestToPrBranch(env, { - installationId: req.installationId, - repoFullName: req.repoFullName, - prNumber: req.pr.number, - headRef: pr.headRef, - headSha: pr.headSha, + installationId: args.installationId, + repoFullName: args.repoFullName, + prNumber: args.pr.number, + headRef: args.pr.headRef, + headSha: args.pr.headSha, testSource, - actor: req.actor, - mode, + actor: args.actor, + mode: args.mode, }); // The render layer only distinguishes committed/declined/blocked -- an "error" (unexpected failure, // vs. an expected can-never-work case) is still surfaced to the maintainer as "declined", with its @@ -11521,32 +11604,35 @@ async function maybeProcessGenerateTestsCommand(env: Env, deliveryId: string, pa } } - const body = buildE2eTestGenCommentBody({ actor: req.actor, testSource, commit: commitOutcome }); + const body = buildE2eTestGenCommentBody({ actor: args.actor, testSource, commit: commitOutcome }); try { - await createIssueComment(env, req.installationId, req.repoFullName, req.pr.number, sanitizePublicComment(body)); + await createIssueComment(env, args.installationId, args.repoFullName, args.pr.number, sanitizePublicComment(body)); } catch (error) { - // sanitizePublicComment THROWS on a forbidden term rather than stripping it -- generated test source is - // far less predictable than this codebase's other curated comment content, so failing closed to a safe - // withheld-content note (never the raw error, never the raw generated text) is the right degrade here. + // Generated test source is far less predictable than this codebase's other curated comment content, so + // a failure posting it (a GitHub API error, a rate limit, or any other unexpected throw) degrades to a + // safe withheld-content note (never the raw error, never the raw generated text) rather than leaving the + // maintainer with silence. await createIssueComment( env, - req.installationId, - req.repoFullName, - req.pr.number, - sanitizePublicComment(buildE2eTestGenCommentBody({ actor: req.actor, testSource: null })), + args.installationId, + args.repoFullName, + args.pr.number, + sanitizePublicComment(buildE2eTestGenCommentBody({ actor: args.actor, testSource: null })), ); - console.log(JSON.stringify({ event: "e2e_test_gen_comment_withheld", repoFullName: req.repoFullName, pr: req.pr.number, error: errorMessage(error) })); + console.log(JSON.stringify({ event: "e2e_test_gen_comment_withheld", repoFullName: args.repoFullName, pr: args.pr.number, error: errorMessage(error) })); } await recordAuditEvent(env, { eventType: "github_app.e2e_tests_generation", - actor: req.actor, - targetKey, + actor: args.actor, + targetKey: args.targetKey, outcome: "completed", detail: testSource ? "Generated an E2E test." : `No usable test generated (${result.status}).`, - metadata: { deliveryId, repoFullName: req.repoFullName, status: result.status, byok: Boolean(providerKey), deliveryMode, ...(commitOutcome ? { commitStatus: commitOutcome.status } : {}) }, + // headSha is included so the #4196 auto-trigger's per-commit double-generation guard (hasAuditEventForHeadSha) + // can find this row again; a null headSha (never observed in practice -- both callers require a truthy one + // before reaching here) degrades to simply never matching that guard, not a thrown error. + metadata: { deliveryId: args.deliveryId, repoFullName: args.repoFullName, status: result.status, byok: Boolean(providerKey), deliveryMode, trigger: args.trigger, headSha: args.pr.headSha ?? null, ...(commitOutcome ? { commitStatus: commitOutcome.status } : {}) }, }); - await recordGithubProductUsage(env, "e2e_tests_generation", { actor: req.actor, repoFullName: req.repoFullName, targetKey, outcome: "completed", metadata: { status: result.status, generated: Boolean(testSource), deliveryMode, ...(commitOutcome ? { commitStatus: commitOutcome.status } : {}) } }); - return true; + await recordGithubProductUsage(env, "e2e_tests_generation", { actor: args.actor, repoFullName: args.repoFullName, targetKey: args.targetKey, outcome: "completed", metadata: { status: result.status, generated: Boolean(testSource), deliveryMode, trigger: args.trigger, ...(commitOutcome ? { commitStatus: commitOutcome.status } : {}) } }); } async function postGenerateTestsNotEnabledComment(env: Env, installationId: number, repoFullName: string, prNumber: number): Promise { diff --git a/test/unit/db-parsers.test.ts b/test/unit/db-parsers.test.ts index f72a35d30f..77cd99228e 100644 --- a/test/unit/db-parsers.test.ts +++ b/test/unit/db-parsers.test.ts @@ -8,6 +8,7 @@ import { countRecentAuditEventsForActorInRepoWithTargetSuffix, findHottestReviewTargetForRepo, hasAuditEventForDelivery, + hasAuditEventForHeadSha, getLatestScorePreview, getRepoAuthorPullRequestHistory, getLatestScoringModelSnapshot, @@ -755,6 +756,26 @@ describe("database row parser hardening", () => { expect(await hasAuditEventForDelivery(env, "maintainer", "github_app.command_invocation", "owner/repo#1#help", "delivery-never-recorded", "2026-06-24T09:00:00.000Z")).toBe(false); }); + it("hasAuditEventForHeadSha finds a matching headSha inside metadata_json, scoped to eventType+targetKey, with no time bound (#4196)", async () => { + const env = createTestEnv(); + await recordAuditEvent(env, { + eventType: "github_app.e2e_tests_generation", + actor: "contributor", + targetKey: "owner/repo#7", + outcome: "completed", + createdAt: "2020-01-01T00:00:00.000Z", // ancient -- there is no time window on this lookup at all + metadata: { headSha: "sha-a" }, + }); + + expect(await hasAuditEventForHeadSha(env, "github_app.e2e_tests_generation", "owner/repo#7", "sha-a")).toBe(true); + // A DIFFERENT head SHA on the same targetKey/eventType must not match -- a genuinely new push is always a fresh miss. + expect(await hasAuditEventForHeadSha(env, "github_app.e2e_tests_generation", "owner/repo#7", "sha-b")).toBe(false); + // The SAME headSha but a different targetKey (a different PR) must not match. + expect(await hasAuditEventForHeadSha(env, "github_app.e2e_tests_generation", "owner/repo#8", "sha-a")).toBe(false); + // The SAME headSha and targetKey but a different eventType must not match. + expect(await hasAuditEventForHeadSha(env, "github_app.some_other_event", "owner/repo#7", "sha-a")).toBe(false); + }); + it("computes complete case-insensitive repo author PR history for gate grace", async () => { const env = createTestEnv(); diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index bf05e31bfb..05513c39d8 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -24934,6 +24934,45 @@ describe("queue processors", () => { expect(denied?.detail).toBe("maintainer_command_requires_maintainer"); }); + it("falls back to a safe withheld-content note when posting the real generated-test comment fails", async () => { + const repoFullName = "JSONbored/gen-tests-4195-post-fails"; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => ({ response: "```typescript\n" + VALID_TEST_SOURCE + "\n```" }) } as unknown as Ai, + GITTENSORY_REVIEW_E2E_TESTS: "true", + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + }); + await seedGenerateTestsPr(env, repoFullName, 4200, "gen-tests-4195-post-fails"); + let postAttempts = 0; + let fallbackBody = ""; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/collaborators/maintainer/permission")) return Response.json({ permission: "admin" }); + if (url.includes("/issues/4200/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/4200/comments") && method === "POST") { + postAttempts += 1; + // The FIRST attempt (the real generated-test comment) fails with a genuine GitHub API error; the + // SECOND attempt (the withheld-content fallback) must still succeed. + if (postAttempts === 1) return new Response("server exploded", { status: 500 }); + fallbackBody = String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? ""); + return Response.json({ id: 42000 }); + } + return new Response("not found", { status: 404 }); + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + await processJob(env, generateTestsWebhook(repoFullName, 4200, "maintainer", { association: "MEMBER" })); + + expect(postAttempts).toBe(2); + expect(fallbackBody).toContain("did not produce a usable result"); + expect(fallbackBody).not.toContain("test('checkout retries on failure'"); + expect(logSpy.mock.calls.map((c) => String(c[0])).some((line) => line.includes("e2e_test_gen_comment_withheld"))).toBe(true); + logSpy.mockRestore(); + }); + it("posts a not-enabled note (no generation call) when features.e2eTests is off for the repo", async () => { const repoFullName = "JSONbored/gen-tests-4195-disabled"; const run = vi.fn(); @@ -25404,6 +25443,352 @@ describe("queue processors", () => { }); }); + // #4196 (part of the #4189 epic): promotes the existing manifest_missing_tests advisory finding into an + // actual auto-trigger for #4192/#4194's generation-and-render path, additive to the explicit + // `@gittensory generate-tests` command (#4195) tested above -- this describe block drives the AUTOMATED + // review pass (maybePublishPrPublicSurface, via a `pull_request` webhook) rather than an issue_comment. + describe("manifest_missing_tests auto-trigger (#4196)", () => { + const AUTO_TEST_SOURCE = "import { test, expect } from '@playwright/test';\n\ntest('auto-generated coverage', async ({ page }) => {\n await page.goto('/');\n await expect(page).toHaveTitle(/./);\n});"; + + async function seedAutoTriggerPr( + env: Env, + repoFullName: string, + prNumber: number, + headSha: string, + opts: { e2eTests?: boolean; hasTestFile?: boolean; validationNote?: boolean; manifestPolicyGateMode?: "advisory" | "block" } = {}, + ) { + const slash = repoFullName.indexOf("/"); + const owner = repoFullName.slice(0, slash); + const name = repoFullName.slice(slash + 1); + await upsertRepositoryFromGitHub(env, { name, full_name: repoFullName, private: false, owner: { login: owner } }, 123); + await upsertRepositorySettings(env, { + repoFullName, + commentMode: "off", + publicSurface: "off", + autoLabelEnabled: false, + checkRunMode: "off", + // "enabled" (not "off") -- resolveRepositorySettings derives reviewCheckMode: "required" from this + // when reviewCheckMode itself is unset, and gateEnabled (which the whole manifestPolicyGateMode block + // this auto-trigger lives inside is downstream of) requires a truthy reviewCheckMode + a headSha. With + // gateCheckMode: "off" the function bails out via its own early-return before ever reaching guidance. + gateCheckMode: "enabled", + requireLinkedIssue: false, + linkedIssueGateMode: "off", + manifestPolicyGateMode: opts.manifestPolicyGateMode ?? "advisory", + aiReviewMode: "off", + typeLabelsEnabled: false, + }); + await upsertPullRequestFromGitHub(env, repoFullName, { + number: prNumber, + title: "Add retry to checkout", + state: "open", + user: { login: "contributor" }, + author_association: "CONTRIBUTOR", + head: { sha: headSha, ref: "feature/checkout-retry" }, + labels: [], + body: opts.validationNote ? "Ran npm run test:ci -- all green." : "No validation evidence mentioned here.", + }); + await upsertPullRequestFile(env, { + repoFullName, + pullNumber: prNumber, + path: opts.hasTestFile ? "test/unit/checkout.test.ts" : "src/checkout.ts", + status: "modified", + additions: 3, + deletions: 0, + changes: 3, + payload: { patch: "+function retryPayment() {\n+ return true;\n+}" }, + }); + // testExpectations is a TOP-LEVEL manifest field (unlike review.e2e_test_delivery's nested snake_case) -- + // both it and features.e2eTests must land in the SAME upsertRepoFocusManifest call, since a second + // separate call replaces rather than merges with the first. + await upsertRepoFocusManifest(env, repoFullName, { + testExpectations: ["Run npm run test:ci."], + features: { e2eTests: opts.e2eTests ?? true }, + }); + } + + const autoTriggerWebhook = (repoFullName: string, prNumber: number, headSha: string, action: "opened" | "synchronize" = "opened", body = "No validation evidence mentioned here.") => ({ + type: "github-webhook" as const, + deliveryId: `auto-e2e-${prNumber}-${headSha}-${action}`, + eventName: "pull_request" as const, + payload: { + action, + installation: { id: 123, account: { login: repoFullName.slice(0, repoFullName.indexOf("/")), id: 1, type: "User" } }, + repository: { name: repoFullName.slice(repoFullName.indexOf("/") + 1), full_name: repoFullName, private: false, owner: { login: repoFullName.slice(0, repoFullName.indexOf("/")) } }, + pull_request: { + number: prNumber, + title: "Add retry to checkout", + state: "open", + user: { login: "contributor" }, + head: { sha: headSha }, + labels: [], + // The incoming webhook payload's own body ALWAYS re-upserts the cached PR record before this pass + // runs, overwriting whatever body seedAutoTriggerPr wrote directly to the DB -- so a test that needs + // a specific validation-note body must pass it here, not rely on the DB seed alone. + body, + }, + }, + }) as unknown as Parameters[1]; + + function stubAutoTriggerFetch(prNumber: number, posted: { count: number; body: string }) { + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url === "https://api.gittensor.io/miners") return Response.json([]); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + // gateCheckMode: "enabled" means this pass ALSO publishes/updates a gate check-run -- these three + // endpoints back that unrelated publish, not the e2e-test-gen comment itself. + if (url.includes("/check-runs") && method === "GET") return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/check-runs") && method === "POST") return Response.json({ id: prNumber * 100 }, { status: 201 }); + if (url.includes("/check-runs") && method === "PATCH") return Response.json({ id: prNumber * 100, html_url: `https://github.com/checks/${prNumber * 100}` }); + if (url.includes(`/issues/${prNumber}/comments`) && method === "GET") return Response.json([]); + if (url.includes(`/issues/${prNumber}/comments`) && method === "POST") { + posted.count += 1; + posted.body = String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? ""); + return Response.json({ id: prNumber * 10 }); + } + return new Response("not found", { status: 404 }); + }); + } + + it("auto-triggers generation when manifest_missing_tests fires and features.e2eTests is enabled", async () => { + const repoFullName = "JSONbored/auto-e2e-4196-ok"; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => ({ response: "```typescript\n" + AUTO_TEST_SOURCE + "\n```" }) } as unknown as Ai, + GITTENSORY_REVIEW_E2E_TESTS: "true", + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + }); + await seedAutoTriggerPr(env, repoFullName, 5001, "auto-4196-ok-sha"); + const posted = { count: 0, body: "" }; + stubAutoTriggerFetch(5001, posted); + + await processJob(env, autoTriggerWebhook(repoFullName, 5001, "auto-4196-ok-sha")); + + expect(posted.count).toBe(1); + expect(posted.body).toContain("test('auto-generated coverage'"); + const audited = await env.DB.prepare("select outcome, metadata_json from audit_events where event_type = ?").bind("github_app.e2e_tests_generation").first<{ outcome: string; metadata_json: string }>(); + expect(audited?.outcome).toBe("completed"); + expect(JSON.parse(audited?.metadata_json ?? "{}")).toMatchObject({ trigger: "auto", headSha: "auto-4196-ok-sha" }); + }); + + it("does not auto-trigger when manifest_missing_tests fires but features.e2eTests is disabled for the repo", async () => { + const repoFullName = "JSONbored/auto-e2e-4196-disabled"; + const run = vi.fn(); + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), AI: { run } as unknown as Ai, GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await seedAutoTriggerPr(env, repoFullName, 5002, "auto-4196-disabled-sha", { e2eTests: false }); + const posted = { count: 0, body: "" }; + stubAutoTriggerFetch(5002, posted); + + await processJob(env, autoTriggerWebhook(repoFullName, 5002, "auto-4196-disabled-sha")); + + expect(run).not.toHaveBeenCalled(); + expect(posted.count).toBe(0); + const audited = await env.DB.prepare("select count(*) as n from audit_events where event_type = ?").bind("github_app.e2e_tests_generation").first<{ n: number }>(); + expect(audited?.n).toBe(0); + }); + + it("does not auto-trigger when the PR already carries a test file (the manifest_missing_tests signal never fires)", async () => { + const repoFullName = "JSONbored/auto-e2e-4196-has-test"; + const run = vi.fn(); + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), AI: { run } as unknown as Ai, GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await seedAutoTriggerPr(env, repoFullName, 5003, "auto-4196-has-test-sha", { hasTestFile: true }); + const posted = { count: 0, body: "" }; + stubAutoTriggerFetch(5003, posted); + + await processJob(env, autoTriggerWebhook(repoFullName, 5003, "auto-4196-has-test-sha")); + + expect(run).not.toHaveBeenCalled(); + expect(posted.count).toBe(0); + }); + + it("does not auto-trigger when the PR body already carries a validation note (the manifest_missing_tests signal never fires)", async () => { + const repoFullName = "JSONbored/auto-e2e-4196-validated"; + const run = vi.fn(); + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), AI: { run } as unknown as Ai, GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await seedAutoTriggerPr(env, repoFullName, 5004, "auto-4196-validated-sha", { validationNote: true }); + const posted = { count: 0, body: "" }; + stubAutoTriggerFetch(5004, posted); + + await processJob(env, autoTriggerWebhook(repoFullName, 5004, "auto-4196-validated-sha", "opened", "Ran npm run test:ci -- all green.")); + + expect(run).not.toHaveBeenCalled(); + expect(posted.count).toBe(0); + }); + + it("does not re-trigger generation on a second automated pass over the SAME unchanged head SHA (double-generation guard)", async () => { + const repoFullName = "JSONbored/auto-e2e-4196-dedup"; + let runCalls = 0; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => { runCalls += 1; return { response: "```typescript\n" + AUTO_TEST_SOURCE + "\n```" }; } } as unknown as Ai, + GITTENSORY_REVIEW_E2E_TESTS: "true", + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + }); + await seedAutoTriggerPr(env, repoFullName, 5005, "auto-4196-dedup-sha"); + const posted = { count: 0, body: "" }; + stubAutoTriggerFetch(5005, posted); + + // Two passes over the identical head SHA -- e.g. a `synchronize` redelivery or a re-review sweep tick + // with no new push in between. + await processJob(env, autoTriggerWebhook(repoFullName, 5005, "auto-4196-dedup-sha", "opened")); + await processJob(env, autoTriggerWebhook(repoFullName, 5005, "auto-4196-dedup-sha", "synchronize")); + + expect(runCalls).toBe(1); + expect(posted.count).toBe(1); + const rows = await env.DB.prepare("select count(*) as n from audit_events where event_type = ?").bind("github_app.e2e_tests_generation").first<{ n: number }>(); + expect(rows?.n).toBe(1); + }); + + it("DOES trigger again for a genuinely NEW head SHA (a real push) even though a prior SHA on the same PR already fired", async () => { + const repoFullName = "JSONbored/auto-e2e-4196-new-push"; + let runCalls = 0; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => { runCalls += 1; return { response: "```typescript\n" + AUTO_TEST_SOURCE + "\n```" }; } } as unknown as Ai, + GITTENSORY_REVIEW_E2E_TESTS: "true", + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + }); + await seedAutoTriggerPr(env, repoFullName, 5006, "auto-4196-first-sha"); + const posted = { count: 0, body: "" }; + stubAutoTriggerFetch(5006, posted); + await processJob(env, autoTriggerWebhook(repoFullName, 5006, "auto-4196-first-sha", "opened")); + expect(runCalls).toBe(1); + + // A genuine new push: the PR's cached head SHA moves, re-seeding the manifest (features.e2eTests stays + // on) and re-running the webhook at the NEW sha. + await upsertPullRequestFromGitHub(env, repoFullName, { number: 5006, title: "Add retry to checkout", state: "open", user: { login: "contributor" }, author_association: "CONTRIBUTOR", head: { sha: "auto-4196-second-sha", ref: "feature/checkout-retry" }, labels: [], body: "No validation evidence mentioned here." }); + await processJob(env, autoTriggerWebhook(repoFullName, 5006, "auto-4196-second-sha", "synchronize")); + + expect(runCalls).toBe(2); + expect(posted.count).toBe(2); + }); + + it("an explicit @gittensory generate-tests command still regenerates on the SAME head SHA the auto-trigger already covered", async () => { + const repoFullName = "JSONbored/auto-e2e-4196-explicit-after-auto"; + let runCalls = 0; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => { runCalls += 1; return { response: "```typescript\n" + AUTO_TEST_SOURCE + "\n```" }; } } as unknown as Ai, + GITTENSORY_REVIEW_E2E_TESTS: "true", + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + }); + await seedAutoTriggerPr(env, repoFullName, 5007, "auto-4196-explicit-sha"); + const posted = { count: 0, body: "" }; + stubAutoTriggerFetch(5007, posted); + await processJob(env, autoTriggerWebhook(repoFullName, 5007, "auto-4196-explicit-sha")); + expect(runCalls).toBe(1); + + // Now the maintainer explicitly asks, on the SAME PR at the SAME (still-unpushed) head SHA. The + // auto-trigger's dedup guard must not leak into the explicit command's own path. + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/collaborators/maintainer/permission")) return Response.json({ permission: "admin" }); + if (url.includes("/issues/5007/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/5007/comments") && method === "POST") { posted.count += 1; posted.body = String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? ""); return Response.json({ id: 50070 }); } + return new Response("not found", { status: 404 }); + }); + await processJob(env, { + type: "github-webhook", + deliveryId: "auto-e2e-4196-explicit-command", + eventName: "issue_comment", + payload: { + action: "created", + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, + repository: { name: "auto-e2e-4196-explicit-after-auto", full_name: repoFullName, private: false, owner: { login: "JSONbored" } }, + issue: { number: 5007, title: "Add retry to checkout", state: "open", user: { login: "contributor" }, pull_request: {} }, + comment: { id: 50071, body: "@gittensory generate-tests", author_association: "MEMBER", user: { login: "maintainer", type: "User" } }, + sender: { login: "maintainer", type: "User" }, + }, + } as unknown as Parameters[1]); + + expect(runCalls).toBe(2); + expect(posted.count).toBe(2); + const rows = await env.DB.prepare("select count(*) as n from audit_events where event_type = ?").bind("github_app.e2e_tests_generation").first<{ n: number }>(); + expect(rows?.n).toBe(2); + }); + + it("respects agentPaused — records a skip and never spends an LLM call, even though the signal fired", async () => { + const repoFullName = "JSONbored/auto-e2e-4196-paused"; + const run = vi.fn(); + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), AI: { run } as unknown as Ai, GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await seedAutoTriggerPr(env, repoFullName, 5008, "auto-4196-paused-sha"); + await upsertRepositorySettings(env, { repoFullName, commentMode: "off", publicSurface: "off", autoLabelEnabled: false, checkRunMode: "off", gateCheckMode: "enabled", requireLinkedIssue: false, linkedIssueGateMode: "off", manifestPolicyGateMode: "advisory", aiReviewMode: "off", agentPaused: true }); + const posted = { count: 0, body: "" }; + stubAutoTriggerFetch(5008, posted); + + await processJob(env, autoTriggerWebhook(repoFullName, 5008, "auto-4196-paused-sha")); + + expect(run).not.toHaveBeenCalled(); + expect(posted.count).toBe(0); + const skipped = await env.DB.prepare("select detail from audit_events where event_type = ?").bind("github_app.e2e_tests_generation_skipped").first<{ detail: string }>(); + expect(skipped?.detail).toBe("agent_paused"); + }); + + it("respects agentDryRun — records a skip with detail dry_run (not agent_paused), and never spends an LLM call", async () => { + const repoFullName = "JSONbored/auto-e2e-4196-dryrun"; + const run = vi.fn(); + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), AI: { run } as unknown as Ai, GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await seedAutoTriggerPr(env, repoFullName, 5009, "auto-4196-dryrun-sha"); + await upsertRepositorySettings(env, { repoFullName, commentMode: "off", publicSurface: "off", autoLabelEnabled: false, checkRunMode: "off", gateCheckMode: "enabled", requireLinkedIssue: false, linkedIssueGateMode: "off", manifestPolicyGateMode: "advisory", aiReviewMode: "off", agentDryRun: true }); + const posted = { count: 0, body: "" }; + stubAutoTriggerFetch(5009, posted); + + await processJob(env, autoTriggerWebhook(repoFullName, 5009, "auto-4196-dryrun-sha")); + + expect(run).not.toHaveBeenCalled(); + expect(posted.count).toBe(0); + const skipped = await env.DB.prepare("select detail from audit_events where event_type = ?").bind("github_app.e2e_tests_generation_skipped").first<{ detail: string }>(); + expect(skipped?.detail).toBe("dry_run"); + }); + + it("attributes the generated test to \"the PR author\" when the cached PR has no author login at all (a ghost/deleted account)", async () => { + const repoFullName = "JSONbored/auto-e2e-4196-no-author"; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => ({ response: "```typescript\n" + AUTO_TEST_SOURCE + "\n```" }) } as unknown as Ai, + GITTENSORY_REVIEW_E2E_TESTS: "true", + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + }); + const slash = repoFullName.indexOf("/"); + await upsertRepositoryFromGitHub(env, { name: repoFullName.slice(slash + 1), full_name: repoFullName, private: false, owner: { login: repoFullName.slice(0, slash) } }, 123); + await upsertRepositorySettings(env, { repoFullName, commentMode: "off", publicSurface: "off", autoLabelEnabled: false, checkRunMode: "off", gateCheckMode: "enabled", requireLinkedIssue: false, linkedIssueGateMode: "off", manifestPolicyGateMode: "advisory", aiReviewMode: "off" }); + // Deliberately no `user` field at all -- authorLogin resolves to null, exercising the `author ?? "the PR + // author"` fallback arm (the explicit command's own `actor` is always a real commenter login, so this + // branch is reachable only from the auto-trigger, which has no comment-invoker to fall back on). + await upsertPullRequestFromGitHub(env, repoFullName, { number: 5010, title: "Add retry to checkout", state: "open", author_association: "CONTRIBUTOR", head: { sha: "auto-4196-no-author-sha", ref: "feature/checkout-retry" }, labels: [], body: "No validation evidence mentioned here." }); + await upsertPullRequestFile(env, { repoFullName, pullNumber: 5010, path: "src/checkout.ts", status: "modified", additions: 3, deletions: 0, changes: 3, payload: { patch: "+function retryPayment() {\n+ return true;\n+}" } }); + await upsertRepoFocusManifest(env, repoFullName, { testExpectations: ["Run npm run test:ci."], features: { e2eTests: true } }); + const posted = { count: 0, body: "" }; + stubAutoTriggerFetch(5010, posted); + + // Built inline (not via autoTriggerWebhook) so the incoming payload's own pull_request sub-object omits + // `user` too -- autoTriggerWebhook always hardcodes a real `user.login`, which would re-upsert (and thus + // restore) an author login before this pass ever runs. + await processJob(env, { + type: "github-webhook", + deliveryId: "auto-e2e-4196-no-author", + eventName: "pull_request", + payload: { + action: "opened", + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, + repository: { name: "auto-e2e-4196-no-author", full_name: repoFullName, private: false, owner: { login: "JSONbored" } }, + pull_request: { number: 5010, title: "Add retry to checkout", state: "open", head: { sha: "auto-4196-no-author-sha" }, labels: [], body: "No validation evidence mentioned here." }, + }, + } as unknown as Parameters[1]); + + expect(posted.count).toBe(1); + expect(posted.body).toContain("AI-generated Playwright test for @the PR author"); + }); + }); + it("ops-alerts job no-ops when GITTENSORY_REVIEW_OPS is OFF (does no anomaly scan)", async () => { const env = createTestEnv(); // flag unset → OFF await env.DB.prepare("INSERT INTO repositories (full_name, owner, name, is_installed, is_registered) VALUES (?, ?, ?, 1, 1)")