From 538e09a35190925deedae91b491a9bee37620bb0 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Tue, 29 Sep 2026 17:11:10 -0700 Subject: [PATCH] test(evals): add director-graded cases on the dir-shop fixture --- evals/capability/cases/dir-counsel/case.json | 8 +++++ evals/capability/cases/dir-counsel/verify.sh | 9 +++++ evals/capability/cases/dir-critic/case.json | 8 +++++ evals/capability/cases/dir-critic/verify.sh | 8 +++++ evals/capability/cases/dir-explorer/case.json | 8 +++++ evals/capability/cases/dir-explorer/verify.sh | 7 ++++ evals/capability/cases/dir-tester/case.json | 8 +++++ evals/capability/cases/dir-tester/verify.sh | 9 +++++ fixtures/dir-shop/package.json | 5 +++ fixtures/dir-shop/src/cart.ts | 25 ++++++++++++++ fixtures/dir-shop/tests/cart.test.ts | 33 +++++++++++++++++++ 11 files changed, 128 insertions(+) create mode 100644 evals/capability/cases/dir-counsel/case.json create mode 100755 evals/capability/cases/dir-counsel/verify.sh create mode 100644 evals/capability/cases/dir-critic/case.json create mode 100755 evals/capability/cases/dir-critic/verify.sh create mode 100644 evals/capability/cases/dir-explorer/case.json create mode 100755 evals/capability/cases/dir-explorer/verify.sh create mode 100644 evals/capability/cases/dir-tester/case.json create mode 100755 evals/capability/cases/dir-tester/verify.sh create mode 100644 fixtures/dir-shop/package.json create mode 100644 fixtures/dir-shop/src/cart.ts create mode 100644 fixtures/dir-shop/tests/cart.test.ts diff --git a/evals/capability/cases/dir-counsel/case.json b/evals/capability/cases/dir-counsel/case.json new file mode 100644 index 000000000..1d781c34e --- /dev/null +++ b/evals/capability/cases/dir-counsel/case.json @@ -0,0 +1,8 @@ +{ + "id": "dir-counsel", + "tier": "easy", + "title": "Counsel plans a coupon feature", + "fixture": "fixtures/dir-shop", + "prompt": "Plan adding coupon codes (percentage off, single use per cart) to this cart. Give an ordered engineering plan. Do not change source. Save your final plan (the same text you return) to PLAN.md.", + "verify": "verify.sh" +} diff --git a/evals/capability/cases/dir-counsel/verify.sh b/evals/capability/cases/dir-counsel/verify.sh new file mode 100755 index 000000000..246e3aacf --- /dev/null +++ b/evals/capability/cases/dir-counsel/verify.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh" +dirty=$(git status --porcelain | grep -v -E "(REPORT|PLAN)\.md$|\.corbits/" || true) +[[ -z "$dirty" ]] || { echo "FAIL: source files changed: $dirty"; exit 1; } +[[ -f PLAN.md ]] || { echo 'FAIL: no PLAN.md'; exit 1; } +grep -q 'src/cart.ts' PLAN.md && grep -q 'tests/' PLAN.md || { echo 'FAIL: plan missing files'; exit 1; } +n=$(grep -c -E '^\s*([0-9]+[.)]|-|\*)\s' PLAN.md || true) +[[ "$n" -ge 4 ]] || { echo 'FAIL: fewer than 4 steps'; exit 1; } +echo 'PASS: counsel' diff --git a/evals/capability/cases/dir-critic/case.json b/evals/capability/cases/dir-critic/case.json new file mode 100644 index 000000000..d207fe38e --- /dev/null +++ b/evals/capability/cases/dir-critic/case.json @@ -0,0 +1,8 @@ +{ + "id": "dir-critic", + "tier": "easy", + "title": "Critic finds planted defects", + "fixture": "fixtures/dir-shop", + "prompt": "Review src/cart.ts for defects. Do not fix anything. Save your final report (the same text you return) to REPORT.md.", + "verify": "verify.sh" +} diff --git a/evals/capability/cases/dir-critic/verify.sh b/evals/capability/cases/dir-critic/verify.sh new file mode 100755 index 000000000..bdf9ec4f3 --- /dev/null +++ b/evals/capability/cases/dir-critic/verify.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh" +dirty=$(git status --porcelain | grep -v -E "(REPORT|PLAN)\.md$|\.corbits/" || true) +[[ -z "$dirty" ]] || { echo "FAIL: source files changed: $dirty"; exit 1; } +[[ -f REPORT.md ]] || { echo 'FAIL: no REPORT.md'; exit 1; } +grep -qi -E 'twice|double|both.*(lineTotal|cartTotal)|applied again|compound' REPORT.md || { echo 'FAIL: missed double discount'; exit 1; } +grep -qi -E 'splice|skip|mutat|index' REPORT.md || { echo 'FAIL: missed removeSku splice bug'; exit 1; } +echo 'PASS: critic found both' diff --git a/evals/capability/cases/dir-explorer/case.json b/evals/capability/cases/dir-explorer/case.json new file mode 100644 index 000000000..6eafe4229 --- /dev/null +++ b/evals/capability/cases/dir-explorer/case.json @@ -0,0 +1,8 @@ +{ + "id": "dir-explorer", + "tier": "easy", + "title": "Explorer locates discount logic", + "fixture": "fixtures/dir-shop", + "prompt": "Where is the discount applied to a cart, and which functions are involved? Do not change any source. Save your final report (the same text you return) to REPORT.md.", + "verify": "verify.sh" +} diff --git a/evals/capability/cases/dir-explorer/verify.sh b/evals/capability/cases/dir-explorer/verify.sh new file mode 100755 index 000000000..1c9079bc3 --- /dev/null +++ b/evals/capability/cases/dir-explorer/verify.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh" +dirty=$(git status --porcelain | grep -v -E "(REPORT|PLAN)\.md$|\.corbits/" || true) +[[ -z "$dirty" ]] || { echo "FAIL: source files changed: $dirty"; exit 1; } +[[ -f REPORT.md ]] || { echo 'FAIL: no REPORT.md'; exit 1; } +grep -q 'src/cart.ts' REPORT.md && grep -q 'lineTotal' REPORT.md && grep -q 'cartTotal' REPORT.md || { echo 'FAIL: report missing path or functions'; exit 1; } +echo 'PASS: explorer' diff --git a/evals/capability/cases/dir-tester/case.json b/evals/capability/cases/dir-tester/case.json new file mode 100644 index 000000000..0c81ea31d --- /dev/null +++ b/evals/capability/cases/dir-tester/case.json @@ -0,0 +1,8 @@ +{ + "id": "dir-tester", + "tier": "easy", + "title": "Tester reports failing test", + "fixture": "fixtures/dir-shop", + "prompt": "Run the test suite and report exactly which test fails and the likely cause. Do not fix anything. Save your final report (the same text you return) to REPORT.md.", + "verify": "verify.sh" +} diff --git a/evals/capability/cases/dir-tester/verify.sh b/evals/capability/cases/dir-tester/verify.sh new file mode 100755 index 000000000..8589ea9e7 --- /dev/null +++ b/evals/capability/cases/dir-tester/verify.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh" +dirty=$(git status --porcelain | grep -v -E "(REPORT|PLAN)\.md$|\.corbits/" || true) +[[ -z "$dirty" ]] || { echo "FAIL: source files changed: $dirty"; exit 1; } +[[ -f REPORT.md ]] || { echo 'FAIL: no REPORT.md'; exit 1; } +grep -qi 'removes all matching skus' REPORT.md || { echo 'FAIL: failing test not named'; exit 1; } +grep -qi -E 'splice|skip|mutat|index' REPORT.md || { echo 'FAIL: cause not identified'; exit 1; } +grep -qi -E 'bun test|exit' REPORT.md || { echo 'FAIL: no command evidence'; exit 1; } +echo 'PASS: tester' diff --git a/fixtures/dir-shop/package.json b/fixtures/dir-shop/package.json new file mode 100644 index 000000000..30c092aab --- /dev/null +++ b/fixtures/dir-shop/package.json @@ -0,0 +1,5 @@ +{ + "name": "dir-shop-fixture", + "private": true, + "type": "module" +} diff --git a/fixtures/dir-shop/src/cart.ts b/fixtures/dir-shop/src/cart.ts new file mode 100644 index 000000000..4ba7b8abe --- /dev/null +++ b/fixtures/dir-shop/src/cart.ts @@ -0,0 +1,25 @@ +export interface Item { + sku: string; + price: number; + qty: number; +} + +export function lineTotal(item: Item, discountPct: number): number { + const discounted = item.price * (1 - discountPct / 100); + return discounted * item.qty; +} + +export function cartTotal(items: Item[], discountPct = 0): number { + const sum = items.reduce( + (acc, item) => acc + lineTotal(item, discountPct), + 0, + ); + return sum * (1 - discountPct / 100); +} + +export function removeSku(items: Item[], sku: string): Item[] { + for (let i = 0; i < items.length; i++) { + if (items[i]?.sku === sku) items.splice(i, 1); + } + return items; +} diff --git a/fixtures/dir-shop/tests/cart.test.ts b/fixtures/dir-shop/tests/cart.test.ts new file mode 100644 index 000000000..bba9829be --- /dev/null +++ b/fixtures/dir-shop/tests/cart.test.ts @@ -0,0 +1,33 @@ +import { expect, test } from "bun:test"; +import { cartTotal, lineTotal, removeSku } from "../src/cart"; + +test("line total without discount", () => { + expect(lineTotal({ sku: "a", price: 10, qty: 2 }, 0)).toBe(20); +}); + +test("cart total without discount", () => { + expect(cartTotal([{ sku: "a", price: 10, qty: 2 }])).toBe(20); +}); + +test("removes a single sku", () => { + const out = removeSku( + [ + { sku: "a", price: 1, qty: 1 }, + { sku: "b", price: 1, qty: 1 }, + ], + "a", + ); + expect(out.map((i) => i.sku)).toEqual(["b"]); +}); + +test("removes all matching skus", () => { + const out = removeSku( + [ + { sku: "a", price: 1, qty: 1 }, + { sku: "a", price: 1, qty: 1 }, + { sku: "b", price: 1, qty: 1 }, + ], + "a", + ); + expect(out.map((i) => i.sku)).toEqual(["b"]); +});