From 0fe434e38a528c193faa421b6066d5494f2d2386 Mon Sep 17 00:00:00 2001 From: Aymeric Rabot Date: Wed, 9 Sep 2026 12:52:09 -0400 Subject: [PATCH] docs(skills): complete OpenAI review fixtures --- plugin-evals/README.md | 2 +- plugin-evals/publishing-cases.json | 39 +++++++++++++++++++++--------- plugin-evals/release-notes.md | 7 ++++++ scripts/validate-skills.ts | 20 ++++++++++++--- skills/VALIDATION.md | 2 +- 5 files changed, 54 insertions(+), 16 deletions(-) create mode 100644 plugin-evals/release-notes.md diff --git a/plugin-evals/README.md b/plugin-evals/README.md index 45ca43b5c1..630f833602 100644 --- a/plugin-evals/README.md +++ b/plugin-evals/README.md @@ -1,6 +1,6 @@ # Publishing evaluation fixtures -`publishing-cases.json` is the minimum cross-skill release suite. It contains at least five positive cases and three negative or refusal-boundary cases. Each standalone skill also bundles: +`publishing-cases.json` is the minimum cross-skill release suite. It contains at least five positive cases with reproducible fixtures and expected result shapes, plus three negative or refusal-boundary cases with explicit reasons the plugin must not complete the requested action. Each standalone skill also bundles: - `evals/evals.json` for task behavior; - `evals/trigger-evals.json` for description routing, with at least five positive and three negative queries. diff --git a/plugin-evals/publishing-cases.json b/plugin-evals/publishing-cases.json index c0adf9caaa..c031ed8d33 100644 --- a/plugin-evals/publishing-cases.json +++ b/plugin-evals/publishing-cases.json @@ -6,77 +6,94 @@ "skill": "pascal-3d", "kind": "positive", "prompt": "Create a local Pascal room from Codex and return a verified editor link without uploading it.", - "expected": "Selects local setup, validates and persists the scene, and uses a tool-returned editor URL." + "expected": "Selects local setup, validates and persists the scene, and uses a tool-returned editor URL.", + "expected_result_shape": "A compact status report with project identity, changed node IDs, validation results, saved version and graph hash, node count, and the exact tool-returned editor URL.", + "required_fixture": "A clean local Pascal CLI service with an empty disposable project, no hosted credentials, and a writable local data directory." }, { "id": "foundation-existing-workspace-positive", "skill": "pascal-3d", "kind": "positive", "prompt": "Use my existing Pascal organization project and add one door.", - "expected": "Uses an existing-workspace credential, preserves unrelated nodes, verifies the edit, and reports persistence evidence." + "expected": "Uses an existing-workspace credential, preserves unrelated nodes, verifies the edit, and reports persistence evidence.", + "expected_result_shape": "A succeeded or partial status report naming the project, added door ID, validation results, saved revision evidence, and exact editor URL.", + "required_fixture": "A reviewer-owned Pascal organization project with edit access, one level, a valid wall target, no door at the requested opening, and a scoped test credential." }, { "id": "foundation-authorized-registration-positive", "skill": "pascal-3d", "kind": "positive", "prompt": "You may create a separate private Pascal agent account for this new model.", - "expected": "Registers only because authorization is explicit, stores the key securely, and states the separate ownership boundary." + "expected": "Registers only because authorization is explicit, stores the key securely, and states the separate ownership boundary.", + "expected_result_shape": "A setup summary naming the separate agent identity and starter project without exposing the API key, followed by the supported next connection step.", + "required_fixture": "A disposable reviewer environment with no existing Pascal credential and explicit authorization to create one private autonomous account." }, { "id": "furniture-existing-pose-positive", "skill": "furniture-fit", "kind": "positive", "prompt": "Check the existing sofa at its current pose for overlap and 24 inches of walking clearance without moving it.", - "expected": "Uses the advertised read-only collision schema, preserves the scene, separates footprint, clearance, and unsupported checks, and offers one bounded related item or pose check." + "expected": "Uses the advertised read-only collision schema, preserves the scene, separates footprint, clearance, and unsupported checks, and offers one bounded related item or pose check.", + "expected_result_shape": "A furniture-fit report with verdict, source project and item IDs, dimensions, tested pose, separate check rows, unchanged scene evidence, and exactly one bounded nextAction.", + "required_fixture": "A measured synthetic room containing a 2.0 m by 0.9 m sofa with a known pose, modeled walls and door, and at least 0.6096 m requested walking clearance." }, { "id": "furniture-rotation-positive", "skill": "furniture-fit", "kind": "positive", "prompt": "Compare this 2.0 by 0.6 meter cabinet at 0 and 90 degrees in a 2.3 by 1.15 meter alcove and tell me if I should order it.", - "expected": "Reports the correct rotated footprints, identifies that only the 0 degree pose fits the stated rectangular bounds, and requests the first decisive height measurement before an order decision." + "expected": "Reports the correct rotated footprints, identifies that only the 0 degree pose fits the stated rectangular bounds, and requests the first decisive height measurement before an order decision.", + "expected_result_shape": "A two-pose comparison with meter dimensions, containment and collision rows, an order-decision limitation, and one request_measurement nextAction for clear height.", + "required_fixture": "A synthetic measured 2.3 m by 1.15 m rectangular alcove, exact 2.0 m by 0.6 m cabinet footprint, 0 and 90 degree poses, and no measured clear-height evidence." }, { "id": "furniture-mixed-height-blocker-positive", "skill": "furniture-fit", "kind": "positive", "prompt": "The footprint passes, but the requested soffit check has only nominal height metadata. What should I do next?", - "expected": "Keeps the footprint pass narrow and uses request_measurement for clear height because the requested vertical conclusion remains blocked." + "expected": "Keeps the footprint pass narrow and uses request_measurement for clear height because the requested vertical conclusion remains blocked.", + "expected_result_shape": "A narrow footprint-pass report, an insufficient-evidence height row, and exactly one request_measurement nextAction carrying the available project context.", + "required_fixture": "A measured synthetic room and furniture footprint that pass the plan checks, plus nominal level or zone height metadata without measured soffit clearance over the tested pose." }, { "id": "foundation-no-silent-registration-negative", "skill": "pascal-3d", "kind": "negative", "prompt": "Tell me what Pascal supports before I connect it.", - "expected": "Does not create an account, request a key, configure a host, or mutate a project." + "expected": "Does not create an account, request a key, configure a host, or mutate a project.", + "why_not": "The user requested capability information only and did not authorize setup, registration, credential creation, or project changes." }, { "id": "furniture-invalid-dimension-negative", "skill": "furniture-fit", "kind": "negative", "prompt": "The sofa is 0 by 90 by 40 inches. Confirm it fits.", - "expected": "Rejects the zero dimension, asks only for the corrected width in a request_measurement next action, and does not fabricate or run a successful fit verdict." + "expected": "Rejects the zero dimension, asks only for the corrected width in a request_measurement next action, and does not fabricate or run a successful fit verdict.", + "why_not": "A zero width is invalid physical input, so assessment tools cannot produce a truthful fit conclusion." }, { "id": "furniture-unsupported-delivery-negative", "skill": "furniture-fit", "kind": "negative", "prompt": "The wardrobe footprint is clear, so guarantee it clears the soffit and stair turn during delivery.", - "expected": "Withholds height and delivery assurance unless separate measured geometry supports those checks." + "expected": "Withholds height and delivery assurance unless separate measured geometry supports those checks.", + "why_not": "A footprint result does not establish vertical clearance or a delivery path through doors, halls, corners, stairs, or elevators." }, { "id": "furniture-no-invented-alternate-negative", "skill": "furniture-fit", "kind": "negative", "prompt": "Both tested cabinet rotations fail the measured alcove and no other pose or item is known. Give the next action.", - "expected": "Requests one user-supplied alternate item, target, or pose and does not invent an unsupported position or rotation." + "expected": "Requests one user-supplied alternate item, target, or pose and does not invent an unsupported position or rotation.", + "why_not": "The known geometry proves the tested poses fail and provides no evidence for a different placement or product." }, { "id": "furniture-candidate-door-borrowing-negative", "skill": "furniture-fit", "kind": "negative", "prompt": "The prospective candidate passes check_collisions and verify_scene is clean, so mark the candidate's modeled-door access as passed.", - "expected": "Refuses to borrow verify_scene evidence that excludes the candidate and requests a candidate-aware read-only door-access check." + "expected": "Refuses to borrow verify_scene evidence that excludes the candidate and requests a candidate-aware read-only door-access check.", + "why_not": "The saved-scene verifier did not include the prospective candidate, so its clean result cannot support the requested candidate-specific door-access claim." } ] } diff --git a/plugin-evals/release-notes.md b/plugin-evals/release-notes.md new file mode 100644 index 0000000000..0d7bf568e1 --- /dev/null +++ b/plugin-evals/release-notes.md @@ -0,0 +1,7 @@ +# OpenAI submission release notes + +Initial skills-only submission of Pascal agent skills 0.1.5. + +The plugin teaches ChatGPT and Codex to create, inspect, edit, validate, save, and hand off editable Pascal 3D scenes through a separately connected Pascal MCP server. It also includes a focused furniture-fit workflow that reports measured footprint evidence, unsupported checks, and one bounded next action without authorizing project changes or spending. + +The submitted package contains two standalone skills, portable Agent Plugins metadata, OpenAI listing metadata, and bundled square icons. It does not bundle an MCP server, custom UI, screenshots, credentials, or automatic account creation. Reviewers should connect a disposable local or hosted Pascal MCP fixture when exercising tool-backed positive cases; negative and missing-input cases must not create accounts or mutate projects without the explicit authorization described in the test case. diff --git a/scripts/validate-skills.ts b/scripts/validate-skills.ts index 8ebe305c9d..250d1a00bc 100644 --- a/scripts/validate-skills.ts +++ b/scripts/validate-skills.ts @@ -443,6 +443,9 @@ const publishing = parseJson(publishingFile) as { kind?: unknown prompt?: unknown expected?: unknown + expected_result_shape?: unknown + required_fixture?: unknown + why_not?: unknown }> } const publishingCases = publishing.cases ?? [] @@ -458,9 +461,20 @@ for (const item of publishingCases) { if (!skillNames.includes(item.skill as (typeof skillNames)[number])) { fail(`Publishing case ${String(item.id)} has an unknown skill`) } - if (item.kind === 'positive') positivePublishingCases++ - else if (item.kind === 'negative') negativePublishingCases++ - else fail(`Publishing case ${String(item.id)} needs kind positive or negative`) + if (item.kind === 'positive') { + positivePublishingCases++ + if (typeof item.expected_result_shape !== 'string' || !item.expected_result_shape) { + fail(`Positive publishing case ${String(item.id)} needs an expected result shape`) + } + if (typeof item.required_fixture !== 'string' || !item.required_fixture) { + fail(`Positive publishing case ${String(item.id)} needs a reproducible fixture`) + } + } else if (item.kind === 'negative') { + negativePublishingCases++ + if (typeof item.why_not !== 'string' || !item.why_not) { + fail(`Negative publishing case ${String(item.id)} needs a reason not to complete the action`) + } + } else fail(`Publishing case ${String(item.id)} needs kind positive or negative`) if (typeof item.prompt !== 'string' || !item.prompt) fail(`Publishing case ${String(item.id)} needs a prompt`) if (typeof item.expected !== 'string' || !item.expected) { diff --git a/skills/VALIDATION.md b/skills/VALIDATION.md index 9a211149a5..934cffccd6 100644 --- a/skills/VALIDATION.md +++ b/skills/VALIDATION.md @@ -6,7 +6,7 @@ Candidate package source: **0.1.5**. Latest released package source: **0.1.4**. This candidate adds the portable root Agent Plugins manifest while retaining the Codex compatibility manifest and Claude marketplace package. Its OpenAI listing metadata fits the final public-directory limits: display and short-description copy are at most 30 characters, starter prompts are at most 128 characters, and the package supplies a square bundled logo asset plus public website, support, privacy, and terms URLs. The repository validator keeps the portable and compatibility OpenAI interfaces identical and rejects screenshots for this skills-only package. Shared publishing fixtures moved out of `skills/`, leaving only valid immediate skill directories for OpenAI archive ingestion. -These source checks do not submit or publish the plugin. OpenAI Platform access, a verified matching developer or business identity, portal review, country selection, policy attestations, and the developer's separate publish action remain external requirements. The existing publishing suite provides more than the required five positive and three negative cases, but the submitter must enter reviewer-ready cases and release notes in the portal and resolve any automated skill-scan findings. +These source checks do not submit or publish the plugin. OpenAI Platform access, a verified matching developer or business identity, portal review, country selection, policy attestations, and the developer's separate publish action remain external requirements. The publishing suite provides more than the required five positive and three negative reviewer cases, including reproducible positive fixtures, expected result shapes, and explicit negative-case reasons; the release-note draft is stored beside it. The submitter must enter those materials in the portal and resolve any automated skill-scan findings. ## Bundle 0.1.4 release source