feat: add provenance-linked real-project evidence

crates/workshop-rs-cli/Cargo.toml

@@ -12,7 +12,9 @@ workspace = true
 
 [dependencies]
 workshop-rs = { path = "../workshop-rs", version = "0.1.1" }
+serde = { version = "1", features = ["derive"] }
 serde_json = "1"
+sha2 = "0.10"
 
 [dev-dependencies]
 serde_json = "1"

crates/workshop-rs-cli/src/corpus.rs

@@ -0,0 +1,366 @@
+//! Offline execution for provenance-linked real-project evidence manifests.
+//!
+//! The manifest describes source and expectation provenance; this runner only
+//! measures the bundled Workshop text with the canonical parser and WIR
+//! validation. It never creates an expectation from the observed output.
+
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+use std::path::Path;
+
+use workshop_rs::catalog::{Catalog, CatalogIdentity, Locale};
+use workshop_rs::conformance::{
+    Comparison, ConformanceReason, ConformanceResult, ConformanceStatus, Equivalence, Evidence,
+    EvidenceArtifact, EvidenceClass, ExpectationSource, FeatureId, ReasonCode,
+};
+use workshop_rs::{parser, validate};
+
+#[derive(Debug, Deserialize)]
+struct CorpusManifest {
+    #[serde(rename = "schemaVersion")]
+    schema_version: u32,
+    id: String,
+    locale: String,
+    expectation: ExpectationSource,
+    cases: Vec<CorpusCase>,
+}
+
+#[derive(Debug, Deserialize)]
+struct CorpusCase {
+    id: String,
+    class: EvidenceClass,
+    fixture: String,
+    source: EvidenceArtifact,
+    features: Vec<FeatureId>,
+    #[serde(rename = "expectedStatus")]
+    expected_status: ExpectedStatus,
+    #[serde(rename = "failureContains")]
+    failure_contains: Option<String>,
+    #[serde(rename = "knownGap")]
+    known_gap: Option<KnownGap>,
+    #[serde(rename = "derivedFrom")]
+    derived_from: Option<String>,
+    expectation: Option<ExpectationSource>,
+}
+
+#[derive(Debug, Clone, Copy, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+enum ExpectedStatus {
+    Success,
+}
+
+#[derive(Debug, Deserialize)]
+struct KnownGap {
+    detail: String,
+    #[serde(rename = "trackingRef")]
+    tracking_ref: String,
+}
+
+#[derive(Debug, Serialize)]
+pub(crate) struct CorpusReport {
+    #[serde(rename = "schemaVersion")]
+    schema_version: u32,
+    manifest: String,
+    catalog: CatalogIdentity,
+    results: Vec<ConformanceResult>,
+    summary: CorpusSummary,
+}
+
+#[derive(Debug, Default, Serialize)]
+struct CorpusSummary {
+    matched: usize,
+    unsupported: usize,
+    #[serde(rename = "known-gap")]
+    known_gap: usize,
+    #[serde(rename = "unexpected-regression")]
+    unexpected_regression: usize,
+    inconclusive: usize,
+}
+
+impl CorpusSummary {
+    fn record(&mut self, status: ConformanceStatus) {
+        match status {
+            ConformanceStatus::Matched => self.matched += 1,
+            ConformanceStatus::Unsupported => self.unsupported += 1,
+            ConformanceStatus::KnownGap => self.known_gap += 1,
+            ConformanceStatus::UnexpectedRegression => self.unexpected_regression += 1,
+            ConformanceStatus::Inconclusive => self.inconclusive += 1,
+        }
+    }
+}
+
+impl CorpusReport {
+    pub(crate) fn has_unexpected_regression(&self) -> bool {
+        self.summary.unexpected_regression != 0
+    }
+
+    pub(crate) fn human_summary(&self) -> String {
+        let mut output = format!("manifest {}\n", self.manifest);
+        for result in &self.results {
+            output.push_str(&format!("{}: {:?}\n", result.case_id, result.status));
+        }
+        output.push_str(&format!(
+            "summary: matched={}, unsupported={}, known-gap={}, unexpected-regression={}, inconclusive={}\n",
+            self.summary.matched,
+            self.summary.unsupported,
+            self.summary.known_gap,
+            self.summary.unexpected_regression,
+            self.summary.inconclusive,
+        ));
+        output
+    }
+}
+
+pub(crate) fn run(manifest_path: &Path) -> Result<CorpusReport, String> {
+    let manifest_text = std::fs::read_to_string(manifest_path)
+        .map_err(|error| format!("cannot read {}: {error}", manifest_path.display()))?;
+    let manifest: CorpusManifest = serde_json::from_str(&manifest_text)
+        .map_err(|error| format!("invalid manifest JSON: {error}"))?;
+    if manifest.schema_version != 1 {
+        return Err(format!(
+            "unsupported manifest schema version {}; expected 1",
+            manifest.schema_version
+        ));
+    }
+    if manifest.cases.is_empty() {
+        return Err("manifest must contain at least one case".to_string());
+    }
+
+    let catalog = Catalog::builtin().map_err(|error| format!("catalog: {error}"))?;
+    let catalog_identity = catalog.identity();
+    let locale = Locale::new(&manifest.locale);
+    let manifest_dir = manifest_path
+        .parent()
+        .ok_or_else(|| "manifest has no parent directory".to_string())?;
+    let mut results = Vec::with_capacity(manifest.cases.len());
+    let mut summary = CorpusSummary::default();
+
+    for case in manifest.cases {
+        let fixture_path = manifest_dir.join(&case.fixture);
+        let input = std::fs::read_to_string(&fixture_path).map_err(|error| {
+            format!(
+                "cannot read case {} at {}: {error}",
+                case.id,
+                fixture_path.display()
+            )
+        })?;
+        validate_source_artifact(&case.id, &case.source, &input)?;
+        let expectation = case
+            .expectation
+            .clone()
+            .unwrap_or_else(|| manifest.expectation.clone());
+        validate_expected_artifact(&case.id, &expectation.artifact)?;
+        let evidence = Evidence {
+            class: case.class,
+            fixture: case.source.clone(),
+            expectation: expectation.clone(),
+            catalog: catalog_identity.clone(),
+            locale: Some(locale.clone()),
+            client: None,
+            implementation: Some(workshop_rs::conformance::ImplementationIdentity {
+                name: "workshop-rs".to_string(),
+                version: env!("CARGO_PKG_VERSION").to_string(),
+                revision: None,
+                artifact: None,
+            }),
+        };
+        let result = execute_case(&case, &input, &catalog, &locale, evidence, expectation)?;
+        result
+            .validate_against(&catalog)
+            .map_err(|error| format!("case {} produced invalid #18 result: {error}", case.id))?;
+        summary.record(result.status);
+        results.push(result);
+    }
+
+    Ok(CorpusReport {
+        schema_version: 1,
+        manifest: manifest.id,
+        catalog: catalog_identity,
+        results,
+        summary,
+    })
+}
+
+fn execute_case(
+    case: &CorpusCase,
+    input: &str,
+    catalog: &Catalog,
+    locale: &Locale,
+    evidence: Evidence,
+    expectation: ExpectationSource,
+) -> Result<ConformanceResult, String> {
+    let expected = expectation.artifact.clone();
+    let parsed = parser::parse_with_context(input, catalog, locale, catalog);
+    match (case.expected_status, parsed) {
+        (ExpectedStatus::Success, Ok(program)) => {
+            if let Err(error) = program.validate() {
+                return Ok(non_match(
+                    case,
+                    evidence,
+                    ConformanceStatus::UnexpectedRegression,
+                    expected,
+                    format!("WIR validation failed: {error}"),
+                    ReasonCode::UnexpectedRegression,
+                    None,
+                ));
+            }
+            if let Err(error) = validate::validate_canonical_ids(&program, catalog) {
+                return Ok(non_match(
+                    case,
+                    evidence,
+                    ConformanceStatus::UnexpectedRegression,
+                    expected,
+                    format!("canonical identity validation failed: {error}"),
+                    ReasonCode::UnexpectedRegression,
+                    None,
+                ));
+            }
+            let observed = EvidenceArtifact {
+                name: "workshop-rs canonical WIR dump".to_string(),
+                revision: None,
+                path: Some(case.fixture.clone()),
+                sha256: Some(sha256(&program.dump())),
+                license: Some("MIT".to_string()),
+            };
+            Ok(ConformanceResult {
+                schema_version: 1,
+                case_id: case.id.clone(),
+                features: case.features.clone(),
+                status: ConformanceStatus::Matched,
+                comparison: Comparison {
+                    mode: Equivalence::Semantic,
+                    expected: Some(expected),
+                    observed: Some(observed),
+                    normalizer: Some("parse-validate-canonical-wir-v1".to_string()),
+                },
+                evidence,
+                reason: None,
+            })
+        }
+        (ExpectedStatus::Success, Err(error)) => {
+            let detail = format!("offline parser observation: {error}");
+            let declared_gap = case.known_gap.as_ref().filter(|_gap| {
+                case.failure_contains
+                    .as_ref()
+                    .is_some_and(|needle| error.to_string().contains(needle))
+            });
+            if let Some(gap) = declared_gap {
+                Ok(non_match(
+                    case,
+                    evidence,
+                    ConformanceStatus::KnownGap,
+                    expected,
+                    detail,
+                    ReasonCode::KnownGap,
+                    Some((gap.detail.clone(), gap.tracking_ref.clone())),
+                ))
+            } else {
+                Ok(non_match(
+                    case,
+                    evidence,
+                    ConformanceStatus::UnexpectedRegression,
+                    expected,
+                    detail,
+                    ReasonCode::UnexpectedRegression,
+                    case.derived_from.as_ref().map(|source| {
+                        (format!("case is derived from {source}"), "#20".to_string())
+                    }),
+                ))
+            }
+        }
+    }
+}
+
+fn non_match(
+    case: &CorpusCase,
+    evidence: Evidence,
+    status: ConformanceStatus,
+    expected: EvidenceArtifact,
+    detail: String,
+    code: ReasonCode,
+    gap: Option<(String, String)>,
+) -> ConformanceResult {
+    let reason = gap.map(|(detail, tracking_ref)| ConformanceReason {
+        code,
+        detail,
+        tracking_ref: Some(tracking_ref),
+    });
+    let reason = reason.or(Some(ConformanceReason {
+        code,
+        detail,
+        tracking_ref: None,
+    }));
+    ConformanceResult {
+        schema_version: 1,
+        case_id: case.id.clone(),
+        features: case.features.clone(),
+        status,
+        comparison: Comparison {
+            mode: Equivalence::Semantic,
+            expected: Some(expected),
+            observed: None,
+            normalizer: Some("parse-validate-canonical-wir-v1".to_string()),
+        },
+        evidence,
+        reason,
+    }
+}
+
+fn sha256(value: &str) -> String {
+    let digest = Sha256::digest(value.as_bytes());
+    format!("{digest:x}")
+}
+
+fn validate_source_artifact(
+    case_id: &str,
+    artifact: &EvidenceArtifact,
+    input: &str,
+) -> Result<(), String> {
+    validate_expected_artifact(case_id, artifact)?;
+    let Some(expected_digest) = artifact.sha256.as_deref() else {
+        return Err(format!("case {case_id} source must pin a SHA-256 digest"));
+    };
+    let actual_digest = sha256(input);
+    if expected_digest != actual_digest {
+        return Err(format!(
+            "case {case_id} source digest mismatch: manifest {expected_digest}, fixture {actual_digest}"
+        ));
+    }
+    Ok(())
+}
+
+fn validate_expected_artifact(case_id: &str, artifact: &EvidenceArtifact) -> Result<(), String> {
+    if artifact.name.trim().is_empty() {
+        return Err(format!(
+            "case {case_id} evidence artifact has no repository identity"
+        ));
+    }
+    if artifact.revision.as_deref().is_none_or(str::is_empty) {
+        return Err(format!(
+            "case {case_id} evidence artifact must pin a revision"
+        ));
+    }
+    if artifact.path.as_deref().is_none_or(str::is_empty) {
+        return Err(format!("case {case_id} evidence artifact must pin a path"));
+    }
+    let Some(digest) = artifact.sha256.as_deref() else {
+        return Err(format!(
+            "case {case_id} evidence artifact must pin a SHA-256 digest"
+        ));
+    };
+    if digest.len() != 64
+        || !digest
+            .chars()
+            .all(|character| character.is_ascii_hexdigit())
+    {
+        return Err(format!(
+            "case {case_id} evidence artifact has an invalid SHA-256 digest"
+        ));
+    }
+    if artifact.license.as_deref().is_none_or(str::is_empty) {
+        return Err(format!(
+            "case {case_id} evidence artifact must record a license"
+        ));
+    }
+    Ok(())
+}