From a8569931cbefe3e6d8737bb9e80497b5062aa7fe Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 12:54:56 +0000 Subject: [PATCH 1/4] docs(medcare): consolidate the two-lineage investigation into one active plan Six-agent read-only investigation + unshallowed git history (5 grafted roots -> 4162 commits) established that lance-graph holds TWO MedCare lineages, not one incomplete runtime: - LIVE (consumer-pull): MedCare-rs medcare-bridge -> vendored lance-graph-ogar -> MedcareBridge = UnifiedBridge -> OGAR canonical Health codebook 0x0901..0x090C. Bridge migration COMPLETED at ddb6c840 (2026-06-21); deprecated alias only. Contract codebook mirror verified in sync slot-for-slot; the 7-alias-vs-12-slot gap is intentional (harvest mints, no OGIT entity). - DEAD (host-side scaffold): modules/medcare/manifest.yaml -> CallcenterSupervisor -> MedcareConsumerActor, frozen since birth 2026-05-13. Manifest compile-time parsed but runtime-orphaned (one caller: a test); entity codes / action_capabilities / message_type discarded pre-codegen; medcare_policy nonexistent; StubConsumerActor hard-coded; Dispatch rejected before any child; MedCareActor / MedCareMessage exist in NEITHER repository. New ACTIVE plan medcare-consumer-pull-thinking-proof-v1: prove one real medical thought over the live consumer-pull path (HealthcarePort classids -> cognitive-shader-driver + real MailboxSoA -> owner_adapter cast -> the #879 sparse cycle loop), falsifiers F1-F4 including the currently-absent Healthcare fail-closed unknown-actor test. ogar-obo (MONDO/HPO/Uberon/PATO, verified real, zero consumer edges) is an optional slice. Older MedCare plans classified (completed / dormant- decision-required / stale-but-unmarked); open decisions carried, not blocked on (OQ-2 retention 2190/3650, Ueberweisung/Anamnese canon gap, dead-lineage retire-vs-revive, .grok lineage, NoopAuditSink default). Documentation-only: no Rust, no tests, no manifests changed. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/EPIPHANIES.md | 10 ++ .claude/board/INTEGRATION_PLANS.md | 24 +++ .claude/board/LATEST_STATE.md | 5 + ...medcare-consumer-pull-thinking-proof-v1.md | 144 ++++++++++++++++++ 4 files changed, 183 insertions(+) create mode 100644 .claude/plans/medcare-consumer-pull-thinking-proof-v1.md diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 2192e4ad7..1b0575df2 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -1,3 +1,13 @@ +## 2026-08-02 — E-TWO-MEDCARE-LINEAGES-THE-LIVE-ONE-PULLS-THE-DEAD-ONE-HOSTS-1 — MedCare is integrated through consumer-pull; the host-side actor/manifest scaffold froze on its birth day + +**Status:** FINDING (six-agent read-only investigation + unshallowed git history, HEAD `71d1db1`). **Confidence:** High — every claim carries a commit sha or file:line; contradicting prose ledgers were checked against code. Plan: `.claude/plans/medcare-consumer-pull-thinking-proof-v1.md`. + +**The finding.** lance-graph does NOT contain "one incomplete MedCare runtime" — it contains **two lineages**. The LIVE one is consumer-pull: MedCare-rs `medcare-bridge` → vendored `lance-graph-ogar` → `MedcareBridge = UnifiedBridge` (collapse commit `ddb6c840`, 2026-06-21; deprecated alias since `10e717d4`) → OGAR canonical Health codebook (`0x0901–0x090C`; the contract mirror verified in sync slot-for-slot; the 7-alias-vs-12-slot gap is INTENTIONAL per OGAR `ports.rs:598-606`). The DEAD one is the host-side scaffold born in the sprint-7 wave (2026-05-13, `927788e9`/`3f67aed6`) and **never functionally touched again**: `modules/medcare/manifest.yaml` is compile-time parsed (`contract/build.rs`) into `MANIFEST_METADATA` whose ONLY caller is a test; its richest fields (entity codes 100–105, all five `action_capabilities`, `actor.message_type`) are parsed then DISCARDED pre-codegen; `rbac_policy: medcare_policy` names a nonexistent symbol; `CallcenterSupervisor::spawn_consumer_actor` unconditionally spawns `StubConsumerActor` (`supervisor.rs:369`); `DispatchToG` returns `DispatchNotImplemented` before any child actor for every non-Health envelope; `MedcareConsumerActor` has ZERO constructors repo-wide; and the manifest's declared `MedCareActor`/`MedCareMessage` exist in NEITHER repository (verified against the MedCare-rs sibling directly). + +**Load-bearing distinctions this pins:** (a) TWO same-named `UnifiedBridge` types — ogar's `` resolver (NO authorize method) vs callcenter's `` authorize/audit wrapper — never composed, and currently UNCOMPOSABLE (callcenter has zero dep on lance-graph-ogar); classify composition as later hardening, never as a shipped edge. (b) `lance-graph-ogar` is workspace-EXCLUDED — a green parent-workspace build compiles none of it (`--manifest-path` required; 83 tests green there). (c) The audit default is `NoopAuditSink` + `SuperDomain::Unknown`/salt 0 with swallowed emit errors, and `MEDCARE_AUDIT_SALT` is read fail-open then discarded — for a HIPAA-regime domain, "configured" vs "recording" is indistinguishable (P0 candidate). (d) Retention 2190d (`StepDomain::Medcare`, tested) vs 3650d (manifest, dead) is the DOCUMENTED open question OQ-2 (HIPAA vs BMV-Ä §57), not an accident. (e) `Anamnese`/`Ueberweisung` are first-class consumer concepts (97/81 refs) with no port alias / no canonical id at all — a real canon gap, not naming noise. (f) A third, unreconciled `.grok/` lineage (Zone-3 via spear, 2026-05-08) contradicts both. + +**The lesson (generalizes):** when a repo hosts a consumer-shaped scaffold AND the consumer separately pulls the repo's libraries, the scaffold can freeze at birth while every ledger keeps narrating it as "the integration" — dormancy is invisible in prose because infra work *around* the frozen files (branch names, "first-thought loop" commits) keeps the term alive. The falsifier that cut through: per-file `git log` dormancy (81 days = age of the file) + constructor-count (zero) + the two-repo actor-name check. Also: the clone was SHALLOW (5 grafted roots, 11-day window) — `git fetch --unshallow` converted a "between 06-23 and 06-26 per prose" estimate into the exact commit `ddb6c840` two days earlier. **Archaeology on a shallow clone is prose archaeology; unshallow first.** + ## 2026-08-02 — E-D-MBX-SPINE-IS-STRAIGHT-TRACK-VERSION-IS-NOT-A-FLEET-STEP-SIGNAL-1 — the D-MBX production ownership map is ratified, and P4's fleet-wide fan-step is corrected to sparse sealed-transition application **Status:** RATIFIED (operator ruling, 2026-08-02) + CORRECTION of a same-session drift. **Confidence:** High for the ownership map (verified against `Cargo.toml` deps) and the sparse correction (it removes a direct contradiction with `E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`). Documentation-only — no Rust/tests changed. Recorded in `.claude/plans/cycle-loop-closure-driver-v1.md` §3, §9–§11. diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index 60cafbabf..64da9605e 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -1,3 +1,27 @@ +## 2026-08-02 — medcare-consumer-pull-thinking-proof v1 — ACTIVE (the single active MedCare plan after the two-lineage investigation) — main thread + +**Plan:** `.claude/plans/medcare-consumer-pull-thinking-proof-v1.md` +Consolidation of the 2026-08-02 six-agent read-only MedCare/Healthcare +investigation (HEAD `71d1db1`, git unshallowed to 4162 commits). Verdict: +**two lineages, not one incomplete runtime.** LIVE = consumer-pull (MedCare-rs +`medcare-bridge` → vendored `lance-graph-ogar` → `UnifiedBridge` +→ OGAR canon `0x0901–0x090C`; bridge migration COMPLETED at `ddb6c840` +2026-06-21, `MedcareBridge` deprecated alias only). DEAD = host-side +manifest/supervisor/actor scaffold (frozen since birth 2026-05-13; manifest +compile-time-parsed but runtime-orphaned; `MedcareConsumerActor` never +constructed; supervisor hard-codes `StubConsumerActor`; Dispatch rejected +pre-child; `MedCareActor`/`MedCareMessage` exist in NEITHER repo; +`medcare_policy` symbol nonexistent). Two same-named, never-composed +`UnifiedBridge` types (ogar resolver — no authorize; callcenter authorize/audit +— no ogar dep) classified as LATER hardening, not a proof prerequisite. +**Active proof target:** one real medical thought over the live consumer-pull +path — HealthcarePort classids → cognitive-shader-driver + REAL MailboxSoA +(closing the qualia-extractor honesty gap) → owner_adapter cast → PR #879 +sparse cycle loop; falsifiers F1–F4 incl. the currently-ABSENT fail-closed +unknown-actor test. `ogar-obo` (MONDO/HPO/Uberon/PATO — verified real, zero +consumer edges) = optional slice only. Older MedCare plans classified in the +plan's §8 ledger (completed / dormant-decision-required / stale-but-unmarked). + ## 2026-08-02 — cycle-loop-closure-driver v1 — PLANNED / CONJECTURE (the seam that makes persist_sink load-bearing at 64k) — main thread **Plan:** `.claude/plans/cycle-loop-closure-driver-v1.md` diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index ada1f2ff2..6bf172263 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,8 @@ +## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — MedCare two-lineage investigation CONSOLIDATED → one active plan (consumer-pull thinking proof) + +- Six-agent read-only investigation (code inventory · callcenter/policy · manifest→runtime chain · plans archaeology · git history · tests+build) + main-thread cross-repo verification (OGAR + MedCare-rs siblings) + `git fetch --unshallow` (5 grafted roots → 4162 commits). Full verdict in `EPIPHANIES.md` `E-TWO-MEDCARE-LINEAGES-THE-LIVE-ONE-PULLS-THE-DEAD-ONE-HOSTS-1`; classification + proof target in **`.claude/plans/medcare-consumer-pull-thinking-proof-v1.md`** (ACTIVE — the single active MedCare plan; INTEGRATION_PLANS prepended). +- Headline: **bridge migration COMPLETED** (`ddb6c840` 2026-06-21, `MedcareBridge` = deprecated alias over `UnifiedBridge`; codebook mirror verified in sync with OGAR); **host-side manifest/supervisor/actor lineage DORMANT since birth 2026-05-13** (manifest runtime-orphaned; `MedcareConsumerActor` never constructed; `StubConsumerActor` hard-coded; Dispatch rejected pre-child; `MedCareActor`/`MedCareMessage` in neither repo; `medcare_policy` nonexistent). Two same-named, uncomposable `UnifiedBridge` types = later hardening. **Active target: one real medical thought over the live consumer-pull path into the #879 cycle loop** (F1–F4 falsifiers, incl. the ABSENT Healthcare fail-closed unknown-actor test). P0 candidates surfaced: NoopAuditSink-by-default for a HIPAA-regime domain; fail-open discarded `MEDCARE_AUDIT_SALT`. Open decisions carried (§6): OQ-2 retention 2190/3650, `Ueberweisung`/`Anamnese` canon gap, dead-lineage retire/revive, `.grok/` lineage. + ## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — PR #879 review round: recovery/data-integrity holes fixed + scope honesty (grain-of-salt audit) Operator-forwarded review (grain of salt); each finding verified against code before acting. **Accepted + fixed (all real):** diff --git a/.claude/plans/medcare-consumer-pull-thinking-proof-v1.md b/.claude/plans/medcare-consumer-pull-thinking-proof-v1.md new file mode 100644 index 000000000..6ecf070d4 --- /dev/null +++ b/.claude/plans/medcare-consumer-pull-thinking-proof-v1.md @@ -0,0 +1,144 @@ +# medcare-consumer-pull-thinking-proof-v1 — one real medical thought through the already-live OGAR consumer path + +> **Status:** ACTIVE (consolidation + proof target). **Date:** 2026-08-02. +> **Supersedes as the single active MedCare plan:** the host-side +> manifest/supervisor/actor revival implied by `pr-e-1-manifest-modules.md`, +> `pr-g1-manifest-modules.md`, `pr-g2-ractor-supervisor.md §8` (those specs stay +> as historical records; see §3). Grounded in the 2026-08-02 read-only +> investigation (six-agent sweep + unshallowed git history, HEAD `71d1db1`). +> **Separation of concerns:** lance-graph is PUBLIC; MedCare-rs is PRIVATE. This +> plan names consumer crates/tests structurally only — no clinical schema +> detail, no sourcing/licensing reasoning, ever (MedCare-rs commitment #9). + +## 1. Canonical classification (the investigation's verdict, pinned) + +There are **two MedCare integration lineages, not one partially-assembled path**: + +| Surface | Classification | Evidence | +|---|---|---| +| Bridge migration (`MedcareBridge = UnifiedBridge`) | **COMPLETED** — deprecated compatibility alias only | commit `ddb6c840` (2026-06-21) collapsed the last bespoke per-tenant bridge; `#[deprecated]` landed `10e717d4` (2026-06-22) | +| Healthcare codebook promotion (`0x0901–0x090C`, 7 port aliases) | **COMPLETED** — canon in OGAR `ogar_vocab`; `contract::ogar_codebook` is a **verified wire-compatible mirror** (checked slot-for-slot this pass) | OGAR `ports.rs:286-309`, `lib.rs:1262-1273`; the 7-vs-12 gap is **intentional** (harvest mints carry no OGIT entity → no alias; pin at `ports.rs:598-606`) — do NOT create a Healthcare subset codebook | +| OGAR vocabulary plug-ins (`ogar-vocab`, `ogar-obo`, `ogar-fma`, `ogar-cpic`, ClassView, DDL adapters) | **EXISTING SUBSTRATE** — available plug-in infrastructure, NOT part of the live MedCare execution path (no `MedCare-rs → ogar-obo` edge exists; verified) | OGAR `crates/ogar-obo` (MONDO/HPO/Uberon/PATO); consumer grep clean | +| Host-side manifest/supervisor/actor path | **DORMANT** — decision required before any revival (see §3) | frozen since birth 2026-05-13 (81 days at investigation HEAD) | +| Callcenter authorization/audit composition (`callcenter::UnifiedBridge` ⊗ `ogar::UnifiedBridge

`) | **LATER integration + hardening** — conceptually right, **currently unbuildable** (callcenter has zero dep on lance-graph-ogar); not a prerequisite for the first thought | dep-graph verified | +| Real cognitive thought over MedCare data (shader-driver / MailboxSoA) | **ACTIVE PROOF TARGET** — this plan's §4 | the honest gap: cognition, not plumbing | +| Generic manifest-driven actor factory | **POST-PROOF generalization** (only if §3 rules "revive") | — | + +## 2. The live path (build on THIS) + +``` +MedCare-rs (private) + └─ crates/medcare-bridge ──(vendor/lance-graph softlink)──► + lance_graph_ogar::MedcareBridge = UnifiedBridge + ├─ entity()/entity_by_uri() — namespace-locked, codebook synthesis + ├─ ogar_vocab::ports::HealthcarePort (class_id("Patient")=0x0901, APP_PREFIX=0x0005) + └─ OntologyRegistry (TTL hydration) + └─ crates/medcare-rbac ──► ogar-vocab (direct) +``` + +Compiles and is tested on the consumer side (`healthcare_hydrate.rs` scope-lock) +and on this side (`bridge_scope_lock.rs`, `medcare_bridge_conforms`, all green — +NOTE: `lance-graph-ogar` is workspace-EXCLUDED; test via +`--manifest-path crates/lance-graph-ogar/Cargo.toml`, a green parent-workspace +build proves nothing about it). + +## 3. The dead lineage (dormant — do NOT revive to prove thinking) + +``` +modules/medcare/manifest.yaml → build.rs → MANIFEST_METADATA ╳ (one caller: a test) +CallcenterSupervisor::spawn_consumer_actor → StubConsumerActor (unconditional) +DispatchToG: Health → ok; ALL else → DispatchNotImplemented (child never reached) +MedcareConsumerActor — never constructed; all 8 arms tracing::debug! + TODO +manifest actor.type = MedCareActor / MedCareMessage — exist in NEITHER repo +rbac_policy: medcare_policy — symbol does not exist +entity codes 100–105 / action_capabilities / actor.message_type — parsed then DISCARDED +``` + +Reviving this is **not a wiring patch — it would be a new implementation based +on a stale declaration.** The literal MedCare surfaces that ARE live and useful +as workbench material — `StepDomain::Medcare::profile()`, `medcare_ontology()`, +the ontology_table/conformance fixtures, `ConsumerEnvelope` types — may be +reused as proof fixtures, but are NOT the canonical runtime composition and +must not be presented as such. + +**Open decision (operator):** retire vs revive. If revived: a real typed actor +factory keyed on live types, never on the fictional YAML names; and the +manifest either emits everything it declares or stops declaring it. + +## 4. The proof target — first medical thought (the drill hole) + +``` +MedCare-rs medical input (existing schema surface, private side) + → existing OGAR HealthcarePort bridge (classid resolution, namespace lock) + → existing MedCare-rs RBAC or a narrow proof policy (fail-closed) + → cognitive-shader-driver + REAL MailboxSoA (the honest gap) + → non-vacuous medical cognitive result (falsifier below) + → owner_adapter::emit_bootstrap_intent / BatchWriter (write-on-behalf cast) + → cycle_driver (PR #879): collect → seal (one WAL write) → apply sparse → next intent +``` + +A small direct adapter is acceptable. The known seam: `MailboxSoaView::qualia()` +is deferred; `run_cognitive_work_gated`'s caller-extractor bridges it today — +the proof should read gate inputs from the REAL SoA qualia column, closing the +"extractor-fed" honesty gap in the #879 ledger. + +**Falsifiers (per the P0 falsifiability rule):** +- F1 — the cognitive result is **non-vacuous**: two different medical inputs + produce two different gate outcomes (discriminates; not a constant). +- F2 — the classid on the thought's carrier is the OGAR canon (`0x0901`-family + via `HealthcarePort::class_id`), not a local literal. +- F3 — the intent round-trips: cast in cycle N is collected, sealed (exactly one + WAL write) and applied in N+1; unrepresented owners byte-identical. +- F4 — the policy leg is fail-closed: an unrecognized actor/role against the + Healthcare classid is DENIED (this test is currently ABSENT — finding h). + +## 5. Optional OBO slice (only if it makes the proof observably medical) + +`ogar-obo` exists (MONDO/HPO/Uberon/PATO). If the proof wants a visibly +clinical hop: `HealthcarePort::Diagnosis → MONDO term → optional HPO phenotype +→ cognitive operation`. No full vocabulary hydration, no ontology federation, +no cathedral. Skip entirely if F1–F4 pass without it. + +## 6. Open decisions carried (not blocked on — the proof proceeds around them) + +1. **OQ-2 retention:** 2190d (`StepDomain::Medcare`, tested) vs 3650d + (manifest, dead path) — HIPAA 6yr vs BMV-Ä §57 10yr. Operator/regulatory. +2. **`Ueberweisung` / `Anamnese` canon:** first-class consumer concepts; + `anamnesis` = `0x0908` (no port alias), `Ueberweisung` has NO canonical id. + OGAR-side mint decision. +3. **Dead-lineage retire/revive** (§3). +4. **Bridge composition edge** (callcenter⊗ogar) — later hardening. +5. **`.grok/` Zone-3/spear lineage** — unreconciled third model; needs a ruling + or a staleness banner. +6. **Audit default = `NoopAuditSink`** with `SuperDomain::Unknown`/salt 0 and + swallowed emit errors — for a HIPAA-regime domain, "configured" and + "recording" are indistinguishable. P0 candidate, own PR. + +## 7. Exclusions (scope fence) + +- NO manifest/supervisor/actor revival for the proof. +- NO new Healthcare subset codebook; NO local classid literals. +- NO callcenter⊗ogar dependency edge as a proof prerequisite. +- NO provisional-intent recovery ledger in #879: pre-commit failure → publish + nothing, mutate nothing, retry the byte-identical frozen cycle (`SealFailure`); + post-commit crash → recover committed history (`recover_fleet` + watermarks). + Committed-history recovery and ordinary failed-write recomputation stay + SEPARATE mechanisms. (Latency figures floating in review prose for these two + paths are **claimed, unverified** — nothing here measured them; do not cite.) +- NO clinical schema detail or sourcing/licensing reasoning in this public repo. + +## 8. Status ledger for the older documents + +| Doc | Disposition | +|---|---| +| `pr-e-1-manifest-modules.md`, `pr-g1-manifest-modules.md`, `pr-g2-ractor-supervisor.md` | historical — describe the dormant lineage | +| `pr-e1-medcare-super-domain.md` | historical + carries the open OQ-2 | +| `foundry-consumer-parity-v1.md`, `foundry-roadmap.md`, `MEDCARE_POLICY_GAP.md`, `td-super-domain-subcrates.md` | stale-but-unmarked (investigation §8-P3); banner candidates | +| `ogar-sink-in-and-consumer-bridge-removal-v1.md` | substantially DONE (the `ddb6c840` collapse) | +| `lance-graph-in-medcare-rs-v1.md`, `unified-bridge-consumer-migration-v1.md` | partially landed; remaining items fold into §6 here | +| `.grok/board/*` | unreconciled second lineage; §6.5 | + +**The central realignment:** OGAR and the bridge pattern are NOT the unfinished +part. The unfinished part is **cognition and composition**. Prove one real +medical thought through the already-live consumer path, then generalize the +exact seam that worked. From 1bb0a594f428fb348c37e90a92009317c0a4cbd7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 13:07:17 +0000 Subject: [PATCH 2/4] supervisor(cycle_driver)+docs: correct the pre-commit failure contract to deterministic regeneration; re-home the proof to the live consumer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 grain-of-salt realignment (operator-ruled), post-#879-merge. 1) Pre-commit failure contract. The authoritative rule is deterministic regeneration, not retained-batch retry: sealed Vn + unchanged Kanban task + deterministic computation = the same provisional intent on the next sweep Commit fails before Vn+1 exists -> publish nothing, mutate no owner, advance no watermark, discard provisional slots / held moves / planning results, rerun the unchanged task from Vn. - SealFailure{casts} reclassified: OPTIONAL retry cache / implementation convenience only — never the correctness mechanism, never a provisional-planning ledger; dropping it is always sound. - recover_fleet doc-pinned as COMMITTED-HISTORY recovery ONLY (Vn+1 exists, application/restart interrupted); explicitly separated from ordinary pre-commit write failure — no shared state. - HeldIntent doc-pinned as within-success scheduling convenience, discarded on a failed seal, regenerated by the next thought pass. - NEW authoritative falsifier pre_commit_failure_discards_everything_and_regenerates_from_vn: derive cycle C deterministically from Vn, inject commit failure, DROP the SealFailure cache, assert no version/phase/watermark change, rerun the unchanged task from Vn, assert the same SEMANTIC sparse cycle regenerates, allow commit, assert exactly one Vn+1 and one advance per represented owner. Object identity of the first heap batch deliberately not asserted. - The prior byte-identical-retry test demoted to an optional-cache probe (secondary, convenience path). 20 cycle_driver tests green; clippy + fmt clean. Latency figures in review prose are operator-provided measurements, not workspace-reproduced benchmarks; their values are not restated. 2) Proof re-homed (medcare-consumer-pull-thinking-proof-v1 section 4): primary home = MedCare-rs (the live composition root); lance-graph contributes only genuinely-missing GENERIC seams — no MedCare-shaped host adapter here (that would rebuild the dead lineage in miniature). Hard requirement: the proof must invoke the existing cognitive-shader-driver + MailboxSoA operational unit; shade_owner only as the driver's existing downstream gate, never a substitute. Trace-and-report obligation added; F1 strengthened to a discriminating driver outcome. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/LATEST_STATE.md | 6 + ...medcare-consumer-pull-thinking-proof-v1.md | 68 +++++-- .../src/cycle_driver.rs | 191 ++++++++++++++++-- 3 files changed, 231 insertions(+), 34 deletions(-) diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 6bf172263..1f6090bd2 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,9 @@ +## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` (restarted on main post-#879-merge) — pre-commit failure contract corrected to deterministic regeneration + proof re-homed to the live consumer + +- **#879 MERGED** (2026-08-02T12:25Z); branch restarted from `origin/main` per protocol, unmerged docs commit rebased on top. Two operator-ruled realignments applied (grain-of-salt round 2): +- **1) Pre-commit failure contract corrected** in `cycle_driver.rs` docs + tests. The authoritative rule: **sealed `Vn` + unchanged Kanban task + deterministic computation = the same provisional intent on the next sweep** — commit fails before `Vn+1` → publish nothing · mutate no owner · advance no watermark · discard provisional slots/held moves/planning results · rerun from `Vn`. `SealFailure{casts}` RECLASSIFIED from "the retry-safety mechanism" to **optional retry cache / implementation convenience** (never a provisional-planning ledger; dropping it is always sound). `recover_fleet` doc-pinned as **committed-history recovery ONLY** (`Vn+1` exists, application interrupted) — the two mechanisms share no state. `HeldIntent` doc-pinned as within-success scheduling convenience, discarded on failure. NEW authoritative falsifier `pre_commit_failure_discards_everything_and_regenerates_from_vn` (derives cycle C from Vn → injects failure → DROPS the SealFailure → asserts no version/phase/watermark change → reruns the unchanged deterministic task → asserts the SAME semantic sparse cycle regenerates → one `Vn+1`, owners advance once; object identity deliberately NOT asserted). The old byte-identical-retry test demoted to "optional-cache probe". **20 cycle_driver tests green**, clippy+fmt clean. Latency figures in review prose = operator-provided measurements, not workspace-reproduced. +- **2) Proof re-homed:** `medcare-consumer-pull-thinking-proof-v1.md` §4 now pins the proof's primary home to **MedCare-rs** (the live composition root) — lance-graph contributes only genuinely-missing GENERIC seams, never a MedCare-shaped host adapter (would reconstruct the dead lineage in miniature). Hard driver requirement added: the proof must invoke the existing **cognitive-shader-driver + MailboxSoA operational unit** — SoA-read qualia fed straight into `shade_owner` proves only the (already-proven) MUL gate; `shade_owner` participates only as the driver's existing downstream gate. Trace-and-report obligation added (every arrow file:symbol-named). F1 strengthened to require a discriminating DRIVER outcome, not gate-only. + ## 2026-08-02 — branch `claude/medcare-rs-continue-ufsazd` — MedCare two-lineage investigation CONSOLIDATED → one active plan (consumer-pull thinking proof) - Six-agent read-only investigation (code inventory · callcenter/policy · manifest→runtime chain · plans archaeology · git history · tests+build) + main-thread cross-repo verification (OGAR + MedCare-rs siblings) + `git fetch --unshallow` (5 grafted roots → 4162 commits). Full verdict in `EPIPHANIES.md` `E-TWO-MEDCARE-LINEAGES-THE-LIVE-ONE-PULLS-THE-DEAD-ONE-HOSTS-1`; classification + proof target in **`.claude/plans/medcare-consumer-pull-thinking-proof-v1.md`** (ACTIVE — the single active MedCare plan; INTEGRATION_PLANS prepended). diff --git a/.claude/plans/medcare-consumer-pull-thinking-proof-v1.md b/.claude/plans/medcare-consumer-pull-thinking-proof-v1.md index 6ecf070d4..3ac33d660 100644 --- a/.claude/plans/medcare-consumer-pull-thinking-proof-v1.md +++ b/.claude/plans/medcare-consumer-pull-thinking-proof-v1.md @@ -67,24 +67,44 @@ manifest either emits everything it declares or stops declaring it. ## 4. The proof target — first medical thought (the drill hole) +**Home: the proof lands primarily in MedCare-rs** — the live composition root +per §2. lance-graph contributes only **generic** seams genuinely missing (e.g. +the deferred `MailboxSoaView::qualia()` when sanctioned, driver entry surface) +— never a new "MedCare-rs-shaped" host adapter in this repo (that would +reconstruct the dead lineage in miniature). Public/private boundary: +MedCare-specific schema and clinical mappings stay in MedCare-rs; generic +SoA/cycle/driver primitives may land here. + ``` -MedCare-rs medical input (existing schema surface, private side) - → existing OGAR HealthcarePort bridge (classid resolution, namespace lock) - → existing MedCare-rs RBAC or a narrow proof policy (fail-closed) - → cognitive-shader-driver + REAL MailboxSoA (the honest gap) - → non-vacuous medical cognitive result (falsifier below) - → owner_adapter::emit_bootstrap_intent / BatchWriter (write-on-behalf cast) - → cycle_driver (PR #879): collect → seal (one WAL write) → apply sparse → next intent +MedCare-rs input (existing schema surface, private side) + → existing medcare-bridge → UnifiedBridge (classid, namespace lock) + → existing MedCare-rs policy/schema path (or narrow proof policy — fail-closed) + → cognitive-shader-driver + REAL MailboxSoA (the honest gap) + → non-vacuous medical cognitive result + → owner_adapter::emit_bootstrap_intent / BatchWriter (write-on-behalf cast) + → cycle_driver (#879, merged): collect → seal (one WAL write) → apply sparse → next intent ``` -A small direct adapter is acceptable. The known seam: `MailboxSoaView::qualia()` -is deferred; `run_cognitive_work_gated`'s caller-extractor bridges it today — -the proof should read gate inputs from the REAL SoA qualia column, closing the -"extractor-fed" honesty gap in the #879 ledger. +**The driver requirement (hard):** the proof must invoke the EXISTING +`cognitive-shader-driver` + `MailboxSoA` operational unit. Reading qualia from +the real SoA column is necessary but NOT sufficient — feeding extracted values +straight into `shade_owner` bypasses the driver and proves only the MUL gate +(already proven). `shade_owner` participates only as the existing downstream +gate where the driver actually reaches it, never as a substitute for the +driver. + +**Trace-and-report obligation:** the proof's deliverable includes the exact +active chain, symbol by symbol: +`MedCare input type → HealthcarePort resolution → SoA projection → +cognitive-shader-driver entry point → material SoA/result change → MUL gate +(if reached) → StrategyOutcome → owner_adapter cast → sparse sealed +transition`. Every arrow named with file:symbol; no arrow asserted without +evidence. **Falsifiers (per the P0 falsifiability rule):** - F1 — the cognitive result is **non-vacuous**: two different medical inputs - produce two different gate outcomes (discriminates; not a constant). + produce two different DRIVER outcomes (a material SoA/result change that + discriminates; not a constant, and not gate-only). - F2 — the classid on the thought's carrier is the OGAR canon (`0x0901`-family via `HealthcarePort::class_id`), not a local literal. - F3 — the intent round-trips: cast in cycle N is collected, sealed (exactly one @@ -119,12 +139,24 @@ no cathedral. Skip entirely if F1–F4 pass without it. - NO manifest/supervisor/actor revival for the proof. - NO new Healthcare subset codebook; NO local classid literals. - NO callcenter⊗ogar dependency edge as a proof prerequisite. -- NO provisional-intent recovery ledger in #879: pre-commit failure → publish - nothing, mutate nothing, retry the byte-identical frozen cycle (`SealFailure`); - post-commit crash → recover committed history (`recover_fleet` + watermarks). - Committed-history recovery and ordinary failed-write recomputation stay - SEPARATE mechanisms. (Latency figures floating in review prose for these two - paths are **claimed, unverified** — nothing here measured them; do not cite.) +- NO provisional-intent recovery ledger in #879 (merged). The authoritative + pre-commit rule (operator-ruled): **sealed `Vn` + unchanged Kanban task + + deterministic computation = the same provisional intent on the next sweep.** + Commit fails before `Vn+1` exists → publish nothing · mutate no owner · + advance no watermark · **discard** provisional slots, held moves and planning + results · rerun the unchanged task from `Vn`. Correctness never requires + retaining a byte-identical frozen cycle — `SealFailure{casts}` is classified + as an **optional retry cache / implementation convenience only** and must not + become a planning ledger (so documented in `cycle_driver.rs`, with the + deterministic-regeneration falsifier + `pre_commit_failure_discards_everything_and_regenerates_from_vn` that DROPS + the cache and proves the same semantic sparse cycle re-derives). + `recover_fleet` is **committed-history recovery ONLY** (`Vn+1` exists, + application/restart interrupted) — never conceptually grouped with ordinary + write failure; the two mechanisms share no state. (Latency figures attached + to these paths in review prose are **operator-provided measurements**, not + workspace-reproduced benchmarks — their exact values are not the + architectural foundation and are not restated here.) - NO clinical schema detail or sourcing/licensing reasoning in this public repo. ## 8. Status ledger for the older documents diff --git a/crates/lance-graph-supervisor/src/cycle_driver.rs b/crates/lance-graph-supervisor/src/cycle_driver.rs index c4de6c6a3..797d2bf04 100644 --- a/crates/lance-graph-supervisor/src/cycle_driver.rs +++ b/crates/lance-graph-supervisor/src/cycle_driver.rs @@ -23,6 +23,31 @@ //! globally-complete cycle physically sparse //! (`E-COMPLETE-CYCLE-IS-PHYSICALLY-SPARSE-NOT-A-FULL-REWRITE-1`). //! +//! ## Failure semantics (authoritative, operator-ruled) — TWO separate mechanisms +//! +//! **Pre-commit failure (no `Vn+1` exists): deterministic regeneration is the +//! correctness mechanism.** The authoritative rule: +//! +//! ```text +//! sealed Vn + unchanged Kanban task + deterministic computation +//! = the same provisional intent on the next sweep +//! ``` +//! +//! So when a commit fails before `Vn+1` exists: publish nothing · mutate no +//! owner · advance no watermark · **discard** provisional slots, held moves and +//! planning results · rerun the unchanged Kanban task from `Vn` — the same +//! semantic sparse cycle regenerates. Correctness NEVER requires retaining the +//! first heap batch: [`SealFailure::casts`] is an **optional retry cache** (an +//! implementation convenience that skips the re-stage pass), never the source +//! of truth and never a provisional-planning ledger. Discarding it is always +//! sound; the regeneration falsifier pins this. +//! +//! **Post-commit interruption (`Vn+1` EXISTS): committed-history recovery.** +//! [`recover_fleet`] replays already-SEALED landings whose application (or a +//! restart) was interrupted, idempotent over per-owner watermarks. It is for +//! committed history ONLY — never a substitute for, or conceptually grouped +//! with, ordinary pre-commit write failure. The two mechanisms share no state. +//! //! ## The ≤1-transition-per-owner rule is enforced BEFORE sealing //! //! [`collect_casts`] partitions: the **first** intent move per owner (in cast @@ -116,16 +141,25 @@ pub struct SealedCycle { pub next_position_base: u64, } -/// A seal that failed at the WAL — **retry-safe**: carries the complete frozen -/// input back to the caller, byte-identical, so a retry submits the SAME cycle. -/// No owner was mutated (apply never ran); the writer stays drained; the frozen -/// casts here are the single surviving copy of the cycle. +/// A seal that failed at the WAL. **No owner was mutated, no version published, +/// no watermark advanced** (apply never ran). +/// +/// **Classification (module docs § Failure semantics): the retained `casts` are +/// an OPTIONAL retry cache, NOT the correctness mechanism.** The authoritative +/// pre-commit rule is deterministic regeneration: discard everything and rerun +/// the unchanged Kanban task from the sealed `Vn` — the same semantic sparse +/// cycle regenerates (sealed `Vn` + unchanged task + deterministic computation +/// = the same provisional intent). Callers that keep a hot loop MAY resubmit +/// `casts` via [`seal_cycle`] to skip the re-stage pass; callers that simply +/// drop this value are equally correct. This must never grow into a +/// provisional-planning ledger. #[derive(Debug)] pub struct SealFailure { - /// The frame the failed seal was submitted under (retry with the same one, - /// or a refreshed `base_version` after a fence conflict). + /// The frame the failed seal was submitted under (regenerate with the same + /// one, or a refreshed `base_version` after a fence conflict). pub frame: CycleFrame, - /// The complete frozen cast set — resubmit via [`seal_cycle`]. + /// Optional retry cache (see type docs) — safe to discard; regeneration + /// from `Vn` is the prescribed path. pub casts: Vec, /// Why the WAL write failed. pub cause: PersistError, @@ -151,7 +185,14 @@ pub struct AppliedCycle { } /// An intent move held back by the pre-seal ≤1-per-owner partition — NOT sealed -/// this cycle, NOT lost: re-stage it into a future cycle via [`restage_held`]. +/// this cycle; re-stage it into a future cycle via [`restage_held`]. +/// +/// **Scheduling convenience within a SUCCESSFUL cycle, not a durable ledger:** +/// on a failed seal, held intents are discarded together with everything else +/// (the pre-commit rule is deterministic regeneration from `Vn`, module docs +/// § Failure semantics — [`run_cycle`]'s seal-error path drops them). Even on +/// success, a dropped held intent is regenerated by the owner's next thought +/// pass; retaining it merely saves that pass. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct HeldIntent { /// The owner whose extra move is held. @@ -277,10 +318,13 @@ pub fn restage_held(writer: &mut BatchWriter>, held: Vec) -> /// (cross-owner-move reject, cast/frame cycle-match). The version is published /// before any owner advances (P4b). /// -/// **Retry-safe:** on a WAL failure the complete frozen cast set is returned in -/// [`SealFailure`] — no owner was mutated, and a retry (`seal_cycle(sink, -/// failure.frame, failure.casts)`) submits the byte-identical cycle. The one -/// clone held until commit success is the price of that guarantee. +/// **On WAL failure:** nothing is published, no owner is mutated, no watermark +/// advances. The prescribed recovery is **deterministic regeneration from the +/// unchanged `Vn`** (module docs § Failure semantics); the [`SealFailure`] +/// carries the frozen casts only as an optional retry cache (dropping it and +/// re-staging is equally correct — the regeneration falsifier proves the same +/// semantic cycle re-derives). The one clone held until commit success is the +/// price of offering that cache. pub async fn seal_cycle( sink: &S, frame: CycleFrame, @@ -426,8 +470,11 @@ pub struct CycleOutcome { /// A [`run_cycle`] failure. #[derive(Debug)] pub enum CycleError { - /// The WAL commit failed — **no owner mutated**; the boxed [`SealFailure`] - /// carries the byte-identical frozen cycle for retry via [`seal_cycle`]. + /// The WAL commit failed — **nothing published, no owner mutated, no + /// watermark advanced**. Held intents from this pass were discarded. + /// Prescribed recovery: rerun the unchanged Kanban task from `Vn` + /// (deterministic regeneration, module docs § Failure semantics); the + /// boxed [`SealFailure`] is an optional retry cache only. Seal(Box), /// A guard tripped mid-apply — the applied prefix (with its watermarks /// already advanced) is preserved; re-drive the tail via [`recover_fleet`]. @@ -686,7 +733,14 @@ pub struct FleetRecovery { pub owners_recovered: usize, } -/// **P4e.** Fleet-level crash recovery: scan the sealed landings ONCE, partition +/// **P4e — COMMITTED-HISTORY recovery ONLY.** Valid solely when a commit +/// SUCCEEDED (`Vn+1` exists as a sealed landing) and its application — or a +/// restart — was interrupted. It is NEVER the path for an ordinary pre-commit +/// write failure: that path is deterministic regeneration from the unchanged +/// `Vn` (module docs § Failure semantics), and the two mechanisms share no +/// state. +/// +/// Scan the sealed landings ONCE, partition /// them per owner (one pass over history, not O(fleet × history)), and replay /// each owner's PENDING tail via `persist_sink::recover_and_apply`, idempotent /// with a **per-owner watermark**. Only unreplayed moves (above the owner's @@ -960,7 +1014,112 @@ mod tests { assert!(again.slots.is_empty(), "drain cleared the writer's staging"); } - // ── RETRY FALSIFIER: a failed seal preserves the byte-identical cycle ─────── + // ── AUTHORITATIVE PRE-COMMIT FALSIFIER: deterministic regeneration from Vn ── + // The operator-ruled contract: sealed Vn + unchanged Kanban task + + // deterministic computation = the same provisional intent on the next sweep. + // Correctness NEVER requires retaining the first heap batch — this test + // DROPS the SealFailure (the optional cache) and proves the same semantic + // sparse cycle re-derives from the unchanged Vn. Object identity of the + // first batch is deliberately NOT asserted. + #[tokio::test] + async fn pre_commit_failure_discards_everything_and_regenerates_from_vn() { + let sink = FakeWalSink::new(); + let mut fleet: HashMap = HashMap::from([ + (3, FakeOwner::at(3, KanbanColumn::Planning)), + (8, FakeOwner::at(8, KanbanColumn::Planning)), + ]); + let before = fleet.clone(); + let mut wm: HashMap> = HashMap::new(); + + // The deterministic Kanban task: a pure function of the (unchanged) + // fleet state — every owner still in Planning stages Planning→CognitiveWork. + let stage = |fleet: &HashMap| -> BatchWriter> { + let mut w: BatchWriter> = BatchWriter::new(); + let mut ids: Vec = fleet + .values() + .filter(|o| o.phase() == KanbanColumn::Planning) + .map(|o| o.mailbox_id()) + .collect(); + ids.sort_unstable(); + for id in ids { + w.cast( + id, + vec![mv(id, KanbanColumn::Planning, KanbanColumn::CognitiveWork)], + vec![id as u8], + ); + } + w + }; + + // Derive cycle C deterministically from Vn; inject a commit failure. + let mut w1 = stage(&fleet); + sink.fail_next_commit(); + let err = run_cycle( + &sink, + &mut fleet, + &mut w1, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + 0, + &mut wm, + u64::from, + ) + .await + .expect_err("injected commit failure"); + let CycleError::Seal(failure) = err else { + panic!("expected a Seal failure"); + }; + // Snapshot the failed cycle's SEMANTIC content, then DISCARD the batch — + // the optional cache is deliberately NOT used for recovery. + let semantic_c1: Vec<(MailboxId, KanbanColumn, KanbanColumn)> = failure + .casts + .iter() + .filter_map(|s| s.paired_move.map(|m| (s.owner, m.from, m.to))) + .collect(); + drop(failure); + + // Publish nothing · mutate no owner · advance no watermark. + assert_eq!(sink.wal_writes(), 0, "no Vn+1 was published"); + assert_eq!(fleet, before, "no owner mutated"); + assert!(wm.is_empty(), "no watermark advanced"); + + // Rerun the UNCHANGED task from the unchanged Vn (fresh writer, fresh + // staging — nothing retained from the failed attempt). + let mut w2 = stage(&fleet); + let out = run_cycle( + &sink, + &mut fleet, + &mut w2, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + 0, + &mut wm, + u64::from, + ) + .await + .expect("regenerated cycle commits"); + + // The SAME semantic sparse cycle regenerated (owner set + from→to). + let semantic_c2: Vec<(MailboxId, KanbanColumn, KanbanColumn)> = out + .sealed + .transitions + .iter() + .map(|t| (t.owner, t.mv.from, t.mv.to)) + .collect(); + assert_eq!( + semantic_c2, semantic_c1, + "deterministic regeneration: the same semantic sparse cycle" + ); + + // Exactly one Vn+1; exactly the represented owners advance once. + assert_eq!(sink.wal_writes(), 1, "exactly one successful WAL write"); + assert_eq!(out.sealed.version, DatasetVersion(1)); + assert_eq!(out.applied.applied.len(), 2); + assert_eq!(fleet[&3].phase(), KanbanColumn::CognitiveWork); + assert_eq!(fleet[&8].phase(), KanbanColumn::CognitiveWork); + } + + // ── Optional-cache probe: the SealFailure retry cache also works ──────────── + // Secondary to the regeneration falsifier above: a caller that DOES use the + // cache gets a byte-identical resubmit. Convenience path, not the contract. #[tokio::test] async fn failed_seal_preserves_the_frozen_cycle_for_byte_identical_retry() { let sink = FakeWalSink::new(); From 7d8d2248bf2fbf5a5662f7a4eaf4ea4b2692c7c3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 10:21:02 +0000 Subject: [PATCH 3/4] =?UTF-8?q?graph/cycle=5Fsink:=20concrete=20Lance-back?= =?UTF-8?q?ed=20WalSink=20=E2=80=94=20the=20storage-proven=20cognitive-cyc?= =?UTF-8?q?le=20sink?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the gap persist_sink deliberately left ('this module builds NO concrete Lance sink'): LanceCycleSink implements lance_graph_planner::persist_sink::WalSink over the official Lance 9 insert path (Dataset::write / Dataset::append — the same InsertBuilder transaction machinery every Lance writer uses). No bespoke ledger, no acknowledgement protocol, no parallel replay system: Lance's own manifest/version chain IS the WAL. The §I.6 invariant made physical: one detached cycle batch → ONE official Lance commit → exactly one real DatasetVersion base+1, with the epistemic fence in both halves — pre-commit head==base (a stale base is refused with nothing written; an empty store's head is DatasetVersion(0)) and post-commit published==base+1 (Lance auto-resolves append-append conflicts, so a foreign interleaved writer surfaces as a loud timeline anomaly instead of silently shifting the cycle↔version identity reads derive). The per-cycle frame row seals the cycle↔version mapping inside the same atomic commit, so the coarse timeline survives restart with zero sidecar state. Order is a write-side property: landings are stored in the already-deinterlaced freeze order and scanned back with scan_in_order(true) — never sorted on read. Domain-0x09 witness contract (module doc): the patient SoA is the ONLY place patient reasoning is written to Lance, so the store is witness-focused and maximally rich — payload carries the 512-byte canonical EpisodicWitness node whose edges point INTO the immutable domain-0x03 ontology address space; the cycle takes ontology immutability for granted for its representation window via base_version, and the sealed versioning is a reflection of the thinking. Gated on the default-on 'planner' feature (the trait lives in the optional planner dep). 6 tokio tests, every guarantee proven against a REOPENED dataset (fresh sink instance + fresh Dataset::open): seal survives restart; stale base fenced writing nothing; sequential cycles chain V1→V2→V3 with strictly-after filtering; a zero-landing cycle advances the timeline only; an empty store reads empty (DatasetNotFound is a state, not an error); move-nullability + payload byte-exact round-trip. Board hygiene (LATEST_STATE + PR_ARC_INVENTORY) in this same commit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/LATEST_STATE.md | 12 + .claude/board/PR_ARC_INVENTORY.md | 9 + Cargo.lock | 1 + crates/lance-graph/src/graph/cycle_sink.rs | 659 +++++++++++++++++++++ crates/lance-graph/src/graph/mod.rs | 5 + 5 files changed, 686 insertions(+) create mode 100644 crates/lance-graph/src/graph/cycle_sink.rs diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 04786891c..370e20157 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,15 @@ +## 2026-08-09 — branch `claude/medcare-rs-continue-ufsazd` — `lance_graph::graph::cycle_sink`: the CONCRETE cognitive-cycle Lance sink (the storage-proven `WalSink`) + +### Current Contract Inventory — new module (lance-graph core, `planner` feature) + +- `lance_graph::graph::cycle_sink` — **the gap `persist_sink` deliberately left is now closed**: a concrete implementation of `lance_graph_planner::persist_sink::WalSink` over the official Lance 9 insert path (`Dataset::write` / `Dataset::append` — the same `InsertBuilder` transaction machinery every Lance writer uses). No bespoke ledger, no acknowledgement protocol, no parallel replay system: Lance's own manifest/version chain IS the WAL. Gated on the default-on `planner` feature (the trait lives in the optional planner dep). + - `LanceCycleSink::new(path)` — holds only the dataset path; opens the dataset per operation, so restart-survival is exercised on EVERY call, not just in tests. + - `cycle_store_schema()` — one dataset, two row kinds: a per-cycle **frame row** (`kind=0`, the cycle ↔ version mapping sealed INSIDE the same atomic commit — zero sidecar state) and **landing rows** (`kind=1`: `stream_position`, `owner`, `row`, nullable `move_*` Rubicon-edge columns, `payload` witness-node bytes). + - **The §I.6 fence, both halves.** Pre-commit: the store's current version must equal the cycle's sealed predecessor `base` (empty store ⇒ head `DatasetVersion(0)`), else refused with NOTHING written. Post-commit: the published version must be exactly `base + 1` — Lance auto-resolves append-append conflicts, so a foreign interleaved writer surfaces as a LOUD timeline anomaly instead of silently shifting the cycle ↔ version identity reads derive (`sealed_version = base_version + 1` is an identity the commit path VERIFIES, never assumes). + - **Order is a write-side property**: landings are stored in the already-deinterlaced `DetachedCycleBatch` order and scanned back with `scan_in_order(true)`; the sink never sorts on read. + - **Domain-0x09 witness contract (module doc, operator-ruled):** the patient SoA at classid domain `0x09` is the ONLY place patient reasoning is written to Lance, so the store is witness-focused and maximally rich — `payload` carries the 512-byte canonical EpisodicWitness node (visited ontology addresses, executed crosswalk mappings, exact RO/ontology edge ids, supporting/contradicting/missing observations, NARS truth+confidence, differential branches), whose edges point INTO the immutable domain-0x03 ontology address space. The cycle takes ontology immutability for granted for its representation window (`base_version` names the sealed predecessor it read) and therefore never restates ontology content — the sealed versioning is a reflection of the thinking; downstream (the Gotham display, differential views) reads the sealed version, never a live recomputation. +- **Gates:** 6 tokio tests, every guarantee proven against a REOPENED dataset (fresh sink instance + fresh `Dataset::open`, never an in-memory echo): seal survives restart; stale base fenced with nothing written (store version chain, landings, timeline all untouched — checked, not assumed); sequential cycles chain V1→V2→V3 with strictly-after filtering; a zero-landing cycle advances the timeline only; an empty store reads empty (a state, not an error — `DatasetNotFound` distinguished from real I/O failure); move-nullability + 512-byte payload byte-exact round-trip. Real Lance version chain cross-checked against the returned `DatasetVersion` in-test. + ## 2026-08-06 — branch `claude/vocab-tenant-bake` — `lance_graph_contract::identity_quad`: four external identifier spaces joined at BAKE time into one 96-bit facet payload ### Current Contract Inventory — new module (lance-graph-contract) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 573128e76..44fe88bb8 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -33,6 +33,15 @@ > - **Docs** — knowledge files produced (immutable) > - **Confidence (YYYY-MM-DD):** — the ONLY mutable field +## 2026-08-09 — branch `claude/medcare-rs-continue-ufsazd` (PR pending) — the concrete cognitive-cycle Lance sink: `graph::cycle_sink::LanceCycleSink` + +- **Added.** `lance_graph::graph::cycle_sink` (~660 LOC incl. 6 reopened-dataset tokio tests) — the concrete `lance_graph_planner::persist_sink::WalSink` over the official Lance 9 insert path; `cycle_store_schema()` (frame row + landing rows, nullable Rubicon `move_*` columns, `payload` witness bytes); `LanceCycleSink`. Module gated on the default-on `planner` feature. +- **Locked.** **Lance's manifest/version chain IS the WAL** — no bespoke ledger / acknowledgement protocol / parallel replay system (operator-ruled). **The §I.6 fence has two halves**: pre-commit head==base (nothing written on refusal) AND post-commit published==base+1 (Lance auto-resolves append-append conflicts, so a foreign writer is a loud timeline anomaly, never a silent identity shift). **`sealed_version = base_version + 1` is a verified identity, not an assumption** — which is what lets reads derive the cycle↔version mapping from the atomically-co-committed frame row with zero sidecar state. **Order is a write-side property** (stored deinterlaced order, `scan_in_order(true)`, no read-time sort). **Domain-0x09 witness contract**: the patient SoA is the ONLY patient-reasoning Lance write target; payload = maximally-rich EpisodicWitness node built ON TOP of the immutable domain-0x03 ontology addresses (immutability taken for granted for the representation window via `base_version`). +- **Deferred.** The MedCare consumer arc (production `drive_cohort_thoughts` caller, witness-seal, views reading the sealed version) — next PR, in MedCare-rs. `recover_and_apply` wiring against this sink in a production driver. Object-store (s3/az/gs) smoke — the path plumbing accepts URIs but only local was exercised. +- **Docs.** Module-level witness/§I.6 contract in `cycle_sink.rs`; this entry + LATEST_STATE inventory (same commit). + +**Confidence (2026-08-09):** tests green against reopened local datasets; not yet merged. + ## 2026-08-05 — the lance 9 / DataFusion 54 / Rust 1.97.1 cross-repo bump (9 repos; lance-graph PR pending, siblings MERGED) Siblings merged same day: OGAR #244/#245, ruff #93, stockfish-rs #14, woa-rs #179, a2ui-rs #19, MedCare-rs #351. lance-graph's own arc rides the branch below. diff --git a/Cargo.lock b/Cargo.lock index 37f8ac75b..88c0a095a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6638,6 +6638,7 @@ dependencies = [ name = "ndarray" version = "0.17.2" dependencies = [ + "blake3", "fractal", "matrixmultiply", "num-complex", diff --git a/crates/lance-graph/src/graph/cycle_sink.rs b/crates/lance-graph/src/graph/cycle_sink.rs new file mode 100644 index 000000000..872b05633 --- /dev/null +++ b/crates/lance-graph/src/graph/cycle_sink.rs @@ -0,0 +1,659 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! The CONCRETE cognitive-cycle Lance sink — the storage-proven implementation of +//! `lance_graph_planner::persist_sink::WalSink` over the official Lance +//! transaction/write path (`lance = 9.0.0`). +//! +//! `persist_sink` (the planner seam) deliberately builds NO concrete sink: its +//! `FakeWalSink` proves the algebra (fence, ordering, coalescing, recovery) in +//! process memory only — "compile+test green ≠ storage proven" (the Ladybug +//! lesson). This module closes that gap: every trait operation here is true +//! against a REOPENED dataset on real storage. +//! +//! # The §I.6 invariant (the ruled contract this sink makes physical) +//! +//! ```text +//! 64k thoughts read sealed Vn → write-side temporal deinterlace → +//! one detached cycle batch → ONE official Lance commit → +//! exactly one real DatasetVersion Vn+1 → no open or partial visibility +//! ``` +//! +//! - **One durable append per cycle.** `commit_cycle` performs a single +//! `Dataset::write` / `Dataset::append` (the official Lance insert path — +//! `InsertBuilder` under the hood, the same transaction machinery every Lance +//! writer uses). No bespoke ledger, no acknowledgement protocol, no parallel +//! replay system: Lance's own manifest/version chain IS the WAL. +//! - **The epistemic fence, both halves.** Pre-commit: the dataset's current +//! version must equal the cycle's sealed predecessor `base` (`Vn`), else the +//! commit is refused with nothing written. Post-commit: the published version +//! must be exactly `base + 1`. Lance's optimistic concurrency auto-resolves +//! append-append conflicts (a foreign interleaved writer would yield +//! `base + 2`), so under the one-writer-per-mailbox doctrine a post-check +//! mismatch is a LOUD timeline anomaly, never silently accepted. +//! - **Order is a write-side property.** Landings arrive already deinterlaced +//! (`DetachedCycleBatch::freeze` ran the loom); they are stored in that +//! canonical order and scanned back with Lance's in-order scan. This sink +//! never sorts on read. +//! - **All-or-nothing visibility.** An unsealed / fenced / failed cycle leaves +//! no rows and no version: after restart + reopen it is simply absent. +//! Recovery is a read of the sealed store (`scan_sealed` + the caller's +//! watermark in `recover_and_apply`) — idempotent without any sidecar state. +//! +//! # Domain 0x09 — the patient SoA witness store (why the schema is rich) +//! +//! The patient SoA at classid domain `0x09` is the ONLY place patient reasoning +//! is ever written to Lance. Everything else the reasoner touches — the +//! interlocked ontologies at domain `0x03`, crosswalks, RO edges — is IMMUTABLE +//! for the duration of a representation window: a cycle takes that immutability +//! for granted (its `base_version` names the sealed ontology-bearing predecessor +//! it read), and therefore never needs to restate ontology content. What it MUST +//! state — maximally richly — is the WITNESSING: +//! +//! - **`payload`** carries the canonical witness node bytes (the 512-byte +//! `key(16) | edges(16) | value(480)` node ABI): the EpisodicWitness row — +//! visited ontology addresses, executed crosswalk mappings, the exact RO / +//! ontology edge identifiers walked, supporting / contradicting / missing +//! observations, NARS truth + confidence, differential branches. Domain-0x09 +//! keys, edges pointing INTO the immutable 0x03 address space. +//! - **The landing columns** carry the dynamic-reasoning-update record: which +//! mailbox reasoned (`owner`), where in the canonical thought stream +//! (`stream_position`), which SoA row the update lands on (`row`), and the +//! Rubicon lifecycle step the thought cast (`move_*` — the sealed reflection +//! of the thinking, applied post-SEAL only). +//! - **The frame row** (one per cycle, `kind = 0`) seals the cycle ↔ version +//! mapping INSIDE the same atomic commit, so the coarse timeline +//! (`versions()`) survives restart with zero sidecar files — the sealed +//! versioning is literally a reflection of the thinking that produced it. +//! +//! Downstream (the Gotham display, differential views, any consumer) reads the +//! sealed version — never a live recomputation: the witness is examined in +//! place, at the version its cycle published. + +use std::sync::Arc; + +use arrow_array::{ + builder::{BinaryBuilder, UInt32Builder, UInt8Builder}, + Array, BinaryArray, RecordBatch, RecordBatchIterator, UInt32Array, UInt64Array, UInt8Array, +}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use futures::TryStreamExt; +use lance::dataset::{Dataset, WriteMode, WriteParams}; + +use lance_graph_contract::collapse_gate::MailboxId; +use lance_graph_contract::kanban::{ExecTarget, KanbanColumn, KanbanMove}; +use lance_graph_contract::scheduler::DatasetVersion; +use lance_graph_planner::persist_sink::{ + CycleId, DetachedCycleBatch, LandedSlot, SweepSlot, WalSink, WriteFailed, +}; + +/// Row kind discriminant: the per-cycle frame row (cycle ↔ version mapping, +/// sealed inside the same atomic commit as its landings). +const KIND_FRAME: u8 = 0; +/// Row kind discriminant: one landing (a thought's persistence record). +const KIND_LANDING: u8 = 1; + +/// The Arrow schema of the cycle store — one dataset, two row kinds. +/// +/// | column | type | frame row | landing row | +/// |-------------------------------|-----------------|-----------|-------------| +/// | `kind` | `UInt8` | 0 | 1 | +/// | `cycle` | `UInt64` | cycle id | cycle id | +/// | `base_version` | `UInt64` | `Vn` | `Vn` | +/// | `stream_position` | `UInt64` | 0 | canonical order key | +/// | `owner` | `UInt32` | 0 | mailbox | +/// | `row` | `UInt64` | 0 | SoA row | +/// | `move_mailbox` | `UInt32?` | null | paired move (or null) | +/// | `move_from` / `move_to` | `UInt8?` | null | Rubicon edge | +/// | `move_witness_chain_position` | `UInt32?` | null | witness pointer (R4) | +/// | `move_exec` | `UInt8?` | null | exec target | +/// | `payload` | `Binary` | empty | witness node bytes | +/// +/// The sealed version of every row's cycle is `base_version + 1` — an identity +/// the commit path VERIFIES against the real published Lance version (it is +/// never assumed), so reads may derive it without a sidecar mapping. +pub fn cycle_store_schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("kind", DataType::UInt8, false), + Field::new("cycle", DataType::UInt64, false), + Field::new("base_version", DataType::UInt64, false), + Field::new("stream_position", DataType::UInt64, false), + Field::new("owner", DataType::UInt32, false), + Field::new("row", DataType::UInt64, false), + Field::new("move_mailbox", DataType::UInt32, true), + Field::new("move_from", DataType::UInt8, true), + Field::new("move_to", DataType::UInt8, true), + Field::new("move_witness_chain_position", DataType::UInt32, true), + Field::new("move_exec", DataType::UInt8, true), + Field::new("payload", DataType::Binary, false), + ])) +} + +/// The concrete Lance-backed cycle sink. +/// +/// Cheap to clone / recreate: it holds only the dataset path and opens the +/// dataset per operation (the restart-survival guarantee is thereby exercised on +/// EVERY call, not just in tests). Point it at the domain-0x09 patient witness +/// store (e.g. `/witness_cycles.lance`) — one sink instance per store. +#[derive(Debug, Clone)] +pub struct LanceCycleSink { + dataset_path: String, +} + +impl LanceCycleSink { + /// A sink over the Lance dataset at `path` (local path or object-store URI — + /// anything `Dataset::open` accepts). The dataset is created on the first + /// committed cycle; a missing dataset is simply "nothing sealed yet". + #[must_use] + pub fn new(path: impl Into) -> Self { + Self { + dataset_path: path.into(), + } + } + + /// The dataset path this sink commits to. + #[must_use] + pub fn dataset_path(&self) -> &str { + &self.dataset_path + } + + /// Open the store if it exists; `None` = nothing sealed yet (a state, not an + /// error — distinguishing it from a real I/O failure is the caller-visible + /// difference between an empty timeline and a broken one). + async fn open_if_exists(&self) -> Result, WriteFailed> { + match Dataset::open(&self.dataset_path).await { + Ok(ds) => Ok(Some(ds)), + Err(lance::Error::DatasetNotFound { .. }) => Ok(None), + Err(e) => Err(WriteFailed(format!("open {}: {e}", self.dataset_path))), + } + } + + /// Build the single atomic RecordBatch for a cycle: the frame row first, + /// then the landings in their ALREADY-canonical order (the loom ran in + /// `DetachedCycleBatch::freeze`; storage order = stream order by contract). + fn build_batch(batch: &DetachedCycleBatch) -> Result { + let n = batch.landings.len() + 1; + let mut kind = Vec::with_capacity(n); + let mut cycle = Vec::with_capacity(n); + let mut base_version = Vec::with_capacity(n); + let mut stream_position = Vec::with_capacity(n); + let mut owner = Vec::with_capacity(n); + let mut row = Vec::with_capacity(n); + let mut move_mailbox = UInt32Builder::with_capacity(n); + let mut move_from = UInt8Builder::with_capacity(n); + let mut move_to = UInt8Builder::with_capacity(n); + let mut move_wcp = UInt32Builder::with_capacity(n); + let mut move_exec = UInt8Builder::with_capacity(n); + let mut payload = BinaryBuilder::new(); + + // Frame row — the cycle ↔ version mapping, sealed atomically with its + // landings (a zero-landing cycle still advances the timeline). + kind.push(KIND_FRAME); + cycle.push(batch.frame.cycle.0); + base_version.push(batch.frame.base_version.0); + stream_position.push(0); + owner.push(0); + row.push(0); + move_mailbox.append_null(); + move_from.append_null(); + move_to.append_null(); + move_wcp.append_null(); + move_exec.append_null(); + payload.append_value([]); + + for s in &batch.landings { + kind.push(KIND_LANDING); + cycle.push(s.cycle.0); + base_version.push(batch.frame.base_version.0); + stream_position.push(s.stream_position); + owner.push(s.owner); + row.push(s.row); + match &s.paired_move { + Some(m) => { + move_mailbox.append_value(m.mailbox); + move_from.append_value(m.from as u8); + move_to.append_value(m.to as u8); + move_wcp.append_value(m.witness_chain_position); + move_exec.append_value(m.exec as u8); + } + None => { + move_mailbox.append_null(); + move_from.append_null(); + move_to.append_null(); + move_wcp.append_null(); + move_exec.append_null(); + } + } + payload.append_value(&s.payload); + } + + RecordBatch::try_new( + cycle_store_schema(), + vec![ + Arc::new(UInt8Array::from(kind)), + Arc::new(UInt64Array::from(cycle)), + Arc::new(UInt64Array::from(base_version)), + Arc::new(UInt64Array::from(stream_position)), + Arc::new(UInt32Array::from(owner)), + Arc::new(UInt64Array::from(row)), + Arc::new(move_mailbox.finish()), + Arc::new(move_from.finish()), + Arc::new(move_to.finish()), + Arc::new(move_wcp.finish()), + Arc::new(move_exec.finish()), + Arc::new(payload.finish()), + ], + ) + .map_err(|e| WriteFailed(format!("build cycle batch: {e}"))) + } + + /// Read the whole store at its LATEST version, in stored (insertion) order — + /// Lance's in-order scan; this sink never sorts on read. + async fn read_all_rows(&self, ds: &Dataset) -> Result, WriteFailed> { + let mut scan = ds.scan(); + scan.scan_in_order(true); + let batches: Vec = scan + .try_into_stream() + .await + .map_err(|e| WriteFailed(format!("scan {}: {e}", self.dataset_path)))? + .try_collect() + .await + .map_err(|e| WriteFailed(format!("collect {}: {e}", self.dataset_path)))?; + + let mut rows = Vec::new(); + for b in &batches { + let col_u8 = |name: &str| -> Result<&UInt8Array, WriteFailed> { + b.column_by_name(name) + .and_then(|c| c.as_any().downcast_ref()) + .ok_or_else(|| WriteFailed(format!("missing column {name}"))) + }; + let col_u32 = |name: &str| -> Result<&UInt32Array, WriteFailed> { + b.column_by_name(name) + .and_then(|c| c.as_any().downcast_ref()) + .ok_or_else(|| WriteFailed(format!("missing column {name}"))) + }; + let col_u64 = |name: &str| -> Result<&UInt64Array, WriteFailed> { + b.column_by_name(name) + .and_then(|c| c.as_any().downcast_ref()) + .ok_or_else(|| WriteFailed(format!("missing column {name}"))) + }; + let kind = col_u8("kind")?; + let cycle = col_u64("cycle")?; + let base_version = col_u64("base_version")?; + let stream_position = col_u64("stream_position")?; + let owner = col_u32("owner")?; + let row = col_u64("row")?; + let move_mailbox = col_u32("move_mailbox")?; + let move_from = col_u8("move_from")?; + let move_to = col_u8("move_to")?; + let move_wcp = col_u32("move_witness_chain_position")?; + let move_exec = col_u8("move_exec")?; + let payload: &BinaryArray = b + .column_by_name("payload") + .and_then(|c| c.as_any().downcast_ref()) + .ok_or_else(|| WriteFailed("missing column payload".into()))?; + + for i in 0..b.num_rows() { + let paired_move = if move_mailbox.is_valid(i) { + Some(KanbanMove { + mailbox: move_mailbox.value(i) as MailboxId, + from: KanbanColumn::from_u8(move_from.value(i)), + to: KanbanColumn::from_u8(move_to.value(i)), + witness_chain_position: move_wcp.value(i), + exec: ExecTarget::from_u8(move_exec.value(i)), + }) + } else { + None + }; + rows.push(StoredRow { + kind: kind.value(i), + cycle: CycleId(cycle.value(i)), + base_version: DatasetVersion(base_version.value(i)), + slot: SweepSlot { + cycle: CycleId(cycle.value(i)), + stream_position: stream_position.value(i), + owner: owner.value(i), + row: row.value(i), + paired_move, + payload: payload.value(i).to_vec(), + }, + }); + } + } + Ok(rows) + } +} + +/// One decoded store row (frame or landing) — internal read shape. +struct StoredRow { + kind: u8, + cycle: CycleId, + base_version: DatasetVersion, + slot: SweepSlot, +} + +impl StoredRow { + /// The version this row's cycle sealed into — `base + 1`, the identity the + /// commit path verified against the real published Lance version. + fn sealed_version(&self) -> DatasetVersion { + DatasetVersion(self.base_version.0 + 1) + } +} + +impl WalSink for LanceCycleSink { + /// THE single amortized durable append for a whole cycle, over the official + /// Lance insert path — one commit, one new `DatasetVersion`, all-or-nothing. + /// + /// The epistemic fence, both halves: + /// 1. **Pre-commit:** the store's current version must equal `base` (`Vn`). + /// An empty store has head `DatasetVersion(0)`, so the first cycle must + /// declare base 0 (it read no sealed predecessor). A stale base is + /// refused with NOTHING written. + /// 2. **Post-commit:** the published version must be exactly `base + 1`. + /// Lance auto-resolves append-append conflicts, so a foreign interleaved + /// writer surfaces here as a loud timeline anomaly instead of silently + /// shifting the cycle ↔ version identity that reads derive. + async fn commit_cycle( + &self, + base: DatasetVersion, + batch: DetachedCycleBatch, + ) -> Result { + if batch.frame.base_version != base { + return Err(WriteFailed(format!( + "frame base {:?} != commit base {base:?}", + batch.frame.base_version + ))); + } + let record_batch = Self::build_batch(&batch)?; + let schema = cycle_store_schema(); + let published = match self.open_if_exists().await? { + None => { + // Empty store: sealed head is DatasetVersion(0) by convention. + if base.0 != 0 { + return Err(WriteFailed(format!( + "stale base {base:?}: sealed head is DatasetVersion(0) (empty store)" + ))); + } + let reader = RecordBatchIterator::new(vec![Ok(record_batch)], schema); + let params = WriteParams { + mode: WriteMode::Create, + ..Default::default() + }; + let ds = Dataset::write(reader, &self.dataset_path, Some(params)) + .await + .map_err(|e| WriteFailed(format!("create commit: {e}")))?; + ds.version().version + } + Some(mut ds) => { + let head = ds.version().version; + if head != base.0 { + return Err(WriteFailed(format!( + "stale base {base:?}: sealed head is DatasetVersion({head})" + ))); + } + let reader = RecordBatchIterator::new(vec![Ok(record_batch)], schema); + ds.append(reader, None) + .await + .map_err(|e| WriteFailed(format!("append commit: {e}")))?; + ds.version().version + } + }; + if published != base.0 + 1 { + // The data IS committed at `published` — but the one-writer §I.6 + // timeline is broken (a foreign writer interleaved). Surface loudly; + // never let a shifted identity pass as a sealed cycle. + return Err(WriteFailed(format!( + "timeline anomaly: committed at DatasetVersion({published}), expected {}", + base.0 + 1 + ))); + } + Ok(DatasetVersion(published)) + } + + /// Committed landings only, in the STORED canonical order, from the + /// REOPENED dataset — never an in-memory echo. `from_version` filters to + /// cycles sealed strictly after it. + async fn scan_sealed( + &self, + from_version: Option, + ) -> Result, WriteFailed> { + let Some(ds) = self.open_if_exists().await? else { + return Ok(Vec::new()); + }; + let rows = self.read_all_rows(&ds).await?; + Ok(rows + .into_iter() + .filter(|r| r.kind == KIND_LANDING) + .filter(|r| from_version.is_none_or(|f| r.sealed_version() > f)) + .map(|r| LandedSlot { + version: r.sealed_version(), + slot: r.slot, + }) + .collect()) + } + + /// The cheap coarse timeline — the per-cycle frame rows, each sealed in the + /// same atomic commit as its landings, read back from the reopened store. + async fn versions(&self) -> Result, WriteFailed> { + let Some(ds) = self.open_if_exists().await? else { + return Ok(Vec::new()); + }; + let rows = self.read_all_rows(&ds).await?; + Ok(rows + .into_iter() + .filter(|r| r.kind == KIND_FRAME) + .map(|r| (r.cycle, r.sealed_version())) + .collect()) + } +} + +// --------------------------------------------------------------------------- +// Tests — every guarantee proven against a REOPENED dataset (fresh sink +// instance, fresh `Dataset::open`), never an in-memory echo. +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use lance_graph_planner::persist_sink::{persist_cycle, CycleFrame}; + + fn mv(owner: MailboxId) -> KanbanMove { + KanbanMove { + mailbox: owner, + from: KanbanColumn::Planning, + to: KanbanColumn::CognitiveWork, + witness_chain_position: 7, + exec: ExecTarget::Elixir, + } + } + + fn slot(cycle: u64, stream_position: u64, owner: MailboxId, row: u64) -> SweepSlot { + SweepSlot { + cycle: CycleId(cycle), + stream_position, + owner, + row, + paired_move: Some(mv(owner)), + payload: vec![stream_position as u8; 4], + } + } + + /// One cycle → ONE official Lance commit → exactly one real DatasetVersion + /// `base + 1`; a fresh sink over the same path (restart) reads the sealed + /// landings and the cycle ↔ version mapping back from storage. + #[tokio::test] + async fn seal_survives_restart_and_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("witness_cycles.lance"); + let sink = LanceCycleSink::new(path.to_str().unwrap()); + + let frame = CycleFrame::new(CycleId(1), DatasetVersion(0)); + let casts = vec![slot(1, 20, 5, 100), slot(1, 10, 5, 101)]; + let v = persist_cycle(&sink, frame, casts).await.unwrap(); + assert_eq!(v, DatasetVersion(1)); + + // The REAL Lance version chain agrees — not a private counter. + let ds = Dataset::open(path.to_str().unwrap()).await.unwrap(); + assert_eq!(ds.version().version, 1); + + // Restart: a brand-new sink instance, nothing shared but the path. + let reopened = LanceCycleSink::new(path.to_str().unwrap()); + let sealed = reopened.scan_sealed(None).await.unwrap(); + assert_eq!(sealed.len(), 2); + // Stored canonical order (deinterlaced at freeze: 10 before 20) — the + // scan preserves it, it does not repair it. + assert_eq!(sealed[0].slot.stream_position, 10); + assert_eq!(sealed[1].slot.stream_position, 20); + assert_eq!(sealed[0].version, DatasetVersion(1)); + assert_eq!(sealed[0].slot.paired_move, Some(mv(5))); + assert_eq!(sealed[0].slot.payload, vec![10u8; 4]); + + let versions = reopened.versions().await.unwrap(); + assert_eq!(versions, vec![(CycleId(1), DatasetVersion(1))]); + } + + /// A stale `base` is fenced with NOTHING written: the store's version chain, + /// landings, and timeline are untouched — proven on reopen. + #[tokio::test] + async fn stale_base_is_fenced_and_writes_nothing() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("witness_cycles.lance"); + let sink = LanceCycleSink::new(path.to_str().unwrap()); + + // Empty store: a caller claiming a sealed predecessor V3 is refused. + let err = sink + .commit_cycle( + DatasetVersion(3), + DetachedCycleBatch::freeze(CycleFrame::new(CycleId(1), DatasetVersion(3)), vec![]), + ) + .await + .unwrap_err(); + assert!(err.0.contains("stale base"), "{err}"); + assert!(Dataset::open(path.to_str().unwrap()).await.is_err()); + + // Seal cycle 1 at base 0 → V1; then a sibling still reading base 0 is + // fenced, and the store is byte-for-byte the sealed head it was. + persist_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![slot(1, 1, 2, 40)], + ) + .await + .unwrap(); + let err = sink + .commit_cycle( + DatasetVersion(0), + DetachedCycleBatch::freeze( + CycleFrame::new(CycleId(2), DatasetVersion(0)), + vec![slot(2, 2, 2, 41)], + ), + ) + .await + .unwrap_err(); + assert!(err.0.contains("stale base"), "{err}"); + + let reopened = LanceCycleSink::new(path.to_str().unwrap()); + let ds = Dataset::open(path.to_str().unwrap()).await.unwrap(); + assert_eq!(ds.version().version, 1, "fenced commit must not publish"); + assert_eq!(reopened.scan_sealed(None).await.unwrap().len(), 1); + assert_eq!(reopened.versions().await.unwrap().len(), 1); + } + + /// Sequential cycles chain the sealed horizon: V1 → V2 → V3; `scan_sealed` + /// filters strictly-after; `versions` is the full coarse timeline. + #[tokio::test] + async fn sequential_cycles_chain_and_filter() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("witness_cycles.lance"); + let sink = LanceCycleSink::new(path.to_str().unwrap()); + + for (cycle, base) in [(1u64, 0u64), (2, 1), (3, 2)] { + let v = persist_cycle( + &sink, + CycleFrame::new(CycleId(cycle), DatasetVersion(base)), + vec![slot(cycle, cycle * 10, 9, cycle)], + ) + .await + .unwrap(); + assert_eq!(v, DatasetVersion(base + 1)); + } + + let reopened = LanceCycleSink::new(path.to_str().unwrap()); + assert_eq!( + reopened.versions().await.unwrap(), + vec![ + (CycleId(1), DatasetVersion(1)), + (CycleId(2), DatasetVersion(2)), + (CycleId(3), DatasetVersion(3)), + ] + ); + // Strictly after V1: cycles 2 and 3 only. + let after_v1 = reopened.scan_sealed(Some(DatasetVersion(1))).await.unwrap(); + assert_eq!(after_v1.len(), 2); + assert!(after_v1.iter().all(|l| l.version > DatasetVersion(1))); + } + + /// A zero-landing cycle still advances the sealed timeline (its frame row + /// commits atomically) while contributing nothing to `scan_sealed`. + #[tokio::test] + async fn empty_cycle_advances_timeline_only() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("witness_cycles.lance"); + let sink = LanceCycleSink::new(path.to_str().unwrap()); + + let v = persist_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![], + ) + .await + .unwrap(); + assert_eq!(v, DatasetVersion(1)); + + let reopened = LanceCycleSink::new(path.to_str().unwrap()); + assert!(reopened.scan_sealed(None).await.unwrap().is_empty()); + assert_eq!( + reopened.versions().await.unwrap(), + vec![(CycleId(1), DatasetVersion(1))] + ); + } + + /// An empty store is a state, not an error: nothing sealed, empty timeline. + #[tokio::test] + async fn empty_store_reads_empty() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("never_created.lance"); + let sink = LanceCycleSink::new(path.to_str().unwrap()); + assert!(sink.scan_sealed(None).await.unwrap().is_empty()); + assert!(sink.versions().await.unwrap().is_empty()); + } + + /// A no-move landing round-trips as `None` (nullable move columns), and a + /// large-ish payload survives byte-exact. + #[tokio::test] + async fn move_nullability_and_payload_roundtrip() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("witness_cycles.lance"); + let sink = LanceCycleSink::new(path.to_str().unwrap()); + + let witness_node = (0..=255u8).cycle().take(512).collect::>(); + let mut s = slot(1, 3, 11, 900); + s.paired_move = None; + s.payload = witness_node.clone(); + persist_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![s], + ) + .await + .unwrap(); + + let reopened = LanceCycleSink::new(path.to_str().unwrap()); + let sealed = reopened.scan_sealed(None).await.unwrap(); + assert_eq!(sealed.len(), 1); + assert_eq!(sealed[0].slot.paired_move, None); + assert_eq!(sealed[0].slot.payload, witness_node); + assert_eq!(sealed[0].slot.owner, 11); + assert_eq!(sealed[0].slot.row, 900); + } +} diff --git a/crates/lance-graph/src/graph/mod.rs b/crates/lance-graph/src/graph/mod.rs index eded4abb0..5fad3a037 100644 --- a/crates/lance-graph/src/graph/mod.rs +++ b/crates/lance-graph/src/graph/mod.rs @@ -10,6 +10,11 @@ pub mod arigraph; pub mod audio; pub mod blasgraph; +/// The concrete Lance-backed `WalSink` (cognitive-cycle sink). Gated on the +/// `planner` feature because the trait it implements lives in +/// `lance-graph-planner` (an optional dep outside that feature). +#[cfg(feature = "planner")] +pub mod cycle_sink; pub mod fingerprint; pub mod graph_router; pub mod hydrate; From ba4f06cd5c3dee0da5a5e28dcf1f8e8abc292bb8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 10:35:40 +0000 Subject: [PATCH 4/4] =?UTF-8?q?cycle=5Fsink:=20review=20round=20=E2=80=94?= =?UTF-8?q?=20retroactive=20fence=20compensation,=20durable=20coalesced=20?= =?UTF-8?q?image,=20512-byte=20witness=20gate,=20projected=20reads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the #911 review findings (Codex P1x2/P2, CodeRabbit): - Fence made effective even post-publication: Lance has no expected- version conditional append (the rebase runs even on a single-attempt Append commit; strict mode is Overwrite-only — measured in lance-9.0.0/src/io/commit.rs), so a foreign interleaved writer can land the batch at base+2. Detecting that now triggers a compensating official Dataset::delete scoped to exactly this cycle's rows (the cycle id is an unsealed identity, so the predicate is exact) BEFORE the retryable WriteFailed returns — 'write failed' is true at the visible head, the driver's regenerate-from-Vn contract stays sound, and no rows survive under a shifted sealed_version identity. The one manual-reconciliation corner (the compensating delete itself fails) names the orphaned version explicitly. - The coalesced image is now DURABLE: kind=2 rows (row -> final payload after the per-row fold) persist in the same atomic commit as the landings, and LanceCycleSink::scan_image(cycle) reads a sealed cycle's coherent end-state (projected row+payload under the kind+cycle predicate) while scan_sealed keeps the per-cast history. - Witness payloads are gated to exactly EPISODIC_WITNESS_BYTES = 512 (the canonical key(16)|edges(16)|value(480) node stride) before anything durable happens; malformed rows refuse the whole cycle. - versions() projects only cycle+base_version under kind=0 and scan_sealed pushes kind=1 into the scan, so the coarse-timeline lookup never materializes a witness payload. - Board entries updated to the corrected contract; MD040 text fences in the medcare plan doc; INTEGRATION_PLANS 2026-08-02 entry moved ahead of its same-date sibling (EPIPHANIES placement already coherent). 8 reopened-dataset tests green (2 new: malformed-payload refusal with nothing written; durable per-cycle image with later-stream-position- wins and intact cast history). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KCGhDYoQBXs3poaR7sFuqp --- .claude/board/INTEGRATION_PLANS.md | 48 +-- .claude/board/LATEST_STATE.md | 3 +- .claude/board/PR_ARC_INVENTORY.md | 2 +- ...medcare-consumer-pull-thinking-proof-v1.md | 6 +- crates/lance-graph/src/graph/cycle_sink.rs | 303 ++++++++++++++++-- 5 files changed, 303 insertions(+), 59 deletions(-) diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index 4ef195277..f163e3068 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -105,6 +105,30 @@ answers — ownership cost, layout cost, WAL amortisation knee (descriptive, never PASS/KILL), and what genuine parallel thought execution adds before the deterministic seal. +## 2026-08-02 — medcare-consumer-pull-thinking-proof v1 — ACTIVE (the single active MedCare plan after the two-lineage investigation) — main thread + +**Plan:** `.claude/plans/medcare-consumer-pull-thinking-proof-v1.md` +Consolidation of the 2026-08-02 six-agent read-only MedCare/Healthcare +investigation (HEAD `71d1db1`, git unshallowed to 4162 commits). Verdict: +**two lineages, not one incomplete runtime.** LIVE = consumer-pull (MedCare-rs +`medcare-bridge` → vendored `lance-graph-ogar` → `UnifiedBridge` +→ OGAR canon `0x0901–0x090C`; bridge migration COMPLETED at `ddb6c840` +2026-06-21, `MedcareBridge` deprecated alias only). DEAD = host-side +manifest/supervisor/actor scaffold (frozen since birth 2026-05-13; manifest +compile-time-parsed but runtime-orphaned; `MedcareConsumerActor` never +constructed; supervisor hard-codes `StubConsumerActor`; Dispatch rejected +pre-child; `MedCareActor`/`MedCareMessage` exist in NEITHER repo; +`medcare_policy` symbol nonexistent). Two same-named, never-composed +`UnifiedBridge` types (ogar resolver — no authorize; callcenter authorize/audit +— no ogar dep) classified as LATER hardening, not a proof prerequisite. +**Active proof target:** one real medical thought over the live consumer-pull +path — HealthcarePort classids → cognitive-shader-driver + REAL MailboxSoA +(closing the qualia-extractor honesty gap) → owner_adapter cast → PR #879 +sparse cycle loop; falsifiers F1–F4 incl. the currently-ABSENT fail-closed +unknown-actor test. `ogar-obo` (MONDO/HPO/Uberon/PATO — verified real, zero +consumer edges) = optional slice only. Older MedCare plans classified in the +plan's §8 ledger (completed / dormant-decision-required / stale-but-unmarked). + ## 2026-08-02 — kanban-64k-inverted-awareness v1 — PLANNED / CONJECTURE (parallel thinking + the inverted-awareness witness) — main thread **Plan:** `.claude/plans/kanban-64k-inverted-awareness-v1.md` @@ -144,30 +168,6 @@ each probe-first with both falsifier halves. Blocked-on external: the consumer physical bake (private repo), validity criterion, `LanceShardSink`. Built on the R1–R15 review list (session 2026-08-02); §0 ground state carries receipts so nothing is re-derived. -## 2026-08-02 — medcare-consumer-pull-thinking-proof v1 — ACTIVE (the single active MedCare plan after the two-lineage investigation) — main thread - -**Plan:** `.claude/plans/medcare-consumer-pull-thinking-proof-v1.md` -Consolidation of the 2026-08-02 six-agent read-only MedCare/Healthcare -investigation (HEAD `71d1db1`, git unshallowed to 4162 commits). Verdict: -**two lineages, not one incomplete runtime.** LIVE = consumer-pull (MedCare-rs -`medcare-bridge` → vendored `lance-graph-ogar` → `UnifiedBridge` -→ OGAR canon `0x0901–0x090C`; bridge migration COMPLETED at `ddb6c840` -2026-06-21, `MedcareBridge` deprecated alias only). DEAD = host-side -manifest/supervisor/actor scaffold (frozen since birth 2026-05-13; manifest -compile-time-parsed but runtime-orphaned; `MedcareConsumerActor` never -constructed; supervisor hard-codes `StubConsumerActor`; Dispatch rejected -pre-child; `MedCareActor`/`MedCareMessage` exist in NEITHER repo; -`medcare_policy` symbol nonexistent). Two same-named, never-composed -`UnifiedBridge` types (ogar resolver — no authorize; callcenter authorize/audit -— no ogar dep) classified as LATER hardening, not a proof prerequisite. -**Active proof target:** one real medical thought over the live consumer-pull -path — HealthcarePort classids → cognitive-shader-driver + REAL MailboxSoA -(closing the qualia-extractor honesty gap) → owner_adapter cast → PR #879 -sparse cycle loop; falsifiers F1–F4 incl. the currently-ABSENT fail-closed -unknown-actor test. `ogar-obo` (MONDO/HPO/Uberon/PATO — verified real, zero -consumer edges) = optional slice only. Older MedCare plans classified in the -plan's §8 ledger (completed / dormant-decision-required / stale-but-unmarked). - ## 2026-08-02 — cycle-loop-closure-driver v1 — PLANNED / CONJECTURE (the seam that makes persist_sink load-bearing at 64k) — main thread **Plan:** `.claude/plans/cycle-loop-closure-driver-v1.md` diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 7a52b1e77..24a814745 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -5,7 +5,8 @@ - `lance_graph::graph::cycle_sink` — **the gap `persist_sink` deliberately left is now closed**: a concrete implementation of `lance_graph_planner::persist_sink::WalSink` over the official Lance 9 insert path (`Dataset::write` / `Dataset::append` — the same `InsertBuilder` transaction machinery every Lance writer uses). No bespoke ledger, no acknowledgement protocol, no parallel replay system: Lance's own manifest/version chain IS the WAL. Gated on the default-on `planner` feature (the trait lives in the optional planner dep). - `LanceCycleSink::new(path)` — holds only the dataset path; opens the dataset per operation, so restart-survival is exercised on EVERY call, not just in tests. - `cycle_store_schema()` — one dataset, two row kinds: a per-cycle **frame row** (`kind=0`, the cycle ↔ version mapping sealed INSIDE the same atomic commit — zero sidecar state) and **landing rows** (`kind=1`: `stream_position`, `owner`, `row`, nullable `move_*` Rubicon-edge columns, `payload` witness-node bytes). - - **The §I.6 fence, both halves.** Pre-commit: the store's current version must equal the cycle's sealed predecessor `base` (empty store ⇒ head `DatasetVersion(0)`), else refused with NOTHING written. Post-commit: the published version must be exactly `base + 1` — Lance auto-resolves append-append conflicts, so a foreign interleaved writer surfaces as a LOUD timeline anomaly instead of silently shifting the cycle ↔ version identity reads derive (`sealed_version = base_version + 1` is an identity the commit path VERIFIES, never assumes). + - **The §I.6 fence, both halves.** Pre-commit: the store's current version must equal the cycle's sealed predecessor `base` (empty store ⇒ head `DatasetVersion(0)`), else refused with NOTHING written. Post-commit: the published version must be exactly `base + 1` (`sealed_version = base_version + 1` is an identity the commit path VERIFIES, never assumes). Lance has NO expected-version conditional append (the rebase runs even on a single-attempt Append commit; strict mode exists only for Overwrite — measured in `lance-9.0.0/src/io/commit.rs`), so a foreign interleaved writer can land the batch at `base + 2`; the review round (#911 Codex P1 + CodeRabbit) made the fence EFFECTIVE retroactively there: the just-published cycle rows are removed again with an official `Dataset::delete` scoped to exactly this cycle (the cycle id is an unsealed identity — the predicate is exact), and only then is the retryable `WriteFailed` returned, so "write failed" is TRUE at the visible head and the driver's regenerate-from-`Vn` contract stays sound. The one manual-reconciliation corner (compensating delete itself fails) names the orphaned version explicitly — reachable only when the one-writer doctrine was already violated by a foreign writer. + - **The coalesced image is DURABLE** (review round): `kind = 2` image rows (`row → final 512-byte payload` after the per-row fold) persist in the same atomic commit as the landings; `LanceCycleSink::scan_image(cycle)` reads a sealed cycle's coherent end-state (projected `row`+`payload` under the kind+cycle predicate) while the per-cast history stays intact via `scan_sealed`. Every landing/image payload is gated to exactly `EPISODIC_WITNESS_BYTES = 512` (the canonical `key(16)|edges(16)|value(480)` node stride) BEFORE anything durable happens; `versions()` projects only `cycle`+`base_version` under `kind = 0` so the coarse-timeline lookup never materializes a witness payload. - **Order is a write-side property**: landings are stored in the already-deinterlaced `DetachedCycleBatch` order and scanned back with `scan_in_order(true)`; the sink never sorts on read. - **Domain-0x09 witness contract (module doc, operator-ruled):** the patient SoA at classid domain `0x09` is the ONLY place patient reasoning is written to Lance, so the store is witness-focused and maximally rich — `payload` carries the 512-byte canonical EpisodicWitness node (visited ontology addresses, executed crosswalk mappings, exact RO/ontology edge ids, supporting/contradicting/missing observations, NARS truth+confidence, differential branches), whose edges point INTO the immutable domain-0x03 ontology address space. The cycle takes ontology immutability for granted for its representation window (`base_version` names the sealed predecessor it read) and therefore never restates ontology content — the sealed versioning is a reflection of the thinking; downstream (the Gotham display, differential views) reads the sealed version, never a live recomputation. - **Gates:** 6 tokio tests, every guarantee proven against a REOPENED dataset (fresh sink instance + fresh `Dataset::open`, never an in-memory echo): seal survives restart; stale base fenced with nothing written (store version chain, landings, timeline all untouched — checked, not assumed); sequential cycles chain V1→V2→V3 with strictly-after filtering; a zero-landing cycle advances the timeline only; an empty store reads empty (a state, not an error — `DatasetNotFound` distinguished from real I/O failure); move-nullability + 512-byte payload byte-exact round-trip. Real Lance version chain cross-checked against the returned `DatasetVersion` in-test. diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 44fe88bb8..366be30e3 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -36,7 +36,7 @@ ## 2026-08-09 — branch `claude/medcare-rs-continue-ufsazd` (PR pending) — the concrete cognitive-cycle Lance sink: `graph::cycle_sink::LanceCycleSink` - **Added.** `lance_graph::graph::cycle_sink` (~660 LOC incl. 6 reopened-dataset tokio tests) — the concrete `lance_graph_planner::persist_sink::WalSink` over the official Lance 9 insert path; `cycle_store_schema()` (frame row + landing rows, nullable Rubicon `move_*` columns, `payload` witness bytes); `LanceCycleSink`. Module gated on the default-on `planner` feature. -- **Locked.** **Lance's manifest/version chain IS the WAL** — no bespoke ledger / acknowledgement protocol / parallel replay system (operator-ruled). **The §I.6 fence has two halves**: pre-commit head==base (nothing written on refusal) AND post-commit published==base+1 (Lance auto-resolves append-append conflicts, so a foreign writer is a loud timeline anomaly, never a silent identity shift). **`sealed_version = base_version + 1` is a verified identity, not an assumption** — which is what lets reads derive the cycle↔version mapping from the atomically-co-committed frame row with zero sidecar state. **Order is a write-side property** (stored deinterlaced order, `scan_in_order(true)`, no read-time sort). **Domain-0x09 witness contract**: the patient SoA is the ONLY patient-reasoning Lance write target; payload = maximally-rich EpisodicWitness node built ON TOP of the immutable domain-0x03 ontology addresses (immutability taken for granted for the representation window via `base_version`). +- **Locked.** **Lance's manifest/version chain IS the WAL** — no bespoke ledger / acknowledgement protocol / parallel replay system (operator-ruled). **The §I.6 fence has two halves**: pre-commit head==base (nothing written on refusal) AND post-commit published==base+1 — and because Lance has NO expected-version conditional append (Append rebases even single-attempt; strict mode is Overwrite-only, measured in `lance-9.0.0/src/io/commit.rs`), a detected shift triggers a compensating official `Dataset::delete` of exactly this cycle's rows BEFORE the retryable `WriteFailed` returns (review round, #911 Codex P1): the fence is effective retroactively, nothing of the cycle stays visible, regenerate-from-`Vn` stays sound. **The coalesced image is durable** (`kind=2` rows + `scan_image`, same atomic commit); **payloads are gated to the canonical 512 bytes**; **`versions()` is a projected frame-only read**. **`sealed_version = base_version + 1` is a verified identity, not an assumption** — which is what lets reads derive the cycle↔version mapping from the atomically-co-committed frame row with zero sidecar state. **Order is a write-side property** (stored deinterlaced order, `scan_in_order(true)`, no read-time sort). **Domain-0x09 witness contract**: the patient SoA is the ONLY patient-reasoning Lance write target; payload = maximally-rich EpisodicWitness node built ON TOP of the immutable domain-0x03 ontology addresses (immutability taken for granted for the representation window via `base_version`). - **Deferred.** The MedCare consumer arc (production `drive_cohort_thoughts` caller, witness-seal, views reading the sealed version) — next PR, in MedCare-rs. `recover_and_apply` wiring against this sink in a production driver. Object-store (s3/az/gs) smoke — the path plumbing accepts URIs but only local was exercised. - **Docs.** Module-level witness/§I.6 contract in `cycle_sink.rs`; this entry + LATEST_STATE inventory (same commit). diff --git a/.claude/plans/medcare-consumer-pull-thinking-proof-v1.md b/.claude/plans/medcare-consumer-pull-thinking-proof-v1.md index 3ac33d660..c5bf438c3 100644 --- a/.claude/plans/medcare-consumer-pull-thinking-proof-v1.md +++ b/.claude/plans/medcare-consumer-pull-thinking-proof-v1.md @@ -26,7 +26,7 @@ There are **two MedCare integration lineages, not one partially-assembled path** ## 2. The live path (build on THIS) -``` +```text MedCare-rs (private) └─ crates/medcare-bridge ──(vendor/lance-graph softlink)──► lance_graph_ogar::MedcareBridge = UnifiedBridge @@ -44,7 +44,7 @@ build proves nothing about it). ## 3. The dead lineage (dormant — do NOT revive to prove thinking) -``` +```text modules/medcare/manifest.yaml → build.rs → MANIFEST_METADATA ╳ (one caller: a test) CallcenterSupervisor::spawn_consumer_actor → StubConsumerActor (unconditional) DispatchToG: Health → ok; ALL else → DispatchNotImplemented (child never reached) @@ -75,7 +75,7 @@ reconstruct the dead lineage in miniature). Public/private boundary: MedCare-specific schema and clinical mappings stay in MedCare-rs; generic SoA/cycle/driver primitives may land here. -``` +```text MedCare-rs input (existing schema surface, private side) → existing medcare-bridge → UnifiedBridge (classid, namespace lock) → existing MedCare-rs policy/schema path (or narrow proof policy — fail-closed) diff --git a/crates/lance-graph/src/graph/cycle_sink.rs b/crates/lance-graph/src/graph/cycle_sink.rs index 872b05633..73ca28e9d 100644 --- a/crates/lance-graph/src/graph/cycle_sink.rs +++ b/crates/lance-graph/src/graph/cycle_sink.rs @@ -92,8 +92,23 @@ use lance_graph_planner::persist_sink::{ const KIND_FRAME: u8 = 0; /// Row kind discriminant: one landing (a thought's persistence record). const KIND_LANDING: u8 = 1; - -/// The Arrow schema of the cycle store — one dataset, two row kinds. +/// Row kind discriminant: one coalesced-image row — the FINAL payload of a +/// dirty SoA row after the per-row fold (`DetachedCycleBatch::image`), made +/// durable in the same atomic commit so the store carries the coherent cycle +/// image itself, not just the per-cast history it folds from. +const KIND_IMAGE: u8 = 2; + +/// The canonical witness-node payload size: `key(16) | edges(16) | value(480)` +/// — the 512-byte node row stride with its 16-byte edge reservation. Every +/// landing / image payload persisted by this sink MUST be exactly this long; +/// a malformed witness row is refused before anything durable happens. +pub const EPISODIC_WITNESS_BYTES: usize = 512; + +/// The Arrow schema of the cycle store — one dataset, three row kinds: the +/// per-cycle frame row (`kind = 0`), the per-cast landing rows (`kind = 1`, +/// the table below), and the coalesced-image rows (`kind = 2`: `row` + the +/// FINAL 512-byte payload after the per-row fold, `stream_position`/`owner` +/// zero, moves null — the durable coherent cycle image). /// /// | column | type | frame row | landing row | /// |-------------------------------|-----------------|-----------|-------------| @@ -170,9 +185,15 @@ impl LanceCycleSink { /// Build the single atomic RecordBatch for a cycle: the frame row first, /// then the landings in their ALREADY-canonical order (the loom ran in - /// `DetachedCycleBatch::freeze`; storage order = stream order by contract). + /// `DetachedCycleBatch::freeze`; storage order = stream order by contract), + /// then the coalesced-image rows (`row → final payload`) so the coherent + /// cycle image is durable alongside the per-cast history it folds from. + /// + /// Every landing / image payload must be exactly [`EPISODIC_WITNESS_BYTES`] + /// — the canonical 512-byte node row — or the whole cycle is refused + /// before anything durable happens. fn build_batch(batch: &DetachedCycleBatch) -> Result { - let n = batch.landings.len() + 1; + let n = batch.landings.len() + batch.image.len() + 1; let mut kind = Vec::with_capacity(n); let mut cycle = Vec::with_capacity(n); let mut base_version = Vec::with_capacity(n); @@ -202,6 +223,13 @@ impl LanceCycleSink { payload.append_value([]); for s in &batch.landings { + if s.payload.len() != EPISODIC_WITNESS_BYTES { + return Err(WriteFailed(format!( + "landing payload for row {} is {} bytes, expected the canonical {EPISODIC_WITNESS_BYTES}", + s.row, + s.payload.len() + ))); + } kind.push(KIND_LANDING); cycle.push(s.cycle.0); base_version.push(batch.frame.base_version.0); @@ -227,6 +255,32 @@ impl LanceCycleSink { payload.append_value(&s.payload); } + // Coalesced-image rows: the final per-row state after the stream-order + // fold. `BTreeMap` iteration gives a deterministic (row-ascending) + // stored order. Same-cycle landings already passed the 512-byte gate, + // and the image is a fold over exactly those payloads — the length + // check here guards the invariant independently rather than assuming it. + for (row_id, image_payload) in &batch.image { + if image_payload.len() != EPISODIC_WITNESS_BYTES { + return Err(WriteFailed(format!( + "image payload for row {row_id} is {} bytes, expected the canonical {EPISODIC_WITNESS_BYTES}", + image_payload.len() + ))); + } + kind.push(KIND_IMAGE); + cycle.push(batch.frame.cycle.0); + base_version.push(batch.frame.base_version.0); + stream_position.push(0); + owner.push(0); + row.push(*row_id); + move_mailbox.append_null(); + move_from.append_null(); + move_to.append_null(); + move_wcp.append_null(); + move_exec.append_null(); + payload.append_value(image_payload); + } + RecordBatch::try_new( cycle_store_schema(), vec![ @@ -247,11 +301,19 @@ impl LanceCycleSink { .map_err(|e| WriteFailed(format!("build cycle batch: {e}"))) } - /// Read the whole store at its LATEST version, in stored (insertion) order — - /// Lance's in-order scan; this sink never sorts on read. - async fn read_all_rows(&self, ds: &Dataset) -> Result, WriteFailed> { + /// Read the store's rows of ONE kind at its LATEST version, in stored + /// (insertion) order — Lance's in-order scan; this sink never sorts on + /// read. The kind predicate is pushed into the scan so frame/image reads + /// never materialize landing payloads (and vice versa). + async fn read_rows_of_kind( + &self, + ds: &Dataset, + kind_filter: u8, + ) -> Result, WriteFailed> { let mut scan = ds.scan(); scan.scan_in_order(true); + scan.filter(&format!("kind = {kind_filter}")) + .map_err(|e| WriteFailed(format!("filter {}: {e}", self.dataset_path)))?; let batches: Vec = scan .try_into_stream() .await @@ -340,6 +402,51 @@ impl StoredRow { } } +impl LanceCycleSink { + /// Read a sealed cycle's durable coalesced image: `row → final payload` + /// after the write-side per-row fold. This is the coherent end-state a + /// downstream consumer (a view, the Gotham display) reads — the per-cast + /// history behind it stays available via [`WalSink::scan_sealed`]. Projects + /// `row` + `payload` under a `kind = 2 AND cycle = …` predicate. An empty + /// map = the cycle is unknown or landed nothing. + pub async fn scan_image( + &self, + cycle: CycleId, + ) -> Result>, WriteFailed> { + let Some(ds) = self.open_if_exists().await? else { + return Ok(std::collections::BTreeMap::new()); + }; + let mut scan = ds.scan(); + scan.scan_in_order(true); + scan.filter(&format!("kind = {KIND_IMAGE} AND cycle = {}", cycle.0)) + .map_err(|e| WriteFailed(format!("filter {}: {e}", self.dataset_path)))?; + scan.project(&["row", "payload"]) + .map_err(|e| WriteFailed(format!("project {}: {e}", self.dataset_path)))?; + let batches: Vec = scan + .try_into_stream() + .await + .map_err(|e| WriteFailed(format!("scan {}: {e}", self.dataset_path)))? + .try_collect() + .await + .map_err(|e| WriteFailed(format!("collect {}: {e}", self.dataset_path)))?; + let mut out = std::collections::BTreeMap::new(); + for b in &batches { + let row: &UInt64Array = b + .column_by_name("row") + .and_then(|c| c.as_any().downcast_ref()) + .ok_or_else(|| WriteFailed("missing column row".into()))?; + let payload: &BinaryArray = b + .column_by_name("payload") + .and_then(|c| c.as_any().downcast_ref()) + .ok_or_else(|| WriteFailed("missing column payload".into()))?; + for i in 0..b.num_rows() { + out.insert(row.value(i), payload.value(i).to_vec()); + } + } + Ok(out) + } +} + impl WalSink for LanceCycleSink { /// THE single amortized durable append for a whole cycle, over the official /// Lance insert path — one commit, one new `DatasetVersion`, all-or-nothing. @@ -350,9 +457,21 @@ impl WalSink for LanceCycleSink { /// declare base 0 (it read no sealed predecessor). A stale base is /// refused with NOTHING written. /// 2. **Post-commit:** the published version must be exactly `base + 1`. - /// Lance auto-resolves append-append conflicts, so a foreign interleaved - /// writer surfaces here as a loud timeline anomaly instead of silently - /// shifting the cycle ↔ version identity that reads derive. + /// Lance auto-resolves append-append conflicts (there is no + /// expected-version conditional append in the official API — the rebase + /// runs even on a single-attempt commit for Append operations), so a + /// foreign interleaved writer can land this batch at `base + 2`. When + /// that is detected, the fence is made EFFECTIVE retroactively: the + /// just-published cycle rows are removed again with an official + /// `Dataset::delete` scoped to exactly this cycle's rows, and only THEN + /// is the retryable [`WriteFailed`] returned — so "write failed" is + /// true at the visible head (nothing of this cycle remains readable), + /// the driver's regenerate-from-`Vn` contract stays sound, and no rows + /// survive under a shifted `sealed_version` identity. If the + /// compensating delete itself fails, the error says so explicitly and + /// names the orphaned version — the one manual-reconciliation corner, + /// reachable only when the one-writer §I.6 doctrine was already + /// violated by a foreign writer. async fn commit_cycle( &self, base: DatasetVersion, @@ -395,18 +514,41 @@ impl WalSink for LanceCycleSink { ds.append(reader, None) .await .map_err(|e| WriteFailed(format!("append commit: {e}")))?; - ds.version().version + let published = ds.version().version; + if published != base.0 + 1 { + // A foreign writer interleaved between the fence check and + // the commit; Lance's append rebase landed this batch at a + // shifted version. Make the fence effective retroactively: + // remove exactly this cycle's just-appended rows, then + // report the (now-true) retryable failure. The cycle id is + // an unsealed identity at this point — no earlier sealed + // rows can carry it — so the predicate is exact. + let compensate = ds + .delete(&format!( + "cycle = {} AND base_version = {}", + batch.frame.cycle.0, base.0 + )) + .await; + return Err(match compensate { + Ok(_) => WriteFailed(format!( + "fenced post-publication: a foreign writer moved the head past \ + {base:?} (batch landed at DatasetVersion({published})); the \ + cycle's rows were deleted again — nothing of cycle {} is \ + visible; regenerate from the current sealed head", + batch.frame.cycle.0 + )), + Err(e) => WriteFailed(format!( + "TIMELINE ANOMALY, MANUAL RECONCILIATION REQUIRED: cycle {} \ + committed at DatasetVersion({published}) (expected {}), and the \ + compensating delete failed: {e}", + batch.frame.cycle.0, + base.0 + 1 + )), + }); + } + published } }; - if published != base.0 + 1 { - // The data IS committed at `published` — but the one-writer §I.6 - // timeline is broken (a foreign writer interleaved). Surface loudly; - // never let a shifted identity pass as a sealed cycle. - return Err(WriteFailed(format!( - "timeline anomaly: committed at DatasetVersion({published}), expected {}", - base.0 + 1 - ))); - } Ok(DatasetVersion(published)) } @@ -420,10 +562,9 @@ impl WalSink for LanceCycleSink { let Some(ds) = self.open_if_exists().await? else { return Ok(Vec::new()); }; - let rows = self.read_all_rows(&ds).await?; + let rows = self.read_rows_of_kind(&ds, KIND_LANDING).await?; Ok(rows .into_iter() - .filter(|r| r.kind == KIND_LANDING) .filter(|r| from_version.is_none_or(|f| r.sealed_version() > f)) .map(|r| LandedSlot { version: r.sealed_version(), @@ -434,16 +575,44 @@ impl WalSink for LanceCycleSink { /// The cheap coarse timeline — the per-cycle frame rows, each sealed in the /// same atomic commit as its landings, read back from the reopened store. + /// Projects only `cycle` + `base_version` under a `kind = 0` predicate, so + /// the lookup never materializes a single landing payload no matter how + /// much witness history the store has accumulated. async fn versions(&self) -> Result, WriteFailed> { let Some(ds) = self.open_if_exists().await? else { return Ok(Vec::new()); }; - let rows = self.read_all_rows(&ds).await?; - Ok(rows - .into_iter() - .filter(|r| r.kind == KIND_FRAME) - .map(|r| (r.cycle, r.sealed_version())) - .collect()) + let mut scan = ds.scan(); + scan.scan_in_order(true); + scan.filter(&format!("kind = {KIND_FRAME}")) + .map_err(|e| WriteFailed(format!("filter {}: {e}", self.dataset_path)))?; + scan.project(&["cycle", "base_version"]) + .map_err(|e| WriteFailed(format!("project {}: {e}", self.dataset_path)))?; + let batches: Vec = scan + .try_into_stream() + .await + .map_err(|e| WriteFailed(format!("scan {}: {e}", self.dataset_path)))? + .try_collect() + .await + .map_err(|e| WriteFailed(format!("collect {}: {e}", self.dataset_path)))?; + let mut out = Vec::new(); + for b in &batches { + let cycle: &UInt64Array = b + .column_by_name("cycle") + .and_then(|c| c.as_any().downcast_ref()) + .ok_or_else(|| WriteFailed("missing column cycle".into()))?; + let base_version: &UInt64Array = b + .column_by_name("base_version") + .and_then(|c| c.as_any().downcast_ref()) + .ok_or_else(|| WriteFailed("missing column base_version".into()))?; + for i in 0..b.num_rows() { + out.push(( + CycleId(cycle.value(i)), + DatasetVersion(base_version.value(i) + 1), + )); + } + } + Ok(out) } } @@ -467,6 +636,12 @@ mod tests { } } + /// A canonical 512-byte witness payload, tagged by `stream_position` so + /// distinct casts stay byte-distinguishable. + fn witness(stream_position: u64) -> Vec { + vec![stream_position as u8; EPISODIC_WITNESS_BYTES] + } + fn slot(cycle: u64, stream_position: u64, owner: MailboxId, row: u64) -> SweepSlot { SweepSlot { cycle: CycleId(cycle), @@ -474,7 +649,7 @@ mod tests { owner, row, paired_move: Some(mv(owner)), - payload: vec![stream_position as u8; 4], + payload: witness(stream_position), } } @@ -506,7 +681,7 @@ mod tests { assert_eq!(sealed[1].slot.stream_position, 20); assert_eq!(sealed[0].version, DatasetVersion(1)); assert_eq!(sealed[0].slot.paired_move, Some(mv(5))); - assert_eq!(sealed[0].slot.payload, vec![10u8; 4]); + assert_eq!(sealed[0].slot.payload, witness(10)); let versions = reopened.versions().await.unwrap(); assert_eq!(versions, vec![(CycleId(1), DatasetVersion(1))]); @@ -656,4 +831,72 @@ mod tests { assert_eq!(sealed[0].slot.owner, 11); assert_eq!(sealed[0].slot.row, 900); } + + /// A malformed witness payload (≠ 512 bytes) is refused with NOTHING + /// written — the canonical node row stride is enforced before anything + /// durable happens, and the store stays exactly as it was. + #[tokio::test] + async fn malformed_payload_is_refused_before_persistence() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("witness_cycles.lance"); + let sink = LanceCycleSink::new(path.to_str().unwrap()); + + let mut bad = slot(1, 1, 3, 50); + bad.payload = vec![0u8; 100]; + let err = persist_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![bad], + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("100 bytes"), "{err}"); + // Nothing durable: the dataset was never even created. + assert!(Dataset::open(path.to_str().unwrap()).await.is_err()); + + // The store still accepts a well-formed cycle afterwards. + persist_cycle( + &sink, + CycleFrame::new(CycleId(1), DatasetVersion(0)), + vec![slot(1, 1, 3, 50)], + ) + .await + .unwrap(); + } + + /// The coalesced image is DURABLE: same-row casts fold to the final + /// payload, persisted as image rows in the same atomic commit and read + /// back per cycle from a reopened store — while the per-cast landing + /// history stays intact alongside it. + #[tokio::test] + async fn coalesced_image_is_durable_per_cycle() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("witness_cycles.lance"); + let sink = LanceCycleSink::new(path.to_str().unwrap()); + + // Three casts, two rows: row 7 is written twice (positions 1 then 3 — + // later stream position wins the image), row 8 once. + let casts = vec![slot(1, 3, 2, 7), slot(1, 1, 2, 7), slot(1, 2, 2, 8)]; + persist_cycle(&sink, CycleFrame::new(CycleId(1), DatasetVersion(0)), casts) + .await + .unwrap(); + + let reopened = LanceCycleSink::new(path.to_str().unwrap()); + let image = reopened.scan_image(CycleId(1)).await.unwrap(); + assert_eq!(image.len(), 2); + assert_eq!(image[&7], witness(3), "later stream position wins"); + assert_eq!(image[&8], witness(2)); + // The per-cast history is still complete and ordered. + let sealed = reopened.scan_sealed(None).await.unwrap(); + assert_eq!(sealed.len(), 3); + assert_eq!( + sealed + .iter() + .map(|l| l.slot.stream_position) + .collect::>(), + vec![1, 2, 3] + ); + // An unknown cycle has no image. + assert!(reopened.scan_image(CycleId(99)).await.unwrap().is_empty()); + } }