From c672856bafb3d5f522b9b68b113ff3c229bf439a Mon Sep 17 00:00:00 2001 From: Evgeny Formanenko Date: Wed, 29 Jul 2026 19:50:24 +0300 Subject: [PATCH] fix(hotblocks): enforce finalized fork floor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fork whose common ancestor lies inside a finality-straddling chunk cannot resume at fin+1 — insert_fork replaces whole chunks, so a mid-chunk position wedges the dataset on a 60s restart loop — and resuming below fin instead silently rewrites the finalized prefix (GAP-22). Resolution now stops at the stored chunk boundary and the write path verifies the finalized region: a replacement reaching that low must span fin and reproduce every block up to it, hash for hash. Comparing only fin's own hash would admit a source rewriting the interior, since hashes come from the source and nothing ties a reproduced boundary to what leads to it. Payload equality is out of reach by construction (INV-13 Scope). The guard only recovers if the replay reaches fin, which was left as an assumption about the flush triggers. It does not hold: the 200k-row bound can cut the replayed chunk one block below fin, the write path refuses it, and every retry repeats the identical cut — an indefinite park, observed. Fork resolution now carries fin to the ingest, which withholds a chunk until it covers that block. A retention trim can also land inside a fork replay, leaving a rollback resolved against a window that no longer exists; insert_fork then drops every surviving chunk and the head ends up under the floor with the trimmed blocks back. Whether the ingest may keep running is read off the trim's own result: predicting it from block numbers covered only a floor moving past the head, while the window is also cleared on an empty dataset, on a parent-hash mismatch and on a gap — none of which move the floor past the head, so a stale ingest outlived all three, and with nothing left stored it committed below the floor against a guard that had no bound to enforce. A surviving head still keeps the ingest — the floor advances continuously through the Api path, so restarting on every trim reconnected the source for nothing — and that replay is refused at commit instead. The bound is the first stored chunk, not first(D): retention trims whole chunks, so its logical floor can sit inside the surviving one, which a fork legitimately resumes at; with nothing stored it falls back to first(D), which is where resolution resumes anyway. The check precedes the finality guard because a trim above fin clears fin, and then it is the only guard left. A push repeating the current floor is answered before the trim runs: nothing below it is left to delete, and reading the verdict off an empty window reports a lost head and restarts an ingest the call never touched — once per push, on a dataset that has not committed its first chunk yet. The report that becomes fin was itself unchecked, and it anchors both that guard and every later rollback. It is a header the source hands over, not a block it served, so where it names a height the committed chunk carries, the chunk now decides (WP-8); where it names a height the chunk skips there is no hash to record, so it is dropped rather than refused — refusing would park a slot-numbered dataset for as long as an honest source keeps reporting a height its own stream skipped. Every refusal shape shared one metric bucket, so a stale-ingest refusal that clears itself on the next epoch read exactly like a source rewriting finalized history; UnapplicableFork::reason is now a slug on a cause label. And a withheld replay is counted only where a flush is attempted, which a source that stops below the floor never reaches — the pending floor is a level-readable gauge, set at fork resolution. Unrelated but folded in: the query slot is released before its caller is woken. The pool thread sent the result and dropped the slot afterwards, so a caller woken in between could be refused admission for a query that had already finished — and the panic-path test flaked on it, 3 failures in 200 local runs. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FD1RAN4sYMB1mgTVMEoASi Co-Authored-By: Codex --- Cargo.lock | 1 + crates/data-source/src/standard.rs | 20 +- crates/hotblocks-harness/README.md | 24 +- crates/hotblocks-harness/src/harness.rs | 4 + crates/hotblocks-harness/src/model.rs | 109 +- crates/hotblocks-harness/src/sim.rs | 168 ++- crates/hotblocks-harness/src/sut.rs | 3 + crates/hotblocks/Cargo.toml | 1 + crates/hotblocks/spec/02-data-model.md | 10 + crates/hotblocks/spec/03-write-path.md | 123 +- crates/hotblocks/spec/06-invariants.md | 25 +- crates/hotblocks/spec/08-failure-model.md | 1 + crates/hotblocks/spec/11-observability.md | 6 +- crates/hotblocks/spec/12-conformance-tdd.md | 62 +- crates/hotblocks/src/cli.rs | 8 +- .../dataset_controller/dataset_controller.rs | 69 +- .../src/dataset_controller/ingest_generic.rs | 177 ++- .../dataset_controller/write_controller.rs | 1322 +++++++++++++++-- crates/hotblocks/src/errors.rs | 65 +- crates/hotblocks/src/metrics.rs | 163 +- crates/hotblocks/src/query/executor.rs | 3 + crates/hotblocks/tests/ct4_finality.rs | 333 +++++ crates/hotblocks/tests/ct9_source_faults.rs | 15 + crates/storage/src/db/db.rs | 13 + crates/storage/src/db/read/blocks_table.rs | 68 +- crates/storage/src/db/write/dataset_update.rs | 12 + crates/storage/src/db/write/tx.rs | 141 +- 27 files changed, 2665 insertions(+), 281 deletions(-) create mode 100644 crates/hotblocks/tests/ct4_finality.rs diff --git a/Cargo.lock b/Cargo.lock index fc2c0b54..33b1b514 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4894,6 +4894,7 @@ name = "sqd-hotblocks" version = "0.1.0" dependencies = [ "anyhow", + "arrow", "async-stream", "axum", "bytes", diff --git a/crates/data-source/src/standard.rs b/crates/data-source/src/standard.rs index 682c2d77..d1361323 100644 --- a/crates/data-source/src/standard.rs +++ b/crates/data-source/src/standard.rs @@ -5,7 +5,7 @@ use futures::{future::BoxFuture, stream::BoxStream, FutureExt, Stream, StreamExt use sqd_data_client::{BlockStreamRequest, BlockStreamResponse, DataClient}; use sqd_primitives::{Block, BlockNumber, BlockRef}; use tokio::time::Sleep; -use tracing::warn; +use tracing::{info, warn}; use crate::types::{DataEvent, DataSource}; @@ -301,11 +301,19 @@ where let forks = self.endpoints.iter().filter(|ep| ep.is_on_fork()).count(); if forks > 0 { - if forks > self.endpoints.len() / 2 - || forks == self.endpoints.iter().filter(|ep| ep.is_active()).count() - || self.fork_consensus_timeout(cx) - { - return Poll::Ready(DataEvent::Fork(self.extract_fork())); + let active = self.endpoints.iter().filter(|ep| ep.is_active()).count(); + if forks > self.endpoints.len() / 2 || forks == active || self.fork_consensus_timeout(cx) { + let chain = self.extract_fork(); + info!( + forked_endpoints = forks, + active_endpoints = active, + total_endpoints = self.endpoints.len(), + hint_count = chain.len(), + oldest_hint =? chain.first().map(|b| b.number), + newest_hint =? chain.last().map(|b| b.number), + "fork consensus reached" + ); + return Poll::Ready(DataEvent::Fork(chain)); } } else { self.state.fork_consensus_timeout = None diff --git a/crates/hotblocks-harness/README.md b/crates/hotblocks-harness/README.md index aa26404c..8e4b03c4 100644 --- a/crates/hotblocks-harness/README.md +++ b/crates/hotblocks-harness/README.md @@ -22,6 +22,7 @@ is what makes the crash/restart and shutdown classes expressible at all. ```bash cargo test -p sqd-hotblocks-harness # the harness's own unit tests (model, chain, simulator) cargo test -p sqd-hotblocks --test ct1_happy_path # CT-1 — the Phase 0 exit criterion +cargo test -p sqd-hotblocks --test ct4_finality # CT-4 — finalized-prefix equivocation cargo test -p sqd-hotblocks --test ct9_source_faults ``` @@ -33,7 +34,7 @@ reusable lives here, so a future soak or benchmark runner can use it outside `ca | Module | What it is | Spec | |---|---|---| -| [`sim`](src/sim.rs) | source simulator: scripted chain, fork signals, finality headers, fault knobs | 13 §7, DEF-12 | +| [`sim`](src/sim.rs) | source simulator: scripted chain, fork signals, finality headers, fault knobs including explicit finalized-prefix equivocation | 13 §7, DEF-12 | | [`model`](src/model.rs) | the reference model — the oracle. Block-exact, well-formedness asserted after every transition | 12 §2 | | [`driver`](src/driver.rs) | client: the read binding, the structural validators, the anchored follower and backfill scanner | 04 §7, 12 §4 | | [`compare`](src/compare.rs) | quiescence comparator: diffs every observable, collects *all* violations before failing | 12 §1 | @@ -131,15 +132,18 @@ Fixed in `crates/data-client/src/reqwest/lines.rs`; pinned by a unit test there - **CT-2 (crash/restart)** — `Sut::crash()`, `Sut::stop()`, `Sut::restart()` already exist and keep the same database directory and port across boots. What is missing is the kill-point matrix. -- **CT-4 (fork/finality corpus)** — `Harness::fork()` and the model's `resolve_fork` / - `Finalize::IntegrityFault` are implemented and unit-tested; the follower implements the - normative CONFLICT recovery of 04 §7. What is missing is most of the scripts. - `ct4_lagging_source` covers the multi-endpoint shape production actually runs — several - sources per dataset, one of them behind (`HarnessConfig::sources`, `Harness::produce_ahead`). - Note `Harness::fork()` refuses to run with peers configured: reorging one endpoint of several - is a source *disagreement*, and what the service should do with it is the fork-consensus - question (`StandardDataSource::poll_next_event` — majority, or all-active, or a 2 s timeout). - That deserves a deliberate script, not an accidental one. +- **CT-4 (fork/finality corpus)** — `ct4_finality` drives equivocation through the real binary at + the retained-window floor and at a two-chunk layout with finality inside the second chunk, plus + the honest duals: a reorg above `fin` recovers, so does one whose replay is cut below `fin`, and + a replay reproducing `fin`'s own hash while rewriting a block below it is refused. Rollback + resumes at a stored chunk boundary, which may sit below `fin`; the finalized prefix is verified + on the write path instead. `ct4_lagging_source` covers the multi-endpoint shape production + actually runs — several sources per dataset, one of them behind (`HarnessConfig::sources`, + `Harness::produce_ahead`). Note `Harness::fork()` refuses to run with peers configured: reorging + one endpoint of several is a source *disagreement*, and what the service should do with it is the + fork-consensus question (`StandardDataSource::poll_next_event` — majority, or all-active, or a + 2 s timeout). That deserves a deliberate script, not an accidental one. The below-window, + malformed-finality, fork-storm and alarm scripts remain. - **CT-5 (error taxonomy)** — `ct5_error_soundness` covers unsupported-dialect containment, error classification, and mid-stream worker-panic abort; the anchored check across large sparse-number holes is deferred (GAP-21, test `#[ignore]`d). `Model::predict_query` supplies diff --git a/crates/hotblocks-harness/src/harness.rs b/crates/hotblocks-harness/src/harness.rs index d0c54b51..bb5e7242 100644 --- a/crates/hotblocks-harness/src/harness.rs +++ b/crates/hotblocks-harness/src/harness.rs @@ -29,6 +29,8 @@ pub struct HarnessConfig { /// Dense (evm, hyperliquid) or sparse (Solana slots) block numbering. pub numbering: Numbering, pub retention: Retention, + /// Keep physical ingest chunks separate when a test needs a deterministic storage layout. + pub disable_compaction: bool, /// Whether the service is told the anchor hash. If not, the anchor is `⊥` (DEF-7) and the /// first block's parent is unverifiable. pub anchored: bool, @@ -59,6 +61,7 @@ impl HarnessConfig { number: start_block, parent_hash: Some(block_hash(start_block - 1, 0)) }, + disable_compaction: false, anchored: true, source_poll: Duration::from_millis(200), sources: 1, @@ -119,6 +122,7 @@ impl Harness { id: cfg.dataset.clone(), kind: cfg.chain.config_kind().to_string(), retention: cfg.retention.clone(), + disable_compaction: cfg.disable_compaction, sources: std::iter::once(&sim) .chain(peers.iter()) .map(|s| s.base_url(&cfg.dataset)) diff --git a/crates/hotblocks-harness/src/model.rs b/crates/hotblocks-harness/src/model.rs index 9c5c36a8..511e4024 100644 --- a/crates/hotblocks-harness/src/model.rs +++ b/crates/hotblocks-harness/src/model.rs @@ -17,9 +17,9 @@ use crate::{ #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Finalize { Applied, - /// Stale or not-yet-applicable report; state unchanged (WP-7). + /// Stale or not-yet-applicable report; state unchanged (WP §2.4/WP-8). Ignored, - /// Finality contradicting stored data — must be alarmed, never accepted (WP-8, GAP-4/5). + /// Finality contradicting stored data — must be alarmed, never accepted (WP-8). IntegrityFault } @@ -176,9 +176,13 @@ impl Model { blocks[0].number ); } + let previous = self.clone(); self.seg.extend_from_slice(blocks); self.ver += 1; - self.apply_finality(fin)?; + if let Err(err) = self.apply_finality(fin) { + *self = previous; + return Err(err); + } self.wf(); Ok(()) } @@ -198,11 +202,25 @@ impl Model { from >= first, "INV-14: REPLACE at {from} is below the window floor {first}" ); - // ... nor below the finalized prefix (INV-13). - if let Some(fin) = &self.fin { + // ... nor alter the finalized prefix (INV-13). Reaching below `fin` is allowed only as an + // identical replay: a batch-granular implementation cannot start at `fin + 1` when finality + // sits inside a batch, and re-committing what is already there changes nothing. + if let Some(fin) = &self.fin + && from <= fin.number + { + let stored: Vec<_> = self + .seg + .iter() + .filter(|b| b.number >= from && b.number <= fin.number) + .collect(); + let replayed: Vec<_> = blocks.iter().filter(|b| b.number <= fin.number).collect(); ensure!( - from > fin.number, - "INV-13: REPLACE at {from} is at or below the finalized head {}", + stored.len() == replayed.len() + && stored + .iter() + .zip(&replayed) + .all(|(a, b)| a.number == b.number && a.hash == b.hash), + "INV-13: REPLACE at {from} alters the finalized prefix up to {}", fin.number ); } @@ -227,14 +245,42 @@ impl Model { } } } + if let Some(report) = fin + && self.replacement_evicts_reported_block(from, blocks, report) + { + bail!("INTEGRITY_FAULT: replacement evicts the block reported final at {report}"); + } + let previous = self.clone(); self.seg.truncate(keep); self.seg.extend_from_slice(blocks); self.ver += 1; - self.apply_finality(fin)?; + if let Err(err) = self.apply_finality(fin) { + *self = previous; + return Err(err); + } self.wf(); Ok(()) } + /// WP-8's non-hole case: the incoming range owns `e`, omits it, and thereby removes a block + /// stored there before the transition. Evaluate only exact, non-regressive reports; a report + /// above the resulting head is clamped and does not assert its own height. + fn replacement_evicts_reported_block(&self, from: BlockNumber, blocks: &[Block], report: &BlockRef) -> bool { + let post_head = blocks.last().expect("REPLACE blocks are non-empty").number; + if report.number > post_head || report.number < self.first().expect("REPLACE window is non-empty") { + return false; + } + if let Some(fin) = &self.fin + && report.number <= fin.number + { + return false; + } + from <= report.number + && report.number <= post_head + && blocks.iter().all(|b| b.number != report.number) + && self.block_at(report.number).is_some() + } + /// WP §2.4 — advance `fin`, monotone and clamped to the stored chain. pub fn finalize(&mut self, r: &BlockRef) -> Finalize { let outcome = self.finalize_inner(r); @@ -260,10 +306,10 @@ impl Model { return Finalize::Ignored; } let at_e = if e == r.number { - // The report names an exact height: a hole there contradicts the stored chain the - // same way a hash mismatch does (WP §2.4), and a different stored hash is WP-8. + // A slot-numbered chain may genuinely skip this height. There is no block reference to + // store as `fin`, so the report is ignored and may re-arrive later (WP-8). let Some(stored) = self.block_at(e) else { - return Finalize::IntegrityFault; + return Finalize::Ignored; }; if stored.hash != r.hash { return Finalize::IntegrityFault; @@ -610,8 +656,7 @@ mod tests { assert_eq!(m.finalize(&BlockRef::new(101, block_hash(101, 0))), Finalize::Ignored); assert_eq!(m.fin, Some(blocks[4].as_ref())); - // A report naming a block we do not have is an integrity fault (WP-8) — this is the - // check GAP-4 says the implementation skips below the head. + // A report naming an existing block with another hash is an integrity fault (WP-8). let (mut m, _) = model_at(100, 5); assert_eq!( m.finalize(&BlockRef::new(102, block_hash(102, 7))), @@ -621,16 +666,25 @@ mod tests { } #[test] - fn finalize_on_a_hole_is_an_integrity_fault() { + fn finalize_on_a_hole_is_ignored() { let anchor = block_hash(99, 0); let mut m = Model::new(Anchor::new(99, Some(anchor.clone()))); m.extend(&chain_of(&[100, 103, 104], 0, (99, &anchor)), None).unwrap(); - // WP §2.4: a report naming a height that is a hole in the stored chain contradicts - // the chain exactly like a hash mismatch. - assert_eq!( - m.finalize(&BlockRef::new(101, block_hash(101, 0))), - Finalize::IntegrityFault - ); + + assert_eq!(m.finalize(&BlockRef::new(101, block_hash(101, 0))), Finalize::Ignored); + assert_eq!(m.fin, None); + } + + #[test] + fn replacement_cannot_evict_the_block_it_reports_finalized() { + let (mut m, blocks) = model_at(100, 3); + let replacement = chain_of(&[102], 1, (100, &blocks[0].hash)); + let before = m.seg.clone(); + + let err = m.replace(101, &replacement, Some(&blocks[1].as_ref())).unwrap_err(); + + assert!(err.to_string().contains("INTEGRITY_FAULT"), "unexpected error: {err:#}"); + assert_eq!(m.seg, before, "a refused composed transition must be atomic"); assert_eq!(m.fin, None); } @@ -639,10 +693,18 @@ mod tests { let (mut m, blocks) = model_at(100, 5); assert_eq!(m.finalize(&BlockRef::new(102, block_hash(102, 0))), Finalize::Applied); - // Below the finalized head — INV-13. + // Below the finalized head, on another branch — INV-13. let bad = run(102, 2, 1, &blocks[0].hash); assert!(m.replace(102, &bad, None).is_err()); + // Reaching that low is allowed when the finalized part is replayed unchanged: a + // batch-granular implementation has no `fin + 1` position when finality sits mid-batch. + let replay = [blocks[2].clone()] + .into_iter() + .chain(run(103, 2, 7, &blocks[2].hash)) + .collect::>(); + assert!(m.replace(102, &replay, None).is_ok()); + // Above it — accepted, and the head moves back by one. let good = run(103, 2, 1, &blocks[2].hash); m.replace(103, &good, None).unwrap(); @@ -672,6 +734,11 @@ mod tests { assert_eq!(m.anchor, Anchor::new(104, Some(blocks[4].hash.clone())), "INV-18"); assert_eq!(m.fin, None, "RS-2: finality below the window is dropped"); assert_eq!(m.head().unwrap().number, 109); + assert_eq!( + m.finalize(&BlockRef::new(104, blocks[4].hash.clone())), + Finalize::Ignored, + "WP §2.4: finality below the window stays ignored" + ); // Trimming at the floor is a no-op. m.retain(105, None); diff --git a/crates/hotblocks-harness/src/sim.rs b/crates/hotblocks-harness/src/sim.rs index c37f8fb9..c880f074 100644 --- a/crates/hotblocks-harness/src/sim.rs +++ b/crates/hotblocks-harness/src/sim.rs @@ -85,13 +85,26 @@ pub struct SimFaults { /// no-data. RP-5b confines the signal to `from == tip + 1`, the one position where the /// assertion is evaluable; a source doing it higher reports a divergence it cannot have /// observed, and a source that is merely behind starts looking like a forked one. - pub fork_signal_above_tip: bool + pub fork_signal_above_tip: bool, + /// Cut every response to at most this many blocks (0 reads as 1), so the service's chunk + /// boundaries stop tracking the source's history. + pub max_blocks_per_response: Option, + /// Report finality no higher than this — a replica lagging behind the one the service already + /// accepted finality from. + pub finality_report_cap: Option } /// Counters a test can assert on (how the SUT actually drove the source). #[derive(Clone, Copy, Debug, Default)] pub struct SimStats { + /// HTTP `/stream` requests. Unlike `stream_requests`, a long-poll wakeup does not increment it. + pub stream_http_requests: u64, + /// Response evaluations, including re-evaluations of one parked request after a source bump. pub stream_requests: u64, + /// `fromBlock` on the most recent request, for rollback-position assertions. + pub last_stream_from: Option, + /// Lowest `fromBlock` observed since [`SourceSim::reset_stream_request_observations`]. + pub lowest_stream_from: Option, pub blocks_served: u64, pub fork_signals: u64, pub no_data: u64, @@ -166,6 +179,30 @@ impl SourceSim { Ok(blocks) } + /// Fault injection for CT-4/FM-SRC-5: replace a suffix that includes the source's own + /// finalized head and claim the replacement tip as final. + /// + /// Unlike [`Self::fork`], this deliberately violates source finality. It has no model-side + /// counterpart: the last accepted model state remains the oracle while the SUT rejects the + /// equivocating source. + pub fn equivocate_finalized_prefix(&self, dataset: &str, from: BlockNumber, len: u32) -> Result<()> { + self.try_with(dataset, |d| d.equivocate_finalized_prefix(from, len))?; + self.bump(); + Ok(()) + } + + /// Fault injection for CT-4/INV-13: rewrite the hash of one block strictly *below* the source's + /// own finalized head, leaving that block and everything above it intact. + /// + /// The chain stays internally linked, so only a comparison against stored history tells this + /// apart from an honest replay. Like [`Self::equivocate_finalized_prefix`] it has no model-side + /// counterpart. + pub fn rewrite_hash_below_finality(&self, dataset: &str, at: BlockNumber) -> Result { + let r = self.try_with(dataset, |d| d.rewrite_hash_below_finality(at))?; + self.bump(); + Ok(r) + } + /// Declare `number` (and everything below it) final. pub fn finalize(&self, dataset: &str, number: BlockNumber) -> Result { let r = self.try_with(dataset, |d| d.finalize(number))?; @@ -181,6 +218,11 @@ impl SourceSim { self.with(dataset, |d| d.stats) } + /// Start a new request-observation window without disturbing lifetime response counters. + pub fn reset_stream_request_observations(&self, dataset: &str) { + self.with(dataset, DatasetSim::reset_stream_request_observations); + } + /// Turn a source-side fault on or off (FM-SRC-*). pub fn inject_fault(&self, dataset: &str, f: impl FnOnce(&mut SimFaults)) { self.with(dataset, |d| f(&mut d.faults)); @@ -308,22 +350,75 @@ impl DatasetSim { } fn fork(&mut self, from: BlockNumber, len: u32) -> Result> { + self.validate_fork_position(from)?; + ensure!( + self.fin.as_ref().is_none_or(|f| f.number < from), + "the script forks at or below the source's own finalized head — an equivocating source \ + belongs to the CT-4 fault corpus, not to a well-formed script" + ); + Ok(self.replace_suffix(from, len)) + } + + fn equivocate_finalized_prefix(&mut self, from: BlockNumber, len: u32) -> Result<()> { + self.validate_fork_position(from)?; + ensure!(len > 0, "a finality-equivocation fault must mint a replacement tip"); + let finalized = self + .fin + .as_ref() + .context("a finality-equivocation fault requires an existing finalized head")?; + ensure!( + from <= finalized.number, + "equivocation at {from} does not replace finalized block {}", + finalized.number + ); + + let replacement = self.replace_suffix(from, len); + self.fin = Some(replacement.last().expect("a non-empty replacement has a tip").as_ref()); + Ok(()) + } + + fn rewrite_hash_below_finality(&mut self, at: BlockNumber) -> Result { + let finalized = self + .fin + .as_ref() + .context("rewriting below finality requires an existing finalized head")? + .number; + ensure!( + at < finalized, + "block {at} is not strictly below the source's finalized head {finalized}" + ); + + let i = self + .chain + .binary_search_by_key(&at, |b| b.number) + .ok() + .with_context(|| format!("the source does not have block {at}"))?; + + let hash = block_hash(at, self.next_fork_id); + self.next_fork_id += 1; + self.chain[i].hash = hash.clone(); + if let Some(child) = self.chain.get_mut(i + 1) { + child.parent_hash = hash.clone(); + } + Ok(BlockRef::new(at, hash)) + } + + fn validate_fork_position(&self, from: BlockNumber) -> Result<()> { ensure!( from >= self.start, "fork at {from} is below the source's first block {}", self.start ); ensure!(from <= self.next_number(), "fork at {from} is above the source's chain"); - ensure!( - self.fin.as_ref().is_none_or(|f| f.number < from), - "the script forks at or below the source's own finalized head — an equivocating source \ - belongs to the CT-4 fault corpus, not to a well-formed script" - ); + Ok(()) + } + + fn replace_suffix(&mut self, from: BlockNumber, len: u32) -> Vec { let keep = self.chain.partition_point(|b| b.number < from); self.chain.truncate(keep); self.fork_id = self.next_fork_id; self.next_fork_id += 1; - Ok(self.produce(len)) + self.produce(len) } fn finalize(&mut self, number: BlockNumber) -> Result { @@ -378,23 +473,54 @@ impl DatasetSim { if req.from_block < self.start { self.stats.below_history += 1; - return Reply::NoData(self.fin.clone()); + return Reply::NoData(self.reported_fin()); } let Some(tip) = self.tip_number() else { self.stats.no_data += 1; - return Reply::NoData(self.fin.clone()); + return Reply::NoData(self.reported_fin()); }; if req.from_block > tip { self.stats.no_data += 1; - return Reply::NoData(self.fin.clone()); + return Reply::NoData(self.reported_fin()); } let lo = self.chain.partition_point(|b| b.number < req.from_block); - let blocks = self.chain[lo..].to_vec(); + let mut blocks = self.chain[lo..].to_vec(); + if let Some(cap) = self.faults.max_blocks_per_response { + blocks.truncate((cap as usize).max(1)); + } self.stats.blocks_served += blocks.len() as u64; Reply::Blocks { blocks, - fin: self.fin.clone() + fin: self.reported_fin() + } + } + + fn observe_stream_request(&mut self, from_block: BlockNumber) { + self.stats.stream_http_requests += 1; + self.stats.last_stream_from = Some(from_block); + self.stats.lowest_stream_from = Some( + self.stats + .lowest_stream_from + .map_or(from_block, |current| current.min(from_block)) + ); + } + + fn reset_stream_request_observations(&mut self) { + self.stats.stream_http_requests = 0; + self.stats.last_stream_from = None; + self.stats.lowest_stream_from = None; + } + + /// What this replica admits to having finalized (see [`SimFaults::finality_report_cap`]). + fn reported_fin(&self) -> Option { + let fin = self.fin.clone()?; + match self.faults.finality_report_cap { + Some(cap) if cap < fin.number => self.chain_hash_at(cap).map(|hash| BlockRef { + number: cap, + hash: hash.to_string() + }), + _ => Some(fin) } } } @@ -413,6 +539,14 @@ async fn stream(State(shared): State>, Path(ds): Path, body: Err(err) => return (StatusCode::BAD_REQUEST, format!("bad stream request: {err}")).into_response() }; + { + let mut guard = shared.datasets.lock().expect("simulator state is poisoned"); + let Some(dataset) = guard.get_mut(&ds) else { + return (StatusCode::NOT_FOUND, format!("unknown dataset '{ds}'")).into_response(); + }; + dataset.observe_stream_request(req.from_block); + } + let mut bump = shared.bump.subscribe(); let deadline = Instant::now() + shared.poll_timeout; @@ -465,7 +599,7 @@ async fn head(State(shared): State>, Path(ds): Path) -> Resp } async fn finalized_head(State(shared): State>, Path(ds): Path) -> Response { - watermark(&shared, &ds, |d| d.fin.clone()) + watermark(&shared, &ds, |d| d.reported_fin()) } fn watermark(shared: &Shared, ds: &str, f: impl FnOnce(&DatasetSim) -> Option) -> Response { @@ -671,5 +805,13 @@ mod tests { started.elapsed() >= Duration::from_millis(100), "the request must long-poll" ); + let stats = sim.stats(DS); + assert_eq!(stats.stream_http_requests, 1); + assert_eq!(stats.last_stream_from, Some(START)); + assert_eq!(stats.lowest_stream_from, Some(START)); + assert!( + stats.stream_requests > stats.stream_http_requests, + "long-poll response evaluations must not masquerade as new HTTP requests" + ); } } diff --git a/crates/hotblocks-harness/src/sut.rs b/crates/hotblocks-harness/src/sut.rs index 3eee2188..d0b9f68f 100644 --- a/crates/hotblocks-harness/src/sut.rs +++ b/crates/hotblocks-harness/src/sut.rs @@ -39,6 +39,8 @@ pub struct DatasetSpec { pub id: String, pub kind: String, pub retention: Retention, + /// Preserve response-aligned chunks for tests that exercise physical layout boundaries. + pub disable_compaction: bool, pub sources: Vec } @@ -262,6 +264,7 @@ impl Sut { Retention::Api => yaml.push_str(" retention_strategy: Api\n"), Retention::None => yaml.push_str(" retention_strategy: None\n") } + yaml.push_str(&format!(" disable_compaction: {}\n", ds.disable_compaction)); yaml.push_str(" data_sources:\n"); for src in &ds.sources { yaml.push_str(&format!(" - \"{src}\"\n")); diff --git a/crates/hotblocks/Cargo.toml b/crates/hotblocks/Cargo.toml index 03ad3da4..764496ca 100644 --- a/crates/hotblocks/Cargo.toml +++ b/crates/hotblocks/Cargo.toml @@ -37,6 +37,7 @@ url = { workspace = true, features = ["serde"] } [dev-dependencies] anyhow = { workspace = true } +arrow = { workspace = true } reqwest = { workspace = true } sqd-hotblocks-harness = { path = "../hotblocks-harness" } tempfile = { workspace = true } diff --git a/crates/hotblocks/spec/02-data-model.md b/crates/hotblocks/spec/02-data-model.md index a65a8652..717f5c5a 100644 --- a/crates/hotblocks/spec/02-data-model.md +++ b/crates/hotblocks/spec/02-data-model.md @@ -74,6 +74,15 @@ next(D) = head(D).number + 1 (if seg empty: anchor.number + 1) span(D) = n (window size in blocks) ``` +A storage implementation that retains and replaces whole chunks may physically carry a +prefix below the logical `first(D)`. `chunk_first(D)` denotes the first block of the physical +chunk that owns `first(D)`; therefore `chunk_first(D) ≤ first(D)`, with equality in a +precisely-trimmed implementation. That retained overshoot does not lower the logical retention +floor: FINALIZE and the reference state use `first(D)`. `chunk_first(D)` is used only where an +atomic whole-chunk REPLACE or its linkage must name the physical owner. `chunk_anchor(D)` is +that chunk's carried parent reference (the chain hash immediately preceding `chunk_first(D)`); +it equals `anchor` when `chunk_first(D) = first(D)`. + A dataset state is **well-formed** iff it satisfies the structural invariants INV-1 … INV-7 of [06-invariants.md](06-invariants.md). Every externally observable state MUST be well-formed. @@ -226,3 +235,4 @@ advance arriving together); invariants are evaluated at commit points only. | fork hints | `ForkSignal.hints` (DEF-12), also the payload of the CONFLICT error (RP-11) | | hash index / `bidx`, `tidx` | DEF-17 | | batch | the unit of one `EXTEND`/`REPLACE` commit (bounded by P-BATCH-ROWS / P-BATCH-BYTES) | +| chunk | a physical storage unit replaced atomically; compaction may merge several committed batches into one chunk | diff --git a/crates/hotblocks/spec/03-write-path.md b/crates/hotblocks/spec/03-write-path.md index 89580957..a6ba3d82 100644 --- a/crates/hotblocks/spec/03-write-path.md +++ b/crates/hotblocks/spec/03-write-path.md @@ -33,11 +33,29 @@ loop: hashes are opaque, source-controlled strings (DEF-2), so hash linkage alone implies no number order (GAP-20). A run failing validation MUST be discarded without any state change, and the offending source penalized ([FM-SRC-4](08-failure-model.md)). -- **WP-3 (Batching).** Blocks MAY be accumulated and committed in batches. A batch is - bounded by `P-BATCH-ROWS` rows / `P-BATCH-BYTES` bytes and MUST be flushed no later - than: the bound being reached, an `OnTip` event, a `ForkSignal`, or an item-availability - change. Batching MUST NOT reorder or drop blocks. (Freshness consequence: the head - advances in batch-sized steps; see HZ-6 and SLI-11.) +- **WP-3 (Batching).** Blocks MAY be accumulated and committed in batches. Outside the + finalized-replay exception below, a batch is bounded by `P-BATCH-ROWS` rows / + `P-BATCH-BYTES` bytes and MUST be flushed no later than: the bound being reached, an + `OnTip` event, a `ForkSignal`, or an item-availability change. Batching MUST NOT reorder + or drop blocks. (Freshness consequence: the head advances in batch-sized steps; see HZ-6 + and SLI-11.) + + **Finalized-replay exception.** When fork resolution (§2.4) resumes at or below + `fin.number`, publishing a partial replay could replace finalized storage before the + candidate reproduces it. Such a candidate MUST therefore remain unpublished until it + has reached at least `fin.number` and can satisfy §2.3's whole-prefix reproduction check. + While it remains below that floor: + + - a row/byte bound or `OnTip` event MUST NOT waive the floor; an implementation MAY spill + intermediate buffers without publishing them; + - a newer `ForkSignal` MAY discard the superseded candidate and resolve again; and + - an item-availability change that cannot share one valid batch MUST abort and retry + without changing stored state. + + The unpublished candidate can consequently exceed the ordinary batch bounds and remain + pending if the source stops below the floor; that liveness risk MUST be level-observable + and is tracked by GAP-43. This exception never permits alteration of a finalized block + and does not relax the ordinary-ingest bounds. - **WP-4 (Multi-source arbitration).** With multiple sources, the service MUST present the effect of a single coherent source: duplicate deliveries deduplicated, positions below `next(D)` ignored, and a `ForkSignal` acted upon only when corroborated — by a majority @@ -91,15 +109,25 @@ position, not a promised block number, DEF-1.) ### 2.3 REPLACE(from, B, f?) — fork application - Pre: `seg ≠ ∅`; `B ≠ ∅` and valid per WP-2; `B[0].number ≥ from`; and - - **fork floor (finality):** `fin = ⊥ ∨ from > fin.number` — MUST (INV-13/14); - - **fork floor (window):** `from ≥ first(D)` — MUST (INV-14). A required rollback below - `first(D)` is not representable as REPLACE; it MUST be handled as RESET (§2.6) and - surfaced as an event ([OB-9](11-observability.md)); - - linkage: `from > first(D)` ⇒ `B[0].parent_hash` = the hash of `from`'s preceding + - **fork floor (finality):** `fin = ⊥ ∨ from > fin.number`, or `B` reproduces the stored + chain over `[from, fin.number]` block for block — MUST (INV-13/14). The second disjunct + exists because `from` snaps to a stored batch boundary, which sits below `fin` whenever + finality falls inside a batch; INV-13 forbids *altering* a finalized block, not + re-writing it identically. Reproduction MUST be checked over the whole overlap, not at + `fin.number` alone: hashes come from the source, so a reproduced boundary says nothing + about the interior; + - **fork floor (window):** `from ≥ chunk_first(D)` — MUST (INV-14). In a precisely-trimmed + implementation `chunk_first(D) = first(D)`; a whole-chunk implementation may start in + retained physical overshoot below the logical floor solely to replace its owner atomically. + A required rollback below `chunk_first(D)` is not representable as REPLACE; it MUST be + handled as RESET (§2.6) and surfaced as an event ([OB-9](11-observability.md)); + - linkage: `from > chunk_first(D)` ⇒ `B[0].parent_hash` = the hash of `from`'s preceding block (`hash_at(from − 1)`, DEF-16 — `from − 1` itself may be a hole on a - slot-numbered chain); `from = first(D)` ⇒ (`anchor.hash ≠ ⊥` ⇒ - `B[0].parent_hash = anchor.hash`). + slot-numbered chain); `from = chunk_first(D)` ⇒ (`chunk_anchor(D).hash ≠ ⊥` ⇒ + `B[0].parent_hash = chunk_anchor(D).hash`). - Post: `seg' = seg[‥from] ⧺ B`; `anchor' = anchor`; `fin'` per composed FINALIZE or `fin`. + When physical `from < first(D)`, the reference-state projection omits `B`'s retained + overshoot below `first(D)`; it exists only to make the owning batch replaceable atomically. - `REPLACE(next(D), B)` degenerates to `EXTEND(B)`. **WP-6 (Fork resolution).** Upon an arbitrated `ForkSignal(hints)` the service MUST @@ -111,21 +139,31 @@ signal contradicts finality and MUST be treated as a source integrity fault the stored chain (or the anchor). If no hint matches stored state, the fallback MUST respect the finality floor: -- `fin ≠ ⊥` → resume from `⟨fin.number + 1, fin.hash⟩`: the finalized block is common to - both chains unless the source contradicts finality, so only the volatile suffix is - replaced (a full-window replacement would violate the fork floor anyway, INV-14). If - the source then repeatedly fails to link at this position, that *is* a finality - contradiction: after `P-SOURCE-STRIKES` consecutive rejections it MUST be classified as - FM-SRC-5 (fault + alarm, keep serving) — never RESET; -- `fin = ⊥` → resume from `⟨first(D), anchor.hash⟩` (full-window replacement); WP-6b +- `fin ≠ ⊥` → resume from `⟨fin.number + 1, fin.hash⟩`, or from any lower physical position + `≥ chunk_first(D)` whose replacement reproduces the finalized prefix (INV-14). The finalized + block is common to both chains unless the source contradicts finality, so only the + volatile suffix carries new data either way. An implementation whose REPLACE granularity + is a storage chunk MUST take the lower position when `fin` falls inside a chunk: `fin + 1` + is then unrepresentable, and retrying it is a wedge rather than a rejection. If the source + repeatedly fails to link at the chosen position, that *is* a finality contradiction: after + `P-SOURCE-STRIKES` consecutive rejections it MUST be classified as FM-SRC-5 (fault + + alarm, keep serving) — never RESET; +- `fin = ⊥` → resume from `⟨chunk_first(D), chunk_anchor(D).hash⟩` (full-window replacement); WP-6b governs escalation if the divergence turns out to lie below the window. The subsequent commit is `REPLACE(m + 1, …)` (resp. `REPLACE(fin.number + 1, …)`, -`REPLACE(first(D), …)`). The resume point MAY be conservatively deeper than the optimal -`m + 1` by at most one storage batch (an implementation that matches hints only at batch -boundaries): the replacement then re-commits blocks identical to those it replaces, which -is correctness-neutral. It MUST NOT be shallower than `m + 1` and MUST NOT cross the -floors of §2.3. +`REPLACE(chunk_first(D), …)`). Both the match and the finality fallback MAY be conservatively +deeper by at most one storage chunk (an implementation that replaces whole chunks): the +replacement then re-commits blocks identical to those it replaces, which is +correctness-neutral, and below `fin` that identity is what INV-14 requires. It MUST NOT be +shallower than `m + 1` and MUST NOT cross the floors of §2.3. + +Background maintenance MAY merge across the selected physical batch boundary before the +replacement commits. A replay that consequently starts inside the newly merged batch is a +**stale rollback plan**, not source equivocation or an unspecified storage failure: the +replacement MUST remain atomic, the refusal MUST be attributable as such, and resolution +MUST be retried against the current layout. A conservative retry may therefore select one +merged batch deeper than the first resolution without changing the logical result. **WP-6b (Divergence below the window → RESET).** WP-6's fallback cannot represent a fork deeper than the window; the service MUST detect that case and escalate to RESET (alarmed, @@ -136,7 +174,8 @@ OB-9) instead of retrying a replacement that can never link: `RESET(⟨anchor.number, hint hash⟩)`; - **indirect evidence:** after a full-window-replacement resume (reachable only when `fin = ⊥` — WP-6 fallback), the source's runs - repeatedly fail attachment at `first(D)` (`B[0].parent_hash ≠ anchor.hash`, WP-2). + repeatedly fail attachment at `chunk_first(D)` (`B[0].parent_hash ≠ chunk_anchor(D).hash`, + WP-2). After `P-SOURCE-STRIKES` consecutive such rejections the condition MUST be classified as a below-window divergence — `RESET(⟨anchor.number, ⊥⟩)` — not retried silently forever (LIV-9b, GAP-3/GAP-5). @@ -153,20 +192,34 @@ Let `e = min(r.number, head(D).number)` (a finality report above the head is cla the head; the excess is not forgotten by sources and will re-arrive). - Pre: `seg ≠ ∅` (else the report is deferred/ignored); `e ≥ first(D)` (a report below the - window is ignored); **hash verification:** when `e = r.number` there MUST be a stored - block at height `e` with hash equal to `r.hash` — a report naming a height that is a - hole in the stored chain (slot-numbered chains, DEF-1) contradicts the stored chain - exactly like a hash mismatch and MUST be treated as a source integrity fault - (FM-SRC-5, WP-8); when clamped (`e < r.number`), the stored head is taken as finalized - on the strength of the source's claim about its descendant. + logical retention floor is ignored, even when its physical chunk remains as retention + overshoot); **hash verification:** when `e = r.number`, resolve the report per + WP-8 — a block carried by the post-state at `e` MUST have hash `r.hash`, while a genuine + hole is ignored; when clamped (`e < r.number`), the stored head is taken as finalized on + the strength of the source's claim about its descendant. - Monotonicity: if `fin ≠ ⊥ ∧ e < fin.number` → ignore (no transition). If `e = fin.number` with a different hash → source integrity fault (FM-SRC-5), no transition. - Post: `fin' = ⟨e, stored hash at e⟩`. -**WP-8** A finality report whose hash contradicts the stored block at the same height -(either `fin` itself or the block at `e`) MUST NOT be applied and MUST raise an integrity -alarm — silently dropping it hides either a source fault or a wrong stored chain. +**WP-8 (Finality reports are verified against blocks, not taken from headers).** `r` is a +header the source hands over; it is not a block it served, and `fin` anchors both the fork +floor (INV-14) and every later fork resolution (WP-6). Resolve the owner of height `e` in +the post-state first: a replacement batch owns its numeric range; stored history owns +everything outside that range. Exactly one outcome then holds: + +- the owner carries a block at `e` with hash `r.hash` → apply `⟨e, r.hash⟩`; +- the owner carries a block at `e` with another hash → integrity fault (FM-SRC-5), not + applied, alarmed (OB-9). Composed with a batch (§2.2/§2.3), the batch is refused with it; +- the owner carries no block at `e`, and no block is being removed from that height → + **ignored, not refused**. This is a genuine hole on a slot-numbered chain (DEF-1); the + post-state `⟨e, stored hash at e⟩` has no value to take. Finality re-arrives, so this + costs a report, never the transition; +- a replacement owns `e` but omits it while stored history currently carries a block there + → integrity fault. This is not a hole: the response deletes a stored block while calling + that same height final. + +A report is logged in every case (OB-9). ### 2.5 RETAIN(from, h?) @@ -186,7 +239,7 @@ history cannot be re-acquired through RETAIN. symmetric with WP-9. *Finality note:* like every RESET (§2.6) and like upward trims (RS-2), this discards the finalized prefix and clears `fin` — retention dominates finality. It is **not** a rollback below `fin`: the fork floor (INV-13/14) is - untouched — *sources* can never replace anything at or below `fin` — and this path is + untouched — *sources* can never alter anything at or below `fin` — and this path is reachable only through an explicit retention instruction (operator-class actor, FM-OP-4), never through source input. - Case `first(D) < from ≤ next(D)`: diff --git a/crates/hotblocks/spec/06-invariants.md b/crates/hotblocks/spec/06-invariants.md index 32ff3ed7..21da44a2 100644 --- a/crates/hotblocks/spec/06-invariants.md +++ b/crates/hotblocks/spec/06-invariants.md @@ -51,7 +51,8 @@ If `seg = ∅` then `fin = ⊥`. Otherwise, if `fin ≠ ⊥` then **INV-6 — Finalized-on-chain.** [state] If `fin ≠ ⊥` then the stored block at height `fin.number` has hash `fin.hash`. *Why:* finality must describe the chain actually served, else finalized-only reads lie. -*Check:* CT-1; CT-4 with equivocating-finality sources (GAP-4). +*Check:* CT-1; CT-4 with equivocating-finality sources; write-controller finality +resolver regressions. **INV-7 — Provenance fidelity.** [state] Every stored block was delivered by a configured source, and all queryable field values @@ -83,13 +84,27 @@ changes. `fin` may become `⊥` only via RETAIN (window passing above it), RESET **INV-13 — Finalized immutability.** [transition] No `REPLACE` removes or alters blocks at heights `≤ fin.number`. Finalized blocks leave the store only via RETAIN / RESET / DROP. +*Scope:* "alters" is decidable only down to block identity — `⟨number, hash⟩` compared +against stored history. No block hash is re-derived from its payload anywhere in the +system, so a source reproducing a height's hash while serving different content for it is +outside what any write-path check can see; sources are trusted for content at every height, +finalized or not, and this holds equally for the first write of a block. *Check:* CT-1/CT-4 fork storms around the finality boundary. **INV-14 — Fork floor.** [transition] -For every `REPLACE(from, B)`: `from > fin.number` (when `fin ≠ ⊥`) **and** -`from ≥ first(D)`. A deeper divergence is representable only as RESET (explicit, alarmed). -*Why:* silent rollback below the window or below finality corrupts continuation clients. -*Check:* CT-4 deep-fork corpus (GAP-3). +For every `REPLACE(from, B)`: `from ≥ chunk_first(D)` **and** either `from > fin.number` +(when `fin ≠ ⊥`) or `B` reproduces the stored chain over `[from, fin.number]` block for +block. `chunk_first(D)` equals logical `first(D)` for a precisely-trimmed store; a +whole-chunk implementation may re-commit retained overshoot below `first(D)` only as part +of replacing that physical owner. A deeper divergence is representable only as RESET +(explicit, alarmed). +*Why:* silent rollback below the window or below finality corrupts continuation clients. An +identical replay of the finalized prefix changes nothing, and permitting it is what lets an +implementation whose REPLACE granularity is a storage chunk resolve a fork at all: when `fin` +falls inside a chunk, `from = fin + 1` is not a representable position. The window half of the +floor has to be enforced at commit, not only where `from` is chosen: a trim between the two +moves `chunk_first(D)` under a resume position that was legal when it was picked. +*Check:* CT-4 deep-fork corpus (GAP-3); `replacement_below_the_retained_window_is_refused`. **INV-15 — Retention trims prefix only.** [transition] `RETAIN` removes only blocks with numbers below its `from`; it never creates gaps, never diff --git a/crates/hotblocks/spec/08-failure-model.md b/crates/hotblocks/spec/08-failure-model.md index 00d63c91..89e54ec4 100644 --- a/crates/hotblocks/spec/08-failure-model.md +++ b/crates/hotblocks/spec/08-failure-model.md @@ -76,6 +76,7 @@ each. "Required response" uses four verbs: | FM-OP-3 | Two service instances over one store | fail-safe: detect divergence, stop the losing writer per dataset, alarm (WP-15); MUST NOT interleave-corrupt | | FM-OP-4 | Retention mistakes (raise far above head, contradictory instructions) | defined semantics (WP-9/RETAIN cases): destructive outcomes are the documented ones only; observable; idempotent | | FM-OP-5 | Restart with changed parameters (window size, budgets) | mask: state re-converges to policy (trim or backfill-forward per WP-10/WP-5); no invariant violations during convergence | +| FM-OP-6 | Data availability changed for a live dataset — sources swapped or reconfigured so the same blocks arrive with a different section set (the data-availability mask) | **out of scope**: a deliberate operator change, sequenced by the operator. No behaviour across it is promised and none should be relied on. Only the standing integrity line holds: two section sets never share a chunk (INV-7), so where the ingest cannot cut cleanly it stops loudly rather than serving blocks stripped of sections | ## 6. Fault → property cross-reference diff --git a/crates/hotblocks/spec/11-observability.md b/crates/hotblocks/spec/11-observability.md index b9554835..92240c63 100644 --- a/crates/hotblocks/spec/11-observability.md +++ b/crates/hotblocks/spec/11-observability.md @@ -45,7 +45,11 @@ surface of the binding (13 §5) with bounded cardinality. - **OB-9 (Alarm states).** Distinct, queryable, per-dataset alarm conditions with reason codes: integrity conflict (WP-8/FM-SRC-5), RESET occurred (WP §2.6), boot validation refusal (INV-43), dataset stopped (CN-10), dual-writer detected (WP-15), disk floor - breached (FM-STOR-2). Alarms are edge-triggered events *and* level-readable states. + breached (FM-STOR-2), replay withheld below finality (WP-6). Alarms are edge-triggered + events *and* level-readable states. Reason codes MUST separate refusals that clear + themselves on the next epoch from refusals that repeat until an operator acts: they carry + the same consequence for a minute and opposite ones over an hour, and a single bucket + cannot be alerted on. - **OB-10 (Bounded cardinality).** All label spaces are bounded by configuration (datasets, sources, classes, outcome enums); unbounded client-derived labels MUST be sanitized (e.g. allowlisted client identities, "other" bucket). diff --git a/crates/hotblocks/spec/12-conformance-tdd.md b/crates/hotblocks/spec/12-conformance-tdd.md index 2c738a46..2e5433ab 100644 --- a/crates/hotblocks/spec/12-conformance-tdd.md +++ b/crates/hotblocks/spec/12-conformance-tdd.md @@ -4,11 +4,13 @@ This document turns the spec into a test program: the reference model (oracle), harness architecture, the test-class taxonomy, the traceability matrix, and the dated gap register that seeds the hardening backlog. -Statuses and the gap register reflect the state of knowledge as of **2026-07-20** and are +Statuses and the gap register reflect the state of knowledge as of **2026-07-30** and are expected to change; everything else in this document is stable methodology. The harness described here exists: [`crates/hotblocks-harness`](../../hotblocks-harness). -Phase 0 of §7 is done — CT-1 runs a happy-path script green against the real binary. +Phase 0 of §7 is done — CT-1 runs a happy-path script green against the real binary, and CT-4 +now includes finality-equivocation regressions for both deep-window and straddling-chunk fallback, +exact rollback-position assertions, and honest-reorg recovery that must converge rather than wedge. ## 1. Harness architecture @@ -95,7 +97,7 @@ model Dataset: replace(from_, B, f=⊥): # WP §2.3 require seg and B and valid_run(B) and B[0].number >= from_ - require fin == ⊥ or from_ > fin.number # INV-13/14 + require fin == ⊥ or from_ > fin.number or reproduces(B, seg, from_, fin.number) # INV-13/14 require from_ >= first() # INV-14 require B[0].parent_hash == hash_at(from_ - 1) # DEF-16 (⊥ accepted at the window edge) seg = [b in seg | b.number < from_] + B; ver += 1; if f: finalize_inline(f) @@ -139,8 +141,11 @@ model Dataset: return RESET((anchor.number, hint hash at that position)) # WP-6b: below-window divergence m = max({x in hints | stored_or_anchor(x)}, default=⊥) if m != ⊥: return (m.number + 1, m.hash) - if fin != ⊥: return (fin.number + 1, fin.hash) # volatile suffix only; repeated - # rejection here = FM-SRC-5, never RESET + if fin != ⊥: return (fin.number + 1, fin.hash) # or any lower position reproducing the + # finalized prefix (INV-14) — a batch-granular + # implementation must, when fin sits inside a + # batch. Repeated rejection here = FM-SRC-5, + # never RESET return (first(), anchor.hash) # full-window replacement *probe*; # repeated WP-2 rejection at first(D) # escalates to RESET per WP-6b @@ -203,7 +208,7 @@ weaken to soundness or it will fail on correct behavior. | CT-1 | **Stateful property tests** | randomized scripts of source events + retention ops + reads; model diff continuously and at quiescence | INV-1..7, 10..18, 20..27, 30/31, 44; WP-*; RP-* | | CT-2 | **Crash-recovery** | kill-point matrix (during batch write, during fork, during trim, during boot, during shutdown) × restart → model diff; repeated-crash convergence | INV-40, 42, 43; CN-6/9/11; LIV-5/6/12; GAP-2 | | CT-3 | **Concurrency** | reader swarms hammering during write/fork/trim/maintenance storms; interleaved HEAD+QUERY sequencing checks | INV-20/21/23/31/41; CN-3/4; LIV-3/4 | -| CT-4 | **Source-fault corpus** | scripted FM-SRC-1..8 scenarios incl. fork storms, deep forks, finality conflicts, equivocation | INV-12/13/14/23/24; WP-6/8; LIV-9; FM-SRC-*; GAP-3/4/5 | +| CT-4 | **Source-fault corpus** | scripted FM-SRC-1..8 scenarios incl. fork storms, deep forks, finality conflicts, equivocation | INV-12/13/14/23/24; WP-6/8; LIV-9; FM-SRC-*; GAP-3/5 | | CT-5 | **Interface conformance** | exhaustive request/response matrix against the binding: error taxonomy, watermark headers, encodings, hash-lookup matrix, boot config matrix | RP-1..16, 19/20; INV-26/43; IB-*; GAP-8/9/39 | | CT-6 | **Performance benchmarks** | reference scenarios S1–S6; SLI capture; SLO gates; saturation knees | SLI-1..12; PF-1..9; LIV-1/3/10; GAP-13 | | CT-7 | **Soak / endurance** | multi-day S4 churn with fault sprinkling; space, memory, stall, residue tracking | LIV-2/7/11; RS-6/10; INV-16/17; HZ-2/5; GAP-1/6 | @@ -227,30 +232,30 @@ INV-21/22/23 (checks in parentheses): 6. anchored continuation across responses never breaks parent-hash chains (INV-23); 7. watermark coherence: `first ≤ fin ≤ head` whenever reported together (INV-5/30). -## 5. Traceability matrix (status @ 2026-07-21) +## 5. Traceability matrix (status @ 2026-07-30) -Legend: **C** covered, **P** partial (some storage-layer or fixture coverage exists; -service-level black-box coverage absent), **U** untested. Rows that changed with Phase 0 name -the test that moved them; unless a row says otherwise, "covered" means *on the happy path* — +Legend: **C** covered, **P** partial (some paths or layers are covered; the full class is not), +**U** untested. Rows that changed name the test that moved them; unless a row says otherwise, +"covered" means *on the happy path* — the same property under forks, crashes and retention is the business of CT-2/CT-4. | Property | CT class | Status | Note | |---|---|---|---| | INV-1..3 structural chain | CT-1 | **C** | `ct1_evm` / `ct1_solana` / `ct1_hyperliquid_fills`: full-window scan, parent linkage + anchor, dense and slot-numbered | | INV-4 kind/schema | CT-1/5 | **C** for evm/solana/hyperliquid-fills | payload round-trips against the emission oracle per kind; bitcoin, tron and hl-replica-cmds unmodeled | -| INV-5/6 watermark bounds/on-chain | CT-1 | **P — known-violated** | happy path C; finality below the window is accepted after a trim clears `fin` (GAP-27); hash below head unverified (GAP-4) | +| INV-5/6 watermark bounds/on-chain | CT-1/4 | **P** | happy path C; the shared finality resolver checks standalone and batch-composed reports against the post-state owner, ignores genuine holes and reports below the logical retention floor, and refuses hash conflicts or a replacement that evicts the block it calls final. Pinned by write-controller regressions; broader CT-4 sequencing remains | | INV-7 provenance | CT-1/6 | **C** | `ct1_happy_path`: what the source served is read back, payload included | | INV-10 atomic transitions | CT-2/3 | U | | | INV-11 append | CT-1 | **C** | | -| INV-12/13 finality monotone/immutable | CT-1/4 | **P — known-violated** | monotone advance observed; composed-finality REPLACE below `fin` admitted (GAP-22); regression/conflict paths await CT-4 | -| INV-14 fork floor | CT-4 | **U — known-violated** | GAP-3; composed-finality bypass (GAP-22) | +| INV-12/13 finality monotone/immutable | CT-1/4 | **P** | monotone advance in CT-1; five `ct4_finality` scenarios + write-controller regressions pin finalized-prefix immutability (whole overlapping range, not just the boundary at `fin`), fixed-height hash immutability, atomic rejection, and convergence after finality advances over an in-flight replay; the ingest regression replaces an already-withheld episode on a second fork. Stale/regressing source reports and sustained fork storms remain | +| INV-14 fork floor | CT-4 | **P — known-violated** | `ct4_finality` asserts the exact source resume position for all-mismatching-hint fallback at both the retained-window floor and a finality-straddling chunk; a storage-level regression classifies a compaction-invalidated physical boundary and proves that deeper re-resolution converges. Below-window RESET and trimmed-anchor cases remain (GAP-3/23) | | INV-15/18 retention trim/anchor | CT-1 | **U — known-violated** | INV-18: trims drop the anchor hash (GAP-23), restart rebuilds it wrong (GAP-2); comparator needs RS-4 slack first (see §7) | | INV-16 frame | CT-1/7 | U | | -| INV-17 maintenance transparency | CT-7 | P | merge-equivalence tested storage-level | +| INV-17 maintenance transparency | CT-7 | P | merge-equivalence and compaction-invalidated rollback recovery tested storage-level | | INV-20 snapshot isolation | CT-3 | P | single-threaded snapshot test only | | INV-21/22 response shape/completeness | CT-1/5/6 | P | structural validators + emission diff under `include_all`; coverage cuts and filtered emission await CT-5; comparator must implement the RP-9 marker exemption (spec change 2026-07-12) | -| INV-23 anchored ancestry | CT-1/4 | P | anchored continuation across responses covered; the CONFLICT path awaits CT-4 | -| INV-24 finalized-only | CT-4 | U | | +| INV-23 anchored ancestry | CT-1/4 | P | anchored continuation across responses, finality-conflict rejection, and honest reorg recovery above finality covered; the broader fork corpus remains | +| INV-24 finalized-only | CT-4 | P | `ct4_finality` pins once-finalized content across source equivocation with a public full-window scan; a dedicated `QUERY-FINALIZED` poller and the broader fork corpus remain | | INV-25 progress | CT-1/6 | P | a successful response must cover ≥ 1 block — asserted by the scanner | | INV-26 error soundness | CT-5 | **P — known-violated** | `ct5_error_soundness`: unsupported dialect containment/accounting and mid-stream worker-panic abort pinned; finalized-snapshot race pinned unit-level. Anchored eval across large holes (GAP-21) reverted; shared-status families keep free-text discrimination (GAP-36/39) | | INV-27 range honesty | CT-1 | **C** | validator: no block outside `[from, min(to, head)]` | @@ -279,7 +284,7 @@ the same property under forks, crashes and retention is the business of CT-2/CT- | RS-6 amplification | CT-7 | U | reclaim path fixed 2026-07 (GAP-6); bound unmeasured under churn | | RS-8 boot maintenance | CT-2/7 | P | unlink/orphan-purge behaviors have storage-level tests | | RS-10/11 residue/deletion cost | CT-7 | P | GAP-6/13 | -| FM-1 robustness | CT-9 | **P — known-violated** | GAP-12 open; unsupported query and query-worker panic classes are closed (§6.1), and the unterminated-record class is pinned by `ct9_source_faults` | +| FM-1 robustness | CT-9 | **P — known-violated** | GAP-12 open; unsupported query and query-worker panic classes are closed (§6.1), while `ct9_source_faults` pins both the unterminated-record and unrepresentable-block-time classes | | FM-SRC-* corpus | CT-4 | **P** | `ct4_lagging_source`: the multi-endpoint shape production runs — several sources per dataset, one far behind — is covered in both the minority and majority laggard shapes, and a *lagging* source is confirmed harmless to the head, independently of its slot in the fixed poll order. A source answering *wrongly* is not: one endpoint of three signalling a fork above its own tip parks ingestion (GAP-5's shape, GAP-41), pinned `#[ignore]`d. Still no strike/quarantine substrate (GAP-30) | | FM-STOR-2/3 disk pressure | CT-7 | U | incident-derived; no automated test | | FM-OP-1..5 | CT-5 | U | | @@ -288,7 +293,7 @@ the same property under forks, crashes and retention is the business of CT-2/CT- | OB-2..11 | all | P | query metrics exist; stall gauges pending on PR #83 (unmerged); OB-2 heartbeat, OB-6 debt accounting, OB-9 alarms, OB-11 forensics absent | | OB-12 index state | CT-1 | **P** | CF-wide estimated keys / live SST bytes are exported for both indexes; per-dataset enabled/count/bytes and lookup hit/miss/latency remain absent (GAP-40) | -## 6. Gap register (dated 2026-07-21, informative) +## 6. Gap register (dated 2026-07-30, informative) Known or strongly suspected divergences between this spec and the current system, from incident history, code-level review, and coverage analysis. Priorities: P0 = active @@ -299,9 +304,8 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po |---|---|---|---|---| | GAP-1 | Whole-service ingest freezes (≈6 min) observed post-deploy; all datasets stall simultaneously; root cause unconfirmed (shared write-path backpressure suspected); no stall observability to attribute it | LIV-2, LIV-8, OB-3/11 | **P0** | CT-7 stall harness: S1 + storage-pressure injection, assert SLI-9 ≤ budget; build OB-11 capture first | | GAP-2 | Recovered anchor hash is reconstructed from the wrong value after restart — `WriteController::new` takes the first batch's *last*-block hash where the correct value sits in its `parent_block_hash` field; latent until the fork-fallback path consumes it, then ingestion resumes with a wrong expected parent (perpetual source-rejection loop) | INV-40, WP-19, LIV-6 | P1 | CT-2: restart, then force full-window fork fallback; assert resume position equals model | -| GAP-3 | Fork floor at the window start is not enforced (marked-as-known in the system); a deeper-than-window divergence may be mishandled instead of becoming an explicit RESET per WP-6b | INV-14, WP-6/6b | P1 | CT-4 deep-fork case: hints strictly below `first(D)` | -| GAP-4 | Finality reports strictly below the head are applied without verifying the hash against the stored block — in both the standalone FINALIZE path and the batch-composed path; the window floor is not checked either (GAP-27) | INV-6, WP §2.4 | P2 | CT-4: finality with corrupted hash below head; assert INTEGRITY_FAULT not acceptance | -| GAP-5 | Unapplicable divergence (fork below finality, finality conflicts) results in silent bounded-pause retry forever — no alarm state, no distinct observable; one such class already caused a crash-loop incident. **Reproduced 2026-07-20** through a second trigger (GAP-41), and reachable from a *single* misbehaving endpoint of three: the head freezes at the position the divergence arrived at and never resumes, while the honest sources keep offering the chain — from the outside, indistinguishable from a hung service | LIV-9b, FM-SRC-5, OB-9 | P1 | `ct4_a_single_source_signalling_a_fork_above_its_tip_does_not_park_ingestion` (`#[ignore]`d); the fork-below-finality script still owes the `P-ALARM` assertion | +| GAP-3 | The write path now refuses a replacement below the physical window, so a stale rollback cannot corrupt retained state. REMAINING: fork resolution still does not classify evidence of a deeper-than-window divergence or execute the explicit alarmed RESET required by WP-6b; it parks the epoch through GAP-5 instead | INV-14, WP-6/6b | P1 | CT-4 deep-fork case: hints strictly below `first(D)`; assert RESET + alarm rather than a refused retry | +| GAP-5 | Unapplicable divergence (fork below finality, finality conflicts) results in bounded-pause retry forever — no alarm state or recovery; one such class already caused a crash-loop incident. The refusal is now attributable through `dataset_epoch_failures{reason="unapplicable_fork",cause=...}`, but that counter is not the required stateful alarm. **Reproduced 2026-07-20** through a second trigger (GAP-41), reachable from a *single* misbehaving endpoint of three: the head freezes at the divergence while honest sources keep offering the chain | LIV-9b, FM-SRC-5, OB-9 | P1 | `ct4_a_single_source_signalling_a_fork_above_its_tip_does_not_park_ingestion` (`#[ignore]`d); the fork-below-finality script still owes the `P-ALARM` assertion | | GAP-6 | ~~Default deployments never reclaim~~ — routine reclaim fixed 2026-07 (PR #79: 10 s point-delete sweep + deletion-collector + periodic-compaction backstop). REMAINING: the interrupted-build residue purge and the whole-file unlink are confined to the gated boot mode (off by default) — a torn build's residue leaks for good in default config and pins the boot-unlink watermark; SLI-8 under churn still unmeasured | RS-10, RS-8 (RS-6 residual) | P2 (was P0) | CT-7: churn soak in default config; assert SLI-8 bound + residue-age bound | | GAP-7 | Serving is gated on full initialization: tens of seconds of refused connections after deploy, scaling with state size and dataset count; readiness not observable per dataset | LIV-5, OB-8 | P1 | CT-6 S5: SLI-5 vs state-size regression curve | | GAP-8 | ~~Zero-emission responses do not convey the coverage end~~ — a de-facto carrier existed all along (the coverage-end block is always emitted, header-only when unmatched) and was adopted as normative RP-9 on 2026-07-12. REMAINING: (a) a zero-emission success now *asserts* full-range coverage, but under time-budget truncation over a blockless range (explicit `to` inside a hole run) the implementation can return an empty 200 having covered only part of it — the client then silently skips the rest; (b) the comparator must implement the INV-22 boundary-marker exemption; (c) in one reachable corner RP-9's clauses are jointly unsatisfiable — an effective range whose covered part contains no stored block and whose coverage cannot legally reach the range end (an availability boundary, RP-8, or a hard `P-QUERY-TIME` stop inside a long hole run): zero emission is legal only at full coverage, and the carrier (highest stored block ≤ `L`) lies below `from`, which INV-27 forbids emitting — here the *spec*, not just the implementation, owes an answer (candidate remedies: an explicit in-band terminal coverage record — a wire change — or forbidding coverage to end inside a hole except at the range end) | RP-9, INV-22, RP-8, INV-27 | P2 | CT-5: hole-range query with explicit `to` + tight budget; assert empty-200 only with full coverage | @@ -311,16 +315,13 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po | GAP-13 | Ingest batch accumulation has content-dependent unbounded memory (no hard byte ceiling on some structures) | PF-1 | P2 | CT-6 adversarial `W-BLOCK-SIZE`/`W-ITEM-DENSITY`; RSS ceiling assertion | | GAP-14 | Read-side capacity (execution slots, waiter slots) is a single global pool: one dataset's query herd can starve all datasets | PF-4, LIV-8 | P2 | CT-8: herd on D′, tip-follower SLOs on D | | GAP-15 | No explicit store-format compatibility gate at boot; incompatibility surfaces as runtime decode errors | CN-12, INV-43 | P3 | CT-5 boot matrix with future-format fixture | -| GAP-16 | ~~The service layer has essentially zero automated tests~~ — **closed by Phase 0**, see §6.1 | all | — | done | | GAP-17 | Shutdown can take a panic-class exit path in ingestion cancellation (observed at redeploy). **2026-07-21 addendum**: replica replacement is client-visible, in two ways that must not be conflated. (a) *Idle pooled connection* — at SIGTERM the server closes keep-alive connections at once while the endpoint removal is still propagating (chart: grace 5 s, no `preStop`, no readiness probe at all), so the portal POSTs into a socket already gone: 502 burst observed one second after SIGTERM with a Ready sibling idle. Nothing was admitted, so LIV-12 does permit it, and masking it belongs outside this spec — portal replay on a fresh connection (`sqd-portal` 13-conformance GAP-22) plus a chart `preStop`. (b) *Active stream cut* — **not** permitted: RP-15 requires the truncation to be "observationally identical to a budget stop", and a process kill leaves an unterminated chunked body and a half-written codec frame, so the client cannot decode the prefix at all, let alone as valid JSONL (IB-5's looser paraphrase is not a licence). A response that has already committed 200 and emitted bytes is unreplayable by any client, so (b) is fixable only here. And (b) is not rare: the drain floor already equals `P-HEAD-WAIT`, because head-waiters run a fixed 5 s timeout with no knowledge of the signal (`query/service.rs`) in violation of LIV-4's explicit shutdown clause — that is the whole 5 s grace, so under tip-follower load the exit is SIGKILL, i.e. FM-PROC-1, and the "clean path" this entry once assumed is not taken in production at all. The drain is also unbounded from the inside: `axum::serve(..).with_graceful_shutdown` carries no deadline and GAP-29 (no response deadline; a stalled reader pins its response) means nothing bounds `P-SHUTDOWN` whatever the grace is. Lastly the RP-15 escape hatch owes a truncation counter that does not exist (GAP-10). **Partly closed 2026-07-21**: SIGTERM now runs the same two-phase sequence as the portal (sqd-portal#113) — `/ready` reports 503 for `--pre-drain-grace-secs` while everything else serves normally, then the drain runs under a hard `--drain-timeout-secs`; SIGINT stays on the default handler so dev Ctrl-C is unaffected. That answers (a) and bounds `P-SHUTDOWN` from the inside, but only takes effect once the chart probes `/ready` and raises the grace above the sum of the two. Still open: (b), since a stream cut at the deadline is still a reset rather than an RP-15 end; LIV-4's shutdown clause, since waiters still sit out their fixed timeout; and the original ingest-cancellation path | LIV-12, LIV-4, RP-15, FM-PROC-4 | **P1** (was P2 on the "bounded or rare" premise; the addendum removes it — it fires on every replica replacement and the clean path is never taken under load) | CT-2 shutdown class: SIGTERM under load ×100 — zero panic exits, exit ≤ `P-SHUTDOWN`, every cut stream decodes as a valid JSONL prefix, every long-poll released ≤ `P-HEAD-WAIT` | | GAP-18 | Dual-writer detection exists only on some paths (finality/head updates), not all mutations | WP-15, FM-OP-3 | P3 | CT-5: two harness-driven writers, assert loser stops on every mutation type | | GAP-20 | `parent_number` linkage is never validated on any layer (the block trait exposes it; nothing reads it): a hash-linked run can claim an arbitrarily higher number for the next block, storing a false hole on a densely-numbered chain — a silent data gap served as if it were a slot gap. (The originally-filed non-monotonic-numbers scenario is unreachable today: the source-position advance forces ascending numbers.) | WP-2, DEF-4, INV-1 | P2 | CT-4/CT-9: hash-linked run with a number jump on a dense chain; the run MUST be rejected with no state change | | GAP-21 | An anchored query whose `from` sits mid-chunk above a number gap larger than the conflict-check lookback (a hard-coded 100 positions in the plan's base-block check) fails `INTERNAL` instead of evaluating the assertion. A >100-position hole with an anchor landing just above it is probably unrealistic, hence low priority. A correct all-predecessors scan was tried and reverted 2026-07-15 — it regressed `check_parent_block` into an unbounded per-chunk scan+sort; needs a lazy sort-desc + limit(100) | RP-11, INV-26 | P3 | CT-5 `ct5_anchor_is_evaluated_across_a_large_number_hole` (`#[ignore]` until fixed): >100-position hole in one chunk; anchored query just above must yield OK/CONFLICT, never 500 | -| GAP-22 | Deep-fork handling can silently replace the finalized prefix: the fork-resolution fallback ignores `fin` (resumes from the window start instead of `⟨fin + 1, fin.hash⟩`), the composed-finality guard admits a REPLACE whose base lies at/below `fin` whenever the pack carries a finality mark ≥ current, and Window trims have dropped the anchor hash (GAP-23) so the replacement attaches unchecked. If the first replacement batch reaches past the old `fin`, the finalized prefix is replaced with no RESET event and no alarm — finalized-only clients observe two hashes at one height (INV-24 broken); otherwise the commit trips the fork-floor check ("can't fork safely") and the epoch parks on the blind 60 s retry loop. Fix: fallback → `fin + 1`; enforce the fork floor at commit unconditionally; carry the anchor hash | WP-6, INV-13/14, INV-24, FM-SRC-5, LIV-9 | **P1** | CT-4: fork with all-mismatching hints on a dataset with `fin` defined; assert REPLACE from `fin + 1` (or alarmed fault) — never a commit whose base ≤ `fin` | -| GAP-23 | Window trims drop the anchor hash: the automatic trim passes no hash and the retained state stores `⊥`, though the correct value sits unused in the first batch's `parent_block_hash`. Disables below-window divergence detection (WP-6b has nothing to contradict) and feeds GAP-22 | INV-18, DEF-7, WP-6b | P1 | CT-1: CONFLICT hints / STATUS at the window edge after a trim; CT-4: below-window fork after a trim must RESET, not absorb silently | +| GAP-23 | Window trims drop the anchor hash: the automatic trim passes no hash and the retained state stores `⊥`, though the correct value sits unused in the first batch's `parent_block_hash`. This disables direct below-window divergence detection because WP-6b has nothing to contradict | INV-18, DEF-7, WP-6b | P1 | CT-1: CONFLICT hints / STATUS at the window edge after a trim; CT-4: below-window fork after a trim must RESET, not absorb silently | | GAP-24 | One dataset's init failure aborts the whole service: startup propagates the first controller error (kind mismatch, retention bail, corrupt state) instead of alarming that dataset and serving the rest | CN-10, FM-OP-1, INV-36, INV-43 | P1 | CT-5 boot matrix: corrupt one dataset's persisted state; assert the others serve and the broken one alarms | | GAP-25 | Downward retention (`from < first(D)`) executes as an *implicit, unobservable* RESET (WP §2.5 as amended 2026-07-12 legalizes the destruction, but requires OB-9 observability) — no event, indistinguishable from a trim; and the boot-time `Pinned` equivalent aborts the entire service (via GAP-24) instead of a dataset-level refusal | WP §2.5, OB-9, INV-43 | P2 | CT-1: SET-RETENTION below `first`; assert a RESET observable + serving continuity; CT-5: boot with lowered `Pinned.from` | -| GAP-27 | FINALIZE never checks `e ≥ first(D)`: with `fin = ⊥` (e.g. after a trim passed above it) a lagging source's report below the window commits `fin < first(D)` | WP §2.4, INV-5 | P2 | CT-4: finality below the window on a trimmed dataset; assert the report is ignored | | GAP-28 | The External retention instruction and the empty-dataset anchor are memory-only (the persisted label holds kind/version/fin) — a restart forgets the instructed bound (the dataset re-idles awaiting a new instruction) and resets an empty dataset's anchor | CN-9, INV-40, WP-11 | P2 | CT-2: SET-RETENTION, restart; assert the bound and anchor are recovered | | GAP-29 | A stalled-but-open connection pins the response's storage snapshot indefinitely (no overall response deadline / server write timeout; only disconnects release it); pinned snapshots block physical reclaim of point-deleted data | CN-7, RP-18, HZ-9, RS-6 | P2 | CT-3/CT-7: zombie client that stops reading; assert snapshot lifetime ≤ `P-QUERY-TIME` and reclaim proceeds | | GAP-30 | No strike counting exists anywhere: a linkage-mismatch rejection resets the source session and re-requests in a zero-backoff hot loop; sources are never quarantined; cross-source equivocation never alarms. `P-SOURCE-STRIKES` has no substrate, so the WP-6b/FM-SRC-5 escalation rules are currently unimplementable. **Slowness is likewise unnameable**: an endpoint leaves the rotation only by erroring (`Endpoint::on_error` → `Backoff`; `is_active` knows no other reason), so an endpoint minutes behind the tip keeps full standing and is streamed and parsed on every cycle, its blocks discarded below the shared cursor. Selection is per *block*, not per source — the first arrival that links onto the cursor wins, ties going to config order — so there is no convergence to a fast source and no memory of who wins. Confirmed harmless to the head on its own — in the minority shape production runs (`ct4_a_lagging_source_does_not_hold_the_head_back`), in the majority shape (`ct4_two_lagging_sources_of_three_do_not_hold_the_head_back`), and independently of the laggard's slot in the fixed poll order, since a laggard sits below the shared cursor and so has nothing to serve. The cost is ingest work paid for every endpoint regardless of standing, and the only lever is removing a source by hand | FM-SRC-3/4/6, WP-2 | P2 | CT-4: source serving a permanently mis-linked run; assert bounded request rate, quarantine after N strikes, alarm. Lag-aware demotion needs per-source observability first — PR #81 (`hotblocks_ingest_source_errors_total`) is the substrate and has been in draft since 2026-07-08 | @@ -333,6 +334,7 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po | GAP-39 | A hash-lookup miss and an unknown dataset are both 404 with only a free-text body between them, and IB-7 forbids keying on text. This is worse than the GAP-36 family it belongs to: RP-19 makes "this hash is not indexed" a *deliberately uninformative* answer, so a client that cannot separate it from "this dataset does not exist" cannot tell a misconfiguration from a legitimate miss at all | INV-26, IB-7, RP-19 | P3 | CT-5: unknown dataset vs unknown hash; assert a structured discriminant | | GAP-40 | Hash-index CFs export only engine-wide estimated keys and live SST bytes. OB-12 still lacks per-dataset enabled state / entry count / bytes and hit-vs-miss lookup counts with latency. Because a miss is uninformative by design, an index empty for a structural reason — enabled after the window had filled, wrong kind — remains indistinguishable from one receiving only unknown hashes | OB-12, OB-6 | P3 | CT-1: scrape per-dataset state and exercise hit/miss counters once exported | | GAP-41 | Fork consensus is credulous: `StandardDataSource::poll_next_event` counts fork-signalling endpoints without asking whether a signalling endpoint has any standing at the contested position, and `extract_fork` then adopts the *longest* hint chain among them. RP-5b confines a legitimate signal to `from == tip + 1`, so an in-spec source can only signal where it is; but FM-1 requires surviving one that is not, and a source signalling above its tip is what a source merely *behind* would look like if it answered wrongly. **One** such endpoint out of three suffices, and not by majority: `forks > endpoints.len() / 2` is false at 1-of-3, but the 2 s `fork_consensus_timeout` fires on any poll where every endpoint returned `Pending`, and `accept_new_block` clears that timer only on a commit — so on a chain with block time ≥ 2 s *every inter-block gap is a firing window*. Measured 2026-07-20, 5/5 runs, by instrumenting `poll_next_event`: `forks=1 endpoints=3 active=3 majority=false all_active=false timeout=true`; the adopted hints end at the liar's stale tip, `compute_rollback` rejects them as below `fin`, and ingestion parks per GAP-5 with two healthy sources still offering the chain. The endpoints' own position is known (`Endpoint::last_committed_block`) and unused | FM-1, WP-6, FM-SRC-4/5 | **P1** (was P2 on the premise that it took a majority — measurement overturned it: a lone bad source is enough, on the ordinary inter-block gap rather than a rare coincidence) | `ct4_a_single_source_signalling_a_fork_above_its_tip_does_not_park_ingestion`; the majority path is pinned separately by `ct4_a_fork_signal_majority_above_the_tip_does_not_park_ingestion` so a fix closing only that clause cannot read as green. Both `#[ignore]`d | +| GAP-43 | A withheld replay is bounded by nothing: it ends when the source finally delivers up to the floor, or not at all. The per-episode warning and counter also only fire where a flush is attempted — a source that simply stops below the floor never gets there, since `MaybeOnHead` is suppressed while the position sits below the highest finality any endpoint reported. The *pending floor* itself is now level-readable per dataset (`ingest_flush_floor`, set at fork resolution rather than at a flush attempt), so the shape is attributable; what is missing is a cap that ends the episode loudly on its own | LIV-2, OB-9, OB-11 | P2 | CT-4: replay below `fin` from a source that stops short; assert an alarmed state within `P-ALARM` independent of the flush triggers | ### 6.1 Closed @@ -341,7 +343,6 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po | GAP-11 | Unsupported `substrate` / `fuel` queries and query-worker panics escaped the HTTP error taxonomy by panicking their request task | Typed `UNSUPPORTED_QUERY` admission errors, panic containment in the query executor, CT-5 dialect requests, and an executor unit test (2026-07-15) | | GAP-16 | No service-level automated tests | [`crates/hotblocks-harness`](../../hotblocks-harness) + `ct1_happy_path` (Phase 0, 2026-07-12) | | GAP-19 | A source response whose final JSONL record carried no trailing newline panicked the line reader (`LineStream::take_final_line` left its scan position past the emptied buffer). The ingest task died, its buffered batch was lost, and the dataset parked for `P-EPOCH-RETRY` — then crash-looped, since the source served the same body on retry. Violated FM-1, LIV-2 | Found by CT-1 on the harness's first run; fixed in `crates/data-client/src/reqwest/lines.rs`; pinned by a unit test there and by `ct9_source_faults` (2026-07-12) | -| GAP-26 | A block timestamp outside the datetime-conversion range killed the ingest flush — the conversion existed only for a log line | Log formatting is best-effort and the raw value is stored unchanged; evm/solana seconds→millis saturate so an absurd source value neither panics nor wraps (PR #100, 2026-07-20). No regression test — CT-9: serve a block with `time = i64::MAX`; assert the batch commits and serving continues | | GAP-32 | A finalized-head trim/reset race could turn an admitted finalized query into `INTERNAL` when its snapshot no longer had a finalized head | Snapshot-time absence now maps to `NO_DATA`; pinned by `finalized_snapshot_without_a_head_is_no_data` (2026-07-15) | | GAP-38 | `TX-BY-HASH` / `tidx` absent; fork re-inclusion ordering unimplemented | Transaction hash CF + independent flag + HTTP binding; storage transition suite and black-box ingest/reorg/re-inclusion tests (2026-07-15) | @@ -353,13 +354,12 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po Delivered as [`crates/hotblocks-harness`](../../hotblocks-harness), driving the real binary as a child process over the binding of 13. Its README records the design decisions the - next phases must not undo. Three things the later phases need are built but unexercised, and - three are missing: + next phases must not undo. The current harness support and remaining corpus are: - | Built, awaiting scripts | Missing | + | Harness support | Remaining | |---|---| | `Sut::crash/stop/restart` (same db, same port); `ct2_shutdown` exercises the bounded SIGTERM path | the remaining CT-2 kill-point matrix | - | `Harness::fork` + `Model::resolve_fork` + the follower's CONFLICT recovery → CT-4 | the CT-4 fork/finality corpus | + | `Harness::fork`, finalized-prefix and below-finality equivocation faults, `Model::resolve_fork` and follower CONFLICT recovery → CT-4 | below-window RESET, malformed finality, fork-storm and alarm cases remain | | `Model::predict_query` + initial `ct5_error_soundness` matrix | remaining CT-5 binding, boot, and overload rows | | `SimFaults` injection point → CT-9 | the rest of the FM-SRC repertoire | @@ -372,7 +372,7 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po accounting via OB-6 (GAP-6). These target the two production incidents. - **Phase 2 — correctness core.** CT-2 crash matrix (GAP-2) and shutdown class (GAP-17 — pulled forward from Phase 3: it is client-visible on every replica replacement), CT-4 - fork/finality corpus (GAP-3/4/5), CT-5 remaining error taxonomy + boot matrix + fork/finality corpus (GAP-3/5), CT-5 remaining error taxonomy + boot matrix (GAP-8/9/15/36/39). - **Phase 3 — robustness.** CT-9 fuzz both surfaces (GAP-12), CT-3 concurrency swarms, CT-8 isolation (GAP-14). diff --git a/crates/hotblocks/src/cli.rs b/crates/hotblocks/src/cli.rs index 8fcc1abf..ed844e8e 100644 --- a/crates/hotblocks/src/cli.rs +++ b/crates/hotblocks/src/cli.rs @@ -81,8 +81,8 @@ pub struct CLI { /// `CF_TABLES` memtable size. RocksDB's 64 MB default flushes small L0 files /// continuously; the resulting compaction churn is most of the device write /// bill, and writes stop whenever the flush of one buffer has not finished - /// before the next fills (measured: `immutable_memtables` pegged at the - /// ceiling on every stalled pod). + /// before the next fills (measured: `immutable_memtables` pegged at its + /// ceiling wherever writes stopped). #[arg(long, value_name = "MB", default_value = "64")] pub rocksdb_write_buffer_mb: usize, @@ -95,8 +95,8 @@ pub struct CLI { /// multiplier of 10, this decides how many levels deep the ladder runs, and /// a byte is rewritten once per level it descends. The default 256 MB is /// smaller than one 512 MB memtable's flush, so L0 currently arrives larger - /// than the level it merges into. Validate boot time and compaction behavior - /// before raising it. + /// than the level it merges into. Unmeasured at full data size; validate boot + /// time and compaction behavior before raising it. #[arg(long, value_name = "MB", default_value = "256")] pub rocksdb_level_base_mb: usize, diff --git a/crates/hotblocks/src/dataset_controller/dataset_controller.rs b/crates/hotblocks/src/dataset_controller/dataset_controller.rs index e5d14774..35cba839 100644 --- a/crates/hotblocks/src/dataset_controller/dataset_controller.rs +++ b/crates/hotblocks/src/dataset_controller/dataset_controller.rs @@ -9,7 +9,12 @@ use tokio::{select, task::JoinHandle, time::Instant}; use tracing::{Instrument, debug, error, info, info_span, instrument, warn}; use crate::{ - dataset_controller::{ingest::ingest, ingest_generic::IngestMessage, write_controller::WriteController}, + dataset_controller::{ + ingest::ingest, + ingest_generic::IngestMessage, + write_controller::{FlushFloorUpdate, WriteController} + }, + metrics::report_flush_floor, types::{DBRef, DatasetKind, RetentionStrategy} }; @@ -191,13 +196,15 @@ enum State { } struct IngestHandle { + dataset_id: DatasetId, msg_recv: tokio::sync::mpsc::Receiver, task: JoinHandle> } impl Drop for IngestHandle { fn drop(&mut self) { - self.task.abort() + self.task.abort(); + report_flush_floor(self.dataset_id, None); } } @@ -341,9 +348,12 @@ impl Ctl { msg = handle.msg_recv.recv() => { if let Some(msg) = msg { let head = *head; - blocking! { + let flush_floor_update = blocking! { write.handle_ingest_msg(msg, head) }?; + if let FlushFloorUpdate::Set(floor) = flush_floor_update { + report_flush_floor(self.dataset_id, floor); + } } else { // ingest task must have failed match (&mut handle.task).await { @@ -410,11 +420,13 @@ impl Ctl { match retention { RetentionStrategy::FromBlock { number, parent_hash } => { let (number, parent_hash) = self.clamp_floor(&write, number, parent_hash); - let will_erase_head = write.head().map_or(false, |h| h.number < number) || // FromBlock is greater than current head, so everything is cleared - write.start_block() > number; // FromBlock is less than current front, dropping everything by design - blocking_write!(write, write.retain(number, parent_hash))?; + let kept_head = blocking_write!(write, write.retain(number, parent_hash))?; + // A surviving head keeps the ingest alive: the floor moves often enough that + // restarting on every trim would reconnect the source for nothing. Its in-flight + // rollback can still aim below the trimmed window — refused at commit instead + // (`new_chunk`). match state { - State::Ingest { .. } if !will_erase_head => {} // Keep ingesting, head is valid + State::Ingest { .. } if kept_head => {} // Keep ingesting, head is valid _ => *state = State::Init { head: self.max_blocks } // New ingest needed } } @@ -431,6 +443,10 @@ impl Ctl { fn spawn_ingest(&self, write: &WriteController) -> IngestHandle { let (msg_sender, msg_recv) = tokio::sync::mpsc::channel(1); + // The controller is the single owner of this per-dataset gauge. Clear any completed + // episode before its replacement can publish a new rollback floor. + report_flush_floor(self.dataset_id, None); + let ingest_span = info_span!("ingest"); let task = tokio::spawn( @@ -446,7 +462,11 @@ impl Ctl { .instrument(ingest_span) ); - IngestHandle { msg_recv, task } + IngestHandle { + dataset_id: self.dataset_id, + msg_recv, + task + } } async fn new_write(&self, maybe_write: Option) -> anyhow::Result { @@ -604,3 +624,36 @@ async fn compaction_loop(db: DBRef, dataset_id: DatasetId, mut enabled: tokio::s } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::metrics::build_metrics_registry; + + fn flush_floor_line(dataset: &str) -> String { + let mut output = String::new(); + prometheus_client::encoding::text::encode(&mut output, &build_metrics_registry()).unwrap(); + output + .lines() + .find(|line| line.starts_with("hotblocks_ingest_flush_floor") && line.contains(dataset)) + .unwrap_or_else(|| panic!("no flush floor series for {dataset}:\n{output}")) + .to_string() + } + + #[tokio::test] + async fn dropping_ingest_handle_clears_a_pending_flush_floor() { + let dataset_id = DatasetId::from_str("ingest-handle-drop-test"); + report_flush_floor(dataset_id, Some(1234)); + assert!(flush_floor_line("ingest-handle-drop-test").ends_with(" 1234")); + + let (_msg_sender, msg_recv) = tokio::sync::mpsc::channel(1); + let task = tokio::spawn(std::future::pending::>()); + drop(IngestHandle { + dataset_id, + msg_recv, + task + }); + + assert!(flush_floor_line("ingest-handle-drop-test").ends_with(" -1")); + } +} diff --git a/crates/hotblocks/src/dataset_controller/ingest_generic.rs b/crates/hotblocks/src/dataset_controller/ingest_generic.rs index c98b5e59..c1791329 100644 --- a/crates/hotblocks/src/dataset_controller/ingest_generic.rs +++ b/crates/hotblocks/src/dataset_controller/ingest_generic.rs @@ -10,11 +10,12 @@ use sqd_data_core::{BlockChunkBuilder, ChunkProcessor, PreparedChunk}; use sqd_data_source::{DataEvent, DataSource}; use sqd_primitives::{Block, BlockNumber, BlockRef, DataMask, DisplayBlockRefOption}; use sqd_storage::db::DatasetId; -use tracing::{debug, field::valuable, info}; +use tracing::{debug, field::valuable, info, warn}; use crate::{ dataset_controller::write_controller::Rollback, - metrics::{WriteStage, report_write_duration} + errors::DataAvailabilityChangedDuringFinalizedReplay, + metrics::{WriteStage, report_withheld_flush, report_write_duration} }; pub enum IngestMessage { @@ -121,7 +122,11 @@ pub struct IngestGeneric { last_block: BlockNumber, last_block_hash: String, last_block_time: Option, - data_mask: DataMask + data_mask: DataMask, + /// No chunk may be emitted before it covers this block ([`Rollback::reach_at_least`]). + flush_floor: Option, + /// Whether the current withholding episode has already been warned about and counted. + withholding: bool } impl IngestGeneric @@ -150,7 +155,9 @@ where last_block: 0, last_block_hash: String::new(), last_block_time: None, - data_mask: DataMask::default() + data_mask: DataMask::default(), + flush_floor: None, + withholding: false } } @@ -167,6 +174,13 @@ where let data_mask = block.data_availability_mask(); if self.data_mask != data_mask { self.flush().await?; + // Masks can't share a chunk, and the flush floor can't be waived. + ensure!( + self.buffered_blocks == 0, + DataAvailabilityChangedDuringFinalizedReplay { + block_number: block.number() + } + ); self.data_mask = data_mask } self.push_block(block, is_final)?; @@ -180,7 +194,11 @@ where } async fn handle_fork(&mut self, prev_blocks: Vec) -> anyhow::Result<()> { - info!(upstream_blocks = valuable(&prev_blocks), "fork received"); + info!( + stream_from = self.first_block, + upstream_blocks = valuable(&prev_blocks), + "fork received" + ); let (rollback_sender, rollback_recv) = tokio::sync::oneshot::channel(); @@ -195,16 +213,19 @@ where let rollback = rollback_recv.await?; info!( - block_number = rollback.first_block, - parent_block_hash =? rollback.parent_block_hash, + resume_from = rollback.resume_from, + expected_parent_hash =? rollback.expected_parent_hash, + reach_at_least =? rollback.reach_at_least, "resetting ingest position" ); self.buffered_blocks = 0; self.finalized_head = None; - self.first_block = rollback.first_block; + self.first_block = rollback.resume_from; + self.flush_floor = rollback.reach_at_least; + self.withholding = false; self.data_source - .set_position(rollback.first_block, rollback.parent_block_hash.as_deref()); + .set_position(rollback.resume_from, rollback.expected_parent_hash.as_deref()); Ok(()) } @@ -236,8 +257,10 @@ where async fn maybe_flush(&mut self) -> anyhow::Result<()> { if self.builder_ref().num_rows() > 200_000 { - return self.flush().await; + self.flush().await?; } + // Not `else`: a withheld flush leaves every row in place, so `num_rows` stays above the + // bound and the replay would grow in memory until it reached the floor. if self.builder_ref().in_memory_buffered_bytes() > self.builder_ref().spill_bound_bytes { return self.with_blocking_builder(|b| b.flush_to_processor()).await; } @@ -249,6 +272,26 @@ where return Ok(()); } + // Emitting here would swap the finalized block out of storage; keep buffering. Warned once + // per episode, since the row bound is re-crossed by every subsequent block — and only where + // a flush is attempted, so `ingest_flush_floor` carries the shape where none ever is. + // FUTURE: nothing caps how long a replay may withhold (GAP-43). + if let Some(floor) = self.flush_floor.filter(|floor| self.last_block < *floor) { + if !self.withholding { + self.withholding = true; + warn!( + buffered_blocks = self.buffered_blocks, + last_block = self.last_block, + reach_at_least = floor, + "withholding a chunk until the replay reaches the finalized head" + ); + report_withheld_flush(self.dataset_id); + } + return Ok(()); + } + self.flush_floor = None; + self.withholding = false; + let parent_block_hash = self.parent_block_hash.clone(); let first_block = self.first_block; let last_block = self.last_block; @@ -332,10 +375,46 @@ where #[cfg(test)] mod tests { + use std::{ + pin::Pin, + task::{Context, Poll} + }; + + use futures::Stream; use sqd_data::hyperliquid_fills::{model::Block, tables::HyperliquidFillsChunkBuilder}; use super::*; + struct TestSource { + next_block: BlockNumber, + parent_block_hash: Option + } + + impl Stream for TestSource { + type Item = DataEvent; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } + } + + impl DataSource for TestSource { + type Block = Block; + + fn set_position(&mut self, next_block: BlockNumber, parent_block_hash: Option<&str>) { + self.next_block = next_block; + self.parent_block_hash = parent_block_hash.map(str::to_string); + } + + fn get_next_block(&self) -> BlockNumber { + self.next_block + } + + fn get_parent_block_hash(&self) -> Option<&str> { + self.parent_block_hash.as_deref() + } + } + fn fills_block() -> Block { serde_json::from_value(serde_json::json!({ "header": { @@ -397,4 +476,82 @@ mod tests { let (_, table) = chunk.pop_first().unwrap(); assert!(table.into_processor().is_err(), "expected the in-memory path"); } + + #[tokio::test] + async fn a_second_fork_replaces_a_withheld_replay_episode() -> anyhow::Result<()> { + let (message_sender, mut message_receiver) = tokio::sync::mpsc::channel(1); + let source = TestSource { + next_block: 10_000, + parent_block_hash: Some("initial-parent".to_string()) + }; + let mut ingest = IngestGeneric::new( + DatasetId::from_str("second-fork-while-withholding"), + source, + HyperliquidFillsChunkBuilder::new(), + message_sender, + DEFAULT_SPILL_BOUND_BYTES + ); + + let first_reply = async { + let Some(IngestMessage::Fork { rollback_sender, .. }) = message_receiver.recv().await else { + panic!("expected the first fork request") + }; + rollback_sender + .send(Rollback { + resume_from: 10_000, + expected_parent_hash: Some("first-parent".to_string()), + reach_at_least: Some(10_001) + }) + .expect("the first fork handler must still be waiting"); + }; + let (first_result, ()) = tokio::join!( + ingest.handle_fork(vec![BlockRef { + number: 10_000, + hash: "first-hint".to_string() + }]), + first_reply + ); + first_result?; + + ingest.push_block(fills_block(), true)?; + ingest.flush().await?; + assert!(ingest.withholding, "the first replay must be withheld below its floor"); + assert_eq!(ingest.buffered_blocks, 1); + assert!(ingest.builder_ref().num_rows() > 0); + + let second_reply = async { + let Some(IngestMessage::Fork { rollback_sender, .. }) = message_receiver.recv().await else { + panic!("expected the second fork request") + }; + rollback_sender + .send(Rollback { + resume_from: 9_990, + expected_parent_hash: Some("second-parent".to_string()), + reach_at_least: Some(10_005) + }) + .expect("the second fork handler must still be waiting"); + }; + let (second_result, ()) = tokio::join!( + ingest.handle_fork(vec![BlockRef { + number: 9_999, + hash: "second-hint".to_string() + }]), + second_reply + ); + second_result?; + + assert_eq!(ingest.buffered_blocks, 0, "the superseded replay must be discarded"); + assert_eq!( + ingest.builder_ref().num_rows(), + 0, + "the superseded builder must be cleared" + ); + assert_eq!(ingest.finalized_head, None); + assert_eq!(ingest.first_block, 9_990); + assert_eq!(ingest.flush_floor, Some(10_005)); + assert!(!ingest.withholding, "the replacement replay starts a fresh episode"); + assert_eq!(ingest.data_source.get_next_block(), 9_990); + assert_eq!(ingest.data_source.get_parent_block_hash(), Some("second-parent")); + Ok(()) + } } diff --git a/crates/hotblocks/src/dataset_controller/write_controller.rs b/crates/hotblocks/src/dataset_controller/write_controller.rs index 44ce65ed..dfb4e9c6 100644 --- a/crates/hotblocks/src/dataset_controller/write_controller.rs +++ b/crates/hotblocks/src/dataset_controller/write_controller.rs @@ -8,15 +8,31 @@ use tracing::{debug, field::valuable, info, instrument, warn}; use crate::{ dataset_controller::ingest_generic::{IngestMessage, NewChunk}, - errors::UnapplicableFork, + errors::{UnapplicableFork, UnapplicableForkReason as ForkReason}, metrics::{WriteStage, report_hash_index_write_metrics, report_write_duration}, types::{DBRef, DatasetKind} }; +/// Source position selected after resolving a fork against stored history. +/// +/// `resume_from` lands on a stored chunk boundary, so it may sit at or below `fin` when the +/// common ancestor is inside a finality-straddling chunk; the finalized prefix is guarded on the +/// write path instead ([`WriteController::new_chunk`]). #[derive(Debug)] pub struct Rollback { - pub first_block: BlockNumber, - pub parent_block_hash: Option + /// Lowest block number the source may return after resolving the fork. + pub resume_from: BlockNumber, + /// Hash that must anchor the first returned block, when an anchor is known. + pub expected_parent_hash: Option, + /// Lowest block the first replayed chunk must reach, set when `resume_from` sits at or below + /// `fin`. The swap deletes the chunk holding the finalized block; a chunk stopping short never + /// puts it back, and the retry repeats the identical cut forever. + pub reach_at_least: Option +} + +pub(super) enum FlushFloorUpdate { + Unchanged, + Set(Option) } /// Single writer for a dataset. Owns head/finalized-head as its working copy of @@ -137,59 +153,68 @@ impl WriteController { .get_label(self.dataset_id)? .ok_or_else(|| anyhow!("dataset {} no longer exists", self.dataset_id))?; - if let Some(finalized_head) = label.finalized_head() { + let finalized_head = label.finalized_head().cloned(); + + if let Some(finalized_head) = finalized_head.as_ref() { let pos = match prev.iter().position(|b| b.number >= finalized_head.number) { Some(pos) => pos, None => bail!(UnapplicableFork { - reason: "all passed prev blocks lie below finalized head" + reason: ForkReason::HintsBelowFinalizedHead }) }; if prev[pos].number == finalized_head.number { ensure!( prev[pos].hash == finalized_head.hash, UnapplicableFork { - reason: "fork disagrees with the finalized head hash" + reason: ForkReason::HintConflictsWithFinalizedHead } ); } prev = &prev[pos..] } - let existing_chunks = snapshot - .list_chunks(self.dataset_id, 0, Some(prev.last().unwrap().number)) - .into_reversed(); + let (resume_from, expected_parent_hash) = 'resume: { + let existing_chunks = snapshot + .list_chunks(self.dataset_id, 0, Some(prev.last().unwrap().number)) + .into_reversed(); - let mut prev_blocks = prev.iter().rev().peekable(); + let mut prev_blocks = prev.iter().rev().peekable(); - for chunk_result in existing_chunks { - let head = chunk_result?; + for chunk_result in existing_chunks { + let head = chunk_result?; - if prev_blocks.peek().map_or(false, |b| b.number < head.last_block()) { - continue; - } + if prev_blocks.peek().map_or(false, |b| b.number < head.last_block()) { + continue; + } - while prev_blocks.peek().map_or(false, |b| b.number > head.last_block()) { - prev_blocks.next(); - } + while prev_blocks.peek().map_or(false, |b| b.number > head.last_block()) { + prev_blocks.next(); + } - if let Some(&b) = prev_blocks.peek() { - if b.number == head.last_block() && b.hash == head.last_block_hash() { - return Ok(Rollback { - first_block: b.number + 1, - parent_block_hash: Some(b.hash.clone()) - }); + if let Some(&b) = prev_blocks.peek() { + if b.number == head.last_block() && b.hash == head.last_block_hash() { + break 'resume (b.number + 1, Some(b.hash.clone())); + } + } else { + break 'resume (head.last_block() + 1, Some(head.last_block_hash().to_string())); } - } else { - return Ok(Rollback { - first_block: head.last_block() + 1, - parent_block_hash: Some(head.last_block_hash().to_string()) - }); } - } + + // Retention trims whole chunks, so `self.first_block` can sit inside the surviving + // one — a position `insert_fork` refuses as overlapping. Fall back to the physical + // start of the window instead. + match snapshot.get_first_chunk(self.dataset_id)? { + Some(chunk) => (chunk.first_block(), Some(chunk.parent_block_hash().to_string())), + None => (self.first_block, self.parent_block_hash.clone()) + } + }; Ok(Rollback { - first_block: self.first_block, - parent_block_hash: self.parent_block_hash.clone() + resume_from, + expected_parent_hash, + reach_at_least: finalized_head + .filter(|fin| resume_from <= fin.number) + .map(|fin| fin.number) }) } @@ -200,7 +225,13 @@ impl WriteController { parent_block_hash: Option, delete_mismatch: bool, metrics: &mut HashIndexWriteMetrics - ) -> anyhow::Result<()> { + ) -> anyhow::Result { + // Nothing left to delete below a floor that is already there, and `Api` repeats the same + // floor often. Answering from the window's emptiness would restart an untouched ingest. + if self.starts_at(from_block, &parent_block_hash) { + return Ok(true); + } + #[derive(Eq, PartialEq)] enum Status { Range { @@ -282,7 +313,7 @@ impl WriteController { Ok(status) })?; - match status { + let kept_head = match status { Status::Range { first_chunk, head, @@ -296,27 +327,31 @@ impl WriteController { first_chunk.first_block(), head.last_block() ); + true } Status::HashMismatch => { self.clear_heads(); - warn!("cleared dataset due to parent block hash mismatch") + warn!("cleared dataset due to parent block hash mismatch"); + false } Status::Gap(existed) => { self.clear_heads(); warn!( "cleared dataset, because there was a gap between first requested block {} and already existed {}", from_block, existed - ) + ); + false } Status::Clear => { self.clear_heads(); - info!("dataset was cleared") + info!("dataset was cleared"); + false } - } + }; self.first_block = from_block; self.parent_block_hash = parent_block_hash; - Ok(()) + Ok(kept_head) } fn clear_heads(&mut self) { @@ -325,14 +360,20 @@ impl WriteController { self.first_chunk_head = None; } - pub fn retain(&mut self, from_block: BlockNumber, parent_block_hash: Option) -> anyhow::Result<()> { + /// `false` when this call cleared the window instead of trimming it: the head a live ingest is + /// building on is gone and it must restart. A call that changes nothing reports `true`. + pub fn retain(&mut self, from_block: BlockNumber, parent_block_hash: Option) -> anyhow::Result { let dataset_id = self.dataset_id; observe_storage_write(dataset_id, WriteStage::Retention, |metrics| { self._retain(from_block, parent_block_hash, true, metrics) }) } - pub fn init_retention(&mut self, from_block: BlockNumber, parent_block_hash: Option) -> anyhow::Result<()> { + pub fn init_retention( + &mut self, + from_block: BlockNumber, + parent_block_hash: Option + ) -> anyhow::Result { let dataset_id = self.dataset_id; observe_storage_write(dataset_id, WriteStage::Retention, |metrics| { self._retain(from_block, parent_block_hash, false, metrics) @@ -345,6 +386,7 @@ impl WriteController { ))] pub fn finalize(&mut self, new_finalized_head: &BlockRef) -> anyhow::Result<()> { let Some(head) = self.head.as_ref() else { return Ok(()) }; + let logical_floor = self.first_block; let update = self.db.update_dataset(self.dataset_id, |tx| { ensure!( @@ -352,46 +394,35 @@ impl WriteController { "seems like the dataset is controlled by multiple processes" ); - if let Some(current) = tx.label().finalized_head() { - if current.number > new_finalized_head.number { - return Ok(None); - } - if current.number == new_finalized_head.number { - ensure!(current.hash == new_finalized_head.hash); - return Ok(None); - } - } - let maybe_head_chunk = tx.list_chunks(0, None).into_reversed().next().transpose()?; - let head_chunk = match maybe_head_chunk { + let _stored_head = match maybe_head_chunk { Some(c) if c.last_block_hash() == head.hash => c, _ => bail!("seems like the dataset is controlled by multiple processes") }; - let new_finalized_head = if new_finalized_head.number > head_chunk.last_block() { - get_chunk_head(&head_chunk) - } else if new_finalized_head.number == head_chunk.last_block() { - ensure!(new_finalized_head.hash == head_chunk.last_block_hash()); - new_finalized_head.clone() - } else { - new_finalized_head.clone() - }; - - tx.set_finalized_head(new_finalized_head.clone()); - - Ok(Some(new_finalized_head)) + match resolve_finality(tx, tx.label().finalized_head(), None, logical_floor, new_finalized_head)? { + FinalityDecision::Applied(new_head) => { + tx.set_finalized_head(new_head.clone()); + Ok(CommittedFinalityUpdate::Applied(new_head)) + } + FinalityDecision::Ignored(reason) => Ok(CommittedFinalityUpdate::Ignored(reason)), + FinalityDecision::IntegrityFault { reason, detail } => Err(unapplicable_fork(reason, detail)) + } })?; - if let Some(new_head) = update { - debug!( - block_number = new_head.number, - block_hash = new_head.hash, - "saved new finalized head" - ); - self.set_finalized_head(Some(new_head)); - } else { - debug!("finalized head was ignored") + match update { + CommittedFinalityUpdate::Applied(new_head) => { + debug!( + block_number = new_head.number, + block_hash = new_head.hash, + "saved new finalized head" + ); + self.set_finalized_head(Some(new_head)); + } + CommittedFinalityUpdate::Ignored(reason) => { + debug!(reason = %reason, "finalized head was ignored") + } } Ok(()) @@ -406,33 +437,125 @@ impl WriteController { pub fn new_chunk(&mut self, finalized_head: Option<&BlockRef>, chunk: &StorageChunk) -> anyhow::Result<()> { // FIXME: accept self.first_block rollback limit let dataset_id = self.dataset_id; - let finalized_head = observe_storage_write(dataset_id, WriteStage::Commit, |metrics| { + let logical_floor = self.first_block; + let commit = observe_storage_write(dataset_id, WriteStage::Commit, |metrics| { self.db .update_dataset_with_hash_index_metrics(dataset_id, metrics, |tx| { - let new_finalized_head = match (finalized_head, tx.label().finalized_head()) { - (Some(new), None) => Some(new), - (Some(new), Some(current)) if new.number >= current.number => Some(new), - (_, Some(current)) if current.number < chunk.first_block() => Some(current), - (_, Some(_)) => bail!( - "can't fork safely, because fork base is below the current finalized head \ - and finalized head of the data pack is below the current" - ), - (None, None) => None + // A rollback resolved before a trim can aim below the window that survived it; + // `insert_fork` would then drop every surviving chunk and leave the head under + // the floor. The bound is the first *stored* chunk where there is one: + // retention trims whole chunks, so the logical floor can sit inside the + // surviving one, which a fork legitimately resumes at. With nothing stored + // there is no such chunk, and `compute_rollback` falls back to the logical + // floor too. + let window_start = match tx.list_chunks(0, None).next().transpose()? { + Some(chunk) => chunk.first_block(), + None => logical_floor }; + ensure!( + chunk.first_block() >= window_start, + unapplicable_fork( + ForkReason::BelowRetainedWindow, + format!( + "chunk {}-{} starts below window start {}", + chunk.first_block(), + chunk.last_block(), + window_start + ) + ) + ); + + // Compaction may merge across a boundary after `compute_rollback` selected it. + // Starting inside that merged chunk is a stale plan, not a storage failure: + // leave the old state intact and let the next epoch resolve against the new + // layout. FUTURE: feed this verdict back to the live ingest and re-resolve + // immediately instead of paying the one-minute epoch restart. + let boundary_owner = tx + .list_chunks(chunk.first_block(), Some(chunk.first_block())) + .next() + .transpose()?; + if let Some(existing) = boundary_owner + && existing.first_block() < chunk.first_block() + && chunk.first_block() <= existing.last_block() + { + return Err(unapplicable_fork( + ForkReason::StaleRollbackBoundary, + format!( + "chunk {}-{} starts inside compacted chunk {}-{}", + chunk.first_block(), + chunk.last_block(), + existing.first_block(), + existing.last_block() + ) + )); + } - let new_finalized_head = new_finalized_head.map(|head| { - if head.number < chunk.last_block() { - head.clone() - } else { - get_chunk_head(&chunk) + let current_finalized_head = tx.label().finalized_head().cloned(); + + // A fork resuming at a chunk boundary can reach below `fin`. Admit it only if + // it reproduces that region exactly — spanning `fin`, matching every stored + // hash up to it. `fin`'s own hash alone proves nothing: hashes come from the + // source, so a reproduced boundary says nothing about what leads to it. + if let Some(current) = current_finalized_head.as_ref() + && chunk.first_block() <= current.number + { + ensure!( + chunk.last_block() >= current.number, + unapplicable_fork( + ForkReason::DropsFinalizedBlock, + format!( + "chunk {}-{} does not reach finalized block {}", + chunk.first_block(), + chunk.last_block(), + current.number + ) + ) + ); + if let Err(divergence) = tx.validate_finalized_prefix(chunk, current.number)? { + return Err(unapplicable_fork(ForkReason::RewritesFinalizedHistory, divergence)); + } + } + + // `fin` anchors the guard above and every later `compute_rollback`, and the + // report behind it is a header, not a block the source served (WP-8). + let new_finalized_head = match finalized_head { + None => current_finalized_head.clone(), + Some(report) => { + match resolve_finality( + tx, + current_finalized_head.as_ref(), + Some(chunk), + logical_floor, + report + )? { + FinalityDecision::Applied(new_head) => Some(new_head), + FinalityDecision::Ignored(reason) => { + debug!(reason = %reason, "finalized head was ignored"); + current_finalized_head.clone() + } + FinalityDecision::IntegrityFault { reason, detail } => { + return Err(unapplicable_fork(reason, detail)); + } + } } - }); + }; tx.set_finalized_head(new_finalized_head.clone()); tx.insert_fork(chunk)?; Ok(new_finalized_head) }) - })?; + }); + + // The caller materialized these tables before we could judge the chunk, and only a + // committed `write_chunk` clears their dirty markers — the orphan sweep runs at startup + // only. A source refused on every retry would leak a chunk a minute. + let finalized_head = match commit { + Ok(head) => head, + Err(err) => { + self.abandon_tables(chunk); + return Err(err); + } + }; debug!(finalized_head = valuable(&finalized_head), "saved new chunk"); @@ -452,10 +575,15 @@ impl WriteController { /// `retain_from_head` is the `Head(n)` window size, if set; on EXTEND the /// window is trimmed to keep at most `n` blocks behind the head. - pub fn handle_ingest_msg(&mut self, msg: IngestMessage, retain_from_head: Option) -> anyhow::Result<()> { - match msg { + pub fn handle_ingest_msg( + &mut self, + msg: IngestMessage, + retain_from_head: Option + ) -> anyhow::Result { + let update = match msg { IngestMessage::FinalizedHead(finalized_head) => { self.finalize(&finalized_head)?; + FlushFloorUpdate::Unchanged } IngestMessage::NewChunk(new_chunk) => { let ctx = format!("failed to write new chunk {}", new_chunk); @@ -463,20 +591,36 @@ impl WriteController { if let Some(n) = retain_from_head { let first_chunk_head = self.first_chunk_head().map(|h| h.number); if let Some(floor) = trim_floor(first_chunk_head, self.next_block(), n) { + // Verdict ignored: `trim_floor` fires above the first chunk's last block + // and `n >= 1` keeps it at or below the head, so with chunks tiling by + // `last_block + 1` the floor always lands inside one. `Head(0)` aside. + self.retain(floor, None)?; } } + FlushFloorUpdate::Set(None) } IngestMessage::Fork { prev_blocks, rollback_sender } => { - self.compute_rollback(&prev_blocks).map(|rollback| { - let _ = rollback_sender.send(rollback); - })?; + let rollback = self.compute_rollback(&prev_blocks)?; + let flush_floor = rollback.reach_at_least; + if rollback_sender.send(rollback).is_ok() { + FlushFloorUpdate::Set(flush_floor) + } else { + FlushFloorUpdate::Set(None) + } } + }; + Ok(update) + } + + fn abandon_tables(&self, chunk: &StorageChunk) { + let tables = chunk.tables().values().copied().collect::>(); + if let Err(err) = self.db.delete_tables(&tables) { + warn!(reason =? err, "failed to abandon the tables of a refused chunk"); } - Ok(()) } fn write_new_chunk(&mut self, mut new_chunk: NewChunk) -> anyhow::Result<()> { @@ -524,6 +668,168 @@ impl WriteController { } } +/// Keeps [`UnapplicableFork`] in the chain — `report_dataset_epoch_failure` buckets on the type, +/// and the message can never carry block numbers or hashes into a metric label. +fn unapplicable_fork(reason: ForkReason, detail: String) -> anyhow::Error { + anyhow::Error::new(UnapplicableFork { reason }).context(detail) +} + +#[derive(Debug)] +enum CommittedFinalityUpdate { + Applied(BlockRef), + Ignored(FinalityIgnoreReason) +} + +#[derive(Debug)] +enum FinalityDecision { + Applied(BlockRef), + Ignored(FinalityIgnoreReason), + IntegrityFault { reason: ForkReason, detail: String } +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +enum FinalityIgnoreReason { + Regressive, + AlreadyFinalized, + BelowRetainedWindow, + NoBlockAtHeight +} + +impl FinalityIgnoreReason { + const fn as_str(self) -> &'static str { + match self { + Self::Regressive => "regressive", + Self::AlreadyFinalized => "already_finalized", + Self::BelowRetainedWindow => "below_retained_window", + Self::NoBlockAtHeight => "no_block_at_height" + } + } +} + +impl std::fmt::Display for FinalityIgnoreReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Resolves one report against the state that will exist after this transaction. +/// +/// A replacement owns its whole numeric range; stored history owns heights below it. A report +/// above the resulting head is clamped to that head. At an exact height, the owner must carry the +/// reported hash. A genuine sparse hole is ignored, while a replacement omitting a stored block +/// that the same response calls final is an integrity fault (WP-8). +fn resolve_finality( + tx: &sqd_storage::db::DatasetUpdate<'_>, + current: Option<&BlockRef>, + replacement: Option<&StorageChunk>, + logical_floor: BlockNumber, + report: &BlockRef +) -> anyhow::Result { + let post_head = match replacement { + Some(chunk) => get_chunk_head(chunk), + None => { + let head = tx + .list_chunks(0, None) + .into_reversed() + .next() + .transpose()? + .ok_or_else(|| anyhow!("can't finalize an empty dataset"))?; + get_chunk_head(&head) + } + }; + + let (effective, clamped) = if report.number > post_head.number { + (post_head, true) + } else { + (report.clone(), false) + }; + + if effective.number < logical_floor { + return Ok(FinalityDecision::Ignored(FinalityIgnoreReason::BelowRetainedWindow)); + } + + if let Some(current) = current { + if effective.number < current.number { + return Ok(FinalityDecision::Ignored(FinalityIgnoreReason::Regressive)); + } + if effective.number == current.number { + if effective.hash != current.hash { + return Ok(FinalityDecision::IntegrityFault { + reason: ForkReason::FinalityHashChanged, + detail: format!( + "block {}: expected {}, got {}", + current.number, current.hash, effective.hash + ) + }); + } + return Ok(FinalityDecision::Ignored(FinalityIgnoreReason::AlreadyFinalized)); + } + } + + // A claim about a descendant proves the resulting head final without asserting the + // descendant's hash exists in this dataset. + if clamped { + return Ok(FinalityDecision::Applied(effective)); + } + + let in_batch = replacement.filter(|c| c.first_block() <= report.number && report.number <= c.last_block()); + + if let Some(chunk) = in_batch { + if let Some(hash) = tx.find_block_hash_in_chunk(chunk, report.number)? { + return Ok(if hash == report.hash { + FinalityDecision::Applied(report.clone()) + } else { + FinalityDecision::IntegrityFault { + reason: ForkReason::FinalityContradictsChunk, + detail: format!( + "block {}: chunk carries {}, finality reports {}", + report.number, hash, report.hash + ) + } + }); + } + // Missing from the batch is only a hole if stored history agrees: otherwise this commit + // deletes a block the same response calls final. + if let Some(hash) = tx.find_stored_block_hash(report.number)? { + return Ok(FinalityDecision::IntegrityFault { + reason: ForkReason::FinalityEvictsFinalizedBlock, + detail: format!( + "block {}#{} is dropped by the chunk whose response reports it finalized", + report.number, hash + ) + }); + } + warn!( + block_number = report.number, + block_hash = %report.hash, + "finality reported at a height neither the chunk nor stored history carries; ignoring it" + ); + return Ok(FinalityDecision::Ignored(FinalityIgnoreReason::NoBlockAtHeight)); + } + + match tx.find_stored_block_hash(report.number)? { + Some(hash) => Ok(if hash == report.hash { + FinalityDecision::Applied(report.clone()) + } else { + FinalityDecision::IntegrityFault { + reason: ForkReason::FinalityContradictsStoredBlock, + detail: format!( + "block {}: stored history carries {}, finality reports {}", + report.number, hash, report.hash + ) + } + }), + None => { + warn!( + block_number = report.number, + block_hash = %report.hash, + "finality reported at a height stored history does not carry; ignoring it" + ); + Ok(FinalityDecision::Ignored(FinalityIgnoreReason::NoBlockAtHeight)) + } + } +} + fn get_chunk_head(chunk: &Chunk) -> BlockRef { BlockRef { number: chunk.last_block(), @@ -574,12 +880,19 @@ fn observe_storage_write( mod tests { use std::{collections::BTreeMap, sync::Arc}; + use arrow::{ + array::{RecordBatch, StringArray, UInt64Array}, + datatypes::{DataType, Field, Schema} + }; use sqd_primitives::BlockRef; - use sqd_storage::db::{Chunk, DatabaseSettings, DatasetId}; + use sqd_storage::db::{Chunk, CompactionStatus, DatabaseSettings, DatasetId}; use tokio::sync::watch; - use super::{WriteController, trim_floor}; - use crate::types::{DBRef, DatasetKind}; + use super::{WriteController, get_chunk_head, trim_floor}; + use crate::{ + errors::{UnapplicableFork, UnapplicableForkReason}, + types::{DBRef, DatasetKind} + }; #[test] fn nothing_is_trimmed_while_the_window_fits() { @@ -661,34 +974,37 @@ mod tests { assert_eq!(stored.last_block_hash(), "h10"); } - // INV-30: the published finalized head equals the storage label. + // INV-30: the published finalized head equals the storage label. Real tables, because a report + // below the head chunk's last block is now checked against the block stored there. #[test] fn finalize_publishes_committed_finalized_head() { let mut f = fixture(); - f.wc.new_chunk(None, &chunk(1, 10, "h10", "h0")).unwrap(); + f.wc.new_chunk(None, &linked_chunk(&f.db, 1, 10, "h0", "h").unwrap()) + .unwrap(); assert_eq!(*f.fin_rx.borrow(), None); - f.wc.finalize(&block(5, "h5")).unwrap(); + f.wc.finalize(&block(5, "h-5")).unwrap(); - assert_eq!(*f.fin_rx.borrow(), Some(block(5, "h5"))); + assert_eq!(*f.fin_rx.borrow(), Some(block(5, "h-5"))); let label = f.db.snapshot().get_label(f.dataset_id).unwrap().unwrap(); - assert_eq!(label.finalized_head(), Some(&block(5, "h5"))); + assert_eq!(label.finalized_head(), Some(&block(5, "h-5"))); } // INV-40/CN-9: a rebuilt writer reseeds subscribers from committed storage. #[test] fn rebuilt_writer_reseeds_watermarks_from_storage() { let mut f = fixture(); - f.wc.new_chunk(None, &chunk(1, 10, "h10", "h0")).unwrap(); - f.wc.finalize(&block(5, "h5")).unwrap(); + f.wc.new_chunk(None, &linked_chunk(&f.db, 1, 10, "h0", "h").unwrap()) + .unwrap(); + f.wc.finalize(&block(5, "h-5")).unwrap(); drop(f.wc); let (head_tx, head_rx) = watch::channel(None); let (fin_tx, fin_rx) = watch::channel(None); let _wc = WriteController::new(f.db.clone(), f.dataset_id, DatasetKind::Evm, head_tx, fin_tx).unwrap(); - assert_eq!(*head_rx.borrow(), Some(block(10, "h10"))); - assert_eq!(*fin_rx.borrow(), Some(block(5, "h5"))); + assert_eq!(*head_rx.borrow(), Some(block(10, "h-10"))); + assert_eq!(*fin_rx.borrow(), Some(block(5, "h-5"))); } // Head-only progress must not fire the finalized channel — no spurious @@ -746,4 +1062,794 @@ mod tests { eprintln!("chunk commit + publish: {commit_us:.1} us/op"); assert_eq!(*f.head_rx.borrow(), Some(block(last_block, &parent))); } + + /// Blocks carry `{tag}-{number}` hashes: two chunks agree on a range exactly when built with + /// the same tag. + fn hashes(tag: &str, first_block: u64, last_block: u64) -> Vec { + (first_block..=last_block).map(|n| format!("{tag}-{n}")).collect() + } + + /// A chunk with a real `blocks` table — the finalized-prefix guard reads hashes out of + /// storage, so the table-less `chunk` above would exercise nothing. + fn chunk_with_hashes(db: &DBRef, first_block: u64, parent_hash: &str, hashes: &[String]) -> anyhow::Result { + let blocks = hashes + .iter() + .enumerate() + .map(|(i, hash)| (first_block + i as u64, hash.clone())) + .collect::>(); + chunk_with_numbers(db, parent_hash, &blocks) + } + + /// The same, with the block numbers given explicitly — a chain with holes (Solana slots) is + /// not expressible as a contiguous range. + fn chunk_with_numbers(db: &DBRef, parent_hash: &str, blocks: &[(u64, String)]) -> anyhow::Result { + let schema = Arc::new(Schema::new(vec![ + Field::new("number", DataType::UInt64, false), + Field::new("hash", DataType::Utf8, false), + Field::new("parent_hash", DataType::Utf8, false), + ])); + + let numbers = blocks.iter().map(|(number, _)| *number).collect::>(); + let hashes = blocks.iter().map(|(_, hash)| hash.clone()).collect::>(); + let parent_hashes = std::iter::once(parent_hash) + .chain(hashes.iter().map(String::as_str)) + .take(hashes.len()) + .collect::>(); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt64Array::from(numbers.clone())), + Arc::new(StringArray::from(hashes.clone())), + Arc::new(StringArray::from(parent_hashes)), + ] + )?; + + let mut builder = db.new_table_builder(schema); + builder.write_record_batch(&batch)?; + let table_id = builder.finish()?; + + Ok(Chunk::V1 { + first_block: *numbers.first().unwrap(), + last_block: *numbers.last().unwrap(), + last_block_hash: hashes.last().unwrap().clone(), + parent_block_hash: parent_hash.to_string(), + first_block_time: None, + last_block_time: None, + tables: BTreeMap::from([("blocks".to_string(), table_id)]) + }) + } + + fn linked_chunk( + db: &DBRef, + first_block: u64, + last_block: u64, + parent_hash: &str, + tag: &str + ) -> anyhow::Result { + chunk_with_hashes(db, first_block, parent_hash, &hashes(tag, first_block, last_block)) + } + + fn seed_chain(f: &mut Fixture) -> anyhow::Result<(Chunk, Chunk)> { + let first = linked_chunk(&f.db, 0, 5, "genesis", "old")?; + let second = linked_chunk(&f.db, 6, 9, "old-5", "old")?; + f.wc.new_chunk(None, &first)?; + f.wc.new_chunk(None, &second)?; + Ok((first, second)) + } + + fn stored_chunks(f: &Fixture) -> anyhow::Result> { + f.db.snapshot() + .list_chunks(f.dataset_id, 0, None) + .collect::>>() + } + + // INV-12/13: a replacement reaching the finalized height with a different hash there is a + // source equivocating below its own finality; refusing it must leave nothing behind. + #[test] + fn replacement_rewriting_the_finalized_block_is_rejected() -> anyhow::Result<()> { + let mut f = fixture(); + let (first, second) = seed_chain(&mut f)?; + let current_finalized = block(5, "old-5"); + f.wc.finalize(¤t_finalized)?; + + let replacement = linked_chunk(&f.db, 0, 5, "other-genesis", "new")?; + let result = f.wc.new_chunk(Some(&block(5, "new-5")), &replacement); + + let err = result.unwrap_err(); + assert!( + err.chain().any(|e| e.is::()), + "finality rejections must stay in the unapplicable_fork metric bucket: {err:#}" + ); + assert_eq!(*f.fin_rx.borrow(), Some(current_finalized.clone())); + assert_eq!(stored_chunks(&f)?, vec![first, second]); + assert_eq!( + f.db.snapshot() + .get_label(f.dataset_id)? + .and_then(|label| label.finalized_head().cloned()), + Some(current_finalized) + ); + Ok(()) + } + + // INV-12: finality is immutable at a fixed height, even when carried by an ordinary append. + #[test] + fn composed_finality_rejects_hash_change_at_fixed_height() -> anyhow::Result<()> { + let mut f = fixture(); + let (first, second) = seed_chain(&mut f)?; + let current_finalized = block(5, "old-5"); + f.wc.finalize(¤t_finalized)?; + + let append = linked_chunk(&f.db, 10, 12, "old-9", "old")?; + let result = f.wc.new_chunk(Some(&block(5, "other-5")), &append); + + assert!(result.is_err(), "finalized hash changed at a fixed height"); + assert_eq!(*f.fin_rx.borrow(), Some(current_finalized)); + assert_eq!(stored_chunks(&f)?, vec![first, second]); + Ok(()) + } + + // Accepting a replacement that stops below `fin` would leave the finalized block absent until + // some later flush caught up, so it is refused instead. + #[test] + fn replacement_below_finalized_that_misses_it_is_rejected() -> anyhow::Result<()> { + let mut f = fixture(); + let (first, second) = seed_chain(&mut f)?; + let current_finalized = block(7, "old-7"); + f.wc.finalize(¤t_finalized)?; + + let replacement = linked_chunk(&f.db, 6, 6, "old-5", "new")?; + let result = f.wc.new_chunk(None, &replacement); + + assert!( + result.is_err(), + "replacement that drops the finalized block was accepted" + ); + assert_eq!(*f.fin_rx.borrow(), Some(current_finalized)); + assert_eq!(stored_chunks(&f)?, vec![first, second]); + Ok(()) + } + + // INV-13: the replacement carries `fin`'s own hash unchanged and a different one below it — + // what a guard checking only the boundary would admit. + #[test] + fn replacement_reproducing_finality_but_rewriting_below_it_is_rejected() -> anyhow::Result<()> { + let mut f = fixture(); + let (first, second) = seed_chain(&mut f)?; + let current_finalized = block(7, "old-7"); + f.wc.finalize(¤t_finalized)?; + + let replacement = chunk_with_hashes( + &f.db, + 6, + "old-5", + &["fork-6", "old-7", "old-8", "old-9"].map(str::to_string) + )?; + let result = f.wc.new_chunk(None, &replacement); + + let err = result.unwrap_err(); + assert!( + err.chain().any(|e| e.is::()), + "finality rejections must stay in the unapplicable_fork metric bucket: {err:#}" + ); + assert_eq!(*f.fin_rx.borrow(), Some(current_finalized)); + assert_eq!(stored_chunks(&f)?, vec![first, second]); + // The refused chunk's tables were already durable; leaving them would leak one chunk + // per 60-second retry, since the orphan sweep only runs at startup. + assert_eq!(f.db.cleanup()?, 1, "the refused chunk's tables were not abandoned"); + assert_eq!(f.db.purge_orphan_dirty_tables()?, 0); + Ok(()) + } + + // The honest dual: a reorg above `fin` rewrites the whole straddling chunk and must land. + #[test] + fn replacement_reproducing_the_finalized_range_is_accepted() -> anyhow::Result<()> { + let mut f = fixture(); + let (first, _) = seed_chain(&mut f)?; + let current_finalized = block(7, "old-7"); + f.wc.finalize(¤t_finalized)?; + + let replacement = chunk_with_hashes( + &f.db, + 6, + "old-5", + &["old-6", "old-7", "new-8", "new-9"].map(str::to_string) + )?; + f.wc.new_chunk(None, &replacement)?; + + assert_eq!(f.wc.head(), Some(&get_chunk_head(&replacement))); + assert_eq!(*f.fin_rx.borrow(), Some(current_finalized)); + assert_eq!(stored_chunks(&f)?, vec![first, replacement]); + // Exactly one table is collected — the replaced chunk's. The accepted chunk keeps its + // own, so an over-eager abandon would show up here as two. + assert_eq!(f.db.cleanup()?, 1); + Ok(()) + } + + // With no matching boundary the fallback is the window start, below `fin`: the resume position + // is not clamped, the write path guards instead. The anchor comes from the stored chunk, so a + // full-window replay is still linkage-checked. + #[test] + fn rollback_without_matching_hints_resumes_from_window_start() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + f.wc.finalize(&block(4, "old-4"))?; + let hints = vec![block(5, "fork-5"), block(9, "fork-9")]; + + let rollback = f.wc.compute_rollback(&hints)?; + + assert_eq!(rollback.resume_from, f.wc.start_block()); + assert_eq!(rollback.expected_parent_hash.as_deref(), Some("genesis")); + assert_eq!(rollback.reach_at_least, Some(4)); + Ok(()) + } + + // The wedge regression: clamping to `fin + 1` (8) gave a mid-chunk position `insert_fork` + // could not satisfy, so resolution stops at the chunk boundary below `fin` instead. + #[test] + fn rollback_from_straddling_chunk_resumes_at_chunk_boundary() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + f.wc.finalize(&block(7, "old-7"))?; + + let rollback = f.wc.compute_rollback(&[block(8, "fork-8")])?; + + assert_eq!(rollback.resume_from, 6); + assert_eq!(rollback.expected_parent_hash.as_deref(), Some("old-5")); + // The replay must carry the chunk back up to 7, or the swap drops the finalized block. + assert_eq!(rollback.reach_at_least, Some(7)); + Ok(()) + } + + // A rollback plan carries the finality seen while it was resolved. If finality advances over + // an unchanged block before the first cut replay flushes, that short chunk is refused; the next + // resolution raises the floor and must converge instead of repeating the stale cut forever. + #[test] + fn replay_re_resolves_after_finality_advances_past_its_floor() -> anyhow::Result<()> { + let mut f = fixture(); + let (first, second) = seed_chain(&mut f)?; + f.wc.finalize(&block(7, "old-7"))?; + + let stale = f.wc.compute_rollback(&[block(9, "new-9")])?; + assert_eq!(stale.resume_from, 6); + assert_eq!(stale.reach_at_least, Some(7)); + + // The source's fork begins at 9, so block 8 is still common and may become final while the + // replay is in flight. + let advanced_finality = block(8, "old-8"); + f.wc.finalize(&advanced_finality)?; + let short_replay = linked_chunk(&f.db, 6, 7, "old-5", "old")?; + + let err = f.wc.new_chunk(None, &short_replay).unwrap_err(); + let refusal = err + .chain() + .find_map(|e| e.downcast_ref::()) + .expect("a stale finality floor is a typed fork refusal"); + assert_eq!(refusal.reason, UnapplicableForkReason::DropsFinalizedBlock); + assert_eq!(stored_chunks(&f)?, vec![first.clone(), second]); + assert_eq!(*f.fin_rx.borrow(), Some(advanced_finality.clone())); + + let fresh = f.wc.compute_rollback(&[block(9, "new-9")])?; + assert_eq!(fresh.resume_from, 6); + assert_eq!(fresh.reach_at_least, Some(8)); + let replacement = chunk_with_hashes( + &f.db, + 6, + "old-5", + &["old-6", "old-7", "old-8", "new-9"].map(str::to_string) + )?; + f.wc.new_chunk(None, &replacement)?; + + assert_eq!(stored_chunks(&f)?, vec![first, replacement]); + assert_eq!(*f.fin_rx.borrow(), Some(advanced_finality)); + Ok(()) + } + + // Compaction is logically transparent but may erase a physical boundary selected by an + // in-flight rollback. The stale replay is attributable and atomic; resolving again against the + // merged layout goes one chunk deeper and succeeds. + #[test] + fn replay_re_resolves_after_compaction_consumes_its_boundary() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + let current_finality = block(7, "old-7"); + f.wc.finalize(¤t_finality)?; + + let stale = f.wc.compute_rollback(&[block(9, "new-9")])?; + assert_eq!(stale.resume_from, 6); + assert_eq!(stale.reach_at_least, Some(7)); + + assert!(matches!( + f.db.perform_dataset_compaction(f.dataset_id, Some(100), Some(1.25), None)?, + CompactionStatus::Ok(_) + )); + let compacted = stored_chunks(&f)?; + assert_eq!(compacted.len(), 1); + assert_eq!((compacted[0].first_block(), compacted[0].last_block()), (0, 9)); + + let stale_replay = chunk_with_hashes( + &f.db, + 6, + "old-5", + &["old-6", "old-7", "new-8", "new-9"].map(str::to_string) + )?; + let err = f.wc.new_chunk(None, &stale_replay).unwrap_err(); + let refusal = err + .chain() + .find_map(|e| e.downcast_ref::()) + .expect("a compacted rollback boundary is a typed fork refusal"); + assert_eq!(refusal.reason, UnapplicableForkReason::StaleRollbackBoundary); + assert_eq!(stored_chunks(&f)?, compacted, "the stale replay must be atomic"); + + let fresh = f.wc.compute_rollback(&[block(9, "new-9")])?; + assert_eq!(fresh.resume_from, 0); + assert_eq!(fresh.expected_parent_hash.as_deref(), Some("genesis")); + assert_eq!(fresh.reach_at_least, Some(7)); + + let mut replacement_hashes = hashes("old", 0, 7); + replacement_hashes.extend(["new-8".to_string(), "new-9".to_string()]); + let replacement = chunk_with_hashes(&f.db, 0, "genesis", &replacement_hashes)?; + f.wc.new_chunk(None, &replacement)?; + + assert_eq!(stored_chunks(&f)?, vec![replacement]); + assert_eq!(*f.fin_rx.borrow(), Some(current_finality)); + Ok(()) + } + + // Storage owns the replacement boundary verdict. In-memory watermarks are only a published + // working copy, so even an impossible drift there must not downgrade a stale rollback to an + // untyped overlap error or let finality resolve against the wrong owner. + #[test] + fn stale_boundary_guard_reads_storage_when_memory_head_is_missing() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + assert!(matches!( + f.db.perform_dataset_compaction(f.dataset_id, Some(100), Some(1.25), None)?, + CompactionStatus::Ok(_) + )); + let compacted = stored_chunks(&f)?; + assert_eq!((compacted[0].first_block(), compacted[0].last_block()), (0, 9)); + f.db.cleanup()?; + + f.wc.head = None; + let stale_replay = linked_chunk(&f.db, 6, 12, "old-5", "new")?; + let err = f.wc.new_chunk(None, &stale_replay).unwrap_err(); + let refusal = err + .chain() + .find_map(|e| e.downcast_ref::()) + .expect("the transaction must classify the stale stored boundary"); + + assert_eq!(refusal.reason, UnapplicableForkReason::StaleRollbackBoundary); + assert_eq!(stored_chunks(&f)?, compacted); + assert_eq!(f.db.cleanup()?, 1, "the refused chunk's tables were not abandoned"); + Ok(()) + } + + // Nothing finalized is being replaced, so the ingest keeps its own flush boundaries. + #[test] + fn rollback_above_finalized_head_sets_no_reach_floor() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + f.wc.finalize(&block(5, "old-5"))?; + + let rollback = f.wc.compute_rollback(&[block(9, "fork-9")])?; + + assert_eq!(rollback.resume_from, 6); + assert_eq!(rollback.reach_at_least, None); + Ok(()) + } + + // Retention trims whole chunks, so its floor can land inside the surviving one. Resuming at + // that logical floor gives `insert_fork` an overlapping position and wedges the dataset the + // same way the `fin + 1` clamp did (GAP-3). + #[test] + fn rollback_fallback_resumes_at_the_physical_window_start() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + f.wc.finalize(&block(7, "old-7"))?; + // Drops [0, 5] and keeps [6, 9] whole, with the logical floor at 7. + f.wc.retain(7, None)?; + assert_eq!(f.wc.start_block(), 7); + + let rollback = f.wc.compute_rollback(&[block(8, "fork-8")])?; + + assert_eq!(rollback.resume_from, 6); + assert_eq!(rollback.expected_parent_hash.as_deref(), Some("old-5")); + assert_eq!(rollback.reach_at_least, Some(7)); + Ok(()) + } + + // A fork that cannot reach up to finality is refused, never resumed below it. + #[test] + fn rollback_with_all_hints_below_finalized_head_is_refused() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + f.wc.finalize(&block(7, "old-7"))?; + + let err = + f.wc.compute_rollback(&[block(3, "fork-3"), block(5, "fork-5")]) + .unwrap_err(); + assert!( + err.to_string().contains("hints_below_finalized_head"), + "unexpected error: {err}" + ); + Ok(()) + } + + // The validated range spans two stored chunks, so `validate_finalized_prefix` has to pull the + // second one in mid-scan. A broken advance shows up here as a false divergence at the boundary. + #[test] + fn replacement_spanning_two_stored_chunks_reproducing_both_is_accepted() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + assert_eq!( + stored_chunks(&f)?.len(), + 2, + "the scan must cross a stored chunk boundary" + ); + let current_finalized = block(7, "old-7"); + f.wc.finalize(¤t_finalized)?; + + let mut replayed = hashes("old", 0, 7); + replayed.extend(hashes("new", 8, 9)); + let replacement = chunk_with_hashes(&f.db, 0, "genesis", &replayed)?; + f.wc.new_chunk(None, &replacement)?; + + assert_eq!(stored_chunks(&f)?, vec![replacement.clone()]); + assert_eq!(f.wc.head(), Some(&get_chunk_head(&replacement))); + assert_eq!(*f.fin_rx.borrow(), Some(current_finalized)); + Ok(()) + } + + // The same layout with the divergence in the *second* stored chunk: reaching it at all proves + // the cursor crossed the boundary, rather than running out of stored blocks at 6. + #[test] + fn replacement_spanning_two_stored_chunks_rewriting_the_second_is_rejected() -> anyhow::Result<()> { + let mut f = fixture(); + let (first, second) = seed_chain(&mut f)?; + let current_finalized = block(7, "old-7"); + f.wc.finalize(¤t_finalized)?; + + let mut replayed = hashes("old", 0, 6); + replayed.push("fork-7".to_string()); + replayed.extend(hashes("new", 8, 9)); + let err = + f.wc.new_chunk(None, &chunk_with_hashes(&f.db, 0, "genesis", &replayed)?) + .unwrap_err(); + + assert!( + format!("{err:#}").contains("expected finalized block 7#old-7"), + "the divergence must be reported at 7, inside the second stored chunk: {err:#}" + ); + assert_eq!(stored_chunks(&f)?, vec![first, second]); + assert_eq!(*f.fin_rx.borrow(), Some(current_finalized)); + Ok(()) + } + + // A rollback resolved before a trim, committed after it. Without the window guard `insert_fork` + // drops every surviving chunk — the head ends up below the retention floor and the trimmed + // blocks are back. + #[test] + fn replacement_below_the_retained_window_is_refused() -> anyhow::Result<()> { + let mut f = fixture(); + let (_, second) = seed_chain(&mut f)?; + f.wc.retain(6, None)?; + // Drain the trimmed chunk's table, so the count below is the refused chunk's alone. + f.db.cleanup()?; + + let stale = linked_chunk(&f.db, 0, 9, "genesis", "new")?; + let err = f.wc.new_chunk(None, &stale).unwrap_err(); + + assert!( + err.chain().any(|e| e.is::()), + "a stale rollback must stay in the unapplicable_fork metric bucket: {err:#}" + ); + assert_eq!(stored_chunks(&f)?, vec![second.clone()]); + assert_eq!(f.wc.head(), Some(&get_chunk_head(&second))); + assert_eq!(f.db.cleanup()?, 1, "the refused chunk's tables were not abandoned"); + assert_eq!(f.db.purge_orphan_dirty_tables()?, 0); + Ok(()) + } + + // The boundary the guard must not reject: retention's logical floor sits inside the surviving + // chunk, so `compute_rollback` resumes at that chunk's first block — below `start_block()`. + #[test] + fn replacement_at_the_physical_window_start_is_accepted() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + f.wc.retain(7, None)?; + assert_eq!(f.wc.start_block(), 7); + + let replacement = linked_chunk(&f.db, 6, 9, "old-5", "new")?; + f.wc.new_chunk(None, &replacement)?; + + assert_eq!(stored_chunks(&f)?, vec![replacement.clone()]); + assert_eq!(f.wc.head(), Some(&get_chunk_head(&replacement))); + Ok(()) + } + + // The controller keeps a live ingest exactly while the head it builds on survives, so `retain` + // reports what it did instead of leaving the caller to predict it from block numbers — which + // missed every path that clears the window without moving the floor past the head. + #[test] + fn retain_reports_a_cleared_window() -> anyhow::Result<()> { + let mut f = fixture(); + // Nothing stored yet: an ingest started at the old floor is already stale. + assert!(!f.wc.retain(200, None)?); + + let mut f = fixture(); + seed_chain(&mut f)?; + assert!(!f.wc.retain(100, None)?, "a floor above the head clears the window"); + + let mut f = fixture(); + seed_chain(&mut f)?; + assert!( + !f.wc.retain(7, Some("fork-6".to_string()))?, + "a parent hash mismatch inside the window clears it" + ); + assert_eq!(f.wc.head(), None); + Ok(()) + } + + #[test] + fn retain_keeps_the_head_it_only_trims_behind() -> anyhow::Result<()> { + let mut f = fixture(); + let (_, second) = seed_chain(&mut f)?; + assert!(f.wc.retain(6, None)?); + assert_eq!(f.wc.head(), Some(&get_chunk_head(&second))); + Ok(()) + } + + // An emptied window leaves `list_chunks` with no bound to offer, so the floor guard has to fall + // back to the logical floor — otherwise an ingest that outlived the clear writes below it. + #[test] + fn replacement_below_the_floor_of_an_empty_window_is_refused() -> anyhow::Result<()> { + let mut f = fixture(); + assert!(!f.wc.retain(200, None)?); + + let stale = linked_chunk(&f.db, 100, 150, "genesis", "stale")?; + let err = f.wc.new_chunk(None, &stale).unwrap_err(); + + assert!( + err.chain().any(|e| e.is::()), + "a stale ingest must stay in the unapplicable_fork metric bucket: {err:#}" + ); + assert_eq!(stored_chunks(&f)?, vec![]); + assert_eq!(f.wc.head(), None); + assert_eq!(f.db.cleanup()?, 1, "the refused chunk's tables were not abandoned"); + Ok(()) + } + + // An equivocation at the finalized height is refused, never absorbed. + #[test] + fn rollback_with_conflicting_hint_at_finalized_height_is_refused() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + f.wc.finalize(&block(7, "old-7"))?; + + let err = + f.wc.compute_rollback(&[block(7, "fork-7"), block(9, "fork-9")]) + .unwrap_err(); + assert!( + err.to_string().contains("hint_conflicts_with_finalized_head"), + "unexpected error: {err}" + ); + Ok(()) + } + + // A push repeating the current floor is the common case, not the corner: answering it from the + // window's emptiness restarts an untouched ingest, once per push on a fresh dataset. + #[test] + fn an_idempotent_trim_reports_no_loss() -> anyhow::Result<()> { + let mut f = fixture(); + + assert!(f.wc.retain(0, None)?, "the floor is already 0 on an empty dataset"); + assert!( + !f.wc.retain(200, None)?, + "a floor moving over an empty window does strand a live ingest" + ); + assert!(f.wc.retain(200, None)?, "repeating that floor changes nothing"); + Ok(()) + } + + #[test] + fn an_idempotent_trim_keeps_the_head_and_the_window() -> anyhow::Result<()> { + let mut f = fixture(); + let (_, second) = seed_chain(&mut f)?; + let current_finalized = block(7, "old-7"); + f.wc.finalize(¤t_finalized)?; + + assert!(f.wc.retain(6, None)?); + assert!(f.wc.retain(6, None)?); + + assert_eq!(stored_chunks(&f)?, vec![second.clone()]); + assert_eq!(f.wc.head(), Some(&get_chunk_head(&second))); + assert_eq!(*f.fin_rx.borrow(), Some(current_finalized)); + Ok(()) + } + + // `fin` anchors the finalized-prefix guard and every later `compute_rollback`, so a report + // naming a hash the chunk does not carry is an equivocation, not a detail. + #[test] + fn finality_report_contradicting_the_chunk_is_rejected() -> anyhow::Result<()> { + let mut f = fixture(); + let chunk = linked_chunk(&f.db, 0, 9, "genesis", "old")?; + + let err = f.wc.new_chunk(Some(&block(5, "forged-5")), &chunk).unwrap_err(); + + assert!( + err.chain().any(|e| e.is::()), + "finality rejections must stay in the unapplicable_fork metric bucket: {err:#}" + ); + assert_eq!(stored_chunks(&f)?, vec![]); + assert_eq!(*f.fin_rx.borrow(), None); + assert_eq!(f.db.cleanup()?, 1, "the refused chunk's tables were not abandoned"); + Ok(()) + } + + #[test] + fn finality_report_matching_the_chunk_is_accepted() -> anyhow::Result<()> { + let mut f = fixture(); + let chunk = linked_chunk(&f.db, 0, 9, "genesis", "old")?; + + f.wc.new_chunk(Some(&block(5, "old-5")), &chunk)?; + + assert_eq!(stored_chunks(&f)?, vec![chunk]); + assert_eq!(*f.fin_rx.borrow(), Some(block(5, "old-5"))); + Ok(()) + } + + // Sparse numbering is ordinary (Solana slots): refusing would park the dataset, recording would + // put a block nobody has behind FINALIZED-HEAD. The chunk still commits. + #[test] + fn finality_at_a_height_the_chunk_skips_is_ignored() -> anyhow::Result<()> { + let mut f = fixture(); + let chunk = chunk_with_numbers(&f.db, "genesis", &[(4, "old-4".to_string()), (6, "old-6".to_string())])?; + + f.wc.new_chunk(Some(&block(5, "reported-5")), &chunk)?; + + assert_eq!(stored_chunks(&f)?, vec![chunk]); + assert_eq!(*f.fin_rx.borrow(), None); + Ok(()) + } + + // Missing from the replacement, so it looks like a hole — but stored history carries it and this + // commit deletes the chunk holding it. A source may not finalize a block and evict it at once. + #[test] + fn finality_report_evicting_a_stored_block_is_rejected() -> anyhow::Result<()> { + let mut f = fixture(); + let stored = chunk_with_numbers( + &f.db, + "genesis", + &[ + (4, "old-4".to_string()), + (5, "old-5".to_string()), + (6, "old-6".to_string()) + ] + )?; + f.wc.new_chunk(None, &stored)?; + let current_finalized = block(4, "old-4"); + f.wc.finalize(¤t_finalized)?; + + // Same range, same finalized prefix up to 4, but block 5 is gone. + let replacement = chunk_with_numbers(&f.db, "genesis", &[(4, "old-4".to_string()), (6, "new-6".to_string())])?; + let err = f.wc.new_chunk(Some(&block(5, "old-5")), &replacement).unwrap_err(); + + assert!( + err.chain().any(|e| e.is::()), + "finality rejections must stay in the unapplicable_fork metric bucket: {err:#}" + ); + assert_eq!(stored_chunks(&f)?, vec![stored]); + assert_eq!(*f.fin_rx.borrow(), Some(current_finalized)); + assert_eq!(f.db.cleanup()?, 1, "the refused chunk's tables were not abandoned"); + Ok(()) + } + + // The dominant arm: finality lags the tip, so the report usually names a height below the batch, + // where stored history owns the hash and survives the commit. + #[test] + fn finality_below_the_chunk_contradicting_stored_history_is_rejected() -> anyhow::Result<()> { + let mut f = fixture(); + let (first, second) = seed_chain(&mut f)?; + + let append = linked_chunk(&f.db, 10, 12, "old-9", "old")?; + let err = f.wc.new_chunk(Some(&block(5, "forged-5")), &append).unwrap_err(); + + assert!( + err.chain().any(|e| e.is::()), + "finality rejections must stay in the unapplicable_fork metric bucket: {err:#}" + ); + assert_eq!(stored_chunks(&f)?, vec![first, second]); + assert_eq!(*f.fin_rx.borrow(), None); + assert_eq!(f.db.cleanup()?, 1, "the refused chunk's tables were not abandoned"); + Ok(()) + } + + #[test] + fn finality_below_the_chunk_matching_stored_history_is_accepted() -> anyhow::Result<()> { + let mut f = fixture(); + let (first, second) = seed_chain(&mut f)?; + + let append = linked_chunk(&f.db, 10, 12, "old-9", "old")?; + f.wc.new_chunk(Some(&block(5, "old-5")), &append)?; + + assert_eq!(stored_chunks(&f)?, vec![first, second, append]); + assert_eq!(*f.fin_rx.borrow(), Some(block(5, "old-5"))); + Ok(()) + } + + // The direct FINALIZE path checked the hash only at the head chunk's last block. + #[test] + fn finalize_rejects_a_report_contradicting_stored_history() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + + let err = f.wc.finalize(&block(5, "forged-5")).unwrap_err(); + + assert!( + err.chain().any(|e| e.is::()), + "finality rejections must stay in the unapplicable_fork metric bucket: {err:#}" + ); + assert_eq!(*f.fin_rx.borrow(), None); + Ok(()) + } + + // The WP-8 exception on the direct path: a height stored history skips is ignored, not a fault. + #[test] + fn finalize_ignores_a_report_at_a_height_stored_history_skips() -> anyhow::Result<()> { + let mut f = fixture(); + let stored = chunk_with_numbers(&f.db, "genesis", &[(4, "old-4".to_string()), (6, "old-6".to_string())])?; + f.wc.new_chunk(None, &stored)?; + + f.wc.finalize(&block(5, "reported-5"))?; + + assert_eq!(*f.fin_rx.borrow(), None); + assert_eq!(stored_chunks(&f)?, vec![stored]); + Ok(()) + } + + // INV-5/WP-8: retention's logical floor can sit inside the first physical chunk. A report + // naming the chunk's retained overshoot still lies outside the dataset window and is ignored. + #[test] + fn finalize_ignores_a_report_below_the_retention_floor() -> anyhow::Result<()> { + let mut f = fixture(); + let (_, retained) = seed_chain(&mut f)?; + f.wc.retain(7, None)?; + + f.wc.finalize(&block(6, "old-6"))?; + + assert_eq!(stored_chunks(&f)?, vec![retained]); + assert_eq!(*f.fin_rx.borrow(), None); + Ok(()) + } + + // The same floor applies when finality is composed with a replacement at the first physical + // chunk boundary: the chunk commits, but its below-window finality report does not. + #[test] + fn composed_finality_below_the_retention_floor_is_ignored() -> anyhow::Result<()> { + let mut f = fixture(); + seed_chain(&mut f)?; + f.wc.retain(7, None)?; + let replacement = linked_chunk(&f.db, 6, 9, "old-5", "new")?; + + f.wc.new_chunk(Some(&block(6, "new-6")), &replacement)?; + + assert_eq!(stored_chunks(&f)?, vec![replacement]); + assert_eq!(*f.fin_rx.borrow(), None); + Ok(()) + } + + // The same report once a real height is named: the drop above must not have poisoned anything. + #[test] + fn finality_recovers_after_a_skipped_height_report() -> anyhow::Result<()> { + let mut f = fixture(); + let chunk = chunk_with_numbers(&f.db, "genesis", &[(4, "old-4".to_string()), (6, "old-6".to_string())])?; + f.wc.new_chunk(Some(&block(5, "reported-5")), &chunk)?; + + let next = linked_chunk(&f.db, 7, 9, "old-6", "old")?; + f.wc.new_chunk(Some(&block(6, "old-6")), &next)?; + + assert_eq!(*f.fin_rx.borrow(), Some(block(6, "old-6"))); + Ok(()) + } } diff --git a/crates/hotblocks/src/errors.rs b/crates/hotblocks/src/errors.rs index 03a74d4f..5ee899cb 100644 --- a/crates/hotblocks/src/errors.rs +++ b/crates/hotblocks/src/errors.rs @@ -38,10 +38,71 @@ impl Display for QueryTaskPanicked { impl std::error::Error for QueryTaskPanicked {} -/// Divergence reaching below finalized data. Kills the update task, which then parks for a minute. +/// A replay below the finalized head cannot publish a partial chunk merely to cut at a +/// data-availability boundary. +#[derive(Debug)] +pub struct DataAvailabilityChangedDuringFinalizedReplay { + pub block_number: BlockNumber +} + +impl Display for DataAvailabilityChangedDuringFinalizedReplay { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "data availability changed at block {} inside the replayed finalized range", + self.block_number + ) + } +} + +impl std::error::Error for DataAvailabilityChangedDuringFinalizedReplay {} + +/// Stable reason for refusing a fork or finality transition. +/// +/// The snake-case representation is exported as the `cause` label of +/// `dataset_epoch_failures`; dynamic details belong in the error context. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum UnapplicableForkReason { + HintsBelowFinalizedHead, + HintConflictsWithFinalizedHead, + BelowRetainedWindow, + StaleRollbackBoundary, + DropsFinalizedBlock, + RewritesFinalizedHistory, + FinalityHashChanged, + FinalityContradictsChunk, + FinalityEvictsFinalizedBlock, + FinalityContradictsStoredBlock +} + +impl UnapplicableForkReason { + pub const fn as_str(self) -> &'static str { + match self { + Self::HintsBelowFinalizedHead => "hints_below_finalized_head", + Self::HintConflictsWithFinalizedHead => "hint_conflicts_with_finalized_head", + Self::BelowRetainedWindow => "below_retained_window", + Self::StaleRollbackBoundary => "stale_rollback_boundary", + Self::DropsFinalizedBlock => "drops_finalized_block", + Self::RewritesFinalizedHistory => "rewrites_finalized_history", + Self::FinalityHashChanged => "finality_hash_changed", + Self::FinalityContradictsChunk => "finality_contradicts_chunk", + Self::FinalityEvictsFinalizedBlock => "finality_evicts_finalized_block", + Self::FinalityContradictsStoredBlock => "finality_contradicts_stored_block" + } + } +} + +impl Display for UnapplicableForkReason { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// A fork or finality transition that cannot be applied safely. Kills the update task, which then +/// parks for a minute. #[derive(Debug)] pub struct UnapplicableFork { - pub reason: &'static str + pub reason: UnapplicableForkReason } impl Display for UnapplicableFork { diff --git a/crates/hotblocks/src/metrics.rs b/crates/hotblocks/src/metrics.rs index 10ed3e0f..1460fc0b 100644 --- a/crates/hotblocks/src/metrics.rs +++ b/crates/hotblocks/src/metrics.rs @@ -8,17 +8,23 @@ use prometheus_client::{ MetricType, counter::Counter, family::Family, + gauge::Gauge, histogram::{Histogram, exponential_buckets} }, registry::Registry }; +use sqd_primitives::BlockNumber; use sqd_storage::db::{ CF_BLOCK_HASHES, CF_CHUNKS, CF_DATASETS, CF_DELETED_TABLES, CF_DIRTY_TABLES, CF_TABLES, CF_TRANSACTION_HASHES, DatasetId, HashIndexWriteMetrics, ReadSnapshot }; use tracing::error; -use crate::{errors::UnapplicableFork, query::QueryExecutorCollector, types::DBRef}; +use crate::{ + errors::{DataAvailabilityChangedDuringFinalizedReplay, UnapplicableFork}, + query::QueryExecutorCollector, + types::DBRef +}; #[derive(Copy, Clone, Hash, Debug, Default, Ord, PartialOrd, Eq, PartialEq, EncodeLabelSet)] struct DatasetLabel { @@ -60,6 +66,10 @@ pub static QUERY_ERROR_WORKER_PANIC: LazyLock = LazyLock::new(Default:: pub static COMPLETED_QUERIES: LazyLock = LazyLock::new(Default::default); +static INGEST_WITHHELD_FLUSHES: LazyLock> = LazyLock::new(Default::default); + +static INGEST_FLUSH_FLOOR: LazyLock> = LazyLock::new(Default::default); + pub static STREAM_DURATIONS: LazyLock> = LazyLock::new(|| Family::new_with_constructor(|| Histogram::new(exponential_buckets(0.01, 2.0, 20)))); pub static STREAM_BYTES: LazyLock> = @@ -132,22 +142,32 @@ struct WriteLabels { #[derive(Copy, Clone, Hash, Debug, Eq, PartialEq, EncodeLabelSet)] struct EpochFailureLabels { dataset: DatasetValue, - reason: &'static str + reason: &'static str, + cause: &'static str } static DATASET_EPOCH_FAILURES: LazyLock> = LazyLock::new(Default::default); pub(crate) fn report_dataset_epoch_failure(dataset_id: DatasetId, err: &anyhow::Error) { - // Never label by message — it carries block numbers and hashes. - let reason = if err.chain().any(|e| e.is::()) { - "unapplicable_fork" - } else { - "other" + // Never label by message — it carries block numbers and hashes. `cause` splits the fork class + // by `UnapplicableFork::reason`: a stale-ingest refusal that self-heals next epoch and a source + // rewriting finalized history are otherwise indistinguishable here. + let fork = err.chain().find_map(|e| e.downcast_ref::()); + let (reason, cause) = match fork { + Some(fork) => ("unapplicable_fork", fork.reason.as_str()), + None if err + .chain() + .any(|e| e.is::()) => + { + ("other", "data_availability_changed_during_finalized_replay") + } + None => ("other", "unspecified") }; DATASET_EPOCH_FAILURES .get_or_create(&EpochFailureLabels { dataset: DatasetValue(dataset_id), - reason + reason, + cause }) .inc(); } @@ -177,6 +197,20 @@ pub(crate) fn report_hash_index_write_metrics(dataset_id: DatasetId, metrics: &H } } +pub(crate) fn report_withheld_flush(dataset_id: DatasetId) { + INGEST_WITHHELD_FLUSHES.get_or_create(&dataset_label!(dataset_id)).inc(); +} + +/// The block a fork replay must reach before it may emit a chunk, `None` once it has. Set where the +/// floor is *set*, not at a flush attempt: a source stopping below the floor never reaches one, and +/// that stall is otherwise invisible (GAP-43). Alert on +/// `min_over_time(hotblocks_ingest_flush_floor[10m]) >= 0`. +pub(crate) fn report_flush_floor(dataset_id: DatasetId, floor: Option) { + INGEST_FLUSH_FLOOR + .get_or_create(&dataset_label!(dataset_id)) + .set(floor.map_or(-1, |block| i64::try_from(block).unwrap_or(i64::MAX))); +} + pub fn report_query_too_many_tasks_error() { QUERY_ERROR_TOO_MANY_TASKS.inc(); } @@ -508,7 +542,8 @@ pub fn build_metrics_registry() -> Registry { "dataset_epoch_failures", "Dataset update task failures, by dataset and cause; each one parks ingestion for \ 60s before a full restart. reason=unapplicable_fork is a divergence reaching below \ - finalized data", + finalized data, and cause names which one (below_retained_window, \ + rewrites_finalized_history, finality_contradicts_chunk, ...)", DATASET_EPOCH_FAILURES.clone() ); @@ -557,6 +592,18 @@ pub fn build_metrics_registry() -> Registry { "Number of completed queries", COMPLETED_QUERIES.clone() ); + registry.register( + "ingest_withheld_flushes", + "Fork replays whose chunk was held back until it covered the finalized head", + INGEST_WITHHELD_FLUSHES.clone() + ); + registry.register( + "ingest_flush_floor", + "Block a fork replay must still reach before it may emit a chunk; -1 when none is \ + pending. Non-negative for long means the replay is not getting there -- a source stopped \ + below the floor is otherwise only visible as a stale last-block timestamp", + INGEST_FLUSH_FLOOR.clone() + ); top_registry } @@ -591,6 +638,7 @@ mod tests { use sqd_storage::db::{Chunk, DatabaseSettings, DatasetId, DatasetKind}; use super::*; + use crate::errors::{DataAvailabilityChangedDuringFinalizedReplay, UnapplicableForkReason}; #[test] fn write_duration_exposes_bounded_stage_and_outcome_labels() { @@ -636,6 +684,103 @@ mod tests { ); } + // `cause` separates refusals inside the `unapplicable_fork` class, without admitting a block + // number or hash into a label. + #[test] + fn epoch_failure_splits_the_fork_bucket_by_cause() { + let dataset_id = DatasetId::from_str("epoch-failure-test"); + + report_dataset_epoch_failure( + dataset_id, + &anyhow::Error::new(UnapplicableFork { + reason: UnapplicableForkReason::BelowRetainedWindow + }) + .context("chunk 0-9 starts below window start 6") + ); + report_dataset_epoch_failure( + dataset_id, + &anyhow::Error::new(UnapplicableFork { + reason: UnapplicableForkReason::StaleRollbackBoundary + }) + .context("chunk 6-9 starts inside compacted chunk 0-9") + ); + report_dataset_epoch_failure( + dataset_id, + &anyhow::Error::new(DataAvailabilityChangedDuringFinalizedReplay { block_number: 42 }) + ); + report_dataset_epoch_failure(dataset_id, &anyhow::anyhow!("storage transaction failed")); + + let registry = build_metrics_registry(); + let mut output = String::new(); + prometheus_client::encoding::text::encode(&mut output, ®istry).unwrap(); + + let epoch_failures = output + .lines() + .filter(|line| line.starts_with("hotblocks_dataset_epoch_failures_total")) + .filter(|line| line.contains("dataset=\"epoch-failure-test\"")) + .collect::>(); + assert!( + epoch_failures + .iter() + .any(|line| line.contains("reason=\"unapplicable_fork\"") + && line.contains("cause=\"below_retained_window\"")), + "missing fork cause:\n{output}" + ); + assert!( + epoch_failures + .iter() + .any(|line| line.contains("reason=\"unapplicable_fork\"") + && line.contains("cause=\"stale_rollback_boundary\"")), + "missing stale-boundary cause:\n{output}" + ); + assert!( + epoch_failures.iter().any(|line| { + line.contains("reason=\"other\"") + && line.contains("cause=\"data_availability_changed_during_finalized_replay\"") + }), + "missing data-availability cause:\n{output}" + ); + assert!( + epoch_failures + .iter() + .any(|line| line.contains("reason=\"other\"") && line.contains("cause=\"unspecified\"")), + "missing non-fork failure:\n{output}" + ); + assert!( + !output.contains("window start 6") && !output.contains("compacted chunk 0-9"), + "the error message reached a label:\n{output}" + ); + } + + // GAP-43: a source stopping below the floor never reaches a flush, so the counter and warning + // stay silent. The pending floor does not depend on a flush being tried. + #[test] + fn flush_floor_gauge_holds_a_pending_replay() { + let dataset_id = DatasetId::from_str("flush-floor-test"); + let floor_line = |output: &str| -> String { + output + .lines() + .find(|line| line.starts_with("hotblocks_ingest_flush_floor") && line.contains("flush-floor-test")) + .unwrap_or_else(|| panic!("no flush floor series:\n{output}")) + .to_string() + }; + + report_flush_floor(dataset_id, Some(1234)); + let mut output = String::new(); + prometheus_client::encoding::text::encode(&mut output, &build_metrics_registry()).unwrap(); + assert!(floor_line(&output).ends_with(" 1234"), "{}", floor_line(&output)); + + report_flush_floor(dataset_id, Some(0)); + let mut output = String::new(); + prometheus_client::encoding::text::encode(&mut output, &build_metrics_registry()).unwrap(); + assert!(floor_line(&output).ends_with(" 0"), "{}", floor_line(&output)); + + report_flush_floor(dataset_id, None); + let mut output = String::new(); + prometheus_client::encoding::text::encode(&mut output, &build_metrics_registry()).unwrap(); + assert!(floor_line(&output).ends_with(" -1"), "{}", floor_line(&output)); + } + #[test] fn rocksdb_collector_exposes_global_and_per_cf_properties() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/hotblocks/src/query/executor.rs b/crates/hotblocks/src/query/executor.rs index 13aadff8..e8341e6e 100644 --- a/crates/hotblocks/src/query/executor.rs +++ b/crates/hotblocks/src/query/executor.rs @@ -80,6 +80,9 @@ impl QuerySlot { sqd_polars::POOL.spawn(move || { let slot = self; let result = catch_unwind(AssertUnwindSafe(|| task(&slot))).map_err(|_| QueryTaskPanicked); + // Release before waking the caller: sending first lets it observe the slot still + // taken and be refused admission for a query that has already finished. + drop(slot); let _ = tx.send(result); }); diff --git a/crates/hotblocks/tests/ct4_finality.rs b/crates/hotblocks/tests/ct4_finality.rs new file mode 100644 index 00000000..02bd6e5c --- /dev/null +++ b/crates/hotblocks/tests/ct4_finality.rs @@ -0,0 +1,333 @@ +//! CT-4 — an equivocating source must not rewrite the accepted finalized prefix, and an honest +//! reorg above finality must recover rather than wedge. +//! +//! Covers INV-12/13/14/24, WP-6 and FM-SRC-5 through the public binding. Conflict windows +//! deliberately omit the old finalized block, forcing fork resolution to resume at a stored chunk +//! boundary; the whole-chunk rewrite that follows is verified on the write path. + +use std::{ + sync::Arc, + time::{Duration, Instant} +}; + +use anyhow::{Context, Result, ensure}; +use sqd_hotblocks_harness::{ + P_CONFLICT_WINDOW, + chain::HlFills, + harness::{Harness, HarnessConfig}, + types::BlockRef +}; + +const START: u64 = 1_000; +const DEEP_FORK_BLOCKS: u32 = (P_CONFLICT_WINDOW + 50) as u32; +const PREFIX_CHUNK_BLOCKS: u32 = 50; +const STRADDLING_CHUNK_BLOCKS: u32 = (P_CONFLICT_WINDOW + 50) as u32; +const FINALITY_LAG: u64 = P_CONFLICT_WINDOW + 20; +const REJECTION_TIMEOUT: Duration = Duration::from_secs(10); +const POLL: Duration = Duration::from_millis(50); + +#[tokio::test(flavor = "multi_thread")] +async fn ct4_finality_equivocation_does_not_replace_finalized_prefix() -> Result<()> { + let mut h = start_harness(false).await?; + + if let Err(err) = run_deep_fork(&mut h).await { + panic!("CT-4 failed: {err:?}"); + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn ct4_straddling_chunk_rollback_respects_finalized_floor() -> Result<()> { + let mut h = start_harness(true).await?; + + if let Err(err) = run_straddling_chunk_fork(&mut h).await { + panic!("CT-4 straddling-chunk scenario failed: {err:?}"); + } + Ok(()) +} + +/// The honest dual: a legitimate reorg above `fin` whose common ancestor lies inside a +/// finality-straddling chunk must recover, not wedge. Before the fix this clamped to `fin + 1`, a +/// mid-chunk position `insert_fork` rejected, freezing the dataset on a 60-second restart loop. +#[tokio::test(flavor = "multi_thread")] +async fn ct4_honest_reorg_into_straddling_chunk_recovers() -> Result<()> { + let mut h = start_harness(true).await?; + + if let Err(err) = run_honest_reorg_recovery(&mut h, false).await { + panic!("honest-reorg recovery failed: {err:?}"); + } + Ok(()) +} + +/// The same reorg, with the source cutting every response short so the replay's first chunk ends +/// one block below `fin`: accepting it drops the finalized block, refusing it parks the epoch. Every +/// retry then repeats the identical cut, so without the flush floor the dataset never recovers. +#[tokio::test(flavor = "multi_thread")] +async fn ct4_honest_reorg_recovers_when_the_replay_is_cut_below_finality() -> Result<()> { + let mut h = start_harness(true).await?; + + if let Err(err) = run_honest_reorg_recovery(&mut h, true).await { + panic!("cut-replay recovery failed: {err:?}"); + } + Ok(()) +} + +/// The subtle equivocation: the source reproduces the finalized block's own hash but rewrites a +/// block *below* it. The replacement stays internally linked, so a guard checking only that hash +/// admits it and the finalized prefix changes under readers (INV-13). +#[tokio::test(flavor = "multi_thread")] +async fn ct4_replacement_rewriting_below_finality_is_refused() -> Result<()> { + let mut h = start_harness(true).await?; + + if let Err(err) = run_rewrite_below_finality(&mut h).await { + panic!("below-finality rewrite was not refused: {err:?}"); + } + Ok(()) +} + +async fn start_harness(disable_compaction: bool) -> Result { + let mut cfg = HarnessConfig::from_block(env!("CARGO_BIN_EXE_sqd-hotblocks"), Arc::new(HlFills), START); + cfg.disable_compaction = disable_compaction; + Harness::start(cfg).await +} + +async fn run_deep_fork(h: &mut Harness) -> Result<()> { + // A finalized head deeper than one conflict-hint window. + h.produce(DEEP_FORK_BLOCKS)?; + h.finalize_with_lag(FINALITY_LAG)?; + h.settle().await?; + h.assert_conforms().await?; + + assert_finality_equivocation_rejected(h, START, DEEP_FORK_BLOCKS, START).await +} + +async fn run_straddling_chunk_fork(h: &mut Harness) -> Result<()> { + // Two separate responses commit as [prefix] [finality-straddling chunk]. + h.produce(PREFIX_CHUNK_BLOCKS)?; + h.settle().await?; + h.assert_conforms().await?; + + let straddling_chunk_start = START + .checked_add(u64::from(PREFIX_CHUNK_BLOCKS)) + .context("the second chunk start overflows")?; + h.produce(STRADDLING_CHUNK_BLOCKS)?; + h.finalize_with_lag(FINALITY_LAG)?; + h.settle().await?; + h.assert_conforms().await?; + + let head = h.model.head().context("the accepted model has no head")?; + let fin = h + .model + .fin + .as_ref() + .context("the accepted model has no finalized head")?; + ensure!( + straddling_chunk_start < fin.number && fin.number < head.number, + "finalized block {} is not strictly inside the second chunk [{straddling_chunk_start}, {}]", + fin.number, + head.number + ); + + assert_finality_equivocation_rejected( + h, + straddling_chunk_start, + STRADDLING_CHUNK_BLOCKS, + straddling_chunk_start + ) + .await +} + +async fn run_rewrite_below_finality(h: &mut Harness) -> Result<()> { + // The [prefix] [finality-straddling] layout puts the resume boundary below `fin`, so the + // replay covers finalized ground. + h.produce(PREFIX_CHUNK_BLOCKS)?; + h.settle().await?; + let straddling_chunk_start = START + .checked_add(u64::from(PREFIX_CHUNK_BLOCKS)) + .context("the second chunk start overflows")?; + h.produce(STRADDLING_CHUNK_BLOCKS)?; + h.finalize_with_lag(FINALITY_LAG)?; + h.settle().await?; + h.assert_conforms().await?; + + let head = h.model.head().context("the accepted model has no head")?; + let fin = h + .model + .fin + .clone() + .context("the accepted model has no finalized head")?; + ensure!( + straddling_chunk_start < fin.number && fin.number < head.number, + "finalized block {} is not strictly inside the second chunk [{straddling_chunk_start}, {}]", + fin.number, + head.number + ); + let tampered_at = straddling_chunk_start + (fin.number - straddling_chunk_start) / 2; + ensure!( + straddling_chunk_start <= tampered_at && tampered_at < fin.number, + "the tampered block {tampered_at} must lie inside the replayed range and below finality" + ); + h.sim.reset_stream_request_observations(&h.dataset); + let tampered = h.sim.rewrite_hash_below_finality(&h.dataset, tampered_at)?; + ensure!( + h.model.hash_at(tampered_at).is_some_and(|h| h != tampered.hash), + "the fault did not change the stored hash at {tampered_at}" + ); + // A source fault, so the model stays on the accepted chain: reorg through the simulator + // rather than `Harness::fork`. + let reorg_from = fin.number + (head.number - fin.number) / 2; + h.sim.fork(&h.dataset, reorg_from, STRADDLING_CHUNK_BLOCKS)?; + + await_finality_fault_rejection(h, &head, &fin, straddling_chunk_start).await?; + h.assert_conforms().await?; + Ok(()) +} + +async fn run_honest_reorg_recovery(h: &mut Harness, cut_replay: bool) -> Result<()> { + // The same layout, with `fin` strictly inside chunk 2. + h.produce(PREFIX_CHUNK_BLOCKS)?; + h.settle().await?; + let straddling_chunk_start = START + .checked_add(u64::from(PREFIX_CHUNK_BLOCKS)) + .context("the second chunk start overflows")?; + h.produce(STRADDLING_CHUNK_BLOCKS)?; + h.finalize_with_lag(FINALITY_LAG)?; + h.settle().await?; + h.assert_conforms().await?; + + let head = h.model.head().context("the accepted model has no head")?; + let fin = h + .model + .fin + .clone() + .context("the accepted model has no finalized head")?; + ensure!( + straddling_chunk_start < fin.number && fin.number < head.number, + "finalized block {} is not strictly inside the second chunk [{straddling_chunk_start}, {}]", + fin.number, + head.number + ); + + // An honest tip reorg above `fin` but inside the straddling chunk: its common ancestor sits + // below `fin`, so the whole chunk is rewritten and must reproduce the finalized block. + let reorg_from = fin.number + (head.number - fin.number) / 2; + ensure!( + fin.number < reorg_from && reorg_from <= head.number, + "the reorg point {reorg_from} must lie strictly above finality and on the chain" + ); + if cut_replay { + // The lagging report is what lets a cut response end a chunk: the client suppresses its + // own end-of-response commit below the finality it has seen, and a restart resets that to + // the lagging replica's view. The 200k-row flush bound is the other way to get there. + let cut = fin.number - straddling_chunk_start; + ensure!(cut > 0, "the cut response must stop strictly below finality"); + h.sim.inject_fault(&h.dataset, |f| { + f.max_blocks_per_response = Some(cut as u32); + f.finality_report_cap = Some(straddling_chunk_start); + }); + h.sut.restart().await?; + } + h.sim.reset_stream_request_observations(&h.dataset); + h.fork(reorg_from, STRADDLING_CHUNK_BLOCKS)?; + + h.settle().await?; + assert_replay_started_at(h, straddling_chunk_start)?; + h.assert_conforms().await?; + let recovered_fin = h + .client + .finalized_head() + .await + .context("failed to read FINALIZED-HEAD after recovery")?; + ensure!( + recovered_fin.as_ref() == Some(&fin), + "finality moved during an honest recovery: expected {fin:?}, got {recovered_fin:?}" + ); + Ok(()) +} + +async fn assert_finality_equivocation_rejected( + h: &Harness, + fork_from: u64, + replacement_blocks: u32, + expected_resume: u64 +) -> Result<()> { + let expected_head = h.model.head().context("the accepted model has no head")?; + let expected_fin = h + .model + .fin + .clone() + .context("the accepted model has no finalized head")?; + ensure!( + expected_head.number.saturating_sub(expected_fin.number) > P_CONFLICT_WINDOW, + "the first conflict window would include finalized block {}", + expected_fin.number + ); + // The source rewrites a suffix including `fin` and claims the new tip final; a source fault, + // so the reference model stays on the accepted fork. + h.sim.reset_stream_request_observations(&h.dataset); + h.sim + .equivocate_finalized_prefix(&h.dataset, fork_from, replacement_blocks)?; + let conflicting_source_head = h.sim.tip(&h.dataset).context("the faulty source has no head")?; + assert_ne!( + conflicting_source_head.hash, expected_head.hash, + "the fault did not mint a distinct source branch" + ); + + await_finality_fault_rejection(h, &expected_head, &expected_fin, expected_resume).await?; + h.assert_conforms().await?; + Ok(()) +} + +async fn await_finality_fault_rejection( + h: &Harness, + expected_head: &BlockRef, + expected_fin: &BlockRef, + expected_resume: u64 +) -> Result<()> { + // Hold the accepted watermarks still for the whole window. Adopting the equivocation would move + // HEAD/FINALIZED-HEAD off the accepted chain within a poll or two; refusing it keeps them fixed. + let deadline = Instant::now() + REJECTION_TIMEOUT; + loop { + let observed_head = h + .client + .head() + .await + .context("failed to read HEAD during fork recovery")?; + let observed_fin = h + .client + .finalized_head() + .await + .context("failed to read FINALIZED-HEAD during fork recovery")?; + ensure!( + observed_head.as_ref() == Some(expected_head), + "finality equivocation changed HEAD: expected {expected_head:?}, got {observed_head:?}" + ); + ensure!( + observed_fin.as_ref() == Some(expected_fin), + "finality equivocation changed FINALIZED-HEAD: expected {expected_fin:?}, got {observed_fin:?}" + ); + + if Instant::now() > deadline { + // Engagement: this must be a new HTTP replay request, not the source mutation waking + // the already-parked long poll. Its exact position also pins the physical rollback + // boundary; final-state equality alone cannot distinguish it from a full-window replay. + assert_replay_started_at(h, expected_resume)?; + return Ok(()); + } + tokio::time::sleep(POLL).await; + } +} + +fn assert_replay_started_at(h: &Harness, expected_resume: u64) -> Result<()> { + let stats = h.sim.stats(&h.dataset); + ensure!( + stats.stream_http_requests > 0, + "the SUT never opened a replay request after the fork: {stats:?}" + ); + ensure!( + stats.lowest_stream_from == Some(expected_resume), + "the SUT resumed from {:?}, expected the stored chunk boundary {expected_resume}: {stats:?}", + stats.lowest_stream_from + ); + Ok(()) +} diff --git a/crates/hotblocks/tests/ct9_source_faults.rs b/crates/hotblocks/tests/ct9_source_faults.rs index f80ab139..0428b029 100644 --- a/crates/hotblocks/tests/ct9_source_faults.rs +++ b/crates/hotblocks/tests/ct9_source_faults.rs @@ -35,6 +35,20 @@ async fn ct9_unterminated_final_record_does_not_stall_ingestion() -> Result<()> Ok(()) } +/// A block time is informational. Even when it cannot be represented as a `chrono::DateTime`, the +/// batch must commit and the next batch must keep flowing. +#[tokio::test(flavor = "multi_thread")] +async fn ct9_unrepresentable_block_time_does_not_stall_ingestion() -> Result<()> { + let mut cfg = HarnessConfig::from_block(env!("CARGO_BIN_EXE_sqd-hotblocks"), Arc::new(HlFills), START); + cfg.base_timestamp_ms = i64::MIN; + let mut h = Harness::start(cfg).await?; + + if let Err(err) = run(&mut h).await { + panic!("CT-9 unrepresentable-time scenario failed: {err:?}"); + } + Ok(()) +} + async fn run(h: &mut Harness) -> Result<()> { h.produce(20)?; h.finalize_with_lag(5)?; @@ -79,6 +93,7 @@ async fn ct9_a_total_source_outage_is_counted_before_ingestion_starts() -> Resul kind: Evm.config_kind().to_string(), // `Head` is what routes the dataset through the probe at all. retention: Retention::Head(100), + disable_compaction: false, sources: vec![format!("http://127.0.0.1:{dead}/{DS}")] }] )) diff --git a/crates/storage/src/db/db.rs b/crates/storage/src/db/db.rs index 79f309f5..65098e83 100644 --- a/crates/storage/src/db/db.rs +++ b/crates/storage/src/db/db.rs @@ -13,6 +13,7 @@ use super::{ use crate::db::{ ops::{perform_dataset_compaction, CompactionStatus}, read::datasets::list_all_datasets, + table_id::TableId, write::{ ops as cleanup_ops, table_builder::TableBuilder, @@ -547,6 +548,18 @@ impl Database { Ok(()) } + /// Schedules `tables` for the ordinary purge. Used to abandon tables whose chunk was + /// refused after they were already written: their dirty marker alone is collected by the + /// startup orphan sweep only, so a caller refused on every retry would leak them. + pub fn delete_tables(&self, tables: &[TableId]) -> anyhow::Result<()> { + Tx::new(&self.db).run(|tx| { + for table_id in tables { + tx.delete_table(table_id)?; + } + Ok(()) + }) + } + /// Phase 1 -- logically purge deleted tables (snapshot-safe point deletes). /// Returns the number of tables logically deleted by this call. pub fn cleanup(&self) -> anyhow::Result { diff --git a/crates/storage/src/db/read/blocks_table.rs b/crates/storage/src/db/read/blocks_table.rs index 92a89c88..de6ef1e3 100644 --- a/crates/storage/src/db/read/blocks_table.rs +++ b/crates/storage/src/db/read/blocks_table.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, bail, ensure}; use arrow::{ - array::{Array, AsArray}, + array::{Array, ArrayRef, AsArray}, datatypes::{DataType, UInt32Type, UInt64Type} }; use sqd_array::{ @@ -15,12 +15,7 @@ pub fn get_parent_block_hash( blocks_table: &TableReader, block_number: BlockNumber ) -> anyhow::Result { - let numbers = { - let col_idx = blocks_table.schema().index_of("number")?; - let mut builder = AnyBuilder::new(blocks_table.schema().field(col_idx).data_type()); - blocks_table.create_column_reader(col_idx)?.read(&mut builder)?; - builder.finish() - }; + let numbers = read_block_numbers(blocks_table)?; let maybe_row_idx = match numbers.data_type() { DataType::UInt32 => find_block_row(numbers.as_primitive::().values(), block_number as u32), @@ -30,18 +25,52 @@ pub fn get_parent_block_hash( let row_index = maybe_row_idx.ok_or_else(|| anyhow!("block {} was not found in the given table", block_number))?; - let parent_hash = { - let col_idx = blocks_table.schema().index_of("parent_hash")?; - let mut builder = AnyBuilder::new(blocks_table.schema().field(col_idx).data_type()); - blocks_table - .create_column_reader(col_idx)? - .read_slice(&mut builder, row_index, 1)?; - builder.finish() + read_string_cell(blocks_table, "parent_hash", row_index) +} + +/// The hash `blocks_table` carries for `block_number`, `None` when it holds no such block. Unlike +/// [`get_parent_block_hash`] the match is exact: a skipped number is an absence, not the next up. +pub fn find_block_hash( + blocks_table: &TableReader, + block_number: BlockNumber +) -> anyhow::Result> { + let numbers = read_block_numbers(blocks_table)?; + + let maybe_row_idx = match numbers.data_type() { + DataType::UInt32 => u32::try_from(block_number) + .ok() + .and_then(|n| find_exact_block_row(numbers.as_primitive::().values(), n)), + DataType::UInt64 => find_exact_block_row(numbers.as_primitive::().values(), block_number), + ty => bail!("'number' column has unexpected data type - {}", ty) }; - Ok(match parent_hash.data_type() { - DataType::Utf8 => parent_hash.as_string::().value(0).to_string(), - ty => bail!("'parent_hash' column has unexpected data type - {}", ty) + maybe_row_idx + .map(|row_index| read_string_cell(blocks_table, "hash", row_index)) + .transpose() +} + +fn read_block_numbers(blocks_table: &TableReader) -> anyhow::Result { + let col_idx = blocks_table.schema().index_of("number")?; + let mut builder = AnyBuilder::new(blocks_table.schema().field(col_idx).data_type()); + blocks_table.create_column_reader(col_idx)?.read(&mut builder)?; + Ok(builder.finish()) +} + +fn read_string_cell( + blocks_table: &TableReader, + column: &str, + row_index: usize +) -> anyhow::Result { + let col_idx = blocks_table.schema().index_of(column)?; + let mut builder = AnyBuilder::new(blocks_table.schema().field(col_idx).data_type()); + blocks_table + .create_column_reader(col_idx)? + .read_slice(&mut builder, row_index, 1)?; + let values = builder.finish(); + + Ok(match values.data_type() { + DataType::Utf8 => values.as_string::().value(0).to_string(), + ty => bail!("'{}' column has unexpected data type - {}", column, ty) }) } @@ -55,6 +84,11 @@ fn find_block_row(numbers: &[BN], block: BN) -> Option { .map(|e| e.0) } +// Linear like `find_block_row`: nothing guarantees the column is sorted. +fn find_exact_block_row(numbers: &[BN], block: BN) -> Option { + numbers.iter().position(|n| *n == block) +} + /// Streams all `(block number, hash)` pairs of a `blocks` table, reading the /// columns in batches so peak memory stays `O(batch)` even for large compacted /// chunks. `number` must be `UInt32`/`UInt64`, `hash` must be `Utf8`, and both diff --git a/crates/storage/src/db/write/dataset_update.rs b/crates/storage/src/db/write/dataset_update.rs index 153a88bd..af9fcd1c 100644 --- a/crates/storage/src/db/write/dataset_update.rs +++ b/crates/storage/src/db/write/dataset_update.rs @@ -47,6 +47,18 @@ impl<'a> DatasetUpdate<'a> { .validate_parent_block_hash(chunk, block_number, expected_parent_hash) } + pub fn validate_finalized_prefix(&self, chunk: &Chunk, up_to: BlockNumber) -> anyhow::Result> { + self.tx.validate_finalized_prefix(self.dataset_id, chunk, up_to) + } + + pub fn find_block_hash_in_chunk(&self, chunk: &Chunk, block_number: BlockNumber) -> anyhow::Result> { + self.tx.find_block_hash_in_chunk(chunk, block_number) + } + + pub fn find_stored_block_hash(&self, block_number: BlockNumber) -> anyhow::Result> { + self.tx.find_stored_block_hash(self.dataset_id, block_number) + } + pub fn delete_chunk(&self, chunk: &Chunk) -> anyhow::Result<()> { self.tx.unindex_hashes(self.dataset_id, chunk)?; self.tx.delete_chunk(self.dataset_id, chunk) diff --git a/crates/storage/src/db/write/tx.rs b/crates/storage/src/db/write/tx.rs index f755330f..2a3ff7bf 100644 --- a/crates/storage/src/db/write/tx.rs +++ b/crates/storage/src/db/write/tx.rs @@ -16,7 +16,7 @@ use crate::db::{ CF_DELETED_TABLES, CF_DIRTY_TABLES, CF_TRANSACTION_HASHES }, read::{ - blocks_table::{for_each_block_hash, get_parent_block_hash}, + blocks_table::{find_block_hash, for_each_block_hash, get_parent_block_hash}, chunk::ChunkIterator, transactions_table::for_each_transaction_hash }, @@ -521,6 +521,145 @@ impl<'a> Tx<'a> { } } + /// The hash `chunk` itself carries for `block_number`, `None` when it holds no such block. Lets + /// the write path check a finality report against the blocks that same response served. + pub fn find_block_hash_in_chunk(&self, chunk: &Chunk, block_number: BlockNumber) -> anyhow::Result> { + let blocks_table_id = chunk + .tables() + .get("blocks") + .copied() + .ok_or_else(|| anyhow!("'blocks' table does not exist in chunk {}", chunk))?; + + find_block_hash( + &ReadSnapshot::new(self.db).create_table_reader(blocks_table_id)?, + block_number + ) + } + + /// The hash stored history carries for `block_number`, `None` when no stored chunk covers it: + /// below the window, above the head, or a height the chain skips. + pub fn find_stored_block_hash( + &self, + dataset_id: DatasetId, + block_number: BlockNumber + ) -> anyhow::Result> { + let Some(chunk) = self + .list_chunks(dataset_id, block_number, Some(block_number)) + .next() + .transpose()? + else { + return Ok(None); + }; + self.find_block_hash_in_chunk(&chunk, block_number) + } + + /// Compares `chunk` against stored history block for block over + /// `[chunk.first_block(), up_to]`, describing the first divergence. + /// + /// `up_to` is the finalized head (INV-13). Checking its hash alone would not + /// do: hashes come from the source, so a reproduced boundary says nothing + /// about the interior. Identical hashes over different payload are likewise + /// invisible here — content is the source's word at every height. + /// + /// Peak memory is one stored chunk's worth of pairs: the replacement streams + /// past a cursor that pulls stored chunks in one at a time. + pub fn validate_finalized_prefix( + &self, + dataset_id: DatasetId, + chunk: &Chunk, + up_to: BlockNumber + ) -> anyhow::Result> { + let from = chunk.first_block(); + + let stored_chunks = self + .list_chunks(dataset_id, from, Some(up_to)) + .collect::>>()?; + let mut stored_chunks = stored_chunks.iter(); + + let mut stored: Vec<(BlockNumber, String)> = Vec::new(); + let mut pos = 0; + let mut divergence = None; + + // `true` while a stored block is available at `stored[pos]`. + let mut seek_stored = |stored: &mut Vec<(BlockNumber, String)>, pos: &mut usize| -> anyhow::Result { + while *pos >= stored.len() { + let Some(next) = stored_chunks.next() else { + return Ok(false); + }; + *stored = self.read_block_hashes(next, from, up_to)?; + *pos = 0; + } + Ok(true) + }; + + let blocks_table_id = chunk + .tables() + .get("blocks") + .copied() + .ok_or_else(|| anyhow!("'blocks' table does not exist in chunk {}", chunk))?; + + let snapshot = ReadSnapshot::new(self.db); + let reader = snapshot.create_table_reader(blocks_table_id)?; + + for_each_block_hash(&reader, |number, hash| { + if divergence.is_some() || number > up_to { + return Ok(()); + } + if !seek_stored(&mut stored, &mut pos)? { + divergence = Some(format!( + "block {}#{} is not part of the stored finalized history", + number, hash + )); + return Ok(()); + } + let (stored_number, stored_hash) = &stored[pos]; + if *stored_number == number && stored_hash == hash { + pos += 1; + } else { + divergence = Some(format!( + "expected finalized block {}#{}, got {}#{}", + stored_number, stored_hash, number, hash + )); + } + Ok(()) + })?; + + if divergence.is_none() && seek_stored(&mut stored, &mut pos)? { + let (stored_number, stored_hash) = &stored[pos]; + divergence = Some(format!( + "finalized block {}#{} is missing from the replacement", + stored_number, stored_hash + )); + } + + Ok(divergence.map_or(Ok(()), Err)) + } + + fn read_block_hashes( + &self, + chunk: &Chunk, + from: BlockNumber, + to: BlockNumber + ) -> anyhow::Result> { + let blocks_table_id = chunk + .tables() + .get("blocks") + .copied() + .ok_or_else(|| anyhow!("'blocks' table does not exist in chunk {}", chunk))?; + + let snapshot = ReadSnapshot::new(self.db); + let reader = snapshot.create_table_reader(blocks_table_id)?; + + let mut hashes = Vec::new(); + for_each_block_hash(&reader, |number, hash| { + if from <= number && number <= to { + hashes.push((number, hash.to_string())); + } + Ok(()) + })?; + Ok(hashes) + } + pub fn list_chunks( &self, dataset_id: DatasetId,