diff --git a/crates/asap-aware-mapping/src/accuracy.rs b/crates/asap-aware-mapping/src/accuracy.rs index 4e6b2d400..a268dae17 100644 --- a/crates/asap-aware-mapping/src/accuracy.rs +++ b/crates/asap-aware-mapping/src/accuracy.rs @@ -32,7 +32,8 @@ //! | `ApproximateAggregate`, all `RelativeValue`, values known non-negative | multiplicative | `ε_in + ε_out + ε_in·ε_out`, `δ` by union bound | //! | `Lipschitz { L }`, one `AbsoluteValue` input | Lipschitz | `L·B_in + B_local`, `δ` by union bound | //! | `ExactSum`, value-like inputs | sum | `Σ B_i` (`AbsoluteValue`), `δ` by union bound over inputs | -//! | `ExactExtremum`, same-metric inputs | max/min | `max B_i`, `δ` by union bound over inputs | +//! | `ExactAverage`, `AbsoluteValue` inputs | average | `max B_i`, `δ` by union bound over inputs | +//! | `ExactExtremum`, `AbsoluteValue` inputs | max/min | `max B_i`, `δ` by union bound over inputs | //! | anything else | — | [`AccuracyError::UnsupportedComposition`] | //! //! Cross-metric compositions (a `Rank` error under a value-additive rule, @@ -68,9 +69,12 @@ //! (unchanged), and an approximate layer can never satisfy it. use asap_types::post_asap::{ - AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, - ResultGuarantee, SketchAlgorithm, SketchParams, SketchQuery, SummaryFamilyType, + AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, ExactOperation, GuaranteeSource, + ProbabilityExpr, ResultGuarantee, SketchAlgorithm, SketchParams, SketchQuery, + SummaryFamilyType, }; +#[cfg(test)] +use asap_types::pre_asap::AggIntent; use asap_types::types::AccuracyTarget; /// Statistics a propagation rule may consult. Every field is optional and @@ -85,7 +89,7 @@ pub struct PropagationStats { /// sign change. pub values_non_negative: Option, /// Number of input rows an exact aggregation consumes (e.g. the number - /// of groups a `sum` folds), for `ExactSum`/`ExactExtremum`'s union + /// of groups a function folds), for exact aggregate union bounds /// bound over per-input failures. pub input_row_count: Option, /// Fresh key-frequency distribution evidence from the data workload. @@ -153,6 +157,13 @@ impl AccuracyEvidenceProvider for WorkloadAccuracyEvidence<'_> { /// this trait and passes it to /// [`crate::replacement::SketchAlgorithmStrategy::with_models`]. pub trait AccuracyModel { + /// The definition-registered rule for applying `operation` to an + /// approximate input. `None` means the function is exact only over exact + /// inputs; callers must fail closed for approximate input. + fn exact_operation_rule(&self, _operation: &ExactOperation) -> Option { + None + } + /// The guarantee of reading `query` out of a summary of family `family` /// built over an **exact** input — derived from the family's committed /// parameters by inverting the same sizing formulas @@ -454,6 +465,53 @@ impl DefaultAccuracyModel { }) } + /// Exact arithmetic mean over values with absolute-error guarantees. + /// Averaging cannot amplify the largest absolute input error. The event + /// that every row respects its bound is still protected conservatively + /// by a union bound over the input row count. + fn exact_average( + op: &CompositionOperator, + inputs: &[ResultGuarantee], + stats: &PropagationStats, + ) -> Result { + if inputs + .iter() + .any(|input| input.metric != ErrorMetric::AbsoluteValue) + { + return Err(AccuracyError::UnsupportedComposition { + operator: op.clone(), + input_metrics: inputs.iter().map(|g| g.metric).collect(), + local_metric: None, + reason: "exact average requires AbsoluteValue input guarantees".into(), + }); + } + let mut provenance = Vec::new(); + let count = row_count(stats, &mut provenance); + let exact_local = ResultGuarantee::exact("ExactAggregate(Average)"); + provenance.extend(composed_provenance( + op, + inputs, + &exact_local, + "exact_average_union_bound", + )); + Ok(ResultGuarantee { + metric: ErrorMetric::AbsoluteValue, + bound: BoundExpr::Max { + terms: inputs.iter().map(|g| g.bound.clone()).collect(), + }, + failure_probability: ProbabilityExpr::Scaled { + count, + inner: Box::new(ProbabilityExpr::UnionBound { + terms: inputs + .iter() + .map(|g| g.failure_probability.clone()) + .collect(), + }), + }, + provenance, + }) + } + /// Exact `max`/`min` over approximate inputs of one shared metric: the /// returned value's error is at most the largest input bound (order /// statistics are monotone under a uniform perturbation), with @@ -465,12 +523,15 @@ impl DefaultAccuracyModel { stats: &PropagationStats, ) -> Result { let metric = inputs[0].metric; - if inputs.iter().any(|g| g.metric != metric) || metric == ErrorMetric::TopKMembership { + if inputs + .iter() + .any(|g| g.metric != ErrorMetric::AbsoluteValue) + { return Err(AccuracyError::UnsupportedComposition { operator: op.clone(), input_metrics: inputs.iter().map(|g| g.metric).collect(), local_metric: None, - reason: "exact max/min needs every input under one value-like metric".into(), + reason: "exact max/min requires AbsoluteValue input guarantees".into(), }); } let mut provenance = Vec::new(); @@ -567,6 +628,18 @@ fn composed_provenance( } impl AccuracyModel for DefaultAccuracyModel { + fn exact_operation_rule(&self, operation: &ExactOperation) -> Option { + let ExactOperation::Aggregate { measures, .. } = operation else { + return None; + }; + match measures.as_slice() { + [intent] => crate::function_rules::function_rules(intent).map(|rules| rules.accuracy), + // The remaining functions are exact over exact samples, but have + // no definition-backed rule over approximate values yet. + _ => None, + } + } + fn local_guarantee( &self, family: &SummaryFamilyType, @@ -692,7 +765,15 @@ impl AccuracyModel for DefaultAccuracyModel { Ok(Self::lipschitz(op, *constant, inputs, local)) } CompositionOperator::ExactSum => Self::exact_sum(op, inputs, stats), + CompositionOperator::ExactAverage => Self::exact_average(op, inputs, stats), CompositionOperator::ExactExtremum => Self::exact_extremum(op, inputs, stats), + CompositionOperator::CounterRate + | CompositionOperator::InstantCounterRate + | CompositionOperator::CounterIncrease => Err(unsupported( + "counter reset detection and boundary extrapolation have no distribution-free \ + accuracy bound over approximate samples; exact samples remain exact" + .into(), + )), CompositionOperator::TopKSelection => { let (Some(selected_lower), Some(excluded_upper), Some(delta)) = ( stats.topk_selected_lower_bound, @@ -1122,6 +1203,46 @@ mod tests { assert!((out.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); } + #[test] + fn exact_average_has_its_own_absolute_error_rule() { + let out = DefaultAccuracyModel + .propagate( + &CompositionOperator::ExactAverage, + &[abs(0.25, 0.01)], + None, + &PropagationStats { + input_row_count: Some(4), + ..PropagationStats::default() + }, + ) + .unwrap(); + assert_eq!(out.metric, ErrorMetric::AbsoluteValue); + assert_eq!(out.bound.evaluate(), Some(0.25)); + assert_eq!(out.failure_probability.evaluate(), Some(0.04)); + } + + #[test] + fn counter_functions_have_distinct_definition_rules() { + let operation = |intent| ExactOperation::Aggregate { + reduction: asap_types::pre_asap::Reduction::PerEntity, + measures: vec![intent], + output_names: vec![], + having: None, + }; + assert_eq!( + DefaultAccuracyModel.exact_operation_rule(&operation(AggIntent::Rate)), + Some(CompositionOperator::CounterRate) + ); + assert_eq!( + DefaultAccuracyModel.exact_operation_rule(&operation(AggIntent::IRate)), + Some(CompositionOperator::InstantCounterRate) + ); + assert_eq!( + DefaultAccuracyModel.exact_operation_rule(&operation(AggIntent::Increase)), + Some(CompositionOperator::CounterIncrease) + ); + } + #[test] fn topk_selection_requires_a_separated_margin_certificate() { let err = DefaultAccuracyModel diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index f2651ebed..fa8caf8a4 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -49,8 +49,8 @@ use std::rc::Rc; use asap_types::post_asap::{ - GroupingStrategy, HydraParams, ResultGuarantee, SketchAlgorithm, SketchParams, SketchQuery, - SummaryExpr, SummaryFamilyType, SummaryMaintenanceLifecycleGuarantee, SummaryNode, + ExactOperation, GroupingStrategy, HydraParams, ResultGuarantee, SketchAlgorithm, SketchParams, + SketchQuery, SummaryExpr, SummaryFamilyType, SummaryMaintenanceLifecycleGuarantee, SummaryNode, SummaryWindowFramework, }; use asap_types::pre_asap::agg_intent::AggIntent; @@ -58,8 +58,10 @@ use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::QueryExpr; use asap_types::types::AccuracyTarget; +use crate::exact_composition::{ExactComposition, OperationPlacement}; use crate::recurrence::{ - self, Horizon, RecurrenceCostExplanation, RecurrenceError, RecurrenceProfile, + self, CostRate, EvaluationRate, Horizon, RecurrenceCostExplanation, RecurrenceError, + RecurrenceProfile, }; use crate::replacement::{ realize_child, Implementation, Replacement, ReplacementProvenance, ReplacementSubDAG, @@ -69,6 +71,211 @@ use crate::summary_maintenance_lifecycle::{ SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCostInputs, }; +// ── Recurring-cost vocabulary for mixed exact/summary plans (issue #171) ── + +/// The unit a recurring cost is expressed in. One variant today; an enum so +/// a JSON/DAG export names the unit explicitly instead of a consumer +/// assuming it, and so a future per-resource unit can be added without +/// changing every hook's signature. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CostUnit { + /// Abstract cost units per wall-clock second — the common currency + /// every recurring alternative (maintain-and-read vs. recompute-per-eval) + /// is compared in. + CostUnitsPerSecond, +} + +impl CostUnit { + /// Stable name for export (`"cost_units_per_second"`). + pub fn as_str(self) -> &'static str { + match self { + Self::CostUnitsPerSecond => "cost_units_per_second", + } + } +} + +/// Who produced a set of [`ExactCompositionCostInputs`], and under which +/// model version — carried into every composed decision's explanation and +/// DAG export so a reviewer can tell a deployment's measured numbers from +/// a placeholder. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CostProvenance { + /// The cost model's own name (e.g. `"DefaultCostModel"`). + pub model: String, + /// The model's own version string, whatever scheme it uses. + pub version: String, +} + +/// Which mixed-execution shapes the downstream runtime can actually +/// execute (issue #171). [`crate::exact_composition::ExactCompositionStrategy`] +/// proposes an `ValueOperationAtReadTime` candidate only when +/// `read_time` is set, and an `ValueOperationAtMaintenanceTime` candidate only +/// when `maintenance_time` is — a runtime that cannot run an exact +/// operator on the update path must never be handed one. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct ValueOperationCapabilities { + /// The runtime can apply an exact operator to summary readouts at + /// query evaluation time. + pub read_time: bool, + /// The runtime can apply an exact row transform on the update path, + /// feeding its output into maintained summary state. + pub maintenance_time: bool, +} + +impl ValueOperationCapabilities { + /// Neither shape supported. + pub const NONE: Self = Self { + read_time: false, + maintenance_time: false, + }; + /// Both shapes supported. + pub const ALL: Self = Self { + read_time: true, + maintenance_time: true, + }; + + pub fn supports(self, placement: OperationPlacement) -> bool { + match placement { + OperationPlacement::Read => self.read_time, + OperationPlacement::Maintenance => self.maintenance_time, + } + } +} + +/// What [`CostModel::exact_composition_cost_inputs`] is asked about: one +/// composed alternative at one site, paired with the concrete summary it +/// composes with. +#[derive(Debug, Clone, Copy)] +pub struct ExactCompositionCostRequest<'a> { + /// The pre-ASAP target the composed candidate replaces. + pub target: &'a QueryExpr, + /// The composition itself — placement, operator, child target. + pub composition: &'a ExactComposition, + /// For [`OperationPlacement::Read`]: the child target's *selected* + /// summary readout candidate the exact operator consumes. For + /// [`OperationPlacement::Maintenance`]: the maintained summary *above* the + /// transform that consumes its output (the `SummaryAgg` this transform + /// feeds). Either way, the summary whose maintenance/read cost the + /// formula charges. + pub summary: &'a SummaryNode, + /// How many times this site actually runs once ancestors' own choices + /// are accounted for (see `PlanSpace::global_selection`). + pub effective_consumer_count: usize, +} + +/// Every input the issue #171 cost formulas need, each individually +/// optional: **an unknown stays `None` — never a zero** — so a formula +/// with a missing input yields no rate at all rather than a spuriously +/// cheap one, and global selection then keeps the conservative +/// `KeepPreAsap` behavior. A deployment model that wants defaults supplies +/// them explicitly by overriding [`CostModel::exact_composition_cost_inputs`]. +#[derive(Debug, Clone, PartialEq)] +pub struct ExactCompositionCostInputs { + /// Exact operator cost per row it processes — per readout row for a + /// read-time operation, per input row for an maintenance-time operation. + pub exact_cost_per_row: Option, + /// Rows the exact operator consumes per evaluation (read-time operation) or + /// per update (transform). + pub expected_input_rows: Option, + /// Rows the exact operator emits per evaluation/update. + pub expected_output_rows: Option, + /// Cost of one update to the composed-with summary's maintained state. + pub summary_maintenance_cost_per_update: Option, + /// Cost of one readout of that summary at evaluation time. + pub summary_read_cost: Option, + /// Update (ingest) events per second reaching this site. + pub update_rate: Option, + /// Evaluations per second across every consumer of this site. + pub evaluation_rate: Option, + /// Cost of one full raw recompute of the target from pre-ASAP data — + /// the `KeepPreAsap` baseline's per-evaluation cost. + pub raw_recompute_cost: Option, + pub unit: CostUnit, + pub provenance: CostProvenance, +} + +impl ExactCompositionCostInputs { + /// Every input unknown, attributed to `provenance` — what a model that + /// has no statistics for a site returns. + pub fn unknown(provenance: CostProvenance) -> Self { + Self { + exact_cost_per_row: None, + expected_input_rows: None, + expected_output_rows: None, + summary_maintenance_cost_per_update: None, + summary_read_cost: None, + update_rate: None, + evaluation_rate: None, + raw_recompute_cost: None, + unit: CostUnit::CostUnitsPerSecond, + provenance, + } + } + + /// The rate for whichever composition placement is requested — + /// [`read_operation_plan_cost_rate`] or [`maintenance_operation_plan_cost_rate`]. + pub fn composed_plan_cost_rate(&self, placement: OperationPlacement) -> Option { + match placement { + OperationPlacement::Read => read_operation_plan_cost_rate(self), + OperationPlacement::Maintenance => maintenance_operation_plan_cost_rate(self), + } + } +} + +/// Outer exact read-time operation over a maintained summary: +/// +/// ```text +/// read_operation_plan_cost_rate = +/// update_rate * summary_maintenance_cost_per_update +/// + evaluation_rate * (summary_read_cost +/// + output_rows_per_eval * exact_read-time operation_cost_per_row) +/// ``` +/// +/// `None` if any input is unknown — see [`ExactCompositionCostInputs`]. +pub fn read_operation_plan_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { + let maintenance = inputs.update_rate? * inputs.summary_maintenance_cost_per_update?; + let per_eval = + inputs.summary_read_cost? + inputs.expected_output_rows? * inputs.exact_cost_per_row?; + let evaluation = inputs.evaluation_rate?.0 * per_eval; + finite_rate(maintenance + evaluation) +} + +/// Outer maintained summary over an exact maintenance-time operation: +/// +/// ```text +/// maintenance_operation_plan_cost_rate = +/// update_rate * (exact_function_cost_per_input_row +/// + summary_maintenance_cost_per_update) +/// + evaluation_rate * summary_read_cost +/// ``` +/// +/// `None` if any input is unknown — see [`ExactCompositionCostInputs`]. +pub fn maintenance_operation_plan_cost_rate( + inputs: &ExactCompositionCostInputs, +) -> Option { + let per_update = inputs.exact_cost_per_row? + inputs.summary_maintenance_cost_per_update?; + let maintenance = inputs.update_rate? * per_update; + let evaluation = inputs.evaluation_rate?.0 * inputs.summary_read_cost?; + finite_rate(maintenance + evaluation) +} + +/// The raw/pre-ASAP fallback baseline: +/// +/// ```text +/// raw_recompute_cost_rate = evaluation_rate * raw_recompute_cost +/// ``` +/// +/// `None` if either input is unknown — see [`ExactCompositionCostInputs`]. +pub fn raw_recompute_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { + finite_rate(inputs.evaluation_rate?.0 * inputs.raw_recompute_cost?) +} + +fn finite_rate(units_per_second: f64) -> Option { + units_per_second + .is_finite() + .then_some(CostRate(units_per_second)) +} + /// A CSE-detected, legality-gated shared subtree with two or more consumers /// — the unit [`CostModel::cse_share_decision`] decides over. Built by /// [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted) @@ -672,6 +879,58 @@ pub trait CostModel { self.raw_query_recompute_cost(target) .map(|per_read| Cost(per_read.0 * expected_reads)) } + /// Which mixed exact/summary execution shapes the downstream runtime + /// advertises (issue #171). Gates candidate *generation* in + /// [`crate::exact_composition::ExactCompositionStrategy`]: a shape the + /// runtime can't execute is never proposed, so it can't be selected + /// either. + /// + /// Default: [`ValueOperationCapabilities::ALL`]. The built-in model + /// describes no particular runtime, and leaving both shapes *visible* + /// in `PlanSpace` (for explanations and the DAG viewer) is the more + /// informative default; selection is still gated separately by + /// [`Self::exact_composition_cost_inputs`], whose default supplies no + /// statistics, so nothing is ever *committed* to under the built-in + /// model. A deployment whose runtime lacks a shape narrows this. + fn value_operation_capabilities(&self) -> ValueOperationCapabilities { + ValueOperationCapabilities::ALL + } + + /// Whether the runtime implements this concrete function at this + /// placement. Deployments override this definition-level hook when + /// support differs between functions; the default delegates to the + /// coarse placement capability for backward compatibility. + fn supports_value_operation( + &self, + _operation: &ExactOperation, + placement: OperationPlacement, + ) -> bool { + self.value_operation_capabilities().supports(placement) + } + + /// The statistics the issue #171 recurring-cost formulas need for one + /// composed alternative — see [`ExactCompositionCostInputs`] for each + /// input and [`read_operation_plan_cost_rate`]/ + /// [`maintenance_operation_plan_cost_rate`]/[`raw_recompute_cost_rate`] for how + /// they combine. One structured hook rather than eight scalar ones, so + /// a deployment answers them all from one place (and can attach its own + /// [`CostProvenance`]). + /// + /// Default: every input unknown ([`ExactCompositionCostInputs::unknown`]) + /// — unknown is never zero, and with no rate derivable + /// `PlanSpace::global_selection` keeps the conservative `KeepPreAsap` + /// behavior for the site. A deployment that wants defaults must supply + /// them here explicitly. + fn exact_composition_cost_inputs( + &self, + request: &ExactCompositionCostRequest<'_>, + ) -> ExactCompositionCostInputs { + let _ = request; + ExactCompositionCostInputs::unknown(CostProvenance { + model: "CostModel::exact_composition_cost_inputs (default)".into(), + version: "unknown".into(), + }) + } } fn sketch_state( @@ -829,6 +1088,12 @@ impl CostModel for DefaultCostModel { (self.cse_recompute_cost(&cse) * consumer_count).0 } } + // A composed candidate is costed in cost-units-per-second by + // `PlanSpace::global_selection` against the child decision it + // is committed with — a different unit from this structural + // estimate, and unknowable here without that child. `NaN` + // keeps it from ever out-ranking a real estimate by accident. + Replacement::ExactComposition(_) => f64::NAN, } } } @@ -973,6 +1238,73 @@ mod tests { ); } + // ── Recurring-cost formulas (issue #171) ───────────────────────────── + + fn known_inputs() -> ExactCompositionCostInputs { + ExactCompositionCostInputs { + exact_cost_per_row: Some(0.1), + expected_input_rows: Some(50.0), + expected_output_rows: Some(10.0), + summary_maintenance_cost_per_update: Some(0.01), + summary_read_cost: Some(1.0), + update_rate: Some(100.0), + evaluation_rate: Some(EvaluationRate(2.0)), + raw_recompute_cost: Some(100.0), + unit: CostUnit::CostUnitsPerSecond, + provenance: CostProvenance { + model: "test".into(), + version: "1".into(), + }, + } + } + + #[test] + fn composition_formulas_match_the_issue_definitions() { + let inputs = known_inputs(); + // 100 * 0.01 + 2 * (1 + 10 * 0.1) = 1 + 4 = 5 + assert_eq!(read_operation_plan_cost_rate(&inputs).unwrap().0, 5.0); + // 100 * (0.1 + 0.01) + 2 * 1 = 11 + 2 = 13 + assert!((maintenance_operation_plan_cost_rate(&inputs).unwrap().0 - 13.0).abs() < 1e-9); + // 2 * 100 + assert_eq!(raw_recompute_cost_rate(&inputs).unwrap().0, 200.0); + assert_eq!( + crate::recurrence::total_cost(CostRate(5.0), Horizon(10.0), Cost(3.0)), + Cost(53.0) + ); + } + + #[test] + fn a_missing_input_yields_no_rate_not_zero() { + let mut inputs = known_inputs(); + inputs.summary_maintenance_cost_per_update = None; + assert_eq!(read_operation_plan_cost_rate(&inputs), None); + assert_eq!(maintenance_operation_plan_cost_rate(&inputs), None); + // The baseline doesn't need maintenance and is still known. + assert!(raw_recompute_cost_rate(&inputs).is_some()); + let unknown = ExactCompositionCostInputs::unknown(known_inputs().provenance); + assert_eq!(raw_recompute_cost_rate(&unknown), None); + } + + #[test] + fn default_model_advertises_capabilities_but_no_statistics() { + assert_eq!( + DefaultCostModel.value_operation_capabilities(), + ValueOperationCapabilities::ALL + ); + assert!(ValueOperationCapabilities::NONE + .supports(OperationPlacement::Read) + .not()); + } + + trait Not { + fn not(self) -> bool; + } + impl Not for bool { + fn not(self) -> bool { + !self + } + } + // ── CSE sharing (issue #237, #223 stage 4) ────────────────────────── use asap_types::post_asap::{ diff --git a/crates/asap-aware-mapping/src/exact_composition.rs b/crates/asap-aware-mapping/src/exact_composition.rs new file mode 100644 index 000000000..ad0717f1a --- /dev/null +++ b/crates/asap-aware-mapping/src/exact_composition.rs @@ -0,0 +1,677 @@ +//! [`ExactCompositionStrategy`] composes an exact function with a summary +//! plan across an explicit maintenance/read-time boundary (issue #171). +//! +//! `construct_summary_agg` already nests accumulator realizations, such as +//! KLL over exact `Sum` state or a quantile over `Rate` state. This strategy +//! covers the more general cases where an exact function must consume a +//! summary readout, or where a maintained summary consumes the values of an +//! exact function that has no accumulator realization. +//! +//! Both cases use the general [`SummaryExpr::ValueOperation`] node. Its +//! semantic [`ValueOperation`] is independent from [`ExecutionTiming`], so +//! adding a function does not require adding a new physical node type. +//! +//! ## Reference, don't select +//! +//! A composed candidate needs a child plan to compose *with* — the inner +//! quantile's own summary readout, say. This strategy deliberately does +//! **not** pick that child itself (the way `construct_summary_agg`'s +//! `realize_child` takes the head of the child's own ranking): a +//! [`Replacement::ExactComposition`] carries only the child *target* +//! (`ExactComposition::child_target`, the same `Rc` whose +//! `MemoGroup` in `PlanSpace` already holds every candidate for it). It is +//! [`PlanSpace::global_selection`](crate::replacement::PlanSpace::global_selection) +//! that commits the compatible parent/child pair — so the child's own +//! cost-model ranking, workload-wide effective consumer count, and shared +//! `Rc` identity (one inner summary serving two outer folds) all stay +//! correct, and a child that is also shared by an unrelated consumer is +//! maintained exactly once. `GlobalSelection::materialize` then links the +//! committed pair into one validated post-ASAP DAG. +//! +//! ## Proposal conditions +//! +//! A candidate is proposed only when all of these hold: +//! +//! - the target is a single-measure, `HAVING`-free exact aggregate; +//! - read-time operation: the child is a bindable aggregate that has at least one +//! readout-producing summary implementation (a sketch/sample/wavelet/ +//! model — the shapes a maintained accumulator can't sit above), and the +//! target's grouping keys resolve in the child's output schema; +//! transform: the target is a per-entity exact function with no +//! accumulator form (its only implementation is `PassThrough`); +//! - the exact operator consumes only `Plain` values in its data_state — checked +//! again, structurally, when the pair is composed; +//! - the plugged-in [`CostModel`] advertises the matching +//! [`ValueOperationCapabilities`](crate::cost_model::ValueOperationCapabilities). +//! +//! `avg` gets a read-time operation candidate *and* keeps +//! [`crate::rewrite::AvgToSumOverCountStrategy`]'s rewrite in the same +//! group; the cost model picks between them, nothing here hard-codes one. +//! +//! ## What this strategy never does +//! +//! - Propose an `ExactRead` for a position beneath a maintained +//! summary — data_state validation at composition rejects it as a typed +//! `ImplementError` regardless. +//! - Decide whether a composition is *worth it*: that is +//! `global_selection`'s job, using the issue's cost-units-per-second +//! formulas (see `crate::cost_model::read_operation_plan_cost_rate` and +//! siblings). Missing statistics keep the conservative `KeepPreAsap`. + +use std::rc::Rc; + +use asap_types::post_asap::execution_data_state::validate_execution_data_states_at; +use asap_types::post_asap::{ + exact_operation_output_schema, produced_data_state, AccuracyError, ExactOperation, + ExecutionDataState, ExecutionDataStateError, ResultGuarantee, SummaryExpr, SummaryNode, + SummarySchema, ValueOperation, +}; +use asap_types::pre_asap::agg_intent::AggIntent; +use asap_types::pre_asap::query_expr::{QueryExpr, Reduction}; +use asap_types::types::AccuracyTarget; + +use crate::cost_model::CostModel; +use crate::replacement::{ + bindable_intent, describe_intent, implementations_for_with, ImplementError, Implementation, + Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, +}; +use crate::{AccuracyModel, DefaultAccuracyModel, PropagationStats}; + +#[cfg(test)] +use asap_types::post_asap::ExecutionTiming; + +/// Which side of the maintenance/read boundary an [`ExactComposition`]'s +/// exact function executes on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum OperationPlacement { + /// After the child's summary readout. + Read, + /// On the maintenance path, feeding + /// maintained state above. + Maintenance, +} + +impl OperationPlacement { + /// The availability the composed operator consumes and produces. + pub fn data_state(self) -> ExecutionDataState { + match self { + Self::Read => ExecutionDataState::READ_ROWS, + Self::Maintenance => ExecutionDataState::MAINTENANCE_ROWS, + } + } + + pub fn provenance(self) -> ReplacementProvenance { + match self { + Self::Read => ReplacementProvenance::ValueOperationAtReadTime, + Self::Maintenance => ReplacementProvenance::ValueOperationAtMaintenanceTime, + } + } +} + +/// The payload of a [`Replacement::ExactComposition`] candidate: an exact +/// operator, the placement it runs at, and a *reference* to the child target +/// it composes over — never an already-selected child plan (see the module +/// docs' "Reference, don't select"). +#[derive(Debug, Clone)] +pub struct ExactComposition { + pub placement: OperationPlacement, + pub op: ExactOperation, + /// The pre-ASAP child the operator consumes; its `MemoGroup` holds the + /// candidates `global_selection` may commit this composition with. + pub child_target: Rc, + /// The composed node's output schema — the target's own pre-ASAP + /// output schema, lifted with every column `Plain` (an exact operator + /// only ever produces plain values). + pub schema: SummarySchema, +} + +impl ExactComposition { + /// Can `child` legally be this composition's input? Phase legality + /// (the child's produced data_state — a `KeepPreAsap` leaf takes the + /// phase this edge assigns) plus the plain-operand rule, checked + /// through the same schema derivation [`Self::compose`] uses. + pub fn accepts_child(&self, child: &SummaryNode) -> bool { + let phase_ok = match produced_data_state(&child.expr) { + None => true, + Some(avail) => avail == self.placement.data_state(), + }; + phase_ok && exact_operation_output_schema(&self.op, &child.schema).is_ok() + } + + /// Build the composed, data_state-validated node over `child`. Every edge of + /// the result (including everything beneath `child`) is checked by + /// `asap_types::post_asap::validate_execution_data_states`; an illegal + /// placement is a typed [`ImplementError::ExecutionDataState`], never deferred to a + /// runtime. + pub fn compose(&self, child: Rc) -> Result, ImplementError> { + self.compose_with_accuracy(child, &DefaultAccuracyModel) + } + + /// Compose using the caller's accuracy algebra. Exact operators do not + /// erase an approximate child's error: supported folds propagate it; + /// unsupported folds fail closed with a typed accuracy error. + pub fn compose_with_accuracy( + &self, + child: Rc, + accuracy_model: &dyn AccuracyModel, + ) -> Result, ImplementError> { + if let Some(produced) = produced_data_state(&child.expr) { + if produced != self.placement.data_state() { + let edge = match self.placement { + OperationPlacement::Maintenance => "ValueOperation.child (maintenance time)", + OperationPlacement::Read => "ValueOperation.child (read time)", + }; + return Err(ImplementError::ExecutionDataState( + ExecutionDataStateError::IllegalChildDataState { + edge, + child: produced, + }, + )); + } + } + let schema = exact_operation_output_schema(&self.op, &child.schema)?; + let guarantee = match &child.guarantee { + None => None, + Some(input) if input.is_exact() => Some(ResultGuarantee::exact(format!( + "exact function {:?} over exact input", + self.op + ))), + Some(input) => match accuracy_model.exact_operation_rule(&self.op) { + Some(operator) => match accuracy_model.propagate( + &operator, + std::slice::from_ref(input), + None, + &PropagationStats::default(), + ) { + Ok(guarantee) => Some(guarantee), + // The plan remains executable without a declared accuracy + // target, but an unknown guarantee cannot satisfy a later + // target check. Never replace this with an exact/default + // bound. + Err(AccuracyError::UnsupportedComposition { .. }) => None, + Err(error) => return Err(ImplementError::Accuracy(error)), + }, + // No definition-registered rule: preserve "unknown". This is + // the fail-closed value used by accuracy-target filtering. + None => None, + }, + }; + let timing = match self.placement { + OperationPlacement::Read => asap_types::post_asap::ExecutionTiming::ReadTime, + OperationPlacement::Maintenance => { + asap_types::post_asap::ExecutionTiming::MaintenanceTime + } + }; + let expr = SummaryExpr::ValueOperation { + child, + operation: ValueOperation::Exact(self.op.clone()), + timing, + }; + let node = Rc::new(SummaryNode { + expr, + schema, + guarantee, + }); + validate_execution_data_states_at(&node, self.placement.data_state())?; + Ok(node) + } + + /// Structural identity for `MemoGroup` dedup: same placement, same + /// operator, same child `Rc`. + pub(crate) fn same_as(&self, other: &Self) -> bool { + self.placement == other.placement + && self.op == other.op + && Rc::ptr_eq(&self.child_target, &other.child_target) + } +} + +/// Which exact reducers may run as a query-time fold over readout rows. +/// `Count` only at `Exact` accuracy (an approximate count is a sketch +/// target, not an exact fold). +fn is_read_time_reducer(intent: &AggIntent) -> bool { + matches!( + intent, + AggIntent::Sum { .. } + | AggIntent::Min { .. } + | AggIntent::Max { .. } + | AggIntent::Avg { .. } + | AggIntent::StdDev { .. } + | AggIntent::Variance { .. } + | AggIntent::Count { + accuracy: AccuracyTarget::Exact + } + ) +} + +/// Does `implementation` need a `SummaryEstimate` readout to yield a value +/// — i.e. is it a shape a maintained accumulator can't legally sit above? +fn needs_readout(implementation: &Implementation) -> bool { + matches!( + implementation, + Implementation::Sketch(_) + | Implementation::Sample { .. } + | Implementation::Wavelet { .. } + | Implementation::StatModel { .. } + ) +} + +/// The `(op, child)` of a read-time operation-shaped target, or `None`. +fn read_time_shape( + root: &QueryExpr, + cost_model: &dyn CostModel, +) -> Option<(ExactOperation, Rc, AggIntent)> { + let QueryExpr::Aggregate { + reduction, + measures, + output_names, + having: None, + child, + } = root + else { + return None; + }; + let Reduction::Reduce(by) = reduction else { + return None; + }; + if by.is_without() { + return None; + } + let [intent] = measures.as_slice() else { + return None; + }; + if !is_read_time_reducer(intent) { + return None; + } + let child_intent = bindable_intent(child)?; + if !implementations_for_with(child_intent, cost_model) + .iter() + .any(needs_readout) + { + return None; + } + // Grouping keys must resolve in the child's output schema — the same + // derivation the composed node's own schema will use. + root.output_schema().ok()?; + Some(( + ExactOperation::Aggregate { + reduction: reduction.clone(), + measures: measures.clone(), + output_names: output_names.clone(), + having: None, + }, + Rc::clone(child), + intent.clone(), + )) +} + +/// The `(op, child)` of a function-shaped target — a per-entity exact +/// transform with no accumulator form — or `None`. +fn maintenance_time_shape( + root: &QueryExpr, + cost_model: &dyn CostModel, +) -> Option<(ExactOperation, Rc, AggIntent)> { + let QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures, + output_names, + having: None, + child, + } = root + else { + return None; + }; + let [intent] = measures.as_slice() else { + return None; + }; + if !intent.is_per_series() { + return None; + } + // Exact accumulators (`Rate`/`Increase`) are already directly nestable + // as `SummaryAgg(ExactAggregate)`; only a pass-through function needs + // an explicit update-path node. + if implementations_for_with(intent, cost_model) + .iter() + .any(|i| *i != Implementation::PassThrough) + { + return None; + } + root.output_schema().ok()?; + Some(( + ExactOperation::Aggregate { + reduction: Reduction::PerEntity, + measures: measures.clone(), + output_names: output_names.clone(), + having: None, + }, + Rc::clone(child), + intent.clone(), + )) +} + +/// Proposes [`Replacement::ExactComposition`] candidates — see the module +/// docs. Holds a [`CostModel`] only to ask it which mixed-execution shapes +/// the runtime advertises and which implementations the child has; it +/// never uses it to *rank* anything. +pub struct ExactCompositionStrategy<'a> { + cost_model: &'a dyn CostModel, +} + +static DEFAULT_COST_MODEL: crate::cost_model::DefaultCostModel = + crate::cost_model::DefaultCostModel; + +impl ExactCompositionStrategy<'static> { + /// A strategy consulting the built-in [`DefaultCostModel`](crate::cost_model::DefaultCostModel). + pub fn default_cost_model() -> Self { + Self { + cost_model: &DEFAULT_COST_MODEL, + } + } +} + +impl<'a> ExactCompositionStrategy<'a> { + pub fn new(cost_model: &'a dyn CostModel) -> Self { + Self { cost_model } + } + + fn candidates(&self, target: &TargetSubDAG<'_>) -> Vec { + let Ok(schema) = target.root.output_schema() else { + return Vec::new(); + }; + let schema = asap_types::post_asap::execution_data_state::lift_plain(&schema); + let mut out = Vec::new(); + + if let Some((op, child, intent)) = read_time_shape(target.root, self.cost_model) { + if self + .cost_model + .supports_value_operation(&op, OperationPlacement::Read) + { + let child_desc = + describe_intent(bindable_intent(&child).expect("checked by read_time_shape")); + out.push(ReplacementSubDAG { + strategy: "ExactCompositionStrategy", + replacement: Replacement::ExactComposition(ExactComposition { + placement: OperationPlacement::Read, + op, + child_target: child, + schema: schema.clone(), + }), + provenance: ReplacementProvenance::ValueOperationAtReadTime, + rationale: format!( + "{} is an exact fold whose input is the readout of {} — a maintained \ + accumulator cannot consume query-time values, so instead of collapsing \ + the whole tree into KeepPreAsap this applies the fold as an \ + ExactRead over whichever summary readout global_selection \ + commits for the child target (asap_aware_mapping::exact_composition)", + describe_intent(&intent), + child_desc + ), + }); + } + } + + if let Some((op, child, intent)) = maintenance_time_shape(target.root, self.cost_model) { + if self + .cost_model + .supports_value_operation(&op, OperationPlacement::Maintenance) + { + out.push(ReplacementSubDAG { + strategy: "ExactCompositionStrategy", + replacement: Replacement::ExactComposition(ExactComposition { + placement: OperationPlacement::Maintenance, + op, + child_target: child, + schema, + }), + provenance: ReplacementProvenance::ValueOperationAtMaintenanceTime, + rationale: format!( + "{} is an exact per-entity function with no accumulator form; as an \ + explicit ExactMaintenance on the update path its output can feed a \ + maintained summary above it instead of being handed over as an opaque \ + raw KeepPreAsap blob (asap_aware_mapping::exact_composition)", + describe_intent(&intent) + ), + }); + } + } + out + } +} + +impl ReplacementStrategy for ExactCompositionStrategy<'_> { + fn matches(&self, target: &TargetSubDAG<'_>) -> bool { + !self.candidates(target).is_empty() + } + + fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { + self.candidates(target) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cost_model::{DefaultCostModel, ValueOperationCapabilities}; + use crate::replacement::keep_pre_asap; + use asap_types::post_asap::{ExecutionDataStateError, SketchAlgorithm, SummaryFamilyType}; + use asap_types::pre_asap::agg_intent::default_quantile; + use asap_types::pre_asap::query_expr::Source; + use asap_types::pre_asap::schema::{Column, DataType, Schema}; + + fn metric_scan(labels: &[&str]) -> QueryExpr { + let mut columns = vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ]; + columns.extend(labels.iter().map(|n| Column::new(*n, DataType::Utf8, true))); + QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index(columns, 0, vec![]), + } + } + + fn agg(by: Vec, intent: AggIntent, child: QueryExpr) -> QueryExpr { + QueryExpr::Aggregate { + reduction: Reduction::by(by), + measures: vec![intent], + output_names: vec![], + having: None, + child: Rc::new(child), + } + } + + fn per_entity(intent: AggIntent, child: QueryExpr) -> QueryExpr { + QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![intent], + output_names: vec![], + having: None, + child: Rc::new(child), + } + } + + /// `max by (zone) (quantile by (zone, host) (m))`. + fn max_over_quantile() -> Rc { + let inner = agg( + vec![2, 3], + default_quantile(0.99), + metric_scan(&["zone", "host"]), + ); + Rc::new(agg(vec![0], AggIntent::Max { col: None }, inner)) + } + + #[test] + fn proposes_read_time_operation_for_max_over_quantile() { + let root = max_over_quantile(); + let target = TargetSubDAG::new(&root); + let strategy = ExactCompositionStrategy::default_cost_model(); + assert!(strategy.matches(&target)); + let candidates = strategy.replacements(&target); + assert_eq!(candidates.len(), 1); + let Replacement::ExactComposition(comp) = &candidates[0].replacement else { + panic!( + "expected a composition, got {:?}", + candidates[0].replacement + ); + }; + assert_eq!(comp.placement, OperationPlacement::Read); + assert_eq!( + candidates[0].provenance, + ReplacementProvenance::ValueOperationAtReadTime + ); + let QueryExpr::Aggregate { child, .. } = root.as_ref() else { + unreachable!() + }; + assert!( + Rc::ptr_eq(&comp.child_target, child), + "the candidate references the child target's own Rc — nothing selected" + ); + let names: Vec<_> = comp.schema.fields.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(names, vec!["zone", "max"]); + } + + #[test] + fn proposes_read_time_operation_for_avg_over_quantile_alongside_the_rewrite() { + let inner = agg(vec![2], default_quantile(0.99), metric_scan(&["zone"])); + let root = Rc::new(agg(vec![0], AggIntent::Avg { col: None }, inner)); + let target = TargetSubDAG::new(&root); + assert_eq!( + ExactCompositionStrategy::default_cost_model() + .replacements(&target) + .len(), + 1 + ); + // `avg` competes with AvgToSumOverCountStrategy in the same group. + assert!(crate::rewrite::AvgToSumOverCountStrategy.matches(&target)); + } + + #[test] + fn proposes_maintenance_time_operation_for_a_per_entity_pass_through_over_raw_input() { + let root = Rc::new(per_entity(AggIntent::Deriv, metric_scan(&["zone"]))); + let target = TargetSubDAG::new(&root); + let candidates = ExactCompositionStrategy::default_cost_model().replacements(&target); + assert_eq!(candidates.len(), 1); + assert_eq!( + candidates[0].provenance, + ReplacementProvenance::ValueOperationAtMaintenanceTime + ); + } + + #[test] + fn does_not_propose_for_shapes_already_covered_by_accumulators() { + // sum by (zone) over an exact Sum child: the child has no readout, + // so SummaryAgg(Sum) over SummaryAgg(Sum) is already legal. + let inner = agg( + vec![2, 3], + AggIntent::Sum { col: None }, + metric_scan(&["zone", "host"]), + ); + let root = Rc::new(agg(vec![0], AggIntent::Sum { col: None }, inner)); + assert!(!ExactCompositionStrategy::default_cost_model().matches(&TargetSubDAG::new(&root))); + // rate is an exact accumulator — directly nestable, no separate value operation. + let rate = Rc::new(per_entity(AggIntent::Rate, metric_scan(&[]))); + assert!(!ExactCompositionStrategy::default_cost_model().matches(&TargetSubDAG::new(&rate))); + // A sketch-capable outer intent is not an exact fold. + let inner = agg(vec![2], default_quantile(0.5), metric_scan(&["zone"])); + let root = Rc::new(agg(vec![0], default_quantile(0.99), inner)); + assert!(!ExactCompositionStrategy::default_cost_model().matches(&TargetSubDAG::new(&root))); + } + + struct NoMixedExecution; + impl CostModel for NoMixedExecution { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + fn value_operation_capabilities(&self) -> ValueOperationCapabilities { + ValueOperationCapabilities::NONE + } + } + + #[test] + fn a_runtime_without_the_capability_gets_no_candidate() { + let root = max_over_quantile(); + let target = TargetSubDAG::new(&root); + let strategy = ExactCompositionStrategy::new(&NoMixedExecution); + assert!(!strategy.matches(&target)); + assert!(strategy.replacements(&target).is_empty()); + let deriv = Rc::new(per_entity(AggIntent::Deriv, metric_scan(&[]))); + assert!(!strategy.matches(&TargetSubDAG::new(&deriv))); + } + + #[test] + fn compose_rejects_a_maintained_state_child_for_a_read_time_operation() { + let root = max_over_quantile(); + let target = TargetSubDAG::new(&root); + let candidates = ExactCompositionStrategy::default_cost_model().replacements(&target); + let Replacement::ExactComposition(comp) = &candidates[0].replacement else { + unreachable!() + }; + // A bare SummaryAgg (state, no readout) is not a legal read-time operation + // input — the operator would be consuming sketch state. + let state_child = + crate::replacement::realize_child(&comp.child_target, &DefaultCostModel).unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &state_child.expr else { + panic!("expected the child to realize to a readout"); + }; + assert!(!comp.accepts_child(summary_input)); + assert!(matches!( + comp.compose(Rc::clone(summary_input)), + Err(ImplementError::ExecutionDataState( + ExecutionDataStateError::IllegalChildDataState { .. } + )) + )); + // The readout itself is accepted and composes to a plain schema. + assert!(comp.accepts_child(&state_child)); + let composed = comp.compose(state_child).unwrap(); + assert!( + composed.guarantee.is_none(), + "rank error has no registered conversion through max" + ); + assert!(matches!( + composed.expr, + SummaryExpr::ValueOperation { + timing: ExecutionTiming::ReadTime, + .. + } + )); + assert!(composed + .schema + .fields + .iter() + .all(|f| matches!(f.dtype, SummaryFamilyType::Plain(_)))); + } + + #[test] + fn compose_rejects_a_readout_child_for_a_maintenance_time_operation() { + let inner = agg(vec![2], default_quantile(0.99), metric_scan(&["zone"])); + let root = Rc::new(per_entity(AggIntent::Deriv, inner)); + let candidates = + ExactCompositionStrategy::default_cost_model().replacements(&TargetSubDAG::new(&root)); + let Replacement::ExactComposition(comp) = &candidates[0].replacement else { + unreachable!() + }; + let readout = + crate::replacement::realize_child(&comp.child_target, &DefaultCostModel).unwrap(); + assert!(!comp.accepts_child(&readout)); + assert!(matches!( + comp.compose(readout), + Err(ImplementError::ExecutionDataState( + ExecutionDataStateError::IllegalChildDataState { .. } + )) + )); + // Raw update input is fine. + let raw = keep_pre_asap(&comp.child_target).unwrap(); + assert!(comp.accepts_child(&raw)); + assert!(matches!( + comp.compose(raw).unwrap().expr, + SummaryExpr::ValueOperation { + timing: ExecutionTiming::MaintenanceTime, + .. + } + )); + } +} diff --git a/crates/asap-aware-mapping/src/explanation.rs b/crates/asap-aware-mapping/src/explanation.rs index 026df9161..3ddf2cbe6 100644 --- a/crates/asap-aware-mapping/src/explanation.rs +++ b/crates/asap-aware-mapping/src/explanation.rs @@ -216,6 +216,13 @@ pub enum ExplanationKind { /// cross-subpopulation reuse entries, all the same underlying structural /// fact. CommonSubexpressionReuse, + /// A `TargetSubDAG`'s candidate list contains at least one + /// [`Replacement::ExactComposition`] — + /// [`crate::exact_composition::ExactCompositionStrategy`] found an exact + /// operator that can be composed with a summary plan across an explicit + /// update/readout boundary instead of collapsing the whole tree into + /// `KeepPreAsap` (issue #171). + ExactComposition, } /// Why a [`Replacement`] of `kind` exists at `location` (a human-readable @@ -319,6 +326,15 @@ fn findings_from_plan_space(space: &PlanSpace) -> Vec) -> Vec Option { + let reasons: Vec<&str> = group + .candidates + .iter() + .filter(|c| matches!(c.replacement, Replacement::ExactComposition(_))) + .map(|c| c.rationale.as_str()) + .collect(); + if reasons.is_empty() { + None + } else { + Some(reasons.join("; ")) + } +} + /// Does `group`'s candidate list contain a genuine sketch-family realization? /// If so, the finding's `reason` is every such candidate's own `rationale`, /// joined — this module does not invent new prose to restate why a candidate diff --git a/crates/asap-aware-mapping/src/function_rules.rs b/crates/asap-aware-mapping/src/function_rules.rs new file mode 100644 index 000000000..82ad4e205 --- /dev/null +++ b/crates/asap-aware-mapping/src/function_rules.rs @@ -0,0 +1,41 @@ +//! Function facts shared by value-operation propagation and accumulator realization. +//! Runtime support remains a deployment decision in `CostModel`. +use asap_types::post_asap::{CompositionOperator, ExactKind, ExactParams}; +use asap_types::pre_asap::AggIntent; + +pub(crate) struct FunctionRules { + pub accuracy: CompositionOperator, + pub accumulator: Option<(ExactKind, ExactParams)>, +} + +/// Unregistered functions have no approximate-input propagation rule. +pub(crate) fn function_rules(intent: &AggIntent) -> Option { + let (accuracy, accumulator) = match intent { + AggIntent::Sum { .. } => ( + CompositionOperator::ExactSum, + Some((ExactKind::Sum, ExactParams::Sum)), + ), + AggIntent::Min { .. } | AggIntent::Max { .. } => ( + CompositionOperator::ExactExtremum, + Some((ExactKind::MinMax, ExactParams::MinMax)), + ), + AggIntent::Avg { .. } => (CompositionOperator::ExactAverage, None), + AggIntent::Rate => ( + CompositionOperator::CounterRate, + Some((ExactKind::Rate, ExactParams::Rate)), + ), + AggIntent::IRate => ( + CompositionOperator::InstantCounterRate, + Some((ExactKind::IRate, ExactParams::IRate)), + ), + AggIntent::Increase => ( + CompositionOperator::CounterIncrease, + Some((ExactKind::Increase, ExactParams::Increase)), + ), + _ => return None, + }; + Some(FunctionRules { + accuracy, + accumulator, + }) +} diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 89a44accc..34a420c33 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -189,7 +189,9 @@ pub mod cost_model; pub mod empirical_comparison; pub mod empirical_cost; pub mod empirical_resources; +pub mod exact_composition; pub mod explanation; +mod function_rules; pub mod grouping; pub mod physical_operator_statistics; pub mod physical_plan_cost_model; @@ -210,7 +212,13 @@ pub use accuracy::{ PropagationStats, WorkloadAccuracyEvidence, }; pub use accuracy_reconciliation::AccuracyReconciliationStrategy; -pub use cost_model::{CompleteSummaryCandidateEstimate, CostModel, DefaultCostModel}; +pub use cost_model::CompleteSummaryCandidateEstimate; +pub use cost_model::{ + maintenance_operation_plan_cost_rate, raw_recompute_cost_rate, read_operation_plan_cost_rate, + CostModel, CostProvenance, CostUnit, DefaultCostModel, ExactCompositionCostInputs, + ExactCompositionCostRequest, ValueOperationCapabilities, +}; +pub use exact_composition::{ExactComposition, ExactCompositionStrategy, OperationPlacement}; pub use explanation::{ explain_replacements, explain_replacements_with, ExplanationKind, ReplacementExplanation, }; @@ -222,11 +230,11 @@ pub use recurrence::{ }; pub use replacement::{ default_strategies, default_strategies_with, search_workload, search_workload_with, - search_workload_with_targets, summary_candidates, GlobalSelection, ImplementError, - Implementation, Matcher, MemoGroup, PlanSpace, Proposals, RankedGroup, RecurrenceProfileMap, - RejectedCandidate, Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, - SelectedGroup, SharedSubtreeStrategy, SketchAlgorithmStrategy, TargetSubDAG, - MAX_SEARCH_ITERATIONS, + search_workload_with_targets, summary_candidates, CompositionDecision, GlobalSelection, + ImplementError, Implementation, Matcher, MemoGroup, PlanSpace, Proposals, RankedGroup, + RecurrenceProfileMap, RejectedCandidate, Replacement, ReplacementProvenance, + ReplacementStrategy, ReplacementSubDAG, SelectedGroup, SharedSubtreeStrategy, + SketchAlgorithmStrategy, TargetSubDAG, MAX_SEARCH_ITERATIONS, }; pub use rewrite::{AvgToSumOverCountStrategy, SemanticEquivalentRewriteStrategy}; pub use summary_maintenance_dag_export::{ diff --git a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs index ad70cdc3e..a416be50c 100644 --- a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs +++ b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs @@ -185,6 +185,9 @@ impl<'a> PhysicalPlanCostModel<'a> { snapshot: &snapshot, }; let replacement = match &candidate.replacement { + Replacement::ExactComposition(_) => { + return Err(AnalyticalCostError::UnsupportedCandidate) + } Replacement::Rewrite(query) => lower_query_physical_dag(query, scope, &evidence)?, Replacement::Summary(summary) => match &summary.expr { SummaryExpr::KeepPreAsap(query) => { diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 4a3b12afc..5b895cfb2 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -345,15 +345,17 @@ //! multi-group joint optimization beyond this per-site recurrence is left //! for whenever that changes. +use std::cell::RefCell; use std::collections::{HashMap, HashSet, VecDeque}; -use asap_types::post_asap::{AccuracyError, CompositionOperator, GuaranteeSource, ResultGuarantee}; use asap_types::post_asap::{ - EntityIdentity, ExactKind, ExactParams, GroupingStrategy, SamplingKind, SamplingParams, - SketchAlgorithm, SketchKind, SketchParams, SketchQuery as PostAsapSketchQuery, StatModelKind, - StatModelParams, SummaryExpr, SummaryFamilyType, SummaryField, SummaryInputExpr, SummaryNode, - SummarySchema, SummaryUpdate, WaveletKind, WaveletParams, + validate_execution_data_states_at, EntityIdentity, ExactKind, ExactOperationSchemaError, + ExactParams, ExecutionDataState, ExecutionDataStateError, GroupingStrategy, SamplingKind, + SamplingParams, SketchAlgorithm, SketchKind, SketchParams, SketchQuery as PostAsapSketchQuery, + StatModelKind, StatModelParams, SummaryExpr, SummaryFamilyType, SummaryField, SummaryInputExpr, + SummaryNode, SummarySchema, SummaryUpdate, WaveletKind, WaveletParams, }; +use asap_types::post_asap::{AccuracyError, CompositionOperator, GuaranteeSource, ResultGuarantee}; use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; use asap_types::pre_asap::cse::{share_common_subtrees, structural_hash, HashCache}; use asap_types::pre_asap::expr_ir::ColumnRef; @@ -370,8 +372,13 @@ use crate::accuracy::{ KLL_RANK_ERROR_EXPONENT_99, }; use crate::accuracy_reconciliation::AccuracyReconciliationStrategy; -use crate::cost_model::{Cost, CostModel, CseCandidate, DefaultCostModel, ShareDecision}; +use crate::cost_model::{ + raw_recompute_cost_rate, Cost, CostModel, CseCandidate, DefaultCostModel, + ExactCompositionCostInputs, ExactCompositionCostRequest, ShareDecision, +}; +use crate::exact_composition::{ExactComposition, ExactCompositionStrategy, OperationPlacement}; use crate::grouping::HydraGroupingStrategy; +use crate::recurrence::CostRate; use crate::recurrence::{ evaluation_rate_of, Horizon, RecurrenceError, RecurrenceProfile, RootRecurrence, UpdateRate, }; @@ -402,6 +409,15 @@ pub enum ImplementError { /// would change its semantics. #[error("unsupported physical summary realization: {0}")] PhysicalRealization(&'static str), + /// A constructed plan violates the update/readout phase contract + /// (issue #171) — e.g. a summary readout placed beneath a maintained + /// `SummaryAgg`. Detected at construction, never at runtime. + #[error("execution-data_state violation in post-ASAP plan: {0}")] + ExecutionDataState(#[from] ExecutionDataStateError), + /// An `ExactOperator`'s output schema could not be derived over its + /// child — the child carries summary state the operator can't read. + #[error("exact operator schema derivation failed: {0}")] + ExactOperationSchema(#[from] ExactOperationSchemaError), } /// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. @@ -461,6 +477,15 @@ pub enum Replacement { /// different from the target's own `root` (e.g. sharing vs. not sharing /// a subtree) but semantically equivalent to it. Rewrite(Rc), + /// An exact operator composed over another target's *own* selected + /// decision across an explicit update/readout boundary (issue #171): + /// `ValueOperationAtReadTime` over a child's summary readout, or + /// `ValueOperationAtMaintenanceTime` feeding a maintained summary above. Carries only a + /// reference to the child target — [`PlanSpace::global_selection`] + /// commits the compatible parent/child pair and + /// [`GlobalSelection::materialize`] links it into one validated + /// `SummaryNode`. See [`crate::exact_composition`]. + ExactComposition(ExactComposition), } /// One candidate replacement for a [`TargetSubDAG`], plus a human-readable @@ -502,6 +527,12 @@ pub enum ReplacementProvenance { /// regardless, so pricing it like a full independent rebuild would be /// the wrong shape of cost, not just the wrong number. AccuracyReconciliation, + /// [`Replacement::ExactComposition`] with + /// [`OperationPlacement::Read`] (issue #171). + ValueOperationAtReadTime, + /// [`Replacement::ExactComposition`] with + /// [`OperationPlacement::Maintenance`] (issue #171). + ValueOperationAtMaintenanceTime, } /// A candidate a strategy considered for a target but refused to propose on @@ -527,6 +558,7 @@ pub struct RejectedCandidate { pub struct Proposals { pub candidates: Vec, pub rejected: Vec, + domain_error: Option, } /// A replacement strategy: given a [`TargetSubDAG`], does this strategy have @@ -572,6 +604,7 @@ pub trait ReplacementStrategy { Proposals { candidates: self.replacements(target), rejected: Vec::new(), + domain_error: None, } } } @@ -749,25 +782,16 @@ pub(crate) fn implementations_for_with( }, // ── Exact mergeable accumulators ───────────────────────────────────── - AggIntent::Sum { .. } => vec![exact_accumulator(intent, ExactKind::Sum, ExactParams::Sum)], - AggIntent::Min { .. } | AggIntent::Max { .. } => { - vec![exact_accumulator( - intent, - ExactKind::MinMax, - ExactParams::MinMax, - )] - } - AggIntent::Rate => vec![exact_accumulator( - intent, - ExactKind::Rate, - ExactParams::Rate, - )], - AggIntent::Increase => { - vec![exact_accumulator( - intent, - ExactKind::Increase, - ExactParams::Increase, - )] + AggIntent::Sum { .. } + | AggIntent::Min { .. } + | AggIntent::Max { .. } + | AggIntent::Rate + | AggIntent::IRate + | AggIntent::Increase => { + let (kind, params) = crate::function_rules::function_rules(intent) + .and_then(|rules| rules.accumulator) + .expect("exact accumulator intents have registered realizations"); + vec![exact_accumulator(intent, kind, params)] } // ── Exact, non-mergeable reducers — richer partial state than a @@ -1391,6 +1415,22 @@ impl<'a> SketchAlgorithmStrategy<'a> { ); } } + if proposals.candidates.is_empty() { + if let Some(error) = &proposals.domain_error { + if let Ok(node) = keep_pre_asap(root) { + proposals.candidates.push(ReplacementSubDAG { + strategy: "SketchAlgorithmStrategy", + replacement: Replacement::Summary(node), + provenance: ReplacementProvenance::SummaryImplementation, + rationale: format!( + "{} stays pre-ASAP because summary construction crosses an illegal \ + execution-data_state boundary ({error})", + describe_intent(intent) + ), + }); + } + } + } proposals } } @@ -1412,7 +1452,14 @@ impl Proposals { description: rationale, error, }), - Err(ImplementError::Schema(_) | ImplementError::PhysicalRealization(_)) => {} + Err(ImplementError::ExecutionDataState(error)) => { + self.domain_error.get_or_insert(error); + } + Err( + ImplementError::Schema(_) + | ImplementError::ExactOperationSchema(_) + | ImplementError::PhysicalRealization(_), + ) => {} } } } @@ -1563,10 +1610,10 @@ pub(crate) fn realize_child_with( .. }) => Ok(node), Some(ReplacementSubDAG { - replacement: Replacement::Rewrite(_), + replacement: Replacement::Rewrite(_) | Replacement::ExactComposition(_), .. }) => { - unreachable!("SketchAlgorithmStrategy never returns a Rewrite candidate") + unreachable!("SketchAlgorithmStrategy never returns a Rewrite/composition candidate") } // No candidate at all: `root` isn't `bindable_intent` shape (or its // intent has no realization `implementations_for_with` can't @@ -2142,8 +2189,6 @@ fn compose_guarantee( let (op, local) = match (family, query) { (SummaryFamilyType::ExactAggregate(kind, _), _) => { let op = match kind { - ExactKind::Sum => CompositionOperator::ExactSum, - ExactKind::MinMax => CompositionOperator::ExactExtremum, // A row count does not depend on the rows' values: exact // regardless of the child's own error. ExactKind::Count => { @@ -2154,9 +2199,11 @@ fn compose_guarantee( // Counter-reset detection over perturbed values has no finite // Lipschitz constant — over an approximate child this is a // deterministic transform with no registered rule. - ExactKind::Increase | ExactKind::Rate => CompositionOperator::Lipschitz { - constant: f64::INFINITY, - }, + _ => { + crate::function_rules::function_rules(intent) + .expect("exact accumulator intents have registered accuracy rules") + .accuracy + } }; ( op, @@ -2430,10 +2477,13 @@ impl MemoGroup { (Replacement::Summary(existing_node), Replacement::Summary(node)) => { is_duplicate_summary(existing_node, node) } - // A `Rewrite` and a `Summary` are never the same candidate — - // they're different `Replacement` variants entirely. - (Replacement::Rewrite(_), Replacement::Summary(_)) - | (Replacement::Summary(_), Replacement::Rewrite(_)) => false, + ( + Replacement::ExactComposition(existing), + Replacement::ExactComposition(candidate), + ) => existing.same_as(candidate), + // Different `Replacement` variants are never the same + // candidate. + _ => false, } }); if is_duplicate { @@ -2528,6 +2578,73 @@ pub struct PlanSpace { /// Discovery order — stable iteration for [`PlanSpace::groups`]/ /// [`PlanSpace::cost_sorted`], since `HashMap` iteration order isn't. order: Vec<*const QueryExpr>, + /// Composition proofs are computed with the search model, then retained + /// through costing and materialization so no later default can replace it. + composition_plans: Vec, +} + +struct PreparedComposition { + target: *const QueryExpr, + operation: ExactComposition, + child: Rc, + plan: Rc, +} + +impl PlanSpace { + fn prepare_compositions( + &mut self, + accuracy: &dyn AccuracyModel, + targets: &HashMap<*const QueryExpr, Vec>, + ) { + self.composition_plans.clear(); + for group in self.groups.values() { + for candidate in &group.candidates { + let Replacement::ExactComposition(operation) = &candidate.replacement else { + continue; + }; + let children: Vec<_> = match operation.placement { + OperationPlacement::Read => self + .groups + .get(&Rc::as_ptr(&operation.child_target)) + .into_iter() + .flat_map(|g| &g.candidates) + .filter_map(|c| match &c.replacement { + Replacement::Summary(child) if operation.accepts_child(child) => { + Some(Rc::clone(child)) + } + _ => None, + }) + .collect(), + OperationPlacement::Maintenance => { + keep_pre_asap(&operation.child_target).into_iter().collect() + } + }; + for child in children { + let Ok(plan) = operation.compose_with_accuracy(Rc::clone(&child), accuracy) + else { + continue; + }; + if let Some(requirements) = targets.get(&Rc::as_ptr(&group.target)) { + if operation.placement == OperationPlacement::Maintenance + || !requirements.iter().all(|target| { + plan.guarantee + .as_ref() + .is_some_and(|g| accuracy.satisfies(g, target)) + }) + { + continue; + } + } + self.composition_plans.push(PreparedComposition { + target: Rc::as_ptr(&group.target), + operation: operation.clone(), + child, + plan, + }); + } + } + } + } } /// Lifecycle-aware whole-subplan costs keyed by target and candidate identity. @@ -3151,7 +3268,7 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R .iter() .map(|c| match &c.replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }) .collect(); if let Some(kinds) = kinds { @@ -3159,7 +3276,7 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R ranked.sort_by_key(|c| { let kind = match &c.replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }; kind.and_then(|k| order.iter().position(|o| *o == k)) .unwrap_or(usize::MAX) @@ -3287,6 +3404,34 @@ pub struct SelectedGroup<'a> { /// registered strategy proposed anything for (mirrors /// [`MemoGroup::candidates`] being possibly empty). pub chosen: Option<&'a ReplacementSubDAG>, + /// When `chosen` is a [`Replacement::ExactComposition`]: the child + /// decision it was committed together with, and the cost comparison + /// that justified it — the explicit target-to-decision provenance + /// chain (issue #171). + pub composition: Option>, +} + +/// Why [`PlanSpace::global_selection`] committed an exact composition at a +/// site: which child candidate it composes with, and the +/// cost-units-per-second comparison against the raw fallback that it won. +#[derive(Debug)] +pub struct CompositionDecision<'a> { + /// The exact child/operation pair validated by the search accuracy model. + pub plan: Rc, + /// The child target the composed operator consumes. + pub child_target: &'a Rc, + /// For a read-time operation: the child's own candidate committed alongside + /// (the summary readout the operator folds). `None` for an update-path + /// transform, whose input is raw update data — its cost is charged to + /// the maintained summary *above* it instead. + pub child_candidate: Option<&'a ReplacementSubDAG>, + /// The composed plan's recurring rate — `read_operation_plan_cost_rate` + /// or `maintenance_operation_plan_cost_rate`. + pub cost_rate: CostRate, + /// `raw_recompute_cost_rate` — the `KeepPreAsap` baseline it beat. + pub baseline_rate: CostRate, + /// The statistics (and their provenance) both rates were computed from. + pub inputs: ExactCompositionCostInputs, } /// [`PlanSpace::global_selection`]'s result: one [`SelectedGroup`] per @@ -3296,6 +3441,10 @@ pub struct SelectedGroup<'a> { pub struct GlobalSelection<'a> { order: Vec<*const QueryExpr>, groups: HashMap<*const QueryExpr, SelectedGroup<'a>>, + /// [`Self::materialize`]'s memo — one bound node per target for the + /// life of this selection, so two parents composing over one shared + /// child get the *same* `Rc`. + materialized: RefCell>>, } impl<'a> GlobalSelection<'a> { @@ -3311,22 +3460,296 @@ impl<'a> GlobalSelection<'a> { self.groups.get(&Rc::as_ptr(target)) } - /// Materialize the selected replacement at `target`. Exact operators - /// that remain in pre-ASAP IR are preserved by `KeepPreAsap`; logical - /// summary candidates are already fully bound post-ASAP nodes. + /// Link this selection's per-site decisions into one data_state-validated + /// post-ASAP DAG rooted at `target` — the one place a committed + /// composition's child *reference* becomes an actual `Rc` + /// edge (issue #171). `None` if `target` is not a discovered site. + /// + /// Per site: a [`Replacement::ExactComposition`] uses its validated + /// operation/child plan, retaining the search model's guarantee; + /// a [`Replacement::Summary`] is + /// re-linked so its `SummaryAgg` child is the child target's own + /// materialization whenever that is phase-legal beneath maintenance + /// (so a child that chose an `ValueOperationAtMaintenanceTime` actually ends up under + /// the summary); a [`Replacement::Rewrite`] or an unmatched site stays + /// the conservative `KeepPreAsap`. Memoized by target identity, so a + /// shared inner summary is one `Rc` no matter how many roots reach it. pub fn materialize( &self, target: &Rc, ) -> Result>, ImplementError> { - let Some(selected) = self.for_target(target) else { + if !self.groups.contains_key(&Rc::as_ptr(target)) { return Ok(None); + } + self.materialize_inner(target).map(Some) + } + + fn materialize_inner(&self, target: &Rc) -> Result, ImplementError> { + let ptr = Rc::as_ptr(target); + if let Some(node) = self.materialized.borrow().get(&ptr) { + return Ok(Rc::clone(node)); + } + let node = match self + .groups + .get(&ptr) + .and_then(|sel| sel.chosen) + .map(|c| &c.replacement) + { + None => keep_pre_asap(target)?, + Some(Replacement::Rewrite(rewritten)) => keep_pre_asap(rewritten)?, + Some(Replacement::Summary(node)) => self.relink_summary(node, target)?, + Some(Replacement::ExactComposition(_)) => Rc::clone( + &self.groups[&ptr] + .composition + .as_ref() + .expect("selected compositions have a validated decision") + .plan, + ), + }; + self.materialized.borrow_mut().insert(ptr, Rc::clone(&node)); + Ok(node) + } + + /// Re-link a bound `Summary` candidate's `SummaryAgg` child to the + /// child target's own materialization when that is legal beneath + /// maintenance; otherwise keep the candidate exactly as constructed. + fn relink_summary( + &self, + node: &Rc, + target: &Rc, + ) -> Result, ImplementError> { + let QueryExpr::Aggregate { + child: pre_child, .. + } = target.as_ref() + else { + return Ok(Rc::clone(node)); + }; + let has_maintenance_operation = self + .groups + .get(&Rc::as_ptr(pre_child)) + .and_then(|selection| selection.chosen) + .is_some_and(|candidate| { + matches!( + &candidate.replacement, + Replacement::ExactComposition(composition) + if composition.placement == OperationPlacement::Maintenance + ) + }); + if !has_maintenance_operation { + return Ok(Rc::clone(node)); + } + let new_child = self.materialize_inner(pre_child)?; + Ok(relink_agg_child(node, &new_child)) + } +} + +/// Rebuild `node` (a `SummaryAgg`, possibly under a `SummaryEstimate`) with +/// `new_child` as the `SummaryAgg`'s child, if the result still validates +/// as maintained state; otherwise return `node` unchanged. +fn relink_agg_child(node: &Rc, new_child: &Rc) -> Rc { + match &node.expr { + SummaryExpr::SummaryEstimate { + summary_input, + query, + } => { + let inner = relink_agg_child(summary_input, new_child); + if Rc::ptr_eq(&inner, summary_input) { + return Rc::clone(node); + } + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryEstimate { + summary_input: inner, + query: query.clone(), + }, + schema: node.schema.clone(), + guarantee: node.guarantee.clone(), + }) + } + SummaryExpr::SummaryAgg { + child, + family, + input, + reduction, + grouping, + } => { + if Rc::ptr_eq(child, new_child) { + return Rc::clone(node); + } + let rebuilt = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child: Rc::clone(new_child), + family: family.clone(), + input: input.clone(), + reduction: reduction.clone(), + grouping: grouping.clone(), + }, + schema: node.schema.clone(), + guarantee: node.guarantee.clone(), + }); + match validate_execution_data_states_at( + &rebuilt, + ExecutionDataState::MAINTENANCE_SUMMARY, + ) { + Ok(_) => rebuilt, + Err(_) => Rc::clone(node), + } + } + _ => Rc::clone(node), + } +} + +/// The maintained `SummaryAgg` a bound `Summary` candidate builds (under +/// its `SummaryEstimate` readout, if any) — the summary an `ValueOperationAtMaintenanceTime` +/// beneath it feeds, for `maintenance_operation_plan_cost_rate`. +fn maintained_summary(node: &Rc) -> Option<&Rc> { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => maintained_summary(summary_input), + SummaryExpr::SummaryAgg { .. } => Some(node), + _ => None, + } +} + +fn is_composition_candidate(candidate: &ReplacementSubDAG) -> bool { + matches!(candidate.replacement, Replacement::ExactComposition(_)) +} + +/// Everything [`PlanSpace::global_selection`] threads between sites for +/// exact compositions (issue #171): child candidates already committed by +/// an earlier parent, and the maintained summary above each site. +#[derive(Default)] +struct CompositionContext { + /// child target ptr → the child's candidate an ancestor's composition + /// already committed to (a later parent must compose with the *same* + /// one, and the child's own selection is forced to it). + committed_child: HashMap<*const QueryExpr, *const ReplacementSubDAG>, + /// site ptr → the maintained `SummaryAgg` directly above it, when its + /// parent chose a bound `Summary` — what an `ValueOperationAtMaintenanceTime` here feeds. + maintaining_parent: HashMap<*const QueryExpr, Rc>, +} + +/// One eligible composed alternative at a site, before the cheapest wins. +struct CompositionOption<'a> { + candidate: &'a ReplacementSubDAG, + decision: CompositionDecision<'a>, +} + +/// Every [`Replacement::ExactComposition`] candidate of `group` whose +/// composed-plan rate is *known* and beats the raw-recompute baseline — +/// costed against each compatible child candidate already in `PlanSpace` +/// (or the one an earlier parent committed). Unknown statistics yield no +/// option at all: the conservative `KeepPreAsap` path stays. +fn composition_options<'a>( + group: &'a MemoGroup, + groups: &'a HashMap<*const QueryExpr, MemoGroup>, + effective: usize, + cost_model: &dyn CostModel, + context: &CompositionContext, + plans: &[PreparedComposition], +) -> Vec> { + let mut options = Vec::new(); + for candidate in &group.candidates { + let Replacement::ExactComposition(composition) = &candidate.replacement else { + continue; + }; + let child_ptr = Rc::as_ptr(&composition.child_target); + let Some(child_group) = groups.get(&child_ptr) else { + continue; + }; + let already_committed = context.committed_child.get(&child_ptr).copied(); + let cost = |summary: &SummaryNode, shared: bool| { + let request = ExactCompositionCostRequest { + target: &group.target, + composition, + summary, + effective_consumer_count: effective, + }; + let mut inputs = cost_model.exact_composition_cost_inputs(&request); + if shared { + // Shared state is counted once: an earlier parent already + // pays this child's maintenance, so the marginal cost here + // is zero — a *known* zero, unlike an unknown input. + if let Some(maintenance) = inputs.summary_maintenance_cost_per_update.as_mut() { + *maintenance = 0.0; + } + } + let rate = inputs.composed_plan_cost_rate(composition.placement)?; + let baseline = raw_recompute_cost_rate(&inputs)?; + (rate < baseline).then_some((rate, baseline, inputs)) }; - match selected.chosen.map(|candidate| &candidate.replacement) { - Some(Replacement::Summary(node)) => Ok(Some(Rc::clone(node))), - Some(Replacement::Rewrite(rewritten)) => keep_pre_asap(rewritten).map(Some), - None => keep_pre_asap(target).map(Some), + match composition.placement { + OperationPlacement::Read => { + let child_candidates: Vec<&'a ReplacementSubDAG> = match already_committed { + // SAFETY-free: the pointer was taken from `groups`'s own + // candidate storage, which outlives this borrow. + Some(ptr) => child_group + .candidates + .iter() + .filter(|c| std::ptr::eq(*c, ptr)) + .collect(), + None => child_group.candidates.iter().collect(), + }; + for child_candidate in child_candidates { + let Replacement::Summary(summary) = &child_candidate.replacement else { + continue; + }; + if !composition.accepts_child(summary) { + continue; + } + let Some(prepared) = plans.iter().find(|p| { + p.target == Rc::as_ptr(&group.target) + && p.operation.same_as(composition) + && Rc::ptr_eq(&p.child, summary) + }) else { + continue; + }; + let Some((rate, baseline, inputs)) = cost(summary, already_committed.is_some()) + else { + continue; + }; + options.push(CompositionOption { + candidate, + decision: CompositionDecision { + plan: Rc::clone(&prepared.plan), + child_target: &composition.child_target, + child_candidate: Some(child_candidate), + cost_rate: rate, + baseline_rate: baseline, + inputs, + }, + }); + } + } + OperationPlacement::Maintenance => { + let Some(prepared) = plans.iter().find(|p| { + p.target == Rc::as_ptr(&group.target) && p.operation.same_as(composition) + }) else { + continue; + }; + // An maintenance-time operation only pays off beneath a + // maintained summary; with nothing above it, its output is + // never read and the raw fallback is the same computation. + let Some(parent) = context.maintaining_parent.get(&Rc::as_ptr(&group.target)) + else { + continue; + }; + let Some((rate, baseline, inputs)) = cost(parent, false) else { + continue; + }; + options.push(CompositionOption { + candidate, + decision: CompositionDecision { + plan: Rc::clone(&prepared.plan), + child_target: &composition.child_target, + child_candidate: None, + cost_rate: rate, + baseline_rate: baseline, + inputs, + }, + }); + } } } + options } impl PlanSpace { @@ -3378,6 +3801,7 @@ impl PlanSpace { let mut effective_uses = graph.external_root_uses.clone(); let mut chosen_share: HashMap<*const QueryExpr, ShareDecision> = HashMap::new(); let mut groups: HashMap<*const QueryExpr, SelectedGroup<'_>> = HashMap::new(); + let mut context = CompositionContext::default(); for ptr in &topo { let group = &self.groups[ptr]; @@ -3385,38 +3809,83 @@ impl PlanSpace { let effective = effective_uses.get(ptr).copied().unwrap_or(0); effective_uses.insert(*ptr, effective); + // ── Exact compositions (issue #171) ───────────────────────── + // A child an earlier parent's composition committed to is + // forced to exactly that candidate — the parent/child pair is + // one decision. Otherwise, a composition here wins only when + // its cost-units-per-second rate is *known* and beats the raw + // recompute baseline; missing statistics keep the conservative + // path below. + let mut composition_decision = None; + let forced = context + .committed_child + .get(ptr) + .and_then(|&cptr| group.candidates.iter().find(|c| std::ptr::eq(*c, cptr))); + let composed = if forced.is_some() { + None + } else { + composition_options( + group, + &self.groups, + effective, + cost_model, + &context, + &self.composition_plans, + ) + .into_iter() + .min_by(|a, b| a.decision.cost_rate.0.total_cmp(&b.decision.cost_rate.0)) + }; + if let Some(option) = &composed { + if let Some(child_candidate) = option.decision.child_candidate { + context.committed_child.insert( + Rc::as_ptr(option.decision.child_target), + child_candidate as *const ReplacementSubDAG, + ); + } + if let Replacement::ExactComposition(composition) = &option.candidate.replacement { + if composition.placement == OperationPlacement::Maintenance { + // A chain of functions feeds the same summary. + if let Some(parent) = context.maintaining_parent.get(ptr).cloned() { + context + .maintaining_parent + .insert(Rc::as_ptr(&composition.child_target), parent); + } + } + } + } + let lifecycle_choice = candidate_costs .filter(|costs| costs.finalizes(&group.target)) .map(|costs| { let summary = group .candidates .iter() + .filter(|candidate| !is_composition_candidate(candidate)) .filter_map(|candidate| { costs .get(&group.target, candidate) .map(|cost| (candidate, cost)) }) - .min_by(|(_, a), (_, b)| a.0.total_cmp(&b.0)); + .min_by(|(_, left), (_, right)| left.0.total_cmp(&right.0)); match (summary, costs.raw(&group.target)) { (Some((_, summary_cost)), Some(raw)) if raw.0 <= summary_cost.0 => None, (Some((candidate, _)), _) => Some(candidate), - (None, Some(_)) => None, - (None, None) => None, + (None, _) => None, } }); - let chosen = if let Some(lifecycle_choice) = lifecycle_choice { - lifecycle_choice - } else if cost_model.candidate_cost_covers_complete_plan() { + + let complete_plan_choice = (!forced.is_some() + && composed.is_none() + && lifecycle_choice.is_none() + && cost_model.candidate_cost_covers_complete_plan()) + .then(|| { let effective_target = TargetSubDAG::with_consumer_count(&group.target, effective); - let bound_physical = group + let bound = group .candidates .iter() - // A logical CSE rewrite does not encode shared retained - // state or independent execution multiplicity. Until it - // is bound as a complete physical DAG, it must not enter - // evidence-backed ranking as though those costs were - // known. - .filter(|candidate| !is_cse_candidate(candidate)) + .filter(|candidate| { + !is_cse_candidate(candidate) && !is_composition_candidate(candidate) + }) .filter_map(|candidate| { cost_model .candidate_cost(candidate, &effective_target) @@ -3424,7 +3893,7 @@ impl PlanSpace { }) .min_by(|(_, left), (_, right)| left.0.total_cmp(&right.0)) .map(|(candidate, _)| candidate); - bound_physical.or_else(|| { + bound.or_else(|| { (effective >= 2) .then(|| { decide_with_effective_count(group, effective, cost_model).and_then( @@ -3436,6 +3905,18 @@ impl PlanSpace { }) .flatten() }) + }) + .flatten(); + + let chosen = if let Some(forced) = forced { + Some(forced) + } else if let Some(option) = composed { + composition_decision = Some(option.decision); + Some(option.candidate) + } else if let Some(choice) = lifecycle_choice { + choice + } else if cost_model.candidate_cost_covers_complete_plan() { + complete_plan_choice } else if effective >= 2 && cse_candidate_pair(group).is_some() { let decision = if let Some(profiles) = profiles { decide_group_with_recurrence( @@ -3456,7 +3937,9 @@ impl PlanSpace { let logical = group .candidates .iter() - .filter(|candidate| !is_cse_candidate(candidate)) + .filter(|candidate| { + !is_cse_candidate(candidate) && !is_composition_candidate(candidate) + }) .filter_map(|candidate| { cost_model .candidate_cost(candidate, &effective_target) @@ -3499,12 +3982,13 @@ impl PlanSpace { // group also contributes no Share collapse to its own // children (see `multiplier`'s `_ => effective` arm). None => rank_group(group, cost_model).into_iter().find(|candidate| { - cost_model - .candidate_cost( - candidate, - &TargetSubDAG::with_consumer_count(&group.target, effective), - ) - .is_some() + !is_composition_candidate(candidate) + && cost_model + .candidate_cost( + candidate, + &TargetSubDAG::with_consumer_count(&group.target, effective), + ) + .is_some() }), } } else { @@ -3513,6 +3997,7 @@ impl PlanSpace { .into_iter() .find(|candidate| { !is_cse_candidate(candidate) + && !is_composition_candidate(candidate) && cost_model .candidate_cost(candidate, &effective_target) .is_some() @@ -3528,6 +4013,19 @@ impl PlanSpace { }) }; + // Record the maintained summary this site's bound candidate + // builds, for a child that may compose an `ValueOperationAtMaintenanceTime` + // beneath it. + if let (Some(Replacement::Summary(node)), QueryExpr::Aggregate { child, .. }) = + (chosen.map(|c| &c.replacement), group.target.as_ref()) + { + if let Some(summary) = maintained_summary(node) { + context + .maintaining_parent + .insert(Rc::as_ptr(child), Rc::clone(summary)); + } + } + let outgoing_multiplier = multiplier(*ptr, &effective_uses, &chosen_share); match chosen { Some(ReplacementSubDAG { @@ -3545,7 +4043,9 @@ impl PlanSpace { _ => { let selected_rewrite = match chosen.map(|candidate| &candidate.replacement) { Some(Replacement::Rewrite(rewrite)) => rewrite, - Some(Replacement::Summary(_)) | None => &group.target, + Some(Replacement::Summary(_) | Replacement::ExactComposition(_)) | None => { + &group.target + } }; for (child, edge_count) in direct_child_counts(selected_rewrite) { *effective_uses.entry(child).or_insert(0) += @@ -3561,6 +4061,7 @@ impl PlanSpace { consumer_count: group.consumer_count, effective_consumer_count: effective, chosen, + composition: composition_decision, }, ); } @@ -3568,6 +4069,7 @@ impl PlanSpace { Ok(GlobalSelection { order: self.order.clone(), groups, + materialized: RefCell::new(HashMap::new()), }) } } @@ -3943,7 +4445,8 @@ pub fn default_strategies() -> Vec> { Box::new(SketchAlgorithmStrategy::default_cost_model()), Box::new(HydraGroupingStrategy::default_cost_model()), Box::new(SharedSubtreeStrategy), - Box::new(crate::rewrite::SemanticEquivalentRewriteStrategy), + Box::new(crate::rewrite::AvgToSumOverCountStrategy), + Box::new(ExactCompositionStrategy::default_cost_model()), ] } @@ -3958,6 +4461,7 @@ pub fn default_strategies_with<'a>( Box::new(HydraGroupingStrategy::new(cost_model)), Box::new(SharedSubtreeStrategy), Box::new(crate::rewrite::SemanticEquivalentRewriteStrategy), + Box::new(ExactCompositionStrategy::new(cost_model)), ] } @@ -3984,6 +4488,7 @@ pub fn default_strategies_with_evidence<'a>( )), Box::new(SharedSubtreeStrategy), Box::new(crate::rewrite::AvgToSumOverCountStrategy), + Box::new(ExactCompositionStrategy::new(cost_model)), ] } @@ -4023,7 +4528,9 @@ pub fn search_workload_with<'s, Id>( roots: Vec<(Id, Rc)>, strategies: &[Box], ) -> PlanSpace { - search_cse_workload_with(cse_workload(roots), strategies) + let mut space = search_cse_workload_with(cse_workload(roots), strategies); + space.prepare_compositions(&DefaultAccuracyModel, &HashMap::new()); + space } /// [`search_workload_with`] plus a per-root end-to-end `AccuracyTarget` @@ -4055,7 +4562,7 @@ pub fn search_workload_with_targets<'s, Id>( (id, root) }) .collect(); - let mut space = search_workload_with(roots, strategies); + let mut space = search_cse_workload_with(cse_workload(roots), strategies); // `cse_workload` preserves root order, so targets zip by position. let root_ptrs: Vec<(*const QueryExpr, AccuracyTarget)> = space .roots @@ -4063,7 +4570,12 @@ pub fn search_workload_with_targets<'s, Id>( .zip(targets) .filter_map(|((_, root), target)| target.map(|t| (Rc::as_ptr(root), t))) .collect(); + let mut composition_targets: HashMap<_, Vec<_>> = HashMap::new(); for (ptr, target) in root_ptrs { + composition_targets + .entry(ptr) + .or_default() + .push(target.clone()); let Some(group) = space.groups.get_mut(&ptr) else { continue; }; @@ -4077,6 +4589,9 @@ pub fn search_workload_with_targets<'s, Id>( .as_ref() .is_some_and(|g| accuracy_model.satisfies(g, &target)), Replacement::Rewrite(_) => true, + // A composition's guarantee depends on the concrete child; + // prepare_compositions checks those pairs after all roots. + Replacement::ExactComposition(_) => true, }); group.candidates = legal; group.rejected.extend(illegal.into_iter().map(|candidate| { @@ -4097,6 +4612,11 @@ pub fn search_workload_with_targets<'s, Id>( None, )), Replacement::Rewrite(_) => unreachable!("rewrites are never rejected here"), + Replacement::ExactComposition(_) => ( + asap_types::post_asap::ErrorMetric::AbsoluteValue, + None, + None, + ), }; RejectedCandidate { strategy: candidate.strategy, @@ -4110,6 +4630,7 @@ pub fn search_workload_with_targets<'s, Id>( } })); } + space.prepare_compositions(accuracy_model, &composition_targets); space } @@ -4267,6 +4788,7 @@ fn search_cse_workload_with<'s, Id>( roots: cse_roots, groups, order, + composition_plans: Vec::new(), } } @@ -4559,6 +5081,7 @@ mod tests { (A::Min { col: None }, Acc(E::MinMax)), (A::Max { col: None }, Acc(E::MinMax)), (A::Rate, Acc(E::Rate)), + (A::IRate, Acc(E::IRate)), (A::Increase, Acc(E::Increase)), // exact but non-mergeable → pass-through (A::Avg { col: None }, Pass), @@ -5050,7 +5573,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert!(kinds.contains(&SketchAlgorithm::Kll), "{kinds:?}"); @@ -5070,7 +5595,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert_eq!( @@ -5098,7 +5625,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert_eq!(kinds, vec![SketchAlgorithm::Theta, SketchAlgorithm::Kmv]); @@ -5176,7 +5705,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert!(kinds.contains(&SketchAlgorithm::Kll)); @@ -5184,13 +5715,10 @@ mod tests { assert_eq!(kinds.len(), 2); } - /// Enumerating candidates for the *target* node must only steer that - /// node's own decision — a nested aggregate underneath it still gets its - /// own independent (`cost_model`-ranked) enumeration, not whatever the - /// caller happened to pick for the outer target. This is the behavior - /// [`construct_summary`]'s recursion (via [`realize_child`]) - /// gets for free: only the top node's `Implementation` is ever forced - /// from outside; the child is always re-enumerated fresh. + /// Constructing the outer target's candidates never leaks its algorithm + /// choice into the nested aggregate. Existing approximate composition + /// remains governed by the accuracy model, independently of #171's exact + /// value-operation candidates. #[test] fn enumerating_the_targets_candidates_does_not_leak_into_a_nested_aggregate() { // outer: quantile(0.99, ...) over inner: quantile(0.5, m) — both @@ -5210,35 +5738,33 @@ mod tests { ) .replacements(&target); - let ddsketch = replacements + assert_eq!(replacements.len(), 2, "{replacements:?}"); + assert!(replacements .iter() - .find(|r| { - matches!(&r.replacement, Replacement::Summary(node) - if summary_family_algorithm(node) == SketchAlgorithm::DDSketch) - }) - .expect("the outer target's DDSketch candidate must be present"); - let Replacement::Summary(node) = &ddsketch.replacement else { - unreachable!("filtered on Replacement::Summary above"); - }; - assert_eq!( - summary_family_algorithm(node), - SketchAlgorithm::DDSketch, - "the outer (target) node must be the DDSketch candidate" + .all(|candidate| { matches!(candidate.replacement, Replacement::Summary(_)) })); + // The inner target is still independently enumerated and ranked — + // a custom cost model that prefers DDSketch for it is honored, and + // nothing about the outer target's choice reaches it. + let space = search_workload_with( + vec![("q", Rc::clone(&outer))], + &default_strategies_with(&PreferDDSketchViaCostModel), ); - - let asap_types::post_asap::SummaryExpr::SummaryEstimate { summary_input, .. } = &node.expr - else { - panic!("expected SummaryEstimate root, got {:?}", node.expr); - }; - let asap_types::post_asap::SummaryExpr::SummaryAgg { child, .. } = &summary_input.expr - else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); + let QueryExpr::Aggregate { child, .. } = space.roots[0].1.as_ref() else { + unreachable!() }; + let inner_group = space.group_for(child).expect("inner quantile is a target"); + let inner_kinds: Vec = inner_group + .candidates + .iter() + .filter_map(|c| match &c.replacement { + Replacement::Summary(node) => sketch_kind_of(node), + _ => None, + }) + .collect(); assert_eq!( - summary_family_algorithm(child), - SketchAlgorithm::Kll, - "the nested inner aggregate must still get the cost-model-ranked \ - default (Kll), not inherit the outer target's DDSketch candidate" + inner_kinds, + vec![SketchAlgorithm::DDSketch, SketchAlgorithm::Kll], + "the nested inner aggregate keeps its own cost-model-ranked candidates" ); } @@ -5765,7 +6291,7 @@ mod tests { assert_eq!(rewrites.len(), 2); let first_shares_target = match &rewrites[0].replacement { Replacement::Rewrite(rc) => Rc::ptr_eq(rc, &group.target), - Replacement::Summary(_) => false, + Replacement::Summary(_) | Replacement::ExactComposition(_) => false, }; assert!( first_shares_target, @@ -5801,7 +6327,7 @@ mod tests { assert_eq!(agg_group.candidates.len(), 2); let first_kind = match &agg_group.candidates[0].replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }; assert_eq!(first_kind, Some(SketchAlgorithm::DDSketch)); } @@ -5997,7 +6523,7 @@ mod tests { .unwrap(); let kind = match &agg_group.chosen.unwrap().replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }; assert_eq!(kind, Some(SketchAlgorithm::DDSketch)); } @@ -6463,6 +6989,7 @@ mod tests { roots, groups, order: order.clone(), + composition_plans: Vec::new(), }; let graph = reference_graph(&space); @@ -7673,6 +8200,7 @@ mod tests { DefaultAccuracyModel.satisfies(g, &AccuracyTarget::Epsilon(0.1)) }), Replacement::Rewrite(_) => false, + Replacement::ExactComposition(_) => false, })); let ranked = space.cost_sorted(&DefaultCostModel); let root_ranked = ranked.iter().find(|g| Rc::ptr_eq(g.target, root)).unwrap(); @@ -7740,6 +8268,7 @@ mod tests { .as_ref() .is_some_and(ResultGuarantee::is_exact), Replacement::Rewrite(_) => true, + Replacement::ExactComposition(_) => false, })); } diff --git a/crates/asap-aware-mapping/src/rollup.rs b/crates/asap-aware-mapping/src/rollup.rs index 5dc3c81e7..a5abae8a4 100644 --- a/crates/asap-aware-mapping/src/rollup.rs +++ b/crates/asap-aware-mapping/src/rollup.rs @@ -690,7 +690,7 @@ mod tests { .iter() .find_map(|candidate| match &candidate.replacement { Replacement::Rewrite(rewrite) => Some(rewrite), - Replacement::Summary(_) => None, + Replacement::Summary(_) | Replacement::ExactComposition(_) => None, }) .expect("default search must include the roll-up rewrite"); let QueryExpr::Aggregate { child, .. } = rewrite.as_ref() else { diff --git a/crates/asap-aware-mapping/src/summary_maintenance_cost/estimator.rs b/crates/asap-aware-mapping/src/summary_maintenance_cost/estimator.rs index d7c28ae1e..840a19ad6 100644 --- a/crates/asap-aware-mapping/src/summary_maintenance_cost/estimator.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_cost/estimator.rs @@ -29,7 +29,9 @@ pub(super) fn estimate_heterogeneous_summary( } match &node.expr { SummaryExpr::KeepPreAsap(query) => query_source_selections(query, out)?, - SummaryExpr::SummaryAgg { child, .. } => summary_source_selections(child, seen, out)?, + SummaryExpr::SummaryAgg { child, .. } | SummaryExpr::ValueOperation { child, .. } => { + summary_source_selections(child, seen, out)? + } SummaryExpr::SummaryMerge { children } => { for child in children { summary_source_selections(child, seen, out)?; @@ -290,6 +292,23 @@ pub(super) fn estimate_heterogeneous_summary( io_bytes, )?; } + SummaryExpr::ValueOperation { child, .. } => { + let operation = summary_operation_evidence(node, evidence)?.resource(); + *cpu_ops += evaluation_count as f64 + * validated_operator_executions("value_operation", operation)? as f64 + * validated_operator_cpu("value_operation", operation.cpu_ops)?; + add_operator_io(io_bytes, operation, evaluation_count)?; + visit_ops( + child, + seen, + by_node, + evidence, + scope, + evaluation_count, + cpu_ops, + io_bytes, + )?; + } SummaryExpr::KeepPreAsap(_) => { let retained = evidence .retained_queries @@ -391,6 +410,7 @@ pub(super) fn estimate_heterogeneous_summary( out.push(node as *const _); collect_aggs(child, seen, out); } + SummaryExpr::ValueOperation { child, .. } => collect_aggs(child, seen, out), SummaryExpr::SummaryMerge { children } => { children .iter() @@ -591,7 +611,9 @@ fn validate_summary_edges_and_physical_ids( fn children(node: &SummaryNode) -> Vec<&SummaryNode> { match &node.expr { SummaryExpr::KeepPreAsap(_) => vec![], - SummaryExpr::SummaryAgg { child, .. } => vec![child], + SummaryExpr::SummaryAgg { child, .. } | SummaryExpr::ValueOperation { child, .. } => { + vec![child] + } SummaryExpr::SummaryMerge { children } => { children.iter().map(|child| child.as_ref()).collect() } @@ -760,7 +782,9 @@ pub(super) fn estimate_transient_liveness( fn children(node: &SummaryNode) -> Vec<&SummaryNode> { match &node.expr { SummaryExpr::KeepPreAsap(_) => vec![], - SummaryExpr::SummaryAgg { child, .. } => vec![child], + SummaryExpr::SummaryAgg { child, .. } | SummaryExpr::ValueOperation { child, .. } => { + vec![child] + } SummaryExpr::SummaryMerge { children } => { children.iter().map(|child| child.as_ref()).collect() } @@ -816,6 +840,7 @@ pub(super) fn estimate_transient_liveness( .ok_or(AnalyticalCostError::MissingOrStale("summary_join")), SummaryExpr::SummaryMerge { .. } | SummaryExpr::BinaryOp { .. } + | SummaryExpr::ValueOperation { .. } | SummaryExpr::SummarySubtract { .. } | SummaryExpr::SummaryDelete { .. } | SummaryExpr::SummaryEstimate { .. } => { @@ -880,6 +905,7 @@ pub(super) fn evidence_nodes(root: &SummaryNode) -> (Vec<&SummaryNode>, Vec<&Sum aggregations.push(node); visit(child, seen, aggregations, joins); } + SummaryExpr::ValueOperation { child, .. } => visit(child, seen, aggregations, joins), SummaryExpr::SummaryMerge { children } => { for child in children { visit(child, seen, aggregations, joins); @@ -1242,6 +1268,7 @@ fn count_operations(root: &SummaryNode) -> Result visit(child, seen, counts)?, SummaryExpr::SummaryDelete { summary_input, .. } => { counts.deletes_per_update = counts .deletes_per_update diff --git a/crates/asap-aware-mapping/src/summary_maintenance_cost/model.rs b/crates/asap-aware-mapping/src/summary_maintenance_cost/model.rs index e7371650c..447659003 100644 --- a/crates/asap-aware-mapping/src/summary_maintenance_cost/model.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_cost/model.rs @@ -677,6 +677,7 @@ impl CostModel for SummaryMaintenanceCostModel { _target: &TargetSubDAG<'_>, ) -> Option { match &candidate.replacement { + Replacement::ExactComposition(_) => None, // Lifecycle selection supplies a complete override. If it cannot, // the candidate remains unavailable rather than receiving this // trait's structural fallback. @@ -3174,7 +3175,8 @@ mod tests { }, ); } - SummaryExpr::SummaryAgg { child, .. } => retained(model, child, seen), + SummaryExpr::SummaryAgg { child, .. } + | SummaryExpr::ValueOperation { child, .. } => retained(model, child, seen), SummaryExpr::SummaryMerge { children } => { for child in children { retained(model, child, seen); @@ -3367,6 +3369,9 @@ mod tests { owners.push(node as *const _); owning_aggs(child, seen, owners); } + SummaryExpr::ValueOperation { child, .. } => { + owning_aggs(child, seen, owners) + } SummaryExpr::SummaryMerge { children } => { for child in children { owning_aggs(child, seen, owners); @@ -3406,7 +3411,10 @@ mod tests { } } match &node.expr { - SummaryExpr::SummaryAgg { child, .. } => bind_ops(model, child, seen, inputs, cpu), + SummaryExpr::SummaryAgg { child, .. } + | SummaryExpr::ValueOperation { child, .. } => { + bind_ops(model, child, seen, inputs, cpu) + } SummaryExpr::SummaryMerge { children } => { for child in children { bind_ops(model, child, seen, inputs, cpu); diff --git a/crates/asap-aware-mapping/src/summary_maintenance_cost/window.rs b/crates/asap-aware-mapping/src/summary_maintenance_cost/window.rs index 41938ab73..a8ae5674a 100644 --- a/crates/asap-aware-mapping/src/summary_maintenance_cost/window.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_cost/window.rs @@ -44,6 +44,7 @@ pub(super) fn summary_aggregation_identities(root: &SummaryNode) -> HashSet<*con out.insert(node as *const _); visit(child, seen, out); } + SummaryExpr::ValueOperation { child, .. } => visit(child, seen, out), SummaryExpr::SummaryMerge { children } => { for child in children { visit(child, seen, out); diff --git a/crates/asap-aware-mapping/src/summary_maintenance_dag_export.rs b/crates/asap-aware-mapping/src/summary_maintenance_dag_export.rs index 156e53168..353ba4619 100644 --- a/crates/asap-aware-mapping/src/summary_maintenance_dag_export.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_dag_export.rs @@ -199,6 +199,7 @@ fn summary_children(expr: &SummaryExpr) -> Vec<&Rc> { SummaryExpr::KeepPreAsap(_) => vec![], SummaryExpr::BinaryOp { lhs, rhs, .. } => vec![lhs, rhs], SummaryExpr::SummaryAgg { child, .. } => vec![child], + SummaryExpr::ValueOperation { child, .. } => vec![child], SummaryExpr::SummaryJoin { outer, inner, .. } | SummaryExpr::SummarySubtract { left: outer, diff --git a/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs b/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs index ccd4e0c48..cd6d60a8c 100644 --- a/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs @@ -960,6 +960,7 @@ fn collect_summary_aggs( output.push(Rc::clone(node)); collect_summary_aggs(child, seen, output); } + SummaryExpr::ValueOperation { child, .. } => collect_summary_aggs(child, seen, output), SummaryExpr::SummaryJoin { outer, inner, .. } | SummaryExpr::BinaryOp { lhs: outer, diff --git a/crates/devtools/src/bin/analyze_corpora.rs b/crates/devtools/src/bin/analyze_corpora.rs index b76a00865..ef1df13cd 100644 --- a/crates/devtools/src/bin/analyze_corpora.rs +++ b/crates/devtools/src/bin/analyze_corpora.rs @@ -378,8 +378,7 @@ fn anomaly_report(all: &[DumpRecord], language: &str, manual_notes: &str) -> Str report.push_str("- **Serialization caveat:** JSON renders non-finite floating-point values such as `NaN` and `±Inf` as `null`; use the `ir_debug` field in each per-query file when reviewing those values. Fingerprints use this lossless debug representation.\n"); if language == "PromQL" { report.push_str("- **Known semantic collapse to review:** `rate(cumulative[5m])` and `"); - report - .push_str("`irate(cumulative[5m])` both produce `AggIntent::Rate`. Confirm that this "); + report.push_str("`irate(cumulative[5m])` produces `AggIntent::IRate`. Confirm that this "); report.push_str( "abstraction is intentional; PromQL defines different sampling behavior for ", ); diff --git a/crates/devtools/src/bin/dag_export.rs b/crates/devtools/src/bin/dag_export.rs index 8c95fab66..08df63beb 100644 --- a/crates/devtools/src/bin/dag_export.rs +++ b/crates/devtools/src/bin/dag_export.rs @@ -1050,6 +1050,9 @@ fn target_replacement( Replacement::Rewrite(rewritten) => { TargetReplacementAfter::Rewrite(dag_export::export(rewritten)) } + Replacement::ExactComposition(_) => { + unreachable!("composition candidates are materialized by GlobalSelection") + } }; let (baseline_cost, selected_cost, benefit) = winner.costs.clone(); // Derived from `selected_cost` so the legacy scalar field and the @@ -1198,6 +1201,14 @@ fn run_post_asap_with_progress( .groups() .filter_map(|group| { let candidate = group.chosen?; + // A composition is a reference-bearing logical choice, not an + // independently exportable replacement. The CLI's default cost + // model has no composition statistics and therefore cannot + // select one; callers that provide such statistics must export + // `GlobalSelection::materialize` instead. + if matches!(candidate.replacement, Replacement::ExactComposition(_)) { + return None; + } if matches!( &candidate.replacement, Replacement::Summary(node) if matches!(node.expr, SummaryExpr::KeepPreAsap(_)) @@ -1276,6 +1287,9 @@ fn run_post_asap_with_progress( replacement: Rc::clone(rc), decision, }, + Replacement::ExactComposition(_) => { + unreachable!("composition winners are excluded above") + } }) }; let post_graphs: Vec<(String, DagGraph)> = lowered_queries @@ -2179,6 +2193,9 @@ mod tests { Replacement::Rewrite(rewrite) => { serde_json::to_value(dag_export::export(rewrite)).unwrap() } + Replacement::ExactComposition(_) => { + unreachable!("cost fixtures select directly materialized candidates") + } } } @@ -2209,6 +2226,9 @@ mod tests { Replacement::Rewrite(rewrite) => { serde_json::to_value(dag_export::export(rewrite)).unwrap() } + Replacement::ExactComposition(_) => { + unreachable!("cost fixtures select directly materialized candidates") + } }; let document = PlannerCostDocument { storage_io: None, @@ -2233,6 +2253,9 @@ mod tests { plan, query_nodes: query_evidence(&query), }, + Replacement::ExactComposition(_) => { + unreachable!("cost fixtures select directly materialized candidates") + } }], }], }; diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs index a38184249..74fc8d2b1 100644 --- a/crates/frontend-promql/src/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -34,7 +34,7 @@ //! | `count_over_time(m[w])` | `Aggregate{[Count], TimeRange{w}}` | //! | `last/first/mad/ts_of_min/ts_of_max/ts_of_first/ts_of_last_over_time(m[w])` | `Aggregate{[Last/First/Mad/TsOf…OverTime], TimeRange{w}}` — per-series range reducers (issue #51) | //! | `sort`/`sort_desc(v)`, `sort_by_label[_desc](v,"l"…)` | `Sort{value \| label…}` (no `Limit`) — row-preserving reorder (issue #51); `min_of`/`max_of` scalar reducers → #89 | -//! | `rate/irate(m[w])` | `Aggregate{[Rate], TimeRange{w}}` — `irate` shares the `rate` *intent*; the avg-vs-last-two-samples difference is a post-ASAP estimation method | +//! | `rate(m[w])` / `irate(m[w])` | `Aggregate{[Rate/IRate], TimeRange{w}}` — distinct function identities; shared physical machinery is a later realization choice | //! | `increase(m[w])` | `Aggregate{[Increase], TimeRange{w}}` | //! | `changes`/`delta`/`idelta`/`deriv`/`resets`/`predict_linear`/`double_exponential_smoothing`(`m[w]`, …) | `Aggregate{[Changes/Delta/…], TimeRange{w}}` — per-series counter-derivative intents (issue #44) | //! | `absent(v)` / `absent_over_time(m[w])` / `present_over_time(m[w])` | `Aggregate{[Absent/AbsentOverTime/PresentOverTime]}` — presence intents; the empty→synthesized-sample logic is a post-ASAP concern (issue #47) | @@ -130,6 +130,7 @@ enum InnerFunc { // window field either; `windowed_aggregate` reads `Inner.window` // uniformly for every intent, so it would be a redundant duplicate here. Rate, + IRate, Increase, // Counter-derivative range functions (issue #44). The window rides on the // enclosing `TimeRange` node (like `*_over_time`), so these carry only @@ -351,10 +352,11 @@ fn range_fn_over_subquery(call: &Call) -> Result> { if subquery_range(arg_expr).is_none() { return Ok(None); } - let inner = if call.func.name == "increase" { - InnerFunc::Increase - } else { - InnerFunc::Rate + let inner = match call.func.name { + "rate" => InnerFunc::Rate, + "irate" => InnerFunc::IRate, + "increase" => InnerFunc::Increase, + _ => unreachable!(), }; return Ok(Some(outer_aggregate( vec![], @@ -1311,7 +1313,11 @@ fn lower_inner_call(call: &Call) -> Result { metric, matchers, window: Some(window), - func: Some(InnerFunc::Rate), + func: Some(if name == "irate" { + InnerFunc::IRate + } else { + InnerFunc::Rate + }), shift, }) } @@ -1645,6 +1651,7 @@ fn inner_intent(f: &InnerFunc) -> AggIntent { accuracy: current_accuracy(), }, InnerFunc::Rate => AggIntent::Rate, + InnerFunc::IRate => AggIntent::IRate, InnerFunc::Increase => AggIntent::Increase, InnerFunc::Changes => AggIntent::Changes, InnerFunc::Delta => AggIntent::Delta, diff --git a/crates/frontend-promql/tests/promql_conformance.rs b/crates/frontend-promql/tests/promql_conformance.rs index ef806de5b..b29e5ea10 100644 --- a/crates/frontend-promql/tests/promql_conformance.rs +++ b/crates/frontend-promql/tests/promql_conformance.rs @@ -266,11 +266,10 @@ fn rate_range_lives_in_time_range_node() { } #[test] -fn irate_maps_to_rate_intent() { - // SEMANTICS: instant rate from the last two samples; same intent vocabulary. +fn irate_maps_to_its_own_intent() { assert!(has(&ok("irate(http_requests_total[1m])"), |i| matches!( i, - AggIntent::Rate + AggIntent::IRate ))); } @@ -1504,7 +1503,7 @@ fn range_functions_over_a_subquery_reduce_per_series() { for (q, want) in [ ("rate(sum(m)[5m:])", AggIntent::Rate), ("increase(sum(m)[5m:])", AggIntent::Increase), - ("irate(sum(m)[5m:])", AggIntent::Rate), // irate shares the Rate intent + ("irate(sum(m)[5m:])", AggIntent::IRate), ("changes(rate(m[5m])[1h:])", AggIntent::Changes), ("delta(sum(m)[5m:])", AggIntent::Delta), ("deriv(sum(m)[10m:])", AggIntent::Deriv), diff --git a/crates/frontend-promql/tests/promql_equivalence.rs b/crates/frontend-promql/tests/promql_equivalence.rs index 3d03fc648..010463223 100644 --- a/crates/frontend-promql/tests/promql_equivalence.rs +++ b/crates/frontend-promql/tests/promql_equivalence.rs @@ -120,17 +120,12 @@ fn distinct_semantics_stay_distinct() { } // ───────────────────────────────────────────────────────────────────────────── -// 3. Intentional intent-level equivalence (documented, not a bug). +// 3. Intent-level distinctions and equivalences. // ───────────────────────────────────────────────────────────────────────────── #[test] -fn rate_and_irate_share_the_same_intent() { - // The canonical tree captures *intent* ("per-second rate of a counter"), - // not the estimation method. `rate` (windowed average) and `irate` (last - // two samples) differ only in HOW the rate is estimated — a - // post-ASAP/execution concern — so they share one canonical intent by - // design. - assert_equiv(&["rate(m[5m])", "irate(m[5m])"]); +fn rate_and_irate_are_distinct_functions() { + assert_distinct("rate(m[5m])", "irate(m[5m])"); } #[test] diff --git a/crates/integration-tests/tests/exact_composition.rs b/crates/integration-tests/tests/exact_composition.rs new file mode 100644 index 000000000..6540ffcce --- /dev/null +++ b/crates/integration-tests/tests/exact_composition.rs @@ -0,0 +1,754 @@ +//! Issue #171 — composing exact operators with summary plans across +//! explicit update/readout boundaries, end to end through +//! `search_workload_with` → `PlanSpace::global_selection` → +//! `GlobalSelection::materialize` → `dag_export`. +//! +//! Covers the issue's integration matrix: both nesting directions, grouped +//! fine-to-coarse and identity folds, one inner summary shared by several +//! queries, illegal readout-under-maintenance rejection, a runtime without +//! the capability, a cost model without statistics, and pre/post-ASAP +//! schemas plus shared `Rc` identity — along with pins for every +//! already-supported exact-accumulator nesting. + +use std::rc::Rc; + +use asap_aware_mapping::cost_model::{ + CostProvenance, CostUnit, ExactCompositionCostInputs, ExactCompositionCostRequest, + ValueOperationCapabilities, +}; +use asap_aware_mapping::replacement::{ + default_strategies_with, search_workload_with, ImplementError, Replacement, + ReplacementProvenance, ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG, +}; +use asap_aware_mapping::{ + CostModel, DefaultCostModel, EvaluationRate, ExplanationKind, OperationPlacement, +}; +use asap_frontend_promql::lower_promql; +use asap_types::dag_export; +use asap_types::post_asap::{ + validate_execution_data_states, ExactKind, ExecutionDataState, ExecutionDataStateError, + ExecutionTiming, SketchAlgorithm, SummaryExpr, SummaryFamilyType, SummaryNode, SummaryUpdate, +}; +use asap_types::pre_asap::agg_intent::{default_quantile, AggIntent}; +use asap_types::pre_asap::query_expr::{QueryExpr, Reduction, Source}; +use asap_types::pre_asap::schema::{Column, DataType, Schema}; +use asap_types::types::AccuracyTarget; + +// ── fixtures ──────────────────────────────────────────────────────────── + +fn metric_scan(labels: &[&str]) -> QueryExpr { + let mut columns = vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ]; + columns.extend(labels.iter().map(|n| Column::new(*n, DataType::Utf8, true))); + QueryExpr::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::with_time_index(columns, 0, vec![]), + } +} + +fn agg(by: Vec, intent: AggIntent, child: Rc) -> Rc { + Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(by), + measures: vec![intent], + output_names: vec![], + having: None, + child, + }) +} + +fn per_entity(intent: AggIntent, child: Rc) -> Rc { + Rc::new(QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![intent], + output_names: vec![], + having: None, + child, + }) +} + +/// `quantile by (zone, host) (latency)` — the fine-grained inner summary. +fn fine_quantile() -> Rc { + agg( + vec![2, 3], + default_quantile(0.99), + Rc::new(metric_scan(&["zone", "host"])), + ) +} + +/// A deployment cost model that supplies every statistic the issue's +/// formulas need, so a composition can actually win — and advertises both +/// mixed-execution shapes. +struct StatsModel; + +/// Search, selection, and materialization must retain the caller's proven rule. +#[test] +fn custom_accuracy_rule_survives_root_target_and_materialization() { + use asap_aware_mapping::{AccuracyModel, DefaultAccuracyModel, PropagationStats}; + use asap_types::post_asap::{ + AccuracyError, CompositionOperator, ExactOperation, ResultGuarantee, SketchQuery, + }; + struct Model; + impl AccuracyModel for Model { + fn exact_operation_rule(&self, _: &ExactOperation) -> Option { + Some(CompositionOperator::ExactExtremum) + } + fn local_guarantee( + &self, + family: &SummaryFamilyType, + query: &SketchQuery, + ) -> Option { + DefaultAccuracyModel.local_guarantee(family, query) + } + fn propagate( + &self, + _: &CompositionOperator, + _: &[ResultGuarantee], + _: Option<&ResultGuarantee>, + _: &PropagationStats, + ) -> Result { + // Test-only oracle: the marker detects accidental use of the default model. + Ok(ResultGuarantee::exact("custom rule oracle")) + } + fn satisfies(&self, g: &ResultGuarantee, t: &AccuracyTarget) -> bool { + DefaultAccuracyModel.satisfies(g, t) + } + } + let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + let space = asap_aware_mapping::replacement::search_workload_with_targets( + vec![("q", root, Some(AccuracyTarget::Exact))], + &default_strategies_with(&StatsModel), + &Model, + ); + let selection = space.global_selection(&StatsModel); + assert!(selection + .for_target(&space.roots[0].1) + .unwrap() + .composition + .is_some()); + let node = selection.materialize(&space.roots[0].1).unwrap().unwrap(); + let guarantee = node.guarantee.as_ref().unwrap(); + assert!(guarantee.is_exact()); + assert!(format!("{:?}", guarantee.provenance).contains("custom rule oracle")); +} + +/// An exact operator must not turn an unknown approximate-input bound into exactness. +#[test] +fn root_target_rejects_unproven_composition() { + let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + let space = asap_aware_mapping::replacement::search_workload_with_targets( + vec![("q", root, Some(AccuracyTarget::Exact))], + &default_strategies_with(&StatsModel), + &asap_aware_mapping::DefaultAccuracyModel, + ); + let selection = space.global_selection(&StatsModel); + assert!(selection + .for_target(&space.roots[0].1) + .unwrap() + .composition + .is_none()); +} + +impl CostModel for StatsModel { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + fn exact_composition_cost_inputs( + &self, + _request: &ExactCompositionCostRequest<'_>, + ) -> ExactCompositionCostInputs { + ExactCompositionCostInputs { + exact_cost_per_row: Some(0.1), + expected_input_rows: Some(50.0), + expected_output_rows: Some(10.0), + summary_maintenance_cost_per_update: Some(0.01), + summary_read_cost: Some(1.0), + update_rate: Some(100.0), + evaluation_rate: Some(EvaluationRate(1.0)), + raw_recompute_cost: Some(100.0), + unit: CostUnit::CostUnitsPerSecond, + provenance: CostProvenance { + model: "StatsModel".into(), + version: "test-1".into(), + }, + } + } +} + +/// Same statistics, but the runtime advertises no mixed-execution shape. +struct NoCapabilityModel; + +impl CostModel for NoCapabilityModel { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + fn value_operation_capabilities(&self) -> ValueOperationCapabilities { + ValueOperationCapabilities::NONE + } + fn exact_composition_cost_inputs( + &self, + request: &ExactCompositionCostRequest<'_>, + ) -> ExactCompositionCostInputs { + StatsModel.exact_composition_cost_inputs(request) + } +} + +fn plan( + roots: Vec<(&'static str, Rc)>, + cost_model: &dyn CostModel, +) -> asap_aware_mapping::PlanSpace<&'static str> { + search_workload_with(roots, &default_strategies_with(cost_model)) +} + +fn is_plain(node: &SummaryNode) -> bool { + node.schema + .fields + .iter() + .all(|f| matches!(f.dtype, SummaryFamilyType::Plain(_))) +} + +fn names(node: &SummaryNode) -> Vec<&str> { + node.schema.fields.iter().map(|f| f.name.as_str()).collect() +} + +// ── step 1: pin every already-supported exact-accumulator nesting ─────── + +#[test] +fn every_exact_accumulator_nests_directly_under_an_outer_sketch() { + use std::time::Duration; + let cases: Vec<(Rc, ExactKind)> = vec![ + ( + agg( + vec![2], + AggIntent::Sum { col: None }, + Rc::new(metric_scan(&["zone"])), + ), + ExactKind::Sum, + ), + ( + agg( + vec![2], + AggIntent::Count { + accuracy: AccuracyTarget::Exact, + }, + Rc::new(metric_scan(&["zone"])), + ), + ExactKind::Count, + ), + ( + agg( + vec![2], + AggIntent::Min { col: None }, + Rc::new(metric_scan(&["zone"])), + ), + ExactKind::MinMax, + ), + ( + agg( + vec![2], + AggIntent::Max { col: None }, + Rc::new(metric_scan(&["zone"])), + ), + ExactKind::MinMax, + ), + ( + per_entity( + AggIntent::Rate, + Rc::new(QueryExpr::TimeRange { + range: Duration::from_secs(300), + child: Rc::new(metric_scan(&["zone"])), + }), + ), + ExactKind::Rate, + ), + ( + per_entity( + AggIntent::Increase, + Rc::new(QueryExpr::TimeRange { + range: Duration::from_secs(300), + child: Rc::new(metric_scan(&["zone"])), + }), + ), + ExactKind::Increase, + ), + ]; + for (inner, kind) in cases { + let outer = agg(vec![], default_quantile(0.9), inner); + let target = TargetSubDAG::new(&outer); + let candidates = SketchAlgorithmStrategy::default_cost_model().replacements(&target); + let Replacement::Summary(root) = &candidates[0].replacement else { + unreachable!() + }; + let SummaryExpr::SummaryEstimate { summary_input, .. } = &root.expr else { + panic!("expected KLL readout, got {:?}", root.expr); + }; + let SummaryExpr::SummaryAgg { child, .. } = &summary_input.expr else { + panic!("expected outer SummaryAgg"); + }; + assert!( + matches!( + &child.expr, + SummaryExpr::SummaryAgg { family: SummaryFamilyType::ExactAggregate(k, _), .. } if *k == kind + ), + "{kind:?}: expected the exact accumulator directly under the outer sketch, got {:?}", + child.expr + ); + validate_execution_data_states(root).expect("accumulator state composes under maintenance"); + } +} + +// ── direction 1: outer exact fold over an inner summary readout ──────── + +/// Before this PR both `max`/`avg` over a quantile collapsed into one +/// opaque `KeepPreAsap`. Now: the outer group holds an `ValueOperationAtReadTime` +/// candidate referencing the inner target, the inner group keeps its own +/// sketch candidates, and with statistics the pair is committed and +/// materializes as `ValueOperationAtReadTime → SummaryEstimate → SummaryAgg`. +#[test] +fn max_and_avg_over_quantile_compose_at_read_time_with_statistics() { + for intent in [AggIntent::Max { col: None }, AggIntent::Avg { col: None }] { + let root = agg(vec![0], intent.clone(), fine_quantile()); + let space = plan(vec![("q", Rc::clone(&root))], &StatsModel); + let root = Rc::clone(&space.roots[0].1); + let QueryExpr::Aggregate { child: inner, .. } = root.as_ref() else { + unreachable!() + }; + + let outer_group = space.group_for(&root).unwrap(); + assert!( + outer_group + .candidates + .iter() + .any(|c| c.provenance == ReplacementProvenance::ValueOperationAtReadTime), + "{intent:?}: outer group must hold an ValueOperationAtReadTime candidate" + ); + let inner_group = space.group_for(inner).unwrap(); + assert!( + inner_group + .candidates + .iter() + .any(|c| matches!(&c.replacement, Replacement::Summary(n) + if matches!(n.expr, SummaryExpr::SummaryEstimate { .. }))), + "{intent:?}: the inner quantile keeps its own readout candidates" + ); + + let selection = space.global_selection(&StatsModel); + let selected = selection.for_target(&root).unwrap(); + let chosen = selected.chosen.expect("a decision"); + assert_eq!( + chosen.provenance, + ReplacementProvenance::ValueOperationAtReadTime + ); + let decision = selected + .composition + .as_ref() + .expect("composition provenance"); + assert!(Rc::ptr_eq(decision.child_target, inner)); + assert!(decision.cost_rate < decision.baseline_rate); + assert_eq!(decision.inputs.unit, CostUnit::CostUnitsPerSecond); + assert_eq!(decision.inputs.provenance.model, "StatsModel"); + // The child was committed to a compatible candidate *from its own + // group* — the same candidate its own selection reports. + let child_candidate = decision.child_candidate.expect("read-time operation child"); + let inner_selected = selection.for_target(inner).unwrap(); + assert!(std::ptr::eq( + inner_selected.chosen.unwrap(), + child_candidate + )); + + let composed = selection.materialize(&root).unwrap().unwrap(); + let SummaryExpr::ValueOperation { + child, + timing: ExecutionTiming::ReadTime, + .. + } = &composed.expr + else { + panic!( + "{intent:?}: expected ValueOperationAtReadTime root, got {:?}", + composed.expr + ); + }; + assert!(matches!(child.expr, SummaryExpr::SummaryEstimate { .. })); + assert!( + child.guarantee.is_some(), + "child has its KLL rank guarantee" + ); + assert!( + composed.guarantee.is_none(), + "rank error has no definition-backed conversion through max/average" + ); + assert!(is_plain(&composed)); + assert_eq!( + names(&composed), + root.output_schema() + .unwrap() + .columns + .iter() + .map(|c| c.name.as_str()) + .collect::>(), + "the composed plan's schema is the pre-ASAP target's own" + ); + validate_execution_data_states(&composed).unwrap(); + } +} + +/// `avg` keeps competing with `AvgToSumOverCountStrategy`: both candidates +/// live in the same group; nothing hard-codes the winner. +#[test] +fn avg_over_quantile_keeps_the_sum_over_count_rewrite_as_a_competitor() { + // `by (zone)` over `by (zone)`: the averaged column resolves to the + // non-null quantile output, which is what the rewrite requires. + let inner = agg( + vec![2], + default_quantile(0.99), + Rc::new(metric_scan(&["zone"])), + ); + let root = agg(vec![0], AggIntent::Avg { col: None }, inner); + let space = plan(vec![("q", root)], &StatsModel); + let group = space.group_for(&space.roots[0].1).unwrap(); + let provenances: Vec<_> = group.candidates.iter().map(|c| c.provenance).collect(); + assert!(provenances.contains(&ReplacementProvenance::LogicalRewrite)); + assert!(provenances.contains(&ReplacementProvenance::ValueOperationAtReadTime)); +} + +/// Grouped fine-to-coarse fold (`by (zone)` over `by (zone, host)`) and the +/// identity fold (`by (zone)` over `by (zone)`) both compose; the operator +/// is the same, only the fold's row multiplicity differs. +#[test] +fn identity_and_genuine_multi_row_folds_both_compose() { + let identity_inner = agg( + vec![2], + default_quantile(0.99), + Rc::new(metric_scan(&["zone"])), + ); + for (label, inner) in [ + ("identity", identity_inner), + ("fine-to-coarse", fine_quantile()), + ] { + let root = agg(vec![0], AggIntent::Max { col: None }, inner); + let space = plan(vec![("q", root)], &StatsModel); + let root = &space.roots[0].1; + let composed = space + .global_selection(&StatsModel) + .materialize(root) + .unwrap() + .unwrap(); + assert!( + matches!( + composed.expr, + SummaryExpr::ValueOperation { + timing: ExecutionTiming::ReadTime, + .. + } + ), + "{label}: {:?}", + composed.expr + ); + assert_eq!(names(&composed), vec!["zone", "max"], "{label}"); + } +} + +/// One inner quantile consumed by two outer folds in two queries: CSE +/// collapses the inner target onto one `Rc`, both compositions commit to +/// the *same* child candidate, and both materializations share one +/// `Rc` for it — the summary is maintained once. +#[test] +fn a_shared_inner_summary_is_materialized_once_for_several_outer_folds() { + let max = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + let min = agg(vec![0], AggIntent::Min { col: None }, fine_quantile()); + let space = plan(vec![("max", max), ("min", min)], &StatsModel); + let selection = space.global_selection(&StatsModel); + + let roots: Vec> = space.roots.iter().map(|(_, r)| Rc::clone(r)).collect(); + let inner_of = |r: &Rc| match r.as_ref() { + QueryExpr::Aggregate { child, .. } => Rc::clone(child), + _ => unreachable!(), + }; + assert!( + Rc::ptr_eq(&inner_of(&roots[0]), &inner_of(&roots[1])), + "CSE must intern the shared inner quantile" + ); + let inner = inner_of(&roots[0]); + assert_eq!(space.group_for(&inner).unwrap().consumer_count, 2); + + let decisions: Vec<_> = roots + .iter() + .map(|r| { + selection + .for_target(r) + .unwrap() + .composition + .as_ref() + .expect("both roots compose") + }) + .collect(); + assert!(std::ptr::eq( + decisions[0].child_candidate.unwrap(), + decisions[1].child_candidate.unwrap() + )); + // Shared state counted once: the second parent sees zero marginal + // maintenance, so its rate is strictly lower than the first's. + assert!(decisions[1].cost_rate < decisions[0].cost_rate); + + let composed: Vec<_> = roots + .iter() + .map(|r| selection.materialize(r).unwrap().unwrap()) + .collect(); + let child_of = |n: &Rc| match &n.expr { + SummaryExpr::ValueOperation { + child, + timing: ExecutionTiming::ReadTime, + .. + } => Rc::clone(child), + other => panic!("expected ValueOperationAtReadTime, got {other:?}"), + }; + assert!( + Rc::ptr_eq(&child_of(&composed[0]), &child_of(&composed[1])), + "both folds compose over the same Rc" + ); +} + +// ── direction 2: outer summary over an inner exact maintenance-time operation ─ + +/// `quantile(0.99, deriv(latency[5m]))`: `deriv` has no accumulator form. +/// The function target gets an `ValueOperationAtMaintenanceTime` candidate; with a +/// maintained summary above it and statistics, it is committed, and the +/// outer summary's materialization is re-linked over it. +#[test] +fn outer_summary_over_an_exact_function_composes_at_maintenance_time() { + use std::time::Duration; + let deriv = per_entity( + AggIntent::Deriv, + Rc::new(QueryExpr::TimeRange { + range: Duration::from_secs(300), + child: Rc::new(metric_scan(&["zone"])), + }), + ); + let root = agg(vec![], default_quantile(0.99), deriv); + let space = plan(vec![("q", root)], &StatsModel); + let root = Rc::clone(&space.roots[0].1); + let QueryExpr::Aggregate { child: deriv, .. } = root.as_ref() else { + unreachable!() + }; + assert!(space + .group_for(deriv) + .unwrap() + .candidates + .iter() + .any(|c| c.provenance == ReplacementProvenance::ValueOperationAtMaintenanceTime)); + + let selection = space.global_selection(&StatsModel); + let deriv_sel = selection.for_target(deriv).unwrap(); + assert_eq!( + deriv_sel.chosen.unwrap().provenance, + ReplacementProvenance::ValueOperationAtMaintenanceTime + ); + let decision = deriv_sel.composition.as_ref().unwrap(); + assert!(decision.child_candidate.is_none(), "function input is raw"); + assert!(decision.cost_rate < decision.baseline_rate); + + let composed = selection.materialize(&root).unwrap().unwrap(); + let SummaryExpr::SummaryEstimate { summary_input, .. } = &composed.expr else { + panic!("expected readout root, got {:?}", composed.expr); + }; + let SummaryExpr::SummaryAgg { child, .. } = &summary_input.expr else { + panic!("expected SummaryAgg"); + }; + let SummaryExpr::ValueOperation { + child: raw, + timing: ExecutionTiming::MaintenanceTime, + .. + } = &child.expr + else { + panic!( + "expected ValueOperationAtMaintenanceTime under the maintained summary, got {:?}", + child.expr + ); + }; + assert!(matches!(raw.expr, SummaryExpr::KeepPreAsap(_))); + let assignment = validate_execution_data_states(&composed).unwrap(); + assert_eq!( + assignment.data_state_of(child), + Some(ExecutionDataState::MAINTENANCE_ROWS) + ); + assert_eq!( + assignment.data_state_of(raw), + Some(ExecutionDataState::MAINTENANCE_ROWS) + ); +} + +// ── rejection, capability, statistics ─────────────────────────────────── + +/// A maintained summary above a query-time readout is a typed plan-time +/// error, both for the construction path and for a hand-built plan. +#[test] +fn readout_under_maintenance_is_rejected_at_construction() { + let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + // A read-time ValueOperation can never be placed under a SummaryAgg: compose a + // read-time operation, then try to maintain a summary over it. + let space = plan(vec![("q", Rc::clone(&root))], &StatsModel); + let post = space + .global_selection(&StatsModel) + .materialize(&space.roots[0].1) + .unwrap() + .unwrap(); + let illegal = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child: post, + family: SummaryFamilyType::ExactAggregate( + ExactKind::MinMax, + asap_types::post_asap::ExactParams::MinMax, + ), + input: SummaryUpdate::column(asap_types::pre_asap::ColumnRef::SampleValue), + reduction: Reduction::by(vec![]), + grouping: Default::default(), + }, + schema: asap_types::post_asap::SummarySchema { + fields: vec![], + time_index: None, + }, + guarantee: None, + }); + assert!(matches!( + validate_execution_data_states(&illegal), + Err(ExecutionDataStateError::ReadoutUnderMaintenance { .. }) + )); + let err: ImplementError = validate_execution_data_states(&illegal).unwrap_err().into(); + assert!(matches!(err, ImplementError::ExecutionDataState(_))); +} + +#[test] +fn a_runtime_without_mixed_execution_gets_no_composition_candidates() { + let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + let space = plan(vec![("q", root)], &NoCapabilityModel); + let root = Rc::clone(&space.roots[0].1); + let group = space.group_for(&root).unwrap(); + assert!(group + .candidates + .iter() + .all(|c| !matches!(c.replacement, Replacement::ExactComposition(_)))); + let selection = space.global_selection(&NoCapabilityModel); + assert!(selection.for_target(&root).unwrap().composition.is_none()); + let node = selection.materialize(&root).unwrap().unwrap(); + assert!(!matches!(node.expr, SummaryExpr::ValueOperation { .. })); + // The inner quantile is still independently selectable. + let QueryExpr::Aggregate { child, .. } = root.as_ref() else { + unreachable!() + }; + assert!(selection.for_target(child).unwrap().chosen.is_some()); +} + +/// Without statistics (the built-in model) the composition is *proposed* +/// — visible in `PlanSpace` and explanations — but never *selected*: the +/// site keeps a non-composed alternative, and the inner summary stays +/// independently selectable. +#[test] +fn missing_cost_statistics_preserve_the_conservative_keep_pre_asap() { + let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + let space = plan(vec![("q", root)], &DefaultCostModel); + let root = Rc::clone(&space.roots[0].1); + assert!(space + .group_for(&root) + .unwrap() + .candidates + .iter() + .any(|c| c.provenance == ReplacementProvenance::ValueOperationAtReadTime)); + let selection = space.global_selection(&DefaultCostModel); + let selected = selection.for_target(&root).unwrap(); + assert!(selected.composition.is_none()); + assert!(!matches!( + selected.chosen.map(|c| &c.replacement), + Some(Replacement::ExactComposition(_)) + )); + let node = selection.materialize(&root).unwrap().unwrap(); + assert!(!matches!(node.expr, SummaryExpr::ValueOperation { .. })); + + let explanations = asap_aware_mapping::explain_replacements(vec![("q", (*root).clone())]); + assert!(explanations + .iter() + .any(|e| e.kind == ExplanationKind::ExactComposition)); +} + +// ── DAG export: explicit stage, schema, provenance ─────────────────────── + +#[test] +fn dag_export_carries_explicit_stage_and_plain_schema_for_a_composed_plan() { + let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); + let space = plan(vec![("q", root)], &StatsModel); + let root = &space.roots[0].1; + let composed = space + .global_selection(&StatsModel) + .materialize(root) + .unwrap() + .unwrap(); + let graph = dag_export::export_summary(&composed); + let node = &graph.nodes[graph.root as usize]; + assert_eq!(node.kind, "ValueOperation"); + assert_eq!(node.detail["timing"], "read_time"); + assert!(node.detail["operation"] + .as_str() + .unwrap() + .starts_with("Exact(Aggregate")); + + // Pre-ASAP export of the same target still describes the same columns. + let pre = dag_export::export(root); + let pre_root = &pre.nodes[pre.root as usize]; + let pre_cols: Vec = pre_root.schema.as_ref().unwrap()["columns"] + .as_array() + .unwrap() + .iter() + .map(|c| c["name"].as_str().unwrap().to_string()) + .collect(); + assert_eq!(pre_cols, names(&composed)); +} + +/// The PromQL front end produces the exact issue shape and it composes. +#[test] +fn promql_max_by_zone_over_quantile_over_time_composes() { + let expr = lower_promql( + "max by (zone) (quantile_over_time(0.99, latency[5m]))", + AccuracyTarget::Epsilon(0.01), + ) + .unwrap(); + let space = plan(vec![("q", Rc::new(expr))], &StatsModel); + let root = &space.roots[0].1; + let selection = space.global_selection(&StatsModel); + let selected = selection.for_target(root).unwrap(); + assert_eq!( + selected.chosen.map(|c| c.provenance), + Some(ReplacementProvenance::ValueOperationAtReadTime), + "{:?}", + space + .group_for(root) + .unwrap() + .candidates + .iter() + .map(|c| (c.strategy, c.provenance)) + .collect::>() + ); + let composed = selection.materialize(root).unwrap().unwrap(); + assert!(matches!( + composed.expr, + SummaryExpr::ValueOperation { + timing: ExecutionTiming::ReadTime, + .. + } + )); + assert_eq!( + selected.composition.as_ref().map(|d| d.inputs.unit), + Some(CostUnit::CostUnitsPerSecond) + ); + let _ = OperationPlacement::Read; +} diff --git a/crates/types/src/dag_export.rs b/crates/types/src/dag_export.rs index 9c6ffe286..2b4f1bf9a 100644 --- a/crates/types/src/dag_export.rs +++ b/crates/types/src/dag_export.rs @@ -475,6 +475,7 @@ macro_rules! define_summary_kind_tags { define_summary_kind_tags! { SummaryExpr::BinaryOp { .. } => "SummaryBinaryOp", + SummaryExpr::ValueOperation { .. } => "ValueOperation", SummaryExpr::SummaryAgg { .. } => "SummaryAgg", SummaryExpr::SummaryJoin { .. } => "SummaryJoin", SummaryExpr::SummarySubtract { .. } => "SummarySubtract", @@ -497,6 +498,16 @@ fn summary_shape(expr: &SummaryExpr) -> (&'static str, String, serde_json::Value }); (kind, label, detail) } + SummaryExpr::ValueOperation { + operation, timing, .. + } => ( + kind, + format!("ValueOperation({operation:?})"), + serde_json::json!({ + "operation": format!("{operation:?}"), + "timing": timing.as_str(), + }), + ), SummaryExpr::SummaryAgg { family, input, @@ -549,6 +560,7 @@ fn summary_children(expr: &SummaryExpr) -> Vec<&Rc> { match expr { SummaryExpr::KeepPreAsap(_) => vec![], SummaryExpr::BinaryOp { lhs, rhs, .. } => vec![lhs, rhs], + SummaryExpr::ValueOperation { child, .. } => vec![child], SummaryExpr::SummaryAgg { child, .. } => vec![child], SummaryExpr::SummaryJoin { outer, inner, .. } => vec![outer, inner], SummaryExpr::SummarySubtract { left, right } => vec![left, right], diff --git a/crates/types/src/post_asap/cse.rs b/crates/types/src/post_asap/cse.rs index 4a3667432..439a4e213 100644 --- a/crates/types/src/post_asap/cse.rs +++ b/crates/types/src/post_asap/cse.rs @@ -38,6 +38,18 @@ fn same_node(left: &SummaryNode, right: &SummaryNode) -> bool { operator: bo, }, ) => Rc::ptr_eq(al, bl) && Rc::ptr_eq(ar, br) && ao == bo, + ( + ValueOperation { + child: ac, + operation: ao, + timing: at, + }, + ValueOperation { + child: bc, + operation: bo, + timing: bt, + }, + ) => Rc::ptr_eq(ac, bc) && same_value(ao, bo) && at == bt, ( SummaryAgg { child: ac, @@ -105,6 +117,7 @@ fn same_node(left: &SummaryNode, right: &SummaryNode) -> bool { ( KeepPreAsap(_) | BinaryOp { .. } + | ValueOperation { .. } | SummaryAgg { .. } | SummaryJoin { .. } | SummarySubtract { .. } @@ -143,6 +156,7 @@ pub fn share_common_summary_subtrees( *lhs = visit(lhs, seen, pool); *rhs = visit(rhs, seen, pool); } + SummaryExpr::ValueOperation { child, .. } => *child = visit(child, seen, pool), SummaryExpr::SummaryJoin { outer, inner, .. } => { *outer = visit(outer, seen, pool); *inner = visit(inner, seen, pool); diff --git a/crates/types/src/post_asap/execution_data_state.rs b/crates/types/src/post_asap/execution_data_state.rs new file mode 100644 index 000000000..dc0a7745c --- /dev/null +++ b/crates/types/src/post_asap/execution_data_state.rs @@ -0,0 +1,942 @@ +//! Execution-data-state contract for mixed exact/summary plans (issue #171). +//! +//! A post-ASAP DAG mixes two very different moments of execution: the +//! **update/ingest path** (rows arrive, maintained summary state is updated) +//! and **query evaluation** (maintained state is read out and a final result +//! is produced). A plan that places a query-time residual *underneath* a +//! maintained summary is not merely expensive — it is unexecutable, because +//! the maintenance loop has no readout values to feed into that summary. +//! [`SummaryExpr::ValueOperation`] represents such work without inventing a +//! node per function or use case. Its [`ExecutionTiming`] makes placement +//! explicit and independent of the semantic [`ValueOperation`]. +//! +//! [`ExecutionDataState`] is what a node's output *is*, at which data_state; +//! [`validate_execution_data_states`] checks every edge of a DAG against the +//! rules below at plan construction, returning a typed [`ExecutionDataStateError`] rather +//! than deferring to a runtime failure. +//! +//! ## Edge rules +//! +//! | Parent | Accepts from `child` | +//! |---|---| +//! | `SummaryAgg.child` | `MAINTENANCE_ROWS`, or `MAINTENANCE_SUMMARY` of an **exact accumulator** family. Never a read-time data_state. | +//! | `SummaryEstimate.summary_input` | `MAINTENANCE_SUMMARY` (any family). Produces `READ_ROWS`. | +//! | `SummaryJoin.outer/inner` | `MAINTENANCE_ROWS` or `MAINTENANCE_SUMMARY`; never a read-time data_state. | +//! | `SummarySubtract`/`SummaryDelete`/`SummaryMerge` | `MAINTENANCE_SUMMARY`. | +//! | `ValueOperation.child` with `MaintenanceTime` | `MAINTENANCE_ROWS`. Produces `MAINTENANCE_ROWS`. | +//! | `ValueOperation.child` with `ReadTime` | `READ_ROWS`. Produces `READ_ROWS`. | +//! +//! ## `KeepPreAsap` declares its data_state through the derivation +//! +//! A [`SummaryExpr::KeepPreAsap`] leaf is a raw pre-ASAP computation that a +//! runtime can execute at either time: as maintenance input beneath a +//! `SummaryAgg`/maintenance-time `ValueOperation`, or as a query-time fallback +//! beneath a read-time `ValueOperation` (or at the root). It carries no timing +//! field of its own +//! — every existing consumer pattern-matches the one-field shape — so its +//! data_state is *assigned* by [`validate_execution_data_states`] from the edge that +//! reaches it and reported in the returned [`ExecutionDataStateAssignment`]. What it may +//! not do is stay ambiguous inside one mixed plan: the same `Rc` +//! reached once as update input and once as query-time fallback is +//! [`ExecutionDataStateError::AmbiguousKeepPreAsap`], because no single execution of that +//! subtree can serve both roles. + +use std::collections::HashMap; +use std::rc::Rc; + +use thiserror::Error; + +use super::expr::{ExactOperation, SummaryExpr, SummaryNode, ValueOperation}; +use super::schema::{SummaryFamilyType, SummaryField, SummarySchema}; +use crate::pre_asap::query_expr::{aggregate_output_schema, QueryExprError}; +use crate::pre_asap::schema::{Column, Schema}; + +/// When a post-ASAP value is produced. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ExecutionTiming { + MaintenanceTime, + ReadTime, +} + +impl ExecutionTiming { + pub fn as_str(self) -> &'static str { + match self { + Self::MaintenanceTime => "maintenance_time", + Self::ReadTime => "read_time", + } + } +} + +/// The primitive representation carried by a post-ASAP edge. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DataPrimitive { + /// Directly usable values, including approximate summary readouts. + /// This does not imply original input data or an exact guarantee. + Raw, + SummaryState, +} + +impl DataPrimitive { + pub fn as_str(self) -> &'static str { + match self { + Self::Raw => "raw", + Self::SummaryState => "summary_state", + } + } +} + +/// The two-dimensional edge contract: when a value exists and which data +/// primitive it carries. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ExecutionDataState { + pub timing: ExecutionTiming, + pub primitive: DataPrimitive, +} + +impl ExecutionDataState { + pub const MAINTENANCE_ROWS: Self = Self { + timing: ExecutionTiming::MaintenanceTime, + primitive: DataPrimitive::Raw, + }; + pub const MAINTENANCE_SUMMARY: Self = Self { + timing: ExecutionTiming::MaintenanceTime, + primitive: DataPrimitive::SummaryState, + }; + pub const READ_ROWS: Self = Self { + timing: ExecutionTiming::ReadTime, + primitive: DataPrimitive::Raw, + }; +} + +impl std::fmt::Display for ExecutionDataState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}/{}", self.timing.as_str(), self.primitive.as_str()) + } +} + +/// Which parent/edge a [`ExecutionDataStateError`] is about — the variant name of the +/// parent `SummaryExpr` plus its field, for a message a plan author can act +/// on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExecutionDataStateEdge { + SummaryAggChild, + SummaryEstimateInput, + SummaryJoinInput, + SummarySubtractInput, + SummaryDeleteInput, + SummaryMergeInput, + ValueOperationChild, +} + +impl ExecutionDataStateEdge { + fn describe(self) -> &'static str { + match self { + Self::SummaryAggChild => "SummaryAgg.child", + Self::SummaryEstimateInput => "SummaryEstimate.summary_input", + Self::SummaryJoinInput => "SummaryJoin.{outer,inner}", + Self::SummarySubtractInput => "SummarySubtract.{left,right}", + Self::SummaryDeleteInput => "SummaryDelete.summary_input", + Self::SummaryMergeInput => "SummaryMerge.children[]", + Self::ValueOperationChild => "ValueOperation.child", + } + } +} + +/// A plan-construction-time data_state violation. Typed (not a string) so a +/// strategy can degrade to a conservative fallback on the specific variant +/// it expects, and so tests can assert the *reason* a plan was rejected. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum ExecutionDataStateError { + /// A query-time value (`SummaryEstimate` / read-time `ValueOperation` output) + /// placed beneath a maintained summary — the one shape issue #171's + /// data_state split exists to make unrepresentable. + #[error( + "readout value under maintenance: {edge} received a {child} input, but a maintained \ + summary can only consume update-path values (or exact accumulator state)" + )] + ReadoutUnderMaintenance { + edge: &'static str, + child: ExecutionDataState, + }, + /// Any other edge whose child data_state the parent does not accept + /// (e.g. plain update rows fed straight into a `SummaryEstimate`, or a + /// sketch's opaque state fed into a read-time `ValueOperation`). + #[error("{edge} does not accept a {child} input")] + IllegalChildDataState { + edge: &'static str, + child: ExecutionDataState, + }, + /// A `SummaryAgg` whose child is summary state of a family other than an + /// exact accumulator — re-accumulating opaque sketch/sample/… state on + /// the update path has no defined semantics here. + #[error( + "SummaryAgg.child carries {family} summary state; only exact accumulator state can be \ + composed into another maintained summary" + )] + UnsupportedStateComposition { family: String }, + /// One shared `KeepPreAsap` node reached both as update-path raw input + /// and as a query-time fallback — see the module docs. + #[error( + "KeepPreAsap subtree is data_state-ambiguous: reached as {first} and as {second} in the same \ + plan" + )] + AmbiguousKeepPreAsap { + first: ExecutionDataState, + second: ExecutionDataState, + }, + /// A maintenance-time value operation at the root of a plan: + /// nothing maintains state above it, so its output is never read. + #[error("A maintenance-time value operation cannot be a plan root: its update-path output feeds nothing")] + MaintenanceRowsAtRoot, + /// An `ExactOperation` whose input columns are not all `Plain` at its + /// declared data_state. + #[error("exact operator consumes non-plain column {column:?} ({dtype})")] + NonPlainOperand { column: String, dtype: String }, +} + +/// The data_state assigned to every node of a validated plan, keyed by +/// `Rc` pointer identity — the explicit per-node "execution_data_state" a +/// runtime or a DAG export reads instead of re-deriving it. For every +/// non-`KeepPreAsap` node this equals [`produced_data_state`]; for a +/// `KeepPreAsap` leaf it is the data_state the reaching edge assigned. +#[derive(Debug, Clone, Default)] +pub struct ExecutionDataStateAssignment { + domains: HashMap<*const SummaryNode, ExecutionDataState>, +} + +impl ExecutionDataStateAssignment { + /// The data_state assigned to `node`, if it was part of the validated plan. + pub fn data_state_of(&self, node: &Rc) -> Option { + self.domains.get(&Rc::as_ptr(node)).copied() + } + + /// The data_state assigned to the node at `ptr` — for callers walking a plan + /// by reference rather than by `Rc`. + pub fn data_state_of_ptr(&self, ptr: *const SummaryNode) -> Option { + self.domains.get(&ptr).copied() + } +} + +/// The data_state `expr` *produces*, independent of context — `None` for +/// [`SummaryExpr::KeepPreAsap`], whose data_state is assigned by the edge reaching +/// it (see the module docs). +pub fn produced_data_state(expr: &SummaryExpr) -> Option { + Some(match expr { + SummaryExpr::KeepPreAsap(_) => return None, + SummaryExpr::BinaryOp { .. } => ExecutionDataState::READ_ROWS, + SummaryExpr::SummaryAgg { .. } + | SummaryExpr::SummaryJoin { .. } + | SummaryExpr::SummarySubtract { .. } + | SummaryExpr::SummaryDelete { .. } + | SummaryExpr::SummaryMerge { .. } => ExecutionDataState::MAINTENANCE_SUMMARY, + SummaryExpr::SummaryEstimate { .. } => ExecutionDataState::READ_ROWS, + SummaryExpr::ValueOperation { timing, .. } => match timing { + ExecutionTiming::MaintenanceTime => ExecutionDataState::MAINTENANCE_ROWS, + ExecutionTiming::ReadTime => ExecutionDataState::READ_ROWS, + }, + }) +} + +/// Is `family` the exact-accumulator family whose partial state *is* the +/// value — the one summary state a `SummaryAgg` may re-accumulate? +fn is_exact_accumulator_state(schema: &SummarySchema) -> Result<(), ExecutionDataStateError> { + for field in &schema.fields { + match &field.dtype { + SummaryFamilyType::Plain(_) | SummaryFamilyType::ExactAggregate(..) => {} + other => { + return Err(ExecutionDataStateError::UnsupportedStateComposition { + family: format!("{other:?}"), + }) + } + } + } + Ok(()) +} + +/// Validate every edge of the DAG rooted at `root` against the module-level +/// rules, returning each node's assigned data_state on success. Shared +/// `Rc`s are visited once per reaching edge (the assignment is +/// per node, so a conflict between two edges is what +/// [`ExecutionDataStateError::AmbiguousKeepPreAsap`] detects). +pub fn validate_execution_data_states( + root: &Rc, +) -> Result { + // The root may be a readable value or bare maintained state (a + // deployment may hand an `ExactAggregate` accumulator straight to a + // consumer) — only an update-path-only root is meaningless. + let root_domain = match produced_data_state(&root.expr) { + None => ExecutionDataState::READ_ROWS, + Some(ExecutionDataState::MAINTENANCE_ROWS) => { + return Err(ExecutionDataStateError::MaintenanceRowsAtRoot) + } + Some(data_state) => data_state, + }; + validate_execution_data_states_at(root, root_domain) +} + +/// [`validate_execution_data_states`] for a *sub*-plan whose root is known to +/// sit at `data_state` — e.g. a maintenance-time `ValueOperation` about to be placed beneath a +/// `SummaryAgg`, which would be rejected as a whole-plan root but is a +/// legal update-path input. Validates every edge beneath `root` exactly +/// as the whole-plan entry point does. +pub fn validate_execution_data_states_at( + root: &Rc, + data_state: ExecutionDataState, +) -> Result { + let mut assignment = ExecutionDataStateAssignment::default(); + visit(root, data_state, &mut assignment)?; + Ok(assignment) +} + +/// Record `data_state` for `node` (detecting a conflicting earlier assignment +/// for a `KeepPreAsap`), then check and recurse into every child edge. +fn visit( + node: &Rc, + data_state: ExecutionDataState, + assignment: &mut ExecutionDataStateAssignment, +) -> Result<(), ExecutionDataStateError> { + let ptr = Rc::as_ptr(node); + if let Some(previous) = assignment.domains.get(&ptr) { + if *previous != data_state { + return Err(ExecutionDataStateError::AmbiguousKeepPreAsap { + first: *previous, + second: data_state, + }); + } + // Already validated through another edge with the same data_state. + return Ok(()); + } + assignment.domains.insert(ptr, data_state); + + match &node.expr { + SummaryExpr::KeepPreAsap(_) => Ok(()), + SummaryExpr::BinaryOp { lhs, rhs, .. } => { + for input in [lhs, rhs] { + let state = + produced_data_state(&input.expr).unwrap_or(ExecutionDataState::READ_ROWS); + if state != ExecutionDataState::READ_ROWS { + return Err(ExecutionDataStateError::IllegalChildDataState { + edge: "BinaryOp operand", + child: state, + }); + } + visit(input, state, assignment)?; + } + Ok(()) + } + SummaryExpr::SummaryAgg { child, .. } => { + let child_domain = child_domain( + child, + ExecutionDataStateEdge::SummaryAggChild, + |avail| match avail { + ExecutionDataState::MAINTENANCE_ROWS => Ok(()), + ExecutionDataState::MAINTENANCE_SUMMARY => { + is_exact_accumulator_state(&child.schema) + } + other => Err(ExecutionDataStateError::ReadoutUnderMaintenance { + edge: ExecutionDataStateEdge::SummaryAggChild.describe(), + child: other, + }), + }, + )?; + visit(child, child_domain, assignment) + } + SummaryExpr::SummaryJoin { outer, inner, .. } => { + for input in [outer, inner] { + let s = child_domain(input, ExecutionDataStateEdge::SummaryJoinInput, |avail| { + match avail { + ExecutionDataState::MAINTENANCE_ROWS + | ExecutionDataState::MAINTENANCE_SUMMARY => Ok(()), + other => Err(ExecutionDataStateError::ReadoutUnderMaintenance { + edge: ExecutionDataStateEdge::SummaryJoinInput.describe(), + child: other, + }), + } + })?; + visit(input, s, assignment)?; + } + Ok(()) + } + SummaryExpr::SummarySubtract { left, right } => { + for input in [left, right] { + let s = state_only(input, ExecutionDataStateEdge::SummarySubtractInput)?; + visit(input, s, assignment)?; + } + Ok(()) + } + SummaryExpr::SummaryDelete { summary_input, .. } => { + let s = state_only(summary_input, ExecutionDataStateEdge::SummaryDeleteInput)?; + visit(summary_input, s, assignment) + } + SummaryExpr::SummaryMerge { children } => { + for input in children { + let s = state_only(input, ExecutionDataStateEdge::SummaryMergeInput)?; + visit(input, s, assignment)?; + } + Ok(()) + } + SummaryExpr::SummaryEstimate { summary_input, .. } => { + let s = state_only(summary_input, ExecutionDataStateEdge::SummaryEstimateInput)?; + visit(summary_input, s, assignment) + } + SummaryExpr::ValueOperation { + child, + operation, + timing, + } => { + let required = match timing { + ExecutionTiming::MaintenanceTime => ExecutionDataState::MAINTENANCE_ROWS, + ExecutionTiming::ReadTime => ExecutionDataState::READ_ROWS, + }; + let s = produced_data_state(&child.expr).unwrap_or(required); + if s != required { + return Err(ExecutionDataStateError::IllegalChildDataState { + edge: ExecutionDataStateEdge::ValueOperationChild.describe(), + child: s, + }); + } + check_plain_operands(operation, &child.schema)?; + visit(child, s, assignment) + } + } +} + +/// The data_state `child` takes as a direct input of `parent`, without +/// validating legality — `child`'s own produced data_state, or for a +/// `KeepPreAsap` leaf the data_state `parent`'s edge assigns it (update-path raw +/// input under maintenance-time operation edges, query-time fallback under a +/// a read-time operation, and — meaninglessly, but for a stable answer — maintenance rows +/// under a state-only edge). For DAG export and other reporting that needs +/// an explicit per-node data_state even on a plan that +/// [`validate_execution_data_states`] would reject. +pub fn assigned_child_data_state(parent: &SummaryExpr, child: &SummaryNode) -> ExecutionDataState { + if let Some(avail) = produced_data_state(&child.expr) { + return avail; + } + match parent { + SummaryExpr::ValueOperation { + timing: ExecutionTiming::ReadTime, + .. + } => ExecutionDataState::READ_ROWS, + SummaryExpr::KeepPreAsap(_) + | SummaryExpr::BinaryOp { .. } + | SummaryExpr::SummaryAgg { .. } + | SummaryExpr::SummaryJoin { .. } + | SummaryExpr::SummarySubtract { .. } + | SummaryExpr::SummaryDelete { .. } + | SummaryExpr::SummaryEstimate { .. } + | SummaryExpr::SummaryMerge { .. } + | SummaryExpr::ValueOperation { + timing: ExecutionTiming::MaintenanceTime, + .. + } => ExecutionDataState::MAINTENANCE_ROWS, + } +} + +/// The data_state `child` takes on `edge`: its own produced data_state +/// (checked via `accept`), or — for a `KeepPreAsap` leaf — the data_state the +/// edge assigns it, derived from what that edge accepts. +fn child_domain( + child: &Rc, + edge: ExecutionDataStateEdge, + accept: impl Fn(ExecutionDataState) -> Result<(), ExecutionDataStateError>, +) -> Result { + match produced_data_state(&child.expr) { + Some(avail) => { + accept(avail)?; + Ok(avail) + } + None => { + // A raw pre-ASAP subtree executes at whichever data_state its consumer + // needs: update-path input for maintenance-time operation edges, + // query-time fallback for a read-time edge. State-only edges + // can't consume plain rows at all. + let assigned = match edge { + ExecutionDataStateEdge::SummaryAggChild + | ExecutionDataStateEdge::SummaryJoinInput + | ExecutionDataStateEdge::ValueOperationChild => { + ExecutionDataState::MAINTENANCE_ROWS + } + ExecutionDataStateEdge::SummaryEstimateInput + | ExecutionDataStateEdge::SummarySubtractInput + | ExecutionDataStateEdge::SummaryDeleteInput + | ExecutionDataStateEdge::SummaryMergeInput => { + return Err(ExecutionDataStateError::IllegalChildDataState { + edge: edge.describe(), + child: ExecutionDataState::MAINTENANCE_ROWS, + }) + } + }; + accept(assigned)?; + Ok(assigned) + } + } +} + +fn state_only( + child: &Rc, + edge: ExecutionDataStateEdge, +) -> Result { + child_domain(child, edge, |avail| match avail { + ExecutionDataState::MAINTENANCE_SUMMARY => Ok(()), + other => Err(ExecutionDataStateError::IllegalChildDataState { + edge: edge.describe(), + child: other, + }), + }) +} + +/// The exact operator must consume only `Plain` columns of its input: for +/// an `Aggregate` payload, every grouping key and every measure's input +/// column. +fn check_plain_operands( + op: &ValueOperation, + input: &SummarySchema, +) -> Result<(), ExecutionDataStateError> { + let ValueOperation::Exact(op) = op else { + return check_all_plain(input); + }; + let ExactOperation::Aggregate { + reduction, + measures, + .. + } = op; + let mut referenced: Vec = reduction + .group_keys() + .map(|keys| keys.keys().to_vec()) + .unwrap_or_default(); + for m in measures { + if let Some(col) = m.input_col() { + referenced.push(col); + } + } + // With no explicit input column (the PromQL sample-value convention) + // the operator reads every non-key column, so all must be plain. + let implicit = measures.iter().any(|m| m.input_col().is_none()); + for (i, field) in input.fields.iter().enumerate() { + if !(implicit || referenced.contains(&i)) { + continue; + } + if !matches!(field.dtype, SummaryFamilyType::Plain(_)) { + return Err(ExecutionDataStateError::NonPlainOperand { + column: field.name.clone(), + dtype: format!("{:?}", field.dtype), + }); + } + } + Ok(()) +} + +fn check_all_plain(input: &SummarySchema) -> Result<(), ExecutionDataStateError> { + for field in &input.fields { + if !matches!(field.dtype, SummaryFamilyType::Plain(_)) { + return Err(ExecutionDataStateError::NonPlainOperand { + column: field.name.clone(), + dtype: format!("{:?}", field.dtype), + }); + } + } + Ok(()) +} + +/// The plain pre-ASAP `Schema` underlying an all-`Plain` `SummarySchema`, or +/// `None` if any column carries summary state. +pub fn plain_schema(schema: &SummarySchema) -> Option { + let mut columns = Vec::with_capacity(schema.fields.len()); + for field in &schema.fields { + let SummaryFamilyType::Plain(dtype) = &field.dtype else { + return None; + }; + columns.push(Column::new(&field.name, dtype.clone(), field.nullable)); + } + Some(Schema { + columns, + time_index: schema.time_index, + unique_keys: Vec::new(), + closed: true, + }) +} + +/// Lift a plain pre-ASAP schema to a `SummarySchema` with every column +/// `Plain` — the output of every exact operator. +pub fn lift_plain(schema: &Schema) -> SummarySchema { + SummarySchema { + fields: schema + .columns + .iter() + .map(|c| SummaryField { + name: c.name.clone(), + dtype: SummaryFamilyType::Plain(c.dtype.clone()), + nullable: c.nullable, + }) + .collect(), + time_index: schema.time_index, + } +} + +/// Output schema of `op` applied to a child whose edge carries `input` — +/// the same canonical derivation the pre-ASAP `Aggregate` node uses, so an +/// exact `ValueOperation` never disagrees with the pre-ASAP +/// target it was lowered from. `Err` when the child carries non-plain +/// state the operator cannot read. +pub fn exact_operation_output_schema( + op: &ExactOperation, + input: &SummarySchema, +) -> Result { + let plain = plain_schema(input).ok_or(ExactOperationSchemaError::NonPlainInput)?; + let ExactOperation::Aggregate { + reduction, + measures, + output_names, + .. + } = op; + let out = aggregate_output_schema(&plain, reduction, measures, output_names)?; + Ok(lift_plain(&out)) +} + +/// Why [`exact_operation_output_schema`] could not derive a schema. +#[derive(Debug, Error)] +pub enum ExactOperationSchemaError { + #[error("exact operator input carries summary state, not plain columns")] + NonPlainInput, + #[error("schema derivation failed: {0}")] + Schema(#[from] QueryExprError), +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::post_asap::{ExactKind, ExactParams, GroupingStrategy, SketchQuery}; + use crate::pre_asap::agg_intent::AggIntent; + use crate::pre_asap::expr_ir::ColumnRef; + use crate::pre_asap::query_expr::{QueryExpr, Reduction, Source}; + use crate::pre_asap::schema::DataType; + + /// Both execution phases use raw values, distinct from maintained state. + #[test] + fn raw_primitive_labels() { + assert_eq!( + ExecutionDataState::MAINTENANCE_ROWS.primitive, + DataPrimitive::Raw + ); + assert_eq!(ExecutionDataState::READ_ROWS.primitive, DataPrimitive::Raw); + assert_eq!( + ExecutionDataState::MAINTENANCE_ROWS.to_string(), + "maintenance_time/raw" + ); + assert_eq!(ExecutionDataState::READ_ROWS.to_string(), "read_time/raw"); + assert_eq!(DataPrimitive::SummaryState.as_str(), "summary_state"); + } + + fn scan() -> Rc { + Rc::new(QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + Column::new("zone", DataType::Utf8, true), + ], + 0, + vec![], + ), + }) + } + + fn keep() -> Rc { + let s = scan(); + let schema = lift_plain(&s.output_schema().unwrap()); + Rc::new(SummaryNode { + expr: SummaryExpr::KeepPreAsap(s), + schema, + guarantee: None, + }) + } + + fn plain(names: &[&str]) -> SummarySchema { + SummarySchema { + fields: names + .iter() + .map(|n| SummaryField { + name: (*n).into(), + dtype: SummaryFamilyType::Plain(DataType::Float64), + nullable: false, + }) + .collect(), + time_index: None, + } + } + + fn agg(child: Rc, family: SummaryFamilyType) -> Rc { + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child, + family: family.clone(), + input: crate::post_asap::SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::by(vec![]), + grouping: GroupingStrategy::default(), + }, + schema: SummarySchema { + fields: vec![SummaryField { + name: "state".into(), + dtype: family, + nullable: false, + }], + time_index: None, + }, + guarantee: None, + }) + } + + fn kll() -> SummaryFamilyType { + use crate::post_asap::{SketchAlgorithm, SketchKind, SketchParams}; + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + GroupingStrategy::default(), + ) + } + + fn estimate(child: Rc) -> Rc { + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryEstimate { + summary_input: child, + query: SketchQuery::Quantile { q: 0.99 }, + }, + schema: plain(&["quantile_0_99"]), + guarantee: None, + }) + } + + fn max_op() -> ExactOperation { + ExactOperation::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Max { col: None }], + output_names: vec![], + having: None, + } + } + + #[test] + fn keep_pre_asap_under_summary_agg_is_update_input() { + let leaf = keep(); + let root = agg(Rc::clone(&leaf), kll()); + let assignment = validate_execution_data_states(&root).unwrap(); + assert_eq!( + assignment.data_state_of(&leaf), + Some(ExecutionDataState::MAINTENANCE_ROWS) + ); + assert_eq!( + assignment.data_state_of(&root), + Some(ExecutionDataState::MAINTENANCE_SUMMARY) + ); + } + + #[test] + fn exact_accumulator_state_may_feed_another_summary_agg() { + let inner = agg( + keep(), + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + ); + let root = estimate(agg(inner, kll())); + assert!(validate_execution_data_states(&root).is_ok()); + } + + #[test] + fn readout_under_summary_agg_is_rejected() { + let inner = estimate(agg(keep(), kll())); + let root = agg(inner, kll()); + assert!(matches!( + validate_execution_data_states(&root), + Err(ExecutionDataStateError::ReadoutUnderMaintenance { .. }) + )); + } + + #[test] + fn read_time_operation_over_readout_is_legal_and_root_is_readout() { + let inner = estimate(agg(keep(), kll())); + let root = Rc::new(SummaryNode { + expr: SummaryExpr::ValueOperation { + child: inner, + operation: ValueOperation::Exact(max_op()), + timing: ExecutionTiming::ReadTime, + }, + schema: plain(&["max"]), + guarantee: None, + }); + let assignment = validate_execution_data_states(&root).unwrap(); + assert_eq!( + assignment.data_state_of(&root), + Some(ExecutionDataState::READ_ROWS) + ); + } + + #[test] + fn non_exact_operator_uses_the_same_read_domain_contract() { + let inner = estimate(agg(keep(), kll())); + let root = Rc::new(SummaryNode { + expr: SummaryExpr::ValueOperation { + child: inner, + operation: ValueOperation::Extension { + name: "approximate_calibration".into(), + }, + timing: ExecutionTiming::ReadTime, + }, + schema: plain(&["calibrated"]), + guarantee: None, + }); + + let assignment = validate_execution_data_states(&root).unwrap(); + assert_eq!( + assignment.data_state_of(&root), + Some(ExecutionDataState::READ_ROWS) + ); + } + + #[test] + fn read_time_operation_under_summary_agg_is_rejected() { + let inner = estimate(agg(keep(), kll())); + let post = Rc::new(SummaryNode { + expr: SummaryExpr::ValueOperation { + child: inner, + operation: ValueOperation::Exact(max_op()), + timing: ExecutionTiming::ReadTime, + }, + schema: plain(&["max"]), + guarantee: None, + }); + let root = agg(post, kll()); + assert_eq!( + validate_execution_data_states(&root).err(), + Some(ExecutionDataStateError::ReadoutUnderMaintenance { + edge: "SummaryAgg.child", + child: ExecutionDataState::READ_ROWS, + }) + ); + } + + #[test] + fn function_under_summary_agg_is_legal_but_not_at_root() { + let operation = Rc::new(SummaryNode { + expr: SummaryExpr::ValueOperation { + child: keep(), + operation: ValueOperation::Exact(max_op()), + timing: ExecutionTiming::MaintenanceTime, + }, + schema: plain(&["max"]), + guarantee: None, + }); + assert_eq!( + validate_execution_data_states(&operation).err(), + Some(ExecutionDataStateError::MaintenanceRowsAtRoot) + ); + let root = estimate(agg(Rc::clone(&operation), kll())); + let assignment = validate_execution_data_states(&root).unwrap(); + assert_eq!( + assignment.data_state_of(&operation), + Some(ExecutionDataState::MAINTENANCE_ROWS) + ); + } + + #[test] + fn function_over_readout_is_rejected() { + let inner = estimate(agg(keep(), kll())); + let operation = Rc::new(SummaryNode { + expr: SummaryExpr::ValueOperation { + child: inner, + operation: ValueOperation::Exact(max_op()), + timing: ExecutionTiming::MaintenanceTime, + }, + schema: plain(&["max"]), + guarantee: None, + }); + let root = agg(operation, kll()); + assert!(matches!( + validate_execution_data_states(&root), + Err(ExecutionDataStateError::IllegalChildDataState { + edge: "ValueOperation.child", + child: ExecutionDataState::READ_ROWS + }) + )); + } + + #[test] + fn a_shared_keep_pre_asap_reached_in_two_domains_is_ambiguous() { + // One raw subtree used both as update input (under a SummaryAgg) and + // as a query-time fallback (under an ExactRead) — no single + // execution can serve both, so the plan is rejected. + let shared = keep(); + let maintained = estimate(agg(Rc::clone(&shared), kll())); + let post_over_raw = Rc::new(SummaryNode { + expr: SummaryExpr::ValueOperation { + child: Rc::clone(&shared), + operation: ValueOperation::Exact(max_op()), + timing: ExecutionTiming::ReadTime, + }, + schema: plain(&["max"]), + guarantee: None, + }); + let root = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryMerge { + children: vec![ + Rc::new(SummaryNode { + expr: SummaryExpr::ValueOperation { + child: maintained, + operation: ValueOperation::Exact(max_op()), + timing: ExecutionTiming::ReadTime, + }, + schema: plain(&["max"]), + guarantee: None, + }), + post_over_raw, + ], + }, + schema: plain(&["max"]), + guarantee: None, + }); + // SummaryMerge only accepts state, so this fails earlier for a + // different reason; probe the ambiguity through a direct visit. + let mut assignment = ExecutionDataStateAssignment::default(); + visit( + &shared, + ExecutionDataState::MAINTENANCE_ROWS, + &mut assignment, + ) + .unwrap(); + assert_eq!( + visit(&shared, ExecutionDataState::READ_ROWS, &mut assignment), + Err(ExecutionDataStateError::AmbiguousKeepPreAsap { + first: ExecutionDataState::MAINTENANCE_ROWS, + second: ExecutionDataState::READ_ROWS, + }) + ); + assert!(validate_execution_data_states(&root).is_err()); + } + + #[test] + fn exact_operator_schema_matches_pre_asap_aggregate_derivation() { + let child_schema = lift_plain(&scan().output_schema().unwrap()); + let op = ExactOperation::Aggregate { + reduction: Reduction::by(vec![2]), + measures: vec![AggIntent::Max { col: None }], + output_names: vec![], + having: None, + }; + let out = exact_operation_output_schema(&op, &child_schema).unwrap(); + let names: Vec<_> = out.fields.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(names, vec!["zone", "max"]); + assert!(out + .fields + .iter() + .all(|f| matches!(f.dtype, SummaryFamilyType::Plain(_)))); + } + + #[test] + fn exact_operator_rejects_non_plain_input() { + let state = agg(keep(), kll()); + assert!(matches!( + exact_operation_output_schema(&max_op(), &state.schema), + Err(ExactOperationSchemaError::NonPlainInput) + )); + } +} diff --git a/crates/types/src/post_asap/expr.rs b/crates/types/src/post_asap/expr.rs index 93be6867c..7c4bc1f12 100644 --- a/crates/types/src/post_asap/expr.rs +++ b/crates/types/src/post_asap/expr.rs @@ -3,8 +3,28 @@ use std::rc::Rc; use super::guarantee::ResultGuarantee; use super::schema::{SummaryFamilyType, SummarySchema}; use super::sketch::{GroupingStrategy, SketchQuery, SummaryUpdate}; +use crate::pre_asap::agg_intent::AggIntent; +use crate::pre_asap::query_expr::Predicate; use crate::pre_asap::{BinaryOpKind, ColumnRef, QueryExpr, Reduction, VectorMatch}; +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[non_exhaustive] +pub enum ExactOperation { + Aggregate { + reduction: Reduction, + measures: Vec, + output_names: Vec, + having: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[non_exhaustive] +pub enum ValueOperation { + Exact(ExactOperation), + Extension { name: String }, +} + // ── Post-ASAP DAG node ─────────────────────────────────────────────────────── /// A node in the post-ASAP DAG: wraps the expression and its derived output @@ -57,6 +77,14 @@ pub enum SummaryExpr { operator: BinaryOperator, }, + /// Plain-row semantics composed with a post-ASAP child. Timing is an + /// independent physical choice, not part of the operation's identity. + ValueOperation { + child: Rc, + operation: ValueOperation, + timing: super::execution_data_state::ExecutionTiming, + }, + /// Summary aggregation. Post-ASAP binding chose `family` — which /// summary family (exact accumulator, sketch, sample, wavelet, or /// statistical model) and its `(kind, params)` — from the catalog for diff --git a/crates/types/src/post_asap/guarantee.rs b/crates/types/src/post_asap/guarantee.rs index 696c6cc9f..770cd2956 100644 --- a/crates/types/src/post_asap/guarantee.rs +++ b/crates/types/src/post_asap/guarantee.rs @@ -223,10 +223,19 @@ pub enum CompositionOperator { Lipschitz { constant: f64 }, /// An exact sum over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. ExactSum, - /// An exact max/min over approximate inputs — bounds the returned + /// An exact arithmetic mean over approximate values. For absolute-value + /// input guarantees, the output bound is the largest per-value bound. + ExactAverage, + /// An exact max/min over inputs carrying absolute-value guarantees — bounds the returned /// *value* (`max` of the input bounds) but does not identify which key /// is the true winner. ExactExtremum, + /// PromQL `rate`: reset correction plus range-boundary extrapolation. + CounterRate, + /// PromQL `irate`: reset-aware slope over the final two samples. + InstantCounterRate, + /// PromQL `increase`: extrapolated reset-corrected increase over a range. + CounterIncrease, /// A top-k selection over approximate inputs. Unsupported by the default /// model until the margin certificate of issue #172 PR 3 exists. TopKSelection, diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index 01b659eb3..9ece4b2c1 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -28,6 +28,7 @@ //! — see `asap_aware_mapping::grouping`'s module docs for why. pub mod cse; +pub mod execution_data_state; pub mod expr; pub mod guarantee; pub mod query_time; @@ -38,7 +39,12 @@ pub mod summary_maintenance_lifecycle; pub mod summary_window; pub use cse::share_common_summary_subtrees; -pub use expr::{BinaryOperator, SummaryExpr, SummaryNode}; +pub use execution_data_state::{ + exact_operation_output_schema, produced_data_state, validate_execution_data_states, + validate_execution_data_states_at, DataPrimitive, ExactOperationSchemaError, + ExecutionDataState, ExecutionDataStateAssignment, ExecutionDataStateError, ExecutionTiming, +}; +pub use expr::{BinaryOperator, ExactOperation, SummaryExpr, SummaryNode, ValueOperation}; pub use guarantee::{ AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, ResultGuarantee, diff --git a/crates/types/src/post_asap/sketch.rs b/crates/types/src/post_asap/sketch.rs index 832afb00a..b4612ca40 100644 --- a/crates/types/src/post_asap/sketch.rs +++ b/crates/types/src/post_asap/sketch.rs @@ -19,6 +19,8 @@ pub enum ExactKind { Increase, /// Rate accumulator (increase / time window duration). Rate, + /// Instant-rate accumulator retaining the final two timestamped samples. + IRate, } /// Parameters for an [`ExactKind`] accumulator. All exact accumulators have @@ -32,6 +34,7 @@ pub enum ExactParams { MinMax, Increase, Rate, + IRate, } // ── Approximate sketches ───────────────────────────────────────────────────── diff --git a/crates/types/src/pre_asap/agg_intent.rs b/crates/types/src/pre_asap/agg_intent.rs index 69875ef75..88c3775a9 100644 --- a/crates/types/src/pre_asap/agg_intent.rs +++ b/crates/types/src/pre_asap/agg_intent.rs @@ -104,6 +104,9 @@ pub enum AggIntent { // The temporal range lives on the enclosing `QueryExpr::TimeRange` node, // not in the intent — this keeps the intent vocabulary range-agnostic. Rate, + /// PromQL `irate(v[w])` — reset-aware rate from the final two samples. + /// Distinct from [`Rate`](Self::Rate), which extrapolates across the range. + IRate, Increase, // ── Counter-derivative / range-vector functions (issue #44) ────────── @@ -344,6 +347,7 @@ impl AggIntent { pub fn requires(&self) -> DataModel { match self { Self::Rate + | Self::IRate | Self::Increase | Self::Changes | Self::Delta @@ -382,6 +386,7 @@ impl AggIntent { matches!( self, Self::Rate + | Self::IRate | Self::Increase | Self::Changes | Self::Delta @@ -457,6 +462,7 @@ impl AggIntent { AggIntent::TopK { k, .. } => col(&format!("topk_{k}"), DataType::Utf8, false), AggIntent::Cardinality { .. } => col("cardinality", DataType::Int64, false), AggIntent::Rate => col("rate", DataType::Float64, false), + AggIntent::IRate => col("irate", DataType::Float64, false), AggIntent::Increase => col("increase", DataType::Float64, false), // Counter-derivative range functions (issue #44) — all yield one // float per series (PromQL values are float64), named after the diff --git a/crates/types/src/pre_asap/resolve.rs b/crates/types/src/pre_asap/resolve.rs index 4d80fb442..28130e07f 100644 --- a/crates/types/src/pre_asap/resolve.rs +++ b/crates/types/src/pre_asap/resolve.rs @@ -548,6 +548,7 @@ fn resolve_agg_intent( accuracy: accuracy.clone(), }, AggIntent::Rate => AggIntent::Rate, + AggIntent::IRate => AggIntent::IRate, AggIntent::Increase => AggIntent::Increase, AggIntent::Changes => AggIntent::Changes, AggIntent::Delta => AggIntent::Delta, diff --git a/tools/dag-viewer/node-style.js b/tools/dag-viewer/node-style.js index b41368e96..d032750f7 100644 --- a/tools/dag-viewer/node-style.js +++ b/tools/dag-viewer/node-style.js @@ -32,6 +32,7 @@ const KIND_CATEGORY_JSON = `{ "SummaryJoin": "summary", "SummarySubtract": "summary", "SummaryBinaryOp": "summary", + "ValueOperation": "summary", "SummaryDelete": "summary", "SummaryEstimate": "summary", "SummaryMerge": "summary" @@ -103,7 +104,7 @@ const CATEGORIES = { // Post-ASAP nodes use a neutral palette; KeepPreAsap has a muted override. summary: { label: 'Summary', - description: 'KeepPreAsap, SummaryBinaryOp, SummaryAgg, SummaryJoin, SummarySubtract, SummaryDelete, SummaryEstimate, SummaryMerge — post-ASAP materialized structures', + description: 'KeepPreAsap, SummaryBinaryOp, ValueOperation, SummaryAgg, SummaryJoin, SummarySubtract, SummaryDelete, SummaryEstimate, SummaryMerge — post-ASAP materialized structures', light: { bg: '#f1f2f4', border: '#4b5563' }, dark: { bg: '#20242b', border: '#9ca3af' }, },