diff --git a/crates/asap-aware-mapping/src/accuracy/reconciliation.rs b/crates/asap-aware-mapping/src/accuracy/reconciliation.rs index 36a219d4e..a8b16bcae 100644 --- a/crates/asap-aware-mapping/src/accuracy/reconciliation.rs +++ b/crates/asap-aware-mapping/src/accuracy/reconciliation.rs @@ -3,7 +3,7 @@ //! //! ## The gap this closes //! -//! `asap_types::pre_asap::cse::share_common_sub_dags` (pre-ASAP CSE) only +//! `asap_types::ir::cse::share_common_sub_dags` (pre-ASAP CSE) only //! ever merges two sub-DAGs that are *exactly* [`PartialEq`]-equal, //! including their [`AggIntent`]'s `accuracy: AccuracyTarget` field. Two //! otherwise-identical aggregates that differ *only* in how tight an @@ -303,7 +303,7 @@ impl AccuracyReconciliationStrategy { /// /// Also requires the candidate's own *output* schema to carry a provable /// unique key ([`Schema::has_unique_key`]) — the exact legality gate - /// `pre_asap::cse::share_common_sub_dags` already applies to its own + /// `ir::cse::share_common_sub_dags` already applies to its own /// sharing decisions, and [`crate::rollup::RollupStrategy`] already /// reuses verbatim for the identical reason (see that module's /// `is_legal_rollup_source` doc, point 4): a producer's output is only diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 8256d6f10..7ba28d0ed 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -2164,8 +2164,6 @@ pub enum AnalyticalCostError { UnsupportedDataArrival(DataArrival), #[error("ingestion rate must be finite and non-negative, got {0}")] InvalidIngestionRate(f64), - #[error("summary lifecycle, maintenance mode, and evaluation schedule are inconsistent")] - IncompatibleLifecycleGuarantee, #[error("bootstrap row and byte evidence must either both be zero or both be non-zero")] InconsistentBootstrapEvidence, #[error("required summary operation cost {0} must be finite and positive, got {1}")] @@ -2196,7 +2194,7 @@ pub enum AnalyticalCostError { UnsupportedQueryOperator, #[error("inconsistent operator statistics: {0}")] InconsistentOperatorStatistics(&'static str), - #[error("summary operation {0} has no lifecycle-aware cost formula")] + #[error("summary operation {0} has no cost formula")] UnsupportedSummaryOperation(&'static str), #[error("required comparison-scope field {0} is missing")] MissingComparisonScope(&'static str), diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 9fc170666..9589d1382 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -51,12 +51,10 @@ use std::rc::Rc; use crate::exact_composition::ExactOperation; use asap_types::ir::{ASAPOp, Operator, OperatorNode}; use asap_types::post_asap::{ - FieldDataType, GroupingStrategy, HydraParams, ResultGuarantee, SketchAlgorithm, SketchParams, - SketchStatistic, SummaryMaintenanceLifecycleGuarantee, SummaryWindowFramework, + FieldDataType, GroupingStrategy, HydraParams, SketchAlgorithm, SketchParams, SketchStatistic, }; use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::expr_ir::ColumnRef; -use asap_types::types::AccuracyTarget; use crate::exact_composition::{ExactComposition, OperationPlacement}; use crate::recurrence::{ @@ -66,9 +64,6 @@ use crate::recurrence::{ use crate::replacement::{ realize_child, Realization, Replacement, ReplacementProvenance, ReplacementSubDAG, TargetSubDAG, }; -use crate::summary_maintenance_lifecycle::{ - SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCostInputs, -}; // ── Recurring-cost vocabulary for mixed exact/summary plans (issue #171) ── @@ -298,38 +293,6 @@ pub struct CseCandidate<'a> { #[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] pub struct Cost(pub f64); -/// One physical summary state and the lifecycle selected for that exact DAG -/// node. Node identity is preserved so whole-DAG models can bind per-state -/// evidence without relying on traversal order. -pub struct CostedSummaryDeployment<'a> { - pub summary: &'a OperatorNode, - pub guarantee: &'a SummaryMaintenanceLifecycleGuarantee, - pub selected_cost: Cost, -} - -/// Complete candidate estimate returned to lifecycle and global plan search. -/// -/// A deployment-aware model may compare abstract summary-window primitives -/// using evidence supplied by downstream implementations. `window_frameworks` -/// is planner IR: it records the selected semantic realization contract. -/// `physical_plan_id` is separate provider-owned provenance for the concrete -/// implementation whose evidence won; it is not interpreted as planner IR. -#[derive(Debug, Clone, PartialEq)] -pub struct CompleteSummaryCandidateEstimate { - pub cost: Cost, - /// Stable provider identity of the complete implementation whose evidence - /// produced this estimate. - pub physical_plan_id: Option, - /// Window choice for each entry of the `deployments` slice passed to the - /// complete-cost hook. `None` explicitly means that deployment does not - /// use a summary-window framework. - pub window_frameworks: Vec>, - /// End-to-end guarantee supplied by the selected window realization. - /// `None` means that the complete model supplied no window-specific - /// guarantee; `Some` may be exact or approximate. - pub window_accuracy_guarantee: Option, -} - impl Cost { /// The cost of an operation that costs nothing at all. pub const ZERO: Cost = Cost(0.0); @@ -770,112 +733,6 @@ pub trait CostModel { f64::NAN } - /// Primitive build, update, read, retention, and retirement costs used to - /// compare physical summary-state lifecycles. Unknown values stay - /// unknown, preventing long-lived deployments from winning through - /// optimistic zeroes. - fn summary_maintenance_lifecycle_cost_inputs( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - SummaryMaintenanceLifecycleCostInputs::default() - } - - /// Horizon-aware form used by lifecycle planning. Models whose retention - /// objective is capacity rather than byte-seconds can normalize their - /// rate so the horizon integral equals one peak-capacity charge. - fn summary_maintenance_lifecycle_cost_inputs_for_horizon( - &self, - summary: &OperatorNode, - _horizon: Option, - ) -> SummaryMaintenanceLifecycleCostInputs { - self.summary_maintenance_lifecycle_cost_inputs(summary) - } - - /// Physical update/merge/delete support for one concrete summary. The - /// conservative default advertises no long-lived maintenance capability. - fn summary_maintenance_capabilities( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceCapabilities { - SummaryMaintenanceCapabilities::default() - } - - /// Replace the sum of selected per-state lifecycle costs with a complete - /// root-DAG cost. The default preserves legacy models. Evidence-strict - /// models return `None` when any root operation is unavailable; callers - /// must not then reuse the partial per-state sum. - fn complete_summary_candidate_cost( - &self, - _root: &OperatorNode, - _target: Option<&OperatorNode>, - deployments: &[CostedSummaryDeployment<'_>], - _horizon: Option, - _expected_reads: Option, - _required_accuracy: &[AccuracyTarget], - ) -> Option { - Some(Cost( - deployments - .iter() - .map(|deployment| deployment.selected_cost.0) - .sum(), - )) - } - - /// Complete cost together with selected implementation provenance and - /// planner-visible window primitives. The default preserves cost models - /// that do not perform either decision. - fn complete_summary_candidate_estimate( - &self, - root: &OperatorNode, - target: Option<&OperatorNode>, - deployments: &[CostedSummaryDeployment<'_>], - horizon: Option, - expected_reads: Option, - required_accuracy: &[AccuracyTarget], - ) -> Option { - self.complete_summary_candidate_cost( - root, - target, - deployments, - horizon, - expected_reads, - required_accuracy, - ) - .map(|cost| CompleteSummaryCandidateEstimate { - cost, - physical_plan_id: None, - window_frameworks: vec![None; deployments.len()], - window_accuracy_guarantee: None, - }) - } - - /// Whether the complete-candidate hook is authoritative for lifecycle - /// costs. When true, lifecycle alternatives rejected only because their - /// legacy per-state cost is missing remain eligible for complete-DAG - /// evaluation. Semantic and runtime-capability rejections still apply. - fn complete_summary_candidate_estimate_covers_lifecycle_costs(&self) -> bool { - false - } - - /// Cost of evaluating `target` directly from its logical/raw inputs once. - /// When known, lifecycle-aware materialization compares this fallback with - /// the aggregate cost of the selected summary deployments. - fn raw_query_recompute_cost(&self, _target: &OperatorNode) -> Option { - None - } - - /// Complete raw cost over the comparison context. The default preserves - /// per-read models; context-aware models override this when raw input - /// cardinality changes between evaluations. - fn raw_query_recompute_total_cost( - &self, - target: &OperatorNode, - expected_reads: f64, - ) -> Option { - self.raw_query_recompute_cost(target) - .map(|per_read| Cost(per_read.0 * expected_reads)) - } /// Physical feasibility evidence for a complete summary candidate. /// `None` defers admission to physical/deployment compilation; `Some(false)` /// excludes the candidate without changing its computation or parameters. diff --git a/crates/asap-aware-mapping/src/empirical_cost.rs b/crates/asap-aware-mapping/src/empirical_cost.rs index 18d1d7ee4..eb9904f77 100644 --- a/crates/asap-aware-mapping/src/empirical_cost.rs +++ b/crates/asap-aware-mapping/src/empirical_cost.rs @@ -2,16 +2,14 @@ //! configuration and environment; they are neither runtime feedback nor proofs //! of an accuracy guarantee. CPU quantities are nanoseconds, never CPU operations. -use asap_types::ir::{ASAPOp, Operator, OperatorNode}; -use asap_types::post_asap::{FieldDataType, GroupingStrategy, SketchAlgorithm, SketchParams}; +use asap_types::post_asap::{SketchAlgorithm, SketchParams}; use asap_types::pre_asap::AggIntent; use serde::{Deserialize, Serialize}; -use crate::cost_model::{Cost, CostModel, DefaultCostModel}; +use crate::cost_model::{CostModel, DefaultCostModel}; use crate::replacement::{ accuracy_budget, accuracy_target, default_size_params, ReplacementSubDAG, TargetSubDAG, }; -use crate::summary_maintenance_lifecycle::SummaryMaintenanceLifecycleCostInputs; pub const EVIDENCE_SCHEMA_VERSION: u32 = 1; pub const EVIDENCE_MODEL_VERSION: &str = "empirical-update-cpu-v1"; @@ -207,40 +205,6 @@ impl EmpiricalEvidenceProvider { costs.sort_by(|a, b| a.1.total_cmp(&b.1)); costs.into_iter().map(|(algorithm, _)| algorithm).collect() } - - /// Costs for one independently instantiated sketch state, in CPU ns. - /// Unknown retention/retirement remain unavailable; CPU time must not be - /// mixed with an existing deployment's unitless or CPU-operation costs. - pub fn lifecycle_cost_inputs( - &self, - summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - let Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance), - grouping: GroupingStrategy::PerSubpopulationInstance, - .. - }) = &summary.operator - else { - return SummaryMaintenanceLifecycleCostInputs::default(); - }; - let Ok(row) = self.lookup(kind.algorithm(), kind.params()) else { - return SummaryMaintenanceLifecycleCostInputs::default(); - }; - SummaryMaintenanceLifecycleCostInputs { - build_cost: snapshot_build_cpu(row).map(Cost), - maintenance_cost_per_update: row - .metrics - .resources - .cpu - .update_cpu_ns - .as_ref() - .map(|m| Cost(m.value)), - // A point-frequency benchmark read does not price a total-count - // or quantile read. There is no query request in this hook. - summary_read_cost: None, - ..Default::default() - } - } } /// Standalone adapter for the existing planner boundary. Empirical data changes @@ -276,13 +240,6 @@ impl CostModel for EmpiricalCostModel { fn estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64 { DefaultCostModel.estimate_cost(candidate, target) } - - fn summary_maintenance_lifecycle_cost_inputs( - &self, - summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - self.provider.lifecycle_cost_inputs(summary) - } } impl EvidenceArtifact { @@ -360,14 +317,6 @@ fn nonnegative(value: f64) -> bool { value.is_finite() && value >= 0.0 } -fn snapshot_build_cpu(row: &OfflineMeasurement) -> Option { - let cpu = row.metrics.resources.cpu.build_cpu_ns.as_ref()?.value - + row.metrics.resources.cpu.update_cpu_ns.as_ref()?.value - * row.distribution.sample_count as f64 - + snapshot_prepare_cpu(row)?; - nonnegative(cpu).then_some(cpu) -} - /// The existing fixed-snapshot CMS/CountSketch contract needs no separate /// preparation. Other families must measure that phase, including an explicit /// zero when no preparation is necessary; absence is not free work. @@ -490,30 +439,6 @@ mod tests { ); } - /// Lifecycle build includes all measured snapshot updates, not just an empty - /// allocation. A missing update measurement cannot become free ingestion. - #[test] - fn lifecycle_build_requires_complete_snapshot_ingestion() { - let (mut artifact, _, _) = fixture(); - let row = &mut artifact.records[0]; - row.metrics.resources.cpu.build_cpu_ns = Some(Measurement { - value: 10.0, - stddev: None, - samples: 1, - method: None, - }); - assert_eq!(snapshot_build_cpu(row), Some(20010.0)); - row.metrics.resources.cpu.prepare_cpu_ns = Some(Measurement { - value: 17.0, - stddev: None, - samples: 1, - method: None, - }); - assert_eq!(snapshot_build_cpu(row), Some(20027.0)); - row.metrics.resources.cpu.update_cpu_ns = None; - assert_eq!(snapshot_build_cpu(row), None); - } - /// Newly shared optional dimensions receive the same numeric validation. #[test] fn optional_prepare_and_scan_measurements_are_validated() { @@ -537,28 +462,21 @@ mod tests { /// Only the established frequency-sketch contract can omit preparation. #[test] - fn unmeasured_preparation_for_other_families_keeps_build_unknown() { + fn unmeasured_preparation_for_other_families_stays_unknown() { let (mut artifact, _, _) = fixture(); let row = &mut artifact.records[0]; - row.metrics.resources.cpu.build_cpu_ns = Some(Measurement { - value: 10.0, - stddev: None, - samples: 1, - method: None, - }); - assert_eq!(snapshot_build_cpu(row), Some(20010.0)); row.algorithm = SketchAlgorithm::CountSketch; assert_eq!(snapshot_prepare_cpu(row), Some(0.0)); row.algorithm = SketchAlgorithm::Kll; row.params = SketchParams::Kll { k: 269 }; - assert_eq!(snapshot_build_cpu(row), None); + assert_eq!(snapshot_prepare_cpu(row), None); row.metrics.resources.cpu.prepare_cpu_ns = Some(Measurement { value: 17.0, stddev: None, samples: 1, method: None, }); - assert_eq!(snapshot_build_cpu(row), Some(20027.0)); + assert_eq!(snapshot_prepare_cpu(row), Some(17.0)); } fn fixture() -> (EvidenceArtifact, EvidenceContext, AggIntent) { diff --git a/crates/asap-aware-mapping/src/exact_composition.rs b/crates/asap-aware-mapping/src/exact_composition.rs index f48bc08b5..618265e9b 100644 --- a/crates/asap-aware-mapping/src/exact_composition.rs +++ b/crates/asap-aware-mapping/src/exact_composition.rs @@ -67,7 +67,7 @@ use std::rc::Rc; use asap_types::ir::aggregate_schema::aggregate_output_schema; use asap_types::ir::operator_properties::Reduction; -use asap_types::ir::timing::{planned_data_state, validate_default}; +use asap_types::ir::timing::{planned_data_state, validate_maintained}; use asap_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate}; use asap_types::post_asap::execution_data_state::lift_plain; use asap_types::post_asap::{ @@ -202,7 +202,7 @@ impl ExactComposition { /// Build the composed, data_state-validated node over `child`. Every edge of /// the result (including everything beneath `child`) is checked by - /// `asap_types::ir::timing::validate_default`; an illegal + /// `asap_types::ir::timing::validate_maintained`; an illegal /// placement is a typed [`RealizationError::ExecutionDataState`], never deferred to a /// runtime. pub fn compose(&self, child: Rc) -> Result, RealizationError> { @@ -261,7 +261,7 @@ impl ExactComposition { OperatorNode::with_schema(Operator::NonASAP(self.op.clone().into_op(child)), schema) .with_guarantee(guarantee), ); - validate_default(&node, self.placement.data_state().timing)?; + validate_maintained(&node, self.placement.data_state().timing)?; Ok(node) } @@ -660,7 +660,7 @@ mod tests { Operator::NonASAP(NonASAPOp::Aggregate { .. }) )); // Timing is no longer stored by composition: under the default - // lifecycle assignment the composed read-time operation runs at + // materialization assignment the composed read-time operation runs at // query time. assert_eq!(timed(&composed).timing, Some(ExecutionTiming::QueryTime)); assert!(composed @@ -701,7 +701,7 @@ mod tests { composed.operator, Operator::NonASAP(NonASAPOp::Aggregate { .. }) )); - validate_default(&composed, ExecutionTiming::IngestionTime).unwrap(); + validate_maintained(&composed, ExecutionTiming::IngestionTime).unwrap(); assert_eq!( planned_data_state(&composed, ExecutionTiming::IngestionTime).timing, ExecutionTiming::IngestionTime diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 15fe06d2b..443fcaca1 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -22,9 +22,9 @@ //! It depends only on the IR crate, never on a front end — the layering //! invariant (arrows point up) holds here too. //! -//! Post-lowering **canonicalization** is *not* here: it landed in -//! `asap_types::pre_asap::canonicalize`, run inside the shared `resolve_root` -//! so every front end normalizes before the pre-ASAP IR leaves resolution +//! Post-lowering **canonicalization** is *not* here: it lives in +//! `asap_types::ir::canonicalize`, run inside `asap_frontend_common`'s shared +//! `resolve_root` so every front end normalizes before the IR leaves resolution //! (issue #34, closed). //! //! ## Planning workflows @@ -41,14 +41,11 @@ //! to obtain ranked views, and perform selection downstream. //! - Call [`CandidateLogicalASAPDAGs::global_selection`] once for the workload, then //! [`GlobalSelection::assemble_selected_dag`] for each query root. This -//! coordinates logical choices and preserves shared nodes, but makes no -//! summary-maintenance lifecycle decision. -//! - When Planner owns maintenance-versus-recomputation decisions, use -//! [`global_selection_with_summary_maintenance_lifecycles`] followed by -//! [`assemble_selected_dag_with_summary_maintenance_lifecycles`] per root. -//! This alternative workflow returns [`SummaryMaintenanceLifecyclePlan`] -//! values containing DAG roots and maintenance decisions; callers do not need -//! to run ordinary selection/assembly first. +//! coordinates logical choices and preserves shared nodes. Whether and when +//! a summary state is materialized is not decided here: every summary runs +//! at query time until Stage 2 materialization (#509) owns that choice. +//! - Run the whole pipeline through [`optimize`] with [`MajorPass`], which +//! performs the two steps above for every root of a parsed workload. //! //! Models and evidence determine which choices the helpers can justify. //! Physical operator binding, placement, storage, deployment, and execution @@ -83,7 +80,7 @@ //! re-deriving it from an already-computed, strictly finer sibling //! `Aggregate` over identical child IR instead of an independent pass //! over the raw source — the cross-aggregate sibling of -//! `pre_asap::cse::share_common_sub_dags`'s identical-sub-DAG sharing. +//! `ir::cse::share_common_sub_dags`'s identical-sub-DAG sharing. //! [`rollup::is_legal_rollup_source`] is the standalone legality predicate //! other axes (e.g. issue #256's `GroupingStrategy`) are expected to //! consult directly, so it and this module's `RollupStrategy` can never @@ -115,7 +112,7 @@ //! //! | Term | Meaning | Entry point | //! |---|---|---| -//! | Schema resolution | Derive input schemas and resolve column names to positions | `asap_types::pre_asap::SchemaResolver::resolve_schema`, `resolve_root` | +//! | Schema resolution | Derive input schemas and resolve column names to positions | `asap_frontend_common::schema_resolver::SchemaResolver::resolve_schema`, `asap_frontend_common::resolve::resolve_root` | //! | Realization | Enumerate ranked physical forms for one aggregate intent | `replacement::realizations_for_intent` | //! | Replacement | Construct each candidate summary sub-DAG | [`replacement::ASAPStrategies`] | //! | Search | Enumerate and compare alternatives across a workload | [`replacement::search_workload`] | @@ -172,9 +169,6 @@ pub mod replacement; pub mod rewrite; pub mod rollup; pub mod storage_io; -pub mod summary_maintenance_cost; -pub mod summary_maintenance_dag_export; -pub mod summary_maintenance_lifecycle; #[cfg(test)] mod test_support; pub mod topk_reuse; @@ -185,7 +179,6 @@ pub use accuracy::{ CompositionShape, DefaultAccuracyModel, EqualSplitAllocator, NoAccuracyEvidence, PropagationStats, WorkloadAccuracyEvidence, }; -pub use cost_model::CompleteSummaryCandidateEstimate; pub use cost_model::{ maintenance_operation_plan_cost_rate, raw_recompute_cost_rate, read_operation_plan_cost_rate, CostModel, CostProvenance, CostUnit, DefaultCostModel, ExactCompositionCostInputs, @@ -197,9 +190,8 @@ pub use explanation::{ }; pub use grouping::{has_subpopulations, HydraGroupingStrategy}; pub use pass::{ - optimize, LifecycleInput, MajorPass, OptimizationInput, OptimizationInputError, - OptimizationPass, OptimizeError, PassNameConflict, PassRegistry, PlanOutput, PlanningModels, - QueryLifecyclePlan, + optimize, MajorPass, OptimizationInput, OptimizationInputError, OptimizationPass, + OptimizeError, PassNameConflict, PassRegistry, PlanOutput, PlanningModels, QueryPlan, }; pub use recurrence::{ evaluation_rate_of, total_cost, update_rate_from_data_workload, CostRate, EvaluationRate, @@ -216,22 +208,6 @@ pub use replacement::{ MAX_SEARCH_ITERATIONS, }; pub use rewrite::{AvgToSumOverCountStrategy, SemanticEquivalentRewriteStrategy}; -pub use summary_maintenance_dag_export::{ - export_summary_maintenance_plan, SummaryMaintenanceDAGExport, - SummaryMaintenanceDeploymentExport, SummaryMaintenanceLifecycleAlternativeExport, -}; -pub use summary_maintenance_lifecycle::{ - assemble_selected_dag_with_summary_maintenance_lifecycles, - enumerate_summary_maintenance_lifecycles, execution_timed_workload_dag, - global_selection_with_summary_maintenance_lifecycles, plan_summary_maintenance_lifecycles, - SummaryMaintenanceCapabilities, SummaryMaintenanceDeployment, - SummaryMaintenanceLifecycleAlternative, SummaryMaintenanceLifecycleAssemblyError, - SummaryMaintenanceLifecycleCandidates, SummaryMaintenanceLifecycleCapabilities, - SummaryMaintenanceLifecycleChoiceError, SummaryMaintenanceLifecycleCostInputs, - SummaryMaintenanceLifecyclePlan, SummaryMaintenanceLifecyclePlanError, - SummaryMaintenanceLifecycleRejection, SummaryMaintenanceLifecycleSelectionError, - SummaryMaintenanceTimingError, WorkloadDemand, -}; pub use topk_reuse::TopKLimitReuseStrategy; pub mod maintained_population; diff --git a/crates/asap-aware-mapping/src/maintained_population.rs b/crates/asap-aware-mapping/src/maintained_population.rs index 49a0a6030..01ae82914 100644 --- a/crates/asap-aware-mapping/src/maintained_population.rs +++ b/crates/asap-aware-mapping/src/maintained_population.rs @@ -306,15 +306,17 @@ mod tests { use crate::test_support::lower_promql; use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::export::compile_physical_asap_dag as export_timed; - use asap_types::ir::timing::{apply_lifecycle_timings, LifecycleAssignment, TimingMemo}; + use asap_types::ir::timing::{ + apply_materialization_timings, MaterializationAssignment, TimingMemo, + }; - /// Time `root` under the default lifecycle assignment (which runs the + /// Time `root` under the default materialization assignment (which runs the /// data-state / population-contract validation) and export it. fn compile_physical_asap_dag(root: &Rc) -> Result<(), String> { root.validate_structure().map_err(|e| e.to_string())?; - let timed = apply_lifecycle_timings( + let timed = apply_materialization_timings( root, - &LifecycleAssignment::default_maintained(), + &MaterializationAssignment::all_query_time(), &mut TimingMemo::new(), ) .map_err(|e| format!("{e:?}"))?; @@ -440,7 +442,7 @@ mod tests { assert_eq!(p.grouping, ["instance"]); assert_eq!(p.matchers[0].operation, CurrentSeriesMatch::Regex); } - // Population timing is a lifecycle choice: a retained or rebuilt + // Population timing is a materialization choice: a retained or rebuilt // population both validate, while its evaluation must stay at query time. #[test] fn population_timing_is_not_structural() { diff --git a/crates/asap-aware-mapping/src/pass/major.rs b/crates/asap-aware-mapping/src/pass/major.rs index 7be53e0f0..b8306c65a 100644 --- a/crates/asap-aware-mapping/src/pass/major.rs +++ b/crates/asap-aware-mapping/src/pass/major.rs @@ -12,12 +12,8 @@ use std::rc::Rc; use asap_types::ir::OperatorNode; use asap_types::types::AccuracyTarget; -use super::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, QueryLifecyclePlan}; +use super::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, QueryPlan}; use crate::replacement::{default_strategies_with_evidence, search_workload_with_targets}; -use crate::summary_maintenance_lifecycle::{ - global_selection_with_summary_maintenance_lifecycles, plan_assembled_dag, shared_state_cost, - summary_states, WorkloadDemand, -}; /// The shipped algorithm. Unit struct: its strategy set is the crate default, /// and a caller who wants a different one now has a better option than @@ -36,8 +32,7 @@ impl OptimizationPass for MajorPass { let strategies = default_strategies_with_evidence(models.cost, models.evidence); // `Id` is the entry's position in `QueryWorkload::entries()`, so the - // search result carries the workload binding the lifecycle stage and - // the output both need. CSE may make two identical queries share one + // search result carries the workload binding the output needs. CSE may make two identical queries share one // `Rc`, but it never drops or reorders a root, so this stays aligned. let roots: Vec<(usize, Rc, Option)> = workload .entries() @@ -53,32 +48,7 @@ impl OptimizationPass for MajorPass { let space = search_workload_with_targets(roots, &strategies, models.accuracy); - let lifecycle = input.lifecycle; - - // One index per root, in `CandidateLogicalASAPDAGs::roots` order — which is the order - // the roots went in, which is `entries()` order. - let entry_indices = workload.operator_indices().to_vec(); - let demand = WorkloadDemand { - workload: workload.query_workload(), - data_workload: workload.data_workload(), - entry_indices: &entry_indices, - }; - - let selection = global_selection_with_summary_maintenance_lifecycles( - &space, - demand, - lifecycle.now_ms, - lifecycle.horizon, - lifecycle.capabilities, - models.cost, - ) - .map_err(OptimizeError::LifecycleSelection)?; - // Each root's lifecycle is planned against the entries that consume - // it — the same binding selection costed it with — not the whole - // workload, so one query's reads never amortize another's state. - let bindings = space - .workload_entries_by_target(demand.workload, &entry_indices) - .map_err(|error| OptimizeError::LifecycleSelection(error.into()))?; + let selection = space.global_selection(models.cost); // Assemble every root, then intern structurally identical summary // producers across them once, so two queries that selected the same @@ -87,101 +57,17 @@ impl OptimizationPass for MajorPass { for (entry_index, root) in &space.roots { let dag = selection .assemble_selected_dag(root) - .map_err(|source| OptimizeError::LifecycleAssembly { + .map_err(|source| OptimizeError::Realization { entry_index: *entry_index, - source: source.into(), + source, })? .ok_or_else(|| self.missing_group(*entry_index))?; - assembled.push(dag); + assembled.push((*entry_index, dag)); } - let interned = share_common_sub_dags(assembled.iter().cloned().enumerate().collect()); - let states: Vec<_> = interned - .iter() - .map(|(_, dag)| summary_states(dag)) + let plans = share_common_sub_dags(assembled) + .into_iter() + .map(|(entry_index, root)| QueryPlan { entry_index, root }) .collect(); - - // A state reached from several roots is planned once against all of - // their entries, in every plan that reaches it, so each plan picks - // the same lifecycle for it. When that union cannot be costed the - // roots keep their own, unshared DAG and entries. - let mut shared_entries: Vec<(Rc, Option>)> = Vec::new(); - for (position, (entry_index, root)) in space.roots.iter().enumerate() { - for state in &states[position] { - if shared_entries.iter().any(|(s, _)| Rc::ptr_eq(s, state)) { - continue; - } - let readers: Vec<_> = (0..space.roots.len()) - .filter(|&other| states[other].iter().any(|s| Rc::ptr_eq(s, state))) - .map(|other| &space.roots[other].1) - .collect(); - if readers.iter().all(|reader| Rc::ptr_eq(reader, root)) { - continue; - } - let mut entries: Vec = readers - .iter() - .flat_map(|reader| bindings[&Rc::as_ptr(reader)].iter().copied()) - .collect(); - entries.sort_unstable(); - entries.dedup(); - let cost = shared_state_cost( - state, - WorkloadDemand { - entry_indices: &entries, - ..demand - }, - lifecycle.now_ms, - lifecycle.horizon, - lifecycle.capabilities, - models.cost, - ) - .map_err(|source| OptimizeError::LifecycleAssembly { - entry_index: *entry_index, - source: source.into(), - })?; - shared_entries.push((Rc::clone(state), cost.map(|_| entries))); - } - } - - let mut plans = Vec::with_capacity(space.roots.len()); - for (position, (entry_index, root)) in space.roots.iter().enumerate() { - let mut entries = bindings[&Rc::as_ptr(root)].clone(); - let mut dag = Rc::clone(&interned[position].1); - for (state, shared) in &shared_entries { - if !states[position].iter().any(|s| Rc::ptr_eq(s, state)) { - continue; - } - match shared { - Some(shared) => entries.extend(shared), - None => { - entries = bindings[&Rc::as_ptr(root)].clone(); - dag = Rc::clone(&assembled[position]); - break; - } - } - } - entries.sort_unstable(); - entries.dedup(); - let plan = plan_assembled_dag( - dag, - root, - WorkloadDemand { - entry_indices: &entries, - ..demand - }, - lifecycle.now_ms, - lifecycle.horizon, - lifecycle.capabilities, - models.cost, - ) - .map_err(|source| OptimizeError::LifecycleAssembly { - entry_index: *entry_index, - source, - })?; - plans.push(QueryLifecyclePlan { - entry_index: *entry_index, - plan, - }); - } let mut output = PlanOutput::new(plans); output.scalar_roots = workload.scalar_roots().to_vec(); Ok(output) diff --git a/crates/asap-aware-mapping/src/pass/mod.rs b/crates/asap-aware-mapping/src/pass/mod.rs index afd63dbdb..6355b8915 100644 --- a/crates/asap-aware-mapping/src/pass/mod.rs +++ b/crates/asap-aware-mapping/src/pass/mod.rs @@ -17,20 +17,20 @@ mod major; use std::collections::BTreeMap; use std::rc::Rc; +use asap_types::ir::export::compile_physical_asap_workload; +use asap_types::ir::timing::{ + apply_materialization_timings, MaterializationAssignment, TimingMemo, +}; use asap_types::ir::OperatorNode; use asap_types::parsed_workload::ParsedWorkload; +use asap_types::post_asap::ExecutionDataStateError; use asap_types::workload::WorkloadError; use crate::accuracy::{ AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, }; use crate::cost_model::{CostModel, DefaultCostModel}; -use crate::recurrence::Horizon; use crate::replacement::RealizationError; -use crate::summary_maintenance_lifecycle::{ - SummaryMaintenanceLifecycleAssemblyError, SummaryMaintenanceLifecycleCapabilities, - SummaryMaintenanceLifecyclePlan, SummaryMaintenanceLifecycleSelectionError, -}; pub use major::MajorPass; @@ -90,66 +90,22 @@ impl<'a> PlanningModels<'a> { } } -/// Supplying this asks the pass to also decide summary maintenance versus raw -/// recomputation; leaving it out asks only for the logical DAG. -#[derive(Clone, Copy)] -#[non_exhaustive] -pub struct LifecycleInput { - /// Planning clock, Unix milliseconds. - pub now_ms: u64, - /// Seconds. Required to turn recurring demand into a finite total. - pub horizon: Option, - pub capabilities: SummaryMaintenanceLifecycleCapabilities, -} - -impl LifecycleInput { - pub fn new(now_ms: u64, capabilities: SummaryMaintenanceLifecycleCapabilities) -> Self { - Self { - now_ms, - horizon: None, - capabilities, - } - } - - pub fn with_horizon(mut self, horizon: Horizon) -> Self { - self.horizon = Some(horizon); - self - } -} - #[derive(Clone, Copy)] #[non_exhaustive] pub struct OptimizationInput<'a> { pub workload: &'a ParsedWorkload, pub models: PlanningModels<'a>, - /// Every plan carries the maintenance-versus-recomputation decision, so - /// the planning clock and runtime capabilities are always required. - pub lifecycle: LifecycleInput, } impl<'a> OptimizationInput<'a> { - pub fn new( - workload: &'a ParsedWorkload, - models: PlanningModels<'a>, - lifecycle: LifecycleInput, - ) -> Self { - Self { - workload, - models, - lifecycle, - } + pub fn new(workload: &'a ParsedWorkload, models: PlanningModels<'a>) -> Self { + Self { workload, models } } pub fn validate(&self) -> Result<(), OptimizationInputError> { self.workload .validate() - .map_err(OptimizationInputError::Workload)?; - if let Some(horizon) = self.lifecycle.horizon { - if !horizon.0.is_finite() || horizon.0 <= 0.0 { - return Err(OptimizationInputError::InvalidHorizon(horizon.0)); - } - } - Ok(()) + .map_err(OptimizationInputError::Workload) } } @@ -158,38 +114,34 @@ impl<'a> OptimizationInput<'a> { pub enum OptimizationInputError { #[error("workload: {0}")] Workload(WorkloadError), - #[error("planning horizon must be finite and positive, got {0}")] - InvalidHorizon(f64), } // ── Output ─────────────────────────────────────────────────────────────── -/// One query's selected post-ASAP DAG plus the maintenance decisions taken -/// for it. The DAG is `plan.root`. +/// One query's selected post-ASAP DAG. #[derive(Debug, Clone)] -pub struct QueryLifecyclePlan { +pub struct QueryPlan { /// Index into `QueryWorkload::entries()`. pub entry_index: usize, - pub plan: SummaryMaintenanceLifecyclePlan, + pub root: Rc, } -/// One multi-root workload DAG with query/lifecycle bindings in entry order; +/// One multi-root workload DAG with query bindings in entry order; /// [`check_contract`] enforces that. /// /// Plans are not deduplicated across entries: a summary state that several -/// queries share appears in each of their plans as the same `Rc` (with the -/// same lifecycle), so a consumer that deploys or costs the workload must -/// dedupe deployments by `Rc::ptr_eq` on the summary node. +/// queries share appears in each of their plans as the same `Rc`, so a +/// consumer that deploys or costs the workload must dedupe by `Rc::ptr_eq`. #[derive(Debug, Clone)] #[non_exhaustive] pub struct PlanOutput { - pub plans: Vec, + pub plans: Vec, /// Exact scalar expressions, keyed by workload entry; embedded plan reads remain visible. pub scalar_roots: Vec<(usize, asap_types::ir::ScalarExpr)>, } impl PlanOutput { - pub fn new(plans: Vec) -> Self { + pub fn new(plans: Vec) -> Self { Self { plans, scalar_roots: Vec::new(), @@ -218,7 +170,7 @@ impl PlanOutput { .map(|p| { ( p.entry_index, - asap_types::ir::QueryRoot::Operator(Rc::clone(&p.plan.root)), + asap_types::ir::QueryRoot::Operator(Rc::clone(&p.root)), ) }) .chain( @@ -233,7 +185,7 @@ impl PlanOutput { /// The selected operator roots. Use `roots()` to include scalar queries. pub fn operator_roots(&self) -> Vec> { - self.plans.iter().map(|p| Rc::clone(&p.plan.root)).collect() + self.plans.iter().map(|p| Rc::clone(&p.root)).collect() } /// Unique operators in the entire workload DAG, including scalar-plan dependencies. @@ -264,9 +216,16 @@ impl PlanOutput { /// roots have no physical form yet and are left out. pub fn execution_timed_dag( &self, - ) -> Result { - let plans: Vec<_> = self.plans.iter().map(|p| &p.plan).collect(); - crate::execution_timed_workload_dag(&plans) + ) -> Result { + // One memo, so a node shared by several roots is timed and exported once. + let mut memo = TimingMemo::new(); + let assignment = MaterializationAssignment::all_query_time(); + let timed = self + .plans + .iter() + .map(|p| apply_materialization_timings(&p.root, &assignment, &mut memo)) + .collect::, _>>()?; + compile_physical_asap_workload(&timed) } pub fn len(&self) -> usize { @@ -288,13 +247,6 @@ pub enum OptimizeError { entry_index: usize, source: RealizationError, }, - #[error("summary-maintenance-lifecycle selection: {0}")] - LifecycleSelection(SummaryMaintenanceLifecycleSelectionError), - #[error("entry {entry_index}: {source}")] - LifecycleAssembly { - entry_index: usize, - source: SummaryMaintenanceLifecycleAssemblyError, - }, /// The pass returned something the downstream contract forbids. This is a /// defect in the pass, not in its input. #[error("pass `{pass}` violated the output contract: {detail}")] diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 509314d5c..2020be8a5 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -56,9 +56,7 @@ //! does not establish a compatible workload plan or physical deployability. //! For Planner-owned logical selection, call [`CandidateLogicalASAPDAGs::global_selection`] //! once and [`GlobalSelection::assemble_selected_dag`] for each wanted query -//! root. Alternatively, use the summary-maintenance-lifecycle-aware helpers -//! when Planner should also compare maintenance against raw recomputation. -//! Physical binding, deployment, and execution remain downstream. +//! root. Physical binding, deployment, and execution remain downstream. //! //! Internally, [`realize_child`] and [`realize_one`] may take a preferred local //! realization while constructing or costing a candidate. That local operation @@ -77,7 +75,7 @@ //! binds (single intent, no `HAVING`), every entry becomes its own bound //! candidate. //! - [`SharedSubDAGStrategy`] wraps -//! `asap_types::pre_asap::cse::share_common_sub_dags`'s sharing decision. +//! `asap_types::ir::cse::share_common_sub_dags`'s sharing decision. //! Wherever a [`TargetSubDAG`] already has two or more consumers (i.e. //! `share_common_sub_dags` already collapsed two or more workload //! locations onto the same `Rc` — [`discover_targets`] below @@ -144,7 +142,7 @@ //! 1. **Per-target candidates, not flat plans.** [`TargetSubDAGCandidates`] //! stores the alternatives for one distinct [`TargetSubDAG`] (identified by //! its own `Rc` pointer identity — the same currency -//! [`asap_types::pre_asap::cse::share_common_sub_dags`] already +//! [`asap_types::ir::cse::share_common_sub_dags`] already //! established across the workload) holding every //! [`ReplacementSubDAG`] alternative discovered for it. [`CandidateLogicalASAPDAGs`] is //! a collection of these groups, keyed by `TargetSubDAG` — a candidate @@ -175,7 +173,7 @@ //! line above stands for: every `TargetSubDAG` this pass discovers is one //! iteration of that loop. It walks every workload root's whole DAG (the //! same **relational-skeleton** operator-child scope -//! `asap_types::pre_asap::cse::share_common_sub_dags` itself uses — see +//! `asap_types::ir::cse::share_common_sub_dags` itself uses — see //! that module's "Algorithm" section), discovering one `TargetSubDAG` per //! distinct `Rc` and a *real* `consumer_count`: how many operator-child //! positions anywhere in the workload reference that exact `Rc`, not just @@ -353,7 +351,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use asap_types::ir::cse::{share_common_sub_dags, structural_hash, HashCache}; use asap_types::ir::operator_properties::{BinaryOpKind, JoinKind, Reduction}; use asap_types::ir::summary_coverage::{CoverageRegion, SummaryCoverage}; -use asap_types::ir::timing::validate_default; +use asap_types::ir::timing::validate_maintained; use asap_types::ir::SchemaDerivationError; use asap_types::ir::{ ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, Predicate, ProjectItem, ScalarExpr, @@ -381,8 +379,8 @@ use crate::accuracy::{ DefaultAccuracyModel, EqualSplitAllocator, NoAccuracyEvidence, }; use crate::cost_model::{ - raw_recompute_cost_rate, Cost, CostModel, CseCandidate, DefaultCostModel, - ExactCompositionCostInputs, ExactCompositionCostRequest, ShareDecision, + raw_recompute_cost_rate, CostModel, CseCandidate, DefaultCostModel, ExactCompositionCostInputs, + ExactCompositionCostRequest, ShareDecision, }; use crate::exact_composition::{ExactComposition, ExactCompositionStrategy, OperationPlacement}; use crate::grouping::HydraGroupingStrategy; @@ -1401,10 +1399,11 @@ impl<'a> ASAPStrategies<'a> { fn place(node: &Rc) -> Option> { retime_rate_finalize(node, ExecutionTiming::IngestionTime, true) } + // Legal only if the candidate stays executable with its states maintained. let timed = |node: &Rc| { - asap_types::ir::timing::apply_lifecycle_timings( + asap_types::ir::timing::apply_materialization_timings( node, - &asap_types::ir::timing::LifecycleAssignment::default_maintained(), + &asap_types::ir::timing::MaterializationAssignment::all_ingestion_time(), &mut asap_types::ir::timing::TimingMemo::new(), ) .ok() @@ -1766,7 +1765,7 @@ impl ReplacementStrategy for ASAPStrategies<'_> { /// the logical root does not expose, so each is a finalized query result /// for the identity-carrying root. Placement variants (for example, /// fixed-window or query-time Rate aggregation) are not listed here: the - /// lifecycle assigns timing and the physical compiler reads it. + /// materialization assigns timing and the physical compiler reads it. fn propose_for_root(&self, root: &Rc, target: &AccuracyTarget) -> Proposals { let Ok(typed) = asap_types::ir::schema_support::with_promql_series_identity(root) else { return Proposals::default(); @@ -1976,7 +1975,7 @@ fn exact_topk_over_temporal_values( ) .with_guarantee(guarantee), ); - validate_default(&node, ExecutionTiming::QueryTime)?; + validate_maintained(&node, ExecutionTiming::QueryTime)?; Ok(Some(node)) } @@ -1995,7 +1994,7 @@ fn realize_temporal_average( return Ok(None); }; operator.checked_finite_division = true; - validate_default(&node, ExecutionTiming::QueryTime)?; + validate_maintained(&node, ExecutionTiming::QueryTime)?; Ok(Some(node)) } @@ -2343,7 +2342,7 @@ pub fn finalize_query_candidate( /// The read boundary's placement is fixed here, where the candidate's /// semantics decide it (a fresh query-time summary over this evaluation's -/// finalized values vs. finalized values feeding maintenance); the lifecycle +/// finalized values vs. finalized values feeding maintenance); the materialization /// timing pass honors it. fn finalize_exact_accumulator( node: Rc, @@ -2751,7 +2750,7 @@ fn finish_weighted_topk( ) .with_guarantee(guarantee), ); - validate_default(&result, ExecutionTiming::QueryTime)?; + validate_maintained(&result, ExecutionTiming::QueryTime)?; Ok(result) } @@ -3132,7 +3131,7 @@ fn construct_summary_agg( } else if snapshot_weighted { // Each evaluation's finalized rates feed a fresh summary; rate snapshots // must never accumulate across evaluations. Query time is only the - // initial layout; a retained summary's lifecycle moves it to ingestion. + // initial layout; a maintained summary's materialization moves it to ingestion. finalize_query_candidate(bound_child, &input.child)? } else { let child = @@ -4207,7 +4206,7 @@ impl CandidateLogicalASAPDAGs { } /// DAG candidates assembled from an unpriced search space. -/// This is an internal planning stage: callers must still validate lifecycle +/// This is an internal planning stage: callers must still validate materialization /// requirements and compile supported physical operators before deployment. /// The caller supplies a finite expansion budget; exceeding it is an error, /// never a silently truncated inventory presented as exhaustive. @@ -4393,50 +4392,6 @@ impl CandidateLogicalASAPDAGs { } } -/// Lifecycle-aware whole-subplan costs keyed by target and candidate identity. -#[derive(Default, Clone)] -pub(crate) struct CandidateCostOverrides { - costs: HashMap<(*const OperatorNode, *const ReplacementSubDAG), Cost>, - raw_costs: HashMap<*const OperatorNode, Cost>, - /// Targets for which the caller requested an atomic raw-vs-summary - /// decision. Other memo groups continue through ordinary CSE selection. - finalized_targets: HashSet<*const OperatorNode>, -} - -impl CandidateCostOverrides { - pub(crate) fn finalize_target(&mut self, target: &Rc) { - self.finalized_targets.insert(Rc::as_ptr(target)); - } - - fn finalizes(&self, target: &Rc) -> bool { - self.finalized_targets.contains(&Rc::as_ptr(target)) - } - - pub(crate) fn insert( - &mut self, - target: &Rc, - candidate: &ReplacementSubDAG, - cost: Cost, - ) { - self.costs - .insert((Rc::as_ptr(target), candidate as *const _), cost); - } - - fn get(&self, target: &Rc, candidate: &ReplacementSubDAG) -> Option { - self.costs - .get(&(Rc::as_ptr(target), candidate as *const _)) - .copied() - } - - pub(crate) fn insert_raw(&mut self, target: &Rc, cost: Cost) { - self.raw_costs.insert(Rc::as_ptr(target), cost); - } - - fn raw(&self, target: &Rc) -> Option { - self.raw_costs.get(&Rc::as_ptr(target)).copied() - } -} - impl CandidateLogicalASAPDAGs { /// One candidate set per discovered target sub-DAG, in discovery order. pub fn target_subdag_candidates(&self) -> impl Iterator { @@ -4851,54 +4806,6 @@ impl CandidateLogicalASAPDAGs { .map(|rate| UpdateRate(rate.0)); self.recurrence_profiles(&recurrences, update_rate) } - - /// Associate every discovered target with the normalized workload entries - /// whose roots can reach it. - pub(crate) fn workload_entries_by_target( - &self, - workload: &QueryWorkload, - root_workload_entries: &[usize], - ) -> Result>, RecurrenceError> { - let entry_count = workload.entries().count(); - if root_workload_entries.len() != self.roots.len() { - return Err(RecurrenceError::RootCountMismatch { - expected: self.roots.len(), - got: root_workload_entries.len(), - }); - } - let mut bindings: HashMap<*const OperatorNode, HashSet> = HashMap::new(); - for ((_, root), &entry_index) in self.roots.iter().zip(root_workload_entries) { - if entry_index >= entry_count { - return Err(RecurrenceError::InvalidWorkloadEntry { - index: entry_index, - entry_count, - }); - } - let mut seen = HashSet::new(); - let mut queue = VecDeque::from([Rc::as_ptr(root)]); - while let Some(ptr) = queue.pop_front() { - if !seen.insert(ptr) { - continue; - } - bindings.entry(ptr).or_default().insert(entry_index); - if let Some(group) = self.groups.get(&ptr) { - queue.extend( - direct_child_counts(&group.target) - .into_iter() - .map(|(child, _)| child), - ); - } - } - } - Ok(bindings - .into_iter() - .map(|(ptr, entries)| { - let mut entries: Vec<_> = entries.into_iter().collect(); - entries.sort_unstable(); - (ptr, entries) - }) - .collect()) - } } /// Record `times` occurrences of `recurrence` against `ptr` — `times > 1` @@ -5314,7 +5221,7 @@ impl<'a> GlobalSelection<'a> { return Ok(Rc::clone(node)); } // A selected summary that realizes its inner aggregate, instead of - // hiding it in `KeepPreAsap`, is kept; lifecycle assignment decides + // hiding it in `KeepPreAsap`, is kept; materialization assignment decides // whether it runs in precompute or at query time. let selected_composed_summary = self .groups @@ -5458,7 +5365,7 @@ impl<'a> GlobalSelection<'a> { let node = Rc::new( OperatorNode::with_schema(operator, target.schema.clone()).with_guarantee(guarantee), ); - validate_default(&node, ExecutionTiming::QueryTime)?; + validate_maintained(&node, ExecutionTiming::QueryTime)?; Ok(node) } @@ -5579,7 +5486,7 @@ fn relink_agg_child(node: &Rc, new_child: &Rc) -> Rc ) .with_guarantee(node.guarantee.clone()) }); - match validate_default(&rebuilt, ExecutionTiming::IngestionTime) { + match validate_maintained(&rebuilt, ExecutionTiming::IngestionTime) { Ok(_) => rebuilt, Err(_) => Rc::clone(node), } @@ -5761,7 +5668,7 @@ impl CandidateLogicalASAPDAGs { /// Uncertified DDSketch ratios remain in [`CandidateLogicalASAPDAGs`] for downstream /// inspection but are not chosen automatically by this selector. pub fn global_selection(&self, cost_model: &dyn CostModel) -> GlobalSelection<'_> { - self.global_selection_impl(cost_model, None, None, None) + self.global_selection_impl(cost_model, None, None) .expect("structural global selection cannot produce a recurrence error") } @@ -5775,17 +5682,7 @@ impl CandidateLogicalASAPDAGs { profiles: &RecurrenceProfileMap, horizon: Option, ) -> Result, RecurrenceError> { - self.global_selection_impl(cost_model, Some(profiles), horizon, None) - } - - pub(crate) fn global_selection_with_candidate_costs( - &self, - cost_model: &dyn CostModel, - profiles: &RecurrenceProfileMap, - horizon: Option, - costs: &CandidateCostOverrides, - ) -> Result, RecurrenceError> { - self.global_selection_impl(cost_model, Some(profiles), horizon, Some(costs)) + self.global_selection_impl(cost_model, Some(profiles), horizon) } fn global_selection_impl( @@ -5793,7 +5690,6 @@ impl CandidateLogicalASAPDAGs { cost_model: &dyn CostModel, profiles: Option<&RecurrenceProfileMap>, horizon: Option, - candidate_costs: Option<&CandidateCostOverrides>, ) -> Result, RecurrenceError> { let dag = reference_dag(self); let topo = topological_order(&self.order, &dag); @@ -5854,30 +5750,8 @@ impl CandidateLogicalASAPDAGs { } } - let lifecycle_choice = candidate_costs - .filter(|costs| costs.finalizes(&group.target)) - .map(|costs| { - let summary = group - .candidates - .iter() - .filter(|candidate| !is_composition_candidate(candidate)) - .filter(|candidate| is_automatically_selectable(candidate, cost_model)) - .filter_map(|candidate| { - costs - .get(&group.target, candidate) - .map(|cost| (candidate, cost)) - }) - .min_by(|(_, left), (_, right)| left.0.total_cmp(&right.0)); - match (summary, costs.raw(&group.target)) { - (Some((_, summary_cost)), Some(raw)) if raw.0 <= summary_cost.0 => None, - (Some((candidate, _)), _) => Some(candidate), - (None, _) => None, - } - }); - let complete_plan_choice = (!forced.is_some() && composed.is_none() - && lifecycle_choice.is_none() && cost_model.candidate_cost_covers_complete_plan()) .then(|| { let effective_target = TargetSubDAG::with_consumer_count(&group.target, effective); @@ -5917,8 +5791,6 @@ impl CandidateLogicalASAPDAGs { } else if let Some(option) = composed { composition_decision = Some(option.decision); Some(option.candidate) - } else if let Some(choice) = lifecycle_choice { - choice } else if cost_model.candidate_cost_covers_complete_plan() { complete_plan_choice } else if effective >= 2 && cse_candidate_pair(group).is_some() { @@ -7027,7 +6899,7 @@ mod tests { use super::*; use crate::accuracy::PropagationStats; use crate::cost_model::Cost; - use crate::test_support::{agg, agg_per_entity, lower_promql, metric_scan, timed}; + use crate::test_support::{agg, agg_per_entity, lower_promql, maintained, metric_scan, timed}; use asap_types::ir::operator_properties::{Reduction as ReductionTy, Source}; use asap_types::ir::TimeRangeKind; use asap_types::pre_asap::agg_intent::{ @@ -7081,9 +6953,9 @@ mod tests { } // Grouped Sum over Rate evaluations stays a summary state in the inventory, - // so lifecycle assignment can place it in precompute or at query time. + // so materialization assignment can place it in precompute or at query time. #[test] - fn grouped_rate_sum_inventory_keeps_sum_state_for_lifecycle_placement() { + fn grouped_rate_sum_inventory_keeps_sum_state_for_materialization_placement() { let root = lower_promql("sum by(job)(rate(m[1m]))", AccuracyTarget::Exact); let inventory = search_workload(vec![(0usize, root)]) .enumerate_candidate_dags(4096) @@ -10200,8 +10072,8 @@ mod tests { let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); let outer = agg(vec![], default_quantile(0.9), inner); // Timing is not stored during realization: time the candidate under - // the default lifecycle assignment to read the maintenance boundary. - let root = timed(&realize(&outer).unwrap()); + // a maintained materialization assignment to read the maintenance boundary. + let root = maintained(&realize(&outer).unwrap()); let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { panic!("expected estimate root, got {:?}", root.operator); diff --git a/crates/asap-aware-mapping/src/rollup.rs b/crates/asap-aware-mapping/src/rollup.rs index 09dc16fa4..572d2c60a 100644 --- a/crates/asap-aware-mapping/src/rollup.rs +++ b/crates/asap-aware-mapping/src/rollup.rs @@ -20,7 +20,7 @@ //! "Rolling up aggregations on a fine-grained group by to get a //! coarse-grained group by (like AHA)," alongside "CSE across aggregations, //! and group by key management" — this strategy is the *cross-aggregate* -//! sibling of `pre_asap::cse::share_common_sub_dags`'s *identical*-sub-DAG +//! sibling of `ir::cse::share_common_sub_dags`'s *identical*-sub-DAG //! sharing: CSE shares two structurally-*equal* aggregates onto one `Rc`; //! this strategy relates two structurally-*different* (differently grouped) //! aggregates over the same shared source. @@ -207,7 +207,7 @@ fn rollup_combinator(intent: &AggIntent, finer_measure_col: ColumnId) -> Option< /// 4. `finer_output_schema` (the finer aggregate's own *output* schema, not /// the shared child's) carries a provable unique key /// ([`Schema::has_unique_key`]) — **the exact legality gate -/// `pre_asap::cse::share_common_sub_dags` already applies to its own +/// `ir::cse::share_common_sub_dags` already applies to its own /// sharing decisions**, reused verbatim here rather than re-invented: /// `share_common_sub_dags`'s own doc ("Legality: gated by /// `Schema::unique_keys`") states a producer's output is only safely diff --git a/crates/asap-aware-mapping/src/summary_maintenance_cost/estimator.rs b/crates/asap-aware-mapping/src/summary_maintenance_cost/estimator.rs deleted file mode 100644 index 911d90275..000000000 --- a/crates/asap-aware-mapping/src/summary_maintenance_cost/estimator.rs +++ /dev/null @@ -1,1219 +0,0 @@ -use super::*; - -pub(super) fn estimate_heterogeneous_summary( - root: &OperatorNode, - deployments: &[CostedSummaryDeployment<'_>], - evidence: &SummaryNodeEvidence, - scope: &ComparisonScope, - raw: &RawInputEvidence, - window_frameworks: &[Option], -) -> Result { - if window_frameworks.len() != deployments.len() { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "window framework assignments", - )); - } - let frameworks_by_node: HashMap<_, _> = deployments - .iter() - .zip(window_frameworks) - .map(|(deployment, framework)| (deployment.summary as *const _, framework)) - .collect(); - validate_summary_edges_and_physical_ids(root, evidence, &frameworks_by_node)?; - let evaluation_count = scope.validate()?; - let by_node: HashMap<_, _> = deployments - .iter() - .map(|deployment| (deployment.summary as *const _, deployment)) - .collect(); - let mut cpu_ops = 0.0; - let mut persistent_bytes = 0_u64; - let mut ephemeral_state_bytes = 0_u64; - let mut scans = HashMap::::new(); - let mut physical_states = HashMap::< - String, - ( - SummaryAggregateEvidence, - SummaryMaintenanceLifecycleGuarantee, - String, - Option, - ), - >::new(); - for deployment in deployments { - let node_evidence = evidence - .aggregation(deployment.summary) - .ok_or(AnalyticalCostError::MissingOrStale("summary_agg"))?; - let Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) = &deployment.summary.operator else { - return Err(AnalyticalCostError::UnsupportedCandidate); - }; - let inputs = node_evidence.inputs.validate()?; - validate_arrival_rate(scope.data_arrival, inputs.ingestion_rate_per_second)?; - match node_evidence.scan_selection_index { - Some(index) => { - let declared = - scope - .sources - .get(index) - .ok_or(AnalyticalCostError::MissingComparisonScope( - "summary scan selection", - ))?; - if !has_retained_subdag_evidence(child, evidence) - || inputs.initial_input_rows != raw.planning_time_input_rows - || inputs.initial_input_bytes != raw.planning_time_input_bytes - || inputs.initial_source_scan_bytes != raw.planning_time_source_scan_bytes - || inputs.ingestion_rate_per_second != raw.ingestion_rate_per_second - { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "source-root bootstrap evolution", - )); - } - let mut actual_selections = Vec::new(); - query_source_selections(child, &mut HashSet::new(), &mut actual_selections)?; - let actual_selections = deduplicate_source_selections(actual_selections); - let expected = ( - declared.source.clone(), - declared.predicates.clone(), - declared.info_matchers.clone(), - ); - if actual_selections.as_slice() != [expected] { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "summary source lineage", - )); - } - } - None => { - if has_retained_subdag_evidence(child, evidence) - || inputs.initial_source_scan_bytes != 0 - || !node_evidence.bootstrap_read_identity.is_empty() - { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "intermediate bootstrap source ownership", - )); - } - } - } - validate_guarantee(deployment.guarantee, scope.data_arrival)?; - let logical_state = format!("{:?}", deployment.summary.operator); - let window_framework = (*frameworks_by_node - .get(&(deployment.summary as *const _)) - .ok_or(AnalyticalCostError::MissingOrStale( - "window framework assignment", - ))?) - .clone(); - match physical_states.entry(node_evidence.physical_id.clone()) { - std::collections::hash_map::Entry::Vacant(entry) => { - entry.insert(( - node_evidence.clone(), - deployment.guarantee.clone(), - logical_state, - window_framework, - )); - } - std::collections::hash_map::Entry::Occupied(entry) - if entry.get() - != &( - node_evidence.clone(), - deployment.guarantee.clone(), - logical_state, - window_framework, - ) => - { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "summary physical identity", - )); - } - std::collections::hash_map::Entry::Occupied(_) => continue, - } - let ephemeral = matches!( - deployment.guarantee.summary_maintenance_lifecycle, - SummaryMaintenanceLifecycle::Ephemeral - ); - let (bootstrap, updates, source_scan_bytes) = if ephemeral { - ( - ephemeral_rows_over_horizon(inputs, scope)?, - 0, - ephemeral_scan_bytes_over_horizon(inputs, raw, scope)?, - ) - } else { - let (bootstrap, updates, _) = - lifecycle_row_counts(inputs, deployment.guarantee, scope)?; - let bootstrap_extra_rows = bootstrap - .checked_sub(inputs.initial_input_rows) - .ok_or(AnalyticalCostError::Overflow)?; - let source_scan_bytes = if node_evidence.scan_selection_index.is_some() { - inputs - .initial_source_scan_bytes - .checked_add( - bootstrap_extra_rows - .checked_mul(raw.arriving_source_row_bytes) - .ok_or(AnalyticalCostError::Overflow)?, - ) - .ok_or(AnalyticalCostError::Overflow)? - } else { - 0 - }; - (bootstrap, updates, source_scan_bytes) - }; - let insert = validated_operator_cpu("insert_cpu_ops", node_evidence.insert_cpu_ops)?; - let insert_calls = bootstrap - .checked_mul(inputs.bootstrap_window_count) - .and_then(|calls| { - updates - .checked_mul(inputs.active_window_count) - .and_then(|updates| calls.checked_add(updates)) - }) - .ok_or(AnalyticalCostError::Overflow)?; - cpu_ops += insert_calls as f64 * insert; - let live_window_count = if ephemeral { - inputs.bootstrap_window_count - } else { - inputs - .active_window_count - .checked_add(inputs.retained_window_count) - .ok_or(AnalyticalCostError::Overflow)? - }; - let state_bytes = live_window_count - .checked_mul(inputs.physical_summary_count) - .and_then(|states| states.checked_mul(inputs.state_bytes_per_summary)) - .ok_or(AnalyticalCostError::Overflow)?; - if ephemeral { - ephemeral_state_bytes = ephemeral_state_bytes - .checked_add(state_bytes) - .ok_or(AnalyticalCostError::Overflow)?; - } else { - persistent_bytes = persistent_bytes - .checked_add(state_bytes) - .ok_or(AnalyticalCostError::Overflow)?; - } - if let Some(source_index) = node_evidence.scan_selection_index { - if node_evidence.bootstrap_read_identity.is_empty() { - return Err(AnalyticalCostError::MissingOrStale( - "bootstrap_read_identity", - )); - } - match scans.entry(node_evidence.bootstrap_read_identity.clone()) { - std::collections::hash_map::Entry::Vacant(entry) => { - entry.insert((source_index, source_scan_bytes)); - } - std::collections::hash_map::Entry::Occupied(entry) - if *entry.get() != (source_index, source_scan_bytes) => - { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "bootstrap source bytes", - )); - } - _ => {} - } - } - } - let covered_sources: HashSet<_> = scans.values().map(|(index, _)| *index).collect(); - if covered_sources.len() != scope.sources.len() - || !(0..scope.sources.len()).all(|index| covered_sources.contains(&index)) - { - return Err(AnalyticalCostError::ComparisonScopeMismatch("sources")); - } - - #[expect(clippy::too_many_arguments, reason = "CPU and I/O traversal state")] - fn visit_ops( - node: &OperatorNode, - seen: &mut HashSet, - by_node: &HashMap<*const OperatorNode, &CostedSummaryDeployment<'_>>, - evidence: &SummaryNodeEvidence, - scope: &ComparisonScope, - evaluation_count: u64, - cpu_ops: &mut f64, - io_bytes: &mut u64, - ) -> Result<(), AnalyticalCostError> { - let physical_id = summary_physical_id(node, evidence)?; - if !seen.insert(physical_id) { - return Ok(()); - } - if has_retained_subdag_evidence(node, evidence) { - let retained = evidence - .retained_queries - .get(&(node as *const _)) - .ok_or(AnalyticalCostError::MissingOrStale("retain_exact"))?; - if !retained.preprocessing_cpu_ops_over_horizon.is_finite() - || retained.preprocessing_cpu_ops_over_horizon < 0.0 - { - return Err(AnalyticalCostError::InvalidOperationCost( - "retain_exact", - retained.preprocessing_cpu_ops_over_horizon, - )); - } - *cpu_ops += retained.preprocessing_cpu_ops_over_horizon; - return Ok(()); - } - match &node.operator { - Operator::NonASAP(NonASAPOp::BinaryOp { lhs, rhs, .. }) - | Operator::NonASAP(NonASAPOp::Join { - left: lhs, - right: rhs, - .. - }) => { - let operation = summary_operation_evidence(node, evidence)?.resource(); - *cpu_ops += evaluation_count as f64 - * validated_operator_executions("exact_binary", operation)? as f64 - * validated_operator_cpu("exact_binary", operation.cpu_ops)?; - add_operator_io(io_bytes, operation, evaluation_count)?; - visit_ops( - lhs, - seen, - by_node, - evidence, - scope, - evaluation_count, - cpu_ops, - io_bytes, - )?; - visit_ops( - rhs, - seen, - by_node, - evidence, - scope, - evaluation_count, - cpu_ops, - io_bytes, - )?; - } - - Operator::NonASAP(_) - | Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { .. } - | ASAPOp::MaintainPopulation { .. } - | ASAPOp::EvaluatePopulation { .. }, - ) => { - let operation = summary_operation_evidence(node, evidence)?.resource(); - *cpu_ops += evaluation_count as f64 - * validated_operator_executions("value_operation", operation)? as f64 - * validated_operator_cpu("value_operation", operation.cpu_ops)?; - add_operator_io(io_bytes, operation, evaluation_count)?; - for child in node.children() { - visit_ops( - child, - seen, - by_node, - evidence, - scope, - evaluation_count, - cpu_ops, - io_bytes, - )?; - } - } - Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) => { - visit_ops( - child, - seen, - by_node, - evidence, - scope, - evaluation_count, - cpu_ops, - io_bytes, - )?; - } - Operator::ASAP(ASAPOp::SummaryMerge { children }) => { - let operation = summary_operation_evidence(node, evidence)?.resource(); - let merge = validated_operator_cpu("summary_merge", operation.cpu_ops)?; - *cpu_ops += evaluation_count as f64 - * validated_operator_executions("summary_merge", operation)? as f64 - * merge; - add_operator_io(io_bytes, operation, evaluation_count)?; - for child in children { - visit_ops( - child, - seen, - by_node, - evidence, - scope, - evaluation_count, - cpu_ops, - io_bytes, - )?; - } - } - Operator::ASAP(ASAPOp::SummarySubtract { left, right }) => { - let operation = summary_operation_evidence(node, evidence)?.resource(); - *cpu_ops += evaluation_count as f64 - * validated_operator_executions("summary_subtract", operation)? as f64 - * validated_operator_cpu("summary_subtract", operation.cpu_ops)?; - add_operator_io(io_bytes, operation, evaluation_count)?; - visit_ops( - left, - seen, - by_node, - evidence, - scope, - evaluation_count, - cpu_ops, - io_bytes, - )?; - visit_ops( - right, - seen, - by_node, - evidence, - scope, - evaluation_count, - cpu_ops, - io_bytes, - )?; - } - Operator::ASAP(ASAPOp::SummaryDelete { summary_input, .. }) => { - let delete = summary_operation_evidence(node, evidence)?; - let SummaryOperatorEvidence::Delete { - resource: operation, - events_per_second, - routing_fanout, - } = delete - else { - unreachable!("operation kind was validated") - }; - let state_ptr = evidence - .operation_state_owners - .get(&(node as *const _)) - .ok_or(AnalyticalCostError::MissingOrStale("summary_delete_owner"))?; - fn collect_aggs( - node: &OperatorNode, - seen: &mut HashSet<*const OperatorNode>, - out: &mut Vec<*const OperatorNode>, - ) { - if !seen.insert(node as *const _) { - return; - } - if matches!(node.operator, Operator::ASAP(ASAPOp::SummaryAgg { .. })) { - out.push(node as *const _); - } - for child in node.children() { - collect_aggs(child, seen, out); - } - } - let mut reachable = Vec::new(); - collect_aggs(summary_input, &mut HashSet::new(), &mut reachable); - if reachable.as_slice() != [*state_ptr] { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "summary delete owner", - )); - } - let deployment = by_node - .get(state_ptr) - .copied() - .ok_or(AnalyticalCostError::MissingOrStale("summary_delete_owner"))?; - let state = evidence - .aggregation(deployment.summary) - .ok_or(AnalyticalCostError::MissingOrStale("summary_delete_owner"))?; - let (_, _, active_ms) = - lifecycle_row_counts(state.inputs, deployment.guarantee, scope)?; - if !events_per_second.is_finite() || *events_per_second < 0.0 { - return Err(AnalyticalCostError::InvalidIngestionRate( - *events_per_second, - )); - } - if *routing_fanout == 0 { - return Err(AnalyticalCostError::MissingOrZero("delete_routing_fanout")); - } - let delete_events = (events_per_second * active_ms as f64 / 1_000.0).ceil() - * *routing_fanout as f64; - if !delete_events.is_finite() || delete_events > u64::MAX as f64 { - return Err(AnalyticalCostError::Overflow); - } - let delete_events = delete_events as u64; - *cpu_ops += delete_events as f64 - * validated_operator_executions("summary_delete", operation)? as f64 - * validated_operator_cpu("summary_delete", operation.cpu_ops)?; - add_operator_io(io_bytes, operation, delete_events)?; - visit_ops( - summary_input, - seen, - by_node, - evidence, - scope, - evaluation_count, - cpu_ops, - io_bytes, - )?; - } - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - let operation = summary_operation_evidence(node, evidence)?.resource(); - *cpu_ops += evaluation_count as f64 - * validated_operator_executions("summary_evaluation", operation)? as f64 - * validated_operator_cpu("summary_evaluation", operation.cpu_ops)?; - add_operator_io(io_bytes, operation, evaluation_count)?; - visit_ops( - summary_input, - seen, - by_node, - evidence, - scope, - evaluation_count, - cpu_ops, - io_bytes, - )?; - } - Operator::ASAP(ASAPOp::SummaryJoin { outer, inner, .. }) => { - let join = evidence - .joins - .get(&(node as *const _)) - .ok_or(AnalyticalCostError::MissingOrStale("summary_join"))?; - if !join.cpu_ops_per_execution.is_finite() - || join.cpu_ops_per_execution <= 0.0 - || join.working_memory_bytes == 0 - || join.executions_per_evaluation == 0 - { - return Err(AnalyticalCostError::MissingOrStale("summary_join")); - } - *cpu_ops += evaluation_count as f64 - * join.executions_per_evaluation as f64 - * join.cpu_ops_per_execution; - let join_io = join - .io_bytes_per_execution - .ok_or(AnalyticalCostError::MissingOrStale("summary_join_io"))?; - *io_bytes = io_bytes - .checked_add( - join_io - .checked_mul(join.executions_per_evaluation) - .and_then(|bytes| bytes.checked_mul(evaluation_count)) - .ok_or(AnalyticalCostError::Overflow)?, - ) - .ok_or(AnalyticalCostError::Overflow)?; - visit_ops( - outer, - seen, - by_node, - evidence, - scope, - evaluation_count, - cpu_ops, - io_bytes, - )?; - visit_ops( - inner, - seen, - by_node, - evidence, - scope, - evaluation_count, - cpu_ops, - io_bytes, - )?; - } - Operator::ASAP(ASAPOp::Extension { .. }) => { - return Err(AnalyticalCostError::UnsupportedCandidate); - } - } - Ok(()) - } - - let mut operator_io_bytes = 0; - visit_ops( - root, - &mut HashSet::new(), - &by_node, - evidence, - scope, - evaluation_count, - &mut cpu_ops, - &mut operator_io_bytes, - )?; - let transient_bytes = estimate_transient_liveness(root, evidence)?; - if !cpu_ops.is_finite() { - return Err(AnalyticalCostError::Overflow); - } - Ok(ResourceEstimate::new( - cpu_ops, - persistent_bytes - .checked_add(transient_bytes) - .and_then(|bytes| bytes.checked_add(ephemeral_state_bytes)) - .ok_or(AnalyticalCostError::Overflow)?, - scans - .values() - .try_fold(operator_io_bytes, |sum, (_, bytes)| { - sum.checked_add(*bytes).ok_or(AnalyticalCostError::Overflow) - })?, - )) -} - -fn add_operator_io( - total: &mut u64, - operation: &SummaryOperatorResourceEvidence, - execution_units: u64, -) -> Result<(), AnalyticalCostError> { - let bytes = operation - .io_bytes_per_execution - .ok_or(AnalyticalCostError::MissingOrStale("summary operator io"))?; - if operation.executions_per_evaluation == 0 { - return Err(AnalyticalCostError::MissingOrStale( - "summary operator executions", - )); - } - *total = total - .checked_add( - bytes - .checked_mul(operation.executions_per_evaluation) - .and_then(|value| value.checked_mul(execution_units)) - .ok_or(AnalyticalCostError::Overflow)?, - ) - .ok_or(AnalyticalCostError::Overflow)?; - Ok(()) -} - -fn validate_summary_edges_and_physical_ids( - root: &OperatorNode, - evidence: &SummaryNodeEvidence, - frameworks_by_node: &HashMap<*const OperatorNode, &Option>, -) -> Result<(), AnalyticalCostError> { - fn children<'a>( - node: &'a OperatorNode, - evidence: &SummaryNodeEvidence, - ) -> Vec<&'a OperatorNode> { - summary_children(node, evidence) - } - fn metadata( - node: &OperatorNode, - evidence: &SummaryNodeEvidence, - ) -> Result<(String, Vec, EdgeStatistics), AnalyticalCostError> { - if let Some(retained) = evidence.retained_queries.get(&(node as *const _)) { - if node.contains_asap() { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "retained sub-DAG evidence covers summary operators", - )); - } - return Ok((retained.physical_id.clone(), vec![], retained.output)); - } - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { .. }) => { - let value = evidence - .aggregations - .get(&(node as *const _)) - .ok_or(AnalyticalCostError::MissingOrStale("summary_agg"))?; - Ok((value.physical_id.clone(), vec![value.input], value.output)) - } - Operator::ASAP(ASAPOp::SummaryJoin { .. }) => { - let value = evidence - .joins - .get(&(node as *const _)) - .ok_or(AnalyticalCostError::MissingOrStale("summary_join"))?; - Ok(( - value.physical_id.clone(), - value.inputs.clone(), - value.output, - )) - } - _ => { - let value = summary_operation_evidence(node, evidence)?.resource(); - Ok(( - value.physical_id.clone(), - value.inputs.clone(), - value.output, - )) - } - } - } - fn visit( - node: &OperatorNode, - evidence: &SummaryNodeEvidence, - frameworks_by_node: &HashMap<*const OperatorNode, &Option>, - seen: &mut HashSet<*const OperatorNode>, - physical: &mut HashMap, EdgeStatistics, String)>, - ) -> Result { - if !seen.insert(node as *const _) { - return metadata(node, evidence).map(|(_, _, output)| output); - } - let child_nodes = children(node, evidence); - let child_outputs = child_nodes - .iter() - .map(|child| visit(child, evidence, frameworks_by_node, seen, physical)) - .collect::, _>>()?; - let child_physical_ids = child_nodes - .iter() - .map(|child| summary_physical_id(child, evidence)) - .collect::, _>>()?; - let (id, inputs, output) = metadata(node, evidence)?; - let local_fingerprint = if has_retained_subdag_evidence(node, evidence) { - format!("{:?}", evidence.retained_queries.get(&(node as *const _))) - } else { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { .. }) => { - format!("{:?}", evidence.aggregations.get(&(node as *const _))) - } - Operator::ASAP(ASAPOp::SummaryJoin { .. }) => { - format!("{:?}", evidence.joins.get(&(node as *const _))) - } - _ => format!("{:?}", evidence.operations.get(&(node as *const _))), - } - }; - // A provider identity names the complete physical operator, including - // its inputs. Equal local widths/costs do not make operators consuming - // different physical children the same deployment. - let framework = frameworks_by_node.get(&(node as *const _)); - let fingerprint = format!( - "logical={:?}|framework={framework:?}|{local_fingerprint}|children={child_physical_ids:?}", - node.operator - ); - if id.is_empty() - || inputs != child_outputs - || !output.is_consistent() - || inputs.iter().any(|edge| !edge.is_consistent()) - { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "summary physical edge statistics", - )); - } - match physical.entry(id) { - std::collections::hash_map::Entry::Vacant(entry) => { - entry.insert((inputs, output, fingerprint)); - } - std::collections::hash_map::Entry::Occupied(entry) - if entry.get() != &(inputs, output, fingerprint) => - { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "summary physical identity", - )); - } - _ => {} - } - Ok(output) - } - visit( - root, - evidence, - frameworks_by_node, - &mut HashSet::new(), - &mut HashMap::new(), - ) - .map(|_| ()) -} - -/// Whether `node` is a retained non-ASAP sub-DAG costed as one unit: the -/// provider bound retained-query evidence to it instead of per-operator -/// evidence. Its children are then not visited. No ASAP descendant may be -/// hidden by this boundary; dag validation rejects such evidence. -fn has_retained_subdag_evidence(node: &OperatorNode, evidence: &SummaryNodeEvidence) -> bool { - evidence.retained_queries.contains_key(&(node as *const _)) && !node.contains_asap() -} - -/// The inputs the estimator visits below `node`: none for a retained -/// sub-DAG, every direct input otherwise. -fn summary_children<'a>( - node: &'a OperatorNode, - evidence: &SummaryNodeEvidence, -) -> Vec<&'a OperatorNode> { - if has_retained_subdag_evidence(node, evidence) { - vec![] - } else { - node.children() - .into_iter() - .map(|child| child.as_ref()) - .collect() - } -} - -fn summary_physical_id( - node: &OperatorNode, - evidence: &SummaryNodeEvidence, -) -> Result { - if has_retained_subdag_evidence(node, evidence) { - return evidence - .retained_queries - .get(&(node as *const _)) - .map(|value| value.physical_id.clone()) - .ok_or(AnalyticalCostError::MissingOrStale( - "summary physical identity", - )); - } - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { .. }) => evidence - .aggregations - .get(&(node as *const _)) - .map(|value| value.physical_id.clone()), - Operator::ASAP(ASAPOp::SummaryJoin { .. }) => evidence - .joins - .get(&(node as *const _)) - .map(|value| value.physical_id.clone()), - _ => summary_operation_evidence(node, evidence) - .ok() - .map(|value| value.resource().physical_id.clone()), - } - .ok_or(AnalyticalCostError::MissingOrStale( - "summary physical identity", - )) -} - -/// Simulate a deterministic child-before-parent physical schedule. Completed -/// child output buffers remain live until their final consumer executes; -/// operator workspace and its output buffer coexist during that execution. -pub(super) fn estimate_transient_liveness( - root: &OperatorNode, - evidence: &SummaryNodeEvidence, -) -> Result { - fn children<'a>( - node: &'a OperatorNode, - evidence: &SummaryNodeEvidence, - ) -> Vec<&'a OperatorNode> { - summary_children(node, evidence) - } - fn visit<'a>( - node: &'a OperatorNode, - evidence: &SummaryNodeEvidence, - seen: &mut HashSet, - uses: &mut HashMap, - order: &mut Vec<&'a OperatorNode>, - ) -> Result<(), AnalyticalCostError> { - if !seen.insert(summary_physical_id(node, evidence)?) { - return Ok(()); - } - for child in children(node, evidence) { - *uses - .entry(summary_physical_id(child, evidence)?) - .or_default() += 1; - visit(child, evidence, seen, uses, order)?; - } - order.push(node); - Ok(()) - } - fn memory( - node: &OperatorNode, - evidence: &SummaryNodeEvidence, - ) -> Result<(u64, u64), AnalyticalCostError> { - if has_retained_subdag_evidence(node, evidence) { - return evidence - .retained_queries - .get(&(node as *const _)) - .map(|value| (value.working_memory_bytes, value.output_buffer_bytes)) - .ok_or(AnalyticalCostError::MissingOrStale("retain_exact")); - } - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { .. }) => Ok((0, 0)), - Operator::ASAP(ASAPOp::SummaryJoin { .. }) => evidence - .joins - .get(&(node as *const _)) - .map(|value| (value.working_memory_bytes, value.output_buffer_bytes)) - .ok_or(AnalyticalCostError::MissingOrStale("summary_join")), - _ => { - let value = summary_operation_evidence(node, evidence)?.resource(); - Ok((value.working_memory_bytes, value.output_buffer_bytes)) - } - } - } - - let mut uses = HashMap::new(); - let mut order = Vec::new(); - visit(root, evidence, &mut HashSet::new(), &mut uses, &mut order)?; - let outputs: HashMap<_, _> = order - .iter() - .map(|node| { - memory(node, evidence) - .and_then(|(_, output)| summary_physical_id(node, evidence).map(|id| (id, output))) - }) - .collect::>()?; - let mut live = 0_u64; - let mut peak = 0_u64; - for node in order { - let (workspace, output) = memory(node, evidence)?; - peak = peak.max( - live.checked_add(workspace) - .and_then(|bytes| bytes.checked_add(output)) - .ok_or(AnalyticalCostError::Overflow)?, - ); - live = live - .checked_add(output) - .ok_or(AnalyticalCostError::Overflow)?; - for child in children(node, evidence) { - let child_id = summary_physical_id(child, evidence)?; - let remaining = - uses.get_mut(&child_id) - .ok_or(AnalyticalCostError::InvalidPhysicalDAG( - "missing summary consumer count", - ))?; - *remaining -= 1; - if *remaining == 0 { - live = live - .checked_sub(outputs[&child_id]) - .ok_or(AnalyticalCostError::Overflow)?; - } - } - } - Ok(peak) -} -#[cfg(test)] -pub(super) fn evidence_nodes(root: &OperatorNode) -> (Vec<&OperatorNode>, Vec<&OperatorNode>) { - fn visit<'a>( - node: &'a OperatorNode, - seen: &mut HashSet<*const OperatorNode>, - aggregations: &mut Vec<&'a OperatorNode>, - joins: &mut Vec<&'a OperatorNode>, - ) { - if !seen.insert(node as *const _) { - return; - } - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { .. }) => aggregations.push(node), - Operator::ASAP(ASAPOp::SummaryJoin { .. }) => joins.push(node), - _ => {} - } - for child in node.children() { - visit(child, seen, aggregations, joins); - } - } - let mut aggregations = Vec::new(); - let mut joins = Vec::new(); - visit(root, &mut HashSet::new(), &mut aggregations, &mut joins); - (aggregations, joins) -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -#[cfg(test)] -struct SummaryOperationCounts { - state_builds: u64, - merges_per_read: u64, - subtracts_per_read: u64, - deletes_per_update: u64, - evaluations_per_read: u64, - joins_per_read: u64, -} - -/// Low-level diagnostic for a homogeneous deployment. Final planner ranking -/// uses the per-node whole-DAG estimator above. Shared `Rc` nodes are visited -/// once; explicit delete frequency comes from deletion evidence. -#[cfg(test)] -pub(super) fn estimate_incremental_summary_maintenance( - root: &OperatorNode, - guarantee: &SummaryMaintenanceLifecycleGuarantee, - inputs: SummaryMaintenanceInputs, - cpu: SummaryOperationCpuEvidence, - scope: &ComparisonScope, -) -> Result { - estimate_incremental_summary_maintenance_with_join(root, guarantee, inputs, cpu, None, scope) -} -#[cfg(test)] -pub(super) fn estimate_incremental_summary_maintenance_with_join( - root: &OperatorNode, - guarantee: &SummaryMaintenanceLifecycleGuarantee, - inputs: SummaryMaintenanceInputs, - cpu: SummaryOperationCpuEvidence, - join: Option, - scope: &ComparisonScope, -) -> Result { - let inputs = inputs.validate()?; - let evaluation_count = scope.validate()?; - validate_guarantee(guarantee, scope.data_arrival)?; - let (bootstrap_input_rows, arriving_input_rows, active_ms) = - lifecycle_row_counts(inputs, guarantee, scope)?; - let counts = count_operations(root)?; - if counts.state_builds == 0 { - return Err(AnalyticalCostError::UnsupportedCandidate); - } - - let insert = required_cpu("insert_cpu_ops", cpu.insert_cpu_ops)?; - let merge = required_cpu_when(counts.merges_per_read, "merge_cpu_ops", cpu.merge_cpu_ops)?; - let subtract = required_cpu_when( - counts.subtracts_per_read, - "subtract_cpu_ops", - cpu.subtract_cpu_ops, - )?; - let delete = required_cpu_when( - counts.deletes_per_update, - "delete_cpu_ops", - cpu.delete_cpu_ops, - )?; - let delete_events = if counts.deletes_per_update == 0 { - 0_u64 - } else { - let rate = cpu - .delete_events_per_second - .filter(|rate| rate.is_finite() && *rate >= 0.0) - .ok_or(AnalyticalCostError::MissingOrStale( - "delete_events_per_second", - ))?; - let fanout = cpu - .delete_routing_fanout - .filter(|fanout| *fanout > 0) - .ok_or(AnalyticalCostError::MissingOrStale("delete_routing_fanout"))?; - let events = (rate * active_ms as f64 / 1_000.0).ceil(); - if !events.is_finite() || events > u64::MAX as f64 { - return Err(AnalyticalCostError::Overflow); - } - (events as u64) - .checked_mul(fanout) - .ok_or(AnalyticalCostError::Overflow)? - }; - let evaluation = required_cpu_when( - counts.evaluations_per_read, - "evaluation_cpu_ops", - cpu.evaluation_cpu_ops, - )?; - let join_cpu = match (counts.joins_per_read, join.as_ref()) { - (0, _) => 0.0, - (_, Some(evidence)) - if evidence.cpu_ops_per_execution.is_finite() - && evidence.cpu_ops_per_execution > 0.0 - && evidence.working_memory_bytes > 0 => - { - evidence.cpu_ops_per_execution - } - (_, Some(evidence)) - if !evidence.cpu_ops_per_execution.is_finite() - || evidence.cpu_ops_per_execution <= 0.0 => - { - return Err(AnalyticalCostError::InvalidOperationCost( - "summary_join_cpu_ops_per_execution", - evidence.cpu_ops_per_execution, - )); - } - _ => return Err(AnalyticalCostError::MissingOrStale("summary_join")), - }; - - let build_inserts = bootstrap_input_rows - .checked_mul(inputs.bootstrap_window_count) - .ok_or(AnalyticalCostError::Overflow)? - .checked_mul(counts.state_builds) - .ok_or(AnalyticalCostError::Overflow)?; - let update_inserts = arriving_input_rows - .checked_mul(inputs.active_window_count) - .and_then(|n| n.checked_mul(counts.state_builds)) - .ok_or(AnalyticalCostError::Overflow)?; - let instances = inputs.physical_summary_count as f64; - let evaluations = evaluation_count as f64; - let cpu_ops = (build_inserts as f64 + update_inserts as f64) * insert - + evaluations * counts.merges_per_read as f64 * instances * merge - + evaluations * counts.subtracts_per_read as f64 * instances * subtract - + delete_events as f64 * counts.deletes_per_update as f64 * delete - + evaluations * counts.evaluations_per_read as f64 * instances * evaluation - + evaluations * counts.joins_per_read as f64 * join_cpu; - if !cpu_ops.is_finite() { - return Err(AnalyticalCostError::Overflow); - } - - let state_instances = inputs - .active_window_count - .checked_add(inputs.retained_window_count) - .and_then(|n| n.checked_mul(inputs.physical_summary_count)) - .and_then(|n| n.checked_mul(counts.state_builds)) - .ok_or(AnalyticalCostError::Overflow)?; - let retained_bytes = state_instances - .checked_mul(inputs.state_bytes_per_summary) - .ok_or(AnalyticalCostError::Overflow)?; - // Merge/subtract may stream over persistent inputs but still needs one - // result state per physical instance. Persistent retained windows are - // already included above and are not loaded a second time. - let transient_bytes = if counts.merges_per_read > 0 || counts.subtracts_per_read > 0 { - inputs - .physical_summary_count - .checked_mul(inputs.state_bytes_per_summary) - .ok_or(AnalyticalCostError::Overflow)? - } else { - 0 - }; - let join_bytes = match (counts.joins_per_read, join.as_ref()) { - (0, _) => 0, - (_, Some(evidence)) => evidence.working_memory_bytes, - _ => return Err(AnalyticalCostError::MissingOrStale("summary_join")), - }; - let bootstrap_row_buffer = if inputs.initial_input_rows == 0 { - 0 - } else { - inputs - .initial_input_bytes - .div_ceil(inputs.initial_input_rows) - }; - Ok(ResourceEstimate::new( - cpu_ops, - retained_bytes - .checked_add(transient_bytes) - .and_then(|bytes| bytes.checked_add(join_bytes)) - .ok_or(AnalyticalCostError::Overflow)? - .max(bootstrap_row_buffer), - inputs.initial_source_scan_bytes, - )) -} - -pub(super) fn lifecycle_row_counts( - inputs: SummaryMaintenanceInputs, - guarantee: &SummaryMaintenanceLifecycleGuarantee, - scope: &ComparisonScope, -) -> Result<(u64, u64, u64), AnalyticalCostError> { - validate_arrival_rate(scope.data_arrival, inputs.ingestion_rate_per_second)?; - let horizon_end = scope - .planning_time - .0 - .checked_add(scope.horizon.0) - .ok_or(AnalyticalCostError::Overflow)?; - let (bootstrap_extra_ms, active_ms) = match guarantee.summary_maintenance_lifecycle { - SummaryMaintenanceLifecycle::Prepared { - activate_at, - retire_at, - } => { - if activate_at.0 >= retire_at.0 { - return Err(AnalyticalCostError::IncompatibleLifecycleGuarantee); - } - let covers_every_evaluation = evaluation_offsets_ms(scope)?.into_iter().all(|offset| { - scope - .planning_time - .0 - .checked_add(offset) - .is_some_and(|at| at >= activate_at.0 && at < retire_at.0) - }); - if !covers_every_evaluation { - return Err(AnalyticalCostError::IncompatibleLifecycleGuarantee); - } - let activation = activate_at.0.max(scope.planning_time.0).min(horizon_end); - let bootstrap_extra_ms = activation.saturating_sub(scope.planning_time.0); - let start = activation; - let end = retire_at.0.min(horizon_end); - (bootstrap_extra_ms, end.saturating_sub(start)) - } - SummaryMaintenanceLifecycle::Shared { .. } => (0, scope.horizon.0), - SummaryMaintenanceLifecycle::ContinuouslyMaintained => (0, scope.horizon.0), - SummaryMaintenanceLifecycle::Ephemeral => { - return Err(AnalyticalCostError::IncompatibleLifecycleGuarantee) - } - }; - let bootstrap_extra = inputs.ingestion_rate_per_second * bootstrap_extra_ms as f64 / 1000.0; - let updates = inputs.ingestion_rate_per_second * active_ms as f64 / 1000.0; - if !bootstrap_extra.is_finite() - || !updates.is_finite() - || bootstrap_extra > u64::MAX as f64 - || updates > u64::MAX as f64 - { - return Err(AnalyticalCostError::Overflow); - } - Ok(( - inputs - .initial_input_rows - .checked_add(bootstrap_extra.ceil() as u64) - .ok_or(AnalyticalCostError::Overflow)?, - updates.ceil() as u64, - active_ms, - )) -} - -fn validate_guarantee( - guarantee: &SummaryMaintenanceLifecycleGuarantee, - arrival: DataArrival, -) -> Result<(), AnalyticalCostError> { - if guarantee.output_representation != asap_types::post_asap::OutputRepresentation::SummaryState - || guarantee.summary_maintenance_mode - != maintenance_mode(&guarantee.summary_maintenance_lifecycle, arrival) - || guarantee.evaluation_schedule - != evaluation_schedule(&guarantee.summary_maintenance_lifecycle, arrival) - { - return Err(AnalyticalCostError::IncompatibleLifecycleGuarantee); - } - Ok(()) -} - -#[cfg(test)] -fn required_cpu(name: &'static str, value: Option) -> Result { - let value = value.ok_or(AnalyticalCostError::MissingOrStale(name))?; - if !value.is_finite() || value <= 0.0 { - return Err(AnalyticalCostError::InvalidOperationCost(name, value)); - } - Ok(value) -} - -pub(super) fn validated_operator_cpu( - name: &'static str, - value: f64, -) -> Result { - if !value.is_finite() || value <= 0.0 { - Err(AnalyticalCostError::InvalidOperationCost(name, value)) - } else { - Ok(value) - } -} - -fn validated_operator_executions( - name: &'static str, - evidence: &SummaryOperatorResourceEvidence, -) -> Result { - if evidence.executions_per_evaluation == 0 { - return Err(AnalyticalCostError::MissingOrZero(name)); - } - Ok(evidence.executions_per_evaluation) -} - -#[cfg(test)] -fn required_cpu_when( - count: u64, - name: &'static str, - value: Option, -) -> Result { - if count == 0 { - return Ok(0.0); - } - required_cpu(name, value) -} - -#[cfg(test)] -fn count_operations(root: &OperatorNode) -> Result { - fn visit( - node: &OperatorNode, - seen: &mut HashSet<*const OperatorNode>, - counts: &mut SummaryOperationCounts, - ) -> Result<(), AnalyticalCostError> { - if !seen.insert(node as *const OperatorNode) { - return Ok(()); - } - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { .. }) => { - counts.state_builds = counts - .state_builds - .checked_add(1) - .ok_or(AnalyticalCostError::Overflow)?; - } - Operator::ASAP(ASAPOp::SummaryMerge { children }) => { - if children.is_empty() { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "summary merge has no children", - )); - } - counts.merges_per_read = counts - .merges_per_read - .checked_add(children.len().saturating_sub(1) as u64) - .ok_or(AnalyticalCostError::Overflow)?; - } - Operator::ASAP(ASAPOp::SummarySubtract { .. }) => { - counts.subtracts_per_read = counts - .subtracts_per_read - .checked_add(1) - .ok_or(AnalyticalCostError::Overflow)?; - } - Operator::ASAP(ASAPOp::SummaryDelete { .. }) => { - counts.deletes_per_update = counts - .deletes_per_update - .checked_add(1) - .ok_or(AnalyticalCostError::Overflow)?; - } - Operator::ASAP( - ASAPOp::SummaryEstimate { .. } | ASAPOp::FinalizeExactAccumulator { .. }, - ) => { - counts.evaluations_per_read = counts - .evaluations_per_read - .checked_add(1) - .ok_or(AnalyticalCostError::Overflow)?; - } - Operator::ASAP(ASAPOp::SummaryJoin { .. }) => { - counts.joins_per_read = counts - .joins_per_read - .checked_add(1) - .ok_or(AnalyticalCostError::Overflow)?; - } - // Retained relational work, accumulator/population boundaries and - // exact query-time operators add no summary operation. - Operator::NonASAP(_) - | Operator::ASAP( - ASAPOp::MaintainPopulation { .. } - | ASAPOp::EvaluatePopulation { .. } - | ASAPOp::Extension { .. }, - ) => {} - } - for child in node.children() { - visit(child, seen, counts)?; - } - Ok(()) - } - - let mut counts = SummaryOperationCounts::default(); - visit(root, &mut HashSet::new(), &mut counts)?; - Ok(counts) -} diff --git a/crates/asap-aware-mapping/src/summary_maintenance_cost/evidence.rs b/crates/asap-aware-mapping/src/summary_maintenance_cost/evidence.rs deleted file mode 100644 index 6ae5999bd..000000000 --- a/crates/asap-aware-mapping/src/summary_maintenance_cost/evidence.rs +++ /dev/null @@ -1,386 +0,0 @@ -use super::*; - -/// Physical evidence that is not represented by [`DataWorkload`] for one -/// summary deployment. Window counts describe the -/// already-selected physical deployment; this layer does not define another -/// tumbling/sliding policy enum. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -pub struct SummaryPhysicalInputEvidence { - /// Logical bytes in the snapshot used to bootstrap the state. - pub initial_input_bytes: u64, - /// Source bytes read while bootstrapping. Arriving stream bytes are not a - /// disk scan and are therefore excluded. - pub initial_source_scan_bytes: u64, - /// Simultaneously open windows receiving each arriving item. - pub active_window_count: u64, - /// Window/state partitions receiving each bootstrap row. - pub bootstrap_window_count: u64, - /// Completed windows retained for query coverage. - pub retained_window_count: u64, - /// Independent state instances per window: one for shared - /// multi-subpopulation state, otherwise the resolved group count. - pub physical_summary_count: u64, - /// Resident bytes of one concrete state instance. - pub state_bytes_per_summary: u64, -} - -/// Workload-normalized inputs for summary construction and maintenance over one finite -/// comparison horizon. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -pub struct SummaryMaintenanceInputs { - pub initial_input_rows: u64, - pub initial_input_bytes: u64, - pub initial_source_scan_bytes: u64, - pub ingestion_rate_per_second: f64, - pub active_window_count: u64, - pub bootstrap_window_count: u64, - pub retained_window_count: u64, - pub physical_summary_count: u64, - pub state_bytes_per_summary: u64, -} - -impl SummaryMaintenanceInputs { - /// Resolve snapshot size, arriving rows, and reads from the canonical - /// workload. Positive fractional expected work rounds up conservatively. - /// - /// `AtRest` needs snapshot cardinality and implies zero arrivals; continuous - /// ingestion additionally requires fresh rate evidence. - /// `Mixed` fails closed because today's workload schema cannot distinguish - /// its at-rest backlog from its continuing-arrival cardinality. - pub fn from_workload( - physical: SummaryPhysicalInputEvidence, - data: &DataWorkload, - scope: &ComparisonScope, - ) -> Result { - let _ = scope.validate()?; - if scope.data_arrival != data.arrival { - return Err(AnalyticalCostError::ComparisonScopeMismatch("data arrival")); - } - let initial_input_rows = data - .input_cardinality - .value_at(scope.planning_time.0) - .copied() - .ok_or(AnalyticalCostError::MissingOrStale("input_cardinality"))?; - let ingestion_rate = match data.arrival { - DataArrival::AtRest => { - // A declared snapshot has no arrivals. Reject contradictory fresh - // evidence rather than silently pricing the wrong workload. - if let Some(rate) = data.ingestion_rate.value_at(scope.planning_time.0) { - validate_arrival_rate(data.arrival, rate.0)?; - } - 0.0 - } - DataArrival::ContinuouslyIngesting => { - data.ingestion_rate - .value_at(scope.planning_time.0) - .copied() - .ok_or(AnalyticalCostError::MissingOrStale("ingestion_rate"))? - .0 - } - arrival => return Err(AnalyticalCostError::UnsupportedDataArrival(arrival)), - }; - validate_arrival_rate(data.arrival, ingestion_rate)?; - Self { - initial_input_rows, - initial_input_bytes: physical.initial_input_bytes, - initial_source_scan_bytes: physical.initial_source_scan_bytes, - ingestion_rate_per_second: ingestion_rate, - active_window_count: physical.active_window_count, - bootstrap_window_count: physical.bootstrap_window_count, - retained_window_count: physical.retained_window_count, - physical_summary_count: physical.physical_summary_count, - state_bytes_per_summary: physical.state_bytes_per_summary, - } - .validate() - } - - pub fn validate(self) -> Result { - for (name, value) in [ - ("active_window_count", self.active_window_count), - ("bootstrap_window_count", self.bootstrap_window_count), - ("physical_summary_count", self.physical_summary_count), - ("state_bytes_per_summary", self.state_bytes_per_summary), - ] { - if value == 0 { - return Err(AnalyticalCostError::MissingOrZero(name)); - } - } - if (self.initial_input_rows == 0) != (self.initial_input_bytes == 0) - || (self.initial_input_rows == 0 && self.initial_source_scan_bytes != 0) - { - return Err(AnalyticalCostError::InconsistentBootstrapEvidence); - } - if !self.ingestion_rate_per_second.is_finite() || self.ingestion_rate_per_second < 0.0 { - return Err(AnalyticalCostError::InvalidIngestionRate( - self.ingestion_rate_per_second, - )); - } - Ok(self) - } -} - -/// CPU operations for one concrete state operation on one state instance. -/// Missing evidence is legal only when the selected summary DAG does not use -/// that operation. -#[cfg(test)] -#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] -pub struct SummaryOperationCpuEvidence { - pub insert_cpu_ops: Option, - pub merge_cpu_ops: Option, - pub subtract_cpu_ops: Option, - pub delete_cpu_ops: Option, - /// Expirations/retractions routed to this DAG per second. Required only - /// when an explicit `SummaryDelete` is present. - pub delete_events_per_second: Option, - /// Concrete state instances touched by one delete event. - pub delete_routing_fanout: Option, - pub evaluation_cpu_ops: Option, -} - -/// Physical evidence for one `SummaryJoin` implementation. Total work, -/// cardinality, and memory cannot be inferred from the logical join key alone. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct SummaryJoinEvidence { - pub physical_id: String, - pub inputs: Vec, - pub output: EdgeStatistics, - /// Total build, probe, match-production, and output CPU for one complete - /// execution of the selected physical join algorithm. - pub cpu_ops_per_execution: f64, - pub working_memory_bytes: u64, - pub output_buffer_bytes: u64, - pub executions_per_evaluation: u64, - pub io_bytes_per_execution: Option, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct SummaryAggregateEvidence { - pub physical_id: String, - pub input: EdgeStatistics, - pub output: EdgeStatistics, - /// Index into `ComparisonScope.sources` when this state bootstraps directly - /// from storage. `None` means its input is an already-materialized child - /// edge and therefore has no additional source read. - pub scan_selection_index: Option, - /// Provider-owned identity of the physical bootstrap read. Equal source - /// coverage alone does not prove two independent builds share I/O. - pub bootstrap_read_identity: String, - pub inputs: SummaryMaintenanceInputs, - /// CPU operations to insert one routed row into one state instance. - pub insert_cpu_ops: f64, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct SummaryOperatorResourceEvidence { - pub physical_id: String, - pub inputs: Vec, - pub output: EdgeStatistics, - pub cpu_ops: f64, - pub working_memory_bytes: u64, - pub output_buffer_bytes: u64, - /// Executions of this physical operator for one query evaluation. - /// This is provider evidence, not inferred from a descendant state. - pub executions_per_evaluation: u64, - pub io_bytes_per_execution: Option, -} - -/// Evidence is structured by logical summary operation so delete-only facts -/// cannot be attached to merge, subtract, or evaluation nodes. -#[derive(Debug, Clone, PartialEq)] -pub enum SummaryOperatorEvidence { - /// Exact query-time arithmetic over two independently realized operands. - Binary(SummaryOperatorResourceEvidence), - /// Query-time or maintenance-time plain-value work. For `Sort`/`Limit`, - /// providers report the actual comparison/heap work and working set here; - /// the estimator charges it at query multiplicity. - ValueOperation(SummaryOperatorResourceEvidence), - Merge(SummaryOperatorResourceEvidence), - Subtract(SummaryOperatorResourceEvidence), - Delete { - resource: SummaryOperatorResourceEvidence, - events_per_second: f64, - routing_fanout: u64, - }, - Evaluation(SummaryOperatorResourceEvidence), -} - -impl SummaryOperatorEvidence { - pub(super) fn resource(&self) -> &SummaryOperatorResourceEvidence { - match self { - Self::Binary(resource) - | Self::ValueOperation(resource) - | Self::Merge(resource) - | Self::Subtract(resource) - | Self::Delete { resource, .. } - | Self::Evaluation(resource) => resource, - } - } - - #[cfg(test)] - pub(super) fn resource_mut(&mut self) -> &mut SummaryOperatorResourceEvidence { - match self { - Self::Binary(resource) - | Self::ValueOperation(resource) - | Self::Merge(resource) - | Self::Subtract(resource) - | Self::Delete { resource, .. } - | Self::Evaluation(resource) => resource, - } - } -} - -/// Non-aggregation work for a retained pre-ASAP sub-DAG over the comparison -/// horizon. Bootstrap/source I/O belongs exclusively to the owning aggregate, -/// and summary insertion belongs exclusively to its insert evidence. -#[derive(Debug, Clone, PartialEq)] -pub struct RetainedSubDAGEvidence { - pub physical_id: String, - /// Logical output edge consumed by the parent summary operator. - pub output: EdgeStatistics, - pub preprocessing_cpu_ops_over_horizon: f64, - /// Execution workspace, excluding the separately declared output buffer. - pub working_memory_bytes: u64, - pub output_buffer_bytes: u64, -} - -/// Physical evidence bound to the selected DAG's `Rc` identity. A copied, -/// structurally equal node is not silently treated as the same deployment. -#[derive(Debug, Clone, Default)] -pub struct SummaryNodeEvidence { - pub(super) aggregations: HashMap<*const OperatorNode, SummaryAggregateEvidence>, - pub(super) joins: HashMap<*const OperatorNode, SummaryJoinEvidence>, - pub(super) operations: HashMap<*const OperatorNode, SummaryOperatorEvidence>, - pub(super) operation_state_owners: HashMap<*const OperatorNode, *const OperatorNode>, - pub(super) retained_queries: HashMap<*const OperatorNode, RetainedSubDAGEvidence>, -} - -impl SummaryNodeEvidence { - pub fn insert_aggregation( - &mut self, - node: &Rc, - evidence: SummaryAggregateEvidence, - ) { - self.aggregations.insert(Rc::as_ptr(node), evidence); - } - - pub fn insert_join(&mut self, node: &Rc, evidence: SummaryJoinEvidence) { - self.joins.insert(Rc::as_ptr(node), evidence); - } - - pub fn insert_operation(&mut self, node: &Rc, evidence: SummaryOperatorEvidence) { - self.operations.insert(Rc::as_ptr(node), evidence); - } - - /// Bind a stateful operation (currently `SummaryDelete`) to the exact - /// aggregation deployment whose active interval it follows. - pub fn insert_state_operation( - &mut self, - node: &Rc, - state: &Rc, - evidence: SummaryOperatorEvidence, - ) { - self.operations.insert(Rc::as_ptr(node), evidence); - self.operation_state_owners - .insert(Rc::as_ptr(node), Rc::as_ptr(state)); - } - - pub fn insert_retained_query( - &mut self, - node: &Rc, - evidence: RetainedSubDAGEvidence, - ) { - self.retained_queries.insert(Rc::as_ptr(node), evidence); - } - - pub(super) fn aggregation(&self, node: &OperatorNode) -> Option { - self.aggregations.get(&(node as *const _)).cloned() - } -} - -pub(super) fn summary_operation_evidence<'a>( - node: &OperatorNode, - evidence: &'a SummaryNodeEvidence, -) -> Result<&'a SummaryOperatorEvidence, AnalyticalCostError> { - let operation = evidence - .operations - .get(&(node as *const _)) - .ok_or(AnalyticalCostError::MissingOrStale("summary operation"))?; - // A binary operator or join over two inputs is `Binary` evidence; every - // other non-ASAP operator, and the accumulator/population boundaries, - // is a `ValueOperation`. - let matches = match (&node.operator, operation) { - ( - Operator::NonASAP(NonASAPOp::BinaryOp { .. } | NonASAPOp::Join { .. }), - SummaryOperatorEvidence::Binary(_), - ) => true, - (Operator::NonASAP(NonASAPOp::BinaryOp { .. } | NonASAPOp::Join { .. }), _) => false, - ( - Operator::NonASAP(_) - | Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { .. } - | ASAPOp::MaintainPopulation { .. } - | ASAPOp::EvaluatePopulation { .. }, - ), - SummaryOperatorEvidence::ValueOperation(_), - ) => true, - (Operator::ASAP(ASAPOp::SummaryMerge { .. }), SummaryOperatorEvidence::Merge(_)) - | (Operator::ASAP(ASAPOp::SummarySubtract { .. }), SummaryOperatorEvidence::Subtract(_)) - | (Operator::ASAP(ASAPOp::SummaryDelete { .. }), SummaryOperatorEvidence::Delete { .. }) - | ( - Operator::ASAP(ASAPOp::SummaryEstimate { .. }), - SummaryOperatorEvidence::Evaluation(_), - ) => true, - _ => false, - }; - if matches { - Ok(operation) - } else { - Err(AnalyticalCostError::InconsistentOperatorStatistics( - "summary operation evidence kind does not match the operator", - )) - } -} - -/// Evidence for recomputing the raw target over the full comparison horizon. -/// Planning-time dimensions describe the initial snapshot. Each scheduled -/// evaluation adds arrivals since planning time; `physical_dag` is therefore -/// a once-counted DAG whose edge statistics already aggregate all evaluations. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct RawInputEvidence { - pub planning_time_input_rows: u64, - pub planning_time_input_bytes: u64, - pub planning_time_source_scan_bytes: u64, - /// Decoded logical bytes added to operator edges by one arriving row. - pub arriving_logical_row_bytes: u64, - /// Physical storage bytes read for one arriving row. Kept separate from - /// logical width so compression and encoding are not silently conflated. - pub arriving_source_row_bytes: u64, - pub ingestion_rate_per_second: f64, - pub physical_dag: EvidenceBackedPhysicalDAG, -} - -/// One complete provider-enumerated physical implementation of the selected -/// summary DAG. The identifier is stable provenance; concrete -/// framework selection is performed by ranking these complete alternatives. -#[derive(Debug, Clone)] -pub struct SummaryPhysicalPlanAlternative { - pub physical_plan_id: String, - pub node_evidence: SummaryNodeEvidence, -} - -/// Apply arrival semantics to both workload-derived and directly bound evidence. -pub(super) fn validate_arrival_rate( - arrival: DataArrival, - rate: f64, -) -> Result<(), AnalyticalCostError> { - if !rate.is_finite() || rate < 0.0 { - return Err(AnalyticalCostError::InvalidIngestionRate(rate)); - } - match arrival { - DataArrival::AtRest if rate != 0.0 => Err(AnalyticalCostError::ComparisonScopeMismatch( - "at-rest ingestion rate", - )), - DataArrival::AtRest | DataArrival::ContinuouslyIngesting => Ok(()), - other => Err(AnalyticalCostError::UnsupportedDataArrival(other)), - } -} diff --git a/crates/asap-aware-mapping/src/summary_maintenance_cost/mod.rs b/crates/asap-aware-mapping/src/summary_maintenance_cost/mod.rs deleted file mode 100644 index ff4c69c1d..000000000 --- a/crates/asap-aware-mapping/src/summary_maintenance_cost/mod.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Analytical resource cost for at-rest and at-rest and incrementally maintained summary deployments. -//! -//! The canonical workload and lifecycle types own deployment semantics. This -//! module only adds physical evidence absent from those schemas: state size, -//! window counts, and per-operation CPU measurements or complexity estimates. - -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate}; -use asap_types::post_asap::{ - BoundExpr, ErrorMetric, ExactKind, FieldDataType, GuaranteeSource, ProbabilityExpr, - ResultGuarantee, SketchAlgorithm, SummaryMaintenanceLifecycle, - SummaryMaintenanceLifecycleGuarantee, SummaryWindowFramework, -}; -use asap_types::pre_asap::{agg_intent::AggIntent, CompareOpKind, InfoMatcher, Source}; -use asap_types::types::AccuracyTarget; -use asap_types::workload::{DataArrival, DataWorkload, QueryRecurrence, RepeatedDemand}; -use serde::{Deserialize, Serialize}; - -use crate::accuracy::{AccuracyModel, DefaultAccuracyModel}; -use crate::analytical_cost::ExecutionMultiplicity; -#[cfg(test)] -use crate::analytical_cost::PhysicalNodeEvidence; -use crate::analytical_cost::{ - estimate_physical_dag, AnalyticalCostError, EvidenceBackedPhysicalDAG, PhysicalDAGNode, - PhysicalOperator, ResourceCalibration, ResourceEstimate, -}; -use crate::cost_model::{ - CompleteSummaryCandidateEstimate, Cost, CostModel, CostedSummaryDeployment, DefaultCostModel, -}; -#[cfg(test)] -use crate::physical_operator_statistics::UnaryEdgeStatistics; -use crate::physical_operator_statistics::{ComparisonScope, EdgeStatistics, OperatorStatistics}; -use crate::recurrence::CostRate; -use crate::replacement::{Replacement, ReplacementSubDAG, TargetSubDAG}; -use crate::summary_maintenance_lifecycle::{ - evaluation_schedule, maintenance_mode, SummaryMaintenanceCapabilities, - SummaryMaintenanceLifecycleCostInputs, -}; - -pub const SUMMARY_MAINTENANCE_COST_MODEL_VERSION: &str = "summary-maintenance-resource-v2"; - -mod estimator; -mod evidence; -mod model; -mod window; - -pub use evidence::*; -pub use model::*; -pub use window::*; diff --git a/crates/asap-aware-mapping/src/summary_maintenance_cost/model.rs b/crates/asap-aware-mapping/src/summary_maintenance_cost/model.rs deleted file mode 100644 index 8d9c2886a..000000000 --- a/crates/asap-aware-mapping/src/summary_maintenance_cost/model.rs +++ /dev/null @@ -1,3664 +0,0 @@ -use super::*; -/// Adapter that supplies the existing lifecycle planner with analytical -/// summary costs across at-rest and continuously ingesting workloads. The -/// planner's existing lifecycle enums and legality checks remain authoritative. -#[derive(Debug, Clone)] -pub struct SummaryMaintenanceCostModel { - pub node_evidence: SummaryNodeEvidence, - pub calibration: ResourceCalibration, - pub capabilities: SummaryMaintenanceCapabilities, - target_comparisons: HashMap<*const OperatorNode, SummaryTargetComparison>, - candidate_comparisons: HashMap, - physical_plan_alternatives: - HashMap>, - window_framework_candidates: - HashMap>, -} - -type CandidateComparisonKey = (*const OperatorNode, *const OperatorNode); - -#[derive(Debug, Clone)] -struct BoundCandidateIdentity { - _target: Rc, - _root: Rc, -} - -#[derive(Debug, Clone)] -struct SummaryTargetComparison { - _target: Rc, - scope: ComparisonScope, - raw: RawInputEvidence, -} - -pub(super) type LogicalSourceSelection = (Source, Vec, Vec); - -pub(super) fn deduplicate_source_selections( - values: Vec, -) -> Vec { - values.into_iter().fold(Vec::new(), |mut unique, value| { - if !unique.contains(&value) { - unique.push(value); - } - unique - }) -} - -fn info_source(selector: &[InfoMatcher]) -> Result { - let mut metric: Option<&str> = None; - for matcher in selector - .iter() - .filter(|matcher| matcher.label == "__name__") - { - if matcher.op != CompareOpKind::Eq || metric.is_some_and(|value| value != matcher.value) { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - } - metric = Some(&matcher.value); - } - Ok(Source::TimeSeries { - metric: metric.unwrap_or("target_info").into(), - }) -} - -/// Collect the source selections (scan sources with their predicates, and -/// info-metric selectors) of every leaf reachable from `node`, visiting a -/// shared node once. -pub(super) fn query_source_selections( - node: &OperatorNode, - seen: &mut HashSet<*const OperatorNode>, - out: &mut Vec, -) -> Result<(), AnalyticalCostError> { - if !seen.insert(node as *const _) { - return Ok(()); - } - match &node.operator { - Operator::NonASAP(NonASAPOp::Scan { - source, predicates, .. - }) => out.push((source.clone(), predicates.clone(), vec![])), - Operator::NonASAP(NonASAPOp::PromqlInfoEnrich { selector, child }) => { - query_source_selections(child, seen, out)?; - out.push((info_source(selector)?, vec![], selector.clone())); - } - _ => { - for child in node.children() { - query_source_selections(child, seen, out)?; - } - } - } - Ok(()) -} - -fn validate_query_scope( - target: &OperatorNode, - scope: &ComparisonScope, -) -> Result<(), AnalyticalCostError> { - let mut actual = Vec::new(); - query_source_selections(target, &mut HashSet::new(), &mut actual)?; - let actual = deduplicate_source_selections(actual); - let mut declared: Vec<_> = scope - .sources - .iter() - .map(|coverage| { - ( - coverage.source.clone(), - coverage.predicates.clone(), - coverage.info_matchers.clone(), - ) - }) - .collect(); - for selection in actual { - let Some(index) = declared.iter().position(|value| value == &selection) else { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "raw target source lineage", - )); - }; - declared.swap_remove(index); - } - if !declared.is_empty() { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "raw target source lineage", - )); - } - Ok(()) -} - -fn validate_physical_scope_coverage( - physical: &EvidenceBackedPhysicalDAG, - scope: &ComparisonScope, -) -> Result<(), AnalyticalCostError> { - let nodes = reachable_physical_nodes(physical)?; - let mut covered = HashSet::new(); - for node in nodes - .into_iter() - .filter(|node| node.operator == PhysicalOperator::Scan) - { - let coverage = node - .scan_selection - .as_ref() - .ok_or_else(|| AnalyticalCostError::MissingScanSelection(node.id.clone()))?; - let Some(index) = scope.sources.iter().position(|value| value == coverage) else { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "physical scan selection", - )); - }; - covered.insert(index); - } - if covered.len() != scope.sources.len() { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "physical scan selection", - )); - } - Ok(()) -} - -fn reachable_physical_nodes( - physical: &EvidenceBackedPhysicalDAG, -) -> Result, AnalyticalCostError> { - let by_id: HashMap<_, _> = physical - .nodes - .iter() - .map(|node| (node.id.as_str(), node)) - .collect(); - if by_id.len() != physical.nodes.len() { - return Err(AnalyticalCostError::InvalidPhysicalDAG("duplicate node id")); - } - fn visit<'a>( - id: &'a str, - by_id: &HashMap<&'a str, &'a PhysicalDAGNode>, - visiting: &mut HashSet<&'a str>, - visited: &mut HashSet<&'a str>, - nodes: &mut Vec<&'a PhysicalDAGNode>, - ) -> Result<(), AnalyticalCostError> { - if visited.contains(id) { - return Ok(()); - } - if !visiting.insert(id) { - return Err(AnalyticalCostError::InvalidPhysicalDAG("cycle")); - } - let node = by_id - .get(id) - .copied() - .ok_or(AnalyticalCostError::InvalidPhysicalDAG("missing node"))?; - for child in &node.children { - visit(child, by_id, visiting, visited, nodes)?; - } - visiting.remove(id); - visited.insert(id); - nodes.push(node); - Ok(()) - } - let mut nodes = Vec::new(); - visit( - physical.root.as_str(), - &by_id, - &mut HashSet::new(), - &mut HashSet::new(), - &mut nodes, - )?; - Ok(nodes) -} - -fn validate_raw_snapshot_dimensions( - raw: &RawInputEvidence, - scope: &ComparisonScope, -) -> Result<(), AnalyticalCostError> { - validate_arrival_rate(scope.data_arrival, raw.ingestion_rate_per_second)?; - if scope.sources.len() != 1 { - return Err(AnalyticalCostError::MissingComparisonScope( - "single-source raw evolution", - )); - } - if !raw.ingestion_rate_per_second.is_finite() || raw.ingestion_rate_per_second < 0.0 { - return Err(AnalyticalCostError::InvalidIngestionRate( - raw.ingestion_rate_per_second, - )); - } - let bootstrap_is_consistent = if raw.planning_time_input_rows == 0 { - raw.planning_time_input_bytes == 0 && raw.planning_time_source_scan_bytes == 0 - } else { - raw.planning_time_input_bytes > 0 && raw.planning_time_source_scan_bytes > 0 - }; - if !bootstrap_is_consistent - || (raw.ingestion_rate_per_second > 0.0 - && (raw.arriving_logical_row_bytes == 0 || raw.arriving_source_row_bytes == 0)) - { - return Err(AnalyticalCostError::InconsistentBootstrapEvidence); - } - let mut rows = 0_u64; - let mut bytes = 0_u64; - let mut scan = 0_u64; - for offset in evaluation_offsets_ms(scope)? { - let arrivals = (raw.ingestion_rate_per_second * offset as f64 / 1_000.0).ceil(); - if !arrivals.is_finite() || arrivals < 0.0 || arrivals > u64::MAX as f64 { - return Err(AnalyticalCostError::Overflow); - } - let arrivals = arrivals as u64; - rows = rows - .checked_add(raw.planning_time_input_rows) - .and_then(|value| value.checked_add(arrivals)) - .ok_or(AnalyticalCostError::Overflow)?; - bytes = bytes - .checked_add(raw.planning_time_input_bytes) - .and_then(|value| { - arrivals - .checked_mul(raw.arriving_logical_row_bytes) - .and_then(|arriving| value.checked_add(arriving)) - }) - .ok_or(AnalyticalCostError::Overflow)?; - scan = scan - .checked_add(raw.planning_time_source_scan_bytes) - .and_then(|value| { - arrivals - .checked_mul(raw.arriving_source_row_bytes) - .and_then(|arriving| value.checked_add(arriving)) - }) - .ok_or(AnalyticalCostError::Overflow)?; - } - let reachable = reachable_physical_nodes(&raw.physical_dag)?; - if reachable - .iter() - .any(|node| node.execution != ExecutionMultiplicity::Once) - { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "streaming raw horizon evidence must use once-counted aggregate statistics", - )); - } - let expected = EdgeStatistics { rows, bytes }; - let mut scan_count = 0; - for scan_node in reachable - .into_iter() - .filter(|node| node.operator == PhysicalOperator::Scan) - { - scan_count += 1; - let evidence = raw - .physical_dag - .evidence - .get(&scan_node.id) - .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(scan_node.id.clone()))?; - let statistics = &evidence.statistics; - let OperatorStatistics::Scan { - edges, - source_read_bytes, - } = statistics - else { - return Err(AnalyticalCostError::InvalidOperatorStatistics { - node: scan_node.id.clone(), - reason: "raw scan evidence uses the wrong statistics variant", - }); - }; - if edges.input != expected || edges.output != expected || *source_read_bytes != scan { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "raw source evolution", - )); - } - } - if scan_count == 0 { - return Err(AnalyticalCostError::MissingComparisonScope("raw scan")); - } - Ok(()) -} - -pub(super) fn ephemeral_rows_over_horizon( - inputs: SummaryMaintenanceInputs, - scope: &ComparisonScope, -) -> Result { - evaluation_offsets_ms(scope)? - .into_iter() - .try_fold(0_u64, |total, offset| { - let arrivals = (inputs.ingestion_rate_per_second * offset as f64 / 1_000.0).ceil(); - if !arrivals.is_finite() || arrivals < 0.0 || arrivals > u64::MAX as f64 { - return Err(AnalyticalCostError::Overflow); - } - total - .checked_add(inputs.initial_input_rows) - .and_then(|value| value.checked_add(arrivals as u64)) - .ok_or(AnalyticalCostError::Overflow) - }) -} - -pub(super) fn ephemeral_scan_bytes_over_horizon( - inputs: SummaryMaintenanceInputs, - raw: &RawInputEvidence, - scope: &ComparisonScope, -) -> Result { - if inputs.initial_source_scan_bytes == 0 { - return Ok(0); - } - evaluation_offsets_ms(scope)? - .into_iter() - .try_fold(0_u64, |total, offset| { - let arrivals = (inputs.ingestion_rate_per_second * offset as f64 / 1_000.0).ceil(); - if !arrivals.is_finite() || arrivals < 0.0 || arrivals > u64::MAX as f64 { - return Err(AnalyticalCostError::Overflow); - } - total - .checked_add(inputs.initial_source_scan_bytes) - .and_then(|value| { - (arrivals as u64) - .checked_mul(raw.arriving_source_row_bytes) - .and_then(|arriving| value.checked_add(arriving)) - }) - .ok_or(AnalyticalCostError::Overflow) - }) -} - -pub(super) fn evaluation_offsets_ms( - scope: &ComparisonScope, -) -> Result, AnalyticalCostError> { - let count = scope.validate()?; - match &scope.recurrence { - QueryRecurrence::OneTime { - invocations, - execute_at, - } => Ok(vec![ - execute_at.map_or(0, |at| at - .0 - .saturating_sub(scope.planning_time.0)); - *invocations as usize - ]), - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) - | QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { interval, .. }) => { - Ok((1..=count).map(|n| n * u64::from(interval.0)).collect()) - } - QueryRecurrence::Repeated(RepeatedDemand::Scheduled(schedule)) => Ok(schedule - .iter() - .filter(|at| { - at.0 >= scope.planning_time.0 - && at.0 <= scope.planning_time.0.saturating_add(scope.horizon.0) - }) - .map(|at| at.0 - scope.planning_time.0) - .collect()), - QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(_)) => Ok((1..=count) - .map(|n| scope.horizon.0.saturating_mul(n) / count) - .collect()), - QueryRecurrence::Unknown => Err(AnalyticalCostError::InvalidRecurrence), - } -} - -impl SummaryMaintenanceCostModel { - pub fn new( - calibration: ResourceCalibration, - capabilities: SummaryMaintenanceCapabilities, - ) -> Self { - Self { - node_evidence: SummaryNodeEvidence::default(), - calibration, - capabilities, - target_comparisons: HashMap::new(), - candidate_comparisons: HashMap::new(), - physical_plan_alternatives: HashMap::new(), - window_framework_candidates: HashMap::new(), - } - } - - /// Bind one candidate and its raw baseline to the same target-specific - /// comparison context. Rebinding a target to different evidence is - /// rejected rather than silently replacing the canonical context. - pub fn bind_candidate_comparison( - &mut self, - target: &Rc, - root: &Rc, - scope: ComparisonScope, - raw: RawInputEvidence, - ) -> Result<(), AnalyticalCostError> { - scope.validate()?; - validate_query_scope(target, &scope)?; - validate_physical_scope_coverage(&raw.physical_dag, &scope)?; - validate_raw_snapshot_dimensions(&raw, &scope)?; - estimate_physical_dag( - &raw.physical_dag.nodes, - &raw.physical_dag.root, - &scope, - &raw.physical_dag, - )?; - let target_ptr = Rc::as_ptr(target); - if let Some(existing) = self.target_comparisons.get(&target_ptr) { - if existing.scope != scope || existing.raw != raw { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "target comparison", - )); - } - } - // Commit only after every validation above succeeds. Shared nodes do - // not carry one owning target; context identity is `(target, root)`. - self.target_comparisons - .entry(target_ptr) - .or_insert(SummaryTargetComparison { - _target: Rc::clone(target), - scope, - raw, - }); - self.candidate_comparisons.insert( - (target_ptr, Rc::as_ptr(root)), - BoundCandidateIdentity { - _target: Rc::clone(target), - _root: Rc::clone(root), - }, - ); - Ok(()) - } - - /// Add one complete physical implementation for an already-bound logical - /// candidate. Duplicate or empty provider identities are rejected. - pub fn bind_physical_plan_alternative( - &mut self, - target: &Rc, - root: &Rc, - alternative: SummaryPhysicalPlanAlternative, - ) -> Result<(), AnalyticalCostError> { - let key = (Rc::as_ptr(target), Rc::as_ptr(root)); - if !self.candidate_comparisons.contains_key(&key) { - return Err(AnalyticalCostError::MissingOrStale( - "candidate comparison binding", - )); - } - if alternative.physical_plan_id.trim().is_empty() { - return Err(AnalyticalCostError::MissingOrZero("physical_plan_id")); - } - if self.window_framework_candidates.contains_key(&key) { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "physical alternative binding mode", - )); - } - let alternatives = self.physical_plan_alternatives.entry(key).or_default(); - if alternatives - .iter() - .any(|existing| existing.physical_plan_id == alternative.physical_plan_id) - { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "physical plan identity", - )); - } - alternatives.push(alternative); - Ok(()) - } - - /// Add one complete abstract window assignment to Planner candidate search. - /// - /// The provider may bind multiple executor-feasible implementations for - /// the same framework assignment; their stable identities and complete - /// evidence keep the implementations distinct during ranking. - pub fn bind_window_framework_candidate( - &mut self, - target: &Rc, - root: &Rc, - candidate: SummaryWindowFrameworkCandidate, - ) -> Result<(), AnalyticalCostError> { - let key = (Rc::as_ptr(target), Rc::as_ptr(root)); - if !self.candidate_comparisons.contains_key(&key) { - return Err(AnalyticalCostError::MissingOrStale( - "candidate comparison binding", - )); - } - if candidate.physical_plan_id.trim().is_empty() { - return Err(AnalyticalCostError::MissingOrZero("physical_plan_id")); - } - if self.physical_plan_alternatives.contains_key(&key) { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "physical alternative binding mode", - )); - } - if candidate.assignments.is_empty() { - return Err(AnalyticalCostError::MissingOrZero( - "window framework assignments", - )); - } - let mut assigned = HashSet::new(); - if candidate - .assignments - .iter() - .any(|assignment| !assigned.insert(Rc::as_ptr(&assignment.summary))) - { - return Err(AnalyticalCostError::MissingOrZero( - "unique window framework assignments", - )); - } - if assigned != summary_aggregation_identities(root) { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "window framework assignments", - )); - } - let candidates = self.window_framework_candidates.entry(key).or_default(); - if candidates - .iter() - .any(|existing| existing.physical_plan_id == candidate.physical_plan_id) - { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "window framework candidate", - )); - } - candidates.push(candidate); - Ok(()) - } - - fn comparison_context( - &self, - root: &OperatorNode, - target: Option<&OperatorNode>, - horizon: Option, - expected_reads: Option, - ) -> Option<(CandidateComparisonKey, &SummaryTargetComparison)> { - let root_ptr = root as *const _; - let target_ptr = match target { - Some(target) => target as *const _, - None => { - let mut targets = self - .candidate_comparisons - .keys() - .filter_map(|(target, candidate)| (*candidate == root_ptr).then_some(*target)); - let only = targets.next()?; - if targets.next().is_some() { - return None; - } - only - } - }; - let key = (target_ptr, root_ptr); - if !self.candidate_comparisons.contains_key(&key) { - return None; - } - let comparison = self.target_comparisons.get(&target_ptr)?; - if horizon.map(|value| value.0 * 1_000.0) != Some(comparison.scope.horizon.0 as f64) - || expected_reads != Some(comparison.scope.validate().ok()? as f64) - { - return None; - } - Some((key, comparison)) - } - - fn complete_cost_with_evidence( - &self, - root: &OperatorNode, - deployments: &[CostedSummaryDeployment<'_>], - comparison: &SummaryTargetComparison, - evidence: &SummaryNodeEvidence, - window_frameworks: &[Option], - ) -> Option { - self.calibrated( - estimate_heterogeneous_summary( - root, - deployments, - evidence, - &comparison.scope, - &comparison.raw, - window_frameworks, - ) - .ok()?, - ) - } - - fn canonical_inputs(&self, summary: &OperatorNode) -> Option { - let evidence = self.node_evidence.aggregation(summary)?; - evidence.inputs.validate().ok()?; - Some(evidence) - } - - fn calibrated(&self, estimate: ResourceEstimate) -> Option { - estimate.calibrated_cost(&self.calibration).ok().map(Cost) - } - - fn lifecycle_inputs( - &self, - summary: &OperatorNode, - horizon: Option, - ) -> Option { - let evidence = self.canonical_inputs(summary)?; - let inputs = evidence.inputs; - let insert = validated_operator_cpu("insert_cpu_ops", evidence.insert_cpu_ops).ok()?; - let build = self.calibrated(ResourceEstimate::new( - inputs.initial_input_rows as f64 * inputs.bootstrap_window_count as f64 * insert, - 0, - inputs.initial_source_scan_bytes, - ))?; - let maintenance = self.calibrated(ResourceEstimate::new( - inputs.active_window_count as f64 * insert, - 0, - 0, - ))?; - let retained = inputs - .active_window_count - .checked_add(inputs.retained_window_count)? - .checked_mul(inputs.physical_summary_count)? - .checked_mul(inputs.state_bytes_per_summary)?; - let retention_total = self.calibrated(ResourceEstimate::new(0.0, retained, 0))?; - let horizon_seconds = horizon.filter(|value| value.0 > 0.0)?.0; - Some(SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(build), - maintenance_cost_per_update: Some(maintenance), - // Evaluation is a separate physical operator in the complete DAG. - // A state-only candidate therefore does not fabricate evaluation - // evidence merely to keep a lifecycle alternative selectable. - summary_read_cost: Some(Cost::ZERO), - retention_cost_rate: Some(CostRate(retention_total.0 / horizon_seconds)), - // Releasing memory has no modeled CPU or I/O. This is not an - // implicit expiration/rebuild policy; those require an explicit - // SummaryDelete or future authoritative lifecycle evidence. - retirement_cost: Some(Cost::ZERO), - }) - } -} - -impl CostModel for SummaryMaintenanceCostModel { - fn candidate_cost( - &self, - candidate: &ReplacementSubDAG, - _target: &TargetSubDAG<'_>, - ) -> Option { - match &candidate.replacement { - Replacement::ExactComposition(_) => None, - // Lifecycle selection supplies a complete override. If it cannot, - // the candidate remains unavailable rather than receiving this - // trait's structural fallback. - Replacement::SubDAG(_) => None, - } - } - - fn rank_candidates( - &self, - intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - DefaultCostModel.rank_candidates(intent, candidates) - } - - fn estimate_cost(&self, _candidate: &ReplacementSubDAG, _target: &TargetSubDAG<'_>) -> f64 { - f64::INFINITY - } - - fn summary_maintenance_lifecycle_cost_inputs( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - SummaryMaintenanceLifecycleCostInputs::default() - } - - fn summary_maintenance_lifecycle_cost_inputs_for_horizon( - &self, - summary: &OperatorNode, - horizon: Option, - ) -> SummaryMaintenanceLifecycleCostInputs { - self.lifecycle_inputs(summary, horizon).unwrap_or_default() - } - - fn summary_maintenance_capabilities( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceCapabilities { - self.capabilities - } - - fn complete_summary_candidate_cost( - &self, - root: &OperatorNode, - target: Option<&OperatorNode>, - deployments: &[CostedSummaryDeployment<'_>], - horizon: Option, - expected_reads: Option, - required_accuracy: &[AccuracyTarget], - ) -> Option { - self.complete_summary_candidate_estimate( - root, - target, - deployments, - horizon, - expected_reads, - required_accuracy, - ) - .map(|estimate| estimate.cost) - } - - fn complete_summary_candidate_estimate( - &self, - root: &OperatorNode, - target: Option<&OperatorNode>, - deployments: &[CostedSummaryDeployment<'_>], - horizon: Option, - expected_reads: Option, - required_accuracy: &[AccuracyTarget], - ) -> Option { - let (key, comparison) = self.comparison_context(root, target, horizon, expected_reads)?; - if let Some(candidates) = self.window_framework_candidates.get(&key) { - return candidates - .iter() - .filter_map(|candidate| { - if candidate.assignments.len() != deployments.len() { - return None; - } - if !candidate - .accuracy - .matches_assignments(&candidate.assignments) - { - return None; - } - let window_frameworks = deployments - .iter() - .map(|deployment| { - candidate - .assignments - .iter() - .find(|assignment| { - std::ptr::eq(assignment.summary.as_ref(), deployment.summary) - }) - .map(|assignment| assignment.framework.clone()) - }) - .collect::>>()?; - let uses_exponential_histogram = window_frameworks.iter().any(|framework| { - matches!( - framework, - Some(SummaryWindowFramework::ExponentialHistogram) - ) - }); - let window_accuracy_guarantee = candidate.accuracy.end_to_end_guarantee( - uses_exponential_histogram, - root.guarantee.as_ref(), - )?; - if !required_accuracy.iter().all(|target| { - DefaultAccuracyModel.satisfies(&window_accuracy_guarantee, target) - }) { - return None; - } - self.complete_cost_with_evidence( - root, - deployments, - comparison, - &candidate.node_evidence, - &window_frameworks, - ) - .map(|cost| CompleteSummaryCandidateEstimate { - cost, - physical_plan_id: Some(candidate.physical_plan_id.clone()), - window_frameworks, - window_accuracy_guarantee: Some(window_accuracy_guarantee), - }) - }) - .min_by(|left, right| left.cost.0.total_cmp(&right.cost.0)); - } - if let Some(alternatives) = self.physical_plan_alternatives.get(&key) { - return alternatives - .iter() - .filter_map(|alternative| { - let frameworks = vec![None; deployments.len()]; - self.complete_cost_with_evidence( - root, - deployments, - comparison, - &alternative.node_evidence, - &frameworks, - ) - .map(|cost| CompleteSummaryCandidateEstimate { - cost, - physical_plan_id: Some(alternative.physical_plan_id.clone()), - window_frameworks: frameworks, - window_accuracy_guarantee: None, - }) - }) - .min_by(|left, right| left.cost.0.total_cmp(&right.cost.0)); - } - let frameworks = vec![None; deployments.len()]; - self.complete_cost_with_evidence( - root, - deployments, - comparison, - &self.node_evidence, - &frameworks, - ) - .map(|cost| CompleteSummaryCandidateEstimate { - cost, - physical_plan_id: None, - window_frameworks: frameworks, - window_accuracy_guarantee: None, - }) - } - - fn complete_summary_candidate_estimate_covers_lifecycle_costs(&self) -> bool { - true - } - - fn raw_query_recompute_cost(&self, target: &OperatorNode) -> Option { - let _ = target; - None - } - - fn raw_query_recompute_total_cost( - &self, - target: &OperatorNode, - expected_reads: f64, - ) -> Option { - let target_ptr = target as *const _; - let comparison = self.target_comparisons.get(&target_ptr)?; - let evaluations = comparison.scope.validate().ok()?; - if expected_reads != evaluations as f64 { - return None; - } - self.calibrated( - estimate_physical_dag( - &comparison.raw.physical_dag.nodes, - &comparison.raw.physical_dag.root, - &comparison.scope, - &comparison.raw.physical_dag, - ) - .ok()?, - ) - } -} - -use super::estimator::*; -#[cfg(test)] -mod tests { - use std::rc::Rc; - - use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode}; - use asap_types::post_asap::{ - EvaluationSchedule, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, - OutputRepresentation, Schema, SummaryMaintenanceLifecycle, - SummaryMaintenanceLifecycleGuarantee, SummaryMaintenanceMode, SummaryUpdate, - }; - use asap_types::pre_asap::{ - agg_intent::AggIntent, ArithmeticOpKind, BinaryOpKind, DataType, Reduction, Source, - }; - use asap_types::workload::{ - DataWorkload, Evidence, EvidenceSource, Predictability, Query, QueryLanguage, - QueryRecurrence, QueryRequirements, QueryTimeScope, QueryWorkload, QueryWorkloadEntry, - Rate, RepeatedDemand, RepeatingEntry, RepetitionInterval, TimeSelection, - }; - - use super::*; - use crate::recurrence::Horizon; - use crate::summary_maintenance_lifecycle::{ - assemble_selected_dag_with_summary_maintenance_lifecycles, - global_selection_with_summary_maintenance_lifecycles, plan_summary_maintenance_lifecycles, - SummaryMaintenanceLifecycleCapabilities, WorkloadDemand, - }; - - fn estimate_test( - root: &OperatorNode, - guarantee: &SummaryMaintenanceLifecycleGuarantee, - inputs: SummaryMaintenanceInputs, - cpu: SummaryOperationCpuEvidence, - ) -> Result { - estimate_incremental_summary_maintenance(root, guarantee, inputs, cpu, &streaming_scope()) - } - - fn estimate_join_test( - root: &OperatorNode, - guarantee: &SummaryMaintenanceLifecycleGuarantee, - inputs: SummaryMaintenanceInputs, - cpu: SummaryOperationCpuEvidence, - join: Option, - ) -> Result { - estimate_incremental_summary_maintenance_with_join( - root, - guarantee, - inputs, - cpu, - join, - &streaming_scope(), - ) - } - - fn scope_for( - data: &DataWorkload, - query: &QueryWorkloadEntry, - planning_time_ms: u64, - horizon_ms: u64, - ) -> ComparisonScope { - ComparisonScope::from_workload( - data, - query, - asap_types::workload::TimestampMs(planning_time_ms), - asap_types::workload::DurationMs(horizon_ms), - vec![crate::physical_operator_statistics::ScanSelection { - source: Source::TimeSeries { - metric: "metrics".into(), - }, - source_snapshot_id: "stream-start".into(), - predicates: vec![], - info_matchers: vec![], - }], - ) - .unwrap() - } - - fn physical() -> SummaryPhysicalInputEvidence { - SummaryPhysicalInputEvidence { - initial_input_bytes: 640, - initial_source_scan_bytes: 640, - active_window_count: 2, - bootstrap_window_count: 1, - retained_window_count: 3, - physical_summary_count: 2, - state_bytes_per_summary: 100, - } - } - - fn query() -> QueryWorkloadEntry { - QueryWorkloadEntry { - query: Query("streaming count".into()), - requirements: QueryRequirements::default(), - predictability: Predictability::Unknown, - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - RepetitionInterval(1_000), - )), - time_selection: TimeSelection { - scope: QueryTimeScope::Unknown, - lookback: None, - as_of: None, - }, - } - } - - fn continuous_guarantee() -> SummaryMaintenanceLifecycleGuarantee { - SummaryMaintenanceLifecycleGuarantee { - summary_maintenance_lifecycle: SummaryMaintenanceLifecycle::ContinuouslyMaintained, - summary_maintenance_mode: SummaryMaintenanceMode::Incremental, - evaluation_schedule: EvaluationSchedule::PerUpdate, - output_representation: OutputRepresentation::SummaryState, - } - } - - /// A fixed snapshot needs cardinality evidence, but no stream-rate evidence. - #[test] - fn at_rest_workload_adapter_builds_once_without_arrivals() { - let mut data = streaming_data_workload(); - data.arrival = DataArrival::AtRest; - data.ingestion_rate = Evidence::default(); - data.input_cardinality = Evidence { - value: Some(10), - source: EvidenceSource::Declared, - ..Default::default() - }; - let scope = scope_for(&data, &query(), 0, 5_000); - let inputs = SummaryMaintenanceInputs::from_workload(physical(), &data, &scope).unwrap(); - assert_eq!(inputs.initial_input_rows, 10); - assert_eq!(inputs.ingestion_rate_per_second, 0.0); - let guarantee = SummaryMaintenanceLifecycleGuarantee { - summary_maintenance_lifecycle: SummaryMaintenanceLifecycle::Shared { - retention: asap_types::workload::DurationMs(5_000), - }, - summary_maintenance_mode: SummaryMaintenanceMode::DirectBuild, - evaluation_schedule: EvaluationSchedule::OnRead, - output_representation: OutputRepresentation::SummaryState, - }; - assert_eq!( - lifecycle_row_counts(inputs, &guarantee, &scope).unwrap(), - (10, 0, 5_000) - ); - } - - /// Arrival semantics cannot be overridden by missing or contradictory rate evidence. - #[test] - fn workload_adapter_checks_arrival_scope_and_rate_evidence() { - let mut data = streaming_data_workload(); - data.input_cardinality = Evidence { - value: Some(10), - source: EvidenceSource::Declared, - ..Default::default() - }; - let mut scope = scope_for(&data, &query(), 0, 5_000); - data.ingestion_rate = Evidence::default(); - assert_eq!( - SummaryMaintenanceInputs::from_workload(physical(), &data, &scope), - Err(AnalyticalCostError::MissingOrStale("ingestion_rate")) - ); - data.arrival = DataArrival::AtRest; - assert_eq!( - SummaryMaintenanceInputs::from_workload(physical(), &data, &scope), - Err(AnalyticalCostError::ComparisonScopeMismatch("data arrival")) - ); - scope.data_arrival = DataArrival::AtRest; - for rate in [1.0, -1.0, f64::INFINITY, f64::NAN] { - data.ingestion_rate = Evidence { - value: Some(Rate(rate)), - source: EvidenceSource::Declared, - ..Default::default() - }; - assert!(SummaryMaintenanceInputs::from_workload(physical(), &data, &scope).is_err()); - } - data.ingestion_rate = Evidence::default(); - data.input_cardinality = Evidence::default(); - assert_eq!( - SummaryMaintenanceInputs::from_workload(physical(), &data, &scope), - Err(AnalyticalCostError::MissingOrStale("input_cardinality")) - ); - } - - /// The real lifecycle planner costs a fixed snapshot with the same node evidence API. - #[test] - fn lifecycle_planner_selects_fully_costed_at_rest_summary() { - let workload = streaming_workload(); - let mut data = streaming_data_workload(); - data.arrival = DataArrival::AtRest; - data.ingestion_rate = Evidence::default(); - data.input_cardinality = Evidence { - value: Some(10), - source: EvidenceSource::Declared, - ..Default::default() - }; - let mut scope = streaming_scope(); - scope.data_arrival = DataArrival::AtRest; - let inputs = SummaryMaintenanceInputs::from_workload(physical(), &data, &scope).unwrap(); - let target = streaming_sum_query(); - let root = summary_with_operations(false, false, false); - let mut provider = streaming_model(); - bind_aggregations(&mut provider, &target, &root, inputs, streaming_cpu()); - let mut model = streaming_model(); - model.node_evidence = provider.node_evidence; - let mut raw = streaming_raw(); - raw.ingestion_rate_per_second = 0.0; - let edge = EdgeStatistics { - rows: 50, - bytes: 3_200, - }; - raw.physical_dag - .evidence - .get_mut("raw-scan") - .unwrap() - .statistics = OperatorStatistics::Scan { - source_read_bytes: 3_200, - edges: UnaryEdgeStatistics { - input: edge, - output: edge, - promql: None, - }, - }; - model - .bind_candidate_comparison(&target, &root, scope.clone(), raw.clone()) - .unwrap(); - let plan = plan_summary_maintenance_lifecycles( - Rc::clone(&root), - WorkloadDemand::new_with_data(&workload, &data, &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert!(plan.summary_total_cost.is_some()); - assert!(!plan.selected_raw_recompute); - assert_eq!( - plan.deployments[0] - .summary_maintenance_lifecycle_guarantee - .as_ref() - .unwrap() - .summary_maintenance_mode, - SummaryMaintenanceMode::DirectBuild - ); - - // Directly supplied raw evidence must not bypass the workload invariant. - raw.ingestion_rate_per_second = 1.0; - assert_eq!( - streaming_model().bind_candidate_comparison(&target, &root, scope, raw), - Err(AnalyticalCostError::ComparisonScopeMismatch( - "at-rest ingestion rate" - )) - ); - // Nor may a provider hide arrivals on a summary edge. - for aggregation in model.node_evidence.aggregations.values_mut() { - aggregation.inputs.ingestion_rate_per_second = 1.0; - } - let invalid = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &data, &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(invalid.summary_total_cost, None); - } - - #[test] - fn workload_adapter_derives_updates_and_reads_over_one_horizon() { - let data = DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - ingestion_rate: Evidence { - value: Some(Rate(2.0)), - source: EvidenceSource::Observed, - observed_at_ms: Some(100), - valid_for_ms: Some(10_000), - }, - input_cardinality: Evidence { - value: Some(10), - source: EvidenceSource::Observed, - observed_at_ms: Some(100), - valid_for_ms: Some(10_000), - }, - ..DataWorkload::default() - }; - - let scope = scope_for(&data, &query(), 100, 5_000); - let inputs = SummaryMaintenanceInputs::from_workload(physical(), &data, &scope).unwrap(); - assert_eq!(inputs.initial_input_rows, 10); - assert_eq!( - lifecycle_row_counts(inputs, &continuous_guarantee(), &scope) - .unwrap() - .1, - 10 - ); - assert_eq!(scope.validate().unwrap(), 5); - } - - #[test] - fn pure_streaming_can_bootstrap_from_an_empty_state() { - let data = DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - ingestion_rate: Evidence { - value: Some(Rate(2.0)), - source: EvidenceSource::Declared, - observed_at_ms: None, - valid_for_ms: None, - }, - input_cardinality: Evidence { - value: Some(0), - source: EvidenceSource::Declared, - observed_at_ms: None, - valid_for_ms: None, - }, - ..DataWorkload::default() - }; - let mut empty = physical(); - empty.initial_input_bytes = 0; - empty.initial_source_scan_bytes = 0; - let scope = scope_for(&data, &query(), 0, 5_000); - let inputs = SummaryMaintenanceInputs::from_workload(empty, &data, &scope).unwrap(); - let estimate = estimate_test( - &summary_with_operations(false, false, false), - &continuous_guarantee(), - inputs, - SummaryOperationCpuEvidence { - insert_cpu_ops: Some(2.0), - evaluation_cpu_ops: Some(1.0), - ..SummaryOperationCpuEvidence::default() - }, - ) - .unwrap(); - // 10 arrivals * 2 active windows * 2 insert ops + 5 reads * 2 summaries. - assert_eq!(estimate.cpu_ops(), 50.0); - assert_eq!(estimate.scan_bytes(), 0); - } - - #[test] - fn bootstrap_rows_and_bytes_must_be_present_together() { - let mut inputs = SummaryMaintenanceInputs { - initial_input_rows: 0, - initial_input_bytes: 8, - initial_source_scan_bytes: 0, - ingestion_rate_per_second: 1.0, - active_window_count: 1, - bootstrap_window_count: 1, - retained_window_count: 1, - physical_summary_count: 1, - state_bytes_per_summary: 8, - }; - assert_eq!( - inputs.validate(), - Err(AnalyticalCostError::InconsistentBootstrapEvidence) - ); - inputs.initial_input_rows = 1; - inputs.initial_input_bytes = 0; - assert_eq!( - inputs.validate(), - Err(AnalyticalCostError::InconsistentBootstrapEvidence) - ); - } - - #[test] - fn no_completed_windows_is_a_valid_streaming_deployment() { - let mut inputs = streaming_inputs(); - inputs.retained_window_count = 0; - assert!(inputs.validate().is_ok()); - } - - #[test] - fn bootstrap_rows_are_routed_to_declared_window_assignments() { - let mut inputs = streaming_inputs(); - inputs.ingestion_rate_per_second = 0.0; - inputs.bootstrap_window_count = 3; - let estimate = estimate_test( - &summary_with_operations(false, false, false), - &continuous_guarantee(), - inputs, - SummaryOperationCpuEvidence { - insert_cpu_ops: Some(2.0), - evaluation_cpu_ops: Some(1.0), - ..SummaryOperationCpuEvidence::default() - }, - ) - .unwrap(); - // 10 bootstrap rows * 3 windows * 2 insert ops + 5 reads * 2 summaries. - assert_eq!(estimate.cpu_ops(), 70.0); - } - - #[test] - fn lifecycle_output_must_remain_summary_state() { - let mut guarantee = continuous_guarantee(); - guarantee.output_representation = OutputRepresentation::FinalizedValue; - assert_eq!( - estimate_test( - &summary_with_operations(false, false, false), - &guarantee, - streaming_inputs(), - streaming_cpu(), - ), - Err(AnalyticalCostError::IncompatibleLifecycleGuarantee) - ); - } - - #[test] - fn existing_lifecycle_planner_selects_a_fully_costed_streaming_alternative() { - let inputs = SummaryMaintenanceInputs { - initial_input_rows: 10, - initial_input_bytes: 640, - initial_source_scan_bytes: 640, - ingestion_rate_per_second: 2.0, - active_window_count: 2, - bootstrap_window_count: 1, - retained_window_count: 3, - physical_summary_count: 2, - state_bytes_per_summary: 100, - }; - let mut model = streaming_model(); - let workload = QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query("streaming count".into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(1_000)), - requirements: QueryRequirements::default(), - predictability: Predictability::Predictable { known_at: None }, - time_selection: TimeSelection::default(), - }]), - }; - let root = summary_with_operations(false, false, false); - let target = streaming_sum_query(); - bind_aggregations(&mut model, &target, &root, inputs, streaming_cpu()); - let plan = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - let selected = plan.deployments[0] - .summary_maintenance_lifecycle_guarantee - .as_ref() - .unwrap(); - assert_eq!( - selected.summary_maintenance_mode, - SummaryMaintenanceMode::Incremental - ); - assert!(matches!( - selected.summary_maintenance_lifecycle, - SummaryMaintenanceLifecycle::Shared { .. } - | SummaryMaintenanceLifecycle::ContinuouslyMaintained - )); - assert!(plan.summary_total_cost.is_some()); - assert_eq!(model.raw_query_recompute_cost(&target), None); - } - - #[test] - fn complete_streaming_cost_can_select_an_ephemeral_direct_build() { - let workload = streaming_workload(); - let target = streaming_sum_query(); - let root = summary_with_operations(false, false, false); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - - let plan = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities { - supports_ephemeral: true, - supports_prepared: false, - supports_shared: false, - supports_continuously_maintained: false, - }, - &model, - ) - .unwrap(); - - assert!(plan.summary_total_cost.is_some()); - assert!(matches!( - plan.deployments[0] - .summary_maintenance_lifecycle_guarantee - .as_ref() - .map(|guarantee| &guarantee.summary_maintenance_lifecycle), - Some(SummaryMaintenanceLifecycle::Ephemeral) - )); - } - - #[test] - fn complete_streaming_cost_ranks_provider_owned_physical_plans() { - let workload = streaming_workload(); - let target = streaming_sum_query(); - let root = summary_with_operations(false, false, false); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - - let mut high_retention = model.node_evidence.clone(); - for aggregate in high_retention.aggregations.values_mut() { - aggregate.inputs.retained_window_count = 20; - } - let mut low_retention = model.node_evidence.clone(); - for aggregate in low_retention.aggregations.values_mut() { - aggregate.inputs.retained_window_count = 2; - } - for alternative in [ - SummaryPhysicalPlanAlternative { - physical_plan_id: "high-retention-layout".into(), - node_evidence: high_retention, - }, - SummaryPhysicalPlanAlternative { - physical_plan_id: "low-retention-layout".into(), - node_evidence: low_retention, - }, - ] { - model - .bind_physical_plan_alternative(&target, &root, alternative) - .unwrap(); - } - - let plan = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities { - supports_ephemeral: false, - supports_prepared: false, - supports_shared: false, - supports_continuously_maintained: true, - }, - &model, - ) - .unwrap(); - - assert_eq!( - plan.selected_window_implementation_id.as_deref(), - Some("low-retention-layout") - ); - assert_eq!( - crate::summary_maintenance_dag_export::export_summary_maintenance_plan(&plan) - .selected_window_implementation_id - .as_deref(), - Some("low-retention-layout") - ); - } - - #[test] - fn global_selection_compares_streaming_summary_and_raw_over_one_horizon() { - let target = streaming_sum_query(); - let space = crate::replacement::search_workload(vec![("q", Rc::clone(&target))]); - let workload = streaming_workload(); - let mut model = streaming_model(); - for group in space.target_subdag_candidates() { - for candidate in &group.candidates { - if let Replacement::SubDAG(root) = &candidate.replacement { - if !root.contains_asap() { - continue; - } - bind_aggregations( - &mut model, - &group.target, - root, - streaming_inputs(), - streaming_cpu(), - ); - } - } - } - let selection = global_selection_with_summary_maintenance_lifecycles( - &space, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - let plan = assemble_selected_dag_with_summary_maintenance_lifecycles( - &selection, - &space.roots[0].1, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap() - .unwrap(); - assert!(!plan.selected_raw_recompute); - assert_eq!(plan.raw_recompute_total_cost, Some(Cost(5_264.0))); - - let mut missing_baseline = model.clone(); - missing_baseline - .target_comparisons - .get_mut(&Rc::as_ptr(&space.roots[0].1)) - .unwrap() - .raw - .physical_dag - .evidence - .clear(); - let unavailable = global_selection_with_summary_maintenance_lifecycles( - &space, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &missing_baseline, - ) - .unwrap(); - assert!(unavailable - .for_target(&space.roots[0].1) - .unwrap() - .chosen - .is_none()); - - let mut raw_cheaper = model; - for evidence in raw_cheaper.node_evidence.aggregations.values_mut() { - evidence.insert_cpu_ops = 10_000.0; - } - let cheap_selection = global_selection_with_summary_maintenance_lifecycles( - &space, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &raw_cheaper, - ) - .unwrap(); - assert!(cheap_selection - .for_target(&space.roots[0].1) - .unwrap() - .chosen - .is_none()); - let cheap_plan = assemble_selected_dag_with_summary_maintenance_lifecycles( - &cheap_selection, - &space.roots[0].1, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &raw_cheaper, - ) - .unwrap() - .unwrap(); - assert!(cheap_plan.selected_raw_recompute); - assert_eq!(cheap_plan.raw_recompute_total_cost, Some(Cost(5_264.0))); - } - - #[test] - fn raw_evolution_is_bound_to_the_requested_target() { - let target_a = streaming_sum_query(); - let target_b = streaming_sum_query(); - let root_a = summary_with_operations(false, false, false); - let root_b = summary_with_operations(false, false, false); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target_a, - &root_a, - streaming_inputs(), - streaming_cpu(), - ); - let mut faster = streaming_inputs(); - // Candidate-local intermediate cardinality is not the raw target's - // planning-time cardinality and must not constrain its baseline. - faster.initial_input_rows = 7; - faster.initial_input_bytes = 448; - faster.initial_source_scan_bytes = 448; - faster.ingestion_rate_per_second = 4.0; - bind_aggregations(&mut model, &target_b, &root_b, faster, streaming_cpu()); - model - .target_comparisons - .get_mut(&Rc::as_ptr(&target_b)) - .unwrap() - .raw = { - let mut raw = streaming_raw(); - raw.ingestion_rate_per_second = 4.0; - let statistics = &mut raw - .physical_dag - .evidence - .get_mut("raw-scan") - .unwrap() - .statistics; - let OperatorStatistics::Scan { - edges, - source_read_bytes, - } = statistics - else { - unreachable!() - }; - *source_read_bytes = 7_040; - edges.input = EdgeStatistics { - rows: 110, - bytes: 7_040, - }; - edges.output = edges.input; - raw - }; - - let a = model.raw_query_recompute_total_cost(&target_a, 5.0); - let b = model.raw_query_recompute_total_cost(&target_b, 5.0); - assert_eq!(a, Some(Cost(5_264.0))); - assert!(b.unwrap().0 > a.unwrap().0); - assert_eq!(model.raw_query_recompute_total_cost(&target_a, 5.0), a); - } - - #[test] - fn raw_validation_uses_reachable_nodes_and_allows_repeated_source_scans() { - let target = streaming_sum_query(); - let root = summary_with_operations(false, false, false); - let scope = streaming_scope(); - let mut raw = streaming_raw(); - let first_scan = raw.physical_dag.nodes[0].clone(); - let mut second_scan = first_scan.clone(); - second_scan.id = "raw-scan-2".into(); - let mut unreachable = first_scan.clone(); - unreachable.id = "unreachable-per-evaluation".into(); - unreachable.execution = ExecutionMultiplicity::PerEvaluation; - raw.physical_dag.nodes = vec![ - first_scan, - second_scan, - unreachable, - PhysicalDAGNode { - id: "raw-concat".into(), - operator: PhysicalOperator::Concat, - children: vec!["raw-scan".into(), "raw-scan-2".into()], - scan_selection: None, - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::Once, - }, - ]; - raw.physical_dag.root = "raw-concat".into(); - let scan_evidence = raw.physical_dag.evidence["raw-scan"].clone(); - let mut second_scan_evidence = scan_evidence.clone(); - second_scan_evidence.physical_id = "raw-scan-2".into(); - raw.physical_dag - .evidence - .insert("raw-scan-2".into(), second_scan_evidence); - let mut unreachable_evidence = scan_evidence; - unreachable_evidence.physical_id = "unreachable-per-evaluation".into(); - raw.physical_dag - .evidence - .insert("unreachable-per-evaluation".into(), unreachable_evidence); - raw.physical_dag.evidence.insert( - "raw-concat".into(), - PhysicalNodeEvidence { - physical_id: "raw-concat".into(), - statistics: OperatorStatistics::Concat { - inputs: vec![ - EdgeStatistics { - rows: 80, - bytes: 5_120, - }, - EdgeStatistics { - rows: 80, - bytes: 5_120, - }, - ], - output: EdgeStatistics { - rows: 160, - bytes: 10_240, - }, - promql: None, - }, - output_buffer_bytes: 0, - }, - ); - let mut model = streaming_model(); - assert!(model - .bind_candidate_comparison(&target, &root, scope, raw) - .is_ok()); - } - - #[test] - fn comparison_binding_is_transactional_and_shared_nodes_allow_two_targets() { - let target_a = streaming_sum_query(); - let target_b = streaming_sum_query(); - let root = summary_with_operations(false, false, false); - let mut model = streaming_model(); - let mut wrong_scope = streaming_scope(); - wrong_scope.sources[0].source = Source::TimeSeries { - metric: "wrong".into(), - }; - assert_eq!( - model.bind_candidate_comparison(&target_a, &root, wrong_scope, streaming_raw(),), - Err(AnalyticalCostError::ComparisonScopeMismatch( - "raw target source lineage" - )) - ); - assert!(model.target_comparisons.is_empty()); - assert!(model.candidate_comparisons.is_empty()); - - model - .bind_candidate_comparison(&target_a, &root, streaming_scope(), streaming_raw()) - .unwrap(); - model - .bind_candidate_comparison(&target_b, &root, streaming_scope(), streaming_raw()) - .unwrap(); - assert_eq!(model.candidate_comparisons.len(), 2); - } - - #[test] - fn target_scope_rejects_extra_sources_and_tracks_info_matchers() { - let target = streaming_sum_query(); - let mut extra = streaming_scope(); - extra - .sources - .push(crate::physical_operator_statistics::ScanSelection { - source: Source::TimeSeries { - metric: "unused".into(), - }, - source_snapshot_id: "stream-start".into(), - predicates: vec![], - info_matchers: vec![], - }); - assert_eq!( - validate_query_scope(&target, &extra), - Err(AnalyticalCostError::ComparisonScopeMismatch( - "raw target source lineage" - )) - ); - - let selector = vec![InfoMatcher { - label: "job".into(), - op: CompareOpKind::Eq, - value: "api".into(), - }]; - let info_target = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::PromqlInfoEnrich { - selector: selector.clone(), - child: target, - }, - )) - .unwrap(); - let mut info_scope = streaming_scope(); - info_scope - .sources - .push(crate::physical_operator_statistics::ScanSelection { - source: Source::TimeSeries { - metric: "target_info".into(), - }, - source_snapshot_id: "info-start".into(), - predicates: vec![], - info_matchers: selector, - }); - validate_query_scope(&info_target, &info_scope).unwrap(); - info_scope.sources[1].info_matchers[0].value = "worker".into(); - assert_eq!( - validate_query_scope(&info_target, &info_scope), - Err(AnalyticalCostError::ComparisonScopeMismatch( - "raw target source lineage" - )) - ); - } - - #[test] - fn summary_delete_dag_fails_closed_before_costing() { - // SummaryDelete is reserved: planning rejects the DAG even with full - // delete evidence, instead of costing (or owner-checking) the delete. - let target = streaming_sum_query(); - let root = summary_with_operations(false, false, true); - let mut cpu = streaming_cpu(); - cpu.delete_cpu_ops = Some(1.0); - cpu.delete_events_per_second = Some(1.0); - cpu.delete_routing_fanout = Some(1); - let mut model = streaming_model(); - model.capabilities.delete = true; - bind_aggregations(&mut model, &target, &root, streaming_inputs(), cpu); - - // Under a evaluation, the reserved delete surfaces as an illegal child. - assert!(matches!( - streaming_planning_error(Rc::clone(&root), &model), - asap_types::post_asap::ExecutionDataStateError::IllegalChildDataState { - edge: "FinalizeExactAccumulator.child", - child: asap_types::post_asap::ExecutionDataState::QUERY_ROWS, - } - )); - // As the root, it is reported as the unimplemented operator itself. - assert!(matches!( - streaming_planning_error(evaluation_state(&root), &model), - asap_types::post_asap::ExecutionDataStateError::UnimplementedOperator { - operator: "SummaryDelete" - } - )); - } - - #[test] - fn summary_edge_and_io_evidence_fail_closed() { - // Over two independent summaries combined by a BinaryOp: a parent - // input edge that disagrees with its child's output, or missing I/O - // evidence on the root, leaves the whole-DAG cost unset. - let workload = streaming_workload(); - let target = streaming_sum_query(); - let root = add_independent_summary_results(); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - model.node_evidence.insert_operation( - &root, - SummaryOperatorEvidence::Binary(test_resource( - "binary-edge", - vec![test_edge(), EdgeStatistics { rows: 2, bytes: 16 }], - 1.0, - 1, - 0, - )), - ); - let bad_edge = plan_summary_maintenance_lifecycles( - Rc::clone(&root), - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(bad_edge.summary_total_cost, None); - - let root_evidence = model - .node_evidence - .operations - .get_mut(&Rc::as_ptr(&root)) - .unwrap() - .resource_mut(); - root_evidence.inputs = vec![test_edge(), test_edge()]; - root_evidence.io_bytes_per_execution = None; - let missing_io = plan_summary_maintenance_lifecycles( - Rc::clone(&root), - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(missing_io.summary_total_cost, None); - - // Control: the same evidence with I/O restored is costable. - model - .node_evidence - .operations - .get_mut(&Rc::as_ptr(&root)) - .unwrap() - .resource_mut() - .io_bytes_per_execution = Some(0); - let complete = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert!(complete.summary_total_cost.is_some()); - } - - #[test] - fn summary_edges_io_and_physical_identity_fail_closed() { - // Evidence bound to a structurally equal clone of the BinaryOp does not - // count for the real node; a bad input edge or missing I/O on the real - // node still fails closed. - let workload = streaming_workload(); - let target = streaming_sum_query(); - let root = add_independent_summary_results(); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - model.node_evidence.insert_operation( - &Rc::new((*root).clone()), - SummaryOperatorEvidence::Binary(test_resource( - "unused", - vec![test_edge(), test_edge()], - 1.0, - 1, - 0, - )), - ); - // Bind the actual BinaryOp, then make one parent input disagree with - // its child's output. - model.node_evidence.insert_operation( - &root, - SummaryOperatorEvidence::Binary(test_resource( - "binary-edge", - vec![test_edge(), EdgeStatistics { rows: 2, bytes: 16 }], - 1.0, - 1, - 0, - )), - ); - let bad_edge = plan_summary_maintenance_lifecycles( - Rc::clone(&root), - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(bad_edge.summary_total_cost, None); - - let root_evidence = model - .node_evidence - .operations - .get_mut(&Rc::as_ptr(&root)) - .unwrap() - .resource_mut(); - root_evidence.inputs = vec![test_edge(), test_edge()]; - root_evidence.io_bytes_per_execution = None; - let missing_io = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(missing_io.summary_total_cost, None); - } - - #[test] - fn liveness_does_not_add_disjoint_execution_workspaces() { - let target = streaming_sum_query(); - let root = summary_join(); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - let join = evidence_nodes(&root).1[0]; - model.node_evidence.joins.insert( - join as *const _, - SummaryJoinEvidence { - physical_id: "huge-join".into(), - inputs: vec![test_edge(), test_edge()], - output: test_edge(), - cpu_ops_per_execution: 1.0, - working_memory_bytes: u64::MAX, - output_buffer_bytes: 0, - executions_per_evaluation: 1, - io_bytes_per_execution: Some(0), - }, - ); - model - .node_evidence - .operations - .get_mut(&Rc::as_ptr(&root)) - .unwrap() - .resource_mut() - .working_memory_bytes = u64::MAX; - assert_eq!( - estimate_transient_liveness(&root, &model.node_evidence), - Ok(u64::MAX) - ); - } - - #[test] - fn conflicting_evidence_cannot_alias_one_provider_physical_identity() { - // Two independent summary states (combined by a BinaryOp) that claim - // one physical id but carry different evidence leave the cost unset. - let workload = streaming_workload(); - let target = streaming_sum_query(); - let root = add_independent_summary_results(); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - let aggregations = evidence_nodes(&root).0; - assert_eq!(aggregations.len(), 2); - let first = aggregations[0] as *const _; - let second = aggregations[1] as *const _; - model - .node_evidence - .aggregations - .get_mut(&first) - .unwrap() - .physical_id = "aliased-state".into(); - let second_evidence = model.node_evidence.aggregations.get_mut(&second).unwrap(); - second_evidence.physical_id = "aliased-state".into(); - second_evidence.insert_cpu_ops = 99.0; - - let plan = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(plan.summary_total_cost, None); - } - - #[test] - fn summary_join_root_fails_closed_even_with_join_evidence() { - // SummaryJoin is reserved: planning rejects the DAG whether or not - // join evidence is bound, so no partial or join cost is produced. - let root = summary_join(); - let target = streaming_sum_query(); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - assert!(matches!( - streaming_planning_error(Rc::clone(&root), &model), - asap_types::post_asap::ExecutionDataStateError::UnimplementedOperator { - operator: "SummaryJoin" - } - )); - - let join_node = evidence_nodes(&root).1[0]; - model.node_evidence.joins.insert( - join_node as *const _, - SummaryJoinEvidence { - physical_id: "costed-join".into(), - inputs: vec![test_edge(), test_edge()], - output: test_edge(), - cpu_ops_per_execution: 6.0, - working_memory_bytes: 64, - output_buffer_bytes: 64, - executions_per_evaluation: 1, - io_bytes_per_execution: Some(0), - }, - ); - assert!(matches!( - streaming_planning_error(root, &model), - asap_types::post_asap::ExecutionDataStateError::UnimplementedOperator { - operator: "SummaryJoin" - } - )); - } - - #[test] - fn whole_dag_cost_requires_and_uses_each_rc_bound_state_evidence() { - // Over two independent summaries combined by a BinaryOp: the cost needs - // evidence for each Rc-bound state, charges peak transient memory, and - // de-duplicates bootstrap scans only on a shared provider read id. - let workload = streaming_workload(); - let root = add_independent_summary_results(); - let (left, right) = binary_operands(&root); - let target = streaming_sum_query(); - let aggregations = [evaluation_state(&left), evaluation_state(&right)]; - let mut model = streaming_model(); - bind_comparison(&mut model, &target, &root); - model.node_evidence.insert_aggregation( - &aggregations[0], - SummaryAggregateEvidence { - physical_id: "left-state".into(), - input: test_edge(), - output: test_edge(), - scan_selection_index: Some(0), - bootstrap_read_identity: "left-bootstrap".into(), - inputs: streaming_inputs(), - insert_cpu_ops: streaming_cpu().insert_cpu_ops.unwrap(), - }, - ); - let incomplete = plan_summary_maintenance_lifecycles( - Rc::clone(&root), - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(incomplete.summary_total_cost, None); - - let mut second_inputs = streaming_inputs(); - second_inputs.state_bytes_per_summary = 250; - let mut second_cpu = streaming_cpu(); - second_cpu.insert_cpu_ops = Some(5.0); - model.node_evidence.insert_aggregation( - &aggregations[1], - SummaryAggregateEvidence { - physical_id: "right-state".into(), - input: test_edge(), - output: test_edge(), - scan_selection_index: Some(0), - bootstrap_read_identity: "right-bootstrap".into(), - inputs: second_inputs, - insert_cpu_ops: second_cpu.insert_cpu_ops.unwrap(), - }, - ); - // The left evaluation plays the old join's role: a 64-byte workspace and a - // 64-byte output that stays live until the root BinaryOp consumes it. - model.node_evidence.insert_operation( - &left, - SummaryOperatorEvidence::ValueOperation(test_resource( - "left-evaluation", - vec![test_edge()], - 6.0, - 64, - 64, - )), - ); - model.node_evidence.insert_operation( - &right, - SummaryOperatorEvidence::ValueOperation(test_resource( - "right-evaluation", - vec![test_edge()], - 3.0, - 0, - 0, - )), - ); - model.node_evidence.insert_operation( - &root, - SummaryOperatorEvidence::Binary(test_resource( - "root-binary", - vec![test_edge(), test_edge()], - 3.0, - 0, - 0, - )), - ); - let complete = plan_summary_maintenance_lifecycles( - Rc::clone(&root), - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert!(complete.summary_total_cost.is_some()); - - model - .node_evidence - .operations - .get_mut(&Rc::as_ptr(&root)) - .unwrap() - .resource_mut() - .working_memory_bytes = 128; - let larger_workspace = plan_summary_maintenance_lifecycles( - Rc::clone(&root), - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - // The left evaluation's 64-byte output remains live while the binary's - // workspace is active (64 + 128 = 192); the evaluation's own workspace is - // released first, so the old peak was 64 + 64 = 128. - assert_eq!( - larger_workspace.summary_total_cost.unwrap().0 - complete.summary_total_cost.unwrap().0, - 64.0 - ); - - // Equal ScanSelection does not imply that two independent state - // builds share one physical read. Only a provider-owned read identity - // permits scan de-duplication. - let mut shared_read = model; - for aggregate in shared_read.node_evidence.aggregations.values_mut() { - aggregate.bootstrap_read_identity = "one-physical-read".into(); - } - let shared = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &shared_read, - ) - .unwrap(); - assert_eq!( - larger_workspace.summary_total_cost.unwrap().0 - shared.summary_total_cost.unwrap().0, - 640.0 - ); - } - - #[test] - fn planner_selects_an_abstract_window_framework_from_downstream_evidence() { - let mut workload = streaming_workload(); - workload.repeating_queries.as_mut().unwrap()[0] - .requirements - .accuracy = - asap_types::workload::AccuracyRequirement::Explicit(AccuracyTarget::EpsilonDelta { - epsilon: 0.10, - delta: 0.01, - }); - let target = streaming_sum_query(); - let root = summary_with_operations(false, false, false); - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: summary_input, - }) = &root.operator - else { - unreachable!(); - }; - let windowed_summary = Rc::clone(summary_input); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - - let mut tumbling = model.node_evidence.clone(); - for aggregate in tumbling.aggregations.values_mut() { - aggregate.inputs.active_window_count = 1; - aggregate.inputs.retained_window_count = 20; - } - let mut sliding = model.node_evidence.clone(); - for aggregate in sliding.aggregations.values_mut() { - aggregate.inputs.active_window_count = 10; - aggregate.inputs.retained_window_count = 10; - } - let mut exponential_histogram = model.node_evidence.clone(); - for aggregate in exponential_histogram.aggregations.values_mut() { - aggregate.inputs.active_window_count = 2; - aggregate.inputs.retained_window_count = 2; - } - for candidate in [ - SummaryWindowFrameworkCandidate { - physical_plan_id: "tumbling-v1".into(), - assignments: vec![SummaryWindowFrameworkAssignment { - summary: Rc::clone(&windowed_summary), - framework: Some(SummaryWindowFramework::Tumbling), - }], - accuracy: SummaryWindowAccuracyEvidence::Exact, - node_evidence: tumbling, - }, - SummaryWindowFrameworkCandidate { - physical_plan_id: "sliding-v1".into(), - assignments: vec![SummaryWindowFrameworkAssignment { - summary: Rc::clone(&windowed_summary), - framework: Some(SummaryWindowFramework::Sliding), - }], - accuracy: SummaryWindowAccuracyEvidence::Exact, - node_evidence: sliding, - }, - SummaryWindowFrameworkCandidate { - physical_plan_id: "eh-v1".into(), - assignments: vec![SummaryWindowFrameworkAssignment { - summary: Rc::clone(&windowed_summary), - framework: Some(SummaryWindowFramework::ExponentialHistogram), - }], - accuracy: SummaryWindowAccuracyEvidence::ExponentialHistogram( - ExponentialHistogramAccuracyEvidence::UniversalGsum { - epsilon: 0.05, - failure_probability: 0.01, - range: ExponentialHistogramQueryRange::MostRecentWindow, - }, - ), - node_evidence: exponential_histogram, - }, - ] { - model - .bind_window_framework_candidate(&target, &root, candidate) - .unwrap(); - } - // Framework candidates are authoritative. Selection must not depend - // on duplicating one arbitrary implementation into the legacy global - // evidence map. - model.node_evidence = SummaryNodeEvidence::default(); - - let plan = plan_summary_maintenance_lifecycles( - Rc::clone(&root), - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities { - supports_ephemeral: false, - supports_prepared: false, - supports_shared: false, - supports_continuously_maintained: true, - }, - &model, - ) - .unwrap(); - - assert_eq!( - plan.deployments[0].selected_window_framework, - Some(SummaryWindowFramework::ExponentialHistogram) - ); - assert_eq!( - plan.selected_window_implementation_id.as_deref(), - Some("eh-v1") - ); - let guarantee = plan.window_accuracy_guarantee.as_ref().unwrap(); - assert_eq!(guarantee.metric, ErrorMetric::RelativeValue); - assert!((guarantee.bound.evaluate().unwrap() - 0.05).abs() < f64::EPSILON); - let exported = - crate::summary_maintenance_dag_export::export_summary_maintenance_plan(&plan); - assert_eq!( - exported.deployments[0].selected_window_framework, - Some(SummaryWindowFramework::ExponentialHistogram) - ); - assert_eq!( - exported.selected_window_implementation_id.as_deref(), - Some("eh-v1") - ); - assert_eq!( - exported.window_accuracy_guarantee.unwrap().metric, - ErrorMetric::RelativeValue - ); - - workload.repeating_queries.as_mut().unwrap()[0] - .requirements - .accuracy = - asap_types::workload::AccuracyRequirement::Explicit(AccuracyTarget::Epsilon(0.01)); - let stricter = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities { - supports_ephemeral: false, - supports_prepared: false, - supports_shared: false, - supports_continuously_maintained: true, - }, - &model, - ) - .unwrap(); - assert_ne!( - stricter.deployments[0].selected_window_framework, - Some(SummaryWindowFramework::ExponentialHistogram) - ); - assert!(stricter.window_accuracy_guarantee.unwrap().is_exact()); - } - - #[test] - fn window_framework_candidates_require_unique_nonempty_planner_primitives() { - let target = streaming_sum_query(); - let root = summary_with_operations(false, false, false); - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: summary_input, - }) = &root.operator - else { - unreachable!(); - }; - let windowed_summary = Rc::clone(summary_input); - let mut model = streaming_model(); - bind_comparison(&mut model, &target, &root); - - let empty = model.bind_window_framework_candidate( - &target, - &root, - SummaryWindowFrameworkCandidate { - physical_plan_id: "empty-assignments".into(), - assignments: vec![], - accuracy: SummaryWindowAccuracyEvidence::Exact, - node_evidence: model.node_evidence.clone(), - }, - ); - assert!(matches!(empty, Err(AnalyticalCostError::MissingOrZero(_)))); - - let candidate = SummaryWindowFrameworkCandidate { - physical_plan_id: "tumbling-v1".into(), - assignments: vec![SummaryWindowFrameworkAssignment { - summary: windowed_summary, - framework: Some(SummaryWindowFramework::Tumbling), - }], - accuracy: SummaryWindowAccuracyEvidence::Exact, - node_evidence: model.node_evidence.clone(), - }; - model - .bind_window_framework_candidate(&target, &root, candidate.clone()) - .unwrap(); - assert!(matches!( - model.bind_window_framework_candidate(&target, &root, candidate), - Err(AnalyticalCostError::ComparisonScopeMismatch(_)) - )); - } - - #[test] - fn one_physical_identity_cannot_alias_different_window_frameworks() { - // Two independent summaries (combined by a BinaryOp) sharing one - // physical state id but assigned different window frameworks leave - // the cost unset. - let workload = streaming_workload(); - let target = streaming_sum_query(); - let root = add_independent_summary_results(); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - let (left, right) = binary_operands(&root); - let aggregation_nodes = [evaluation_state(&left), evaluation_state(&right)]; - - let mut shared_aggregation = - model.node_evidence.aggregations[&Rc::as_ptr(&aggregation_nodes[0])].clone(); - shared_aggregation.physical_id = "shared-window-state".into(); - for aggregate in &aggregation_nodes { - model - .node_evidence - .insert_aggregation(aggregate, shared_aggregation.clone()); - } - - let retained_children: Vec<_> = aggregation_nodes - .iter() - .map(|aggregate| match &aggregate.operator { - Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) => Rc::clone(child), - _ => unreachable!(), - }) - .collect(); - let mut shared_retained = - model.node_evidence.retained_queries[&Rc::as_ptr(&retained_children[0])].clone(); - shared_retained.physical_id = "shared-retained-input".into(); - for child in &retained_children { - model - .node_evidence - .retained_queries - .insert(Rc::as_ptr(child), shared_retained.clone()); - } - - let candidate = SummaryWindowFrameworkCandidate { - physical_plan_id: "mixed-framework-binary".into(), - assignments: vec![ - SummaryWindowFrameworkAssignment { - summary: Rc::clone(&aggregation_nodes[0]), - framework: Some(SummaryWindowFramework::Tumbling), - }, - SummaryWindowFrameworkAssignment { - summary: Rc::clone(&aggregation_nodes[1]), - framework: Some(SummaryWindowFramework::Sliding), - }, - ], - accuracy: SummaryWindowAccuracyEvidence::Exact, - node_evidence: model.node_evidence.clone(), - }; - model - .bind_window_framework_candidate(&target, &root, candidate) - .unwrap(); - - let plan = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(plan.summary_total_cost, None); - } - - #[test] - fn promsketch_eh_accuracy_composes_registered_full_and_subwindow_bounds() { - let full = SummaryWindowAccuracyEvidence::ExponentialHistogram( - ExponentialHistogramAccuracyEvidence::KllRank { - eh_epsilon: 0.01, - kll_epsilon: 0.02, - failure_probability: 0.01, - range: ExponentialHistogramQueryRange::MostRecentWindow, - }, - ) - .guarantee(true) - .unwrap(); - assert_eq!(full.metric, ErrorMetric::Rank); - assert!((full.bound.evaluate().unwrap() - 0.04).abs() < f64::EPSILON); - - let subwindow = SummaryWindowAccuracyEvidence::ExponentialHistogram( - ExponentialHistogramAccuracyEvidence::KllRank { - eh_epsilon: 0.01, - kll_epsilon: 0.02, - failure_probability: 0.01, - range: ExponentialHistogramQueryRange::SubWindow { - suffix_rows: 100, - query_rows: 25, - }, - }, - ) - .guarantee(true) - .unwrap(); - assert!((subwindow.bound.evaluate().unwrap() - 0.10).abs() < f64::EPSILON); - - let gsum = SummaryWindowAccuracyEvidence::ExponentialHistogram( - ExponentialHistogramAccuracyEvidence::UniversalGsum { - epsilon: 0.05, - failure_probability: 0.30, - range: ExponentialHistogramQueryRange::SubWindow { - suffix_rows: 100, - query_rows: 25, - }, - }, - ) - .guarantee(true) - .unwrap(); - assert_eq!(gsum.metric, ErrorMetric::RelativeValue); - assert!((gsum.bound.evaluate().unwrap() - 0.20).abs() < f64::EPSILON); - } - - #[test] - fn eh_accuracy_rejects_negative_components_and_mismatched_summary_guarantees() { - let evidence = SummaryWindowAccuracyEvidence::ExponentialHistogram( - ExponentialHistogramAccuracyEvidence::KllRank { - eh_epsilon: -0.01, - kll_epsilon: 0.03, - failure_probability: 0.01, - range: ExponentialHistogramQueryRange::MostRecentWindow, - }, - ); - assert!(evidence.guarantee(true).is_none()); - - let evidence = SummaryWindowAccuracyEvidence::ExponentialHistogram( - ExponentialHistogramAccuracyEvidence::KllRank { - eh_epsilon: 0.01, - kll_epsilon: 0.02, - failure_probability: 0.01, - range: ExponentialHistogramQueryRange::MostRecentWindow, - }, - ); - let actual_summary = ResultGuarantee { - metric: ErrorMetric::Rank, - bound: BoundExpr::Constant { value: 0.03 }, - failure_probability: ProbabilityExpr::Constant { value: 0.01 }, - provenance: vec![], - }; - assert!(evidence - .end_to_end_guarantee(true, Some(&actual_summary)) - .is_none()); - } - - #[test] - fn exponential_histogram_without_registered_accuracy_composition_fails_closed() { - let mut workload = streaming_workload(); - workload.repeating_queries.as_mut().unwrap()[0] - .requirements - .accuracy = - asap_types::workload::AccuracyRequirement::Explicit(AccuracyTarget::Epsilon(1.0)); - let target = streaming_sum_query(); - let root = summary_with_operations(false, false, false); - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: summary_input, - }) = &root.operator - else { - unreachable!(); - }; - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - model - .bind_window_framework_candidate( - &target, - &root, - SummaryWindowFrameworkCandidate { - physical_plan_id: "invalid-eh".into(), - assignments: vec![SummaryWindowFrameworkAssignment { - summary: Rc::clone(summary_input), - framework: Some(SummaryWindowFramework::ExponentialHistogram), - }], - accuracy: SummaryWindowAccuracyEvidence::Exact, - node_evidence: model.node_evidence.clone(), - }, - ) - .unwrap(); - - let plan = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(plan.summary_total_cost, None); - } - - /// Bulk relational evidence cannot hide summary operators below an ordinary root. - #[test] - fn retained_subdag_evidence_cannot_hide_summary_work() { - let workload = streaming_workload(); - let target = streaming_sum_query(); - let root = add_shared_summary_result(); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - model.node_evidence.insert_retained_query( - &root, - RetainedSubDAGEvidence { - physical_id: "false-retained-root".into(), - output: test_edge(), - preprocessing_cpu_ops_over_horizon: 0.0, - working_memory_bytes: 0, - output_buffer_bytes: 0, - }, - ); - let plan = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(plan.summary_total_cost, None); - } - - #[test] - fn whole_dag_fails_closed_for_missing_retained_work_or_false_source_lineage() { - let workload = streaming_workload(); - let target = streaming_sum_query(); - let root = summary_with_operations(false, false, false); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - model.node_evidence.retained_queries.clear(); - let missing_retained = plan_summary_maintenance_lifecycles( - Rc::clone(&root), - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(missing_retained.summary_total_cost, None); - - bind_comparison(&mut model, &target, &root); - model - .target_comparisons - .get_mut(&Rc::as_ptr(&target)) - .unwrap() - .scope - .sources[0] - .source = Source::TimeSeries { - metric: "other_metric".into(), - }; - let false_lineage = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &model, - ) - .unwrap(); - assert_eq!(false_lineage.summary_total_cost, None); - } - - #[test] - fn state_only_needs_no_evaluation_and_summary_merge_child_fails_closed() { - // A state-only root is costable without evaluation evidence; a - // SummaryAgg over a reserved SummaryMerge is rejected at planning. - let workload = streaming_workload(); - let target = streaming_sum_query(); - let estimated = summary_with_operations(false, false, false); - let state_only = evaluation_state(&estimated); - let mut no_evaluation_cpu = streaming_cpu(); - no_evaluation_cpu.evaluation_cpu_ops = None; - let mut state_model = streaming_model(); - bind_aggregations( - &mut state_model, - &target, - &state_only, - streaming_inputs(), - no_evaluation_cpu, - ); - let state_plan = plan_summary_maintenance_lifecycles( - state_only, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &state_model, - ) - .unwrap(); - assert!(state_plan.summary_total_cost.is_some()); - - let nested = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: evaluation_state(&summary_with_operations(true, false, false)), - family: FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count), - input: SummaryUpdate { - item: None, - weight: asap_types::post_asap::SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), - }, - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::PerSubpopulationInstance, - filter: None, - }), - count_state_schema(), - ) - .with_guarantee(None), - ); - let mut nested_cpu = streaming_cpu(); - nested_cpu.merge_cpu_ops = Some(1.0); - let mut nested_model = streaming_model(); - bind_aggregations( - &mut nested_model, - &target, - &nested, - streaming_inputs(), - nested_cpu, - ); - assert!(matches!( - streaming_planning_error(nested, &nested_model), - asap_types::post_asap::ExecutionDataStateError::UnimplementedOperator { - operator: "SummaryMerge" - } - )); - } - - #[test] - fn mixed_arrival_fails_closed_until_backlog_and_stream_are_separate() { - let data = DataWorkload { - arrival: DataArrival::Mixed, - ..DataWorkload::default() - }; - let mut scope = streaming_scope(); - scope.data_arrival = DataArrival::Mixed; - assert_eq!( - SummaryMaintenanceInputs::from_workload(physical(), &data, &scope), - Err(AnalyticalCostError::UnsupportedDataArrival( - DataArrival::Mixed - )) - ); - } - - #[test] - fn direct_read_costs_build_updates_windows_and_recurrence() { - let estimate = estimate_test( - &summary_with_operations(false, false, false), - &continuous_guarantee(), - SummaryMaintenanceInputs { - initial_input_rows: 10, - initial_input_bytes: 640, - initial_source_scan_bytes: 640, - ingestion_rate_per_second: 2.0, - active_window_count: 2, - bootstrap_window_count: 1, - retained_window_count: 3, - physical_summary_count: 2, - state_bytes_per_summary: 100, - }, - SummaryOperationCpuEvidence { - insert_cpu_ops: Some(2.0), - evaluation_cpu_ops: Some(3.0), - ..SummaryOperationCpuEvidence::default() - }, - ) - .unwrap(); - // 10 bootstrap + 10 arrivals into two active windows; two states read 5 times. - assert_eq!(estimate.cpu_ops(), 90.0); - assert_eq!(estimate.peak_memory_bytes(), 1_000); - assert_eq!(estimate.scan_bytes(), 640); - } - - #[test] - fn operations_use_update_or_read_multiplicity_and_shared_state_once() { - let estimate = estimate_test( - &summary_with_operations(true, true, true), - &continuous_guarantee(), - SummaryMaintenanceInputs { - initial_input_rows: 1, - initial_input_bytes: 8, - initial_source_scan_bytes: 8, - ingestion_rate_per_second: 4.0, - active_window_count: 1, - bootstrap_window_count: 1, - retained_window_count: 2, - physical_summary_count: 2, - state_bytes_per_summary: 10, - }, - SummaryOperationCpuEvidence { - insert_cpu_ops: Some(1.0), - merge_cpu_ops: Some(2.0), - subtract_cpu_ops: Some(3.0), - delete_cpu_ops: Some(5.0), - delete_events_per_second: Some(4.0), - delete_routing_fanout: Some(2), - evaluation_cpu_ops: Some(7.0), - }, - ) - .unwrap(); - assert_eq!(estimate.cpu_ops(), 21.0 + 20.0 + 30.0 + 200.0 + 70.0); - // Three persistent windows plus one transient result, for two instances. - assert_eq!(estimate.peak_memory_bytes(), 80); - } - - #[test] - fn lifecycle_mode_and_schedule_must_match_existing_planner_semantics() { - let mut guarantee = continuous_guarantee(); - guarantee.evaluation_schedule = EvaluationSchedule::OnRead; - assert_eq!( - estimate_test( - &summary_with_operations(false, false, false), - &guarantee, - SummaryMaintenanceInputs { - initial_input_rows: 1, - initial_input_bytes: 8, - initial_source_scan_bytes: 8, - ingestion_rate_per_second: 1.0, - active_window_count: 1, - bootstrap_window_count: 1, - retained_window_count: 1, - physical_summary_count: 1, - state_bytes_per_summary: 8, - }, - SummaryOperationCpuEvidence { - insert_cpu_ops: Some(1.0), - evaluation_cpu_ops: Some(1.0), - ..SummaryOperationCpuEvidence::default() - }, - ), - Err(AnalyticalCostError::IncompatibleLifecycleGuarantee) - ); - } - - #[test] - fn missing_cost_for_an_operation_in_the_dag_fails_closed() { - assert_eq!( - estimate_test( - &summary_with_operations(true, false, false), - &continuous_guarantee(), - SummaryMaintenanceInputs { - initial_input_rows: 1, - initial_input_bytes: 8, - initial_source_scan_bytes: 8, - ingestion_rate_per_second: 1.0, - active_window_count: 1, - bootstrap_window_count: 1, - retained_window_count: 1, - physical_summary_count: 1, - state_bytes_per_summary: 8, - }, - SummaryOperationCpuEvidence { - insert_cpu_ops: Some(1.0), - evaluation_cpu_ops: Some(1.0), - ..SummaryOperationCpuEvidence::default() - }, - ), - Err(AnalyticalCostError::MissingOrStale("merge_cpu_ops")) - ); - } - - #[test] - fn direct_build_mode_is_not_mispriced_as_incremental_maintenance() { - let mut guarantee = continuous_guarantee(); - guarantee.summary_maintenance_lifecycle = SummaryMaintenanceLifecycle::Ephemeral; - guarantee.summary_maintenance_mode = SummaryMaintenanceMode::DirectBuild; - guarantee.evaluation_schedule = EvaluationSchedule::OneShot; - assert_eq!( - estimate_test( - &summary_with_operations(false, false, false), - &guarantee, - SummaryMaintenanceInputs { - initial_input_rows: 1, - initial_input_bytes: 8, - initial_source_scan_bytes: 8, - ingestion_rate_per_second: 1.0, - active_window_count: 1, - bootstrap_window_count: 1, - retained_window_count: 1, - physical_summary_count: 1, - state_bytes_per_summary: 8, - }, - SummaryOperationCpuEvidence { - insert_cpu_ops: Some(1.0), - evaluation_cpu_ops: Some(1.0), - ..SummaryOperationCpuEvidence::default() - }, - ), - Err(AnalyticalCostError::IncompatibleLifecycleGuarantee) - ); - } - - #[test] - fn prepared_maintenance_charges_only_its_active_interval() { - let guarantee = SummaryMaintenanceLifecycleGuarantee { - summary_maintenance_lifecycle: SummaryMaintenanceLifecycle::Prepared { - activate_at: asap_types::workload::TimestampMs(1_000), - retire_at: asap_types::workload::TimestampMs(6_000), - }, - summary_maintenance_mode: SummaryMaintenanceMode::Incremental, - evaluation_schedule: EvaluationSchedule::PerUpdate, - output_representation: OutputRepresentation::SummaryState, - }; - let estimate = estimate_test( - &summary_with_operations(false, false, false), - &guarantee, - SummaryMaintenanceInputs { - initial_input_rows: 10, - initial_input_bytes: 80, - initial_source_scan_bytes: 80, - ingestion_rate_per_second: 2.0, - active_window_count: 1, - bootstrap_window_count: 1, - retained_window_count: 1, - physical_summary_count: 1, - state_bytes_per_summary: 8, - }, - SummaryOperationCpuEvidence { - insert_cpu_ops: Some(1.0), - evaluation_cpu_ops: Some(1.0), - ..SummaryOperationCpuEvidence::default() - }, - ) - .unwrap(); - // Two pre-activation arrivals join the bootstrap; eight more are - // maintained through the horizon; five reads are served. - assert_eq!(estimate.cpu_ops(), 25.0); - } - - #[test] - fn shared_retention_is_not_the_comparison_horizon() { - let guarantee = SummaryMaintenanceLifecycleGuarantee { - summary_maintenance_lifecycle: SummaryMaintenanceLifecycle::Shared { - retention: asap_types::workload::DurationMs(999), - }, - summary_maintenance_mode: SummaryMaintenanceMode::Incremental, - evaluation_schedule: EvaluationSchedule::PerUpdate, - output_representation: OutputRepresentation::SummaryState, - }; - assert!(estimate_test( - &summary_with_operations(false, false, false), - &guarantee, - SummaryMaintenanceInputs { - initial_input_rows: 1, - initial_input_bytes: 8, - initial_source_scan_bytes: 8, - ingestion_rate_per_second: 1.0, - active_window_count: 1, - bootstrap_window_count: 1, - retained_window_count: 1, - physical_summary_count: 1, - state_bytes_per_summary: 8, - }, - SummaryOperationCpuEvidence { - insert_cpu_ops: Some(1.0), - evaluation_cpu_ops: Some(1.0), - ..SummaryOperationCpuEvidence::default() - }, - ) - .is_ok()); - } - - #[test] - fn lifecycle_retention_rate_integrates_to_one_peak_capacity_charge() { - let target = streaming_sum_query(); - let root = summary_with_operations(false, false, false); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - let aggregation = evidence_nodes(&root).0[0]; - let inputs = model - .lifecycle_inputs(aggregation, Some(Horizon(5.0))) - .unwrap(); - let integrated = inputs.retention_cost_rate.unwrap().0 * 5.0; - // (2 active + 3 retained) * 2 states * 100 bytes, calibrated once. - assert_eq!(integrated, 1_000.0); - } - - #[test] - fn summary_join_requires_cardinality_and_working_memory_evidence() { - let joined = summary_join(); - let inputs = SummaryMaintenanceInputs { - initial_input_rows: 1, - initial_input_bytes: 8, - initial_source_scan_bytes: 8, - ingestion_rate_per_second: 1.0, - active_window_count: 1, - bootstrap_window_count: 1, - retained_window_count: 1, - physical_summary_count: 1, - state_bytes_per_summary: 8, - }; - let cpu = SummaryOperationCpuEvidence { - insert_cpu_ops: Some(1.0), - evaluation_cpu_ops: Some(1.0), - ..SummaryOperationCpuEvidence::default() - }; - assert_eq!( - estimate_join_test(&joined, &continuous_guarantee(), inputs, cpu, None,), - Err(AnalyticalCostError::MissingOrStale("summary_join")) - ); - let estimate = estimate_join_test( - &joined, - &continuous_guarantee(), - inputs, - cpu, - Some(SummaryJoinEvidence { - physical_id: "diagnostic-join".into(), - inputs: vec![test_edge(), test_edge()], - output: test_edge(), - cpu_ops_per_execution: 12.0, - working_memory_bytes: 32, - output_buffer_bytes: 0, - executions_per_evaluation: 1, - io_bytes_per_execution: Some(0), - }), - ) - .unwrap(); - assert_eq!(estimate.cpu_ops(), 77.0); - assert_eq!(estimate.peak_memory_bytes(), 64); // 4 persistent states + join memory. - } - - fn count_state_schema() -> Schema { - Schema::lifted( - vec![Field::new( - "count", - FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count), - false, - )], - None, - ) - } - - fn count_evaluation_schema() -> Schema { - Schema::lifted(vec![Field::plain("count", DataType::Int64, false)], None) - } - - /// The retained relational input of every test summary: a bare scan of - /// the `metrics` series. - fn metrics_scan() -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "metrics".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap() - } - - fn summary_with_operations(merge: bool, subtract: bool, delete: bool) -> Rc { - let state_type = FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count); - let schema = count_state_schema(); - let child = metrics_scan(); - let coverage = crate::replacement::whole_source_coverage(&child).unwrap(); - let agg = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: state_type, - input: SummaryUpdate { - item: None, - weight: asap_types::post_asap::SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), - }, - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::PerSubpopulationInstance, - filter: None, - }), - schema.clone(), - ) - .with_guarantee(None) - .with_coverage(coverage) - .unwrap(), - ); - let mut root = Rc::clone(&agg); - if merge { - root = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(&agg), Rc::clone(&agg)], - }), - schema.clone(), - ) - .with_guarantee(None), - ); - } - if subtract { - root = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummarySubtract { - left: Rc::clone(&root), - right: Rc::clone(&agg), - }), - schema.clone(), - ) - .with_guarantee(None), - ); - } - if delete { - root = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryDelete { - summary_input: root, - key: 0, - }), - schema.clone(), - ) - .with_guarantee(None), - ); - } - std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: root }), - count_evaluation_schema(), - ) - .with_guarantee(Some(ResultGuarantee::exact("exact count evaluation"))), - ) - } - - fn summary_join() -> Rc { - let left = summary_with_operations(false, false, false); - let right = summary_with_operations(false, false, false); - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: left }) = &left.operator - else { - unreachable!() - }; - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: right }) = &right.operator - else { - unreachable!() - }; - let join = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryJoin { - outer: Rc::clone(left), - inner: Rc::clone(right), - key: 0, - family: FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count), - }), - left.schema.clone(), - ) - .with_guarantee(None), - ); - std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: join }), - count_evaluation_schema(), - ) - .with_guarantee(None), - ) - } - - fn add_shared_summary_result() -> Rc { - let operand = summary_with_operations(false, false, false); - Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - }, - return_bool: false, - lhs: Rc::clone(&operand), - rhs: operand, - })) - .unwrap() - .with_guarantee(Some(ResultGuarantee::exact("test binary"))), - ) - } - - /// Two independent summary states, each read out, combined by an ordinary - /// `BinaryOp`: the non-reserved replacement for a `SummaryJoin` fixture. - #[test] - fn independent_summary_results_form_a_valid_dag() { - add_independent_summary_results() - .validate_structure() - .unwrap(); - } - - fn add_independent_summary_results() -> Rc { - Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - }, - return_bool: false, - lhs: summary_with_operations(false, false, false), - rhs: summary_with_operations(false, false, false), - })) - .unwrap() - .with_guarantee(Some(ResultGuarantee::exact("test binary"))), - ) - } - - /// The `(lhs, rhs)` evaluations of [`add_independent_summary_results`]. - fn binary_operands(root: &OperatorNode) -> (Rc, Rc) { - let Operator::NonASAP(NonASAPOp::BinaryOp { lhs, rhs, .. }) = &root.operator else { - unreachable!(); - }; - (Rc::clone(lhs), Rc::clone(rhs)) - } - - /// The `SummaryAgg` under one `SummaryEstimate` evaluation. - fn evaluation_state(evaluation: &OperatorNode) -> Rc { - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: summary_input, - }) = &evaluation.operator - else { - unreachable!(); - }; - Rc::clone(summary_input) - } - - /// The DAG-validation error that streaming lifecycle planning of `root` - /// fails closed with. - fn streaming_planning_error( - root: Rc, - model: &SummaryMaintenanceCostModel, - ) -> asap_types::post_asap::ExecutionDataStateError { - let workload = streaming_workload(); - match plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - model, - ) { - Err( - crate::summary_maintenance_lifecycle::SummaryMaintenanceLifecyclePlanError::InvalidPostAsapDAG( - error, - ), - ) => error, - Err(other) => panic!("unexpected planning error: {other}"), - Ok(_) => panic!("planning must fail closed"), - } - } - - fn test_resource( - physical_id: &str, - inputs: Vec, - cpu_ops: f64, - working_memory_bytes: u64, - output_buffer_bytes: u64, - ) -> SummaryOperatorResourceEvidence { - SummaryOperatorResourceEvidence { - physical_id: physical_id.into(), - inputs, - output: test_edge(), - cpu_ops, - working_memory_bytes, - output_buffer_bytes, - executions_per_evaluation: 1, - io_bytes_per_execution: Some(0), - } - } - - #[test] - fn exact_binary_is_costable_with_explicit_physical_evidence() { - let workload = streaming_workload(); - let target = streaming_sum_query(); - let root = add_shared_summary_result(); - let mut model = streaming_model(); - bind_aggregations( - &mut model, - &target, - &root, - streaming_inputs(), - streaming_cpu(), - ); - - let plan = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &streaming_data_workload(), &[0]), - 0, - Some(Horizon(5.0)), - SummaryMaintenanceLifecycleCapabilities { - supports_ephemeral: true, - supports_prepared: false, - supports_shared: false, - supports_continuously_maintained: false, - }, - &model, - ) - .expect("binary physical evidence should produce a complete cost"); - assert!(plan.summary_total_cost.is_some()); - } - - fn streaming_sum_query() -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: metrics_scan(), - })) - .unwrap() - } - - fn streaming_workload() -> QueryWorkload { - QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query("sum(metrics)".into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(1_000)), - requirements: QueryRequirements::default(), - predictability: Predictability::Predictable { known_at: None }, - time_selection: TimeSelection::default(), - }]), - } - } - - fn streaming_data_workload() -> DataWorkload { - DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - data_ingestion_interval: Evidence { - value: Some(asap_types::workload::DurationMs(1_000)), - ..Default::default() - }, - ingestion_rate: Evidence { - value: Some(Rate(2.0)), - source: EvidenceSource::Declared, - observed_at_ms: None, - valid_for_ms: None, - }, - ..Default::default() - } - } - - fn streaming_model() -> SummaryMaintenanceCostModel { - SummaryMaintenanceCostModel::new( - ResourceCalibration { - cost_per_cpu_op: 1.0, - cost_per_scan_byte: 1.0, - cost_per_retained_byte: 1.0, - version: "test".into(), - }, - SummaryMaintenanceCapabilities { - incremental_update: true, - merge: false, - delete: false, - }, - ) - } - - fn streaming_raw() -> RawInputEvidence { - let scope = streaming_scope(); - let node = PhysicalDAGNode { - id: "raw-scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(scope.sources[0].clone()), - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::Once, - }; - let edge = EdgeStatistics { - rows: 80, - bytes: 5_120, - }; - let statistics = OperatorStatistics::Scan { - source_read_bytes: 5_120, - edges: UnaryEdgeStatistics { - input: edge, - output: edge, - promql: None, - }, - }; - RawInputEvidence { - planning_time_input_rows: 10, - planning_time_input_bytes: 640, - planning_time_source_scan_bytes: 640, - arriving_logical_row_bytes: 64, - arriving_source_row_bytes: 64, - ingestion_rate_per_second: 2.0, - physical_dag: EvidenceBackedPhysicalDAG { - nodes: vec![node], - root: "raw-scan".into(), - evidence: HashMap::from([( - "raw-scan".into(), - PhysicalNodeEvidence { - physical_id: "raw-scan".into(), - statistics, - output_buffer_bytes: 0, - }, - )]), - }, - } - } - - /// A retained relational sub-DAG: a non-ASAP node with no summary below - /// it, costed as one unit through retained-query evidence. - fn is_retained(node: &OperatorNode) -> bool { - !node.contains_asap() - } - - fn bind_comparison( - model: &mut SummaryMaintenanceCostModel, - target: &Rc, - root: &Rc, - ) { - model - .bind_candidate_comparison(target, root, streaming_scope(), streaming_raw()) - .unwrap(); - fn retained( - model: &mut SummaryMaintenanceCostModel, - node: &Rc, - seen: &mut HashSet<*const OperatorNode>, - ) { - if !seen.insert(Rc::as_ptr(node)) { - return; - } - if is_retained(node) { - model.node_evidence.insert_retained_query( - node, - RetainedSubDAGEvidence { - physical_id: format!("retained-{node:p}"), - output: test_edge(), - preprocessing_cpu_ops_over_horizon: 1.0, - working_memory_bytes: 8, - output_buffer_bytes: 0, - }, - ); - return; - } - for child in node.children() { - retained(model, child, seen); - } - } - retained(model, root, &mut HashSet::new()); - } - - fn streaming_inputs() -> SummaryMaintenanceInputs { - SummaryMaintenanceInputs { - initial_input_rows: 10, - initial_input_bytes: 640, - initial_source_scan_bytes: 640, - ingestion_rate_per_second: 2.0, - active_window_count: 2, - bootstrap_window_count: 1, - retained_window_count: 3, - physical_summary_count: 2, - state_bytes_per_summary: 100, - } - } - - fn test_edge() -> EdgeStatistics { - EdgeStatistics { rows: 1, bytes: 8 } - } - - fn streaming_cpu() -> SummaryOperationCpuEvidence { - SummaryOperationCpuEvidence { - insert_cpu_ops: Some(2.0), - evaluation_cpu_ops: Some(3.0), - ..SummaryOperationCpuEvidence::default() - } - } - - fn bind_aggregations( - model: &mut SummaryMaintenanceCostModel, - target: &Rc, - root: &Rc, - inputs: SummaryMaintenanceInputs, - cpu: SummaryOperationCpuEvidence, - ) { - bind_comparison(model, target, root); - for node in evidence_nodes(root).0 { - let source_root = matches!( - &node.operator, - Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) if is_retained(child) - ); - let mut node_inputs = inputs; - if !source_root { - node_inputs.initial_input_rows = test_edge().rows; - node_inputs.initial_input_bytes = test_edge().bytes; - node_inputs.initial_source_scan_bytes = 0; - } - model.node_evidence.aggregations.insert( - node as *const _, - SummaryAggregateEvidence { - physical_id: format!("agg-{node:p}"), - input: test_edge(), - output: test_edge(), - scan_selection_index: source_root.then_some(0), - bootstrap_read_identity: if source_root { - "shared-bootstrap".into() - } else { - String::new() - }, - inputs: node_inputs, - insert_cpu_ops: cpu.insert_cpu_ops.unwrap(), - }, - ); - } - fn resource( - physical_id: String, - inputs: Vec, - cpu_ops: f64, - working_memory_bytes: u64, - ) -> SummaryOperatorResourceEvidence { - SummaryOperatorResourceEvidence { - physical_id, - inputs, - output: test_edge(), - cpu_ops, - working_memory_bytes, - output_buffer_bytes: 0, - executions_per_evaluation: 1, - io_bytes_per_execution: Some(0), - } - } - fn bind_ops( - model: &mut SummaryMaintenanceCostModel, - node: &OperatorNode, - seen: &mut HashSet<*const OperatorNode>, - inputs: SummaryMaintenanceInputs, - cpu: SummaryOperationCpuEvidence, - ) { - if !seen.insert(node as *const _) { - return; - } - if is_retained(node) { - return; - } - let operation = match &node.operator { - Operator::NonASAP(NonASAPOp::BinaryOp { .. }) => { - cpu.evaluation_cpu_ops.map(|cpu_ops| { - SummaryOperatorEvidence::Binary(resource( - format!("binary-{node:p}"), - vec![test_edge(), test_edge()], - cpu_ops, - 0, - )) - }) - } - Operator::NonASAP(NonASAPOp::Join { .. }) => None, - Operator::NonASAP(_) - | Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { .. } - | ASAPOp::MaintainPopulation { .. } - | ASAPOp::EvaluatePopulation { .. }, - ) => cpu.evaluation_cpu_ops.map(|cpu_ops| { - SummaryOperatorEvidence::ValueOperation(resource( - format!("value-operation-{node:p}"), - vec![test_edge()], - cpu_ops, - 0, - )) - }), - Operator::ASAP(ASAPOp::SummaryMerge { children }) => { - cpu.merge_cpu_ops.map(|cpu_ops| { - SummaryOperatorEvidence::Merge(resource( - format!("merge-{node:p}"), - vec![test_edge(); children.len()], - cpu_ops, - inputs.state_bytes_per_summary, - )) - }) - } - Operator::ASAP(ASAPOp::SummarySubtract { .. }) => { - cpu.subtract_cpu_ops.map(|cpu_ops| { - SummaryOperatorEvidence::Subtract(resource( - format!("subtract-{node:p}"), - vec![test_edge(), test_edge()], - cpu_ops, - inputs.state_bytes_per_summary, - )) - }) - } - Operator::ASAP(ASAPOp::SummaryDelete { .. }) => { - cpu.delete_cpu_ops.and_then(|cpu_ops| { - Some(SummaryOperatorEvidence::Delete { - resource: resource( - format!("delete-{node:p}"), - vec![test_edge()], - cpu_ops, - 0, - ), - events_per_second: cpu.delete_events_per_second?, - routing_fanout: cpu.delete_routing_fanout?, - }) - }) - } - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) => { - cpu.evaluation_cpu_ops.map(|cpu_ops| { - SummaryOperatorEvidence::Evaluation(resource( - format!("evaluation-{node:p}"), - vec![test_edge()], - cpu_ops, - 0, - )) - }) - } - Operator::ASAP( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryJoin { .. } - | ASAPOp::Extension { .. }, - ) => None, - }; - if let Some(operation) = operation { - model - .node_evidence - .operations - .insert(node as *const _, operation); - if let Operator::ASAP(ASAPOp::SummaryDelete { summary_input, .. }) = &node.operator - { - fn owning_aggs( - node: &OperatorNode, - seen: &mut HashSet<*const OperatorNode>, - owners: &mut Vec<*const OperatorNode>, - ) { - if !seen.insert(node as *const _) { - return; - } - if matches!(node.operator, Operator::ASAP(ASAPOp::SummaryAgg { .. })) { - owners.push(node as *const _); - } - for child in node.children() { - owning_aggs(child, seen, owners); - } - } - let mut owners = Vec::new(); - owning_aggs(summary_input, &mut HashSet::new(), &mut owners); - owners.sort_unstable(); - owners.dedup(); - if let [owner] = owners.as_slice() { - model - .node_evidence - .operation_state_owners - .insert(node as *const _, *owner); - } - } - } - for child in node.children() { - bind_ops(model, child, seen, inputs, cpu); - } - } - bind_ops(model, root, &mut HashSet::new(), inputs, cpu); - } - - fn streaming_scope() -> ComparisonScope { - let workload = streaming_workload(); - let entry = workload.entries().next().unwrap(); - ComparisonScope::from_workload( - &DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - data_ingestion_interval: Evidence { - value: Some(asap_types::workload::DurationMs(1_000)), - ..Default::default() - }, - ingestion_rate: Evidence { - value: Some(Rate(2.0)), - source: EvidenceSource::Declared, - observed_at_ms: None, - valid_for_ms: None, - }, - ..Default::default() - }, - &entry, - asap_types::workload::TimestampMs(0), - asap_types::workload::DurationMs(5_000), - vec![crate::physical_operator_statistics::ScanSelection { - source: Source::TimeSeries { - metric: "metrics".into(), - }, - source_snapshot_id: "stream-start".into(), - predicates: vec![], - info_matchers: vec![], - }], - ) - .unwrap() - } -} diff --git a/crates/asap-aware-mapping/src/summary_maintenance_cost/window.rs b/crates/asap-aware-mapping/src/summary_maintenance_cost/window.rs deleted file mode 100644 index fb80d5ee0..000000000 --- a/crates/asap-aware-mapping/src/summary_maintenance_cost/window.rs +++ /dev/null @@ -1,252 +0,0 @@ -use super::*; - -/// One per-state window choice within a complete Planner candidate. -#[derive(Debug, Clone)] -pub struct SummaryWindowFrameworkAssignment { - pub summary: Rc, - /// `None` explicitly means that this state is not window-organized. - pub framework: Option, -} - -/// Cost evidence for one complete abstract window-framework assignment across -/// a summary DAG in Planner search. -/// -/// The provider derives this evidence from a concrete downstream -/// implementation under the current data workload. The stable identity is -/// provenance for the chosen implementation, while deployment placement and -/// runtime configuration remain downstream concerns. -#[derive(Debug, Clone)] -pub struct SummaryWindowFrameworkCandidate { - /// Stable identity of the complete provider implementation whose evidence - /// is bound to this planner-visible framework assignment. - pub physical_plan_id: String, - /// Exactly one assignment for every summary deployment in the DAG. - pub assignments: Vec, - /// Registered end-to-end accuracy composition for this complete window - /// assignment. EH combinations must use one of the specialized proofs; - /// unknown combinations fail closed. - pub accuracy: SummaryWindowAccuracyEvidence, - pub node_evidence: SummaryNodeEvidence, -} - -pub(super) fn summary_aggregation_identities(root: &OperatorNode) -> HashSet<*const OperatorNode> { - fn visit( - node: &OperatorNode, - seen: &mut HashSet<*const OperatorNode>, - out: &mut HashSet<*const OperatorNode>, - ) { - if !seen.insert(node as *const _) { - return; - } - if matches!(node.operator, Operator::ASAP(ASAPOp::SummaryAgg { .. })) { - out.insert(node as *const _); - } - for child in node.children() { - visit(child, seen, out); - } - } - - let mut out = HashSet::new(); - visit(root, &mut HashSet::new(), &mut out); - out -} - -/// Cardinality normalization used by the PromSketch EH bounds. The paper's -/// sub-window error is stated relative to the suffix beginning at the query's -/// left endpoint, so a query-relative bound needs `suffix_rows/query_rows`. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum ExponentialHistogramQueryRange { - MostRecentWindow, - SubWindow { suffix_rows: u64, query_rows: u64 }, -} - -/// Registered accuracy compositions for Exponential Histogram realizations. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum ExponentialHistogramAccuracyEvidence { - /// PromSketch EHKLL normalized rank error. - KllRank { - eh_epsilon: f64, - kll_epsilon: f64, - failure_probability: f64, - range: ExponentialHistogramQueryRange, - }, - /// PromSketch EHUniv/GSum relative error. - UniversalGsum { - epsilon: f64, - failure_probability: f64, - range: ExponentialHistogramQueryRange, - }, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum SummaryWindowAccuracyEvidence { - /// The window implementation preserves exact query-time coverage and adds - /// no error. Used for exact tumbling/sliding realizations. - Exact, - ExponentialHistogram(ExponentialHistogramAccuracyEvidence), -} - -impl ExponentialHistogramQueryRange { - fn suffix_to_query_ratio(self) -> Option { - match self { - Self::MostRecentWindow => Some(1.0), - Self::SubWindow { - suffix_rows, - query_rows, - } if query_rows > 0 && suffix_rows >= query_rows => { - Some(suffix_rows as f64 / query_rows as f64) - } - Self::SubWindow { .. } => None, - } - } -} - -impl SummaryWindowAccuracyEvidence { - pub(super) fn matches_assignments( - &self, - assignments: &[SummaryWindowFrameworkAssignment], - ) -> bool { - let eh_summaries: Vec<_> = assignments - .iter() - .filter(|assignment| { - assignment.framework == Some(SummaryWindowFramework::ExponentialHistogram) - }) - .collect(); - match self { - Self::Exact => eh_summaries.is_empty(), - Self::ExponentialHistogram(ExponentialHistogramAccuracyEvidence::KllRank { - .. - }) => { - eh_summaries.len() == 1 - && eh_summaries.iter().all(|assignment| { - matches!( - &assignment.summary.operator, - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if kind.algorithm() == &SketchAlgorithm::Kll - ) - }) - } - Self::ExponentialHistogram(ExponentialHistogramAccuracyEvidence::UniversalGsum { - .. - }) => { - eh_summaries.len() == 1 - && eh_summaries.iter().all(|assignment| { - matches!( - &assignment.summary.operator, - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate( - ExactKind::Count | ExactKind::Sum, - _ - ), - .. - }) - ) - }) - } - } - } - - /// Compose the two EH combinations proved by PromSketch - /// (doi:10.14778/3742728.3742732). Unknown EH combinations deliberately - /// have no catch-all arm. - pub(super) fn guarantee(&self, uses_exponential_histogram: bool) -> Option { - match self { - Self::Exact if !uses_exponential_histogram => { - Some(ResultGuarantee::exact("exact window coverage")) - } - Self::Exact => None, - Self::ExponentialHistogram(evidence) if uses_exponential_histogram => { - let (metric, bound, failure_probability, rule) = match *evidence { - ExponentialHistogramAccuracyEvidence::KllRank { - eh_epsilon, - kll_epsilon, - failure_probability, - range, - } => { - if !eh_epsilon.is_finite() - || eh_epsilon < 0.0 - || !kll_epsilon.is_finite() - || kll_epsilon < 0.0 - { - return None; - } - ( - ErrorMetric::Rank, - 2.0 * eh_epsilon * range.suffix_to_query_ratio()? + kll_epsilon, - failure_probability, - "promsketch_eh_kll_rank", - ) - } - ExponentialHistogramAccuracyEvidence::UniversalGsum { - epsilon, - failure_probability, - range, - } => { - if !epsilon.is_finite() || epsilon < 0.0 { - return None; - } - ( - ErrorMetric::RelativeValue, - epsilon * range.suffix_to_query_ratio()?, - failure_probability, - "promsketch_eh_universal_gsum", - ) - } - }; - if !bound.is_finite() - || bound < 0.0 - || !failure_probability.is_finite() - || !(0.0..=1.0).contains(&failure_probability) - { - return None; - } - Some(ResultGuarantee { - metric, - bound: BoundExpr::Constant { value: bound }, - failure_probability: ProbabilityExpr::Constant { - value: failure_probability, - }, - provenance: vec![GuaranteeSource::RuntimeObservation { - source: rule.into(), - detail: serde_json::json!({ - "reference": "doi:10.14778/3742728.3742732" - }), - }], - }) - } - Self::ExponentialHistogram(_) => None, - } - } - - pub(super) fn end_to_end_guarantee( - &self, - uses_exponential_histogram: bool, - summary_guarantee: Option<&ResultGuarantee>, - ) -> Option { - let summary = summary_guarantee?; - match self { - Self::Exact if !uses_exponential_histogram => Some(summary.clone()), - Self::ExponentialHistogram(ExponentialHistogramAccuracyEvidence::KllRank { - kll_epsilon, - failure_probability, - .. - }) if uses_exponential_histogram => { - let summary_bound = summary.bound.evaluate()?; - let summary_failure = summary.failure_probability.evaluate()?; - if summary.metric != ErrorMetric::Rank - || summary_bound != *kll_epsilon - || summary_failure != *failure_probability - { - return None; - } - self.guarantee(true) - } - Self::ExponentialHistogram(ExponentialHistogramAccuracyEvidence::UniversalGsum { - .. - }) if uses_exponential_histogram && summary.is_exact() => self.guarantee(true), - _ => None, - } - } -} diff --git a/crates/asap-aware-mapping/src/summary_maintenance_dag_export.rs b/crates/asap-aware-mapping/src/summary_maintenance_dag_export.rs deleted file mode 100644 index eae562c59..000000000 --- a/crates/asap-aware-mapping/src/summary_maintenance_dag_export.rs +++ /dev/null @@ -1,132 +0,0 @@ -//! Serializable DAG export for a materialized summary-maintenance plan. -//! -//! `asap-types::dag_export` owns the crate-neutral post-ASAP DAG shape. This -//! adapter lives in the mapping layer, where summary-maintenance lifecycle -//! alternatives and their typed rejection reasons are available, and emits -//! both views together. - -use std::collections::HashMap; -use std::rc::Rc; - -use asap_types::ir::OperatorNode; -use serde::Serialize; - -use asap_types::dag_export::{self, SummaryDAG}; -use asap_types::ir::export::PhysicalASAPNodeId; -use asap_types::post_asap::{ - ResultGuarantee, SummaryMaintenanceLifecycle, SummaryMaintenanceLifecycleGuarantee, - SummaryWindowFramework, -}; - -use crate::summary_maintenance_lifecycle::{ - SummaryMaintenanceLifecyclePlan, SummaryMaintenanceLifecycleRejection, -}; - -#[derive(Debug, Clone, Serialize)] -pub struct SummaryMaintenanceDAGExport { - pub dag: SummaryDAG, - pub deployments: Vec, - pub horizon_seconds: Option, - pub evaluation_rate_per_second: Option, - pub update_rate_per_second: Option, - pub expected_reads: Option, - pub selected_raw_recompute: bool, - #[serde(skip_serializing_if = "Option::is_none")] - /// Provider implementation key. The legacy JSON field name is retained - /// until the surrounding export receives its own schema-version bump. - #[serde(rename = "selected_physical_plan_id")] - pub selected_window_implementation_id: Option, - pub summary_total_cost: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub window_accuracy_guarantee: Option, - pub raw_recompute_total_cost: Option, -} - -#[derive(Debug, Clone, Serialize)] -pub struct SummaryMaintenanceDeploymentExport { - pub post_asap_node_id: PhysicalASAPNodeId, - #[serde(skip_serializing_if = "Option::is_none")] - pub selected_window_framework: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub selected: Option, - pub alternatives: Vec, -} - -#[derive(Debug, Clone, Serialize)] -pub struct SummaryMaintenanceLifecycleAlternativeExport { - pub lifecycle: SummaryMaintenanceLifecycle, - pub total_cost: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub rejection: Option, - pub assumptions: Vec, -} - -pub type SummaryMaintenanceLifecycleGuaranteeExport = SummaryMaintenanceLifecycleGuarantee; - -pub fn export_summary_maintenance_plan( - plan: &SummaryMaintenanceLifecyclePlan, -) -> SummaryMaintenanceDAGExport { - let deployments: Vec<_> = plan - .deployments - .iter() - .map(|deployment| SummaryMaintenanceDeploymentExport { - post_asap_node_id: deployment.post_asap_node_id, - selected_window_framework: deployment.selected_window_framework.clone(), - selected: deployment - .summary_maintenance_lifecycle_guarantee - .as_ref() - .cloned(), - alternatives: deployment - .alternatives - .iter() - .map(|alternative| SummaryMaintenanceLifecycleAlternativeExport { - lifecycle: alternative.summary_maintenance_lifecycle.clone(), - total_cost: alternative.total_cost.map(|cost| cost.0), - rejection: alternative.rejection.clone(), - assumptions: alternative.assumptions.clone(), - }) - .collect(), - }) - .collect(); - let mut dag = dag_export::export_summary(&plan.root); - let deployment_by_summary: HashMap<_, _> = plan - .deployments - .iter() - .zip(&deployments) - .map(|(deployment, export)| (Rc::as_ptr(&deployment.summary), export)) - .collect(); - annotate_lifecycle_deployments(&mut dag, &deployment_by_summary); - - SummaryMaintenanceDAGExport { - dag, - deployments, - horizon_seconds: plan.horizon.map(|horizon| horizon.0), - evaluation_rate_per_second: plan.evaluation_rate.map(|rate| rate.0), - update_rate_per_second: plan.update_rate.map(|rate| rate.0), - expected_reads: plan.expected_reads, - selected_raw_recompute: plan.selected_raw_recompute, - selected_window_implementation_id: plan.selected_window_implementation_id.clone(), - summary_total_cost: plan.summary_total_cost.map(|cost| cost.0), - window_accuracy_guarantee: plan.window_accuracy_guarantee.clone(), - raw_recompute_total_cost: plan.raw_recompute_total_cost.map(|cost| cost.0), - } -} - -/// Attach a deployment directly to the exported node of its `SummaryAgg`, -/// matched by the `Rc` identity every exported node carries. This makes the -/// decision visible to dag consumers without asking them to reconstruct -/// pointer identity from dag position. -fn annotate_lifecycle_deployments( - dag: &mut SummaryDAG, - deployments: &HashMap<*const OperatorNode, &SummaryMaintenanceDeploymentExport>, -) { - for dag_node in &mut dag.nodes { - let Some(source) = &dag_node.source_node else { - continue; - }; - if let Some(deployment) = deployments.get(&Rc::as_ptr(source)) { - dag_node.detail["summary_maintenance"] = - serde_json::to_value(deployment).expect("lifecycle export is serializable"); - } - } -} diff --git a/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs b/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs deleted file mode 100644 index 51bb7e29a..000000000 --- a/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs +++ /dev/null @@ -1,3909 +0,0 @@ -//! Workload-aware summary-maintenance lifecycle planning. -//! -//! A **summary-maintenance lifecycle** is the planner policy for when one -//! materialized summary state is created, retained or shared, updated as data -//! arrives, and retired. It is deliberately narrower than the end-to-end data -//! lifecycle and independent of query recurrence. Recurrence says when and how -//! often queries will read the result. The planner converts that demand into -//! expected reads and an evaluation rate, then uses those quantities to compare -//! rebuilding per query with retaining or continuously maintaining state. -//! Recurrence does not itself prescribe a state-maintenance policy. -//! -//! This module enumerates and costs `Ephemeral`, `Prepared`, `Shared`, and -//! `ContinuouslyMaintained` alternatives for every unique `SummaryAgg` in a -//! materialized plan, and for every maintained population (`MaintainPopulation`) -//! that is not an input of a `SummaryAgg`. [`SummaryMaintenanceMode`] is an orthogonal detail of -//! the selected deployment: state is either built directly or updated -//! incrementally. Unknown evidence stays unknown and therefore cannot make a -//! long-lived alternative win. - -use asap_types::ir::cse::share_common_sub_dags; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::export::{ - compile_physical_asap_dag_with_node_ids, compile_physical_asap_workload_with_node_ids, - PhysicalASAPDAG, PhysicalASAPDAGValidationError, PhysicalASAPNodeId, -}; -use asap_types::ir::timing::{apply_lifecycle_timings, LifecycleAssignment, TimingMemo}; -use asap_types::ir::{ASAPOp, Operator, OperatorNode}; -use asap_types::post_asap::{ - EvaluationSchedule, ExecutionDataStateError, ExecutionTiming, OutputRepresentation, - ResultGuarantee, SummaryMaintenanceLifecycle, SummaryMaintenanceLifecycleGuarantee, - SummaryMaintenanceMode, SummaryWindowFramework, -}; -use asap_types::types::AccuracyTarget; -use asap_types::workload::{ - DataArrival, DataWorkload, Predictability, QueryRecurrence, QueryWorkload, RepeatedDemand, - TimestampMs, WorkloadError, -}; - -use crate::analytical_cost::AnalyticalCostError; -use crate::cost_model::{ - CompleteSummaryCandidateEstimate, Cost, CostModel, CostedSummaryDeployment, -}; -use crate::physical_operator_statistics::evaluations_in_horizon; -use crate::recurrence::{ - CostRate, EvaluationRate, Horizon, RecurrenceError, RecurrenceProfile, UpdateRate, -}; -use crate::replacement::{ - CandidateCostOverrides, CandidateLogicalASAPDAGs, GlobalSelection, RealizationError, - Replacement, ReplacementProvenance, -}; - -/// Summary-maintenance lifecycle shapes supported by the target runtime. -/// -/// These independent flags describe the set of lifecycle alternatives the -/// runtime implements, not simultaneous states of one deployment. Multiple -/// flags may be `true` (a runtime can support both ephemeral and prepared -/// state, for example); the planner still selects exactly one mutually -/// exclusive [`SummaryMaintenanceLifecycle`] for each deployment. A supported -/// alternative may still be rejected because workload evidence is missing or -/// its cost is unknown. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct SummaryMaintenanceLifecycleCapabilities { - /// The runtime can build a fresh state for each invocation and retire it - /// after that invocation finishes. - pub supports_ephemeral: bool, - /// The runtime can build state before a predictable execution and retain - /// it until that scheduled execution window ends. - pub supports_prepared: bool, - /// The runtime can retain one state and reuse it across multiple reads. - pub supports_shared: bool, - /// The runtime can keep state current by applying arriving data updates. - pub supports_continuously_maintained: bool, -} - -/// State operations supported by one concrete summary family and -/// representation. -/// -/// This differs from [`SummaryMaintenanceLifecycleCapabilities`]: these flags -/// describe what the summary algorithm itself can do, while lifecycle -/// capabilities describe what deployment policies the target runtime can -/// orchestrate. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct SummaryMaintenanceCapabilities { - /// Existing state can incorporate arriving input without a full rebuild. - pub incremental_update: bool, - /// Two independently built states can be combined into one equivalent - /// state. - pub merge: bool, - /// Expired or retracted input can be removed from existing state. - pub delete: bool, -} - -impl SummaryMaintenanceLifecycleCapabilities { - pub const ALL: Self = Self { - supports_ephemeral: true, - supports_prepared: true, - supports_shared: true, - supports_continuously_maintained: true, - }; -} - -impl Default for SummaryMaintenanceLifecycleCapabilities { - fn default() -> Self { - Self::ALL - } -} - -/// Primitive costs for one concrete summary state. Every field is optional: -/// missing statistics produce an uncosted alternative, never a zero. -/// -/// The lifecycle planner combines these state-specific inputs with workload -/// rates, invocation counts, and the optimization horizon. All `Cost` fields -/// are one-time costs unless their name explicitly says otherwise. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct SummaryMaintenanceLifecycleCostInputs { - /// One-time cost to construct the state from its input. - pub build_cost: Option, - /// Cost to incorporate one arriving input update into existing state. - pub maintenance_cost_per_update: Option, - /// Cost of one read or finalization from already-built summary state. - pub summary_read_cost: Option, - /// Cost per second for retaining the state over a lifecycle window. - pub retention_cost_rate: Option, - /// One-time cost to release or retire the state. - pub retirement_cost: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] -#[serde(rename_all = "snake_case")] -pub enum SummaryMaintenanceLifecycleRejection { - UnsupportedByRuntime, - RequiresPredictableOneTimeQuery, - RequiresMultipleReads, - RequiresHorizon, - RequiresContinuousData, - MissingOrStaleIngestionRate, - SummaryDoesNotSupportIncrementalUpdates, - SummaryDoesNotSupportDeletion, - MissingCostEvidence, -} - -/// One candidate lifecycle policy for a particular summary deployment. -/// -/// `total_cost: None` never means zero: it means the planner lacks enough -/// evidence to cost the candidate. Such a candidate is not selectable and its -/// `rejection` explains why. -#[derive(Debug, Clone, PartialEq)] -pub struct SummaryMaintenanceLifecycleAlternative { - /// State creation, retention, sharing, update, and retirement policy. - pub summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, - /// Complete cost over the requested horizon, when every input is known. - pub total_cost: Option, - /// Why this alternative cannot be selected; `None` means it is legal and - /// fully costed. - pub rejection: Option, - /// Human-readable premises used when deriving and costing the alternative. - pub assumptions: Vec, -} - -impl SummaryMaintenanceLifecycleAlternative { - fn selectable(&self) -> bool { - self.rejection.is_none() && self.total_cost.is_some() - } -} - -/// One unique retained-state deployment. Shared `Rc` nodes are emitted once. -#[derive(Debug, Clone)] -pub struct SummaryMaintenanceDeployment { - /// Identity of this summary in the exported post-ASAP semantic DAG. - /// It is scoped to one plan version and is not a summary definition or - /// summary instance identity. - pub post_asap_node_id: PhysicalASAPNodeId, - /// The unique materialized `SummaryAgg` represented by this deployment. - pub summary: Rc, - /// Lifecycle, evaluation, and representation commitment selected for this - /// state, or `None` when no alternative is selectable. - pub summary_maintenance_lifecycle_guarantee: Option, - /// Abstract window primitive selected for this state. Concrete runtime - /// implementation, placement, and identity remain downstream decisions. - pub selected_window_framework: Option, - /// Every lifecycle shape considered, including rejected and uncosted ones. - pub alternatives: Vec, -} - -/// Workload-aware lifecycle and window-framework decisions for every unique -/// summary state reachable from one materialized post-ASAP root. -#[derive(Debug, Clone)] -pub struct SummaryMaintenanceLifecyclePlan { - /// Root of the materialized post-ASAP DAG being deployed. - pub root: Rc, - /// One entry per unique reachable `SummaryAgg`; shared `Rc` nodes appear - /// only once. - pub deployments: Vec, - /// Caller-supplied optimization horizon used to turn rates into total - /// costs. `None` keeps horizon-dependent alternatives unselectable. - pub horizon: Option, - /// Aggregate recurring query-evaluation rate derived from the workload. - pub evaluation_rate: Option, - /// Fresh source-data ingestion rate, when supplied by the workload. - pub update_rate: Option, - /// Total demand inside the horizon, when every recurrence is known. - pub expected_reads: Option, - /// Whether global costing preferred rebuilding the raw expression over all - /// summary deployments. - pub selected_raw_recompute: bool, - /// Provider-owned identity of the selected complete physical deployment - /// (for example a tumbling, sliding, or exponential-histogram plan). - pub selected_window_implementation_id: Option, - /// Cost of the selected set of summary deployments, when fully known. - pub summary_total_cost: Option, - /// Composed accuracy guarantee supplied by the selected physical window - /// evidence, when the window framework introduces approximation. - pub window_accuracy_guarantee: Option, - /// Cost of evaluating the original expression for the same demand, when - /// fully known. - pub raw_recompute_total_cost: Option, -} - -/// Why a lifecycle plan cannot assign execution timing to its DAG. -#[derive(Debug, thiserror::Error, PartialEq)] -pub enum SummaryMaintenanceTimingError { - #[error(transparent)] - InvalidPostAsapDAG(#[from] ExecutionDataStateError), - #[error("summary {0:?} has no selected lifecycle")] - UnselectedLifecycle(PhysicalASAPNodeId), - /// A maintained population outside any `SummaryAgg`'s inputs has no - /// deployment, so its timing would be guessed. Enumeration always emits - /// one; this arises only for a plan whose root or deployments were edited. - #[error("node {0:?} maintains state that has no summary-maintenance lifecycle")] - UnplannedMaintainedState(PhysicalASAPNodeId), - #[error(transparent)] - InvalidPhases(#[from] PhysicalASAPDAGValidationError), -} - -impl SummaryMaintenanceLifecyclePlan { - /// The post-ASAP DAG of [`Self::root`] with every node's timing derived - /// from the selected lifecycles, so physical compilation places it. - /// - /// A retained (non-`Ephemeral`) state outlives one query, so it and every - /// input it consumes run at ingestion time. Every other node runs at query - /// time: evaluations and consumers of retained state, and each `Ephemeral` - /// state not consumed by retained state together with its inputs, whose - /// raw data the deployment must supply as a query source. This applies to - /// maintained populations as to `SummaryAgg` states; a population feeding - /// a `SummaryAgg` is one of its inputs. Timings already on the root are - /// ignored. - pub fn execution_timed_dag(&self) -> Result { - execution_timed_workload_dag(&[self]) - } -} - -/// One physical ASAP DAG for a workload: a root per plan, in order, with -/// sub-DAGs shared between plans exported once. Timing follows the selected -/// lifecycles of every plan's deployments, as in -/// [`SummaryMaintenanceLifecyclePlan::execution_timed_dag`]. -pub fn execution_timed_workload_dag( - plans: &[&SummaryMaintenanceLifecyclePlan], -) -> Result { - // One memo, so a node shared by several roots is timed and exported once. - let mut memo = TimingMemo::new(); - let assignment = LifecycleAssignment::default_maintained(); - let timed = plans - .iter() - .map(|plan| apply_lifecycle_timings(&plan.root, &assignment, &mut memo)) - .collect::, _>>()?; - let compiled = compile_physical_asap_workload_with_node_ids(&timed)?; - let id_of = |node: &Rc| { - compiled - .node_ids - .node_id(memo.timed(node).expect("plan node was timed")) - .expect("timed plan node belongs to the compiled DAG") - }; - let deployments: Vec<_> = plans - .iter() - .flat_map(|plan| &plan.deployments) - .map(|deployment| (id_of(&deployment.summary), deployment)) - .collect(); - for plan in plans { - for population in &standalone_populations(&plan.root) { - let id = id_of(population); - if !deployments.iter().any(|(deployed, _)| *deployed == id) { - return Err(SummaryMaintenanceTimingError::UnplannedMaintainedState(id)); - } - } - } - let dag = compiled.dag; - let mut pending = Vec::new(); - for (id, deployment) in &deployments { - let guarantee = deployment - .summary_maintenance_lifecycle_guarantee - .as_ref() - .ok_or(SummaryMaintenanceTimingError::UnselectedLifecycle(*id))?; - if guarantee.summary_maintenance_lifecycle != SummaryMaintenanceLifecycle::Ephemeral { - pending.push(*id); - } - } - let mut ingestion = HashSet::new(); - while let Some(id) = pending.pop() { - if ingestion.insert(id) { - pending.extend( - dag.edges - .iter() - .filter(|edge| edge.consumer == id) - .map(|edge| edge.producer), - ); - } - } - let phases = dag - .nodes - .iter() - .map(|node| { - let timing = if ingestion.contains(&node.id) { - ExecutionTiming::IngestionTime - } else { - ExecutionTiming::QueryTime - }; - (node.id, timing) - }) - .collect(); - Ok(dag.with_execution_phases(&phases)?) -} - -/// Explicit association between a materialized target and the normalized -/// workload entries whose demand consumes it. -/// -/// [`QueryWorkload`] remains the source of query demand, while source-data -/// evidence is supplied independently. Indices avoid copying normalized entry -/// definitions while ensuring unrelated entries do not influence a target's -/// lifecycle decision. -#[derive(Debug, Clone, Copy)] -pub struct WorkloadDemand<'a> { - /// Original normalized query workload. - pub workload: &'a QueryWorkload, - /// Independent source-data evidence, when the caller has it. - pub data_workload: Option<&'a DataWorkload>, - /// Indices from [`QueryWorkload::entries`] that consume this target. - pub entry_indices: &'a [usize], -} - -impl<'a> WorkloadDemand<'a> { - /// Bind query demand without source-data evidence. Callers that have a - /// [`DataWorkload`] should use [`Self::new_with_data`] so ingestion facts - /// are not silently discarded. - pub const fn new_without_data(workload: &'a QueryWorkload, entry_indices: &'a [usize]) -> Self { - Self { - workload, - data_workload: None, - entry_indices, - } - } - - pub const fn new_with_data( - workload: &'a QueryWorkload, - data_workload: &'a DataWorkload, - entry_indices: &'a [usize], - ) -> Self { - Self { - workload, - data_workload: Some(data_workload), - entry_indices, - } - } -} - -#[derive(Debug, thiserror::Error)] -pub enum SummaryMaintenanceLifecyclePlanError { - #[error(transparent)] - InvalidWorkload(#[from] WorkloadError), - #[error("optimization horizon must be finite and strictly positive")] - InvalidHorizon, - #[error("workload entry index {index} is out of bounds for {entry_count} entries")] - InvalidWorkloadEntry { index: usize, entry_count: usize }, - #[error("a workload-demand binding must contain at least one entry")] - EmptyWorkloadDemand, - #[error("workload entry index {index} appears more than once in one demand binding")] - DuplicateWorkloadEntry { index: usize }, - #[error(transparent)] - InvalidPostAsapDAG(#[from] ExecutionDataStateError), -} - -#[derive(Debug, thiserror::Error)] -pub enum SummaryMaintenanceLifecycleAssemblyError { - #[error(transparent)] - AssembleDAG(#[from] RealizationError), - #[error(transparent)] - SummaryMaintenance(#[from] SummaryMaintenanceLifecyclePlanError), -} - -/// Failure while deriving workload-aware candidate costs before global -/// selection. -#[derive(Debug, thiserror::Error)] -pub enum SummaryMaintenanceLifecycleSelectionError { - #[error(transparent)] - Recurrence(#[from] RecurrenceError), - #[error(transparent)] - SummaryMaintenance(#[from] SummaryMaintenanceLifecyclePlanError), -} - -/// Every lifecycle alternative for each unique retained state of one fixed -/// root, before any lifecycle is chosen. -/// -/// Planner selection ([`plan_summary_maintenance_lifecycles`]) and a -/// deployment's explicit choice ([`Self::select`]) both finish from this value, -/// so they produce the same [`SummaryMaintenanceLifecyclePlan`] shape. -pub struct SummaryMaintenanceLifecycleCandidates<'a> { - /// Unselected plan: deployments carry alternatives but no guarantee or - /// window framework. - plan: SummaryMaintenanceLifecyclePlan, - components: Vec, - arrival: DataArrival, - required_accuracy: Vec, - cost_model: &'a dyn CostModel, - comparison_target: Option<&'a OperatorNode>, -} - -/// Why an explicit per-state lifecycle choice cannot be bound. -#[derive(Debug, thiserror::Error, PartialEq)] -pub enum SummaryMaintenanceLifecycleChoiceError { - #[error("summary {0:?} is not a deployment of this root")] - UnknownSummary(PhysicalASAPNodeId), - #[error("summary {0:?} is chosen more than once")] - DuplicateChoice(PhysicalASAPNodeId), - #[error("summary {0:?} has no chosen lifecycle")] - MissingChoice(PhysicalASAPNodeId), - #[error("chosen lifecycle is not an enumerated alternative of summary {0:?}")] - NotAnAlternative(PhysicalASAPNodeId), - #[error("chosen lifecycle of summary {post_asap_node_id:?} is rejected: {rejection:?}")] - Rejected { - post_asap_node_id: PhysicalASAPNodeId, - rejection: Option, - }, - #[error("summary states on one maintenance path have different evaluation schedules")] - IncompatibleEvaluationSchedules, - #[error("the cost model supplied no complete estimate for the chosen combination")] - NoCompleteEstimate, -} - -impl SummaryMaintenanceLifecycleCandidates<'_> { - /// One entry per unique retained state (see - /// [`SummaryMaintenanceLifecyclePlan::deployments`]), with every - /// alternative and its rejection; no lifecycle or window framework is - /// selected. - pub fn deployments(&self) -> &[SummaryMaintenanceDeployment] { - &self.plan.deployments - } - - /// Guarantee that binding `lifecycle` would attach under this workload's - /// data arrival, so a caller can price an alternative before choosing it. - pub fn guarantee( - &self, - lifecycle: &SummaryMaintenanceLifecycle, - ) -> SummaryMaintenanceLifecycleGuarantee { - lifecycle_guarantee(lifecycle, self.arrival) - } - - fn context(&self) -> CompleteCostContext<'_> { - CompleteCostContext { - root: &self.plan.root, - components: &self.components, - cost_model: self.cost_model, - comparison_target: self.comparison_target, - horizon: self.plan.horizon, - expected_reads: self.plan.expected_reads, - required_accuracy: &self.required_accuracy, - } - } - - fn finish( - mut self, - estimate: Option, - ) -> SummaryMaintenanceLifecyclePlan { - if let Some(estimate) = estimate { - self.plan.summary_total_cost = Some(estimate.cost); - self.plan.selected_window_implementation_id = estimate.physical_plan_id; - self.plan.window_accuracy_guarantee = estimate.window_accuracy_guarantee; - } - self.plan - } - - /// Planner's choice: the cheapest complete combination of eligible - /// alternatives. - fn select_cheapest(mut self) -> SummaryMaintenanceLifecyclePlan { - let estimate = select_complete_lifecycle_combination( - &self.plan.root, - &mut self.plan.deployments, - &self.components, - self.arrival, - self.cost_model, - self.comparison_target, - self.plan.horizon, - self.plan.expected_reads, - &self.required_accuracy, - ); - self.finish(estimate) - } - - /// Bind one caller-chosen lifecycle per summary state. Each choice must be - /// an alternative Planner itself could select; the complete estimate is - /// then obtained exactly as for Planner selection, so window framework and - /// cost are the model's and unknown cost is never replaced by zero. - pub fn select( - mut self, - choices: &[(PhysicalASAPNodeId, SummaryMaintenanceLifecycle)], - ) -> Result { - use SummaryMaintenanceLifecycleChoiceError as E; - let deployments = &self.plan.deployments; - let mut chosen: Vec> = - vec![None; deployments.len()]; - let context = self.context(); - for (id, lifecycle) in choices { - let index = deployments - .iter() - .position(|deployment| deployment.post_asap_node_id == *id) - .ok_or(E::UnknownSummary(*id))?; - if chosen[index].is_some() { - return Err(E::DuplicateChoice(*id)); - } - let alternative = deployments[index] - .alternatives - .iter() - .find(|alternative| alternative.summary_maintenance_lifecycle == *lifecycle) - .ok_or(E::NotAnAlternative(*id))?; - if !context.eligible(alternative) { - return Err(E::Rejected { - post_asap_node_id: *id, - rejection: alternative.rejection.clone(), - }); - } - chosen[index] = Some(alternative); - } - let selected = chosen - .into_iter() - .enumerate() - .map(|(index, alternative)| { - let alternative = - alternative.ok_or(E::MissingChoice(deployments[index].post_asap_node_id))?; - Ok(( - index, - lifecycle_guarantee(&alternative.summary_maintenance_lifecycle, self.arrival), - // Reached only for costed alternatives or when the - // complete hook is authoritative, matching Planner search. - alternative.total_cost.unwrap_or(Cost::ZERO), - )) - }) - .collect::, E>>()?; - if selected.is_empty() { - return Ok(self.finish(None)); - } - if !context.schedules_compatible(&selected) { - return Err(E::IncompatibleEvaluationSchedules); - } - let estimate = context - .estimate(deployments, &selected) - .ok_or(E::NoCompleteEstimate)?; - let guarantees = selected - .into_iter() - .map(|(index, guarantee, _)| (index, guarantee)) - .collect(); - apply_selection(&mut self.plan.deployments, guarantees, &estimate); - Ok(self.finish(Some(estimate))) - } -} - -/// Workload-wide evidence derived specifically for summary-maintenance -/// lifecycle enumeration and costing. -/// -/// This is not another workload input model. [`QueryWorkload`] and its -/// normalized entries remain the source of truth. Unlike one -/// [`asap_types::workload::QueryWorkloadEntry`], these values aggregate all -/// entries at a particular planning time and optional horizon. It also cannot -/// reuse [`crate::recurrence::RecurrenceProfile`], which describes recurrence -/// for one candidate target and counts consumers rather than invocations. -#[derive(Debug)] -struct SummaryMaintenanceWorkloadFacts { - required_accuracy: Vec, - /// Total one-time and recurring reads inside the horizon. `None` means a - /// recurrence or horizon was unknown, not zero reads. - reads: Option, - /// Sum of declared invocations across all one-time workload entries. - one_time_invocations: u64, - /// Sum of usable recurring query rates in evaluations per second. - evaluation_rate: Option, - /// Fresh workload-level ingestion rate in updates per second. - update_rate: Option, - /// Whether the workload's source data is static, arriving, mixed, or - /// unknown. - arrival: DataArrival, - /// Earliest known activation and latest scheduled execution across - /// predictable one-time entries. `None` means no valid preparation window. - prepared_window: Option<(TimestampMs, TimestampMs)>, - /// Whether every bound consumer is a predictable one-time query suitable - /// for prepared state. - prepared_eligible: bool, - /// Whether maintaining the selected moving time scope requires deleting - /// expired input from summary state. - requires_deletion: bool, -} - -/// Validate a materialized plan, enumerate lifecycle alternatives for each -/// unique summary state, and select the cheapest legal alternative whose cost -/// is fully known. -pub fn plan_summary_maintenance_lifecycles( - root: Rc, - demand: WorkloadDemand<'_>, - now_ms: u64, - horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, - cost_model: &dyn CostModel, -) -> Result { - Ok(enumerate_summary_maintenance_lifecycles( - root, - demand, - now_ms, - horizon, - capabilities, - cost_model, - )? - .select_cheapest()) -} - -/// Validate a materialized plan and enumerate lifecycle alternatives for each -/// unique summary state without choosing one. A deployment that prices the -/// alternatives itself binds its choice with -/// [`SummaryMaintenanceLifecycleCandidates::select`]. -pub fn enumerate_summary_maintenance_lifecycles<'a>( - root: Rc, - demand: WorkloadDemand<'_>, - now_ms: u64, - horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, - cost_model: &'a dyn CostModel, -) -> Result, SummaryMaintenanceLifecyclePlanError> { - enumerate_with_profile( - root, - demand, - now_ms, - horizon, - capabilities, - cost_model, - None, - None, - ) -} - -/// Internal candidate-costing form. The workload binding supplies temporal -/// eligibility and data-arrival facts; `profile` supplies effective uses after -/// DAG path multiplicity has been propagated by `CandidateLogicalASAPDAGs`. -#[expect(clippy::too_many_arguments, reason = "internal bound planning context")] -fn enumerate_with_profile<'a>( - root: Rc, - demand: WorkloadDemand<'_>, - now_ms: u64, - horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, - cost_model: &'a dyn CostModel, - profile: Option, - comparison_target: Option<&'a OperatorNode>, -) -> Result, SummaryMaintenanceLifecyclePlanError> { - demand.workload.validate()?; - if let Some(data) = demand.data_workload { - data.validate()?; - } - if horizon.is_some_and(|h| !h.0.is_finite() || h.0 <= 0.0) { - return Err(SummaryMaintenanceLifecyclePlanError::InvalidHorizon); - } - let mut facts = workload_facts( - demand.workload, - demand.data_workload, - demand.entry_indices, - now_ms, - horizon, - )?; - if let Some(profile) = profile { - facts.one_time_invocations = u64::try_from(profile.one_shot_consumers).unwrap_or(u64::MAX); - facts.evaluation_rate = profile.evaluation_rate; - facts.update_rate = profile.update_rate; - facts.reads = match (profile.evaluation_rate, horizon) { - (Some(rate), Some(horizon)) => { - Some(profile.one_shot_consumers as f64 + rate.0 * horizon.0) - } - (Some(_), None) => None, - (None, _) if profile.one_shot_consumers > 0 => Some(profile.one_shot_consumers as f64), - // Preserve unknown recurrence from the normalized workload. An - // empty profile does not prove that the target is never read. - (None, _) => facts.reads, - }; - } - let mut summaries = Vec::new(); - collect_states( - &root, - &mut HashSet::new(), - &mut summaries, - StateKind::SummaryAgg, - ); - summaries.extend(standalone_populations(&root)); - let mut timing_memo = TimingMemo::new(); - let timed_root = apply_lifecycle_timings( - &root, - &LifecycleAssignment::default_maintained(), - &mut timing_memo, - )?; - let node_ids = compile_physical_asap_dag_with_node_ids(&timed_root)?.node_ids; - let components = summary_state_components(&summaries); - let deployments: Vec = summaries - .into_iter() - .map(|summary| { - let capabilities = if OperatorNode::reachable(&summary).iter().any(|node| { - matches!( - node.non_asap(), - Some(asap_types::ir::NonASAPOp::BinaryOp { .. }) - ) && asap_types::ir::timing::validate_default(node, ExecutionTiming::IngestionTime) - .is_err() - }) { - SummaryMaintenanceLifecycleCapabilities { - supports_ephemeral: capabilities.supports_ephemeral, - supports_prepared: false, - supports_shared: false, - supports_continuously_maintained: false, - } - } else { - capabilities - }; - let alternatives = alternatives_for( - &facts, - horizon, - capabilities, - cost_model.summary_maintenance_capabilities(&summary), - cost_model.summary_maintenance_lifecycle_cost_inputs_for_horizon(&summary, horizon), - ); - SummaryMaintenanceDeployment { - post_asap_node_id: timing_memo - .timed(&summary) - .and_then(|timed| node_ids.node_id(timed)) - .expect("collected summary belongs to the compiled DAG"), - summary, - summary_maintenance_lifecycle_guarantee: None, - selected_window_framework: None, - alternatives, - } - }) - .collect(); - let selected_raw_recompute = !root.contains_asap(); - Ok(SummaryMaintenanceLifecycleCandidates { - plan: SummaryMaintenanceLifecyclePlan { - root, - deployments, - horizon, - evaluation_rate: facts.evaluation_rate, - update_rate: facts.update_rate, - expected_reads: facts.reads, - selected_raw_recompute, - selected_window_implementation_id: None, - summary_total_cost: None, - window_accuracy_guarantee: None, - raw_recompute_total_cost: None, - }, - components, - arrival: facts.arrival, - required_accuracy: facts.required_accuracy, - cost_model, - comparison_target, - }) -} - -/// Rank semantic summary siblings using the cheapest legal -/// summary-maintenance lifecycle for each candidate before final global -/// selection. The candidate space stays compact; only cost overrides are -/// attached, so shared `Rc` identity and exact-composition commitments remain -/// the responsibility of `GlobalSelection`. -/// -/// Summary candidates of different targets whose outermost `SummaryAgg` is -/// structurally identical (for example p50 and p99 over one KLL) form a class. -/// When [`shared_state_cost`] can cost that state once against the union of -/// the targets' entries, each member is offered an equal split of it instead -/// of its independent cost. If selection then leaves any member of a class on -/// another choice, that class reverts to independent costs and selection runs -/// once more. -pub fn global_selection_with_summary_maintenance_lifecycles<'a, Id>( - space: &'a CandidateLogicalASAPDAGs, - demand: WorkloadDemand<'_>, - now_ms: u64, - horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, - cost_model: &dyn CostModel, -) -> Result, SummaryMaintenanceLifecycleSelectionError> { - let WorkloadDemand { - workload, - data_workload, - entry_indices: root_workload_entries, - } = demand; - let profiles = space.recurrence_profiles_from_workload( - workload, - data_workload, - root_workload_entries, - now_ms, - horizon, - )?; - let bindings = space.workload_entries_by_target(workload, root_workload_entries)?; - let mut costs = CandidateCostOverrides::default(); - // Finalized summary candidates, as sharing-class members. - let mut members = Vec::new(); - for group in space.target_subdag_candidates() { - let Some(entry_indices) = bindings.get(&Rc::as_ptr(&group.target)) else { - continue; - }; - for candidate in &group.candidates { - // Only summary realizations carry a maintenance lifecycle; a - // logical rewrite or CSE share/recompute candidate does not. - let Replacement::SubDAG(summary) = &candidate.replacement else { - continue; - }; - if candidate.provenance != ReplacementProvenance::SummaryRealization { - continue; - } - costs.finalize_target(&group.target); - let plan = enumerate_with_profile( - Rc::clone(summary), - WorkloadDemand { - workload, - data_workload, - entry_indices, - }, - now_ms, - horizon, - capabilities, - cost_model, - Some(profiles.for_target(&group.target)), - Some(&group.target), - )? - .select_cheapest(); - let raw = plan - .expected_reads - .and_then(|reads| cost_model.raw_query_recompute_total_cost(&group.target, reads)); - // Final comparison is atomic: without the raw side, no summary - // override is published even when that summary alone is costed. - if let Some(raw) = raw { - costs.insert_raw(&group.target, raw); - if !plan.deployments.is_empty() { - if let Some(total) = plan.summary_total_cost { - costs.insert(&group.target, candidate, total); - } - members.push((group, candidate, Rc::clone(summary))); - } - } - } - } - - // Intern every member once; members whose outermost state (the - // `SummaryAgg` every other state of the candidate feeds) interns to the - // same node share it. Classes are kept in first-member order. - let interned = share_common_sub_dags( - members - .iter() - .enumerate() - .map(|(index, (_, _, summary))| (index, Rc::clone(summary))) - .collect(), - ); - let mut classes: Vec<(Rc, Vec)> = Vec::new(); - for (index, root) in interned { - let states = summary_states(&root); - let Some(state) = states - .iter() - .find(|state| summary_states(state).len() == states.len()) - else { - continue; - }; - if !standalone_populations(&root).is_empty() { - continue; - } - match classes.iter_mut().find(|(s, _)| Rc::ptr_eq(s, state)) { - Some((_, class)) => class.push(index), - None => classes.push((Rc::clone(state), vec![index])), - } - } - let mut shared = Vec::new(); - for (state, class) in classes { - let mut targets: Vec<&Rc> = Vec::new(); - for &index in &class { - let target = &members[index].0.target; - if !targets.iter().any(|t| Rc::ptr_eq(t, target)) { - targets.push(target); - } - } - if targets.len() < 2 { - continue; - } - let mut entries: Vec = targets - .iter() - .flat_map(|target| bindings[&Rc::as_ptr(target)].iter().copied()) - .collect(); - entries.sort_unstable(); - entries.dedup(); - let Some(cost) = shared_state_cost( - &state, - WorkloadDemand { - workload, - data_workload, - entry_indices: &entries, - }, - now_ms, - horizon, - capabilities, - cost_model, - )? - else { - continue; - }; - shared.push((class, Cost(cost.0 / targets.len() as f64))); - } - - let with_shared = |kept: &[(Vec, Cost)]| { - let mut costs = costs.clone(); - for (class, split) in kept { - for &index in class { - let (group, candidate, _) = &members[index]; - costs.insert(&group.target, candidate, *split); - } - } - costs - }; - let selection = space.global_selection_with_candidate_costs( - cost_model, - &profiles, - horizon, - &with_shared(&shared), - )?; - let before = shared.len(); - shared.retain(|(class, _)| { - class.iter().all(|&index| { - let target = &members[index].0.target; - let chosen = selection.for_target(target).and_then(|s| s.chosen); - class.iter().any(|&other| { - Rc::ptr_eq(&members[other].0.target, target) - && chosen.is_some_and(|chosen| std::ptr::eq(chosen, members[other].1)) - }) - }) - }); - if shared.len() == before { - return Ok(selection); - } - Ok(space.global_selection_with_candidate_costs( - cost_model, - &profiles, - horizon, - &with_shared(&shared), - )?) -} - -/// Cost of one `SummaryAgg` state maintained once for every entry in -/// `demand`, or `None` when no lifecycle alternative is selectable for it. -/// No comparison target is supplied: the state serves several queries. -pub(crate) fn shared_state_cost( - state: &Rc, - demand: WorkloadDemand<'_>, - now_ms: u64, - horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, - cost_model: &dyn CostModel, -) -> Result, SummaryMaintenanceLifecyclePlanError> { - Ok(enumerate_with_profile( - Rc::clone(state), - demand, - now_ms, - horizon, - capabilities, - cost_model, - None, - None, - )? - .select_cheapest() - .summary_total_cost) -} - -/// Every unique `SummaryAgg` reachable from `root`. -pub(crate) fn summary_states(root: &Rc) -> Vec> { - let mut states = Vec::new(); - collect_states( - root, - &mut HashSet::new(), - &mut states, - StateKind::SummaryAgg, - ); - states -} - -/// Assemble a globally selected phase-valid DAG and attach workload-aware -/// summary maintenance decisions. This does not create or maintain runtime state. -pub fn assemble_selected_dag_with_summary_maintenance_lifecycles( - selection: &GlobalSelection<'_>, - target: &Rc, - demand: WorkloadDemand<'_>, - now_ms: u64, - horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, - cost_model: &dyn CostModel, -) -> Result, SummaryMaintenanceLifecycleAssemblyError> { - selection - .assemble_selected_dag(target)? - .map(|root| { - plan_assembled_dag( - root, - target, - demand, - now_ms, - horizon, - capabilities, - cost_model, - ) - }) - .transpose() -} - -/// The lifecycle half of -/// [`assemble_selected_dag_with_summary_maintenance_lifecycles`], for a root -/// the caller already assembled (and possibly interned across queries). -pub(crate) fn plan_assembled_dag( - root: Rc, - target: &Rc, - demand: WorkloadDemand<'_>, - now_ms: u64, - horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, - cost_model: &dyn CostModel, -) -> Result { - let mut plan = enumerate_with_profile( - root, - demand, - now_ms, - horizon, - capabilities, - cost_model, - None, - Some(target), - )? - .select_cheapest(); - plan.raw_recompute_total_cost = plan - .expected_reads - .and_then(|reads| cost_model.raw_query_recompute_total_cost(target, reads)); - if !plan.selected_raw_recompute - && plan.raw_recompute_total_cost.is_none_or(|raw| { - plan.summary_total_cost - .is_none_or(|summary| raw.0 <= summary.0) - }) - { - plan.root = crate::replacement::retain_exact(target)?; - plan.deployments.clear(); - plan.selected_raw_recompute = true; - plan.selected_window_implementation_id = None; - plan.summary_total_cost = None; - plan.window_accuracy_guarantee = None; - } - Ok(plan) -} - -fn workload_facts( - workload: &QueryWorkload, - data_workload: Option<&DataWorkload>, - workload_entry_indices: &[usize], - now_ms: u64, - horizon: Option, -) -> Result { - let mut one_time_invocations = 0u64; - let mut recurring_reads = 0.0; - let mut recurring_known = true; - let mut evaluation_rate = 0.0; - let mut has_evaluation_rate = false; - let mut prepared_start: Option = None; - let mut prepared_end: Option = None; - let mut prepared_eligible = true; - let mut requires_deletion = false; - let mut required_accuracy = Vec::new(); - - let entries: Vec<_> = workload.entries().collect(); - if workload_entry_indices.is_empty() { - return Err(SummaryMaintenanceLifecyclePlanError::EmptyWorkloadDemand); - } - let mut seen_indices = HashSet::new(); - for &index in workload_entry_indices { - if !seen_indices.insert(index) { - return Err(SummaryMaintenanceLifecyclePlanError::DuplicateWorkloadEntry { index }); - } - let entry = entries.get(index).ok_or( - SummaryMaintenanceLifecyclePlanError::InvalidWorkloadEntry { - index, - entry_count: entries.len(), - }, - )?; - required_accuracy.push(entry.requirements.accuracy.target()); - requires_deletion |= entry.time_selection.lookback.is_some() - && entry.time_selection.as_of.is_none() - && matches!( - entry.time_selection.scope, - asap_types::workload::QueryTimeScope::RealTime - | asap_types::workload::QueryTimeScope::Mixed - ); - match &entry.recurrence { - QueryRecurrence::OneTime { - invocations, - execute_at, - } => { - one_time_invocations = one_time_invocations.saturating_add(*invocations); - let covered = if let ( - Predictability::Predictable { - known_at: Some(known), - }, - Some(execute), - ) = (&entry.predictability, execute_at) - { - if known < execute && now_ms < execute.0 { - let activate = TimestampMs(known.0.max(now_ms)); - prepared_start = - Some(prepared_start.map_or(activate, |old| old.min(activate))); - prepared_end = Some(prepared_end.map_or(*execute, |old| old.max(*execute))); - true - } else { - false - } - } else { - false - }; - prepared_eligible &= covered; - } - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) - | QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { interval, .. }) => { - prepared_eligible = false; - let rate = 1000.0 / f64::from(interval.0); - evaluation_rate += rate; - has_evaluation_rate = true; - if let Some(h) = horizon { - recurring_reads += h.0 * rate; - } else { - recurring_known = false; - } - } - QueryRecurrence::Repeated(RepeatedDemand::Scheduled(schedule)) => { - prepared_eligible = false; - if let Some(h) = horizon { - let end_ms = now_ms.saturating_add((h.0 * 1000.0) as u64); - let reads_in_horizon = schedule - .iter() - .filter(|at| at.0 >= now_ms && at.0 <= end_ms) - .count() as f64; - recurring_reads += reads_in_horizon; - evaluation_rate += reads_in_horizon / h.0; - has_evaluation_rate = true; - } else { - recurring_known = false; - } - } - QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { - prepared_eligible = false; - if !estimate.is_fresh_at(now_ms) { - recurring_known = false; - continue; - } - let rate = estimate.expected_rate.0; - evaluation_rate += rate; - has_evaluation_rate = true; - if let Some(h) = horizon { - recurring_reads += h.0 * rate; - } else { - recurring_known = false; - } - } - QueryRecurrence::Unknown => { - prepared_eligible = false; - recurring_known = false; - } - } - } - - let data = data_workload; - let arrival = data.map_or(DataArrival::Unknown, |data| data.arrival); - let update_rate = data - .and_then(|data| data.ingestion_rate.value_at(now_ms)) - .map(|rate| UpdateRate(rate.0)); - let reads = if let Some(horizon) = horizon { - let horizon_ms = horizon.0 * 1_000.0; - if !horizon_ms.is_finite() - || horizon_ms <= 0.0 - || horizon_ms > u64::MAX as f64 - || horizon_ms.fract() != 0.0 - { - None - } else { - workload_entry_indices - .iter() - .try_fold(0_u64, |total, index| { - let entry = entries.get(*index)?; - match evaluations_in_horizon(&entry.recurrence, now_ms, horizon_ms as u64) { - Ok(count) => total.checked_add(count), - Err(AnalyticalCostError::NoEvaluationsInHorizon) => Some(total), - Err(_) => None, - } - }) - .map(|count| count as f64) - } - } else { - recurring_known.then_some(one_time_invocations as f64 + recurring_reads) - }; - Ok(SummaryMaintenanceWorkloadFacts { - required_accuracy, - reads, - one_time_invocations, - evaluation_rate: has_evaluation_rate.then_some(EvaluationRate(evaluation_rate)), - update_rate, - arrival, - prepared_window: prepared_start.zip(prepared_end), - prepared_eligible, - requires_deletion, - }) -} - -fn alternatives_for( - facts: &SummaryMaintenanceWorkloadFacts, - horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, - summary_capabilities: SummaryMaintenanceCapabilities, - costs: SummaryMaintenanceLifecycleCostInputs, -) -> Vec { - let alternatives = vec![ - ephemeral(facts, capabilities, &costs), - prepared(facts, capabilities, summary_capabilities, &costs), - shared(facts, horizon, capabilities, summary_capabilities, &costs), - continuous(facts, horizon, capabilities, summary_capabilities, &costs), - ]; - alternatives -} - -fn ephemeral( - facts: &SummaryMaintenanceWorkloadFacts, - capabilities: SummaryMaintenanceLifecycleCapabilities, - costs: &SummaryMaintenanceLifecycleCostInputs, -) -> SummaryMaintenanceLifecycleAlternative { - let lifecycle = SummaryMaintenanceLifecycle::Ephemeral; - if !capabilities.supports_ephemeral { - return rejected( - lifecycle, - SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, - ); - } - let total_cost = zip_costs(&[ - costs.build_cost, - costs.summary_read_cost, - costs.retirement_cost, - ]) - .zip(facts.reads) - .map(|(per_read, reads)| Cost(per_read * reads)); - costed_or_unknown( - lifecycle, - total_cost, - vec!["state is rebuilt per invocation".into()], - ) -} - -fn prepared( - facts: &SummaryMaintenanceWorkloadFacts, - capabilities: SummaryMaintenanceLifecycleCapabilities, - summary_capabilities: SummaryMaintenanceCapabilities, - costs: &SummaryMaintenanceLifecycleCostInputs, -) -> SummaryMaintenanceLifecycleAlternative { - if !facts.prepared_eligible { - return rejected( - SummaryMaintenanceLifecycle::Prepared { - activate_at: TimestampMs(0), - retire_at: TimestampMs(0), - }, - SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery, - ); - } - let Some((activate_at, retire_at)) = facts.prepared_window else { - return rejected( - SummaryMaintenanceLifecycle::Prepared { - activate_at: TimestampMs(0), - retire_at: TimestampMs(0), - }, - SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery, - ); - }; - let lifecycle = SummaryMaintenanceLifecycle::Prepared { - activate_at, - retire_at, - }; - if !capabilities.supports_prepared { - return rejected( - lifecycle, - SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, - ); - } - if let Some(rejection) = maintenance_capability_rejection(facts, summary_capabilities) { - return rejected(lifecycle, rejection); - } - let seconds = retire_at.0.saturating_sub(activate_at.0) as f64 / 1000.0; - let maintenance = maintenance_cost(facts, costs, seconds); - let total_cost = match ( - costs.build_cost, - costs.summary_read_cost, - costs.retention_cost_rate, - costs.retirement_cost, - maintenance, - ) { - (Some(build), Some(read), Some(retention), Some(retire), Some(maintenance)) => Some(Cost( - build.0 - + read.0 * facts.one_time_invocations as f64 - + retention.0 * seconds - + retire.0 - + maintenance, - )), - _ => None, - }; - costed_or_unknown( - lifecycle, - total_cost, - vec!["activation and retirement come from the declared schedule".into()], - ) -} - -fn shared( - facts: &SummaryMaintenanceWorkloadFacts, - horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, - summary_capabilities: SummaryMaintenanceCapabilities, - costs: &SummaryMaintenanceLifecycleCostInputs, -) -> SummaryMaintenanceLifecycleAlternative { - let lifecycle = SummaryMaintenanceLifecycle::Shared { - retention: asap_types::workload::DurationMs(horizon.map_or(0, |h| (h.0 * 1000.0) as u64)), - }; - if !capabilities.supports_shared { - return rejected( - lifecycle, - SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, - ); - } - if let Some(rejection) = maintenance_capability_rejection(facts, summary_capabilities) { - return rejected(lifecycle, rejection); - } - if facts.reads.is_none_or(|reads| reads <= 1.0) { - return rejected( - lifecycle, - SummaryMaintenanceLifecycleRejection::RequiresMultipleReads, - ); - } - let Some(horizon) = horizon else { - return rejected( - lifecycle, - SummaryMaintenanceLifecycleRejection::RequiresHorizon, - ); - }; - let total_cost = retained_cost(facts, costs, horizon.0); - costed_or_unknown( - lifecycle, - total_cost, - vec!["one state is shared across reads".into()], - ) -} - -fn continuous( - facts: &SummaryMaintenanceWorkloadFacts, - horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, - summary_capabilities: SummaryMaintenanceCapabilities, - costs: &SummaryMaintenanceLifecycleCostInputs, -) -> SummaryMaintenanceLifecycleAlternative { - let lifecycle = SummaryMaintenanceLifecycle::ContinuouslyMaintained; - if !capabilities.supports_continuously_maintained { - return rejected( - lifecycle, - SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, - ); - } - if !matches!( - facts.arrival, - DataArrival::ContinuouslyIngesting | DataArrival::Mixed - ) { - return rejected( - lifecycle, - SummaryMaintenanceLifecycleRejection::RequiresContinuousData, - ); - } - if facts.update_rate.is_none() { - return rejected( - lifecycle, - SummaryMaintenanceLifecycleRejection::MissingOrStaleIngestionRate, - ); - } - if let Some(rejection) = maintenance_capability_rejection(facts, summary_capabilities) { - return rejected(lifecycle, rejection); - } - let Some(horizon) = horizon else { - return rejected( - lifecycle, - SummaryMaintenanceLifecycleRejection::RequiresHorizon, - ); - }; - let total_cost = retained_cost(facts, costs, horizon.0); - costed_or_unknown( - lifecycle, - total_cost, - vec!["updates are applied for the optimization horizon".into()], - ) -} - -fn maintenance_capability_rejection( - facts: &SummaryMaintenanceWorkloadFacts, - capabilities: SummaryMaintenanceCapabilities, -) -> Option { - if matches!( - facts.arrival, - DataArrival::ContinuouslyIngesting | DataArrival::Mixed - ) && !capabilities.incremental_update - { - Some(SummaryMaintenanceLifecycleRejection::SummaryDoesNotSupportIncrementalUpdates) - } else if matches!( - facts.arrival, - DataArrival::ContinuouslyIngesting | DataArrival::Mixed - ) && facts.requires_deletion - && !capabilities.delete - { - Some(SummaryMaintenanceLifecycleRejection::SummaryDoesNotSupportDeletion) - } else { - None - } -} - -fn retained_cost( - facts: &SummaryMaintenanceWorkloadFacts, - costs: &SummaryMaintenanceLifecycleCostInputs, - seconds: f64, -) -> Option { - let reads = facts.reads?; - let maintenance = maintenance_cost(facts, costs, seconds)?; - Some(Cost( - costs.build_cost?.0 - + maintenance - + reads * costs.summary_read_cost?.0 - + seconds * costs.retention_cost_rate?.0 - + costs.retirement_cost?.0, - )) -} - -fn maintenance_cost( - facts: &SummaryMaintenanceWorkloadFacts, - costs: &SummaryMaintenanceLifecycleCostInputs, - seconds: f64, -) -> Option { - match facts.arrival { - DataArrival::AtRest => Some(0.0), - DataArrival::ContinuouslyIngesting | DataArrival::Mixed => { - Some(seconds * facts.update_rate?.0 * costs.maintenance_cost_per_update?.0) - } - DataArrival::Unknown => None, - } -} - -fn zip_costs(costs: &[Option]) -> Option { - costs - .iter() - .try_fold(0.0, |sum, cost| Some(sum + cost.as_ref()?.0)) -} - -fn costed_or_unknown( - summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, - total_cost: Option, - assumptions: Vec, -) -> SummaryMaintenanceLifecycleAlternative { - SummaryMaintenanceLifecycleAlternative { - summary_maintenance_lifecycle, - total_cost, - rejection: total_cost - .is_none() - .then_some(SummaryMaintenanceLifecycleRejection::MissingCostEvidence), - assumptions, - } -} - -fn rejected( - summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, - rejection: SummaryMaintenanceLifecycleRejection, -) -> SummaryMaintenanceLifecycleAlternative { - SummaryMaintenanceLifecycleAlternative { - summary_maintenance_lifecycle, - total_cost: None, - rejection: Some(rejection), - assumptions: Vec::new(), - } -} - -#[derive(Clone, Copy, PartialEq)] -enum StateKind { - SummaryAgg, - Population, -} - -/// Collect every unique node of `kind` reachable from `node`. -fn collect_states( - node: &Rc, - seen: &mut HashSet<*const OperatorNode>, - output: &mut Vec>, - kind: StateKind, -) { - if !seen.insert(Rc::as_ptr(node)) { - return; - } - if matches!( - (&node.operator, kind), - ( - Operator::ASAP(ASAPOp::SummaryAgg { .. }), - StateKind::SummaryAgg - ) | ( - Operator::ASAP(ASAPOp::MaintainPopulation { .. }), - StateKind::Population - ) - ) { - output.push(Rc::clone(node)); - } - for child in node.children() { - collect_states(child, seen, output, kind); - } -} - -/// Maintained populations that are not an input of any `SummaryAgg`. A -/// population feeding summary state is on that state's maintenance path, so -/// that state's lifecycle times it, even when a evaluation also reads it directly. -fn standalone_populations(root: &Rc) -> Vec> { - let mut summaries = Vec::new(); - collect_states( - root, - &mut HashSet::new(), - &mut summaries, - StateKind::SummaryAgg, - ); - let mut nested = Vec::new(); - let mut seen = HashSet::new(); - for summary in &summaries { - collect_states(summary, &mut seen, &mut nested, StateKind::Population); - } - let nested: HashSet<_> = nested.iter().map(Rc::as_ptr).collect(); - let mut populations = Vec::new(); - collect_states( - root, - &mut HashSet::new(), - &mut populations, - StateKind::Population, - ); - populations.retain(|population| !nested.contains(&Rc::as_ptr(population))); - populations -} - -pub(crate) fn evaluation_schedule( - lifecycle: &SummaryMaintenanceLifecycle, - arrival: DataArrival, -) -> EvaluationSchedule { - match lifecycle { - SummaryMaintenanceLifecycle::Ephemeral => EvaluationSchedule::OneShot, - SummaryMaintenanceLifecycle::Prepared { .. } - | SummaryMaintenanceLifecycle::Shared { .. } - if matches!( - arrival, - DataArrival::ContinuouslyIngesting | DataArrival::Mixed - ) => - { - EvaluationSchedule::PerUpdate - } - SummaryMaintenanceLifecycle::Prepared { .. } => EvaluationSchedule::OneShot, - SummaryMaintenanceLifecycle::Shared { .. } => EvaluationSchedule::OnRead, - SummaryMaintenanceLifecycle::ContinuouslyMaintained => EvaluationSchedule::PerUpdate, - } -} - -/// Summary states composed on one maintenance path must be produced on the -/// same schedule. Return a component id for each collected state. -fn summary_state_components(summaries: &[Rc]) -> Vec { - let indices: HashMap<_, _> = summaries - .iter() - .enumerate() - .map(|(index, summary)| (Rc::as_ptr(summary), index)) - .collect(); - let mut parents: Vec<_> = (0..summaries.len()).collect(); - - fn find(parents: &mut [usize], index: usize) -> usize { - if parents[index] != index { - parents[index] = find(parents, parents[index]); - } - parents[index] - } - - for (parent_index, summary) in summaries.iter().enumerate() { - let Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) = &summary.operator else { - continue; - }; - if !matches!( - child.operator, - Operator::ASAP( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryJoin { .. } - | ASAPOp::SummarySubtract { .. } - | ASAPOp::SummaryDelete { .. } - | ASAPOp::SummaryMerge { .. } - ) - ) { - continue; - } - let mut descendants = Vec::new(); - collect_states( - child, - &mut HashSet::new(), - &mut descendants, - StateKind::SummaryAgg, - ); - for descendant in descendants { - let child_index = indices[&Rc::as_ptr(&descendant)]; - let parent_root = find(&mut parents, parent_index); - let child_root = find(&mut parents, child_index); - parents[child_root] = parent_root; - } - } - (0..parents.len()) - .map(|index| find(&mut parents, index)) - .collect() -} - -/// Inputs shared by every complete lifecycle-combination evaluation of one -/// root, whether Planner searches combinations or a caller supplies one. -struct CompleteCostContext<'a> { - root: &'a OperatorNode, - components: &'a [usize], - cost_model: &'a dyn CostModel, - comparison_target: Option<&'a OperatorNode>, - horizon: Option, - expected_reads: Option, - required_accuracy: &'a [AccuracyTarget], -} - -impl CompleteCostContext<'_> { - /// Planner's own admission rule for one alternative. Uncosted alternatives - /// are admitted only when the complete-candidate hook is authoritative. - fn eligible(&self, alternative: &SummaryMaintenanceLifecycleAlternative) -> bool { - alternative.selectable() - || (self - .cost_model - .complete_summary_candidate_estimate_covers_lifecycle_costs() - && alternative.rejection - == Some(SummaryMaintenanceLifecycleRejection::MissingCostEvidence)) - } - - /// `selected` holds one entry per deployment, in deployment order. - fn schedules_compatible( - &self, - selected: &[(usize, SummaryMaintenanceLifecycleGuarantee, Cost)], - ) -> bool { - !selected.iter().enumerate().any(|(left, (_, a, _))| { - selected.iter().enumerate().any(|(right, (_, b, _))| { - self.components[left] == self.components[right] - && a.evaluation_schedule != b.evaluation_schedule - }) - }) - } - - fn estimate( - &self, - deployments: &[SummaryMaintenanceDeployment], - selected: &[(usize, SummaryMaintenanceLifecycleGuarantee, Cost)], - ) -> Option { - if !self.schedules_compatible(selected) { - return None; - } - let costed: Vec<_> = selected - .iter() - .map(|(index, guarantee, cost)| CostedSummaryDeployment { - summary: &deployments[*index].summary, - guarantee, - selected_cost: *cost, - }) - .collect(); - let estimate = self.cost_model.complete_summary_candidate_estimate( - self.root, - self.comparison_target, - &costed, - self.horizon, - self.expected_reads, - self.required_accuracy, - )?; - (estimate.window_frameworks.len() == deployments.len()).then_some(estimate) - } -} - -fn lifecycle_guarantee( - lifecycle: &SummaryMaintenanceLifecycle, - arrival: DataArrival, -) -> SummaryMaintenanceLifecycleGuarantee { - SummaryMaintenanceLifecycleGuarantee { - summary_maintenance_mode: maintenance_mode(lifecycle, arrival), - evaluation_schedule: evaluation_schedule(lifecycle, arrival), - summary_maintenance_lifecycle: lifecycle.clone(), - output_representation: OutputRepresentation::SummaryState, - } -} - -fn apply_selection( - deployments: &mut [SummaryMaintenanceDeployment], - guarantees: Vec<(usize, SummaryMaintenanceLifecycleGuarantee)>, - estimate: &CompleteSummaryCandidateEstimate, -) { - for (index, guarantee) in guarantees { - deployments[index].summary_maintenance_lifecycle_guarantee = Some(guarantee); - } - for (deployment, framework) in deployments - .iter_mut() - .zip(estimate.window_frameworks.iter().cloned()) - { - deployment.selected_window_framework = framework; - } -} - -#[expect(clippy::too_many_arguments, reason = "complete combination context")] -fn select_complete_lifecycle_combination( - root: &OperatorNode, - deployments: &mut [SummaryMaintenanceDeployment], - components: &[usize], - arrival: DataArrival, - cost_model: &dyn CostModel, - comparison_target: Option<&OperatorNode>, - horizon: Option, - expected_reads: Option, - required_accuracy: &[AccuracyTarget], -) -> Option { - const MAX_COMPLETE_LIFECYCLE_COMBINATIONS: usize = 4_096; - if deployments.is_empty() { - return None; - } - let context = CompleteCostContext { - root, - components, - cost_model, - comparison_target, - horizon, - expected_reads, - required_accuracy, - }; - // The whole-candidate hook is intentionally arbitrary, so partial costs - // cannot soundly prune the search. Bound exhaustive enumeration and fail - // closed instead of allowing an adversarial DAG to consume exponential - // planner time. - let combinations = deployments - .iter() - .try_fold(1_usize, |product, deployment| { - let selectable = deployment - .alternatives - .iter() - .filter(|alternative| context.eligible(alternative)) - .count(); - product.checked_mul(selectable) - })?; - if combinations == 0 || combinations > MAX_COMPLETE_LIFECYCLE_COMBINATIONS { - return None; - } - type Best = Option<( - CompleteSummaryCandidateEstimate, - Vec<(usize, SummaryMaintenanceLifecycleGuarantee)>, - )>; - fn visit( - index: usize, - context: &CompleteCostContext<'_>, - deployments: &[SummaryMaintenanceDeployment], - arrival: DataArrival, - selected: &mut Vec<(usize, SummaryMaintenanceLifecycleGuarantee, Cost)>, - best: &mut Best, - ) { - if index == deployments.len() { - let Some(estimate) = context.estimate(deployments, selected) else { - return; - }; - if best - .as_ref() - .is_none_or(|(best_estimate, _)| estimate.cost.0 < best_estimate.cost.0) - { - *best = Some(( - estimate, - selected - .iter() - .map(|(index, guarantee, _)| (*index, guarantee.clone())) - .collect(), - )); - } - return; - } - for alternative in deployments[index] - .alternatives - .iter() - .filter(|alternative| context.eligible(alternative)) - { - selected.push(( - index, - lifecycle_guarantee(&alternative.summary_maintenance_lifecycle, arrival), - alternative.total_cost.unwrap_or(Cost::ZERO), - )); - visit(index + 1, context, deployments, arrival, selected, best); - selected.pop(); - } - } - - let mut best = None; - visit( - 0, - &context, - deployments, - arrival, - &mut Vec::new(), - &mut best, - ); - let (estimate, guarantees) = best?; - apply_selection(deployments, guarantees, &estimate); - Some(estimate) -} - -pub(crate) fn maintenance_mode( - lifecycle: &SummaryMaintenanceLifecycle, - arrival: DataArrival, -) -> SummaryMaintenanceMode { - match lifecycle { - SummaryMaintenanceLifecycle::Ephemeral => SummaryMaintenanceMode::DirectBuild, - SummaryMaintenanceLifecycle::ContinuouslyMaintained => SummaryMaintenanceMode::Incremental, - SummaryMaintenanceLifecycle::Prepared { .. } - | SummaryMaintenanceLifecycle::Shared { .. } => match arrival { - DataArrival::ContinuouslyIngesting | DataArrival::Mixed => { - SummaryMaintenanceMode::Incremental - } - DataArrival::AtRest | DataArrival::Unknown => SummaryMaintenanceMode::DirectBuild, - }, - } -} - -#[cfg(test)] -mod tests { - // Independent data evidence must be validated at both planning boundaries. - #[test] - fn rejects_invalid_parallel_data_evidence() { - let query = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); - let space = crate::replacement::search_workload(vec![("q", quantile_query())]); - for rate in [1.0, -1.0, f64::NAN, f64::INFINITY] { - let mut data = at_rest(); - data.ingestion_rate.value = Some(Rate(rate)); - assert!(space - .recurrence_profiles_from_workload(&query, Some(&data), &[0], 0, None) - .is_err()); - assert!(plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_with_data(&query, &data, &[0]), - 0, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &crate::cost_model::DefaultCostModel, - ) - .is_err()); - } - } - use super::*; - use asap_types::ir::export::{NonASAPOpKind, PhysicalASAPOperatorPayload}; - use asap_types::ir::{BinaryOperator, NonASAPOp}; - use asap_types::post_asap::{ - ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, ResultGuarantee, Schema, - SketchAlgorithm, - }; - use asap_types::pre_asap::AggIntent; - use asap_types::pre_asap::{ - ArithmeticOpKind, BinaryOpKind, ColumnRef, DataType, Reduction, Source, - }; - use asap_types::types::AccuracyTarget; - use asap_types::workload::{ - BatchEntry, DataWorkload, DurationMs, Evidence, EvidenceSource, Predictability, Query, - QueryLanguage, QueryRequirements, Rate, RepeatingEntry, RepetitionInterval, TimeSelection, - }; - - struct UnitCosts; - - impl CostModel for UnitCosts { - fn rank_candidates( - &self, - _intent: &asap_types::pre_asap::AggIntent, - candidates: &[asap_types::post_asap::SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - - fn summary_maintenance_lifecycle_cost_inputs( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(10.0)), - maintenance_cost_per_update: Some(Cost(1.0)), - summary_read_cost: Some(Cost(1.0)), - retention_cost_rate: Some(CostRate(0.1)), - retirement_cost: Some(Cost(1.0)), - } - } - - fn summary_maintenance_capabilities( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceCapabilities { - SummaryMaintenanceCapabilities { - incremental_update: true, - merge: true, - delete: true, - } - } - } - - struct RawCheaper; - - impl CostModel for RawCheaper { - fn rank_candidates( - &self, - _intent: &asap_types::pre_asap::AggIntent, - candidates: &[asap_types::post_asap::SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - - fn summary_maintenance_lifecycle_cost_inputs( - &self, - summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - UnitCosts.summary_maintenance_lifecycle_cost_inputs(summary) - } - - fn summary_maintenance_capabilities( - &self, - summary: &OperatorNode, - ) -> SummaryMaintenanceCapabilities { - UnitCosts.summary_maintenance_capabilities(summary) - } - - fn raw_query_recompute_cost(&self, _target: &OperatorNode) -> Option { - Some(Cost(1.0)) - } - } - - struct NoDelete; - - impl CostModel for NoDelete { - fn rank_candidates( - &self, - _intent: &asap_types::pre_asap::AggIntent, - candidates: &[asap_types::post_asap::SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - - fn summary_maintenance_lifecycle_cost_inputs( - &self, - summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - UnitCosts.summary_maintenance_lifecycle_cost_inputs(summary) - } - - fn summary_maintenance_capabilities( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceCapabilities { - SummaryMaintenanceCapabilities { - incremental_update: true, - merge: true, - delete: false, - } - } - } - - struct SummaryMaintenancePrefersDdSketch; - - impl CostModel for SummaryMaintenancePrefersDdSketch { - fn raw_query_recompute_total_cost( - &self, - _target: &OperatorNode, - _expected_reads: f64, - ) -> Option { - Some(Cost(1_000.0)) - } - - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - // Preserve semantic mapping's KLL-first order. The lifecycle - // total below must be what changes the final choice. - candidates.to_vec() - } - - fn summary_maintenance_lifecycle_cost_inputs( - &self, - summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - let build = match sketch_algorithm(summary) { - Some(SketchAlgorithm::Kll) => 100.0, - Some(SketchAlgorithm::DDSketch) => 1.0, - _ => 10.0, - }; - SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(build)), - maintenance_cost_per_update: Some(Cost(1.0)), - summary_read_cost: Some(Cost(1.0)), - retention_cost_rate: Some(CostRate(0.1)), - retirement_cost: Some(Cost(1.0)), - } - } - } - - struct IncompatibleNestedCosts; - - struct WholeCandidatePrefersContinuous; - - impl CostModel for WholeCandidatePrefersContinuous { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - - fn complete_summary_candidate_cost( - &self, - _root: &OperatorNode, - _target: Option<&OperatorNode>, - deployments: &[CostedSummaryDeployment<'_>], - _horizon: Option, - _expected_reads: Option, - _required_accuracy: &[AccuracyTarget], - ) -> Option { - Some( - if deployments.iter().all(|deployment| { - matches!( - deployment.guarantee.summary_maintenance_lifecycle, - SummaryMaintenanceLifecycle::ContinuouslyMaintained - ) - }) { - Cost(1.0) - } else { - Cost(100.0) - }, - ) - } - } - - impl CostModel for IncompatibleNestedCosts { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - - fn summary_maintenance_lifecycle_cost_inputs( - &self, - summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - // A leaf summary is one built directly over kept pre-ASAP rows - // (its child is not an ASAP node); a nested one reads state. - let is_leaf = matches!( - &summary.operator, - Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) if !child.is_asap() - ); - SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(if is_leaf { 1.0 } else { 100.0 })), - maintenance_cost_per_update: Some(Cost(if is_leaf { 100.0 } else { 0.0 })), - summary_read_cost: Some(Cost::ZERO), - retention_cost_rate: Some(CostRate(0.0)), - retirement_cost: Some(Cost::ZERO), - } - } - - fn summary_maintenance_capabilities( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceCapabilities { - SummaryMaintenanceCapabilities { - incremental_update: true, - merge: true, - delete: true, - } - } - } - - /// The sketch algorithm of the first `SummaryAgg` reachable from `node` - /// (through a evaluation or any relational operator kept above it). - fn sketch_algorithm(node: &OperatorNode) -> Option { - if let Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = &node.operator - { - return Some(kind.algorithm().clone()); - } - node.children() - .into_iter() - .find_map(|child| sketch_algorithm(child)) - } - - fn query_root() -> Rc { - query_root_for("m") - } - - fn query_root_for(metric: &str) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap() - } - - fn sum_query() -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: query_root(), - })) - .unwrap() - } - - fn quantile_query() -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.1), - }], - output_names: vec![], - filters: vec![], - having: None, - child: query_root(), - })) - .unwrap() - } - - /// An exact sum accumulator over the kept pre-ASAP scan. - fn summary() -> Rc { - let child = Rc::new( - query_root() - .as_ref() - .clone() - .with_guarantee(Some(ResultGuarantee::exact("raw"))), - ); - let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: family.clone(), - input: asap_types::post_asap::SummaryUpdate::column(ColumnRef::Named( - "value".into(), - )), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }), - Schema::lifted(vec![Field::new("state", family, false)], None), - ) - .with_guarantee(Some(ResultGuarantee::exact("sum"))), - ) - } - - fn nested_summary() -> Rc { - let child = summary(); - let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: family.clone(), - input: asap_types::post_asap::SummaryUpdate::column(ColumnRef::Named( - "state".into(), - )), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }), - Schema::lifted(vec![Field::new("state", family, false)], None), - ) - .with_guarantee(Some(ResultGuarantee::exact("nested sum"))), - ) - } - - fn batch(predictability: Predictability) -> BatchEntry { - BatchEntry { - query: Query("sum(m)".into()), - requirements: QueryRequirements::default(), - predictability, - invocations: 1, - execute_at: None, - time_selection: TimeSelection::default(), - } - } - - fn workload( - batches: Vec, - repeating: Vec, - _data: DataWorkload, - ) -> QueryWorkload { - QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: (!batches.is_empty()).then_some(batches), - repeating_queries: (!repeating.is_empty()).then_some(repeating), - } - } - - fn at_rest() -> DataWorkload { - DataWorkload { - arrival: DataArrival::AtRest, - data_ingestion_interval: Evidence { - value: Some(DurationMs(1_000)), - ..Default::default() - }, - ..Default::default() - } - } - - fn continuous(observed_at_ms: u64, valid_for_ms: u64) -> DataWorkload { - DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - data_ingestion_interval: Evidence { - value: Some(DurationMs(1_000)), - ..Default::default() - }, - ingestion_rate: Evidence { - value: Some(Rate(1.0)), - source: EvidenceSource::Observed, - observed_at_ms: Some(observed_at_ms), - valid_for_ms: Some(valid_for_ms), - }, - ..Default::default() - } - } - - fn repeating() -> RepeatingEntry { - RepeatingEntry { - query: Query("sum(m)".into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(1_000)), - requirements: QueryRequirements::default(), - predictability: Predictability::Predictable { known_at: None }, - time_selection: TimeSelection::default(), - } - } - - fn selected_summary_maintenance_lifecycle( - deployment: &SummaryMaintenanceDeployment, - ) -> Option<&SummaryMaintenanceLifecycle> { - deployment - .summary_maintenance_lifecycle_guarantee - .as_ref() - .map(|guarantee| &guarantee.summary_maintenance_lifecycle) - } - - #[test] - fn fixed_interval_reads_use_the_physical_horizon_multiplicity() { - let mut query = repeating(); - query.demand = RepeatedDemand::FixedInterval(RepetitionInterval(600)); - let workload = workload(vec![], vec![query], at_rest()); - - let facts = - workload_facts(&workload, Some(&at_rest()), &[0], 0, Some(Horizon(1.0))).unwrap(); - - assert_eq!(facts.reads, Some(1.0)); - } - - #[test] - fn unpredictable_one_time_at_rest_selects_ephemeral() { - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_without_data( - &workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()), - &[0], - ), - 1_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - assert_eq!(plan.deployments.len(), 1); - assert_eq!( - selected_summary_maintenance_lifecycle(&plan.deployments[0]), - Some(&SummaryMaintenanceLifecycle::Ephemeral) - ); - let guarantee = plan.deployments[0] - .summary_maintenance_lifecycle_guarantee - .as_ref() - .unwrap(); - assert_eq!(guarantee.evaluation_schedule, EvaluationSchedule::OneShot); - assert_eq!( - guarantee.summary_maintenance_mode, - SummaryMaintenanceMode::DirectBuild - ); - assert_eq!( - guarantee.output_representation, - OutputRepresentation::SummaryState - ); - assert_eq!( - plan.deployments[0].alternatives[0].total_cost, - Some(Cost(12.0)) - ); - } - - #[test] - fn predictable_scheduled_one_time_offers_prepared_state() { - let mut entry = batch(Predictability::Predictable { - known_at: Some(TimestampMs(1_000)), - }); - entry.execute_at = Some(TimestampMs(11_000)); - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_with_data( - &workload(vec![entry], vec![], at_rest()), - &at_rest(), - &[0], - ), - 1_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - let prepared = &plan.deployments[0].alternatives[1]; - assert!(prepared.rejection.is_none()); - assert_eq!(prepared.total_cost, Some(Cost(13.0))); - } - - #[test] - fn prepared_state_starts_no_earlier_than_planning_time() { - let mut entry = batch(Predictability::Predictable { - known_at: Some(TimestampMs(1_000)), - }); - entry.execute_at = Some(TimestampMs(11_000)); - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_with_data( - &workload(vec![entry], vec![], at_rest()), - &at_rest(), - &[0], - ), - 6_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - let prepared = &plan.deployments[0].alternatives[1]; - assert_eq!( - prepared.summary_maintenance_lifecycle, - SummaryMaintenanceLifecycle::Prepared { - activate_at: TimestampMs(6_000), - retire_at: TimestampMs(11_000), - } - ); - assert_eq!(prepared.total_cost, Some(Cost(12.5))); - } - - #[test] - fn expired_one_time_execution_cannot_select_prepared_state() { - let mut entry = batch(Predictability::Predictable { - known_at: Some(TimestampMs(1_000)), - }); - entry.execute_at = Some(TimestampMs(2_000)); - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_without_data(&workload(vec![entry], vec![], at_rest()), &[0]), - 3_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - assert_eq!( - plan.deployments[0].alternatives[1].rejection, - Some(SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery) - ); - } - - #[test] - fn nested_summary_lifecycles_have_compatible_evaluation_schedules() { - let workload = workload(vec![], vec![repeating()], continuous(1_000, 20_000)); - let plan = plan_summary_maintenance_lifecycles( - nested_summary(), - WorkloadDemand::new_without_data(&workload, &[0]), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &IncompatibleNestedCosts, - ) - .unwrap(); - - assert_eq!(plan.deployments.len(), 2); - let schedules: HashSet<_> = plan - .deployments - .iter() - .map(|deployment| { - deployment - .summary_maintenance_lifecycle_guarantee - .as_ref() - .unwrap() - .evaluation_schedule - }) - .collect(); - assert_eq!(schedules.len(), 1); - } - - #[test] - fn repeated_at_rest_selects_shared_without_inventing_updates() { - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_with_data( - &workload(vec![], vec![repeating()], at_rest()), - &at_rest(), - &[0], - ), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - assert_eq!( - selected_summary_maintenance_lifecycle(&plan.deployments[0]), - Some(&SummaryMaintenanceLifecycle::Shared { - retention: DurationMs(10_000) - }) - ); - assert_eq!( - plan.deployments[0] - .summary_maintenance_lifecycle_guarantee - .as_ref() - .unwrap() - .summary_maintenance_mode, - SummaryMaintenanceMode::DirectBuild - ); - assert_eq!( - plan.deployments[0].alternatives[3].rejection, - Some(SummaryMaintenanceLifecycleRejection::RequiresContinuousData) - ); - assert_eq!(plan.update_rate, None); - } - - #[test] - fn repeated_continuous_workload_can_select_continuous_maintenance() { - let capabilities = SummaryMaintenanceLifecycleCapabilities { - supports_shared: false, - ..SummaryMaintenanceLifecycleCapabilities::ALL - }; - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_with_data( - &workload(vec![], vec![repeating()], continuous(1_000, 60_000)), - &continuous(1_000, 60_000), - &[0], - ), - 1_000, - Some(Horizon(10.0)), - capabilities, - &UnitCosts, - ) - .unwrap(); - assert_eq!( - selected_summary_maintenance_lifecycle(&plan.deployments[0]), - Some(&SummaryMaintenanceLifecycle::ContinuouslyMaintained) - ); - assert_eq!( - plan.deployments[0] - .summary_maintenance_lifecycle_guarantee - .as_ref() - .unwrap() - .summary_maintenance_mode, - SummaryMaintenanceMode::Incremental - ); - assert_eq!(plan.evaluation_rate, Some(EvaluationRate(1.0))); - assert_eq!(plan.update_rate, Some(UpdateRate(1.0))); - } - - #[test] - fn stale_ingestion_evidence_cannot_enable_continuous_maintenance() { - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_with_data( - &workload(vec![], vec![repeating()], continuous(1_000, 1_000)), - &continuous(1_000, 1_000), - &[0], - ), - 3_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - assert_eq!( - plan.deployments[0].alternatives[3].rejection, - Some(SummaryMaintenanceLifecycleRejection::MissingOrStaleIngestionRate) - ); - assert_eq!(plan.update_rate, None); - } - - #[test] - fn unknown_costs_do_not_make_a_long_lived_lifecycle_win() { - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_without_data( - &workload(vec![], vec![repeating()], continuous(1_000, 60_000)), - &[0], - ), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &crate::cost_model::DefaultCostModel, - ) - .unwrap(); - assert_eq!( - selected_summary_maintenance_lifecycle(&plan.deployments[0]), - None - ); - assert!(plan.deployments[0] - .alternatives - .iter() - .all(|alternative| alternative.rejection.is_some())); - } - - #[test] - fn unrelated_workload_entries_do_not_create_reuse_for_a_target() { - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_without_data( - &workload( - vec![batch(Predictability::AdHoc), batch(Predictability::AdHoc)], - vec![], - at_rest(), - ), - &[0], - ), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - assert_eq!( - selected_summary_maintenance_lifecycle(&plan.deployments[0]), - Some(&SummaryMaintenanceLifecycle::Ephemeral) - ); - assert_eq!( - plan.deployments[0].alternatives[2].rejection, - Some(SummaryMaintenanceLifecycleRejection::RequiresMultipleReads) - ); - } - - #[test] - fn scheduled_rate_counts_only_executions_inside_the_horizon() { - let mut entry = repeating(); - entry.demand = RepeatedDemand::Scheduled(vec![ - TimestampMs(999), - TimestampMs(5_000), - TimestampMs(20_000), - ]); - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_without_data(&workload(vec![], vec![entry], at_rest()), &[0]), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - assert_eq!(plan.evaluation_rate, Some(EvaluationRate(0.1))); - } - - #[test] - fn demand_binding_rejects_empty_and_duplicate_entries() { - let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); - assert!(matches!( - plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_without_data(&workload, &[]), - 1_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ), - Err(SummaryMaintenanceLifecyclePlanError::EmptyWorkloadDemand) - )); - assert!(matches!( - plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_without_data(&workload, &[0, 0]), - 1_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ), - Err(SummaryMaintenanceLifecyclePlanError::DuplicateWorkloadEntry { index: 0 }) - )); - } - - #[test] - fn prepared_requires_every_bound_consumer_to_be_scheduled_and_predictable() { - let mut predictable = batch(Predictability::Predictable { - known_at: Some(TimestampMs(1_000)), - }); - predictable.execute_at = Some(TimestampMs(2_000)); - let workload = workload( - vec![predictable, batch(Predictability::AdHoc)], - vec![], - at_rest(), - ); - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_without_data(&workload, &[0, 1]), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - assert_eq!( - plan.deployments[0].alternatives[1].rejection, - Some(SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery) - ); - } - - #[test] - fn moving_realtime_maintenance_requires_summary_deletion_support() { - let mut entry = repeating(); - entry.time_selection = TimeSelection { - scope: asap_types::workload::QueryTimeScope::RealTime, - lookback: Some(DurationMs(60_000)), - as_of: None, - }; - let plan = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_with_data( - &workload(vec![], vec![entry], continuous(1_000, 60_000)), - &continuous(1_000, 60_000), - &[0], - ), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &NoDelete, - ) - .unwrap(); - assert_eq!( - plan.deployments[0].alternatives[3].rejection, - Some(SummaryMaintenanceLifecycleRejection::SummaryDoesNotSupportDeletion) - ); - } - - #[test] - fn lifecycle_cost_can_fall_back_to_raw_recomputation() { - let target = sum_query(); - let space = crate::replacement::search_workload(vec![("q", Rc::clone(&target))]); - let selection = space.global_selection(&RawCheaper); - let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); - let plan = assemble_selected_dag_with_summary_maintenance_lifecycles( - &selection, - &space.roots[0].1, - WorkloadDemand::new_without_data(&workload, &[0]), - 1_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &RawCheaper, - ) - .unwrap() - .unwrap(); - assert!(plan.selected_raw_recompute); - assert_eq!(plan.raw_recompute_total_cost, Some(Cost(1.0))); - assert_eq!(plan.summary_total_cost, None); - assert!(plan.deployments.is_empty()); - // The logical query stays exact; deployment assigns execution timing. - assert!(!plan.root.contains_asap()); - assert!(matches!( - plan.root.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - )); - assert!(plan.root.timing.is_none()); - - let exported = - crate::summary_maintenance_dag_export::export_summary_maintenance_plan(&plan); - assert!(exported.selected_raw_recompute); - assert_eq!(exported.raw_recompute_total_cost, Some(1.0)); - assert_eq!(exported.summary_total_cost, None); - assert!(exported.deployments.is_empty()); - } - - #[test] - fn whole_candidate_cost_is_evaluated_before_selecting_a_lifecycle() { - let root = summary(); - let mut deployments = vec![SummaryMaintenanceDeployment { - post_asap_node_id: asap_types::ir::export::LogicalASAPNodeId(0), - summary: Rc::clone(&root), - summary_maintenance_lifecycle_guarantee: None, - selected_window_framework: None, - alternatives: vec![ - SummaryMaintenanceLifecycleAlternative { - summary_maintenance_lifecycle: SummaryMaintenanceLifecycle::Ephemeral, - total_cost: Some(Cost(1.0)), - rejection: None, - assumptions: vec![], - }, - SummaryMaintenanceLifecycleAlternative { - summary_maintenance_lifecycle: - SummaryMaintenanceLifecycle::ContinuouslyMaintained, - total_cost: Some(Cost(10.0)), - rejection: None, - assumptions: vec![], - }, - ], - }]; - - let total = select_complete_lifecycle_combination( - &root, - &mut deployments, - &[0], - DataArrival::ContinuouslyIngesting, - &WholeCandidatePrefersContinuous, - None, - Some(Horizon(10.0)), - Some(2.0), - &[], - ); - - assert_eq!(total.map(|estimate| estimate.cost), Some(Cost(1.0))); - assert!(matches!( - deployments[0] - .summary_maintenance_lifecycle_guarantee - .as_ref() - .unwrap() - .summary_maintenance_lifecycle, - SummaryMaintenanceLifecycle::ContinuouslyMaintained - )); - } - - #[test] - fn complete_lifecycle_enumeration_fails_closed_above_safe_bound() { - let root = summary(); - let alternatives = vec![ - SummaryMaintenanceLifecycleAlternative { - summary_maintenance_lifecycle: SummaryMaintenanceLifecycle::Ephemeral, - total_cost: Some(Cost(1.0)), - rejection: None, - assumptions: vec![], - }, - SummaryMaintenanceLifecycleAlternative { - summary_maintenance_lifecycle: SummaryMaintenanceLifecycle::ContinuouslyMaintained, - total_cost: Some(Cost(2.0)), - rejection: None, - assumptions: vec![], - }, - ]; - let mut deployments: Vec<_> = (0..13) - .map(|summary_index| SummaryMaintenanceDeployment { - post_asap_node_id: asap_types::ir::export::LogicalASAPNodeId(summary_index as u32), - summary: Rc::clone(&root), - summary_maintenance_lifecycle_guarantee: None, - selected_window_framework: None, - alternatives: alternatives.clone(), - }) - .collect(); - assert_eq!( - select_complete_lifecycle_combination( - &root, - &mut deployments, - &(0..13).collect::>(), - DataArrival::ContinuouslyIngesting, - &WholeCandidatePrefersContinuous, - None, - Some(Horizon(10.0)), - Some(2.0), - &[], - ), - None - ); - assert!(deployments - .iter() - .all(|deployment| deployment.summary_maintenance_lifecycle_guarantee.is_none())); - } - - #[test] - fn materialization_falls_back_to_raw_when_raw_cost_is_unavailable() { - let target = quantile_query(); - let space = crate::replacement::search_workload(vec![("q", target)]); - let selection = space.global_selection(&UnitCosts); - let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); - let plan = assemble_selected_dag_with_summary_maintenance_lifecycles( - &selection, - &space.roots[0].1, - WorkloadDemand::new_without_data(&workload, &[0]), - 1_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap() - .unwrap(); - assert!(plan.selected_raw_recompute); - assert!(plan.raw_recompute_total_cost.is_none()); - assert!(!plan.root.contains_asap()); - assert!(matches!( - plan.root.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - )); - } - - #[test] - fn unmatched_target_is_reported_as_raw_recomputation() { - let target = query_root(); - let space = crate::replacement::search_workload(vec![("q", Rc::clone(&target))]); - let selection = space.global_selection(&RawCheaper); - let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); - let plan = assemble_selected_dag_with_summary_maintenance_lifecycles( - &selection, - &space.roots[0].1, - WorkloadDemand::new_without_data(&workload, &[0]), - 1_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &RawCheaper, - ) - .unwrap() - .unwrap(); - - assert!(plan.selected_raw_recompute); - assert_eq!(plan.raw_recompute_total_cost, Some(Cost(1.0))); - assert_eq!(plan.summary_total_cost, None); - assert!(plan.deployments.is_empty()); - assert!(!plan.root.contains_asap()); - assert!(matches!(plan.root.non_asap(), Some(NonASAPOp::Scan { .. }))); - } - - #[test] - fn lifecycle_cost_reorders_semantic_summary_candidates_before_materialization() { - let target = quantile_query(); - let space = crate::replacement::search_workload(vec![("q", target)]); - let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); - - let selection = global_selection_with_summary_maintenance_lifecycles( - &space, - WorkloadDemand::new_with_data(&workload, &at_rest(), &[0]), - 1_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &SummaryMaintenancePrefersDdSketch, - ) - .unwrap(); - let materialized = selection - .assemble_selected_dag(&space.roots[0].1) - .unwrap() - .unwrap(); - - assert_eq!( - sketch_algorithm(&materialized), - Some(SketchAlgorithm::DDSketch) - ); - } - - #[test] - fn lifecycle_cost_counts_one_shared_summary_node_once() { - // One shared exact accumulator read twice by the same root: a - // query-time `sum + sum` over one finalized state. (`SummaryMerge` - // is reserved in the unified IR, so the sharing is expressed through - // a relational consumer instead.) - let shared = summary(); - let finalized = Rc::new( - OperatorNode::new(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: Rc::clone(&shared), - })) - .unwrap(), - ); - let root = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - }, - return_bool: false, - lhs: Rc::clone(&finalized), - rhs: finalized, - })) - .unwrap(); - let workload = workload( - vec![batch(Predictability::AdHoc), batch(Predictability::AdHoc)], - vec![], - at_rest(), - ); - let horizon = Some(Horizon(10.0)); - let plan = plan_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &at_rest(), &[0, 1]), - 1_000, - horizon, - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - assert_eq!(plan.deployments.len(), 1); - assert!(matches!( - selected_summary_maintenance_lifecycle(&plan.deployments[0]), - Some(SummaryMaintenanceLifecycle::Shared { .. }) - )); - } - - /// A state costs 10 however often it is read. Recomputing p50 raw costs - /// 1 and p99 costs 8. - struct P50PrefersRaw; - - impl CostModel for P50PrefersRaw { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - - fn summary_maintenance_lifecycle_cost_inputs( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(10.0)), - maintenance_cost_per_update: Some(Cost::ZERO), - summary_read_cost: Some(Cost::ZERO), - retention_cost_rate: Some(CostRate(0.0)), - retirement_cost: Some(Cost::ZERO), - } - } - - fn summary_maintenance_capabilities( - &self, - summary: &OperatorNode, - ) -> SummaryMaintenanceCapabilities { - UnitCosts.summary_maintenance_capabilities(summary) - } - - fn raw_query_recompute_total_cost( - &self, - target: &OperatorNode, - _expected_reads: f64, - ) -> Option { - match target.non_asap() { - Some(NonASAPOp::Aggregate { measures, .. }) => match measures[..] { - [AggIntent::Quantile { q: 0.5, .. }] => Some(Cost(1.0)), - _ => Some(Cost(8.0)), - }, - _ => None, - } - } - } - - /// p50 and p99 form a sharing class over one state (5 each), but p50's - /// raw recompute (1) still wins. The class reverts, so p99 is reselected - /// at its independent cost (10) and recomputes raw (8), as it does alone. - /// Checked at selection: the assembled plan's own raw comparison would - /// recompute p99 raw either way. - #[test] - fn sharing_class_reverts_when_a_member_selects_elsewhere() { - let quantile = |q| { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Quantile { - col: None, - q, - accuracy: AccuracyTarget::Epsilon(0.1), - }], - // A shared output name keeps p50 and p99 on one state. - output_names: vec!["value".into()], - filters: vec![], - having: None, - child: query_root(), - })) - .unwrap() - }; - let workload = workload(vec![], vec![repeating(), repeating()], at_rest()); - // Whether each root selected a summary rather than raw recompute. - let summaries = |space: &CandidateLogicalASAPDAGs<&str>, entries: &[usize]| { - let selection = global_selection_with_summary_maintenance_lifecycles( - space, - WorkloadDemand::new_with_data(&workload, &at_rest(), entries), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &P50PrefersRaw, - ) - .unwrap(); - space - .roots - .iter() - .map(|(_, target)| selection.for_target(target).unwrap().chosen.is_some()) - .collect::>() - }; - - let space = crate::replacement::search_workload(vec![ - ("p50", quantile(0.5)), - ("p99", quantile(0.99)), - ]); - let alone = crate::replacement::search_workload(vec![("p99", quantile(0.99))]); - assert_eq!(summaries(&space, &[0, 1]), vec![false, false]); - assert_eq!(summaries(&alone, &[1]), vec![false]); - } - - #[test] - fn normalized_workload_drives_candidate_logical_asap_dags_recurrence_profiles() { - let root = query_root(); - let space = crate::replacement::search_workload(vec![("dashboard", Rc::clone(&root))]); - let workload = workload(vec![], vec![repeating()], continuous(1_000, 60_000)); - let profiles = space - .recurrence_profiles_from_workload( - &workload, - Some(&continuous(1_000, 60_000)), - &[0], - 1_000, - Some(Horizon(10.0)), - ) - .unwrap(); - // `search_workload` canonicalizes roots through CSE; recurrence - // profiles are keyed by that canonical post-CSE node. - let profile = profiles.for_target(&space.roots[0].1); - assert_eq!(profile.evaluation_rate, Some(EvaluationRate(1.0))); - assert_eq!(profile.update_rate, Some(UpdateRate(1.0))); - assert_eq!(profile.one_shot_consumers, 0); - } - - #[test] - fn recurrence_binding_is_explicit_when_root_order_differs_from_workload_order() { - let repeating_root = query_root_for("dashboard"); - let batch_root = query_root_for("batch"); - let space = crate::replacement::search_workload(vec![ - ("dashboard", repeating_root), - ("batch", batch_root), - ]); - let workload = workload( - vec![batch(Predictability::AdHoc)], - vec![repeating()], - at_rest(), - ); - let profiles = space - .recurrence_profiles_from_workload(&workload, None, &[1, 0], 1_000, Some(Horizon(10.0))) - .unwrap(); - let dashboard = profiles.for_target(&space.roots[0].1); - let batch = profiles.for_target(&space.roots[1].1); - assert_eq!(dashboard.evaluation_rate, Some(EvaluationRate(1.0))); - assert_eq!(dashboard.one_shot_consumers, 0); - assert_eq!(batch.evaluation_rate, None); - assert_eq!(batch.one_shot_consumers, 1); - } - - fn continuous_candidates<'a>( - workload: &QueryWorkload, - data: &DataWorkload, - model: &'a dyn CostModel, - ) -> SummaryMaintenanceLifecycleCandidates<'a> { - enumerate_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_with_data(workload, data, &[0]), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities { - supports_shared: false, - ..SummaryMaintenanceLifecycleCapabilities::ALL - }, - model, - ) - .unwrap() - } - - fn choose( - candidates: &SummaryMaintenanceLifecycleCandidates<'_>, - lifecycle: SummaryMaintenanceLifecycle, - ) -> Vec<(PhysicalASAPNodeId, SummaryMaintenanceLifecycle)> { - candidates - .deployments() - .iter() - .map(|deployment| (deployment.post_asap_node_id, lifecycle.clone())) - .collect() - } - - // Enumeration reports all four lifecycle kinds with their rejections and - // selects nothing. - #[test] - fn enumeration_exposes_every_lifecycle_without_selecting() { - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - let candidates = continuous_candidates(&workload, &data, &UnitCosts); - let [deployment] = candidates.deployments() else { - panic!("one summary state"); - }; - assert_eq!(deployment.summary_maintenance_lifecycle_guarantee, None); - assert_eq!(deployment.selected_window_framework, None); - let outcome: Vec<_> = deployment - .alternatives - .iter() - .map(|alternative| { - ( - &alternative.summary_maintenance_lifecycle, - alternative.rejection.clone(), - alternative.total_cost.is_some(), - ) - }) - .collect(); - assert!(matches!( - outcome.as_slice(), - [ - (SummaryMaintenanceLifecycle::Ephemeral, None, true), - ( - SummaryMaintenanceLifecycle::Prepared { .. }, - Some(SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery), - false - ), - ( - SummaryMaintenanceLifecycle::Shared { .. }, - Some(SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime), - false - ), - ( - SummaryMaintenanceLifecycle::ContinuouslyMaintained, - None, - true - ), - ] - )); - let guarantee = candidates.guarantee(&SummaryMaintenanceLifecycle::ContinuouslyMaintained); - assert_eq!( - guarantee.summary_maintenance_mode, - SummaryMaintenanceMode::Incremental - ); - assert_eq!(guarantee.evaluation_schedule, EvaluationSchedule::PerUpdate); - } - - // Explicitly choosing Planner's own selection reproduces Planner's plan. - #[test] - fn explicit_choice_of_planner_selection_reproduces_planner_plan() { - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - let planned = plan_summary_maintenance_lifecycles( - summary(), - WorkloadDemand::new_with_data(&workload, &data, &[0]), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities { - supports_shared: false, - ..SummaryMaintenanceLifecycleCapabilities::ALL - }, - &UnitCosts, - ) - .unwrap(); - let candidates = continuous_candidates(&workload, &data, &UnitCosts); - let choice = choose( - &candidates, - SummaryMaintenanceLifecycle::ContinuouslyMaintained, - ); - let chosen = candidates.select(&choice).unwrap(); - assert_eq!(format!("{chosen:?}"), format!("{planned:?}")); - } - - // A deployment may bind a legal alternative Planner's estimate does not - // prefer; the plan carries that alternative's guarantee and cost. - #[test] - fn explicit_choice_may_bind_a_costlier_legal_alternative() { - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - let candidates = continuous_candidates(&workload, &data, &UnitCosts); - let ephemeral_cost = candidates.deployments()[0].alternatives[0].total_cost; - let choice = choose(&candidates, SummaryMaintenanceLifecycle::Ephemeral); - let plan = candidates.select(&choice).unwrap(); - assert_eq!( - selected_summary_maintenance_lifecycle(&plan.deployments[0]), - Some(&SummaryMaintenanceLifecycle::Ephemeral) - ); - assert_eq!(plan.summary_total_cost, ephemeral_cost); - } - - // Choices that Planner could not select, or that do not cover exactly the - // enumerated states, are refused rather than bound. - #[test] - fn explicit_choice_rejects_illegal_or_incomplete_choices() { - use SummaryMaintenanceLifecycleChoiceError as E; - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - let select = |model: &dyn CostModel, choice: &dyn Fn(PhysicalASAPNodeId) -> Vec<_>| { - let candidates = continuous_candidates(&workload, &data, model); - let id = candidates.deployments()[0].post_asap_node_id; - (id, candidates.select(&choice(id)).unwrap_err()) - }; - let shared = SummaryMaintenanceLifecycle::Shared { - retention: DurationMs(10_000), - }; - let (id, error) = select(&UnitCosts, &|id| vec![(id, shared.clone())]); - assert_eq!( - error, - E::Rejected { - post_asap_node_id: id, - rejection: Some(SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime), - } - ); - let continuous = SummaryMaintenanceLifecycle::ContinuouslyMaintained; - let (id, error) = select(&crate::cost_model::DefaultCostModel, &|id| { - vec![(id, SummaryMaintenanceLifecycle::Ephemeral)] - }); - assert_eq!( - error, - E::Rejected { - post_asap_node_id: id, - rejection: Some(SummaryMaintenanceLifecycleRejection::MissingCostEvidence), - } - ); - let (id, error) = select(&UnitCosts, &|id| { - vec![( - id, - SummaryMaintenanceLifecycle::Shared { - retention: DurationMs(1), - }, - )] - }); - assert_eq!(error, E::NotAnAlternative(id)); - let (id, error) = select(&UnitCosts, &|_| vec![]); - assert_eq!(error, E::MissingChoice(id)); - let (id, error) = select(&UnitCosts, &|id| { - vec![(id, continuous.clone()), (id, continuous.clone())] - }); - assert_eq!(error, E::DuplicateChoice(id)); - let (_, error) = select(&UnitCosts, &|_| { - vec![( - asap_types::ir::export::LogicalASAPNodeId(u32::MAX), - continuous.clone(), - )] - }); - assert_eq!( - error, - E::UnknownSummary(asap_types::ir::export::LogicalASAPNodeId(u32::MAX)) - ); - } - - // Nested states on one maintenance path must share an evaluation schedule. - #[test] - fn explicit_choice_rejects_incompatible_nested_schedules() { - let workload = workload(vec![], vec![repeating()], continuous(1_000, 20_000)); - let candidates = enumerate_summary_maintenance_lifecycles( - nested_summary(), - WorkloadDemand::new_with_data(&workload, &continuous(1_000, 20_000), &[0]), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &IncompatibleNestedCosts, - ) - .unwrap(); - let [outer, inner] = candidates.deployments() else { - panic!("two summary states"); - }; - let choice = vec![ - ( - outer.post_asap_node_id, - SummaryMaintenanceLifecycle::Ephemeral, - ), - ( - inner.post_asap_node_id, - SummaryMaintenanceLifecycle::ContinuouslyMaintained, - ), - ]; - assert_eq!( - candidates.select(&choice).unwrap_err(), - SummaryMaintenanceLifecycleChoiceError::IncompatibleEvaluationSchedules - ); - } - - // A multi-summary root yields one candidate entry per unique state, with - // a shared `Rc` state listed once. - #[test] - fn enumeration_lists_each_unique_summary_state_once() { - let shared = summary(); - let root = test_binary( - test_binary(evaluation(&shared), evaluation(&shared)), - evaluation(&summary()), - ); - let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); - let candidates = enumerate_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(&workload, &at_rest(), &[0]), - 1_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - let ids: HashSet<_> = candidates - .deployments() - .iter() - .map(|deployment| deployment.post_asap_node_id) - .collect(); - assert_eq!(candidates.deployments().len(), 2); - assert_eq!(ids.len(), 2); - assert!(candidates - .deployments() - .iter() - .any(|deployment| Rc::ptr_eq(&deployment.summary, &shared))); - } - - fn evaluation(state: &Rc) -> Rc { - std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: Rc::clone(state), - }), - Schema::lifted( - vec![Field::new( - "value", - FieldDataType::Plain(DataType::Float64), - false, - )], - None, - ), - ) - .with_guarantee(Some(ResultGuarantee::exact("sum"))), - ) - } - - fn test_binary(lhs: Rc, rhs: Rc) -> Rc { - let schema = lhs.schema.clone(); - Rc::new(OperatorNode::with_schema( - Operator::NonASAP(NonASAPOp::BinaryOp { - lhs, - rhs, - return_bool: false, - operator: BinaryOperator { - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - checked_relative_division: false, - checked_finite_division: false, - }, - }), - schema, - )) - } - - fn lifecycle_matching( - alternatives: &[SummaryMaintenanceLifecycleAlternative], - kind: fn(&SummaryMaintenanceLifecycle) -> bool, - ) -> SummaryMaintenanceLifecycle { - alternatives - .iter() - .map(|alternative| &alternative.summary_maintenance_lifecycle) - .find(|lifecycle| kind(lifecycle)) - .expect("lifecycle kind is an alternative") - .clone() - } - - /// Bind the lifecycle `choose` picks for every state of `root`, then - /// derive the timed DAG. - fn timed_dag( - root: Rc, - workload: &QueryWorkload, - data: &DataWorkload, - horizon: Option, - choose: impl Fn(&SummaryMaintenanceDeployment) -> SummaryMaintenanceLifecycle, - ) -> PhysicalASAPDAG { - let candidates = enumerate_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data(workload, data, &[0]), - 1_000, - horizon, - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - let choice: Vec<_> = candidates - .deployments() - .iter() - .map(|deployment| (deployment.post_asap_node_id, choose(deployment))) - .collect(); - let dag = candidates - .select(&choice) - .unwrap() - .execution_timed_dag() - .unwrap(); - dag.validate().unwrap(); - dag - } - - /// Operator kinds in node-id order, each paired with its timing. - fn timings(dag: &PhysicalASAPDAG) -> Vec<(&'static str, ExecutionTiming)> { - dag.nodes - .iter() - .map(|node| { - let kind = match node.payload { - PhysicalASAPOperatorPayload::Relational { - operator: NonASAPOpKind::BinaryOp { .. }, - } => "binary", - PhysicalASAPOperatorPayload::Relational { .. } => "raw", - PhysicalASAPOperatorPayload::SummaryAgg { .. } => "state", - PhysicalASAPOperatorPayload::FinalizeExactAccumulator - | PhysicalASAPOperatorPayload::EvaluatePopulation { .. } => "evaluation", - - _ => "other", - }; - (kind, node.output_state.timing) - }) - .collect() - } - - const INGEST: ExecutionTiming = ExecutionTiming::IngestionTime; - const QUERY: ExecutionTiming = ExecutionTiming::QueryTime; - - // Every retained lifecycle kind runs its state and inputs at ingestion - // time and its evaluation at query time. - #[test] - fn retained_lifecycles_time_state_and_inputs_at_ingestion() { - let mut scheduled = batch(Predictability::Predictable { - known_at: Some(TimestampMs(1_000)), - }); - scheduled.execute_at = Some(TimestampMs(11_000)); - type Case = ( - QueryWorkload, - DataWorkload, - Option, - fn(&SummaryMaintenanceLifecycle) -> bool, - ); - let cases: [Case; 3] = [ - ( - workload(vec![], vec![repeating()], continuous(1_000, 60_000)), - continuous(1_000, 60_000), - Some(Horizon(10.0)), - |lifecycle| { - matches!( - lifecycle, - SummaryMaintenanceLifecycle::ContinuouslyMaintained - ) - }, - ), - ( - workload(vec![], vec![repeating()], at_rest()), - at_rest(), - Some(Horizon(10.0)), - |lifecycle| matches!(lifecycle, SummaryMaintenanceLifecycle::Shared { .. }), - ), - ( - workload(vec![scheduled], vec![], at_rest()), - at_rest(), - None, - |lifecycle| matches!(lifecycle, SummaryMaintenanceLifecycle::Prepared { .. }), - ), - ]; - for (workload, data, horizon, kind) in cases { - let dag = timed_dag( - evaluation(&summary()), - &workload, - &data, - horizon, - |deployment| lifecycle_matching(&deployment.alternatives, kind), - ); - assert_eq!( - timings(&dag), - [("raw", INGEST), ("state", INGEST), ("evaluation", QUERY)] - ); - } - } - - // An Ephemeral state, its raw input, and its evaluation all run at query time. - #[test] - fn ephemeral_lifecycle_times_state_and_downstream_at_query() { - let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); - let dag = timed_dag(evaluation(&summary()), &workload, &at_rest(), None, |_| { - SummaryMaintenanceLifecycle::Ephemeral - }); - assert_eq!( - timings(&dag), - [("raw", QUERY), ("state", QUERY), ("evaluation", QUERY)] - ); - } - - // One state read by two consumers is one deployment; its timing follows - // that single choice while both consumers run at query time. - #[test] - fn shared_state_is_timed_once_for_all_consumers() { - let state = summary(); - let lhs = evaluation(&state); - let rhs = Rc::new(lhs.as_ref().clone()); - let root = test_binary(lhs, rhs); - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - let dag = timed_dag(root, &workload, &data, Some(Horizon(10.0)), |deployment| { - assert!(Rc::ptr_eq(&deployment.summary, &state)); - SummaryMaintenanceLifecycle::ContinuouslyMaintained - }); - assert_eq!( - timings(&dag), - [ - ("raw", INGEST), - ("state", INGEST), - ("evaluation", QUERY), - ("evaluation", QUERY), - ("binary", QUERY), - ] - ); - } - - // An Ephemeral state consumed by retained state is built on the retained - // state's ingestion path; it is not retained, but cannot run at query time. - #[test] - fn ephemeral_state_feeding_retained_state_runs_at_ingestion() { - let mut scheduled = batch(Predictability::Predictable { - known_at: Some(TimestampMs(1_000)), - }); - scheduled.execute_at = Some(TimestampMs(11_000)); - let root = nested_summary(); - let workload = workload(vec![scheduled], vec![], at_rest()); - let dag = timed_dag( - Rc::clone(&root), - &workload, - &at_rest(), - None, - |deployment| { - if Rc::ptr_eq(&deployment.summary, &root) { - lifecycle_matching(&deployment.alternatives, |lifecycle| { - matches!(lifecycle, SummaryMaintenanceLifecycle::Prepared { .. }) - }) - } else { - SummaryMaintenanceLifecycle::Ephemeral - } - }, - ); - assert_eq!( - timings(&dag), - [("raw", INGEST), ("state", INGEST), ("state", INGEST)] - ); - } - - // Timing is not derived for a state without a selected lifecycle, and a - // raw-recompute plan runs entirely at query time. - #[test] - fn timing_requires_a_selected_lifecycle_for_every_state() { - let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); - let data = at_rest(); - let demand = WorkloadDemand::new_with_data(&workload, &data, &[0]); - let plan = plan_summary_maintenance_lifecycles( - evaluation(&summary()), - demand, - 1_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &crate::cost_model::DefaultCostModel, - ) - .unwrap(); - assert_eq!( - plan.execution_timed_dag().unwrap_err(), - SummaryMaintenanceTimingError::UnselectedLifecycle( - plan.deployments[0].post_asap_node_id - ) - ); - let raw = plan_summary_maintenance_lifecycles( - crate::replacement::retain_exact(&sum_query()).unwrap(), - demand, - 1_000, - None, - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - assert_eq!( - timings(&raw.execution_timed_dag().unwrap()), - [("raw", QUERY), ("raw", QUERY)] - ); - } - - /// A strategy-built `sum(a)` over one maintained current-series population. - fn population_evaluation() -> Rc { - let target = crate::test_support::lower_promql("sum(a)", AccuracyTarget::Exact); - crate::maintained_population::MaintainedPopulationStrategy::new(std::slice::from_ref( - &target, - )) - .candidate(&target) - .unwrap() - } - - fn is_population(node: &OperatorNode) -> bool { - matches!( - node.operator, - Operator::ASAP(ASAPOp::MaintainPopulation { .. }) - ) - } - - fn population_timings(dag: &PhysicalASAPDAG) -> Vec<(&'static str, ExecutionTiming)> { - dag.nodes - .iter() - .zip(timings(dag)) - .map(|(node, (kind, timing))| match node.payload { - PhysicalASAPOperatorPayload::MaintainPopulation { .. } => ("population", timing), - _ => (kind, timing), - }) - .collect() - } - - // A maintained population is enumerated as retained state, with costs - // from the caller's model for both the maintained and the rebuilt choice. - #[test] - fn enumeration_includes_maintained_population() { - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - let candidates = enumerate_summary_maintenance_lifecycles( - population_evaluation(), - WorkloadDemand::new_with_data(&workload, &data, &[0]), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - let [deployment] = candidates.deployments() else { - panic!("one population state"); - }; - assert!(is_population(&deployment.summary)); - let cost = |lifecycle: SummaryMaintenanceLifecycle| { - deployment - .alternatives - .iter() - .find(|alternative| alternative.summary_maintenance_lifecycle == lifecycle) - .and_then(|alternative| alternative.total_cost) - }; - // Ephemeral: (build 10 + read 1 + retire 1) x 10 reads. Maintained over - // 10 s at 1 update/s: build 10 + updates 10 + reads 10 + retention 1 + retire 1. - assert_eq!( - cost(SummaryMaintenanceLifecycle::Ephemeral), - Some(Cost(120.0)) - ); - assert_eq!( - cost(SummaryMaintenanceLifecycle::ContinuouslyMaintained), - Some(Cost(32.0)) - ); - } - - // Without cost evidence a population's alternatives stay unknown: Planner - // selects none and timing is refused rather than guessed. - #[test] - fn population_without_cost_evidence_stays_unselected() { - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - let plan = plan_summary_maintenance_lifecycles( - population_evaluation(), - WorkloadDemand::new_with_data(&workload, &data, &[0]), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &crate::cost_model::DefaultCostModel, - ) - .unwrap(); - let [deployment] = plan.deployments.as_slice() else { - panic!("one population state"); - }; - assert!(deployment - .alternatives - .iter() - .all(|alternative| alternative.total_cost.is_none())); - assert!(deployment.summary_maintenance_lifecycle_guarantee.is_none()); - assert_eq!( - plan.execution_timed_dag().unwrap_err(), - SummaryMaintenanceTimingError::UnselectedLifecycle(deployment.post_asap_node_id) - ); - } - - // A retained population and its raw input run at ingestion time; an - // Ephemeral population is rebuilt from raw input at query time. - #[test] - fn population_lifecycle_choice_decides_its_timing() { - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - let timed = |lifecycle: SummaryMaintenanceLifecycle| { - population_timings(&timed_dag( - population_evaluation(), - &workload, - &data, - Some(Horizon(10.0)), - |_| lifecycle.clone(), - )) - }; - assert_eq!( - timed(SummaryMaintenanceLifecycle::ContinuouslyMaintained), - [ - ("raw", INGEST), - ("raw", INGEST), - ("population", INGEST), - ("evaluation", QUERY) - ] - ); - assert_eq!( - timed(SummaryMaintenanceLifecycle::Ephemeral), - [ - ("raw", QUERY), - ("raw", QUERY), - ("population", QUERY), - ("evaluation", QUERY) - ] - ); - } - - // When retaining is cheaper, Planner's own selection keeps the population - // maintained at ingestion time, as realization strategies placed it before - // population timing became a lifecycle decision. - #[test] - fn planner_selection_retains_population_at_ingestion() { - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - let plan = plan_summary_maintenance_lifecycles( - population_evaluation(), - WorkloadDemand::new_with_data(&workload, &data, &[0]), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - // Shared and ContinuouslyMaintained tie at 32; the first wins. - assert!(matches!( - selected_summary_maintenance_lifecycle(&plan.deployments[0]), - Some(SummaryMaintenanceLifecycle::Shared { .. }) - )); - assert_eq!( - population_timings(&plan.execution_timed_dag().unwrap()), - [ - ("raw", INGEST), - ("raw", INGEST), - ("population", INGEST), - ("evaluation", QUERY) - ] - ); - } - - // A population feeding summary state is that state's input, not a separate - // deployment: the state's lifecycle times it. - #[test] - fn population_feeding_summary_state_follows_that_state() { - let Operator::ASAP(ASAPOp::EvaluatePopulation { - child: population, .. - }) = &population_evaluation().operator - else { - unreachable!() - }; - let state = summary(); - let Operator::ASAP(ASAPOp::SummaryAgg { - family, - input, - reduction, - grouping, - .. - }) = &state.operator - else { - unreachable!() - }; - let state = Rc::new(OperatorNode { - operator: Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(population), - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: None, - }), - ..state.as_ref().clone() - }); - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - let timed = |lifecycle: SummaryMaintenanceLifecycle| { - population_timings(&timed_dag( - evaluation(&state), - &workload, - &data, - Some(Horizon(10.0)), - |deployment| { - assert!(Rc::ptr_eq(&deployment.summary, &state)); - lifecycle.clone() - }, - )) - }; - assert_eq!( - timed(SummaryMaintenanceLifecycle::ContinuouslyMaintained), - [ - ("raw", INGEST), - ("raw", INGEST), - ("population", INGEST), - ("state", INGEST), - ("evaluation", QUERY) - ] - ); - assert_eq!( - timed(SummaryMaintenanceLifecycle::Ephemeral), - [ - ("raw", QUERY), - ("raw", QUERY), - ("population", QUERY), - ("state", QUERY), - ("evaluation", QUERY) - ] - ); - } - - // A population both read directly and consumed by summary state is that - // state's input in either traversal order: not a separate deployment, and - // timed by the state's lifecycle. - #[test] - fn shared_population_follows_its_summary_consumer() { - let direct = population_evaluation(); - let Operator::ASAP(ASAPOp::EvaluatePopulation { - child: population, .. - }) = &direct.operator - else { - unreachable!() - }; - let state = summary(); - let Operator::ASAP(ASAPOp::SummaryAgg { - family, - input, - reduction, - grouping, - .. - }) = &state.operator - else { - unreachable!() - }; - let state = Rc::new(OperatorNode { - operator: Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(population), - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: None, - }), - ..state.as_ref().clone() - }); - let binary = test_binary; - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - for root in [ - binary(Rc::clone(&direct), evaluation(&state)), - binary(evaluation(&state), Rc::clone(&direct)), - ] { - for lifecycle in [ - SummaryMaintenanceLifecycle::ContinuouslyMaintained, - SummaryMaintenanceLifecycle::Ephemeral, - ] { - let dag = timed_dag( - Rc::clone(&root), - &workload, - &data, - Some(Horizon(10.0)), - |deployment| { - assert!(Rc::ptr_eq(&deployment.summary, &state)); - lifecycle.clone() - }, - ); - let expected = if lifecycle == SummaryMaintenanceLifecycle::Ephemeral { - QUERY - } else { - INGEST - }; - for (kind, timing) in population_timings(&dag) { - if matches!(kind, "raw" | "population" | "state") { - assert_eq!(timing, expected, "{kind}"); - } else { - assert_eq!(timing, QUERY, "{kind}"); - } - } - } - } - } - - // A plan whose population deployment was removed after enumeration is - // refused rather than timed by a guess. - #[test] - fn timing_refuses_population_without_deployment() { - let data = continuous(1_000, 60_000); - let workload = workload(vec![], vec![repeating()], data.clone()); - let mut plan = plan_summary_maintenance_lifecycles( - population_evaluation(), - WorkloadDemand::new_with_data(&workload, &data, &[0]), - 1_000, - Some(Horizon(10.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &UnitCosts, - ) - .unwrap(); - let id = plan.deployments.remove(0).post_asap_node_id; - assert_eq!( - plan.execution_timed_dag(), - Err(SummaryMaintenanceTimingError::UnplannedMaintainedState(id)) - ); - } -} diff --git a/crates/asap-aware-mapping/src/test_support.rs b/crates/asap-aware-mapping/src/test_support.rs index b4d0fd99a..f410f915f 100644 --- a/crates/asap-aware-mapping/src/test_support.rs +++ b/crates/asap-aware-mapping/src/test_support.rs @@ -51,7 +51,9 @@ pub(crate) fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc) -> Rc) -> Rc { - apply_lifecycle_timings( + apply_materialization_timings( + root, + &MaterializationAssignment::all_query_time(), + &mut TimingMemo::new(), + ) + .expect("default materialization timings apply") +} + +/// `root` timed with every summary maintained at ingestion time. +pub(crate) fn maintained(root: &Rc) -> Rc { + apply_materialization_timings( root, - &LifecycleAssignment::default_maintained(), + &MaterializationAssignment::all_ingestion_time(), &mut TimingMemo::new(), ) - .expect("default lifecycle timings apply") + .expect("maintained materialization timings apply") } -/// Time `root` under the default lifecycle assignment (which runs every +/// Time `root` under the default materialization assignment (which runs every /// data-state / population-contract check) and export it as a physical ASAP DAG. pub(crate) fn time_and_export( root: &Rc, @@ -192,9 +204,9 @@ pub(crate) fn time_and_export( asap_types::ir::export::PhysicalASAPDAG, asap_types::post_asap::execution_data_state::ExecutionDataStateError, > { - let timed = apply_lifecycle_timings( + let timed = apply_materialization_timings( root, - &LifecycleAssignment::default_maintained(), + &MaterializationAssignment::all_query_time(), &mut TimingMemo::new(), )?; asap_types::ir::export::compile_physical_asap_dag(&timed) diff --git a/crates/asap-physical-operators/src/expressions/unified_planner.rs b/crates/asap-physical-operators/src/expressions/unified_planner.rs deleted file mode 100644 index a23b3dab1..000000000 --- a/crates/asap-physical-operators/src/expressions/unified_planner.rs +++ /dev/null @@ -1,738 +0,0 @@ -//! Planner scalar expressions evaluated over native typed rows. -use crate::{ - values::{SchemaRef, Value}, - Error, -}; -use planner_types::pre_asap::{ArithmeticOpKind, CompareOpKind, DataType, ScalarValue}; - -use planner_types::ir::ScalarExpr; -use std::{cmp::Ordering, sync::Arc}; - -pub(super) fn evaluate( - expr: &ScalarExpr, - row: &[Value], - schema: &planner_types::pre_asap::Schema, -) -> Result { - match expr { - ScalarExpr::Column(index) => row.get(*index).cloned().ok_or(Error::Invalid(format!( - "column {index} outside row width {}", - row.len() - ))), - ScalarExpr::Literal(value) => Ok(match value { - ScalarValue::Interval { - months, - days, - nanos, - } => Value::Interval { - months: *months, - days: *days, - nanos: *nanos, - }, - ScalarValue::Int64(value) => Value::Int64(*value), - ScalarValue::Float64(value) => Value::Float64(*value), - ScalarValue::Utf8(value) => Value::Utf8(value.clone().into()), - ScalarValue::Boolean(value) => Value::Bool(*value), - ScalarValue::Null => Value::Null, - }), - ScalarExpr::Cast { expr, to, .. } => { - let value = evaluate(expr, row, schema)?; - match (value, to) { - (Value::Null, _) => Ok(Value::Null), - (Value::Int64(value), DataType::Float64) => Ok(Value::Float64(value as f64)), - (value, _) - if expr - .scalar_type(schema) - .map_err(|e| Error::Invalid(e.to_string()))? - .0 - == *to => - { - Ok(value) - } - _ => Err(Error::Invalid("unsupported cast".into())), - } - } - ScalarExpr::Negative { expr, .. } => match evaluate(expr, row, schema)? { - Value::Float64(v) => Ok(Value::Float64(-v)), - Value::Int64(v) => v - .checked_neg() - .map(Value::Int64) - .ok_or_else(|| Error::Invalid("integer negation overflow".into())), - Value::Null => Ok(Value::Null), - _ => Err(Error::Invalid("invalid negation input".into())), - }, - ScalarExpr::Compare { - left, op, right, .. - } => { - let left = evaluate(left, row, schema)?; - let right = evaluate(right, row, schema)?; - compare(op, left, right) - } - ScalarExpr::Arithmetic { - op, left, right, .. - } => arithmetic( - op, - evaluate(left, row, schema)?, - evaluate(right, row, schema)?, - ), - ScalarExpr::Case { - operand: None, - branches, - else_expr, - } => { - for (condition, value) in branches { - if matches!(evaluate(condition, row, schema)?, Value::Bool(true)) { - return evaluate(value, row, schema); - } - } - else_expr - .as_ref() - .map_or(Ok(Value::Null), |e| evaluate(e, row, schema)) - } - ScalarExpr::BoolAnd(parts) | ScalarExpr::BoolOr(parts) => { - let and = matches!(expr, ScalarExpr::BoolAnd(_)); - let mut null = false; - for part in parts { - match evaluate(part, row, schema)? { - Value::Bool(value) if value != and => return Ok(Value::Bool(value)), - Value::Bool(_) => {} - Value::Null => null = true, - _ => return Err(Error::Invalid("boolean predicate required".into())), - } - } - Ok(if null { Value::Null } else { Value::Bool(and) }) - } - ScalarExpr::Not(value) => match evaluate(value, row, schema)? { - Value::Bool(value) => Ok(Value::Bool(!value)), - Value::Null => Ok(Value::Null), - _ => Err(Error::Invalid("boolean predicate required".into())), - }, - ScalarExpr::IsNull(value) => Ok(Value::Bool(matches!( - evaluate(value, row, schema)?, - Value::Null - ))), - ScalarExpr::IsNotNull(value) => Ok(Value::Bool(!matches!( - evaluate(value, row, schema)?, - Value::Null - ))), - ScalarExpr::FunctionCall { name, args } => { - use planner_types::pre_asap::scalar_type_rules::MapScalarFunction; - if planner_types::pre_asap::scalar_type_rules::promql_function_arity(name).is_some() { - let values = args - .iter() - .map(|arg| match evaluate(arg, row, schema)? { - Value::Float64(v) => Ok(v), - _ => Err(Error::Invalid("PromQL function requires floats".into())), - }) - .collect::, _>>()?; - return Ok(Value::Float64(promql_function(name, &values)?)); - } - if name == "promql_drop_metric_name" { - let Value::Utf8(encoded) = evaluate(&args[0], row, schema)? else { - return Err(Error::Invalid("series identity must be Utf8".into())); - }; - let mut labels: std::collections::BTreeMap = - serde_json::from_str(&encoded).map_err(|e| Error::Invalid(e.to_string()))?; - labels.remove("__name__"); - return Ok(Value::Utf8( - serde_json::to_string(&labels) - .map_err(|e| Error::Invalid(e.to_string()))? - .into(), - )); - } - if name.eq_ignore_ascii_case("asap_struct_field") { - expr.scalar_type(schema) - .map_err(|error| Error::Invalid(error.to_string()))?; - let DataType::Struct { fields } = args[0] - .scalar_type(schema) - .map_err(|error| Error::Invalid(error.to_string()))? - .0 - else { - unreachable!() - }; - let offset = match &args[1] { - ScalarExpr::Literal(ScalarValue::Int64(index)) => { - usize::try_from(index - 1).ok() - } - ScalarExpr::Literal(ScalarValue::Utf8(name)) => { - fields.iter().position(|field| &field.name == name) - } - _ => None, - } - .ok_or_else(|| Error::Invalid("struct field selector".into()))?; - let Value::Struct(values) = evaluate(&args[0], row, schema)? else { - return Err(Error::Invalid("struct field input".into())); - }; - return values - .get(offset) - .cloned() - .ok_or_else(|| Error::Invalid("struct field value".into())); - } - if name.eq_ignore_ascii_case("asap_element_access") { - let (output_type, _) = expr - .scalar_type(schema) - .map_err(|error| Error::Invalid(error.to_string()))?; - if let DataType::List { element } = args[0] - .scalar_type(schema) - .map_err(|error| Error::Invalid(error.to_string()))? - .0 - { - let Value::List(values) = evaluate(&args[0], row, schema)? else { - return Err(Error::Invalid("array access input".into())); - }; - let index = match evaluate(&args[1], row, schema)? { - Value::Null => return Ok(Value::Null), - Value::Int64(index) => index, - _ => return Err(Error::Invalid("array access index".into())), - }; - let offset = if index > 0 { - usize::try_from(index - 1).ok() - } else if index < 0 { - usize::try_from(index.unsigned_abs()) - .ok() - .and_then(|distance| values.len().checked_sub(distance)) - } else { - None - }; - return match offset.and_then(|offset| values.get(offset)) { - Some(value) => Ok(value.clone()), - None => default_collection_element(&output_type, element.nullable), - }; - } - } - let function = (if name.eq_ignore_ascii_case("asap_element_access") { - Some(MapScalarFunction::Access) - } else { - MapScalarFunction::from_name(name) - }) - .ok_or_else(|| Error::Invalid(format!("scalar function {name}")))?; - expr.scalar_type(schema) - .map_err(|error| Error::Invalid(error.to_string()))?; - let values = args - .iter() - .map(|arg| evaluate(arg, row, schema)) - .collect::, _>>()?; - match function { - MapScalarFunction::Construct => { - let mut values = values.into_iter(); - let mut entries = Vec::new(); - while let Some(key) = values.next() { - if !matches!(key, Value::Int64(_) | Value::Utf8(_) | Value::Bool(_)) { - return Err(Error::Invalid("map key value type".into())); - } - entries.push(( - key, - values - .next() - .ok_or_else(|| Error::Invalid("odd map argument count".into()))?, - )); - } - Ok(Value::Map(entries.into())) - } - MapScalarFunction::Concat => { - let mut entries = Vec::new(); - for value in values { - let Value::Map(next) = value else { - return Err(Error::Invalid("map concat argument".into())); - }; - entries.extend(next.iter().cloned()); - } - Ok(Value::Map(entries.into())) - } - MapScalarFunction::Access => { - let [Value::Map(entries), key] = values.as_slice() else { - return Err(Error::Invalid("map access arguments".into())); - }; - if matches!(key, Value::Null) { - return Ok(Value::Null); - } - if !matches!(key, Value::Int64(_) | Value::Utf8(_) | Value::Bool(_)) { - return Err(Error::Invalid("map lookup key type".into())); - } - if let Some((_, value)) = entries - .iter() - .find(|(candidate, _)| cell_cmp(candidate, key) == Some(Ordering::Equal)) - { - return Ok(value.clone()); - } - let ( - DataType::Map { - value, - value_nullable, - .. - }, - _, - ) = args[0] - .scalar_type(schema) - .map_err(|error| Error::Invalid(error.to_string()))? - else { - unreachable!() - }; - default_collection_element(&value, value_nullable) - } - } - } - other => Err(Error::Invalid(format!("scalar expression {other:?}"))), - } -} - -fn default_collection_element(dtype: &DataType, nullable: bool) -> Result { - if nullable { - return Ok(Value::Null); - } - Ok(match dtype { - DataType::Interval | DataType::Date => { - return Err(Error::Invalid("temporal value transport".into())) - } - DataType::Null => Value::Null, - DataType::Int64 => Value::Int64(0), - DataType::Float64 => Value::Float64(0.0), - DataType::Utf8 => Value::Utf8("".into()), - DataType::Bool => Value::Bool(false), - DataType::Map { .. } => Value::Map(Arc::from([])), - DataType::List { .. } => Value::List(Arc::from([])), - DataType::Struct { fields } => Value::Struct( - fields - .iter() - .map(|field| default_collection_element(&field.dtype, field.nullable)) - .collect::, _>>()? - .into(), - ), - _ => { - return Err(Error::Invalid( - "collection missing-element default type".into(), - )) - } - }) -} - -fn compare(op: &CompareOpKind, left: Value, right: Value) -> Result { - if matches!(left, Value::Null) || matches!(right, Value::Null) { - return Ok(Value::Null); - } - // NaN is unordered, not a type mismatch. Match the native scalar path. - if matches!(&left, Value::Float64(v) if v.is_nan()) - || matches!(&right, Value::Float64(v) if v.is_nan()) - { - return match op { - CompareOpKind::Ne => Ok(Value::Bool(true)), - CompareOpKind::Eq - | CompareOpKind::Lt - | CompareOpKind::Le - | CompareOpKind::Gt - | CompareOpKind::Ge => Ok(Value::Bool(false)), - _ => Err(Error::Invalid(format!("comparison {op:?}"))), - }; - } - let ordering = cell_cmp(&left, &right) - .ok_or_else(|| Error::Invalid("comparison of incompatible values".into()))?; - let value = match op { - CompareOpKind::Eq => ordering == Ordering::Equal, - CompareOpKind::Ne => ordering != Ordering::Equal, - CompareOpKind::Lt => ordering == Ordering::Less, - CompareOpKind::Le => ordering != Ordering::Greater, - CompareOpKind::Gt => ordering == Ordering::Greater, - CompareOpKind::Ge => ordering != Ordering::Less, - _ => return Err(Error::Invalid(format!("comparison {op:?}"))), - }; - Ok(Value::Bool(value)) -} - -fn arithmetic(op: &ArithmeticOpKind, left: Value, right: Value) -> Result { - let (left, right) = match (left, right) { - (Value::Int64(a), Value::Float64(b)) => (Value::Float64(a as f64), Value::Float64(b)), - (Value::Float64(a), Value::Int64(b)) => (Value::Float64(a), Value::Float64(b as f64)), - pair => pair, - }; - super::numeric(op, left, right) -} - -fn integer_float_cmp(integer: i64, float: f64) -> Option { - if float.is_nan() { - return None; - } - // These bounds are powers of two, exactly representable as Float64. - if float >= 9_223_372_036_854_775_808.0 { - return Some(Ordering::Less); - } - if float < -9_223_372_036_854_775_808.0 { - return Some(Ordering::Greater); - } - let integral = float as i64; - match integer.cmp(&integral) { - Ordering::Equal => 0.0_f64.partial_cmp(&float.fract()), - other => Some(other), - } -} - -fn cell_cmp(left: &Value, right: &Value) -> Option { - match (left, right) { - (Value::Int64(left), Value::Int64(right)) => Some(left.cmp(right)), - (Value::Float64(left), Value::Float64(right)) => left.partial_cmp(right), - (Value::Int64(left), Value::Float64(right)) => integer_float_cmp(*left, *right), - (Value::Float64(left), Value::Int64(right)) => { - integer_float_cmp(*right, *left).map(Ordering::reverse) - } - (Value::Utf8(left), Value::Utf8(right)) => Some(left.cmp(right)), - (Value::Bool(left), Value::Bool(right)) => Some(left.cmp(right)), - (Value::Timestamp(left), Value::Timestamp(right)) => Some(left.cmp(right)), - (Value::Map(left), Value::Map(right)) => { - for ((left_key, left_value), (right_key, right_value)) in left.iter().zip(right.iter()) - { - let order = cell_cmp(left_key, right_key)?; - if order != Ordering::Equal { - return Some(order); - } - let order = match (left_value, right_value) { - (Value::Null, Value::Null) => Ordering::Equal, - (Value::Null, _) => Ordering::Greater, - (_, Value::Null) => Ordering::Less, - _ => cell_cmp(left_value, right_value)?, - }; - if order != Ordering::Equal { - return Some(order); - } - } - Some(left.len().cmp(&right.len())) - } - _ => None, - } -} - -fn promql_function(name: &str, args: &[f64]) -> Result { - let x = args[0]; - Ok(match &name[7..] { - "abs" => x.abs(), - "ceil" => x.ceil(), - "floor" => x.floor(), - "exp" => x.exp(), - "ln" => x.ln(), - "log2" => x.log2(), - "log10" => x.log10(), - "sqrt" => x.sqrt(), - "sgn" => { - if x.is_nan() { - f64::NAN - } else if x == 0.0 { - 0.0 - } else { - x.signum() - } - } - "sin" => x.sin(), - "cos" => x.cos(), - "tan" => x.tan(), - "asin" => x.asin(), - "acos" => x.acos(), - "atan" => x.atan(), - "sinh" => x.sinh(), - "cosh" => x.cosh(), - "tanh" => x.tanh(), - "asinh" => x.asinh(), - "acosh" => x.acosh(), - "atanh" => x.atanh(), - "deg" => x.to_degrees(), - "rad" => x.to_radians(), - "round" => { - let inverse = 1.0 / args[1]; - (x * inverse + 0.5).floor() / inverse - } - "clamp_min" => { - if x.is_nan() || args[1].is_nan() { - f64::NAN - } else { - x.max(args[1]) - } - } - "clamp_max" => { - if x.is_nan() || args[1].is_nan() { - f64::NAN - } else { - x.min(args[1]) - } - } - "clamp" => { - if args.iter().any(|x| x.is_nan()) { - f64::NAN - } else { - x.max(args[1]).min(args[2]) - } - } - part => { - use chrono::{Datelike, Timelike}; - if !x.is_finite() || x < i64::MIN as f64 || x >= i64::MAX as f64 { - return Ok(f64::NAN); - } - let Some(date) = chrono::DateTime::from_timestamp(x as i64, 0) else { - return Ok(f64::NAN); - }; - match part { - "minute" => date.minute() as f64, - "hour" => date.hour() as f64, - "day_of_week" => date.weekday().num_days_from_sunday() as f64, - "day_of_month" => date.day() as f64, - "day_of_year" => date.ordinal() as f64, - "month" => date.month() as f64, - "year" => date.year() as f64, - "days_in_month" => { - let year = date.year(); - let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); - match date.month() { - 2 => { - if leap { - 29.0 - } else { - 28.0 - } - } - 4 | 6 | 9 | 11 => 30.0, - _ => 31.0, - } - } - _ => return Err(Error::Invalid("unregistered PromQL function".into())), - } - } - }) -} - -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -pub struct CompiledExpression { - expression: ScalarExpr, - schema: planner_types::pre_asap::Schema, - output: (DataType, bool), -} -impl CompiledExpression { - pub fn compile(expression: &ScalarExpr, input: &SchemaRef) -> Result { - if !input.is_all_plain() { - return Err(Error::Invalid( - "scalar expression cannot consume summary state".into(), - )); - } - let schema = input.as_ref().clone(); - validate(expression, &schema)?; - let output = expression - .scalar_type(&schema) - .map_err(|e| Error::Invalid(e.to_string()))?; - Ok(Self { - expression: expression.clone(), - schema, - output, - }) - } - pub(crate) fn dtype(&self) -> (DataType, bool) { - self.output.clone() - } - pub(crate) fn validate_input(&self, input: &SchemaRef) -> Result<(), Error> { - let checked = Self::compile(&self.expression, input)?; - if checked.output != self.output { - return Err(Error::Invalid( - "persisted expression type differs from its semantics".into(), - )); - } - if input.fields.len() != self.schema.fields.len() - || input - .fields - .iter() - .zip(&self.schema.fields) - .any(|(field, column)| { - field.dtype != column.dtype.clone() || field.nullable != column.nullable - }) - { - return Err(Error::Invalid( - "expression input differs from its bound schema".into(), - )); - } - Ok(()) - } - /// Evaluate a row under the same typed schema used when binding the expression. - pub fn evaluate(&self, row: &[Value]) -> Result { - if row.len() != self.schema.fields.len() - || row.iter().zip(&self.schema.fields).any(|(value, column)| { - !column - .plain_dtype() - .is_some_and(|dtype| value.matches(dtype, column.nullable)) - }) - { - return Err(Error::Invalid( - "expression input differs from its bound schema".into(), - )); - } - evaluate(&self.expression, row, &self.schema) - } -} -fn validate(expr: &ScalarExpr, schema: &planner_types::pre_asap::Schema) -> Result<(), Error> { - let invalid = || Error::Invalid(format!("unsupported scalar expression: {expr:?}")); - expr.scalar_type(schema) - .map_err(|e| Error::Invalid(e.to_string()))?; - match expr { - ScalarExpr::Column(_) | ScalarExpr::Literal(_) => Ok(()), - ScalarExpr::Cast { expr, to, .. } => { - let source = expr - .scalar_type(schema) - .map_err(|e| Error::Invalid(e.to_string()))? - .0; - if source != *to - && source != DataType::Null - && !(source == DataType::Int64 && *to == DataType::Float64) - { - return Err(invalid()); - } - validate(expr, schema) - } - ScalarExpr::Negative { expr, .. } => validate(expr, schema), - ScalarExpr::Arithmetic { left, right, .. } => { - for value in [left, right] { - validate(value, schema)?; - if !matches!( - value - .scalar_type(schema) - .map_err(|e| Error::Invalid(e.to_string()))? - .0, - DataType::Int64 | DataType::Float64 | DataType::Null - ) { - return Err(invalid()); - } - } - Ok(()) - } - ScalarExpr::Compare { - left, right, op, .. - } => { - if !matches!( - op, - CompareOpKind::Eq - | CompareOpKind::Ne - | CompareOpKind::Lt - | CompareOpKind::Le - | CompareOpKind::Gt - | CompareOpKind::Ge - ) { - return Err(invalid()); - } - validate(left, schema)?; - validate(right, schema)?; - let (a, _) = left - .scalar_type(schema) - .map_err(|e| Error::Invalid(e.to_string()))?; - let (b, _) = right - .scalar_type(schema) - .map_err(|e| Error::Invalid(e.to_string()))?; - fn comparable(dtype: &DataType) -> bool { - match dtype { - DataType::Null - | DataType::Int64 - | DataType::Float64 - | DataType::Utf8 - | DataType::Bool - | DataType::Timestamp => true, - DataType::Map { key, value, .. } => comparable(key) && comparable(value), - _ => false, - } - } - let numeric = |dtype: &DataType| matches!(dtype, DataType::Int64 | DataType::Float64); - if !comparable(&a) - || !comparable(&b) - || (a != b - && !matches!(a, DataType::Null) - && !matches!(b, DataType::Null) - && !(numeric(&a) && numeric(&b))) - { - return Err(invalid()); - } - Ok(()) - } - ScalarExpr::FunctionCall { name, args } => { - if name != "promql_drop_metric_name" - && planner_types::pre_asap::scalar_type_rules::promql_function_arity(name).is_none() - && name != "asap_struct_field" - && name != "asap_element_access" - && planner_types::pre_asap::scalar_type_rules::MapScalarFunction::from_name(name) - .is_none() - { - return Err(invalid()); - } - for arg in args { - validate(arg, schema)?; - } - Ok(()) - } - ScalarExpr::Case { - operand: None, - branches, - else_expr, - } => { - for (condition, value) in branches { - validate(condition, schema)?; - if condition - .scalar_type(schema) - .map_err(|e| Error::Invalid(e.to_string()))? - .0 - != DataType::Bool - { - return Err(invalid()); - } - validate(value, schema)?; - } - if let Some(value) = else_expr { - validate(value, schema)?; - } - Ok(()) - } - ScalarExpr::BoolAnd(parts) | ScalarExpr::BoolOr(parts) => { - for part in parts { - validate(part, schema)?; - if !matches!( - part.scalar_type(schema) - .map_err(|e| Error::Invalid(e.to_string()))? - .0, - DataType::Bool | DataType::Null - ) { - return Err(invalid()); - } - } - Ok(()) - } - ScalarExpr::Not(value) => { - validate(value, schema)?; - if !matches!( - value - .scalar_type(schema) - .map_err(|e| Error::Invalid(e.to_string()))? - .0, - DataType::Bool | DataType::Null - ) { - return Err(invalid()); - } - Ok(()) - } - ScalarExpr::IsNull(value) | ScalarExpr::IsNotNull(value) => validate(value, schema), - _ => Err(invalid()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn mixed_comparison_preserves_integer_precision_and_boundaries() { - assert_eq!( - integer_float_cmp(9_007_199_254_740_993, 9_007_199_254_740_992.0), - Some(Ordering::Greater) - ); - assert_eq!( - integer_float_cmp(i64::MAX, 9_223_372_036_854_775_808.0), - Some(Ordering::Less) - ); - assert_eq!( - integer_float_cmp(i64::MIN, -9_223_372_036_854_775_808.0), - Some(Ordering::Equal) - ); - assert_eq!(integer_float_cmp(-1, -1.5), Some(Ordering::Greater)); - assert_eq!(integer_float_cmp(1, 1.5), Some(Ordering::Less)); - assert_eq!(integer_float_cmp(0, f64::INFINITY), Some(Ordering::Less)); - assert_eq!( - integer_float_cmp(0, f64::NEG_INFINITY), - Some(Ordering::Greater) - ); - assert_eq!(integer_float_cmp(0, f64::NAN), None); - } -} diff --git a/crates/asap-physical-operators/src/physical_planner/candidates.rs b/crates/asap-physical-operators/src/physical_planner/candidates.rs index eacd1cfa2..afc674418 100644 --- a/crates/asap-physical-operators/src/physical_planner/candidates.rs +++ b/crates/asap-physical-operators/src/physical_planner/candidates.rs @@ -1,7 +1,7 @@ //! Compile maintenance-selected frontiers without deployment-specific DAG rewrites. use super::*; -/// One computation realization; lifecycle/window/revision requirements accompany +/// One computation realization; materialization/window/revision requirements accompany /// it during optimization and deployment. Stored outputs have no storage identity. /// Deserialization validates the producer/reader boundary. #[derive(Clone, serde::Serialize, serde::Deserialize)] @@ -86,7 +86,7 @@ pub fn cut_candidate( }) } -/// Materialization frontier implied by lifecycle-assigned timing: ingestion-time +/// Materialization frontier implied by materialization-assigned timing: ingestion-time /// nodes read by a query-time node, plus the root when it is ingestion-timed. /// `cut_candidate` of one [`compile`] result with this frontier realizes the /// assignment, so different assignments are different cuts of one lowering. @@ -125,7 +125,7 @@ pub fn frontier_from_timing(dag: &PhysicalASAPDAG) -> Result, Error> /// Enumerate bounded, reachable materialization frontiers above explicit inputs. /// Each frontier is an antichain: storing an output and its ancestor together -/// would leave the ancestor unused by query execution. Lifecycle eligibility +/// would leave the ancestor unused by query execution. Materialization eligibility /// and deployment feasibility are evaluated separately before cost selection. /// Exceeding the search budget returns an error, never a partial inventory. pub fn enumerate_frontiers( @@ -370,9 +370,9 @@ mod tests { .assemble_selected_dag(&space.roots[0].1) .unwrap() .unwrap(); - let selected = planner_types::ir::apply_lifecycle_timings( + let selected = planner_types::ir::apply_materialization_timings( &selected, - &Default::default(), + &planner_types::ir::MaterializationAssignment::all_ingestion_time(), &mut Default::default(), ) .unwrap(); @@ -440,8 +440,8 @@ mod tests { )]) } - /// Cutting one compilation by a retained-state timing and by the all - /// query-time timing (what ContinuouslyMaintained and Ephemeral assign) + /// Cutting one compilation by a maintained-state timing and by the all + /// query-time timing /// lowers each Planner node once and matches `compile_candidate`. #[test] fn timing_cuts_share_one_lowering() { diff --git a/crates/asap-physical-operators/src/physical_planner/promql_rows.rs b/crates/asap-physical-operators/src/physical_planner/promql_rows.rs index 20992dfe7..f85d187d5 100644 --- a/crates/asap-physical-operators/src/physical_planner/promql_rows.rs +++ b/crates/asap-physical-operators/src/physical_planner/promql_rows.rs @@ -88,9 +88,10 @@ pub fn compile_current_series_evaluation( use planner_types::post_asap::{ maintained_population::PopulationStatistic, Field as SummaryField, }; - let selected = planner_types::ir::apply_lifecycle_timings( + // This compiler emits maintained precompute: every summary at ingestion time. + let selected = planner_types::ir::apply_materialization_timings( selected, - &planner_types::ir::LifecycleAssignment::default_maintained(), + &planner_types::ir::MaterializationAssignment::all_ingestion_time(), &mut planner_types::ir::TimingMemo::new(), ) .map_err(|e| invalid(e.to_string()))?; @@ -201,9 +202,10 @@ pub fn compile_rate_ranking( } node.children().into_iter().find_map(frontier) } - let selected = planner_types::ir::apply_lifecycle_timings( + // This compiler emits maintained precompute: every summary at ingestion time. + let selected = planner_types::ir::apply_materialization_timings( selected, - &planner_types::ir::LifecycleAssignment::default_maintained(), + &planner_types::ir::MaterializationAssignment::all_ingestion_time(), &mut planner_types::ir::TimingMemo::new(), ) .map_err(|e| invalid(e.to_string()))?; @@ -239,7 +241,7 @@ pub fn compile_rate_ranking( Ok((source, program)) } -/// Compile a lifecycle-timed DAG whose heap or grouped Sum over per-series +/// Compile a materialization-timed DAG whose heap or grouped Sum over per-series /// Rate evaluations runs at ingestion time: fresh aggregate state per closed /// window. The input is the complete collection of per-series counter states. pub fn compile_fixed_window_rate_aggregation( diff --git a/crates/asap-physical-operators/src/readout.rs b/crates/asap-physical-operators/src/readout.rs deleted file mode 100644 index e84d8c828..000000000 --- a/crates/asap-physical-operators/src/readout.rs +++ /dev/null @@ -1,111 +0,0 @@ -//! Readouts over merged exact summary states. -use crate::summary_kernels::exact::ExactAccumulator; -use crate::{AggregateCore, KeyByLabelValues, Statistic}; -use std::sync::Arc; - -fn merge_exact_states( - states: impl IntoIterator>, -) -> Result { - let mut states = states.into_iter(); - let exact = |state: &Arc| { - state - .as_any() - .downcast_ref::() - .cloned() - .ok_or_else(|| "readout requires Planner exact state".to_string()) - }; - let mut merged = exact(&states.next().ok_or("empty exact state input")?)?; - for state in states { - merged - .merge_from(&exact(&state)?) - .map_err(|error| error.to_string())?; - } - Ok(merged) -} - -/// PromQL counter readouts omit a series with fewer than two samples. Other -/// state/type/range failures remain errors rather than empty results. -pub fn insufficient_counter_samples(state: &dyn AggregateCore, statistic: Statistic) -> bool { - matches!(statistic, Statistic::Rate | Statistic::Increase) - && state - .as_any() - .downcast_ref::() - .is_some_and(|state| state.insufficient_counter_samples(statistic, &None)) -} - -/// Merge already selected exact panes and read one population. `None` means -/// the population is absent from the result: a counter with too few samples, -/// or an empty MIN/MAX. -pub fn exact_readout( - states: impl IntoIterator>, - statistic: Statistic, - range_ms: Option<(i64, i64)>, - key: Option<&KeyByLabelValues>, -) -> Result, String> { - let merged = merge_exact_states(states)?; - if merged.insufficient_counter_samples(statistic, &key.cloned()) { - return Ok(None); - } - merged - .readout(statistic, range_ms, key) - .map_err(|error| error.to_string()) -} - -#[cfg(test)] -mod counter_tests { - use super::*; - use planner_types::post_asap::{ExactKind, ExactParams, FieldDataType}; - - fn counter(kind: ExactKind, params: ExactParams, keyed: bool) -> ExactAccumulator { - ExactAccumulator::new(FieldDataType::ExactAggregate(kind, params), keyed).unwrap() - } - - // A counter population with a single sample is absent, keyed or not. - #[test] - fn planner_counter_population_omits_insufficient_samples() { - for (kind, params, statistic) in [ - (ExactKind::Rate, ExactParams::Rate, Statistic::Rate), - ( - ExactKind::Increase, - ExactParams::Increase, - Statistic::Increase, - ), - ] { - for keyed in [false, true] { - let mut state = counter(kind.clone(), params.clone(), keyed); - let key = keyed.then(|| KeyByLabelValues::new_with_labels(vec!["checkout".into()])); - state.update(key.as_ref(), 10., 10_000); - assert_eq!( - exact_readout( - [Arc::new(state) as Arc], - statistic, - None, - key.as_ref() - ) - .unwrap(), - None - ); - } - } - } - - // Two ordered samples read a rate; an inverted range and empty input fail. - #[test] - fn sparse_counter_is_absent_but_invalid_ranges_still_fail() { - let mut state = counter(ExactKind::Rate, ExactParams::Rate, false); - state.update(None, 10., 10_000); - let rate = Statistic::Rate; - let one = [Arc::new(state.clone()) as Arc]; - assert_eq!( - exact_readout(one, rate, Some((0, 60_000)), None).unwrap(), - None - ); - state.update(None, 20., 20_000); - let two = || [Arc::new(state.clone()) as Arc]; - assert!(exact_readout(two(), rate, Some((0, 60_000)), None) - .unwrap() - .is_some()); - assert!(exact_readout(two(), rate, Some((60_000, 0)), None).is_err()); - assert!(exact_readout([], rate, Some((0, 60_000)), None).is_err()); - } -} diff --git a/crates/asap-physical-operators/src/unified_physical_planner/candidates.rs b/crates/asap-physical-operators/src/unified_physical_planner/candidates.rs deleted file mode 100644 index 406013eb9..000000000 --- a/crates/asap-physical-operators/src/unified_physical_planner/candidates.rs +++ /dev/null @@ -1,329 +0,0 @@ -//! Compile maintenance-selected frontiers without deployment-specific dag rewrites. -use super::*; - -/// One computation realization; lifecycle/window/revision requirements accompany -/// it during optimization and deployment. Stored outputs have no storage identity. -/// Deserialization validates the producer/reader boundary. -#[derive(Clone, serde::Serialize, serde::Deserialize)] -#[serde(try_from = "UncheckedCompiledPhysicalPlan")] -pub struct CompiledPhysicalPlan { - pub precompute: Option, - pub query: CompiledPhysicalDAG, - pub materialized_outputs: BTreeMap, -} - -/// Compile an explicit materialization frontier selected by Planner maintenance -/// search. Operators upstream of that frontier run in precompute, including -/// evaluations/reductions; query execution receives their typed output values. -/// Empty frontiers retain the full computation in the query DAG. -/// -/// Repeated windows must be instantiated with the same evaluation/population -/// contract used to build each output. This API never treats a result from a -/// different window or revision as interchangeable merely because types match. -pub fn compile_candidate( - dag: &PhysicalASAPDAG, - inputs: BTreeMap, - roots: &[NodeId], - frontier: &[NodeId], -) -> Result { - cut_candidate(&compile(dag, inputs, roots)?, frontier) -} - -/// Derive one frontier's candidate from a complete [`compile`] result by -/// partitioning its operators; nothing is lowered again. A deployment compiles -/// each query DAG once and derives every placement choice from that result. -/// The candidate is identical to [`compile_candidate`] for the same frontier. -pub fn cut_candidate( - compiled: &CompiledPhysicalDAG, - frontier: &[NodeId], -) -> Result { - if frontier.is_empty() { - return Ok(CompiledPhysicalPlan { - precompute: None, - query: compiled.clone(), - materialized_outputs: BTreeMap::new(), - }); - } - let frontier_set: BTreeSet<_> = frontier.iter().copied().collect(); - // `compile` retains only reachable nodes and numbers its helper operators - // above the u32 Planner ID range; only Planner outputs are boundaries. - if frontier_set.len() != frontier.len() - || frontier - .iter() - .any(|&id| !compiled.is_operator(id) || u32::try_from(id).is_err()) - { - return Err(invalid("frontier must contain distinct computed outputs")); - } - let inputs: BTreeMap<_, _> = compiled - .input_contracts() - .map(|(id, contract)| (id, contract.clone())) - .collect(); - let precompute = compiled.cut(&inputs, frontier)?; - let mut materialized_outputs = BTreeMap::new(); - for &id in frontier { - let mut output = precompute.output_contract(id)?; - if output.properties.boundedness != Boundedness::Bounded { - return Err(invalid("materialized output requires bounded execution")); - } - // A stored reader may stream batches even when the producer blocked. - // Its timing is independent; the retained result still must be finite. - output.properties.emission = Emission::Unknown; - materialized_outputs.insert(id, output); - } - let mut query_inputs = inputs; - query_inputs.extend(materialized_outputs.clone()); - let query = compiled.cut(&query_inputs, compiled.roots())?; - let used: BTreeSet<_> = query.input_contracts().map(|(id, _)| id).collect(); - if !frontier.iter().all(|id| used.contains(id)) { - return Err(invalid( - "frontier contains an output shadowed by another boundary", - )); - } - Ok(CompiledPhysicalPlan { - precompute: Some(precompute), - query, - materialized_outputs, - }) -} - -/// Materialization frontier implied by lifecycle-assigned timing: ingestion-time -/// nodes read by a query-time node, plus the root when it is ingestion-timed. -/// `cut_candidate` of one [`compile`] result with this frontier realizes the -/// assignment, so different assignments are different cuts of one lowering. -/// That holds while timing-dependent lowering (an ingestion-time `Binary` -/// aligns by value column) has the same timing at compile time as here. -/// A query-time node feeding an ingestion-time node has no valid placement. -pub fn frontier_from_timing(dag: &PhysicalASAPDAG) -> Result, Error> { - use planner_types::post_asap::ExecutionTiming::IngestionTime; - let timing = dag - .nodes - .iter() - .map(|node| (node.id, node.output_state.timing)) - .collect::>(); - let mut frontier = BTreeSet::new(); - for root in &dag.roots { - if timing.get(root) == Some(&IngestionTime) { - frontier.insert(u64::from(root.0)); - } - } - for edge in &dag.edges { - let (Some(&producer), Some(&consumer)) = - (timing.get(&edge.producer), timing.get(&edge.consumer)) - else { - return Err(invalid("timed DAG edge names an unknown node")); - }; - match (producer == IngestionTime, consumer == IngestionTime) { - (true, false) => { - frontier.insert(u64::from(edge.producer.0)); - } - (false, true) => return Err(invalid("query-time node feeds an ingestion-time node")), - _ => {} - } - } - Ok(frontier.into_iter().collect()) -} - -/// Enumerate bounded, reachable materialization frontiers above explicit inputs. -/// Each frontier is an antichain: storing an output and its ancestor together -/// would leave the ancestor unused by query execution. Lifecycle eligibility -/// and deployment feasibility are evaluated separately before cost selection. -/// Exceeding the search budget returns an error, never a partial inventory. -pub fn enumerate_frontiers( - dag: &PhysicalASAPDAG, - inputs: &BTreeMap, - roots: &[NodeId], - max_candidates: usize, -) -> Result>, Error> { - enumerate_compiled_frontiers(&compile(dag, inputs.clone(), roots)?, max_candidates) -} - -fn enumerate_compiled_frontiers( - compiled: &CompiledPhysicalDAG, - max_candidates: usize, -) -> Result>, Error> { - if max_candidates == 0 { - return Err(invalid( - "frontier search requires a positive candidate budget", - )); - } - let mut ancestors = BTreeMap::>::new(); - let mut eligible = Vec::new(); - for (id, properties) in compiled.output_properties()? { - if !compiled.is_operator(id) - || u32::try_from(id).is_err() - || properties.boundedness != Boundedness::Bounded - { - continue; - } - let mut seen = BTreeSet::new(); - let mut pending = vec![id]; - while let Some(current) = pending.pop() { - if seen.insert(current) { - pending.extend(compiled.dependencies(current)); - } - } - ancestors.insert(id, seen); - eligible.push(id); - } - let mut frontiers = vec![vec![]]; - for id in eligible { - let additions = frontiers - .iter() - .filter(|frontier| { - frontier.iter().all(|previous| { - !ancestors[&id].contains(previous) && !ancestors[previous].contains(&id) - }) - }) - .map(|frontier| { - let mut next = frontier.clone(); - next.push(id); - next - }) - .collect::>(); - if additions.len() > max_candidates.saturating_sub(frontiers.len()) { - return Err(invalid( - "materialization frontier search exceeds candidate budget", - )); - } - frontiers.extend(additions); - } - Ok(frontiers) -} - -/// Lower every maintenance candidate before feasibility/cost evaluation. Keep -/// individual failures visible; do not substitute another computation on error. -/// The DAG is lowered once; each frontier is a [`cut_candidate`] of it. -pub fn compile_candidates( - dag: &PhysicalASAPDAG, - inputs: BTreeMap, - roots: &[NodeId], - frontiers: &[Vec], -) -> Vec> { - match compile(dag, inputs, roots) { - Ok(compiled) => frontiers - .iter() - .map(|frontier| cut_candidate(&compiled, frontier)) - .collect(), - Err(error) => frontiers.iter().map(|_| Err(error.clone())).collect(), - } -} - -/// Complete workload cost supplied by scoped optimizer/deployment evidence. -/// The evaluator includes build/update work, retained state, shared producers -/// and recurrent reads over the same horizon; these are not per-query timings. -#[derive(Clone, Debug)] -pub struct CandidateCost { - pub workload_scope: String, - pub horizon_seconds: f64, - pub total_cost: f64, -} - -pub struct CandidateSelection { - pub candidate: T, - pub candidate_index: usize, - pub cost: CandidateCost, -} - -/// Select only compiled and deployment-feasible physical candidates. `None` -/// rejects an unbindable candidate before pricing. Comparable scoped costs are -/// required; deployment never rewrites the selected frontier after this step. -/// The payload is generic so deployments can retain binding/diagnostic metadata -/// alongside each compiled computation without duplicating winner selection. -pub fn select_candidate( - candidates: Vec>, - mut evaluate: impl FnMut(&T) -> Result, Error>, -) -> Result, Error> { - let mut scope: Option<(String, f64)> = None; - let mut selected: Option> = None; - for (candidate_index, candidate) in candidates.into_iter().enumerate() { - let Ok(candidate) = candidate else { continue }; - let Some(cost) = evaluate(&candidate)? else { - continue; - }; - if cost.workload_scope.is_empty() - || !cost.horizon_seconds.is_finite() - || cost.horizon_seconds <= 0. - || !cost.total_cost.is_finite() - || cost.total_cost < 0. - { - return Err(invalid( - "candidate cost lacks a valid workload scope/horizon", - )); - } - let current_scope = (cost.workload_scope.clone(), cost.horizon_seconds); - if scope.as_ref().is_some_and(|scope| scope != ¤t_scope) { - return Err(invalid( - "candidate costs describe different workloads or horizons", - )); - } - scope = Some(current_scope); - if selected - .as_ref() - .is_none_or(|selected| cost.total_cost < selected.cost.total_cost) - { - selected = Some(CandidateSelection { - candidate, - candidate_index, - cost, - }); - } - } - selected.ok_or_else(|| invalid("no feasible priced physical candidate")) -} - -#[derive(serde::Deserialize)] -#[serde(deny_unknown_fields)] -struct UncheckedCompiledPhysicalPlan { - precompute: Option, - query: CompiledPhysicalDAG, - materialized_outputs: BTreeMap, -} -impl TryFrom for CompiledPhysicalPlan { - type Error = Error; - fn try_from(candidate: UncheckedCompiledPhysicalPlan) -> Result { - let result = Self { - precompute: candidate.precompute, - query: candidate.query, - materialized_outputs: candidate.materialized_outputs, - }; - result.validate()?; - Ok(result) - } -} - -impl CompiledPhysicalPlan { - /// Validate the physical handoff, including the producer/reader boundary. - pub fn validate(&self) -> Result<(), Error> { - self.query.validate()?; - let Some(precompute) = &self.precompute else { - return if self.materialized_outputs.is_empty() { - Ok(()) - } else { - Err(invalid("materialized outputs have no producer DAG")) - }; - }; - precompute.validate()?; - let outputs: BTreeSet<_> = self.materialized_outputs.keys().copied().collect(); - if outputs.is_empty() || outputs != precompute.roots().iter().copied().collect() { - return Err(invalid("physical frontier differs from precompute outputs")); - } - let readers: BTreeMap<_, _> = self.query.input_contracts().collect(); - for (&id, contract) in &self.materialized_outputs { - let produced = precompute.output_contract(id)?; - // Direct frontiers retain their node IDs. Temporal candidates can - // read several window instances through distinct input slots; - // their deployment bindings must validate those slots separately. - let reader = readers.get(&id); - if contract.schema != produced.schema - || reader.is_some_and(|reader| contract.schema != reader.schema) - || produced.properties.boundedness != Boundedness::Bounded - || contract.properties.boundedness != Boundedness::Bounded - || reader - .is_some_and(|reader| reader.properties.boundedness != Boundedness::Bounded) - { - return Err(invalid("physical frontier schema or boundedness mismatch")); - } - } - Ok(()) - } -} diff --git a/crates/asap-physical-operators/src/unified_physical_planner/compiled.rs b/crates/asap-physical-operators/src/unified_physical_planner/compiled.rs deleted file mode 100644 index 2b9244806..000000000 --- a/crates/asap-physical-operators/src/unified_physical_planner/compiled.rs +++ /dev/null @@ -1,352 +0,0 @@ -//! Reader-independent physical computation and checked deployment instantiation. -use super::*; - -/// A typed execution boundary, without storage identity or a live reader. -#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] -pub struct InputContract { - pub schema: SchemaRef, - pub properties: PlanProperties, -} -impl InputContract { - pub fn bounded(schema: SchemaRef) -> Self { - Self { - schema, - properties: PlanProperties { - boundedness: Boundedness::Bounded, - emission: Emission::Unknown, - }, - } - } - pub fn from_source(source: &dyn PhysicalOperator) -> Self { - Self { - schema: source.output_schema(), - properties: source.properties(&[]), - } - } -} -#[derive(Clone, serde::Serialize, serde::Deserialize)] -enum Node { - Input(InputContract), - Operator { - inputs: Vec, - operator: Operator, - }, -} - -/// Selected native operators and input slots. Rebinding never repeats lowering. -/// Serde is format-agnostic; deployments choose the encoding and its versioning. -/// Deserialization validates the dag before it is usable. -#[derive(Clone, serde::Serialize, serde::Deserialize)] -#[serde(try_from = "UncheckedDAG")] -pub struct CompiledPhysicalDAG { - nodes: BTreeMap, - roots: Vec, -} -#[derive(serde::Deserialize)] -#[serde(deny_unknown_fields)] -struct UncheckedDAG { - nodes: BTreeMap, - roots: Vec, -} -impl TryFrom for CompiledPhysicalDAG { - type Error = Error; - fn try_from(dag: UncheckedDAG) -> Result { - let result = Self { - nodes: dag.nodes, - roots: dag.roots, - }; - result.validate()?; - Ok(result) - } -} - -impl CompiledPhysicalDAG { - /// Link already-selected physical fragments without lowering operators again. - /// Fragment keys and source keys share a namespace; repeated dependency IDs - /// therefore remain one producer in the composed dag. - pub fn compose( - sources: BTreeMap, - fragments: BTreeMap, Self)>, - roots: Vec, - ) -> Result { - if sources.keys().any(|id| fragments.contains_key(id)) { - return Err(invalid("physical source and fragment IDs overlap")); - } - let mut contracts = sources.clone(); - for (&id, (_, fragment)) in &fragments { - fragment.validate()?; - let [root] = fragment.roots() else { - return Err(invalid("composed fragment requires one root")); - }; - if fragment.input_contracts().any(|(id, _)| id == *root) { - return Err(invalid("fragment root must be a computed output")); - } - contracts.insert(id, fragment.output_contract(*root)?); - } - let mut next = contracts - .keys() - .next_back() - .copied() - .unwrap_or(0) - .checked_add(1) - .ok_or_else(|| invalid("physical node ID overflow"))?; - let mut result = Self::new(roots); - for (id, contract) in sources { - result.add_input(id, contract)?; - } - for (id, (inputs, fragment)) in fragments { - if inputs.len() != fragment.input_contracts().count() { - return Err(invalid("physical fragment input arity mismatch")); - } - let mut mapping = BTreeMap::new(); - for ((local, expected), global) in fragment.input_contracts().zip(inputs) { - let actual = contracts - .get(&global) - .ok_or_else(|| invalid("missing physical fragment dependency"))?; - if expected.schema != actual.schema - || (expected.properties.boundedness == Boundedness::Bounded - && actual.properties.boundedness != Boundedness::Bounded) - { - return Err(invalid("physical fragment dependency contract mismatch")); - } - mapping.insert(local, global); - } - mapping.insert(fragment.roots[0], id); - for local in fragment.nodes.keys() { - if !mapping.contains_key(local) { - mapping.insert(*local, next); - next = next - .checked_add(1) - .ok_or_else(|| invalid("physical node ID overflow"))?; - } - } - for (local, node) in fragment.nodes { - if let Node::Operator { inputs, operator } = node { - result.add( - mapping[&local], - inputs.into_iter().map(|input| mapping[&input]).collect(), - operator, - )?; - } - } - } - result.validate()?; - Ok(result) - } - - /// Assemble already-lowered operators and typed external inputs. This is - /// useful for engines that compose multiple compiled computation fragments. - pub fn from_operators( - inputs: BTreeMap, - operators: BTreeMap, Operator)>, - roots: Vec, - ) -> Result { - let mut result = Self::new(roots); - for (id, contract) in inputs { - result.add_input(id, contract)?; - } - for (id, (inputs, operator)) in operators { - result.add(id, inputs, operator)?; - } - result.validate()?; - Ok(result) - } - pub(super) fn new(roots: Vec) -> Self { - Self { - nodes: BTreeMap::new(), - roots, - } - } - pub(super) fn add_input(&mut self, id: NodeId, contract: InputContract) -> Result<(), Error> { - self.insert(id, Node::Input(contract)) - } - pub(super) fn add( - &mut self, - id: NodeId, - inputs: Vec, - operator: Operator, - ) -> Result<(), Error> { - self.insert(id, Node::Operator { inputs, operator }) - } - fn insert(&mut self, id: NodeId, node: Node) -> Result<(), Error> { - if self.nodes.insert(id, node).is_some() { - return Err(invalid(format!("duplicate physical node {id}"))); - } - Ok(()) - } - /// Identify the external input whose rows survive unchanged at this output. - /// Protocol adapters can retain labels that are outside a closed physical schema. - pub fn row_source(&self, id: NodeId) -> Option { - match self.nodes.get(&id)? { - Node::Input(_) => Some(id), - Node::Operator { inputs, operator } => { - let index = operator.row_preserving_input()?; - self.row_source(*inputs.get(index)?) - } - } - } - - /// Selected operator name, for plan inspection without decoding its wire format. - /// Certified candidate pruning checks authoritative-key coverage inside this operator. - pub fn certified_pruning_keys(&self, id: NodeId) -> Option<&[(usize, usize)]> { - match self.nodes.get(&id)? { - Node::Operator { operator, .. } => operator.certified_pruning_keys(), - Node::Input(_) => None, - } - } - pub fn operator_name(&self, id: NodeId) -> Option<&str> { - match self.nodes.get(&id)? { - Node::Input(_) => Some("Input"), - Node::Operator { operator, .. } => Some(operator.name()), - } - } - - pub fn roots(&self) -> &[NodeId] { - &self.roots - } - pub fn input_contracts(&self) -> impl Iterator { - self.nodes.iter().filter_map(|(&id, node)| match node { - Node::Input(contract) => Some((id, contract)), - Node::Operator { .. } => None, - }) - } - /// Derive a reachable output contract without opening deployment readers. - pub fn output_contract(&self, id: NodeId) -> Result { - let properties = *self - .output_properties()? - .get(&id) - .ok_or_else(|| invalid("output is not reachable"))?; - let schema = match self - .nodes - .get(&id) - .ok_or_else(|| invalid("missing output"))? - { - Node::Input(contract) => contract.schema.clone(), - Node::Operator { operator, .. } => operator.output_schema(), - }; - Ok(InputContract { schema, properties }) - } - /// Properties of every reachable node, derived in one contract-only pass. - pub(super) fn output_properties(&self) -> Result, Error> { - let sources = self - .input_contracts() - .map(|(id, contract)| (id, Box::new(contract.clone()) as Source<'_>)) - .collect(); - self.instantiate(sources)?.properties(&self.roots) - } - /// Direct physical dependencies; empty for inputs and unknown IDs. - pub(super) fn dependencies(&self, id: NodeId) -> &[NodeId] { - match self.nodes.get(&id) { - Some(Node::Operator { inputs, .. }) => inputs, - _ => &[], - } - } - pub(super) fn is_operator(&self, id: NodeId) -> bool { - matches!(self.nodes.get(&id), Some(Node::Operator { .. })) - } - /// Keep the already-lowered operators reachable from `roots`, replacing - /// each node in `boundaries` by a typed input. Nothing is lowered again. - pub(super) fn cut( - &self, - boundaries: &BTreeMap, - roots: &[NodeId], - ) -> Result { - let mut result = Self::new(roots.to_vec()); - let mut pending = roots.to_vec(); - while let Some(id) = pending.pop() { - if result.nodes.contains_key(&id) { - continue; - } - let node = match boundaries.get(&id) { - Some(contract) => Node::Input(contract.clone()), - None => self - .nodes - .get(&id) - .cloned() - .ok_or_else(|| invalid(format!("missing physical node {id}")))?, - }; - if let Node::Operator { inputs, .. } = &node { - pending.extend(inputs); - } - result.nodes.insert(id, node); - } - result.validate()?; - Ok(result) - } - /// Validate using contract-only sources. No deployment reader is available. - pub fn validate(&self) -> Result<(), Error> { - let sources = self - .input_contracts() - .map(|(id, c)| (id, Box::new(c.clone()) as Source<'_>)) - .collect(); - self.instantiate(sources).map(|_| ()) - } - /// Resolve exactly the declared inputs and validate before any source starts. - pub fn instantiate<'a>( - &self, - mut sources: BTreeMap>, - ) -> Result, Error> { - let mut dag = PhysicalDAG::default(); - for (&id, node) in &self.nodes { - match node { - Node::Input(contract) => { - let source = sources - .remove(&id) - .ok_or_else(|| invalid(format!("missing physical input {id}")))?; - let actual = source.properties(&[]); - if !source.input_schemas().is_empty() - || source.output_schema() != contract.schema - || (contract.properties.boundedness != Boundedness::Unknown - && actual.boundedness != contract.properties.boundedness) - || (contract.properties.emission != Emission::Unknown - && actual.emission != contract.properties.emission) - { - return Err(invalid(format!( - "physical input {id} violates its compiled contract" - ))); - } - dag.add_boxed( - id, - vec![], - Box::new(CheckedSource { - source, - output: contract.schema.clone(), - }), - )?; - } - Node::Operator { inputs, operator } => { - dag.add(id, inputs.clone(), operator.clone())?; - } - } - } - if !sources.is_empty() { - return Err(invalid("unexpected physical input binding")); - } - dag.validate(&self.roots)?; - Ok(dag) - } -} -impl PhysicalOperator for InputContract { - fn name(&self) -> &str { - "UnresolvedInput" - } - fn input_schemas(&self) -> Vec { - vec![] - } - fn output_schema(&self) -> SchemaRef { - self.schema.clone() - } - fn properties(&self, _: &[PlanProperties]) -> PlanProperties { - self.properties - } - fn output_bytes(&self, batch: &Batch) -> usize { - batch.bytes() - } - fn start<'a>( - &'a self, - _: Vec>, - _: crate::runtime::RunContext, - ) -> Result, Error> { - Err(invalid("physical input must be resolved before execution")) - } -} diff --git a/crates/asap-physical-operators/src/unified_physical_planner/logical.rs b/crates/asap-physical-operators/src/unified_physical_planner/logical.rs deleted file mode 100644 index dbc541895..000000000 --- a/crates/asap-physical-operators/src/unified_physical_planner/logical.rs +++ /dev/null @@ -1,374 +0,0 @@ -//! Reconstruct shared operator references from the transport DAG for native lowering. -use super::*; -use planner_types::ir::export::{EdgeRole, NonASAPOpKind as N, PhysicalASAPNodeId, WireScalarExpr}; -use planner_types::ir::{ - ASAPOp, NonASAPOp, Operator as LogicalOperator, OperatorNode, Predicate, ProjectItem, - ScalarExpr, SortKey as LogicalSortKey, -}; -use std::rc::Rc; -pub(super) fn scalar( - expr: &WireScalarExpr, - id_of: &mut impl FnMut(PhysicalASAPNodeId) -> Rc, -) -> ScalarExpr { - fn boxed( - e: &WireScalarExpr, - id_of: &mut impl FnMut(PhysicalASAPNodeId) -> Rc, - ) -> Box { - Box::new(scalar(e, id_of)) - } - fn list( - es: &[WireScalarExpr], - id_of: &mut impl FnMut(PhysicalASAPNodeId) -> Rc, - ) -> Vec { - es.iter().map(|e| scalar(e, id_of)).collect() - } - match expr { - WireScalarExpr::Column(id) => ScalarExpr::Column(*id), - WireScalarExpr::Literal(v) => ScalarExpr::Literal(v.clone()), - WireScalarExpr::Negative { expr, semantics } => ScalarExpr::Negative { - expr: boxed(expr, id_of), - semantics: *semantics, - }, - WireScalarExpr::Compare { - left, - op, - right, - semantics, - } => ScalarExpr::Compare { - left: boxed(left, id_of), - op: op.clone(), - right: boxed(right, id_of), - semantics: *semantics, - }, - WireScalarExpr::BoolAnd(parts) => ScalarExpr::BoolAnd(list(parts, id_of)), - WireScalarExpr::BoolOr(parts) => ScalarExpr::BoolOr(list(parts, id_of)), - WireScalarExpr::Not(e) => ScalarExpr::Not(boxed(e, id_of)), - WireScalarExpr::IsNull(e) => ScalarExpr::IsNull(boxed(e, id_of)), - WireScalarExpr::IsNotNull(e) => ScalarExpr::IsNotNull(boxed(e, id_of)), - WireScalarExpr::Cast { expr, to, try_cast } => ScalarExpr::Cast { - expr: boxed(expr, id_of), - to: to.clone(), - try_cast: *try_cast, - }, - WireScalarExpr::InList { - expr, - list: items, - negated, - } => ScalarExpr::InList { - expr: boxed(expr, id_of), - list: list(items, id_of), - negated: *negated, - }, - WireScalarExpr::FunctionCall { name, args } => ScalarExpr::FunctionCall { - name: name.clone(), - args: list(args, id_of), - }, - WireScalarExpr::Arithmetic { - op, - left, - right, - semantics, - } => ScalarExpr::Arithmetic { - op: op.clone(), - left: boxed(left, id_of), - right: boxed(right, id_of), - semantics: *semantics, - }, - WireScalarExpr::Case { - operand, - branches, - else_expr, - } => ScalarExpr::Case { - operand: operand.as_ref().map(|e| boxed(e, id_of)), - branches: branches - .iter() - .map(|(w, t)| (scalar(w, id_of), scalar(t, id_of))) - .collect(), - else_expr: else_expr.as_ref().map(|e| boxed(e, id_of)), - }, - WireScalarExpr::CurrentTimestamp => ScalarExpr::CurrentTimestamp, - WireScalarExpr::EvalTimestamp => ScalarExpr::EvalTimestamp, - WireScalarExpr::PromqlScalarFromVector(node) => { - ScalarExpr::PromqlScalarFromVector(id_of(*node)) - } - WireScalarExpr::ScalarSubquery(node) => ScalarExpr::ScalarSubquery(id_of(*node)), - WireScalarExpr::Exists { subquery, negated } => ScalarExpr::Exists { - subquery: id_of(*subquery), - negated: *negated, - }, - WireScalarExpr::InSubquery { - expr, - subquery, - negated, - } => ScalarExpr::InSubquery { - expr: boxed(expr, id_of), - subquery: id_of(*subquery), - negated: *negated, - }, - } -} - -pub(super) fn restore(dag: &PhysicalASAPDAG) -> Result>, Error> { - dag.validate().map_err(|e| invalid(e.to_string()))?; - let mut done = BTreeMap::new(); - let mut remaining: Vec<_> = dag.nodes.iter().collect(); - while !remaining.is_empty() { - let before = remaining.len(); - let mut next = Vec::new(); - for node in remaining { - let mut edges: Vec<_> = dag.edges.iter().filter(|e| e.consumer == node.id).collect(); - if edges - .iter() - .any(|e| !done.contains_key(&u64::from(e.producer.0))) - { - next.push(node); - continue; - } - edges.sort_by_key(|e| match e.role { - EdgeRole::Left => 0, - EdgeRole::Input => 1, - EdgeRole::Right => 2, - EdgeRole::ScalarRef => 3, - }); - let inputs: Vec<_> = edges - .iter() - .filter(|e| e.role != EdgeRole::ScalarRef) - .map(|e| Rc::clone(&done[&u64::from(e.producer.0)])) - .collect(); - let input = |index: usize| { - inputs - .get(index) - .cloned() - .ok_or_else(|| invalid("operator is missing an input")) - }; - let mut missing = false; - let mut ref_node = |id: PhysicalASAPNodeId| { - if let Some(node) = done.get(&u64::from(id.0)) { - Rc::clone(node) - } else { - missing = true; - Rc::new(OperatorNode::with_schema( - LogicalOperator::NonASAP(NonASAPOp::Values { - rows: vec![], - schema: Default::default(), - }), - Default::default(), - )) - } - }; - let mut value = |expr: &WireScalarExpr| scalar(expr, &mut ref_node); - let operator = match &node.payload { - Payload::Relational { operator } => LogicalOperator::NonASAP(match operator { - N::Scan { - source, - predicates, - schema, - } => NonASAPOp::Scan { - source: source.clone(), - predicates: predicates.iter().map(|p| Predicate(value(&p.0))).collect(), - schema: schema.clone(), - }, - N::Values { rows, schema } => NonASAPOp::Values { - rows: rows - .iter() - .map(|r| r.iter().map(&mut value).collect()) - .collect(), - schema: schema.clone(), - }, - N::Filter { pred } => NonASAPOp::Filter { - pred: Predicate(value(&pred.0)), - child: input(0)?, - }, - N::Project { cols, qualifier } => NonASAPOp::Project { - cols: cols - .iter() - .map(|c| ProjectItem { - alias: c.alias.clone(), - expr: value(&c.expr), - }) - .collect(), - qualifier: qualifier.clone(), - child: input(0)?, - }, - N::Aggregate { - reduction, - measures, - output_names, - filters, - having, - } => NonASAPOp::Aggregate { - reduction: reduction.clone(), - measures: measures.clone(), - output_names: output_names.clone(), - filters: filters - .iter() - .map(|p| p.as_ref().map(|p| Predicate(value(&p.0)))) - .collect(), - having: having.as_ref().map(|p| Predicate(value(&p.0))), - child: input(0)?, - }, - N::Join { join_kind, pred } => NonASAPOp::Join { - kind: join_kind.clone(), - pred: Predicate(value(&pred.0)), - left: input(0)?, - right: input(1)?, - }, - N::SetOp { set_kind, all } => NonASAPOp::SetOp { - kind: set_kind.clone(), - all: *all, - left: input(0)?, - right: input(1)?, - }, - N::Concat { - discriminator_unique_key, - } => NonASAPOp::Concat { - children: inputs.clone(), - discriminator_unique_key: discriminator_unique_key.clone(), - }, - N::Dedup { cols } => NonASAPOp::Dedup { - cols: cols.clone(), - child: input(0)?, - }, - N::Sort { keys, partition_by } => NonASAPOp::Sort { - keys: keys - .iter() - .map(|k| LogicalSortKey { - expr: value(&k.expr), - ascending: k.ascending, - nulls_first: k.nulls_first, - }) - .collect(), - partition_by: partition_by.clone(), - child: input(0)?, - }, - N::Limit { - n, - offset, - partition_by, - } => NonASAPOp::Limit { - n: *n, - offset: *offset, - partition_by: partition_by.clone(), - child: input(0)?, - }, - N::BinaryOp { - operator, - return_bool, - } => NonASAPOp::BinaryOp { - operator: operator.clone(), - return_bool: *return_bool, - lhs: input(0)?, - rhs: input(1)?, - }, - N::SQLWindowFunc { - func, - args, - partition_by, - order_by, - frame, - output_name, - } => NonASAPOp::SQLWindowFunc { - func: func.clone(), - args: args.iter().map(&mut value).collect(), - partition_by: partition_by.clone(), - order_by: order_by - .iter() - .map(|k| LogicalSortKey { - expr: value(&k.expr), - ascending: k.ascending, - nulls_first: k.nulls_first, - }) - .collect(), - frame: frame.clone(), - output_name: output_name.clone(), - child: input(0)?, - }, - N::TimeRange { range, range_kind } => NonASAPOp::TimeRange { - range: *range, - kind: *range_kind, - child: input(0)?, - }, - N::TimeShift { shift } => NonASAPOp::TimeShift { - shift: *shift, - child: input(0)?, - }, - N::PromqlVectorFromScalar { expr } => { - NonASAPOp::PromqlVectorFromScalar(value(expr)) - } - N::PromqlRelabel { dst, value: expr } => NonASAPOp::PromqlRelabel { - dst: dst.clone(), - value: value(expr), - child: input(0)?, - }, - N::PromqlInfoEnrich { selector } => NonASAPOp::PromqlInfoEnrich { - selector: selector.clone(), - child: input(0)?, - }, - N::PromqlSeriesSample { by, sample_kind } => NonASAPOp::PromqlSeriesSample { - by: by.clone(), - kind: *sample_kind, - child: input(0)?, - }, - N::PromqlSubquery { range, resolution } => NonASAPOp::PromqlSubquery { - range: *range, - resolution: *resolution, - child: input(0)?, - }, - }), - Payload::SummaryAgg { - family, - input: update, - reduction, - grouping, - filter, - } => LogicalOperator::ASAP(ASAPOp::SummaryAgg { - child: input(0)?, - family: family.clone(), - input: update.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.as_ref().map(|p| Predicate(value(&p.0))), - }), - Payload::SummaryEstimate { query } => { - LogicalOperator::ASAP(ASAPOp::SummaryEstimate { - summary_input: input(0)?, - query: query.clone(), - }) - } - Payload::FinalizeExactAccumulator => { - LogicalOperator::ASAP(ASAPOp::FinalizeExactAccumulator { child: input(0)? }) - } - Payload::MaintainPopulation { population } => { - LogicalOperator::ASAP(ASAPOp::MaintainPopulation { - child: input(0)?, - population: population.clone(), - }) - } - Payload::EvaluatePopulation { evaluation } => { - LogicalOperator::ASAP(ASAPOp::EvaluatePopulation { - child: input(0)?, - evaluation: evaluation.clone(), - }) - } - Payload::SummaryMerge => LogicalOperator::ASAP(ASAPOp::SummaryMerge { - children: inputs.clone(), - }), - _ => return Err(invalid("reserved ASAP operation has no native lowering")), - }; - if missing { - return Err(invalid( - "scalar reference is not a preceding DAG dependency", - )); - } - let mut rebuilt = OperatorNode::with_schema(operator, node.output_schema.clone()); - rebuilt.guarantee = node.guarantee.clone(); - rebuilt.timing = Some(node.output_state.timing); - done.insert(u64::from(node.id.0), Rc::new(rebuilt)); - } - if next.len() == before { - return Err(invalid("operator DAG is cyclic")); - } - remaining = next; - } - Ok(done) -} diff --git a/crates/asap-physical-operators/src/unified_physical_planner/mod.rs b/crates/asap-physical-operators/src/unified_physical_planner/mod.rs deleted file mode 100644 index 8d680fcc9..000000000 --- a/crates/asap-physical-operators/src/unified_physical_planner/mod.rs +++ /dev/null @@ -1,1169 +0,0 @@ -//! Compile logical computation to native operators with typed external inputs. -//! Compilation needs no readers; deployment resolves inputs after selection. -use crate::operators::ReadoutQuery; -use crate::summary_kernels::exact::ExactReadout; -use crate::{ - operators::{Expression, Operator, Reduction, SortKey}, - plan::{Boundedness, Emission, NodeId, PhysicalDAG, PhysicalOperator, PlanProperties}, - values::{Batch, SchemaRef}, - Error, -}; -use planner_types::ir::export::{ - NonASAPOpKind, PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPOperatorPayload as Payload, - WireScalarExpr, -}; -use planner_types::ir::{ASAPOp, NonASAPOp, Operator as LogicalOperator, OperatorNode, ScalarExpr}; -use planner_types::{ - post_asap::{FieldDataType, SketchStatistic, SummaryInputExpr}, - pre_asap::{ - AggIntent, ColumnRef, CompareOpKind, DataType, GroupKeys, Reduction as PlannerReduction, - }, -}; -mod logical; -use std::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, -}; -fn invalid(message: impl Into) -> Error { - Error::Invalid(message.into()) -} - -/// Source nodes cut the DAG at an installed storage/ingestion frontier. The -/// binding must have exactly the declared schema and no upstream dependencies. -/// A deployment must authorize these frontiers before calling this function. -pub type Source<'a> = Box + 'a>; - -pub mod precompute; -pub mod promql_fallback; -pub mod promql_rows; -pub mod promql_values; - -mod candidates; -pub use candidates::{ - compile_candidate, compile_candidates, cut_candidate, enumerate_frontiers, - frontier_from_timing, select_candidate, CandidateCost, CandidateSelection, - CompiledPhysicalPlan, -}; - -mod compiled; -pub use compiled::{CompiledPhysicalDAG, InputContract}; - -mod row_values; - -/// Compile computation without opening or retaining deployment readers. -/// Input contracts identify explicit boundaries selected by maintenance planning. -pub fn compile( - dag: &PhysicalASAPDAG, - inputs: BTreeMap, - roots: &[NodeId], -) -> Result { - compile_internal(dag, inputs, roots) -} - -/// Convenience for callers that already resolved inputs. Lowering still uses -/// only their contracts, and instantiation checks those contracts again. -pub fn bind<'a>( - dag: &PhysicalASAPDAG, - sources: BTreeMap>, - roots: &[NodeId], -) -> Result, Error> { - let inputs = sources - .iter() - .map(|(&id, source)| (id, InputContract::from_source(source.as_ref()))) - .collect(); - compile(dag, inputs, roots)?.instantiate(sources) -} - -/// Resolve raw scan connectors before invoking the reader-independent compiler. -pub fn bind_with_data_sources<'a>( - dag: &PhysicalASAPDAG, - mut sources: BTreeMap>, - roots: &[NodeId], - data_sources: &crate::unified_sources::DataSources, -) -> Result, Error> { - let restored = logical::restore(dag)?; - // Only resolve scans reachable below the selected input boundaries. - let mut pending = roots.to_vec(); - let mut seen = BTreeSet::new(); - while let Some(id) = pending.pop() { - if !seen.insert(id) || sources.contains_key(&id) { - continue; - } - let _node = dag - .nodes - .iter() - .find(|n| u64::from(n.id.0) == id) - .ok_or_else(|| invalid(format!("missing node {id}")))?; - if matches!(restored[&id].non_asap(), Some(NonASAPOp::Scan { .. })) { - sources.insert(id, Box::new(data_sources.bind(&restored[&id])?)); - } else { - pending.extend( - dag.edges - .iter() - .filter(|e| u64::from(e.consumer.0) == id) - .map(|e| u64::from(e.producer.0)), - ); - } - } - bind(dag, sources, roots) -} - -#[cfg(test)] -thread_local! { - /// Planner nodes lowered by this thread, for compile-once tests. - static LOWERED_NODES: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -/// Helper operators are numbered from their Planner node alone, above the u32 -/// Planner ID range, so every boundary choice yields a subgraph of the same -/// lowering and candidate cuts need not renumber operators. A node lowering to -/// several helpers takes consecutive indices below its base. -fn helper_id(node: NodeId, index: u64) -> NodeId { - debug_assert!(node <= u64::from(u32::MAX) && index < 1 << 16); - u64::MAX - (node << 16) - index -} - -fn compile_internal( - dag: &PhysicalASAPDAG, - mut sources: BTreeMap, - roots: &[NodeId], -) -> Result { - preflight_depth(dag)?; - let restored = logical::restore(dag)?; - dag.validate().map_err(|e| invalid(e.to_string()))?; - let nodes = dag - .nodes - .iter() - .map(|node| (u64::from(node.id.0), node)) - .collect::>(); - let mut dependencies = BTreeMap::>::new(); - // Binary input order is semantic; serialized edge order is not. - let mut edges = dag.edges.iter().collect::>(); - edges.sort_by_key(|edge| { - ( - edge.consumer.0, - match edge.role { - planner_types::ir::export::EdgeRole::Left => 0, - planner_types::ir::export::EdgeRole::Input => 1, - planner_types::ir::export::EdgeRole::Right => 2, - planner_types::ir::export::EdgeRole::ScalarRef => 3, - }, - ) - }); - let literals = BTreeMap::::new(); - for edge in edges { - dependencies - .entry(u64::from(edge.consumer.0)) - .or_default() - .push(u64::from(edge.producer.0)); - } - let mut fallback = BTreeMap::new(); - for (&id, root) in &restored { - let raw_summary_input = matches!(root.non_asap(), Some(NonASAPOp::TimeRange { .. })) - && dag.edges.iter().any(|e| { - u64::from(e.producer.0) == id - && matches!( - nodes[&u64::from(e.consumer.0)].payload, - Payload::SummaryAgg { .. } - ) - }); - if !root.contains_asap() && !raw_summary_input { - if let Ok(lowered) = promql_fallback::lower(root) { - fallback.insert(id, lowered); - } - } - } - let known = |id: &NodeId| { - nodes.contains_key(id) - || promql_fallback::raw_series_owner(*id).is_some_and(|owner| { - matches!( - nodes.get(&owner), - Some(PhysicalASAPDAGNode { - payload: Payload::Relational { .. }, - .. - }) - ) - }) - }; - if !sources.keys().all(known) { - return Err(invalid("source binding names an unknown node")); - } - let mut ordered = Vec::new(); - let mut seen = BTreeSet::new(); - let mut pending = roots.iter().map(|&id| (id, false)).collect::>(); - while let Some((id, expanded)) = pending.pop() { - if expanded { - ordered.push(id); - continue; - } - if !seen.insert(id) { - continue; - } - if !nodes.contains_key(&id) { - return Err(invalid(format!("missing root {id}"))); - } - pending.push((id, true)); - if !sources.contains_key(&id) && !fallback.contains_key(&id) { - for &input in dependencies.get(&id).into_iter().flatten() { - pending.push((input, false)); - } - } - } - let mut dag = CompiledPhysicalDAG::new(roots.to_vec()); - for id in ordered { - let node = nodes[&id]; - let mut auxiliary = helper_id(id, 0); - let output = Arc::new(node.output_schema.clone()); - crate::values::validate_schema(&output)?; - if let Some(source) = sources.remove(&id) { - if source.schema != output { - return Err(invalid("frontier does not have the declared schema")); - } - dag.add_input(id, source)?; - } else { - #[cfg(test)] - LOWERED_NODES.with(|count| count.set(count.get() + 1)); - let mut inputs = dependencies.get(&id).cloned().unwrap_or_default(); - let mut schemas = inputs - .iter() - .map(|id| Arc::new(nodes[id].output_schema.clone())) - .collect::>(); - if matches!(node.payload, Payload::SummaryMerge) && inputs.len() > 1 { - if schemas.iter().any(|s| s != &schemas[0]) { - return Err(invalid("summary merge inputs have different schemas")); - } - dag.add( - auxiliary, - inputs, - Operator::union(schemas[0].clone(), schemas.len())?, - )?; - inputs = vec![auxiliary]; - schemas.truncate(1); - } - if let Some(promql_fallback::Lowering { - selectors, - mut steps, - }) = fallback.remove(&id) - { - let mut slots = Vec::new(); - for (i, (_, schema)) in selectors.iter().enumerate() { - let slot = promql_fallback::raw_series_input(id, i); - match sources.remove(&slot) { - Some(contract) if &contract.schema == schema => { - dag.add_input(slot, contract)? - } - Some(_) => { - return Err(invalid(format!( - "node {id}: raw series input {slot} differs from the selector schema" - ))) - } - None => { - return Err(invalid(format!( - "node {id}: PromQL fallback requires raw series input {slot}" - ))) - } - } - slots.push(slot); - } - let (last, last_inputs) = steps - .pop() - .ok_or_else(|| invalid("empty PromQL lowering"))?; - let mut ids = Vec::new(); - let resolve = |inputs: Vec, ids: &[NodeId]| { - inputs - .into_iter() - .map(|input| match input { - promql_fallback::Input::Raw(i) => slots[i], - promql_fallback::Input::Step(i) => ids[i], - }) - .collect::>() - }; - for (operator, inputs) in steps { - dag.add(auxiliary, resolve(inputs, &ids), operator)?; - ids.push(auxiliary); - auxiliary -= 1; - } - dag.add( - id, - resolve(last_inputs, &ids), - last.with_output_schema(output)?, - )?; - continue; - } - if let Payload::MaintainPopulation { population } = &node.payload { - use planner_types::post_asap::maintained_population::PopulationInput; - let PopulationInput::CurrentSeries(spec) = &population.input else { - return Err(invalid( - "native maintained population requires a current-series input", - )); - }; - let [input] = schemas.as_slice() else { - return Err(invalid("current-series population requires one input")); - }; - if spec.without { - return Err(invalid( - "dynamic without grouping requires label-set projection", - )); - } - let identity = named_column( - input, - &ColumnRef::Named(promql_rows::SERIES_IDENTITY_COLUMN.into()), - )?; - let coordinate = input - .time_index - .ok_or_else(|| invalid("current-series input lacks timestamp"))?; - let value = named_column(input, &ColumnRef::SampleValue)?; - let lookback = i64::try_from(spec.lookback_ms) - .map_err(|_| invalid("current-series lookback overflows"))?; - dag.add( - id, - inputs, - Operator::current_series(input.clone(), identity, coordinate, value, lookback)? - .with_output_schema(output)?, - )?; - continue; - } - if let Payload::EvaluatePopulation { evaluation } = &node.payload { - use planner_types::post_asap::maintained_population::{ - PopulationInput, PopulationStatistic, - }; - let [producer] = inputs.as_slice() else { - return Err(invalid("population evaluation requires one input")); - }; - let Payload::MaintainPopulation { population } = &nodes[producer].payload else { - return Err(invalid( - "population evaluation requires its declared population", - )); - }; - let PopulationInput::CurrentSeries(spec) = &population.input else { - return Err(invalid("current-series population required")); - }; - if spec.without { - return Err(invalid( - "dynamic without ranking requires label-set projection", - )); - } - let input = schemas[0].clone(); - let PopulationStatistic::TopK { k } = evaluation else { - let mut chain = - row_values::population_aggregate(&input, &spec.grouping, evaluation)?; - let last = chain.pop().expect("nonempty chain"); - let mut inputs = inputs; - for operator in chain { - dag.add(auxiliary, inputs, operator)?; - inputs = vec![auxiliary]; - auxiliary -= 1; - } - dag.add(id, inputs, last.with_output_schema(output)?)?; - continue; - }; - let groups = spec - .grouping - .iter() - .map(|name| named_column(&input, &ColumnRef::Named(name.clone()))) - .collect::, _>>()?; - let value = named_column(&input, &ColumnRef::SampleValue)?; - dag.add( - auxiliary, - inputs, - Operator::sort( - input.clone(), - vec![SortKey { - column: value, - descending: true, - nulls_first: false, - }], - groups.clone(), - )?, - )?; - dag.add( - id, - vec![auxiliary], - Operator::limit(input, *k as u64, 0, groups)?.with_output_schema(output)?, - )?; - continue; - } - // A closed row must include either all source labels or the explicit - // complete-label identity. Projected labels alone are insufficient. - if let Payload::SummaryAgg { - family, - input: update, - reduction: PlannerReduction::PerEntity, - grouping, - filter: None, - } = &node.payload - { - let [input_id] = inputs.as_slice() else { - return Err(invalid("per-entity summary requires one input")); - }; - let Some(NonASAPOp::TimeRange { child, .. }) = restored[input_id].non_asap() else { - return Err(invalid( - "per-entity summary requires a resolved raw time range", - )); - }; - let Some(NonASAPOp::Scan { schema, .. }) = child.non_asap() else { - return Err(invalid("per-entity summary requires a resolved source")); - }; - if !schema.closed || update.item.is_some() { - return Err(invalid( - "per-entity summary requires complete source identity", - )); - } - crate::capability::validate_summary_kernel(family, update, grouping) - .map_err(Error::Invalid)?; - let SummaryInputExpr::Column(value) = &update.weight else { - return Err(invalid( - "per-entity update requires a projected value column", - )); - }; - let input = schemas[0].clone(); - let value = named_column(&input, value)?; - let coordinate = input - .time_index - .ok_or_else(|| invalid("temporal input lacks time"))?; - let groups = (0..input.fields.len()) - .filter(|&column| column != value && column != coordinate) - .collect(); - let build = Operator::summary_build( - input, - family.clone(), - value, - Some(coordinate), - groups, - )?; - let compact = build.schema(); - dag.add(auxiliary, inputs, build)?; - dag.add( - id, - vec![auxiliary], - Operator::scope_timestamp(compact, output)?, - )?; - continue; - } - if let Payload::Relational { - operator: - NonASAPOpKind::BinaryOp { - operator, - return_bool, - }, - } = &node.payload - { - let operator = crate::expressions::binary::BinaryOperator::from_logical( - operator, - *return_bool, - ); - let query_time = node.output_state.timing - == planner_types::post_asap::ExecutionTiming::QueryTime; - if let Some(&(value, left)) = literals.get(&id) { - let [input] = schemas.as_slice() else { - return Err(invalid("scalar binary requires one row input")); - }; - if !query_time { - return Err(invalid("scalar literal binary must run at query time")); - } - let scalar = - Operator::scalar(crate::values::Value::Float64(value), DataType::Float64)?; - let (sides, scalars, operands) = if left { - ( - [scalar.schema(), input.clone()], - [true, false], - vec![auxiliary, inputs[0]], - ) - } else { - ( - [input.clone(), scalar.schema()], - [false, true], - vec![inputs[0], auxiliary], - ) - }; - let [l, r] = sides; - let binary = Operator::series_binary(l, r, operator.clone(), scalars) - .map_err(|error| invalid(format!("node {id}: {error}")))?; - dag.add(auxiliary, vec![], scalar)?; - dag.add(id, operands, binary.with_output_schema(output)?)?; - auxiliary -= 1; - continue; - } - let label_map = |schema: &SchemaRef| { - schema - .fields - .iter() - .any(|f| matches!(f.dtype, FieldDataType::Plain(DataType::Map { .. }))) - }; - // Grouped rows carry their labels as columns; per-series rows - // carry the series identity. - if let (true, [left, right]) = (query_time, schemas.as_slice()) { - if !label_map(left) && !label_map(right) { - let binary = Operator::series_binary( - left.clone(), - right.clone(), - operator.clone(), - [false, false], - ) - .map_err(|error| invalid(format!("node {id}: {error}")))?; - dag.add(id, inputs, binary.with_output_schema(output)?)?; - continue; - } - } - } - if let Payload::FinalizeExactAccumulator = &node.payload { - // Exact counts read out as Int64; PromQL declares a Float64 sample. - let evaluation = bind_operation(node, &schemas) - .map_err(|error| invalid(format!("node {id}: {error}")))?; - let actual = evaluation.schema(); - let converted = actual.fields.iter().zip(&output.fields).position(|(a, d)| { - a.dtype == FieldDataType::Plain(DataType::Int64) - && d.dtype == FieldDataType::Plain(DataType::Float64) - }); - if let Some(column) = converted { - let columns = actual - .fields - .iter() - .enumerate() - .map(|(i, field)| { - ( - field.name.clone(), - if i == column { - Expression::ExactFloat64(i) - } else { - Expression::Column(i) - }, - ) - }) - .collect(); - let project = - Operator::project(actual, columns)?.with_output_schema(output.clone())?; - dag.add(auxiliary, inputs, evaluation)?; - if temporal_evaluation_drops_name(node) { - dag.add(auxiliary - 1, vec![auxiliary], project)?; - dag.add( - id, - vec![auxiliary - 1], - Operator::series_without_name(output)?, - )?; - } else { - dag.add(id, vec![auxiliary], project)?; - } - auxiliary -= 1; - continue; - } - } - let mut operator = compile_node(node, &schemas) - .map_err(|error| invalid(format!("node {id}: {error}")))?; - if operator.is_counter_readout() { - let mut pending = vec![id]; - let mut visited = BTreeSet::new(); - let mut ranges = BTreeSet::new(); - while let Some(ancestor) = pending.pop() { - if !visited.insert(ancestor) { - continue; - } - if let Payload::Relational { - operator: NonASAPOpKind::TimeRange { range, .. }, - } = &nodes[&ancestor].payload - { - ranges.insert( - i64::try_from(range.as_millis()) - .map_err(|_| invalid("counter lookback exceeds Int64"))?, - ); - continue; - } - pending.extend(dependencies.get(&ancestor).into_iter().flatten().copied()); - } - if ranges.len() > 1 { - return Err(invalid("counter evaluation has ambiguous logical windows")); - } - if let Some(lookback) = ranges.into_iter().next() { - operator = operator.with_counter_lookback(lookback)?; - } - } - if temporal_evaluation_drops_name(node) { - dag.add(auxiliary, inputs, operator)?; - dag.add(id, vec![auxiliary], Operator::series_without_name(output)?)?; - } else { - dag.add(id, inputs, operator)?; - } - } - } - dag.validate()?; - Ok(dag) -} - -// Temporal summary evaluations produce PromQL vectors, whose range functions drop -// the metric name before matching/filtering. Stored state retains its full identity. -fn temporal_evaluation_drops_name(node: &PhysicalASAPDAGNode) -> bool { - node.output_schema - .fields - .iter() - .any(|field| field.name == promql_rows::SERIES_IDENTITY_COLUMN) - && matches!( - &node.payload, - Payload::FinalizeExactAccumulator - | Payload::SummaryEstimate { - query: SketchStatistic::Quantile { .. } - | SketchStatistic::Cardinality - | SketchStatistic::PointCount { .. } - | SketchStatistic::FrequencyL2 - | SketchStatistic::FrequencyEntropy - } - ) -} - -/// Bind a Planner node against the schemas supplied by its deployment edges. -/// This is the same checked path used by complete DAG binding. -pub fn compile_node(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { - for schema in inputs { - crate::values::validate_schema(schema)?; - } - bind_operation(node, inputs)?.with_output_schema(Arc::new(node.output_schema.clone())) -} - -fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { - if let Payload::Relational { - operator: NonASAPOpKind::BinaryOp { - operator, - return_bool, - }, - } = &node.payload - { - let operator = - crate::expressions::binary::BinaryOperator::from_logical(operator, *return_bool); - let [left, right] = inputs else { - return Err(invalid("binary requires two inputs")); - }; - if node.output_state.timing == planner_types::post_asap::ExecutionTiming::IngestionTime { - let value = |schema: &SchemaRef| -> Result { - let columns = schema - .fields - .iter() - .enumerate() - .filter(|(_, field)| { - field.dtype - == FieldDataType::Plain(planner_types::pre_asap::DataType::Float64) - }) - .map(|(i, _)| i) - .collect::>(); - match columns.as_slice() { - [value] => Ok(*value), - _ => Err(invalid("aligned binary requires one value column")), - } - }; - let (l, r) = (value(left)?, value(right)?); - let keys = left - .fields - .iter() - .enumerate() - .filter(|(i, _)| *i != l) - .map(|(i, field)| { - right - .fields - .iter() - .position(|other| other.name == field.name && other.dtype == field.dtype) - .map(|j| (i, j)) - .ok_or_else(|| invalid("aligned input identities differ")) - }) - .collect::, _>>()?; - return Operator::aligned_binary( - left.clone(), - right.clone(), - keys, - (l, r), - operator.clone(), - ); - } - return Operator::vector_binary(left.clone(), right.clone(), operator.clone(), false); - } - if let Payload::Relational { - operator: NonASAPOpKind::Join { join_kind, pred }, - } = &node.payload - { - let [left, right] = inputs else { - return Err(invalid("join requires two inputs")); - }; - let pred = planner_types::ir::Predicate(local_scalar(&pred.0)?); - if *join_kind == planner_types::pre_asap::JoinKind::Semi { - if let Ok(keys) = equijoin_keys(&pred, left, right) { - return Operator::semi_join(left.clone(), right.clone(), keys); - } - } - return Operator::unified_relational_join( - left.clone(), - right.clone(), - join_kind.clone(), - &pred, - Arc::new(node.output_schema.clone()), - ); - } - if let Payload::Relational { - operator: NonASAPOpKind::Values { rows, schema }, - } = &node.payload - { - if !inputs.is_empty() { - return Err(invalid("Values takes no relational inputs")); - } - let empty = Arc::new(planner_types::pre_asap::Schema::default()); - let rows = rows - .iter() - .map(|row| { - row.iter() - .map(|expr| expression(expr, &empty)?.evaluate(&[])) - .collect::, Error>>() - }) - .collect::, Error>>()?; - let schema = Arc::new(schema.clone()); - return Operator::source( - schema.clone(), - vec![crate::values::Batch::try_new(schema, rows)?], - ); - } - let [input] = inputs else { - return Err(invalid( - "native Planner binding currently requires a unary operation or an explicit source", - )); - }; - match &node.payload { - Payload::FinalizeExactAccumulator => { - let state = summary_column(input)?; - use crate::Statistic as S; - use planner_types::post_asap::ExactKind as E; - let statistic = match &input.fields[state].dtype { - FieldDataType::ExactAggregate(kind, _) => match kind { - E::Sum => S::Sum, - E::Count => S::Count, - E::Min => S::Min, - E::Max => S::Max, - E::Rate => S::Rate, - E::Increase => S::Increase, - _ => return Err(invalid("exact family evaluation is unsupported")), - }, - _ => return Err(invalid("exact finalization requires exact state")), - }; - Operator::readout( - input.clone(), - state, - ReadoutQuery::Exact(ExactReadout { - statistic, - lookback_ms: None, - }), - ) - } - - Payload::Relational { operator } => match operator { - NonASAPOpKind::Project { cols, .. } => Operator::project( - input.clone(), - cols.iter() - .enumerate() - .map(|(i, col)| { - Ok(( - node.output_schema - .fields - .get(i) - .ok_or_else(|| invalid("projection width mismatch"))? - .name - .clone(), - match &col.expr { - WireScalarExpr::Column(index) => Expression::Column(*index), - expr => expression(expr, input)?, - }, - )) - }) - .collect::>()?, - ), - NonASAPOpKind::Filter { pred } => { - Operator::filter(input.clone(), expression(&pred.0, input)?) - } - NonASAPOpKind::Sort { keys, partition_by } => Operator::sort( - input.clone(), - keys.iter() - .map(|key| { - let WireScalarExpr::Column(column) = key.expr else { - return Err(invalid( - "sort expression must be projected before sorting", - )); - }; - Ok(SortKey { - column, - descending: !key.ascending, - nulls_first: key.nulls_first, - }) - }) - .collect::>()?, - groups(input, partition_by)?, - ), - NonASAPOpKind::Limit { - n, - offset, - partition_by, - } => Operator::limit( - input.clone(), - n.unwrap_or(usize::MAX) as u64, - *offset as u64, - groups(input, partition_by)?, - ), - NonASAPOpKind::Aggregate { - reduction, - measures, - output_names, - filters, - having: None, - } => { - if filters.iter().any(Option::is_some) { - return Err(invalid("filtered aggregate has no native implementation")); - } - if measures.len() != output_names.len() { - return Err(invalid("aggregate output names differ from measures")); - } - let PlannerReduction::Reduce(keys) = reduction else { - return Err(invalid( - "per-entity aggregate requires an explicit entity binding", - )); - }; - let measures = measures - .iter() - .zip(output_names) - .map(|(m, name)| { - let column = |col: Option| { - col.map(Ok) - .unwrap_or_else(|| named_column(input, &ColumnRef::SampleValue)) - }; - let m = match m { - AggIntent::Count { .. } => Reduction::Count, - AggIntent::Sum { col } => Reduction::Sum(column(*col)?), - AggIntent::Avg { col } => Reduction::Avg(column(*col)?), - AggIntent::Min { col } => Reduction::Min(column(*col)?), - AggIntent::Max { col } => Reduction::Max(column(*col)?), - _ => { - return Err(invalid( - "aggregate intent has no native implementation", - )) - } - }; - Ok((name.clone(), m)) - }) - .collect::>()?; - Operator::aggregate(input.clone(), groups(input, keys)?, measures) - } - _ => Err(invalid("value operation has no native implementation")), - }, - Payload::SummaryAgg { - family, - input: update, - reduction, - grouping, - filter, - } => { - if filter.is_some() { - return Err(invalid( - "filtered summary update has no native implementation", - )); - } - if let Some(item) = &update.item { - let PlannerReduction::Reduce(keys) = reduction else { - return Err(invalid("keyed summary requires explicit partitions")); - }; - let SummaryInputExpr::Column(weight) = &update.weight else { - return Err(invalid( - "keyed summary weight must be a finalized value column", - )); - }; - if matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::CmsWithHeap) - && !matches!( - update.weight_domain, - planner_types::post_asap::WeightDomain::NonNegative { .. } - ) - { - return Err(invalid("CMS requires a nonnegative weight contract")); - } - fn columns( - expr: &SummaryInputExpr, - input: &SchemaRef, - result: &mut Vec, - ) -> Result<(), Error> { - match expr { - SummaryInputExpr::Column(column) => { - result.push(named_column(input, column)?) - } - SummaryInputExpr::Tuple(items) => { - for item in items { - columns(item, input, result)?; - } - } - _ => return Err(invalid("keyed summary needs explicit item columns")), - } - Ok(()) - } - let mut items = Vec::new(); - columns(item, input, &mut items)?; - return Operator::keyed_summary_build( - input.clone(), - family.clone(), - named_column(input, weight)?, - items, - groups(input, keys)?, - ); - } - crate::capability::validate_summary_kernel(family, update, grouping) - .map_err(Error::Invalid)?; - let SummaryInputExpr::Column(column) = &update.weight else { - return Err(invalid( - "summary update expression must be projected to a column", - )); - }; - let PlannerReduction::Reduce(keys) = reduction else { - return Err(invalid( - "summary construction requires explicit grouping columns", - )); - }; - Operator::summary_build( - input.clone(), - family.clone(), - named_column(input, column)?, - input.time_index, - groups(input, keys)?, - ) - } - Payload::SummaryMerge => { - let state = summary_column(input)?; - Operator::summary_merge( - input.clone(), - state, - (0..input.fields.len()) - .filter(|&i| i != state && Some(i) != input.time_index) - .collect(), - ) - } - Payload::SummaryEstimate { query } => { - if let SketchStatistic::TopK { k } = query { - return Operator::keyed_readout( - input.clone(), - summary_column(input)?, - *k, - Arc::new(node.output_schema.clone()), - ); - } - Operator::readout( - input.clone(), - summary_column(input)?, - ReadoutQuery::Sketch(query.clone()), - ) - } - _ => Err(invalid( - "physical operation has no native binding; no fallback is installed", - )), - } -} -fn summary_column(input: &SchemaRef) -> Result { - let columns = input - .fields - .iter() - .enumerate() - .filter(|(_, f)| !matches!(f.dtype, FieldDataType::Plain(_))) - .map(|(i, _)| i) - .collect::>(); - match columns.as_slice() { - [column] => Ok(*column), - _ => Err(invalid("one summary state column required")), - } -} -fn named_column(input: &SchemaRef, column: &ColumnRef) -> Result { - let name = match column { - // Executable SchemaRef retains column names, not table qualifiers. - // Frontend binding has resolved the qualifier; still reject ambiguous - // names here rather than guessing a join side. - ColumnRef::Named(name) | ColumnRef::Qualified { name, .. } => name.as_str(), - ColumnRef::SampleValue => "value", - _ => { - return Err(invalid( - "summary update requires an unambiguous bound column", - )) - } - }; - let matches = input - .fields - .iter() - .enumerate() - .filter(|(_, field)| field.name == name) - .map(|(i, _)| i) - .collect::>(); - match matches.as_slice() { - [column] => Ok(*column), - _ => Err(invalid("summary update column missing or ambiguous")), - } -} -fn groups(input: &SchemaRef, groups: &GroupKeys) -> Result, Error> { - if groups.is_without() { - return Err(invalid("grouping without requires resolved label columns")); - } - if groups.keys().iter().any(|&i| i >= input.fields.len()) { - return Err(invalid("grouping column out of range")); - } - Ok(groups.keys().to_vec()) -} -fn expression(expr: &WireScalarExpr, input: &SchemaRef) -> Result { - let expr = local_scalar(expr)?; - Ok(Expression::unified_planner( - crate::expressions::unified_planner::CompiledExpression::compile(&expr, input)?, - )) -} - -struct CheckedSource<'a> { - source: Source<'a>, - output: SchemaRef, -} -impl PhysicalOperator for CheckedSource<'_> { - fn properties(&self, inputs: &[crate::plan::PlanProperties]) -> crate::plan::PlanProperties { - self.source.properties(inputs) - } - - fn name(&self) -> &str { - self.source.name() - } - fn input_schemas(&self) -> Vec { - vec![] - } - fn output_schema(&self) -> SchemaRef { - self.output.clone() - } - fn output_bytes(&self, batch: &Batch) -> usize { - self.source.output_bytes(batch) - } - fn start<'a>( - &'a self, - inputs: Vec>, - context: crate::runtime::RunContext, - ) -> Result, Error> { - use futures::StreamExt; - Ok(self - .source - .start(inputs, context)? - .map(|batch| { - let batch = batch?; - if batch.schema() != &self.output { - return Err(invalid("source batch differs from its bound schema")); - } - Ok(batch) - }) - .boxed_local()) - } -} - -// Bound recursion before invoking the upstream recursive provenance validator. -fn preflight_depth(dag: &PhysicalASAPDAG) -> Result<(), Error> { - let mut remaining = dag - .nodes - .iter() - .map(|node| (node.id, 0usize)) - .collect::>(); - if remaining.len() != dag.nodes.len() { - return Err(invalid("duplicate Planner node")); - } - let mut consumers = BTreeMap::<_, Vec<_>>::new(); - for edge in &dag.edges { - if !remaining.contains_key(&edge.producer) { - return Err(invalid("missing Planner edge producer")); - } - *remaining - .get_mut(&edge.consumer) - .ok_or_else(|| invalid("missing Planner edge consumer"))? += 1; - consumers - .entry(edge.producer) - .or_default() - .push(edge.consumer); - } - let mut ready = remaining - .iter() - .filter(|(_, n)| **n == 0) - .map(|(id, _)| *id) - .collect::>(); - let mut depths = BTreeMap::new(); - let mut visited = 0; - while let Some(id) = ready.pop_front() { - visited += 1; - let depth = *depths.get(&id).unwrap_or(&1usize); - if depth > 128 { - return Err(invalid("DAG exceeds the supported execution depth of 128")); - } - for &consumer in consumers.get(&id).into_iter().flatten() { - let next = depths.entry(consumer).or_insert(1); - *next = (*next).max(depth + 1); - let count = remaining.get_mut(&consumer).expect("validated endpoint"); - *count -= 1; - if *count == 0 { - ready.push_back(consumer); - } - } - } - if visited != dag.nodes.len() { - return Err(invalid("Planner DAG contains a cycle")); - } - Ok(()) -} - -/// Join predicates address the concatenated left/right schema. -fn semi_join_keys( - expr: &ScalarExpr, - left: usize, - right: usize, - keys: &mut Vec<(usize, usize)>, -) -> Result<(), Error> { - match expr { - ScalarExpr::BoolAnd(parts) => { - for part in parts { - semi_join_keys(part, left, right, keys)?; - } - } - ScalarExpr::Compare { - left: a, - op: CompareOpKind::Eq, - right: b, - .. - } => { - let (ScalarExpr::Column(a), ScalarExpr::Column(b)) = (a.as_ref(), b.as_ref()) else { - return Err(invalid("semi-join requires column equality keys")); - }; - let (a, b) = if a < b { (*a, *b) } else { (*b, *a) }; - if a >= left || b < left || b >= left + right { - return Err(invalid("semi-join key must match left to right")); - } - keys.push((a, b - left)); - } - _ => return Err(invalid("unsupported semi-join predicate")), - } - Ok(()) -} - -/// Resolve equality keys against the Planner join's concatenated input schema. -/// Deployments may use these positions to bind their source columns. -pub fn equijoin_keys( - pred: &planner_types::ir::Predicate, - left: &planner_types::post_asap::Schema, - right: &planner_types::post_asap::Schema, -) -> Result, Error> { - let mut keys = Vec::new(); - semi_join_keys(&pred.0, left.fields.len(), right.fields.len(), &mut keys)?; - if keys.is_empty() { - return Err(invalid("semi-join requires explicit matching keys")); - } - Ok(keys) -} - -fn local_scalar(expr: &WireScalarExpr) -> Result { - let mut missing = false; - let result = logical::scalar(expr, &mut |_| { - missing = true; - std::rc::Rc::new(OperatorNode::with_schema( - LogicalOperator::NonASAP(NonASAPOp::Values { - rows: vec![], - schema: Default::default(), - }), - Default::default(), - )) - }); - if missing { - Err(invalid( - "scalar plan reads require explicit execution bindings", - )) - } else { - Ok(result) - } -} diff --git a/crates/asap-physical-operators/src/unified_physical_planner/precompute.rs b/crates/asap-physical-operators/src/unified_physical_planner/precompute.rs deleted file mode 100644 index 12e7564d8..000000000 --- a/crates/asap-physical-operators/src/unified_physical_planner/precompute.rs +++ /dev/null @@ -1,643 +0,0 @@ -//! Compile immutable summary-input computation with explicit population and pane identity. -use super::promql_rows::SERIES_IDENTITY_COLUMN as SERIES_IDENTITY; -use super::*; -use planner_types::post_asap::FieldDataType as SummaryFamilyType; -use planner_types::{ - post_asap::{ExecutionTiming, GroupingStrategy, Schema}, - pre_asap::DataType, -}; - -/// Physical rows carry the population and pane coordinate alongside the logical value. -/// These fields preserve identities which are implicit in a stored summary instance. -pub fn population_schema(family: SummaryFamilyType) -> SchemaRef { - Arc::new(Schema { - fields: vec![ - planner_types::post_asap::Field { - name: "$population".into(), - dtype: SummaryFamilyType::Plain(DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Utf8), - value_nullable: false, - }), - nullable: false, - table: None, - }, - planner_types::post_asap::Field { - name: "$window_end".into(), - dtype: SummaryFamilyType::Plain(DataType::Timestamp), - nullable: false, - table: None, - }, - planner_types::post_asap::Field { - name: "value".into(), - dtype: family, - nullable: false, - table: None, - }, - ], - time_index: Some(1), - unique_keys: vec![], - closed: false, - }) -} - -/// Raw sample rows at a precompute boundary. `$population` holds the series' -/// complete label set, so it is the complete source identity of per-series -/// summaries; `$timestamp` is the sample time and `value` a finite sample -/// (stale markers are not samples). Rows are what the boundary's source scan -/// selected; the deployment decides which rows and panes they are. Label sets -/// must be canonical (sorted, unique, no empty values), since they are the -/// population identity: build rows with [`raw_sample_row`]. -pub fn raw_sample_schema() -> SchemaRef { - let mut schema = (*population_schema(SummaryFamilyType::Plain(DataType::Float64))).clone(); - schema.fields[1].name = "$timestamp".into(); - Arc::new(schema) -} - -/// A raw sample row whose label set is sorted, unique and omits empty values, -/// so one series always has one population identity. -pub fn raw_sample_row( - labels: &BTreeMap, - timestamp_ms: i64, - value: f64, -) -> Vec { - use crate::values::Value; - vec![ - Value::Map( - labels - .iter() - .filter(|(_, v)| !v.is_empty()) - .map(|(k, v)| { - ( - Value::Utf8(k.as_str().into()), - Value::Utf8(v.as_str().into()), - ) - }) - .collect::>() - .into(), - ), - Value::Timestamp(timestamp_ms), - Value::Float64(value), - ] -} - -/// Input contract of a precompute boundary: raw sample rows for a raw time -/// series scan, otherwise the stored population of its summary state. -pub fn boundary_schema(node: &PhysicalASAPDAGNode) -> Result { - if !matches!( - &node.payload, - Payload::Relational { - operator: NonASAPOpKind::Scan { - source: planner_types::pre_asap::Source::TimeSeries { .. }, - .. - } | NonASAPOpKind::TimeRange { .. } - } - ) { - return source_schema(&node.output_schema); - } - let logical = &node.output_schema; - // Labels may be absent from a series; its label map then omits them. - let valid = logical - .fields - .iter() - .enumerate() - .all(|(i, field)| match &field.dtype { - SummaryFamilyType::Plain(DataType::Timestamp) => { - Some(i) == logical.time_index && !field.nullable - } - SummaryFamilyType::Plain(DataType::Float64) => field.name == "value" && !field.nullable, - SummaryFamilyType::Plain(DataType::Utf8) => true, - _ => false, - }) - && !logical - .fields - .iter() - .any(|f| f.name.starts_with('$') && f.name != SERIES_IDENTITY) - && logical.time_index.is_some() - && logical.fields.iter().filter(|f| f.name == "value").count() == 1; - if !valid { - return Err(invalid( - "raw sample boundary requires labels, a timestamp and one Float64 value", - )); - } - Ok(raw_sample_schema()) -} - -/// Validate the adapter layout during installed-plan recovery without lowering operators. -pub fn source_schema(logical: &Schema) -> Result { - let states = logical - .fields - .iter() - .filter(|f| !matches!(f.dtype, SummaryFamilyType::Plain(_))) - .collect::>(); - let [state] = states.as_slice() else { - return Err(invalid( - "stored population requires one typed summary state", - )); - }; - if logical.fields.iter().enumerate().any(|(i, field)| matches!(&field.dtype, SummaryFamilyType::Plain(dtype) - if field.nullable || !matches!(dtype, DataType::Utf8) && !(Some(i) == logical.time_index && *dtype == DataType::Timestamp))) { - return Err(invalid("stored population metadata cannot reconstruct extra value columns")); - } - if state.nullable { - return Err(invalid("stored population state cannot be null")); - } - Ok(population_schema(state.dtype.clone())) -} - -pub fn is_population_schema(schema: &SchemaRef) -> bool { - schema - .fields - .get(2) - .is_some_and(|field| *schema == population_schema(field.dtype.clone())) -} - -/// Compile a complete selected precompute sub-DAG. Inputs are already-computed -/// state boundaries; the deployment supplies groups, panes and states, never operations. -pub fn compile( - dag: &PhysicalASAPDAG, - frontiers: &[NodeId], - roots: &[NodeId], -) -> Result { - preflight_depth(dag)?; - dag.validate().map_err(|e| invalid(e.to_string()))?; - let nodes = dag - .nodes - .iter() - .map(|n| (u64::from(n.id.0), n)) - .collect::>(); - let frontier = frontiers.iter().copied().collect::>(); - if frontier.len() != frontiers.len() || roots.iter().any(|r| frontier.contains(r)) { - return Err(invalid( - "precompute boundaries must be distinct from outputs", - )); - } - let mut dependencies = BTreeMap::>::new(); - let mut edges = dag.edges.iter().collect::>(); - edges.sort_by_key(|edge| { - ( - edge.consumer.0, - match edge.role { - planner_types::ir::export::EdgeRole::Left => 0, - planner_types::ir::export::EdgeRole::Input => 1, - planner_types::ir::export::EdgeRole::Right => 2, - planner_types::ir::export::EdgeRole::ScalarRef => 3, - }, - ) - }); - for edge in edges { - dependencies - .entry(u64::from(edge.consumer.0)) - .or_default() - .push(u64::from(edge.producer.0)); - } - let mut ordered = Vec::new(); - let mut seen = BTreeSet::new(); - let mut pending = roots.iter().map(|&id| (id, false)).collect::>(); - while let Some((id, expanded)) = pending.pop() { - if expanded { - ordered.push(id); - continue; - } - if !seen.insert(id) { - continue; - } - if !nodes.contains_key(&id) { - return Err(invalid("missing precompute node")); - } - pending.push((id, true)); - if !frontier.contains(&id) { - pending.extend( - dependencies - .get(&id) - .into_iter() - .flatten() - .map(|id| (*id, false)), - ); - } - } - let mut sources = BTreeMap::new(); - let mut fragments = BTreeMap::new(); - let mut outputs = BTreeMap::::new(); - for id in ordered { - let node = nodes[&id]; - if frontier.contains(&id) { - let schema = boundary_schema(node)?; - sources.insert(id, InputContract::bounded(schema.clone())); - outputs.insert(id, schema); - continue; - } - if node.output_state.timing != ExecutionTiming::IngestionTime { - return Err(invalid("precompute dag contains a query-time operation")); - } - let inputs = dependencies.get(&id).cloned().unwrap_or_default(); - let schemas = inputs - .iter() - .map(|id| { - outputs - .get(id) - .cloned() - .ok_or_else(|| invalid("missing precompute input")) - }) - .collect::, _>>()?; - let dag = fragment( - node, - &schemas, - &inputs.iter().map(|id| nodes[id]).collect::>(), - )?; - outputs.insert(id, dag.output_contract(dag.roots()[0])?.schema); - fragments.insert(id, (inputs, dag)); - } - CompiledPhysicalDAG::compose(sources, fragments, roots.to_vec()) -} - -fn validate_value_output(node: &PhysicalASAPDAGNode) -> Result<(), Error> { - let schema = &node.output_schema; - // Physical population rows already carry the complete identity in `$population`. - // Typed logical plans may expose its opaque series-identity column as metadata. - let identity = planner_types::pre_asap::schema::PROMQL_SERIES_IDENTITY; - let identities = schema - .fields - .iter() - .filter(|field| field.name == identity) - .collect::>(); - if identities.len() > 1 - || identities - .iter() - .any(|field| field.nullable || field.dtype != SummaryFamilyType::Plain(DataType::Utf8)) - { - return Err(invalid( - "precompute series identity requires one non-null Utf8 column", - )); - } - let values = schema - .fields - .iter() - .enumerate() - .filter(|(i, field)| Some(*i) != schema.time_index && field.name != identity) - .collect::>(); - if !matches!(values.as_slice(), [(_, field)] if !field.nullable && field.dtype == SummaryFamilyType::Plain(DataType::Float64)) - || schema.time_index.is_some_and(|i| { - schema.fields.get(i).is_none_or(|f| { - f.nullable || f.dtype != SummaryFamilyType::Plain(DataType::Timestamp) - }) - }) - { - return Err(invalid( - "precompute value schema requires Float64 and an optional declared timestamp", - )); - } - Ok(()) -} - -fn fragment( - node: &PhysicalASAPDAGNode, - schemas: &[SchemaRef], - parents: &[&PhysicalASAPDAGNode], -) -> Result { - let sources = schemas - .iter() - .enumerate() - .map(|(id, schema)| (id as u64, InputContract::bounded(schema.clone()))) - .collect(); - let mut operators = BTreeMap::new(); - let mut next = schemas.len() as u64; - let mut add = |inputs: Vec, op: Operator| -> Result { - let id = next; - next += 1; - operators.insert(id, (inputs, op)); - Ok(id) - }; - let root = match &node.payload { - Payload::Relational { - operator: - NonASAPOpKind::BinaryOp { - operator, - return_bool, - }, - } => { - let operator = - crate::expressions::binary::BinaryOperator::from_logical(operator, *return_bool); - validate_value_output(node)?; - if node.output_schema.time_index.is_none() - || parents.iter().any(|p| p.output_schema.time_index.is_none()) - { - return Err(invalid( - "precompute binary requires declared window timestamps", - )); - } - let [left, right] = schemas else { - return Err(invalid("precompute binary requires two inputs")); - }; - add( - vec![0, 1], - Operator::aligned_binary( - left.clone(), - right.clone(), - vec![(0, 0), (1, 1)], - (2, 2), - operator.clone(), - )?, - )? - } - Payload::FinalizeExactAccumulator => { - let [input] = schemas else { - return Err(invalid("finalize requires one state input")); - }; - validate_value_output(node)?; - let statistic = match &input.fields[2].dtype { - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _) => { - crate::Statistic::Sum - } - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Count, - _, - ) => crate::Statistic::Count, - _ => { - return Err(invalid( - "precompute finalization requires explicit Sum or Count semantics", - )) - } - }; - let read = Operator::readout( - input.clone(), - 2, - ReadoutQuery::Exact(ExactReadout { - statistic, - lookback_ms: None, - }), - )?; - let output = read.schema(); - let read = add(vec![0], read)?; - let project = Operator::project( - output, - vec![ - ("$population".into(), Expression::Column(0)), - ("$window_end".into(), Expression::Column(1)), - ( - "value".into(), - Expression::FiniteFloat64(Box::new(Expression::ExactFloat64(2))), - ), - ], - )? - .with_output_schema(population_schema(SummaryFamilyType::Plain( - DataType::Float64, - )))?; - add(vec![read], project)? - } - Payload::SummaryAgg { - family, - input: update, - reduction, - grouping, - filter, - } => { - if filter.is_some() { - return Err(invalid( - "filtered summary update has no native implementation", - )); - } - let [input] = schemas else { - return Err(invalid("summary update requires one input")); - }; - // Item identities resolve against the complete label set of raw - // samples; finalized evaluations carry no such identity. - let raw = *input == raw_sample_schema(); - // A unit-frequency summary (HLL) observes each raw sample value. - let unit_frequency = raw - && crate::capability::is_unit_sample_frequency(update) - && matches!(family, SummaryFamilyType::Sketch(kind, _) if !matches!( - kind.algorithm(), - planner_types::post_asap::SketchAlgorithm::Cms - | planner_types::post_asap::SketchAlgorithm::CountSketch - | planner_types::post_asap::SketchAlgorithm::CmsWithHeap - | planner_types::post_asap::SketchAlgorithm::CountSketchWithHeap - )); - let keyed = update.item.is_some() && !unit_frequency; - if (keyed && !raw) || !matches!(grouping, GroupingStrategy::PerSubpopulationInstance) { - return Err(invalid( - "precompute keyed/shared update needs its dedicated physical candidate", - )); - } - crate::capability::validate_summary_kernel(family, update, grouping) - .map_err(Error::Invalid)?; - if raw - && matches!( - update.weight_domain, - planner_types::post_asap::WeightDomain::NonNegative { - proof: planner_types::post_asap::NonNegativeWeightProof::ResetAwareCounterDerivative - } - ) - { - return Err(invalid( - "a counter-derivative weight cannot be read from raw cumulative samples", - )); - } - if keyed - && matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::CmsWithHeap) - && !matches!( - update.weight_domain, - planner_types::post_asap::WeightDomain::NonNegative { .. } - ) - { - return Err(invalid("CMS requires a nonnegative weight contract")); - } - let labels = match reduction { - PlannerReduction::PerEntity => Expression::Column(0), - PlannerReduction::Reduce(keys) => Expression::LabelSet { - column: 0, - labels: keys - .keys() - .iter() - .map(|key| { - parents[0] - .output_schema - .fields - .get(*key) - // A raw label map omits absent labels; the - // series identity is not one of its labels. - .filter(|field| { - (raw || !field.nullable) - && field.name != SERIES_IDENTITY - && field.dtype == SummaryFamilyType::Plain(DataType::Utf8) - }) - .map(|f| f.name.clone()) - .ok_or_else(|| { - invalid("summary grouping must identify population labels") - }) - }) - .collect::, _>>()?, - without: keys.is_without(), - }, - }; - let weight = match &update.weight { - _ if unit_frequency => Expression::Column(2), - SummaryInputExpr::Constant(value) => Expression::Literal { - value: crate::values::Value::Float64(*value), - dtype: DataType::Float64, - }, - SummaryInputExpr::Column(ColumnRef::SampleValue) => Expression::Column(2), - SummaryInputExpr::Column(ColumnRef::Named(name)) - if parents[0].output_schema.fields.iter().any(|f| { - f.name == *name && f.dtype == SummaryFamilyType::Plain(DataType::Float64) - }) => - { - Expression::Column(2) - } - _ => { - return Err(invalid( - "summary weight does not resolve to the input value", - )) - } - }; - let mut columns = vec![ - ("$population".into(), labels), - ("$window_end".into(), Expression::Column(1)), - ("value".into(), Expression::FiniteFloat64(Box::new(weight))), - ]; - let mut fields = population_schema(SummaryFamilyType::Plain(DataType::Float64)) - .fields - .clone(); - if keyed { - let mut items = Vec::new(); - raw_items( - update.item.as_ref().expect("keyed item"), - &parents[0].output_schema, - &mut items, - )?; - for (index, (expression, dtype)) in items.into_iter().enumerate() { - let name = format!("$item{index}"); - fields.push(planner_types::post_asap::Field { - name: name.clone(), - dtype: SummaryFamilyType::Plain(dtype), - nullable: false, - table: None, - }); - columns.push((name, expression)); - } - } - let item_columns = (3..fields.len()).collect::>(); - let project = Operator::project(input.clone(), columns)?.with_output_schema( - Arc::new(Schema { - fields, - unique_keys: vec![], - closed: false, - time_index: Some(1), - }), - )?; - let projected = project.schema(); - let project = add(vec![0], project)?; - let build = if keyed { - Operator::keyed_summary_build(projected, family.clone(), 2, item_columns, vec![0])? - } else { - Operator::summary_build(projected, family.clone(), 2, Some(1), vec![0])? - }; - let built = build.schema(); - let build = add(vec![project], build)?; - add( - vec![build], - Operator::scope_timestamp(built, population_schema(family.clone()))?, - )? - } - Payload::SummaryMerge => { - let Some(input) = schemas.first() else { - return Err(invalid("summary merge requires inputs")); - }; - if schemas.iter().any(|s| s != input) { - return Err(invalid("summary merge inputs differ")); - } - let union = add( - (0..schemas.len() as u64).collect(), - Operator::union(input.clone(), schemas.len())?, - )?; - let merge = Operator::summary_merge(input.clone(), 2, vec![0])?; - let merged = merge.schema(); - let merge = add(vec![union], merge)?; - add( - vec![merge], - Operator::scope_timestamp(merged, input.clone())?, - )? - } - _ => { - return Err(invalid( - "precompute operation has no native population implementation", - )) - } - }; - CompiledPhysicalDAG::from_operators(sources, operators, vec![root]) -} - -/// Resolve keyed item identities over raw sample rows: labels (absent labels -/// read as empty, as in PromQL), the sample value, or the canonical encoding -/// of the label set less excluded labels. -fn raw_items( - expr: &SummaryInputExpr, - scan: &Schema, - items: &mut Vec<(Expression, DataType)>, -) -> Result<(), Error> { - // Open PromQL scans need not list every label, so any name that is not - // another scan column (value, time, series identity) reads as a label. - let label = |column: &ColumnRef| match column { - ColumnRef::Named(name) | ColumnRef::Qualified { name, .. } - if !name.starts_with('$') - && scan.fields.iter().all(|f| { - &f.name != name || f.dtype == SummaryFamilyType::Plain(DataType::Utf8) - }) => - { - Some(name.clone()) - } - _ => None, - }; - let identity = |excluding: Vec| { - ( - Expression::LabelIdentity { - column: 0, - excluding, - }, - DataType::Utf8, - ) - }; - match expr { - SummaryInputExpr::Column(ColumnRef::SampleValue) => { - items.push((Expression::Column(2), DataType::Float64)) - } - SummaryInputExpr::Column(ColumnRef::Named(name) | ColumnRef::Qualified { name, .. }) - if name == "value" => - { - items.push((Expression::Column(2), DataType::Float64)) - } - SummaryInputExpr::Column(ColumnRef::Named(name) | ColumnRef::Qualified { name, .. }) - if name == SERIES_IDENTITY => - { - items.push(identity(vec![])) - } - SummaryInputExpr::Column(column) if label(column).is_some() => items.push(( - Expression::Label { - column: 0, - name: label(column).expect("resolved label"), - }, - DataType::Utf8, - )), - SummaryInputExpr::EntityIdentity( - planner_types::post_asap::EntityIdentity::PromqlLabelSet { excluding }, - ) => items.push(identity( - excluding - .iter() - .map(|column| { - label(column).ok_or_else(|| invalid("excluded identity label is not a label")) - }) - .collect::>()?, - )), - SummaryInputExpr::Tuple(parts) if !parts.is_empty() => { - for part in parts { - raw_items(part, scan, items)?; - } - } - _ => { - return Err(invalid( - "keyed summary item does not resolve over raw samples", - )) - } - } - Ok(()) -} diff --git a/crates/asap-physical-operators/src/unified_physical_planner/promql_fallback.rs b/crates/asap-physical-operators/src/unified_physical_planner/promql_fallback.rs deleted file mode 100644 index 5d03bb0b9..000000000 --- a/crates/asap-physical-operators/src/unified_physical_planner/promql_fallback.rs +++ /dev/null @@ -1,859 +0,0 @@ -//! Compile a retained PromQL sub-DAG (`Fallback`) from its typed expression. -//! The deployment supplies the raw series of each selector; the Planner -//! computes selection, range functions, subqueries, matching and aggregation. -use super::*; -use crate::operators::SubquerySteps; -use planner_types::post_asap::execution_data_state::lift_plain; -use planner_types::pre_asap::{AtModifier, VectorMatchKind}; - -/// Input slot for the raw series read by the `selector`th selector (in -/// [`raw_series`] order) of Fallback node `node`. The node's own ID names its -/// computed output, so the raw rows need another. -pub fn raw_series_input(node: NodeId, selector: usize) -> NodeId { - node | ((selector as u64 + 1) << 32) -} - -/// The Fallback node that owns a raw-series input slot. -pub(super) fn raw_series_owner(slot: NodeId) -> Option { - (slot >> 32 != 0).then_some(slot & u64::from(u32::MAX)) -} - -/// A selector expression and its raw-series row schema. -pub type Selector = (OperatorNode, SchemaRef); - -/// The selectors a Fallback expression reads, left to right, and the row -/// schema of the raw series the deployment supplies for each at -/// [`raw_series_input`]. The rows must cover the selector's window at every -/// evaluation instant `T`, or at its `@` time: `(T - offset - range, T - offset]`; -/// under a subquery `[R:S] offset O` that is `(T - O - R - offset - range, T - O - offset]`. -pub fn raw_series(expression: &OperatorNode) -> Result, Error> { - Ok(lower(expression)?.selectors) -} - -/// An operator input: a selector's raw rows or an earlier step. -pub(super) enum Input { - Raw(usize), - Step(usize), -} - -/// Operators computing an expression; the last step is its result. -#[derive(Default)] -pub(super) struct Lowering { - pub selectors: Vec, - pub steps: Vec<(Operator, Vec)>, -} - -pub(super) fn lower(expression: &OperatorNode) -> Result { - let mut lowering = Lowering::default(); - lowering.value(expression)?; - Ok(lowering) -} - -/// Compile a standalone scalar expression and expose its real series dependencies. -/// Input slots use root 0; no logical wrapper node is introduced. -pub fn compile_scalar_root( - expr: &ScalarExpr, -) -> Result<(CompiledPhysicalDAG, Vec), Error> { - let mut lowering = Lowering::default(); - lowering.scalar_value(expr)?; - let mut inputs = BTreeMap::new(); - for (i, (_, schema)) in lowering.selectors.iter().enumerate() { - inputs.insert( - raw_series_input(0, i), - InputContract::bounded(schema.clone()), - ); - } - let last = lowering.steps.len() - 1; - let mut operators = BTreeMap::new(); - for (i, (operator, dependencies)) in lowering.steps.into_iter().enumerate() { - let id = if i == last { 0 } else { i as u64 + 1 }; - let dependencies = dependencies - .into_iter() - .map(|input| match input { - Input::Raw(i) => raw_series_input(0, i), - Input::Step(i) => i as u64 + 1, - }) - .collect(); - operators.insert(id, (dependencies, operator)); - } - Ok(( - CompiledPhysicalDAG::from_operators(inputs, operators, vec![0])?, - lowering.selectors, - )) -} - -fn declared(expression: &OperatorNode) -> Result { - let schema = expression.schema.clone(); - Ok(Arc::new(lift_plain(&schema))) -} - -fn millis(duration: &std::time::Duration) -> Result { - i64::try_from(duration.as_millis()).map_err(|_| invalid("PromQL duration exceeds Int64")) -} - -/// A fixed `@` time. `start()`/`end()` depend on the deployment's range query. -fn at(shift: &planner_types::pre_asap::TimeShift) -> Result, Error> { - match shift.at { - None => Ok(None), - Some(AtModifier::Timestamp(at)) => Ok(Some(at)), - Some(AtModifier::Start | AtModifier::End) => Ok(None), - } -} - -fn range_anchor(expression: &OperatorNode) -> Option { - match expression.expect_non_asap() { - NonASAPOp::TimeRange { child, .. } => range_anchor(child), - NonASAPOp::TimeShift { shift, .. } => shift - .at - .filter(|at| matches!(at, AtModifier::Start | AtModifier::End)), - _ => None, - } -} - -/// `TimeRange { range, [TimeShift { offset, @ }], Scan }`: range, offset, `@`. -fn selector(expression: &OperatorNode) -> Result<(i64, i64, Option), Error> { - let NonASAPOp::TimeRange { range, child, .. } = expression.expect_non_asap() else { - return Err(invalid("PromQL operand must be a series selector")); - }; - let (offset, at, scan) = match child.expect_non_asap() { - NonASAPOp::TimeShift { shift, child } => { - (shift.offset_ms, at(shift)?, child.expect_non_asap()) - } - scan => (0, None, scan), - }; - if !matches!(scan, NonASAPOp::Scan { .. }) { - return Err(invalid("PromQL selector must read one scan")); - } - Ok((millis(range)?, offset, at)) -} - -impl Lowering { - fn schema(&self, input: &Input) -> SchemaRef { - match input { - Input::Raw(i) => self.selectors[*i].1.clone(), - Input::Step(i) => self.steps[*i].0.schema(), - } - } - - fn add(&mut self, operator: Operator, inputs: Vec) -> Input { - self.steps.push((operator, inputs)); - Input::Step(self.steps.len() - 1) - } - - /// Conform `operator` to the logical schema of the expression it computes. - fn push( - &mut self, - operator: Operator, - inputs: Vec, - logical: &OperatorNode, - ) -> Result { - Ok(self.add(operator.with_output_schema(declared(logical)?)?, inputs)) - } - - fn read(&mut self, selector: &OperatorNode) -> Result { - let schema = declared(selector)?; - if !schema - .fields - .iter() - .any(|f| f.name == promql_rows::SERIES_IDENTITY_COLUMN) - { - return Err(invalid( - "PromQL fallback requires the complete series identity", - )); - } - self.selectors.push((selector.clone(), schema)); - Ok(Input::Raw(self.selectors.len() - 1)) - } - - /// An instant vector, or a scalar for scalar-valued expressions. - fn value(&mut self, expression: &OperatorNode) -> Result { - match expression.expect_non_asap() { - NonASAPOp::Concat { children, .. } => { - if !children.iter().all(|branch| matches!(branch.expect_non_asap(), - NonASAPOp::PromqlRelabel { child, .. } if matches!(child.expect_non_asap(), - NonASAPOp::Aggregate { measures, .. } if matches!(measures.as_slice(), [AggIntent::HistogramQuantile { .. }])))) { - return Err(invalid("PromQL concatenation requires classic histogram quantile branches")); - } - let inputs = children - .iter() - .map(|child| self.value(child)) - .collect::, _>>()?; - let output = declared(expression)?; - if inputs.iter().any(|input| self.schema(input) != output) { - return Err(invalid( - "concatenated PromQL branches require equal schemas", - )); - } - let union = self.add(Operator::union(output.clone(), inputs.len())?, inputs); - // Multi-quantile branches drop the metric name and form one vector. - self.push( - Operator::series_without_name(output)?, - vec![union], - expression, - ) - } - NonASAPOp::PromqlRelabel { dst, value, child } => { - let step = self.value(child)?; - let input = self.schema(&step); - let (replacement, source_regex) = match value { - ScalarExpr::Literal(planner_types::pre_asap::ScalarValue::Utf8(value)) => { - (value.clone(), None) - } - ScalarExpr::FunctionCall { name, args } if name == "label_replace" => { - let [ScalarExpr::Column(source), ScalarExpr::Literal(planner_types::pre_asap::ScalarValue::Utf8( - pattern, - )), ScalarExpr::Literal(planner_types::pre_asap::ScalarValue::Utf8( - replacement, - ))] = args.as_slice() - else { - return Err(invalid("invalid label_replace arguments")); - }; - let source = input - .fields - .get(*source) - .ok_or_else(|| invalid("label_replace source missing"))? - .name - .clone(); - (replacement.clone(), Some((source, pattern.clone()))) - } - _ => return Err(invalid("unsupported PromQL label rewrite")), - }; - let operator = Operator::series_relabel( - input, - declared(expression)?, - dst.clone(), - replacement, - source_regex, - )?; - self.push(operator, vec![step], expression) - } - NonASAPOp::TimeRange { .. } => { - let (range, offset, at) = selector(expression)?; - let input = self.read(expression)?; - let schema = self.schema(&input); - self.push( - Operator::series_window(schema, None, range, offset, at, None)? - .with_series_range_bounds(range_anchor(expression), None)?, - vec![input], - expression, - ) - } - NonASAPOp::Aggregate { - reduction: planner_types::pre_asap::Reduction::PerEntity, - measures, - having: None, - child, - filters, - .. - } if filters.iter().all(Option::is_none) => { - let [function] = measures.as_slice() else { - return Err(invalid("range function requires one measure")); - }; - let step = self.range_function(function, child, expression)?; - if matches!(function, AggIntent::LastOverTime) { - return Ok(step); - } - // Other range functions drop the name; equal label sets then error. - let input = self.schema(&step); - Ok(self.add(Operator::series_without_name(input)?, vec![step])) - } - NonASAPOp::Aggregate { - reduction: planner_types::pre_asap::Reduction::Reduce(keys), - measures, - having: None, - child, - filters, - .. - } if filters.iter().all(Option::is_none) => { - let [measure] = measures.as_slice() else { - return Err(invalid("vector aggregation requires one measure")); - }; - let input = self.value(child)?; - self.aggregate(input, measure, keys, expression) - } - NonASAPOp::Project { - cols, - child, - qualifier, - } => { - let value = planner_types::pre_asap::column_resolution::resolve_column_ref( - &ColumnRef::SampleValue, - &child.schema, - ) - .map_err(|e| invalid(e.to_string()))?; - let sample = cols - .iter() - .find(|col| { - col.alias.as_deref() == Some(child.schema.fields[value].name.as_str()) - }) - .ok_or_else(|| invalid("missing sample projection"))?; - let keep_name = matches!(sample.expr, ScalarExpr::Negative { .. }); - let fields: Vec<_> = child - .schema - .fields - .iter() - .enumerate() - .filter(|(_, field)| keep_name || field.name != "__name__") - .collect(); - if qualifier.is_some() || cols.len() != fields.len() { - return Err(invalid("unsupported temporal projection shape")); - } - let mut computed = None; - for (col, (index, field)) in cols.iter().zip(fields) { - if col.alias.as_deref() != Some(field.name.as_str()) { - return Err(invalid("unsupported temporal projection alias")); - } - if index == value { - computed = Some(col); - } else { - let expected = if !keep_name - && field.name == planner_types::pre_asap::schema::PROMQL_SERIES_IDENTITY - { - ScalarExpr::FunctionCall { - name: "promql_drop_metric_name".into(), - args: vec![ScalarExpr::Column(index)], - } - } else { - ScalarExpr::Column(index) - }; - if col.expr != expected { - return Err(invalid("unsupported temporal projection expression")); - } - } - } - let computed = computed.ok_or_else(|| invalid("no computed sample"))?; - if matches!( - computed.expr, - ScalarExpr::Negative { .. } | ScalarExpr::FunctionCall { .. } - ) { - return self.pointwise_projection(cols, child, value, expression, keep_name); - } - self.sample_scalar_operation(&computed.expr, child, value, expression) - } - NonASAPOp::Filter { pred, child } => { - let value = planner_types::pre_asap::column_resolution::resolve_column_ref( - &ColumnRef::SampleValue, - &child.schema, - ) - .map_err(|e| invalid(e.to_string()))?; - self.sample_scalar_operation(&pred.0, child, value, expression) - } - NonASAPOp::Sort { - keys, - partition_by, - child, - } => { - let step = self.value(child)?; - let input = self.schema(&step); - let keys = keys - .iter() - .map(|key| match key.expr { - ScalarExpr::Column(column) => Ok(SortKey { - column, - descending: !key.ascending, - nulls_first: key.nulls_first, - }), - _ => Err(invalid("sort key must be a column")), - }) - .collect::>()?; - let groups = groups(&input, partition_by)?; - self.push(Operator::sort(input, keys, groups)?, vec![step], expression) - } - NonASAPOp::Limit { - n, offset, child, .. - } => { - let step = self.value(child)?; - let input = self.schema(&step); - // `topk by (...)` partitions through the Sort it limits. - let groups = match child.expect_non_asap() { - NonASAPOp::Sort { partition_by, .. } => groups(&input, partition_by)?, - _ => vec![], - }; - self.push( - Operator::limit( - input, - n.unwrap_or(usize::MAX) as u64, - *offset as u64, - groups, - )?, - vec![step], - expression, - ) - } - NonASAPOp::BinaryOp { - operator, - lhs, - rhs, - return_bool, - } => { - let sides = vec![self.value(lhs)?, self.value(rhs)?]; - let operator = crate::expressions::binary::BinaryOperator::from_logical( - operator, - *return_bool, - ); - let binary = Operator::series_binary( - self.schema(&sides[0]), - self.schema(&sides[1]), - operator, - [false, false], - )?; - self.push(binary, sides, expression) - } - NonASAPOp::PromqlVectorFromScalar(expr) => { - let step = self.scalar_value(expr)?; - let input = self.schema(&step); - Ok(self.add( - Operator::scope_timestamp(input, declared(expression)?)?, - vec![step], - )) - } - _ => Err(invalid("PromQL expression has no native fallback lowering")), - } - } - - fn pointwise_projection( - &mut self, - cols: &[planner_types::ir::ProjectItem], - child: &OperatorNode, - value: usize, - output: &OperatorNode, - keep_name: bool, - ) -> Result { - let mut input = self.value(child)?; - let mut projected = cols.to_vec(); - for col in &mut projected { - if col.alias.as_deref() != Some(child.schema.fields[value].name.as_str()) { - continue; - } - if let ScalarExpr::FunctionCall { name, args } = &mut col.expr { - if planner_types::pre_asap::scalar_type_rules::promql_function_arity(name).is_none() - || args.first() != Some(&ScalarExpr::Column(value)) - { - return Err(invalid("unsupported pointwise function")); - } - for arg in args.iter_mut().skip(1) { - let scalar = self.scalar_value(arg)?; - let left = self.schema(&input); - let right = self.schema(&scalar); - let index = left.fields.len(); - let mut schema = (*left).clone(); - schema.fields.extend(right.fields.clone()); - let join = Operator::unified_relational_join( - left, - right, - planner_types::pre_asap::JoinKind::Inner, - &planner_types::ir::Predicate(ScalarExpr::Literal( - planner_types::pre_asap::ScalarValue::Boolean(true), - )), - Arc::new(schema), - )?; - input = self.add(join, vec![input, scalar]); - *arg = ScalarExpr::Column(index); - } - if name == "promql_clamp" { - let predicate = ScalarExpr::Not(Box::new(ScalarExpr::Compare { - left: Box::new(args[1].clone()), - right: Box::new(args[2].clone()), - op: planner_types::pre_asap::CompareOpKind::Gt, - semantics: planner_types::ir::ExprSemantics::Promql, - })); - let schema = self.schema(&input); - let predicate = - crate::expressions::unified_planner::CompiledExpression::compile( - &predicate, &schema, - )?; - input = self.add( - Operator::filter( - schema, - crate::expressions::Expression::unified_planner(predicate), - )?, - vec![input], - ); - } - } - } - let schema = self.schema(&input); - let columns = projected - .iter() - .map(|col| { - Ok(( - col.alias.clone().unwrap(), - crate::expressions::Expression::unified_planner( - crate::expressions::unified_planner::CompiledExpression::compile( - &col.expr, &schema, - )?, - ), - )) - }) - .collect::, Error>>()?; - let project = Operator::project(schema, columns)?; - let result = self.push(project, vec![input], output)?; - if keep_name { - Ok(result) - } else { - self.push( - Operator::series_without_name(self.schema(&result))?, - vec![result], - output, - ) - } - } - - fn sample_scalar_operation( - &mut self, - expr: &ScalarExpr, - child: &OperatorNode, - value: usize, - output: &OperatorNode, - ) -> Result { - let (left, right, kind) = scalar_binary(expr)?; - let (scalar, scalar_left) = match (left, right) { - (ScalarExpr::Column(i), scalar) if *i == value => (scalar, false), - (scalar, ScalarExpr::Column(i)) if *i == value => (scalar, true), - _ => { - return Err(invalid( - "sample projection requires one vector sample and one scalar", - )) - } - }; - let vector = self.value(child)?; - let scalar = self.scalar_value(scalar)?; - let sides = if scalar_left { - vec![scalar, vector] - } else { - vec![vector, scalar] - }; - let operator = Operator::series_binary( - self.schema(&sides[0]), - self.schema(&sides[1]), - kernel(kind), - [scalar_left, !scalar_left], - )?; - self.push(operator, sides, output) - } - - fn scalar_value(&mut self, expr: &ScalarExpr) -> Result { - match expr { - ScalarExpr::Literal(planner_types::pre_asap::ScalarValue::Float64(value)) => Ok(self - .add( - Operator::scalar(crate::values::Value::Float64(*value), DataType::Float64)?, - vec![], - )), - ScalarExpr::EvalTimestamp => Ok(self.add(Operator::evaluation_time(), vec![])), - ScalarExpr::PromqlScalarFromVector(child) => { - let step = self.value(child)?; - let input = self.schema(&step); - let values: Vec<_> = input - .fields - .iter() - .enumerate() - .filter(|(_, f)| f.dtype == FieldDataType::Plain(DataType::Float64)) - .map(|(i, _)| i) - .collect(); - let [value] = values.as_slice() else { - return Err(invalid("scalar() requires one float sample column")); - }; - let value = *value; - Ok(self.add(Operator::vector_to_scalar(input, value)?, vec![step])) - } - ScalarExpr::Negative { expr, .. } => { - let value = self.scalar_value(expr)?; - let minus = self.scalar_value(&ScalarExpr::literal_f64(-1.0))?; - let op = Operator::series_binary( - self.schema(&value), - self.schema(&minus), - kernel(crate::expressions::binary::BinaryOpKind::Arithmetic( - planner_types::pre_asap::ArithmeticOpKind::Mul, - )), - [true, true], - )?; - Ok(self.add(op, vec![value, minus])) - } - _ => { - let (left, right, kind) = scalar_binary(expr)?; - let sides = vec![self.scalar_value(left)?, self.scalar_value(right)?]; - let op = Operator::series_binary( - self.schema(&sides[0]), - self.schema(&sides[1]), - kernel(kind), - [true, true], - )?; - Ok(self.add(op, sides)) - } - } - } - - /// `function(matrix)`, where the matrix is a range selector or a subquery. - fn range_function( - &mut self, - function: &AggIntent, - matrix: &OperatorNode, - logical: &OperatorNode, - ) -> Result { - let function = unbound(function)?; - let (subquery, offset, at_ms) = match matrix.expect_non_asap() { - NonASAPOp::TimeShift { shift, child } => (child.as_ref(), shift.offset_ms, at(shift)?), - _ => (matrix, 0, None), - }; - let NonASAPOp::PromqlSubquery { - range: outer, - resolution, - child, - } = subquery.expect_non_asap() - else { - let (range, offset, at) = selector(matrix)?; - let input = self.read(matrix)?; - let schema = self.schema(&input); - return self.push( - Operator::series_window(schema, Some(function), range, offset, at, None)? - .with_series_range_bounds(range_anchor(matrix), None)?, - vec![input], - logical, - ); - }; - let step = resolution.as_ref().ok_or_else(|| { - invalid("subquery resolution defaults to the deployment evaluation interval") - })?; - let steps = SubquerySteps { - range_ms: millis(outer)?, - step_ms: millis(step)?, - offset_ms: offset, - at_ms, - }; - // Each step evaluates a per-series selection or range function. - let (inner, selected) = match child.expect_non_asap() { - NonASAPOp::Aggregate { - reduction: planner_types::pre_asap::Reduction::PerEntity, - measures, - having: None, - child: selected, - .. - } => match measures.as_slice() { - [inner] => (Some(unbound(inner)?), selected.as_ref()), - _ => return Err(invalid("range function requires one measure")), - }, - _ => (None, child.as_ref()), - }; - let (range, inner_offset, inner_at) = selector(selected)?; - let raw = self.read(selected)?; - let schema = self.schema(&raw); - let mut step = self.push( - Operator::series_window( - schema, - inner.clone(), - range, - inner_offset, - inner_at, - Some(steps), - )? - .with_series_range_bounds(range_anchor(selected), range_anchor(matrix))?, - vec![raw], - child, - )?; - // Name removal must validate each subquery evaluation step. - if inner.is_some() && !matches!(inner, Some(AggIntent::LastOverTime)) { - let input = self.schema(&step); - let relabel = Operator::series_without_name(input)?; - step = self.add(relabel, vec![step]); - } - let input = self.schema(&step); - self.push( - Operator::series_window(input, Some(function), steps.range_ms, offset, at_ms, None)? - .with_series_range_bounds(range_anchor(matrix), None)?, - vec![step], - logical, - ) - } - - /// Cross-series aggregation. A global aggregate groups by one constant so - /// that no input series yields an empty vector, not one row. - fn aggregate( - &mut self, - mut step: Input, - measure: &AggIntent, - keys: &GroupKeys, - logical: &OperatorNode, - ) -> Result { - let mut input = self.schema(&step); - if let AggIntent::HistogramQuantile { q, le } = measure { - if !keys.is_without() || keys.keys() != [*le] { - return Err(invalid("histogram_quantile must group without (le)")); - } - let operator = Operator::series_histogram_quantile(input, *q, *le)?; - return self.push(operator, vec![step], logical); - } - let value = input - .fields - .iter() - .enumerate() - .filter(|(_, f)| f.dtype == FieldDataType::Plain(DataType::Float64)) - .map(|(i, _)| i) - .collect::>(); - let [value] = value.as_slice() else { - return Err(invalid("PromQL aggregation requires one Float64 value")); - }; - let value = *value; - let reduction = match measure { - AggIntent::Sum { .. } => Reduction::Sum(value), - AggIntent::Avg { .. } => Reduction::Avg(value), - AggIntent::Min { .. } => Reduction::Min(value), - AggIntent::Max { .. } => Reduction::Max(value), - AggIntent::Count { .. } => Reduction::Count, - _ => return Err(invalid("vector aggregate has no native lowering")), - }; - let mut groups = if keys.is_without() { - // Group by every remaining label, including the rewritten identity. - let excluded = keys.keys(); - if excluded.iter().any(|&i| i >= input.fields.len()) { - return Err(invalid("grouping column out of range")); - } - let names = excluded.iter().map(|&i| input.fields[i].name.clone()); - let relabel = - Operator::series_labels(input.clone(), VectorMatchKind::Ignoring, names.collect())?; - step = self.add(relabel, vec![step]); - (0..input.fields.len()) - .filter(|&i| Some(i) != input.time_index && i != value && !excluded.contains(&i)) - .collect() - } else { - groups(&input, keys)? - }; - let global = groups.is_empty(); - if global { - let mut columns = (0..input.fields.len()) - .map(|i| (input.fields[i].name.clone(), Expression::Column(i))) - .collect::>(); - columns.push(( - "$promql_global_group".into(), - Expression::Literal { - value: crate::values::Value::Utf8("".into()), - dtype: DataType::Utf8, - }, - )); - let project = Operator::project(input, columns)?; - input = project.schema(); - groups = vec![input.fields.len() - 1]; - step = self.add(project, vec![step]); - } - let output = declared(logical)?; - let name = output - .fields - .last() - .ok_or_else(|| invalid("aggregate output lacks a value"))? - .name - .clone(); - let aggregate = Operator::aggregate(input, groups, vec![(name, reduction)])?; - let actual = aggregate.schema(); - let step = self.add(aggregate, vec![step]); - // Drop the constant group; convert counts where PromQL declares Float64. - let skip = usize::from(global); - let columns = actual.fields[skip..] - .iter() - .zip(&output.fields) - .enumerate() - .map(|(i, (field, declared))| { - let column = i + skip; - let expression = if field.dtype != declared.dtype { - Expression::ExactFloat64(column) - } else { - Expression::Column(column) - }; - (field.name.clone(), expression) - }) - .collect(); - self.push(Operator::project(actual, columns)?, vec![step], logical) - } -} - -fn unbound(intent: &AggIntent) -> Result, Error> { - Ok(match intent { - AggIntent::Rate => AggIntent::Rate, - AggIntent::Deriv => AggIntent::Deriv, - AggIntent::PredictLinear { seconds } => AggIntent::PredictLinear { seconds: *seconds }, - AggIntent::Increase => AggIntent::Increase, - AggIntent::Delta => AggIntent::Delta, - AggIntent::Count { accuracy } => AggIntent::Count { - accuracy: accuracy.clone(), - }, - AggIntent::Sum { .. } => AggIntent::Sum { col: None }, - AggIntent::Avg { .. } => AggIntent::Avg { col: None }, - AggIntent::Min { .. } => AggIntent::Min { col: None }, - AggIntent::Max { .. } => AggIntent::Max { col: None }, - AggIntent::IRate => AggIntent::IRate, - AggIntent::IDelta => AggIntent::IDelta, - AggIntent::Changes => AggIntent::Changes, - AggIntent::Resets => AggIntent::Resets, - AggIntent::LastOverTime => AggIntent::LastOverTime, - AggIntent::Quantile { - col: None, - q, - accuracy, - } => AggIntent::Quantile { - col: None, - q: *q, - accuracy: accuracy.clone(), - }, - _ => return Err(invalid("unsupported PromQL range function")), - }) -} - -fn kernel( - kind: crate::expressions::binary::BinaryOpKind, -) -> crate::expressions::binary::BinaryOperator { - crate::expressions::binary::BinaryOperator { - kind, - vector_match: None, - checked_relative_division: false, - checked_finite_division: false, - } -} - -fn scalar_binary( - expr: &ScalarExpr, -) -> Result< - ( - &ScalarExpr, - &ScalarExpr, - crate::expressions::binary::BinaryOpKind, - ), - Error, -> { - use crate::expressions::binary::BinaryOpKind as K; - match expr { - ScalarExpr::Arithmetic { - left, - right, - op, - semantics: planner_types::ir::ExprSemantics::Promql, - } => Ok((left, right, K::Arithmetic(op.clone()))), - ScalarExpr::Compare { - left, - right, - op, - semantics: planner_types::ir::ExprSemantics::Promql, - } => Ok((left, right, K::Compare(op.clone()))), - ScalarExpr::Case { - operand: None, - branches, - else_expr, - } if matches!(else_expr.as_deref(), Some(ScalarExpr::Literal(planner_types::pre_asap::ScalarValue::Float64(v))) if *v == 0.0) => - { - let [( - ScalarExpr::Compare { - left, - right, - op, - semantics: planner_types::ir::ExprSemantics::Promql, - }, - ScalarExpr::Literal(planner_types::pre_asap::ScalarValue::Float64(v)), - )] = branches.as_slice() - else { - return Err(invalid("unsupported scalar case")); - }; - if *v != 1.0 { - return Err(invalid("unsupported scalar case result")); - } - Ok((left, right, K::CompareBool(op.clone()))) - } - _ => Err(invalid("scalar expression has no native temporal lowering")), - } -} diff --git a/crates/asap-physical-operators/src/unified_physical_planner/promql_rows.rs b/crates/asap-physical-operators/src/unified_physical_planner/promql_rows.rs deleted file mode 100644 index 20992dfe7..000000000 --- a/crates/asap-physical-operators/src/unified_physical_planner/promql_rows.rs +++ /dev/null @@ -1,308 +0,0 @@ -//! A bounded PromQL source row carries the entire label set, not just labels -//! mentioned by the query. The source adapter owns this lossless encoding. -use super::*; -use planner_types::ir::export::{ - compile_physical_asap_dag, compile_physical_asap_dag_with_node_ids, -}; -use planner_types::post_asap::FieldDataType as SummaryFamilyType; -use planner_types::pre_asap::DataType; -use std::rc::Rc; - -/// Not a legal PromQL label name, so it cannot shadow a user label. -pub use planner_types::pre_asap::schema::PROMQL_SERIES_IDENTITY as SERIES_IDENTITY_COLUMN; - -/// Canonical, reversible identity. JSON object encoding preserves label names, -/// empty values and escaping; sorting makes ingestion order irrelevant. -pub fn encode_series_identity(labels: &BTreeMap) -> Result { - serde_json::to_string(labels).map_err(|error| invalid(error.to_string())) -} - -pub fn decode_series_identity(encoded: &str) -> Result, Error> { - let labels: BTreeMap = - serde_json::from_str(encoded).map_err(|error| invalid(error.to_string()))?; - if encode_series_identity(&labels)? != encoded { - return Err(invalid("series identity is not canonically encoded")); - } - Ok(labels) -} - -/// Resolve the row representation before candidate search; see -/// [`planner_types::ir::schema_support::with_promql_series_identity`]. -pub fn with_series_identity(root: &Rc) -> Result, Error> { - planner_types::ir::schema_support::with_promql_series_identity(root).map_err(invalid) -} - -/// Construct source rows only from full identities. The named label columns -/// are projections of that same identity and cannot independently redefine it. -pub fn series_row( - schema: &SchemaRef, - labels: &BTreeMap, - timestamp: i64, - value: f64, -) -> Result, Error> { - use crate::values::Value; - let identity = encode_series_identity(labels)?; - let mut found = false; - let row = schema - .fields - .iter() - .enumerate() - .map(|(index, field)| { - if field.name == SERIES_IDENTITY_COLUMN { - if field.dtype != SummaryFamilyType::Plain(DataType::Utf8) - || field.nullable - || found - { - return Err(invalid("invalid series identity column")); - } - found = true; - Ok(Value::Utf8(identity.clone().into())) - } else if Some(index) == schema.time_index { - Ok(Value::Timestamp(timestamp)) - } else if field.name == "value" - && field.dtype == SummaryFamilyType::Plain(DataType::Float64) - { - Ok(Value::Float64(value)) - } else if field.dtype == SummaryFamilyType::Plain(DataType::Utf8) { - Ok(labels.get(&field.name).map_or_else( - || Value::Utf8("".into()), - |value| Value::Utf8(value.clone().into()), - )) - } else { - Err(invalid("unsupported PromQL source column")) - } - }) - .collect::, _>>()?; - if !found { - return Err(invalid("source lacks its full series identity")); - } - Ok(row) -} - -/// Compile the selected TopK computation above an existing maintained-population -/// source. The boundary supplies the complete eligible vector, not a truncated -/// TopK result; ranking remains a native physical operator. -pub fn compile_current_series_evaluation( - selected: &Rc, -) -> Result { - use planner_types::post_asap::{ - maintained_population::PopulationStatistic, Field as SummaryField, - }; - let selected = planner_types::ir::apply_lifecycle_timings( - selected, - &planner_types::ir::LifecycleAssignment::default_maintained(), - &mut planner_types::ir::TimingMemo::new(), - ) - .map_err(|e| invalid(e.to_string()))?; - let mut dag = - compile_physical_asap_dag(&selected).map_err(|error| invalid(error.to_string()))?; - // Typed snapshot candidates already carry full identity throughout the DAG. - // Cut at the population output, preserving all selected heap/evaluation nodes. - let populations = dag.nodes.iter().filter(|node| matches!(&node.payload, - Payload::MaintainPopulation { population } - if matches!(population.input, planner_types::post_asap::maintained_population::PopulationInput::CurrentSeries(_)) - )).collect::>(); - if let [population] = populations.as_slice() { - if population - .output_schema - .fields - .iter() - .any(|field| field.name == SERIES_IDENTITY_COLUMN) - { - return compile( - &dag, - BTreeMap::from([( - u64::from(population.id.0), - InputContract::bounded(Arc::new(population.output_schema.clone())), - )]), - &dag.roots.iter().map(|r| u64::from(r.0)).collect::>(), - ); - } - } - let mut frontier = None; - for node in &mut dag.nodes { - match &mut node.payload { - Payload::Relational { operator } => { - if let NonASAPOpKind::Scan { schema, .. } = operator { - schema.fields.push(SummaryField::new( - SERIES_IDENTITY_COLUMN, - SummaryFamilyType::Plain(DataType::Utf8), - false, - )); - schema.closed = true; - } - } - Payload::MaintainPopulation { .. } => { - frontier = Some(u64::from(node.id.0)); - } - Payload::EvaluatePopulation { - evaluation: PopulationStatistic::TopK { .. }, - } => {} - _ => return Err(invalid("unsupported current-series evaluation dependency")), - } - if node - .output_schema - .fields - .iter() - .any(|field| field.name == SERIES_IDENTITY_COLUMN) - { - return Err(invalid( - "current-series input already has a physical identity column", - )); - } - node.output_schema.fields.push(SummaryField { - name: SERIES_IDENTITY_COLUMN.into(), - dtype: SummaryFamilyType::Plain(DataType::Utf8), - nullable: false, - table: None, - }); - } - for edge in &mut dag.edges { - edge.intermediate_schema = dag - .nodes - .iter() - .find(|node| node.id == edge.producer) - .unwrap() - .output_schema - .clone(); - } - let frontier = frontier.ok_or_else(|| invalid("missing current-series population"))?; - let schema = Arc::new( - dag.nodes - .iter() - .find(|node| u64::from(node.id.0) == frontier) - .unwrap() - .output_schema - .clone(), - ); - compile( - &dag, - BTreeMap::from([(frontier, InputContract::bounded(schema))]), - &dag.roots.iter().map(|r| u64::from(r.0)).collect::>(), - ) -} - -/// Compile selected ranking or aggregation above an exact per-series Rate -/// evaluation. Deployments bind complete window evaluations at this boundary; -/// the heap is rebuilt independently for each evaluation. This does not move -/// that frontier to ingestion time or authorize combining finalized rates. -pub fn compile_rate_ranking( - selected: &Rc, -) -> Result<(Rc, CompiledPhysicalDAG), Error> { - use planner_types::post_asap::ExactKind; - fn frontier(node: &Rc) -> Option> { - if matches!(&node.operator, LogicalOperator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) - if matches!(&child.operator, LogicalOperator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate, _), - reduction: planner_types::pre_asap::Reduction::PerEntity, child: raw, .. - }) if matches!(raw.non_asap(), Some(NonASAPOp::TimeRange { .. })))) - { - return Some(Rc::clone(node)); - } - node.children().into_iter().find_map(frontier) - } - let selected = planner_types::ir::apply_lifecycle_timings( - selected, - &planner_types::ir::LifecycleAssignment::default_maintained(), - &mut planner_types::ir::TimingMemo::new(), - ) - .map_err(|e| invalid(e.to_string()))?; - let source = frontier(&selected) - .ok_or_else(|| invalid("ranking requires one exact per-series Rate frontier"))?; - if !source - .schema - .fields - .iter() - .any(|field| field.name == SERIES_IDENTITY_COLUMN) - { - return Err(invalid("Rate ranking requires complete series identity")); - } - let compiled = compile_physical_asap_dag_with_node_ids(&selected) - .map_err(|error| invalid(error.to_string()))?; - let id = u64::from( - compiled - .node_ids - .node_id(&source) - .ok_or_else(|| invalid("missing Rate frontier"))? - .0, - ); - let program = compile( - &compiled.dag, - BTreeMap::from([(id, InputContract::bounded(Arc::new(source.schema.clone())))]), - &compiled - .dag - .roots - .iter() - .map(|r| u64::from(r.0)) - .collect::>(), - )?; - Ok((source, program)) -} - -/// Compile a lifecycle-timed DAG whose heap or grouped Sum over per-series -/// Rate evaluations runs at ingestion time: fresh aggregate state per closed -/// window. The input is the complete collection of per-series counter states. -pub fn compile_fixed_window_rate_aggregation( - dag: &planner_types::ir::export::PhysicalASAPDAG, -) -> Result { - use planner_types::post_asap::{ExactKind, ExecutionTiming, SketchAlgorithm}; - let sources = dag - .nodes - .iter() - .filter(|n| { - matches!( - &n.payload, - Payload::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Rate, _), - reduction: planner_types::pre_asap::Reduction::PerEntity, - .. - } - ) - }) - .collect::>(); - let heaps = dag - .nodes - .iter() - .filter(|n| { - n.output_state.timing == ExecutionTiming::IngestionTime - && match &n.payload { - Payload::SummaryAgg { - family: SummaryFamilyType::Sketch(kind, _), - .. - } => matches!( - kind.algorithm(), - SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap - ), - Payload::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), - .. - } => true, - _ => false, - } - }) - .collect::>(); - let ([source], [heap]) = (sources.as_slice(), heaps.as_slice()) else { - return Err(invalid( - "expected one selected fixed-window Rate aggregation", - )); - }; - if !source - .output_schema - .fields - .iter() - .any(|f| f.name == SERIES_IDENTITY_COLUMN) - { - return Err(invalid( - "fixed-window Rate aggregation requires complete series identity", - )); - } - compile_candidate( - dag, - BTreeMap::from([( - u64::from(source.id.0), - InputContract::bounded(Arc::new(source.output_schema.clone())), - )]), - &dag.roots.iter().map(|r| u64::from(r.0)).collect::>(), - &[u64::from(heap.id.0)], - ) -} diff --git a/crates/asap-physical-operators/src/unified_physical_planner/promql_values.rs b/crates/asap-physical-operators/src/unified_physical_planner/promql_values.rs deleted file mode 100644 index 223d7f15c..000000000 --- a/crates/asap-physical-operators/src/unified_physical_planner/promql_values.rs +++ /dev/null @@ -1,281 +0,0 @@ -//! Physical scalar/vector contracts preserve complete label sets across native computation. -use super::*; -use planner_types::post_asap::FieldDataType as SummaryFamilyType; - -pub fn scalar_schema() -> SchemaRef { - crate::operators::vector_binary::value_schema(true) -} -pub fn vector_schema() -> SchemaRef { - crate::operators::vector_binary::value_schema(false) -} - -pub fn matrix_schema() -> SchemaRef { - crate::operators::vector_window::matrix_schema() -} - -pub fn compile_scalar(value: f64) -> Result { - let operator = Operator::scalar( - crate::values::Value::Float64(value), - planner_types::pre_asap::DataType::Float64, - )? - .with_output_schema(scalar_schema())?; - CompiledPhysicalDAG::from_operators( - BTreeMap::new(), - BTreeMap::from([(0, (vec![], operator))]), - vec![0], - ) -} - -pub fn compile_temporal( - intent: &AggIntent, - preserve_metric_name: bool, -) -> Result { - let operator = Operator::range_window(intent.clone())?; - let mut operators = vec![operator]; - if !preserve_metric_name { - operators.push(Operator::project( - vector_schema(), - vec![ - ( - "labels".into(), - Expression::LabelSet { - column: 0, - labels: vec![], - without: true, - }, - ), - ("value".into(), Expression::Column(1)), - ], - )?); - } - unary(operators, matrix_schema()) -} - -pub fn compile_histogram_quantile() -> Result { - CompiledPhysicalDAG::from_operators( - BTreeMap::from([ - (0, InputContract::bounded(scalar_schema())), - (1, InputContract::bounded(vector_schema())), - ]), - BTreeMap::from([(2, (vec![0, 1], Operator::histogram_quantile()))]), - vec![2], - ) -} - -/// Compile before deployment chooses readers. Input slots 0 and 1 retain operand order. -pub fn compile_binary( - operator: &crate::expressions::binary::BinaryOperator, - return_bool: bool, - left_scalar: bool, - right_scalar: bool, -) -> Result { - let left = crate::operators::vector_binary::value_schema(left_scalar); - let right = crate::operators::vector_binary::value_schema(right_scalar); - let op = Operator::vector_binary(left.clone(), right.clone(), operator.clone(), return_bool)?; - CompiledPhysicalDAG::from_operators( - BTreeMap::from([ - (0, InputContract::bounded(left)), - (1, InputContract::bounded(right)), - ]), - BTreeMap::from([(2, (vec![0, 1], op))]), - vec![2], - ) -} - -fn unary(operators: Vec, input: SchemaRef) -> Result { - let root = operators.len() as u64; - CompiledPhysicalDAG::from_operators( - BTreeMap::from([(0, InputContract::bounded(input))]), - operators - .into_iter() - .enumerate() - .map(|(i, op)| ((i + 1) as u64, (vec![i as u64], op))) - .collect(), - vec![root], - ) -} - -fn grouped(grouping: &GroupKeys) -> Result { - let labels = grouping - .keys() - .iter() - .map(|key| match key { - ColumnRef::Named(label) => Ok(label.clone()), - _ => Err(invalid("vector grouping requires label names")), - }) - .collect::, _>>()?; - Operator::project( - vector_schema(), - vec![ - ("labels".into(), Expression::Column(0)), - ("value".into(), Expression::Column(1)), - ( - "group".into(), - Expression::LabelSet { - column: 0, - labels, - without: grouping.is_without(), - }, - ), - ], - ) -} - -fn vector_output(input: SchemaRef, labels: usize, value: usize) -> Result { - let value = Expression::ExactFloat64(value); - Operator::project( - input, - vec![ - ("labels".into(), Expression::Column(labels)), - ("value".into(), value), - ], - ) -} - -pub fn compile_aggregate( - intent: &AggIntent, - grouping: &GroupKeys, -) -> Result { - let project = grouped(grouping)?; - let reduction = match intent { - AggIntent::Sum { .. } => Reduction::Sum(1), - AggIntent::Avg { .. } => Reduction::Avg(1), - AggIntent::Count { .. } => Reduction::Count, - AggIntent::Min { .. } => Reduction::Min(1), - AggIntent::Max { .. } => Reduction::Max(1), - _ => return Err(invalid("unsupported vector aggregate")), - }; - let aggregate = - Operator::aggregate(project.schema(), vec![2], vec![("value".into(), reduction)])?; - let output = vector_output(aggregate.schema(), 0, 1)?; - unary(vec![project, aggregate, output], vector_schema()) -} - -pub fn compile_sort( - descending: bool, - grouping: &GroupKeys, -) -> Result { - let project = grouped(grouping)?; - let sort = Operator::sort( - project.schema(), - vec![SortKey { - column: 1, - descending, - nulls_first: false, - }], - vec![2], - )?; - let output = vector_output(sort.schema(), 0, 1)?; - unary(vec![project, sort, output], vector_schema()) -} - -pub fn compile_limit( - n: u64, - offset: u64, - grouping: &GroupKeys, -) -> Result { - let project = grouped(grouping)?; - let limit = Operator::limit(project.schema(), n, offset, vec![2])?; - let output = vector_output(limit.schema(), 0, 1)?; - unary(vec![project, limit, output], vector_schema()) -} - -pub fn compile_negate(scalar: bool) -> Result { - let input = if scalar { - scalar_schema() - } else { - vector_schema() - }; - let mut columns = Vec::new(); - if !scalar { - columns.push(("labels".into(), Expression::Column(0))); - } - columns.push(( - if scalar { - "$promql_scalar".into() - } else { - "value".into() - }, - Expression::Negate(Box::new(Expression::Column(if scalar { 0 } else { 1 }))), - )); - unary(vec![Operator::project(input.clone(), columns)?], input) -} - -pub fn compile_vector_to_scalar() -> Result { - unary( - vec![Operator::vector_to_scalar(vector_schema(), 1)?.with_output_schema(scalar_schema())?], - vector_schema(), - ) -} - -/// A stored exact-state input retains the complete population identity. The -/// deployment supplies eligible panes; merging and finalization are computation. -pub fn exact_state_schema(family: SummaryFamilyType) -> Result { - if !matches!(family, SummaryFamilyType::ExactAggregate(..)) { - return Err(invalid("exact-state input requires an exact family")); - } - crate::values::validate_family(&family)?; - let mut schema = (*vector_schema()).clone(); - schema.fields[1].dtype = family; - Ok(Arc::new(schema)) -} - -/// Retain exact evaluation semantics before any deployment state is opened. -pub fn compile_exact_evaluation( - family: SummaryFamilyType, - lookback_ms: u64, - preserve_metric_name: bool, -) -> Result { - use planner_types::post_asap::ExactKind; - let statistic = match &family { - SummaryFamilyType::ExactAggregate(kind, _) => match kind { - ExactKind::Sum => crate::Statistic::Sum, - ExactKind::Count => crate::Statistic::Count, - ExactKind::Min => crate::Statistic::Min, - ExactKind::Max => crate::Statistic::Max, - ExactKind::Rate => crate::Statistic::Rate, - ExactKind::Increase => crate::Statistic::Increase, - ExactKind::IRate => { - return Err(invalid("instant-rate state evaluation is not supported")) - } - }, - _ => return Err(invalid("exact evaluation requires an exact family")), - }; - let input = exact_state_schema(family)?; - let merge = Operator::summary_merge(input.clone(), 1, vec![0])?; - let mut evaluation = Operator::readout( - merge.schema(), - 1, - ReadoutQuery::Exact(ExactReadout { - statistic, - lookback_ms: None, - }), - )?; - if matches!( - statistic, - crate::Statistic::Rate | crate::Statistic::Increase - ) { - evaluation = evaluation.with_counter_lookback( - i64::try_from(lookback_ms).map_err(|_| invalid("counter lookback exceeds Int64"))?, - )?; - } - let project = Operator::project( - evaluation.schema(), - vec![ - ( - "labels".into(), - if preserve_metric_name { - Expression::Column(0) - } else { - Expression::LabelSet { - column: 0, - labels: vec![], - without: true, - } - }, - ), - ("value".into(), Expression::ExactFloat64(1)), - ], - )?; - unary(vec![merge, evaluation, project], input) -} diff --git a/crates/asap-physical-operators/src/unified_physical_planner/row_values.rs b/crates/asap-physical-operators/src/unified_physical_planner/row_values.rs deleted file mode 100644 index 14f3d4813..000000000 --- a/crates/asap-physical-operators/src/unified_physical_planner/row_values.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! Query-time PromQL value computation over logical row schemas. -use super::*; -use planner_types::post_asap::maintained_population::PopulationStatistic; -use planner_types::pre_asap::DataType; - -/// Aggregate evaluations of a maintained current-series population, as a chain. -pub(super) fn population_aggregate( - input: &SchemaRef, - grouping: &[String], - evaluation: &PopulationStatistic, -) -> Result, Error> { - let groups = grouping - .iter() - .map(|name| named_column(input, &ColumnRef::Named(name.clone()))) - .collect::, _>>()?; - let value = named_column(input, &ColumnRef::SampleValue)?; - let reduction = match evaluation { - PopulationStatistic::Sum => Reduction::Sum(value), - PopulationStatistic::Count => Reduction::Count, - PopulationStatistic::Average => Reduction::Avg(value), - PopulationStatistic::Quantile { q } => Reduction::Quantile { - column: value, - q: *q, - }, - PopulationStatistic::TopK { .. } => { - return Err(invalid( - "TopK population evaluation ranks; it does not aggregate", - )) - } - }; - if !groups.is_empty() { - return Ok(vec![Operator::aggregate( - input.clone(), - groups, - vec![("value".into(), reduction)], - )?]); - } - // A global aggregate over no members is an empty PromQL vector, not one row. - let aggregate = Operator::aggregate( - input.clone(), - vec![], - vec![ - ("value".into(), reduction), - ("members".into(), Reduction::Count), - ], - )?; - let zero = Expression::Literal { - value: crate::values::Value::Int64(0), - dtype: DataType::Int64, - }; - let filter = Operator::filter( - aggregate.schema(), - Expression::Less(Box::new(zero), Box::new(Expression::Column(1))), - )?; - let project = Operator::project( - filter.schema(), - vec![("value".into(), Expression::Column(0))], - )?; - Ok(vec![aggregate, filter, project]) -} diff --git a/crates/asap-physical-operators/src/unified_sources/memory.rs b/crates/asap-physical-operators/src/unified_sources/memory.rs deleted file mode 100644 index 856c73cb0..000000000 --- a/crates/asap-physical-operators/src/unified_sources/memory.rs +++ /dev/null @@ -1,44 +0,0 @@ -use super::*; -/// Immutable in-memory raw data. The connector owns the resident input; each -/// cursor clones only the next requested batch, not the entire data set. -pub struct MemorySource { - schema: SchemaRef, - batches: Vec, -} -impl MemorySource { - pub fn new(schema: SchemaRef, batches: Vec) -> Result { - crate::values::validate_schema(&schema)?; - if schema - .fields - .iter() - .any(|f| !matches!(f.dtype, SummaryFamilyType::Plain(_))) - { - return Err(Error::Invalid( - "raw source cannot contain summary states".into(), - )); - } - if batches.iter().any(|batch| batch.schema() != &schema) { - return Err(Error::Invalid("memory source batch schema mismatch".into())); - } - Ok(Self { schema, batches }) - } -} -impl RawSource for MemorySource { - fn boundedness(&self) -> crate::plan::Boundedness { - crate::plan::Boundedness::Bounded - } - fn schema(&self) -> SchemaRef { - self.schema.clone() - } - fn scan(&self, context: RunContext) -> Result, Error> { - Ok(stream::iter(self.batches.iter()) - .map(move |batch| { - if context.is_cancelled() { - return Err(Error::Cancelled); - } - let _allocation = context.reserve(batch.bytes())?; - Ok(batch.clone()) - }) - .boxed_local()) - } -} diff --git a/crates/asap-physical-operators/src/unified_sources/mod.rs b/crates/asap-physical-operators/src/unified_sources/mod.rs deleted file mode 100644 index 9dbe62c15..000000000 --- a/crates/asap-physical-operators/src/unified_sources/mod.rs +++ /dev/null @@ -1,177 +0,0 @@ -//! Raw data access. Connectors provide rows; Scan owns Planner predicate semantics. -use crate::{ - expressions::unified_planner::CompiledExpression, - plan::PhysicalOperator, - runtime::{Input, OutputStream, RunContext}, - values::{Batch, SchemaRef, Value}, - Error, -}; -use futures::{stream, StreamExt}; -use planner_types::ir::{NonASAPOp, OperatorNode}; -use planner_types::{ - post_asap::FieldDataType as SummaryFamilyType, - pre_asap::{DataType, Source}, -}; -use std::sync::Arc; - -/// A bound data source. Metadata must be stable for the lifetime of the binding. -/// Each scan opens an independent cursor. Connectors return raw, unfiltered rows -/// and must honor cancellation and bound their own I/O buffers. Dropping a cursor -/// must release its resources. A connector error is never an empty successful scan. -pub trait RawSource { - fn schema(&self) -> SchemaRef; - /// Declare a finite snapshot/window explicitly; execution scope alone does not bound a cursor. - fn boundedness(&self) -> crate::plan::Boundedness { - crate::plan::Boundedness::Unknown - } - fn scan(&self, context: RunContext) -> Result, Error>; -} - -/// Explicit source identities; no implicit network discovery or fallback. -#[derive(Default)] -pub struct DataSources { - sources: Vec<(Source, Arc)>, -} -impl DataSources { - pub fn register(&mut self, identity: Source, source: Arc) -> Result<(), Error> { - if self.sources.iter().any(|(key, _)| key == &identity) { - return Err(Error::Invalid("duplicate data source".into())); - } - crate::values::validate_schema(&source.schema())?; - self.sources.push((identity, source)); - Ok(()) - } - pub fn bind(&self, expression: &OperatorNode) -> Result { - let Some(NonASAPOp::Scan { - source, - predicates, - schema, - }) = expression.non_asap() - else { - return Err(Error::Invalid( - "raw Scan requires a Planner Scan leaf".into(), - )); - }; - let output = Arc::new(schema.clone()); - crate::values::validate_schema(&output)?; - let reader = self - .sources - .iter() - .find(|(key, _)| key == source) - .map(|(_, reader)| reader.clone()) - .ok_or_else(|| Error::Invalid(format!("unbound raw source: {source:?}")))?; - if reader.schema() != output { - return Err(Error::Invalid( - "raw source differs from Planner Scan schema".into(), - )); - } - let predicates = predicates - .iter() - .map(|predicate| { - let predicate = CompiledExpression::compile(&predicate.0, &output)?; - if predicate.dtype().0 != DataType::Bool { - return Err(Error::Invalid("Scan predicate must be boolean".into())); - } - Ok(predicate) - }) - .collect::, Error>>()?; - Ok(Scan { - reader, - output, - predicates, - }) - } -} - -pub struct Scan { - reader: Arc, - output: SchemaRef, - predicates: Vec, -} -impl PhysicalOperator for Scan { - fn properties(&self, _: &[crate::plan::PlanProperties]) -> crate::plan::PlanProperties { - crate::plan::PlanProperties { - boundedness: self.reader.boundedness(), - emission: crate::plan::Emission::Incremental, - } - } - - fn name(&self) -> &str { - "Scan" - } - fn input_schemas(&self) -> Vec { - vec![] - } - fn output_schema(&self) -> SchemaRef { - self.output.clone() - } - fn output_bytes(&self, batch: &Batch) -> usize { - batch.bytes() - } - fn start<'a>( - &'a self, - inputs: Vec>, - context: RunContext, - ) -> Result, Error> { - if !inputs.is_empty() { - return Err(Error::Invalid("Scan cannot have inputs".into())); - } - if context.is_cancelled() { - return Err(Error::Cancelled); - } - // Opening is lazy: validation and construction of a run perform no I/O. - let opening = context.clone(); - let stream = stream::once(async move { - if opening.is_cancelled() { - return Err(Error::Cancelled); - } - self.reader.scan(opening) - }); - use futures::TryStreamExt; - Ok(stream - .try_flatten() - .map(move |batch| { - if context.is_cancelled() { - return Err(Error::Cancelled); - } - let batch = batch?; - if batch.schema() != &self.output { - return Err(Error::Invalid( - "connector returned a different Scan schema".into(), - )); - } - if self.predicates.is_empty() { - return Ok(batch); - } - let _workspace = - context.reserve(batch.bytes().checked_mul(2).ok_or(Error::MemoryLimit)?)?; - let mut rows = Vec::new(); - for row in batch.rows() { - if context.is_cancelled() { - return Err(Error::Cancelled); - } - let mut keep = true; - for predicate in &self.predicates { - match predicate.evaluate(row)? { - Value::Bool(true) => {} - Value::Bool(false) | Value::Null => { - keep = false; - break; - } - _ => { - return Err(Error::Invalid("Scan predicate is not boolean".into())) - } - } - } - if keep { - rows.push(row.clone()); - } - } - Batch::try_new(self.output.clone(), rows) - }) - .boxed_local()) - } -} - -mod memory; -pub use memory::MemorySource; diff --git a/crates/asap-physical-operators/tests/blocking_resources.rs b/crates/asap-physical-operators/tests/blocking_resources.rs index 66702afdb..09cee7a4e 100644 --- a/crates/asap-physical-operators/tests/blocking_resources.rs +++ b/crates/asap-physical-operators/tests/blocking_resources.rs @@ -8,7 +8,7 @@ use asap_physical_operators::{ }; use futures::{executor::block_on, FutureExt, StreamExt}; use planner_types::ir::Predicate; -use planner_types::ir::ScalarExpr as QueryExpr; +use planner_types::ir::ScalarExpr; use planner_types::{ post_asap::{Field, FieldDataType}, pre_asap::{DataType, JoinKind, ScalarValue}, @@ -62,7 +62,7 @@ fn cross_join() -> Operator { schema(1), schema(1), JoinKind::Cross, - &Predicate(QueryExpr::Literal(ScalarValue::Boolean(true))), + &Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), schema(2), ) .unwrap() diff --git a/crates/asap-physical-operators/tests/common/mod.rs b/crates/asap-physical-operators/tests/common/mod.rs index 35ba496b4..b3602adbe 100644 --- a/crates/asap-physical-operators/tests/common/mod.rs +++ b/crates/asap-physical-operators/tests/common/mod.rs @@ -1,14 +1,16 @@ #![allow(dead_code)] use planner_types::ir::export::PhysicalASAPDAG; -use planner_types::ir::{apply_lifecycle_timings, LifecycleAssignment, OperatorNode, TimingMemo}; +use planner_types::ir::{ + apply_materialization_timings, MaterializationAssignment, OperatorNode, TimingMemo, +}; use std::rc::Rc; pub fn compile_physical_asap_dag( root: &Rc, ) -> Result> { - let root = apply_lifecycle_timings( + let root = apply_materialization_timings( root, - &LifecycleAssignment::default(), + &MaterializationAssignment::default(), &mut TimingMemo::default(), )?; Ok(planner_types::ir::export::compile_physical_asap_dag(&root)?) diff --git a/crates/asap-physical-operators/tests/physical_dag.rs b/crates/asap-physical-operators/tests/physical_dag.rs index 450b95652..93a8b7a86 100644 --- a/crates/asap-physical-operators/tests/physical_dag.rs +++ b/crates/asap-physical-operators/tests/physical_dag.rs @@ -14,7 +14,7 @@ use planner_types::ir::export::{ PhysicalASAPOperatorPayload, WindowEdgeCompatibility, }; use planner_types::ir::BinaryOperator; -use planner_types::ir::ScalarExpr as QueryExpr; +use planner_types::ir::ScalarExpr; use planner_types::{ post_asap::{ExactKind, ExactParams, Field, FieldDataType}, pre_asap::DataType, @@ -908,11 +908,11 @@ fn planner_expressions_preserve_collection_and_nullable_types() { }, false, )]); - let access = QueryExpr::FunctionCall { + let access = ScalarExpr::FunctionCall { name: "asap_element_access".into(), args: vec![ - QueryExpr::Column(0), - QueryExpr::Literal(ScalarValue::Utf8("count".into())), + ScalarExpr::Column(0), + ScalarExpr::Literal(ScalarValue::Utf8("count".into())), ], }; let project = Operator::project( @@ -945,11 +945,11 @@ fn planner_expressions_preserve_collection_and_nullable_types() { .unwrap(); let projected = project.schema(); dag.add(1, vec![0], project).unwrap(); - let predicate = QueryExpr::Compare { + let predicate = ScalarExpr::Compare { semantics: planner_types::ir::ExprSemantics::Sql, - left: Box::new(QueryExpr::Column(0)), + left: Box::new(ScalarExpr::Column(0)), op: CompareOpKind::Ge, - right: Box::new(QueryExpr::Literal(ScalarValue::Int64(1))), + right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(1))), }; dag.add( 2, @@ -963,9 +963,9 @@ fn planner_expressions_preserve_collection_and_nullable_types() { .unwrap(); let rows = run(&dag, 2, query()); assert!(matches!(rows.as_slice(),[row] if matches!(row.as_slice(),[Value::Int64(7)]))); - let unknown = QueryExpr::FunctionCall { + let unknown = ScalarExpr::FunctionCall { name: "unregistered_function".into(), - args: vec![QueryExpr::Column(0)], + args: vec![ScalarExpr::Column(0)], }; assert!(CompiledExpression::compile(&unknown, &input_schema).is_err()); } @@ -977,11 +977,11 @@ fn native_relational_join_kinds_preserve_unmatched_rows() { use planner_types::pre_asap::{CompareOpKind, JoinKind}; let input = schema(&[("key", DataType::Int64, true)]); - let predicate = Predicate(QueryExpr::Compare { + let predicate = Predicate(ScalarExpr::Compare { semantics: planner_types::ir::ExprSemantics::Sql, - left: Box::new(QueryExpr::Column(0)), + left: Box::new(ScalarExpr::Column(0)), op: CompareOpKind::Eq, - right: Box::new(QueryExpr::Column(1)), + right: Box::new(ScalarExpr::Column(1)), }); for (kind, count) in [ (JoinKind::Inner, 1), diff --git a/crates/asap-physical-operators/tests/physical_semantics.rs b/crates/asap-physical-operators/tests/physical_semantics.rs index b35d10a9f..498e8b3f0 100644 --- a/crates/asap-physical-operators/tests/physical_semantics.rs +++ b/crates/asap-physical-operators/tests/physical_semantics.rs @@ -12,7 +12,7 @@ use futures::{executor::block_on, StreamExt}; use planner_types::ir::export::NonASAPOpKind as ValueOperation; use planner_types::ir::export::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; use planner_types::ir::Predicate; -use planner_types::ir::ScalarExpr as QueryExpr; +use planner_types::ir::ScalarExpr; use planner_types::{ post_asap::{Field, FieldDataType}, pre_asap::{CompareOpKind, DataType, JoinKind}, @@ -78,11 +78,11 @@ fn keys(rows: &[Vec]) -> Vec>> { .collect() } fn eq_predicate() -> Predicate { - Predicate(QueryExpr::Compare { + Predicate(ScalarExpr::Compare { semantics: planner_types::ir::ExprSemantics::Sql, - left: Box::new(QueryExpr::Column(0)), + left: Box::new(ScalarExpr::Column(0)), op: CompareOpKind::Eq, - right: Box::new(QueryExpr::Column(1)), + right: Box::new(ScalarExpr::Column(1)), }) } fn join(left: Vec, right: Vec, kind: JoinKind, keyed: bool) -> Vec> { @@ -340,7 +340,7 @@ fn aggregate_empty_and_all_null_follow_asap_contract() { fn projection_rejects_expression_bound_to_another_schema() { let original = schema(&[("a", DataType::Int64, false), ("b", DataType::Int64, false)]); let current = schema(&[("a", DataType::Int64, false)]); - let expr = CompiledExpression::compile(&QueryExpr::Column(1), &original).unwrap(); + let expr = CompiledExpression::compile(&ScalarExpr::Column(1), &original).unwrap(); assert!(Operator::project(current, vec![("b".into(), Expression::planner(expr))]).is_err()); } @@ -417,11 +417,11 @@ fn planner_comparisons_handle_nan_without_execution_errors() { CompareOpKind::Gt, CompareOpKind::Ge, ] { - let expression = QueryExpr::Compare { + let expression = ScalarExpr::Compare { semantics: planner_types::ir::ExprSemantics::Sql, - left: Box::new(QueryExpr::Column(0)), + left: Box::new(ScalarExpr::Column(0)), op: op.clone(), - right: Box::new(QueryExpr::Column(1)), + right: Box::new(ScalarExpr::Column(1)), }; let compiled = CompiledExpression::compile(&expression, &input).unwrap(); for row in [ @@ -485,11 +485,11 @@ fn mixed_numeric_comparisons_preserve_large_integer_precision() { ("a", DataType::Int64, false), ("b", DataType::Float64, false), ]); - let expr = QueryExpr::Compare { + let expr = ScalarExpr::Compare { semantics: planner_types::ir::ExprSemantics::Sql, - left: Box::new(QueryExpr::Column(0)), + left: Box::new(ScalarExpr::Column(0)), op: CompareOpKind::Gt, - right: Box::new(QueryExpr::Column(1)), + right: Box::new(ScalarExpr::Column(1)), }; let compiled = CompiledExpression::compile(&expr, &input).unwrap(); for (a, b, expected) in [ @@ -510,11 +510,11 @@ fn boolean_truth_tables_agree_between_expression_paths() { for and in [true, false] { for a in [None, Some(false), Some(true)] { for b in [None, Some(false), Some(true)] { - let parts = vec![QueryExpr::Column(0), QueryExpr::Column(1)]; + let parts = vec![ScalarExpr::Column(0), ScalarExpr::Column(1)]; let planner = if and { - QueryExpr::BoolAnd(parts) + ScalarExpr::BoolAnd(parts) } else { - QueryExpr::BoolOr(parts) + ScalarExpr::BoolOr(parts) }; let native = if and { Expression::And( diff --git a/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs b/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs index 228332cbc..e33de530f 100644 --- a/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs +++ b/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs @@ -4,7 +4,7 @@ //! ranking, or workload Cartesian expansion. Placement variants are not listed. mod common; use common::compile_physical_asap_dag; -use planner_types::ir::OperatorNode as QueryExpr; +use planner_types::ir::OperatorNode; use asap_aware_mapping::{ accuracy::{AccuracyEvidenceProvider, DefaultAccuracyModel, PropagationStats}, @@ -28,7 +28,7 @@ use std::rc::Rc; struct Evidence; impl AccuracyEvidenceProvider for Evidence { - fn topk_max_distinct_items(&self, _: &QueryExpr) -> Option { + fn topk_max_distinct_items(&self, _: &OperatorNode) -> Option { Some(1000) } fn propagation_stats( @@ -67,7 +67,7 @@ impl ReplacementStrategy for LogicalOnly { } } -fn lower(query: &str, accuracy: &AccuracyTarget) -> Rc { +fn lower(query: &str, accuracy: &AccuracyTarget) -> Rc { let workload = PlanningWorkload { query_workload: QueryWorkload { language: QueryLanguage::PromQL, diff --git a/crates/asap-physical-operators/tests/weighted_topk_binding.rs b/crates/asap-physical-operators/tests/weighted_topk_binding.rs index b4d8c85d3..c14d16979 100644 --- a/crates/asap-physical-operators/tests/weighted_topk_binding.rs +++ b/crates/asap-physical-operators/tests/weighted_topk_binding.rs @@ -760,102 +760,56 @@ fn spatial_topk_exposes_signed_heap_candidate_over_complete_snapshot() { } } -/// Deployment-side lifecycle choice: every summary state of `candidate` is -/// continuously maintained, and the chosen lifecycles set execution timing. +/// Every summary state of `candidate` maintained at ingestion time, the +/// materialization a deployment would assign for a continuously served query: +/// each `SummaryAgg` and every input it consumes run at ingestion time, the +/// rest at query time. The phases are assigned on the exported DAG because +/// the candidate pins its finalize boundary to query time. fn continuously_maintained_dag(candidate: &Rc) -> PhysicalASAPDAG { - use asap_aware_mapping::{ - cost_model::{Cost, CostModel}, - enumerate_summary_maintenance_lifecycles, CostRate, Horizon, - SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCapabilities, - SummaryMaintenanceLifecycleCostInputs, WorkloadDemand, - }; - use planner_types::workload::{ - DataArrival, Rate, RepeatedDemand, RepeatingEntry, RepetitionInterval, - }; - struct Costed; - impl CostModel for Costed { - fn rank_candidates( - &self, - _: &planner_types::pre_asap::agg_intent::AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - fn summary_maintenance_lifecycle_cost_inputs( - &self, - _: &planner_types::ir::OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(10.)), - maintenance_cost_per_update: Some(Cost(1.)), - summary_read_cost: Some(Cost(1.)), - retention_cost_rate: Some(CostRate(0.1)), - retirement_cost: Some(Cost(1.)), - } - } - fn summary_maintenance_capabilities( - &self, - _: &planner_types::ir::OperatorNode, - ) -> SummaryMaintenanceCapabilities { - SummaryMaintenanceCapabilities { - incremental_update: true, - merge: true, - delete: true, - } - } - } - const NOW_MS: u64 = 1_000_000; - let queries = QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query("topk by(job)(2, rate(m[1m]))".into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(60_000)), - requirements: QueryRequirements::default(), - predictability: Predictability::Predictable { known_at: None }, - time_selection: TimeSelection::default(), - }]), - }; - let data = DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - ingestion_rate: WorkloadEvidence { - value: Some(Rate(1.)), - source: planner_types::workload::EvidenceSource::Observed, - observed_at_ms: Some(NOW_MS), - valid_for_ms: Some(60_000), - }, - ..Default::default() - }; - let lifecycles = enumerate_summary_maintenance_lifecycles( - Rc::clone(candidate), - WorkloadDemand::new_with_data(&queries, &data, &[0]), - NOW_MS, - Some(Horizon(100.)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &Costed, + use planner_types::ir::{apply_materialization_timings, MaterializationAssignment, TimingMemo}; + let timed = apply_materialization_timings( + candidate, + &MaterializationAssignment::all_query_time(), + &mut TimingMemo::new(), ) .unwrap(); - let choices = lifecycles - .deployments() + let dag = planner_types::ir::export::compile_physical_asap_dag(&timed).unwrap(); + let mut pending: Vec<_> = dag + .nodes .iter() - .map(|deployment| { - ( - deployment.post_asap_node_id, - SummaryMaintenanceLifecycle::ContinuouslyMaintained, - ) + .filter(|node| matches!(node.payload, PhysicalASAPOperatorPayload::SummaryAgg { .. })) + .map(|node| node.id) + .collect(); + let mut ingestion = std::collections::HashSet::new(); + while let Some(id) = pending.pop() { + if ingestion.insert(id) { + pending.extend( + dag.edges + .iter() + .filter(|edge| edge.consumer == id) + .map(|edge| edge.producer), + ); + } + } + let phases = dag + .nodes + .iter() + .map(|node| { + let timing = if ingestion.contains(&node.id) { + ExecutionTiming::IngestionTime + } else { + ExecutionTiming::QueryTime + }; + (node.id, timing) }) - .collect::>(); - lifecycles - .select(&choices) - .unwrap() - .execution_timed_dag() - .unwrap() + .collect(); + dag.with_execution_phases(&phases).unwrap() } // A maintained heap over finalized per-series Rate is the fixed-window -// placement: lifecycle timing, not a separate candidate, puts it in precompute. +// placement: materialization timing, not a separate candidate, puts it in precompute. #[test] -fn maintained_rate_heap_lifecycle_compiles_fixed_window_precompute() { +fn maintained_rate_heap_compiles_fixed_window_precompute() { use asap_physical_operators::physical_planner::{ compile_candidate, promql_rows::with_series_identity, }; diff --git a/crates/devtools/src/bin/show_post_asap_ir.rs b/crates/devtools/src/bin/show_post_asap_ir.rs index df6c2bfd5..4565657f9 100644 --- a/crates/devtools/src/bin/show_post_asap_ir.rs +++ b/crates/devtools/src/bin/show_post_asap_ir.rs @@ -176,9 +176,9 @@ mod tests { candidates[0].guarantee.is_none(), "missing evidence must not claim a certified ratio bound" ); - let timed = asap_types::ir::timing::apply_lifecycle_timings( + let timed = asap_types::ir::timing::apply_materialization_timings( &candidates[0], - &asap_types::ir::timing::LifecycleAssignment::default_maintained(), + &asap_types::ir::timing::MaterializationAssignment::all_query_time(), &mut asap_types::ir::timing::TimingMemo::new(), ) .expect("the demo candidate has a legal default timing"); diff --git a/crates/devtools/tests/viewer_contract.rs b/crates/devtools/tests/viewer_contract.rs new file mode 100644 index 000000000..b34b472fe --- /dev/null +++ b/crates/devtools/tests/viewer_contract.rs @@ -0,0 +1,128 @@ +//! `tools/dag-viewer` ↔ `asap_types::dag_export` contract: the viewer's +//! `KIND_CATEGORY_JSON` must categorize exactly the `kind` strings +//! [`asap_types::dag_export::export`] can emit — `Operator::kind_name()` of +//! every `NonASAPOp` and `ASAPOp` variant — no more (a stale kind the IR no +//! longer has) and no less (an exported kind the viewer would render +//! uncategorized). + +use std::collections::{BTreeMap, BTreeSet}; + +use asap_types::ir::{ASAPOp, NonASAPOp}; + +/// Every `NonASAPOp::kind_name()`. +const NON_ASAP_KINDS: &[&str] = &[ + "Scan", + "Values", + "Filter", + "Project", + "Aggregate", + "Join", + "SetOp", + "Concat", + "Dedup", + "Sort", + "Limit", + "BinaryOp", + "SQLWindowFunc", + "TimeRange", + "TimeShift", + "PromqlVectorFromScalar", + "PromqlRelabel", + "PromqlInfoEnrich", + "PromqlSeriesSample", + "PromqlSubquery", +]; + +/// Every `ASAPOp::kind_name()`. +const ASAP_KINDS: &[&str] = &[ + "SummaryAgg", + "SummaryEstimate", + "FinalizeExactAccumulator", + "MaintainPopulation", + "EvaluatePopulation", + "SummaryMerge", + "SummarySubtract", + "SummaryDelete", + "SummaryJoin", + "Extension", +]; + +/// Compile-time tripwire: adding an operator variant fails these exhaustive +/// matches until the matching `*_KINDS` list above is extended too. Never +/// called; the match arms are the point. +#[allow(dead_code)] +fn kind_lists_track_every_variant(non_asap: &NonASAPOp, asap: &ASAPOp) { + let listed = |name: &str, list: &[&str]| assert!(list.contains(&name)); + listed( + match non_asap { + NonASAPOp::Scan { .. } => "Scan", + NonASAPOp::Values { .. } => "Values", + NonASAPOp::Filter { .. } => "Filter", + NonASAPOp::Project { .. } => "Project", + NonASAPOp::Aggregate { .. } => "Aggregate", + NonASAPOp::Join { .. } => "Join", + NonASAPOp::SetOp { .. } => "SetOp", + NonASAPOp::Concat { .. } => "Concat", + NonASAPOp::Dedup { .. } => "Dedup", + NonASAPOp::Sort { .. } => "Sort", + NonASAPOp::Limit { .. } => "Limit", + NonASAPOp::BinaryOp { .. } => "BinaryOp", + NonASAPOp::SQLWindowFunc { .. } => "SQLWindowFunc", + NonASAPOp::TimeRange { .. } => "TimeRange", + NonASAPOp::TimeShift { .. } => "TimeShift", + NonASAPOp::PromqlVectorFromScalar(_) => "PromqlVectorFromScalar", + NonASAPOp::PromqlRelabel { .. } => "PromqlRelabel", + NonASAPOp::PromqlInfoEnrich { .. } => "PromqlInfoEnrich", + NonASAPOp::PromqlSeriesSample { .. } => "PromqlSeriesSample", + NonASAPOp::PromqlSubquery { .. } => "PromqlSubquery", + }, + NON_ASAP_KINDS, + ); + listed( + match asap { + ASAPOp::SummaryAgg { .. } => "SummaryAgg", + ASAPOp::SummaryEstimate { .. } => "SummaryEstimate", + ASAPOp::FinalizeExactAccumulator { .. } => "FinalizeExactAccumulator", + ASAPOp::MaintainPopulation { .. } => "MaintainPopulation", + ASAPOp::EvaluatePopulation { .. } => "EvaluatePopulation", + ASAPOp::SummaryMerge { .. } => "SummaryMerge", + ASAPOp::SummarySubtract { .. } => "SummarySubtract", + ASAPOp::SummaryDelete { .. } => "SummaryDelete", + ASAPOp::SummaryJoin { .. } => "SummaryJoin", + ASAPOp::Extension { .. } => "Extension", + }, + ASAP_KINDS, + ); +} + +/// The viewer's `kind -> category` table, parsed out of the JS source the +/// same way the viewer itself does (`JSON.parse(KIND_CATEGORY_JSON)`). +fn viewer_kind_categories() -> BTreeMap { + const START: &str = "const KIND_CATEGORY_JSON = `"; + let source = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../tools/dag-viewer/node-style.js" + )); + let json = source + .split_once(START) + .expect("node-style.js must declare KIND_CATEGORY_JSON") + .1 + .split_once("`;") + .expect("KIND_CATEGORY_JSON must be a template literal") + .0; + serde_json::from_str(json).expect("KIND_CATEGORY_JSON must be valid JSON") +} + +#[test] +fn viewer_categorizes_exactly_the_exported_node_kinds() { + let expected: BTreeSet<&str> = NON_ASAP_KINDS.iter().chain(ASAP_KINDS).copied().collect(); + assert_eq!( + expected.len(), + NON_ASAP_KINDS.len() + ASAP_KINDS.len(), + "exported kind names must be unique" + ); + let categories = viewer_kind_categories(); + let actual: BTreeSet<&str> = categories.keys().map(String::as_str).collect(); + + assert_eq!(actual, expected); +} diff --git a/crates/frontend-metricsql/src/unified/mod.rs b/crates/frontend-metricsql/src/unified/mod.rs deleted file mode 100644 index 3544a9801..000000000 --- a/crates/frontend-metricsql/src/unified/mod.rs +++ /dev/null @@ -1,389 +0,0 @@ -//! MetricsQL AST → the name-based `UnresolvedOp` tree → the unified operator DAG. - -use std::{rc::Rc, time::Duration}; - -use asap_frontend_common::{ - resolve_root, UnresolvedOp as U, UnresolvedPredicate, UnresolvedScalar, -}; -use asap_types::ir::{BinaryOperator, ExprSemantics, OperatorNode, TimeRangeKind}; -use asap_types::pre_asap::{ - AggIntent, ArithmeticOpKind, BinaryOpKind, ColumnRef, CompareOpKind, GroupKeys, - PromQLVectorSetOpKind, Reduction, ScalarValue, Source, -}; -use asap_types::types::AccuracyTarget; -use metricsql_parser::ast::{AggregateModifier, DurationExpr, Expr, MetricExpr, RollupExpr}; -use metricsql_parser::functions::{AggregateFunction, BuiltinFunction, RollupFunction}; -use metricsql_parser::label::{LabelFilter, LabelFilterOp, NAME_LABEL}; -use thiserror::Error; - -pub use metricsql_parser::ast::Expr as MetricsqlExpr; - -#[derive(Debug, Error)] -pub enum MetricsqlError { - #[error("MetricsQL parse error: {0}")] - Parse(String), - #[error("unsupported MetricsQL feature: {0}")] - UnsupportedFeature(String), - #[error("MetricsQL column resolution failed: {0}")] - Resolve(String), -} - -pub fn parse_metricsql(query: &str) -> Result { - metricsql_parser::parser::parse(query).map_err(|e| MetricsqlError::Parse(e.to_string())) -} - -pub fn canonical_metricsql(query: &str) -> Result { - Ok(parse_metricsql(query)?.to_string()) -} - -pub fn lower_metricsql( - query: &str, - accuracy: AccuracyTarget, -) -> Result, MetricsqlError> { - match lower_metricsql_query(query, accuracy)? { - asap_types::ir::QueryRoot::Operator(node) => Ok(node), - _ => Err(unsupported("scalar root: use lower_metricsql_query")), - } -} - -/// Lower scalar constants without fabricating a relational operator. -pub fn lower_metricsql_query( - query: &str, - accuracy: AccuracyTarget, -) -> Result { - let ast = parse_metricsql(query)?; - if let Expr::NumberLiteral(number) = &ast { - return Ok(asap_types::ir::QueryRoot::Scalar( - asap_types::ir::ScalarExpr::literal_f64(number.value), - )); - } - let unresolved = Lowerer { accuracy }.lower(&ast)?; - resolve_root(&unresolved) - .map(asap_types::ir::QueryRoot::Operator) - .map_err(|e| MetricsqlError::Resolve(e.to_string())) -} - -struct Lowerer { - accuracy: AccuracyTarget, -} - -impl Lowerer { - fn lower(&self, expr: &Expr) -> Result { - match expr { - Expr::MetricExpression(e) => self.metric(e), - Expr::Rollup(e) => self.rollup(e), - Expr::Function(e) => self.function(e), - Expr::Aggregation(e) => self.aggregate(e), - Expr::NumberLiteral(_) => { - Err(unsupported("scalar root requires lower_metricsql_query")) - } - // Vector negation is `x * -1` (as in the PromQL front end). - Expr::UnaryOperator(e) => Ok(U::PromqlScalarOp { - child: Rc::new(self.lower(&e.expr)?), - scalar: UnresolvedScalar::Literal(ScalarValue::Float64(-1.0)), - op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), - scalar_left: false, - return_bool: false, - }), - Expr::BinaryOperator(e) => self.binary(e), - Expr::Parens(e) if e.expressions.len() == 1 => self.lower(&e.expressions[0]), - Expr::With(e) => self.lower(&e.expr), - other => Err(unsupported(format!("AST node `{other}`"))), - } - } - - fn metric(&self, metric: &MetricExpr) -> Result { - if metric.has_or_matchers() { - return Err(unsupported("or-delimited selector matchers")); - } - let name = metric - .metric_name() - .ok_or_else(|| unsupported("selector without one exact metric name"))?; - let mut filters: Vec<_> = metric - .matchers - .filter_iter() - .filter(|f| f.label != NAME_LABEL) - .collect(); - filters.sort_by(|a, b| a.label.cmp(&b.label).then(a.value.cmp(&b.value))); - Ok(U::Scan { - source: Source::TimeSeries { - metric: name.to_owned(), - }, - predicates: filters - .into_iter() - .map(|f| UnresolvedPredicate(matcher(f))) - .collect(), - schema: None, - }) - } - - fn rollup(&self, rollup: &RollupExpr) -> Result { - if rollup.offset.is_some() || rollup.at.is_some() { - return Err(unsupported("offset and @ modifiers")); - } - if rollup.for_subquery() { - return Err(unsupported("subquery step or inherited step")); - } - let child = self.lower(&rollup.expr)?; - match &rollup.window { - None => Ok(child), - Some(window) => Ok(U::TimeRange { - range: duration(window)?, - kind: TimeRangeKind::Range, - child: Rc::new(child), - }), - } - } - - fn function( - &self, - function: &metricsql_parser::ast::FunctionExpr, - ) -> Result { - if function.keep_metric_names { - return Err(unsupported( - "keep_metric_names requires metric-name lineage", - )); - } - let BuiltinFunction::Rollup(rollup) = function.function else { - return Err(unsupported(format!("function `{}`", function.name()))); - }; - let expected_args = if rollup == RollupFunction::QuantileOverTime { - 2 - } else { - 1 - }; - require_arity(function.name(), function.args.len(), expected_args)?; - let child_index = usize::from(rollup == RollupFunction::QuantileOverTime); - let child = function - .args - .get(child_index) - .ok_or_else(|| unsupported(format!("missing argument for `{}`", function.name())))?; - let intent = match rollup { - RollupFunction::DefaultRollup | RollupFunction::LastOverTime => AggIntent::LastOverTime, - RollupFunction::FirstOverTime => AggIntent::FirstOverTime, - RollupFunction::AvgOverTime => AggIntent::Avg { col: None }, - RollupFunction::MinOverTime => AggIntent::Min { col: None }, - RollupFunction::MaxOverTime => AggIntent::Max { col: None }, - RollupFunction::SumOverTime => AggIntent::Sum { col: None }, - RollupFunction::CountOverTime => AggIntent::Count { - accuracy: self.accuracy.clone(), - }, - RollupFunction::StddevOverTime => AggIntent::StdDev { - col: None, - population: true, - }, - RollupFunction::StdvarOverTime => AggIntent::Variance { - col: None, - population: true, - }, - RollupFunction::Rate => AggIntent::Rate, - RollupFunction::IRate => AggIntent::IRate, - RollupFunction::Increase => AggIntent::Increase, - RollupFunction::Changes => AggIntent::Changes, - RollupFunction::Delta => AggIntent::Delta, - RollupFunction::IDelta => AggIntent::IDelta, - RollupFunction::Deriv => AggIntent::Deriv, - RollupFunction::Resets => AggIntent::Resets, - RollupFunction::MadOverTime => AggIntent::MadOverTime, - RollupFunction::PresentOverTime => AggIntent::PresentOverTime, - RollupFunction::AbsentOverTime => AggIntent::AbsentOverTime, - RollupFunction::QuantileOverTime => AggIntent::Quantile { - col: None, - q: number_arg(&function.args, 0)?, - accuracy: self.accuracy.clone(), - }, - _ => { - return Err(unsupported(format!( - "rollup function `{}`", - function.name() - ))) - } - }; - let child = self.lower(child)?; - if rollup == RollupFunction::DefaultRollup && !matches!(child, U::TimeRange { .. }) { - return Err(unsupported( - "default_rollup without an explicit range requires an evaluation step", - )); - } - Ok(aggregate(Reduction::PerEntity, intent, child)) - } - - fn aggregate( - &self, - expr: &metricsql_parser::ast::AggregationExpr, - ) -> Result { - if expr.limit != 0 || expr.keep_metric_names { - return Err(unsupported("aggregate limit or keep_metric_names")); - } - let expected_args = if expr.function == AggregateFunction::Quantile { - 2 - } else { - 1 - }; - require_arity(expr.name(), expr.args.len(), expected_args)?; - let child_index = expr - .arg_idx_for_optimization() - .ok_or_else(|| unsupported(format!("aggregate `{}` arguments", expr.name())))?; - let intent = match expr.function { - AggregateFunction::Sum => AggIntent::Sum { col: None }, - AggregateFunction::Avg => AggIntent::Avg { col: None }, - AggregateFunction::Min => AggIntent::Min { col: None }, - AggregateFunction::Max => AggIntent::Max { col: None }, - AggregateFunction::Count => AggIntent::Cardinality { - cols: vec![], - accuracy: self.accuracy.clone(), - }, - AggregateFunction::StdDev => AggIntent::StdDev { - col: None, - population: true, - }, - AggregateFunction::StdVar => AggIntent::Variance { - col: None, - population: true, - }, - AggregateFunction::Group => AggIntent::Group, - AggregateFunction::Quantile => AggIntent::Quantile { - col: None, - q: number_arg(&expr.args, 0)?, - accuracy: self.accuracy.clone(), - }, - _ => return Err(unsupported(format!("aggregate `{}`", expr.name()))), - }; - let reduction = match &expr.modifier { - None => Reduction::by(vec![]), - Some(AggregateModifier::By(v)) => Reduction::by(names(v)), - Some(AggregateModifier::Without(v)) => Reduction::Reduce(GroupKeys::without(names(v))), - }; - let child = expr - .args - .get(child_index) - .ok_or_else(|| unsupported("missing aggregate input"))?; - Ok(aggregate(reduction, intent, self.lower(child)?)) - } - - fn binary(&self, expr: &metricsql_parser::ast::BinaryExpr) -> Result { - if expr.modifier.is_some() { - return Err(unsupported("binary vector matching modifiers")); - } - use metricsql_parser::ast::Operator as O; - let op = match expr.op { - O::Add => BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - O::Sub => BinaryOpKind::Arithmetic(ArithmeticOpKind::Sub), - O::Mul => BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), - O::Div => BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), - O::Mod => BinaryOpKind::Arithmetic(ArithmeticOpKind::Mod), - O::Pow => BinaryOpKind::Arithmetic(ArithmeticOpKind::Pow), - O::Atan2 => BinaryOpKind::Arithmetic(ArithmeticOpKind::Atan2), - O::Eql => BinaryOpKind::Compare(CompareOpKind::Eq), - O::NotEq => BinaryOpKind::Compare(CompareOpKind::Ne), - O::Lt => BinaryOpKind::Compare(CompareOpKind::Lt), - O::Lte => BinaryOpKind::Compare(CompareOpKind::Le), - O::Gt => BinaryOpKind::Compare(CompareOpKind::Gt), - O::Gte => BinaryOpKind::Compare(CompareOpKind::Ge), - O::And => BinaryOpKind::Set(PromQLVectorSetOpKind::And), - O::Or => BinaryOpKind::Set(PromQLVectorSetOpKind::Or), - O::Unless => BinaryOpKind::Set(PromQLVectorSetOpKind::Unless), - O::If | O::IfNot | O::Default => { - return Err(unsupported(format!("MetricsQL operator `{}`", expr.op))) - } - }; - for (scalar, vector, scalar_left) in [ - (&expr.left, &expr.right, true), - (&expr.right, &expr.left, false), - ] { - if let Expr::NumberLiteral(n) = scalar.as_ref() { - return Ok(U::PromqlScalarOp { - child: Rc::new(self.lower(vector)?), - scalar: UnresolvedScalar::Literal(ScalarValue::Float64(n.value)), - op, - scalar_left, - return_bool: false, - }); - } - } - Ok(binary_op( - op, - self.lower(&expr.left)?, - self.lower(&expr.right)?, - )) - } -} - -/// A `BinaryOp` with default matching; MetricsQL modifiers (including `bool`) -/// are rejected before reaching here. -fn binary_op(kind: BinaryOpKind, lhs: U, rhs: U) -> U { - U::BinaryOp { - operator: BinaryOperator { - kind, - vector_match: None, - checked_relative_division: false, - checked_finite_division: false, - }, - return_bool: false, - lhs: Rc::new(lhs), - rhs: Rc::new(rhs), - } -} - -fn names(values: &[String]) -> Vec { - values.iter().cloned().map(ColumnRef::Named).collect() -} - -fn aggregate(reduction: Reduction, intent: AggIntent, child: U) -> U { - U::Aggregate { - reduction, - measures: vec![intent], - output_names: vec![String::new()], - filters: vec![], - having: None, - child: Rc::new(child), - } -} - -fn matcher(filter: &LabelFilter) -> UnresolvedScalar { - let op = match filter.op { - LabelFilterOp::Equal => CompareOpKind::Eq, - LabelFilterOp::NotEqual => CompareOpKind::Ne, - LabelFilterOp::RegexEqual => CompareOpKind::Regex, - LabelFilterOp::RegexNotEqual => CompareOpKind::NotRegex, - }; - UnresolvedScalar::Compare { - left: Box::new(UnresolvedScalar::Column(ColumnRef::Named( - filter.label.clone(), - ))), - op, - right: Box::new(UnresolvedScalar::Literal(ScalarValue::Utf8( - filter.value.clone(), - ))), - semantics: ExprSemantics::Promql, - } -} - -fn duration(value: &DurationExpr) -> Result { - match value { - DurationExpr::Millis(ms) if *ms >= 0 => Ok(Duration::from_millis(*ms as u64)), - DurationExpr::StepValue(_) => Err(unsupported("step-relative duration")), - DurationExpr::Millis(_) => Err(unsupported("negative duration")), - } -} - -fn number_arg(args: &[Expr], index: usize) -> Result { - match args.get(index) { - Some(Expr::NumberLiteral(v)) if v.value.is_finite() => Ok(v.value), - _ => Err(unsupported(format!("numeric argument #{index}"))), - } -} - -fn require_arity(name: &str, actual: usize, expected: usize) -> Result<(), MetricsqlError> { - if actual == expected { - Ok(()) - } else { - Err(unsupported(format!( - "`{name}` with {actual} arguments; canonical lowering requires exactly {expected}" - ))) - } -} - -fn unsupported(message: impl Into) -> MetricsqlError { - MetricsqlError::UnsupportedFeature(message.into()) -} diff --git a/crates/frontend-promql/src/unified/error.rs b/crates/frontend-promql/src/unified/error.rs deleted file mode 100644 index a889d6710..000000000 --- a/crates/frontend-promql/src/unified/error.rs +++ /dev/null @@ -1,81 +0,0 @@ -use std::fmt; - -use asap_frontend_common::ResolveDAGError; -use asap_types::workload::WorkloadError; - -/// Errors from lowering a PromQL query (parse → the name-based unresolved -/// tree, built directly → -/// [`resolve_root`](asap_frontend_common::resolve_root) binds it to the -/// unified operator DAG, issue #179). -/// -/// Carries no DataFusion type — the PromQL front end never depends on the SQL -/// stack. The language-neutral variants (`UnsupportedFeature` / `WrongLanguage` -/// / `Convert`) are mirrored by [`asap_frontend_sql::SqlError`] rather than -/// shared, so neither front end pulls the other's parser. -#[derive(Debug)] -pub enum PromqlError { - /// The workload omitted information required for plan-ready PromQL lowering. - InvalidWorkload(WorkloadError), - /// The `promql-parser` crate rejected the query string (parse failure). - Parse(String), - /// A PromQL function (`rate`, `*_over_time`, …) not supported in this version. - UnsupportedFunction(String), - /// A PromQL aggregation operator (`sum`, `topk`, …) not supported. - UnsupportedAggregateOp(String), - /// A structural feature (offset / `@` / `without`) not supported in this - /// version. - UnsupportedFeature(String), - /// A required function / aggregator argument was missing. - MissingArgument(String), - /// An argument had the wrong shape (e.g. a non-numeric `topk` parameter). - InvalidParameter(String), - /// The workload's query language is not PromQL. - WrongLanguage(String), - /// Resolving the canonical unresolved tree failed (name resolution - /// against the bound schema). - Convert(ResolveDAGError), -} - -impl fmt::Display for PromqlError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::InvalidWorkload(e) => write!(f, "invalid PromQL workload: {e}"), - Self::Parse(e) => write!(f, "PromQL parse error: {e}"), - Self::UnsupportedFunction(n) => write!(f, "unsupported PromQL function: {n}"), - Self::UnsupportedAggregateOp(n) => write!(f, "unsupported PromQL aggregate op: {n}"), - Self::UnsupportedFeature(m) => write!(f, "unsupported feature: {m}"), - Self::MissingArgument(m) => write!(f, "missing argument: {m}"), - Self::InvalidParameter(m) => write!(f, "invalid parameter: {m}"), - Self::WrongLanguage(l) => write!(f, "unsupported query language: {l}"), - Self::Convert(e) => write!(f, "column resolution failed: {e}"), - } - } -} - -impl std::error::Error for PromqlError {} - -impl From for PromqlError { - fn from(e: ResolveDAGError) -> Self { - Self::Convert(e) - } -} - -impl From for PromqlError { - fn from(e: WorkloadError) -> Self { - Self::InvalidWorkload(e) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn unsupported_feature_label_is_language_neutral() { - // `UnsupportedFeature` shares a Display label with the SQL side, so it - // must not hardcode "PromQL". - let msg = PromqlError::UnsupportedFeature("subquery".into()).to_string(); - assert_eq!(msg, "unsupported feature: subquery"); - assert!(!msg.contains("PromQL"), "got: {msg}"); - } -} diff --git a/crates/frontend-promql/src/unified/histogram.rs b/crates/frontend-promql/src/unified/histogram.rs deleted file mode 100644 index ecb8cd2c4..000000000 --- a/crates/frontend-promql/src/unified/histogram.rs +++ /dev/null @@ -1,129 +0,0 @@ -//! Sample-type metadata for the `histogram_quantile` discrimination (issue #79). -//! -//! Classic cumulative buckets use exact interpolation. The explicitly declared -//! `RawSamples` extension permits generic quantile sketches; it is not standard -//! PromQL histogram semantics. Native samples are rejected until the IR has a -//! native histogram sample type. Undeclared metrics require classic bucket -//! evidence (`by (le)`, a `_bucket` metric, or an `le` matcher). - -use std::cell::RefCell; -use std::collections::HashMap; - -/// The physical sample type behind a histogram metric — the true signal for -/// whether `histogram_quantile` over it can be re-sketched. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HistogramKind { - /// Classic cumulative `le` buckets — pre-aggregated counts. The - /// distribution can't be reconstructed from them, so it is **not** - /// sketch-able: `histogram_quantile` is exact bucket interpolation. - ClassicBucket, - /// Native histogram samples; currently rejected because the IR lacks their type. - Native, - /// Raw float samples the client retains — sketch-able. This is the case the - /// generic `Quantile` lowering exists for (a client holding raw samples can - /// build a quantile sketch even though the user wrote `histogram_quantile`). - RawSamples, -} - -impl HistogramKind { - /// Whether `histogram_quantile` over this kind lowers to the sketch-able - /// generic `Quantile` (`true`) rather than exact bucket interpolation. - pub fn is_sketchable(self) -> bool { - matches!(self, HistogramKind::RawSamples) - } -} - -/// Metric-name → declared [`HistogramKind`]. Supplied by a client that knows its -/// sample types, to drive the `histogram_quantile` discrimination from metadata -/// instead of query structure (issue #79). -#[derive(Debug, Clone, Default)] -pub struct HistogramCatalog(HashMap); - -impl HistogramCatalog { - pub fn new() -> Self { - Self::default() - } - - /// Declare `metric`'s sample type (builder style). - pub fn with(mut self, metric: impl Into, kind: HistogramKind) -> Self { - self.0.insert(metric.into(), kind); - self - } - - /// The declared kind for `metric`, if any. - pub fn kind_of(&self, metric: &str) -> Option { - self.0.get(metric).copied() - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -/// RAII guard installing `catalog` as the ambient histogram catalog for the -/// current thread, restoring the prior value on drop. -/// -/// Lowering is synchronous and processes one query at a time, so a thread-local -/// ambient catalog cleanly injects this read-only metadata into the deep, -/// free-function `walk` recursion without threading a parameter through every -/// signature (the discrimination is consulted in exactly one place, -/// `walk_histogram`). -pub(crate) struct CatalogGuard(Option); - -impl CatalogGuard { - pub(crate) fn install(catalog: HistogramCatalog) -> Self { - let prev = CURRENT.with(|c| c.borrow_mut().replace(catalog)); - CatalogGuard(prev) - } -} - -impl Drop for CatalogGuard { - fn drop(&mut self) { - CURRENT.with(|c| *c.borrow_mut() = self.0.take()); - } -} - -/// The ambient catalog's declared kind for `metric`, or `None` when no catalog -/// is installed or the metric is undeclared (→ fall back to the heuristic). -pub(crate) fn current_kind_of(metric: &str) -> Option { - CURRENT.with(|c| c.borrow().as_ref().and_then(|cat| cat.kind_of(metric))) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn only_explicit_raw_samples_are_sketchable() { - assert!(!HistogramKind::ClassicBucket.is_sketchable()); - assert!(!HistogramKind::Native.is_sketchable()); - assert!(HistogramKind::RawSamples.is_sketchable()); - } - - #[test] - fn catalog_lookup() { - let cat = HistogramCatalog::new() - .with("classic", HistogramKind::ClassicBucket) - .with("raw", HistogramKind::RawSamples); - assert_eq!(cat.kind_of("classic"), Some(HistogramKind::ClassicBucket)); - assert_eq!(cat.kind_of("raw"), Some(HistogramKind::RawSamples)); - assert_eq!(cat.kind_of("unknown"), None); - } - - #[test] - fn guard_installs_and_restores_the_ambient_catalog() { - assert_eq!(current_kind_of("m"), None); - { - let _g = CatalogGuard::install( - HistogramCatalog::new().with("m", HistogramKind::ClassicBucket), - ); - assert_eq!(current_kind_of("m"), Some(HistogramKind::ClassicBucket)); - } - // Restored to empty after the guard drops. - assert_eq!(current_kind_of("m"), None); - } -} diff --git a/crates/frontend-promql/src/unified/mod.rs b/crates/frontend-promql/src/unified/mod.rs deleted file mode 100644 index e7d99fe4c..000000000 --- a/crates/frontend-promql/src/unified/mod.rs +++ /dev/null @@ -1,233 +0,0 @@ -//! PromQL front end: parse (via `promql-parser`) → the name-based -//! [`UnresolvedOp`](asap_frontend_common::UnresolvedOp) tree, built directly -//! in canonical shape (issue #179) → [`resolve_root`]. -//! -//! `resolve_root` runs the -//! [`SchemaResolver`](asap_frontend_common::SchemaResolver) for positional -//! name resolution and returns the unified -//! [`OperatorNode`](asap_types::ir::OperatorNode) DAG. Depends on the PromQL -//! parser only — never on the SQL / DataFusion stack. - -pub mod error; -pub mod histogram; -pub mod promql; - -use std::rc::Rc; - -use asap_types::ir::OperatorNode; -use asap_types::workload::{DurationMs, PlanningWorkload, QueryLanguage, WorkloadError}; - -pub use error::PromqlError; -pub use histogram::{HistogramCatalog, HistogramKind}; - -/// Lower every normalized PromQL workload entry to a plan-ready operator DAG. -/// -/// PromQL workloads must declare a non-zero `data_ingestion_interval`; it is -/// injected around each bare instant selector. Explicit range selectors keep -/// their query-specified range. -/// `now_ms` is the planning time in Unix milliseconds; cadence evidence must -/// be valid at that time, using the same clock as downstream planning. -pub fn lower_promql_workload( - workload: &PlanningWorkload, - now_ms: u64, -) -> Result>, PromqlError> { - lower_promql_workload_inner(workload, now_ms) -} - -/// Like [`lower_promql_workload`], but uses `histograms` to distinguish classic -/// bucket interpolation from generic sketchable quantiles. -pub fn lower_promql_workload_with_histograms( - workload: &PlanningWorkload, - histograms: HistogramCatalog, - now_ms: u64, -) -> Result>, PromqlError> { - let _guard = histogram::CatalogGuard::install(histograms); - lower_promql_workload_inner(workload, now_ms) -} - -/// Lower scalar and vector query roots without introducing constant operators. -pub fn lower_promql_query_workload( - workload: &PlanningWorkload, - now_ms: u64, -) -> Result, PromqlError> { - lower_promql_query_workload_inner(workload, now_ms) -} - -pub fn lower_promql_query_workload_with_histograms( - workload: &PlanningWorkload, - histograms: HistogramCatalog, - now_ms: u64, -) -> Result, PromqlError> { - let _guard = histogram::CatalogGuard::install(histograms); - lower_promql_query_workload_inner(workload, now_ms) -} - -fn lower_promql_workload_inner( - workload: &PlanningWorkload, - now_ms: u64, -) -> Result>, PromqlError> { - lower_promql_query_workload_inner(workload, now_ms)? - .into_iter() - .map(|root| match root { - asap_types::ir::QueryRoot::Operator(node) => Ok(node), - asap_types::ir::QueryRoot::Scalar(_) => Err(PromqlError::UnsupportedFeature( - "scalar root: use lower_promql_query_workload".into(), - )), - }) - .collect() -} - -fn lower_promql_query_workload_inner( - workload: &PlanningWorkload, - now_ms: u64, -) -> Result, PromqlError> { - if !matches!(workload.query_workload.language, QueryLanguage::PromQL) { - return Err(PromqlError::WrongLanguage(format!( - "{:?}", - workload.query_workload.language - ))); - } - workload.validate()?; - let &DurationMs(interval_ms) = workload - .data_workload - .as_ref() - .expect("validated PromQL workload has data_workload") - .data_ingestion_interval - .value_at(now_ms) - .ok_or(WorkloadError::UnavailableDataIngestionInterval)?; - workload - .query_workload - .entries() - .map(|entry| { - let root = promql::PromqlLowerer::lower_query_with_ingestion_interval( - &entry.query.0, - &entry.requirements.accuracy.target(), - std::time::Duration::from_millis(interval_ms), - )?; - Ok(root) - }) - .collect() -} - -#[cfg(test)] -mod tests { - // Expiring evidence without an observation timestamp is never usable. - #[test] - fn rejects_unusable_ingestion_evidence() { - let mut input = workload("sum(data)"); - input - .data_workload - .as_mut() - .unwrap() - .data_ingestion_interval - .valid_for_ms = Some(100); - assert!(lower_promql_workload(&input, 0).is_err()); - } - - // Cadence expiry is inclusive; future and expired evidence cannot set a horizon. - #[test] - fn ingestion_evidence_respects_planning_time_with_and_without_histograms() { - let mut input = workload("sum(data)"); - let evidence = &mut input - .data_workload - .as_mut() - .unwrap() - .data_ingestion_interval; - evidence.observed_at_ms = Some(1_000); - evidence.valid_for_ms = Some(100); - for (now_ms, usable) in [(999, false), (1_000, true), (1_100, true), (1_101, false)] { - assert_eq!(lower_promql_workload(&input, now_ms).is_ok(), usable); - assert_eq!( - lower_promql_workload_with_histograms(&input, HistogramCatalog::default(), now_ms) - .is_ok(), - usable - ); - } - input - .data_workload - .as_mut() - .unwrap() - .data_ingestion_interval - .observed_at_ms = None; - assert!( - lower_promql_workload_with_histograms(&input, HistogramCatalog::default(), 1_000) - .is_err() - ); - } - use std::time::Duration; - - use asap_types::ir::{NonASAPOp, TimeRangeKind}; - use asap_types::workload::{ - BatchEntry, DataWorkload, Evidence, PlanningWorkload, Query, QueryRequirements, - QueryWorkload, TimeSelection, - }; - - use super::*; - - fn workload(query: &str) -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements::default(), - predictability: Default::default(), - invocations: 1, - execute_at: None, - time_selection: TimeSelection::default(), - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload { - data_ingestion_interval: Evidence { - value: Some(DurationMs(1_000)), - ..Default::default() - }, - ..Default::default() - }), - } - } - - // A bare instant selector reads the latest sample within the declared - // ingestion interval: an `Instant` lookback of that length. - #[test] - fn instant_selector_uses_declared_ingestion_interval() { - let query = lower_promql_workload(&workload("sum by (job) (data)"), 0).unwrap(); - let NonASAPOp::Aggregate { child, .. } = query[0].expect_non_asap() else { - panic!("expected aggregate") - }; - assert!( - matches!(child.expect_non_asap(), NonASAPOp::TimeRange { range, kind, child } - if *range == Duration::from_secs(1) - && *kind == TimeRangeKind::Instant - && matches!(child.expect_non_asap(), NonASAPOp::Scan { .. })) - ); - } - - // An explicit `m[5m]` keeps its own window as a `Range` selection. - #[test] - fn explicit_range_selector_keeps_its_query_range() { - let query = lower_promql_workload(&workload("sum_over_time(data[5m])"), 0).unwrap(); - let NonASAPOp::Aggregate { child, .. } = query[0].expect_non_asap() else { - panic!("expected aggregate") - }; - assert!( - matches!(child.expect_non_asap(), NonASAPOp::TimeRange { range, kind, child } - if *range == Duration::from_secs(300) - && *kind == TimeRangeKind::Range - && matches!(child.expect_non_asap(), NonASAPOp::Scan { .. })) - ); - } - - #[test] - fn workload_without_interval_fails_loudly() { - let mut workload = workload("sum(data)"); - workload.data_workload = Some(DataWorkload::default()); - assert!(matches!( - lower_promql_workload(&workload, 0), - Err(PromqlError::InvalidWorkload( - asap_types::workload::WorkloadError::MissingDataIngestionInterval - )) - )); - } -} diff --git a/crates/frontend-promql/src/unified/promql.rs b/crates/frontend-promql/src/unified/promql.rs deleted file mode 100644 index f97e4fb9d..000000000 --- a/crates/frontend-promql/src/unified/promql.rs +++ /dev/null @@ -1,2236 +0,0 @@ -//! PromQL string → the name-based -//! [`UnresolvedOp`](asap_frontend_common::UnresolvedOp) tree. -//! -//! - **Parsing** is delegated to `promql-parser` 0.8. -//! - **Lowering** builds *directly in canonical shape* here (issue #179): the -//! walk interprets PromQL semantics (range vectors, aggregate operators, -//! label matchers) and emits `UnresolvedOp` / `UnresolvedScalar` nodes with -//! unresolved `ColumnRef`s — the same tree shape -//! [`resolve_root`](asap_frontend_common::resolve_root) later binds to the -//! positional [`OperatorNode`](asap_types::ir::OperatorNode) DAG. The structural decisions a -//! separate converter stage would otherwise have to make (heavy-hitter -//! `topk` recognition, the `PerEntity`/`Reduce` reduction choice, -//! `without(...)` grouping) are made right here, since a front end -//! building this shape already knows the answer at parse time — see -//! `reduction_for` and `mark_without`. `resolve_root` is left with exactly -//! the schema-*dependent* work: binding every `ColumnRef` to its -//! positional `ColumnId`. -//! -//! # PromQL → canonical unresolved-tree mapping (summary) -//! -//! | PromQL | Canonical shape | -//! |---|---| -//! | `quantile_over_time(φ, m{f}[w])` | `Aggregate{[Quantile(φ)], TimeRange{w, Scan{predicates}}}` | -//! | `histogram_quantile(φ, )` | `Aggregate{without(le), [HistogramQuantile(φ, le)]}` — cumulative-bucket interpolation (classic form recognised by `by (le)` / a `_bucket` metric / an `le` matcher) | -//! | `histogram_quantile(φ, )` | `Aggregate{[Quantile(φ)]}` over the fully-lowered arg (generic, sketch-able with an accuracy target) | -//! | `histogram_quantiles(v, "l", φ…)` | `Concat{PromqlRelabel{l=φᵢ, }…}` — one branch per φ (issue #109) | -//! | `histogram_count/sum/avg/stddev/stdvar(v)`, `histogram_fraction(l,u,v)` | `Aggregate{[Histogram*]}` — per-series native-histogram accessors (issue #43) | -//! | `OUTER_op(inner_func(m[w]))` (e.g. `sum(rate(m[w]))`) | `Aggregate{[OUTER_op]}` over `Aggregate{[inner_func]}` — two levels | -//! | `OUTER_op()` (e.g. `max(sum by (job) (rate(m[w])))`, `sum(rate(a[w]) + rate(b[w]))`) | `Aggregate{[OUTER_op]}` over the fully-lowered `` — arbitrary function nesting (issue #27) | -//! | `topk(k, )` / `bottomk(k, )` | `Sort{value} → Limit{k}` over the fully-lowered argument | -//! | `avg/min/max/sum_over_time(m[w])` | `Aggregate{[Avg/Min/Max/Sum], TimeRange{w}}` | -//! | `stddev/stdvar_over_time(m[w])` | `Aggregate{[StdDev/Variance], TimeRange{w}}` | -//! | `count_over_time(m[w])` | `Aggregate{[Count], TimeRange{w}}` | -//! | `last/first/mad/ts_of_min/ts_of_max/ts_of_first/ts_of_last_over_time(m[w])` | `Aggregate{[Last/First/Mad/TsOf…OverTime], TimeRange{w}}` — per-series range reducers (issue #51) | -//! | `sort`/`sort_desc(v)`, `sort_by_label[_desc](v,"l"…)` | `Sort{value \| label…}` (no `Limit`) — row-preserving reorder (issue #51); `min_of`/`max_of` scalar reducers → #89 | -//! | `rate(m[w])` / `irate(m[w])` | `Aggregate{[Rate/IRate], TimeRange{w}}` — distinct function identities; shared physical machinery is a later realization choice | -//! | `increase(m[w])` | `Aggregate{[Increase], TimeRange{w}}` | -//! | `changes`/`delta`/`idelta`/`deriv`/`resets`/`predict_linear`/`double_exponential_smoothing`(`m[w]`, …) | `Aggregate{[Changes/Delta/…], TimeRange{w}}` — per-series counter-derivative intents (issue #44) | -//! | `absent(v)` / `absent_over_time(m[w])` / `present_over_time(m[w])` | `Aggregate{[Absent/AbsentOverTime/PresentOverTime]}` — presence intents; the empty→synthesized-sample logic is a post-ASAP concern (issue #47) | -//! | `abs`/`ceil`/`sqrt`/`ln`/`clamp*`/`round`/trig(`v`), `pi()` | typed scalar `Project` (issue #45); `pi()` → a `ScalarExpr::Literal` root | -//! | `time()` / `timestamp`/`hour`/`day_of_week`/… (`v`) | `ScalarExpr::EvalTimestamp` root / `Aggregate{[TimeFn(f)]}` (issue #46) | -//! | `vector(s)` / `scalar(v)` | `PromqlVectorFromScalar(s)` / `ScalarExpr::PromqlScalarFromVector(v)` — the scalar⇄vector bridges (issue #48) | -//! | ` op ` (`time() - 1`, `1 < bool 2`, `-time()`) | `ScalarExpr::{Arithmetic, Case, Negative}` — a scalar expression, never an operator | -//! | `v op `, `a op bool b`, `v > bool 0` | `Project`/`Filter` with owned scalar expressions; vector/vector uses `BinaryOp{return_bool}` | -//! | `label_replace(v,…)` / `label_join(v,…)` | `PromqlRelabel{dst, value}` — per-series label rewrite; value unchanged (issue #50) | -//! | `info(v, [selector])` | `PromqlInfoEnrich{selector}` — label-enrichment join against the info metric(s); join keys resolved during post-ASAP binding (issue #84) | -//! | `group` / `offset` / `@` / `info` | **rejected** — distinct semantics with no intent-algebra representation yet (`info` label-join → #84) | -//! | `OUTER by (dims) (…)` | `Aggregate.reduction = Reduce(by = dims)` (generic `topk by`/`bottomk` grouping → `Sort.partition_by`) | -//! | `count by (d) (…)` | `Aggregate{[Count], …}` | -//! | `group(v)` / `count_values("l", v)` | `Aggregate{[Group]}` (constant 1) / `Aggregate{[CountValues{l}]}` (group-by-value + count, new label `l`) — issue #49 | -//! | `limitk(k, v)` / `limit_ratio(r, v)` | `PromqlSeriesSample{LimitK(k) \| LimitRatio(r)}` — series-sampling selection, whole series kept unchanged (issue #86) | -//! | `topk(k, count_over_time(…))` / `topk(k, sum_over_time(…))` | `Aggregate{[TopK{k}]}` (heavy-hitter intent) over the explicit inner `Aggregate{[Count/Sum]}` | -//! | `topk(k, )` / `bottomk(k, …)` | `Sort{value} → Limit{k}` | -//! | `m{f}` / `m{f}[w]` | `TimeRange{ingestion, Instant, Scan{predicates}}` / `TimeRange{w, Range, Scan}` | -//! | `a OP b` | `BinaryOp{vector_match}` | -//! | `expr[r:res]` | `PromqlSubquery{r, res}` | -//! | ` offset ` / ` @ `/`start()`/`end()` | `TimeShift{shift}` over the selector's `Scan` — pass-through schema; a ranged selector shifts under its `TimeRange` (issue #40) | - -use std::rc::Rc; -use std::time::{Duration, SystemTime}; - -use promql_parser::label::{MatchOp, Matcher}; -use promql_parser::parser::value::ValueType; -use promql_parser::parser::{ - self, token, AggregateExpr, AtModifier as ParserAtModifier, BinaryExpr, Call, Expr, - LabelModifier, Offset, VectorMatchCardinality, VectorSelector, -}; - -use asap_frontend_common::{ - UnresolvedOp as Unresolved, UnresolvedPredicate, UnresolvedScalar as Scalar, UnresolvedSortKey, -}; -use asap_types::ir::operator_properties::{ - AtModifier, BinaryOpKind, GroupKeys, GroupSide, PromQLVectorSetOpKind, Reduction, Source, - TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, -}; -use asap_types::ir::{BinaryOperator, ExprSemantics, TimeRangeKind}; -use asap_types::pre_asap::agg_intent::{topk, AggIntent, TimeFunc}; - -use asap_types::pre_asap::{ - ArithmeticOpKind, ColumnRef, CompareOpKind, InfoMatcher, SampleKind, ScalarValue, -}; -use asap_types::types::AccuracyTarget; - -/// Every scalar expression this front end builds follows PromQL's numeric rules. -const PROMQL: ExprSemantics = ExprSemantics::Promql; - -use crate::unified::error::PromqlError as LoweringError; - -type Result = std::result::Result; - -/// Parses and lowers (→ the canonical, unresolved tree) a PromQL query string. -pub(crate) struct PromqlLowerer; - -#[derive(Debug, Clone)] -enum Outer { - None, - Plain(OuterIntent), - Count, - /// `count_values("l", v)` — group by value + count, emitting the value as a - /// new label `l` (issue #49). - CountValues { - label: String, - }, - TopK { - k: u64, - descending: bool, - }, - /// `limitk`/`limit_ratio` — series-sampling selection (issue #86). - Sample { - kind: SampleKind, - }, -} - -#[derive(Debug, Clone)] -enum OuterIntent { - Sum, - Avg, - Min, - Max, - StdDev, - Variance, - Quantile(f64), - /// `group(v)` — constant 1 per group (issue #49). - Group, -} - -#[derive(Debug, Clone)] -enum InnerFunc { - FrequencyL2, - FrequencyEntropy, - Cardinality, - Quantile(f64), - Avg, - Min, - Max, - Sum, - StdDev, - Variance, - Count, - // `Rate`/`Increase` carry no window of their own — unlike the old Unresolved - // `AggFunc::Rate{window}`, canonical `AggIntent::Rate`/`Increase` have no - // window field either; `windowed_aggregate` reads `Inner.window` - // uniformly for every intent, so it would be a redundant duplicate here. - Rate, - IRate, - Increase, - // Counter-derivative range functions (issue #44). The window rides on the - // enclosing `TimeRange` node (like `*_over_time`), so these carry only - // their non-window scalar params. - Changes, - Delta, - IDelta, - Deriv, - Resets, - PredictLinear(f64), - DoubleExp { smoothing: f64, trend: f64 }, - // Additional range-vector reducers (issue #51). Per-series over the window - // (like `*_over_time`); the window rides on the enclosing Unresolved `Window`. - LastOverTime, - FirstOverTime, - MadOverTime, - TsOfMinOverTime, - TsOfMaxOverTime, - TsOfFirstOverTime, - TsOfLastOverTime, -} - -struct Inner { - metric: String, - matchers: Vec, - window: Option, - func: Option, - /// `offset` / `@` on the selector, carried to the `Source` (issue #40). - shift: TimeShift, -} - -/// Maximum PromQL expression nesting depth the walker accepts. Real queries -/// nest only a handful deep; this bounds the recursive descent (`walk` and the -/// mutually-recursive helpers) so a pathologically nested query is rejected -/// rather than overflowing the stack. -const MAX_DEPTH: usize = 256; - -impl PromqlLowerer { - pub(crate) fn lower_query_with_ingestion_interval( - query: &str, - accuracy: &AccuracyTarget, - interval: Duration, - ) -> Result { - let _guard = AccuracyGuard::install(accuracy.clone()); - let _interval = IngestionIntervalGuard::install(interval); - let ast = parser::parse(query).map_err(LoweringError::Parse)?; - check_depth(&ast, MAX_DEPTH)?; - let mut metrics = Vec::new(); - collect_metric_names(&ast, &mut metrics); - if metrics.iter().any(|metric| { - crate::unified::histogram::current_kind_of(metric) - == Some(crate::unified::histogram::HistogramKind::Native) - }) { - return Err(LoweringError::UnsupportedFeature( - "native histogram samples have no IR representation".into(), - )); - } - - if ast.value_type() == ValueType::Scalar { - Ok(asap_types::ir::QueryRoot::Scalar( - asap_frontend_common::resolve_scalar_root(&lower_scalar(&ast)?)?, - )) - } else { - Ok(asap_types::ir::QueryRoot::Operator( - asap_frontend_common::resolve_root(&walk(&ast)?)?, - )) - } - } -} - -std::thread_local! { - static ACCURACY: std::cell::RefCell = - const { std::cell::RefCell::new(AccuracyTarget::Exact) }; - static INGESTION_INTERVAL: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; -} - -/// RAII guard installing `accuracy` as the ambient accuracy target for the -/// current thread's lowering, restoring the prior value on drop — same shape -/// as `histogram::CatalogGuard`. -struct AccuracyGuard(AccuracyTarget); - -impl AccuracyGuard { - fn install(accuracy: AccuracyTarget) -> Self { - let prev = ACCURACY.with(|a| a.replace(accuracy)); - AccuracyGuard(prev) - } -} - -impl Drop for AccuracyGuard { - fn drop(&mut self) { - ACCURACY.with(|a| *a.borrow_mut() = std::mem::replace(&mut self.0, AccuracyTarget::Exact)); - } -} - -/// The ambient accuracy target installed by the current [`PromqlLowerer::lower`] call. -fn current_accuracy() -> AccuracyTarget { - ACCURACY.with(|a| a.borrow().clone()) -} - -struct IngestionIntervalGuard(Option); - -impl IngestionIntervalGuard { - fn install(interval: Duration) -> Self { - Self(INGESTION_INTERVAL.with(|current| current.replace(Some(interval)))) - } -} - -impl Drop for IngestionIntervalGuard { - fn drop(&mut self) { - INGESTION_INTERVAL.with(|current| *current.borrow_mut() = self.0.take()); - } -} - -fn current_ingestion_interval() -> Duration { - INGESTION_INTERVAL.with(|current| { - current - .borrow() - .expect("ingestion interval is installed for workload lowering") - }) -} - -/// Bounded depth check over the parser AST: errors once nesting would exceed -/// `budget` frames, descending into every child expression. -fn check_depth(expr: &Expr, budget: usize) -> Result<()> { - let Some(budget) = budget.checked_sub(1) else { - return Err(LoweringError::UnsupportedFeature(format!( - "query nesting exceeds the {MAX_DEPTH}-level limit" - ))); - }; - match expr { - Expr::Aggregate(a) => { - check_depth(&a.expr, budget)?; - if let Some(p) = &a.param { - check_depth(p, budget)?; - } - } - Expr::Unary(u) => check_depth(&u.expr, budget)?, - Expr::Binary(b) => { - check_depth(&b.lhs, budget)?; - check_depth(&b.rhs, budget)?; - } - Expr::Paren(p) => check_depth(&p.expr, budget)?, - Expr::Subquery(s) => check_depth(&s.expr, budget)?, - Expr::Call(c) => { - for arg in &c.args.args { - check_depth(arg, budget)?; - } - } - Expr::MatrixSelector(_) - | Expr::VectorSelector(_) - | Expr::NumberLiteral(_) - | Expr::StringLiteral(_) - | Expr::Extension(_) => {} - } - Ok(()) -} - -fn walk(expr: &Expr) -> Result { - // A scalar-typed expression (`5`, `time() - 1`, `scalar(v)`, `1 < bool 2`) - // is a scalar expression at an operator position, never an operator tree. - if expr.value_type() == ValueType::Scalar { - return Err(LoweringError::UnsupportedFeature( - "scalar root requires query-root lowering".into(), - )); - } - match expr { - Expr::Aggregate(agg) => walk_aggregate(agg), - Expr::Call(call) if call.func.name.starts_with("histogram_") => walk_histogram(call), - Expr::Call(call) if is_math_fn(call.func.name) => walk_math(call), - Expr::Call(call) if is_presence_fn(call.func.name) => walk_presence(call), - Expr::Call(call) if is_time_fn(call.func.name) => walk_time(call), - Expr::Call(call) if is_typeconv_fn(call.func.name) => walk_typeconv(call), - Expr::Call(call) if is_label_fn(call.func.name) => walk_label(call), - Expr::Call(call) if is_sort_fn(call.func.name) => walk_sort(call), - Expr::Call(call) if call.func.name == "info" => walk_info(call), - Expr::Call(call) => walk_call(call), - Expr::Binary(bin) => walk_binary(bin), - Expr::Paren(p) => walk(&p.expr), - // `UnaryExpr` is built only by negation (`Neg`); unary `+` is folded to - // identity and `-` to a negated `NumberLiteral`. A scalar - // operand was dispatched to `lower_scalar` above (→ `Negative`), so this - // is a vector projection. Unary negation retains the metric name. - Expr::Unary(u) => Ok(Unresolved::PromqlMap { - child: Rc::new(walk(&u.expr)?), - sample: Scalar::Negative { - expr: Box::new(Scalar::Column(ColumnRef::SampleValue)), - semantics: ExprSemantics::Promql, - }, - drop_metric_name: false, - }), - Expr::Subquery(sq) => { - let subquery = Unresolved::PromqlSubquery { - range: sq.range, - resolution: sq.step, - child: Rc::new(walk(&sq.expr)?), - }; - // `offset`/`@` move the whole subquery, including its step grid. - let shift = time_shift(sq.offset.as_ref(), sq.at.as_ref())?; - Ok(if shift.is_identity() { - subquery - } else { - Unresolved::TimeShift { - shift, - child: Rc::new(subquery), - } - }) - } - Expr::VectorSelector(vs) => { - let (metric, matchers, shift) = vs_parts(vs)?; - Ok(instant_source(metric, matchers, shift)) - } - Expr::MatrixSelector(ms) => { - let (metric, matchers, shift) = vs_parts(&ms.vs)?; - Ok(Unresolved::TimeRange { - range: ms.range, - kind: TimeRangeKind::Range, - child: Rc::new(filtered_source(metric, matchers, shift)), - }) - } - // Scalar-typed, dispatched above; kept for exhaustiveness. String - // literals only appear as function args (`label_replace`, …), so a - // bare one is rejected (issue #35). - Expr::NumberLiteral(_) => unreachable!("scalar handled above"), - Expr::StringLiteral(_) => Err(LoweringError::UnsupportedFeature( - "bare string literal".into(), - )), - Expr::Extension(_) => Err(LoweringError::UnsupportedFeature( - "extension expression".into(), - )), - } -} - -/// Lower a scalar-typed PromQL expression to a scalar expression. A constant -/// sub-expression folds to one `Literal` (as `num_expr` always did); anything -/// else keeps its structure: `-time()` → `Negative`, `time() - 1` → -/// `Arithmetic`, `scalar(v)` → `PromqlScalarFromVector`, and a `bool` -/// comparison → `Case(Compare → 1, else 0)` (PromQL yields `0`/`1`). -fn lower_scalar(expr: &Expr) -> Result { - if let Ok(v) = num_expr(expr) { - return Ok(Scalar::Literal(ScalarValue::Float64(v))); - } - match expr { - Expr::Paren(p) => lower_scalar(&p.expr), - Expr::Unary(u) => Ok(Scalar::Negative { - expr: Box::new(lower_scalar(&u.expr)?), - semantics: PROMQL, - }), - Expr::Binary(bin) => lower_scalar_binary(bin), - Expr::Call(call) => match call.func.name { - "time" => Ok(Scalar::EvalTimestamp), - "pi" => Ok(Scalar::Literal(ScalarValue::Float64(std::f64::consts::PI))), - "scalar" => Ok(Scalar::PromqlScalarFromVector(Rc::new(walk(arg( - call, 0, - )?)?))), - // `min_of`/`max_of` fold only over constants (#89); the fold above - // failed, so surface its error for the non-constant argument. - name if is_scalar_reducer_fn(name) => Err(num_expr(expr).unwrap_err()), - other => Err(LoweringError::UnsupportedFunction(other.to_string())), - }, - other => Err(LoweringError::UnsupportedFeature(format!( - "scalar expression `{other}`" - ))), - } -} - -/// ` op `: arithmetic is an `Arithmetic` expression; a -/// comparison needs the `bool` modifier (PromQL has no scalar filter) and -/// becomes `Case(Compare → 1.0, else 0.0)`. The parser already rejects both a -/// bool-less scalar comparison and a scalar set op; both are re-checked here. -fn lower_scalar_binary(bin: &BinaryExpr) -> Result { - let left = Box::new(lower_scalar(&bin.lhs)?); - let right = Box::new(lower_scalar(&bin.rhs)?); - match binop(bin.op.id())? { - BinaryOpKind::Arithmetic(op) => Ok(Scalar::Arithmetic { - op, - left, - right, - semantics: PROMQL, - }), - BinaryOpKind::Compare(op) | BinaryOpKind::CompareBool(op) => { - if !bin.return_bool() { - return Err(LoweringError::InvalidParameter( - "a comparison between two scalars requires the `bool` modifier".into(), - )); - } - let compare = Scalar::Compare { - left, - op, - right, - semantics: PROMQL, - }; - Ok(Scalar::Case { - operand: None, - branches: vec![(compare, Scalar::Literal(ScalarValue::Float64(1.0)))], - else_expr: Some(Box::new(Scalar::Literal(ScalarValue::Float64(0.0)))), - }) - } - BinaryOpKind::Set(_) => Err(LoweringError::UnsupportedFeature( - "set operator between two scalars".into(), - )), - } -} - -/// A binary operation over two vectors. -fn vector_binary( - kind: BinaryOpKind, - vector_match: Option, - return_bool: bool, - lhs: Unresolved, - rhs: Unresolved, -) -> Unresolved { - Unresolved::BinaryOp { - operator: BinaryOperator { - kind, - vector_match, - checked_relative_division: false, - checked_finite_division: false, - }, - return_bool, - lhs: Rc::new(lhs), - rhs: Rc::new(rhs), - } -} - -/// Lower a bare function call (`rate(m[5m])`, `max_over_time(m[5m])`, …). -/// -/// The common case routes through the flat `lower_inner_call` template. The one -/// exception is a `*_over_time`/`quantile_over_time` function applied to a -/// **sub-query** (`max_over_time(rate(m[5m])[1h:])`): its argument is a -/// `PromQLSubquery`, not a matrix selector, so the flat template's -/// `extract_matrix` can't accept it. Lower the sub-query recursively and reduce -/// it per series (issue #27). -fn walk_call(call: &Call) -> Result { - if let Some(tree) = range_fn_over_subquery(call)? { - return Ok(tree); - } - build(lower_inner_call(call)?, vec![], Outer::None) -} - -/// A range-vector function applied to a **sub-query** — `f([range:res])`. -/// -/// Covers the whole range-vector family: the `*_over_time` reducers, -/// `rate`/`irate`/`increase`, and the counter-derivatives -/// (`changes`/`delta`/`idelta`/`deriv`/`resets`/`predict_linear`/ -/// `double_exponential_smoothing`). Each lowers to a per-series `Aggregate{[f]}` -/// directly over the `PromqlSubquery` — the sub-query is the range context, so -/// there is no separate `Window`/`TimeRange` (this walk treats the `PromqlSubquery` -/// node itself as the range marker). Returns `None` when `call` isn't a range -/// function or its argument isn't a sub-query, so the flat matrix-selector -/// template still handles `f(m[w])` (issues #42, #55). -fn range_fn_over_subquery(call: &Call) -> Result> { - // `rate`/`increase`/`irate` carry their window in the `AggFunc`; over a - // sub-query that window is the sub-query's own range. - if let "rate" | "irate" | "increase" = call.func.name { - let arg_expr = arg(call, 0)?; - if subquery_range(arg_expr).is_none() { - return Ok(None); - } - let inner = match call.func.name { - "rate" => InnerFunc::Rate, - "irate" => InnerFunc::IRate, - "increase" => InnerFunc::Increase, - _ => unreachable!(), - }; - return Ok(Some(per_series_aggregate( - vec![], - inner_intent(&inner), - walk(arg_expr)?, - ))); - } - - // `*_over_time` reducers + counter-derivatives: the func-kind, and the index - // of the matrix/sub-query argument (`quantile_over_time` reads φ from arg 0, - // so its matrix is arg 1; the rest take arg 0 + trailing scalar params). - let (inner, matrix_idx): (InnerFunc, usize) = match call.func.name { - "avg_over_time" => (InnerFunc::Avg, 0), - "min_over_time" => (InnerFunc::Min, 0), - "max_over_time" => (InnerFunc::Max, 0), - "sum_over_time" => (InnerFunc::Sum, 0), - "stddev_over_time" => (InnerFunc::StdDev, 0), - "stdvar_over_time" => (InnerFunc::Variance, 0), - "count_over_time" => (InnerFunc::Count, 0), - "distinct_over_time" => (InnerFunc::Cardinality, 0), - "entropy_over_time" => (InnerFunc::FrequencyEntropy, 0), - "l2_over_time" => (InnerFunc::FrequencyL2, 0), - "quantile_over_time" => (InnerFunc::Quantile(quantile_param(num_arg(call, 0)?)?), 1), - "changes" => (InnerFunc::Changes, 0), - "delta" => (InnerFunc::Delta, 0), - "idelta" => (InnerFunc::IDelta, 0), - "deriv" => (InnerFunc::Deriv, 0), - "resets" => (InnerFunc::Resets, 0), - "last_over_time" => (InnerFunc::LastOverTime, 0), - "first_over_time" => (InnerFunc::FirstOverTime, 0), - "mad_over_time" => (InnerFunc::MadOverTime, 0), - "ts_of_min_over_time" => (InnerFunc::TsOfMinOverTime, 0), - "ts_of_max_over_time" => (InnerFunc::TsOfMaxOverTime, 0), - "ts_of_first_over_time" => (InnerFunc::TsOfFirstOverTime, 0), - "ts_of_last_over_time" => (InnerFunc::TsOfLastOverTime, 0), - "predict_linear" => (InnerFunc::PredictLinear(num_arg(call, 1)?), 0), - "double_exponential_smoothing" => ( - InnerFunc::DoubleExp { - smoothing: num_arg(call, 1)?, - trend: num_arg(call, 2)?, - }, - 0, - ), - _ => return Ok(None), - }; - let arg_expr = arg(call, matrix_idx)?; - if !is_subquery(arg_expr) { - return Ok(None); - } - Ok(Some(per_series_aggregate( - vec![], - inner_intent(&inner), - walk(arg_expr)?, - ))) -} - -/// A (parenthesised) PromQL sub-query — `[range:res]`. -fn is_subquery(expr: &Expr) -> bool { - subquery_range(expr).is_some() -} - -/// The `range` of a (parenthesised) sub-query argument, if it is one. -fn subquery_range(expr: &Expr) -> Option { - match expr { - Expr::Subquery(sq) => Some(sq.range), - Expr::Paren(p) => subquery_range(&p.expr), - _ => None, - } -} - -fn walk_aggregate(agg: &AggregateExpr) -> Result { - let (keys, without) = resolve_group(agg)?; - let outer = outer_kind(agg)?; - - // `without(...)` grouping is modelled only for the reducing aggregations - // (sum/avg/count/…), whose grouping lives on an `Aggregate` node. `topk`/ - // `bottomk` (→ `Sort.partition_by`) and `limitk`/`limit_ratio` (→ `PromqlSeriesSample`) - // would need without-partitioning too; reject rather than silently lower - // them as a `by` grouping (issue #39). - if without && matches!(outer, Outer::TopK { .. } | Outer::Sample { .. }) { - return Err(LoweringError::UnsupportedFeature( - "`without(...)` is only supported on reducing aggregations, not \ - topk/bottomk/limitk" - .into(), - )); - } - - // Fast path — the argument is a bare selector or a single range-vector - // function (`rate`/`increase`/`*_over_time`). `lower_inner` lowers it via the - // flat selector/call template, which also recognises the heavy-hitter - // `topk(k, count_over_time(...))` shape. This is the common two-level case - // (`sum by (job) (rate(m[5m]))`). - // - // General nesting — the argument is itself a composite expression: another - // aggregate (`max(sum by (job) (rate(m[5m])))`), a binary op, a sub-query, or - // a function lowered elsewhere. Lower it recursively with the same `walk` - // used at the top level, then wrap it in the outer aggregation (issue #27; a - // negated argument `sum(-m)` lowers here too, #36). A genuinely unsupported - // inner expression surfaces its own error rather than being mislowered. - // - // Either way, `mark_without` flips the resulting outer `Aggregate` to the - // exclusion form when the modifier was `without(...)`. - let built = match lower_inner(&agg.expr) { - Ok(inner) => build(inner, keys, outer)?, - Err(_) => build_over_sub_dag(outer, keys, walk(&agg.expr)?)?, - }; - Ok(mark_without(built, without)) -} - -/// Map an `AggregateExpr`'s operator (`sum`/`avg`/`topk`/…) to the [`Outer`] -/// shape, independent of what the argument is — so both the flat fast path and -/// the general recursive path share one operator-dispatch. -fn outer_kind(agg: &AggregateExpr) -> Result { - let op = agg.op.id(); - - Ok(if op == token::T_TOPK { - Outer::TopK { - k: count_param(agg)?, - descending: true, - } - } else if op == token::T_BOTTOMK { - Outer::TopK { - k: count_param(agg)?, - descending: false, - } - } else if op == token::T_COUNT { - Outer::Count - } else if op == token::T_SUM { - Outer::Plain(OuterIntent::Sum) - } else if op == token::T_GROUP { - // `group(v)` yields a constant 1 per group (presence), not a sum of - // values — a distinct intent, never folded onto `Sum` (issue #49). - Outer::Plain(OuterIntent::Group) - } else if op == token::T_COUNT_VALUES { - // `count_values("l", v)` groups by sample value and counts, emitting the - // value as a new label `l` (the string parameter) — issue #49. - Outer::CountValues { - label: str_param(agg)?, - } - } else if op == token::T_LIMITK { - // `limitk(k, v)` — up to k series per group (issue #86). - Outer::Sample { - kind: SampleKind::LimitK(count_param(agg)? as usize), - } - } else if op == token::T_LIMIT_RATIO { - // `limit_ratio(r, v)` — an r-fraction of series per group (issue #86). - Outer::Sample { - kind: SampleKind::LimitRatio(ratio_param(agg)?), - } - } else if op == token::T_AVG { - Outer::Plain(OuterIntent::Avg) - } else if op == token::T_MIN { - Outer::Plain(OuterIntent::Min) - } else if op == token::T_MAX { - Outer::Plain(OuterIntent::Max) - } else if op == token::T_STDDEV { - Outer::Plain(OuterIntent::StdDev) - } else if op == token::T_STDVAR { - Outer::Plain(OuterIntent::Variance) - } else if op == token::T_QUANTILE { - Outer::Plain(OuterIntent::Quantile(quantile_param(num_param(agg)?)?)) - } else { - return Err(LoweringError::UnsupportedAggregateOp(format!( - "aggregate token {op}" - ))); - }) -} - -/// Wrap an already-lowered Unresolved sub-DAG in the outer aggregation. This is the -/// general-nesting counterpart to [`build`]: where `build` assembles the -/// two-level shape from a flat [`Inner`], this composes the outer operator over -/// an arbitrary child (`max(sum by (job) (…))`, `sum(a + b)`, …). -/// -/// A heavy-hitter `TopK` is only recognised on the flat `count_over_time` shape -/// (handled in `build`); over a general sub-DAG, `topk`/`bottomk` is a generic -/// order-by-value + limit — the same `Sort{partition_by} → Limit` pair `build` -/// emits for any non-heavy-hitter ranking. -/// Flip the outer `Aggregate` produced for a `without(...)` grouping into the -/// exclusion form. The reducing-aggregation `build` paths place that aggregate -/// at the root; `walk_aggregate` has already rejected the non-aggregate outers -/// (topk/limitk), so a `without` grouping always has an `Aggregate` here (issue -/// #39). A no-op when the modifier was `by`. -/// Flip the outer `Aggregate` produced for a `without(...)` grouping into the -/// exclusion form. A no-op when the modifier was `by`. -/// -/// `reduction_for` (used by [`windowed_aggregate`]/[`outer_aggregate`] to -/// build this node) decides `PerEntity` vs `Reduce(by)` *without* knowing -/// about `without` yet — it only ever sees `by`-mode keys, since `without`'s -/// excluded-labels list is applied here, after the fact, exactly like the -/// pre-#179 legacy relational tree's own `mark_without` did (its -/// converter read `without` only after this front-end step had already set -/// it). Whether -/// `reduction_for` picked `PerEntity` (only possible when `keys` was empty) -/// or `Reduce(by)`, the correct answer under `without(...)` is always -/// `Reduce(without(keys))`: a `without` grouping is never label-preserving — -/// per-entity requires `!by.is_without()` — so this both re-tags an existing -/// `Reduce` and upgrades a wrongly-early `PerEntity` guess, uniformly. -fn mark_without(tree: Unresolved, without: bool) -> Unresolved { - if !without { - return tree; - } - match tree { - Unresolved::Aggregate { - reduction, - measures, - output_names, - filters, - having, - child, - } => { - let keys = match reduction { - Reduction::Reduce(by) => by.keys().to_vec(), - Reduction::PerEntity => vec![], - }; - Unresolved::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(keys)), - measures, - output_names, - filters, - having, - child, - } - } - other => other, - } -} - -fn build_over_sub_dag(outer: Outer, keys: Vec, child: Unresolved) -> Result { - Ok(match outer { - // `walk_aggregate` always passes a real aggregator; `None` can't occur. - Outer::None => child, - Outer::Plain(intent) => outer_aggregate(keys, outer_intent(&intent), child), - Outer::Count => outer_aggregate(keys, count(), child), - Outer::CountValues { label } => { - outer_aggregate(keys, AggIntent::CountValues { label }, child) - } - Outer::Sample { kind } => Unresolved::PromqlSeriesSample { - by: keys.into(), - kind, - child: Rc::new(child), - }, - Outer::TopK { k, descending } => { - let weighted_counter_ranking = matches!( - &child, - Unresolved::Aggregate { - measures, - child: sum_child, - .. - } if matches!(measures.as_slice(), [AggIntent::Sum { .. }]) - && matches!(sum_child.as_ref(), Unresolved::Aggregate { measures, .. } - if matches!(measures.as_slice(), [AggIntent::Rate | AggIntent::Increase])) - ); - let direct_counter_ranking = matches!(&child, Unresolved::Aggregate { - measures, reduction: Reduction::PerEntity, .. - } if matches!(measures.as_slice(), [AggIntent::Rate | AggIntent::Increase])); - if descending && (weighted_counter_ranking || direct_counter_ranking) { - return Ok(outer_aggregate( - keys, - AggIntent::TopK { - k: k as usize, - accuracy: current_accuracy(), - }, - child, - )); - } - ranked_by_value(keys, k, descending, child) - } - }) -} - -/// Generic `topk`/`bottomk`: `Limit{k} → Sort{value, partition_by: keys}` over -/// `child` — an order-by-value ranking, not a heavy-hitter intent. -fn ranked_by_value( - keys: Vec, - k: u64, - descending: bool, - child: Unresolved, -) -> Unresolved { - let sorted = Unresolved::Sort { - keys: vec![UnresolvedSortKey { - expr: Scalar::Column(ColumnRef::SampleValue), - ascending: !descending, - nulls_first: false, - }], - partition_by: keys.into(), - child: Rc::new(child), - }; - Unresolved::Limit { - n: Some(k as usize), - offset: 0, - partition_by: GroupKeys::none(), - child: Rc::new(sorted), - } -} - -/// The `histogram_*` function family (issues #43, histogram_quantile). -/// -/// `histogram_quantile(φ, )` lowers `` in full — preserving any -/// `sum by (le)` / `rate` structure inside it. The classic `le`-bucket form -/// becomes [`classic_histogram_quantile`]; a native histogram or raw samples -/// become a `Quantile` over the whole argument. -/// The native-histogram accessors (`histogram_count`/`sum`/`avg`/`stddev`/ -/// `stdvar`/`fraction`) each extract one float per series, lowering to a -/// per-series `Aggregate{[accessor]}` directly over the (instant) argument. -/// `histogram_fraction(lower, upper, v)` reads its bounds from args 0/1 and the -/// vector from arg 2; the rest take the vector at arg 0. -fn walk_histogram(call: &Call) -> Result { - if call.func.name == "histogram_quantiles" { - return walk_histogram_quantiles(call); - } - if call.func.name == "histogram_quantile" { - let phi = quantile_param(num_arg(call, 0)?)?; - let arg_expr = arg(call, 1)?; - // Two lowerings of `histogram_quantile(φ, …)`: - // - classic `le`-bucket form → `HistogramQuantile`, exact interpolation - // over cumulative buckets (not sketch-able). - // - native-histogram / raw-samples form → the generic `Quantile` intent - // (sketch-able). - // The true signal is the argument's sample type: a declared - // `HistogramKind` (issue #79) drives the choice when available, else we - // fall back to the structural `by (le)`/`_bucket` heuristic (issue #43). - if !histogram_arg_is_sketchable(arg_expr)? { - return Ok(classic_histogram_quantile(phi, "", walk(arg_expr)?)); - } - let func = AggIntent::Quantile { - col: None, - q: phi, - accuracy: current_accuracy(), - }; - return Ok(outer_aggregate(vec![], func, walk(arg_expr)?)); - } - Err(LoweringError::UnsupportedFeature( - "native histogram samples have no IR representation".into(), - )) -} - -/// Classic-bucket `histogram_quantile(φ, child)`. One histogram is the set of -/// series that differ only in `le`, so the aggregate groups `without (le)`. -/// That grouping also seeds `le` into a usage-derived source schema, even -/// when no matcher names it. An empty `output_name` keeps the intent-keyed name. -fn classic_histogram_quantile(q: f64, output_name: &str, child: Unresolved) -> Unresolved { - let le = ColumnRef::Named("le".into()); - Unresolved::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![le.clone()])), - measures: vec![AggIntent::HistogramQuantile { q, le }], - output_names: vec![output_name.into()], - filters: vec![], - having: None, - child: Rc::new(child), - } -} - -/// `histogram_quantiles(v, "label", φ₀, φ₁, …)` — the experimental multi-quantile -/// form (issue #109). It is `histogram_quantile(φᵢ, v)` fanned out over the -/// quantiles, each branch's output series tagged with `label = φᵢ`. -/// -/// Lowers to a `Concat` of one `PromqlRelabel`-wrapped quantile branch per φ, reusing -/// the single-quantile decision — classic `le`-buckets interpolate -/// (`HistogramQuantile`), native histograms / raw samples take the sketch-able -/// `Quantile` (issues #43 / #79) — so the two functions cannot diverge. -/// -/// The vector argument is lowered once per branch, duplicating the sub-DAG — -/// a future workload-level reuse pass could hoist it back into a single -/// producer. -/// -/// Each branch aliases its value column to `value` rather than taking the -/// intent-keyed name (`quantile_0_5`, `quantile_0_9`, …). `Concat` derives its -/// schema from the first child, so branches that disagree on a column *name* -/// would make the merged schema silently misdescribe every branch but one. The -/// quantile is carried by the `label` column, which is exactly where Prometheus -/// puts it. -fn walk_histogram_quantiles(call: &Call) -> Result { - let vec_expr = arg(call, 0)?; - let label = str_arg(call, 1)?; - if call.args.args.len() < 3 { - return Err(LoweringError::MissingArgument( - "histogram_quantiles(v, label, φ…) needs at least one quantile".into(), - )); - } - // The bucket-vs-native choice is a property of the argument, not of φ. - let sketchable = histogram_arg_is_sketchable(vec_expr)?; - let branches = (2..call.args.args.len()) - .map(|i| { - let phi = bounded_quantile_param(num_arg(call, i)?)?; - let child = walk(vec_expr)?; - // Each branch aliases its value column to "value" (not the - // intent-keyed default) so `Concat` — which derives its schema - // from the first branch — doesn't silently misdescribe the rest. - let quantile = if sketchable { - let intent = AggIntent::Quantile { - col: None, - q: phi, - accuracy: current_accuracy(), - }; - Unresolved::Aggregate { - reduction: reduction_for(&[], intent.is_per_series()), - measures: vec![intent], - output_names: vec!["value".into()], - filters: vec![], - having: None, - child: Rc::new(child), - } - } else { - classic_histogram_quantile(phi, "value", child) - }; - Ok(Unresolved::PromqlRelabel { - dst: label.clone(), - value: Scalar::Literal(ScalarValue::Utf8(open_metrics_float(phi))), - child: Rc::new(quantile), - }) - }) - .collect::>>()?; - // No discriminator asserted here today (issue #228): the φ value each - // branch carries via `PromqlRelabel` *is* structurally a distinct - // per-branch discriminator, but nothing downstream currently needs the - // resulting compound unique key — see - // `docs/design_docs/concat-unique-keys-decision.md`. `Unresolved::concat` - // keeps `output_schema`'s default (drop `unique_keys` entirely). - Ok(Unresolved::concat(branches)) -} - -/// Prometheus's `labels.FormatOpenMetricsFloat` — how `histogram_quantiles` -/// renders each φ into its label value. Go's `%g` shortest round-trip, switching -/// to exponent form outside `[1e-4, 1e21)`, with `.0` appended when the result -/// would otherwise look like an integer. -fn open_metrics_float(v: f64) -> String { - // The cases upstream hardcodes. - if v == 1.0 { - return "1.0".into(); - } - if v == 0.0 { - return "0.0".into(); - } - if v == -1.0 { - return "-1.0".into(); - } - if v.is_nan() { - return "NaN".into(); - } - if v.is_infinite() { - return if v.is_sign_positive() { "+Inf" } else { "-Inf" }.into(); - } - let sci = format!("{v:e}"); - let exp: i32 = sci - .split_once('e') - .and_then(|(_, e)| e.parse().ok()) - .unwrap_or(0); - if !(-4..21).contains(&exp) { - // Go writes a signed, zero-padded two-digit exponent: `1e-05`. - let (mantissa, _) = sci.split_once('e').unwrap_or((sci.as_str(), "0")); - let sign = if exp < 0 { '-' } else { '+' }; - return format!("{mantissa}e{sign}{:02}", exp.abs()); - } - let s = format!("{v}"); - if s.contains(['e', '.']) { - s - } else { - format!("{s}.0") - } -} - -/// The calendar functions (issue #46); `time()` is scalar-typed and lowers in -/// `lower_scalar`. -fn is_time_fn(name: &str) -> bool { - matches!( - name, - "timestamp" - | "minute" - | "hour" - | "day_of_week" - | "day_of_month" - | "day_of_year" - | "month" - | "year" - | "days_in_month" - ) -} - -/// `timestamp(v)` and the calendar accessors → `Aggregate{[TimeFn(f)]}` over -/// the argument vector, or over `PromqlVectorFromScalar(EvalTimestamp)` for the -/// no-argument calendar forms (`hour()`, `day_of_week()`, …). Issue #46. -fn walk_time(call: &Call) -> Result { - // timestamp() reads the selected sample's timestamp, not its value. - if call.func.name == "timestamp" { - return Ok(outer_aggregate( - vec![], - AggIntent::TimeFn(TimeFunc::Timestamp), - walk(arg(call, 0)?)?, - )); - } - let child = if call.args.args.is_empty() { - Unresolved::PromqlVectorFromScalar(Scalar::EvalTimestamp) - } else { - walk(arg(call, 0)?)? - }; - Ok(Unresolved::PromqlMap { - child: Rc::new(child), - sample: Scalar::FunctionCall { - name: format!("promql_{}", call.func.name), - args: vec![Scalar::Column(ColumnRef::SampleValue)], - }, - drop_metric_name: true, - }) -} - -/// The presence functions (issue #47). -fn is_presence_fn(name: &str) -> bool { - matches!(name, "absent" | "absent_over_time" | "present_over_time") -} - -/// `absent(v)` / `absent_over_time(m[w])` / `present_over_time(m[w])` — lowered -/// to an `Aggregate{[Absent/…]}` over the (instant or range) argument. The -/// empty-result → synthesized-1-sample logic is a post-ASAP/runtime concern; -/// the canonical tree only marks the operation (issue #47). -fn walk_presence(call: &Call) -> Result { - let func = match call.func.name { - "absent" => AggIntent::Absent, - "absent_over_time" => AggIntent::AbsentOverTime, - "present_over_time" => AggIntent::PresentOverTime, - other => return Err(LoweringError::UnsupportedFunction(other.to_string())), - }; - // arg 0 is the instant vector (`absent`) or range vector (`*_over_time`); - // `walk` produces a `Window` for the matrix-selector forms. - Ok(outer_aggregate(vec![], func, walk(arg(call, 0)?)?)) -} - -/// The scalar→vector conversion (issue #48); `scalar(v)` is scalar-typed and -/// lowers in `lower_scalar`. `info` is *not* here: it is a label-enrichment -/// join, not a type conversion (#84). -fn is_typeconv_fn(name: &str) -> bool { - name == "vector" -} - -/// `vector(s)` — promote a scalar to a label-less instant vector carrying the -/// scalar expression `s` (issue #48). -fn walk_typeconv(call: &Call) -> Result { - Ok(Unresolved::PromqlVectorFromScalar(lower_scalar(arg( - call, 0, - )?)?)) -} - -/// The instant-vector reordering functions (issue #51). -fn is_sort_fn(name: &str) -> bool { - matches!( - name, - "sort" | "sort_desc" | "sort_by_label" | "sort_by_label_desc" - ) -} - -/// `sort`/`sort_desc(v)` reorder an instant vector by sample value; -/// `sort_by_label`/`sort_by_label_desc(v, "l"…)` reorder by label values. All -/// lower to a bare `Sort` (no `Limit`) over the vector argument — a faithful, -/// row-preserving reordering (issue #51). -fn walk_sort(call: &Call) -> Result { - let child = Rc::new(walk(arg(call, 0)?)?); - let (by_value, ascending) = match call.func.name { - "sort" => (true, true), - "sort_desc" => (true, false), - "sort_by_label" => (false, true), - "sort_by_label_desc" => (false, false), - other => return Err(LoweringError::UnsupportedFunction(other.to_string())), - }; - let sort_key = |expr| UnresolvedSortKey { - expr, - ascending, - nulls_first: false, - }; - let keys = if by_value { - vec![sort_key(Scalar::Column(ColumnRef::SampleValue))] - } else { - // `sort_by_label(v, "l1", "l2", …)` — one key per label arg, in order. - if call.args.args.len() < 2 { - return Err(LoweringError::MissingArgument( - "sort_by_label needs at least one label".into(), - )); - } - (1..call.args.args.len()) - .map(|i| { - Ok(sort_key(Scalar::Column(ColumnRef::Named(str_arg( - call, i, - )?)))) - }) - .collect::>>()? - }; - Ok(Unresolved::Sort { - keys, - partition_by: GroupKeys::none(), - child, - }) -} - -/// `info(v, [selector])` — a label-enrichment join. Lowers the input vector and -/// wraps it in an `PromqlInfoEnrich` carrying the (optional) data-label selector's -/// matchers; the actual join against the info metric — on shared identifying -/// labels — is resolved during post-ASAP binding (issue #84). -fn walk_info(call: &Call) -> Result { - let child = Rc::new(walk(arg(call, 0)?)?); - let selector = match call.args.args.get(1) { - Some(sel) => info_selector(sel)?, - None => Vec::new(), // default: enrich from `target_info` - }; - Ok(Unresolved::PromqlInfoEnrich { selector, child }) -} - -/// Extract the `info` data-label selector's matchers. Unlike an ordinary -/// selector these are **info-metric-side** and may carry regex / multiple -/// `__name__` matchers (which pick the info metric(s)), so they bypass the -/// single-metric `vs_parts` restriction and are kept symbolic. -fn info_selector(expr: &Expr) -> Result> { - match expr { - Expr::VectorSelector(vs) => Ok(vs - .matchers - .matchers - .iter() - .map(|m| InfoMatcher { - label: m.name.clone(), - op: match &m.op { - MatchOp::Equal => CompareOpKind::Eq, - MatchOp::NotEqual => CompareOpKind::Ne, - MatchOp::Re(_) => CompareOpKind::Regex, - MatchOp::NotRe(_) => CompareOpKind::NotRegex, - }, - value: m.value.clone(), - }) - .collect()), - Expr::Paren(p) => info_selector(&p.expr), - other => Err(LoweringError::UnsupportedFeature(format!( - "`info` data-label selector must be a label-matcher set, got `{other}`" - ))), - } -} - -/// The label-rewrite functions (issue #50). -fn is_label_fn(name: &str) -> bool { - matches!(name, "label_replace" | "label_join") -} - -/// `label_replace(v, dst, replacement, src, regex)` / -/// `label_join(v, dst, sep, src…)` — per-series label rewrites. Both lower to a -/// `PromqlRelabel` over the fully-lowered vector argument, differing only in the -/// expression that computes the destination label: `label_replace` a regex -/// capture-expansion, `label_join` a separator-joined concatenation. Sample -/// values are untouched; the regex-match-or-passthrough and capture-expansion -/// are post-ASAP/runtime concerns (issue #50). -fn walk_label(call: &Call) -> Result { - let child = Rc::new(walk(arg(call, 0)?)?); - match call.func.name { - "label_replace" => { - let dst = str_arg(call, 1)?; - let replacement = str_arg(call, 2)?; - let src = str_arg(call, 3)?; - let regex = str_arg(call, 4)?; - let value = Scalar::FunctionCall { - name: "label_replace".into(), - args: vec![ - Scalar::Column(ColumnRef::Named(src)), - Scalar::Literal(ScalarValue::Utf8(regex)), - Scalar::Literal(ScalarValue::Utf8(replacement)), - ], - }; - Ok(Unresolved::PromqlRelabel { dst, value, child }) - } - "label_join" => { - // label_join(v, dst, sep, src_1, …, src_n) — needs ≥1 source label. - if call.args.args.len() < 4 { - return Err(LoweringError::MissingArgument( - "label_join(v, dst, sep, src…) needs at least one source label".into(), - )); - } - let dst = str_arg(call, 1)?; - let sep = str_arg(call, 2)?; - let mut args = vec![Scalar::Literal(ScalarValue::Utf8(sep))]; - for i in 3..call.args.args.len() { - args.push(Scalar::Column(ColumnRef::Named(str_arg(call, i)?))); - } - let value = Scalar::FunctionCall { - name: "label_join".into(), - args, - }; - Ok(Unresolved::PromqlRelabel { dst, value, child }) - } - other => Err(LoweringError::UnsupportedFunction(other.to_string())), - } -} - -/// The element-wise math / trig functions (issue #45). -fn is_math_fn(name: &str) -> bool { - matches!( - name, - "abs" - | "ceil" - | "floor" - | "exp" - | "ln" - | "log2" - | "log10" - | "sqrt" - | "sgn" - | "sin" - | "cos" - | "tan" - | "asin" - | "acos" - | "atan" - | "sinh" - | "cosh" - | "tanh" - | "asinh" - | "acosh" - | "atanh" - | "deg" - | "rad" - | "round" - | "clamp" - | "clamp_min" - | "clamp_max" - ) -} - -/// A math / trig function — a per-series element-wise value transform, lowered -/// to a typed scalar projection over the instant-vector argument. -/// `pi()` is scalar-typed and lowers in `lower_scalar` (issue #45). -fn walk_math(call: &Call) -> Result { - let mut args = vec![Scalar::Column(ColumnRef::SampleValue)]; - for index in 1..call.args.args.len() { - args.push(lower_scalar(arg(call, index)?)?); - } - if call.func.name == "round" && args.len() == 1 { - args.push(Scalar::Literal(ScalarValue::Float64(1.0))); - } - Ok(Unresolved::PromqlMap { - child: Rc::new(walk(arg(call, 0)?)?), - sample: Scalar::FunctionCall { - name: format!("promql_{}", call.func.name), - args, - }, - drop_metric_name: true, - }) -} - -/// Whether `expr` is a **classic cumulative-bucket** `histogram_quantile` -/// argument — as opposed to a native histogram or raw samples. Recognised -/// structurally, by any of: -/// - a `by (le)` grouping (`sum by (le) (…)`), -/// - a selector on a classic `_bucket` metric (`http_request_…_bucket`), -/// - a selector with an `le` label matcher (`{le="…"}`). -/// -/// The bucket form must be *interpolated* (`HistogramQuantile`); everything -/// else is a sketch-able generic `Quantile`. This is a heuristic proxy for the -/// real signal — the argument's sample type — which isn't visible at lowering; -/// see the follow-up issue on the discrimination criteria (issue #43). -/// Whether `histogram_quantile(φ, arg)` lowers to the sketch-able generic -/// `Quantile` (`true`) or exact classic-bucket interpolation (`false`). -/// -/// Metadata wins: if any metric referenced in `arg` has a declared -/// [`HistogramKind`](crate::unified::histogram::HistogramKind), that decides it (issue -/// #79) — this fixes both the false-positive (a `…_bucket`-named non-histogram -/// declared `RawSamples`) and the false-negative (a suffix-less classic -/// histogram declared `ClassicBucket`) of the structural heuristic. With no -/// declaration, fall back to the structural `by (le)`/`_bucket` heuristic. -fn histogram_arg_is_sketchable(arg: &Expr) -> Result { - let mut metrics = Vec::new(); - collect_metric_names(arg, &mut metrics); - let kinds = metrics - .iter() - .filter_map(|metric| crate::unified::histogram::current_kind_of(metric)) - .collect::>(); - if kinds.contains(&crate::unified::histogram::HistogramKind::Native) { - return Err(LoweringError::UnsupportedFeature( - "native histogram samples have no IR representation".into(), - )); - } - if let Some(kind) = kinds.first() { - if kinds.iter().any(|other| other != kind) { - return Err(LoweringError::UnsupportedFeature( - "mixed histogram sample contracts".into(), - )); - } - return Ok(kind.is_sketchable()); - } - if is_classic_bucket_arg(arg) { - Ok(false) - } else { - Err(LoweringError::UnsupportedFeature("histogram_quantile requires classic buckets; use quantile for float samples or explicitly declare the RawSamples extension".into())) - } -} - -/// Collect the metric names of every vector/matrix selector reachable in `expr` -/// (for the metadata lookup in [`histogram_arg_is_sketchable`]). Skips -/// name-less selectors like `{le="…"}`. -fn collect_metric_names(expr: &Expr, out: &mut Vec) { - match expr { - Expr::VectorSelector(vs) => { - if let Ok((metric, ..)) = vs_parts(vs) { - if !metric.is_empty() { - out.push(metric); - } - } - } - Expr::MatrixSelector(ms) => { - if let Ok((metric, ..)) = vs_parts(&ms.vs) { - if !metric.is_empty() { - out.push(metric); - } - } - } - Expr::Paren(p) => collect_metric_names(&p.expr, out), - Expr::Unary(u) => collect_metric_names(&u.expr, out), - Expr::Subquery(s) => collect_metric_names(&s.expr, out), - Expr::Aggregate(a) => collect_metric_names(&a.expr, out), - Expr::Binary(b) => { - collect_metric_names(&b.lhs, out); - collect_metric_names(&b.rhs, out); - } - Expr::Call(c) => c - .args - .args - .iter() - .for_each(|a| collect_metric_names(a, out)), - _ => {} - } -} - -fn is_classic_bucket_arg(expr: &Expr) -> bool { - match expr { - Expr::Paren(p) => is_classic_bucket_arg(&p.expr), - Expr::Unary(u) => is_classic_bucket_arg(&u.expr), - Expr::Subquery(s) => is_classic_bucket_arg(&s.expr), - Expr::Aggregate(agg) => { - matches!( - &agg.modifier, - Some(LabelModifier::Include(ls)) if ls.labels.iter().any(|l| l == "le") - ) || is_classic_bucket_arg(&agg.expr) - } - Expr::Binary(b) => is_classic_bucket_arg(&b.lhs) || is_classic_bucket_arg(&b.rhs), - Expr::Call(c) => c.args.args.iter().any(|a| is_classic_bucket_arg(a)), - Expr::VectorSelector(vs) => selector_is_bucket(vs), - Expr::MatrixSelector(ms) => selector_is_bucket(&ms.vs), - _ => false, - } -} - -/// A classic histogram bucket selector — a `_bucket`-named metric (via bare name -/// or `__name__` matcher) or an explicit `le` label matcher. -fn selector_is_bucket(vs: &VectorSelector) -> bool { - let name = vs.name.as_deref().or_else(|| { - vs.matchers - .matchers - .iter() - .find(|m| m.name == "__name__") - .map(|m| m.value.as_str()) - }); - name.is_some_and(|n| n.ends_with("_bucket")) - || vs.matchers.matchers.iter().any(|m| m.name == "le") -} - -/// A binary op with at least one vector operand (a scalar/scalar op is -/// scalar-typed and never reaches here). A scalar side lowers to a -/// scalar expression; mixed operations resolve to Project or Filter. -fn walk_binary(bin: &BinaryExpr) -> Result { - let op = binop(bin.op.id())?; - let scalar_left = bin.lhs.value_type() == ValueType::Scalar; - if scalar_left || bin.rhs.value_type() == ValueType::Scalar { - let (scalar, vector) = if scalar_left { - (&bin.lhs, &bin.rhs) - } else { - (&bin.rhs, &bin.lhs) - }; - return Ok(Unresolved::PromqlScalarOp { - child: Rc::new(walk(vector)?), - scalar: lower_scalar(scalar)?, - op, - scalar_left, - return_bool: bin.return_bool(), - }); - } - let lhs = walk(&bin.lhs)?; - let rhs = walk(&bin.rhs)?; - // `VectorMatch` has no fill field; dropping fill would change which series - // are emitted and their values, so the query must fall back to exact - // execution instead. - if let Some(m) = &bin.modifier { - if m.fill_values.lhs.is_some() || m.fill_values.rhs.is_some() { - return Err(LoweringError::UnsupportedFeature(format!( - "`fill` vector-matching modifier: `{bin}`" - ))); - } - } - let vector_match = bin.modifier.as_ref().map(|m| { - let (kind, labels) = match &m.matching { - Some(LabelModifier::Include(ls)) => (VectorMatchKind::On, ls.labels.clone()), - Some(LabelModifier::Exclude(ls)) => (VectorMatchKind::Ignoring, ls.labels.clone()), - // No explicit `on(…)`/`ignoring(…)` — the parser attaches a default - // modifier to every set op (`and`/`or`/`unless`). The default is - // "match on all shared labels", which is exactly `ignoring([])` - // (ignore no labels). Representing it as `Ignoring([])` — not - // `On([])` — keeps it distinct from an explicit `on()` (match on the - // empty label set) while making it correctly equal to an explicit - // `ignoring()` (issue #68). - None => (VectorMatchKind::Ignoring, vec![]), - }; - let grouping = match &m.card { - VectorMatchCardinality::ManyToOne(ls) => Some(VectorGrouping { - side: GroupSide::Left, - labels: ls.labels.clone(), - }), - VectorMatchCardinality::OneToMany(ls) => Some(VectorGrouping { - side: GroupSide::Right, - labels: ls.labels.clone(), - }), - _ => None, - }; - VectorMatch { - kind, - labels, - grouping, - } - }); - Ok(vector_binary(op, vector_match, bin.return_bool(), lhs, rhs)) -} - -fn lower_inner(expr: &Expr) -> Result { - match expr { - Expr::VectorSelector(vs) => { - let (metric, matchers, shift) = vs_parts(vs)?; - Ok(Inner { - metric, - matchers, - window: None, - func: None, - shift, - }) - } - Expr::MatrixSelector(ms) => { - let (metric, matchers, shift) = vs_parts(&ms.vs)?; - Ok(Inner { - metric, - matchers, - window: Some(ms.range), - func: None, - shift, - }) - } - Expr::Paren(p) => lower_inner(&p.expr), - Expr::Call(call) => lower_inner_call(call), - other => Err(LoweringError::UnsupportedFeature(format!( - "aggregate argument: `{other}`" - ))), - } -} - -fn lower_inner_call(call: &Call) -> Result { - let name = call.func.name; - let at0 = |func: InnerFunc| -> Result { - let (metric, matchers, window, shift) = extract_matrix(arg(call, 0)?)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(func), - shift, - }) - }; - match name { - "rate" | "irate" => { - let (metric, matchers, window, shift) = extract_matrix(arg(call, 0)?)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(if name == "irate" { - InnerFunc::IRate - } else { - InnerFunc::Rate - }), - shift, - }) - } - "increase" => { - let (metric, matchers, window, shift) = extract_matrix(arg(call, 0)?)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(InnerFunc::Increase), - shift, - }) - } - "quantile_over_time" => { - let phi = quantile_param(num_arg(call, 0)?)?; - let (metric, matchers, window, shift) = extract_matrix(arg(call, 1)?)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(InnerFunc::Quantile(phi)), - shift, - }) - } - "avg_over_time" => at0(InnerFunc::Avg), - "min_over_time" => at0(InnerFunc::Min), - "max_over_time" => at0(InnerFunc::Max), - "sum_over_time" => at0(InnerFunc::Sum), - "stddev_over_time" => at0(InnerFunc::StdDev), - "stdvar_over_time" => at0(InnerFunc::Variance), - "count_over_time" => at0(InnerFunc::Count), - "distinct_over_time" => at0(InnerFunc::Cardinality), - "entropy_over_time" => at0(InnerFunc::FrequencyEntropy), - "l2_over_time" => at0(InnerFunc::FrequencyL2), - // Counter-derivative range functions (issue #44). Each has its own - // intent — `changes` (value-change count) and `resets` (counter-reset - // count) are NOT sample counts, so they are not aliased to - // `count_over_time`. The window is arg 0's matrix; scalar params follow. - "changes" => at0(InnerFunc::Changes), - "delta" => at0(InnerFunc::Delta), - "idelta" => at0(InnerFunc::IDelta), - "deriv" => at0(InnerFunc::Deriv), - "resets" => at0(InnerFunc::Resets), - // Additional range-vector reducers (issue #51) — same windowed - // per-series shape as the `*_over_time` family above. - "last_over_time" => at0(InnerFunc::LastOverTime), - "first_over_time" => at0(InnerFunc::FirstOverTime), - "mad_over_time" => at0(InnerFunc::MadOverTime), - "ts_of_min_over_time" => at0(InnerFunc::TsOfMinOverTime), - "ts_of_max_over_time" => at0(InnerFunc::TsOfMaxOverTime), - "ts_of_first_over_time" => at0(InnerFunc::TsOfFirstOverTime), - "ts_of_last_over_time" => at0(InnerFunc::TsOfLastOverTime), - "predict_linear" => { - let (metric, matchers, window, shift) = extract_matrix(arg(call, 0)?)?; - let seconds = num_arg(call, 1)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(InnerFunc::PredictLinear(seconds)), - shift, - }) - } - "double_exponential_smoothing" => { - let (metric, matchers, window, shift) = extract_matrix(arg(call, 0)?)?; - let smoothing = num_arg(call, 1)?; - let trend = num_arg(call, 2)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(InnerFunc::DoubleExp { smoothing, trend }), - shift, - }) - } - other => Err(LoweringError::UnsupportedFunction(other.to_string())), - } -} - -/// Assemble the Layer-2 tree from a lowered inner vector, the resolved group -/// keys, and the enclosing aggregator shape. -fn build(inner: Inner, keys: Vec, outer: Outer) -> Result { - match outer { - Outer::None => match &inner.func { - None => Ok(instant_source(inner.metric, inner.matchers, inner.shift)), - Some(f) => { - let intent = inner_intent(f); - Ok(windowed_aggregate(inner, keys, intent)) - } - }, - // An OUTER aggregation operator (`sum`/`avg`/…/`count`) over an inner - // range-vector function (`rate`/`increase`/`*_over_time`) is a - // two-level reduction: the inner func runs per series, the outer op - // then aggregates across series. Collapsing them into one aggregate - // silently drops a level — e.g. `sum(rate(m[w]))` must keep the `sum`. - Outer::Plain(intent) => Ok(match &inner.func { - None => windowed_aggregate(inner, keys, outer_intent(&intent)), - Some(f) => { - let inner_i = inner_intent(f); - let inner_agg = windowed_aggregate(inner, vec![], inner_i); - outer_aggregate(keys, outer_intent(&intent), inner_agg) - } - }), - Outer::Count => Ok(match &inner.func { - None => windowed_aggregate(inner, keys, count()), - Some(f) => { - let inner_i = inner_intent(f); - let inner_agg = windowed_aggregate(inner, vec![], inner_i); - outer_aggregate(keys, count(), inner_agg) - } - }), - Outer::CountValues { label } => Ok(match &inner.func { - None => windowed_aggregate(inner, keys, AggIntent::CountValues { label }), - Some(f) => { - let inner_i = inner_intent(f); - let inner_agg = windowed_aggregate(inner, vec![], inner_i); - outer_aggregate(keys, AggIntent::CountValues { label }, inner_agg) - } - }), - Outer::Sample { kind } => { - // Series sampling selects whole series unchanged — like generic - // `topk`, a range-vector argument reduces per series first (label- - // preserving), a bare selector is sampled directly; neither is - // wrapped in a reducing aggregate (issue #86). - let base = match inner.func.as_ref().map(inner_intent) { - Some(intent) => windowed_aggregate(inner, vec![], intent), - None => instant_source(inner.metric, inner.matchers, inner.shift), - }; - Ok(Unresolved::PromqlSeriesSample { - by: keys.into(), - kind, - child: Rc::new(base), - }) - } - Outer::TopK { k, descending } => { - // Preserve the counter-value ranking intent. Physical candidates - // may rebuild a heap over finalized rates or use exact Sort/Limit; - // neither is allowed to sum raw counter samples as ranking weights. - if descending && matches!(inner.func, Some(InnerFunc::Rate | InnerFunc::Increase)) { - let intent = inner_intent(inner.func.as_ref().expect("counter function")); - let ranked = windowed_aggregate(inner, vec![], intent); - return Ok(Unresolved::Aggregate { - reduction: Reduction::Reduce(keys.into()), - measures: vec![AggIntent::TopK { - k: k as usize, - accuracy: current_accuracy(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(ranked), - }); - } - // Heavy-hitter only when ranking by an additive measure (`count` - // or `sum`): that is a - // first-class aggregate intent → `TopK`. Any other ranking (topk - // over avg/quantile, a bare selector's raw value, all bottomk) - // is a generic order-by-value + limit and stays as the `Sort + Limit` - // operator pair. The descending-plus-measure rule is shared with the - // canonicalize-pass promotion so the two cannot drift (issue #38). - let measure = match inner.func { - Some(InnerFunc::Count) => topk::Ranking::Frequency, - Some(InnerFunc::Sum) => topk::Ranking::WeightedSum, - _ => topk::Ranking::NonAdditive, - }; - let additive_ranking = measure.is_supported(descending); - if additive_ranking { - // Preserve the ranked aggregate intent in the canonical tree so the - // intent algebra is explicit about what is being computed. - // Post-ASAP binding may fuse the Count and TopK into a - // single-pass heavy-hitter sketch (SpaceSaving / - // CMS-with-heap), but that is a cost-model decision, not a - // canonical-IR concern. - let ranked = match measure { - topk::Ranking::Frequency => InnerFunc::Count, - topk::Ranking::WeightedSum => InnerFunc::Sum, - topk::Ranking::NonAdditive => { - unreachable!("heavy-hitter gate rejected non-additive ranking") - } - }; - let ranked_agg = windowed_aggregate(inner, vec![], inner_intent(&ranked)); - Ok(Unresolved::Aggregate { - // A ranking always reduces (a `by`-empty TopK ranks the - // whole input into one ordering, never per-entity). - reduction: Reduction::Reduce(keys.into()), - measures: vec![AggIntent::TopK { - k: k as usize, - accuracy: current_accuracy(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(ranked_agg), - }) - } else { - // The base over which we rank. A range-vector-function argument - // (`topk(k, rate(m[5m]))`) reduces *per series* first — that is - // label-preserving, so the `by (host)` partition labels survive. - // A **bare instant selector** (`topk(k, m)`) ranks its own - // samples directly: it must NOT be wrapped in a reducing - // aggregate. Defaulting it to `Sum` was both semantically wrong - // (PromQL `topk` ranks the raw samples, it does not sum them) and - // destructive — the cross-series `Sum` collapses every label, - // including the `by (…)` partition keys, so they no longer - // resolve (issue #30). Keep the selector label-preserving so - // `Sort.partition_by` can rank within each group (issue #12). - let base = match inner.func.as_ref().map(inner_intent) { - Some(intent) => windowed_aggregate(inner, vec![], intent), - None => instant_source(inner.metric, inner.matchers, inner.shift), - }; - Ok(ranked_by_value(keys, k, descending, base)) - } - } - } -} - -/// Decide `PerEntity` vs `Reduce(by)` for a canonical `Aggregate`, entirely -/// from local PromQL semantics: the keys and whether this operation preserves -/// each input series. It never infers entity reduction from the child tree's -/// temporal shape. `without()` is applied -/// separately, post-hoc, by `mark_without` — see its doc for why that's still -/// correct here. -fn reduction_for(keys: &[ColumnRef], per_entity: bool) -> Reduction { - if keys.is_empty() && per_entity { - Reduction::PerEntity - } else { - Reduction::Reduce(GroupKeys::by(keys.to_vec())) - } -} - -/// `Aggregate{reduction, [intent]}` over `[TimeRange{w}] → Scan`. Always wraps -/// in `TimeRange` when there's a window — including for `Rate`/`Increase`, -/// whose window rides on `inner.window` too (set redundantly alongside the -/// intent itself): canonical `AggIntent::Rate`/`Increase` carry no window -/// field of their own, unlike the old Unresolved `AggFunc::Rate{window}` — "the range -/// is on the enclosing `TimeRange` node" is now true unconditionally, so -/// there's no more `skip_window` special case. -fn windowed_aggregate( - inner: Inner, - keys: Vec, - intent: AggIntent, -) -> Unresolved { - let base = filtered_source(inner.metric, inner.matchers, inner.shift); - let child = match inner.window { - Some(w) => Unresolved::TimeRange { - range: w, - kind: TimeRangeKind::Range, - child: Rc::new(base), - }, - None => ingestion_lookback(base), - }; - let reduction = reduction_for(&keys, inner.window.is_some() || intent.is_per_series()); - Unresolved::Aggregate { - reduction, - measures: vec![intent], - // A single empty entry — never an override — so the resolver keeps - // PromQL's intent-keyed output names ("sum", "quantile_0_99", …) - // instead. - output_names: vec![String::new()], - filters: vec![], - having: None, - child: Rc::new(child), - } -} - -/// `Aggregate{reduction, [intent]}` directly over an existing Unresolved sub-DAG — the -/// OUTER level of a two-level aggregation such as `sum(rate(…))` or the -/// `Aggregate{[Quantile]}` that wraps a `histogram_quantile` argument. -fn outer_aggregate( - keys: Vec, - intent: AggIntent, - child: Unresolved, -) -> Unresolved { - let reduction = reduction_for(&keys, intent.is_per_series()); - Unresolved::Aggregate { - reduction, - measures: vec![intent], - output_names: vec![String::new()], - filters: vec![], - having: None, - child: Rc::new(child), - } -} - -/// A temporal range function over a subquery consumes each series' subquery -/// samples independently. Unlike an ordinary outer aggregate, this cannot be -/// inferred from the intent: `max` is cross-series in `max(v)`, but per-series -/// in `max_over_time(v[...])`. -fn per_series_aggregate( - keys: Vec, - intent: AggIntent, - child: Unresolved, -) -> Unresolved { - let reduction = reduction_for(&keys, true); - Unresolved::Aggregate { - reduction, - measures: vec![intent], - output_names: vec![String::new()], - filters: vec![], - having: None, - child: Rc::new(child), - } -} - -fn filtered_source(metric: String, matchers: Vec, shift: TimeShift) -> Unresolved { - let scan = Unresolved::Scan { - source: Source::TimeSeries { metric }, - predicates: matchers.into_iter().map(UnresolvedPredicate).collect(), - // Usage-derived (PromQL is schemaless) — the SchemaResolver fills this in. - schema: None, - }; - if shift.is_identity() { - scan - } else { - Unresolved::TimeShift { - shift, - child: Rc::new(scan), - } - } -} - -/// An instant selector: the latest sample per series within the workload's -/// ingestion interval, so the lookback is an `Instant` `TimeRange`. -fn instant_source(metric: String, matchers: Vec, shift: TimeShift) -> Unresolved { - ingestion_lookback(filtered_source(metric, matchers, shift)) -} - -fn ingestion_lookback(child: Unresolved) -> Unresolved { - Unresolved::TimeRange { - range: current_ingestion_interval(), - kind: TimeRangeKind::Instant, - child: Rc::new(child), - } -} - -/// Count vector elements regardless of their sample values. -fn count() -> AggIntent { - AggIntent::Count { - accuracy: current_accuracy(), - } -} - -fn inner_intent(f: &InnerFunc) -> AggIntent { - match f { - InnerFunc::FrequencyL2 => AggIntent::FrequencyL2 { - col: None, - accuracy: current_accuracy(), - }, - InnerFunc::FrequencyEntropy => AggIntent::FrequencyEntropy { - col: None, - accuracy: current_accuracy(), - }, - InnerFunc::Cardinality => AggIntent::Cardinality { - cols: vec![], - accuracy: current_accuracy(), - }, - InnerFunc::Quantile(q) => AggIntent::Quantile { - col: None, - q: *q, - accuracy: current_accuracy(), - }, - InnerFunc::Avg => AggIntent::Avg { col: None }, - InnerFunc::Min => AggIntent::Min { col: None }, - InnerFunc::Max => AggIntent::Max { col: None }, - InnerFunc::Sum => AggIntent::Sum { col: None }, - InnerFunc::StdDev => AggIntent::StdDev { - col: None, - population: true, - }, - InnerFunc::Variance => AggIntent::Variance { - col: None, - population: true, - }, - InnerFunc::Count => AggIntent::Count { - accuracy: current_accuracy(), - }, - InnerFunc::Rate => AggIntent::Rate, - InnerFunc::IRate => AggIntent::IRate, - InnerFunc::Increase => AggIntent::Increase, - InnerFunc::Changes => AggIntent::Changes, - InnerFunc::Delta => AggIntent::Delta, - InnerFunc::IDelta => AggIntent::IDelta, - InnerFunc::Deriv => AggIntent::Deriv, - InnerFunc::Resets => AggIntent::Resets, - InnerFunc::PredictLinear(s) => AggIntent::PredictLinear { seconds: *s }, - InnerFunc::DoubleExp { smoothing, trend } => AggIntent::DoubleExpSmoothing { - smoothing: *smoothing, - trend: *trend, - }, - InnerFunc::LastOverTime => AggIntent::LastOverTime, - InnerFunc::FirstOverTime => AggIntent::FirstOverTime, - InnerFunc::MadOverTime => AggIntent::MadOverTime, - InnerFunc::TsOfMinOverTime => AggIntent::TsOfMinOverTime, - InnerFunc::TsOfMaxOverTime => AggIntent::TsOfMaxOverTime, - InnerFunc::TsOfFirstOverTime => AggIntent::TsOfFirstOverTime, - InnerFunc::TsOfLastOverTime => AggIntent::TsOfLastOverTime, - } -} - -fn outer_intent(o: &OuterIntent) -> AggIntent { - match o { - OuterIntent::Sum => AggIntent::Sum { col: None }, - OuterIntent::Avg => AggIntent::Avg { col: None }, - OuterIntent::Min => AggIntent::Min { col: None }, - OuterIntent::Max => AggIntent::Max { col: None }, - OuterIntent::StdDev => AggIntent::StdDev { - col: None, - population: true, - }, - OuterIntent::Variance => AggIntent::Variance { - col: None, - population: true, - }, - OuterIntent::Quantile(q) => AggIntent::Quantile { - col: None, - q: *q, - accuracy: current_accuracy(), - }, - OuterIntent::Group => AggIntent::Group, - } -} - -/// Unwrap a (possibly parenthesised) string literal — `count_values` labels and -/// `label_replace`/`label_join` arguments are all string literals, sometimes -/// wrapped in parens (`count_values((("v")), …)`). -fn expr_str(expr: &Expr) -> Result { - match expr { - Expr::StringLiteral(s) => Ok(s.val.clone()), - Expr::Paren(p) => expr_str(&p.expr), - other => Err(LoweringError::InvalidParameter(format!( - "expected a string literal, got `{other}`" - ))), - } -} - -/// A `count_values` string parameter (the synthesized label name). -fn str_param(agg: &AggregateExpr) -> Result { - match &agg.param { - Some(e) => expr_str(e), - None => Err(LoweringError::MissingArgument( - "`count_values` label parameter".into(), - )), - } -} - -/// A call's `idx`-th argument as a string literal (`label_replace`/`label_join`). -fn str_arg(call: &Call, idx: usize) -> Result { - expr_str(arg(call, idx)?) -} - -/// Resolve an aggregation's grouping modifier into a `(keys, without)` pair. -/// -/// `by(labels)` → the kept labels, `without = false`. `without(labels)` → the -/// **excluded** labels, `without = true`: the kept set (the complement) can't be -/// enumerated under an open usage-derived schema, so it is deferred to the -/// runtime and only the excluded positions are carried (issue #39). Both forms -/// canonicalise their label set (sort + dedup) so equivalent groupings lower -/// identically. PromQL labels have no table qualifier → `ColumnRef::Named`. -fn resolve_group(agg: &AggregateExpr) -> Result<(Vec, bool)> { - let canon = |labels: &[String]| -> Vec { - let mut keys = labels.to_vec(); - keys.sort(); - keys.dedup(); - keys.into_iter().map(ColumnRef::Named).collect() - }; - match &agg.modifier { - None => Ok((vec![], false)), - Some(LabelModifier::Include(ls)) => Ok((canon(&ls.labels), false)), - Some(LabelModifier::Exclude(ls)) => Ok((canon(&ls.labels), true)), - } -} - -// ── Free helpers ────────────────────────────────────────────────────────────── - -fn vs_parts(vs: &VectorSelector) -> Result<(String, Vec, TimeShift)> { - // A non-equality `__name__` matcher (`=~` / `!~` / `!=`) selects *across* - // metric names. `Source::TimeSeries { metric }` carries a single concrete - // metric name, so there is no representation for a regex/negated name - // match — reject rather than mislower it to a literal metric named after - // the pattern (issue #67). An equality `__name__` (`{__name__="up"}`) - // still names the metric below. - if let Some(m) = vs - .matchers - .matchers - .iter() - .find(|m| m.name == "__name__" && !matches!(m.op, MatchOp::Equal)) - { - return Err(LoweringError::UnsupportedFeature(format!( - "non-equality `__name__` matcher ({}{:?}) selects across metric names, \ - which has no single-metric canonical representation", - m.name, m.op - ))); - } - let metric = vs.name.clone().unwrap_or_else(|| { - vs.matchers - .matchers - .iter() - .find(|m| m.name == "__name__") - .map(|m| m.value.clone()) - .unwrap_or_default() - }); - // Label matchers are an unordered set: `{a="1",b="2"}` and `{b="2",a="1"}` - // select the same series. Canonicalise by (name, value) so equivalent - // selectors lower to identical predicates. - let mut ms: Vec<&Matcher> = vs - .matchers - .matchers - .iter() - .filter(|m| m.name != "__name__") - .collect(); - ms.sort_by(|a, b| a.name.cmp(&b.name).then_with(|| a.value.cmp(&b.value))); - let matchers = ms.into_iter().map(matcher_to_compare).collect(); - let shift = time_shift(vs.offset.as_ref(), vs.at.as_ref())?; - Ok((metric, matchers, shift)) -} - -/// Convert the parser's `offset` / `@` modifiers into a [`TimeShift`] (issue -/// #40). Offset is signed milliseconds; `@ ` (parser seconds → ms) becomes -/// an absolute anchor, `@ start()`/`@ end()` the range bounds. -fn time_shift(offset: Option<&Offset>, at: Option<&ParserAtModifier>) -> Result { - let offset_ms = match offset { - None => 0, - Some(Offset::Pos(d)) => duration_ms(*d)?, - Some(Offset::Neg(d)) => -duration_ms(*d)?, - }; - let at = match at { - None => None, - Some(ParserAtModifier::Start) => Some(AtModifier::Start), - Some(ParserAtModifier::End) => Some(AtModifier::End), - Some(ParserAtModifier::At(t)) => Some(AtModifier::Timestamp(system_time_ms(*t)?)), - }; - Ok(TimeShift { offset_ms, at }) -} - -/// A `Duration` as `i64` milliseconds, rejecting an overflow rather than -/// silently truncating a pathologically large `offset`. -fn duration_ms(d: Duration) -> Result { - i64::try_from(d.as_millis()).map_err(|_| { - LoweringError::InvalidParameter("offset duration overflows i64 milliseconds".into()) - }) -} - -/// A `SystemTime` (`@ `) as `i64` milliseconds since the Unix epoch, signed -/// so pre-epoch anchors (the parser permits them) are preserved. -fn system_time_ms(t: SystemTime) -> Result { - let ms = match t.duration_since(std::time::UNIX_EPOCH) { - Ok(d) => i64::try_from(d.as_millis()), - Err(e) => i64::try_from(e.duration().as_millis()).map(|ms| -ms), - }; - ms.map_err(|_| { - LoweringError::InvalidParameter("`@` timestamp overflows i64 milliseconds".into()) - }) -} - -fn matcher_to_compare(m: &Matcher) -> Scalar { - let op = match &m.op { - MatchOp::Equal => CompareOpKind::Eq, - MatchOp::NotEqual => CompareOpKind::Ne, - MatchOp::Re(_) => CompareOpKind::Regex, - MatchOp::NotRe(_) => CompareOpKind::NotRegex, - }; - Scalar::Compare { - left: Box::new(Scalar::Column(ColumnRef::Named(m.name.clone()))), - op, - right: Box::new(Scalar::Literal(ScalarValue::Utf8(m.value.clone()))), - semantics: PROMQL, - } -} - -fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration, TimeShift)> { - match expr { - Expr::MatrixSelector(ms) => { - let (metric, matchers, shift) = vs_parts(&ms.vs)?; - Ok((metric, matchers, ms.range, shift)) - } - Expr::Paren(p) => extract_matrix(&p.expr), - // A range-vector function argument must be a (parenthesised) matrix - // selector. Do NOT descend through an arbitrary `Call` — that would - // silently strip an unsupported wrapper (`rate(deriv(m[5m]))` lowering - // as `rate(m[5m])`). Reject instead. - other => Err(LoweringError::UnsupportedFeature(format!( - "expected a range-vector (matrix) argument, got `{other}`" - ))), - } -} - -fn arg(call: &Call, idx: usize) -> Result<&Expr> { - call.args - .args - .get(idx) - .map(|b| b.as_ref()) - .ok_or_else(|| LoweringError::MissingArgument(format!("{} arg #{idx}", call.func.name))) -} - -fn num_arg(call: &Call, idx: usize) -> Result { - num_expr(arg(call, idx)?) -} - -fn num_param(agg: &AggregateExpr) -> Result { - match &agg.param { - Some(e) => num_expr(e), - None => Err(LoweringError::MissingArgument( - "aggregate parameter (k / φ)".into(), - )), - } -} - -fn num_expr(expr: &Expr) -> Result { - match expr { - Expr::NumberLiteral(n) => Ok(n.val), - Expr::Paren(p) => num_expr(&p.expr), - Expr::Unary(u) => Ok(-num_expr(&u.expr)?), - // Constant-fold a pure scalar arithmetic expression — the parser does - // not fold `10*1024*1024` / `24 * 3600`. A `modifier` (vector matching) - // or a non-arithmetic operator means it is not a pure scalar. - Expr::Binary(b) if b.modifier.is_none() => { - let (l, r) = (num_expr(&b.lhs)?, num_expr(&b.rhs)?); - let id = b.op.id(); - if id == token::T_ADD { - Ok(l + r) - } else if id == token::T_SUB { - Ok(l - r) - } else if id == token::T_MUL { - Ok(l * r) - } else if id == token::T_DIV { - Ok(l / r) - } else if id == token::T_MOD { - Ok(l % r) - } else if id == token::T_POW { - Ok(l.powf(r)) - } else { - Err(LoweringError::InvalidParameter( - "non-arithmetic operator in scalar expression".into(), - )) - } - } - // `min_of`/`max_of` are n-ary *scalar* reducers (issue #89). Fold them - // when every argument is itself a constant scalar — this is the only - // form the intent algebra can hold (there is no scalar min/max node). A - // non-constant argument (`min_of(step(), 1s)`) fails the recursive fold - // and propagates the error, so it stays rejected. `f64::min`/`max` - // ignore NaN, matching PromQL's `min`/`max` NaN semantics. - Expr::Call(c) if is_scalar_reducer_fn(c.func.name) => { - let reduce = if c.func.name == "min_of" { - f64::min - } else { - f64::max - }; - c.args - .args - .iter() - .map(|a| num_expr(a)) - .reduce(|acc, v| Ok(reduce(acc?, v?))) - .ok_or_else(|| { - LoweringError::MissingArgument(format!("{} needs an argument", c.func.name)) - })? - } - other => Err(LoweringError::InvalidParameter(format!( - "expected a numeric scalar, got `{other}`" - ))), - } -} - -/// The n-ary scalar min/max reducers, foldable when all arguments are constant -/// scalars (issue #89). -fn is_scalar_reducer_fn(name: &str) -> bool { - matches!(name, "min_of" | "max_of") -} - -/// `topk`/`bottomk` count parameter — a non-negative integer. Rejects -/// fractional / negative / non-finite values rather than silently truncating -/// or saturating them via `as u64` (`topk(2.7, …)` ≠ `topk(2, …)`). -fn count_param(agg: &AggregateExpr) -> Result { - let v = num_param(agg)?; - if v.is_finite() && v >= 0.0 && v.fract() == 0.0 && v <= u64::MAX as f64 { - Ok(v as u64) - } else { - Err(LoweringError::InvalidParameter(format!( - "topk/bottomk k must be a non-negative integer, got {v}" - ))) - } -} - -/// `limit_ratio` ratio parameter — a finite value; Prometheus clamps it to -/// `[-1, 1]` (a negative ratio selects the complementary fraction). A non-finite -/// ratio (`limit_ratio(NaN, …)`) or a dynamic one (`time() % 17/17`, which -/// `num_param` can't fold) is rejected (issue #86). -fn ratio_param(agg: &AggregateExpr) -> Result { - let r = num_param(agg)?; - if !r.is_finite() { - return Err(LoweringError::InvalidParameter(format!( - "limit_ratio ratio must be finite, got {r}" - ))); - } - Ok(r.clamp(-1.0, 1.0)) -} - -/// Preserve the full Prometheus quantile parameter domain, including special values. -fn quantile_param(q: f64) -> Result { - // Prometheus returns NaN/-Inf/+Inf for these parameters at execution time. - Ok(q) -} - -// The non-standard histogram_quantiles extension keeps its bounded label contract. -fn bounded_quantile_param(q: f64) -> Result { - if q.is_finite() && (0.0..=1.0).contains(&q) { - Ok(q) - } else { - Err(LoweringError::InvalidParameter(format!( - "quantile φ must be in [0, 1], got {q}" - ))) - } -} - -fn binop(id: token::TokenId) -> Result { - Ok(if id == token::T_ADD { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Add) - } else if id == token::T_SUB { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Sub) - } else if id == token::T_MUL { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul) - } else if id == token::T_DIV { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Div) - } else if id == token::T_MOD { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Mod) - } else if id == token::T_POW { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Pow) - } else if id == token::T_ATAN2 { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Atan2) - } else if id == token::T_EQLC { - BinaryOpKind::Compare(CompareOpKind::Eq) - } else if id == token::T_NEQ { - BinaryOpKind::Compare(CompareOpKind::Ne) - } else if id == token::T_LSS { - BinaryOpKind::Compare(CompareOpKind::Lt) - } else if id == token::T_LTE { - BinaryOpKind::Compare(CompareOpKind::Le) - } else if id == token::T_GTR { - BinaryOpKind::Compare(CompareOpKind::Gt) - } else if id == token::T_GTE { - BinaryOpKind::Compare(CompareOpKind::Ge) - } else if id == token::T_LAND { - BinaryOpKind::Set(PromQLVectorSetOpKind::And) - } else if id == token::T_LOR { - BinaryOpKind::Set(PromQLVectorSetOpKind::Or) - } else if id == token::T_LUNLESS { - BinaryOpKind::Set(PromQLVectorSetOpKind::Unless) - } else { - return Err(LoweringError::UnsupportedFeature(format!( - "binary operator token {id}" - ))); - }) -} diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index bba2656f2..c095eae2d 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -69,18 +69,18 @@ pub fn promql_scalar(node: &ScalarExpr) -> Option { } } -/// Time `root` under the default (every summary maintained) lifecycle -/// assignment and export the post-ASAP DAG — the wire-6 export needs every -/// node timed first. +/// Time `root` under the default materialization assignment (every summary +/// at query time) and export the physical DAG — export needs every node +/// timed first. #[allow(dead_code)] pub fn post_asap_dag(root: &Rc) -> asap_types::ir::export::PhysicalASAPDAG { - use asap_types::ir::{apply_lifecycle_timings, LifecycleAssignment, TimingMemo}; - let timed = apply_lifecycle_timings( + use asap_types::ir::{apply_materialization_timings, MaterializationAssignment, TimingMemo}; + let timed = apply_materialization_timings( root, - &LifecycleAssignment::default_maintained(), + &MaterializationAssignment::all_query_time(), &mut TimingMemo::new(), ) - .expect("default lifecycle timings"); + .expect("default materialization timings"); asap_types::ir::export::compile_physical_asap_dag(&timed).expect("post-ASAP DAG export") } diff --git a/crates/frontend-sql/src/unified/error.rs b/crates/frontend-sql/src/unified/error.rs deleted file mode 100644 index f819cfd46..000000000 --- a/crates/frontend-sql/src/unified/error.rs +++ /dev/null @@ -1,63 +0,0 @@ -use std::fmt; - -use asap_frontend_common::ResolveDAGError; - -/// Errors from lowering a SQL query (parse + plan via DataFusion → the -/// name-based [`UnresolvedOp`](asap_frontend_common::UnresolvedOp) tree → -/// [`resolve_root`](asap_frontend_common::resolve_root) binds it into the -/// unified IR). -/// -/// Carries no PromQL type — the SQL front end never depends on the PromQL -/// parser. The language-neutral variants (`UnsupportedFeature` / `WrongLanguage` -/// / `Convert`) are mirrored by [`asap_frontend_promql::PromqlError`] rather -/// than shared, so neither front end pulls the other's parser. -#[derive(Debug)] -pub enum SqlError { - /// DataFusion failed to parse / plan the SQL query. - DataFusion(datafusion::error::DataFusionError), - /// A table referenced by the query is absent from the catalog. - TableNotFound(String), - /// A SQL aggregate function not supported in this version. - UnsupportedAggregate(String), - /// A SQL scalar expression that could not be lowered. - InvalidExpression(String), - /// The SQL dialect is not supported (only DataFusionSQL is implemented). - UnsupportedDialect(String), - /// A structural feature (JOIN type / subquery / derived table) not - /// supported in this version. - UnsupportedFeature(String), - /// The workload's query language is not SQL. - WrongLanguage(String), - /// Resolving the name-based tree failed (name resolution against the - /// bound schema, or schema derivation). - Convert(ResolveDAGError), -} - -impl fmt::Display for SqlError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::DataFusion(e) => write!(f, "DataFusion error: {e}"), - Self::TableNotFound(t) => write!(f, "table not found in catalog: {t}"), - Self::UnsupportedAggregate(n) => write!(f, "unsupported aggregate: {n}"), - Self::InvalidExpression(m) => write!(f, "invalid expression: {m}"), - Self::UnsupportedDialect(d) => write!(f, "unsupported SQL dialect: {d}"), - Self::UnsupportedFeature(m) => write!(f, "unsupported feature: {m}"), - Self::WrongLanguage(l) => write!(f, "unsupported query language: {l}"), - Self::Convert(e) => write!(f, "column resolution failed: {e}"), - } - } -} - -impl std::error::Error for SqlError {} - -impl From for SqlError { - fn from(e: ResolveDAGError) -> Self { - Self::Convert(e) - } -} - -impl From for SqlError { - fn from(e: datafusion::error::DataFusionError) -> Self { - Self::DataFusion(e) - } -} diff --git a/crates/frontend-sql/src/unified/mod.rs b/crates/frontend-sql/src/unified/mod.rs deleted file mode 100644 index 279585fb9..000000000 --- a/crates/frontend-sql/src/unified/mod.rs +++ /dev/null @@ -1,103 +0,0 @@ -//! SQL front end: parse + plan (via DataFusion) → the name-based -//! [`UnresolvedOp`](asap_frontend_common::UnresolvedOp) tree, built directly -//! (issue #179) → [`resolve_root`]. -//! -//! Emits the shared front-end tree (`UnresolvedOp` / `UnresolvedScalar`, -//! name-based [`ColumnRef`](asap_types::pre_asap::ColumnRef)s) directly, rather -//! than a separate per-language relational tree; `resolve_root` binds it into -//! the unified [`OperatorNode`] IR, deriving every schema on the way. -//! Depends on DataFusion only — never on the PromQL parser. - -pub mod error; -pub mod sql; - -use std::rc::Rc; - -use asap_frontend_common::resolve_root; -use asap_types::ir::OperatorNode; -use asap_types::types::AccuracyTarget; -use asap_types::workload::{QueryLanguage, QueryWorkload, SqlDialect}; - -pub use error::SqlError; -pub use sql::{SqlCatalog, SqlLowerer}; - -/// Lower a single SQL query string to the resolved, canonical operator DAG, -/// parsed as `SqlDialect::DataFusionSQL`. -/// -/// The `catalog` supplies table schemas (used both to plan the SQL with -/// DataFusion and to carry positional column identity into the resolved -/// tree). `accuracy` is threaded onto every approximate intent as it's built. -pub async fn lower_sql( - query: &str, - catalog: &SqlCatalog, - accuracy: AccuracyTarget, -) -> Result, SqlError> { - lower_sql_dialect(query, catalog, SqlDialect::DataFusionSQL, accuracy).await -} - -/// Lower a single SQL query string under an explicit [`SqlDialect`]. -/// -/// `ClickhouseSQL` parses via sqlparser's vendored `ClickHouseDialect` -/// (array-lambda syntax, `arr[-1]` indexing). It also teaches DataFusion's -/// planner the ClickHouse-only builtin functions listed in -/// `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` (`uniqExact`, `countIf`) -/// — every other ClickHouse-only builtin still fails to plan. -/// `ElasticSQL` has no vendored parser and always returns `UnsupportedDialect`. -pub async fn lower_sql_dialect( - query: &str, - catalog: &SqlCatalog, - dialect: SqlDialect, - accuracy: AccuracyTarget, -) -> Result, SqlError> { - let unresolved = SqlLowerer::with_dialect(catalog, dialect) - .lower(query, &accuracy) - .await?; - // Binding resolves names and derives every node's schema; result-type - // checks (such as temporal subtraction, whose duration unit the IR cannot - // represent) surface here as `ResolveDAGError::Schema`. - Ok(resolve_root(&unresolved)?) -} - -/// Lower every SQL batch entry in `workload` to an operator DAG. -/// -/// One `Result` per entry — errors are per-query, not fatal for the batch. -/// Returns `WrongLanguage` for every entry if the workload is not SQL, and -/// `UnsupportedDialect` for `ElasticSQL` (no vendored parser). -pub async fn lower_sql_batch( - workload: &QueryWorkload, - catalog: &SqlCatalog, -) -> Vec, SqlError>> { - let entries = match &workload.query_batch { - Some(e) if !e.is_empty() => e, - _ => return vec![], - }; - - // `DataFusion` is a legacy alias for `SQL(DataFusionSQL)`; accept both. - if !matches!( - workload.language, - QueryLanguage::SQL(_) | QueryLanguage::DataFusion - ) { - let lang = format!("{:?}", workload.language); - return entries - .iter() - .map(|_| Err(SqlError::WrongLanguage(lang.clone()))) - .collect(); - } - let dialect = match &workload.language { - QueryLanguage::SQL(d) => d.clone(), - _ => SqlDialect::DataFusionSQL, - }; - if matches!(dialect, SqlDialect::ElasticSQL) { - return entries - .iter() - .map(|_| Err(SqlError::UnsupportedDialect("ElasticSQL".into()))) - .collect(); - } - - let mut results = Vec::with_capacity(entries.len()); - for entry in entries { - let accuracy = entry.requirements.accuracy.target(); - results.push(lower_sql_dialect(&entry.query.0, catalog, dialect.clone(), accuracy).await); - } - results -} diff --git a/crates/frontend-sql/src/unified/sql/clickhouse_ast.rs b/crates/frontend-sql/src/unified/sql/clickhouse_ast.rs deleted file mode 100644 index c68a803be..000000000 --- a/crates/frontend-sql/src/unified/sql/clickhouse_ast.rs +++ /dev/null @@ -1,139 +0,0 @@ -//! Structural ClickHouse syntax normalization before DataFusion type inference. -use datafusion::sql::sqlparser::ast::{ - visit_expressions, visit_expressions_mut, BinaryOperator, Expr, Function, FunctionArg, - FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, MapAccessSyntax, ObjectName, - Query, SelectItem, SetExpr, Statement, VisitMut, VisitorMut, -}; -use std::ops::ControlFlow; - -pub(super) fn normalize(statement: &mut Statement) { - struct PreserveNames; - impl VisitorMut for PreserveNames { - type Break = (); - fn pre_visit_query(&mut self, query: &mut Query) -> ControlFlow<()> { - fn preserve(body: &mut SetExpr) { - match body { - SetExpr::Select(select) => { - for item in &mut select.projection { - if let SelectItem::UnnamedExpr(expr) = item { - let mut changed = false; - let _: ControlFlow<()> = visit_expressions_mut(expr, |node| { - changed |= normalize_map_access(node); - ControlFlow::Continue(()) - }); - let _: ControlFlow<()> = visit_expressions(expr, |candidate| { - if let Expr::Function(function) = candidate { - changed |= function.name.0.len() == 1 - && function.name.0[0].quote_style.is_none() - && matches!( - function.name.0[0] - .value - .to_ascii_lowercase() - .as_str(), - "modulo" - | "map" - | "mapconcat" - | "arrayelement" - | "tupleelement" - ); - } - ControlFlow::Continue(()) - }); - if changed { - let alias = Ident::with_quote('"', expr.to_string()); - let value = std::mem::replace( - expr, - Expr::Value(datafusion::sql::sqlparser::ast::Value::Null), - ); - *item = SelectItem::ExprWithAlias { expr: value, alias }; - } - } - } - } - SetExpr::SetOperation { left, right, .. } => { - preserve(left); - preserve(right); - } - _ => {} - } - } - preserve(&mut query.body); - ControlFlow::Continue(()) - } - } - let _: ControlFlow<()> = statement.visit(&mut PreserveNames); - let _: ControlFlow<()> = visit_expressions_mut(statement, |expr| { - normalize_map_access(expr); - let Expr::Function(function) = expr else { - return ControlFlow::Continue(()); - }; - if function.name.0.len() != 1 - || function.name.0[0].quote_style.is_some() - || !function.name.0[0].value.eq_ignore_ascii_case("modulo") - || !matches!(function.parameters, FunctionArguments::None) - || function.filter.is_some() - || function.over.is_some() - || function.null_treatment.is_some() - || !function.within_group.is_empty() - { - return ControlFlow::Continue(()); - } - let FunctionArguments::List(arguments) = &function.args else { - return ControlFlow::Continue(()); - }; - if arguments.duplicate_treatment.is_some() || !arguments.clauses.is_empty() { - return ControlFlow::Continue(()); - } - let [FunctionArg::Unnamed(FunctionArgExpr::Expr(left)), FunctionArg::Unnamed(FunctionArgExpr::Expr(right))] = - arguments.args.as_slice() - else { - return ControlFlow::Continue(()); - }; - *expr = Expr::BinaryOp { - left: Box::new(left.clone()), - op: BinaryOperator::Modulo, - right: Box::new(right.clone()), - }; - ControlFlow::Continue(()) - }); -} - -fn normalize_map_access(expression: &mut Expr) -> bool { - let Expr::MapAccess { keys, .. } = expression else { - return false; - }; - if keys.is_empty() - || keys - .iter() - .any(|key| key.syntax != MapAccessSyntax::Bracket) - { - return false; - } - let Expr::MapAccess { column, keys } = std::mem::replace( - expression, - Expr::Value(datafusion::sql::sqlparser::ast::Value::Null), - ) else { - unreachable!() - }; - let mut input = *column; - for key in keys { - input = Expr::Function(Function { - name: ObjectName(vec![Ident::new("arrayElement")]), - parameters: FunctionArguments::None, - args: FunctionArguments::List(FunctionArgumentList { - duplicate_treatment: None, - clauses: vec![], - args: vec![ - FunctionArg::Unnamed(FunctionArgExpr::Expr(input)), - FunctionArg::Unnamed(FunctionArgExpr::Expr(key.key)), - ], - }), - filter: None, - null_treatment: None, - over: None, - within_group: vec![], - }); - } - *expression = input; - true -} diff --git a/crates/frontend-sql/src/unified/sql/collection_planning.rs b/crates/frontend-sql/src/unified/sql/collection_planning.rs deleted file mode 100644 index 75d0450cc..000000000 --- a/crates/frontend-sql/src/unified/sql/collection_planning.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! DataFusion planning adapters. Types come from the canonical signature rules; -//! physical evaluation deliberately remains the query engine's responsibility. -use super::types::{arrow_to_dtype, dtype_to_arrow, scalar_value_to_asap}; -use asap_types::ir::scalar::{element_access_type, struct_field_type}; -use asap_types::ir::ScalarExpr; -use asap_types::pre_asap::scalar_type_rules::MapScalarFunction; -use asap_types::pre_asap::{Field, Schema}; -use datafusion::arrow::datatypes::DataType; -use datafusion::common::{DataFusionError, ExprSchema, Result}; -use datafusion::logical_expr::{ - ColumnarValue, Expr, ExprSchemable, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, - Volatility, -}; -use datafusion::prelude::SessionContext; - -#[derive(Debug, Clone, Copy)] -enum PlanningFunction { - Map(MapScalarFunction), - Element, - StructField, -} - -pub(super) fn register(context: &SessionContext) { - for (name, function) in [ - ("map", PlanningFunction::Map(MapScalarFunction::Construct)), - ( - "mapconcat", - PlanningFunction::Map(MapScalarFunction::Concat), - ), - ("arrayelement", PlanningFunction::Element), - ("tupleelement", PlanningFunction::StructField), - ] { - context.register_udf(ScalarUDF::from(CollectionPlanningFunction { - name, - function, - signature: match function { - PlanningFunction::Map(MapScalarFunction::Construct) => Signature::one_of( - vec![TypeSignature::Exact(vec![]), TypeSignature::VariadicAny], - Volatility::Immutable, - ), - PlanningFunction::Map(MapScalarFunction::Access) - | PlanningFunction::Element - | PlanningFunction::StructField => Signature::any(2, Volatility::Immutable), - PlanningFunction::Map(MapScalarFunction::Concat) => { - Signature::variadic_any(Volatility::Immutable) - } - }, - })); - } -} -#[derive(Debug)] -struct CollectionPlanningFunction { - name: &'static str, - function: PlanningFunction, - signature: Signature, -} -impl CollectionPlanningFunction { - fn output( - &self, - args: &[DataType], - nullable: &[bool], - expressions: Option<&[Expr]>, - ) -> Result<(DataType, bool)> { - let inputs = args - .iter() - .zip(nullable) - .map(|(dtype, null)| { - arrow_to_dtype(dtype) - .map(|dtype| (dtype, *null)) - .map_err(|e| DataFusionError::Plan(e.to_string())) - }) - .collect::>>()?; - let (dtype, nullable) = if matches!( - self.function, - PlanningFunction::Element | PlanningFunction::StructField - ) { - // DataFusion asks for argument-dependent types before canonical - // expression binding. Reuse the shared resolver over typed argument - // slots; final canonical binding also validates literal selectors. - let schema = Schema::new( - inputs - .into_iter() - .enumerate() - .map(|(index, (dtype, nullable))| { - Field::plain(format!("argument_{index}"), dtype, nullable) - }) - .collect(), - ); - let args = (0..schema.fields.len()) - .map(|index| { - if let Some(Expr::Literal(value)) = expressions.and_then(|args| args.get(index)) - { - scalar_value_to_asap(value) - .map(ScalarExpr::Literal) - .map_err(|error| DataFusionError::Plan(error.to_string())) - } else { - Ok(ScalarExpr::Column(index)) - } - }) - .collect::>>()?; - match self.function { - PlanningFunction::Element => element_access_type(&args, &schema), - PlanningFunction::StructField => struct_field_type(&args, &schema), - PlanningFunction::Map(_) => unreachable!(), - } - } else if let PlanningFunction::Map(function) = self.function { - function.output_type(&inputs) - } else { - unreachable!() - } - .map_err(DataFusionError::Plan)?; - Ok((dtype_to_arrow(&dtype), nullable)) - } -} -impl ScalarUDFImpl for CollectionPlanningFunction { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn name(&self) -> &str { - self.name - } - fn signature(&self) -> &Signature { - &self.signature - } - fn return_type(&self, args: &[DataType]) -> Result { - self.output( - args, - &args - .iter() - .map(|dtype| *dtype == DataType::Null) - .collect::>(), - None, - ) - .map(|output| output.0) - } - fn return_type_from_exprs( - &self, - args: &[Expr], - schema: &dyn ExprSchema, - types: &[DataType], - ) -> Result { - let nullable = args - .iter() - .map(|arg| arg.nullable(schema)) - .collect::>>()?; - self.output(types, &nullable, Some(args)) - .map(|output| output.0) - } - fn is_nullable(&self, args: &[Expr], schema: &dyn ExprSchema) -> bool { - let types = args - .iter() - .map(|arg| arg.get_type(schema)) - .collect::>>(); - let nullable = args - .iter() - .map(|arg| arg.nullable(schema)) - .collect::>>(); - match (types, nullable) { - (Ok(types), Ok(nullable)) => self - .output(&types, &nullable, Some(args)) - .map(|out| out.1) - .unwrap_or(true), - _ => true, - } - } - fn invoke_batch(&self, _args: &[ColumnarValue], _number_rows: usize) -> Result { - Err(DataFusionError::NotImplemented("collection planning adapter cannot execute; use a capable query engine or external exact sub_dag".into())) - } -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn planning_adapter_explicitly_refuses_physical_execution() { - let adapter = CollectionPlanningFunction { - name: "map", - function: PlanningFunction::Map(MapScalarFunction::Construct), - signature: Signature::any(0, Volatility::Immutable), - }; - assert!(matches!( - adapter.invoke_batch(&[], 1), - Err(DataFusionError::NotImplemented(_)) - )); - let result = adapter.return_type(&[]).unwrap(); - let (expected, _) = MapScalarFunction::Construct.output_type(&[]).unwrap(); - assert_eq!(result, dtype_to_arrow(&expected)); - } -} diff --git a/crates/frontend-sql/src/unified/sql/dialect.rs b/crates/frontend-sql/src/unified/sql/dialect.rs deleted file mode 100644 index 03d9253e0..000000000 --- a/crates/frontend-sql/src/unified/sql/dialect.rs +++ /dev/null @@ -1,112 +0,0 @@ -//! The parser dialect for `SqlDialect::DataFusionSQL`. -//! -//! sqlparser's `GenericDialect` leaves `FILTER (WHERE …)` on aggregate calls -//! off (`supports_filter_during_aggregation`), and DataFusion only selects a -//! dialect by name — so `count(x) FILTER (WHERE p)` cannot reach the planner -//! through `SessionContext::sql`. This wrapper is `GenericDialect` with that -//! one switch flipped (issue #466); `lower` parses through -//! `DFParser::parse_sql_with_dialect` with it and plans the statement itself, -//! exactly as the ClickHouse path already does. - -use std::any::TypeId; - -use datafusion::sql::sqlparser::dialect::{Dialect, GenericDialect}; - -#[derive(Debug, Default)] -pub(crate) struct GenericWithAggregateFilter; - -/// Forward every boolean switch `GenericDialect` overrides, so the only -/// behavioural difference is `supports_filter_during_aggregation`. -macro_rules! forward_to_generic { - ($($method:ident),* $(,)?) => { - $(fn $method(&self) -> bool { - GenericDialect.$method() - })* - }; -} - -impl Dialect for GenericWithAggregateFilter { - /// The parser's own `dialect_of!(… is GenericDialect)` checks keep - /// matching, so generic-only syntax paths stay enabled. - fn dialect(&self) -> TypeId { - GenericDialect.dialect() - } - - fn is_delimited_identifier_start(&self, ch: char) -> bool { - GenericDialect.is_delimited_identifier_start(ch) - } - - fn is_identifier_start(&self, ch: char) -> bool { - GenericDialect.is_identifier_start(ch) - } - - fn is_identifier_part(&self, ch: char) -> bool { - GenericDialect.is_identifier_part(ch) - } - - fn supports_filter_during_aggregation(&self) -> bool { - true - } - - forward_to_generic!( - supports_unicode_string_literal, - supports_group_by_expr, - supports_connect_by, - supports_match_recognize, - supports_start_transaction_modifier, - supports_window_function_null_treatment_arg, - supports_dictionary_syntax, - supports_window_clause_named_window_reference, - supports_parenthesized_set_variables, - supports_select_wildcard_except, - support_map_literal_syntax, - allow_extract_custom, - allow_extract_single_quotes, - supports_create_index_with_clause, - ); -} - -#[cfg(test)] -mod tests { - use super::*; - use datafusion::sql::parser::DFParser; - - // Every switch `GenericDialect` sets is mirrored, and only the aggregate - // FILTER switch differs. - #[test] - fn mirrors_generic_except_for_aggregate_filter() { - let ours = GenericWithAggregateFilter; - let generic = GenericDialect; - assert_eq!(ours.dialect(), generic.dialect()); - for ch in ['"', '`', '_', '#', '@', '$', 'a', '1', ' '] { - assert_eq!( - ours.is_delimited_identifier_start(ch), - generic.is_delimited_identifier_start(ch) - ); - assert_eq!( - ours.is_identifier_start(ch), - generic.is_identifier_start(ch) - ); - assert_eq!(ours.is_identifier_part(ch), generic.is_identifier_part(ch)); - } - assert_eq!( - ours.supports_group_by_expr(), - generic.supports_group_by_expr() - ); - assert!(!generic.supports_filter_during_aggregation()); - assert!(ours.supports_filter_during_aggregation()); - } - - // The generic dialect rejects an aggregate FILTER clause; ours parses it. - #[test] - fn parses_aggregate_filter_clause() { - let sql = "SELECT count(*) FILTER (WHERE a > 1) FROM t"; - assert!(DFParser::parse_sql_with_dialect(sql, &GenericDialect).is_err()); - assert_eq!( - DFParser::parse_sql_with_dialect(sql, &GenericWithAggregateFilter) - .unwrap() - .len(), - 1 - ); - } -} diff --git a/crates/frontend-sql/src/unified/sql/expr.rs b/crates/frontend-sql/src/unified/sql/expr.rs deleted file mode 100644 index b726bfc12..000000000 --- a/crates/frontend-sql/src/unified/sql/expr.rs +++ /dev/null @@ -1,354 +0,0 @@ -use std::rc::Rc; - -use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; - -use asap_frontend_common::UnresolvedScalar as Unresolved; -use asap_types::ir::ExprSemantics; -use asap_types::pre_asap::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; - -use crate::unified::error::SqlError as LoweringError; - -use super::types::{arrow_to_dtype, scalar_value_to_asap}; -use super::SqlLowerer; - -pub(super) fn split_conjuncts(expr: &Expr) -> Vec<&Expr> { - match expr { - Expr::BinaryExpr(BinaryExpr { - left, - op: Operator::And, - right, - }) => { - let mut v = split_conjuncts(left); - v.extend(split_conjuncts(right)); - v - } - _ => vec![expr], - } -} - -impl SqlLowerer<'_> { - /// Translate a DataFusion `Expr` to the name-based scalar tree. Every - /// `Compare` / `Arithmetic` / `Negative` carries `ExprSemantics::Sql`. - /// Subquery-valued expressions lower their plan as a root of its own - /// (which is why this is a method: the plan walk needs the catalog). - /// Returns `UnsupportedFeature` for anything not needed in v1. - pub(super) fn lower_expr(&self, expr: &Expr) -> Result { - let bx = |e: &Expr| self.lower_expr(e).map(Box::new); - match expr { - // Preserve DataFusion's relation qualifier so a column name shared - // across a join (`a.k` vs `b.k`) resolves to the correct side. - Expr::Column(col) => Ok(Unresolved::Column(match &col.relation { - Some(rel) => ColumnRef::Qualified { - table: rel.to_string(), - name: col.name.clone(), - }, - None => ColumnRef::Named(col.name.clone()), - })), - - // Keep Arrow date literals equivalent to SQL CAST('YYYY-MM-DD' AS DATE), - // including typed nulls, without adding another canonical scalar variant. - Expr::Literal( - sv @ (datafusion::common::ScalarValue::Date32(_) - | datafusion::common::ScalarValue::Date64(_)), - ) => { - let text = sv.cast_to(&datafusion::arrow::datatypes::DataType::Utf8)?; - // Arrow formats Date64 with a time suffix; the canonical Date has - // no time-of-day, just like Date64 catalog registration as Date32. - let text = match text { - datafusion::common::ScalarValue::Utf8(Some(value)) => { - ScalarValue::Utf8(value.split('T').next().unwrap().to_owned()) - } - other => scalar_value_to_asap(&other)?, - }; - Ok(Unresolved::Cast { - expr: Box::new(Unresolved::Literal(text)), - to: asap_types::pre_asap::schema::DataType::Date, - try_cast: false, - }) - } - Expr::Literal(sv) => scalar_value_to_asap(sv).map(Unresolved::Literal), - - Expr::Alias(a) => self.lower_expr(&a.expr), - - Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op { - Operator::And => { - let parts = split_conjuncts(expr); - let lowered: Result, _> = - parts.iter().map(|e| self.lower_expr(e)).collect(); - Ok(Unresolved::BoolAnd(lowered?)) - } - Operator::Or => { - let parts = split_disjuncts(expr); - let lowered: Result, _> = - parts.iter().map(|e| self.lower_expr(e)).collect(); - Ok(Unresolved::BoolOr(lowered?)) - } - Operator::Eq => self.compare(left, CompareOpKind::Eq, right), - Operator::NotEq => self.compare(left, CompareOpKind::Ne, right), - Operator::Lt => self.compare(left, CompareOpKind::Lt, right), - Operator::LtEq => self.compare(left, CompareOpKind::Le, right), - Operator::Gt => self.compare(left, CompareOpKind::Gt, right), - Operator::GtEq => self.compare(left, CompareOpKind::Ge, right), - // BinaryExpr LIKE/ILIKE operators (from optimizer rewrites) - Operator::LikeMatch => self.compare(left, CompareOpKind::Like, right), - Operator::ILikeMatch => self.compare(left, CompareOpKind::ILike, right), - Operator::NotLikeMatch => self.compare(left, CompareOpKind::NotLike, right), - Operator::NotILikeMatch => self.compare(left, CompareOpKind::NotILike, right), - // Arithmetic - Operator::Plus => self.arith(left, ArithmeticOpKind::Add, right), - Operator::Minus => self.arith(left, ArithmeticOpKind::Sub, right), - Operator::Multiply => self.arith(left, ArithmeticOpKind::Mul, right), - Operator::Divide => self.arith(left, ArithmeticOpKind::Div, right), - Operator::Modulo => self.arith(left, ArithmeticOpKind::Mod, right), - other => Err(LoweringError::UnsupportedFeature(format!( - "operator: {other:?}" - ))), - }, - - // SQL LIKE / ILIKE (dedicated expr node from the SQL parser) - Expr::Like(like) => { - let op = match (like.negated, like.case_insensitive) { - (false, false) => CompareOpKind::Like, - (true, false) => CompareOpKind::NotLike, - (false, true) => CompareOpKind::ILike, - (true, true) => CompareOpKind::NotILike, - }; - self.compare(&like.expr, op, &like.pattern) - } - - // Unary minus. (DataFusion's planner already folds `-` - // into a negative literal, so this is a non-literal operand.) - Expr::Negative(inner) => Ok(Unresolved::Negative { - expr: bx(inner)?, - semantics: ExprSemantics::Sql, - }), - - // SQL CASE expression - Expr::Case(c) => { - let operand = c.expr.as_deref().map(bx).transpose()?; - let branches = c - .when_then_expr - .iter() - .map(|(when, then)| Ok((self.lower_expr(when)?, self.lower_expr(then)?))) - .collect::, LoweringError>>()?; - let else_expr = c.else_expr.as_deref().map(bx).transpose()?; - Ok(Unresolved::Case { - operand, - branches, - else_expr, - }) - } - - Expr::Not(inner) => Ok(Unresolved::Not(bx(inner)?)), - - Expr::IsNull(inner) => Ok(Unresolved::IsNull(bx(inner)?)), - - Expr::IsNotNull(inner) => Ok(Unresolved::IsNotNull(bx(inner)?)), - - Expr::Cast(c) => Ok(Unresolved::Cast { - expr: bx(&c.expr)?, - to: arrow_to_dtype(&c.data_type)?, - try_cast: false, - }), - - // TRY_CAST returns NULL on conversion failure; preserve that semantic. - Expr::TryCast(c) => Ok(Unresolved::Cast { - expr: bx(&c.expr)?, - to: arrow_to_dtype(&c.data_type)?, - try_cast: true, - }), - - Expr::InList(il) => { - let list: Result, _> = il.list.iter().map(|e| self.lower_expr(e)).collect(); - Ok(Unresolved::InList { - expr: bx(&il.expr)?, - list: list?, - negated: il.negated, - }) - } - - Expr::Between(b) => { - // Normalize: `x BETWEEN low AND high` → `x >= low AND x <= high`. - // `x NOT BETWEEN low AND high` → `x < low OR x > high`. - if b.negated { - let lt = self.compare(&b.expr, CompareOpKind::Lt, &b.low)?; - let gt = self.compare(&b.expr, CompareOpKind::Gt, &b.high)?; - Ok(Unresolved::BoolOr(vec![lt, gt])) - } else { - let x_low = self.compare(&b.expr, CompareOpKind::Ge, &b.low)?; - let x_high = self.compare(&b.expr, CompareOpKind::Le, &b.high)?; - Ok(Unresolved::BoolAnd(vec![x_low, x_high])) - } - } - - // `NOW()` / `CURRENT_TIMESTAMP` read the SQL statement evaluation - // time. Keep this timestamp-typed leaf distinct from PromQL's - // Float64 Unix-seconds `EvalTimestamp`. Issue #184. - Expr::ScalarFunction(sf) - if sf.args.is_empty() - && matches!( - sf.func.name().to_ascii_lowercase().as_str(), - "now" | "current_timestamp" - ) => - { - Ok(Unresolved::CurrentTimestamp) - } - - Expr::ScalarFunction(sf) => { - let args: Result, _> = sf.args.iter().map(|e| self.lower_expr(e)).collect(); - Ok(Unresolved::FunctionCall { - name: if sf.func.name().eq_ignore_ascii_case("arrayelement") { - "asap_element_access".into() - } else if sf.func.name().eq_ignore_ascii_case("tupleelement") { - "asap_struct_field".into() - } else { - sf.func.name().to_string() - }, - args: args?, - }) - } - - // Subquery-valued expressions. Each subquery plan is lowered as a - // root of its own; `resolve_root` binds it in its own scope, so an - // outer reference inside it has nothing to resolve against — a - // correlated subquery is rejected rather than mislowered. - Expr::ScalarSubquery(sq) => Ok(Unresolved::ScalarSubquery(Rc::new( - self.lower_uncorrelated_subquery(sq, "scalar subquery")?, - ))), - Expr::Exists(ex) => Ok(Unresolved::Exists { - subquery: Rc::new(self.lower_uncorrelated_subquery(&ex.subquery, "EXISTS")?), - negated: ex.negated, - }), - Expr::InSubquery(is) => { - let fields = is.subquery.subquery.schema().fields().len(); - if fields != 1 { - return Err(LoweringError::InvalidExpression(format!( - "IN (subquery) must select exactly one column, got {fields}" - ))); - } - Ok(Unresolved::InSubquery { - expr: bx(&is.expr)?, - subquery: Rc::new( - self.lower_uncorrelated_subquery(&is.subquery, "IN (subquery)")?, - ), - negated: is.negated, - }) - } - - other => Err(LoweringError::UnsupportedFeature(format!( - "expression: {}", - other - ))), - } - } - - fn lower_uncorrelated_subquery( - &self, - sq: &datafusion::logical_expr::Subquery, - what: &str, - ) -> Result { - if !sq.outer_ref_columns.is_empty() { - return Err(LoweringError::UnsupportedFeature(format!( - "correlated {what}" - ))); - } - self.lower_plan(&sq.subquery) - } - - pub(super) fn compare( - &self, - left: &Expr, - op: CompareOpKind, - right: &Expr, - ) -> Result { - Ok(Unresolved::Compare { - left: Box::new(self.lower_expr(left)?), - op, - right: Box::new(self.lower_expr(right)?), - semantics: ExprSemantics::Sql, - }) - } - - fn arith( - &self, - left: &Expr, - op: ArithmeticOpKind, - right: &Expr, - ) -> Result { - Ok(Unresolved::Arithmetic { - op, - left: Box::new(self.lower_expr(left)?), - right: Box::new(self.lower_expr(right)?), - semantics: ExprSemantics::Sql, - }) - } -} - -pub(super) fn split_disjuncts(expr: &Expr) -> Vec<&Expr> { - match expr { - Expr::BinaryExpr(BinaryExpr { - left, - op: Operator::Or, - right, - }) => { - let mut v = split_disjuncts(left); - v.extend(split_disjuncts(right)); - v - } - _ => vec![expr], - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::unified::sql::SqlCatalog; - use asap_types::pre_asap::schema::DataType; - use datafusion::common::ScalarValue as DfScalarValue; - - // Typed Arrow dates normalize to the same typed form as SQL date casts. - #[test] - fn arrow_date_literals_preserve_value_and_type() { - let catalog = SqlCatalog::new(); - let lowerer = SqlLowerer::new(&catalog); - for (value, expected) in [ - ( - DfScalarValue::Date32(Some(0)), - ScalarValue::Utf8("1970-01-01".into()), - ), - ( - DfScalarValue::Date64(Some(-86_400_000)), - ScalarValue::Utf8("1969-12-31".into()), - ), - (DfScalarValue::Date32(None), ScalarValue::Null), - (DfScalarValue::Date64(None), ScalarValue::Null), - ] { - let actual = lowerer.lower_expr(&Expr::Literal(value)).unwrap(); - assert_eq!( - actual, - Unresolved::Cast { - expr: Box::new(Unresolved::Literal(expected)), - to: DataType::Date, - try_cast: false, - } - ); - } - } - - // Unary minus over a non-literal is the `Negative` scalar, SQL-flavoured. - #[test] - fn unary_minus_lowers_to_negative_with_sql_semantics() { - let catalog = SqlCatalog::new(); - let lowerer = SqlLowerer::new(&catalog); - let expr = Expr::Negative(Box::new(Expr::Column( - datafusion::common::Column::new_unqualified("x"), - ))); - assert_eq!( - lowerer.lower_expr(&expr).unwrap(), - Unresolved::Negative { - expr: Box::new(Unresolved::Column(ColumnRef::Named("x".into()))), - semantics: ExprSemantics::Sql, - } - ); - } -} diff --git a/crates/frontend-sql/src/unified/sql/mod.rs b/crates/frontend-sql/src/unified/sql/mod.rs deleted file mode 100644 index 1f58f92c0..000000000 --- a/crates/frontend-sql/src/unified/sql/mod.rs +++ /dev/null @@ -1,2525 +0,0 @@ -//! SQL → the name-based front-end tree -//! ([`UnresolvedOp`](asap_frontend_common::UnresolvedOp) / -//! [`UnresolvedScalar`](asap_frontend_common::UnresolvedScalar)). -//! -//! Parses SQL via DataFusion (over the catalog's registered tables), then -//! walks the unoptimized `LogicalPlan` and emits `UnresolvedOp` nodes with -//! unresolved `ColumnRef`s directly (issue #179) — the same tree shape -//! [`resolve_root`](asap_frontend_common::resolve_root) binds into the -//! positional, unified `OperatorNode` IR. Unlike PromQL's front end, SQL's -//! Ordinary SQL `Aggregate` nodes are `Reduction::Reduce`. The explicit -//! `asap_rate`/`asap_increase` bridge is the narrow exception: it -//! spells a time-series range reducer with an explicit value, time-index, and -//! window and therefore lowers to the same `TimeRange` + `PerEntity` shape as -//! its PromQL counterpart. The front end also has to fold a `WHERE` directly -//! over a bare table scan onto -//! `Scan.predicates` itself (`filter_or_fold`) — canonical's invariant that a -//! `Filter` never sits directly over a `Scan` — since front ends producing -//! this shape are responsible for it now, not a converter. -//! -//! Heavy-hitter `topk` recognition (`ORDER BY count(...) DESC LIMIT k`) is -//! *not* done here: SQL emits a plain `Sort`/`Limit`, and the shared -//! `canonicalize` pass (issue #34, run by `resolve_root`) recognises the -//! count-ranked shape positionally, so a SQL `ORDER BY`/`LIMIT` and a PromQL -//! `topk(...)` converge without either front end special-casing the other's -//! syntax. - -use std::rc::Rc; -use std::sync::Arc; -use std::time::Duration; - -use datafusion::arrow::compute::kernels::cast_utils::parse_interval_month_day_nano; -use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field}; -use datafusion::catalog_common::MemorySchemaProvider; -use datafusion::common::config::ConfigOptions; -use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; -use datafusion::common::{Column as DfColumn, DFSchema, ScalarValue as DfScalarValue}; -use datafusion::datasource::MemTable; -use datafusion::functions_aggregate::count::count_udaf; -use datafusion::functions_aggregate::sum::sum_udaf; -use datafusion::logical_expr::expr::AggregateFunction; -use datafusion::logical_expr::expr_rewriter::FunctionRewrite; -use datafusion::logical_expr::function::{PartitionEvaluatorArgs, WindowUDFFieldArgs}; -use datafusion::logical_expr::{ - self, lit, AggregateUDF, Case, Distinct, Expr, ExprSchemable, JoinType, LogicalPlan, - PartitionEvaluator, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF, TypeSignature, - Volatility, WindowFrameBound as DfWindowFrameBound, WindowFrameUnits as DfWindowFrameUnits, - WindowFunctionDefinition, WindowUDF, WindowUDFImpl, -}; -use datafusion::optimizer::analyzer::function_rewrite::ApplyFunctionRewrites; -use datafusion::optimizer::{AnalyzerRule, OptimizerConfig}; -use datafusion::prelude::{SessionConfig, SessionContext}; -use datafusion::sql::parser::DFParser; - -use asap_frontend_common::{ - resolve_root, UnresolvedOp as Unresolved, UnresolvedPredicate as Predicate, - UnresolvedProjectItem as ProjectItem, UnresolvedScalar as Scalar, UnresolvedSortKey as SortKey, -}; -use asap_sql_function_catalog::{AggSemantic, Arity, RewriteKind}; -use asap_types::ir::operator_properties::{ - GroupKeys, Reduction, Source, WindowFrame, WindowFrameBound, WindowFrameOffset, - WindowFrameUnits, -}; -use asap_types::ir::TimeRangeKind; -use asap_types::pre_asap::agg_intent::AggIntent; -use asap_types::pre_asap::schema::{DataType, FieldDataType, Schema}; - -use asap_types::pre_asap::{ - resolve_column_ref, ColumnRef, CompareOpKind, JoinKind, RelationalSetOpKind, ScalarValue, - WindowFuncKind, -}; -use asap_types::types::AccuracyTarget; -use asap_types::workload::SqlDialect; - -use crate::unified::error::SqlError as LoweringError; - -mod clickhouse_ast; -mod collection_planning; -mod dialect; -mod expr; -mod types; - -pub use types::SqlCatalog; - -use self::dialect::GenericWithAggregateFilter; -use self::types::{arrow_to_dtype, scalar_value_to_asap, schema_to_arrow}; - -std::thread_local! { - static ACCURACY: std::cell::RefCell = - const { std::cell::RefCell::new(AccuracyTarget::Exact) }; -} - -/// RAII guard installing `accuracy` as the ambient accuracy target for the -/// current thread's lowering, restoring the prior value on drop — same -/// ambient-thread-local shape as `asap_frontend_promql::promql`'s -/// `AccuracyGuard`, for the same reason: it injects `accuracy` into the deep -/// `lower_plan` recursion without a parameter on every one of its -/// signatures, consulted only at the couple of sites that build an -/// accuracy-bearing `AggIntent`. -struct AccuracyGuard(AccuracyTarget); - -impl AccuracyGuard { - fn install(accuracy: AccuracyTarget) -> Self { - let prev = ACCURACY.with(|a| a.replace(accuracy)); - AccuracyGuard(prev) - } -} - -impl Drop for AccuracyGuard { - fn drop(&mut self) { - ACCURACY.with(|a| *a.borrow_mut() = std::mem::replace(&mut self.0, AccuracyTarget::Exact)); - } -} - -fn current_accuracy() -> AccuracyTarget { - ACCURACY.with(|a| a.borrow().clone()) -} - -/// Lowers SQL strings to the name-based [`UnresolvedOp`](asap_frontend_common::UnresolvedOp) -/// tree over a table [`SqlCatalog`]. Call -/// [`resolve_root`](asap_frontend_common::resolve_root) on the result for -/// the resolved operator DAG. -pub struct SqlLowerer<'a> { - catalog: &'a SqlCatalog, - dialect: SqlDialect, -} - -impl<'a> SqlLowerer<'a> { - pub fn new(catalog: &'a SqlCatalog) -> Self { - Self { - catalog, - dialect: SqlDialect::DataFusionSQL, - } - } - - /// Parse under a specific SQL dialect (e.g. `ClickhouseSQL`, which maps to - /// sqlparser's vendored `ClickHouseDialect` — array-lambda syntax and - /// `arr[-1]` indexing parse under it that don't parse generically). This - /// only changes *parsing*: a ClickHouse-only builtin function not listed - /// in `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` (`uniqExact` and - /// `countIf` are; most of ClickHouse's builtin surface isn't yet) is - /// still unknown to DataFusion's planner and still fails there, and - /// `ElasticSQL` has no vendored parser at all. - pub fn with_dialect(catalog: &'a SqlCatalog, dialect: SqlDialect) -> Self { - Self { catalog, dialect } - } - - /// Parse + lower a SQL query to the name-based tree, threading - /// `accuracy` onto every approximate intent (`Count`, `Quantile`, - /// `Cardinality`) as it is built. - /// - /// The `AccuracyGuard` installs *after* the only `.await` point - /// (DataFusion statement planning) — `lower_plan` itself is synchronous, so once it starts - /// there is no further suspension point that could move this task to a - /// different OS thread out from under a thread-local set beforehand. - /// - /// Runs `ApplyFunctionRewrites` — the single `AnalyzerRule` DataFusion's - /// own `Analyzer` uses internally to apply `FunctionRewrite`s, called - /// directly rather than through `Analyzer::execute_and_check` — over the - /// raw parsed plan before lowering, carrying only - /// `ClickHouseBuiltinRewrite` (catalog-driven, see its own doc — it - /// covers every `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` entry, - /// not just one). `ctx.sql(...).into_unoptimized_plan()` alone returns - /// `SqlToRel`'s output untouched, and a `FunctionRewrite` only ever runs - /// as part of this rule, so calling it directly is unavoidable to make - /// the rewrite fire. Its `analyze()` already does a full - /// `transform_up_with_subqueries` over the whole plan, so it needs no - /// wrapping `Analyzer` at all — deliberately not - /// `Analyzer::execute_and_check` (whether with the default 5-rule - /// analyzer or an empty one carrying just this rewrite): that method - /// runs an unconditional post-check (`check_plan`, hardcoded, not itself - /// a rule) that isn't wanted here — e.g. it independently rejects a - /// multi-column `IN (subquery)` before `lower_in_subquery`'s own arity - /// check would. Going straight to `ApplyFunctionRewrites` avoids that - /// entirely. TypeCoercion then records implicit conversions explicitly, - /// including timestamp literals in predicates, before IR validation. - pub async fn lower( - &self, - sql: &str, - accuracy: &AccuracyTarget, - ) -> Result { - let ctx = self.build_context()?; - let state = ctx.state(); - let statement = if matches!(self.dialect, SqlDialect::ClickhouseSQL) { - let mut statement = state.sql_to_statement(sql, "ClickHouse")?; - if let datafusion::sql::parser::Statement::Statement(ast) = &mut statement { - clickhouse_ast::normalize(ast); - } - statement - } else { - // Not `ctx.sql(sql)`: that parses under the by-name `generic` - // dialect, which cannot see an aggregate `FILTER (WHERE …)`. - let mut statements = DFParser::parse_sql_with_dialect(sql, &GenericWithAggregateFilter) - .map_err(|e| datafusion::error::DataFusionError::SQL(e, None))?; - let (Some(statement), true) = (statements.pop_front(), statements.is_empty()) else { - return Err(LoweringError::UnsupportedFeature( - "exactly one SQL statement per query".into(), - )); - }; - statement - }; - let plan = state.statement_to_plan(statement).await?; - let rewriter = ApplyFunctionRewrites::new(vec![Arc::new(ClickHouseBuiltinRewrite)]); - let plan = rewriter.analyze(plan, ctx.state().options())?; - let plan = datafusion::optimizer::analyzer::type_coercion::TypeCoercion::new() - .analyze(plan, ctx.state().options())?; - // Output schemas omit predicate and nested-expression types. Check the - // typed SQL plan before lowering erases fixed-duration units. - plan.apply_with_subqueries(|node| { - let mut schema = DFSchema::empty(); - for input in node.inputs() { - schema.merge(input.schema()); - } - schema.merge(node.schema()); - node.apply_expressions(|expr| { - expr.apply(|nested| { - if let Expr::BinaryExpr(binary) = nested { - if binary.op == logical_expr::Operator::Minus - && matches!(nested.get_type(&schema)?, ArrowDataType::Duration(_)) - { - return Err(datafusion::common::DataFusionError::Plan( - "temporal subtraction produces an unsupported duration type".into(), - )); - } - } - Ok(TreeNodeRecursion::Continue) - }) - }) - })?; - let _guard = AccuracyGuard::install(accuracy.clone()); - self.lower_plan(&plan) - } - - /// Register the catalog tables (empty Arrow `MemTable`s) so DataFusion can - /// resolve table/column references during planning. - fn build_context(&self) -> Result { - let dialect_name = match &self.dialect { - SqlDialect::DataFusionSQL => "generic", - SqlDialect::ClickhouseSQL => "ClickHouse", - SqlDialect::ElasticSQL => { - return Err(LoweringError::UnsupportedDialect("ElasticSQL".into())) - } - }; - let config = SessionConfig::new().set_str("datafusion.sql_parser.dialect", dialect_name); - let ctx = SessionContext::new_with_config(config); - if matches!(self.dialect, SqlDialect::ClickhouseSQL) { - collection_planning::register(&ctx); - } - // A catalog key like "bgp.bgp_updates" schema-qualifies the table - // (e.g. a ClickHouse database name). DataFusion requires the parent - // schema to be registered before a qualified table can be, so create - // it on demand. - let catalog_provider = ctx.catalog("datafusion").ok_or_else(|| { - LoweringError::InvalidExpression("default \"datafusion\" catalog missing".into()) - })?; - for (name, schema) in &self.catalog.tables { - if let Some((schema_name, _)) = name.split_once('.') { - if catalog_provider.schema(schema_name).is_none() { - catalog_provider - .register_schema(schema_name, Arc::new(MemorySchemaProvider::new()))?; - } - } - let arrow_schema = Arc::new(schema_to_arrow(schema)); - let mem_table = MemTable::try_new(arrow_schema, vec![])?; - ctx.register_table(name.as_str(), Arc::new(mem_table))?; - } - // Register a stub `AggregateUDF` for every catalog-listed - // ClickHouse-only builtin, purely so DataFusion's planner can - // resolve its name during parsing — `lower()` rewrites every call - // site to a native DataFusion aggregate via `ClickHouseBuiltinRewrite` - // before `lower_plan` sees it. - for builtin in asap_sql_function_catalog::CLICKHOUSE_BUILTINS { - ctx.register_udaf(clickhouse_builtin_stub_udaf(builtin.name, builtin.arity)); - } - // Register a stub `ScalarUDF` for every catalog-listed ClickHouse-only - // *scalar* builtin — same reason as the `AggregateUDF` loop above - // (DataFusion otherwise rejects the call as an unknown function - // during `SqlToRel` conversion), but with no rewrite step to follow: - // `lower_expr`'s `Expr::ScalarFunction` arm already lowers any - // scalar call generically to `UnresolvedScalar::FunctionCall { name, - // args }`, so registering the stub is the entire fix (issue #230). - for builtin in asap_sql_function_catalog::CLICKHOUSE_SCALAR_BUILTINS { - ctx.register_udf(clickhouse_scalar_builtin_stub_udf( - builtin.name, - builtin.arity, - )); - } - // Planning-only relation markers. They let a workload author state - // the PromQL temporal/classic-histogram semantics of an equivalent SQL - // rewrite without teaching the canonical IR a second, SQL-specific - // spelling of either operation. `lower_projection` consumes these - // calls; they can never survive as executable scalar functions. - for (name, arity) in [ - ("asap_promql_subquery", Arity::Exact(2)), - ("asap_histogram_quantile", Arity::Exact(1)), - ] { - ctx.register_udf(clickhouse_scalar_builtin_stub_udf(name, arity)); - } - // Register a stub `WindowUDF` for every catalog-listed ClickHouse-only - // *window* builtin — same reason as the two loops above, but with no - // rewrite step to follow: `lower_window_func_kind` already maps each - // name directly to its own `WindowFuncKind` variant (issue #267). - for builtin in asap_sql_function_catalog::CLICKHOUSE_WINDOW_BUILTINS { - ctx.register_udwf(clickhouse_window_builtin_stub_udwf( - builtin.name, - builtin.arity, - )); - } - Ok(ctx) - } - - pub(super) fn lower_plan(&self, plan: &LogicalPlan) -> Result { - match plan { - LogicalPlan::TableScan(scan) => self.lower_table_scan(scan), - // The one empty input row of a `SELECT` without `FROM`. - LogicalPlan::EmptyRelation(empty) => Ok(Unresolved::Values { - rows: if empty.produce_one_row { - vec![vec![]] - } else { - vec![] - }, - schema: Schema { - fields: vec![], - time_index: None, - unique_keys: vec![], - closed: true, - }, - }), - LogicalPlan::Values(values) => self.lower_values(values), - LogicalPlan::Filter(filter) => self.lower_filter(filter), - LogicalPlan::Projection(proj) => self.lower_projection(proj), - LogicalPlan::Aggregate(agg) => self.lower_aggregate(agg), - LogicalPlan::Sort(sort) => self.lower_sort(sort), - LogicalPlan::Limit(limit) => self.lower_limit(limit), - LogicalPlan::Distinct(d) => match d { - Distinct::On(_) => Err(LoweringError::UnsupportedFeature("DISTINCT ON".into())), - Distinct::All(input) => Ok(Unresolved::Dedup { - cols: vec![], - child: Rc::new(self.lower_plan(input)?), - }), - }, - LogicalPlan::Union(u) => { - // Fold n inputs left-associatively into SetOp { Union, all: true }. - let mut iter = u.inputs.iter(); - let first = iter - .next() - .ok_or_else(|| LoweringError::InvalidExpression("empty union".into()))?; - let first_expr = self.lower_plan(first)?; - iter.try_fold(first_expr, |left, right_plan| { - Ok(Unresolved::SetOp { - kind: RelationalSetOpKind::Union, - all: true, - left: Rc::new(left), - right: Rc::new(self.lower_plan(right_plan)?), - }) - }) - } - LogicalPlan::Window(window) => self.lower_window(window), - LogicalPlan::Join(join) => self.lower_join(join), - LogicalPlan::Subquery(_) => Err(LoweringError::UnsupportedFeature("subquery".into())), - LogicalPlan::SubqueryAlias(alias) => { - // An alias over a table re-qualifies the scan's columns with the - // alias (so `a.col` / `b.col` in a self-join disambiguate). - match alias.input.as_ref() { - LogicalPlan::TableScan(scan) => { - self.scan_source(&scan.table_name.to_string(), &alias.alias.to_string()) - } - // A *derived table* / inline view — `FROM (SELECT …) t`, the - // SQL counterpart of PromQL function nesting (an aggregate - // over an aggregate, a filter over a derived aggregate, …). - // Lower the inner plan, then re-qualify its output columns - // with the alias so `t.col` resolves to *this* relation — and, - // critically, so a join over two derived tables disambiguates - // its keys instead of both binding to the first bare-name - // match (issue #66). The inner column *names* are unchanged; - // only the qualifier is stamped. - other => { - let alias_name = alias.alias.to_string(); - match self.lower_plan(other)? { - // The derived SELECT list already lowered to a - // Projection — stamp the alias onto it, no extra node. - Unresolved::Project { cols, child, .. } => Ok(Unresolved::Project { - cols, - qualifier: Some(alias_name), - child, - }), - // Otherwise (e.g. `SELECT *` unwrapped to a scan) wrap - // in an identity projection that re-qualifies each - // output column. Names come from the sub-plan's schema. - inner => { - let cols = alias - .input - .schema() - .fields() - .iter() - .map(|f| ProjectItem { - alias: Some(f.name().clone()), - expr: Scalar::Column(ColumnRef::Named(f.name().clone())), - }) - .collect(); - Ok(Unresolved::Project { - cols, - qualifier: Some(alias_name), - child: Rc::new(inner), - }) - } - } - } - } - } - other => Err(LoweringError::UnsupportedFeature(format!( - "plan node: {}", - other.display() - ))), - } - } - - /// `WHERE` — a conjunction of ordinary predicates plus, possibly, subquery - /// predicates (issue #111). - /// - /// The ordinary conjuncts stay one predicate, folded onto a bare `Scan` - /// (`filter_or_fold`). A subquery conjunct — `c IN (SELECT …)`, `EXISTS - /// (…)`, `x > (SELECT …)` — is a row filter whose predicate reads another - /// operator (`UnresolvedScalar::InSubquery` / `Exists` / - /// `ScalarSubquery`); each one becomes its own `Filter` **above** the - /// ordinary predicate, so the shared `canonicalize` pass can turn it into - /// the join it is without having to peel it out of a conjunction or off - /// a `Scan` (it only lifts subqueries out of `Filter` / `Project`). A - /// semi-join only ever drops left rows, so the two orders agree. - /// - /// The one subquery shape still lowered to a join here is a *correlated* - /// `EXISTS`: its correlation references both sides, which only a join - /// predicate can bind (a subquery referenced from a scalar position is - /// resolved as a root in its own scope). - fn lower_filter(&self, filter: &logical_expr::Filter) -> Result { - let mut conjuncts = Vec::new(); - split_conjunction(&filter.predicate, &mut conjuncts); - let (subqueries, residual): (Vec<_>, Vec<_>) = - conjuncts.into_iter().partition(|e| reads_subquery(e)); - - let input = self.lower_plan(&filter.input)?; - let mut node = match rebuild_conjunction(&residual) { - Some(pred) => filter_or_fold(self.lower_expr(&pred)?, input), - None => input, - }; - for sq in subqueries { - node = match sq { - Expr::Exists(ex) if !ex.subquery.outer_ref_columns.is_empty() => { - self.lower_correlated_exists(ex, node)? - } - other => Unresolved::Filter { - pred: Predicate(self.lower_expr(other)?), - child: Rc::new(node), - }, - }; - } - Ok(node) - } - - /// `[NOT] EXISTS (SELECT … WHERE inner.k = outer.k)` → a semi- / anti-join - /// on the correlation predicate (issue #111). - fn lower_correlated_exists( - &self, - ex: &logical_expr::expr::Exists, - left: Unresolved, - ) -> Result { - let kind = if ex.negated { - JoinKind::Anti - } else { - JoinKind::Semi - }; - // A semi-join discards the right side's columns, and `SELECT 1` projects - // the correlation columns away — so drop the subquery's projections and - // join against what they sit on. - let mut inner = ex.subquery.subquery.as_ref(); - while let LogicalPlan::Projection(p) = inner { - inner = &p.input; - } - // Lift the correlated conjuncts out of the subquery's filter; they are - // the join predicate. Whatever is left stays an ordinary inner filter. - let (inner, correlation) = split_correlation(inner)?; - let right = self.lower_plan(&inner)?; - let pred = match correlation { - Some(e) => Predicate(self.lower_expr(&e)?), - None => Predicate(Scalar::Literal(ScalarValue::Boolean(true))), - }; - Ok(Unresolved::Join { - kind, - pred, - left: Rc::new(left), - right: Rc::new(right), - }) - } - - /// `VALUES (…), (…)` — one row per values row, typed by DataFusion's - /// declared schema. Row expressions have no input-column scope. - fn lower_values(&self, values: &logical_expr::Values) -> Result { - let rows = values - .values - .iter() - .map(|row| row.iter().map(|e| self.lower_expr(e)).collect()) - .collect::>, LoweringError>>()?; - let fields = values - .schema - .fields() - .iter() - .map(|f| { - Ok(asap_types::pre_asap::Field::plain( - f.name().clone(), - arrow_to_dtype(f.data_type())?, - f.is_nullable(), - )) - }) - .collect::, LoweringError>>()?; - Ok(Unresolved::Values { - rows, - schema: Schema { - fields, - time_index: None, - unique_keys: vec![], - closed: true, - }, - }) - } - - /// Table leaf — carries the catalog's resolved schema directly on `Scan` - /// (`schema: Some(_)`), so `resolve_root`'s SchemaResolver doesn't need to - /// usage-derive it (SQL is never schemaless). Projection pushdown is left - /// to the enclosing `Project` (DataFusion's unoptimized plan sets no - /// projection). - fn lower_table_scan( - &self, - scan: &logical_expr::TableScan, - ) -> Result { - let table = scan.table_name.to_string(); - self.scan_source(&table, &table) - } - - /// A `Scan` over catalog table `table`, with its columns qualified by - /// `qualifier` (the table name, or an alias from a `SubqueryAlias`) so - /// `Qualified` column refs resolve to the right side across a join. - fn scan_source(&self, table: &str, qualifier: &str) -> Result { - let schema = self - .catalog - .tables - .get(table) - .ok_or_else(|| LoweringError::TableNotFound(table.to_string()))?; - let qualified = Schema { - fields: schema - .fields - .iter() - .cloned() - .map(|c| c.with_table(qualifier)) - .collect(), - time_index: schema.time_index, - unique_keys: schema.unique_keys.clone(), - // Catalog-backed: the table's columns are fully declared → closed. - closed: true, - }; - Ok(Unresolved::Scan { - source: Source::Table { - table_ref: table.to_string(), - }, - predicates: vec![], - schema: Some(qualified), - }) - } - - /// ⋈ — equijoin. The `on` key pairs become `left = right` comparisons, - /// AND-ed with any non-equi `filter`, into the join predicate — still - /// name-based here (like a `WHERE`); `resolve_root` derives the - /// concatenated output schema downstream. Semi/anti/mark joins have no - /// canonical counterpart yet and are rejected. - fn lower_join(&self, join: &logical_expr::Join) -> Result { - let kind = match join.join_type { - JoinType::Inner => JoinKind::Inner, - JoinType::Left => JoinKind::Left, - JoinType::Right => JoinKind::Right, - JoinType::Full => JoinKind::Full, - other => { - return Err(LoweringError::UnsupportedFeature(format!( - "join type: {other:?}" - ))) - } - }; - let mut conjuncts = join - .on - .iter() - .map(|(l, r)| self.compare(l, CompareOpKind::Eq, r)) - .collect::, LoweringError>>()?; - if let Some(filter) = &join.filter { - conjuncts.push(self.lower_expr(filter)?); - } - let pred = Predicate(match conjuncts.len() { - // No condition (a CROSS JOIN) is unconditionally true. - 0 => Scalar::Literal(ScalarValue::Boolean(true)), - 1 => conjuncts.pop().unwrap(), - _ => Scalar::BoolAnd(conjuncts), - }); - Ok(Unresolved::Join { - kind, - pred, - left: Rc::new(self.lower_plan(&join.left)?), - right: Rc::new(self.lower_plan(&join.right)?), - }) - } - - /// `func(args) OVER (PARTITION BY … ORDER BY … ROWS/RANGE BETWEEN …)`. One - /// window function per plan node. - fn lower_window(&self, window: &logical_expr::Window) -> Result { - if window.window_expr.len() > 1 { - return Err(LoweringError::UnsupportedFeature(format!( - "multiple window functions in one plan node (got {}); split them", - window.window_expr.len() - ))); - } - let child = Rc::new(self.lower_plan(&window.input)?); - let first = window - .window_expr - .first() - .ok_or_else(|| LoweringError::InvalidExpression("empty window expression".into()))?; - let first = match first { - Expr::Alias(alias) => alias.expr.as_ref(), - other => other, - }; - let Expr::WindowFunction(wf) = first else { - return Err(LoweringError::InvalidExpression( - "expected a window function in Window plan node".into(), - )); - }; - let func = lower_window_func_kind(&wf.fun)?; - let mut args = wf - .args - .iter() - .map(|e| self.lower_expr(e)) - .collect::, _>>()?; - // Nth_value: lift N from the (literal) 2nd arg, keep only the column. - let func = if matches!(func, WindowFuncKind::NthValue(None)) { - let n = match args.get(1) { - Some(Scalar::Literal(ScalarValue::Int64(n))) if *n > 0 => *n as u64, - other => { - return Err(LoweringError::InvalidExpression(format!( - "NTH_VALUE requires a positive integer literal 2nd arg, got {other:?}" - ))) - } - }; - args.truncate(1); - WindowFuncKind::NthValue(Some(n)) - } else { - func - }; - let partition_by = wf - .partition_by - .iter() - .map(expr_to_group_ref) - .collect::, _>>()?; - let order_by = wf - .order_by - .iter() - .map(|s| { - self.lower_expr(&s.expr).map(|expr| SortKey { - expr, - ascending: s.asc, - nulls_first: s.nulls_first, - }) - }) - .collect::, _>>()?; - let frame = lower_window_frame(&wf.window_frame)?; - // The window plan's schema is `[input fields …, window output]`; the last - // field is the window column's name (what an enclosing Project references). - let output_name = window - .schema - .fields() - .last() - .map(|f| f.name().clone()) - .unwrap_or_else(|| "window".into()); - Ok(Unresolved::SQLWindowFunc { - func, - args, - partition_by: partition_by.into(), - order_by, - frame: Some(frame), - output_name, - child, - }) - } - - fn lower_projection( - &self, - proj: &logical_expr::Projection, - ) -> Result { - if let Some(bridge) = planning_bridge(proj)? { - let input = self.lower_plan(&proj.input)?; - return Ok(match bridge { - PlanningBridge::PromqlSubquery { range, resolution } => { - let child = Rc::new(self.temporal_bridge_projection(proj, input)?); - Unresolved::PromqlSubquery { - range, - resolution: Some(resolution), - child, - } - } - PlanningBridge::HistogramQuantile { q } => Unresolved::Aggregate { - // The marker is the projection's only column: one histogram. - reduction: Reduction::Reduce(GroupKeys::none()), - measures: vec![AggIntent::HistogramQuantile { - q, - le: ColumnRef::Named("le".into()), - }], - output_names: vec!["value".into()], - filters: vec![], - having: None, - child: Rc::new(input), - }, - }); - } - // SELECT * — no column constraint; pass through without a Project. - if proj.expr.iter().any(|e| matches!(e, Expr::Wildcard { .. })) { - return self.lower_plan(&proj.input); - } - let child = Rc::new(self.lower_plan(&proj.input)?); - let temporal_input = plan_has_temporal_aggregate(&proj.input); - let cols = proj - .expr - .iter() - .map(|e| match e { - Expr::Alias(a) => { - let expr = if temporal_input && is_temporal_output_column(&a.expr) { - Scalar::Column(ColumnRef::Named("value".into())) - } else { - self.lower_expr(&a.expr)? - }; - Ok::(ProjectItem { - expr, - alias: Some(a.name.clone()), - }) - } - _ => { - let expr = if temporal_input && is_temporal_output_column(e) { - Scalar::Column(ColumnRef::Named("value".into())) - } else { - self.lower_expr(e)? - }; - Ok::(ProjectItem { expr, alias: None }) - } - }) - .collect::, _>>()?; - Ok(Unresolved::Project { - cols, - qualifier: None, - child, - }) - } - - fn lower_aggregate(&self, agg: &logical_expr::Aggregate) -> Result { - let input = self.lower_plan(&agg.input)?; - // Each measure's row predicate (`FILTER (WHERE …)`, or the NULL-skip - // a `count(expr)` implies), read off the original typed expression - // before derived-column rewriting erases the argument's nullability. - let measure_filters = agg - .aggr_expr - .iter() - .map(|e| measure_filter(e, agg.input.schema())) - .collect::, LoweringError>>()?; - - if agg.aggr_expr.iter().any(is_temporal_aggregate) { - if measure_filters.iter().any(Option::is_some) { - return Err(LoweringError::UnsupportedFeature( - "FILTER on an ASAP temporal aggregate".into(), - )); - } - return self.lower_temporal_aggregate(agg, input); - } - - // `GROUPING SETS`/`ROLLUP`/`CUBE` emit several grouping levels from one - // scan. `Aggregate.by` is a single key set, so each level becomes its own - // `Aggregate` and they are merged (issue #118). - if let Some(gs) = agg.group_expr.iter().find_map(as_grouping_set) { - if measure_filters.iter().any(Option::is_some) { - return Err(LoweringError::UnsupportedFeature( - "FILTER on a measure inside a multi-level grouping".into(), - )); - } - return self.lower_grouping_sets(agg, gs, input); - } - - // `Aggregate.by` and the reducers index *columns*, so a grouping or - // reducer expression (`GROUP BY date_trunc(…)`, `SUM(a * 8)`) has no - // slot. Materialize each one as a derived column in a `Project` beneath - // the aggregate, then group/reduce over that column (issue #110). - let mut derived = DerivedCols::new(self); - - // DataFusion strips `AS m` from a grouping expression, so the aggregate - // schema's field name is what the enclosing Projection references — - // the derived column has to carry exactly that name. - let group_names: Vec = agg - .schema - .fields() - .iter() - .take(agg.group_expr.len()) - .map(|f| f.name().to_string()) - .collect(); - - let mut keys = Vec::with_capacity(agg.group_expr.len()); - for (i, e) in agg.group_expr.iter().enumerate() { - match unalias(e) { - Expr::Column(_) => { - derived.passthrough(e)?; - keys.push(expr_to_group_ref(e)?); - } - other => { - let name = group_names - .get(i) - .cloned() - .unwrap_or_else(|| other.to_string()); - derived.materialize(name.clone(), self.lower_expr(other)?)?; - keys.push(ColumnRef::Named(name)); - } - } - } - - // Reducer arguments get the same treatment; `rewrite_agg` returns the - // aggregate with its argument repointed at the derived column. - let aggr_expr = agg - .aggr_expr - .iter() - .map(|e| derived.rewrite_agg(e)) - .collect::, LoweringError>>()?; - // A measure filter reads the aggregate's input rows, so the columns - // it names must survive any derived-column `Project` inserted below. - for column in measure_filters.iter().flatten().flat_map(Expr::column_refs) { - derived.passthrough(&Expr::Column(column.clone()))?; - } - - let child = Rc::new(derived.wrap(input)?); - // DataFusion names the aggregate outputs in its own schema (e.g. - // "sum(metrics.bytes)") — the same names the enclosing Projection - // references. The schema is [group fields …, aggregate fields …], so - // skip the group fields and thread the rest straight through as - // `Aggregate.output_names`, letting that Projection resolve them. - let output_names: Vec = agg - .schema - .fields() - .iter() - .skip(agg.group_expr.len()) - .map(|f| f.name().to_string()) - .collect(); - let measures = aggr_expr - .iter() - .map(lower_agg_intent) - .collect::, LoweringError>>()?; - // Empty when nothing is filtered — the one canonical unfiltered shape. - let filters = if measure_filters.iter().any(Option::is_some) { - measure_filters - .iter() - .map(|f| { - f.as_ref() - .map(|f| Ok(Predicate(self.lower_expr(f)?))) - .transpose() - }) - .collect::, LoweringError>>()? - } else { - Vec::new() - }; - Ok(Unresolved::Aggregate { - // SQL `GROUP BY` is always an inclusion list, never PromQL's - // `without(...)` exclusion form — and always a genuine reduction, - // never `PerEntity` (there's no windowed/subquery-child concept - // in SQL for that to apply to). - reduction: Reduction::Reduce(GroupKeys::by(keys)), - measures, - output_names, - filters, - having: None, - child, - }) - } - - fn lower_temporal_aggregate( - &self, - agg: &logical_expr::Aggregate, - input: Unresolved, - ) -> Result { - if agg.aggr_expr.len() != 1 { - return Err(LoweringError::UnsupportedFeature( - "an ASAP temporal aggregate cannot share an Aggregate node with another reducer" - .into(), - )); - } - let Expr::AggregateFunction(call) = unalias(&agg.aggr_expr[0]) else { - unreachable!("is_temporal_aggregate accepted a non-aggregate expression") - }; - let name = call.func.name().to_lowercase(); - let [value, timestamp, window] = call.args.as_slice() else { - unreachable!("ASAP temporal UDAF signatures require exactly three arguments") - }; - - let value_ref = reducer_col(&name, std::slice::from_ref(value))?; - let timestamp_ref = reducer_col(&name, std::slice::from_ref(timestamp))?; - let Expr::Literal(window) = unalias(window) else { - return Err(LoweringError::InvalidExpression(format!( - "{name} window_ms must be a positive integer literal" - ))); - }; - let window_ms = scalar_positive_u64(window).ok_or_else(|| { - LoweringError::InvalidExpression(format!( - "{name} window_ms must be a positive integer literal" - )) - })?; - - let input_schema = resolve_root(&input)?.schema.clone(); - let timestamp_id = resolve_column_ref(×tamp_ref, &input_schema).map_err(|error| { - LoweringError::InvalidExpression(format!("{name} timestamp argument: {error}")) - })?; - if input_schema.time_index != Some(timestamp_id) { - return Err(LoweringError::InvalidExpression(format!( - "{name} timestamp argument must name the input schema's time-index column" - ))); - } - let value_id = resolve_column_ref(&value_ref, &input_schema).map_err(|error| { - LoweringError::InvalidExpression(format!("{name} value argument: {error}")) - })?; - if value_id == timestamp_id - || !matches!( - input_schema.fields[value_id].dtype, - FieldDataType::Plain(DataType::Int64 | DataType::Float64) - ) - { - return Err(LoweringError::InvalidExpression(format!( - "{name} value argument must name a numeric non-time column" - ))); - } - - let mut group_ids = Vec::with_capacity(agg.group_expr.len()); - let mut group_refs = Vec::with_capacity(agg.group_expr.len()); - for group in &agg.group_expr { - let group_ref = expr_to_group_ref(group)?; - let group_id = resolve_column_ref(&group_ref, &input_schema).map_err(|error| { - LoweringError::InvalidExpression(format!("{name} GROUP BY column: {error}")) - })?; - if group_id == timestamp_id || group_id == value_id { - return Err(LoweringError::InvalidExpression(format!( - "{name} GROUP BY cannot contain its timestamp or value column" - ))); - } - if group_ids.contains(&group_id) { - return Err(LoweringError::InvalidExpression(format!( - "{name} GROUP BY contains the same resolved column more than once" - ))); - } - group_ids.push(group_id); - group_refs.push(group_ref); - } - // Minimal series-identity contract without adding SQL-only metadata to - // the shared Schema: a declared row-unique key must contain the time - // index, and removing that index yields the complete series key. The - // GROUP BY must match that key exactly. A unique key that omits time is - // only row identity and proves nothing about time-series continuity. - let identifies_one_series = input_schema - .unique_keys - .iter() - .filter(|key| key.contains(×tamp_id)) - .any(|key| { - let mut series_key: Vec<_> = key - .iter() - .copied() - .filter(|id| *id != timestamp_id) - .collect(); - series_key.sort_unstable(); - series_key.dedup(); - let mut grouped = group_ids.clone(); - grouped.sort_unstable(); - series_key == grouped - }); - if !identifies_one_series { - return Err(LoweringError::InvalidExpression(format!( - "{name} GROUP BY must exactly match a declared series identity (a unique key without the time index)" - ))); - } - - let mut cols = vec![ - ProjectItem { - alias: Some("ts".into()), - expr: Scalar::Column(timestamp_ref.clone()), - }, - ProjectItem { - alias: Some("value".into()), - expr: Scalar::Column(value_ref.clone()), - }, - ]; - for group_ref in group_refs { - let group_name = named_ref(&group_ref).to_string(); - cols.push(ProjectItem { - alias: Some(group_name), - expr: Scalar::Column(group_ref), - }); - } - let child = Unresolved::Project { - cols, - qualifier: None, - child: Rc::new(input), - }; - // The explicit window is a range selector over the series, the same - // shape PromQL's `rate(m[5m])` lowers to. - let child = Unresolved::TimeRange { - range: Duration::from_millis(window_ms), - kind: TimeRangeKind::Range, - child: Rc::new(child), - }; - let intent = match name.as_str() { - "asap_rate" => AggIntent::Rate, - "asap_increase" => AggIntent::Increase, - - _ => unreachable!("is_temporal_aggregate admitted {name}"), - }; - Ok(Unresolved::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(child), - }) - } - - /// `GROUP BY ROLLUP/CUBE/GROUPING SETS` — multi-level grouping (issue #118). - /// - /// One scan produces several grouping levels; `Aggregate.by` holds a single - /// key set. So each level becomes its own `Aggregate`, and the levels are - /// `Concat`ed. A level that omits a key still has to *emit* it — as `NULL`, per - /// SQL — so each branch is wrapped in a `Project` that reinstates the missing - /// keys as typed nulls and restores the canonical column order. That keeps - /// the branches union-compatible, which `Concat` requires (it derives its - /// schema from the first child). - /// - /// `Aggregate.child` is duplicated per level — the same trade - /// `histogram_quantiles` makes (#109); a future workload-level reuse pass - /// could hoist it back into a single producer. - /// - /// DataFusion's `__grouping_id` discriminator is dropped: it only exists to - /// tell a subtotal's `NULL` apart from a data `NULL`, which is observable - /// solely through `GROUPING(col)` — an aggregate this front end rejects. - fn lower_grouping_sets( - &self, - agg: &logical_expr::Aggregate, - gs: &logical_expr::GroupingSet, - input: Unresolved, - ) -> Result { - // DataFusion normalizes every mixed form (`GROUP BY g, ROLLUP(d)`) into a - // single `GroupingSets`, so one grouping expression is the only shape. - if agg.group_expr.len() != 1 { - return Err(LoweringError::UnsupportedFeature( - "a grouping set alongside plain GROUP BY keys".into(), - )); - } - - // `distinct_expr()` is ordered exactly like the aggregate's leading - // schema fields, which is the column order the enclosing Projection - // expects. The field after them is `__grouping_id`. - let distinct = gs.distinct_expr(); - for e in &distinct { - if !matches!(unalias(e), Expr::Column(_)) { - return Err(LoweringError::UnsupportedFeature(format!( - "non-column key inside a multi-level grouping: {e}" - ))); - } - } - let keys: Vec<(String, DataType)> = agg - .schema - .fields() - .iter() - .take(distinct.len()) - .map(|f| Ok((f.name().to_string(), arrow_to_dtype(f.data_type())?))) - .collect::>()?; - - let output_names: Vec = agg - .schema - .fields() - .iter() - .skip(distinct.len() + 1) // + `__grouping_id` - .map(|f| f.name().to_string()) - .collect(); - - // Reducer arguments still materialize as derived columns (#110); the - // grouping keys are plain columns, so they only need carrying through. - let mut derived = DerivedCols::new(self); - for e in &distinct { - derived.passthrough(e)?; - } - let aggr_expr = agg - .aggr_expr - .iter() - .map(|e| derived.rewrite_agg(e)) - .collect::, LoweringError>>()?; - let measures = aggr_expr - .iter() - .map(lower_agg_intent) - .collect::, LoweringError>>()?; - let input = derived.wrap(input)?; - - let branches = expand_grouping_set(gs) - .iter() - .map(|level| { - let level_keys = distinct - .iter() - .filter(|e| level.contains(e)) - .map(|e| expr_to_group_ref(e)) - .collect::, LoweringError>>()?; - let aggregate = Unresolved::Aggregate { - reduction: Reduction::Reduce(GroupKeys::by(level_keys)), - measures: measures.clone(), - output_names: output_names.clone(), - filters: vec![], - having: None, - child: Rc::new(input.clone()), - }; - // Reinstate omitted keys as typed nulls, in canonical order. - let cols = keys - .iter() - .zip(&distinct) - .map(|((name, dtype), e)| ProjectItem { - alias: Some(name.clone()), - expr: if level.contains(e) { - Scalar::Column(ColumnRef::Named(name.clone())) - } else { - Scalar::Cast { - expr: Box::new(Scalar::Literal(ScalarValue::Null)), - to: dtype.clone(), - try_cast: false, - } - }, - }) - .chain(output_names.iter().map(|n| ProjectItem { - alias: Some(n.clone()), - expr: Scalar::Column(ColumnRef::Named(n.clone())), - })) - .collect(); - Ok(Unresolved::Project { - cols, - qualifier: None, - child: Rc::new(aggregate), - }) - }) - .collect::, LoweringError>>()?; - - // No discriminator asserted here today (issue #228): DataFusion's own - // `__grouping_id` would be the natural one, but this front end - // already discards it (see above — `GROUPING()` itself is rejected), - // so there is no distinct-per-branch column available to name yet. - // `Unresolved::concat` keeps `output_schema`'s default (drop - // `unique_keys` entirely). See - // `docs/design_docs/concat-unique-keys-decision.md`. - Ok(Unresolved::concat(branches)) - } - - fn lower_sort(&self, sort: &logical_expr::Sort) -> Result { - // A count-ranked `ORDER BY … LIMIT k` is the frequency heavy-hitter the - // `TopK` intent represents, but that promotion now happens in the shared - // `canonicalize` pass (issue #34) — the same one both front ends run — - // so SQL emits a plain `Sort` (+ `Limit`) here and lets canonicalization - // recognise the count-ranked shape positionally. This removes the gate's - // alias blind spot (#20). - let keys = sort - .expr - .iter() - .map(|s| { - self.lower_expr(&s.expr).map(|expr| SortKey { - expr, - ascending: s.asc, - nulls_first: s.nulls_first, - }) - }) - .collect::, _>>()?; - Ok(Unresolved::Sort { - keys, - // SQL `ORDER BY` is a global sort; per-group ranking would come from a - // window function (`SQLWindowFunc`), not a bare Sort. - partition_by: GroupKeys::none(), - child: Rc::new(self.lower_plan(&sort.input)?), - }) - } - - fn lower_limit(&self, limit: &logical_expr::Limit) -> Result { - // Count-ranked `LIMIT k` over a `Sort` is promoted to the heavy-hitter - // `TopK` by the shared `canonicalize` pass (issue #34), not here. - Ok(Unresolved::Limit { - // No (literal) fetch is offset-only. - n: eval_fetch(&limit.fetch), - offset: eval_fetch(&limit.skip).unwrap_or(0), - partition_by: GroupKeys::none(), - child: Rc::new(self.lower_plan(&limit.input)?), - }) - } -} - -/// A deliberately explicit marker accepted only in a projection of planning -/// SQL. The marker describes a relation operator, so it is removed rather than -/// lowered to the ordinary scalar `FunctionCall` variant. -enum PlanningBridge { - PromqlSubquery { - range: Duration, - resolution: Duration, - }, - HistogramQuantile { - q: f64, - }, -} - -fn planning_bridge( - projection: &logical_expr::Projection, -) -> Result, LoweringError> { - let mut found = None; - for expr in &projection.expr { - let Expr::ScalarFunction(call) = unalias(expr) else { - continue; - }; - let name = call.func.name().to_ascii_lowercase(); - let bridge = match name.as_str() { - "asap_promql_subquery" => { - let [range, resolution] = call.args.as_slice() else { - return Err(LoweringError::InvalidExpression( - "asap_promql_subquery requires (range_ms, resolution_ms)".into(), - )); - }; - let range = positive_millis_literal(range, "range_ms")?; - let resolution = positive_millis_literal(resolution, "resolution_ms")?; - PlanningBridge::PromqlSubquery { range, resolution } - } - "asap_histogram_quantile" => { - let [q] = call.args.as_slice() else { - return Err(LoweringError::InvalidExpression( - "asap_histogram_quantile requires one literal quantile".into(), - )); - }; - let q = float_literal(q).ok_or_else(|| { - LoweringError::InvalidExpression( - "asap_histogram_quantile quantile must be a numeric literal".into(), - ) - })?; - if !q.is_finite() || !(0.0..=1.0).contains(&q) { - return Err(LoweringError::InvalidExpression(format!( - "asap_histogram_quantile quantile must be finite and in [0,1], got {q}" - ))); - } - PlanningBridge::HistogramQuantile { q } - } - _ => continue, - }; - if found.is_some() { - return Err(LoweringError::InvalidExpression( - "a planning projection may contain only one asap_* relation marker".into(), - )); - } - found = Some(bridge); - } - if matches!(found, Some(PlanningBridge::HistogramQuantile { .. })) && projection.expr.len() != 1 - { - return Err(LoweringError::InvalidExpression( - "asap_histogram_quantile must be the projection's only expression".into(), - )); - } - Ok(found) -} - -/// Rebuild the SQL projection around the relation sampled by the temporal -/// marker. The marker's alias names the existing child column that occupies -/// its output slot (`... asap_promql_subquery(...) AS value ...`). This makes -/// the bridge schema-preserving without silently retaining columns that SQL -/// projected away. -impl SqlLowerer<'_> { - fn temporal_bridge_projection( - &self, - projection: &logical_expr::Projection, - child: Unresolved, - ) -> Result { - let cols = projection - .expr - .iter() - .map(|expr| { - if let Expr::ScalarFunction(call) = unalias(expr) { - if call - .func - .name() - .eq_ignore_ascii_case("asap_promql_subquery") - { - let Expr::Alias(alias) = expr else { - return Err(LoweringError::InvalidExpression( - "asap_promql_subquery must have an alias naming its child value column" - .into(), - )); - }; - return Ok(ProjectItem { - expr: Scalar::Column(ColumnRef::Named(alias.name.clone())), - alias: Some(alias.name.clone()), - }); - } - } - match expr { - Expr::Alias(alias) => Ok(ProjectItem { - expr: self.lower_expr(&alias.expr)?, - alias: Some(alias.name.clone()), - }), - other => Ok(ProjectItem { - expr: self.lower_expr(other)?, - alias: None, - }), - } - }) - .collect::, LoweringError>>()?; - Ok(Unresolved::Project { - cols, - qualifier: None, - child: Rc::new(child), - }) - } -} - -fn positive_millis_literal(expr: &Expr, argument: &str) -> Result { - let millis = match unalias(expr) { - Expr::Literal(DfScalarValue::Int64(Some(value))) if *value > 0 => *value as u64, - Expr::Literal(DfScalarValue::UInt64(Some(value))) if *value > 0 => *value, - Expr::Literal(DfScalarValue::Int32(Some(value))) if *value > 0 => *value as u64, - other => { - return Err(LoweringError::InvalidExpression(format!( - "{argument} must be a positive integer millisecond literal, got {other}" - ))) - } - }; - Ok(Duration::from_millis(millis)) -} - -fn float_literal(expr: &Expr) -> Option { - match unalias(expr) { - Expr::Literal(DfScalarValue::Float64(Some(value))) => Some(*value), - Expr::Literal(DfScalarValue::Float32(Some(value))) => Some(*value as f64), - Expr::Literal(DfScalarValue::Int64(Some(value))) => Some(*value as f64), - Expr::Literal(DfScalarValue::UInt64(Some(value))) => Some(*value as f64), - Expr::Literal(DfScalarValue::Int32(Some(value))) => Some(*value as f64), - _ => None, - } -} - -// ── ClickHouse-builtin compatibility, taught to DataFusion itself ────────────── -// -// Generalized over `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` (issue -// #225): adding support for one more ClickHouse-only builtin DataFusion -// doesn't know at all is a catalog data entry (name, arity, `RewriteKind`) -// plus, only if its rewrite target is a genuinely new shape, one match arm -// in `ClickHouseBuiltinRewrite::rewrite` below — never a new stub-UDAF -// constructor or a new `FunctionRewrite`-implementing type. `uniqExact` -// (issue #221) and `countIf` both go through this one mechanism. - -/// A stub `AggregateUDF` for one `CLICKHOUSE_BUILTINS` entry, registered -/// purely so DataFusion's planner can resolve the function name during -/// `SqlToRel` conversion (it errors on an unknown function before a rewrite -/// ever gets a chance to run). Every call site is replaced by -/// `ClickHouseBuiltinRewrite` — via the `Analyzer` `lower()` runs after -/// parsing — before physical planning could ever ask this UDAF for an -/// `Accumulator`, so `accumulator` is unreachable for every catalog entry. -fn clickhouse_builtin_stub_udaf(name: &'static str, arity: Arity) -> AggregateUDF { - AggregateUDF::from(SimpleAggregateUDF::new_with_signature( - name, - arity_to_signature(arity), - ArrowDataType::Int64, - Arc::new(move |_| { - // ponytail: dead code by construction (see doc comment above) — - // a real accumulator would just reimplement whatever native - // shape `ClickHouseBuiltinRewrite` rewrites this call to. - unimplemented!( - "{name} has no accumulator: every call site is rewritten to a native \ - DataFusion aggregate before physical planning" - ) - }), - vec![], - )) -} - -/// A catalog [`Arity`] as the DataFusion `Signature` a stub UDAF/UDF is -/// registered with — shared by the aggregate stub above and the scalar stub -/// below, since neither wants to model per-argument types, only how many -/// arguments a call may take. -fn arity_to_signature(arity: Arity) -> Signature { - match arity { - Arity::Exact(n) => Signature::any(n, Volatility::Immutable), - Arity::Range { min, max } => Signature::one_of( - (min..=max).map(TypeSignature::Any).collect(), - Volatility::Immutable, - ), - } -} - -// ── ClickHouse scalar-builtin compatibility ───────────────────────────────── -// -// The scalar counterpart of the aggregate mechanism above, but simpler: -// `asap_sql_function_catalog::CLICKHOUSE_SCALAR_BUILTINS` carries no -// `RewriteKind`, because a scalar call needs none. Unlike an aggregate call -// (which must become a real `AggIntent`, hence the rewrite to a native -// DataFusion aggregate shape `lower_agg_intent` can classify), a scalar -// function call in this IR is already deliberately opaque — -// `SqlLowerer::lower_expr`'s `Expr::ScalarFunction` arm lowers *any* -// scalar call generically to `UnresolvedScalar::FunctionCall { name, args }`, with -// zero name-specific logic. So teaching DataFusion's planner to accept a -// ClickHouse scalar builtin's name — a stub `ScalarUDF`, registered below — -// is the entire fix; the existing generic lowering already does the rest. - -/// A stub `ScalarUDF` for one `CLICKHOUSE_SCALAR_BUILTINS` entry, registered -/// purely so DataFusion's planner can resolve the function name during -/// `SqlToRel` conversion (it errors on an unknown function otherwise), and so -/// it can keep building the surrounding expression's type from a plausible -/// return type. Unlike `clickhouse_builtin_stub_udaf`, no `FunctionRewrite` -/// ever fires for these — the call survives to `lower_plan` as-is and lowers -/// through the generic `Expr::ScalarFunction` arm — so `invoke`/`invoke_batch` -/// (left at their default, which returns a `NotImplemented` `DataFusionError`) -/// are unreachable for every catalog entry: this front end only ever uses -/// DataFusion for planning/type-checking, never physical execution. -fn clickhouse_scalar_builtin_stub_udf(name: &'static str, arity: Arity) -> ScalarUDF { - ScalarUDF::from(ClickHouseScalarBuiltinStub { - name, - signature: arity_to_signature(arity), - return_type: clickhouse_scalar_builtin_return_type(name), - }) -} - -/// A plausible Arrow return type for one `CLICKHOUSE_SCALAR_BUILTINS` entry — -/// just precise enough that DataFusion's planner can keep building the type -/// of whatever expression the call sits inside (e.g. a `WHERE` predicate -/// wants `Boolean`), not a claim about ClickHouse's actual return type. -/// Real function typing happens downstream, at post-ASAP binding. -fn clickhouse_scalar_builtin_return_type(name: &str) -> ArrowDataType { - match name { - // Array(String) in ClickHouse; a plain `Utf8` element list is close - // enough for planning purposes here. - "splitbychar" => ArrowDataType::List(Arc::new(datafusion::arrow::datatypes::Field::new( - "item", - ArrowDataType::Utf8, - true, - ))), - "todate" => ArrowDataType::Date32, - // ClickHouse returns UInt8 (0/1), but every corpus use is a boolean - // predicate — `Boolean` keeps that context type-checking. - "match" | "startswith" => ArrowDataType::Boolean, - "tostartofhour" - | "tostartofweek" - | "tostartofminute" - | "tostartoffiveminutes" - | "tostartofinterval" => { - ArrowDataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None) - } - // 1-based match position, 0 if not found. - "positioncaseinsensitive" => ArrowDataType::UInt64, - // Relation markers are removed by `lower_projection`; Float64 merely - // lets DataFusion type the temporary SELECT list. - "asap_promql_subquery" | "asap_histogram_quantile" => ArrowDataType::Float64, - other => unreachable!( - "{other}: every CLICKHOUSE_SCALAR_BUILTINS entry must have a return type listed here" - ), - } -} - -/// A stub `ScalarUDFImpl` carrying only what DataFusion's planner needs: -/// name, arity-only [`Signature`], and a fixed return type. `invoke`/ -/// `invoke_batch` are left at their trait defaults (a `NotImplemented` -/// `DataFusionError`) — see [`clickhouse_scalar_builtin_stub_udf`]'s doc for -/// why that is unreachable in practice. -#[derive(Debug)] -struct ClickHouseScalarBuiltinStub { - name: &'static str, - signature: Signature, - return_type: ArrowDataType, -} - -impl ScalarUDFImpl for ClickHouseScalarBuiltinStub { - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn name(&self) -> &str { - self.name - } - - fn signature(&self) -> &Signature { - &self.signature - } - - fn return_type( - &self, - _arg_types: &[ArrowDataType], - ) -> datafusion::common::Result { - Ok(self.return_type.clone()) - } -} - -// ── ClickHouse window-builtin compatibility ───────────────────────────────── -// -// The window counterpart of the scalar mechanism above: a stub `WindowUDF` -// registered purely so DataFusion's planner accepts the call name during -// `SqlToRel` conversion. No rewrite step follows — `lower_window_func_kind` -// already maps each `asap_sql_function_catalog::CLICKHOUSE_WINDOW_BUILTINS` -// name directly to its own `WindowFuncKind` variant (issue #267). - -/// A stub `WindowUDF` for one `CLICKHOUSE_WINDOW_BUILTINS` entry, registered -/// purely so DataFusion's planner can resolve the function name inside an -/// `OVER (...)` clause. This front end only ever uses DataFusion for -/// planning/type-checking, never physical execution, so -/// `partition_evaluator` (which physical execution alone would call) is -/// unreachable in practice. -fn clickhouse_window_builtin_stub_udwf(name: &'static str, arity: Arity) -> WindowUDF { - WindowUDF::from(ClickHouseWindowBuiltinStub { - name, - signature: arity_to_signature(arity), - }) -} - -/// A stub `WindowUDFImpl` carrying only what DataFusion's planner needs: -/// name, arity-only [`Signature`], and a field type derived from the first -/// argument (matching `lag`/`lead`'s own "output type = input type" -/// behavior). `partition_evaluator` is left `unimplemented!()` — see -/// [`clickhouse_window_builtin_stub_udwf`]'s doc for why that is unreachable. -#[derive(Debug)] -struct ClickHouseWindowBuiltinStub { - name: &'static str, - signature: Signature, -} - -impl WindowUDFImpl for ClickHouseWindowBuiltinStub { - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn name(&self) -> &str { - self.name - } - - fn signature(&self) -> &Signature { - &self.signature - } - - fn field(&self, field_args: WindowUDFFieldArgs) -> datafusion::common::Result { - let dtype = field_args.get_input_type(0).unwrap_or(ArrowDataType::Null); - Ok(Field::new(field_args.name(), dtype, true)) - } - - fn partition_evaluator( - &self, - _partition_evaluator_args: PartitionEvaluatorArgs, - ) -> datafusion::common::Result> { - let name = self.name; - unimplemented!( - "{name} has no partition evaluator: this front end never runs DataFusion's \ - physical planner, only SqlToRel + the unoptimized LogicalPlan" - ) - } -} - -/// Rewrites every `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` call to -/// the native DataFusion aggregate shape its entry's `RewriteKind` names — -/// so a ClickHouse-only builtin DataFusion doesn't know at all becomes an -/// ordinary DataFusion aggregate before the plan ever reaches -/// `lower_agg_intent`, which needs no ClickHouse-specific name of its own. -#[derive(Debug)] -struct ClickHouseBuiltinRewrite; - -impl FunctionRewrite for ClickHouseBuiltinRewrite { - fn name(&self) -> &str { - "clickhouse builtin -> native DataFusion aggregate" - } - - fn rewrite( - &self, - expr: Expr, - _schema: &DFSchema, - _config: &ConfigOptions, - ) -> datafusion::common::Result> { - let Expr::AggregateFunction(f) = expr else { - return Ok(Transformed::no(expr)); - }; - let Some(builtin) = asap_sql_function_catalog::lookup_clickhouse_builtin(f.func.name()) - else { - return Ok(Transformed::no(Expr::AggregateFunction(f))); - }; - let rewritten = match builtin.rewrite { - // No native DataFusion shape to become — leave the call exactly - // as DataFusion's planner parsed it. `lower_agg_intent` handles - // the ClickHouse name (`argMax`/`argMin`) directly (issue #232). - RewriteKind::PassThrough => return Ok(Transformed::no(Expr::AggregateFunction(f))), - // `f(args...)` -> `count(args...) DISTINCT` — `lower_agg_intent` - // already maps `count` + `DISTINCT` to `AggIntent::Cardinality`, - // at whatever arity the call carries. - RewriteKind::CountDistinct => AggregateFunction::new_udf( - count_udaf(), - f.args, - true, - f.filter, - f.order_by, - f.null_treatment, - ), - // `f(cond)` -> `sum(CASE WHEN cond THEN 1 ELSE 0 END)` — see - // `RewriteKind::CountIfToSum`'s doc; moving the `-If` family onto - // `Aggregate.filters` (issue #466) is a follow-up. - RewriteKind::CountIfToSum => { - let cond = f.args.into_iter().next().expect( - "countif's stub signature fixes its arity at 1 -- the planner \ - already rejected any other argument count before this rewrite runs", - ); - let indicator = Expr::Case(Case::new( - None, - vec![(Box::new(cond), Box::new(lit(1i64)))], - Some(Box::new(lit(0i64))), - )); - AggregateFunction::new_udf( - sum_udaf(), - vec![indicator], - false, - f.filter, - f.order_by, - f.null_treatment, - ) - } - }; - Ok(Transformed::yes(Expr::AggregateFunction(rewritten))) - } -} - -// ── Aggregate / group-key helpers ─────────────────────────────────────────────── - -/// The row predicate one aggregate call carries (issue #466): its explicit -/// `FILTER (WHERE p)`, plus — for a plain `count(expr)`, which canonical -/// `AggIntent::Count` lowers to a row count that never looks at `expr` — the -/// NULL-skipping SQL gives it. `count(CASE WHEN p THEN x END)` is the -/// conditional-count idiom, so it becomes `p [AND x IS NOT NULL]` rather -/// than the opaque `CASE … IS NOT NULL`; any other nullable argument becomes -/// `expr IS NOT NULL`. `None` when the call updates on every row. -fn measure_filter(expr: &Expr, input: &DFSchema) -> Result, LoweringError> { - let Expr::AggregateFunction(agg_fn) = unalias(expr) else { - return Ok(None); - }; - let mut conjuncts: Vec = agg_fn.filter.iter().map(|f| (**f).clone()).collect(); - let counts_rows = agg_fn.func.name().eq_ignore_ascii_case("count") && !agg_fn.distinct; - if counts_rows { - for argument in &agg_fn.args { - let nullable = argument - .nullable(input) - .map_err(|error| LoweringError::UnsupportedFeature(error.to_string()))?; - if !nullable { - continue; - } - match conditional_count_arm(argument) { - Some((when, then)) => { - conjuncts.push(when.clone()); - if then - .nullable(input) - .map_err(|error| LoweringError::UnsupportedFeature(error.to_string()))? - { - conjuncts.push(then.clone().is_not_null()); - } - } - None => conjuncts.push(argument.clone().is_not_null()), - } - } - } - Ok(conjuncts.into_iter().reduce(Expr::and)) -} - -/// `CASE WHEN p THEN x END` (searched, one arm, no `ELSE` or `ELSE NULL`) -/// as `(p, x)`. -fn conditional_count_arm(expr: &Expr) -> Option<(&Expr, &Expr)> { - let Expr::Case(case) = unalias(expr) else { - return None; - }; - if case.expr.is_some() { - return None; - } - let else_is_null = match case.else_expr.as_deref() { - None => true, - Some(Expr::Literal(value)) => value.is_null(), - Some(_) => false, - }; - if !else_is_null { - return None; - } - let [(when, then)] = case.when_then_expr.as_slice() else { - return None; - }; - Some((when, then)) -} - -/// Map a DataFusion aggregate expression directly to the canonical -/// [`AggIntent`] — issue #179's "dedicated function → canonical -/// intent directly" front-end construction, no `AggFunc` intermediate. The -/// name → semantic mapping itself lives in `asap_sql_function_catalog` -/// (issue #225) as flat data (`NATIVE_FUNCTIONS`); what stays here is -/// call-site logic that isn't a function of the name alone — the DISTINCT -/// modifier rule, the "reducer argument must be a bare column" rule -/// (`reducer_col`), φ extraction from a literal argument, and the ambient -/// `AccuracyTarget`. `resolve_root` resolves `col` to a positional -/// `ColumnId`; the output name (DataFusion's own, e.g. -/// `"sum(metrics.bytes)"`) is threaded separately as `Aggregate.output_names`, -/// not carried here. -fn lower_agg_intent(expr: &Expr) -> Result, LoweringError> { - match expr { - Expr::Alias(a) => lower_agg_intent(&a.expr), - Expr::AggregateFunction(agg_fn) => { - let name = agg_fn.func.name().to_lowercase(); - // ClickHouse's row-selecting `argMax`/`argMin` — `RewriteKind:: - // PassThrough` in the catalog, so the call reaches here under its - // own name rather than a native DataFusion aggregate. Handled - // before the `NATIVE_FUNCTIONS` lookup below since neither name - // is in that table (issue #232). - if let Some(intent) = lower_arg_selector(&name, &agg_fn.args)? { - return Ok(intent); - } - let semantic = asap_sql_function_catalog::lookup_native(&name) - .ok_or_else(|| LoweringError::UnsupportedAggregate(name.clone()))?; - // The canonical intent algebra has no DISTINCT modifier for the - // value reducers; only - // COUNT(DISTINCT) maps (to Cardinality). Reject DISTINCT elsewhere - // rather than silently lowering `SUM(DISTINCT x)` as `SUM(x)`. - if agg_fn.distinct && !matches!(semantic, AggSemantic::Count) { - return Err(LoweringError::UnsupportedAggregate(format!( - "DISTINCT {name}" - ))); - } - // Value reducers (`reducer_col`) require a real column — `SUM(a*b)` - // is rejected, not silently reduced over a probe column. Quantile - // and CountDistinct reduce a column too, so they take the same path: - // `col` is `Option` once resolved, where `None` means "the - // PromQL sample value", which a SQL query never has. Taking an - // expression here would set `col: None` and silently drop it (#115). - let col = |args: &[Expr]| -> Result, LoweringError> { - reducer_col(&name, args).map(Some) - }; - Ok(match semantic { - AggSemantic::Correlation => { - if agg_fn.order_by.is_some() || agg_fn.null_treatment.is_some() { - return Err(LoweringError::UnsupportedAggregate( - "corr with ORDER BY or explicit null treatment".into(), - )); - } - let [left, right] = agg_fn.args.as_slice() else { - return Err(LoweringError::UnsupportedAggregate( - "corr requires two arguments".into(), - )); - }; - AggIntent::PearsonCorr { - left: expr_to_group_ref(left)?, - right: expr_to_group_ref(right)?, - } - } - // Every argument reaches the intent: `COUNT(DISTINCT a, b)` - // counts distinct *tuples*, which is a different quantity from - // the distinct count of either column. - AggSemantic::Count if agg_fn.distinct => match agg_fn.args.as_slice() { - // DataFusion's planner rejects a bare `COUNT(DISTINCT)` - // before lowering. Guarded anyway: an empty `cols` is the - // PromQL sample-value convention, which SQL never has. - [] => { - return Err(LoweringError::UnsupportedAggregate( - "COUNT(DISTINCT) without an argument".into(), - )) - } - args => AggIntent::Cardinality { - cols: args.iter().map(distinct_col).collect::>()?, - accuracy: current_accuracy(), - }, - }, - AggSemantic::Count => AggIntent::Count { - accuracy: current_accuracy(), - }, - AggSemantic::Sum => AggIntent::Sum { - col: col(&agg_fn.args)?, - }, - AggSemantic::Min => AggIntent::Min { - col: col(&agg_fn.args)?, - }, - AggSemantic::Max => AggIntent::Max { - col: col(&agg_fn.args)?, - }, - AggSemantic::Avg => AggIntent::Avg { - col: col(&agg_fn.args)?, - }, - AggSemantic::StdDev { population } => AggIntent::StdDev { - col: col(&agg_fn.args)?, - population, - }, - AggSemantic::Variance { population } => AggIntent::Variance { - col: col(&agg_fn.args)?, - population, - }, - // `fixed_q = Some(0.5)` is `median`/`approx_median`. As with - // `approx_distinct` and `approx_percentile_cont`, the - // `approx_` prefix does not force an approximation: the - // sketch-vs-exact choice is the AccuracyTarget's (see - // `plan::boundary`), so both spellings share one intent - // (#111). - AggSemantic::Quantile { fixed_q } => AggIntent::Quantile { - col: col(&agg_fn.args)?, - q: match fixed_q { - Some(q) => q, - None => extract_percentile_q(&agg_fn.args)?, - }, - accuracy: current_accuracy(), - }, - AggSemantic::Cardinality => AggIntent::Cardinality { - cols: vec![reducer_col(&name, &agg_fn.args)?], - accuracy: current_accuracy(), - }, - }) - } - _ => Err(LoweringError::UnsupportedAggregate(format!( - "measure is not an aggregate function call: {expr}" - ))), - } -} - -fn temporal_aggregate_name(expr: &Expr) -> Option { - let Expr::AggregateFunction(call) = unalias(expr) else { - return None; - }; - let name = call.func.name().to_lowercase(); - matches!(name.as_str(), "asap_rate" | "asap_increase").then_some(name) -} - -fn is_temporal_aggregate(expr: &Expr) -> bool { - temporal_aggregate_name(expr).is_some() -} - -fn is_temporal_output_column(expr: &Expr) -> bool { - let Expr::Column(col) = unalias(expr) else { - return false; - }; - let name = col.name.to_lowercase(); - ["asap_rate(", "asap_increase("] - .iter() - .any(|prefix| name.starts_with(prefix)) -} - -fn plan_has_temporal_aggregate(plan: &LogicalPlan) -> bool { - match plan { - LogicalPlan::Aggregate(agg) => agg.aggr_expr.iter().any(is_temporal_aggregate), - LogicalPlan::Filter(filter) => plan_has_temporal_aggregate(&filter.input), - LogicalPlan::SubqueryAlias(alias) => plan_has_temporal_aggregate(&alias.input), - _ => false, - } -} - -fn named_ref(col: &ColumnRef) -> &str { - match col { - ColumnRef::Named(name) | ColumnRef::Qualified { name, .. } => name, - ColumnRef::SampleValue | ColumnRef::Wildcard => { - unreachable!("reducer_col only returns named column references") - } - } -} - -fn scalar_positive_u64(value: &DfScalarValue) -> Option { - match value { - DfScalarValue::Int64(Some(v)) if *v > 0 => Some(*v as u64), - DfScalarValue::Int32(Some(v)) if *v > 0 => Some(*v as u64), - DfScalarValue::UInt64(Some(v)) if *v > 0 => Some(*v), - DfScalarValue::UInt32(Some(v)) if *v > 0 => Some(*v as u64), - _ => None, - } -} - -/// ClickHouse's row-selecting `argMax(arg, val)` / `argMin(arg, val)` — -/// "return `arg`'s value from the row where `val` is maximal/minimal". -/// `Some(name)` for `"argmax"`/`"argmin"`, `None` for every other name (the -/// caller falls through to the ordinary `NATIVE_FUNCTIONS` path). -/// -/// Unlike every existing `AggIntent` reducer (`Sum`/`Min`/`Max`/`Avg`/…), -/// which folds *one* column to a value derived from itself, this is a -/// two-column, row-selecting aggregate: it returns a *different* column's -/// value, selected by which row maximizes/minimizes a second column. No -/// existing `AggIntent` shape fits, and — per its own doc comment's -/// "core only grows for intents ≥2 deployment models actually use" bar — -/// a repo-wide search (PromQL front end, the other SQL dialects, docs) found -/// no second deployment model wanting this shape, so this lowers to -/// `AggIntent::Extension` rather than earning a first-class `ArgMax`/`ArgMin` -/// core variant (issue #232). Core treats `Extension` opaquely: both columns -/// are kept only as validated bare-column names in `payload` (`reducer_col`'s -/// same "no expression arguments" rule, issue #115) — they are **not** run -/// through `resolve_agg_intent`'s positional `ColumnRef` -> `ColumnId` -/// binding the way a real reducer's `col` is, since `Extension` carries no -/// typed column field for core to resolve. Shared `arg_selector_columns` validates -/// and resolves those names during aggregate schema derivation, preserving the -/// selected argument's type and nullability for downstream exact execution. -/// -/// DerivedCols preserves both bare-column arguments when grouping expressions -/// introduce an intermediate Project. Shared aggregate schema derivation resolves -/// the payload and preserves the selected argument's type and nullability. -fn lower_arg_selector( - name: &str, - args: &[Expr], -) -> Result>, LoweringError> { - let ext_kind = match name { - "argmax" => "arg_max", - "argmin" => "arg_min", - _ => return Ok(None), - }; - let [arg, val] = args else { - unreachable!( - "{name}'s stub signature (asap_sql_function_catalog::CLICKHOUSE_BUILTINS) fixes \ - its arity at 2 -- the planner already rejected any other argument count before \ - lower_agg_intent runs" - ); - }; - let arg_col = reducer_col(name, std::slice::from_ref(arg))?; - let val_col = reducer_col(name, std::slice::from_ref(val))?; - Ok(Some(AggIntent::Extension { - ext_kind: ext_kind.to_string(), - payload: serde_json::json!({ "arg_col": arg_col, "val_col": val_col }), - })) -} - -/// Fold `pred` directly onto `child.predicates` when `child` is a bare `Scan` -/// (a `WHERE` directly over a table), otherwise wrap it in an ordinary -/// `Filter` — canonical's invariant that a `Filter` never sits directly over a -/// `Scan`. A front end emitting the canonical shape directly is responsible -/// for maintaining that invariant itself (issue #179). -fn filter_or_fold(pred: Scalar, child: Unresolved) -> Unresolved { - match child { - Unresolved::Scan { - source, - mut predicates, - schema, - } => { - predicates.push(Predicate(pred)); - Unresolved::Scan { - source, - predicates, - schema, - } - } - other => Unresolved::Filter { - pred: Predicate(pred), - child: Rc::new(other), - }, - } -} - -/// Flatten a top-level `AND` chain into its conjuncts. -fn split_conjunction<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) { - match expr { - Expr::BinaryExpr(b) if b.op == logical_expr::Operator::And => { - split_conjunction(&b.left, out); - split_conjunction(&b.right, out); - } - other => out.push(other), - } -} - -/// Whether `expr` reads another operator anywhere inside it (`EXISTS`, -/// `IN (…)`, a scalar subquery). -fn reads_subquery(expr: &Expr) -> bool { - expr.exists(|e| { - Ok(matches!( - e, - Expr::ScalarSubquery(_) | Expr::InSubquery(_) | Expr::Exists(_) - )) - }) - .expect("the predicate never fails") -} - -/// Re-`AND` the conjuncts, or `None` when there are none left. -fn rebuild_conjunction(conjuncts: &[&Expr]) -> Option { - conjuncts - .iter() - .map(|e| (*e).clone()) - .reduce(|acc, e| acc.and(e)) -} - -/// Split a correlated subquery's plan into `(uncorrelated plan, correlation)`. -/// -/// The correlation is the conjunction of the filter conjuncts that mention an -/// outer column, rewritten so `outer_ref(t.c)` becomes a plain `t.c` — it then -/// resolves against the join's concatenated `left ++ right` schema, like any -/// other join predicate. Everything else stays an ordinary inner `Filter`. -/// -/// An outer reference anywhere but a top-level filter conjunct is rejected: it -/// would need real decorrelation, not a predicate lift. -fn split_correlation(plan: &LogicalPlan) -> Result<(LogicalPlan, Option), LoweringError> { - let LogicalPlan::Filter(filter) = plan else { - return if plan_has_outer_ref(plan) { - Err(LoweringError::UnsupportedFeature( - "correlated subquery whose outer reference is not a filter conjunct".into(), - )) - } else { - Ok((plan.clone(), None)) - }; - }; - - let mut conjuncts = Vec::new(); - split_conjunction(&filter.predicate, &mut conjuncts); - let (correlated, inner): (Vec<_>, Vec<_>) = - conjuncts.into_iter().partition(|e| expr_has_outer_ref(e)); - - let input = filter.input.as_ref(); - if plan_has_outer_ref(input) { - return Err(LoweringError::UnsupportedFeature( - "correlated subquery whose outer reference is below its filter".into(), - )); - } - - let correlation = rebuild_conjunction(&correlated) - .map(|e| strip_outer_refs(&e)) - .transpose()?; - let plan = match rebuild_conjunction(&inner) { - Some(pred) => LogicalPlan::Filter( - logical_expr::Filter::try_new(pred, filter.input.clone()) - .map_err(LoweringError::DataFusion)?, - ), - None => input.clone(), - }; - Ok((plan, correlation)) -} - -/// Rewrite `outer_ref(t.c)` to `t.c` so the expression resolves against the -/// join's concatenated schema. -fn strip_outer_refs(expr: &Expr) -> Result { - expr.clone() - .transform(|e| { - Ok(match e { - Expr::OuterReferenceColumn(_, col) => Transformed::yes(Expr::Column(col)), - other => Transformed::no(other), - }) - }) - .map(|t| t.data) - .map_err(LoweringError::DataFusion) -} - -fn expr_has_outer_ref(expr: &Expr) -> bool { - let mut found = false; - expr.apply(|e| { - if matches!(e, Expr::OuterReferenceColumn(..)) { - found = true; - return Ok(TreeNodeRecursion::Stop); - } - Ok(TreeNodeRecursion::Continue) - }) - .expect("infallible visitor"); - found -} - -fn plan_has_outer_ref(plan: &LogicalPlan) -> bool { - let mut found = false; - plan.apply(|p| { - if p.expressions().iter().any(expr_has_outer_ref) { - found = true; - return Ok(TreeNodeRecursion::Stop); - } - Ok(TreeNodeRecursion::Continue) - }) - .expect("infallible visitor"); - found -} - -/// Strip `AS alias` wrappers. -fn unalias(expr: &Expr) -> &Expr { - match expr { - Expr::Alias(a) => unalias(&a.expr), - other => other, - } -} - -/// The `GroupingSet` inside a grouping expression, if any. -fn as_grouping_set(expr: &Expr) -> Option<&logical_expr::GroupingSet> { - match unalias(expr) { - Expr::GroupingSet(gs) => Some(gs), - _ => None, - } -} - -/// The grouping levels a `GroupingSet` stands for, widest first (issue #118). -/// -/// `ROLLUP(a, b)` → `(a,b), (a), ()` — the prefixes. -/// `CUBE(a, b)` → `(a,b), (a), (b), ()` — the power set. -/// `GROUPING SETS` is already the explicit list. -fn expand_grouping_set(gs: &logical_expr::GroupingSet) -> Vec> { - match gs { - logical_expr::GroupingSet::Rollup(exprs) => (0..=exprs.len()) - .rev() - .map(|n| exprs[..n].to_vec()) - .collect(), - logical_expr::GroupingSet::Cube(exprs) => { - // Bitmask descending, so the full set leads and `()` trails. - (0..(1u32 << exprs.len())) - .rev() - .map(|mask| { - exprs - .iter() - .enumerate() - .filter(|(i, _)| mask & (1 << i) != 0) - .map(|(_, e)| e.clone()) - .collect() - }) - .collect() - } - logical_expr::GroupingSet::GroupingSets(sets) => sets.clone(), - } -} - -/// Derived columns materialized in a `Project` beneath an `Aggregate` (#110). -/// -/// `Aggregate.by` holds positional `ColumnId`s and each reducer holds one input -/// column, so neither can hold an expression. `GROUP BY date_trunc('minute', t)` -/// and `SUM(bytes * 8)` are therefore rewritten to group/reduce over a projected -/// column that carries the expression's value. -/// -/// The projection also has to carry through the plain columns the aggregate -/// still references, since a `Project` replaces its child's schema rather than -/// extending it. -struct DerivedCols<'l> { - lowerer: &'l SqlLowerer<'l>, - cols: Vec, - /// Whether any column is genuinely derived. Without one the aggregate keeps - /// its original child, so trees that lower today keep their exact shape. - any: bool, - /// First same-name-different-value collision, reported only if the - /// projection is actually inserted (see [`Self::wrap`]). - collision: Option, -} - -impl<'l> DerivedCols<'l> { - fn new(lowerer: &'l SqlLowerer<'l>) -> Self { - Self { - lowerer, - cols: Vec::new(), - any: false, - collision: None, - } - } - - /// Add `alias := expr`, or note a collision if `alias` already means - /// something else. `Project` carries one relation qualifier for all its - /// columns, so `a.k` and `b.k` cannot both survive it — but that only - /// matters when a projection gets inserted at all. - fn push(&mut self, alias: String, expr: Scalar) { - let existing = self - .cols - .iter() - .find(|c| c.alias.as_deref() == Some(&alias)); - match existing { - // Same name, same value — one projected column serves both uses. - Some(e) if e.expr == expr => {} - Some(_) => { - self.collision.get_or_insert(alias); - } - None => self.cols.push(ProjectItem { - alias: Some(alias), - expr, - }), - } - } - - /// A plain column the aggregate references — carried through unchanged. - fn passthrough(&mut self, expr: &Expr) -> Result<(), LoweringError> { - let Expr::Column(c) = unalias(expr) else { - return Ok(()); - }; - self.push(c.name.clone(), self.lowerer.lower_expr(expr)?); - Ok(()) - } - - /// A genuinely derived column: `alias` now names `expr`'s value. - fn materialize(&mut self, alias: String, expr: Scalar) -> Result<(), LoweringError> { - self.any = true; - self.push(alias, expr); - Ok(()) - } - - /// Repoint a reducer's argument at a derived column when it is an - /// expression; otherwise carry its plain input column through. - fn rewrite_agg(&mut self, expr: &Expr) -> Result { - let Expr::AggregateFunction(agg_fn) = unalias(expr) else { - return Ok(expr.clone()); - }; - if matches!( - asap_sql_function_catalog::lookup_native(&agg_fn.func.name().to_lowercase()), - Some(AggSemantic::Correlation) - ) { - // Give each value argument its own projected name, including casts - // and qualified columns. This retains both inputs and avoids losing - // relation qualifiers when the projection becomes an unqualified schema. - let mut rewritten = agg_fn.clone(); - for arg in &mut rewritten.args { - let alias = unalias(arg).to_string(); - self.materialize(alias.clone(), self.lowerer.lower_expr(arg)?)?; - *arg = Expr::Column(DfColumn::new_unqualified(alias)); - } - return Ok(Expr::AggregateFunction(rewritten)); - } - // `COUNT(*)` reduces no column; `agg_col_name` covers bare/aliased/cast - // columns, so `None` here means the argument really is an expression. - let counts_rows = agg_fn.func.name().eq_ignore_ascii_case("count") && !agg_fn.distinct; - let Some(arg) = agg_fn.args.first() else { - return Ok(expr.clone()); - }; - if counts_rows { - return Ok(expr.clone()); - } - // Preserve every additional column dependency (e.g. argMax's ordering - // column) when an unrelated grouping expression creates a Project. - // Literal parameters need no source column and remain untouched. - for argument in agg_fn.args.iter().skip(1) { - self.passthrough(argument)?; - } - match agg_col_name(&agg_fn.args) { - Some(name) => { - self.push(name, self.lowerer.lower_expr(arg)?); - Ok(expr.clone()) - } - None => { - let alias = unalias(arg).to_string(); - self.materialize(alias.clone(), self.lowerer.lower_expr(arg)?)?; - let mut agg_fn = agg_fn.clone(); - agg_fn.args[0] = Expr::Column(DfColumn::new_unqualified(alias)); - Ok(Expr::AggregateFunction(agg_fn)) - } - } - } - - /// Wrap `input` in the materializing `Project`, or return it untouched when - /// nothing needed deriving — so a query that lowers today keeps its exact - /// tree, and a name collision that the projection would have flattened only - /// matters once the projection exists. - fn wrap(self, input: Unresolved) -> Result { - if !self.any { - return Ok(input); - } - if let Some(alias) = self.collision { - return Err(LoweringError::UnsupportedFeature(format!( - "ambiguous column `{alias}` beneath an expression GROUP BY / \ - aggregate — alias the relations apart" - ))); - } - Ok(Unresolved::Project { - cols: self.cols, - qualifier: None, - child: Rc::new(input), - }) - } -} - -/// The first aggregate argument's column name (bare / aliased / cast column), -/// or `None` for `*` / a non-column expression. -fn agg_col_name(args: &[Expr]) -> Option { - fn col_name(e: &Expr) -> Option { - match e { - Expr::Column(c) => Some(c.name.clone()), - Expr::Alias(a) => col_name(&a.expr), - Expr::Cast(c) => col_name(&c.expr), - _ => None, - } - } - args.first().and_then(col_name) -} - -/// The single input column of a value reducer (`SUM`/`MIN`/`MAX`/`AVG`/stddev/ -/// variance/quantile/count-distinct). Errors if the argument is not a column: -/// the canonical `AggIntent` reduces a column, not an arbitrary expression -/// (`SUM(a*b)`), so silently picking a probe column would compute the wrong -/// result. -fn reducer_col(name: &str, args: &[Expr]) -> Result { - agg_col_name(args).map(ColumnRef::Named).ok_or_else(|| { - LoweringError::UnsupportedAggregate(format!("{name} over a non-column expression")) - }) -} - -/// One argument of a `COUNT(DISTINCT ...)`. Resolved the way a grouping key is -/// — what is being counted is an identity, and its qualifier has to survive a -/// join (`a.k` vs `b.k`) — but reported as an aggregate restriction, since an -/// aggregate call is what the user wrote. -fn distinct_col(expr: &Expr) -> Result { - expr_to_group_ref(expr).map_err(|_| { - LoweringError::UnsupportedAggregate( - "COUNT(DISTINCT ...) over a non-column expression".into(), - ) - }) -} - -fn expr_to_group_ref(expr: &Expr) -> Result { - match expr { - // Preserve the relation qualifier so a GROUP BY / PARTITION BY key over a - // join (`b.k` vs `a.k`) resolves to the correct side — the same rule the - // scalar predicate path uses (`lower_expr`). - Expr::Column(col) => Ok(match &col.relation { - Some(rel) => ColumnRef::Qualified { - table: rel.to_string(), - name: col.name.clone(), - }, - None => ColumnRef::Named(col.name.clone()), - }), - Expr::Alias(a) => expr_to_group_ref(&a.expr), - other => Err(LoweringError::UnsupportedFeature(format!( - "non-column GROUP BY expression: {other}" - ))), - } -} - -fn extract_percentile_q(args: &[Expr]) -> Result { - let q = match args.get(1) { - Some(Expr::Literal(DfScalarValue::Float64(Some(q)))) => *q, - Some(Expr::Literal(DfScalarValue::Float32(Some(q)))) => *q as f64, - _ => { - return Err(LoweringError::InvalidExpression( - "percentile value must be a float literal (2nd arg)".into(), - )) - } - }; - if q.is_finite() && (0.0..=1.0).contains(&q) { - Ok(q) - } else { - Err(LoweringError::InvalidExpression(format!( - "percentile must be in [0, 1], got {q}" - ))) - } -} - -// ── LogicalPlan navigation helpers ────────────────────────────────────────────── - -fn eval_fetch(expr_opt: &Option>) -> Option { - expr_opt.as_ref().and_then(|e| match e.as_ref() { - Expr::Literal(DfScalarValue::Int64(Some(v))) if *v >= 0 => Some(*v as usize), - Expr::Literal(DfScalarValue::UInt64(Some(v))) => Some(*v as usize), - Expr::Literal(DfScalarValue::Int32(Some(v))) if *v >= 0 => Some(*v as usize), - _ => None, - }) -} - -/// Map a DataFusion window-function definition to the canonical -/// [`WindowFuncKind`]. -/// `NthValue` is returned with `None`; `lower_window` fills in `n` from args. -fn lower_window_func_kind(fun: &WindowFunctionDefinition) -> Result { - let unsupported = |what: &str, name: &str| { - LoweringError::UnsupportedFeature(format!("window {what}: {name}")) - }; - match fun { - WindowFunctionDefinition::WindowUDF(udf) => match udf.name().to_lowercase().as_str() { - "row_number" => Ok(WindowFuncKind::RowNumber), - "rank" => Ok(WindowFuncKind::Rank), - "dense_rank" => Ok(WindowFuncKind::DenseRank), - "lag" => Ok(WindowFuncKind::Lag), - "lead" => Ok(WindowFuncKind::Lead), - // ClickHouse: frame-respecting variants, not plain Lag/Lead (#267). - "laginframe" => Ok(WindowFuncKind::LagInFrame), - "leadinframe" => Ok(WindowFuncKind::LeadInFrame), - "first_value" => Ok(WindowFuncKind::FirstValue), - "last_value" => Ok(WindowFuncKind::LastValue), - "nth_value" => Ok(WindowFuncKind::NthValue(None)), - other => Err(unsupported("function", other)), - }, - WindowFunctionDefinition::AggregateUDF(udf) => match udf.name().to_lowercase().as_str() { - "sum" => Ok(WindowFuncKind::Sum), - "avg" | "mean" => Ok(WindowFuncKind::Avg), - "count" => Ok(WindowFuncKind::Count), - "min" => Ok(WindowFuncKind::Min), - "max" => Ok(WindowFuncKind::Max), - other => Err(unsupported("aggregate", other)), - }, - WindowFunctionDefinition::BuiltInWindowFunction(biwf) => { - use datafusion::logical_expr::BuiltInWindowFunction; - match biwf { - BuiltInWindowFunction::FirstValue => Ok(WindowFuncKind::FirstValue), - BuiltInWindowFunction::LastValue => Ok(WindowFuncKind::LastValue), - BuiltInWindowFunction::NthValue => Ok(WindowFuncKind::NthValue(None)), - } - } - } -} - -/// Map DataFusion's resolved `WindowFrame` (issue #268) to the canonical -/// [`WindowFrame`]. DataFusion's planner always fills in the SQL-standard -/// default frame before the logical plan is built, so this never sees an -/// "absent" frame — only `ROWS`/`RANGE`/`GROUPS` with concrete bounds. -/// `GROUPS` is rejected: no query in this repo's SQL corpora uses it, and -/// nothing downstream interprets frame semantics yet, so it isn't worth -/// modelling untested. -fn lower_window_frame( - frame: &datafusion::logical_expr::WindowFrame, -) -> Result { - let units = match frame.units { - DfWindowFrameUnits::Rows => WindowFrameUnits::Rows, - DfWindowFrameUnits::Range => WindowFrameUnits::Range, - DfWindowFrameUnits::Groups => { - return Err(LoweringError::UnsupportedFeature( - "window frame unit: GROUPS".into(), - )) - } - }; - let offset = |v: &DfScalarValue| -> Result { - Ok(match v { - DfScalarValue::IntervalYearMonth(Some(months)) => WindowFrameOffset::Interval { - months: *months, - days: 0, - nanoseconds: 0, - }, - DfScalarValue::IntervalDayTime(Some(value)) => WindowFrameOffset::Interval { - months: 0, - days: value.days, - nanoseconds: i64::from(value.milliseconds) * 1_000_000, - }, - DfScalarValue::IntervalMonthDayNano(Some(value)) => WindowFrameOffset::Interval { - months: value.months, - days: value.days, - nanoseconds: value.nanoseconds, - }, - // DataFusion 43 keeps every RANGE offset as text: both numeric - // bounds such as `1.5` and normalized interval literals such as - // `"1 HOUR"`. Arrow's interval parser accepts bare numbers and - // interprets them as months, so classify numeric text first. - DfScalarValue::Utf8(Some(value)) | DfScalarValue::LargeUtf8(Some(value)) => { - if let Ok(value) = value.parse::() { - WindowFrameOffset::Scalar(ScalarValue::Int64(value)) - } else if let Ok(value) = value.parse::() { - WindowFrameOffset::Scalar(ScalarValue::Float64(value)) - } else { - match parse_interval_month_day_nano(value) { - Ok(interval) => WindowFrameOffset::Interval { - months: interval.months, - days: interval.days, - nanoseconds: interval.nanoseconds, - }, - Err(_) => WindowFrameOffset::Scalar(scalar_value_to_asap(v)?), - } - } - } - _ => WindowFrameOffset::Scalar(scalar_value_to_asap(v)?), - }) - }; - let bound = |b: &DfWindowFrameBound| -> Result { - Ok(match b { - DfWindowFrameBound::Preceding(v) => WindowFrameBound::Preceding(offset(v)?), - DfWindowFrameBound::CurrentRow => WindowFrameBound::CurrentRow, - DfWindowFrameBound::Following(v) => WindowFrameBound::Following(offset(v)?), - }) - }; - Ok(WindowFrame { - units, - start_bound: bound(&frame.start_bound)?, - end_bound: bound(&frame.end_bound)?, - }) -} - -// ── Issue #225, item 3: DataFusion registry drift detection ──────────────── -// -// `asap_sql_function_catalog::NATIVE_FUNCTIONS` is hand-maintained data -// mirroring what DataFusion's own aggregate-function registry resolves. That -// mirror can only silently drift out of sync — a DataFusion version bump -// that adds, renames, or removes a builtin aggregate leaves the catalog -// looking fine while `lower_agg_intent` quietly gains or loses coverage. Of -// the two introspectable sources the issue names, DataFusion's own registry -// is the one with no external dependency: `SessionContext` already lists its -// aggregate UDFs in-process, so the check below builds a real context the -// same way `build_context` does and walks it directly — no live database, -// no new CI infra, just `cargo test`. (ClickHouse's `system.functions` is -// the other source; it needs a live ClickHouse instance, which is handled -// separately by the dev-only `tools/clickhouse/extract_functions.py` script, -// deliberately not wired into this test or into CI.) -#[cfg(test)] -mod catalog_drift { - use super::*; - - /// Every aggregate function name DataFusion's planner resolves inside a - /// context built the same way `build_context` builds one must be - /// *accounted for* by the catalog: either `lookup_native` maps it to a - /// canonical semantic, it is one of our own `CLICKHOUSE_BUILTINS` stub - /// registrations (`build_context` registers those into the very same - /// context, so they show up here too), or it is explicitly listed in - /// `KNOWN_UNMAPPED_NATIVE_FUNCTIONS` with a reason. - /// - /// This does *not* assert the reverse (that every `NATIVE_FUNCTIONS` - /// entry is resolvable) — a name that stops resolving after a DataFusion - /// bump just becomes permanently unreachable dead data, not a lowering - /// hazard, so it's out of scope for a regression gate. It also does not - /// try to derive `AggSemantic` from anything DataFusion reports — that - /// judgment call stays with whoever adds the catalog entry. - #[test] - fn every_datafusion_aggregate_name_is_covered_by_the_catalog() { - let catalog = SqlCatalog::new(); - let ctx = SqlLowerer::new(&catalog) - .build_context() - .expect("build_context with an empty table catalog cannot fail"); - let state = ctx.state(); - let mut uncovered: Vec<&str> = state - .aggregate_functions() - .keys() - .map(String::as_str) - .filter(|name| { - asap_sql_function_catalog::lookup_native(name).is_none() - && asap_sql_function_catalog::lookup_clickhouse_builtin(name).is_none() - && !asap_sql_function_catalog::KNOWN_UNMAPPED_NATIVE_FUNCTIONS.contains(name) - }) - .collect(); - uncovered.sort_unstable(); - assert!( - uncovered.is_empty(), - "DataFusion resolves these aggregate names but the catalog doesn't know about them \ - (crates/sql-function-catalog/src/lib.rs): {uncovered:?}\n\ - Either add a `NativeFunction` entry mapping each to its `AggSemantic`, or -- if it's \ - a deliberate non-goal (no `AggIntent` shape for it, or it's rejected elsewhere) -- \ - add it to `KNOWN_UNMAPPED_NATIVE_FUNCTIONS` with a reason. This usually means a \ - DataFusion version bump added or renamed a builtin aggregate." - ); - } - - /// Every `KNOWN_UNMAPPED_NATIVE_FUNCTIONS` entry earns its place by - /// actually being a name DataFusion resolves today — otherwise it is - /// stale documentation for a name that no longer exists (e.g. a prior - /// DataFusion version renamed it), not a real "deliberately not mapped" - /// decision, and should be removed. - #[test] - fn known_unmapped_entries_are_all_real_datafusion_names() { - let catalog = SqlCatalog::new(); - let ctx = SqlLowerer::new(&catalog) - .build_context() - .expect("build_context with an empty table catalog cannot fail"); - let resolved = ctx.state().aggregate_functions().clone(); - for name in asap_sql_function_catalog::KNOWN_UNMAPPED_NATIVE_FUNCTIONS { - assert!( - resolved.contains_key(*name), - "`{name}` is listed in KNOWN_UNMAPPED_NATIVE_FUNCTIONS but DataFusion no longer \ - resolves it -- remove the stale entry" - ); - } - } -} diff --git a/crates/frontend-sql/src/unified/sql/types.rs b/crates/frontend-sql/src/unified/sql/types.rs deleted file mode 100644 index 05d46f938..000000000 --- a/crates/frontend-sql/src/unified/sql/types.rs +++ /dev/null @@ -1,378 +0,0 @@ -//! Type bridges between DataFusion's Arrow types and the canonical `DataType`, plus -//! the SQL table catalog used to register tables with DataFusion and to carry -//! resolved leaf schemas into the canonical, unresolved tree. - -use std::collections::HashMap; - -use datafusion::arrow::datatypes::{ - DataType as ArrowDataType, Field as ArrowField, Fields, Schema as ArrowSchema, -}; -use datafusion::common::ScalarValue as DfScalarValue; - -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::pre_asap::ScalarValue; - -use crate::unified::error::SqlError as LoweringError; - -/// Table catalog for SQL lowering: table name → resolved canonical [`Schema`]. -/// -/// Used twice: to register Arrow-backed `MemTable`s so DataFusion can resolve -/// `SELECT … FROM t`, and to attach each table's schema directly onto the -/// canonical `Scan` (`schema: Some(_)`) so the SchemaResolver doesn't need to -/// usage-derive it. -#[derive(Debug, Clone, Default)] -pub struct SqlCatalog { - pub tables: HashMap, -} - -impl SqlCatalog { - pub fn new() -> Self { - Self::default() - } - - /// Builder: register `name` with its resolved canonical schema. - pub fn with_table(mut self, name: impl Into, schema: Schema) -> Self { - self.tables.insert(name.into(), schema); - self - } -} - -pub(super) fn scalar_value_to_asap(sv: &DfScalarValue) -> Result { - match sv { - DfScalarValue::Int64(Some(v)) => Ok(ScalarValue::Int64(*v)), - DfScalarValue::Int32(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), - DfScalarValue::Int16(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), - DfScalarValue::Int8(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), - DfScalarValue::UInt64(Some(v)) => i64::try_from(*v).map(ScalarValue::Int64).map_err(|_| { - LoweringError::InvalidExpression(format!("UInt64 value {v} overflows i64")) - }), - DfScalarValue::UInt32(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), - DfScalarValue::Float64(Some(v)) => Ok(ScalarValue::Float64(*v)), - DfScalarValue::Float32(Some(v)) => Ok(ScalarValue::Float64(*v as f64)), - DfScalarValue::Utf8(Some(s)) | DfScalarValue::LargeUtf8(Some(s)) => { - Ok(ScalarValue::Utf8(s.clone())) - } - DfScalarValue::Boolean(Some(b)) => Ok(ScalarValue::Boolean(*b)), - // All three of DataFusion's interval scalars land on one canonical - // shape; the narrower two simply leave the fields they do not carry - // at zero. - DfScalarValue::IntervalYearMonth(Some(months)) => Ok(ScalarValue::Interval { - months: *months, - days: 0, - nanos: 0, - }), - DfScalarValue::IntervalDayTime(Some(v)) => Ok(ScalarValue::Interval { - months: 0, - days: v.days, - nanos: i64::from(v.milliseconds) * 1_000_000, - }), - DfScalarValue::IntervalMonthDayNano(Some(v)) => Ok(ScalarValue::Interval { - months: v.months, - days: v.days, - nanos: v.nanoseconds, - }), - _ if sv.is_null() => Ok(ScalarValue::Null), - _ => Err(LoweringError::InvalidExpression(format!( - "unsupported scalar: {sv:?}" - ))), - } -} - -/// Arrow → the canonical `DataType` (used for `CAST` targets). Deliberately narrow. -pub(super) fn arrow_to_dtype(dt: &ArrowDataType) -> Result { - match dt { - ArrowDataType::Null => Ok(DataType::Null), - ArrowDataType::Int64 - | ArrowDataType::Int32 - | ArrowDataType::Int16 - | ArrowDataType::Int8 => Ok(DataType::Int64), - ArrowDataType::Float64 | ArrowDataType::Float32 => Ok(DataType::Float64), - ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => Ok(DataType::Utf8), - ArrowDataType::Boolean => Ok(DataType::Bool), - ArrowDataType::Timestamp(_, _) => Ok(DataType::Timestamp), - ArrowDataType::Date32 | ArrowDataType::Date64 => Ok(DataType::Date), - ArrowDataType::Interval(_) => Ok(DataType::Interval), - ArrowDataType::List(element) => Ok(DataType::List { - element: Box::new(Field::new( - element.name(), - arrow_to_dtype(element.data_type())?, - element.is_nullable(), - )), - }), - ArrowDataType::Struct(fields) => Ok(DataType::Struct { - fields: fields - .iter() - .map(|field| { - Ok(Field::new( - field.name(), - arrow_to_dtype(field.data_type())?, - field.is_nullable(), - )) - }) - .collect::, LoweringError>>()?, - }), - ArrowDataType::Map(entries, _) => { - let ArrowDataType::Struct(fields) = entries.data_type() else { - return Err(LoweringError::UnsupportedFeature( - "map entries must be a struct".into(), - )); - }; - if fields.len() != 2 || fields[0].is_nullable() { - return Err(LoweringError::UnsupportedFeature( - "map entries require a non-null key and a value".into(), - )); - } - Ok(DataType::Map { - key: Box::new(arrow_to_dtype(fields[0].data_type())?), - value: Box::new(arrow_to_dtype(fields[1].data_type())?), - value_nullable: fields[1].is_nullable(), - }) - } - other => Err(LoweringError::UnsupportedFeature(format!( - "Arrow type: {other:?}" - ))), - } -} - -/// The canonical `DataType` → Arrow (for registering catalog tables with DataFusion). -pub(super) fn dtype_to_arrow(dt: &DataType) -> ArrowDataType { - match dt { - DataType::Null => ArrowDataType::Null, - DataType::Int64 => ArrowDataType::Int64, - DataType::Float64 => ArrowDataType::Float64, - DataType::Utf8 => ArrowDataType::Utf8, - DataType::Bool => ArrowDataType::Boolean, - DataType::List { element } => ArrowDataType::List(std::sync::Arc::new(ArrowField::new( - &element.name, - dtype_to_arrow(&element.dtype), - element.nullable, - ))), - DataType::Struct { fields } => ArrowDataType::Struct( - fields - .iter() - .map(|field| { - ArrowField::new(&field.name, dtype_to_arrow(&field.dtype), field.nullable) - }) - .collect::>() - .into(), - ), - DataType::Map { - key, - value, - value_nullable, - } => ArrowDataType::Map( - std::sync::Arc::new(ArrowField::new( - "entries", - ArrowDataType::Struct( - vec![ - ArrowField::new("key", dtype_to_arrow(key), false), - ArrowField::new("value", dtype_to_arrow(value), *value_nullable), - ] - .into(), - ), - false, - )), - false, - ), - DataType::Timestamp => { - ArrowDataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None) - } - // Deliberately narrowing: `Date64` lowers to `DataType::Date` and comes - // back as `Date32`. Both spell the same calendar date and nothing in - // the planner reads the width; a catalog that wants `Date64` back would - // need a second variant carrying no planning information. - DataType::Date => ArrowDataType::Date32, - // Only reachable through a hand-built schema: `Interval` types a - // literal, and no catalog declares a column with it. Mapped to the - // same three-field shape `ScalarValue::Interval` carries rather than - // left to panic. - DataType::Interval => { - ArrowDataType::Interval(datafusion::arrow::datatypes::IntervalUnit::MonthDayNano) - } - } -} - -/// Build an Arrow schema from a canonical [`Schema`] (column name + type + nullability). -pub(super) fn schema_to_arrow(schema: &Schema) -> ArrowSchema { - let fields: Fields = schema - .fields - .iter() - .map(|c: &Field| { - ArrowField::new(&c.name, dtype_to_arrow(c.expect_plain_dtype()), c.nullable) - }) - .collect(); - ArrowSchema::new(fields) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Both Arrow date widths bridge to the one canonical `Date`, and it - /// registers back as `Date32` — the documented narrowing. - #[test] - fn both_arrow_date_widths_bridge_to_date() { - assert_eq!( - arrow_to_dtype(&ArrowDataType::Date32).unwrap(), - DataType::Date - ); - assert_eq!( - arrow_to_dtype(&ArrowDataType::Date64).unwrap(), - DataType::Date - ); - assert_eq!(dtype_to_arrow(&DataType::Date), ArrowDataType::Date32); - } - - /// Every Arrow interval width shares the canonical calendar interval type. - #[test] - fn interval_types_round_trip_through_the_catalog_bridge() { - use datafusion::arrow::datatypes::IntervalUnit; - for unit in [ - IntervalUnit::YearMonth, - IntervalUnit::DayTime, - IntervalUnit::MonthDayNano, - ] { - assert_eq!( - arrow_to_dtype(&ArrowDataType::Interval(unit)).unwrap(), - DataType::Interval - ); - } - assert_eq!( - arrow_to_dtype(&dtype_to_arrow(&DataType::Interval)).unwrap(), - DataType::Interval - ); - } - - /// All three of DataFusion's interval scalars carry into the one canonical - /// three-field shape, with the fields they do not spell left at zero. - #[test] - fn every_datafusion_interval_scalar_carries_across() { - use datafusion::arrow::datatypes::{IntervalDayTime, IntervalMonthDayNano}; - - assert_eq!( - scalar_value_to_asap(&DfScalarValue::IntervalYearMonth(Some(14))).unwrap(), - ScalarValue::Interval { - months: 14, - days: 0, - nanos: 0 - } - ); - assert_eq!( - scalar_value_to_asap(&DfScalarValue::IntervalDayTime(Some(IntervalDayTime::new( - 30, 500 - )))) - .unwrap(), - ScalarValue::Interval { - months: 0, - days: 30, - nanos: 500_000_000 - } - ); - assert_eq!( - scalar_value_to_asap(&DfScalarValue::IntervalMonthDayNano(Some( - IntervalMonthDayNano::new(1, 2, 3) - ))) - .unwrap(), - ScalarValue::Interval { - months: 1, - days: 2, - nanos: 3 - } - ); - } - - /// Nested map values and value nullability survive catalog registration. - #[test] - fn nested_map_schema_round_trip() { - let map = DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Map { - key: Box::new(DataType::Int64), - value: Box::new(DataType::Float64), - value_nullable: true, - }), - value_nullable: false, - }; - assert_eq!(arrow_to_dtype(&dtype_to_arrow(&map)).unwrap(), map); - let encoded = serde_json::to_string(&map).unwrap(); - assert_eq!(serde_json::from_str::(&encoded).unwrap(), map); - } -} - -#[cfg(test)] -mod collection_tests { - use super::*; - #[test] - fn nested_collections_preserve_field_names_order_and_nullability() { - let dtype = DataType::Struct { - fields: vec![ - Field::new( - "samples", - DataType::List { - element: Box::new(Field::new( - "sample", - DataType::Struct { - fields: vec![ - Field::new("timestamp", DataType::Timestamp, false), - Field::new("value", DataType::Float64, true), - Field::new( - "labels", - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::List { - element: Box::new(Field::new( - "label_value", - DataType::Utf8, - false, - )), - }), - value_nullable: true, - }, - true, - ), - ], - }, - true, - )), - }, - false, - ), - Field::new("optional", DataType::Int64, true), - ], - }; - let arrow = dtype_to_arrow(&dtype); - assert_eq!(arrow_to_dtype(&arrow).unwrap(), dtype); - assert_eq!(dtype_to_arrow(&arrow_to_dtype(&arrow).unwrap()), arrow); - let encoded = serde_json::to_string(&dtype).unwrap(); - assert_eq!(serde_json::from_str::(&encoded).unwrap(), dtype); - } - #[test] - fn empty_struct_and_nonnullable_list_element_roundtrip() { - let dtype = DataType::List { - element: Box::new(Field::new( - "empty", - DataType::Struct { fields: vec![] }, - false, - )), - }; - assert_eq!(arrow_to_dtype(&dtype_to_arrow(&dtype)).unwrap(), dtype); - } -} - -#[cfg(test)] -mod bottom_map_tests { - use super::*; - #[test] - fn empty_map_bottom_types_roundtrip_without_string_defaults() { - let (map, nullable) = asap_types::pre_asap::scalar_type_rules::MapScalarFunction::Construct - .output_type(&[]) - .unwrap(); - assert!(!nullable); - let arrow = dtype_to_arrow(&map); - assert_eq!(arrow_to_dtype(&arrow).unwrap(), map); - assert_eq!( - arrow_to_dtype(&ArrowDataType::Null).unwrap(), - DataType::Null - ); - } -} diff --git a/crates/frontend-sql/tests/maintained_population.rs b/crates/frontend-sql/tests/maintained_population.rs index 391426ae9..3abd11888 100644 --- a/crates/frontend-sql/tests/maintained_population.rs +++ b/crates/frontend-sql/tests/maintained_population.rs @@ -3,8 +3,9 @@ use asap_aware_mapping::maintained_population::MaintainedPopulationStrategy; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::{ ir::{ - apply_lifecycle_timings, cse::share_common_sub_dags, export::compile_physical_asap_dag, - ASAPOp, LifecycleAssignment, NonASAPOp, Operator, OperatorNode, TimingMemo, + apply_materialization_timings, cse::share_common_sub_dags, + export::compile_physical_asap_dag, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, + OperatorNode, TimingMemo, }, post_asap::maintained_population::{MaintainedPopulation, PopulationInput}, pre_asap::{DataType, Field, Schema}, @@ -37,12 +38,12 @@ fn population(mut node: &OperatorNode) -> (&Rc, &MaintainedPopulat (child, population) } -/// Export `plan` the way the planner does: assign the default lifecycle +/// Export `plan` the way the planner does: assign the default materialization /// timings, then compile the timed DAG. fn compile(plan: &Rc) -> Result<(), String> { - let timed = apply_lifecycle_timings( + let timed = apply_materialization_timings( plan, - &LifecycleAssignment::default_maintained(), + &MaterializationAssignment::all_query_time(), &mut TimingMemo::new(), ) .map_err(|e| e.to_string())?; diff --git a/crates/frontend-sql/tests/pearson_corr.rs b/crates/frontend-sql/tests/pearson_corr.rs index 8ef1fb415..560da2b48 100644 --- a/crates/frontend-sql/tests/pearson_corr.rs +++ b/crates/frontend-sql/tests/pearson_corr.rs @@ -3,8 +3,8 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::{ - apply_lifecycle_timings, export::compile_physical_asap_dag, LifecycleAssignment, NonASAPOp, - OperatorNode, ScalarExpr, TimingMemo, + apply_materialization_timings, export::compile_physical_asap_dag, MaterializationAssignment, + NonASAPOp, OperatorNode, ScalarExpr, TimingMemo, }; use asap_types::pre_asap::{AggIntent, DataType, Field, Schema}; use asap_types::types::AccuracyTarget; @@ -152,9 +152,9 @@ async fn corr_survives_exact_plan_compilation() { // The exact fallback is the query's own operator DAG, no ASAP node added. assert!(!plan.contains_asap(), "expected exact fallback"); assert_eq!(aggregate(&plan).0, aggregate(&query).0); - let timed = apply_lifecycle_timings( + let timed = apply_materialization_timings( &plan, - &LifecycleAssignment::default_maintained(), + &MaterializationAssignment::all_query_time(), &mut TimingMemo::new(), ) .unwrap(); diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 7ded8981b..af9598cd3 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -100,22 +100,37 @@ pub mod fixtures { /// Timing and export helpers for post-ASAP plans. pub mod post_asap { use asap_types::ir::export::{compile_physical_asap_dag, PhysicalASAPDAG}; - use asap_types::ir::{apply_lifecycle_timings, LifecycleAssignment, OperatorNode, TimingMemo}; + use asap_types::ir::{ + apply_materialization_timings, MaterializationAssignment, OperatorNode, TimingMemo, + }; use std::rc::Rc; - /// Time `root` under the default (every summary maintained) lifecycle - /// assignment. Returns the timed copy; read `node.timing` on it. + /// Time `root` under the default assignment (every summary computed at + /// query time). Returns the timed copy; read `node.timing` on it. pub fn timed(root: &Rc) -> Rc { - apply_lifecycle_timings( - root, - &LifecycleAssignment::default_maintained(), - &mut TimingMemo::new(), - ) - .expect("default lifecycle timing failed") + timed_with(root, &MaterializationAssignment::all_query_time()) + } + + /// Time `root` with every summary maintained at ingestion time. + pub fn maintained(root: &Rc) -> Rc { + timed_with(root, &MaterializationAssignment::all_ingestion_time()) } - /// Time `root` (default assignment), then export the wire-6 DAG. + fn timed_with( + root: &Rc, + assignment: &MaterializationAssignment, + ) -> Rc { + apply_materialization_timings(root, assignment, &mut TimingMemo::new()) + .expect("materialization timing failed") + } + + /// Time `root` (default assignment), then export the physical DAG. pub fn post_asap_dag(root: &Rc) -> PhysicalASAPDAG { compile_physical_asap_dag(&timed(root)).expect("post-ASAP DAG export failed") } + + /// Time `root` with every summary maintained, then export the physical DAG. + pub fn maintained_post_asap_dag(root: &Rc) -> PhysicalASAPDAG { + compile_physical_asap_dag(&maintained(root)).expect("post-ASAP DAG export failed") + } } diff --git a/crates/integration-tests/tests/exact_composition.rs b/crates/integration-tests/tests/exact_composition.rs index f72458377..f8e069a3d 100644 --- a/crates/integration-tests/tests/exact_composition.rs +++ b/crates/integration-tests/tests/exact_composition.rs @@ -25,7 +25,7 @@ use asap_aware_mapping::{ CostModel, DefaultCostModel, EvaluationRate, ExplanationKind, OperationPlacement, }; use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::post_asap::{post_asap_dag, timed}; +use asap_integration_tests::post_asap::{maintained, post_asap_dag, timed}; use asap_types::dag_export; use asap_types::ir::export::{NonASAPOpKind, PhysicalASAPOperatorPayload}; use asap_types::ir::operator_properties::{Reduction, Source}; @@ -385,9 +385,9 @@ fn every_exact_accumulator_is_finalized_before_an_outer_sketch() { let Replacement::SubDAG(root) = &candidates[0].replacement else { unreachable!() }; - // Timing is not stored on the plan: time it (default lifecycle, - // which also validates every edge) and inspect the timed copy. - let root = timed(root); + // Timing is not stored on the plan: time it with the outer summary + // maintained (which also validates every edge) and inspect the copy. + let root = maintained(root); let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { panic!("expected KLL evaluation, got {:?}", root.operator); }; @@ -668,8 +668,8 @@ fn outer_summary_over_an_exact_function_composes_at_ingestion_time() { assert!(decision.cost_rate < decision.baseline_rate); let composed = selection.assemble_selected_dag(&root).unwrap().unwrap(); - // Walk the timed copy: timing is written by the lifecycle assignment. - let composed = timed(&composed); + // Walk the timed copy, with the outer summary maintained at ingestion time. + let composed = maintained(&composed); let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &composed.operator else { panic!("expected evaluation root, got {:?}", composed.operator); }; @@ -723,7 +723,7 @@ fn summary_construction_follows_its_value_input_phase() { // time, because a evaluation sits below it. let state = asap_types::ir::planned_data_state(&illegal, ExecutionTiming::IngestionTime); assert_eq!(state.timing, ExecutionTiming::QueryTime); - asap_types::ir::validate_default(&illegal, state.timing).unwrap(); + asap_types::ir::validate_maintained(&illegal, state.timing).unwrap(); } #[test] diff --git a/crates/integration-tests/tests/operator_design_examples.rs b/crates/integration-tests/tests/operator_design_examples.rs index 4b52026b4..3c4c028d6 100644 --- a/crates/integration-tests/tests/operator_design_examples.rs +++ b/crates/integration-tests/tests/operator_design_examples.rs @@ -270,46 +270,15 @@ async fn sql_window_and_filtered_aggregate_types() { /// and executes both selected plans. No replacement dag is constructed by the test. #[tokio::test] async fn batch_planning_replaces_and_shares_summary_operators() { - use asap_aware_mapping::cost_model::{Cost, DefaultCostModel}; use asap_aware_mapping::pass::PlanningModels; - use asap_aware_mapping::{ - CostModel, CostRate, LifecycleInput, SummaryMaintenanceLifecycleCapabilities, - SummaryMaintenanceLifecycleCostInputs, - }; use asap_physical_operators::{ physical_planner::{compile, InputContract}, runtime::Scope, values::{Batch, Value}, }; use asap_planner::{e2e_plan, FrontendInput, UserInput}; - use asap_types::post_asap::SketchAlgorithm; use asap_types::workload::*; use std::{collections::BTreeMap, sync::Arc}; - struct Costs; - impl CostModel for Costs { - fn rank_candidates( - &self, - intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - DefaultCostModel.rank_candidates(intent, candidates) - } - fn summary_maintenance_lifecycle_cost_inputs( - &self, - _: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(1.0)), - maintenance_cost_per_update: Some(Cost::ZERO), - summary_read_cost: Some(Cost::ZERO), - retention_cost_rate: Some(CostRate(0.0)), - retirement_cost: Some(Cost::ZERO), - } - } - fn raw_query_recompute_cost(&self, _: &OperatorNode) -> Option { - Some(Cost(1000.0)) - } - } let queries = [ "SELECT SUM(bytes) + 1 AS result FROM requests", "SELECT SUM(bytes) * 2 AS result FROM requests", @@ -347,8 +316,7 @@ async fn batch_planning_replaces_and_shares_summary_operators() { let output = e2e_plan(UserInput::new( &workload, FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin().with_cost(&Costs), - LifecycleInput::new(0, SummaryMaintenanceLifecycleCapabilities::default()), + PlanningModels::builtin(), )) .await .unwrap(); @@ -361,8 +329,7 @@ async fn batch_planning_replaces_and_shares_summary_operators() { .collect(); assert_eq!(states.len(), 1, "the batch owns one shared SUM state"); for (plan, expected) in output.plans.iter().zip([31.0, 60.0]) { - assert!(!plan.plan.selected_raw_recompute); - let root = &plan.plan.root; + let root = &plan.root; root.validate_structure().unwrap(); assert!(OperatorNode::reachable(root) .iter() diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index e23e9721d..fc4c2b655 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -46,10 +46,29 @@ pub fn execute( pub fn compile_physical_asap_dag( root: &std::rc::Rc, ) -> Result> { - let root = asap_types::ir::apply_lifecycle_timings( + compile_with( root, - &Default::default(), - &mut Default::default(), - )?; + &asap_types::ir::MaterializationAssignment::all_query_time(), + ) +} + +/// As [`compile_physical_asap_dag`], with every summary maintained at +/// ingestion time, the placement precompute compilation requires. +#[allow(dead_code)] // Not every test binary sharing this module compiles precompute. +pub fn compile_maintained_physical_asap_dag( + root: &std::rc::Rc, +) -> Result> { + compile_with( + root, + &asap_types::ir::MaterializationAssignment::all_ingestion_time(), + ) +} + +fn compile_with( + root: &std::rc::Rc, + assignment: &asap_types::ir::MaterializationAssignment, +) -> Result> { + let root = + asap_types::ir::apply_materialization_timings(root, assignment, &mut Default::default())?; Ok(asap_types::ir::export::compile_physical_asap_dag(&root)?) } diff --git a/crates/integration-tests/tests/precompute_raw_samples.rs b/crates/integration-tests/tests/precompute_raw_samples.rs index 4bc8f7ace..f0a270b0d 100644 --- a/crates/integration-tests/tests/precompute_raw_samples.rs +++ b/crates/integration-tests/tests/precompute_raw_samples.rs @@ -3,7 +3,7 @@ mod physical_common; use asap_types::ir::export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; use asap_types::ir::OperatorNode; -use physical_common::compile_physical_asap_dag; +use physical_common::compile_maintained_physical_asap_dag; use std::{collections::BTreeMap, collections::BTreeSet, rc::Rc, sync::Arc}; use asap_aware_mapping::cost_model::DefaultCostModel; @@ -421,7 +421,7 @@ fn raw_sample_summaries_compile_and_match_their_kernels() { let mut checked = BTreeMap::new(); for (query, accuracy) in queries { for candidate in candidates(query, accuracy.clone()) { - let dag = compile_physical_asap_dag(&candidate).unwrap(); + let dag = compile_maintained_physical_asap_dag(&candidate).unwrap(); for (source, root) in raw_summaries(&dag) { match check(query, &dag, source, root, &rows) { Ok(family) => { @@ -474,7 +474,7 @@ fn grouped_raw_summary(family: FieldDataType, input: SummaryUpdate) -> (Physical ) .pop() .unwrap(); - let mut dag = compile_physical_asap_dag(&candidate).unwrap(); + let mut dag = compile_maintained_physical_asap_dag(&candidate).unwrap(); let (source, root) = raw_summaries(&dag)[0]; let node = dag .nodes diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index 9ce1d5d72..24c4209ec 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -19,7 +19,9 @@ use asap_aware_mapping::{ ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::post_asap::{post_asap_dag, timed}; +use asap_integration_tests::post_asap::{ + maintained, maintained_post_asap_dag, post_asap_dag, timed, +}; use asap_types::ir::export::{NonASAPOpKind, PhysicalASAPOperatorPayload}; use asap_types::ir::operator_properties::Reduction; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; @@ -1063,8 +1065,9 @@ fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { .assemble_selected_dag(&space.roots[0].1) .unwrap() .unwrap(); - // Stored timings are gone: time the plan and read the timed copy. - let timed_plan = timed(&plan); + // Stored timings are gone: time the plan with its outer summary + // maintained and read the timed copy. + let timed_plan = maintained(&plan); let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = timed_plan.asap() else { panic!("expected selected quantile summary"); }; @@ -1123,7 +1126,7 @@ fn physical_node_owns_phase_independently_of_binary_payload() { .assemble_selected_dag(&search.roots[0].1) .unwrap() .unwrap(); - let dag = post_asap_dag(&plan); + let dag = maintained_post_asap_dag(&plan); let node = dag .nodes .iter() diff --git a/crates/integration-tests/tests/summary_maintenance_lifecycle_e2e.rs b/crates/integration-tests/tests/summary_maintenance_lifecycle_e2e.rs deleted file mode 100644 index 411644d4b..000000000 --- a/crates/integration-tests/tests/summary_maintenance_lifecycle_e2e.rs +++ /dev/null @@ -1,1155 +0,0 @@ -//! End-to-end coverage for workload-aware summary-maintenance planning: -//! source workload -> PromQL lowering -> candidate search -> -//! summary-maintenance lifecycle selection -> materialized deployment guarantees. - -use asap_types::ir::export::PhysicalASAPOperatorPayload; -use asap_types::ir::ASAPOp; -use physical_common::compile_physical_asap_dag; -use std::rc::Rc; - -use asap_aware_mapping::cost_model::Cost; -use asap_aware_mapping::CostRate; -use asap_aware_mapping::{ - assemble_selected_dag_with_summary_maintenance_lifecycles, export_summary_maintenance_plan, - global_selection_with_summary_maintenance_lifecycles, search_workload_with, CostModel, Horizon, - SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCapabilities, - SummaryMaintenanceLifecycleCostInputs, SummaryMaintenanceLifecycleRejection, WorkloadDemand, -}; -use asap_frontend_promql::lower_promql_workload; -use asap_types::ir::OperatorNode; -use asap_types::post_asap::{ - EvaluationSchedule, SummaryMaintenanceLifecycle, SummaryMaintenanceMode, -}; -use asap_types::pre_asap::agg_intent::AggIntent; -use asap_types::types::AccuracyTarget; -use asap_types::workload::{ - AccuracyRequirement, BatchEntry, DataArrival, DataWorkload, DurationMs, Evidence, - EvidenceSource, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, - QueryTimeScope, QueryWorkload, Rate, RepeatedDemand, RepeatingEntry, RepetitionInterval, - TimeSelection, -}; - -const NOW_MS: u64 = 1_000_000; - -struct FullyCostedRuntime; - -impl CostModel for FullyCostedRuntime { - fn raw_query_recompute_total_cost( - &self, - _target: &OperatorNode, - _expected_reads: f64, - ) -> Option { - Some(Cost(1_000.0)) - } - - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[asap_types::post_asap::SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - - fn summary_maintenance_lifecycle_cost_inputs( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(10.0)), - maintenance_cost_per_update: Some(Cost(1.0)), - summary_read_cost: Some(Cost(1.0)), - retention_cost_rate: Some(CostRate(0.1)), - retirement_cost: Some(Cost(1.0)), - } - } - - fn summary_maintenance_capabilities( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceCapabilities { - SummaryMaintenanceCapabilities { - incremental_update: true, - merge: true, - delete: true, - } - } -} - -fn dashboard_workload() -> PlanningWorkload { - let query = Query("quantile_over_time(0.99, latency[5m])".into()); - let requirements = QueryRequirements { - accuracy: AccuracyRequirement::Explicit(AccuracyTarget::Epsilon(0.01)), - ..QueryRequirements::default() - }; - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: query.clone(), - requirements: requirements.clone(), - predictability: Predictability::AdHoc, - invocations: 1, - execute_at: None, - time_selection: TimeSelection::default(), - }]), - repeating_queries: Some(vec![RepeatingEntry { - query, - demand: RepeatedDemand::FixedInterval(RepetitionInterval(1_000)), - requirements, - predictability: Predictability::Predictable { known_at: None }, - time_selection: TimeSelection { - scope: QueryTimeScope::RealTime, - ..TimeSelection::default() - }, - }]), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - ingestion_rate: Evidence { - value: Some(Rate(1.0)), - source: EvidenceSource::Observed, - observed_at_ms: Some(NOW_MS), - valid_for_ms: Some(60_000), - }, - data_ingestion_interval: Evidence { - value: Some(DurationMs(1_000)), - ..Default::default() - }, - ..DataWorkload::default() - }), - } -} - -#[test] -fn promql_dashboard_materializes_continuous_summary_with_explained_rejections() { - let workload = dashboard_workload(); - let plan = selected_plan(&workload); - - assert!(!plan.selected_raw_recompute); - assert_eq!(plan.expected_reads, Some(100.0)); - assert_eq!(plan.deployments.len(), 1); - - let deployment = &plan.deployments[0]; - let guarantee = deployment - .summary_maintenance_lifecycle_guarantee - .as_ref() - .expect("selected lifecycle guarantee"); - assert_eq!( - guarantee.summary_maintenance_lifecycle, - SummaryMaintenanceLifecycle::ContinuouslyMaintained - ); - assert_eq!(guarantee.evaluation_schedule, EvaluationSchedule::PerUpdate); - assert_eq!( - guarantee.summary_maintenance_mode, - SummaryMaintenanceMode::Incremental - ); - assert!(deployment.alternatives.iter().any(|alternative| { - matches!( - alternative.summary_maintenance_lifecycle, - SummaryMaintenanceLifecycle::Prepared { .. } - ) && alternative.rejection - == Some(SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery) - })); - assert!(deployment.alternatives.iter().any(|alternative| { - matches!( - alternative.summary_maintenance_lifecycle, - SummaryMaintenanceLifecycle::Shared { .. } - ) && alternative.rejection - == Some(SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime) - })); - - let exported = serde_json::to_value(export_summary_maintenance_plan(&plan)).unwrap(); - assert_eq!( - exported["deployments"][0]["selected"]["lifecycle"]["kind"], - "continuously_maintained" - ); - assert_eq!( - exported["deployments"][0]["selected"]["maintenance_mode"], - "incremental" - ); - let alternatives = exported["deployments"][0]["alternatives"] - .as_array() - .expect("exported lifecycle alternatives"); - assert!(alternatives.iter().any(|alternative| { - alternative["lifecycle"]["kind"] == "prepared" - && alternative["rejection"] == "requires_predictable_one_time_query" - })); - assert!(alternatives.iter().any(|alternative| { - alternative["lifecycle"]["kind"] == "shared" - && alternative["rejection"] == "unsupported_by_runtime" - })); - assert!(exported["dag"]["nodes"].as_array().is_some()); - let summary_node = exported["dag"]["nodes"] - .as_array() - .unwrap() - .iter() - .find(|node| node["kind"] == "summary_agg") - .expect("exported summary_agg node"); - assert_eq!( - summary_node["detail"]["summary_maintenance"]["selected"]["lifecycle"]["kind"], - "continuously_maintained" - ); -} - -fn selected_plan( - workload: &PlanningWorkload, -) -> asap_aware_mapping::SummaryMaintenanceLifecyclePlan { - selected_plan_with_model(workload, &FullyCostedRuntime) -} - -fn selected_plan_with_model( - workload: &PlanningWorkload, - model: &dyn CostModel, -) -> asap_aware_mapping::SummaryMaintenanceLifecyclePlan { - selected_plan_with_horizon(workload, model, Horizon(100.)) -} - -fn selected_plan_with_horizon( - workload: &PlanningWorkload, - model: &dyn CostModel, - horizon: Horizon, -) -> asap_aware_mapping::SummaryMaintenanceLifecyclePlan { - workload.validate().unwrap(); - - let lowered = lower_promql_workload(workload, 0) - .expect("valid PromQL workload") - .into_iter() - .next() - .expect("one normalized workload entry"); - selected_plan_for_lowered(workload, lowered, model, horizon) -} - -fn selected_plan_for_lowered( - workload: &PlanningWorkload, - lowered: Rc, - model: &dyn CostModel, - horizon: Horizon, -) -> asap_aware_mapping::SummaryMaintenanceLifecyclePlan { - let root = lowered; - let strategies = asap_aware_mapping::default_strategies_with(model); - let space = search_workload_with(vec![("dashboard", Rc::clone(&root))], &strategies); - let target = Rc::clone(&space.roots[0].1); - let capabilities = SummaryMaintenanceLifecycleCapabilities { - supports_ephemeral: true, - supports_prepared: false, - supports_shared: false, - supports_continuously_maintained: true, - }; - - let selection = global_selection_with_summary_maintenance_lifecycles( - &space, - WorkloadDemand { - workload: &workload.query_workload, - data_workload: workload.data_workload.as_ref(), - entry_indices: &[1], - }, - NOW_MS, - Some(horizon), - capabilities, - model, - ) - .unwrap(); - assemble_selected_dag_with_summary_maintenance_lifecycles( - &selection, - &target, - WorkloadDemand::new_with_data( - &workload.query_workload, - workload.data_workload.as_ref().unwrap(), - &[1], - ), - NOW_MS, - Some(horizon), - capabilities, - model, - ) - .unwrap() - .expect("selected summary plan") -} - -mod physical_common; - -/// A selected continuous lifecycle supplies a materialization boundary; its -/// maintenance and query DAGs execute the selected KLL computation in fresh runs. -#[test] -fn continuous_lifecycle_compiles_and_executes_spatial_kll() { - use asap_physical_operators::{ - physical_planner::{compile_candidate, InputContract}, - runtime::Scope, - values::{Batch, Value}, - }; - use asap_types::{post_asap::FieldDataType, pre_asap::DataType}; - use std::{collections::BTreeMap, sync::Arc}; - let mut workload = dashboard_workload(); - workload.query_workload.query_batch.as_mut().unwrap()[0].query = - Query("quantile(0.99, latency)".into()); - workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = - Query("quantile(0.99, latency)".into()); - let selected = selected_plan(&workload); - assert_eq!( - selected.deployments[0] - .summary_maintenance_lifecycle_guarantee - .as_ref() - .unwrap() - .summary_maintenance_lifecycle, - SummaryMaintenanceLifecycle::ContinuouslyMaintained - ); - let dag = compile_physical_asap_dag(&selected.root).unwrap(); - let build = dag - .nodes - .iter() - .find(|node| matches!(node.payload, PhysicalASAPOperatorPayload::SummaryAgg { .. })) - .unwrap(); - let input = dag - .edges - .iter() - .find(|edge| edge.consumer == build.id) - .unwrap() - .producer; - let raw = dag.nodes.iter().find(|node| node.id == input).unwrap(); - let schema = Arc::new(raw.output_schema.clone()); - let candidate = compile_candidate( - &dag, - BTreeMap::from([(u64::from(input.0), InputContract::bounded(schema.clone()))]), - &[u64::from(dag.roots[0].0)], - &[u64::from(build.id.0)], - ) - .unwrap(); - - // A continuous input without a finite pane boundary cannot implement this - // blocking builder. Retain lifecycle ownership in the candidate payload; - // only the legal bounded request candidate reaches workload pricing. - let mut unbounded = InputContract::bounded(schema.clone()); - unbounded.properties.boundedness = asap_physical_operators::plan::Boundedness::Unbounded; - let rejected = compile_candidate( - &dag, - BTreeMap::from([(u64::from(input.0), unbounded)]), - &[u64::from(dag.roots[0].0)], - &[u64::from(build.id.0)], - ); - assert!(rejected.is_err()); - let request = compile_candidate( - &dag, - BTreeMap::from([(u64::from(input.0), InputContract::bounded(schema.clone()))]), - &[u64::from(dag.roots[0].0)], - &[], - ) - .unwrap(); - let mut priced = 0; - let feedback = asap_physical_operators::physical_planner::select_candidate( - vec![ - rejected.map(|candidate| { - ( - SummaryMaintenanceLifecycle::ContinuouslyMaintained, - candidate, - ) - }), - Ok((SummaryMaintenanceLifecycle::Ephemeral, request)), - ], - |_| { - priced += 1; - Ok(Some( - asap_physical_operators::physical_planner::CandidateCost { - workload_scope: "dashboard".into(), - horizon_seconds: 100., - total_cost: 1000., - }, - )) - }, - ) - .unwrap(); - assert_eq!(priced, 1); - assert_eq!(feedback.candidate.0, SummaryMaintenanceLifecycle::Ephemeral); - for revision in [1, 2] { - let rows = (1..=100) - .map(|value| { - schema - .fields - .iter() - .map(|field| match field.dtype { - FieldDataType::Plain(DataType::Float64) => Value::Float64(f64::from(value)), - FieldDataType::Plain(DataType::Timestamp) => Value::Timestamp(300_000), - _ => panic!("unexpected field {field:?}"), - }) - .collect() - }) - .collect(); - let raw_batch = Batch::try_new(schema.clone(), rows).unwrap(); - let direct = physical_common::execute( - &feedback.candidate.1.query, - BTreeMap::from([(u64::from(input.0), raw_batch.clone())]), - Scope::Query { - evaluation_time_ms: 300_000, - revision, - }, - ); - let state = physical_common::execute( - candidate.precompute.as_ref().unwrap(), - BTreeMap::from([(u64::from(input.0), raw_batch)]), - Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 300_000, - revision, - }, - ); - let result = physical_common::execute( - &candidate.query, - BTreeMap::from([(u64::from(build.id.0), state[0][0].clone())]), - Scope::Query { - evaluation_time_ms: 300_000, - revision, - }, - ); - let values: Vec<_> = result[0] - .iter() - .flat_map(|batch| batch.rows()) - .flat_map(|row| row.iter()) - .filter_map(|value| { - if let Value::Float64(value) = value { - Some(*value) - } else { - None - } - }) - .collect(); - let direct_values: Vec<_> = direct[0] - .iter() - .flat_map(|batch| batch.rows()) - .flat_map(|row| row.iter()) - .filter_map(|value| { - if let Value::Float64(value) = value { - Some(*value) - } else { - None - } - }) - .collect(); - assert_eq!( - values, direct_values, - "maintenance and request candidates preserve the same population" - ); - assert_eq!(values.len(), 1); - assert!( - (98. ..=100.).contains(&values[0]), - "p99 rank must reflect the supplied population" - ); - } -} - -fn quantile_workload(query: &str) -> PlanningWorkload { - let mut workload = dashboard_workload(); - workload.query_workload.query_batch.as_mut().unwrap()[0].query = Query(query.into()); - workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = Query(query.into()); - workload -} - -/// Timed DAG for `query` after binding every summary state to `lifecycle`. -/// Grouped queries carry a physical series identity, as per-entity state needs. -fn lifecycle_timed_dag( - query: &str, - lifecycle: &SummaryMaintenanceLifecycle, -) -> (asap_types::ir::export::PhysicalASAPDAG, Vec) { - use asap_aware_mapping::enumerate_summary_maintenance_lifecycles; - let workload = quantile_workload(query); - let mut lowered = lower_promql_workload(&workload, 0).unwrap().remove(0); - if query.contains(" by(") { - lowered = - asap_physical_operators::physical_planner::promql_rows::with_series_identity(&lowered) - .unwrap(); - } - let root = - selected_plan_for_lowered(&workload, lowered, &FullyCostedRuntime, Horizon(100.)).root; - let candidates = enumerate_summary_maintenance_lifecycles( - root, - WorkloadDemand::new_with_data( - &workload.query_workload, - workload.data_workload.as_ref().unwrap(), - &[1], - ), - NOW_MS, - Some(Horizon(100.)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &FullyCostedRuntime, - ) - .unwrap(); - let choices: Vec<_> = candidates - .deployments() - .iter() - .map(|deployment| (deployment.post_asap_node_id, lifecycle.clone())) - .collect(); - let mut states: Vec<_> = choices.iter().map(|(id, _)| u64::from(id.0)).collect(); - states.sort_unstable(); - let dag = candidates - .select(&choices) - .unwrap() - .execution_timed_dag() - .unwrap(); - (dag, states) -} - -/// Compile inputs for a timed DAG: its raw source, available at either phase. -fn raw_inputs( - dag: &asap_types::ir::export::PhysicalASAPDAG, -) -> std::collections::BTreeMap { - let raw = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - asap_types::ir::export::PhysicalASAPOperatorPayload::Relational { - operator: asap_types::ir::export::NonASAPOpKind::TimeRange { .. } - } - ) - }) - .unwrap(); - std::collections::BTreeMap::from([( - u64::from(raw.id.0), - asap_physical_operators::physical_planner::InputContract::bounded(std::sync::Arc::new( - raw.output_schema.clone(), - )), - )]) -} - -/// For existing PromQL fixtures, Planner's own retained lifecycle selection -/// reproduces the timing that realization strategies assign today. -#[test] -fn planner_lifecycle_selection_reproduces_strategy_timing() { - for query in [ - "quantile_over_time(0.99, latency[5m])", - "quantile(0.99, latency)", - "sum by(job)(rate(m[1m]))", - ] { - let plan = selected_plan(&quantile_workload(query)); - assert!(!plan.selected_raw_recompute, "{query}"); - assert!(plan.deployments.iter().all(|deployment| { - deployment - .summary_maintenance_lifecycle_guarantee - .as_ref() - .is_some_and(|guarantee| { - guarantee.summary_maintenance_lifecycle - != SummaryMaintenanceLifecycle::Ephemeral - }) - })); - let strategy = compile_physical_asap_dag(&plan.root).unwrap(); - assert_eq!(plan.execution_timed_dag().unwrap(), strategy, "{query}"); - } -} - -/// An explicitly chosen lifecycle reaches physical compilation through timing: -/// ContinuouslyMaintained puts the state in precompute, Ephemeral leaves -/// precompute empty and reads the raw source at query time; both answer alike. -#[test] -fn chosen_lifecycle_timing_decides_precompute_contents() { - use asap_physical_operators::{ - physical_planner::{compile_candidate, frontier_from_timing}, - runtime::Scope, - values::{Batch, Value}, - }; - use asap_types::{post_asap::FieldDataType, pre_asap::DataType}; - use std::collections::BTreeMap; - - let mut answers = Vec::new(); - for lifecycle in [ - SummaryMaintenanceLifecycle::ContinuouslyMaintained, - SummaryMaintenanceLifecycle::Ephemeral, - ] { - let (dag, states) = lifecycle_timed_dag("quantile(0.99, latency)", &lifecycle); - let [state] = states[..] else { - panic!("one summary state"); - }; - let inputs = raw_inputs(&dag); - let (&raw_id, contract) = inputs.iter().next().unwrap(); - let schema = contract.schema.clone(); - let frontier = frontier_from_timing(&dag).unwrap(); - let candidate = - compile_candidate(&dag, inputs, &[u64::from(dag.roots[0].0)], &frontier).unwrap(); - let rows = (1..=100) - .map(|value| { - schema - .fields - .iter() - .map(|field| match field.dtype { - FieldDataType::Plain(DataType::Float64) => Value::Float64(f64::from(value)), - FieldDataType::Plain(DataType::Timestamp) => Value::Timestamp(300_000), - _ => panic!("unexpected field {field:?}"), - }) - .collect() - }) - .collect(); - let raw_batch = Batch::try_new(schema.clone(), rows).unwrap(); - let query_scope = Scope::Query { - evaluation_time_ms: 300_000, - revision: 1, - }; - let result = if lifecycle == SummaryMaintenanceLifecycle::Ephemeral { - assert!(frontier.is_empty()); - assert!(candidate.precompute.is_none()); - physical_common::execute( - &candidate.query, - BTreeMap::from([(raw_id, raw_batch)]), - query_scope, - ) - } else { - assert_eq!(frontier, [state]); - assert_eq!( - candidate - .materialized_outputs - .keys() - .copied() - .collect::>(), - [state] - ); - let stored = physical_common::execute( - candidate.precompute.as_ref().unwrap(), - BTreeMap::from([(raw_id, raw_batch)]), - Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 300_000, - revision: 1, - }, - ); - physical_common::execute( - &candidate.query, - BTreeMap::from([(state, stored[0][0].clone())]), - query_scope, - ) - }; - answers.push( - result[0] - .iter() - .flat_map(|batch| batch.rows()) - .flat_map(|row| row.iter()) - .filter_map(|value| match value { - Value::Float64(value) => Some(*value), - _ => None, - }) - .collect::>(), - ); - } - assert_eq!(answers[0], answers[1]); - assert_eq!(answers[0].len(), 1); -} - -/// One compilation, cut by each lifecycle assignment's timing, yields exactly -/// the candidate `compile_candidate` builds for that timed DAG: the retained -/// state is the frontier under ContinuouslyMaintained, and nothing under -/// Ephemeral. Covers the KLL quantile fixture and grouped Rate→Sum. -#[test] -fn lifecycle_timing_cuts_one_compilation() { - use asap_physical_operators::physical_planner::{ - compile, compile_candidate, cut_candidate, frontier_from_timing, - }; - for query in ["quantile(0.99, latency)", "sum by(job)(rate(m[1m]))"] { - let ephemeral = SummaryMaintenanceLifecycle::Ephemeral; - let (compiled_dag, _) = lifecycle_timed_dag(query, &ephemeral); - let inputs = raw_inputs(&compiled_dag); - let roots = [u64::from(compiled_dag.roots[0].0)]; - let compiled = compile(&compiled_dag, inputs.clone(), &roots).unwrap(); - for lifecycle in [ - SummaryMaintenanceLifecycle::ContinuouslyMaintained, - ephemeral, - ] { - let (dag, states) = lifecycle_timed_dag(query, &lifecycle); - let frontier = frontier_from_timing(&dag).unwrap(); - // Retained states read by a query-time consumer, or the root itself. - let query_time = |id: u64| { - dag.nodes.iter().any(|node| { - u64::from(node.id.0) == id - && node.output_state.timing - == asap_types::post_asap::ExecutionTiming::QueryTime - }) - }; - let expected_frontier = if lifecycle == SummaryMaintenanceLifecycle::Ephemeral { - vec![] - } else { - states - .iter() - .copied() - .filter(|state| { - *state == u64::from(dag.roots[0].0) - || dag.edges.iter().any(|edge| { - u64::from(edge.producer.0) == *state - && query_time(u64::from(edge.consumer.0)) - }) - }) - .collect() - }; - assert_eq!(frontier, expected_frontier, "{query} {lifecycle:?}"); - let cut = cut_candidate(&compiled, &frontier).unwrap(); - let expected = compile_candidate(&dag, inputs.clone(), &roots, &frontier).unwrap(); - assert_eq!( - serde_json::to_vec(&cut).unwrap(), - serde_json::to_vec(&expected).unwrap(), - "{query} {lifecycle:?}" - ); - } - } -} - -/// A maintained current-series population is placed by its lifecycle choice: -/// ContinuouslyMaintained stores the population in precompute, Ephemeral -/// rebuilds it from the raw source at query time; both rank alike. -#[test] -fn chosen_population_lifecycle_decides_precompute_contents() { - use asap_aware_mapping::{ - enumerate_summary_maintenance_lifecycles, - maintained_population::MaintainedPopulationStrategy, - }; - use asap_physical_operators::{ - physical_planner::{ - compile_candidate, - promql_rows::{series_row, with_series_identity}, - InputContract, - }, - runtime::Scope, - values::{Batch, Value}, - }; - use asap_types::post_asap::maintained_population::PopulationInput; - use std::{collections::BTreeMap, sync::Arc}; - - let workload = quantile_workload("topk by(job)(1, m)"); - let root = - with_series_identity(&lower_promql_workload(&workload, 0).unwrap().remove(0)).unwrap(); - let root = MaintainedPopulationStrategy::new(std::slice::from_ref(&root)) - .candidate(&root) - .unwrap(); - let mut answers = Vec::new(); - for lifecycle in [ - SummaryMaintenanceLifecycle::ContinuouslyMaintained, - SummaryMaintenanceLifecycle::Ephemeral, - ] { - let candidates = enumerate_summary_maintenance_lifecycles( - Rc::clone(&root), - WorkloadDemand::new_with_data( - &workload.query_workload, - workload.data_workload.as_ref().unwrap(), - &[1], - ), - NOW_MS, - Some(Horizon(100.)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &FullyCostedRuntime, - ) - .unwrap(); - let [deployment] = candidates.deployments() else { - panic!("one population state"); - }; - let id = deployment.post_asap_node_id; - let dag = candidates - .select(&[(id, lifecycle.clone())]) - .unwrap() - .execution_timed_dag() - .unwrap(); - let population = dag.nodes.iter().find(|node| node.id == id).unwrap(); - let PhysicalASAPOperatorPayload::MaintainPopulation { population } = &population.payload - else { - panic!("the deployment is the maintained population"); - }; - let PopulationInput::CurrentSeries(spec) = &population.input else { - panic!("current-series population"); - }; - let lookback = i64::try_from(spec.lookback_ms).unwrap(); - let raw = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::Relational { - operator: asap_types::ir::export::NonASAPOpKind::TimeRange { .. } - } - ) - }) - .unwrap(); - let (raw_id, schema) = (u64::from(raw.id.0), Arc::new(raw.output_schema.clone())); - let frontier = - asap_physical_operators::physical_planner::frontier_from_timing(&dag).unwrap(); - let candidate = compile_candidate( - &dag, - BTreeMap::from([(raw_id, InputContract::bounded(schema.clone()))]), - &[u64::from(dag.roots[0].0)], - &frontier, - ) - .unwrap(); - let end = 60_000; - let rows = [("a", end - 1, 100.), ("a", end, 1.), ("b", end, 20.)] - .into_iter() - .map(|(instance, at, value)| { - series_row( - &schema, - &BTreeMap::from([ - ("job".into(), "api".into()), - ("instance".into(), instance.into()), - ]), - at, - value, - ) - .unwrap() - }) - .collect(); - let raw_batch = Batch::try_new(schema.clone(), rows).unwrap(); - let query_scope = Scope::Query { - evaluation_time_ms: end, - revision: 1, - }; - let result = if lifecycle == SummaryMaintenanceLifecycle::Ephemeral { - assert!(frontier.is_empty()); - assert!(candidate.precompute.is_none()); - physical_common::execute( - &candidate.query, - BTreeMap::from([(raw_id, raw_batch)]), - query_scope, - ) - } else { - let state = u64::from(id.0); - assert_eq!(frontier, [state]); - let stored = physical_common::execute( - candidate.precompute.as_ref().unwrap(), - BTreeMap::from([(raw_id, raw_batch)]), - Scope::Ingestion { - window_start_ms: end - lookback, - window_end_ms: end, - revision: 1, - }, - ); - physical_common::execute( - &candidate.query, - BTreeMap::from([(state, stored[0][0].clone())]), - query_scope, - ) - }; - answers.push( - result[0] - .iter() - .flat_map(|batch| batch.rows()) - .flat_map(|row| row.iter()) - .filter_map(|value| match value { - Value::Float64(value) => Some(*value), - _ => None, - }) - .collect::>(), - ); - } - assert_eq!(answers[0], answers[1]); - assert_eq!(answers[0], [20.]); -} - -/// Grouped Rate→Sum is one inventory candidate: retaining the Sum state puts -/// Rate and Sum in precompute, while an `Ephemeral` Sum over a retained Rate -/// state leaves Sum in the query DAG. -#[test] -fn grouped_rate_sum_placement_is_a_lifecycle_choice() { - use asap_aware_mapping::enumerate_summary_maintenance_lifecycles; - use asap_physical_operators::physical_planner::{compile_candidate, InputContract}; - use asap_types::post_asap::{ExactKind, FieldDataType}; - use std::{collections::BTreeMap, sync::Arc}; - - let workload = quantile_workload("sum by(job)(rate(m[1m]))"); - let root = asap_physical_operators::physical_planner::promql_rows::with_series_identity( - &lower_promql_workload(&workload, 0).unwrap().remove(0), - ) - .unwrap(); - let is_exact = |node: &OperatorNode, kind: ExactKind| { - matches!(&node.operator, asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(k, _), .. - }) if *k == kind) - }; - let inventory = asap_aware_mapping::search_workload(vec![("q", root)]) - .enumerate_candidate_dags(4096) - .unwrap(); - let candidates = inventory - .candidates - .into_iter() - .map(|mut forest| forest.remove(0).1) - .filter(|candidate| { - matches!(&candidate.operator, asap_types::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) - if is_exact(child, ExactKind::Sum)) - }) - .collect::>(); - let [candidate] = candidates.as_slice() else { - panic!("one grouped Sum candidate, got {}", candidates.len()); - }; - let mut placements = Vec::new(); - for sum_lifecycle in [ - SummaryMaintenanceLifecycle::ContinuouslyMaintained, - SummaryMaintenanceLifecycle::Ephemeral, - ] { - let lifecycles = enumerate_summary_maintenance_lifecycles( - Rc::clone(candidate), - WorkloadDemand::new_with_data( - &workload.query_workload, - workload.data_workload.as_ref().unwrap(), - &[1], - ), - NOW_MS, - Some(Horizon(100.)), - SummaryMaintenanceLifecycleCapabilities::ALL, - &FullyCostedRuntime, - ) - .unwrap(); - let choices = lifecycles - .deployments() - .iter() - .map(|deployment| { - let lifecycle = if is_exact(&deployment.summary, ExactKind::Sum) { - sum_lifecycle.clone() - } else { - SummaryMaintenanceLifecycle::ContinuouslyMaintained - }; - (deployment.post_asap_node_id, lifecycle) - }) - .collect::>(); - assert_eq!(choices.len(), 2, "Rate and Sum states"); - let dag = lifecycles - .select(&choices) - .unwrap() - .execution_timed_dag() - .unwrap(); - let raw = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::Relational { - operator: asap_types::ir::export::NonASAPOpKind::TimeRange { .. } - } - ) - }) - .unwrap(); - let frontier = - asap_physical_operators::physical_planner::frontier_from_timing(&dag).unwrap(); - let [boundary] = frontier.as_slice() else { - panic!("one precompute output, got {frontier:?}"); - }; - let boundary = dag - .nodes - .iter() - .find(|node| u64::from(node.id.0) == *boundary) - .unwrap(); - let physical = compile_candidate( - &dag, - BTreeMap::from([( - u64::from(raw.id.0), - InputContract::bounded(Arc::new(raw.output_schema.clone())), - )]), - &[u64::from(dag.roots[0].0)], - &frontier, - ) - .unwrap(); - let json = |value| String::from_utf8(serde_json::to_vec(value).unwrap()).unwrap(); - placements.push(( - boundary.payload.clone(), - json(physical.precompute.as_ref().unwrap()), - json(&physical.query), - )); - } - let builds = |json: &str, kind: &str| { - json.contains(&format!( - r#"{{"SummaryBuild":{{"family":{{"ExactAggregate":["{kind}","{kind}"]}}"# - )) - }; - let [(retained, retained_pre, retained_query), (ephemeral, ephemeral_pre, ephemeral_query)] = - placements.as_slice() - else { - unreachable!() - }; - let state = |payload: &PhysicalASAPOperatorPayload, kind: ExactKind| { - matches!(payload, PhysicalASAPOperatorPayload::SummaryAgg { - family: FieldDataType::ExactAggregate(k, _), .. - } if *k == kind) - }; - assert!(state(retained, ExactKind::Sum)); - assert!(builds(retained_pre, "Rate") && builds(retained_pre, "Sum")); - assert!(!retained_query.contains("SummaryBuild")); - assert!(state(ephemeral, ExactKind::Rate)); - assert!(builds(ephemeral_pre, "Rate") && !builds(ephemeral_pre, "Sum")); - assert!(builds(ephemeral_query, "Sum")); -} - -/// The lifecycle-timed DAG Planner selects for `query` with upfront series -/// typing, and whether it keeps an ingestion-time Binary. -fn typed_selection(query: &str) -> (asap_types::ir::export::PhysicalASAPDAG, bool) { - use asap_types::post_asap::ExecutionTiming; - let workload = quantile_workload(query); - let lowered = asap_types::ir::schema_support::with_promql_series_identity( - &lower_promql_workload(&workload, 0).unwrap().remove(0), - ) - .unwrap(); - let dag = selected_plan_for_lowered(&workload, lowered, &FullyCostedRuntime, Horizon(100.)) - .execution_timed_dag() - .unwrap(); - let ingestion_binary = dag.nodes.iter().any(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::Relational { - operator: asap_types::ir::export::NonASAPOpKind::BinaryOp { .. } - } - ) && node.output_state.timing == ExecutionTiming::IngestionTime - }); - (dag, ingestion_binary) -} - -/// Execute a timed DAG's precompute and query DAGs over `samples` -/// (`(metric, job, seconds, value)`) at 300s; returns the root's values. -fn execute_timed( - dag: &asap_types::ir::export::PhysicalASAPDAG, - samples: &[(&str, &str, i64, f64)], -) -> Vec { - use asap_physical_operators::{ - physical_planner::{compile_candidate, frontier_from_timing, promql_rows, InputContract}, - runtime::Scope, - values::{Batch, Value}, - }; - use asap_types::{ir::export::PhysicalASAPOperatorPayload, pre_asap::Source}; - use std::{collections::BTreeMap, sync::Arc}; - // Raw inputs: a selector Fallback is itself the input; a retained - // expression reads each of its selectors through its raw-series slots. - let mut raw = BTreeMap::new(); - for node in &dag.nodes { - if !matches!( - node.payload, - PhysicalASAPOperatorPayload::Relational { - operator: asap_types::ir::export::NonASAPOpKind::TimeRange { .. } - | asap_types::ir::export::NonASAPOpKind::Scan { .. } - } - ) { - continue; - } - let mut id = node.id; - loop { - let n = dag.nodes.iter().find(|n| n.id == id).unwrap(); - if let PhysicalASAPOperatorPayload::Relational { - operator: - asap_types::ir::export::NonASAPOpKind::Scan { - source: Source::TimeSeries { metric }, - .. - }, - } = &n.payload - { - raw.insert( - u64::from(node.id.0), - (Arc::new(node.output_schema.clone()), metric.clone()), - ); - break; - } - id = dag - .edges - .iter() - .find(|e| e.consumer == id) - .unwrap() - .producer; - } - } - let batch = |schema: &asap_physical_operators::values::SchemaRef, name: &str| { - let rows = samples - .iter() - .filter(|sample| sample.0 == name) - .map(|(metric, job, seconds, value)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), metric.to_string()), - ("job".to_string(), job.to_string()), - ]); - promql_rows::series_row(schema, &labels, seconds * 1000, *value).unwrap() - }) - .collect(); - Batch::try_new(schema.clone(), rows).unwrap() - }; - let frontier = frontier_from_timing(dag).unwrap(); - let candidate = compile_candidate( - dag, - raw.iter() - .map(|(id, (schema, _))| (*id, InputContract::bounded(schema.clone()))) - .collect(), - &[u64::from(dag.roots[0].0)], - &frontier, - ) - .unwrap(); - let raw_sources = |plan: &asap_physical_operators::physical_planner::CompiledPhysicalDAG| { - plan.input_contracts() - .filter_map(|(id, _)| raw.get(&id).map(|(schema, name)| (id, batch(schema, name)))) - .collect::>() - }; - let mut query_sources = raw_sources(&candidate.query); - if let Some(precompute) = &candidate.precompute { - let stored = physical_common::execute( - precompute, - raw_sources(precompute), - Scope::Ingestion { - window_start_ms: 240_000, - window_end_ms: 300_000, - revision: 1, - }, - ); - for (root, batches) in precompute.roots().iter().zip(stored) { - query_sources.insert(*root, batches[0].clone()); - } - } - let result = physical_common::execute( - &candidate.query, - query_sources, - Scope::Query { - evaluation_time_ms: 300_000, - revision: 1, - }, - ); - result[0] - .iter() - .flat_map(|batch| batch.rows()) - .flat_map(|row| row.iter()) - .filter_map(|value| match value { - Value::Float64(value) => Some(*value), - _ => None, - }) - .collect() -} - -/// Prometheus drops series without a match: arithmetic over different -/// selectors keeps only label sets present on both sides (none when disjoint), -/// and such arithmetic never becomes aligned maintenance. -#[test] -fn maintained_arithmetic_over_different_selectors_matches_prometheus() { - let query = "sum(sum_over_time(m[1m]) + sum_over_time(n[1m]))"; - let (dag, ingestion_binary) = typed_selection(query); - let disjoint = [ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("n", "b", 250, 5.0), - ]; - let values = execute_timed(&dag, &disjoint); - assert!(values.is_empty(), "{values:?}"); - // Only job a is on both sides: m_a + n_a = (1 + 2) + 7; m{job="b"} is dropped. - let overlapping = [ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("m", "b", 250, 5.0), - ("n", "a", 250, 7.0), - ]; - assert_eq!(execute_timed(&dag, &overlapping), [10.0]); - assert!(!ingestion_binary); - // The quantile's exact fallback runs outside Planner; it must not be maintained either. - let (_, ingestion_binary) = - typed_selection("quantile(0.9, sum_over_time(m[1m]) + sum_over_time(n[1m]))"); - assert!(!ingestion_binary); -} - -/// Arithmetic over one selector keeps its maintained layout and adds each -/// series' two evaluations before the quantile. -#[test] -fn maintained_arithmetic_over_one_selector_executes() { - let (dag, ingestion_binary) = - typed_selection("quantile(0.9, sum_over_time(m[1m]) + sum_over_time(m[1m]))"); - assert!(ingestion_binary, "one selector shares its key set"); - let values = execute_timed( - &dag, - &[ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("m", "b", 250, 5.0), - ], - ); - // job a: 3 + 3 = 6; job b: 5 + 5 = 10 (mispairing a with b gives 8 and 8). - // KLL at epsilon 0.01 returns an input value within 0.01 of rank 0.9; of - // two values only the larger is. - assert_eq!(values, [10.0]); -} diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index 4f527f148..506ed12ea 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -25,8 +25,8 @@ use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError}; // configures the same models and reads the same output whether it goes through // `e2e_plan` or straight to `optimize`. pub use asap_aware_mapping::pass::{ - optimize, LifecycleInput, MajorPass, OptimizationInput, OptimizationPass, OptimizeError, - PassRegistry, PlanOutput, PlanningModels, QueryLifecyclePlan, + optimize, MajorPass, OptimizationInput, OptimizationPass, OptimizeError, PassRegistry, + PlanOutput, PlanningModels, QueryPlan, }; // ── Input ──────────────────────────────────────────────────────────────── @@ -53,9 +53,6 @@ pub struct UserInput<'a> { pub workload: &'a PlanningWorkload, pub frontend_specific: FrontendInput<'a>, pub models: PlanningModels<'a>, - /// Planning clock and runtime capabilities for the - /// maintenance-versus-recomputation decision every plan carries. - pub lifecycle: LifecycleInput, /// `None` uses [`MajorPass`]. A black-box caller never sets this. pub pass: Option<&'a dyn OptimizationPass>, } @@ -65,13 +62,11 @@ impl<'a> UserInput<'a> { workload: &'a PlanningWorkload, frontend_specific: FrontendInput<'a>, models: PlanningModels<'a>, - lifecycle: LifecycleInput, ) -> Self { Self { workload, frontend_specific, models, - lifecycle, pass: None, } } @@ -101,21 +96,6 @@ impl<'a> UserInput<'a> { }); } - if let Some(horizon) = self.lifecycle.horizon { - if !horizon.0.is_finite() || horizon.0 <= 0.0 { - return Err(UserInputError::InvalidHorizon(horizon.0)); - } - } - // Two clocks would let the DAG be built for one instant and priced - // for another, with neither stage able to notice. - if let FrontendInput::Promql { now_ms, .. } = &self.frontend_specific { - if *now_ms != self.lifecycle.now_ms { - return Err(UserInputError::PlanningTimeMismatch { - frontend: *now_ms, - lifecycle: self.lifecycle.now_ms, - }); - } - } Ok(()) } } @@ -142,10 +122,6 @@ pub enum UserInputError { language: String, frontend: &'static str, }, - #[error("planning horizon must be finite and positive, got {0}")] - InvalidHorizon(f64), - #[error("frontend planning time {frontend} ms disagrees with lifecycle planning time {lifecycle} ms")] - PlanningTimeMismatch { frontend: u64, lifecycle: u64 }, } #[derive(Debug, thiserror::Error)] @@ -194,7 +170,7 @@ pub async fn e2e_plan(input: UserInput<'_>) -> Result { let fallback = MajorPass; let pass: &dyn OptimizationPass = input.pass.unwrap_or(&fallback); - let optimization = OptimizationInput::new(&parsed, input.models, input.lifecycle); + let optimization = OptimizationInput::new(&parsed, input.models); optimize(pass, optimization).map_err(PlanError::Optimize) } @@ -202,8 +178,7 @@ pub async fn e2e_plan(input: UserInput<'_>) -> Result { /// /// The SQL and MetricsQL frontends are driven one entry at a time rather than /// through `lower_sql_batch`, which walks `query_batch` alone and would drop -/// every repeating query — exactly the entries whose recurrence the lifecycle -/// stage needs. +/// every repeating query, and the output must cover every entry. async fn lower(input: &UserInput<'_>) -> Result, PlanError> { let entries = || input.workload.query_workload.entries(); diff --git a/crates/planner/tests/e2e_plan.rs b/crates/planner/tests/e2e_plan.rs index 3ab55b101..0f929c96d 100644 --- a/crates/planner/tests/e2e_plan.rs +++ b/crates/planner/tests/e2e_plan.rs @@ -7,9 +7,7 @@ use asap_aware_mapping::pass::{ OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, PlanningModels, }; use asap_aware_mapping::replacement::default_strategies_with_evidence; -use asap_aware_mapping::{ - search_workload_with_targets, Horizon, LifecycleInput, SummaryMaintenanceLifecycleCapabilities, -}; +use asap_aware_mapping::search_workload_with_targets; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; use asap_planner::{e2e_plan, FrontendInput, PlanError, UserInput, UserInputError}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; @@ -40,12 +38,6 @@ fn batch(sql: &str) -> BatchEntry { } } -/// The planning clock and default capabilities, no horizon: the least a -/// caller can supply. -fn lifecycle() -> LifecycleInput { - LifecycleInput::new(NOW_MS, SummaryMaintenanceLifecycleCapabilities::default()) -} - fn lineitem_catalog() -> SqlCatalog { SqlCatalog::new().with_table( "lineitem", @@ -89,7 +81,6 @@ async fn plans_every_query_in_entry_order() { &workload, FrontendInput::Sql { catalog: &catalog }, PlanningModels::builtin(), - lifecycle(), ); let output = e2e_plan(input).await.expect("workload plans"); @@ -97,13 +88,10 @@ async fn plans_every_query_in_entry_order() { assert_eq!(output.entry_indices(), vec![0, 1]); } -/// With the built-in cost model no lifecycle cost is ever known, and -/// lifecycle-aware selection then finalizes every summary target as raw -/// recompute: the cost-only selection picks a sketch for the same workload. -/// This pins that behavior so the facade's output is not mistaken for a -/// decision; it is a defect of `DefaultCostModel`, not addressed here. +/// The facade selects what workload-wide cost selection selects over the +/// same search space: here a summary for both approximate queries. #[tokio::test] -async fn builtin_cost_model_cannot_price_lifecycles_and_falls_back_to_raw_recompute() { +async fn facade_plans_match_cost_only_selection() { let workload = sql_workload( vec![ batch("SELECT COUNT(DISTINCT l_orderkey) FROM lineitem"), @@ -118,7 +106,6 @@ async fn builtin_cost_model_cannot_price_lifecycles_and_falls_back_to_raw_recomp &workload, FrontendInput::Sql { catalog: &catalog }, models, - lifecycle(), )) .await .expect("workload plans"); @@ -153,21 +140,17 @@ async fn builtin_cost_model_cannot_price_lifecycles_and_falls_back_to_raw_recomp "entry {}: cost-only selection was expected to pick a summary", plan.entry_index ); - assert!( - !plan.plan.root.contains_asap() - && plan.plan.selected_raw_recompute - && plan.plan.deployments.is_empty() - && plan.plan.summary_total_cost.is_none() - && plan.plan.raw_recompute_total_cost.is_none(), - "entry {}: the built-in model priced a lifecycle", + assert_eq!( + plan.root, cost_only, + "entry {}: the facade selected a different DAG", plan.entry_index ); } } /// A repeating SQL query reaches the optimizer. `lower_sql_batch` walks -/// `query_batch` alone, so driving the frontend through it would drop exactly -/// the entries whose recurrence the lifecycle stage reads. +/// `query_batch` alone, so driving the frontend through it would drop the +/// repeating entries. #[tokio::test] async fn lowers_repeating_sql_entries_too() { let workload = sql_workload( @@ -187,7 +170,6 @@ async fn lowers_repeating_sql_entries_too() { &workload, FrontendInput::Sql { catalog: &catalog }, PlanningModels::builtin(), - lifecycle(), ); let output = e2e_plan(input).await.expect("workload plans"); @@ -229,7 +211,6 @@ async fn runs_a_caller_supplied_pass_instead_of_the_shipped_one() { &workload, FrontendInput::Sql { catalog: &catalog }, PlanningModels::builtin(), - lifecycle(), ) .with_pass(&pass); @@ -270,7 +251,6 @@ async fn harness_rejects_a_pass_that_mislabels_entry_indices() { &workload, FrontendInput::Sql { catalog: &catalog }, PlanningModels::builtin(), - lifecycle(), ) .with_pass(&pass); @@ -302,7 +282,6 @@ async fn rejects_a_frontend_that_does_not_match_the_workload_language() { histograms: None, }, PlanningModels::builtin(), - lifecycle(), ); let err = e2e_plan(input).await.unwrap_err(); @@ -312,48 +291,9 @@ async fn rejects_a_frontend_that_does_not_match_the_workload_language() { )); } -/// Two planning clocks would let the DAG be built for one instant and priced -/// for another; the input check refuses that before lowering. -#[test] -fn rejects_disagreeing_planning_clocks() { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![batch("up")]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - data_ingestion_interval: Evidence { - value: Some(DurationMs(15_000)), - ..Default::default() - }, - ..Default::default() - }), - }; - let input = UserInput::new( - &workload, - FrontendInput::Promql { - now_ms: NOW_MS, - histograms: None, - }, - PlanningModels::builtin(), - LifecycleInput::new( - NOW_MS + 1, - SummaryMaintenanceLifecycleCapabilities::default(), - ), - ); - - assert!(matches!( - input.validate(), - Err(UserInputError::PlanningTimeMismatch { .. }) - )); -} - -/// The maintenance decisions ride inside each plan, and the DAG is still -/// there — inside the plan's `root`, not alongside it. +/// A repeating PromQL query yields one plan carrying its selected DAG root. #[tokio::test] -async fn lifecycle_decisions_ride_inside_each_plan() { +async fn each_plan_carries_its_selected_root() { let workload = PlanningWorkload { query_workload: QueryWorkload { language: QueryLanguage::PromQL, @@ -382,61 +322,15 @@ async fn lifecycle_decisions_ride_inside_each_plan() { histograms: None, }, PlanningModels::builtin(), - lifecycle().with_horizon(Horizon(3_600.0)), ); let output = e2e_plan(input).await.expect("workload plans"); assert_eq!(output.plans.len(), 1); assert_eq!(output.plans[0].entry_index, 0); - let _: &Rc<_> = &output.plans[0].plan.root; + let _: &Rc<_> = &output.plans[0].root; assert_eq!(output.operator_roots().len(), 1); } -/// Each root's lifecycle is planned against the entries that read it: a -/// query polled every minute and an unrelated one polled every ten minutes -/// each see only their own reads over the hour, not the workload's 66. -#[tokio::test] -async fn each_plan_counts_only_its_own_entries_reads() { - let repeating = |query: &str, interval_ms: u32| RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(interval_ms)), - requirements: approximate(), - predictability: Predictability::Unknown, - time_selection: TimeSelection::default(), - }; - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![ - repeating("count_over_time(up[5m])", 60_000), - repeating("sum_over_time(latency[5m])", 600_000), - ]), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - data_ingestion_interval: Evidence { - value: Some(DurationMs(15_000)), - ..Default::default() - }, - ..Default::default() - }), - }; - let input = UserInput::new( - &workload, - FrontendInput::Promql { - now_ms: NOW_MS, - histograms: None, - }, - PlanningModels::builtin(), - lifecycle().with_horizon(Horizon(3_600.0)), - ); - - let output = e2e_plan(input).await.expect("workload plans"); - let reads: Vec<_> = output.plans.iter().map(|p| p.plan.expected_reads).collect(); - assert_eq!(reads, vec![Some(60.0), Some(6.0)]); -} - /// Scalar-only and mixed workloads preserve entry bindings without wrapper nodes. #[tokio::test] async fn scalar_roots_survive_planning_in_workload_order() { @@ -465,7 +359,6 @@ async fn scalar_roots_survive_planning_in_workload_order() { histograms: None, }, PlanningModels::builtin(), - lifecycle(), )) .await .unwrap(); diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index 7c198493b..8eb9ed4f8 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -1,5 +1,5 @@ //! Structurally identical summary producers chosen by different queries are -//! shared after Pass 1: one `Rc` across their plans, costed once. +//! shared after Pass 1: one `Rc` across their plans. use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::{ASAPOp, OperatorNode}; @@ -8,16 +8,11 @@ use std::rc::Rc; use asap_aware_mapping::accuracy::{ AccuracyModel, DefaultAccuracyModel, EqualSplitAllocator, PropagationStats, }; -use asap_aware_mapping::cost_model::Cost; -use asap_aware_mapping::pass::{PlanOutput, PlanningModels}; +use asap_aware_mapping::pass::{PlanOutput, PlanningModels, QueryPlan}; use asap_aware_mapping::replacement::{default_size_params, DEFAULT_DELTA}; use asap_aware_mapping::{ - global_selection_with_summary_maintenance_lifecycles, search_workload_with_targets, - ASAPStrategies, ReplacementStrategy, WorkloadDemand, -}; -use asap_aware_mapping::{ - CostModel, CostRate, DefaultCostModel, Horizon, LifecycleInput, SummaryMaintenanceCapabilities, - SummaryMaintenanceLifecycleCapabilities, SummaryMaintenanceLifecycleCostInputs, + search_workload_with_targets, ASAPStrategies, CostModel, DefaultCostModel, Replacement, + ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; use asap_frontend_promql::lower_promql_workload; use asap_frontend_sql::SqlCatalog; @@ -26,7 +21,7 @@ use asap_types::post_asap::{ AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, ProbabilityExpr, ResultGuarantee, SketchStatistic, }; -use asap_types::post_asap::{FieldDataType, SketchAlgorithm, SketchParams}; +use asap_types::post_asap::{FieldDataType, SketchAlgorithm, SketchKind, SketchParams}; use asap_types::pre_asap::agg_intent::default_quantile; use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::pre_asap::AggIntent; @@ -38,16 +33,18 @@ use asap_types::workload::{ }; const NOW_MS: u64 = 1_700_000_000_000; -const HORIZON_S: f64 = 3_600.0; -/// A state costs `build` once however often it is read; raw recomputation -/// costs `raw_per_read` per read. -struct FixedCosts { - build: f64, - raw_per_read: f64, -} +/// Stand-in for the workload-level amortization Stage 2 materialization will +/// price: a sketch candidate costs `preference(kind)` per sketch state, any +/// other candidate more than every sketch. Ranking is otherwise built-in. +struct PreferSketch(fn(&SketchKind) -> f64); + +impl CostModel for PreferSketch { + // Selection takes the cheapest candidate by `estimate_cost`. + fn candidate_cost_covers_complete_plan(&self) -> bool { + true + } -impl CostModel for FixedCosts { fn rank_candidates( &self, intent: &AggIntent, @@ -56,41 +53,42 @@ impl CostModel for FixedCosts { DefaultCostModel.rank_candidates(intent, candidates) } - fn summary_maintenance_lifecycle_cost_inputs( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(self.build)), - maintenance_cost_per_update: Some(Cost::ZERO), - summary_read_cost: Some(Cost::ZERO), - retention_cost_rate: Some(CostRate(0.0)), - retirement_cost: Some(Cost::ZERO), - } - } - - fn summary_maintenance_capabilities( - &self, - _summary: &OperatorNode, - ) -> SummaryMaintenanceCapabilities { - SummaryMaintenanceCapabilities { - incremental_update: true, - merge: true, - delete: true, + fn estimate_cost(&self, candidate: &ReplacementSubDAG, _: &TargetSubDAG<'_>) -> f64 { + let Replacement::SubDAG(root) = &candidate.replacement else { + return 1e9; + }; + let kinds: Vec<_> = OperatorNode::reachable(root) + .into_iter() + .filter_map(|node| match &node.operator { + asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(kind, _), + .. + }) => Some(kind.clone()), + _ => None, + }) + .collect(); + if kinds.is_empty() { + 1e9 + } else { + kinds.iter().map(self.0).sum() } } - - fn raw_query_recompute_cost(&self, _target: &OperatorNode) -> Option { - Some(Cost(self.raw_per_read)) - } } -/// Summaries are far cheaper than raw recomputation, so every query selects -/// one independently and only sharing is under test. -const CHEAP_SUMMARY: FixedCosts = FixedCosts { - build: 1.0, - raw_per_read: 1_000.0, -}; +/// Prefers the largest KLL, i.e. one sized for the strictest consumer. +const PREFER_LARGE_KLL: PreferSketch = PreferSketch(|kind| match kind.params() { + SketchParams::Kll { k } => 1.0 / f64::from(*k), + _ => 1.0, +}); + +/// Prefers UnivMon, which can serve every frequency moment from one state. +const PREFER_UNIVMON: PreferSketch = PreferSketch(|kind| { + if kind.algorithm() == &SketchAlgorithm::UnivMon { + 0.0 + } else { + 1.0 + } +}); fn requirements(epsilon: f64) -> QueryRequirements { QueryRequirements { @@ -110,11 +108,6 @@ fn repeating(query: &str, epsilon: f64) -> RepeatingEntry { } } -fn lifecycle() -> LifecycleInput { - LifecycleInput::new(NOW_MS, SummaryMaintenanceLifecycleCapabilities::default()) - .with_horizon(Horizon(HORIZON_S)) -} - fn promql_workload(queries: &[(&str, f64)]) -> PlanningWorkload { PlanningWorkload { query_workload: QueryWorkload { @@ -142,7 +135,11 @@ fn promql_workload(queries: &[(&str, f64)]) -> PlanningWorkload { } } -async fn plan_promql(queries: &[(&str, f64)], costs: &FixedCosts) -> PlanOutput { +async fn plan_promql(queries: &[(&str, f64)]) -> PlanOutput { + plan_promql_with(queries, &DefaultCostModel).await +} + +async fn plan_promql_with(queries: &[(&str, f64)], cost: &dyn CostModel) -> PlanOutput { let workload = promql_workload(queries); let input = UserInput::new( &workload, @@ -150,13 +147,12 @@ async fn plan_promql(queries: &[(&str, f64)], costs: &FixedCosts) -> PlanOutput now_ms: NOW_MS, histograms: None, }, - PlanningModels::builtin().with_cost(costs), - lifecycle(), + PlanningModels::builtin().with_cost(cost), ); e2e_plan(input).await.expect("workload plans") } -async fn plan_sql(queries: &[&str], costs: &FixedCosts) -> PlanOutput { +async fn plan_sql(queries: &[&str]) -> PlanOutput { let workload = PlanningWorkload { query_workload: QueryWorkload { language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), @@ -178,25 +174,33 @@ async fn plan_sql(queries: &[&str], costs: &FixedCosts) -> PlanOutput { let input = UserInput::new( &workload, FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin().with_cost(costs), - lifecycle(), + PlanningModels::builtin(), ); e2e_plan(input).await.expect("workload plans") } -/// Every summary state each plan deploys. +/// Every summary state (`SummaryAgg`) each plan reaches, in traversal order. +fn plan_states(plan: &QueryPlan) -> Vec> { + OperatorNode::reachable(&plan.root) + .into_iter() + .filter(|node| { + matches!( + node.operator, + asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { .. }) + ) + }) + .collect() +} + +/// Every summary state each plan reaches; each plan selects at least one. fn states(output: &PlanOutput) -> Vec>> { output .plans .iter() .map(|plan| { - assert!(!plan.plan.selected_raw_recompute, "{:?}", plan.plan.root); - assert!(!plan.plan.deployments.is_empty()); - plan.plan - .deployments - .iter() - .map(|deployment| Rc::clone(&deployment.summary)) - .collect() + let states = plan_states(plan); + assert!(!states.is_empty(), "{:?}", plan.root); + states }) .collect() } @@ -210,12 +214,12 @@ fn same_states(states: &[Vec>]) -> bool { .all(|(left, right)| Rc::ptr_eq(left, right)) } -/// The deployments a consumer would run, deduplicated by pointer. +/// The summary states a consumer would run, deduplicated by pointer. fn unique_deployments(output: &PlanOutput) -> usize { let mut seen: Vec<*const OperatorNode> = Vec::new(); for plan in &output.plans { - for deployment in &plan.plan.deployments { - let ptr = Rc::as_ptr(&deployment.summary); + for state in plan_states(plan) { + let ptr = Rc::as_ptr(&state); if !seen.contains(&ptr) { seen.push(ptr); } @@ -225,39 +229,18 @@ fn unique_deployments(output: &PlanOutput) -> usize { } /// p50 and p99 over the same window and accuracy read one KLL: the -/// equal-params subset of summary capability. Both plans hold the same `Rc` -/// with the same lifecycle, so a consumer maintains it once. +/// equal-params subset of summary capability. Both plans hold the same `Rc`, +/// so a consumer maintains it once. #[tokio::test] async fn quantiles_with_equal_params_share_one_producer() { - let output = plan_promql( - &[ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[5m])", 0.01), - ], - &CHEAP_SUMMARY, - ) + let output = plan_promql(&[ + ("quantile_over_time(0.5, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[5m])", 0.01), + ]) .await; assert!(same_states(&states(&output))); - assert!(!Rc::ptr_eq( - &output.plans[0].plan.root, - &output.plans[1].plan.root - )); + assert!(!Rc::ptr_eq(&output.plans[0].root, &output.plans[1].root)); assert_eq!(unique_deployments(&output), 1); - let lifecycles: Vec<_> = output - .plans - .iter() - .map(|plan| { - plan.plan.deployments[0] - .summary_maintenance_lifecycle_guarantee - .clone() - }) - .collect(); - assert_eq!(lifecycles[0], lifecycles[1]); - assert!(lifecycles[0].is_some()); - // Each plan is planned against both queries' reads. - for plan in &output.plans { - assert_eq!(plan.plan.expected_reads, Some(12.0)); - } } /// A different window or label selector is a different producer, even when @@ -282,27 +265,27 @@ async fn different_producers_are_not_shared() { ("quantile_over_time(0.99, lat{job=\"b\"}[5m])", 0.01), ], ] { - let output = plan_promql(&queries, &CHEAP_SUMMARY).await; + let output = plan_promql(&queries).await; assert!(!same_states(&states(&output)), "{queries:?}"); assert_eq!(unique_deployments(&output), 2, "{queries:?}"); for (plan, (_, epsilon)) in output.plans.iter().zip(queries) { - assert_eq!(plan.plan.expected_reads, Some(6.0), "{queries:?}"); assert_eq!(kll_k(plan), kll_k_for(epsilon), "{queries:?}"); } } } /// The KLL `k` of the one state a plan deploys. -fn kll_k(plan: &asap_aware_mapping::pass::QueryLifecyclePlan) -> u32 { - let [deployment] = plan.plan.deployments.as_slice() else { - panic!("one state: {:?}", plan.plan.deployments.len()); +fn kll_k(plan: &QueryPlan) -> u32 { + let states = plan_states(plan); + let [deployment] = states.as_slice() else { + panic!("one state: {:?}", states.len()); }; let asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. - }) = &deployment.summary.operator + }) = &deployment.operator else { - panic!("sketch state: {:?}", deployment.summary.operator); + panic!("sketch state: {:?}", deployment.operator); }; let SketchParams::Kll { k } = kind.params() else { panic!("KLL state: {kind:?}"); @@ -324,19 +307,20 @@ fn kll_k_for(epsilon: f64) -> u32 { } /// p50 at ε=0.01 and p99 at ε=0.001 over the same input share one KLL sized -/// for the strictest consumer; each reader's guarantee meets its own target. +/// for the strictest consumer when the cost model prefers that candidate; each +/// reader's guarantee meets its own target. #[tokio::test] async fn quantiles_share_one_producer_sized_for_the_strictest_consumer() { let p50 = ("quantile_over_time(0.5, lat[5m])", 0.01); let p99 = ("quantile_over_time(0.99, lat[5m])", 0.001); assert!(kll_k_for(0.001) > kll_k_for(0.01)); - let output = plan_promql(&[p50, p99], &CHEAP_SUMMARY).await; + let output = plan_promql_with(&[p50, p99], &PREFER_LARGE_KLL).await; assert!(same_states(&states(&output))); assert_eq!(unique_deployments(&output), 1); for (plan, (_, epsilon)) in output.plans.iter().zip([p50, p99]) { assert_eq!(kll_k(plan), kll_k_for(0.001)); - let guarantee = plan.plan.root.guarantee.as_ref().expect("certified"); + let guarantee = plan.root.guarantee.as_ref().expect("certified"); assert!( guarantee.bound.evaluate().unwrap() <= epsilon, "{guarantee:?}" @@ -344,7 +328,7 @@ async fn quantiles_share_one_producer_sized_for_the_strictest_consumer() { } // Alone, the looser query keeps its own, smaller KLL. - let alone = plan_promql(&[p50], &CHEAP_SUMMARY).await; + let alone = plan_promql_with(&[p50], &PREFER_LARGE_KLL).await; assert_eq!(kll_k(&alone.plans[0]), kll_k_for(0.01)); } @@ -352,11 +336,7 @@ async fn quantiles_share_one_producer_sized_for_the_strictest_consumer() { /// quantile, so p50 and p99 over one selector share it. #[tokio::test] async fn cross_series_p50_and_p99_share_one_producer() { - let output = plan_promql( - &[("quantile(0.5, lat)", 0.01), ("quantile(0.99, lat)", 0.01)], - &CHEAP_SUMMARY, - ) - .await; + let output = plan_promql(&[("quantile(0.5, lat)", 0.01), ("quantile(0.99, lat)", 0.01)]).await; assert!(same_states(&states(&output))); assert_eq!(unique_deployments(&output), 1); } @@ -366,7 +346,7 @@ async fn cross_series_p50_and_p99_share_one_producer() { #[tokio::test] async fn identical_ungrouped_queries_share_their_producers() { let query = ("sum(rate(x[5m]))", 0.01); - let output = plan_promql(&[query, query], &CHEAP_SUMMARY).await; + let output = plan_promql(&[query, query]).await; assert!(same_states(&states(&output))); assert_eq!(unique_deployments(&output), 2); } @@ -377,7 +357,7 @@ async fn identical_ungrouped_queries_share_their_producers() { async fn identical_sql_percentiles_share_one_producer() { let query = "SELECT approx_percentile_cont(l_extendedprice, 0.5) FROM lineitem WHERE l_orderkey > 10"; - let output = plan_sql(&[query, query], &CHEAP_SUMMARY).await; + let output = plan_sql(&[query, query]).await; assert!(same_states(&states(&output))); assert_eq!(unique_deployments(&output), 1); } @@ -391,15 +371,14 @@ async fn sql_p50_and_p99_share_one_producer() { "SELECT approx_percentile_cont(l_extendedprice, 0.5) FROM lineitem WHERE l_orderkey > 10"; let p99 = "SELECT approx_percentile_cont(l_extendedprice, 0.99) FROM lineitem WHERE l_orderkey > 10"; - let output = plan_sql(&[p50, p99], &CHEAP_SUMMARY).await; + let output = plan_sql(&[p50, p99]).await; assert!(same_states(&states(&output))); assert_eq!(unique_deployments(&output), 1); let names: Vec<_> = output .plans .iter() .map(|plan| { - plan.plan - .root + plan.root .schema .fields .iter() @@ -425,32 +404,12 @@ async fn sql_p50_and_p99_share_one_producer() { "SELECT approx_percentile_cont(l_orderkey, 0.99) FROM lineitem WHERE l_orderkey > 10", ], ] { - let output = plan_sql(&queries, &CHEAP_SUMMARY).await; + let output = plan_sql(&queries).await; assert!(!same_states(&states(&output)), "{queries:?}"); assert_eq!(unique_deployments(&output), 2, "{queries:?}"); } } -/// A state costs 100 and recomputing a query costs 60 over its six reads: -/// alone, the query recomputes raw. Shared by p50 and p99, the state costs 50 -/// per query, so both keep it. -#[tokio::test] -async fn shared_amortization_alone_can_beat_raw_recompute() { - let costs = FixedCosts { - build: 100.0, - raw_per_read: 10.0, - }; - let p50 = ("quantile_over_time(0.5, lat[5m])", 0.01); - let p99 = ("quantile_over_time(0.99, lat[5m])", 0.01); - - let alone = plan_promql(&[p50], &costs).await; - assert!(alone.plans[0].plan.selected_raw_recompute); - - let output = plan_promql(&[p50, p99], &costs).await; - assert!(same_states(&states(&output))); - assert_eq!(unique_deployments(&output), 1); -} - /// Synthetic evidence certifying UnivMon evaluations; it exercises sharing, never /// runtime accuracy. struct UnivMonEvidence; @@ -489,7 +448,7 @@ impl AccuracyModel for UnivMonEvidence { } /// Distinct count, entropy and L2 over one input, certified by an accuracy -/// model, read one UnivMon state: #515 sharing is the summary-capability rule +/// model and selected by a cost model preferring UnivMon, read one UnivMon state: #515 sharing is the summary-capability rule /// when the states are identical. `MajorPass` builds candidates with the /// built-in accuracy model, so this runs its pipeline with the test model. #[test] @@ -509,25 +468,12 @@ fn certified_frequency_evaluations_share_one_univmon_state() { .collect(); let strategies: Vec> = vec![Box::new(ASAPStrategies::new_with_planning_inputs( - &CHEAP_SUMMARY, + &PREFER_UNIVMON, &UnivMonEvidence, &EqualSplitAllocator, ))]; let space = search_workload_with_targets(roots, &strategies, &UnivMonEvidence); - let entry_indices: Vec = (0..queries.len()).collect(); - let selection = global_selection_with_summary_maintenance_lifecycles( - &space, - WorkloadDemand { - workload: &workload.query_workload, - data_workload: workload.data_workload.as_ref(), - entry_indices: &entry_indices, - }, - NOW_MS, - Some(Horizon(HORIZON_S)), - SummaryMaintenanceLifecycleCapabilities::default(), - &CHEAP_SUMMARY, - ) - .expect("selects"); + let selection = space.global_selection(&PREFER_UNIVMON); let assembled = space .roots .iter() diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index 8572854a9..0e10ccdf4 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" # execution logic — removed, no real implementor existed; see issue #190). # No internal deps. [dependencies] -# "rc" — QueryExpr's child fields are Rc> (issue #212, #222: +# "rc" — OperatorNode child fields are Rc (issue #212, #222: # shared sub-expressions), and Rc's Serialize/Deserialize impls live behind # this feature flag. dag_export.rs / DAGNode already flatten the DAG to a # node+edge list for JSON export, so this does not change wire format — a diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index f95d71902..01d1da021 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -1,5 +1,4 @@ -//! Unified operator and scalar representation from #511. -//! Legacy consumers remain on their existing representation until the planner cutover. +//! The operator IR from #511: one operator DAG for every planning stage. pub mod aggregate_schema; pub mod asap; pub mod error; @@ -20,11 +19,11 @@ pub mod cse; pub mod export; /// Physical ASAP DAG transport: the logical payloads plus execution timing. pub mod physical_export; -/// Execution timing for physical plans: a lifecycle assignment expanded onto every node. +/// Execution timing for physical plans: a materialization assignment expanded onto every node. pub mod timing; pub use timing::{ - apply_lifecycle_timings, data_state, planned_data_state, split_shared_by_phase, - validate_default, LifecycleAssignment, TimingMemo, + apply_materialization_timings, data_state, planned_data_state, split_shared_by_phase, + validate_maintained, MaterializationAssignment, TimingMemo, }; /// Semantic observation coverage, separate from field layout and physical timing. pub mod summary_coverage; diff --git a/crates/types/src/ir/node.rs b/crates/types/src/ir/node.rs index cc71d32e5..15431962f 100644 --- a/crates/types/src/ir/node.rs +++ b/crates/types/src/ir/node.rs @@ -89,7 +89,7 @@ impl Operator { /// `schema` and `result_kind` are derived from `operator` and its children /// at construction and retained. `guarantee` is `None` until accuracy /// assessment establishes one (`None` never means exact). `timing` is `None` -/// until a lifecycle assignment is applied; export rejects an executable +/// until a materialization assignment is applied; export rejects an executable /// node without one. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OperatorNode { diff --git a/crates/types/src/ir/operator_properties.rs b/crates/types/src/ir/operator_properties.rs index 4737e6278..27648351b 100644 --- a/crates/types/src/ir/operator_properties.rs +++ b/crates/types/src/ir/operator_properties.rs @@ -1,8 +1,564 @@ -//! Operator parameters shared with the existing dag during migration. -//! Definitions move here when legacy dag consumers are removed. -pub use crate::pre_asap::query_expr::{ - AtModifier, BinaryOpKind, ColState, ConcatDiscriminatorKey, DataModel, GroupKeys, GroupSide, - InfoMatcher, JoinKind, PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, - Source, TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, - WindowFrameOffset, WindowFrameUnits, WindowFuncKind, -}; +//! Supporting parameter types used inside operator payloads. +//! +//! For example, `Aggregate.by` uses [`GroupKeys`], a join chooses [`JoinKind`], +//! and a SQL window carries [`WindowFrame`]. These types describe what an +//! operator does. Derived node metadata (schema, guarantee, timing) lives on +//! [`super::OperatorNode`], not in this module. +use crate::pre_asap::{ArithmeticOpKind, ColumnId, ColumnRef, CompareOpKind, ScalarValue}; +use serde::{Deserialize, Serialize}; +/// The column-reference type an operator parameter is generic over: +/// positional [`ColumnId`] once bound, name-based [`ColumnRef`] before. +pub trait ColState: + Clone + std::fmt::Debug + PartialEq + Serialize + for<'de> Deserialize<'de> +{ +} + +impl ColState for ColumnId {} + +impl ColState for ColumnRef {} + +// ── Leaf / supporting types ─────────────────────────────────────────────────── + +/// Positional grouping keys, shared by every "operate per group" operator: +/// `Aggregate.by` (reduce per group), `Sort.partition_by` (rank per group — +/// including generic `topk`/`bottomk`), and `SQLWindowFunc.partition_by` (window +/// per group). One spelling so grouping has a single home to evolve. Empty +/// (and `by`) = no grouping (a global operation). +/// +/// Heavy-hitter `AggIntent::TopK` carries its grouping here too, via the +/// enclosing `Aggregate.by` (issue #13) — so reduce, rank, and window groupings +/// all share this one type. +/// +/// ## `by` vs `without` (issue #39) +/// +/// The stored [`keys`](Self::keys) are **kept** labels for `by(...)` and +/// **excluded** labels for `without(...)`. PromQL's `without(labels)` groups by +/// every label *except* those listed; the complement can't be enumerated at +/// lowering time under an open (usage-derived) schema, so it is deferred to the +/// runtime — the excluded positions are stored, the kept set stays open. Only +/// `Aggregate` ever produces the `without` form; `Sort` / `SQLWindowFunc` / +/// `PromqlSeriesSample` groupings are always `by`. +/// +/// Serialises as a bare array for the (overwhelmingly common) `by` case — +/// wire-compatible with the `Vec` this field held before — and as +/// `{"without": [...]}` for the exclusion case. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct GroupKeys { + keys: Vec, + without: bool, +} + +// Not `#[derive(Default)]`: derive would add a `C: Default` bound, but an +// empty key set needs nothing from `C` — `ColumnRef` has no meaningful +// default anyway. +impl Default for GroupKeys { + fn default() -> Self { + Self { + keys: Vec::new(), + without: false, + } + } +} + +impl GroupKeys { + /// An empty key set — a global (ungrouped) operation. + pub fn none() -> Self { + Self::default() + } + /// `by(keys)` — group by exactly these columns. + pub fn by(keys: Vec) -> Self { + Self { + keys, + without: false, + } + } + /// `without(keys)` — group by every label *except* these (issue #39). The + /// kept set is runtime-resolved; only the excluded positions are stored. + pub fn without(keys: Vec) -> Self { + Self { + keys, + without: true, + } + } + /// Whether this is a `without(...)` exclusion grouping. + pub fn is_without(&self) -> bool { + self.without + } + /// The named keys — kept labels for `by`, excluded labels for `without`. + pub fn keys(&self) -> &[C] { + &self.keys + } +} + +impl std::ops::Deref for GroupKeys { + type Target = [C]; + fn deref(&self) -> &Self::Target { + &self.keys + } +} + +impl From> for GroupKeys { + fn from(keys: Vec) -> Self { + Self::by(keys) + } +} + +impl FromIterator for GroupKeys { + fn from_iter>(iter: I) -> Self { + Self::by(iter.into_iter().collect()) + } +} + +impl<'a, C> IntoIterator for &'a GroupKeys { + type Item = &'a C; + type IntoIter = std::slice::Iter<'a, C>; + fn into_iter(self) -> Self::IntoIter { + self.keys.iter() + } +} + +/// Compare directly against a `Vec` so call sites and tests can keep +/// writing `keys == vec![..]` / `assert_eq!(keys, &vec![..])`. A `without` +/// grouping never equals a bare `by` list. +impl PartialEq> for GroupKeys { + fn eq(&self, other: &Vec) -> bool { + !self.without && &self.keys == other + } +} + +/// (De)serialise as a bare array for `by`, or `{"without": [...]}` for the +/// exclusion form — keeping the `by` wire format identical to the old newtype. +/// Borrowed for `Serialize` (no `C: Clone` needed to write one out), owned for +/// `Deserialize` (there's nothing to borrow from). +#[derive(Serialize)] +#[serde(untagged)] +enum GroupKeysReprRef<'a, C> { + By(&'a [C]), + Without { without: &'a [C] }, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum GroupKeysRepr { + By(Vec), + Without { without: Vec }, +} + +impl Serialize for GroupKeys { + fn serialize(&self, serializer: S) -> Result { + if self.without { + GroupKeysReprRef::Without { + without: self.keys.as_slice(), + } + .serialize(serializer) + } else { + GroupKeysReprRef::By(self.keys.as_slice()).serialize(serializer) + } + } +} + +impl<'de, C: Deserialize<'de>> Deserialize<'de> for GroupKeys { + fn deserialize>(deserializer: D) -> Result { + Ok(match GroupKeysRepr::deserialize(deserializer)? { + GroupKeysRepr::By(keys) => Self::by(keys), + GroupKeysRepr::Without { without } => Self::without(without), + }) + } +} + +/// Which data model a `Source` / `AggIntent` operates over. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum DataModel { + TimeSeries, + Tabular, + Any, +} + +/// The leaf data source of a `Scan`. The schema itself rides on the +/// `Scan.schema` field (SchemaResolver-built); `Source` carries only the leaf's +/// identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum Source { + /// Time-series leaf — PromQL / DC lifecycle. Produces `(ts, value, *labels)`. + TimeSeries { metric: String }, + /// Tabular leaf — asap-fusion / future OLAP. Columns ride on `Scan.schema`. + Table { table_ref: String }, +} + +impl Source { + pub fn data_model(&self) -> DataModel { + match self { + Source::TimeSeries { .. } => DataModel::TimeSeries, + Source::Table { .. } => DataModel::Tabular, + } + } +} + +/// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's +/// [`ArithmeticOpKind`] / [`CompareOpKind`] so every arithmetic/comparison +/// operator has exactly one representation (and one `Display`) across the IR. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum BinaryOpKind { + /// Arithmetic — `Add/Sub/Mul/Div/Mod` (shared with `ScalarExpr::Arithmetic`). + Arithmetic(ArithmeticOpKind), + /// Comparison — `Eq/Ne/Lt/Le/Gt/Ge` + `Like/ILike/Regex` family (shared + /// with `ScalarExpr::Compare`). PromQL keeps the matched series whose + /// comparison holds. + Compare(CompareOpKind), + /// PromQL comparison with the `bool` modifier: every matched series + /// yields 1 or 0 and loses its metric name. A separate variant, not a + /// flag, because only comparisons take `bool`. + CompareBool(CompareOpKind), + /// PromQL vector-set operation. + Set(PromQLVectorSetOpKind), +} + +impl std::fmt::Display for BinaryOpKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BinaryOpKind::Arithmetic(op) => write!(f, "{op}"), + BinaryOpKind::Compare(op) => write!(f, "{op}"), + BinaryOpKind::CompareBool(op) => write!(f, "{op} bool"), + BinaryOpKind::Set(PromQLVectorSetOpKind::And) => f.write_str("AND"), + BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => f.write_str("OR"), + BinaryOpKind::Set(PromQLVectorSetOpKind::Unless) => f.write_str("unless"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum JoinKind { + Inner, + Left, + Right, + Full, + Cross, + /// Left semi-join — each left row that has **at least one** match, once. + /// `WHERE c IN (SELECT …)` / `WHERE EXISTS (…)` (issue #111). + /// + /// Output schema is the **left's alone**; the right side is a filter, not a + /// source of columns. The join predicate still resolves against the + /// concatenated `left ++ right` schema — its scope is deliberately wider + /// than the node's output. + Semi, + /// Left anti-join — each left row with **no** match. `WHERE NOT EXISTS (…)`. + /// Same schema rule as [`JoinKind::Semi`]. + /// + /// Note this is *not* `NOT IN (SELECT …)`: under SQL's three-valued logic a + /// NULL on the right makes `NOT IN` yield no rows at all, where an anti-join + /// yields every left row. The SQL front end rejects `NOT IN (subquery)` + /// rather than lower it here. + Anti, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum RelationalSetOpKind { + Union, + Intersect, + Except, +} + +/// PromQL vector-set operator used by [`BinaryOpKind::Set`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PromQLVectorSetOpKind { + And, + Or, + Unless, +} + +/// SQL analytic window function (`fn(...) OVER (…)`). Distinct from a streaming +/// time `Window`: this is an analytic frame over already-materialised rows. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowFuncKind { + RowNumber, + Rank, + DenseRank, + Lag, + Lead, + /// ClickHouse `lagInFrame`/`leadInFrame`: unlike [`Lag`](Self::Lag)/[`Lead`](Self::Lead), + /// these respect the window frame bounds (NULL/default past the frame edge) + /// rather than reaching arbitrarily far back/forward. Kept as distinct + /// variants so the frame clause is never silently discarded by conflating + /// them with `Lag`/`Lead` (#267). `WindowFuncKind` still has no frame + /// representation, so today these lower and behave exactly like + /// `Lag`/`Lead` — the tag is correct, the frame-respecting behavior isn't + /// implemented yet. See #231 for modeling window frames properly. + LagInFrame, + LeadInFrame, + FirstValue, + LastValue, + /// `NTH_VALUE(expr, n)` — `n` is resolved from the (literal) 2nd argument. + NthValue(Option), + Sum, + Avg, + Count, + Min, + Max, +} + +/// A window's frame-spec (`ROWS`/`RANGE BETWEEN … AND …`) — which rows around +/// the current one an analytic window function reads. `GROUPS` is rejected at +/// lowering time (issue #268): every SQL corpus in this repo uses only `ROWS`, +/// and nothing downstream interprets frame semantics yet, so it isn't worth +/// modelling untested. +/// +/// Meaningless (but harmless) on the rank-only and navigation functions +/// (`ROW_NUMBER`/`RANK`/`DENSE_RANK`/`LAG`/`LEAD`), which ignore the frame per +/// SQL semantics — DataFusion still attaches one, stored here verbatim. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct WindowFrame { + pub units: WindowFrameUnits, + pub start_bound: WindowFrameBound, + pub end_bound: WindowFrameBound, +} + +/// A finite window-frame displacement. Intervals are normalized to Arrow's +/// month/day/nanosecond representation so SQL `RANGE INTERVAL ...` bounds +/// survive lowering without leaking DataFusion types into the canonical IR. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowFrameOffset { + Scalar(ScalarValue), + Interval { + months: i32, + days: i32, + nanoseconds: i64, + }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowFrameUnits { + /// Boundaries count physical rows: `ROWS BETWEEN 2 PRECEDING AND CURRENT ROW`. + Rows, + /// Boundaries count by value-distance on the (single) `ORDER BY` column: + /// `RANGE BETWEEN INTERVAL '1' HOUR PRECEDING AND CURRENT ROW`. + Range, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowFrameBound { + /// `UNBOUNDED PRECEDING` is + /// `Preceding(WindowFrameOffset::Scalar(ScalarValue::Null))`. + Preceding(WindowFrameOffset), + CurrentRow, + /// `UNBOUNDED FOLLOWING` is + /// `Following(WindowFrameOffset::Scalar(ScalarValue::Null))`. + Following(WindowFrameOffset), +} + +/// A symbolic label matcher on the **info metric** side of an +/// [`crate::ir::NonASAPOp::PromqlInfoEnrich`] (issue #84). Unlike a `Scan` predicate it is not +/// resolved positionally — it references the info metric's labels (`__name__` +/// picks the metric, the rest constrain data labels), which aren't in the input +/// vector's schema; the post-ASAP realization pass applies it against the info metric. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct InfoMatcher { + pub label: String, + /// One of `Eq` / `Ne` / `Regex` / `NotRegex` (PromQL `=`/`!=`/`=~`/`!~`). + pub op: CompareOpKind, + pub value: String, +} + +/// Series-sampling selection mode (PromQL `limitk` / `limit_ratio`, issue #86). +/// A [`crate::ir::NonASAPOp::PromqlSeriesSample`] keeps a *subset of whole series*, unchanged — it does +/// not rank or reduce, so it is distinct from `TopK` and from `Sort → Limit`. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SampleKind { + /// `limitk(k, v)` — up to `k` series per group. Which series survive is + /// deterministic across evaluations but otherwise unspecified (no ordering). + LimitK(usize), + /// `limit_ratio(r, v)` — a deterministic `r`-fraction of series per group. + /// `r ∈ [-1, 1]`; a negative `r` selects the complementary fraction. + LimitRatio(f64), +} + +/// PromQL vector-match modifier (`on`/`ignoring` + `group_left`/`group_right`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VectorMatch { + pub kind: VectorMatchKind, + pub labels: Vec, + pub grouping: Option, +} + +/// PromQL `@` modifier — pins a selector's evaluation time to an anchor instead +/// of the query evaluation time (issue #40). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum AtModifier { + /// `@ start()` — the query range's start instant. + Start, + /// `@ end()` — the query range's end instant. + End, + /// `@ ` — an absolute instant, milliseconds since the Unix epoch (may be + /// negative). PromQL writes the timestamp in seconds; the front end scales it. + Timestamp(i64), +} + +/// PromQL per-selector **time-shift** modifiers — `offset` and `@` (issue #40). +/// Neither changes a selector's *schema*; both move *when* it is evaluated, so +/// the shift is a pass-through wrapper ([`crate::ir::NonASAPOp::TimeShift`]) over the +/// selector rather than a new leaf shape. The runtime resolves the anchor and +/// applies the offset. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct TimeShift { + /// `offset ` as signed milliseconds — a positive value shifts the + /// lookback *back* in time (`offset 5m`), a negative value shifts it + /// *forward* (`offset -5m`). `0` = no offset. + pub offset_ms: i64, + /// `@` anchor; `None` = evaluate at the query time. + pub at: Option, +} + +impl TimeShift { + /// Whether this shift is the identity (no `offset`, no `@`) — the state of + /// every selector that carries neither modifier. + pub fn is_identity(&self) -> bool { + self.offset_ms == 0 && self.at.is_none() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum VectorMatchKind { + On, + Ignoring, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VectorGrouping { + pub side: GroupSide, + pub labels: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum GroupSide { + Left, + Right, +} + +// ── Intent algebra IR ──────────────────────────────────────────────────────── + +/// What kind of computation an `Aggregate` node performs — orthogonal to +/// *which* columns it groups by (that's still [`GroupKeys`], inside +/// `Reduce`). Explicit, decided once by whichever pass constructs the node +/// (structural, at front-end lowering time), rather than inferred downstream from +/// whether a grouping-key list happens to be empty or from a neighboring +/// node's shape. See design proposal #165. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum Reduction { + /// Collapses input rows via `by` — `by`/`without` semantics are exactly + /// [`GroupKeys`]'s. May still collapse every row into one (an empty, + /// non-`without` `by`) — that's a genuine reduction with zero grouping + /// columns, not "no grouping concept." + Reduce(GroupKeys), + /// No grouping concept at all: preserves one output row per input + /// entity (e.g. a per-series windowed computation with no `by(...)` + /// clause to begin with, because there's no aggregation operator here + /// for such a clause to attach to). Never merges across entities, and + /// never collapses an entity's own row structure (e.g. a time axis) — + /// unlike `Reduce(GroupKeys::without(vec![]))` ("group by every + /// label"), which is still a genuine reduction and does collapse it. + PerEntity, +} + +impl Reduction { + /// Shorthand for the common case — group by these (possibly empty) + /// keys, kept rather than excluded. + pub fn by(keys: Vec) -> Self { + Self::Reduce(GroupKeys::by(keys)) + } + + /// The grouping keys, if this is a genuine reduction — `None` for + /// `PerEntity`, which has no grouping-keys concept to report. + pub fn group_keys(&self) -> Option<&GroupKeys> { + match self { + Self::Reduce(by) => Some(by), + Self::PerEntity => None, + } + } + + /// The grouping keys, panicking if this is `PerEntity` — for call sites + /// (tests, mostly) that already know, from the shape they built or are + /// asserting on, that this must be a genuine reduction. Prefer + /// [`group_keys`](Self::group_keys) wherever the caller can't assume that. + pub fn expect_reduce(&self) -> &GroupKeys { + match self { + Self::Reduce(by) => by, + Self::PerEntity => panic!("expected Reduction::Reduce, got PerEntity"), + } + } +} + +/// A caller-proven compound unique key for a [`crate::ir::NonASAPOp::Concat`] (issue +/// #228) — built only via [`ConcatDiscriminatorKey::new`], never by naming `discriminator` directly +/// in a struct literal (both fields are private): from *other Rust code*, +/// the only way to end up with one of these is to hand over a specific +/// column as the discriminator, by name, at the call site. +/// +/// Caveat: this is a Rust-API-level guarantee, not a data-level one. The +/// derived `Deserialize` impl below builds a `ConcatDiscriminatorKey` +/// directly from field values, bypassing `new()`. Deserialization is therefore +/// equivalent to a caller supplying the assertion directly; it does not prove +/// either fact below. An external boundary accepting IR data must +/// reject this field or validate both obligations before treating it as +/// uniqueness evidence. +/// +/// # Soundness +/// +/// `Concat`'s default (see its own doc) is to drop `unique_keys` +/// unconditionally, because a key unique **within** one branch is not unique +/// **across** the concatenation unless the branches' value sets for that key +/// are provably disjoint — nothing about matching schemas or matching +/// per-branch keys establishes that on its own. Two different branches can +/// trivially emit the same `inner_key` value (e.g. two PromQL +/// `histogram_quantiles` branches keyed on `(host, le)` can both produce a +/// `(host, le)` pair for different φ). +/// +/// Prepending `discriminator` restores a compound key only when two facts +/// hold: `inner_key` uniquely identifies rows **within every branch**, and +/// `discriminator`'s value is **guaranteed to differ between branches** — a +/// literal the producer just tagged the branch with (PromQL φ riding along via +/// [`crate::ir::NonASAPOp::PromqlRelabel`], a Postgres-style synthetic `GROUPING()` id +/// for `ROLLUP`/`CUBE`, …), never something inferred structurally from the +/// branches' own data — then `discriminator` alone partitions rows into +/// disjoint sets independent of what the branches actually contain, so +/// `(discriminator, inner_key)` is sound even when otherwise-identical +/// `inner_key` values occur in different branches. Neither fact is verified +/// here; both are part of the caller-proven claim. +/// +/// This is a **caller-proven claim, not something `Concat` can verify**: +/// nothing stops a caller from asserting a discriminator that in fact +/// repeats across branches, in which case the resulting `unique_keys` claim +/// is simply wrong — `output_schema` trusts it without checking. The +/// obligation is on the constructor call site, exactly as it is on +/// [`crate::ir::NonASAPOp::Dedup`]'s `cols` or any other unverified `unique_keys` +/// producer in this module. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] +pub struct ConcatDiscriminatorKey { + discriminator: C, + inner_key: Vec, +} + +impl ConcatDiscriminatorKey { + /// The only constructor — `discriminator` must be named explicitly by + /// the caller. See the type's doc for the soundness obligation this + /// puts on that caller. + pub fn new(discriminator: C, inner_key: Vec) -> Self { + Self { + discriminator, + inner_key, + } + } + + pub fn discriminator(&self) -> &C { + &self.discriminator + } + + pub fn inner_key(&self) -> &[C] { + &self.inner_key + } +} diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index 2a8c8fd5f..cc38ddf7c 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -2,7 +2,7 @@ //! //! Same operator payloads as the logical export, plus the execution timing //! (data state) of every node and edge. The input must already be timed -//! ([`super::timing::apply_lifecycle_timings`]); export reads each node's +//! ([`super::timing::apply_materialization_timings`]); export reads each node's //! timing and does not re-run data-state validation. use std::collections::{BTreeMap, HashMap, HashSet}; @@ -336,7 +336,7 @@ pub fn compile_physical_asap_dag( } /// Export the timed DAG below `root`. Every reachable node must carry a -/// timing (see [`super::timing::apply_lifecycle_timings`]); the data-state +/// timing (see [`super::timing::apply_materialization_timings`]); the data-state /// rules were checked by that pass and are not re-run here. pub fn compile_physical_asap_dag_with_node_ids( root: &Rc, diff --git a/crates/types/src/ir/schema_support.rs b/crates/types/src/ir/schema_support.rs index c710c8654..95c42070d 100644 --- a/crates/types/src/ir/schema_support.rs +++ b/crates/types/src/ir/schema_support.rs @@ -1,5 +1,14 @@ //! Series-identity realization for the unified dag. use crate::pre_asap::schema::*; +/// Resolve a PromQL root to rows carrying [`PROMQL_SERIES_IDENTITY`] before +/// candidate search. `closed` describes physical columns here: the final +/// column contains every dynamic source label. It does not assert that the +/// query's projected labels are the full label set. +/// +/// This realization supports explicit `by` grouping and per-series computation. +/// Operators that rewrite or implicitly match dynamic label sets require their +/// own realization; they must not accidentally treat the opaque identity as a +/// user label or silently discard it. pub fn with_promql_series_identity( root: &std::rc::Rc, ) -> Result, String> { diff --git a/crates/types/src/ir/timing.rs b/crates/types/src/ir/timing.rs index e19513ba3..eb6f5a9d5 100644 --- a/crates/types/src/ir/timing.rs +++ b/crates/types/src/ir/timing.rs @@ -1,16 +1,18 @@ -//! Execution timing: written into every node from a lifecycle assignment, -//! then validated against each operator's kind and its consuming edges. +//! Execution timing: written into every node from a materialization +//! assignment, then validated against each operator's kind and its consuming +//! edges. //! -//! The logical DAG carries no timing. Summary materialization chooses a -//! lifecycle per summary state; [`LifecycleAssignment`] records that choice -//! (ingestion-time maintenance or query-time recomputation per `SummaryAgg`) -//! and [`apply_lifecycle_timings`] expands it into a timing on every node: +//! The logical DAG carries no timing. Materialization decides per summary +//! state whether it is maintained at ingestion time or computed at query +//! time; [`MaterializationAssignment`] records that choice per `SummaryAgg` +//! and [`apply_materialization_timings`] expands it into a timing on every node: //! //! - a node of fixed kind takes its kind's timing (`SummaryEstimate` and //! `EvaluatePopulation` run at query time, `MaintainPopulation` at ingestion //! time); -//! - a `SummaryAgg` takes the assignment's timing (default: ingestion time), -//! unless something below it can only exist at query time; +//! - a `SummaryAgg` takes the assignment's timing (default: query time, until +//! Stage 2 materialization (#509) chooses otherwise), unless something below +//! it can only exist at query time; //! - every other node runs when its consumer runs: everything that feeds a //! maintained state runs at ingestion time, everything above a evaluation at //! query time. @@ -43,23 +45,31 @@ use crate::post_asap::execution_data_state::{ }; use crate::pre_asap::schema::{DataType, FieldDataType, Schema}; -/// The per-state lifecycle choice summary materialization made: for each -/// `SummaryAgg` node (by identity), whether its state is maintained at -/// ingestion time or recomputed at query time. A state absent from the map -/// takes the default, ingestion-time maintenance. +/// The per-state materialization choice: for each `SummaryAgg` node (by +/// identity), whether its state is maintained at ingestion time or computed +/// at query time. A state absent from the map takes the assignment's default. +/// `Default` is [`Self::all_query_time`]: nothing is materialized until Stage 2 +/// materialization (#509) decides otherwise. #[derive(Debug, Clone, Default)] -pub struct LifecycleAssignment { +pub struct MaterializationAssignment { summary_timings: HashMap<*const OperatorNode, ExecutionTiming>, + default_timing: ExecutionTiming, } -impl LifecycleAssignment { - /// The assignment under which every summary state is maintained at - /// ingestion time — the timings every plan carried before lifecycles - /// became a planning choice. - pub fn default_maintained() -> Self { +impl MaterializationAssignment { + /// Every summary state computed at query time. + pub fn all_query_time() -> Self { Self::default() } + /// Every summary state maintained at ingestion time. + pub fn all_ingestion_time() -> Self { + Self { + summary_timings: HashMap::new(), + default_timing: ExecutionTiming::IngestionTime, + } + } + pub fn set(&mut self, summary: &Rc, timing: ExecutionTiming) { self.summary_timings.insert(Rc::as_ptr(summary), timing); } @@ -68,11 +78,11 @@ impl LifecycleAssignment { self.summary_timings .get(&Rc::as_ptr(summary)) .copied() - .unwrap_or(ExecutionTiming::IngestionTime) + .unwrap_or(self.default_timing) } } -/// Memo of one [`apply_lifecycle_timings`] pass: `input node → timed node`, +/// Memo of one [`apply_materialization_timings`] pass: `input node → timed node`, /// shared by every root of a workload so a node shared by two roots stays /// one `Rc`. Re-reaching a node with a different timing is a conflict. #[derive(Default)] @@ -137,9 +147,9 @@ fn forces_query_time(node: &OperatorNode, seen: &mut HashMap<*const OperatorNode /// Write the timings of `assignment` into every node reachable from `root`, /// top-down, then validate every edge. Returns the timed copy of `root`; /// `memo` carries the sharing across the roots of one workload. -pub fn apply_lifecycle_timings( +pub fn apply_materialization_timings( root: &Rc, - assignment: &LifecycleAssignment, + assignment: &MaterializationAssignment, memo: &mut TimingMemo, ) -> Result, ExecutionDataStateError> { let mut forced = HashMap::new(); @@ -159,24 +169,26 @@ pub fn apply_lifecycle_timings( Ok(timed) } -/// Validate the sub-DAG below `root` under the default (every summary -/// maintained) assignment, with `root` consumed at `root_timing`. For -/// planning-time legality checks of a candidate before it is assembled into -/// a workload DAG; nothing is kept. -pub fn validate_default( +/// Validate the sub-DAG below `root` with every summary maintained at +/// ingestion time ([`MaterializationAssignment::all_ingestion_time`]) and +/// `root` consumed at `root_timing`. For planning-time legality checks of a +/// candidate before it is assembled into a workload DAG: a candidate must stay +/// executable if materialization later maintains its states. Nothing is kept. +pub fn validate_maintained( root: &Rc, root_timing: ExecutionTiming, ) -> Result<(), ExecutionDataStateError> { - let assignment = LifecycleAssignment::default_maintained(); + let assignment = MaterializationAssignment::all_ingestion_time(); let mut memo = TimingMemo::new(); let mut forced = HashMap::new(); let timed = write(root, root_timing, &assignment, &mut memo, &mut forced)?; validate(&timed, &mut HashMap::new()) } -/// The data state `node` produces under the default assignment when its -/// consumer runs at `consumer` — the planning-time answer to "what does this -/// candidate's output look like" before any assignment is applied. +/// The data state `node` produces with every summary maintained at ingestion +/// time when its consumer runs at `consumer` — the planning-time answer to +/// "what does this candidate's output look like", consistent with +/// [`validate_maintained`]. pub fn planned_data_state( node: &Rc, consumer: ExecutionTiming, @@ -185,7 +197,7 @@ pub fn planned_data_state( let timing = own_timing( node, consumer, - &LifecycleAssignment::default_maintained(), + &MaterializationAssignment::all_ingestion_time(), &mut forced, ); ExecutionDataState { @@ -202,7 +214,7 @@ pub fn planned_data_state( fn own_timing( node: &Rc, consumer: ExecutionTiming, - assignment: &LifecycleAssignment, + assignment: &MaterializationAssignment, forced: &mut HashMap<*const OperatorNode, bool>, ) -> ExecutionTiming { // A placement fixed when the candidate was built (an exact-state read @@ -229,7 +241,7 @@ fn own_timing( fn write( node: &Rc, consumer: ExecutionTiming, - assignment: &LifecycleAssignment, + assignment: &MaterializationAssignment, memo: &mut TimingMemo, forced: &mut HashMap<*const OperatorNode, bool>, ) -> Result, ExecutionDataStateError> { @@ -560,7 +572,7 @@ fn validate_non_asap( /// one timing stays one `Rc`. Returns the (possibly rewritten) root. pub fn split_shared_by_phase( root: &Rc, - assignment: &LifecycleAssignment, + assignment: &MaterializationAssignment, ) -> Rc { // First pass: the set of timings each node is reached with. let mut reached: HashMap<*const OperatorNode, Vec> = HashMap::new(); @@ -568,7 +580,7 @@ pub fn split_shared_by_phase( fn collect( node: &Rc, consumer: ExecutionTiming, - assignment: &LifecycleAssignment, + assignment: &MaterializationAssignment, reached: &mut HashMap<*const OperatorNode, Vec>, forced: &mut HashMap<*const OperatorNode, bool>, ) { @@ -598,7 +610,7 @@ pub fn split_shared_by_phase( fn rebuild( node: &Rc, consumer: ExecutionTiming, - assignment: &LifecycleAssignment, + assignment: &MaterializationAssignment, reached: &HashMap<*const OperatorNode, Vec>, copies: &mut HashMap<(*const OperatorNode, ExecutionTiming), Rc>, forced: &mut HashMap<*const OperatorNode, bool>, @@ -764,10 +776,12 @@ mod tests { ) } + /// Apply with every summary maintained, the placement whose edge rules + /// these tests exercise. fn apply(root: &Rc) -> Result, ExecutionDataStateError> { - apply_lifecycle_timings( + apply_materialization_timings( root, - &LifecycleAssignment::default_maintained(), + &MaterializationAssignment::all_ingestion_time(), &mut TimingMemo::new(), ) } @@ -776,6 +790,38 @@ mod tests { Rc::clone(node.children()[0]) } + /// Without a materialization decision, a summary and its input run at + /// query time; an explicit per-state choice overrides the default. + #[test] + fn default_assignment_materializes_nothing() { + let summary = agg(scan(), kll()); + let root = apply_materialization_timings( + &summary, + &MaterializationAssignment::default(), + &mut TimingMemo::new(), + ) + .unwrap(); + assert_eq!( + data_state(&root), + Some(ExecutionDataState { + timing: ExecutionTiming::QueryTime, + primitive: DataPrimitive::SummaryState, + }) + ); + assert_eq!( + data_state(&child(&root)), + Some(ExecutionDataState::QUERY_ROWS) + ); + let mut assignment = MaterializationAssignment::all_query_time(); + assignment.set(&summary, ExecutionTiming::IngestionTime); + let root = + apply_materialization_timings(&summary, &assignment, &mut TimingMemo::new()).unwrap(); + assert_eq!( + data_state(&root), + Some(ExecutionDataState::INGESTION_SUMMARY) + ); + } + #[test] fn summary_agg_input_runs_at_ingestion_time() { let root = apply(&agg(scan(), kll())).unwrap(); @@ -873,7 +919,7 @@ mod tests { second: ExecutionDataState::QUERY_ROWS, }) ); - let split = split_shared_by_phase(&root, &LifecycleAssignment::default_maintained()); + let split = split_shared_by_phase(&root, &MaterializationAssignment::all_ingestion_time()); assert!(apply(&split).is_ok()); } @@ -896,10 +942,10 @@ mod tests { }; for operand in [1, 2] { assert!(matches!( - validate_default(&corr_over(operand), ExecutionTiming::QueryTime), + validate_maintained(&corr_over(operand), ExecutionTiming::QueryTime), Err(ExecutionDataStateError::NonPlainOperand { .. }) )); } - validate_default(&corr_over(3), ExecutionTiming::QueryTime).unwrap(); + validate_maintained(&corr_over(3), ExecutionTiming::QueryTime).unwrap(); } } diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 9fc450c7f..9252ac806 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -1,26 +1,22 @@ //! `asap-types` — shared vocabulary for the whole workspace. //! -//! Merges the former `asap-ir` crate (the pre-ASAP intent algebra, -//! workload/batch types, and DAG export) with the data-type-only modules of -//! the former `asap-sketch` crate (the post-ASAP sketch-bound IR types, -//! under [`post_asap`]). -//! -//! - [`pre_asap`] / [`types`] / [`workload`] / [`dag_export`] — the -//! pre-ASAP IR: language-agnostic query intent, independent of any -//! sketch decision. -//! - [`post_asap`] — the post-ASAP IR: sketch-bound types -//! ([`post_asap::sketch`], [`post_asap::expr`], [`post_asap::schema`]) -//! that commit to a concrete `SummaryKind`/`SummaryParams` realization. -//! No execution logic lives in this workspace (see issue #190) — a -//! downstream deployment crate is expected to supply that. -//! [`post_asap::query_time`] is the one exception, folder-separated from -//! the rest of `post_asap` on purpose: pure, sketch-object-agnostic -//! posterior error-bound math (issue #239) that a future real sketch -//! runtime's readout path can call directly — see that module's docs -//! for the planning-time/execution-time boundary and why it's unwired -//! today. +//! - [`ir`] — the unified operator IR: one operator language before and +//! after ASAP optimization ([`ir::OperatorNode`]), plus its passes +//! (canonicalize, CSE, timing) and the wire export ([`ir::export`]). +//! - [`pre_asap`] — the shared field vocabulary the IR's operators are +//! built from (grouping keys, reductions, sources, aggregation intents, +//! scalar literal / operator kinds, [`pre_asap::Schema`]). +//! - [`post_asap`] — summary-state types (families, kinds, parameters, +//! grouping strategy), accuracy guarantees, and the execution-timing +//! vocabulary. No execution logic lives in this workspace (issue #190). +//! [`post_asap::query_time`] holds pure posterior error-bound math +//! (issue #239) a future sketch runtime's evaluation path can call; see its +//! docs for why it is unwired today. +//! - [`types`] / [`workload`] / [`parsed_workload`] / [`dag_export`] / +//! [`cost`] / [`resources`] — workload, batch, export and cost types. pub mod cost; pub mod dag_export; +pub mod ir; pub mod parsed_workload; pub mod post_asap; pub mod pre_asap; @@ -28,5 +24,3 @@ pub mod resources; pub mod serde_f64; pub mod types; pub mod workload; - -pub mod ir; diff --git a/crates/types/src/post_asap/cse.rs b/crates/types/src/post_asap/cse.rs deleted file mode 100644 index 55889008c..000000000 --- a/crates/types/src/post_asap/cse.rs +++ /dev/null @@ -1,442 +0,0 @@ -//! Structural sharing for a selected workload in one execution/data scope. -//! -//! This is not candidate selection or a cross-request cache. Callers opt into -//! common producer execution only after agreeing on lifecycle and data scope. -//! Typed equality includes schemas, guarantees and complete source expressions. - -use std::collections::HashMap; -use std::rc::Rc; - -use super::{SummaryExpr, SummaryNode}; - -/// Numeric PartialEq alone conflates signed zeros. The serialized check is -/// additional evidence, never a replacement for typed equality (JSON maps -/// nonfinite floats to null). Keep this rule local to structural sharing. -fn same_value(left: &T, right: &T) -> bool { - left == right - && match (serde_json::to_string(left), serde_json::to_string(right)) { - (Ok(left), Ok(right)) => left == right, - _ => false, - } -} - -/// Children have already been interned. Comparing their identities avoids -/// recursively expanding a shared DAG once for every path to each descendant. -fn same_node(left: &SummaryNode, right: &SummaryNode) -> bool { - use SummaryExpr::*; - let expression_equal = match (&left.expr, &right.expr) { - (KeepPreAsap(a), KeepPreAsap(b)) => Rc::ptr_eq(a, b) || same_value(a, b), - ( - BinaryOp { - lhs: al, - rhs: ar, - operator: ao, - timing: at, - }, - BinaryOp { - lhs: bl, - rhs: br, - operator: bo, - timing: bt, - }, - ) => Rc::ptr_eq(al, bl) && Rc::ptr_eq(ar, br) && ao == bo && at == bt, - ( - ValueOperation { - child: ac, - operation: ao, - timing: at, - }, - ValueOperation { - child: bc, - operation: bo, - timing: bt, - }, - ) => Rc::ptr_eq(ac, bc) && same_value(ao, bo) && at == bt, - ( - RelationalJoin { - left: al, - right: ar, - kind: ak, - pred: ap, - pruning: ax, - }, - RelationalJoin { - left: bl, - right: br, - kind: bk, - pred: bp, - pruning: bx, - }, - ) => { - Rc::ptr_eq(al, bl) - && Rc::ptr_eq(ar, br) - && ak == bk - && same_value(ap, bp) - && same_value(ax, bx) - } - ( - SummaryAgg { - child: ac, - family: af, - input: ai, - reduction: ar, - grouping: ag, - filter: afl, - }, - SummaryAgg { - child: bc, - family: bf, - input: bi, - reduction: br, - grouping: bg, - filter: bfl, - }, - ) => { - Rc::ptr_eq(ac, bc) - && af == bf - && same_value(ai, bi) - && ar == br - && ag == bg - && same_value(afl, bfl) - } - ( - SummaryJoin { - outer: ao, - inner: ai, - key: ak, - family: af, - }, - SummaryJoin { - outer: bo, - inner: bi, - key: bk, - family: bf, - }, - ) => Rc::ptr_eq(ao, bo) && Rc::ptr_eq(ai, bi) && ak == bk && af == bf, - ( - SummarySubtract { - left: al, - right: ar, - }, - SummarySubtract { - left: bl, - right: br, - }, - ) => Rc::ptr_eq(al, bl) && Rc::ptr_eq(ar, br), - ( - SummaryEstimate { - summary_input: ai, - query: aq, - }, - SummaryEstimate { - summary_input: bi, - query: bq, - }, - ) => Rc::ptr_eq(ai, bi) && same_value(aq, bq), - ( - SummaryDelete { - summary_input: ai, - key: ak, - }, - SummaryDelete { - summary_input: bi, - key: bk, - }, - ) => Rc::ptr_eq(ai, bi) && ak == bk, - ( - SummaryMerge { - children: a, - timing: at, - }, - SummaryMerge { - children: b, - timing: bt, - }, - ) => at == bt && a.len() == b.len() && a.iter().zip(b).all(|(a, b)| Rc::ptr_eq(a, b)), - // Keep this exhaustive on the left: new variants require a sharing rule. - ( - KeepPreAsap(_) - | BinaryOp { .. } - | ValueOperation { .. } - | RelationalJoin { .. } - | SummaryAgg { .. } - | SummaryJoin { .. } - | SummarySubtract { .. } - | SummaryEstimate { .. } - | SummaryDelete { .. } - | SummaryMerge { .. }, - _, - ) => false, - }; - expression_equal && left.schema == right.schema && same_value(&left.guarantee, &right.guarantee) -} - -/// Intern equal selected sub-DAGs across roots while preserving every root ID. -/// -/// Only structural equality is used: no grouping, parameter, accuracy or source -/// coercions are performed. All roots must belong to the same data snapshot or -/// maintenance scope. Downstream realization must still check physical -/// implementation compatibility. Use separate calls for independent executions. -/// -/// When the selected states are identical, this is the planner's -/// summary-capability rule (#509 Pass 2): one summary build node feeds every -/// readout it supports, e.g. one KLL for p50 and p99, or one UnivMon for -/// distinct count, entropy and L2. Candidate generation sizes a variant for -/// the strictest sibling consumer so differing accuracy targets can reach -/// identical states here. -pub fn share_common_summary_sub_dags( - roots: Vec<(Id, Rc)>, -) -> Vec<(Id, Rc)> { - fn visit( - node: &Rc, - seen: &mut HashMap>, - pool: &mut Vec>, - ) -> Rc { - let identity = Rc::as_ptr(node) as usize; - if let Some(node) = seen.get(&identity) { - return Rc::clone(node); - } - let mut result = node.as_ref().clone(); - match &mut result.expr { - SummaryExpr::KeepPreAsap(_) => {} - SummaryExpr::SummaryAgg { child, .. } => *child = visit(child, seen, pool), - SummaryExpr::BinaryOp { lhs, rhs, .. } => { - *lhs = visit(lhs, seen, pool); - *rhs = visit(rhs, seen, pool); - } - - SummaryExpr::ValueOperation { child, .. } => *child = visit(child, seen, pool), - SummaryExpr::RelationalJoin { left, right, .. } => { - *left = visit(left, seen, pool); - *right = visit(right, seen, pool); - } - SummaryExpr::SummaryJoin { outer, inner, .. } => { - *outer = visit(outer, seen, pool); - *inner = visit(inner, seen, pool); - } - SummaryExpr::SummarySubtract { left, right } => { - *left = visit(left, seen, pool); - *right = visit(right, seen, pool); - } - SummaryExpr::SummaryEstimate { summary_input, .. } - | SummaryExpr::SummaryDelete { summary_input, .. } => { - *summary_input = visit(summary_input, seen, pool); - } - SummaryExpr::SummaryMerge { children, .. } => { - for child in children { - *child = visit(child, seen, pool); - } - } - } - let result = match pool.iter().find(|existing| same_node(existing, &result)) { - Some(existing) => Rc::clone(existing), - None => { - let result = Rc::new(result); - pool.push(Rc::clone(&result)); - result - } - }; - seen.insert(identity, Rc::clone(&result)); - result - } - let mut seen = HashMap::new(); - let mut pool = Vec::new(); - roots - .into_iter() - .map(|(id, root)| (id, visit(&root, &mut seen, &mut pool))) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::post_asap::{ResultGuarantee, Schema}; - use crate::pre_asap::{QueryExpr, ScalarValue}; - - fn leaf(value: f64) -> Rc { - Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(Rc::new(QueryExpr::Literal(ScalarValue::Float64( - value, - )))), - schema: Schema::lifted(vec![], None), - guarantee: Some(ResultGuarantee::exact("fixture")), - }) - } - - // Equal separately constructed roots preserve both IDs but share identity. - #[test] - fn shares_equal_roots_and_preserves_ids() { - let roots = share_common_summary_sub_dags(vec![("a", leaf(1.0)), ("b", leaf(1.0))]); - assert_eq!(roots[0].0, "a"); - assert_eq!(roots[1].0, "b"); - assert!(Rc::ptr_eq(&roots[0].1, &roots[1].1)); - } - - // A diamond is retained across the returned roots, not copied per consumer. - #[test] - fn shares_children_across_distinct_roots() { - let merge = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { - timing: crate::post_asap::ExecutionTiming::IngestionTime, - children: vec![leaf(1.0), leaf(2.0)], - }, - schema: Schema::lifted(vec![], None), - guarantee: None, - }); - let roots = share_common_summary_sub_dags(vec![(0, leaf(1.0)), (1, merge)]); - let SummaryExpr::SummaryMerge { children, .. } = &roots[1].1.expr else { - panic!() - }; - assert!(Rc::ptr_eq(&roots[0].1, &children[0])); - assert!(!Rc::ptr_eq(&children[0], &children[1])); - } - - // Unknown guarantees must not be replaced by an equal expression's exact guarantee. - #[test] - fn distinct_guarantees_and_values_are_not_shared() { - let mut unknown = leaf(1.0).as_ref().clone(); - unknown.guarantee = None; - let roots = share_common_summary_sub_dags(vec![ - (0, leaf(1.0)), - (1, Rc::new(unknown)), - (2, leaf(2.0)), - ]); - assert!(!Rc::ptr_eq(&roots[0].1, &roots[1].1)); - assert!(!Rc::ptr_eq(&roots[0].1, &roots[2].1)); - assert!(roots[1].1.guarantee.is_none()); - } - - // Sharing must preserve IEEE signed zero, including inside exact expressions. - #[test] - fn signed_zero_is_not_coalesced() { - for values in [[0.0, -0.0], [-0.0, 0.0]] { - let roots = - share_common_summary_sub_dags(vec![(0, leaf(values[0])), (1, leaf(values[1]))]); - assert!(!Rc::ptr_eq(&roots[0].1, &roots[1].1)); - for ((_, root), expected) in roots.iter().zip(values) { - let SummaryExpr::KeepPreAsap(expr) = &root.expr else { - panic!() - }; - let QueryExpr::Literal(ScalarValue::Float64(actual)) = expr.as_ref() else { - panic!() - }; - assert_eq!(actual.to_bits(), expected.to_bits()); - assert_eq!(1.0 / actual, 1.0 / expected); - } - } - } - - // Exact expression wrappers must retain signed zero too; JSON's null - // encoding of nonfinite floats must never become the equality decision. - #[test] - fn nested_values_and_nonfinite_values_remain_distinct() { - let wrapped = |value| { - Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(Rc::new(QueryExpr::promql_scalar(value))), - ..leaf(1.0).as_ref().clone() - }) - }; - for (a, b) in [ - (0.0, -0.0), - (f64::INFINITY, f64::NEG_INFINITY), - (f64::NAN, f64::NAN), - ] { - let roots = share_common_summary_sub_dags(vec![(0, wrapped(a)), (1, wrapped(b))]); - assert!(!Rc::ptr_eq(&roots[0].1, &roots[1].1)); - } - let roots = share_common_summary_sub_dags(vec![ - (0, wrapped(f64::INFINITY)), - (1, wrapped(f64::INFINITY)), - ]); - assert!(Rc::ptr_eq(&roots[0].1, &roots[1].1)); - } - - // Distinct quantile readouts share only a compatible typed sketch producer. - #[test] - fn quantile_roots_share_producer_but_not_readout_or_parameters() { - use crate::post_asap::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, - }; - use crate::pre_asap::{ColumnRef, Reduction}; - fn readout(q: f64, alpha: f64) -> Rc { - let producer = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryAgg { - child: leaf(1.0), - family: FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha }, - ), - GroupingStrategy::default(), - ), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), - filter: None, - }, - schema: Schema::lifted(vec![], None), - guarantee: None, - }); - Rc::new(SummaryNode { - expr: SummaryExpr::SummaryEstimate { - summary_input: producer, - query: SketchStatistic::Quantile { q }, - }, - schema: Schema::lifted(vec![], None), - guarantee: None, - }) - } - let roots = share_common_summary_sub_dags(vec![ - ("p95", readout(0.95, 0.01)), - ("p99", readout(0.99, 0.01)), - ("strict", readout(0.95, 0.001)), - ]); - let producer = |root: &Rc| match &root.expr { - SummaryExpr::SummaryEstimate { summary_input, .. } => Rc::clone(summary_input), - _ => panic!(), - }; - assert!(!Rc::ptr_eq(&roots[0].1, &roots[1].1)); - assert!(Rc::ptr_eq(&producer(&roots[0].1), &producer(&roots[1].1))); - assert!(!Rc::ptr_eq(&producer(&roots[0].1), &producer(&roots[2].1))); - } - - // Fifty unique input nodes must not require walking an expanded 2^24 DAG. - // The timeout is a coarse runaway guard, not a performance SLA. - #[test] - fn shared_diamond_does_not_expand_during_comparison() { - let (done, completion) = std::sync::mpsc::channel(); - let worker = std::thread::spawn(move || { - fn diamond() -> Rc { - let mut current = leaf(1.0); - for _ in 0..24 { - current = Rc::new(SummaryNode { - expr: SummaryExpr::BinaryOp { - timing: super::super::ExecutionTiming::QueryTime, - lhs: Rc::clone(¤t), - rhs: current, - operator: super::super::BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: crate::pre_asap::BinaryOpKind::Arithmetic( - crate::pre_asap::ArithmeticOpKind::Add, - ), - vector_match: None, - }, - }, - schema: super::super::Schema::lifted(vec![], None), - guarantee: None, - }); - } - current - } - let roots = share_common_summary_sub_dags(vec![(0, diamond()), (1, diamond())]); - assert!(Rc::ptr_eq(&roots[0].1, &roots[1].1)); - done.send(()).unwrap(); - }); - completion - .recv_timeout(std::time::Duration::from_secs(5)) - .expect("comparison expanded the shared DAG"); - worker.join().unwrap(); - } -} diff --git a/crates/types/src/post_asap/execution_data_state.rs b/crates/types/src/post_asap/execution_data_state.rs index 38219fa2c..ad6e28781 100644 --- a/crates/types/src/post_asap/execution_data_state.rs +++ b/crates/types/src/post_asap/execution_data_state.rs @@ -1,56 +1,14 @@ -//! Execution-data-state contract for mixed exact/summary plans (issue #171). +//! Execution timing and data-state vocabulary of the operator IR. //! -//! A post-ASAP DAG mixes two very different moments of execution: the -//! **update/ingest path** (rows arrive, maintained summary state is updated) -//! and **query evaluation** (maintained state is read out and a final result -//! is produced). A plan that places a query-time residual *underneath* a -//! maintained summary is not merely expensive — it is unexecutable, because -//! the maintenance loop has no readout values to feed into that summary. -//! [`SummaryExpr::ValueOperation`] represents such work without inventing a -//! node per function or use case. Its [`ExecutionTiming`] makes placement -//! explicit and independent of the semantic [`ValueOperation`]. -//! -//! [`ExecutionDataState`] is what a node's output *is*, at which data_state; -//! [`validate_execution_data_states`] checks every edge of a DAG against the -//! rules below at plan construction, returning a typed [`ExecutionDataStateError`] rather -//! than deferring to a runtime failure. -//! -//! ## Edge rules -//! -//! | Parent | Accepts from `child` | -//! |---|---| -//! | `SummaryAgg.child` | Rows or exact accumulator state at either phase. The initial construction phase follows the input; deployment assigns final phases. | -//! | `SummaryEstimate.summary_input` | Summary state at either phase (any family). Initial readout produces `QUERY_ROWS`. | -//! | `SummaryJoin.outer/inner` | `INGESTION_ROWS` or `INGESTION_SUMMARY`; never a read-time data_state. | -//! | `SummarySubtract`/`SummaryDelete` | `INGESTION_SUMMARY`. | -//! | `SummaryMerge` | Summary state at its explicit ingestion or read timing. | -//! | `ValueOperation.child` with `IngestionTime` | `INGESTION_ROWS`; explicit `FinalizeExactAccumulator` also accepts exact accumulator state. Produces `INGESTION_ROWS`. | -//! | `ValueOperation.child` with `QueryTime` | `QUERY_ROWS`. Produces `QUERY_ROWS`. | -//! -//! ## `KeepPreAsap` declares its data_state through the derivation -//! -//! A [`SummaryExpr::KeepPreAsap`] leaf is a raw pre-ASAP computation that a -//! runtime can execute at either time: as maintenance input beneath a -//! `SummaryAgg`/maintenance-time `ValueOperation`, or as a query-time fallback -//! beneath a read-time `ValueOperation` (or at the root). It carries no timing -//! field of its own -//! — every existing consumer pattern-matches the one-field shape — so its -//! data_state is *assigned* by [`validate_execution_data_states`] from the edge that -//! reaches it and reported in the returned [`ExecutionDataStateAssignment`]. What it may -//! not do is stay ambiguous inside one mixed plan: the same `Rc` -//! reached once as update input and once as query-time fallback is -//! [`ExecutionDataStateError::AmbiguousKeepPreAsap`], because no single execution of that -//! sub-DAG can serve both roles. - -use std::collections::HashMap; -use std::rc::Rc; +//! [`ExecutionTiming`] says when a node's value is produced (ingestion vs. +//! query time); [`ExecutionDataState`] pairs it with the [`DataPrimitive`] +//! the edge carries (raw values vs. summary state). The rules that assign +//! and check them over a DAG live in [`crate::ir::timing`], which reports +//! violations as [`ExecutionDataStateError`]. use thiserror::Error; -use super::expr::{ExactOperation, SummaryExpr, SummaryNode, ValueOperation}; -use crate::pre_asap::schema::FieldDataType; - -use crate::pre_asap::query_expr::{aggregate_output_schema, Predicate, QueryExprError}; +use crate::ir::SchemaDerivationError; use crate::pre_asap::schema::Schema; /// When a post-ASAP value is produced. @@ -79,7 +37,7 @@ impl ExecutionTiming { /// The primitive representation carried by a post-ASAP edge. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub enum DataPrimitive { - /// Directly usable values, including approximate summary readouts. + /// Directly usable values, including approximate summary evaluations. /// This does not imply original input data or an exact guarantee. Raw, SummaryState, @@ -123,67 +81,27 @@ impl std::fmt::Display for ExecutionDataState { } } -/// Which parent/edge a [`ExecutionDataStateError`] is about — the variant name of the -/// parent `SummaryExpr` plus its field, for a message a plan author can act -/// on. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ExecutionDataStateEdge { - SummaryAggChild, - SummaryEstimateInput, - SummaryJoinInput, - SummarySubtractInput, - SummaryDeleteInput, - SummaryMergeInput, - ValueOperationChild, -} - -impl ExecutionDataStateEdge { - fn describe(self) -> &'static str { - match self { - Self::SummaryAggChild => "SummaryAgg.child", - Self::SummaryEstimateInput => "SummaryEstimate.summary_input", - Self::SummaryJoinInput => "SummaryJoin.{outer,inner}", - Self::SummarySubtractInput => "SummarySubtract.{left,right}", - Self::SummaryDeleteInput => "SummaryDelete.summary_input", - Self::SummaryMergeInput => "SummaryMerge.children[]", - Self::ValueOperationChild => "ValueOperation.child", - } - } -} - /// A plan-construction-time data_state violation. Typed (not a string) so a /// strategy can degrade to a conservative fallback on the specific variant /// it expects, and so tests can assert the *reason* a plan was rejected. #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum ExecutionDataStateError { - #[error("operator reached with conflicting execution timings: {first:?} and {second:?}")] - ConflictingTiming { - first: ExecutionDataState, - second: ExecutionDataState, - }, - #[error("{operator} node has no execution timing")] - UntimedNode { operator: &'static str }, - #[error("evaluation value under maintenance: {edge} received {child}")] - EvaluationUnderMaintenance { - edge: &'static str, - child: ExecutionDataState, - }, - #[error("invalid maintained-population maintenance/readout contract")] + #[error("invalid maintained-population maintenance/evaluation contract")] InvalidMaintainedPopulation, - /// A query-time value (`SummaryEstimate` / read-time `ValueOperation` output) + /// A query-time value (a `SummaryEstimate` or query-time operator output) /// placed beneath a maintained summary — the one shape issue #171's /// data_state split exists to make unrepresentable. #[error( - "readout value under maintenance: {edge} received a {child} input, but a maintained \ + "evaluation value under maintenance: {edge} received a {child} input, but a maintained \ summary can only consume update-path values (or exact accumulator state)" )] - ReadoutUnderMaintenance { + EvaluationUnderMaintenance { edge: &'static str, child: ExecutionDataState, }, /// Any other edge whose child data_state the parent does not accept /// (e.g. plain update rows fed straight into a `SummaryEstimate`, or a - /// sketch's opaque state fed into a read-time `ValueOperation`). + /// sketch's opaque state fed into a query-time operator). #[error("{edge} does not accept a {child} input")] IllegalChildDataState { edge: &'static str, @@ -197,13 +115,10 @@ pub enum ExecutionDataStateError { composed into another maintained summary" )] UnsupportedStateComposition { family: String }, - /// One shared `KeepPreAsap` node reached both as update-path raw input - /// and as a query-time fallback — see the module docs. - #[error( - "KeepPreAsap sub-DAG is data_state-ambiguous: reached as {first} and as {second} in the same \ - plan" - )] - AmbiguousKeepPreAsap { + /// One shared node assigned two different execution timings by its + /// consumers; no single execution of it can serve both. + #[error("shared node is assigned conflicting timings: {first} and {second}")] + ConflictingTiming { first: ExecutionDataState, second: ExecutionDataState, }, @@ -215,7 +130,7 @@ pub enum ExecutionDataStateError { InvalidMaintenanceBinary, #[error("checked division requires one valid guard on a read-time division operator")] InvalidCheckedDivision, - /// An `ExactOperation` whose input columns are not all `Plain` at its + /// An exact operator whose input columns are not all `Plain` at its /// declared data_state. #[error("exact operator consumes non-plain column {column:?} ({dtype})")] NonPlainOperand { column: String, dtype: String }, @@ -223,564 +138,12 @@ pub enum ExecutionDataStateError { /// `SummaryDelete`, `SummaryJoin`, `Extension`) in an executable plan. #[error("{operator} is a reserved operator with no execution contract yet")] UnimplementedOperator { operator: &'static str }, -} - -/// The data_state assigned to every node of a validated plan, keyed by -/// `Rc` pointer identity — the explicit per-node "execution_data_state" a -/// runtime or a DAG export reads instead of re-deriving it. For every -/// non-`KeepPreAsap` node this equals [`produced_data_state`]; for a -/// `KeepPreAsap` leaf it is the data_state the reaching edge assigned. -#[derive(Debug, Clone, Default)] -pub struct ExecutionDataStateAssignment { - domains: HashMap<*const SummaryNode, ExecutionDataState>, -} - -impl ExecutionDataStateAssignment { - /// The data_state assigned to `node`, if it was part of the validated plan. - pub fn data_state_of(&self, node: &Rc) -> Option { - self.domains.get(&Rc::as_ptr(node)).copied() - } - - /// The data_state assigned to the node at `ptr` — for callers walking a plan - /// by reference rather than by `Rc`. - pub fn data_state_of_ptr(&self, ptr: *const SummaryNode) -> Option { - self.domains.get(&ptr).copied() - } -} - -/// Initial layout proposed by semantic realization, not a restriction on physical -/// operator placement. `PostAsapDAG::with_execution_phases` assigns the final -/// phase independently of payload kind. Returns `None` for -/// [`SummaryExpr::KeepPreAsap`], whose data_state is assigned by the edge reaching -/// it (see the module docs). -pub fn produced_data_state(expr: &SummaryExpr) -> Option { - Some(match expr { - SummaryExpr::KeepPreAsap(_) => return None, - SummaryExpr::BinaryOp { timing, .. } => ExecutionDataState { - timing: *timing, - primitive: DataPrimitive::Raw, - }, - SummaryExpr::RelationalJoin { .. } => ExecutionDataState::QUERY_ROWS, - SummaryExpr::SummaryAgg { child, .. } => ExecutionDataState { - timing: produced_data_state(&child.expr) - .map_or(ExecutionTiming::IngestionTime, |state| state.timing), - primitive: DataPrimitive::SummaryState, - }, - SummaryExpr::SummaryJoin { .. } - | SummaryExpr::SummarySubtract { .. } - | SummaryExpr::SummaryDelete { .. } => ExecutionDataState::INGESTION_SUMMARY, - SummaryExpr::SummaryMerge { timing, .. } => ExecutionDataState { - timing: *timing, - primitive: DataPrimitive::SummaryState, - }, - SummaryExpr::SummaryEstimate { .. } => ExecutionDataState::QUERY_ROWS, - SummaryExpr::ValueOperation { timing, .. } => match timing { - ExecutionTiming::IngestionTime => ExecutionDataState::INGESTION_ROWS, - ExecutionTiming::QueryTime => ExecutionDataState::QUERY_ROWS, - }, - }) -} - -/// Is `family` the exact-accumulator family whose partial state *is* the -/// value — the one summary state a `SummaryAgg` may re-accumulate? -fn is_exact_accumulator_state(schema: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &schema.fields { - match &field.dtype { - FieldDataType::Plain(_) | FieldDataType::ExactAggregate(..) => {} - other => { - return Err(ExecutionDataStateError::UnsupportedStateComposition { - family: format!("{other:?}"), - }) - } - } - } - Ok(()) -} - -/// Validate every edge of the DAG rooted at `root` against the module-level -/// rules, returning each node's assigned data_state on success. Shared -/// `Rc`s are visited once per reaching edge (the assignment is -/// per node, so a conflict between two edges is what -/// [`ExecutionDataStateError::AmbiguousKeepPreAsap`] detects). -pub fn validate_execution_data_states( - root: &Rc, -) -> Result { - // The root may be a readable value or bare maintained state (a - // deployment may hand an `ExactAggregate` accumulator straight to a - // consumer) — only an update-path-only root is meaningless. - let root_domain = match produced_data_state(&root.expr) { - None => ExecutionDataState::QUERY_ROWS, - Some(ExecutionDataState::INGESTION_ROWS) => { - return Err(ExecutionDataStateError::MaintenanceRowsAtRoot) - } - Some(data_state) => data_state, - }; - validate_execution_data_states_at(root, root_domain) -} - -/// [`validate_execution_data_states`] for a *sub*-plan whose root is known to -/// sit at `data_state` — e.g. a maintenance-time `ValueOperation` about to be placed beneath a -/// `SummaryAgg`, which would be rejected as a whole-plan root but is a -/// legal update-path input. Validates every edge beneath `root` exactly -/// as the whole-plan entry point does. -pub fn validate_execution_data_states_at( - root: &Rc, - data_state: ExecutionDataState, -) -> Result { - let mut assignment = ExecutionDataStateAssignment::default(); - visit(root, data_state, &mut assignment)?; - Ok(assignment) -} - -/// The source rows whose series a maintenance operand has one row for: a -/// finalized per-series Sum or Count of those rows, or aligned arithmetic of -/// operands over the same rows. Each emits exactly the series with a sample. -fn per_series_rows(node: &SummaryNode) -> Option<&crate::pre_asap::QueryExpr> { - use crate::post_asap::ExactKind; - match &node.expr { - SummaryExpr::ValueOperation { - child, - operation: ValueOperation::FinalizeExactAccumulator, - .. - } => match &child.expr { - SummaryExpr::SummaryAgg { - child, - family: FieldDataType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), - reduction: crate::pre_asap::query_expr::Reduction::PerEntity, - .. - } => match &child.expr { - SummaryExpr::KeepPreAsap(rows) => Some(rows.as_ref()), - _ => None, - }, - _ => None, - }, - SummaryExpr::BinaryOp { - lhs, - rhs, - timing: ExecutionTiming::IngestionTime, - .. - } => { - let rows = per_series_rows(lhs)?; - (per_series_rows(rhs) == Some(rows)).then_some(rows) - } - _ => None, - } -} - -/// Record `data_state` for `node` (detecting a conflicting earlier assignment -/// for a `KeepPreAsap`), then check and recurse into every child edge. -fn visit( - node: &Rc, - data_state: ExecutionDataState, - assignment: &mut ExecutionDataStateAssignment, -) -> Result<(), ExecutionDataStateError> { - let ptr = Rc::as_ptr(node); - if let Some(previous) = assignment.domains.get(&ptr) { - if *previous != data_state { - return Err(ExecutionDataStateError::AmbiguousKeepPreAsap { - first: *previous, - second: data_state, - }); - } - // Already validated through another edge with the same data_state. - return Ok(()); - } - assignment.domains.insert(ptr, data_state); - - match &node.expr { - SummaryExpr::KeepPreAsap(_) => Ok(()), - SummaryExpr::BinaryOp { - lhs, - rhs, - timing, - operator, - } => { - if (operator.checked_relative_division && operator.checked_finite_division) - || (operator.checked_relative_division || operator.checked_finite_division) - && (*timing != ExecutionTiming::QueryTime - || !matches!( - operator.kind, - crate::pre_asap::BinaryOpKind::Arithmetic( - crate::pre_asap::ArithmeticOpKind::Div - ) - )) - { - return Err(ExecutionDataStateError::InvalidCheckedDivision); - } - if *timing == ExecutionTiming::IngestionTime { - use crate::pre_asap::{BinaryOpKind, DataType}; - if operator.vector_match.is_some() - || !matches!(operator.kind, BinaryOpKind::Arithmetic(_)) - || lhs.schema != rhs.schema - || lhs.schema != node.schema - // The opaque identity is a key, not an extra maintenance value. - || node.schema.fields.iter().filter(|field| { - field.name == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY - }).count() > 1 - // Maintenance arithmetic pairs every row by identity, while - // Prometheus drops unmatched series; it is exact only when - // both operands provably produce the same series. - || node.schema.fields.iter().any(|field| { - field.name == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY - }) && per_series_rows(lhs).is_none_or(|rows| per_series_rows(rhs) != Some(rows)) - || !node.schema.fields.iter().all(|field| { - !field.nullable - && if field.name == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY { - field.dtype == FieldDataType::Plain(DataType::Utf8) - } else { - matches!( - field.dtype, - FieldDataType::Plain(DataType::Float64 | DataType::Timestamp) - ) - } - }) - || node - .schema - .fields - .iter() - .filter(|field| { - matches!(field.dtype, FieldDataType::Plain(DataType::Float64)) - }) - .count() - != 1 - { - return Err(ExecutionDataStateError::InvalidMaintenanceBinary); - } - } - let expected = ExecutionDataState { - timing: *timing, - primitive: DataPrimitive::Raw, - }; - for input in [lhs, rhs] { - let state = produced_data_state(&input.expr).unwrap_or(expected); - if state != expected { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "BinaryOp operand", - child: state, - }); - } - visit(input, state, assignment)?; - } - Ok(()) - } - - SummaryExpr::RelationalJoin { left, right, .. } => { - for input in [left, right] { - let state = - produced_data_state(&input.expr).unwrap_or(ExecutionDataState::QUERY_ROWS); - if state != ExecutionDataState::QUERY_ROWS { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "RelationalJoin input", - child: state, - }); - } - visit(input, state, assignment)?; - } - Ok(()) - } - SummaryExpr::SummaryAgg { child, .. } => { - let child_domain = child_domain( - child, - ExecutionDataStateEdge::SummaryAggChild, - |avail| match avail { - ExecutionDataState::INGESTION_ROWS | ExecutionDataState::QUERY_ROWS => Ok(()), - state if state.primitive == DataPrimitive::SummaryState => { - is_exact_accumulator_state(&child.schema) - } - other => Err(ExecutionDataStateError::ReadoutUnderMaintenance { - edge: ExecutionDataStateEdge::SummaryAggChild.describe(), - child: other, - }), - }, - )?; - visit(child, child_domain, assignment) - } - SummaryExpr::SummaryJoin { outer, inner, .. } => { - for input in [outer, inner] { - let s = child_domain(input, ExecutionDataStateEdge::SummaryJoinInput, |avail| { - match avail { - ExecutionDataState::INGESTION_ROWS - | ExecutionDataState::INGESTION_SUMMARY => Ok(()), - other => Err(ExecutionDataStateError::ReadoutUnderMaintenance { - edge: ExecutionDataStateEdge::SummaryJoinInput.describe(), - child: other, - }), - } - })?; - visit(input, s, assignment)?; - } - Ok(()) - } - SummaryExpr::SummarySubtract { left, right } => { - for input in [left, right] { - let s = state_only(input, ExecutionDataStateEdge::SummarySubtractInput)?; - visit(input, s, assignment)?; - } - Ok(()) - } - SummaryExpr::SummaryDelete { summary_input, .. } => { - let s = state_only(summary_input, ExecutionDataStateEdge::SummaryDeleteInput)?; - visit(summary_input, s, assignment) - } - SummaryExpr::SummaryMerge { children, timing } => { - for input in children { - let s = child_domain(input, ExecutionDataStateEdge::SummaryMergeInput, |state| { - if state.primitive == DataPrimitive::SummaryState - && (*timing == ExecutionTiming::QueryTime || state.timing == *timing) - { - Ok(()) - } else { - Err(ExecutionDataStateError::IllegalChildDataState { - edge: ExecutionDataStateEdge::SummaryMergeInput.describe(), - child: state, - }) - } - })?; - visit(input, s, assignment)?; - } - Ok(()) - } - SummaryExpr::SummaryEstimate { summary_input, .. } => { - let s = state_only(summary_input, ExecutionDataStateEdge::SummaryEstimateInput)?; - visit(summary_input, s, assignment) - } - SummaryExpr::ValueOperation { - child, - operation, - timing, - } => { - // Population timing is a lifecycle decision: a retained population - // is maintained at ingestion time, an ephemeral one is rebuilt - // from raw input per query. Its input and readout contracts are - // structural and hold either way. - let valid_population = match operation { - ValueOperation::MaintainPopulation { population } => { - matches!(&child.expr, SummaryExpr::KeepPreAsap(input) if population.matches_input(input)) - } - ValueOperation::ReadPopulation { readout } => { - *timing == ExecutionTiming::QueryTime - && matches!(&child.expr, SummaryExpr::ValueOperation { operation: ValueOperation::MaintainPopulation { population }, .. } if population.supports(readout)) - } - _ => true, - }; - if !valid_population { - return Err(ExecutionDataStateError::InvalidMaintainedPopulation); - } - let required = match timing { - ExecutionTiming::IngestionTime => ExecutionDataState::INGESTION_ROWS, - ExecutionTiming::QueryTime => ExecutionDataState::QUERY_ROWS, - }; - let s = produced_data_state(&child.expr).unwrap_or(required); - let exact_readout = (*timing == ExecutionTiming::QueryTime - || matches!(operation, ValueOperation::FinalizeExactAccumulator)) - && s.primitive == DataPrimitive::SummaryState - && (*timing == ExecutionTiming::QueryTime || s.timing == *timing) - && is_exact_accumulator_state(&child.schema).is_ok(); - // A query-time readout may read a population retained at ingestion. - let population_readout = matches!(operation, ValueOperation::ReadPopulation { .. }) - && *timing == ExecutionTiming::QueryTime - && matches!( - &child.expr, - SummaryExpr::ValueOperation { - operation: ValueOperation::MaintainPopulation { .. }, - .. - } - ); - if s != required && !exact_readout && !population_readout { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: ExecutionDataStateEdge::ValueOperationChild.describe(), - child: s, - }); - } - check_plain_operands(operation, &child.schema)?; - visit(child, s, assignment) - } - } -} - -/// The data_state `child` takes as a direct input of `parent`, without -/// validating legality — `child`'s own produced data_state, or for a -/// `KeepPreAsap` leaf the data_state `parent`'s edge assigns it (update-path raw -/// input under maintenance-time operation edges, query-time fallback under a -/// a read-time operation, and — meaninglessly, but for a stable answer — maintenance rows -/// under a state-only edge). For DAG export and other reporting that needs -/// an explicit per-node data_state even on a plan that -/// [`validate_execution_data_states`] would reject. -pub fn assigned_child_data_state(parent: &SummaryExpr, child: &SummaryNode) -> ExecutionDataState { - if let Some(avail) = produced_data_state(&child.expr) { - return avail; - } - match parent { - SummaryExpr::ValueOperation { - timing: ExecutionTiming::QueryTime, - .. - } - | SummaryExpr::BinaryOp { - timing: ExecutionTiming::QueryTime, - .. - } => ExecutionDataState::QUERY_ROWS, - SummaryExpr::KeepPreAsap(_) - | SummaryExpr::BinaryOp { - timing: ExecutionTiming::IngestionTime, - .. - } - | SummaryExpr::RelationalJoin { .. } - | SummaryExpr::SummaryAgg { .. } - | SummaryExpr::SummaryJoin { .. } - | SummaryExpr::SummarySubtract { .. } - | SummaryExpr::SummaryDelete { .. } - | SummaryExpr::SummaryEstimate { .. } - | SummaryExpr::SummaryMerge { .. } - | SummaryExpr::ValueOperation { - timing: ExecutionTiming::IngestionTime, - .. - } => ExecutionDataState::INGESTION_ROWS, - } -} - -/// The data_state `child` takes on `edge`: its own produced data_state -/// (checked via `accept`), or — for a `KeepPreAsap` leaf — the data_state the -/// edge assigns it, derived from what that edge accepts. -fn child_domain( - child: &Rc, - edge: ExecutionDataStateEdge, - accept: impl Fn(ExecutionDataState) -> Result<(), ExecutionDataStateError>, -) -> Result { - match produced_data_state(&child.expr) { - Some(avail) => { - accept(avail)?; - Ok(avail) - } - None => { - // A raw pre-ASAP sub-DAG executes at whichever data_state its consumer - // needs: update-path input for maintenance-time operation edges, - // query-time fallback for a read-time edge. State-only edges - // can't consume plain rows at all. - let assigned = match edge { - ExecutionDataStateEdge::SummaryAggChild - | ExecutionDataStateEdge::SummaryJoinInput - | ExecutionDataStateEdge::ValueOperationChild => ExecutionDataState::INGESTION_ROWS, - ExecutionDataStateEdge::SummaryEstimateInput - | ExecutionDataStateEdge::SummarySubtractInput - | ExecutionDataStateEdge::SummaryDeleteInput - | ExecutionDataStateEdge::SummaryMergeInput => { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: edge.describe(), - child: ExecutionDataState::INGESTION_ROWS, - }) - } - }; - accept(assigned)?; - Ok(assigned) - } - } -} - -fn state_only( - child: &Rc, - edge: ExecutionDataStateEdge, -) -> Result { - child_domain(child, edge, |avail| match avail { - state - if state.primitive == DataPrimitive::SummaryState - && (state.timing == ExecutionTiming::IngestionTime - || matches!(edge, ExecutionDataStateEdge::SummaryEstimateInput)) => - { - Ok(()) - } - other => Err(ExecutionDataStateError::IllegalChildDataState { - edge: edge.describe(), - child: other, - }), - }) -} - -/// The exact operator must consume only `Plain` columns of its input: for -/// an `Aggregate` payload, every grouping key and every measure's input -/// column. -fn check_plain_operands( - op: &ValueOperation, - input: &Schema, -) -> Result<(), ExecutionDataStateError> { - if matches!( - op, - ValueOperation::Sort { .. } - | ValueOperation::Limit { .. } - | ValueOperation::Project { .. } - | ValueOperation::Filter { .. } - | ValueOperation::FinalizeExactAccumulator - ) { - return check_plain_or_exact_values(input); - } - let ValueOperation::Exact(op) = op else { - return check_all_plain(input); - }; - let ExactOperation::Aggregate { - reduction, - measures, - filters, - .. - } = op; - let mut referenced: Vec = reduction - .group_keys() - .map(|keys| keys.keys().to_vec()) - .unwrap_or_default(); - for m in measures { - referenced.extend(m.input_cols()); - } - for Predicate(f) in filters.iter().flatten() { - referenced.extend(f.columns_referenced().into_iter().copied()); - } - // With no explicit input column (the PromQL sample-value convention) - // the operator reads every non-key column, so all must be plain. - let implicit = measures.iter().any(|m| m.input_cols().is_empty()); - for (i, field) in input.fields.iter().enumerate() { - if !(implicit || referenced.contains(&i)) { - continue; - } - if !matches!(field.dtype, FieldDataType::Plain(_)) { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -fn check_plain_or_exact_values(input: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &input.fields { - if !matches!( - field.dtype, - FieldDataType::Plain(_) | FieldDataType::ExactAggregate(..) - ) { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -fn check_all_plain(input: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &input.fields { - if !matches!(field.dtype, FieldDataType::Plain(_)) { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -/// `schema` with its reuse metadata dropped, or `None` if any field carries -/// summary state — the shape an exact operator reads. -pub fn plain_schema(schema: &Schema) -> Option { - schema - .is_all_plain() - .then(|| Schema::lifted(schema.fields.clone(), schema.time_index)) + /// A node reached by export without a timing: the materialization timing pass + /// was not applied to the DAG first. + #[error( + "{operator} node has no execution timing; apply materialization timings before export" + )] + UntimedNode { operator: &'static str }, } /// `schema` as a summary-planning node output: fields and time axis kept, @@ -789,45 +152,18 @@ pub fn lift_plain(schema: &Schema) -> Schema { Schema::lifted(schema.fields.clone(), schema.time_index) } -/// Output schema of `op` applied to a child whose edge carries `input` — -/// the same canonical derivation the pre-ASAP `Aggregate` node uses, so an -/// exact `ValueOperation` never disagrees with the pre-ASAP -/// target it was lowered from. `Err` when the child carries non-plain -/// state the operator cannot read. -pub fn exact_operation_output_schema( - op: &ExactOperation, - input: &Schema, -) -> Result { - let plain = plain_schema(input).ok_or(ExactOperationSchemaError::NonPlainInput)?; - let ExactOperation::Aggregate { - reduction, - measures, - output_names, - .. - } = op; - let out = aggregate_output_schema(&plain, reduction, measures, output_names)?; - Ok(lift_plain(&out)) -} - -/// Why [`exact_operation_output_schema`] could not derive a schema. +/// Why an exact operator's output schema could not be derived. #[derive(Debug, Error)] pub enum ExactOperationSchemaError { #[error("exact operator input carries summary state, not plain columns")] NonPlainInput, #[error("schema derivation failed: {0}")] - Schema(#[from] QueryExprError), - #[error("schema derivation failed: {0}")] - Derivation(#[from] crate::ir::SchemaDerivationError), + Schema(#[from] SchemaDerivationError), } #[cfg(test)] mod tests { use super::*; - use crate::post_asap::{ExactKind, ExactParams, GroupingStrategy, SketchStatistic}; - use crate::pre_asap::agg_intent::AggIntent; - use crate::pre_asap::expr_ir::ColumnRef; - use crate::pre_asap::query_expr::{QueryExpr, Reduction, Source}; - use crate::pre_asap::schema::{DataType, Field}; /// Both execution phases use raw values, distinct from maintained state. #[test] @@ -845,347 +181,6 @@ mod tests { assert_eq!(DataPrimitive::SummaryState.as_str(), "summary_state"); } - fn scan() -> Rc { - Rc::new(QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("zone", DataType::Utf8, true), - ], - 0, - vec![], - ), - }) - } - - fn keep() -> Rc { - let s = scan(); - let schema = lift_plain(&s.output_schema().unwrap()); - Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(s), - schema, - guarantee: None, - }) - } - - fn plain(names: &[&str]) -> Schema { - Schema::lifted( - names - .iter() - .map(|n| Field { - name: (*n).into(), - dtype: FieldDataType::Plain(DataType::Float64), - nullable: false, - table: None, - }) - .collect(), - None, - ) - } - - fn agg(child: Rc, family: FieldDataType) -> Rc { - Rc::new(SummaryNode { - expr: SummaryExpr::SummaryAgg { - child, - family: family.clone(), - input: crate::post_asap::SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }, - schema: Schema::lifted( - vec![Field { - name: "state".into(), - dtype: family, - nullable: false, - table: None, - }], - None, - ), - guarantee: None, - }) - } - - fn kll() -> FieldDataType { - use crate::post_asap::{SketchAlgorithm, SketchKind, SketchParams}; - FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), - GroupingStrategy::default(), - ) - } - - fn estimate(child: Rc) -> Rc { - Rc::new(SummaryNode { - expr: SummaryExpr::SummaryEstimate { - summary_input: child, - query: SketchStatistic::Quantile { q: 0.99 }, - }, - schema: plain(&["quantile_0_99"]), - guarantee: None, - }) - } - - fn max_op() -> ExactOperation { - ExactOperation::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Max { col: None }], - output_names: vec![], - having: None, - filters: vec![], - } - } - - #[test] - fn keep_pre_asap_under_summary_agg_is_update_input() { - let leaf = keep(); - let root = agg(Rc::clone(&leaf), kll()); - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&leaf), - Some(ExecutionDataState::INGESTION_ROWS) - ); - assert_eq!( - assignment.data_state_of(&root), - Some(ExecutionDataState::INGESTION_SUMMARY) - ); - } - - // Typed derived updates retain one opaque series identity only when both - // operands cover the same series; arbitrary labels are never admitted. - #[test] - fn maintenance_binary_accepts_only_well_typed_series_identity() { - use crate::pre_asap::{ArithmeticOpKind, BinaryOpKind}; - let identity = crate::pre_asap::schema::PROMQL_SERIES_IDENTITY; - // A finalized per-series Sum of `metric`'s rows. - let operand = |metric: &str, schema: &Schema| { - let rows = Rc::new(QueryExpr::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - }); - let mut state = schema.clone(); - state.fields[0].dtype = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let sum = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryAgg { - child: Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(rows), - schema: schema.clone(), - guarantee: None, - }), - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: crate::post_asap::SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), - filter: None, - }, - schema: state, - guarantee: None, - }); - Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: sum, - operation: ValueOperation::FinalizeExactAccumulator, - timing: ExecutionTiming::IngestionTime, - }, - schema: schema.clone(), - guarantee: None, - }) - }; - let validate = |schema: Schema, rhs: &str| { - let binary = Rc::new(SummaryNode { - expr: SummaryExpr::BinaryOp { - lhs: operand("m", &schema), - rhs: operand(rhs, &schema), - timing: ExecutionTiming::IngestionTime, - operator: crate::post_asap::BinaryOperator { - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - checked_relative_division: false, - checked_finite_division: false, - }, - }, - schema, - guarantee: None, - }); - validate_execution_data_states(&estimate(agg(binary, kll()))).map(|_| ()) - }; - let mut schema = plain(&["value"]); - schema.fields.push(Field { - table: None, - name: "ts".into(), - dtype: FieldDataType::Plain(DataType::Timestamp), - nullable: false, - }); - schema.time_index = Some(1); - assert!(validate(schema.clone(), "m").is_ok()); - assert!( - validate(schema.clone(), "n").is_ok(), - "no identity to align" - ); - schema.fields.push(Field { - table: None, - name: identity.into(), - dtype: FieldDataType::Plain(DataType::Utf8), - nullable: false, - }); - assert!(validate(schema.clone(), "m").is_ok()); - assert_eq!( - validate(schema.clone(), "n"), - Err(ExecutionDataStateError::InvalidMaintenanceBinary), - "different selectors may cover different series" - ); - for mutation in 0..4 { - let mut invalid = schema.clone(); - match mutation { - 0 => invalid.fields[2].nullable = true, - 1 => invalid.fields[2].dtype = FieldDataType::Plain(DataType::Timestamp), - 2 => invalid.fields.push(invalid.fields[2].clone()), - _ => invalid.fields[2].name = "label".into(), - } - assert_eq!( - validate(invalid, "m"), - Err(ExecutionDataStateError::InvalidMaintenanceBinary) - ); - } - } - - #[test] - fn exact_accumulator_state_may_feed_another_summary_agg() { - let inner = agg( - keep(), - FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - ); - let root = estimate(agg(inner, kll())); - assert!(validate_execution_data_states(&root).is_ok()); - } - - #[test] - fn readout_can_feed_summary_construction_at_query_time() { - let inner = estimate(agg(keep(), kll())); - let summary = agg(inner, kll()); - let root = estimate(summary.clone()); - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&summary).unwrap().timing, - ExecutionTiming::QueryTime - ); - } - - #[test] - fn query_time_operation_over_readout_is_legal_and_root_is_readout() { - let inner = estimate(agg(keep(), kll())); - let root = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: inner, - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::QueryTime, - }, - schema: plain(&["max"]), - guarantee: None, - }); - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&root), - Some(ExecutionDataState::QUERY_ROWS) - ); - } - - #[test] - fn non_exact_operator_uses_the_same_read_domain_contract() { - let inner = estimate(agg(keep(), kll())); - let root = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: inner, - operation: ValueOperation::Extension { - name: "approximate_calibration".into(), - }, - timing: ExecutionTiming::QueryTime, - }, - schema: plain(&["calibrated"]), - guarantee: None, - }); - - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&root), - Some(ExecutionDataState::QUERY_ROWS) - ); - } - - #[test] - fn query_time_values_can_feed_query_time_summary_construction() { - let inner = estimate(agg(keep(), kll())); - let post = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: inner, - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::QueryTime, - }, - schema: plain(&["max"]), - guarantee: None, - }); - let root = agg(post, kll()); - let root = estimate(root); - validate_execution_data_states(&root).unwrap(); - } - - #[test] - fn function_under_summary_agg_is_legal_but_not_at_root() { - let operation = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: keep(), - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::IngestionTime, - }, - schema: plain(&["max"]), - guarantee: None, - }); - assert_eq!( - validate_execution_data_states(&operation).err(), - Some(ExecutionDataStateError::MaintenanceRowsAtRoot) - ); - let root = estimate(agg(Rc::clone(&operation), kll())); - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&operation), - Some(ExecutionDataState::INGESTION_ROWS) - ); - } - - #[test] - fn function_over_readout_is_rejected() { - let inner = estimate(agg(keep(), kll())); - let operation = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: inner, - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::IngestionTime, - }, - schema: plain(&["max"]), - guarantee: None, - }); - let root = agg(operation, kll()); - assert!(matches!( - validate_execution_data_states(&root), - Err(ExecutionDataStateError::IllegalChildDataState { - edge: "ValueOperation.child", - child: ExecutionDataState::QUERY_ROWS - }) - )); - } - #[test] fn execution_phase_wire_names_are_ingestion_and_query_time() { for (phase, name) in [ @@ -1202,154 +197,4 @@ mod tests { assert!(serde_json::from_str::("\"maintenance_time\"").is_err()); assert!(serde_json::from_str::("\"MaintenanceTime\"").is_err()); } - - #[test] - fn summary_merge_runs_at_ingestion_or_query_time() { - for timing in [ExecutionTiming::IngestionTime, ExecutionTiming::QueryTime] { - let input = agg(keep(), kll()); - let merged = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { - children: vec![input.clone()], - timing, - }, - schema: input.schema.clone(), - guarantee: None, - }); - let root = estimate(merged.clone()); - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&merged), - Some(ExecutionDataState { - timing, - primitive: DataPrimitive::SummaryState, - }) - ); - let exported = crate::post_asap::compile_post_asap_dag(&root).unwrap(); - assert!(exported.nodes.iter().any(|node| matches!(node.payload, - crate::post_asap::PostAsapOperatorPayload::SummaryMerge - if node.output_state.timing == timing))); - } - } - - #[test] - fn ingestion_merge_cannot_depend_on_query_execution() { - let input = agg(keep(), kll()); - let query_merge = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { - children: vec![input.clone()], - timing: ExecutionTiming::QueryTime, - }, - schema: input.schema.clone(), - guarantee: None, - }); - let ingestion_merge = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { - children: vec![query_merge], - timing: ExecutionTiming::IngestionTime, - }, - schema: input.schema.clone(), - guarantee: None, - }); - assert!(validate_execution_data_states(&ingestion_merge).is_err()); - } - - #[test] - fn a_shared_keep_pre_asap_reached_in_two_domains_is_ambiguous() { - // One raw sub-DAG used both as update input (under a SummaryAgg) and - // as a query-time fallback (under an ExactRead) — no single - // execution can serve both, so the plan is rejected. - let shared = keep(); - let maintained = estimate(agg(Rc::clone(&shared), kll())); - let post_over_raw = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: Rc::clone(&shared), - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::QueryTime, - }, - schema: plain(&["max"]), - guarantee: None, - }); - let root = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { - timing: ExecutionTiming::IngestionTime, - children: vec![ - Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: maintained, - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::QueryTime, - }, - schema: plain(&["max"]), - guarantee: None, - }), - post_over_raw, - ], - }, - schema: plain(&["max"]), - guarantee: None, - }); - // SummaryMerge only accepts state, so this fails earlier for a - // different reason; probe the ambiguity through a direct visit. - let mut assignment = ExecutionDataStateAssignment::default(); - visit(&shared, ExecutionDataState::INGESTION_ROWS, &mut assignment).unwrap(); - assert_eq!( - visit(&shared, ExecutionDataState::QUERY_ROWS, &mut assignment), - Err(ExecutionDataStateError::AmbiguousKeepPreAsap { - first: ExecutionDataState::INGESTION_ROWS, - second: ExecutionDataState::QUERY_ROWS, - }) - ); - assert!(validate_execution_data_states(&root).is_err()); - } - - // Both paired operands must be plain; an unrelated state column is not an input. - #[test] - fn pearson_corr_checks_both_operand_states() { - let operation = ValueOperation::Exact(ExactOperation::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::PearsonCorr { left: 0, right: 1 }], - output_names: vec![], - having: None, - filters: vec![], - }); - for operand in [0, 1] { - let mut input = plain(&["x", "y", "unused"]); - input.fields[operand].dtype = kll(); - assert!(matches!( - check_plain_operands(&operation, &input), - Err(ExecutionDataStateError::NonPlainOperand { .. }) - )); - } - let mut input = plain(&["x", "y", "unused"]); - input.fields[2].dtype = kll(); - check_plain_operands(&operation, &input).unwrap(); - } - - #[test] - fn exact_operator_schema_matches_pre_asap_aggregate_derivation() { - let child_schema = lift_plain(&scan().output_schema().unwrap()); - let op = ExactOperation::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::Max { col: None }], - output_names: vec![], - having: None, - filters: vec![], - }; - let out = exact_operation_output_schema(&op, &child_schema).unwrap(); - let names: Vec<_> = out.fields.iter().map(|f| f.name.as_str()).collect(); - assert_eq!(names, vec!["zone", "max"]); - assert!(out - .fields - .iter() - .all(|f| matches!(f.dtype, FieldDataType::Plain(_)))); - } - - #[test] - fn exact_operator_rejects_non_plain_input() { - let state = agg(keep(), kll()); - assert!(matches!( - exact_operation_output_schema(&max_op(), &state.schema), - Err(ExactOperationSchemaError::NonPlainInput) - )); - } } diff --git a/crates/types/src/post_asap/expr.rs b/crates/types/src/post_asap/expr.rs deleted file mode 100644 index 732a09db5..000000000 --- a/crates/types/src/post_asap/expr.rs +++ /dev/null @@ -1,303 +0,0 @@ -use super::ExecutionTiming; -use std::rc::Rc; - -use super::guarantee::ResultGuarantee; -use super::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; -use crate::pre_asap::agg_intent::AggIntent; -use crate::pre_asap::query_expr::Predicate; -use crate::pre_asap::schema::{FieldDataType, Schema}; -use crate::pre_asap::{ - BinaryOpKind, ColumnRef, GroupKeys, JoinKind, ProjectItem, QueryExpr, Reduction, SortKey, - VectorMatch, -}; - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[non_exhaustive] -pub enum ExactOperation { - Aggregate { - reduction: Reduction, - measures: Vec, - output_names: Vec, - /// Per-measure row predicates parallel to `measures`, positional - /// against the child's output rows — the same contract as - /// `QueryExpr::Aggregate.filters` (issue #466). - #[serde(default)] - filters: Vec>, - having: Option, - }, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[non_exhaustive] -pub enum ValueOperation { - /// Maintain the full declared population, including membership changes, - /// so removing a TopK member can promote another. - MaintainPopulation { - population: super::maintained_population::MaintainedPopulation, - }, - /// Read an aggregate or TopK prefix from the maintained population. - ReadPopulation { - readout: super::maintained_population::PopulationStatistic, - }, - Exact(ExactOperation), - /// Read an exact accumulator's state as its finalized scalar value. - /// - /// Exact accumulators do not need an estimator, but the explicit node - /// marks the maintenance-to-read boundary before query-time operators - /// such as PromQL binary arithmetic, sorting, and limiting. - FinalizeExactAccumulator, - /// Query-time column projection. SQL lowering retains the SELECT list as - /// a `Project` above its aggregate, so the post-ASAP DAG must preserve - /// its expressions, aliases, and optional derived-table qualifier while - /// allowing the aggregate child to be planned independently. - Project { - cols: Vec, - qualifier: Option, - }, - /// Query-time row filtering. The predicate remains positional against - /// the child's output schema and is evaluated only after any summary - /// state below it has been read out to rows. - Filter { - pred: Predicate, - }, - /// Query-time ordering of the child's value rows. This is deliberately - /// distinct from frequency-sketch heavy-hitter readout: PromQL `topk` - /// ranks the values produced by its child at the evaluation timestamp. - Sort { - keys: Vec, - partition_by: GroupKeys, - }, - /// Query-time row selection, normally composed over [`Self::Sort`] for - /// PromQL `topk`/`bottomk` and SQL `ORDER BY … LIMIT`. - Limit { - n: usize, - offset: usize, - /// Apply the offset and limit independently to each group. - partition_by: GroupKeys, - }, - Extension { - name: String, - }, -} - -/// Whether a candidate-membership sidecar is proven to contain every true -/// top-k key or is an explicitly approximate optimization. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum CandidateCompleteness { - Certified { guarantee: ResultGuarantee }, - BestEffort { guarantee: Option }, -} - -// ── Post-ASAP DAG node ─────────────────────────────────────────────────────── - -/// A node in the post-ASAP DAG: wraps the expression and its derived output -/// schema so every edge carries a typed schema. `Schema` may contain -/// summary-state-typed columns (`FieldDataType`'s non-`Plain` variants); -/// the pre-ASAP `Schema` cannot. -#[derive(Debug, Clone, PartialEq)] -pub struct SummaryNode { - pub expr: SummaryExpr, - /// Output schema of `expr` — the schema of the data flowing on the edge - /// leading *from* this node to its parent(s). - pub schema: Schema, - /// The machine-readable accuracy guarantee of the *value* this node - /// produces (issue #172) — `Some` on every finalized, caller-visible - /// value: a `SummaryEstimate` readout, an `ExactAggregate`-family - /// `SummaryAgg` (its state *is* the value), or a `KeepPreAsap` sub-DAG - /// (executed exactly). `None` on raw summary state — a sketch-family - /// `SummaryAgg`, `SummaryMerge`, `SummarySubtract`, `SummaryDelete`, - /// `SummaryJoin` — whose guarantee only exists once something reads it - /// out; and `None` on a readout of a family the plugged-in - /// `AccuracyModel` has no local guarantee for (`Sample`/`Wavelet`/ - /// `StatModel`), which a fail-closed consumer must treat as "unknown", - /// never as exact. - pub guarantee: Option, -} - -// ── Post-ASAP sketch-bound IR ──────────────────────────────────────────────── - -/// Sketch-bound IR produced by post-ASAP binding and final selection. Binding -/// rules selectively replace logical aggregates and joins in the pre-ASAP -/// `QueryExpr` with summary-bound counterparts. Final selection can retain -/// supported read-time value operations around independently planned children; -/// other unsupported sub-DAGs pass through as `KeepPreAsap(Rc)`. -/// -/// Traversing from the root node yields a DAG; shared sub-expressions appear -/// as multiple `Rc` references to the same `SummaryNode`. -#[derive(Debug, Clone, PartialEq)] -pub enum SummaryExpr { - /// A pre-ASAP sub-DAG kept as-is because it has no selected implementation - /// or supported residual decomposition. Output schema is the inner node's - /// schema, lifted to `Schema` with all fields as - /// `FieldDataType::Plain`. - KeepPreAsap(Rc), - - /// A PromQL binary operation whose operands were planned independently. - /// This keeps realizable summary/readout leaves visible instead of - /// hiding the complete expression inside `KeepPreAsap`. - BinaryOp { - timing: ExecutionTiming, - lhs: Rc, - rhs: Rc, - operator: BinaryOperator, - }, - - /// Plain-row semantics composed with a post-ASAP child. Timing is an - /// independent physical choice, not part of the operation's identity. - ValueOperation { - child: Rc, - operation: ValueOperation, - timing: super::execution_data_state::ExecutionTiming, - }, - - /// Read-time relational join over two row-producing children. This is - /// distinct from [`SummaryJoin`](Self::SummaryJoin), which combines - /// summary states for join estimation during maintenance. - RelationalJoin { - left: Rc, - right: Rc, - kind: JoinKind, - pred: Predicate, - /// Optional proof for candidate pruning; ranking remains a separate operation. - pruning: Option, - }, - - /// Summary aggregation. Post-ASAP binding chose `family` — which - /// summary family (exact accumulator, sketch, sample, wavelet, or - /// statistical model) and its `(kind, params)` — from the catalog for - /// `AggIntent` under `DeploymentConstraints`. - /// Output schema: grouping columns (verbatim) + one field carrying - /// partial summary state per group, typed `family`. - SummaryAgg { - child: Rc, - /// Which summary family realizes this aggregation, and that - /// family's own `(kind, params)`. Never `FieldDataType::Plain` - /// — this node always produces summary state, not a plain value. - family: FieldDataType, - /// Optional multidimensional item identity and the observation/update - /// weight fed into each state update. Subpopulation semantics remain - /// on `reduction`; physical sharing remains on `grouping`. - input: SummaryUpdate, - /// How this aggregation's output rows relate to `child`'s — the - /// same [`Reduction`] the pre-ASAP `Aggregate` node it was bound - /// from carried (issue #165), reused verbatim rather than - /// flattened to a bare `Vec`. `Reduction::Reduce(by)` - /// with an empty `by` is a genuine full reduction (merge every - /// candidate into one group); `Reduction::PerEntity` has no - /// grouping concept at all (never merge across entities) — the - /// two collapsed to the same ambiguous `by: []` before this field - /// existed (issue #163). - reduction: Reduction, - /// How this aggregation's summary state is physically instantiated - /// across `reduction`'s subpopulations — one independent instance - /// per `by` key (today's only behavior, and this field's default), - /// or one shared Hydra-family structure serving all of them (issue - /// #256). Lives here, next to `reduction`, for planning, and is also - /// encoded in sketch-valued `family`/output-schema state so merges - /// can reject incompatible layouts. `reduction` is the field that - /// carries the `by` keys this axis's legality depends on (a - /// `SharedMultiSubpopulation` choice only makes sense when - /// `reduction` actually has a subpopulation concept — see - /// `asap_aware_mapping::grouping`'s module docs for the legality - /// rules). Every existing producer of a `SummaryAgg` sets this to - /// `GroupingStrategy::PerSubpopulationInstance` (its `Default`), - /// so no existing behavior changes. - grouping: GroupingStrategy, - /// Row predicate gating this summary's updates (issue #466): only - /// rows where it is `TRUE` update the state; grouping keys are - /// still read from every row. Positional against `child`'s output. - /// A field rather than a `Filter` child so summaries that differ - /// only in predicate can still share one child. No binding rule - /// sets it yet — a filtered pre-ASAP measure stays `KeepPreAsap` — - /// so every producer today writes `None`. - filter: Option, - }, - - /// Summary-aware join (KMV / theta for join-cardinality; join-sample for - /// sampling). Emitted only when a `Bind*OnJoin` rule fires. - /// Output schema: one field typed `family`, read by a downstream - /// `SummaryEstimate`. - SummaryJoin { - outer: Rc, - inner: Rc, - key: ColumnRef, - /// Never `FieldDataType::Plain` — see [`SummaryAgg::family`](SummaryExpr::SummaryAgg). - family: FieldDataType, - }, - - /// Subtract one summary from another. Valid only for families with a - /// linear-inverse property (CMS, theta, count-based). Catalog flag - /// `subtractable` must be true for the family. - /// Output schema: one field (same family + params as inputs). - SummarySubtract { - left: Rc, - right: Rc, - }, - - /// Delete a key from a summary (CMS update with −1, deletable Bloom - /// filter). Catalog flag `deletable` must be true. Output schema = - /// input schema unchanged in type (same field type as input). - SummaryDelete { - summary_input: Rc, - key: ColumnRef, - }, - - /// Read out a query result from a built summary. The summary-state field - /// type does *not* propagate downstream of an estimate — the output - /// schema is a regular row-shaped schema (Float64 for quantile, Int64 - /// for count/cardinality, `[(key, count)]` for top-k). - SummaryEstimate { - summary_input: Rc, - query: SketchStatistic, - }, - - /// ⊕ — union of summaries across stages / shards. Distinct from the - /// pre-ASAP `Concat` because summary union has type constraints: all - /// inputs must agree on `family` (kind + params) and the catalog flag - /// `mergeable` must be true. Inserted by a deployment's own stage - /// allocator (not modeled in this crate) on cut edges. - /// Output schema: one field (same family + params as inputs). - SummaryMerge { - children: Vec>, - timing: ExecutionTiming, - }, -} - -/// All semantics owned by a post-ASAP binary operator. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct BinaryOperator { - /// Execute division only for finite operands, a nonzero divisor, and a - /// normal finite result; otherwise use exact execution. Required by the - /// relative-value division certificate, including floating-point range. - #[serde(default)] - pub checked_relative_division: bool, - /// Conditional exact rewrites (such as temporal average from sum/count) - /// require finite operands and quotient. Zero/subnormal results are valid; - /// overflow must fall back to the original query rather than emit infinity. - #[serde(default)] - pub checked_finite_division: bool, - pub kind: BinaryOpKind, - /// `None` is the only currently supported vector/vector matching mode. - /// The field is retained so execution never has to recover semantics by - /// re-parsing PromQL. - pub vector_match: Option, -} - -impl BinaryOperator { - pub fn from_logical(operator: &crate::ir::BinaryOperator, return_bool: bool) -> Self { - use crate::pre_asap::BinaryOpKind as L; - Self { - kind: match &operator.kind { - L::Arithmetic(op) => BinaryOpKind::Arithmetic(op.clone()), - L::Compare(op) if return_bool => BinaryOpKind::CompareBool(op.clone()), - L::Compare(op) => BinaryOpKind::Compare(op.clone()), - L::CompareBool(op) => BinaryOpKind::CompareBool(op.clone()), - L::Set(op) => BinaryOpKind::Set(op.clone()), - }, - vector_match: operator.vector_match.clone(), - checked_relative_division: operator.checked_relative_division, - checked_finite_division: operator.checked_finite_division, - } - } -} diff --git a/crates/types/src/post_asap/maintained_population.rs b/crates/types/src/post_asap/maintained_population.rs index 577544bd7..5e64b5436 100644 --- a/crates/types/src/post_asap/maintained_population.rs +++ b/crates/types/src/post_asap/maintained_population.rs @@ -1,4 +1,4 @@ -//! Language-independent maintained populations and their readouts. +//! Language-independent maintained populations and their evaluations. //! Resource limits, ingestion placement and data structures belong to the executor. use serde::{Deserialize, Serialize}; @@ -35,82 +35,6 @@ pub enum PopulationStatistic { Average, } -impl CurrentSeriesInput { - /// Verify the named contract against the canonical maintenance input. - pub fn matches_input(&self, input: &crate::pre_asap::QueryExpr) -> bool { - use crate::pre_asap::{CompareOpKind, DataType, QueryExpr, ScalarValue, Source}; - // PromQL instant selectors carry an ingestion-interval `TimeRange` as - // their input scope. The population must use the same expiry horizon; - // shifted and otherwise transformed inputs still fail below. - let input = match input { - QueryExpr::TimeRange { range, child } - if self.lookback_ms > 0 - && *range == std::time::Duration::from_millis(self.lookback_ms) => - { - child.as_ref() - } - QueryExpr::TimeRange { .. } => return false, - other if self.lookback_ms == 300_000 => other, - _ => return false, - }; - let QueryExpr::Scan { - source: Source::TimeSeries { metric }, - predicates, - schema, - } = input - else { - return false; - }; - if self.metric.is_empty() - || *metric != self.metric - || (schema.closed && !schema.has_promql_series_identity()) - || schema.time_index.is_none() - { - return false; - } - if self.grouping.iter().any(|label| { - !schema - .fields - .iter() - .any(|c| c.name == *label && c.dtype == DataType::Utf8) - }) { - return false; - } - let mut matchers = Vec::new(); - for predicate in predicates { - let QueryExpr::Compare { left, op, right } = predicate.0.as_ref() else { - return false; - }; - let (QueryExpr::Column(col), QueryExpr::Literal(ScalarValue::Utf8(value))) = - (left.as_ref(), right.as_ref()) - else { - return false; - }; - let Some(column) = schema.fields.get(*col) else { - return false; - }; - if column.dtype != DataType::Utf8 { - return false; - } - let operation = match op { - CompareOpKind::Eq => CurrentSeriesMatch::Equal, - CompareOpKind::Ne => CurrentSeriesMatch::NotEqual, - CompareOpKind::Regex => CurrentSeriesMatch::Regex, - CompareOpKind::NotRegex => CurrentSeriesMatch::NotRegex, - _ => return false, - }; - matchers.push(CurrentSeriesMatcher { - label: column.name.clone(), - value: value.clone(), - operation, - }); - } - matchers.sort(); - matchers.dedup(); - self.matchers == matchers && self.grouping.windows(2).all(|w| w[0] < w[1]) - } -} - /// Membership is part of state identity. Table rows must never acquire implicit /// latest-per-series selection, stale markers, or a PromQL lookback. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -130,35 +54,6 @@ pub struct MaintainedPopulation { pub quantiles: bool, } -impl MaintainedPopulation { - pub fn matches_input(&self, input: &crate::pre_asap::QueryExpr) -> bool { - match &self.input { - PopulationInput::CurrentSeries(spec) => spec.matches_input(input), - PopulationInput::Rows { - input: expected, - value_column, - grouping, - } => { - use crate::pre_asap::{DataType, QueryExpr, Source}; - expected.as_ref() == input - && matches!(input, QueryExpr::Scan { source: Source::Table { .. }, schema, .. } - if schema.closed && schema.fields.get(*value_column).is_some_and(|c| c.dtype == DataType::Float64 && !c.nullable) - && !grouping.is_without() && grouping.keys().iter().all(|k| *k < schema.fields.len())) - } - } - } - - pub fn supports(&self, readout: &PopulationStatistic) -> bool { - match readout { - PopulationStatistic::Quantile { q } => self.quantiles && q.is_finite(), - PopulationStatistic::TopK { k } => *k <= self.max_k, - PopulationStatistic::Sum - | PopulationStatistic::Count - | PopulationStatistic::Average => true, - } - } -} - impl CurrentSeriesInput { /// Verify the named contract against the canonical maintenance input. pub fn matches_node(&self, input: &crate::ir::OperatorNode) -> bool { diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index b1a7da67a..04e3264c2 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -1,43 +1,33 @@ -//! The post-ASAP IR: summary-bound types, distinct from -//! [`crate::pre_asap`]'s pre-ASAP IR. +//! Summary-state types and the execution-timing vocabulary of the operator +//! IR ([`crate::ir`]). //! -//! Where [`crate::pre_asap`] carries *intent* only ("compute a -//! quantile to ε accuracy"), this module is the summary-bound IR: the -//! summary family, kind/algorithm, and parameters are committed (one -//! `(Kind, Params)` pair per family — [`sketch::ExactKind`]/[`sketch::ExactParams`], +//! Where an intent ([`crate::pre_asap::AggIntent`]) says *what* to compute +//! ("a quantile to ε accuracy"), these types say *how* a summary realizes it: +//! the family, kind/algorithm and parameters are committed — one +//! `(Kind, Params)` pair per family ([`sketch::ExactKind`]/[`sketch::ExactParams`], //! [`sketch::SamplingKind`]/[`sketch::SamplingParams`], //! [`sketch::WaveletKind`]/[`sketch::WaveletParams`], -//! [`sketch::StatModelKind`]/[`sketch::StatModelParams`]), and -//! [`expr::SummaryNode`] / [`expr::SummaryExpr`] describe the summary -//! computation. The `Sketch` family is the one exception to that -//! one-pair-per-family shape: it nests a third level, [`sketch::SketchKind`] -//! (quantile/cardinality/frequency/top-k), which itself carries the -//! committed [`sketch::SketchAlgorithm`] and [`sketch::SketchParams`] — -//! `FieldDataType::Sketch(SketchKind, GroupingStrategy)`, not a flat -//! `(kind, params)` pair -//! — because `Sketch` is the one family with more than one algorithm per -//! purpose today; no other family needs that extra level yet. +//! [`sketch::StatModelKind`]/[`sketch::StatModelParams`]). The `Sketch` family +//! nests a third level, [`sketch::SketchKind`] (quantile/cardinality/ +//! frequency/top-k), carrying the committed [`sketch::SketchAlgorithm`] and +//! [`sketch::SketchParams`], because it is the one family with more than one +//! algorithm per purpose. //! -//! A second, orthogonal axis lives here too: [`sketch::GroupingStrategy`] -//! (issue #256) — *how many* physical instances of a chosen family/kind -//! exist across a grouped aggregate's `by` subpopulations -//! (`PerSubpopulationInstance`, today's only behavior, vs. -//! `SharedMultiSubpopulation`/Hydra — see [`sketch::HydraKind`]/ -//! [`sketch::HydraParams`]), carried on [`expr::SummaryExpr::SummaryAgg`] -//! alongside `reduction` and on sketch-valued edge types -//! — see `asap_aware_mapping::grouping`'s module docs for why. +//! [`sketch::GroupingStrategy`] is a second, orthogonal axis: how many +//! physical instances of a summary exist across a grouped aggregate's `by` +//! subpopulations (per-subpopulation vs. one shared Hydra instance — see +//! `asap_aware_mapping::grouping`). It rides on `ASAPOp::SummaryAgg` and on +//! sketch-valued edge types. +//! +//! The rest: accuracy guarantees ([`guarantee`]), maintained populations, +//! summary-window panes, and the execution timing / +//! data-state vocabulary ([`execution_data_state`]). -// Legacy summary IR: no longer re-exported; removed by the cleanup PR. -pub mod cse; pub mod execution_data_state; -pub mod expr; pub mod guarantee; pub mod maintained_population; -pub mod post_asap_dag; pub mod query_time; pub mod sketch; -pub mod summary_maintenance; -pub mod summary_maintenance_lifecycle; pub mod summary_window; pub use crate::pre_asap::schema::{Field, FieldDataType, Schema}; @@ -45,31 +35,10 @@ pub use execution_data_state::{ lift_plain, DataPrimitive, ExactOperationSchemaError, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, }; -// Legacy summary IR names, kept for the legacy modules above only. -#[allow(unused_imports)] -pub(crate) use cse::share_common_summary_sub_dags; -#[allow(unused_imports)] -pub(crate) use execution_data_state::{ - assigned_child_data_state, exact_operation_output_schema, produced_data_state, - validate_execution_data_states, validate_execution_data_states_at, - ExecutionDataStateAssignment, -}; -#[allow(unused_imports)] -pub(crate) use expr::{ - BinaryOperator, CandidateCompleteness, ExactOperation, SummaryExpr, SummaryNode, ValueOperation, -}; pub use guarantee::{ AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, ResultGuarantee, }; -#[allow(unused_imports)] -pub(crate) use post_asap_dag::{ - compile_post_asap_dag, compile_post_asap_dag_with_node_ids, EdgeRole, - GroupingEdgeCompatibility, PostAsapDAG, PostAsapDAGCompilation, PostAsapDAGDocument, - PostAsapDAGEdge, PostAsapDAGNode, PostAsapDAGValidationError, PostAsapNodeId, - PostAsapNodeIdentityMap, PostAsapOperatorPayload, WindowEdgeCompatibility, - POST_ASAP_DAG_WIRE_VERSION, -}; pub use query_time::{ classic_cms_sizing, cms_posterior_error_bound, count_sketch_posterior_error_bound, cu_sketch_posterior_error_bound, traditional_a_priori_bound, @@ -80,12 +49,6 @@ pub use sketch::{ SketchCategory, SketchKind, SketchParams, SketchStatistic, StatModelKind, StatModelParams, SummaryInputExpr, SummaryUpdate, WaveletKind, WaveletParams, WeightDomain, }; -pub use summary_maintenance::SummaryMaintenanceMode; -pub use summary_maintenance_lifecycle::{ - EvaluationSchedule, OutputRepresentation, SummaryMaintenanceLifecycle, - SummaryMaintenanceLifecycleGuarantee, -}; pub use summary_window::{ - plan_pane_phase, validate_pane_coverage, PaneCoverageError, PaneLayout, SummaryWindowFramework, - WindowEdgeCoverage, + plan_pane_phase, validate_pane_coverage, PaneCoverageError, PaneLayout, WindowEdgeCoverage, }; diff --git a/crates/types/src/post_asap/post_asap_dag.rs b/crates/types/src/post_asap/post_asap_dag.rs deleted file mode 100644 index d489c055e..000000000 --- a/crates/types/src/post_asap/post_asap_dag.rs +++ /dev/null @@ -1,867 +0,0 @@ -//! Runtime-neutral post-ASAP DAG contract shared by precompute and query engines. - -use std::collections::HashMap; -use std::rc::Rc; - -use super::{ - validate_execution_data_states, ExecutionDataState, ExecutionDataStateError, ResultGuarantee, - Schema, SummaryExpr, SummaryNode, -}; -use super::{ - BinaryOperator, CandidateCompleteness, ExecutionTiming, FieldDataType, GroupingStrategy, - SketchStatistic, SummaryUpdate, ValueOperation, -}; -use crate::pre_asap::{ColumnRef, JoinKind, Predicate, QueryExpr, Reduction}; -use thiserror::Error; - -pub const POST_ASAP_DAG_WIRE_VERSION: u32 = 6; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub enum EdgeRole { - Input, - Left, - Right, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub enum GroupingEdgeCompatibility { - Identical, - ConsumerCoarsensProducer, - Incompatible, - NotApplicable, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub enum WindowEdgeCompatibility { - /// Physical lowering must prove equal pane/query phase or install an - /// exact boundary residual. The logical DAG alone cannot make that claim. - #[serde(rename = "RequiresAlignedPanePhaseOrExactBoundaryResidual")] - RequiresAlignedPanePhaseOrExactWindowEdgeResidual, - NotApplicable, -} - -/// Stable identity of a node within one exported post-ASAP semantic DAG. -#[derive( - Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, serde::Deserialize, -)] -#[serde(transparent)] -pub struct PostAsapNodeId(pub u32); - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum PostAsapOperatorPayload { - Fallback { - expression: QueryExpr, - }, - Binary { - operator: BinaryOperator, - }, - Value { - operation: ValueOperation, - }, - RelationalJoin { - join_kind: JoinKind, - pred: Predicate, - pruning: Option, - }, - SummaryAgg { - family: FieldDataType, - input: SummaryUpdate, - reduction: Reduction, - grouping: GroupingStrategy, - /// See `SummaryExpr::SummaryAgg::filter`. Wire version 6 added it; - /// a version-5 reader would otherwise take a filtered summary as - /// unfiltered. - filter: Option, - }, - SummaryJoin { - key: ColumnRef, - family: FieldDataType, - }, - SummarySubtract, - SummaryDelete { - key: ColumnRef, - }, - SummaryEstimate { - query: SketchStatistic, - }, - SummaryMerge, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PostAsapDAGNode { - pub id: PostAsapNodeId, - /// The payload variant is the sole operator identity (`payload.kind` in JSON). - pub payload: PostAsapOperatorPayload, - /// Phase is a placement choice for every operator, independent of payload kind. - pub output_state: ExecutionDataState, - pub output_schema: Schema, - pub guarantee: Option, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PostAsapDAGEdge { - pub producer: PostAsapNodeId, - pub consumer: PostAsapNodeId, - pub role: EdgeRole, - pub intermediate_schema: Schema, - pub data_state: ExecutionDataState, - pub grouping: GroupingEdgeCompatibility, - pub window: WindowEdgeCompatibility, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PostAsapDAG { - pub nodes: Vec, - pub edges: Vec, - /// Semantic workload root. Physical query/precompute sinks are selected - /// downstream by the control plane. - pub root: PostAsapNodeId, -} - -/// Versioned transport envelope for a post-ASAP semantic DAG. -/// -/// Process boundaries exchange this envelope and call [`Self::validate`]. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PostAsapDAGDocument { - pub schema_version: u32, - pub dag: PostAsapDAG, -} - -#[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum PostAsapDAGValidationError { - #[error("phase assignment must name every DAG node exactly once")] - IncompletePhaseAssignment, - #[error("ingestion node {consumer:?} depends on query node {producer:?}")] - QueryDependencyInIngestion { - producer: PostAsapNodeId, - consumer: PostAsapNodeId, - }, - #[error("unsupported post-ASAP DAG schema version {0}")] - UnsupportedVersion(u32), - #[error("duplicate post-ASAP node id {0:?}")] - DuplicateNodeId(PostAsapNodeId), - #[error("post-ASAP DAG root {0:?} does not name a node")] - MissingRoot(PostAsapNodeId), - #[error("edge endpoint {0:?} does not name a node")] - MissingEdgeEndpoint(PostAsapNodeId), - #[error("edge {producer:?}->{consumer:?} schema differs from producer output")] - EdgeSchemaMismatch { - producer: PostAsapNodeId, - consumer: PostAsapNodeId, - }, - #[error("edge {producer:?}->{consumer:?} data state differs from producer output")] - EdgeDataStateMismatch { - producer: PostAsapNodeId, - consumer: PostAsapNodeId, - }, - #[error("post-ASAP DAG contains a cycle")] - Cycle, - #[error("post-ASAP node {0:?} is not reachable from the root")] - UnreachableNode(PostAsapNodeId), - #[error("summary aggregate node {node:?} output schema does not contain its declared family")] - SummaryFamilySchemaMismatch { node: PostAsapNodeId }, - #[error( - "summary aggregate node {node:?} declares grouping inconsistent with its sketch state" - )] - SummaryGroupingMismatch { node: PostAsapNodeId }, -} - -impl PostAsapDAGDocument { - pub fn new(dag: PostAsapDAG) -> Self { - Self { - schema_version: POST_ASAP_DAG_WIRE_VERSION, - dag, - } - } - - pub fn validate(&self) -> Result<(), PostAsapDAGValidationError> { - if self.schema_version != POST_ASAP_DAG_WIRE_VERSION { - return Err(PostAsapDAGValidationError::UnsupportedVersion( - self.schema_version, - )); - } - self.dag.validate() - } -} - -impl PostAsapDAG { - /// Assign execution phases without changing operator semantics. Phase choices - /// do not prove deployment support: callers must bind concrete implementations - /// and storage boundaries before installing this plan. - pub fn with_execution_phases( - &self, - phases: &std::collections::BTreeMap, - ) -> Result { - self.validate()?; - if phases.len() != self.nodes.len() - || self.nodes.iter().any(|node| !phases.contains_key(&node.id)) - { - return Err(PostAsapDAGValidationError::IncompletePhaseAssignment); - } - let mut dag = self.clone(); - for node in &mut dag.nodes { - node.output_state.timing = phases[&node.id]; - } - let states: HashMap<_, _> = dag.nodes.iter().map(|n| (n.id, n.output_state)).collect(); - for edge in &mut dag.edges { - edge.data_state = states[&edge.producer]; - } - dag.validate()?; - Ok(dag) - } - - pub fn validate(&self) -> Result<(), PostAsapDAGValidationError> { - use std::collections::{HashMap, HashSet}; - let mut nodes = HashMap::new(); - for node in &self.nodes { - if nodes.insert(node.id, node).is_some() { - return Err(PostAsapDAGValidationError::DuplicateNodeId(node.id)); - } - if let PostAsapOperatorPayload::SummaryAgg { - family, grouping, .. - } = &node.payload - { - let mut found_family = false; - for field in &node.output_schema.fields { - if &field.dtype == family { - found_family = true; - } - if let FieldDataType::Sketch(_, schema_grouping) = &field.dtype { - if schema_grouping != grouping { - return Err(PostAsapDAGValidationError::SummaryGroupingMismatch { - node: node.id, - }); - } - } - } - if !found_family { - return Err(PostAsapDAGValidationError::SummaryFamilySchemaMismatch { - node: node.id, - }); - } - } - } - if !nodes.contains_key(&self.root) { - return Err(PostAsapDAGValidationError::MissingRoot(self.root)); - } - let mut children: HashMap> = HashMap::new(); - for edge in &self.edges { - let producer = nodes.get(&edge.producer).ok_or( - PostAsapDAGValidationError::MissingEdgeEndpoint(edge.producer), - )?; - if !nodes.contains_key(&edge.consumer) { - return Err(PostAsapDAGValidationError::MissingEdgeEndpoint( - edge.consumer, - )); - } - if producer.output_state.timing == ExecutionTiming::QueryTime - && nodes[&edge.consumer].output_state.timing == ExecutionTiming::IngestionTime - { - return Err(PostAsapDAGValidationError::QueryDependencyInIngestion { - producer: edge.producer, - consumer: edge.consumer, - }); - } - if edge.intermediate_schema != producer.output_schema { - return Err(PostAsapDAGValidationError::EdgeSchemaMismatch { - producer: edge.producer, - consumer: edge.consumer, - }); - } - if edge.data_state != producer.output_state { - return Err(PostAsapDAGValidationError::EdgeDataStateMismatch { - producer: edge.producer, - consumer: edge.consumer, - }); - } - children - .entry(edge.consumer) - .or_default() - .push(edge.producer); - } - fn visit( - id: PostAsapNodeId, - children: &HashMap>, - visiting: &mut HashSet, - visited: &mut HashSet, - ) -> bool { - if visited.contains(&id) { - return true; - } - if !visiting.insert(id) { - return false; - } - if children - .get(&id) - .into_iter() - .flatten() - .any(|child| !visit(*child, children, visiting, visited)) - { - return false; - } - visiting.remove(&id); - visited.insert(id); - true - } - if !visit( - self.root, - &children, - &mut HashSet::new(), - &mut HashSet::new(), - ) { - return Err(PostAsapDAGValidationError::Cycle); - } - let mut reachable = HashSet::new(); - fn mark( - id: PostAsapNodeId, - children: &HashMap>, - reachable: &mut HashSet, - ) { - if !reachable.insert(id) { - return; - } - for child in children.get(&id).into_iter().flatten() { - mark(*child, children, reachable); - } - } - mark(self.root, &children, &mut reachable); - if let Some(id) = nodes.keys().find(|id| !reachable.contains(id)) { - return Err(PostAsapDAGValidationError::UnreachableNode(*id)); - } - Ok(()) - } -} - -/// Compiler-local identity assignment. It deliberately retains `Rc` handles -/// and is not serialized; deployed artifacts persist the post-ASAP node ID -/// together with their physical materialization/query IDs. -#[derive(Debug, Clone)] -pub struct PostAsapNodeIdentityMap { - nodes_by_id: Vec>, -} - -impl PostAsapNodeIdentityMap { - pub fn node_id(&self, node: &Rc) -> Option { - self.nodes_by_id - .iter() - .position(|candidate| Rc::ptr_eq(candidate, node)) - .map(|id| PostAsapNodeId(id as u32)) - } - - pub fn summary_node(&self, id: PostAsapNodeId) -> Option<&Rc> { - self.nodes_by_id.get(id.0 as usize) - } -} - -#[derive(Debug, Clone)] -pub struct PostAsapDAGCompilation { - pub dag: PostAsapDAG, - pub node_ids: PostAsapNodeIdentityMap, -} - -pub fn compile_post_asap_dag( - root: &Rc, -) -> Result { - Ok(compile_post_asap_dag_with_node_ids(root)?.dag) -} - -pub fn compile_post_asap_dag_with_node_ids( - root: &Rc, -) -> Result { - let assignment = validate_execution_data_states(root)?; - let mut nodes = Vec::new(); - let mut edges = Vec::new(); - let mut ids = HashMap::new(); - let mut nodes_by_id = Vec::new(); - - fn visit( - node: &Rc, - assignment: &super::ExecutionDataStateAssignment, - ids: &mut HashMap<*const SummaryNode, PostAsapNodeId>, - nodes: &mut Vec, - edges: &mut Vec, - nodes_by_id: &mut Vec>, - ) -> PostAsapNodeId { - if let Some(id) = ids.get(&Rc::as_ptr(node)) { - return *id; - } - let children: Vec<(&Rc, EdgeRole)> = match &node.expr { - SummaryExpr::KeepPreAsap(_) => vec![], - SummaryExpr::BinaryOp { lhs, rhs, .. } => { - vec![(lhs, EdgeRole::Left), (rhs, EdgeRole::Right)] - } - - SummaryExpr::ValueOperation { child, .. } | SummaryExpr::SummaryAgg { child, .. } => { - vec![(child, EdgeRole::Input)] - } - SummaryExpr::RelationalJoin { left, right, .. } => { - vec![(left, EdgeRole::Left), (right, EdgeRole::Right)] - } - SummaryExpr::SummaryJoin { outer, inner, .. } => { - vec![(outer, EdgeRole::Left), (inner, EdgeRole::Right)] - } - SummaryExpr::SummarySubtract { left, right } => { - vec![(left, EdgeRole::Left), (right, EdgeRole::Right)] - } - SummaryExpr::SummaryDelete { summary_input, .. } - | SummaryExpr::SummaryEstimate { summary_input, .. } => { - vec![(summary_input, EdgeRole::Input)] - } - SummaryExpr::SummaryMerge { children, .. } => { - children.iter().map(|c| (c, EdgeRole::Input)).collect() - } - }; - let child_ids: Vec<_> = children - .iter() - .map(|(c, r)| (visit(c, assignment, ids, nodes, edges, nodes_by_id), *c, *r)) - .collect(); - let id = PostAsapNodeId(nodes.len() as u32); - let state = assignment - .data_state_of(node) - .expect("validated node has state"); - let payload = match &node.expr { - SummaryExpr::KeepPreAsap(expression) => PostAsapOperatorPayload::Fallback { - expression: (**expression).clone(), - }, - SummaryExpr::BinaryOp { operator, .. } => PostAsapOperatorPayload::Binary { - operator: operator.clone(), - }, - - SummaryExpr::ValueOperation { operation, .. } => PostAsapOperatorPayload::Value { - operation: operation.clone(), - }, - SummaryExpr::RelationalJoin { - kind, - pred, - pruning, - .. - } => PostAsapOperatorPayload::RelationalJoin { - join_kind: kind.clone(), - pred: pred.clone(), - pruning: pruning.clone(), - }, - SummaryExpr::SummaryAgg { - family, - input, - reduction, - grouping, - filter, - .. - } => PostAsapOperatorPayload::SummaryAgg { - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }, - SummaryExpr::SummaryJoin { key, family, .. } => PostAsapOperatorPayload::SummaryJoin { - key: key.clone(), - family: family.clone(), - }, - SummaryExpr::SummarySubtract { .. } => PostAsapOperatorPayload::SummarySubtract, - SummaryExpr::SummaryDelete { key, .. } => { - PostAsapOperatorPayload::SummaryDelete { key: key.clone() } - } - SummaryExpr::SummaryEstimate { query, .. } => { - PostAsapOperatorPayload::SummaryEstimate { - query: query.clone(), - } - } - SummaryExpr::SummaryMerge { .. } => PostAsapOperatorPayload::SummaryMerge, - }; - nodes.push(PostAsapDAGNode { - id, - payload, - output_state: state, - output_schema: node.schema.clone(), - guarantee: node.guarantee.clone(), - }); - nodes_by_id.push(Rc::clone(node)); - ids.insert(Rc::as_ptr(node), id); - for (producer, child, role) in child_ids { - let maintenance_dependency = nodes[producer.0 as usize].output_state.timing - == ExecutionTiming::IngestionTime - && nodes[id.0 as usize].output_state.timing == ExecutionTiming::IngestionTime; - let grouping = match (&child.expr, &node.expr) { - ( - SummaryExpr::SummaryAgg { - reduction: producer, - .. - }, - SummaryExpr::SummaryAgg { - reduction: consumer, - .. - }, - ) if producer == consumer => GroupingEdgeCompatibility::Identical, - ( - SummaryExpr::SummaryAgg { - reduction: crate::pre_asap::Reduction::PerEntity, - .. - }, - SummaryExpr::SummaryAgg { - reduction: crate::pre_asap::Reduction::Reduce(_), - .. - }, - ) => GroupingEdgeCompatibility::ConsumerCoarsensProducer, - ( - SummaryExpr::SummaryAgg { - reduction: crate::pre_asap::Reduction::Reduce(producer), - .. - }, - SummaryExpr::SummaryAgg { - reduction: crate::pre_asap::Reduction::Reduce(consumer), - .. - }, - ) if !producer.is_without() - && !consumer.is_without() - && consumer.iter().all(|key| producer.contains(key)) => - { - GroupingEdgeCompatibility::ConsumerCoarsensProducer - } - (SummaryExpr::SummaryAgg { .. }, SummaryExpr::SummaryAgg { .. }) => { - GroupingEdgeCompatibility::Incompatible - } - _ => GroupingEdgeCompatibility::NotApplicable, - }; - edges.push(PostAsapDAGEdge { - producer, - consumer: id, - role, - intermediate_schema: child.schema.clone(), - // The whole-DAG validator owns contextual state assignment, - // especially for shared KeepPreAsap leaves. Export that - // authoritative result instead of independently deriving the - // edge state a second time. - data_state: assignment - .data_state_of(child) - .expect("validated child has data state"), - grouping, - window: if maintenance_dependency { - WindowEdgeCompatibility::RequiresAlignedPanePhaseOrExactWindowEdgeResidual - } else { - WindowEdgeCompatibility::NotApplicable - }, - }); - } - id - } - - let root = visit( - root, - &assignment, - &mut ids, - &mut nodes, - &mut edges, - &mut nodes_by_id, - ); - let dag = PostAsapDAG { nodes, edges, root }; - dag.validate() - .expect("compiler emits a valid post-ASAP DAG"); - Ok(PostAsapDAGCompilation { - dag, - node_ids: PostAsapNodeIdentityMap { nodes_by_id }, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::post_asap::{ - ExactKind, ExactParams, ExecutionTiming, FieldDataType, GroupingStrategy, SummaryUpdate, - ValueOperation, - }; - use crate::pre_asap::schema::{Field, Schema}; - use crate::pre_asap::{ColumnRef, DataType, QueryExpr, Reduction, Source}; - use std::collections::BTreeMap; - - #[test] - fn every_physical_payload_can_be_assigned_either_phase() { - use crate::post_asap::DataPrimitive; - use crate::pre_asap::{ArithmeticOpKind, BinaryOpKind, JoinKind, Predicate, ScalarValue}; - let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let predicate = Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(true)))); - let payloads = vec![ - PostAsapOperatorPayload::Fallback { - expression: QueryExpr::Literal(ScalarValue::Int64(1)), - }, - PostAsapOperatorPayload::Binary { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - }, - }, - PostAsapOperatorPayload::Value { - operation: ValueOperation::Limit { - n: 1, - offset: 0, - partition_by: Default::default(), - }, - }, - PostAsapOperatorPayload::RelationalJoin { - join_kind: JoinKind::Semi, - pred: predicate, - pruning: None, - }, - PostAsapOperatorPayload::SummaryAgg { - family: family.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }, - PostAsapOperatorPayload::SummaryJoin { - key: ColumnRef::SampleValue, - family: family.clone(), - }, - PostAsapOperatorPayload::SummarySubtract, - PostAsapOperatorPayload::SummaryDelete { - key: ColumnRef::SampleValue, - }, - PostAsapOperatorPayload::SummaryEstimate { - query: SketchStatistic::Cardinality, - }, - PostAsapOperatorPayload::SummaryMerge, - ]; - for payload in payloads { - // This checks physical identity and placement, not kernel availability. - let primitive = match &payload { - PostAsapOperatorPayload::Fallback { .. } - | PostAsapOperatorPayload::Binary { .. } - | PostAsapOperatorPayload::Value { .. } - | PostAsapOperatorPayload::RelationalJoin { .. } - | PostAsapOperatorPayload::SummaryEstimate { .. } => DataPrimitive::Raw, - PostAsapOperatorPayload::SummaryAgg { .. } - | PostAsapOperatorPayload::SummaryJoin { .. } - | PostAsapOperatorPayload::SummarySubtract - | PostAsapOperatorPayload::SummaryDelete { .. } - | PostAsapOperatorPayload::SummaryMerge => DataPrimitive::SummaryState, - }; - let dag = PostAsapDAG { - root: PostAsapNodeId(0), - edges: vec![], - nodes: vec![PostAsapDAGNode { - id: PostAsapNodeId(0), - payload: payload.clone(), - output_state: ExecutionDataState { - timing: ExecutionTiming::QueryTime, - primitive, - }, - output_schema: Schema::lifted( - vec![Field { - name: "value".into(), - dtype: family.clone(), - nullable: false, - table: None, - }], - None, - ), - guarantee: None, - }], - }; - for phase in [ExecutionTiming::IngestionTime, ExecutionTiming::QueryTime] { - let placed = dag - .with_execution_phases(&BTreeMap::from([(dag.root, phase)])) - .unwrap(); - assert_eq!(placed.nodes[0].payload, payload); - assert_eq!(placed.nodes[0].output_state.timing, phase); - let wire = serde_json::to_value(&placed).unwrap(); - assert!(wire["nodes"][0]["payload"].get("timing").is_none()); - assert_eq!(serde_json::from_value::(wire).unwrap(), placed); - } - assert!(dag.with_execution_phases(&BTreeMap::new()).is_err()); - } - } - - #[test] - fn phase_assignment_updates_edges_and_rejects_query_dependencies_in_ingestion() { - use crate::pre_asap::ScalarValue; - let schema = Schema::lifted(vec![], None); - let nodes = [0, 1] - .into_iter() - .map(|id| PostAsapDAGNode { - id: PostAsapNodeId(id), - payload: PostAsapOperatorPayload::Fallback { - expression: QueryExpr::Literal(ScalarValue::Int64(1)), - }, - output_state: ExecutionDataState::QUERY_ROWS, - output_schema: schema.clone(), - guarantee: None, - }) - .collect(); - let dag = PostAsapDAG { - nodes, - root: PostAsapNodeId(1), - edges: vec![PostAsapDAGEdge { - producer: PostAsapNodeId(0), - consumer: PostAsapNodeId(1), - role: EdgeRole::Input, - intermediate_schema: schema, - data_state: ExecutionDataState::QUERY_ROWS, - grouping: GroupingEdgeCompatibility::NotApplicable, - window: WindowEdgeCompatibility::NotApplicable, - }], - }; - let placed = dag - .with_execution_phases(&BTreeMap::from([ - (PostAsapNodeId(0), ExecutionTiming::IngestionTime), - (PostAsapNodeId(1), ExecutionTiming::QueryTime), - ])) - .unwrap(); - assert_eq!( - placed.edges[0].data_state.timing, - ExecutionTiming::IngestionTime - ); - assert_eq!(dag.edges[0].data_state.timing, ExecutionTiming::QueryTime); - assert!(matches!( - dag.with_execution_phases(&BTreeMap::from([ - (PostAsapNodeId(0), ExecutionTiming::QueryTime), - (PostAsapNodeId(1), ExecutionTiming::IngestionTime), - ])), - Err(PostAsapDAGValidationError::QueryDependencyInIngestion { .. }) - )); - } - - #[test] - fn exports_summary_over_summary_as_typed_precompute_edges() { - let scan = Rc::new(QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::new(vec![Field::plain("value", DataType::Float64, false)]), - }); - let raw = Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(scan), - schema: Schema::lifted( - vec![Field { - name: "value".into(), - dtype: FieldDataType::Plain(DataType::Float64), - nullable: false, - table: None, - }], - None, - ), - guarantee: None, - }); - let make_agg = |child: Rc, kind, params| { - let family = FieldDataType::ExactAggregate(kind, params); - Rc::new(SummaryNode { - expr: SummaryExpr::SummaryAgg { - child, - family: family.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }, - schema: Schema::lifted( - vec![Field { - name: "value".into(), - dtype: family, - nullable: false, - table: None, - }], - None, - ), - guarantee: None, - }) - }; - let inner = make_agg(raw, ExactKind::Sum, ExactParams::Sum); - let outer = make_agg(Rc::clone(&inner), ExactKind::Sum, ExactParams::Sum); - let root = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: outer, - operation: ValueOperation::FinalizeExactAccumulator, - timing: ExecutionTiming::QueryTime, - }, - schema: Schema::lifted( - vec![Field { - name: "value".into(), - dtype: FieldDataType::Plain(DataType::Float64), - nullable: false, - table: None, - }], - None, - ), - guarantee: None, - }); - - let compiled = compile_post_asap_dag_with_node_ids(&root).unwrap(); - assert_eq!(compiled.node_ids.node_id(&root), Some(PostAsapNodeId(3))); - assert!(Rc::ptr_eq( - compiled.node_ids.summary_node(PostAsapNodeId(1)).unwrap(), - &inner - )); - let dag = compiled.dag; - assert_eq!(dag.root, PostAsapNodeId(3)); - assert_eq!( - dag.nodes[1].output_state, - ExecutionDataState::INGESTION_SUMMARY - ); - assert_eq!( - dag.nodes[2].output_state, - ExecutionDataState::INGESTION_SUMMARY - ); - let dependency = dag - .edges - .iter() - .find(|e| e.producer == PostAsapNodeId(1) && e.consumer == PostAsapNodeId(2)) - .unwrap(); - assert_eq!(dependency.data_state, ExecutionDataState::INGESTION_SUMMARY); - assert_eq!(dependency.grouping, GroupingEdgeCompatibility::Identical); - assert_eq!( - dependency.window, - WindowEdgeCompatibility::RequiresAlignedPanePhaseOrExactWindowEdgeResidual - ); - assert!(matches!( - dependency.intermediate_schema.fields[0].dtype, - FieldDataType::ExactAggregate(ExactKind::Sum, _) - )); - let encoded = serde_json::to_string(&dag).expect("serialize post-ASAP DAG"); - let decoded: PostAsapDAG = - serde_json::from_str(&encoded).expect("deserialize post-ASAP DAG"); - assert_eq!(decoded, dag); - let document = PostAsapDAGDocument::new(decoded); - document.validate().unwrap(); - let mut invalid = serde_json::to_value(&document).unwrap(); - invalid["dag"]["nodes"][0]["operator"] = serde_json::json!("Binary"); - assert!(serde_json::from_value::(invalid).is_err()); - assert!(document.dag.nodes.iter().all(|node| { - let wire = serde_json::to_value(node).unwrap(); - wire.get("operator").is_none() && wire["payload"]["kind"].is_string() - })); - let mut old_version = document.clone(); - old_version.schema_version = 1; - assert_eq!( - old_version.validate(), - Err(PostAsapDAGValidationError::UnsupportedVersion(1)) - ); - let mut unknown = serde_json::to_value(&document).unwrap(); - unknown["unexpected"] = serde_json::json!(true); - assert!(serde_json::from_value::(unknown).is_err()); - assert!(matches!( - dag.nodes[2].payload, - PostAsapOperatorPayload::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - reduction: Reduction::Reduce(_), - .. - } - )); - } - - #[test] - fn post_asap_node_ids_serialize_in_deterministic_binding_order() { - let mut bindings = BTreeMap::new(); - bindings.insert(PostAsapNodeId(10), "materialization-10"); - bindings.insert(PostAsapNodeId(2), "query-2"); - assert_eq!( - serde_json::to_string(&bindings).unwrap(), - r#"{"2":"query-2","10":"materialization-10"}"# - ); - } -} diff --git a/crates/types/src/post_asap/summary_maintenance.rs b/crates/types/src/post_asap/summary_maintenance.rs deleted file mode 100644 index 4e7eaf205..000000000 --- a/crates/types/src/post_asap/summary_maintenance.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! Planner-level construction mode for a materialized summary. -//! -//! An ASAP [`crate::ir::OperatorNode`] is a logical summary expression and deliberately -//! does not carry this choice: the same candidate may be built directly for -//! one workload or maintained incrementally for another. Planner search -//! attaches the selected mode to its lifecycle guarantee; downstream physical -//! compilation chooses its concrete implementation. - -/// How a summary deployment obtains its state, independent of implementation. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SummaryMaintenanceMode { - /// Rebuild the summary from its complete input when the deployment needs - /// a value. No update stream is required. - DirectBuild, - /// Create the state once and apply input changes as they arrive. - Incremental, -} - -impl SummaryMaintenanceMode { - pub const fn as_str(self) -> &'static str { - match self { - Self::DirectBuild => "direct_build", - Self::Incremental => "incremental", - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn maintenance_modes_have_stable_export_names() { - assert_eq!(SummaryMaintenanceMode::DirectBuild.as_str(), "direct_build"); - assert_eq!(SummaryMaintenanceMode::Incremental.as_str(), "incremental"); - } -} diff --git a/crates/types/src/post_asap/summary_maintenance_lifecycle.rs b/crates/types/src/post_asap/summary_maintenance_lifecycle.rs deleted file mode 100644 index 798d861f6..000000000 --- a/crates/types/src/post_asap/summary_maintenance_lifecycle.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Planner-level summary-maintenance lifecycle vocabulary. -//! -//! A **summary-maintenance lifecycle** describes when one materialized summary -//! state is created, retained or shared, updated, and retired. It does not -//! describe the broader data lifecycle (collection, transport, and storage), -//! and it is not implied by a logical `SummaryAgg`. Physical planning compares -//! alternatives using the expected number and timing of reads, the source-data -//! arrival/update rate, state-operation costs, and runtime capabilities. - -use super::SummaryMaintenanceMode; -use crate::workload::{DurationMs, TimestampMs}; - -/// When an operator is evaluated. This is independent of whether it owns -/// state and how long that state is retained. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum EvaluationSchedule { - OneShot, - PerUpdate, - OnRead, -} - -/// The form in which this deployment exposes its result to its consumer. The -/// consumer is the next operator in the execution plan that reads the -/// summary's output; for example, `Estimate` is the consumer in -/// `SummaryAgg -> Estimate`. The exposed result is ordinary rows, reusable -/// summary state, or a finalized value. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum OutputRepresentation { - PlainRows, - SummaryState, - FinalizedValue, -} - -/// Abstract policy for when one materialized summary state is created, -/// retained or shared, updated as data arrives, and retired. -/// -/// This is not the lifecycle of the source data or query. Query recurrence -/// provides the expected number and timing of reads; data arrival provides the -/// expected state-update demand. The planner combines those quantities with -/// costs and runtime capabilities to compare these policies. -#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum SummaryMaintenanceLifecycle { - Ephemeral, - Prepared { - #[serde(rename = "activate_at_ms")] - activate_at: TimestampMs, - #[serde(rename = "retire_at_ms")] - retire_at: TimestampMs, - }, - Shared { - #[serde(rename = "retention_ms")] - retention: DurationMs, - }, - ContinuouslyMaintained, -} - -/// The lifecycle commitment emitted for one materialized summary deployment. -/// -/// This names the summary-maintenance promise explicitly so consumers do not -/// confuse it with guarantees about the broader data lifecycle. Accuracy is a -/// separate [`super::ResultGuarantee`]. -#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SummaryMaintenanceLifecycleGuarantee { - #[serde(rename = "lifecycle")] - pub summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, - #[serde(rename = "maintenance_mode")] - pub summary_maintenance_mode: SummaryMaintenanceMode, - pub evaluation_schedule: EvaluationSchedule, - pub output_representation: OutputRepresentation, -} diff --git a/crates/types/src/post_asap/summary_window.rs b/crates/types/src/post_asap/summary_window.rs index 6d45649b4..724a8d595 100644 --- a/crates/types/src/post_asap/summary_window.rs +++ b/crates/types/src/post_asap/summary_window.rs @@ -1,29 +1,13 @@ -//! Planner-level summary-window primitives. +//! Planner-level summary-window pane primitives. //! -//! These values identify the abstract window framework selected during -//! candidate search. They do not identify a runtime library, process, -//! placement, shard layout, storage backend, or deployment instance; those -//! choices belong to downstream physical compilation. +//! These values describe pane layout and window-edge coverage. They do not +//! identify a runtime library, process, placement, shard layout, storage +//! backend, or deployment instance; those choices belong to downstream +//! physical compilation. use crate::workload::RepeatedDemand; use serde::{Deserialize, Serialize}; -/// Abstract framework used to organize incrementally maintained summary -/// state over time. -/// -/// The built-in variants name semantics that the planner can compare across -/// implementations with defined planning and accuracy behavior. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SummaryWindowFramework { - /// Disjoint, fixed-width windows. - Tumbling, - /// Overlapping logical windows, commonly realized from reusable panes. - Sliding, - /// Hierarchical buckets with exponentially increasing coverage. - ExponentialHistogram, -} - /// Concrete pane phase recorded in a catalog layout or inventory snapshot. /// Milliseconds are canonical throughout the shared contract. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -122,30 +106,6 @@ pub fn plan_pane_phase( mod tests { use super::*; - #[test] - fn built_in_frameworks_round_trip() { - for framework in [ - SummaryWindowFramework::Tumbling, - SummaryWindowFramework::Sliding, - SummaryWindowFramework::ExponentialHistogram, - ] { - let encoded = serde_json::to_string(&framework).unwrap(); - assert_eq!( - serde_json::from_str::(&encoded).unwrap(), - framework - ); - } - } - - /// Opaque names cannot enter planning without defined window semantics. - #[test] - fn unimplemented_window_extensions_are_rejected() { - assert!(serde_json::from_value::( - serde_json::json!({"extension": "learned_window"}) - ) - .is_err()); - } - #[test] fn pane_only_evaluation_rejects_source_and_query_phase_mismatch() { let layout = PaneLayout { diff --git a/crates/types/src/pre_asap/agg_intent.rs b/crates/types/src/pre_asap/agg_intent.rs index c60dd55e3..1e926bd44 100644 --- a/crates/types/src/pre_asap/agg_intent.rs +++ b/crates/types/src/pre_asap/agg_intent.rs @@ -10,28 +10,28 @@ //! heavy-hitter sketch when approximate — is a post-ASAP cost-aware decision, //! not encoded here. The semantic distinction that *is* made at lowering is //! intent vs operator: a heavy-hitter aggregate becomes `TopK`, whereas a -//! generic `ORDER BY value LIMIT k` stays as the `QueryExpr::Sort + Limit` +//! generic `ORDER BY value LIMIT k` stays as the `NonASAPOp::Sort + Limit` //! operator pair. use serde::{Deserialize, Serialize}; -use crate::pre_asap::query_expr::DataModel; +use crate::ir::operator_properties::DataModel; use crate::pre_asap::schema::{ColumnId, DataType, Field, FieldDataType}; use crate::types::AccuracyTarget; /// "What to compute" — the vocabulary the planner pivots on. /// -/// Grouping for `TopK` rides on the enclosing `QueryExpr::Aggregate.by` +/// Grouping for `TopK` rides on the enclosing `NonASAPOp::Aggregate`'s `reduction` /// (positional `ColumnId`s), like every other aggregate; the intent itself /// carries only `k` + the accuracy target. /// /// The single-column reducers (`Sum` / `Min` / `Max` / `Avg` / `StdDev` / /// `Variance` / `Quantile`) carry `col: Option` — the input /// column they reduce, generic over the column-reference state the same way -/// [`QueryExpr`](super::query_expr::QueryExpr) is: positional `ColumnId` once -/// bound (the default, and every existing use of the bare `AggIntent` name), -/// or an unresolved name-based `ColumnRef` for a front end constructing this -/// intent directly, before the [`SchemaResolver`](super::schema_resolver::SchemaResolver) has run. +/// the rest of the vocabulary is: positional `ColumnId` once bound (the +/// default, and every existing use of the bare `AggIntent` name), or an +/// unresolved name-based `ColumnRef` for a front end constructing this +/// intent directly, before name resolution (`asap_frontend_common`) has run. /// `None` is the PromQL convention "the time-series sample value"; SQL /// `SUM(bytes), AVG(latency)` sets distinct `Some(_)`s so a multi-aggregate /// node binds each reducer to the right column, and `plan::bind` knows which @@ -128,7 +128,7 @@ pub enum AggIntent { // ── Time-series streaming derivatives ──────────────────────────────── // Counter-reset adjustment; not equivalent to Sum/Count over a window. - // The temporal range lives on the enclosing `QueryExpr::TimeRange` node, + // The temporal range lives on the enclosing `NonASAPOp::TimeRange` node, // not in the intent — this keeps the intent vocabulary range-agnostic. Rate, /// PromQL `irate(v[w])` — reset-aware rate from the final two samples. @@ -234,7 +234,7 @@ pub enum AggIntent { /// A time / calendar accessor (issue #46) — `timestamp`, `minute`, `hour`, /// `day_of_week`, … over each sample's timestamp (or, for the no-arg forms, /// over the evaluation time). Label-preserving per-series value transform. - /// (`time()` is the evaluation time itself — a `QueryExpr::EvalTimestamp` leaf, + /// (`time()` is the evaluation time itself — a `ScalarExpr::EvalTimestamp` leaf, /// not this.) TimeFn(TimeFunc), @@ -378,9 +378,8 @@ pub enum MathFunc { } // `requires` / `is_per_series` / `output_column` never read `col`'s value — -// only its presence via a `{ .. }` pattern — so, unlike -// `QueryExpr::output_schema` (which genuinely cannot compile for an -// unresolved DAG — see its own doc), nothing stops these from being generic +// only its presence via a `{ .. }` pattern — so, unlike schema derivation +// (which needs bound positions), nothing stops these from being generic // over every `C`. And a front end constructing `AggIntent` // directly (issue #179) does need `is_per_series` pre-binding — it decides // the `PerEntity`/`Reduce` reduction shape right at construction time (see @@ -529,7 +528,7 @@ impl AggIntent { impl AggIntent { /// Output column name + type produced by this intent over `input`. - /// Used by `QueryExpr::Aggregate`'s schema-derivation rule. The PromQL + /// Used by `NonASAPOp::Aggregate`'s schema-derivation rule. The PromQL /// convention names the column after the intent kind so consumers can /// locate it without an alias lookup. pub fn output_column(&self, input: &Field) -> Field { diff --git a/crates/types/src/pre_asap/canonicalize.rs b/crates/types/src/pre_asap/canonicalize.rs deleted file mode 100644 index b9e6a653a..000000000 --- a/crates/types/src/pre_asap/canonicalize.rs +++ /dev/null @@ -1,782 +0,0 @@ -//! Shared post-lowering canonicalization of the resolved [`QueryExpr`]. -//! -//! Both language front ends funnel through [`resolve_root`](super::resolve::resolve_root), -//! which runs this pass over the resolved DAG. Its job is to erase -//! *structural* differences between semantically identical queries so a -//! post-ASAP binding rule matching on the intent algebra sees one canonical -//! spelling regardless of source language (issue #34). -//! -//! ## Heavy-hitter promotion -//! -//! An additive-ranked "order by the aggregate, take the top k" is a -//! heavy-hitter represented by [`AggIntent::TopK`]. Front ends may -//! emit it as an ordinary `Limit { Sort { … Aggregate } }`; this pass promotes -//! that shape to the canonical -//! -//! ```text -//! Aggregate { reduction: Reduce(), measures: [TopK{k}], -//! child: Aggregate { measures: [Count | Sum], … } } -//! ``` -//! -//! Count supplies unit weights and Sum supplies value weights. Because the -//! match is positional, aliases do not affect it. Other ranked expressions -//! retain Sort + Limit. - -use std::rc::Rc; - -use super::agg_intent::{topk, AggIntent}; -use super::expr_ir::{CompareOpKind, ScalarValue}; -use super::query_expr::{Predicate, QueryExpr, Reduction, SortKey, WindowFuncKind}; -use crate::types::AccuracyTarget; - -/// Rewrite `expr` into its canonical form (bottom-up). Idempotent: a DAG that -/// is already canonical is returned unchanged. -pub fn canonicalize(mut expr: QueryExpr) -> QueryExpr { - canon(&mut expr); - expr -} - -fn canon(expr: &mut QueryExpr) { - // A `Concat` asserting a caller-proven `discriminator_unique_key` (issue - // #228) had that key's `ColumnId`s resolved, in `resolve.rs`, against - // exactly the first branch's output schema *as it stood before this - // pass ran*. `try_promote_additive_top_ranking`/`try_rewrite_rownumber_topk` - // below can restructure that branch (anywhere within it — not only at - // its own top level, since the same recursive walk can rewrite a node - // nested under a pass-through wrapper too) into a shape with a - // different output schema, which would leave those `ColumnId`s - // pointing at the wrong column, or out of bounds, of the - // post-canonicalize schema. Snapshot the schema the discriminator key - // was actually resolved against, right here, before recursing into the - // children — this is the exact DAG state `resolve.rs` saw. - let discriminator_branch_schema_before = match expr { - QueryExpr::Concat { - children, - discriminator_unique_key: Some(_), - } => children.first().and_then(|c| c.output_schema().ok()), - _ => None, - }; - - // Bottom-up: canonicalize every child before matching at this node, so an - // inner heavy-hitter is promoted before an enclosing rewrite inspects it. - for child in children_mut(expr) { - canon(child); - } - - // If the first branch's output schema moved out from under the asserted - // key, the key can no longer be trusted — drop it (never re-derive it by - // guessing at name/position: the two rewrites above don't preserve - // column identity in a way that's safe to infer). A wrong `unique_keys` - // claim is a wrong query answer, not a missed optimization — see - // `ConcatDiscriminatorKey`'s soundness doc — so this errs conservatively: - // any difference at all (not just a column-count/type change) drops the - // key, including the schema becoming undecidable in either direction. - if let QueryExpr::Concat { - children, - discriminator_unique_key: key @ Some(_), - } = expr - { - let discriminator_branch_schema_after = - children.first().and_then(|c| c.output_schema().ok()); - if discriminator_branch_schema_before != discriminator_branch_schema_after { - *key = None; - } - } - - // Local rewrites chain: a `ROW_NUMBER()`-partitioned top-k rewrites to a - // `Limit{Sort}`, which the heavy-hitter rule may then promote to an - // `Aggregate([TopK])`. Each rule strictly simplifies the node, so applying - // them to a fixpoint terminates. - while let Some(rewritten) = - try_rewrite_rownumber_topk(expr).or_else(|| try_promote_additive_top_ranking(expr)) - { - *expr = rewritten; - } -} - -/// A `&mut QueryExpr` out of a child `Rc` — clone-on-write via -/// [`Rc::make_mut`]: free (no clone) while `r` is uniquely owned, which is -/// the overwhelmingly common case (a DAG `canonicalize` was just handed by -/// value); falls back to cloning just *this* node (its own fields — the -/// grandchildren stay shared `Rc`s, not deep-copied) only when some other -/// owner still holds the same `Rc`, e.g. a caller that kept its own clone -/// around (`once.clone()` in `is_idempotent` below — `QueryExpr::clone()` is -/// now a cheap `Rc`-bump, not a deep copy, so that clone shares structure -/// with `once` until a rewrite here needs to touch it). `Rc::get_mut` would -/// panic on exactly that case; `make_mut` degrades to a shallow copy instead -/// of requiring sole ownership as a precondition. Once a workload-level CSE -/// pass runs (issue #212, #222) and canonicalize sees an already-shared -/// sub-DAG from a *different* query, this is also the mechanism that keeps -/// canonicalizing one query from silently corrupting another's view of the -/// same shared node. -fn rc_mut(r: &mut Rc) -> &mut QueryExpr { - Rc::make_mut(r) -} - -/// Mutable references to the direct **operator** `QueryExpr` children of a -/// node — `canon`'s own top-down/bottom-up walk only ever visits the -/// relational skeleton, never descending into a scalar position (`Filter.pred`, -/// `ProjectItem.expr`, …): none of the three rewrite rules rewrite anything -/// inside a scalar sub-DAG, so there's nothing to gain by recursing into one, -/// and every scalar variant (issue #205) hits the catch-all below. -fn children_mut(expr: &mut QueryExpr) -> Vec<&mut QueryExpr> { - use QueryExpr::*; - match expr { - // `PromqlScalarBridge`'s child is a scalar-sub-language node (issue - // #220), not the relational skeleton — same "no children to recurse - // into" treatment as the scalar variants below. - Scan { .. } | EvalTimestamp | CurrentTimestamp | PromqlScalarBridge(_) => vec![], - PromqlVectorFromScalar(c) | PromqlScalarFromVector(c) => vec![rc_mut(c)], - PromqlRelabel { child, .. } - | Filter { child, .. } - | Project { child, .. } - | Aggregate { child, .. } - | Dedup { child, .. } - | PromqlSubquery { child, .. } - | TimeRange { child, .. } - | TimeShift { child, .. } - | SQLWindowFunc { child, .. } - | PromqlSeriesSample { child, .. } - | PromqlInfoEnrich { child, .. } - | Sort { child, .. } - | Limit { child, .. } => vec![rc_mut(child)], - Concat { children, .. } => children.iter_mut().collect(), - Join { left, right, .. } | SetOp { left, right, .. } => { - vec![rc_mut(left), rc_mut(right)] - } - BinaryOp { lhs, rhs, .. } => vec![rc_mut(lhs), rc_mut(rhs)], - Column(_) - | Literal(_) - | Compare { .. } - | BoolAnd(_) - | BoolOr(_) - | Not(_) - | IsNull(_) - | IsNotNull(_) - | Cast { .. } - | InList { .. } - | FunctionCall { .. } - | Arithmetic { .. } - | Case { .. } => vec![], - } -} - -/// Recognise an additive-ranked -/// `Limit { Sort { [Project] Aggregate([Count | Sum]) } }` and rewrite it to -/// the canonical heavy-hitter `Aggregate([TopK])` over the explicit inner -/// aggregate. Returns `None` when the shape does not match. -fn try_promote_additive_top_ranking(expr: &QueryExpr) -> Option { - // Limit k, no offset (an OFFSET means "not the top k"). - let QueryExpr::Limit { - n: k, - offset: 0, - child, - } = expr - else { - return None; - }; - // A single ordering key on a column. - let QueryExpr::Sort { - keys, - partition_by, - child: sort_child, - } = child.as_ref() - else { - return None; - }; - let [SortKey { - expr: QueryExpr::Column(sort_col), - ascending, - .. - }] = keys.as_slice() - else { - return None; - }; - - // The ordered relation is an `Aggregate`, optionally behind a passthrough - // projection (a bare-column SELECT list). Map the sort key through the - // projection to the aggregate's own output column. - let (agg_expr, ranked_col) = match sort_child.as_ref() { - QueryExpr::Project { cols, child, .. } => { - let QueryExpr::Column(underlying) = &cols.get(*sort_col)?.expr else { - return None; - }; - (child.as_ref(), *underlying) - } - other => (other, *sort_col), - }; - - // Exactly one aggregate, ranked by *its* output column — the measure sits at - // index `by.len()` (after the group keys). A `PerEntity` reduction has no - // `by` to rank a measure against — this shape can't be heavy-hitter - // promoted, so it's a non-match rather than an error. - let QueryExpr::Aggregate { - reduction, - measures, - filters, - child: aggregate_child, - .. - } = agg_expr - else { - return None; - }; - let Reduction::Reduce(by) = reduction else { - return None; - }; - let [ranked_agg] = measures.as_slice() else { - return None; - }; - // A heavy-hitter sketch ranks the raw update stream; a filtered measure - // only counts part of it, and no binding rule applies the filter (#466). - if filters.iter().any(Option::is_some) { - return None; - } - if ranked_col != by.len() { - return None; - } - // The heavy-hitter decision — descending, over a measure with a realised - // heavy-hitter sketch — is the shared rule both front ends' promotions - // consult (issue #38). So an ascending additive-ranked limit - // (`ORDER BY COUNT(*) ASC LIMIT k` = bottom-k) stays generic, exactly as - // PromQL `bottomk(k, count_over_time(…))` does. - if !topk::Ranking::from_aggregate(ranked_agg).is_supported(!ascending) { - return None; - } - // A direct Sum is a stream of additive observation weights. A Sum over a - // derived child such as Rate/Increase is different: a heavy-hitter sketch - // may propose candidate membership, but PromQL still requires exact - // reset-aware/extrapolated values to rerank those candidates. The current - // post-ASAP IR has no candidate-sidecar + exact-rerank node, so keep that - // shape as Sort + Limit instead of treating a sketch estimate as final. - if matches!(ranked_agg, AggIntent::Sum { .. }) - && matches!(aggregate_child.as_ref(), QueryExpr::Aggregate { .. }) - { - return None; - } - // Count ranks unit updates; a direct Sum ranks weighted updates. - let accuracy = match ranked_agg { - AggIntent::Count { accuracy } => accuracy.clone(), - AggIntent::Sum { .. } => AccuracyTarget::Exact, - _ => unreachable!("additive ranking gate admitted a non-additive measure"), - }; - - // Outer heavy-hitter `TopK`, grouped by the ranking's partition (empty for a - // global `ORDER BY … LIMIT k`; the `by` labels for a partitioned `topk by`), - // over the unchanged inner additive aggregate. - Some(QueryExpr::Aggregate { - reduction: Reduction::by(partition_by.to_vec()), - measures: vec![AggIntent::TopK { k: *k, accuracy }], - output_names: Vec::new(), - filters: Vec::new(), - having: None, - child: Rc::new(agg_expr.clone()), - }) -} - -/// Recognise the SQL partitioned-top-k idiom — `WHERE rn <= k` over a -/// `ROW_NUMBER() OVER (PARTITION BY p ORDER BY o)` — and rewrite it to the -/// generic partitioned top-k `Limit{k} { Sort{ o, partition_by: p } }` (issue -/// #24). The count-ranked case is then promoted to a heavy-hitter `TopK` by -/// [`try_promote_additive_top_ranking`], so a SQL `ROW_NUMBER` top-k and the PromQL -/// `topk by (…)` it mirrors converge on the same canonical shape. -fn try_rewrite_rownumber_topk(expr: &QueryExpr) -> Option { - // Filter { pred: `Column(rn) <= k` }. - let QueryExpr::Filter { pred, child } = expr else { - return None; - }; - let Predicate(pred_expr) = pred; - let QueryExpr::Compare { left, op, right } = pred_expr.as_ref() else { - return None; - }; - // `rn <= k` (top-k). `rn < k` would be off-by-one; require `<=`. - if *op != CompareOpKind::Le { - return None; - } - let (QueryExpr::Column(rn_col), QueryExpr::Literal(ScalarValue::Int64(k))) = - (left.as_ref(), right.as_ref()) - else { - return None; - }; - if *k < 0 { - return None; - } - - // Optionally strip a passthrough projection (the derived table's SELECT that - // re-exposes the aggregate columns + rn), mapping the rn column through it. - let (wf_expr, rn_in_wf) = match child.as_ref() { - QueryExpr::Project { cols, child, .. } => { - let QueryExpr::Column(underlying) = &cols.get(*rn_col)?.expr else { - return None; - }; - (child.as_ref(), *underlying) - } - other => (other, *rn_col), - }; - - // The filtered column must be a `ROW_NUMBER()` window output — the single - // column the SQLWindowFunc appends after its input, i.e. the last one. - let QueryExpr::SQLWindowFunc { - func: WindowFuncKind::RowNumber, - partition_by, - order_by, - child: inner, - .. - } = wf_expr - else { - return None; - }; - if order_by.is_empty() { - return None; - } - let inner_cols = inner.output_schema().ok()?.fields.len(); - if rn_in_wf != inner_cols { - return None; // the predicate ranks some other column, not the row number - } - - // Generic partitioned top-k. The window's ORDER BY keys are relative to its - // input (`inner`), so they transfer directly to a `Sort` over `inner`. - Some(QueryExpr::Limit { - n: *k as usize, - offset: 0, - child: Rc::new(QueryExpr::Sort { - keys: order_by.clone(), - partition_by: partition_by.clone(), - child: Rc::new(inner.as_ref().clone()), - }), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pre_asap::query_expr::{ - GroupKeys, ProjectItem, Source, WindowFrame, WindowFrameBound, WindowFrameOffset, - WindowFrameUnits, - }; - use crate::pre_asap::schema::{DataType, Field, Schema}; - use crate::types::AccuracyTarget; - - fn scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - } - } - - /// `Aggregate{ by: [1], [Count] }` over the scan — output cols `[service, count]`. - fn count_by_service() -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan()), - } - } - - fn desc(col: usize) -> Vec { - vec![SortKey { - expr: QueryExpr::Column(col), - ascending: false, - nulls_first: false, - }] - } - - fn limit(n: usize, offset: usize, child: QueryExpr) -> QueryExpr { - QueryExpr::Limit { - n, - offset, - child: Rc::new(child), - } - } - - fn sort(keys: Vec, child: QueryExpr) -> QueryExpr { - QueryExpr::Sort { - keys, - partition_by: GroupKeys::by(vec![]), - child: Rc::new(child), - } - } - - fn is_topk_over_count(qe: &QueryExpr) -> bool { - matches!(qe, - QueryExpr::Aggregate { measures, child, .. } - if matches!(measures.as_slice(), [AggIntent::TopK { k: 5, .. }]) - && matches!(child.as_ref(), QueryExpr::Aggregate { measures, .. } - if matches!(measures.as_slice(), [AggIntent::Count { .. }]))) - } - - #[test] - fn promotes_count_ranked_limit_sort() { - // Limit 5 { Sort DESC by count-col (1) { Aggregate[Count] by [1] } }. - let q = limit(5, 0, sort(desc(1), count_by_service())); - assert!(is_topk_over_count(&canonicalize(q))); - } - - // A heavy-hitter sketch ranks every row; a count that only counts some - // rows (#466) is not that, so the generic Sort + Limit stays. - #[test] - fn does_not_promote_a_filtered_count_ranking() { - let mut filtered = count_by_service(); - let QueryExpr::Aggregate { filters, .. } = &mut filtered else { - unreachable!() - }; - *filters = vec![Some(Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(2)), - op: CompareOpKind::Gt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(1.0))), - })))]; - let q = limit(5, 0, sort(desc(1), filtered)); - let canonical = canonicalize(q.clone()); - assert!(!is_topk_over_count(&canonical)); - assert_eq!(canonical, q); - } - - #[test] - fn promotes_through_a_passthrough_projection() { - // …with a `SELECT service, count` projection between the Sort and the Agg. - let proj = QueryExpr::Project { - cols: vec![ - ProjectItem { - alias: None, - expr: QueryExpr::Column(0), - }, - ProjectItem { - alias: Some("c".into()), - expr: QueryExpr::Column(1), - }, - ], - qualifier: None, - child: Rc::new(count_by_service()), - }; - let q = limit(5, 0, sort(desc(1), proj)); - assert!(is_topk_over_count(&canonicalize(q))); - } - - #[test] - fn is_idempotent() { - let q = limit(5, 0, sort(desc(1), count_by_service())); - let once = canonicalize(q); - let twice = canonicalize(once.clone()); - assert_eq!(once, twice, "canonicalize must be idempotent"); - } - - // ── Concat's discriminator_unique_key vs. canonicalize (issue #228 review) ── - // - // `resolve.rs` resolves `discriminator_unique_key`'s `ColumnId`s against - // the first branch's *pre-canonicalize* output schema. If canonicalize - // then restructures that branch (heavy-hitter promotion, the - // `ROW_NUMBER()` top-k rewrite), those `ColumnId`s can end up pointing at - // the wrong column — or out of bounds — of the new schema. The two tests - // below pin the fix: the key is dropped whenever the branch's schema - // actually changed, and survives untouched otherwise. Never guessed at. - - #[test] - fn concat_discriminator_key_survives_canonicalize_when_first_branch_is_unaffected() { - // A plain `Aggregate` first branch matches neither rewrite trigger, - // so its schema is identical before and after canonicalize. - let q = QueryExpr::concat_with_discriminator( - vec![count_by_service(), count_by_service()], - /* discriminator */ 0, - /* inner_key */ vec![1], - ); - let QueryExpr::Concat { - discriminator_unique_key, - .. - } = canonicalize(q) - else { - panic!("expected Concat"); - }; - assert!( - discriminator_unique_key.is_some(), - "an untouched first branch's discriminator key must survive canonicalize" - ); - } - - #[test] - fn concat_discriminator_key_is_dropped_when_first_branch_gets_rewritten() { - // The first branch is exactly the heavy-hitter promotion trigger — - // `Limit{Sort{Aggregate([Count])}}`, with an empty (global) - // `partition_by` — so canonicalize rewrites it in place to - // `Aggregate{TopK}`, whose own output is a single column, not the - // original two (`[service, count]`). A discriminator key resolved - // against the original 2-column shape (`discriminator` = `service` - // at index 0, `inner_key` = `count` at index 1) must not silently - // survive pointing at the new 1-column schema. - let promotable_branch = limit(5, 0, sort(desc(1), count_by_service())); - let q = QueryExpr::concat_with_discriminator( - vec![promotable_branch, count_by_service()], - /* discriminator */ 0, - /* inner_key */ vec![1], - ); - let QueryExpr::Concat { - children, - discriminator_unique_key, - } = canonicalize(q) - else { - panic!("expected Concat"); - }; - assert!( - is_topk_over_count(&children[0]), - "the first branch is still promoted normally" - ); - assert!( - discriminator_unique_key.is_none(), - "a stale discriminator key must be dropped, never silently kept wrong" - ); - } - - #[test] - fn does_not_promote_ascending_sort() { - // Ascending = bottom-k: the Top-K operator's ranking rule - // rejects it (needs descending), so it stays a generic Sort+Limit — the - // same call PromQL `bottomk` makes (issue #38). - let asc = vec![SortKey { - expr: QueryExpr::Column(1), - ascending: true, - nulls_first: false, - }]; - let q = limit(5, 0, sort(asc, count_by_service())); - assert!(!is_topk_over_count(&canonicalize(q))); - } - - #[test] - fn does_not_promote_with_offset() { - let q = limit(5, 2, sort(desc(1), count_by_service())); - assert!(!is_topk_over_count(&canonicalize(q))); - } - - #[test] - fn does_not_promote_ranking_by_a_group_key() { - // DESC by col 0 (the `service` group key), not the count → not a - // frequency heavy-hitter. - let q = limit(5, 0, sort(desc(0), count_by_service())); - assert!(!is_topk_over_count(&canonicalize(q))); - } - - #[test] - fn promotes_sum_ranked_limit_sort_as_weighted_heavy_hitter() { - let sum = QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan()), - }; - let q = limit(5, 0, sort(desc(1), sum)); - let out = canonicalize(q); - let QueryExpr::Aggregate { - measures, child, .. - } = out - else { - panic!("expected weighted TopK aggregate"); - }; - assert!(matches!( - measures.as_slice(), - [AggIntent::TopK { k: 5, .. }] - )); - assert!( - matches!(child.as_ref(), QueryExpr::Aggregate { measures, .. } - if matches!(measures.as_slice(), [AggIntent::Sum { .. }])) - ); - } - - #[test] - fn keeps_sum_over_counter_reduction_as_exact_value_ranking() { - for counter in [AggIntent::Rate, AggIntent::Increase] { - let derived = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![counter], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan()), - }; - let sum = QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(derived), - }; - let out = canonicalize(limit(5, 0, sort(desc(1), sum))); - assert!(matches!(out, QueryExpr::Limit { child, .. } - if matches!(child.as_ref(), QueryExpr::Sort { child, .. } - if matches!(child.as_ref(), QueryExpr::Aggregate { measures, child, .. } - if matches!(measures.as_slice(), [AggIntent::Sum { .. }]) - && matches!(child.as_ref(), QueryExpr::Aggregate { .. }))))); - } - } - - // ── ROW_NUMBER() partitioned top-k (issue #24) ────────────────────────── - - /// A scan with `[ts, service, region, value]`. - fn scan4() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - } - } - - /// `Aggregate{ by: [1,2] (service, region), [agg] }` — output `[service, - /// region, ]` (3 cols), so a ROW_NUMBER over it appends `rn` at index 3. - fn grouped(agg: AggIntent) -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::by(vec![1, 2]), - measures: vec![agg], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan4()), - } - } - - /// `ROW_NUMBER` ignores its frame clause, so the top-k rewrite doesn't care - /// what's in it; any concrete frame works as fixture data. - fn rownumber_frame() -> WindowFrame { - WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(WindowFrameOffset::Scalar(ScalarValue::Null)), - end_bound: WindowFrameBound::Following(WindowFrameOffset::Scalar(ScalarValue::Null)), - } - } - - /// `Filter{ rn(3) <= 5 } { SQLWindowFunc{ RowNumber, PARTITION BY region(2), - /// ORDER BY col(2) DESC } { agg } }`. - fn rownumber_topk(agg: QueryExpr) -> QueryExpr { - let wf = QueryExpr::SQLWindowFunc { - func: WindowFuncKind::RowNumber, - args: vec![], - partition_by: GroupKeys::by(vec![2]), // region - order_by: vec![SortKey { - expr: QueryExpr::Column(2), // the aggregate output column - ascending: false, - nulls_first: true, - }], - frame: Some(rownumber_frame()), - output_name: "rn".into(), - child: Rc::new(agg), - }; - QueryExpr::Filter { - pred: Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(3)), // rn = the appended window column - op: CompareOpKind::Le, - right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(5))), - })), - child: Rc::new(wf), - } - } - - #[test] - fn rownumber_count_topk_becomes_a_partitioned_heavy_hitter() { - // Count-ranked ROW_NUMBER top-k → outer TopK grouped by the partition - // (region, col 2) over the explicit inner Count. - let q = rownumber_topk(grouped(AggIntent::Count { - accuracy: AccuracyTarget::Exact, - })); - let out = canonicalize(q); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &out - else { - panic!("expected outer Aggregate([TopK]), got {out:?}"); - }; - let Reduction::Reduce(by) = reduction else { - panic!("expected a Reduce grouping, got {reduction:?}"); - }; - assert!(matches!( - measures.as_slice(), - [AggIntent::TopK { k: 5, .. }] - )); - assert_eq!(**by, vec![2], "outer TopK partitioned by region"); - assert!( - matches!(child.as_ref(), QueryExpr::Aggregate { measures, .. } - if matches!(measures.as_slice(), [AggIntent::Count { .. }])) - ); - } - - #[test] - fn rownumber_avg_topk_becomes_a_partitioned_sort_limit() { - // Avg-ranked (not a frequency heavy-hitter) → generic partitioned - // top-k: Limit{5}{ Sort{ partition_by: [region] } }. - let q = rownumber_topk(grouped(AggIntent::Avg { col: None })); - let out = canonicalize(q); - let QueryExpr::Limit { n, child, .. } = &out else { - panic!("expected a Limit, got {out:?}"); - }; - assert_eq!(*n, 5); - let QueryExpr::Sort { - partition_by, - child, - .. - } = child.as_ref() - else { - panic!("expected a Sort under the Limit"); - }; - assert_eq!(**partition_by, vec![2], "partitioned by region"); - assert!( - matches!(child.as_ref(), QueryExpr::Aggregate { measures, .. } - if matches!(measures.as_slice(), [AggIntent::Avg { .. }])) - ); - } - - #[test] - fn filter_on_a_non_rownumber_column_is_left_alone() { - // `WHERE service_len <= 5` (col 0, not the rn window column) must not be - // mistaken for a top-k. - let wf = QueryExpr::SQLWindowFunc { - func: WindowFuncKind::RowNumber, - args: vec![], - partition_by: GroupKeys::by(vec![2]), - order_by: vec![SortKey { - expr: QueryExpr::Column(2), - ascending: false, - nulls_first: true, - }], - frame: Some(rownumber_frame()), - output_name: "rn".into(), - child: Rc::new(grouped(AggIntent::Count { - accuracy: AccuracyTarget::Exact, - })), - }; - let q = QueryExpr::Filter { - pred: Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(0)), // NOT the rn column (index 3) - op: CompareOpKind::Le, - right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(5))), - })), - child: Rc::new(wf), - }; - assert!( - matches!(canonicalize(q), QueryExpr::Filter { .. }), - "left as a Filter" - ); - } -} diff --git a/crates/types/src/pre_asap/column_resolution.rs b/crates/types/src/pre_asap/column_resolution.rs index cfafbe9e0..e2d8120d6 100644 --- a/crates/types/src/pre_asap/column_resolution.rs +++ b/crates/types/src/pre_asap/column_resolution.rs @@ -1,22 +1,13 @@ //! Schema-driven column resolution. //! -//! Front ends (issue #179) emit `ColumnRef` (name-based, optionally -//! table-qualified); the canonical DAG uses positional [`ColumnId`] resolved -//! against a per-node [`Schema`]. These helpers bridge the two — the -//! [`SchemaResolver`](super::schema_resolver) builds the schema, and [`resolve_column_refs`] -//! turns name-based refs (group keys, dedup columns) into positional ids, -//! qualifier-aware. - -use std::rc::Rc; +//! Front ends emit `ColumnRef` (name-based, optionally table-qualified); the +//! IR uses positional [`ColumnId`] resolved against a per-node [`Schema`]. +//! These helpers turn name-based refs into positional ids, qualifier-aware. +//! Front-end name resolution (`asap_frontend_common::resolve`) calls them. use thiserror::Error; -use super::agg_intent::AggIntent; use super::expr_ir::ColumnRef; -use super::query_expr::{ - aggregate_output_schema, GroupKeys, QueryExpr, QueryExprError, Reduction, ResolvedQueryExpr, - UnresolvedQueryExpr, -}; use super::schema::{ColumnId, DataType, FieldDataType, Schema}; /// Errors returned by the resolution helpers. @@ -115,102 +106,6 @@ pub fn resolve_group_keys_promql( .collect() } -/// Resolve a name-based scalar [`UnresolvedQueryExpr`] (one of `QueryExpr`'s scalar -/// variants, issue #205) into a positional [`ResolvedQueryExpr`] by resolving every -/// column reference against `schema`. Structural otherwise. `expr` must be -/// one of the scalar variants — an operator variant here is a construction -/// bug, not a shape this needs to handle silently. -pub fn resolve_expr( - expr: &UnresolvedQueryExpr, - schema: &Schema, -) -> Result { - let rc = |e: &UnresolvedQueryExpr| -> Result, ResolveError> { - Ok(Rc::new(resolve_expr(e, schema)?)) - }; - let each = |es: &[UnresolvedQueryExpr]| -> Result, ResolveError> { - es.iter().map(|e| resolve_expr(e, schema)).collect() - }; - Ok(match expr { - QueryExpr::Column(c) => QueryExpr::Column(resolve_column_ref(c, schema)?), - QueryExpr::Literal(s) => QueryExpr::Literal(s.clone()), - QueryExpr::EvalTimestamp => QueryExpr::EvalTimestamp, - QueryExpr::CurrentTimestamp => QueryExpr::CurrentTimestamp, - QueryExpr::Compare { left, op, right } => QueryExpr::Compare { - left: rc(left)?, - op: op.clone(), - right: rc(right)?, - }, - QueryExpr::BoolAnd(v) => QueryExpr::BoolAnd(each(v)?), - QueryExpr::BoolOr(v) => QueryExpr::BoolOr(each(v)?), - QueryExpr::Not(e) => QueryExpr::Not(rc(e)?), - QueryExpr::IsNull(e) => QueryExpr::IsNull(rc(e)?), - QueryExpr::IsNotNull(e) => QueryExpr::IsNotNull(rc(e)?), - QueryExpr::Cast { expr, to, try_cast } => QueryExpr::Cast { - expr: rc(expr)?, - to: to.clone(), - try_cast: *try_cast, - }, - QueryExpr::InList { - expr, - list, - negated, - } => QueryExpr::InList { - expr: rc(expr)?, - list: each(list)?, - negated: *negated, - }, - QueryExpr::FunctionCall { name, args } => QueryExpr::FunctionCall { - name: name.clone(), - args: each(args)?, - }, - QueryExpr::Arithmetic { op, left, right } => QueryExpr::Arithmetic { - op: op.clone(), - left: rc(left)?, - right: rc(right)?, - }, - QueryExpr::Case { - operand, - branches, - else_expr, - } => QueryExpr::Case { - operand: operand.as_deref().map(rc).transpose()?, - branches: branches - .iter() - .map(|(w, t)| Ok((resolve_expr(w, schema)?, resolve_expr(t, schema)?))) - .collect::, ResolveError>>()?, - else_expr: else_expr.as_deref().map(rc).transpose()?, - }, - other => unreachable!("resolve_expr called on a non-scalar QueryExpr variant: {other:?}"), - }) -} - -/// Output schema produced by an `Aggregate { by, measures }` over `input`. -/// Mirrors `QueryExpr::output_schema_in`'s `Aggregate` arm; out-of-range `by` -/// ids are silently dropped (callers needing the strict check resolve `by` -/// via [`resolve_column_refs`], which surfaces `NotFound`). -pub fn output_schema_for_aggregate( - input: &Schema, - by: &GroupKeys, - measures: &[AggIntent], - output_names: &[String], -) -> Result { - // Delegate to the single canonical derivation so HAVING resolution can never - // drift from `QueryExpr::output_schema_in` (issue #41). HAVING is SQL-only - // and cross-series (SQL has no `without`), but detect the child-independent - // per-entity case anyway (a lone `rate`/`increase`/`*_over_time` intent) so - // the two agree on every shared input — the range-window child marker the - // canonical arm also keys off is not visible here, and never co-occurs with - // HAVING. - let per_entity = - by.is_empty() && !by.is_without() && measures.len() == 1 && measures[0].is_per_series(); - let reduction = if per_entity { - Reduction::PerEntity - } else { - Reduction::Reduce(by.clone()) - }; - aggregate_output_schema(input, &reduction, measures, output_names) -} - #[cfg(test)] mod tests { use super::*; @@ -326,77 +221,4 @@ mod tests { Err(ResolveError::NotFound { .. }) )); } - - #[test] - fn aggregate_strips_time_and_keeps_unique_keys() { - let mut input = ts_value_schema(); - input - .fields - .push(Field::plain("host", DataType::Utf8, false)); - let out = output_schema_for_aggregate( - &input, - &GroupKeys::by(vec![2]), - &[AggIntent::Sum { col: None }], - &[], - ) - .expect("valid group-by column"); - assert_eq!(out.fields.len(), 2); // host, sum - assert_eq!(out.fields[0].name, "host"); - assert_eq!(out.fields[1].name, "sum"); - assert!(out.time_index.is_none()); - assert_eq!(out.unique_keys, vec![vec![0]]); - } - - #[test] - fn having_schema_agrees_with_canonical_for_a_per_series_reduction() { - // Issue #41: `output_schema_for_aggregate` (HAVING resolution) and the - // canonical `QueryExpr::output_schema_in` must produce identical schemas - // for the same aggregate. Before the dedup this diverged on a per-series - // reduction — the HAVING mirror lacked the per-series branch and would - // collapse `[ts, value]` to a single `rate` column. - use crate::pre_asap::query_expr::Source; - use std::time::Duration; - - let leaf_schema = Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ); - let scan = QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: leaf_schema.clone(), - }; - // Aggregate{ reduction: PerEntity, [Rate], child: TimeRange{ Scan } } — - // a per-series reduction (label-preserving). - let agg = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Rate], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(QueryExpr::TimeRange { - range: Duration::from_secs(300), - child: Rc::new(scan), - }), - }; - let canonical = agg.output_schema().expect("canonical schema"); - - // The HAVING-resolution derivation gets only the input schema (the - // TimeRange passes the leaf schema through). - let having_side = - output_schema_for_aggregate(&leaf_schema, &GroupKeys::none(), &[AggIntent::Rate], &[]) - .unwrap(); - - assert_eq!( - canonical, having_side, - "the two aggregate-schema derivations must agree (issue #41)" - ); - // Sanity: it really is the label-preserving per-series shape, not `[rate]`. - assert!(having_side.fields.iter().any(|c| c.name == "value")); - assert!(having_side.time_index.is_some()); - } } diff --git a/crates/types/src/pre_asap/cse.rs b/crates/types/src/pre_asap/cse.rs deleted file mode 100644 index 0a00475e6..000000000 --- a/crates/types/src/pre_asap/cse.rs +++ /dev/null @@ -1,1111 +0,0 @@ -//! Pre-ASAP structural common-subexpression elimination: bottom-up -//! hash-consing over an already-`resolve_root`'d [`QueryExpr`] DAG (issue -//! #212, #222, #223). -//! -//! CSE only runs on an already-bound, already-canonicalized DAG — -//! structural matching is meaningless before canonicalization has converged -//! semantically-equivalent queries onto one shape (`docs/develop_docs/pre-asap-ir.md` -//! design principle 3; `median(latency)` and `approx_percentile_cont(latency, -//! 0.5)` already lower to an identical `AggIntent::Quantile` today, per -//! `sql_lowering.rs`'s `median_is_the_same_intent_as_an_explicit_half_percentile` -//! test). [`share_common_sub_dags`] is the single entry point, run once per -//! workload batch (or once per query — see "Single-query CSE" below) *after* -//! `resolve_root`, *before* the pre-ASAP → post-ASAP replacement/search pass -//! (`asap_aware_mapping::replacement`). -//! -//! ## Algorithm: classic hash-consing / value-numbering -//! -//! Bottom-up: every child is interned before its parent, so a parent's -//! candidacy for sharing naturally incorporates whether its own children were -//! themselves shared — two parents whose children were independently -//! deduplicated down to the same `Rc`s are structurally identical iff their -//! own fields also match, without re-walking the sub-DAGs. -//! -//! Only the **relational skeleton** participates — the same set of "operator" -//! children [`canonicalize`](super::canonicalize)'s `children_mut` walks -//! (`Filter`/`Project`/`Aggregate`/`Concat`/`Join`/`BinaryOp`/…). A scalar -//! subexpression reachable only through a wrapper position (`Predicate`, -//! `ProjectItem.expr`, `Aggregate.having`, `SQLWindowFunc.args`, …) stays -//! embedded as opaque data on its owning operator node, compared by -//! `QueryExpr`'s derived `PartialEq` along with the rest of that node's -//! fields, rather than separately hash-consed — the same scope -//! `canonicalize.rs` settled on ("none of the rewrite rules touch a scalar -//! sub-DAG, so there's nothing to gain by recursing into one"). Widening this -//! to scalar positions is future work, not attempted here. -//! -//! ## Correctness: hash is a filter, `PartialEq` is the decision -//! -//! This is the one non-negotiable rule. A **false positive** here — two -//! sub-DAGs wrongly judged shareable — is a wrong query answer, not a missed -//! optimization: two different queries would read each other's data. -//! [`structural_hash`] (`DefaultHasher`/SipHash over a canonical -//! serialization, no collision-freedom guarantee) may only narrow the -//! candidate set within one bucket; [`InternTable::intern`]'s `PartialEq` -//! check on that bucket is what actually decides sharing, every time, no -//! exceptions for "the hash probably didn't collide." -//! -//! This also means the pass is safe by construction against the case #212 -//! flagged as a real historical bug (issue #115): `AggIntent::Quantile` -//! carries its input column and its `AccuracyTarget`, both `PartialEq` -//! fields, so `Quantile(x, 0.99, ε=0.01)` and `Quantile(x, 0.99, ε=0.001)` — -//! or `Quantile(x, ..)` vs `Quantile(y, ..)` — are never merged. This is -//! intentionally conservative: it only recognizes *exact* structural -//! matches, not "a stricter-accuracy summary could also answer a looser -//! request." That subsumption question already has a documented, -//! deliberately-unfilled home (`asap_aware_mapping::Matcher`) — -//! CSE here does not attempt it. -//! -//! ## Legality: gated by `Schema::unique_keys` -//! -//! Structural equality alone is necessary but not sufficient. Per -//! [`Schema::unique_keys`](super::schema::Schema::unique_keys)'s own doc: "a -//! producer's output can only be safely shared across consumers when its row -//! identity is provably stable across reads." A candidate node with no -//! provable unique key (`Schema::has_unique_key()` false, or `output_schema` -//! not even defined for that node, e.g. a `Concat`/`SetOp` branch whose union -//! drops `unique_keys`, or an ungrouped/global `Aggregate`, whose empty `by` -//! also reports no unique key today) is **never** hoisted, even when it is -//! structurally identical to something already interned — it is always -//! inserted fresh, matching the rule the (now-deleted) prior CSE attempt -//! already encoded and the doc comment on `Aggregate`'s `child` field -//! ("`unique_keys` feeds CSE's producer-sharing legality check"). -//! -//! ## Single-query CSE falls out for free -//! -//! A repeated sub-expression within *one* query (e.g. the same grouped -//! `Aggregate` referenced twice on two `BinaryOp` branches) is deduplicated -//! by the exact same bottom-up interning — a workload of size one still -//! interns bottom-up within that one DAG. No separate mechanism is needed; -//! see the `single_query_shares_its_own_repeated_sub_dag` test below. -//! -//! ## Landing plan (issue #223) -//! -//! This module is stage 1 of a 4-stage plan. Stage 2 -//! (`asap_aware_mapping::replacement::search_workload_with`, which runs -//! [`share_common_sub_dags`] itself before searching) is a real caller, -//! wired at the same time so this never becomes unwired dead code again -//! (the original `asap-plan::cse::dedupe_subtrees` was deleted in #192 for -//! exactly that). Stage 3 — [`dag_export`](crate::dag_export) computing its -//! per-node `hash` by calling this module's [`structural_hash`] directly, -//! instead of a parallel reimplementation — is also done, so -//! `tools/dag-viewer`'s "shared sub-DAG" highlighting now flags exactly the -//! candidate pairs this module's own `InternTable` would bucket together -//! (still only a hash match, not a guarantee of -//! `share_common_sub_dags`-actual sharing — see `dag_export`'s module doc). -//! Stage 4 (issue #237) is implemented in -//! `asap_aware_mapping::cost_model::CostModel::cse_share_decision`, called -//! from `asap_aware_mapping::replacement::CandidateLogicalASAPDAGs::cost_sorted` (via that -//! module's own `cse_preference`) — a real, Volcano/Cascades-style cost -//! comparison over what this module detects, not a fixed rule. See -//! `docs/design_docs/cost-model.md`. This module's own -//! unconditional "share whenever legal" behavior is unchanged: detection -//! stays cost-agnostic by construction (this crate cannot depend on -//! `asap-aware-mapping`'s `CostModel`), and the cost-aware decision is -//! applied downstream, after detection, over what this module finds. - -use std::collections::HashMap; -use std::hash::{Hash, Hasher}; -use std::rc::Rc; - -use super::query_expr::QueryExpr; - -/// Bottom-up hash-consing table: structurally-equal, sharing-legal -/// [`QueryExpr`] nodes collapse onto one `Rc`. -/// -/// `buckets` is keyed by [`structural_hash`] — a coarse candidate filter -/// only (see the module-level "Correctness" section). Every entry within one -/// bucket is a full node kept around for the `PartialEq` comparison that -/// actually decides a match; a hash collision between structurally different -/// nodes just means a (harmless) linear scan of a few extra candidates. -struct InternTable { - buckets: HashMap>>, - /// Memoizes [`structural_hash`] per already-hashed `Rc` pointer, shared - /// across every [`intern`](Self::intern) call for the table's whole - /// lifetime — see [`structural_hash`]'s own doc on why this matters: - /// without it, hashing an `N`-node bottom-up pass costs `O(N)` work - /// *per node* (every already-interned descendant gets re-walked), not - /// `O(1)` amortized. - hash_cache: HashCache, -} - -impl InternTable { - fn new() -> Self { - Self { - buckets: HashMap::new(), - hash_cache: HashMap::new(), - } - } - - /// Intern one already-children-rebuilt node: look it up by - /// [`structural_hash`], confirm with `PartialEq`, and — only when - /// sharing is legal (see "Legality" above) — return the existing `Rc` - /// instead of allocating a new one. - fn intern(&mut self, node: QueryExpr) -> Rc { - let hash = structural_hash(&node, &mut self.hash_cache); - // A node with no provable unique key is never *returned* as a match - // for something else — it may still go on to occupy a fresh slot in - // the bucket (harmless; it just never gets found by a later - // `PartialEq` scan that also requires `reusable`). - let reusable = node - .output_schema() - .is_ok_and(|schema| schema.has_unique_key()); - let bucket = self.buckets.entry(hash).or_default(); - if reusable { - if let Some(existing) = bucket.iter().find(|candidate| candidate.as_ref() == &node) { - return Rc::clone(existing); - } - } - let rc = Rc::new(node); - bucket.push(Rc::clone(&rc)); - rc - } -} - -/// [`structural_hash`]'s memoization cache: maps an already-hashed node's -/// `Rc` pointer to its computed hash. Not tied to any one `QueryExpr` — a -/// fresh, empty cache is correct to start with anywhere; what matters is -/// letting it *persist* across every node in one bottom-up pass (as -/// [`InternTable`] does via its own `hash_cache` field), rather than -/// starting a new one per call. -/// -/// `pub` (not `pub(crate)`) so `asap_aware_mapping`'s workload-search MEMO -/// engine (`replacement::is_duplicate_rewrite`) can reuse this exact -/// candidate-narrowing filter for its own dedup, instead of maintaining a -/// parallel reimplementation — the same "one real hash, reused everywhere -/// it's needed" rationale [`structural_hash`]'s own doc gives for -/// [`dag_export`](crate::dag_export)'s `pub(crate)` reuse. -pub type HashCache = HashMap<*const QueryExpr, u64>; - -/// Coarse structural hash used only to bucket [`InternTable::intern`]'s -/// candidate search — never the actual sharing decision (`PartialEq` is). -/// -/// `QueryExpr` carries `f64`s (`Literal(ScalarValue::Float64)`, `AggIntent::Quantile.q`, …), so it -/// cannot derive `std::hash::Hash`. Serializing to a canonical JSON string -/// and hashing that sidesteps the `f64` problem — but only for `node`'s own -/// tag and non-child fields, *not* its children's full values: each -/// `Rc`-backed child's contribution is its own [`structural_hash`], looked -/// up in `cache` if already computed there (memoized by `Rc` pointer -/// identity) rather than recursed into again. -/// -/// This is the DAG-aware fix a naive "just serialize the whole sub-DAG" -/// hash would get wrong: after [`share_common_sub_dags`] (or even before -/// it — a front end can emit internal `Rc` sharing directly, e.g. a -/// repeated subexpression within one query), `node` generally has internal -/// sharing. A full-sub-DAG serialization re-serializes — re-walks — -/// any descendant `node` already shares internally once per parent that -/// references it; called once per node in a bottom-up pass (as -/// [`InternTable::intern`] and [`dag_export`](crate::dag_export) both do), -/// that costs `O(sub-DAG size)` *per node* instead of `O(1)` amortized — -/// quadratic-or-worse for a deep chain, compounding further with any real -/// internal sharing. Memoizing each child's hash by pointer identity in -/// `cache` (persisted across the whole pass by the caller, not reset per -/// node) makes each node's own contribution `O(1)` beyond its children's -/// already-known hashes, giving `O(N)` total for `N` nodes — matching -/// [`dag_node_count`]'s own shared-node counting fix (issue #212/#223/#237's stage -/// 4) in spirit, applied to hashing instead of counting. -/// -/// `pub` (not private) so [`dag_export`](crate::dag_export) can call -/// this exact function for its exported nodes' `hash` field instead of -/// maintaining its own parallel reimplementation — issue #223 stage 3. That -/// makes `tools/dag-viewer`'s "shared sub-DAG" highlighting reflect this -/// module's real hashing, not a lookalike computed a different way; see the -/// module doc's "Landing plan" section. A NaN/infinite `f64` makes JSON -/// serialization fail; falling back to a fixed hash just puts every such -/// node in one (larger, still `PartialEq`-disambiguated) bucket. Made `pub` -/// (rather than staying `pub(crate)`) for one more reuse across the crate -/// boundary: `asap_aware_mapping`'s workload-search MEMO engine -/// (`replacement::is_duplicate_rewrite`) needs the identical -/// candidate-narrowing filter this module's own [`InternTable::intern`] -/// already uses, so it doesn't have to reinvent (and risk drifting from) it. -/// -/// Exhaustive over every `QueryExpr` variant, matching [`rebuild_children`] -/// in which fields count as an operator child (must stay in sync — a new -/// variant fails to compile in both places until both are extended). -pub fn structural_hash(node: &QueryExpr, cache: &mut HashCache) -> u64 { - use QueryExpr::*; - - fn child_hash(child: &Rc, cache: &mut HashCache) -> u64 { - let ptr = Rc::as_ptr(child); - if let Some(&h) = cache.get(&ptr) { - return h; - } - let h = structural_hash(child, cache); - cache.insert(ptr, h); - h - } - - /// Hash `own_fields` (this node's own tag and non-child scalar - /// fields — anything JSON-serializable and small, i.e. never a - /// `QueryExpr` sub-DAG) via the same canonical-JSON-string trick the - /// whole-sub-DAG version used, just applied to `O(1)` fields instead - /// of `O(sub-DAG size)`. - fn hash_own_fields(hasher: &mut impl Hasher, own_fields: &impl serde::Serialize) { - serde_json::to_string(own_fields) - .unwrap_or_default() - .hash(hasher); - } - - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - match node { - Scan { - source, - predicates, - schema, - } => hash_own_fields(&mut hasher, &("Scan", source, predicates, schema)), - PromqlVectorFromScalar(c) => { - "PromqlVectorFromScalar".hash(&mut hasher); - child_hash(c, cache).hash(&mut hasher); - } - PromqlScalarFromVector(c) => { - "PromqlScalarFromVector".hash(&mut hasher); - child_hash(c, cache).hash(&mut hasher); - } - PromqlRelabel { dst, value, child } => { - hash_own_fields(&mut hasher, &("PromqlRelabel", dst, value)); - child_hash(child, cache).hash(&mut hasher); - } - PromqlInfoEnrich { selector, child } => { - hash_own_fields(&mut hasher, &("PromqlInfoEnrich", selector)); - child_hash(child, cache).hash(&mut hasher); - } - PromqlSeriesSample { by, kind, child } => { - hash_own_fields(&mut hasher, &("PromqlSeriesSample", by, kind)); - child_hash(child, cache).hash(&mut hasher); - } - Filter { pred, child } => { - hash_own_fields(&mut hasher, &("Filter", pred)); - child_hash(child, cache).hash(&mut hasher); - } - Project { - cols, - qualifier, - child, - } => { - hash_own_fields(&mut hasher, &("Project", cols, qualifier)); - child_hash(child, cache).hash(&mut hasher); - } - Aggregate { - reduction, - measures, - output_names, - filters, - having, - child, - } => { - hash_own_fields( - &mut hasher, - &( - "Aggregate", - reduction, - measures, - output_names, - filters, - having, - ), - ); - child_hash(child, cache).hash(&mut hasher); - } - Dedup { cols, child } => { - hash_own_fields(&mut hasher, &("Dedup", cols)); - child_hash(child, cache).hash(&mut hasher); - } - Concat { - children, - discriminator_unique_key, - } => { - hash_own_fields(&mut hasher, &("Concat", discriminator_unique_key)); - for c in children { - // Stored by value, not `Rc` — see `rebuild_children`'s - // `intern_owned` use for this variant — so there's no - // pointer to memoize on here; recurse directly. Any - // `Rc`-typed descendant beneath `c` still gets memoized - // once this call reaches it. - structural_hash(c, cache).hash(&mut hasher); - } - } - Join { - kind, - pred, - left, - right, - } => { - hash_own_fields(&mut hasher, &("Join", kind, pred)); - child_hash(left, cache).hash(&mut hasher); - child_hash(right, cache).hash(&mut hasher); - } - SetOp { - kind, - all, - left, - right, - } => { - hash_own_fields(&mut hasher, &("SetOp", kind, all)); - child_hash(left, cache).hash(&mut hasher); - child_hash(right, cache).hash(&mut hasher); - } - Sort { - keys, - partition_by, - child, - } => { - hash_own_fields(&mut hasher, &("Sort", keys, partition_by)); - child_hash(child, cache).hash(&mut hasher); - } - Limit { n, offset, child } => { - hash_own_fields(&mut hasher, &("Limit", n, offset)); - child_hash(child, cache).hash(&mut hasher); - } - PromqlSubquery { - range, - resolution, - child, - } => { - hash_own_fields(&mut hasher, &("PromqlSubquery", range, resolution)); - child_hash(child, cache).hash(&mut hasher); - } - TimeRange { range, child } => { - hash_own_fields(&mut hasher, &("TimeRange", range)); - child_hash(child, cache).hash(&mut hasher); - } - TimeShift { shift, child } => { - hash_own_fields(&mut hasher, &("TimeShift", shift)); - child_hash(child, cache).hash(&mut hasher); - } - SQLWindowFunc { - func, - args, - partition_by, - order_by, - frame, - output_name, - child, - } => { - hash_own_fields( - &mut hasher, - &( - "SQLWindowFunc", - func, - args, - partition_by, - order_by, - frame, - output_name, - ), - ); - child_hash(child, cache).hash(&mut hasher); - } - BinaryOp { - op, - lhs, - rhs, - vector_match, - } => { - hash_own_fields(&mut hasher, &("BinaryOp", op, vector_match)); - child_hash(lhs, cache).hash(&mut hasher); - child_hash(rhs, cache).hash(&mut hasher); - } - // `EvalTimestamp`, `PromqlScalarBridge`, and the scalar variants - // (issue #205) are all leaves for this traversal's purposes — none - // has an operator child to look up in `cache` — so hashing the - // whole node via `serde_json` in one shot is already `O(node - // size)`, not `O(sub-DAG size)`: exactly the same cost the - // per-variant `hash_own_fields` calls above pay, just without - // needing to spell out each field individually. Matches - // `rebuild_children`'s and `dag_node_count`'s identical scope - // decision for these variants ("never descended into"). - EvalTimestamp - | CurrentTimestamp - | PromqlScalarBridge(_) - | Column(_) - | Literal(_) - | Compare { .. } - | BoolAnd(_) - | BoolOr(_) - | Not(_) - | IsNull(_) - | IsNotNull(_) - | Cast { .. } - | InList { .. } - | FunctionCall { .. } - | Arithmetic { .. } - | Case { .. } => hash_own_fields(&mut hasher, node), - } - hasher.finish() -} - -/// Count of *unique* nodes reachable from `root`, deduplicated by `Rc` -/// pointer identity (`Rc::as_ptr`) — the real size of the DAG rooted at -/// `root`, not a per-path walk count. -/// -/// After [`share_common_sub_dags`] runs (or even before it, for a DAG a -/// front end already built with internal `Rc` sharing — e.g. re-running -/// CSE, or a single-query repeated subexpression), `root` is generally a -/// **DAG** with internal sharing — that is this whole module's premise. Anything that -/// walks `root` as if every reference were a fresh sub-DAG (a naive -/// recursive walk with no identity tracking, or a naive full -/// `serde_json` serialization — `Rc`'s `Serialize` impl serializes the -/// pointee's *value* at every occurrence, it does not dedupe by identity) -/// re-visits/re-counts an already-shared descendant once per parent that -/// references it, over-counting relative to the actual work of holding it -/// in memory or recomputing it once. This function is the DAG-correct -/// alternative: each unique node is counted exactly once, regardless of -/// how many places within `root` reference it. -/// -/// `pub` so cost-aware callers outside this crate (e.g. -/// `asap_aware_mapping::CostModel::cse_recompute_cost`'s default) have a -/// DAG-correct structural-size proxy available, instead of reaching for -/// something per-path like a raw serialization length. -/// -/// Same operator-child traversal scope as [`share_common_sub_dags`] itself -/// (see the module doc's "Algorithm" section, and this module's private -/// `rebuild_children`) — a scalar subexpression embedded in a wrapper -/// position (`Predicate`, `ProjectItem.expr`, `Aggregate.having`, …) is not -/// separately visited, matching this module's own stated scope; it's -/// counted as part of its owning operator node, the same node -/// `rebuild_children` treats as a single opaque leaf for interning -/// purposes. -pub fn dag_node_count(root: &QueryExpr) -> usize { - let mut seen: std::collections::HashSet<*const QueryExpr> = std::collections::HashSet::new(); - count_unique(root, &mut seen) -} - -/// One node's own contribution (`1`) plus each *not-yet-seen* operator -/// child's contribution — exhaustive over every `QueryExpr` variant, -/// enumerating the same fields [`rebuild_children`] does (kept as a -/// separate, read-only traversal rather than threaded through -/// `rebuild_children` itself, since that function consumes and rebuilds -/// its input while this one only ever reads it). -fn count_unique(node: &QueryExpr, seen: &mut std::collections::HashSet<*const QueryExpr>) -> usize { - use QueryExpr::*; - - /// Visit one `Rc`-held child: counts (and recurses into) it only the - /// first time its pointer is seen, `0` on every later occurrence — - /// this is the actual dedup step. - fn visit( - child: &Rc, - seen: &mut std::collections::HashSet<*const QueryExpr>, - ) -> usize { - if seen.insert(Rc::as_ptr(child)) { - count_unique(child, seen) - } else { - 0 - } - } - - 1 + match node { - // `PromqlScalarBridge`'s child is a scalar-sub-language node (issue - // #220), never descended into — same treatment `rebuild_children` - // gives it (see that function's comment on this same variant). - Scan { .. } | PromqlScalarBridge(_) | EvalTimestamp | CurrentTimestamp => 0, - PromqlVectorFromScalar(c) | PromqlScalarFromVector(c) => visit(c, seen), - PromqlRelabel { child, .. } - | PromqlInfoEnrich { child, .. } - | PromqlSeriesSample { child, .. } - | Filter { child, .. } - | Project { child, .. } - | Aggregate { child, .. } - | Dedup { child, .. } - | Sort { child, .. } - | Limit { child, .. } - | PromqlSubquery { child, .. } - | TimeRange { child, .. } - | TimeShift { child, .. } - | SQLWindowFunc { child, .. } => visit(child, seen), - // `Concat`'s branches are stored by value (`Vec`, not - // `Rc` — see `rebuild_children`'s `intern_owned` use for - // this variant), so a branch has no `Rc` identity of its own to - // dedup on at this position; still recurse into each in case an - // `Rc`-shared descendant appears further down. - Concat { children, .. } => children.iter().map(|c| count_unique(c, seen)).sum(), - Join { left, right, .. } | SetOp { left, right, .. } => { - visit(left, seen) + visit(right, seen) - } - BinaryOp { lhs, rhs, .. } => visit(lhs, seen) + visit(rhs, seen), - // Scalar variants (issue #205) — never descended into, matching - // `rebuild_children`'s own scope exactly (see its trailing match - // arm and this module's "Algorithm" section). - Column(_) - | Literal(_) - | Compare { .. } - | BoolAnd(_) - | BoolOr(_) - | Not(_) - | IsNull(_) - | IsNotNull(_) - | Cast { .. } - | InList { .. } - | FunctionCall { .. } - | Arithmetic { .. } - | Case { .. } => 0, - } -} - -/// Recurse into `child`, then intern the result. `Rc::try_unwrap` recovers -/// the owned node without cloning in the overwhelmingly common case — a -/// DAG freshly built by a front end / `resolve_root`, not yet shared by any -/// prior CSE pass, where every `Rc` is uniquely owned. Falls back to cloning -/// this node's own fields (its children stay `Rc`s, not deep-copied) only -/// when `child` is already shared — e.g. re-running CSE over a DAG that -/// went through a previous `share_common_sub_dags` pass; a structural -/// duplicate collapses right back onto `child` itself via `PartialEq`, an -/// already-optimal no-op. -fn intern_child(table: &mut InternTable, child: Rc) -> Rc { - match Rc::try_unwrap(child) { - Ok(owned) => intern_bottom_up(table, owned), - Err(shared) => intern_bottom_up(table, (*shared).clone()), - } -} - -/// Like [`intern_child`], for a `Concat` branch — stored by value -/// (`Vec`, not `Rc`), so this position itself can never -/// alias another parent. Interning it anyway still lets any `Rc`-typed -/// descendant of the branch participate in sharing, and registers the -/// branch's own hash/value in the table for a *different* `Concat` elsewhere -/// with a structurally identical branch (which — being in its own `Vec` -/// slot too — still can't literally share the `Rc`, but this keeps the -/// interning behavior uniform and the table's bucket contents consistent). -fn intern_owned(table: &mut InternTable, expr: QueryExpr) -> QueryExpr { - let rc = intern_bottom_up(table, expr); - Rc::try_unwrap(rc).unwrap_or_else(|shared| (*shared).clone()) -} - -/// Bottom-up: rebuild `expr`'s children (recursively interning each), then -/// intern the rebuilt node itself. -fn intern_bottom_up(table: &mut InternTable, expr: QueryExpr) -> Rc { - let rebuilt = rebuild_children(table, expr); - table.intern(rebuilt) -} - -/// Rebuild `expr` with each **operator** child (see the module doc on scope) -/// replaced by its interned `Rc`. Exhaustive over every `QueryExpr` variant, -/// matching `canonicalize.rs`'s `children_mut` exactly in which fields count -/// as an operator child — new variants fail to compile here until this match -/// is extended. -fn rebuild_children(table: &mut InternTable, expr: QueryExpr) -> QueryExpr { - use QueryExpr::*; - match expr { - Scan { .. } | EvalTimestamp | CurrentTimestamp => expr, - PromqlVectorFromScalar(c) => PromqlVectorFromScalar(intern_child(table, c)), - PromqlScalarFromVector(c) => PromqlScalarFromVector(intern_child(table, c)), - PromqlRelabel { dst, value, child } => PromqlRelabel { - dst, - value, - child: intern_child(table, child), - }, - PromqlInfoEnrich { selector, child } => PromqlInfoEnrich { - selector, - child: intern_child(table, child), - }, - PromqlSeriesSample { by, kind, child } => PromqlSeriesSample { - by, - kind, - child: intern_child(table, child), - }, - Filter { pred, child } => Filter { - pred, - child: intern_child(table, child), - }, - Project { - cols, - qualifier, - child, - } => Project { - cols, - qualifier, - child: intern_child(table, child), - }, - Aggregate { - reduction, - measures, - output_names, - filters, - having, - child, - } => Aggregate { - reduction, - measures, - output_names, - filters, - having, - child: intern_child(table, child), - }, - Dedup { cols, child } => Dedup { - cols, - child: intern_child(table, child), - }, - Concat { - children, - discriminator_unique_key, - } => Concat { - children: children - .into_iter() - .map(|c| intern_owned(table, c)) - .collect(), - discriminator_unique_key, - }, - Join { - kind, - pred, - left, - right, - } => Join { - kind, - pred, - left: intern_child(table, left), - right: intern_child(table, right), - }, - SetOp { - kind, - all, - left, - right, - } => SetOp { - kind, - all, - left: intern_child(table, left), - right: intern_child(table, right), - }, - Sort { - keys, - partition_by, - child, - } => Sort { - keys, - partition_by, - child: intern_child(table, child), - }, - Limit { n, offset, child } => Limit { - n, - offset, - child: intern_child(table, child), - }, - PromqlSubquery { - range, - resolution, - child, - } => PromqlSubquery { - range, - resolution, - child: intern_child(table, child), - }, - TimeRange { range, child } => TimeRange { - range, - child: intern_child(table, child), - }, - TimeShift { shift, child } => TimeShift { - shift, - child: intern_child(table, child), - }, - SQLWindowFunc { - func, - args, - partition_by, - order_by, - frame, - output_name, - child, - } => SQLWindowFunc { - func, - args, - partition_by, - order_by, - frame, - output_name, - child: intern_child(table, child), - }, - BinaryOp { - op, - lhs, - rhs, - vector_match, - } => BinaryOp { - op, - lhs: intern_child(table, lhs), - rhs: intern_child(table, rhs), - vector_match, - }, - // `PromqlScalarBridge`'s child is a scalar-sub-language node (issue - // #220) — same "never descended into" treatment as the scalar - // variants below; the whole bridge node is still interned as a unit - // by the `table.intern(rebuilt)` call in `intern_bottom_up`. - PromqlScalarBridge(_) => expr, - // Scalar variants (issue #205) — never descended into; see the - // module doc's "Algorithm" section on scope. Left byte-for-byte - // unchanged: predicate / project-list / sort-key / window-arg - // expressions stay embedded as opaque leaf data, compared by the - // enclosing operator node's derived `PartialEq`. - Column(_) - | Literal(_) - | Compare { .. } - | BoolAnd(_) - | BoolOr(_) - | Not(_) - | IsNull(_) - | IsNotNull(_) - | Cast { .. } - | InList { .. } - | FunctionCall { .. } - | Arithmetic { .. } - | Case { .. } => expr, - } -} - -/// Share structurally-identical, sharing-legal sub-DAGs across a workload's -/// query roots (or within one query, for `roots.len() == 1` — see the -/// module doc's "Single-query CSE" section). Every root's *value* is -/// unchanged (`PartialEq`-equal to its input) — only its internal `Rc` -/// structure may now alias another root's, or another part of its own DAG. -/// -/// `roots` must already be bound + canonicalized (post-`resolve_root`). -/// `Id` is caller-chosen — a `QueryWorkload` entry's own key, an index, a -/// query name, whatever identifies one root through the pipeline; this -/// module has no opinion on its shape. -pub fn share_common_sub_dags(roots: Vec<(Id, QueryExpr)>) -> Vec<(Id, Rc)> { - let mut table = InternTable::new(); - roots - .into_iter() - .map(|(id, expr)| (id, intern_bottom_up(&mut table, expr))) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pre_asap::agg_intent::AggIntent; - use crate::pre_asap::expr_ir::{CompareOpKind, ScalarValue}; - use crate::pre_asap::query_expr::{BinaryOpKind, GroupKeys, Predicate, Reduction, Source}; - use crate::pre_asap::schema::{DataType, Field, Schema}; - use crate::types::AccuracyTarget; - - /// `[ts, service, value, latency]`. - fn scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - Field::plain("latency", DataType::Float64, false), - ], - 0, - vec![], - ), - } - } - - fn quantile_agg(by: Vec, col: Option, q: f64) -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::by(by), - measures: vec![AggIntent::Quantile { - col, - q, - accuracy: AccuracyTarget::Exact, - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan()), - } - } - - #[test] - fn distinct_column_quantiles_do_not_merge() { - // Grouped (unique_keys present) so the legality gate isn't what's - // blocking the merge — only the differing `col` is. - let a = quantile_agg(vec![1], Some(2), 0.5); - let b = quantile_agg(vec![1], Some(3), 0.5); - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!( - !Rc::ptr_eq(ra, rb), - "distinct-column Quantiles must not be shared" - ); - assert_ne!(ra, rb); - } - - // Two aggregates that differ only in one measure's `FILTER` predicate - // compute different values, so structural sharing must keep them apart. - #[test] - fn filtered_and_unfiltered_aggregates_do_not_merge() { - let a = quantile_agg(vec![1], Some(2), 0.5); - let mut b = quantile_agg(vec![1], Some(2), 0.5); - let QueryExpr::Aggregate { filters, .. } = &mut b else { - unreachable!() - }; - *filters = vec![Some(Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(3)), - op: CompareOpKind::Gt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(1.0))), - })))]; - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!(!Rc::ptr_eq(ra, rb), "a filtered measure must not be shared"); - assert_ne!(ra, rb); - } - - #[test] - fn no_unique_keys_means_no_merge_even_when_structurally_identical() { - // Ungrouped (global) aggregate: `by` is empty, so - // `aggregate_output_schema` reports no unique key today — not - // hoistable even though `a` and `b` are structurally identical. - let a = quantile_agg(vec![], Some(2), 0.9); - let b = quantile_agg(vec![], Some(2), 0.9); - assert_eq!(a, b, "fixture sanity: the two DAGs are structurally equal"); - assert!( - !a.output_schema().unwrap().has_unique_key(), - "fixture sanity: an ungrouped aggregate has no provable unique key" - ); - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!( - !Rc::ptr_eq(ra, rb), - "no unique key ⇒ never hoisted, even for an identical structural match" - ); - } - - #[test] - fn median_and_explicit_half_percentile_merge() { - // Two front-end spellings ("median" and "approx_percentile_cont(., - // 0.5)") already lower to the identical `AggIntent::Quantile { q: - // 0.5, .. }` today (see `sql_lowering.rs`'s - // `median_is_the_same_intent_as_an_explicit_half_percentile`) — here - // built directly (grouped, so a unique key is provable) as two - // independently-constructed but structurally identical DAGs, the - // way two different call sites in a workload would produce them. - let median = quantile_agg(vec![1], Some(2), 0.5); - let approx_percentile_cont_half = quantile_agg(vec![1], Some(2), 0.5); - let shared = share_common_sub_dags(vec![ - ("median", median), - ("percentile", approx_percentile_cont_half), - ]); - let [(_, m), (_, p)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!( - Rc::ptr_eq(m, p), - "median and an explicit 0.5 percentile must merge onto one Rc" - ); - } - - #[test] - fn single_query_shares_its_own_repeated_sub_dag() { - // One query root referencing the same grouped Aggregate on both - // BinaryOp branches — built as two separately-allocated but - // structurally identical sub-DAGs (`.clone()` into two distinct - // `Rc::new` calls), the shape a front end emitting a repeated - // sub-expression would actually produce (no sharing yet). A - // workload of size 1 still interns bottom-up within this one DAG — - // no separate single-query mechanism needed. - let agg = quantile_agg(vec![1], Some(2), 0.5); - let root = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(crate::pre_asap::expr_ir::CompareOpKind::Eq), - lhs: Rc::new(agg.clone()), - rhs: Rc::new(agg), - vector_match: None, - }; - let shared = share_common_sub_dags(vec![("q", root)]); - let [(_, root)] = shared.as_slice() else { - panic!("expected 1 root"); - }; - let QueryExpr::BinaryOp { lhs, rhs, .. } = root.as_ref() else { - panic!("expected BinaryOp root, got {root:?}"); - }; - assert!( - Rc::ptr_eq(lhs, rhs), - "the two structurally identical branches must collapse onto one Rc" - ); - } - - // ── structural_hash (DAG-aware memoization) ───────────────────────── - - #[test] - fn structural_hash_is_stable_across_cache_states() { - // The hash of a given *value* must not depend on whether its cache - // started warm or cold — memoization changes how much work is - // redone, never what a node's hash actually is. - let agg = quantile_agg(vec![1], Some(2), 0.5); - let mut cold = HashMap::new(); - let mut warm = HashMap::new(); - // Prime `warm` with an unrelated node first, so it's non-empty but - // holds nothing relevant to `agg`. - structural_hash(&scan(), &mut warm); - assert_eq!( - structural_hash(&agg, &mut cold), - structural_hash(&agg, &mut warm), - "hash must be independent of unrelated cache state" - ); - } - - #[test] - fn structural_hash_of_an_internally_shared_dag_matches_the_unshared_equivalent() { - // The same BinaryOp-with-shared-branches shape as - // `dag_node_count_deduplicates_an_internally_shared_sub_dag` below: - // hashing it (however the memoization internally short-circuits the - // second branch) must produce the exact same value as hashing a - // structurally-identical DAG built with *no* sharing at all — the - // whole point of memoization is not changing the answer, only the - // work needed to reach it. - let agg = quantile_agg(vec![1], Some(2), 0.5); - let shared_root = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(crate::pre_asap::expr_ir::CompareOpKind::Eq), - lhs: Rc::new(agg.clone()), - rhs: Rc::new(agg.clone()), - vector_match: None, - }; - let unshared_root = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(crate::pre_asap::expr_ir::CompareOpKind::Eq), - lhs: Rc::new(agg.clone()), - rhs: Rc::new(agg), // a second, independently-allocated Rc with an equal value - vector_match: None, - }; - let mut cache = HashMap::new(); - assert_eq!( - structural_hash(&shared_root, &mut cache), - structural_hash(&unshared_root, &mut HashMap::new()), - ); - } - - #[test] - fn structural_hash_memoizes_a_shared_descendant_exactly_once() { - // Direct proof the cache is actually doing its job: hashing a - // BinaryOp whose two branches are the *same* Rc (2 underlying - // nodes: Scan + Aggregate) should populate the cache with exactly - // 2 entries — the shared branch's nodes, cached once each when - // first reached — not a fresh entry (or a fresh, redundant - // recursive walk) for the second occurrence. - let agg = quantile_agg(vec![1], Some(2), 0.5); - let shared = Rc::new(agg); - let root = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(crate::pre_asap::expr_ir::CompareOpKind::Eq), - lhs: Rc::clone(&shared), - rhs: Rc::clone(&shared), - vector_match: None, - }; - let mut cache = HashMap::new(); - structural_hash(&root, &mut cache); - assert_eq!( - cache.len(), - 2, - "expected exactly one cache entry per unique node in the shared \ - branch (Aggregate + its Scan child), got {} entries: {:?}", - cache.len(), - cache - ); - } - - // ── dag_node_count ─────────────────────────────────────────────────── - - #[test] - fn dag_node_count_is_the_naive_count_when_nothing_is_shared() { - // scan() alone: 1 node. - assert_eq!(dag_node_count(&scan()), 1); - // quantile_agg's own child is a fresh, unshared scan(): 2 nodes. - assert_eq!(dag_node_count(&quantile_agg(vec![1], Some(2), 0.5)), 2); - } - - #[test] - fn dag_node_count_deduplicates_an_internally_shared_sub_dag() { - // Same shape as `single_query_shares_its_own_repeated_sub_dag`: a - // BinaryOp whose two branches are the *same* Rc after - // `share_common_sub_dags` (2 nodes: Scan + Aggregate) — the root - // itself makes 3 unique nodes total (BinaryOp, Aggregate, Scan), - // not 5 (which a per-path walk / naive serialization, counting the - // shared branch's 2 nodes twice, would report). - let agg = quantile_agg(vec![1], Some(2), 0.5); - let root = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(crate::pre_asap::expr_ir::CompareOpKind::Eq), - lhs: Rc::new(agg.clone()), - rhs: Rc::new(agg), - vector_match: None, - }; - let shared = share_common_sub_dags(vec![("q", root)]); - let [(_, root)] = shared.as_slice() else { - panic!("expected 1 root"); - }; - assert_eq!( - dag_node_count(root), - 3, - "the shared branch's 2 nodes must be counted once, not once per \ - occurrence — got {} for {root:?}", - dag_node_count(root) - ); - } - - #[test] - fn dag_node_count_deduplicates_across_two_workload_roots() { - // Two workload roots sharing one Aggregate after - // `share_common_sub_dags` (the `duplicate_workload_queries_...` - // shape from `crates/integration-tests/tests/cse.rs`, built - // directly here): each root's own `dag_node_count` must report the - // shared sub-DAG's real size once, not double-count anything — - // there's nothing *to* double-count from a single root's own count - // in this case (no root references the shared node twice), so this - // pins the simpler, more common case that a per-candidate cost - // proxy (`CseCandidate::sub-DAG` in `asap-aware-mapping`) actually - // exercises: counting one occurrence's own reachable DAG size. - let a = quantile_agg(vec![1], Some(2), 0.5); - let b = quantile_agg(vec![1], Some(2), 0.5); - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!(Rc::ptr_eq(ra, rb), "fixture sanity: the two roots merged"); - assert_eq!(dag_node_count(ra), 2); - assert_eq!(dag_node_count(rb), 2); - } - - #[test] - fn dedup_gates_sharing_the_same_as_aggregate() { - // `Dedup { cols }` adds `cols` as a unique key — so two identical - // `Dedup` sub-DAGs over a keyed column *do* merge, exercising the - // legality gate on a non-`Aggregate` node. - let dedup = |cols: Vec| QueryExpr::Dedup { - cols, - child: Rc::new(scan()), - }; - let a = dedup(vec![1]); - let b = dedup(vec![1]); - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!( - Rc::ptr_eq(ra, rb), - "Dedup on the same cols has a provable unique key and should merge" - ); - } - - #[test] - fn group_keys_gate_still_prevented_when_partition_by_without_used() { - // Sanity on the module's advertised precedent: a `without(...)` - // grouping stays open (no unique key) even though `by` is - // non-empty-shaped structurally, so two identical `without` groups - // do not merge under the same gate that blocks the ungrouped case. - let without_agg = || QueryExpr::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![0])), - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan()), - }; - let a = without_agg(); - let b = without_agg(); - assert!(!a.output_schema().unwrap().has_unique_key()); - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!(!Rc::ptr_eq(ra, rb)); - } -} diff --git a/crates/types/src/pre_asap/expr_ir.rs b/crates/types/src/pre_asap/expr_ir.rs index 21ffe21bf..fe373e58f 100644 --- a/crates/types/src/pre_asap/expr_ir.rs +++ b/crates/types/src/pre_asap/expr_ir.rs @@ -1,29 +1,20 @@ -//! Column-reference and scalar-operator vocabulary shared by the whole -//! canonical [`QueryExpr`](super::query_expr::QueryExpr) DAG. -//! -//! Issue #205: the scalar expression shapes (`Column`/`Literal`/`Compare`/…) -//! used to live in a separate, self-recursive `Expr` DAG here, reachable -//! from `QueryExpr` only through wrapper fields (`Predicate`, `ProjectItem`, -//! `SortKey`). They're variants of `QueryExpr` itself now — one recursive -//! DAG, not two type families joined by wrappers — generic over the same -//! column-reference state `C` the rest of `QueryExpr` already carries -//! (issue #179): [`ColumnRef`] (name-based, front-end-emitted) or -//! [`ColumnId`](super::schema::ColumnId) (positional, once bound). -//! -//! What's left here is the vocabulary those scalar variants are built from — -//! [`ScalarValue`], [`CompareOpKind`], [`ArithmeticOpKind`] — the **union** of what the two +//! Column-reference and scalar-operator vocabulary shared by the IR's scalar +//! expressions ([`crate::ir::ScalarExpr`]) and the front ends' unresolved +//! form: [`ColumnRef`] (name-based, front-end-emitted; positional +//! [`ColumnId`](super::schema::ColumnId) once bound), and [`ScalarValue`], +//! [`CompareOpKind`], [`ArithmeticOpKind`] — the **union** of what the two //! front ends need: PromQL contributes `Regex` / `NotRegex` (`=~` / `!~`); SQL //! contributes arithmetic, `CASE`, `IN`, `CAST`, `IS [NOT] NULL`, scalar //! function calls, and the `LIKE` / `ILIKE` comparison family. use serde::{Deserialize, Serialize}; -/// A name-based column reference — the front-end-emitted, unresolved state of -/// [`QueryExpr::Column`](super::query_expr::QueryExpr::Column) (`C = -/// ColumnRef`); the [`SchemaResolver`](super::schema_resolver::SchemaResolver) resolves it to a -/// positional [`ColumnId`](super::schema::ColumnId). This is a logical reference, -/// not schema metadata or a runtime data array. `SampleValue` names the implicit -/// PromQL sample column; `Wildcard` represents an all-columns/rows request. +/// A name-based column reference — the front-end-emitted, unresolved form of +/// [`ScalarExpr::Column`](crate::ir::ScalarExpr::Column); front-end name +/// resolution turns it into a positional [`ColumnId`](super::schema::ColumnId). +/// This is a logical reference, not schema metadata or a runtime data array. +/// `SampleValue` names the implicit PromQL sample column; `Wildcard` represents +/// an all-columns/rows request. #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum ColumnRef { Named(String), diff --git a/crates/types/src/pre_asap/mod.rs b/crates/types/src/pre_asap/mod.rs index 1a017ec27..c309c7b30 100644 --- a/crates/types/src/pre_asap/mod.rs +++ b/crates/types/src/pre_asap/mod.rs @@ -1,66 +1,34 @@ -//! The canonical pre-ASAP intent algebra IR. +//! Shared vocabulary of the operator IR. The operators themselves live in +//! [`crate::ir`]; this module holds the field types they are built from. //! -//! - [`query_expr`] — the canonical, language- and deployment-independent -//! intent algebra: one recursive [`QueryExpr`] DAG (relational operators -//! *and* scalar expression shapes both, since issue #205) + [`AggIntent`], -//! generic over the column-reference state (positional [`ColumnId`] once -//! bound, name-based [`ColumnRef`] before). -//! - [`agg_intent`] — the aggregation-intent vocabulary. -//! - [`expr_ir`] — the [`ColumnRef`] column-reference type and the scalar -//! operator/literal vocabulary ([`ScalarValue`], [`CompareOpKind`], [`ArithmeticOpKind`]) -//! [`QueryExpr`]'s scalar variants are built from. +//! Operator parameters and schema derivation live in [`crate::ir`]. +//! - [`agg_intent`] — the aggregation-intent vocabulary ([`AggIntent`]). +//! - [`expr_ir`] — [`ColumnRef`] and the scalar literal / operator kinds +//! ([`ScalarValue`], [`CompareOpKind`], [`ArithmeticOpKind`]). //! - [`schema`] — the per-edge [`Schema`] every node carries. -//! - [`schema_resolver`] / [`column_resolution`] — name resolution: turn a `ColumnRef` -//! into a positional `ColumnId` against an in-scope [`Schema`]. -//! - [`resolve`] — binds a whole front-end-emitted [`UnresolvedQueryExpr`] DAG to -//! canonical [`ResolvedQueryExpr`] (issue #179): both front ends -//! (`asap-frontend-promql`, `asap-frontend-sql`) construct `UnresolvedQueryExpr` -//! directly during their own `interpret` step and call -//! [`resolve_root`] on the result — there is no separate per-language -//! relational DAG or converter anymore. -//! - [`canonicalize`] — post-lowering structural normalization of [`QueryExpr`] -//! (issue #34), run by [`resolve_root`]. -//! - [`cse`] — workload-level structural common-subexpression elimination -//! over an already-`resolve_root`'d DAG (issue #212, #222, #223), run -//! *after* `resolve_root` / `canonicalize` and *before* implementation -//! (`asap_aware_mapping::replacement`). -//! -//! Formerly the separate `asap-l2` crate; folded in here since -//! `schema_resolver`/`column_resolution`/`canonicalize`/`resolve` have no -//! front-end-specific logic — they operate directly on this crate's own -//! `QueryExpr`. +//! - [`column_resolution`] — turn a name-based `ColumnRef` into a positional +//! `ColumnId` against a [`Schema`] (used by front-end name resolution). +//! - [`scalar_type_rules`] — shared scalar type and nullability rules. pub mod agg_intent; -pub mod canonicalize; pub mod column_resolution; -pub mod cse; pub mod expr_ir; -pub mod query_expr; -pub mod resolve; pub mod scalar_type_rules; pub mod schema; -pub mod schema_resolver; +pub use crate::ir::operator_properties::{ + AtModifier, BinaryOpKind, ColState, ConcatDiscriminatorKey, DataModel, GroupKeys, GroupSide, + InfoMatcher, JoinKind, PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, + Source, TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, + WindowFrameOffset, WindowFrameUnits, WindowFuncKind, +}; pub use agg_intent::{ agg_accuracy, agg_is_exact, agg_is_mergeable, default_cardinality, default_quantile, AggIntent, MathFunc, TimeFunc, }; -pub use canonicalize::canonicalize; -pub use column_resolution::{ - output_schema_for_aggregate, resolve_column_ref, resolve_column_refs, resolve_expr, - ResolveError, -}; -pub use cse::share_common_sub_dags; +pub use column_resolution::{resolve_column_ref, resolve_column_refs, ResolveError}; pub use expr_ir::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; -pub use query_expr::{ - aggregate_output_schema, any_measure_filtered, AtModifier, BinaryOpKind, ColState, DataModel, - GroupKeys, GroupSide, InfoMatcher, JoinKind, Predicate, ProjectItem, PromQLVectorSetOpKind, - QueryExpr, QueryExprError, Reduction, RelationalSetOpKind, ResolvedQueryExpr, SampleKind, - SortKey, Source, TimeShift, UnresolvedQueryExpr, VectorGrouping, VectorMatch, VectorMatchKind, - WindowFrame, WindowFrameBound, WindowFrameOffset, WindowFrameUnits, WindowFuncKind, -}; -pub use resolve::{resolve_root, ResolveDAGError}; pub use schema::{ColumnId, DataType, Field, FieldDataType, Schema}; -pub use schema_resolver::{SchemaCatalog, SchemaResolver, UsageDerivedCatalog}; +pub use crate::ir::aggregate_schema::aggregate_output_schema; pub use crate::ir::SchemaDerivationError; diff --git a/crates/types/src/pre_asap/query_expr.rs b/crates/types/src/pre_asap/query_expr.rs deleted file mode 100644 index e389f84a2..000000000 --- a/crates/types/src/pre_asap/query_expr.rs +++ /dev/null @@ -1,2774 +0,0 @@ -//! The canonical pre-ASAP intent algebra IR. -//! -//! Language- and deployment-independent. `Rc`-owned DAG — a child field is -//! `Rc>` rather than `Box>` so a structurally -//! identical sub-expression can be shared (the same `Rc`) across more than -//! one parent, within one query or across a `QueryWorkload` batch, instead of -//! being duplicated. Nothing in this module produces that sharing on its -//! own — construction still allocates a fresh `Rc` per node, the same shape -//! as the old `Box` DAG — a separate CSE pass is what turns two -//! independently constructed, structurally-equal sub-DAGs into two -//! references to one `Rc` (issue #212, #222). Field identity is -//! **positional** (`Aggregate.reduction: Reduction`, wrapping `GroupKeys` -//! for the grouped case), resolved by the [`SchemaResolver`](super::schema_resolver) against -//! the self-contained [`Schema`] carried on each `Scan`. - -use std::rc::Rc; -use std::time::Duration; - -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -use super::agg_intent::AggIntent; -use super::expr_ir::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; -use super::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; - -/// The column-reference resolution state a [`QueryExpr`] DAG carries — -/// [`ColumnId`] (the default, and what the bare `QueryExpr` name has always -/// meant) once the [`SchemaResolver`](super::schema_resolver::SchemaResolver) has resolved every -/// reference positionally, or the front-end-emitted, name-based [`ColumnRef`] -/// before binding. The only place the two states differ in *shape* rather -/// than just in which type fills `C` is [`QueryExpr::Scan`]'s `schema` field: -/// a bound DAG's binding schema is always known (the SchemaResolver is total, so -/// [`ScanSchema`](Self::ScanSchema) `= Schema`); an unresolved front-end -/// `Scan` knows its schema only when the front end already has it without -/// binding — a SQL leaf, catalog-backed (`Some`) — `None` (PromQL) defers to -/// the SchemaResolver, so `ScanSchema = Option`. -pub trait ColState: - Clone + std::fmt::Debug + PartialEq + Serialize + for<'de> Deserialize<'de> -{ - /// What [`QueryExpr::Scan`]'s `schema` field holds for a DAG in this state. - type ScanSchema: Clone + std::fmt::Debug + PartialEq + Serialize + for<'de> Deserialize<'de>; -} - -impl ColState for ColumnId { - type ScanSchema = Schema; -} - -impl ColState for ColumnRef { - type ScanSchema = Option; -} - -/// Errors from schema derivation over a canonical DAG. -#[derive(Debug, Error)] -pub enum QueryExprError { - #[error("invalid scalar function signature: {0}")] - InvalidScalarSignature(String), - #[error("by-column id {0} out of range (input has {1} columns)")] - InvalidGroupByColumn(ColumnId, usize), - #[error("Concat requires at least one child")] - EmptyConcat, - /// [`QueryExpr::output_schema`] called on (or reached, while recursing, a - /// child that is) one of the scalar variants (issue #205) — those have no - /// independent row schema of their own; a scalar expression's *type* only - /// makes sense against the schema it's embedded in (see `infer_expr_type`, - /// used by `Project`'s own `output_schema` arm instead). - #[error("a scalar expression has no row schema of its own")] - ScalarHasNoRowSchema, - #[error("invalid per-series sample column: {0}")] - InvalidSampleColumn(String), -} - -// ── Leaf / supporting types ─────────────────────────────────────────────────── - -/// Positional grouping keys, shared by every "operate per group" operator: -/// `Aggregate.by` (reduce per group), `Sort.partition_by` (rank per group — -/// including generic `topk`/`bottomk`), and `SQLWindowFunc.partition_by` (window -/// per group). One spelling so grouping has a single home to evolve. Empty -/// (and `by`) = no grouping (a global operation). -/// -/// Heavy-hitter `AggIntent::TopK` carries its grouping here too, via the -/// enclosing `Aggregate.by` (issue #13) — so reduce, rank, and window groupings -/// all share this one type. -/// -/// ## `by` vs `without` (issue #39) -/// -/// The stored [`keys`](Self::keys) are **kept** labels for `by(...)` and -/// **excluded** labels for `without(...)`. PromQL's `without(labels)` groups by -/// every label *except* those listed; the complement can't be enumerated at -/// lowering time under an open (usage-derived) schema, so it is deferred to the -/// runtime — the excluded positions are stored, the kept set stays open. Only -/// `Aggregate` ever produces the `without` form; `Sort` / `SQLWindowFunc` / -/// `PromqlSeriesSample` groupings are always `by`. -/// -/// Serialises as a bare array for the (overwhelmingly common) `by` case — -/// wire-compatible with the `Vec` this field held before — and as -/// `{"without": [...]}` for the exclusion case. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct GroupKeys { - keys: Vec, - without: bool, -} - -// Not `#[derive(Default)]`: derive would add a `C: Default` bound, but an -// empty key set needs nothing from `C` — `ColumnRef` has no meaningful -// default anyway. -impl Default for GroupKeys { - fn default() -> Self { - Self { - keys: Vec::new(), - without: false, - } - } -} - -impl GroupKeys { - /// An empty key set — a global (ungrouped) operation. - pub fn none() -> Self { - Self::default() - } - /// `by(keys)` — group by exactly these columns. - pub fn by(keys: Vec) -> Self { - Self { - keys, - without: false, - } - } - /// `without(keys)` — group by every label *except* these (issue #39). The - /// kept set is runtime-resolved; only the excluded positions are stored. - pub fn without(keys: Vec) -> Self { - Self { - keys, - without: true, - } - } - /// Whether this is a `without(...)` exclusion grouping. - pub fn is_without(&self) -> bool { - self.without - } - /// The named keys — kept labels for `by`, excluded labels for `without`. - pub fn keys(&self) -> &[C] { - &self.keys - } -} - -impl std::ops::Deref for GroupKeys { - type Target = [C]; - fn deref(&self) -> &Self::Target { - &self.keys - } -} - -impl From> for GroupKeys { - fn from(keys: Vec) -> Self { - Self::by(keys) - } -} - -impl FromIterator for GroupKeys { - fn from_iter>(iter: I) -> Self { - Self::by(iter.into_iter().collect()) - } -} - -impl<'a, C> IntoIterator for &'a GroupKeys { - type Item = &'a C; - type IntoIter = std::slice::Iter<'a, C>; - fn into_iter(self) -> Self::IntoIter { - self.keys.iter() - } -} - -/// Compare directly against a `Vec` so call sites and tests can keep -/// writing `keys == vec![..]` / `assert_eq!(keys, &vec![..])`. A `without` -/// grouping never equals a bare `by` list. -impl PartialEq> for GroupKeys { - fn eq(&self, other: &Vec) -> bool { - !self.without && &self.keys == other - } -} - -/// (De)serialise as a bare array for `by`, or `{"without": [...]}` for the -/// exclusion form — keeping the `by` wire format identical to the old newtype. -/// Borrowed for `Serialize` (no `C: Clone` needed to write one out), owned for -/// `Deserialize` (there's nothing to borrow from). -#[derive(Serialize)] -#[serde(untagged)] -enum GroupKeysReprRef<'a, C> { - By(&'a [C]), - Without { without: &'a [C] }, -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum GroupKeysRepr { - By(Vec), - Without { without: Vec }, -} - -impl Serialize for GroupKeys { - fn serialize(&self, serializer: S) -> Result { - if self.without { - GroupKeysReprRef::Without { - without: self.keys.as_slice(), - } - .serialize(serializer) - } else { - GroupKeysReprRef::By(self.keys.as_slice()).serialize(serializer) - } - } -} - -impl<'de, C: Deserialize<'de>> Deserialize<'de> for GroupKeys { - fn deserialize>(deserializer: D) -> Result { - Ok(match GroupKeysRepr::deserialize(deserializer)? { - GroupKeysRepr::By(keys) => Self::by(keys), - GroupKeysRepr::Without { without } => Self::without(without), - }) - } -} - -/// Which data model a `Source` / `AggIntent` operates over. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum DataModel { - TimeSeries, - Tabular, - Any, -} - -/// The leaf data source of a `Scan`. The schema itself rides on the -/// `Scan.schema` field (SchemaResolver-built); `Source` carries only the leaf's -/// identity. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum Source { - /// Time-series leaf — PromQL / DC lifecycle. Produces `(ts, value, *labels)`. - TimeSeries { metric: String }, - /// Tabular leaf — asap-fusion / future OLAP. Columns ride on `Scan.schema`. - Table { table_ref: String }, -} - -impl Source { - pub fn data_model(&self) -> DataModel { - match self { - Source::TimeSeries { .. } => DataModel::TimeSeries, - Source::Table { .. } => DataModel::Tabular, - } - } -} - -/// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's -/// [`ArithmeticOpKind`] / [`CompareOpKind`] so every arithmetic/comparison -/// operator has exactly one representation (and one `Display`) across the IR. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum BinaryOpKind { - /// Arithmetic — `Add/Sub/Mul/Div/Mod` (shared with `QueryExpr::Arithmetic`). - Arithmetic(ArithmeticOpKind), - /// Comparison — `Eq/Ne/Lt/Le/Gt/Ge` + `Like/ILike/Regex` family (shared - /// with `QueryExpr::Compare`). PromQL keeps the matched series whose - /// comparison holds. - Compare(CompareOpKind), - /// PromQL comparison with the `bool` modifier: every matched series - /// yields 1 or 0 and loses its metric name. A separate variant, not a - /// flag, because only comparisons take `bool`. - CompareBool(CompareOpKind), - /// PromQL vector-set operation. - Set(PromQLVectorSetOpKind), -} - -impl std::fmt::Display for BinaryOpKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - BinaryOpKind::Arithmetic(op) => write!(f, "{op}"), - BinaryOpKind::Compare(op) => write!(f, "{op}"), - BinaryOpKind::CompareBool(op) => write!(f, "{op} bool"), - BinaryOpKind::Set(PromQLVectorSetOpKind::And) => f.write_str("AND"), - BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => f.write_str("OR"), - BinaryOpKind::Set(PromQLVectorSetOpKind::Unless) => f.write_str("unless"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum JoinKind { - Inner, - Left, - Right, - Full, - Cross, - /// Left semi-join — each left row that has **at least one** match, once. - /// `WHERE c IN (SELECT …)` / `WHERE EXISTS (…)` (issue #111). - /// - /// Output schema is the **left's alone**; the right side is a filter, not a - /// source of columns. The join predicate still resolves against the - /// concatenated `left ++ right` schema — its scope is deliberately wider - /// than the node's output. - Semi, - /// Left anti-join — each left row with **no** match. `WHERE NOT EXISTS (…)`. - /// Same schema rule as [`JoinKind::Semi`]. - /// - /// Note this is *not* `NOT IN (SELECT …)`: under SQL's three-valued logic a - /// NULL on the right makes `NOT IN` yield no rows at all, where an anti-join - /// yields every left row. The SQL front end rejects `NOT IN (subquery)` - /// rather than lower it here. - Anti, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum RelationalSetOpKind { - Union, - Intersect, - Except, -} - -/// PromQL vector-set operator used by [`BinaryOpKind::Set`]. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum PromQLVectorSetOpKind { - And, - Or, - Unless, -} - -/// SQL analytic window function (`fn(...) OVER (…)`). Distinct from a streaming -/// time `Window`: this is an analytic frame over already-materialised rows. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WindowFuncKind { - RowNumber, - Rank, - DenseRank, - Lag, - Lead, - /// ClickHouse `lagInFrame`/`leadInFrame`: unlike [`Lag`](Self::Lag)/[`Lead`](Self::Lead), - /// these respect the window frame bounds (NULL/default past the frame edge) - /// rather than reaching arbitrarily far back/forward. Kept as distinct - /// variants so the frame clause is never silently discarded by conflating - /// them with `Lag`/`Lead` (#267). `WindowFuncKind` still has no frame - /// representation, so today these lower and behave exactly like - /// `Lag`/`Lead` — the tag is correct, the frame-respecting behavior isn't - /// implemented yet. See #231 for modeling window frames properly. - LagInFrame, - LeadInFrame, - FirstValue, - LastValue, - /// `NTH_VALUE(expr, n)` — `n` is resolved from the (literal) 2nd argument. - NthValue(Option), - Sum, - Avg, - Count, - Min, - Max, -} - -/// A window's frame-spec (`ROWS`/`RANGE BETWEEN … AND …`) — which rows around -/// the current one an analytic window function reads. `GROUPS` is rejected at -/// lowering time (issue #268): every SQL corpus in this repo uses only `ROWS`, -/// and nothing downstream interprets frame semantics yet, so it isn't worth -/// modelling untested. -/// -/// Meaningless (but harmless) on the rank-only and navigation functions -/// (`ROW_NUMBER`/`RANK`/`DENSE_RANK`/`LAG`/`LEAD`), which ignore the frame per -/// SQL semantics — DataFusion still attaches one, stored here verbatim. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct WindowFrame { - pub units: WindowFrameUnits, - pub start_bound: WindowFrameBound, - pub end_bound: WindowFrameBound, -} - -/// A finite window-frame displacement. Intervals are normalized to Arrow's -/// month/day/nanosecond representation so SQL `RANGE INTERVAL ...` bounds -/// survive lowering without leaking DataFusion types into the canonical IR. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WindowFrameOffset { - Scalar(ScalarValue), - Interval { - months: i32, - days: i32, - nanoseconds: i64, - }, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WindowFrameUnits { - /// Boundaries count physical rows: `ROWS BETWEEN 2 PRECEDING AND CURRENT ROW`. - Rows, - /// Boundaries count by value-distance on the (single) `ORDER BY` column: - /// `RANGE BETWEEN INTERVAL '1' HOUR PRECEDING AND CURRENT ROW`. - Range, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WindowFrameBound { - /// `UNBOUNDED PRECEDING` is - /// `Preceding(WindowFrameOffset::Scalar(ScalarValue::Null))`. - Preceding(WindowFrameOffset), - CurrentRow, - /// `UNBOUNDED FOLLOWING` is - /// `Following(WindowFrameOffset::Scalar(ScalarValue::Null))`. - Following(WindowFrameOffset), -} - -/// A symbolic label matcher on the **info metric** side of an -/// [`QueryExpr::PromqlInfoEnrich`] (issue #84). Unlike a `Scan` predicate it is not -/// resolved positionally — it references the info metric's labels (`__name__` -/// picks the metric, the rest constrain data labels), which aren't in the input -/// vector's schema; the post-ASAP realization pass applies it against the info metric. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct InfoMatcher { - pub label: String, - /// One of `Eq` / `Ne` / `Regex` / `NotRegex` (PromQL `=`/`!=`/`=~`/`!~`). - pub op: CompareOpKind, - pub value: String, -} - -/// Series-sampling selection mode (PromQL `limitk` / `limit_ratio`, issue #86). -/// A [`QueryExpr::PromqlSeriesSample`] keeps a *subset of whole series*, unchanged — it does -/// not rank or reduce, so it is distinct from `TopK` and from `Sort → Limit`. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SampleKind { - /// `limitk(k, v)` — up to `k` series per group. Which series survive is - /// deterministic across evaluations but otherwise unspecified (no ordering). - LimitK(usize), - /// `limit_ratio(r, v)` — a deterministic `r`-fraction of series per group. - /// `r ∈ [-1, 1]`; a negative `r` selects the complementary fraction. - LimitRatio(f64), -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] -pub struct SortKey { - pub expr: QueryExpr, - pub ascending: bool, - pub nulls_first: bool, -} - -/// PromQL vector-match modifier (`on`/`ignoring` + `group_left`/`group_right`). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct VectorMatch { - pub kind: VectorMatchKind, - pub labels: Vec, - pub grouping: Option, -} - -/// PromQL `@` modifier — pins a selector's evaluation time to an anchor instead -/// of the query evaluation time (issue #40). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum AtModifier { - /// `@ start()` — the query range's start instant. - Start, - /// `@ end()` — the query range's end instant. - End, - /// `@ ` — an absolute instant, milliseconds since the Unix epoch (may be - /// negative). PromQL writes the timestamp in seconds; the front end scales it. - Timestamp(i64), -} - -/// PromQL per-selector **time-shift** modifiers — `offset` and `@` (issue #40). -/// Neither changes a selector's *schema*; both move *when* it is evaluated, so -/// the shift is a pass-through wrapper ([`QueryExpr::TimeShift`]) over the -/// selector rather than a new leaf shape. The runtime resolves the anchor and -/// applies the offset. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] -pub struct TimeShift { - /// `offset ` as signed milliseconds — a positive value shifts the - /// lookback *back* in time (`offset 5m`), a negative value shifts it - /// *forward* (`offset -5m`). `0` = no offset. - pub offset_ms: i64, - /// `@` anchor; `None` = evaluate at the query time. - pub at: Option, -} - -impl TimeShift { - /// Whether this shift is the identity (no `offset`, no `@`) — the state of - /// every selector that carries neither modifier. - pub fn is_identity(&self) -> bool { - self.offset_ms == 0 && self.at.is_none() - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum VectorMatchKind { - On, - Ignoring, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct VectorGrouping { - pub side: GroupSide, - pub labels: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum GroupSide { - Left, - Right, -} - -/// A row-level filter predicate (WHERE clause / PromQL label matcher). -/// Boxed: `Predicate` sits directly (not behind a `Vec`) in -/// `Filter.pred`/`Join.pred`/`Aggregate.having`, and `QueryExpr` is -/// self-recursive without further indirection once the scalar variants are -/// part of it — the box is what makes the recursive type's size finite there. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] -pub struct Predicate(pub Rc>); - -/// Whether any entry of an `Aggregate.filters` vector is set — the shape -/// no binding rule accepts yet (issue #466): a filtered measure stays -/// `KeepPreAsap`, and heavy-hitter promotion skips it. -pub fn any_measure_filtered(filters: &[Option>]) -> bool { - filters.iter().any(Option::is_some) -} - -/// One item in a SELECT projection list. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] -pub struct ProjectItem { - pub alias: Option, - pub expr: QueryExpr, -} - -// ── Intent algebra IR ──────────────────────────────────────────────────────── - -/// What kind of computation an `Aggregate` node performs — orthogonal to -/// *which* columns it groups by (that's still [`GroupKeys`], inside -/// `Reduce`). Explicit, decided once by whichever pass constructs the node -/// (structural, at front-end lowering time), rather than inferred downstream from -/// whether a grouping-key list happens to be empty or from a neighboring -/// node's shape. See design proposal #165. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum Reduction { - /// Collapses input rows via `by` — `by`/`without` semantics are exactly - /// [`GroupKeys`]'s. May still collapse every row into one (an empty, - /// non-`without` `by`) — that's a genuine reduction with zero grouping - /// columns, not "no grouping concept." - Reduce(GroupKeys), - /// No grouping concept at all: preserves one output row per input - /// entity (e.g. a per-series windowed computation with no `by(...)` - /// clause to begin with, because there's no aggregation operator here - /// for such a clause to attach to). Never merges across entities, and - /// never collapses an entity's own row structure (e.g. a time axis) — - /// unlike `Reduce(GroupKeys::without(vec![]))` ("group by every - /// label"), which is still a genuine reduction and does collapse it. - PerEntity, -} - -impl Reduction { - /// Shorthand for the common case — group by these (possibly empty) - /// keys, kept rather than excluded. - pub fn by(keys: Vec) -> Self { - Self::Reduce(GroupKeys::by(keys)) - } - - /// The grouping keys, if this is a genuine reduction — `None` for - /// `PerEntity`, which has no grouping-keys concept to report. - pub fn group_keys(&self) -> Option<&GroupKeys> { - match self { - Self::Reduce(by) => Some(by), - Self::PerEntity => None, - } - } - - /// The grouping keys, panicking if this is `PerEntity` — for call sites - /// (tests, mostly) that already know, from the shape they built or are - /// asserting on, that this must be a genuine reduction. Prefer - /// [`group_keys`](Self::group_keys) wherever the caller can't assume that. - pub fn expect_reduce(&self) -> &GroupKeys { - match self { - Self::Reduce(by) => by, - Self::PerEntity => panic!("expected Reduction::Reduce, got PerEntity"), - } - } -} - -/// A caller-proven compound unique key for a [`QueryExpr::Concat`] (issue -/// #228) — built only via [`QueryExpr::concat_with_discriminator`] / -/// [`ConcatDiscriminatorKey::new`], never by naming `discriminator` directly -/// in a struct literal (both fields are private): from *other Rust code*, -/// the only way to end up with one of these is to hand over a specific -/// column as the discriminator, by name, at the call site. -/// -/// Caveat: this is a Rust-API-level guarantee, not a data-level one. The -/// derived `Deserialize` impl below builds a `ConcatDiscriminatorKey` -/// directly from field values, bypassing `new()`. Deserialization is therefore -/// equivalent to a caller supplying the assertion directly; it does not prove -/// either fact below. An external boundary accepting `QueryExpr` data must -/// reject this field or validate both obligations before treating it as -/// uniqueness evidence. -/// -/// # Soundness -/// -/// `Concat`'s default (see its own doc) is to drop `unique_keys` -/// unconditionally, because a key unique **within** one branch is not unique -/// **across** the concatenation unless the branches' value sets for that key -/// are provably disjoint — nothing about matching schemas or matching -/// per-branch keys establishes that on its own. Two different branches can -/// trivially emit the same `inner_key` value (e.g. two PromQL -/// `histogram_quantiles` branches keyed on `(host, le)` can both produce a -/// `(host, le)` pair for different φ). -/// -/// Prepending `discriminator` restores a compound key only when two facts -/// hold: `inner_key` uniquely identifies rows **within every branch**, and -/// `discriminator`'s value is **guaranteed to differ between branches** — a -/// literal the producer just tagged the branch with (PromQL φ riding along via -/// [`QueryExpr::PromqlRelabel`], a Postgres-style synthetic `GROUPING()` id -/// for `ROLLUP`/`CUBE`, …), never something inferred structurally from the -/// branches' own data — then `discriminator` alone partitions rows into -/// disjoint sets independent of what the branches actually contain, so -/// `(discriminator, inner_key)` is sound even when otherwise-identical -/// `inner_key` values occur in different branches. Neither fact is verified -/// here; both are part of the caller-proven claim. -/// -/// This is a **caller-proven claim, not something `Concat` can verify**: -/// nothing stops a caller from asserting a discriminator that in fact -/// repeats across branches, in which case the resulting `unique_keys` claim -/// is simply wrong — `output_schema` trusts it without checking. The -/// obligation is on the constructor call site, exactly as it is on -/// [`QueryExpr::Dedup`]'s `cols` or any other unverified `unique_keys` -/// producer in this module. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] -pub struct ConcatDiscriminatorKey { - discriminator: C, - inner_key: Vec, -} - -impl ConcatDiscriminatorKey { - /// The only constructor — `discriminator` must be named explicitly by - /// the caller. See the type's doc for the soundness obligation this - /// puts on that caller. - pub fn new(discriminator: C, inner_key: Vec) -> Self { - Self { - discriminator, - inner_key, - } - } - - pub fn discriminator(&self) -> &C { - &self.discriminator - } - - pub fn inner_key(&self) -> &[C] { - &self.inner_key - } -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] -pub enum QueryExpr { - /// Outermost leaf. `schema` is the **binding schema** — the resolved column - /// set every positional `ColumnId` in the DAG indexes into, *not* a full - /// description of the runtime row — once bound (`schema: Schema`, always - /// present: the [`SchemaResolver`](super::schema_resolver) is total). Before binding, a - /// front-end-emitted `Scan` (`C = ColumnRef`) knows it only when the front - /// end already has it without binding — a catalog-backed SQL leaf — `None` - /// (PromQL) defers to the SchemaResolver; see [`ColState::ScanSchema`]. Complete - /// when catalog-backed (SQL); for schemaless PromQL the bound schema is - /// usage-derived (the `(ts, value)` floor + the labels the query - /// references), since a metric's label set is open and known only at - /// runtime. That distinction is carried explicitly by - /// [`Schema::closed`](super::schema::Schema::closed) (SQL leaf → `true`, - /// PromQL leaf → `false`). `predicates` are leaf-level row filters (PromQL - /// label matchers, pushed-down `WHERE` conjuncts). - Scan { - source: Source, - #[serde(default)] - predicates: Vec>, - schema: C::ScanSchema, - }, - /// A scalar sub-expression sitting in an **operator-DAG position** — a - /// [`BinaryOp`](Self::BinaryOp) operand for ` op ` - /// thresholds / unit conversions (#35), a - /// [`PromqlVectorFromScalar`](Self::PromqlVectorFromScalar) child, or a - /// whole query's root (a bare PromQL scalar query, e.g. `5`). - /// - /// Formerly its own leaf variant, `PromqlScalar(f64)`. Issue #220: that - /// variant held exactly the same value [`Literal`](Self::Literal) does - /// (every PromQL scalar is `f64`), duplicating it for no reason but - /// *which DAG position* it was allowed to appear in. This wrapper - /// carries that position instead of the value — the inner node is an - /// ordinary scalar sub-language expression (in practice always - /// `Literal(ScalarValue::Float64(_))`, since a front end only ever - /// constructs this fully constant-folded — see - /// [`promql_scalar`](Self::promql_scalar)) — and is what `output_schema`, - /// `canonicalize`, and `resolve` now key off to tell "this operand has - /// its own row schema" from "this is a nested scalar leaf with none," - /// in place of the old `PromqlScalar` vs. `Literal` variant tag. - PromqlScalarBridge(Rc>), - - /// The query **evaluation timestamp** as Unix seconds, exposed by PromQL - /// `time()`. This is not inherently the current wall-clock time: its value - /// is the instant or range-step at which the expression is evaluated. It - /// is also the implicit input of no-argument calendar functions. Issue #46. - EvalTimestamp, - - /// The SQL statement evaluation time (`NOW()` / `CURRENT_TIMESTAMP`) as - /// a SQL [`DataType::Timestamp`]. Kept distinct from [`EvalTimestamp`], - /// whose PromQL `time()` contract is Unix seconds as `Float64`. - CurrentTimestamp, - - /// PromQL `vector(s)` — the scalar→instant-vector bridge. Promotes a - /// scalar-typed child to a single label-less series carrying the scalar's - /// value at every step. Lets a scalar participate where a vector is required - /// (`up or vector(0)` dead-man's-switch). Issue #48. - PromqlVectorFromScalar(Rc>), - - /// PromQL `scalar(v)` — the instant-vector→scalar bridge. Collapses a - /// single-element vector to its value (NaN at runtime if the input is not - /// exactly one series). Lets a vector feed a scalar position (`vector` / - /// aggregation `k` args, thresholds). Issue #48. - PromqlScalarFromVector(Rc>), - - /// ρ — a per-series **label rewrite** (PromQL `label_replace` / - /// `label_join`). Every input row passes through unchanged except for the - /// destination label `dst`, whose new value is computed by `value` — a - /// scalar expression over the child's (source) label columns: - /// `label_replace` → a `label_replace(src, regex, replacement)` function - /// call (regex capture-expansion), `label_join` → a `label_join(sep, srcs…)` - /// concatenation. Sample values and the time axis are untouched. Issue #50. - PromqlRelabel { - /// The label written by this rewrite (PromQL `dst_label`). - dst: String, - value: Rc>, - child: Rc>, - }, - - /// PromQL `info(v, [selector])` — left-join **label enrichment** (#84). Each - /// series in `child` is enriched with labels from the matching info metric(s) - /// (`target_info` by default; `selector`'s `__name__` matchers pick the - /// metric(s), the rest constrain the data labels), joined on their shared - /// identifying labels. Those join keys are the info metric's identifying - /// labels — runtime/metadata-resolved, since an open PromQL schema can't - /// enumerate them — so they are NOT carried here; the post-ASAP realization pass - /// resolves them from the info metric's schema. The output keeps - /// `child`'s (open) schema: the - /// grafted labels appear at runtime. - PromqlInfoEnrich { - #[serde(default)] - selector: Vec, - child: Rc>, - }, - - /// Series-sampling **selection** — PromQL `limitk` / `limit_ratio` (#86). - /// Keeps a subset of whole series per `by` group (empty = global), passing - /// each surviving series through unchanged. Not a ranking (`TopK`) and not a - /// reduction: the output schema equals the child's. - PromqlSeriesSample { - #[serde(default)] - by: GroupKeys, - kind: SampleKind, - child: Rc>, - }, - - /// σ — row-level filter. Output schema = child schema. - Filter { - pred: Predicate, - child: Rc>, - }, - /// π — column projection. - Project { - cols: Vec>, - /// Re-qualifies every output column with this table alias (a derived - /// table / inline view). `None` for an ordinary SELECT list. - #[serde(default)] - qualifier: Option, - child: Rc>, - }, - - /// γ + α — GROUP BY (positional) + aggregate intents. - Aggregate { - reduction: Reduction, - measures: Vec>, - /// Output column names parallel to `measures`. A non-empty entry overrides - /// the synthetic intent-keyed name — SQL threads DataFusion's generated - /// name (e.g. `"sum(metrics.bytes)"`) here so an enclosing `Project` - /// resolves the aggregate output by the name it references. An empty - /// entry (or empty vec) falls back to `AggIntent::output_column`'s name - /// (PromQL's convention). - #[serde(default)] - output_names: Vec, - /// Per-measure row predicates, parallel to `measures` — SQL - /// `FILTER (WHERE …)` semantics (issue #466): only rows where - /// `filters[i]` is `TRUE` update `measures[i]`; groups are still - /// formed from every row. Positional against `child`'s output - /// schema, like `Filter.pred` — not against this node's output like - /// `having`. `None` (or an entry past the end of a shorter vec) is - /// an unfiltered measure, so an empty vec is the pre-#466 shape. - #[serde(default)] - filters: Vec>>, - #[serde(default)] - having: Option>, - child: Rc>, - }, - - /// δ — SQL `DISTINCT` / row deduplication. Positional like every other - /// column reference here; empty = dedup on all columns (`SELECT DISTINCT *`). - Dedup { - cols: Vec, - child: Rc>, - }, - /// ⊕ — exact, n-ary `UNION ALL` of independent branches. Rows are - /// concatenated, never deduplicated; SQL's `UNION`/`INTERSECT`/`EXCEPT` are - /// [`QueryExpr::SetOp`], not this. - /// - /// Used for the branches of one query that a single `Aggregate` cannot - /// express — PromQL `histogram_quantiles` (one branch per φ, issue #109) and - /// SQL `ROLLUP`/`CUBE`/`GROUPING SETS` (one branch per grouping level, issue - /// #118) — as well as for sharded / fan-in plans. - /// - /// **The branches must be union-compatible; nothing here enforces it.** The - /// output schema is the *first* child's, so branches that disagree on a - /// column name or type leave the merged schema silently misdescribing every - /// branch but one. A producer that cannot guarantee compatibility must - /// project the branches into a common shape first. - /// - /// A row may appear in several branches, so no branch's unique key survives - /// the union — `unique_keys` is dropped, as in `SetOp`. **Unless** the - /// constructor asserted `discriminator_unique_key` (issue #228, - /// [`QueryExpr::concat_with_discriminator`]): a caller-proven claim that - /// one column's value is guaranteed distinct per branch, which makes - /// `(discriminator, inner_key)` a sound compound unique key regardless of - /// whether `inner_key` alone repeats across branches. `None` — every - /// ordinary construction path, including the plain struct literal and - /// [`QueryExpr::concat`] — reproduces the old, unconditional-drop - /// behavior exactly; see [`ConcatDiscriminatorKey`]'s doc for the - /// soundness argument and the obligation this puts on whoever asserts it. - /// - /// Empty children is an error ([`QueryExprError::EmptyConcat`]), not an - /// empty relation: there would be no schema to derive. - Concat { - children: Vec>, - /// See the field-level doc above and [`ConcatDiscriminatorKey`]. - #[serde(default)] - discriminator_unique_key: Option>, - }, - - /// Logical join. Post-ASAP binding picks the physical alternative. - Join { - kind: JoinKind, - pred: Predicate, - left: Rc>, - right: Rc>, - }, - SetOp { - kind: RelationalSetOpKind, - all: bool, - left: Rc>, - right: Rc>, - }, - - /// Generic order-by for non-heavy-hitter cases. - /// - /// `partition_by` makes the ordering **per-group**: a non-empty set means - /// "rank within each `partition_by` group" — the semantics behind PromQL - /// `topk by (host) (…)` / SQL `… OVER (PARTITION BY host ORDER BY …)`. It is - /// row-preserving (schema pass-through) and is where the grouping of a - /// generic (non-heavy-hitter) ranking lives, so there is no separate - /// `Partition` node (issue #12: reducing GROUP BY → `Aggregate.by`, per-group - /// ranking → here, parallel sharding → a deployment's own physical - /// stage). Empty = a global order-by. - Sort { - keys: Vec>, - #[serde(default)] - partition_by: GroupKeys, - child: Rc>, - }, - Limit { - n: usize, - offset: usize, - child: Rc>, - }, - - /// PromQL sub-query (`[range:resolution]`). Logical pass-through. - PromqlSubquery { - range: Duration, - #[serde(default)] - resolution: Option, - child: Rc>, - }, - - /// Temporal range selection — "look back `range` of history for this - /// computation." Used for all range-vector functions: `rate`, `increase`, - /// `*_over_time`. The range is distinct from a row-level `Filter`. - /// - /// Structural marker: an `Aggregate` whose direct child is a `TimeRange` - /// is a *per-series* reduction (label-preserving); one whose child is a - /// plain `Scan` or another `Aggregate` is a *cross-series* reduction. - TimeRange { - range: Duration, - child: Rc>, - }, - - /// PromQL `offset` / `@` **time shift** on a selector (issue #40). A - /// pass-through wrapper: it moves *when* `child` is evaluated (the runtime - /// resolves the `@` anchor and applies the offset) but leaves its schema - /// unchanged. Wraps the shifted selector directly — `m offset 1h` → - /// `TimeShift { Scan }`; a ranged selector `m[5m] offset 1h` → - /// `TimeRange { 5m, TimeShift { Scan } }` (the range is taken at the shifted - /// time). A shifted subquery wraps the `PromqlSubquery`, moving its step - /// grid. Never carries the identity shift (the converter emits a bare - /// selector when neither modifier is present). - TimeShift { - shift: TimeShift, - child: Rc>, - }, - - /// SQL analytic window function: `func(args) OVER (PARTITION BY … ORDER BY … - /// ROWS/RANGE BETWEEN …)`. Output schema = child schema + one column named - /// `output_name` (the name the enclosing `Project` references). - SQLWindowFunc { - func: WindowFuncKind, - /// Operand expressions (`LAG(value)` → `[Column(value_id)]`); empty for - /// the rank-only functions (`ROW_NUMBER`/`RANK`/`DENSE_RANK`). - args: Vec>, - partition_by: GroupKeys, - order_by: Vec>, - /// `None` is accepted only for backward compatibility with serialized - /// pre-#268 IR, where the engine's implicit frame was not retained. - /// Newly lowered SQL always carries `Some` with DataFusion's resolved - /// concrete default or explicit frame. - #[serde(default)] - frame: Option, - /// The output column's name — DataFusion's window-expr field name, so a - /// `Project` above resolves it (cf. `Aggregate.output_names`). - output_name: String, - child: Rc>, - }, - - /// Arithmetic / comparison / boolean composition (PromQL binary ops). - BinaryOp { - op: BinaryOpKind, - lhs: Rc>, - rhs: Rc>, - #[serde(default)] - vector_match: Option, - }, - - // ── Scalar expression shapes (issue #205) ─────────────────────────── - // - // Formerly a separate, self-recursive `Expr` DAG, reachable from the - // operator variants above only through wrapper fields (`Predicate`, - // `ProjectItem`, `SortKey`). They're variants of this same DAG now — a - // scalar sub-expression is only ever reachable through one of those same - // wrapper positions (`Filter.pred`, `ProjectItem.expr`, `Aggregate.having`, - // `PromqlRelabel.value`, `SQLWindowFunc.args`, …), which is a *convention* this - // type no longer enforces at compile time the way the old, closed - // `Expr` variant set did — nothing stops constructing, say, a `Scan` - // where a `Compare`'s `left` operand belongs. `output_schema` and every - // scalar-position consumer (`resolve`, `canonicalize`, `infer_expr_type`) - // reject a non-scalar variant found there instead (a `QueryExprError` or - // an `unreachable!`, depending on the call site) — the accepted - // replacement, since the alternative (a marker-trait/sub-enum bound - // restricting which variants are constructible in a scalar position) adds - // real type-level machinery for a distinction every constructor already - // has to get right structurally anyway (a `Filter` is never built with an - // operator sub-DAG as its `pred`). - /// A column reference — unresolved [`ColumnRef`] (front-end-emitted, `C = - /// ColumnRef`) or positional [`ColumnId`] (once bound, `C = ColumnId`). - Column(C), - /// A constant literal value. - Literal(ScalarValue), - /// `left op right` — binary comparison. - Compare { - left: Rc>, - op: CompareOpKind, - right: Rc>, - }, - /// Flat conjunction (logical AND). An empty list is vacuously true. - BoolAnd(Vec>), - /// Flat disjunction (logical OR). An empty list is vacuously false. - BoolOr(Vec>), - /// Logical NOT. - Not(Rc>), - /// `expr IS NULL`. - IsNull(Rc>), - /// `expr IS NOT NULL`. - IsNotNull(Rc>), - /// `CAST(expr AS to)`; `try_cast` for SQL `TRY_CAST` (NULL on failure). - Cast { - expr: Rc>, - to: DataType, - try_cast: bool, - }, - /// `expr [NOT] IN (v1, v2, …)`. - InList { - expr: Rc>, - list: Vec>, - negated: bool, - }, - /// Scalar function call, e.g. `LOWER(col)`, `ABS(x)`. - FunctionCall { - name: String, - args: Vec>, - }, - /// Binary arithmetic: `left op right`. - Arithmetic { - op: ArithmeticOpKind, - left: Rc>, - right: Rc>, - }, - /// SQL `CASE` (both searched and simple forms). `operand` present for the - /// simple form (`CASE expr WHEN …`), absent for searched. - Case { - operand: Option>>, - branches: Vec<(QueryExpr, QueryExpr)>, - else_expr: Option>>, - }, -} - -impl QueryExpr { - /// Construct the [`PromqlScalarBridge`](Self::PromqlScalarBridge) leaf - /// for a bare PromQL numeric literal / folded constant scalar (issue - /// #220) — `Literal(ScalarValue::Float64(v))` at an operator-DAG - /// position. The one constructor every front end / test that used to - /// write `QueryExpr::PromqlScalar(v)` should use instead. - pub fn promql_scalar(v: f64) -> Self { - QueryExpr::PromqlScalarBridge(Rc::new(QueryExpr::Literal(ScalarValue::Float64(v)))) - } - - /// Build an ordinary [`Concat`](Self::Concat) — the ordinary/default - /// construction path every call site should prefer over the bare struct - /// literal: `output_schema` drops `unique_keys` unconditionally, exactly - /// as before issue #228. Use - /// [`concat_with_discriminator`](Self::concat_with_discriminator) instead - /// when the caller can prove branch disjointness via a discriminator - /// column. - pub fn concat(children: Vec>) -> Self { - QueryExpr::Concat { - children, - discriminator_unique_key: None, - } - } - - /// Build a [`Concat`](Self::Concat) whose output schema carries the - /// caller-proven compound unique key `(discriminator, inner_key)` (issue - /// #228). See [`ConcatDiscriminatorKey`]'s doc for the soundness - /// argument and the obligation this puts on the caller — - /// `output_schema` trusts this claim without verifying it: nothing here - /// checks that `inner_key` is unique within every branch or that - /// `discriminator`'s value is distinct between branches. - pub fn concat_with_discriminator( - children: Vec>, - discriminator: C, - inner_key: Vec, - ) -> Self { - QueryExpr::Concat { - children, - discriminator_unique_key: Some(ConcatDiscriminatorKey::new(discriminator, inner_key)), - } - } - - /// The value of a [`PromqlScalarBridge`](Self::PromqlScalarBridge) leaf - /// wrapping a plain `Literal(ScalarValue::Float64(_))` — every one a - /// front end constructs today (see [`promql_scalar`](Self::promql_scalar)). - /// `None` for any other shape, including a `PromqlScalarBridge` wrapping - /// something else (not constructed today, but not precluded by the type). - pub fn as_promql_scalar(&self) -> Option { - match self { - QueryExpr::PromqlScalarBridge(inner) => match inner.as_ref() { - QueryExpr::Literal(ScalarValue::Float64(v)) => Some(*v), - _ => None, - }, - _ => None, - } - } - - /// If this expression is a `BoolAnd`, return its elements; otherwise a - /// single-element slice containing `self`. - pub fn conjuncts(&self) -> &[QueryExpr] { - match self { - QueryExpr::BoolAnd(v) => v.as_slice(), - _ => std::slice::from_ref(self), - } - } - - /// If this expression is a `BoolOr`, return its elements; otherwise a - /// single-element slice containing `self`. - pub fn disjuncts(&self) -> &[QueryExpr] { - match self { - QueryExpr::BoolOr(v) => v.as_slice(), - _ => std::slice::from_ref(self), - } - } - - /// Recursively collect every column reference in a **scalar** sub-DAG — - /// used by the [`SchemaResolver`](super::schema_resolver::SchemaResolver) to seed usage-derived - /// leaf schemas, and available to post-ASAP binding for column-lineage / - /// selectivity. - /// `self` must be one of the scalar variants (see the module doc on - /// [`QueryExpr`]'s scalar shapes) — every caller already only reaches - /// this through a scalar-typed position (`Predicate`, `ProjectItem.expr`, - /// …), so an operator variant here indicates a construction bug, not a - /// shape this needs to handle silently. - pub fn columns_referenced(&self) -> Vec<&C> { - match self { - QueryExpr::Column(c) => vec![c], - QueryExpr::Literal(_) => vec![], - QueryExpr::EvalTimestamp => vec![], - QueryExpr::CurrentTimestamp => vec![], - QueryExpr::Compare { left, right, .. } | QueryExpr::Arithmetic { left, right, .. } => { - let mut v = left.columns_referenced(); - v.extend(right.columns_referenced()); - v - } - QueryExpr::BoolAnd(parts) | QueryExpr::BoolOr(parts) => { - parts.iter().flat_map(|e| e.columns_referenced()).collect() - } - QueryExpr::Not(e) | QueryExpr::IsNull(e) | QueryExpr::IsNotNull(e) => { - e.columns_referenced() - } - QueryExpr::Cast { expr, .. } => expr.columns_referenced(), - QueryExpr::InList { expr, list, .. } => { - let mut v = expr.columns_referenced(); - v.extend(list.iter().flat_map(|e| e.columns_referenced())); - v - } - QueryExpr::FunctionCall { args, .. } => { - args.iter().flat_map(|e| e.columns_referenced()).collect() - } - QueryExpr::Case { - operand, - branches, - else_expr, - } => { - let mut v = vec![]; - if let Some(op) = operand { - v.extend(op.columns_referenced()); - } - for (when, then) in branches { - v.extend(when.columns_referenced()); - v.extend(then.columns_referenced()); - } - if let Some(e) = else_expr { - v.extend(e.columns_referenced()); - } - v - } - other => unreachable!( - "columns_referenced called on a non-scalar QueryExpr variant: {other:?}" - ), - } - } -} - -/// The canonical, positional, resolved DAG — what the bare `QueryExpr` name -/// has always meant (the default `C = ColumnId`). Every existing consumer -/// keeps using `QueryExpr` unparameterized; this alias exists only to name -/// the resolved state explicitly at a use site that also wants to name -/// [`UnresolvedQueryExpr`] nearby. -pub type ResolvedQueryExpr = QueryExpr; - -/// The front-end-emitted, name-based, unresolved DAG — -/// `QueryExpr`: front ends construct this directly during their -/// own `interpret` step (issue #179), and the [`SchemaResolver`](super::schema_resolver) -/// resolves it into [`ResolvedQueryExpr`]. -pub type UnresolvedQueryExpr = QueryExpr; - -// `output_schema` needs a fully bound DAG — it reads `Scan.schema` as a plain -// `Schema` and resolves every scalar `Expr::Column` positionally — so it lives -// only on the resolved instantiation, not `impl QueryExpr`. -// Same reasoning as `AggIntent`'s `output_column`/`requires`/`is_per_series` -// (#205): a schema-shaped property that is only meaningful post-binding. -impl QueryExpr { - /// Infer a scalar expression against its input relation using the same - /// canonical rules as projection schema derivation. - pub fn scalar_type(&self, input: &Schema) -> Result<(DataType, bool), QueryExprError> { - infer_expr_type(self, input) - } - - /// Output schema of the root of a canonical DAG. - pub fn output_schema(&self) -> Result { - match self { - QueryExpr::Scan { schema, .. } => Ok(schema.clone()), - - QueryExpr::Aggregate { - reduction, - measures, - output_names, - child, - .. - } => { - let in_schema = child.output_schema()?; - aggregate_output_schema(&in_schema, reduction, measures, output_names) - } - - QueryExpr::Filter { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - // Series sampling keeps a subset of whole series unchanged, so the - // output schema (and row-uniqueness) is exactly the child's (#86). - | QueryExpr::PromqlSeriesSample { child, .. } - // Info enrichment adds runtime info labels — the statically-known - // schema is the child's (open), so it passes through (#84). - | QueryExpr::PromqlInfoEnrich { child, .. } - | QueryExpr::TimeRange { child, .. } - // A time shift (`offset`/`@`) moves *when* the child is evaluated, - // never its columns — schema passes through (#40). - | QueryExpr::TimeShift { child, .. } => child.output_schema(), - - // ρ — relabel preserves every input column and writes one label - // `dst` (Utf8): overwritten in place if it already exists, else - // appended (nullable — a `label_replace` regex non-match leaves it - // unset). The schema stays open (other labels remain runtime-only). - // A rewrite can collapse two label sets into one, so row-uniqueness - // is no longer provable — drop unique_keys. - QueryExpr::PromqlRelabel { dst, child, .. } => { - let mut out = child.output_schema()?; - if let Some(existing) = out.fields.iter_mut().find(|c| c.name == *dst) { - existing.dtype = FieldDataType::Plain(DataType::Utf8); - existing.nullable = true; - } else { - out.fields.push(Field::plain(dst.clone(), DataType::Utf8, true)); - } - out.unique_keys.clear(); - Ok(out) - } - - // π — one output column per projection item. Each item's type is - // inferred from its expression against the child schema; the name - // is the explicit alias or a derived default. A child unique key - // survives exactly when every one of its columns is passed through - // as a bare `Field` item (possibly reordered or aliased). Derived - // expressions cannot carry key identity. `time_index` is re-found - // by name. - QueryExpr::Project { cols, qualifier, child } => { - let in_schema = child.output_schema()?; - let columns: Vec = cols - .iter() - .enumerate() - .map(|(i, item)| { - let (dtype, nullable) = infer_expr_type(&item.expr, &in_schema)?; - let name = item - .alias - .clone() - .unwrap_or_else(|| default_proj_name(&item.expr, i, &in_schema)); - let c = Field::plain(name, dtype, nullable); - // A derived table re-qualifies its output columns with - // its alias, so `t.col` (and a join over two derived - // tables) resolves to the right relation. - Ok(match qualifier { - Some(q) => c.with_table(q), - None => c, - }) - }) - .collect::, QueryExprError>>()?; - let time_index = columns.iter().position(|c| c.name == "ts"); - let unique_keys = in_schema - .unique_keys - .iter() - .filter_map(|key| { - key.iter() - .map(|input_col| { - cols.iter().position(|item| { - matches!(&item.expr, QueryExpr::Column(col) if col == input_col) - }) - }) - .collect::>>() - }) - .collect(); - Ok(Schema { - fields: columns, - time_index, - unique_keys, - // Projection enumerates exactly its items → closed. - closed: true, - }) - } - - QueryExpr::Dedup { cols, child } => { - let mut out = child.output_schema()?; - // Deduplicating on `cols` makes them a unique key of the result. - if !cols.is_empty() { - out.add_unique_key(cols.clone()); - } - Ok(out) - } - - // ⊕ — the branches are union-compatible by construction, so the - // output shape is the first child's. A row can appear in more than - // one branch, so no key of one branch is a key of the union: drop - // unique_keys, exactly as `SetOp` does — unless the constructor - // asserted `discriminator_unique_key` (issue #228), in which case - // `(discriminator, inner_key)` becomes the sole unique key. That - // assertion is trusted verbatim here, never checked: see - // `ConcatDiscriminatorKey`'s doc for the soundness argument and - // whose obligation it is. - QueryExpr::Concat { - children, - discriminator_unique_key, - } => { - let mut s = children - .first() - .ok_or(QueryExprError::EmptyConcat) - .and_then(|c| c.output_schema())?; - s.unique_keys.clear(); - if let Some(key) = discriminator_unique_key { - let mut compound = vec![*key.discriminator()]; - compound.extend(key.inner_key().iter().copied()); - s.add_unique_key(compound); - } - Ok(s) - } - // Set operations are union-compatible: both sides share the left's - // column shape, so the output schema is the left's. (Row identity - // is not preserved across a UNION, so unique_keys are dropped.) - QueryExpr::SetOp { left, .. } => { - let mut s = left.output_schema()?; - s.unique_keys.clear(); - Ok(s) - } - // ⋈ — output is the concatenation of both inputs' columns. Outer - // joins make the non-preserved side nullable. Post-join row - // identity isn't provable in general, so unique_keys reset. - QueryExpr::Join { - kind, left, right, .. - } => { - let l = left.output_schema()?; - let r = right.output_schema()?; - // Semi / anti joins filter the left side; the right contributes - // no columns, so the output is the left's schema unchanged. Row - // identity *is* preserved (each left row appears at most once), - // but a left row can be dropped, so unique_keys still reset. - if matches!(kind, JoinKind::Semi | JoinKind::Anti) { - return Ok(Schema { - unique_keys: Vec::new(), - ..l - }); - } - let (left_null, right_null) = match kind { - JoinKind::Left => (false, true), - JoinKind::Right => (true, false), - JoinKind::Full => (true, true), - JoinKind::Inner | JoinKind::Cross => (false, false), - JoinKind::Semi | JoinKind::Anti => unreachable!("handled above"), - }; - let l_len = l.fields.len(); - let mut columns = Vec::with_capacity(l_len + r.fields.len()); - columns.extend(l.fields.iter().cloned().map(|mut c| { - c.nullable |= left_null; - c - })); - columns.extend(r.fields.iter().cloned().map(|mut c| { - c.nullable |= right_null; - c - })); - let time_index = l.time_index.or(r.time_index.map(|i| i + l_len)); - Ok(Schema { - fields: columns, - time_index, - unique_keys: Vec::new(), - // The concatenation is complete only if both sides are. - closed: l.closed && r.closed, - }) - } - // ψ-analytic — child schema + one appended window-output column. - QueryExpr::SQLWindowFunc { - func, - args, - output_name, - child, - .. - } => { - let mut out = child.output_schema()?; - // First operand's (dtype, nullable) from the child schema, owned - // so the borrow ends before we append. - let arg = args.first().and_then(|a| match a { - QueryExpr::Column(id) => out.fields.get(*id), - _ => None, - }); - let arg_dtype = || { - arg.and_then(|c| c.plain_dtype().cloned()) - .unwrap_or(DataType::Float64) - }; - let (dtype, nullable) = match func { - WindowFuncKind::RowNumber - | WindowFuncKind::Rank - | WindowFuncKind::DenseRank - | WindowFuncKind::Count => (DataType::Int64, false), - WindowFuncKind::Sum | WindowFuncKind::Avg => (DataType::Float64, true), - // Navigation funcs: arg type, nullable (boundary rows are NULL). - WindowFuncKind::Lag - | WindowFuncKind::Lead - | WindowFuncKind::LagInFrame - | WindowFuncKind::LeadInFrame - | WindowFuncKind::FirstValue - | WindowFuncKind::LastValue - | WindowFuncKind::NthValue(_) => (arg_dtype(), true), - WindowFuncKind::Min | WindowFuncKind::Max => { - (arg_dtype(), arg.is_none_or(|c| c.nullable)) - } - }; - out.fields - .push(Field::plain(output_name.clone(), dtype, nullable)); - Ok(out) - } - - // A scalar bridge has no series — model it as a single `value` - // column so it can sit as a `BinaryOp` operand. Both scalar - // leaves — a bridged scalar sub-expression and the eval time — - // are a single `value` column with no labels. Every - // `PromqlScalarBridge` constructed today wraps a plain - // `Literal(Float64)` (issue #220), so the schema doesn't need to - // inspect the inner node. - QueryExpr::PromqlScalarBridge(_) | QueryExpr::EvalTimestamp => Ok(Schema { - fields: vec![Field::plain("value", DataType::Float64, false)], - time_index: None, - unique_keys: Vec::new(), - closed: true, - }), - - QueryExpr::CurrentTimestamp => Ok(Schema { - fields: vec![Field::plain("value", DataType::Timestamp, false)], - time_index: None, - unique_keys: Vec::new(), - closed: true, - }), - - // `vector(s)` yields a label-less instant vector: the (ts, value) - // floor and nothing else. `closed` — its full label set (empty) is - // known statically (#48). - QueryExpr::PromqlVectorFromScalar(_) => Ok(Schema { - fields: vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - time_index: Some(0), - unique_keys: Vec::new(), - closed: true, - }), - - // `scalar(v)` collapses to a single `value`, no time index — the same - // scalar shape as a constant or `time()` (#48). - QueryExpr::PromqlScalarFromVector(_) => Ok(Schema { - fields: vec![Field::plain("value", DataType::Float64, false)], - time_index: None, - unique_keys: Vec::new(), - closed: true, - }), - - // The output shape of ` op ` (or ` op - // `) is the vector side's — a scalar operand (a constant or - // `time()`) contributes only its value, no labels. Prefer the - // non-scalar side. - QueryExpr::BinaryOp { lhs, rhs, op, vector_match } => { - fn scalar(expression: &QueryExpr) -> bool { - match expression { - QueryExpr::PromqlScalarBridge(_) | QueryExpr::EvalTimestamp | QueryExpr::PromqlScalarFromVector(_) => true, - QueryExpr::BinaryOp { lhs, rhs, .. } => scalar(lhs) && scalar(rhs), - _ => false, - } - } - let left = lhs.output_schema()?; - let right = rhs.output_schema()?; - if scalar(lhs) { return Ok(right); } - let mut output = left; - let grouping = vector_match.as_ref().and_then(|m| m.grouping.as_ref()); - let right_rows = matches!(op, BinaryOpKind::Set(PromQLVectorSetOpKind::Or)) - || matches!(grouping, Some(g) if g.side == GroupSide::Right); - let mut additions = Vec::new(); - if right_rows { - additions.extend(right.fields.iter().filter(|c| c.dtype == DataType::Utf8).cloned()); - } - if let Some(grouping) = grouping { - additions.extend(grouping.labels.iter().map(|name| Field::plain(name.clone(), DataType::Utf8, true))); - } - for column in additions { - if !output.fields.iter().any(|c| c.name == column.name) { - output.fields.push(column); - } - } - Ok(output) - }, - - // The scalar variants (issue #205) — see `QueryExprError::ScalarHasNoRowSchema`. - QueryExpr::Column(_) - | QueryExpr::Literal(_) - | QueryExpr::Compare { .. } - | QueryExpr::BoolAnd(_) - | QueryExpr::BoolOr(_) - | QueryExpr::Not(_) - | QueryExpr::IsNull(_) - | QueryExpr::IsNotNull(_) - | QueryExpr::Cast { .. } - | QueryExpr::InList { .. } - | QueryExpr::FunctionCall { .. } - | QueryExpr::Arithmetic { .. } - | QueryExpr::Case { .. } => Err(QueryExprError::ScalarHasNoRowSchema), - } - } -} - -/// Output schema of a *per-series* window/range reduction (`rate`/`increase`, -/// or an `*_over_time` reducer under a time `Window`). Such a reduction emits -/// one value per series, so every label column of `input` is preserved and only -/// the sample value is replaced — kept named `value` so the PromQL sample-value -/// convention (and any outer `SampleValue` reference) still resolves it by name. -fn per_series_reduction_schema(input: &Schema, agg: &AggIntent) -> Result { - let vi = if let Some(index) = agg.input_cols().first() { - *index - } else { - super::column_resolution::resolve_column_ref(&ColumnRef::SampleValue, input) - .map_err(|error| QueryExprError::InvalidSampleColumn(error.to_string()))? - }; - if !matches!( - input.fields.get(vi).map(|column| &column.dtype), - Some(FieldDataType::Plain(DataType::Float64 | DataType::Int64)) - ) { - return Err(QueryExprError::InvalidSampleColumn(format!( - "column {vi} is not numeric" - ))); - } - let mut columns = input.fields.clone(); - { - let mut out = agg.output_column(&columns[vi]); - out.name = "value".into(); - // A per-series range reduction produces a PromQL sample value, which is - // always `float64` — override the reducer's own output dtype so - // `count_over_time` (whose `Count` intent types `Int64`) matches every - // other range reducer instead of leaking an `Int64` value column (#69). - out.dtype = FieldDataType::Plain(DataType::Float64); - columns[vi] = out; - } - Ok(Schema { - fields: columns, - time_index: input.time_index, - unique_keys: input.unique_keys.clone(), - // Per-series reduction is label-preserving: it inherits its input's - // completeness (an open scan stays open; a closed one stays closed). - closed: input.closed, - }) -} - -/// The output schema of an `Aggregate { reduction, measures }` over `in_schema` — -/// the **single** canonical derivation shared by -/// [`QueryExpr::output_schema`]'s `Aggregate` arm and the converter's -/// HAVING-resolution path (`column_resolution::output_schema_for_aggregate`), -/// so the two can never drift (issue #41). -/// -/// `Reduction::PerEntity` selects the label-preserving -/// [`per_series_reduction_schema`] (`rate`/`increase`/`*_over_time`) instead -/// of the cross-series `by ++ measures` shape. Which one applies is read directly -/// off `reduction` — decided once, at construction, by whoever built the -/// `Aggregate` node (issue #165) — not re-derived here from `by`/child shape. -pub fn aggregate_output_schema( - in_schema: &Schema, - reduction: &Reduction, - measures: &[AggIntent], - output_names: &[String], -) -> Result { - let by = match reduction { - Reduction::PerEntity => { - debug_assert_eq!( - measures.len(), - 1, - "a per-entity reduction is single-aggregate" - ); - return per_series_reduction_schema(in_schema, &measures[0]); - } - Reduction::Reduce(by) => by, - }; - - // `without(excluded)` groups by every label *except* those listed: the kept - // labels are the input's label columns minus the excluded positions (and the - // ts / sample-value columns), and the schema stays **open** because the full - // runtime label set isn't known. The `by(...)` path instead enumerates its - // kept columns and freezes to closed (issue #39). - if by.is_without() { - return without_output_schema(in_schema, by.keys(), measures, output_names); - } - - let mut out_cols: Vec = Vec::with_capacity(by.len() + measures.len()); - for &id in by.keys() { - let c = in_schema - .fields - .get(id) - .ok_or(QueryExprError::InvalidGroupByColumn( - id, - in_schema.fields.len(), - ))?; - out_cols.push(c.clone()); - } - let value_col_idx = - super::column_resolution::resolve_column_ref(&ColumnRef::SampleValue, in_schema) - .ok() - .or_else(|| (0..in_schema.fields.len()).find(|i| !by.contains(i))); - let probe = value_col_idx - .and_then(|i| in_schema.fields.get(i)) - .cloned() - .unwrap_or_else(|| Field::plain("value", DataType::Float64, false)); - // Each reducer types off its own input column (`SUM(bytes)` vs `AVG(latency)` - // in one node); `None` falls back to the sample-value probe (PromQL's - // single-column convention). A non-empty `output_names[i]` overrides the - // synthetic output column name. - for (i, intent) in measures.iter().enumerate() { - // `count_values("l", v)` emits TWO columns: the synthesized `Utf8` label - // `l` (the stringified sample value it groups by) and the per-value - // count. If `l` collides with a group-by key of the same name, PromQL's - // synthesized label takes precedence — emit a single column, never a - // duplicate. - if let AggIntent::CountValues { label } = intent { - if !out_cols.iter().any(|c| c.name == *label) { - out_cols.push(Field::plain(label.clone(), DataType::Utf8, false)); - } - let mut cnt = intent.output_column(&probe); - if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { - cnt.name = name.clone(); - } - out_cols.push(cnt); - continue; - } - // Only the output *type* is read from here, so the leading column is - // enough for the multi-column intents: `Cardinality` and `PearsonCorr` - // both have a fixed output type that ignores it. - let in_col = intent - .input_cols() - .first() - .and_then(|id| in_schema.fields.get(*id)) - .unwrap_or(&probe); - let mut out = intent.output_column(in_col); - // A global extremum emits NULL for an empty input, even if its input - // column is non-nullable. Grouped extrema only emit existing groups. - if by.is_empty() && matches!(intent, AggIntent::Min { .. } | AggIntent::Max { .. }) { - out.nullable = true; - } - if let Some((arg, _)) = intent - .arg_selector_columns(in_schema) - .map_err(QueryExprError::InvalidScalarSignature)? - { - out.dtype = in_schema.fields[arg].dtype.clone(); - out.nullable = in_schema.fields[arg].nullable; - } - if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { - out.name = name.clone(); - } - out_cols.push(out); - } - // `count_values` groups by (by-keys ∪ the synthesized value label), so the - // by-keys alone are not a unique key — be conservative and claim none. - let has_count_values = measures - .iter() - .any(|a| matches!(a, AggIntent::CountValues { .. })); - let unique_keys = if by.is_empty() || has_count_values { - Vec::new() - } else { - vec![(0..by.len()).collect()] - }; - Ok(Schema { - fields: out_cols, - time_index: None, - unique_keys, - // A cross-series aggregate enumerates exactly `by ++ measures`, so its output - // is closed even over an open input — this is where an open schema - // freezes to closed. - closed: true, - }) -} - -/// Output schema of a `without(excluded)` aggregate: the kept labels (every -/// input label column except the `excluded` positions, the time axis, and the -/// sample-value column) followed by the aggregate output column(s). Unlike the -/// `by` path this stays **open** — the excluded set is enumerable but the kept -/// set is not (the runtime carries labels the usage-derived schema never saw), -/// so the schema can't freeze to closed and claims no unique key (issue #39). -fn without_output_schema( - in_schema: &Schema, - excluded: &[ColumnId], - measures: &[AggIntent], - output_names: &[String], -) -> Result { - for &id in excluded { - if id >= in_schema.fields.len() { - return Err(QueryExprError::InvalidGroupByColumn( - id, - in_schema.fields.len(), - )); - } - } - // A nested aggregate renames the sample value (`sum by (le) (…)` → `sum`); - // it is still the value, not a kept label. - let value = - super::column_resolution::resolve_column_ref(&ColumnRef::SampleValue, in_schema).ok(); - let mut out_cols: Vec = Vec::new(); - for (i, col) in in_schema.fields.iter().enumerate() { - let is_time = in_schema.time_index == Some(i); - if !is_time && value != Some(i) && !excluded.contains(&i) { - out_cols.push(col.clone()); - } - } - let probe = value - .and_then(|i| in_schema.fields.get(i)) - .cloned() - .unwrap_or_else(|| Field::plain("value", DataType::Float64, false)); - for (i, intent) in measures.iter().enumerate() { - // Only the output *type* is read from here, so the leading column is - // enough for the multi-column intents: `Cardinality` and `PearsonCorr` - // both have a fixed output type that ignores it. - let in_col = intent - .input_cols() - .first() - .and_then(|id| in_schema.fields.get(*id)) - .unwrap_or(&probe); - let mut out = intent.output_column(in_col); - if let Some((arg, _)) = intent - .arg_selector_columns(in_schema) - .map_err(QueryExprError::InvalidScalarSignature)? - { - out.dtype = in_schema.fields[arg].dtype.clone(); - out.nullable = in_schema.fields[arg].nullable; - } - if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { - out.name = name.clone(); - } - out_cols.push(out); - } - Ok(Schema { - fields: out_cols, - time_index: None, - unique_keys: Vec::new(), - // The kept label set is runtime-only, so — unlike `by` — this does not - // freeze the open schema to closed. - closed: false, - }) -} - -/// Infer the `(DataType, nullable)` a scalar [`QueryExpr`] produces against an -/// input [`Schema`]. Used by `Project` schema derivation. Approximate here: -/// unknown columns and bare `FunctionCall`s fall back to a permissive default -/// (post-ASAP binding refines with a real function/type registry). `expr` -/// must be one of the scalar variants (issue #205) — an operator variant here -/// is a construction bug, not a shape this needs to handle silently. -fn infer_expr_type( - expr: &QueryExpr, - schema: &Schema, -) -> Result<(DataType, bool), QueryExprError> { - Ok(match expr { - QueryExpr::CurrentTimestamp => (DataType::Timestamp, false), - QueryExpr::Column(id) => match schema.fields.get(*id) { - Some(c) => match c.plain_dtype() { - Some(dtype) => (dtype.clone(), c.nullable), - // Summary state is not a scalar value: it has to be read - // out (estimated / finalized) before an expression can use it. - None => { - return Err(QueryExprError::InvalidScalarSignature(format!( - "column `{}` carries summary state and cannot be read as a value", - c.name - ))) - } - }, - None => (DataType::Float64, true), - }, - QueryExpr::Literal(s) => match s { - ScalarValue::Int64(_) => (DataType::Int64, false), - ScalarValue::Float64(_) => (DataType::Float64, false), - ScalarValue::Utf8(_) => (DataType::Utf8, false), - ScalarValue::Boolean(_) => (DataType::Bool, false), - ScalarValue::Null => (DataType::Null, true), - ScalarValue::Interval { .. } => (DataType::Interval, false), - }, - // Boolean-valued expressions (SQL three-valued logic → nullable). - QueryExpr::Compare { .. } - | QueryExpr::BoolAnd(_) - | QueryExpr::BoolOr(_) - | QueryExpr::Not(_) - | QueryExpr::IsNull(_) - | QueryExpr::IsNotNull(_) - | QueryExpr::InList { .. } => (DataType::Bool, true), - QueryExpr::Arithmetic { op, left, right } => { - let (lt, ln) = infer_expr_type(left, schema)?; - let (rt, rn) = infer_expr_type(right, schema)?; - // Temporal subtraction yields a fixed duration with a unit, not a - // calendar interval or a floating-point number. Until the IR can - // preserve that unit, fail instead of publishing a numeric schema. - if matches!(op, ArithmeticOpKind::Sub) - && matches!(lt, DataType::Date | DataType::Timestamp) - && matches!(rt, DataType::Date | DataType::Timestamp) - { - return Err(QueryExprError::InvalidScalarSignature( - "temporal subtraction produces an unsupported duration type".into(), - )); - } - - // Operand order is not checked: the orders that are not valid SQL - // (`Interval - Timestamp`) are rejected by the planner upstream, so - // a pair rule stays as small as the numeric one it sits beside. - let dtype = match (<, &rt) { - // SQL unary minus lowers to -1 * expression, including intervals. - (DataType::Int64, DataType::Interval) | (DataType::Interval, DataType::Int64) - if matches!(op, ArithmeticOpKind::Mul) => - { - DataType::Interval - } - (DataType::Timestamp, DataType::Interval) - | (DataType::Interval, DataType::Timestamp) => DataType::Timestamp, - (DataType::Date, DataType::Interval) | (DataType::Interval, DataType::Date) => { - DataType::Date - } - (DataType::Interval, DataType::Interval) => DataType::Interval, - (DataType::Int64, DataType::Int64) => DataType::Int64, - _ => DataType::Float64, - }; - (dtype, ln || rn) - } - QueryExpr::Cast { to, try_cast, expr } => { - let (_, nullable) = infer_expr_type(expr, schema)?; - (to.clone(), *try_cast || nullable) - } - QueryExpr::FunctionCall { name, args } => { - if name == "asap_element_access" { - super::scalar_type_rules::element_access_type(args, schema) - .map_err(QueryExprError::InvalidScalarSignature)? - } else if name == "asap_struct_field" { - super::scalar_type_rules::struct_field_type(args, schema) - .map_err(QueryExprError::InvalidScalarSignature)? - } else if let Some(function) = - super::scalar_type_rules::MapScalarFunction::from_name(name) - { - let arguments = args - .iter() - .map(|arg| infer_expr_type(arg, schema)) - .collect::, _>>()?; - function - .output_type(&arguments) - .map_err(QueryExprError::InvalidScalarSignature)? - } else { - // Legacy unknown functions retain their existing policy. - (DataType::Float64, true) - } - } - QueryExpr::Case { - branches, - else_expr, - .. - } => { - if let Some((_, then)) = branches.first() { - (infer_expr_type(then, schema)?.0, true) - } else if let Some(other) = else_expr { - infer_expr_type(other, schema)? - } else { - (DataType::Null, true) - } - } - other => { - unreachable!("infer_expr_type called on a non-scalar QueryExpr variant: {other:?}") - } - }) -} - -/// Default output-column name for a projection item with no explicit alias: -/// a bare column keeps its (schema) name; anything else gets `col_{i}`. -fn default_proj_name(expr: &QueryExpr, idx: usize, schema: &Schema) -> String { - match expr { - QueryExpr::Column(id) => schema - .fields - .get(*id) - .map(|c| c.name.clone()) - .unwrap_or_else(|| format!("col_{idx}")), - _ => format!("col_{idx}"), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pre_asap::expr_ir::{ArithmeticOpKind, CompareOpKind}; - use crate::types::AccuracyTarget; - - fn col(name: &str, dtype: DataType, nullable: bool) -> Field { - Field::plain(name, dtype, nullable) - } - - /// Shifting an instant by a duration stays an instant, and shifting a date - /// stays a date — neither falls through to the numeric default, which is - /// what `l_shipdate + INTERVAL '30' DAY` would otherwise be typed as. - #[test] - fn interval_arithmetic_keeps_the_temporal_type() { - let schema = Schema::new(vec![ - col("ts", DataType::Timestamp, false), - col("d", DataType::Date, false), - ]); - let thirty_days = || { - Rc::new(QueryExpr::Literal(ScalarValue::Interval { - months: 0, - days: 30, - nanos: 0, - })) - }; - let shift = |column, op| QueryExpr::Arithmetic { - op, - left: Rc::new(QueryExpr::Column(column)), - right: thirty_days(), - }; - - assert_eq!( - shift(0, ArithmeticOpKind::Add) - .scalar_type(&schema) - .unwrap() - .0, - DataType::Timestamp - ); - assert_eq!( - shift(1, ArithmeticOpKind::Sub) - .scalar_type(&schema) - .unwrap() - .0, - DataType::Date - ); - assert_eq!( - QueryExpr::Arithmetic { - op: ArithmeticOpKind::Add, - left: thirty_days(), - right: thirty_days(), - } - .scalar_type(&schema) - .unwrap() - .0, - DataType::Interval - ); - } - - fn scan( - columns: Vec, - time_index: Option, - uk: Vec>, - ) -> QueryExpr { - QueryExpr::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema { - fields: columns, - time_index, - unique_keys: uk, - closed: true, - }, - } - } - - #[test] - fn project_preserves_unique_keys_that_are_passed_through() { - let input = Rc::new(scan( - vec![ - col("tenant", DataType::Utf8, false), - col("region", DataType::Utf8, false), - col("value", DataType::Int64, false), - ], - None, - vec![vec![0, 1]], - )); - let projected = QueryExpr::Project { - cols: vec![ - ProjectItem { - alias: Some("r".into()), - expr: QueryExpr::Column(1), - }, - ProjectItem { - alias: Some("t".into()), - expr: QueryExpr::Column(0), - }, - ProjectItem { - alias: None, - expr: QueryExpr::Arithmetic { - op: ArithmeticOpKind::Add, - left: Rc::new(QueryExpr::Column(2)), - right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(1))), - }, - }, - ], - qualifier: None, - child: input, - }; - - assert_eq!( - projected.output_schema().unwrap().unique_keys, - vec![vec![1, 0]] - ); - } - - #[test] - fn project_drops_a_unique_key_when_a_key_column_is_omitted() { - let input = Rc::new(scan( - vec![ - col("tenant", DataType::Utf8, false), - col("region", DataType::Utf8, false), - ], - None, - vec![vec![0, 1]], - )); - let projected = QueryExpr::Project { - cols: vec![ProjectItem { - alias: None, - expr: QueryExpr::Column(0), - }], - qualifier: None, - child: input, - }; - - assert!(projected.output_schema().unwrap().unique_keys.is_empty()); - } - - #[test] - fn legacy_window_json_without_frame_deserializes_as_unspecified() { - let window = QueryExpr::SQLWindowFunc { - func: WindowFuncKind::RowNumber, - args: vec![], - partition_by: GroupKeys::by(vec![]), - order_by: vec![], - frame: Some(WindowFrame { - units: WindowFrameUnits::Range, - start_bound: WindowFrameBound::Preceding(WindowFrameOffset::Scalar( - ScalarValue::Null, - )), - end_bound: WindowFrameBound::CurrentRow, - }), - output_name: "row_number".into(), - child: Rc::new(scan(vec![col("v", DataType::Int64, false)], None, vec![])), - }; - let mut json = serde_json::to_value(window).unwrap(); - json.get_mut("SQLWindowFunc") - .and_then(serde_json::Value::as_object_mut) - .unwrap() - .remove("frame"); - - let decoded: QueryExpr = serde_json::from_value(json).unwrap(); - assert!(matches!( - decoded, - QueryExpr::SQLWindowFunc { frame: None, .. } - )); - } - - /// A row can appear in more than one branch, so no branch's unique key is a - /// key of the union. `Concat` took the first child's schema verbatim, which - /// let a `Dedup`'s key leak out and claim a uniqueness the merged rows do - /// not have — `unique_keys` feeds CSE's producer-sharing legality check. - #[test] - fn merge_drops_the_branches_unique_keys() { - let branch = || QueryExpr::Dedup { - cols: vec![0], - child: Rc::new(scan( - vec![ - col("k", DataType::Utf8, false), - col("v", DataType::Int64, false), - ], - None, - vec![], - )), - }; - assert_eq!( - branch().output_schema().unwrap().unique_keys, - vec![vec![0]], - "a Dedup branch does have a unique key on its own" - ); - - let merged = QueryExpr::concat(vec![branch(), branch()]); - let schema = merged.output_schema().unwrap(); - assert!( - schema.unique_keys.is_empty(), - "the union of two deduplicated branches is not deduplicated" - ); - // The column shape is still the first branch's. - assert_eq!(schema.fields.len(), 2); - } - - /// Same rule as `SetOp`, which already dropped them. - #[test] - fn merge_and_setop_agree_on_unique_keys() { - let branch = || QueryExpr::Dedup { - cols: vec![0], - child: Rc::new(scan(vec![col("k", DataType::Utf8, false)], None, vec![])), - }; - let merged = QueryExpr::concat(vec![branch(), branch()]); - let setop = QueryExpr::SetOp { - kind: RelationalSetOpKind::Union, - all: true, - left: Rc::new(branch()), - right: Rc::new(branch()), - }; - assert_eq!( - merged.output_schema().unwrap().unique_keys, - setop.output_schema().unwrap().unique_keys, - ); - } - - #[test] - fn an_empty_merge_has_no_schema() { - assert!(matches!( - QueryExpr::concat(vec![]).output_schema(), - Err(QueryExprError::EmptyConcat) - )); - } - - /// Issue #228: a `Concat` built via `concat_with_discriminator` gets a - /// sound compound `(discriminator, inner_key)` unique key, even though - /// each branch's own `inner_key` alone repeats across branches (exactly - /// the shape `merge_drops_the_branches_unique_keys` shows is unsafe - /// *without* a discriminator). - #[test] - fn discriminator_override_produces_a_compound_unique_key() { - // Two branches, each individually deduplicated on column 0 (`k`) — - // but, per `merge_drops_the_branches_unique_keys`, that alone proves - // nothing about the union. Field 1 (`branch_id`) stands in for a - // discriminator the constructor has separately proven distinct per - // branch (PromQL φ, a synthetic `GROUPING()` id, ...) — this - // schema-level test only checks the shape `output_schema` derives - // from asserting one, not how a real caller proves distinctness. - let branch = || QueryExpr::Dedup { - cols: vec![0], - child: Rc::new(scan( - vec![ - col("k", DataType::Utf8, false), - col("branch_id", DataType::Int64, false), - ], - None, - vec![], - )), - }; - let merged = QueryExpr::concat_with_discriminator( - vec![branch(), branch()], - /* discriminator */ 1, - /* inner_key */ vec![0], - ); - let schema = merged.output_schema().unwrap(); - assert_eq!( - schema.unique_keys, - vec![vec![1, 0]], - "(discriminator, inner_key) is the sole asserted unique key" - ); - assert_eq!( - schema.fields.len(), - 2, - "column shape is still the first branch's" - ); - } - - #[test] - fn discriminator_assertion_rejects_unknown_wire_fields() { - let json = r#"{"discriminator":1,"inner_key":[0],"unverified":true}"#; - assert!(serde_json::from_str::(json).is_err()); - } - - /// The override is opt-in: building a `Concat` without asserting a - /// discriminator — via the plain struct literal, exactly like every call - /// site before issue #228 — still drops `unique_keys` by default, - /// unchanged. - #[test] - fn ordinary_concat_struct_literal_still_drops_unique_keys_by_default() { - let branch = || QueryExpr::Dedup { - cols: vec![0], - child: Rc::new(scan(vec![col("k", DataType::Utf8, false)], None, vec![])), - }; - let merged = QueryExpr::Concat { - children: vec![branch(), branch()], - discriminator_unique_key: None, - }; - assert!(merged.output_schema().unwrap().unique_keys.is_empty()); - } - - /// Misuse check (issue #228): there is no way to end up with a - /// discriminator-backed unique key without a call site literally naming - /// a column as the discriminator. Neither the ordinary `concat` - /// constructor nor a bare struct literal with `discriminator_unique_key: - /// None` can be coaxed into fabricating one — the only path that - /// produces `Some` is `concat_with_discriminator` / - /// `ConcatDiscriminatorKey::new`, both of which require `discriminator` - /// as an explicit, named argument. - #[test] - fn no_way_to_fabricate_a_unique_key_without_naming_a_discriminator() { - let branch = || QueryExpr::Dedup { - cols: vec![0], - child: Rc::new(scan(vec![col("k", DataType::Utf8, false)], None, vec![])), - }; - // The ordinary builder. - assert_eq!( - QueryExpr::concat(vec![branch(), branch()]) - .output_schema() - .unwrap() - .unique_keys, - Vec::>::new() - ); - // The bare struct literal, explicitly opting out. - assert_eq!( - QueryExpr::Concat { - children: vec![branch(), branch()], - discriminator_unique_key: None, - } - .output_schema() - .unwrap() - .unique_keys, - Vec::>::new() - ); - } - - #[test] - fn project_retypes_and_renames_per_item() { - let child = scan( - vec![ - col("ts", DataType::Timestamp, false), - col("host", DataType::Utf8, false), - col("value", DataType::Float64, false), - ], - Some(0), - vec![vec![0, 1]], - ); - let q = QueryExpr::Project { - qualifier: None, - cols: vec![ - // bare column passthrough keeps its (schema) name + type: host=col 1 - ProjectItem { - alias: None, - expr: QueryExpr::Column(1), - }, - // arithmetic over value (col 2) → Float64 - ProjectItem { - alias: Some("dbl".into()), - expr: QueryExpr::Arithmetic { - op: ArithmeticOpKind::Add, - left: Rc::new(QueryExpr::Column(2)), - right: Rc::new(QueryExpr::Column(2)), - }, - }, - // comparison → Bool (nullable under 3-valued logic) - ProjectItem { - alias: Some("flag".into()), - expr: QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(2)), - op: CompareOpKind::Gt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(0.0))), - }, - }, - ], - child: Rc::new(child), - }; - let s = q.output_schema().unwrap(); - assert_eq!(s.fields.len(), 3); - assert_eq!(s.fields[0], col("host", DataType::Utf8, false)); - assert_eq!(s.fields[1], col("dbl", DataType::Float64, false)); - assert_eq!(s.fields[2], col("flag", DataType::Bool, true)); - // projection drops the time axis + unique keys (ts not retained) - assert!(s.time_index.is_none()); - assert!(s.unique_keys.is_empty()); - } - - #[test] - fn group_keys_by_vs_without_semantics() { - let by = GroupKeys::by(vec![1, 2]); - let without = GroupKeys::without(vec![1, 2]); - assert!(!by.is_without()); - assert!(without.is_without()); - // Deref / iteration expose the stored keys regardless of mode. - assert_eq!(by.len(), 2); - assert_eq!(without.keys(), &[1, 2]); - // A `by` compares equal to its bare vec; a `without` never does. - assert_eq!(by, vec![1, 2]); - assert_ne!(without, vec![1, 2]); - assert_ne!(by, without); - } - - #[test] - fn group_keys_serde_by_is_bare_array_without_is_tagged() { - // `by` keeps the pre-#39 bare-array wire format; `without` uses an object. - let by = serde_json::to_string(&GroupKeys::by(vec![2, 3])).unwrap(); - assert_eq!(by, "[2,3]"); - let without = serde_json::to_string(&GroupKeys::without(vec![2])).unwrap(); - assert_eq!(without, r#"{"without":[2]}"#); - // Round-trip both. - for g in [GroupKeys::by(vec![2, 3]), GroupKeys::without(vec![2])] { - let json = serde_json::to_string(&g).unwrap(); - let back: GroupKeys = serde_json::from_str(&json).unwrap(); - assert_eq!(back, g); - } - } - - #[test] - fn without_aggregate_keeps_open_schema_minus_excluded() { - // `sum without (instance) (m)` over `[ts, value, instance, job]`: the - // kept labels are the input labels minus the excluded `instance` (and ts - // / value), followed by the `sum` column, and the schema stays OPEN - // (issue #39). `job` survives; `instance` is dropped. - let scan_node = QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - col("instance", DataType::Utf8, true), - col("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - }; - let agg = QueryExpr::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![2])), // exclude `instance` - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan_node), - }; - let s = agg.output_schema().unwrap(); - let names: Vec<_> = s.fields.iter().map(|c| c.name.as_str()).collect(); - assert_eq!(names, vec!["job", "sum"], "kept `job`, dropped `instance`"); - assert!(!s.closed, "a `without` result stays open"); - assert!(s.time_index.is_none()); - assert!(s.unique_keys.is_empty(), "kept set unknown → no unique key"); - } - - // A nested aggregate's renamed sample value is not a kept label. - #[test] - fn without_aggregate_drops_a_renamed_sample_value() { - // `sum without (inst) (sum by (inst, job) (m))` over `[inst, job, sum]`. - let inner = QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::new(vec![ - col("inst", DataType::Utf8, true), - col("job", DataType::Utf8, true), - col("sum", DataType::Float64, false), - ]), - }; - let agg = QueryExpr::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![0])), - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(inner), - }; - let s = agg.output_schema().unwrap(); - let names: Vec<_> = s.fields.iter().map(|c| c.name.as_str()).collect(); - assert_eq!(names, vec!["job", "sum"]); - } - - #[test] - fn time_shift_is_schema_pass_through() { - // `offset`/`@` move *when* a selector is evaluated, never its columns — - // a `TimeShift` output schema equals its child's (issue #40). - let scan_node = scan( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - col("job", DataType::Utf8, true), - ], - Some(0), - vec![], - ); - let shifted = QueryExpr::TimeShift { - shift: TimeShift { - offset_ms: 3_600_000, - at: Some(AtModifier::Timestamp(1_609_746_000_000)), - }, - child: Rc::new(scan_node.clone()), - }; - assert_eq!( - shifted.output_schema().unwrap(), - scan_node.output_schema().unwrap(), - ); - } - - #[test] - fn time_shift_identity_and_serde() { - let offset_only = TimeShift { - offset_ms: 1, - at: None, - }; - let at_only = TimeShift { - offset_ms: 0, - at: Some(AtModifier::End), - }; - assert!(TimeShift::default().is_identity()); - assert!(!offset_only.is_identity()); - assert!(!at_only.is_identity()); - // Round-trip the shift + anchor. - let s = TimeShift { - offset_ms: -300_000, - at: Some(AtModifier::Timestamp(60_000)), - }; - let back: TimeShift = serde_json::from_str(&serde_json::to_string(&s).unwrap()).unwrap(); - assert_eq!(back, s); - } - - // Nested temporal aggregation must replace the sample, never the grouping label. - #[test] - fn temporal_reduction_of_grouped_sum_preserves_job() { - let input = Schema::new(vec![ - col("job", DataType::Utf8, true), - col("sum", DataType::Float64, false), - ]); - for aggregate in [ - AggIntent::Avg { col: None }, - AggIntent::Avg { col: Some(1) }, - AggIntent::Rate, - ] { - let output = - aggregate_output_schema(&input, &Reduction::PerEntity, &[aggregate], &[]).unwrap(); - assert_eq!(output.fields[0], input.fields[0]); - assert_eq!(output.fields[1].name, "value"); - assert_eq!(output.fields[1].dtype, DataType::Float64); - } - } - - #[test] - fn per_series_rate_preserves_labels() { - // A per-series range reduction (`rate`) is label-preserving: it produces - // one value per series, so every label survives and only the sample - // value is replaced (kept named `value`). The TimeRange child is the - // structural marker; the outer Aggregate carries the Rate intent. - let scan_node = scan( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - col("job", DataType::Utf8, true), - ], - Some(0), - vec![], - ); - let rate = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Rate], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(QueryExpr::TimeRange { - range: Duration::from_secs(300), - child: Rc::new(scan_node), - }), - }; - let s = rate.output_schema().unwrap(); - assert_eq!( - s.fields.iter().map(|c| c.name.as_str()).collect::>(), - vec!["ts", "value", "job"], - "rate preserves all labels; only the sample value is replaced" - ); - assert_eq!(s.time_index, Some(0)); - assert!(s.column_id("job").is_some(), "label survives the reduction"); - } - - #[test] - fn over_time_reduction_preserves_labels() { - // `*_over_time` lowers to `Aggregate { by:[], [reducer], TimeRange { Scan } }`: - // a per-series time-range reduction. The TimeRange child confers per-series - // semantics on otherwise cross-series intents like `Avg`, so an outer - // `sum by(job)(avg_over_time(...))` resolves its key positionally. - let scan_node = scan( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - col("job", DataType::Utf8, true), - ], - Some(0), - vec![], - ); - let avg_over_time = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Avg { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(QueryExpr::TimeRange { - range: Duration::from_secs(300), - child: Rc::new(scan_node), - }), - }; - let s = avg_over_time.output_schema().unwrap(); - assert_eq!( - s.fields.iter().map(|c| c.name.as_str()).collect::>(), - vec!["ts", "value", "job"], - "TimeRange-child marks per-series: labels preserved, value renamed" - ); - assert!( - s.column_id("job").is_some(), - "outer Aggregate.by can resolve it" - ); - } - - #[test] - fn completeness_open_leaf_freezes_to_closed_at_cross_series_aggregate() { - // A schemaless (PromQL-style) leaf is *open*; it stays open through a - // per-series reduction (`rate`), then is **frozen to closed** by a - // cross-series aggregate (which enumerates exactly its output columns). - let open_leaf = QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - // `with_time_index` defaults to `closed: false` (open). - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - col("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - }; - assert!( - !open_leaf.output_schema().unwrap().closed, - "schemaless leaf is open" - ); - - let rate = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Rate], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(open_leaf), - }; - assert!( - !rate.output_schema().unwrap().closed, - "per-series rate is label-preserving → stays open" - ); - - let sum_by_job = QueryExpr::Aggregate { - reduction: Reduction::by(vec![2]), // `job` - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(rate), - }; - assert!( - sum_by_job.output_schema().unwrap().closed, - "cross-series aggregate enumerates `by ++ measures` → frozen to closed" - ); - } - - #[test] - fn project_keeps_time_index_when_ts_passed_through() { - let child = scan( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - ], - Some(0), - vec![], - ); - let q = QueryExpr::Project { - qualifier: None, - cols: vec![ - // value=col 1, ts=col 0 - ProjectItem { - alias: None, - expr: QueryExpr::Column(1), - }, - ProjectItem { - alias: None, - expr: QueryExpr::Column(0), - }, - ], - child: Rc::new(child), - }; - let s = q.output_schema().unwrap(); - assert_eq!(s.fields[0].name, "value"); - assert_eq!(s.fields[1].name, "ts"); - assert_eq!(s.time_index, Some(1)); - } - - fn join(kind: JoinKind) -> QueryExpr { - let left = scan(vec![col("a", DataType::Int64, false)], None, vec![vec![0]]); - let right = scan(vec![col("b", DataType::Utf8, false)], None, vec![]); - QueryExpr::Join { - kind, - pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(true)))), - left: Rc::new(left), - right: Rc::new(right), - } - } - - #[test] - fn inner_join_concatenates_both_sides() { - let s = join(JoinKind::Inner).output_schema().unwrap(); - assert_eq!(s.fields.len(), 2); - assert_eq!(s.fields[0], col("a", DataType::Int64, false)); - assert_eq!(s.fields[1], col("b", DataType::Utf8, false)); - // post-join row identity not provable → no unique keys - assert!(s.unique_keys.is_empty()); - } - - #[test] - fn left_join_makes_right_side_nullable() { - let s = join(JoinKind::Left).output_schema().unwrap(); - assert!(!s.fields[0].nullable, "preserved left side stays non-null"); - assert!(s.fields[1].nullable, "right side nullable under LEFT JOIN"); - } - - #[test] - fn full_join_makes_both_sides_nullable() { - let s = join(JoinKind::Full).output_schema().unwrap(); - assert!(s.fields[0].nullable); - assert!(s.fields[1].nullable); - } - - #[test] - fn setop_takes_left_shape_and_drops_unique_keys() { - let left = scan( - vec![ - col("k", DataType::Utf8, false), - col("v", DataType::Int64, false), - ], - None, - vec![vec![0]], - ); - let right = scan( - vec![ - col("k", DataType::Utf8, false), - col("v", DataType::Int64, false), - ], - None, - vec![vec![0]], - ); - let q = QueryExpr::SetOp { - kind: RelationalSetOpKind::Union, - all: false, - left: Rc::new(left), - right: Rc::new(right), - }; - let s = q.output_schema().unwrap(); - assert_eq!(s.fields.len(), 2); - assert_eq!(s.fields[0].name, "k"); - assert!( - s.unique_keys.is_empty(), - "UNION does not preserve row identity" - ); - } - - // ── PromqlScalarBridge / Literal dedup (issue #220) ───────────────────── - - /// `QueryExpr::promql_scalar(v)` — what every front end now constructs in - /// place of the old `PromqlScalar(v)` leaf — wraps exactly - /// `Literal(ScalarValue::Float64(v))`: the same value a SQL-emitted typed - /// float literal in a scalar-sub-language position would carry, just at a - /// different DAG position. `as_promql_scalar` is the round-trip inverse. - #[test] - fn promql_scalar_bridges_a_literal_float_at_an_operator_position() { - let bridge = QueryExpr::::promql_scalar(2.5); - assert_eq!( - bridge, - QueryExpr::PromqlScalarBridge(Rc::new(QueryExpr::Literal(ScalarValue::Float64(2.5)))) - ); - assert_eq!(bridge.as_promql_scalar(), Some(2.5)); - - // The same value a SQL `Compare`/`Arithmetic` operand would carry, in - // its native (unwrapped, no row schema) scalar-sub-language position — - // no longer a different variant, just not bridged to this DAG - // position. - let sql_literal = QueryExpr::::Literal(ScalarValue::Float64(2.5)); - assert_eq!(bridge.as_promql_scalar(), Some(2.5)); - assert_ne!( - bridge, sql_literal, - "bridge and bare literal are distinct nodes" - ); - // Not every shape is a scalar bridge: neither a bare `Literal` nor an - // operator node reports a value. - assert_eq!(sql_literal.as_promql_scalar(), None); - assert_eq!(scan(vec![], None, vec![]).as_promql_scalar(), None); - } - - /// Pins the DAG-position distinction issue #220 asks for: the very same - /// `Literal(ScalarValue::Float64(_))` value has a row schema when it sits - /// at the operator-DAG position (wrapped in `PromqlScalarBridge` — a - /// `BinaryOp` operand, `PromqlVectorFromScalar` child, or a query root), - /// and has none when it sits bare, in a scalar-sub-language position - /// (`Compare`/`Arithmetic`/… operand) — no longer decided by which of two - /// duplicate variants was used, only by whether the wrapper is present. - #[test] - fn row_schema_rides_on_the_bridge_wrapper_not_the_literal_variant() { - let bridged = QueryExpr::::promql_scalar(42.0); - let schema = bridged.output_schema().expect("bridge has a row schema"); - assert_eq!(schema.fields.len(), 1); - assert_eq!(schema.fields[0].name, "value"); - assert_eq!(schema.fields[0].dtype, DataType::Float64); - assert!(schema.time_index.is_none()); - - // The identical value, unwrapped (the scalar-sub-language position a - // `Compare`/`Arithmetic` operand would occupy) has no row schema of - // its own — it's a construction bug to call `output_schema` on it - // directly, caught as `ScalarHasNoRowSchema` rather than panicking. - let bare = QueryExpr::::Literal(ScalarValue::Float64(42.0)); - assert!(matches!( - bare.output_schema(), - Err(QueryExprError::ScalarHasNoRowSchema) - )); - } - - /// `BinaryOp`'s schema derivation follows the non-scalar (vector) side - /// when the other operand is a `PromqlScalarBridge`, and a `VectorMatch` - /// modifier survives unchanged alongside it — the relational binary-op - /// path (issue #220's Instance 2, left as follow-up) is untouched by the - /// Instance-1 `PromqlScalar` → `PromqlScalarBridge` collapse. - // `filters` (#466) round-trips, and an `Aggregate` serialized before the - // field existed still deserializes as unfiltered. - #[test] - fn aggregate_filters_serde_round_trip_and_default() { - let child = Rc::new(scan( - vec![ - col("service", DataType::Utf8, false), - col("latency", DataType::Float64, false), - ], - None, - vec![], - )); - let filtered = QueryExpr::Aggregate { - reduction: Reduction::by(vec![0]), - measures: vec![ - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - AggIntent::Sum { col: Some(1) }, - ], - output_names: vec![], - filters: vec![ - Some(Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(1)), - op: CompareOpKind::Gt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(1.0))), - }))), - None, - ], - having: None, - child: Rc::clone(&child), - }; - let json = serde_json::to_value(&filtered).unwrap(); - assert_eq!( - serde_json::from_value::(json.clone()).unwrap(), - filtered - ); - - let mut legacy = json; - legacy["Aggregate"] - .as_object_mut() - .unwrap() - .remove("filters") - .expect("fixture sanity: filters was serialized"); - let decoded: QueryExpr = serde_json::from_value(legacy).unwrap(); - let QueryExpr::Aggregate { filters, .. } = &decoded else { - unreachable!() - }; - assert!(filters.is_empty()); - } - - #[test] - fn binary_op_schema_follows_the_vector_side_over_a_scalar_bridge_with_vector_match_intact() { - let vector = scan( - vec![ - col("host", DataType::Utf8, false), - col("value", DataType::Float64, false), - ], - None, - vec![], - ); - let vm = VectorMatch { - kind: VectorMatchKind::On, - labels: vec!["host".into()], - grouping: None, - }; - let op = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOpKind::Gt), - lhs: Rc::new(vector.clone()), - rhs: Rc::new(QueryExpr::promql_scalar(1.0)), - vector_match: Some(vm.clone()), - }; - assert_eq!(op.output_schema().unwrap(), vector.output_schema().unwrap()); - let QueryExpr::BinaryOp { vector_match, .. } = &op else { - unreachable!() - }; - assert_eq!(vector_match.as_ref(), Some(&vm)); - } -} diff --git a/crates/types/src/pre_asap/resolve.rs b/crates/types/src/pre_asap/resolve.rs deleted file mode 100644 index b4a5c87a6..000000000 --- a/crates/types/src/pre_asap/resolve.rs +++ /dev/null @@ -1,857 +0,0 @@ -//! Resolve a front-end-emitted, unresolved [`UnresolvedQueryExpr`] (`QueryExpr`) -//! into the canonical, positional [`ResolvedQueryExpr`] (`QueryExpr`). -//! -//! Both front ends (`asap-frontend-promql`, `asap-frontend-sql`) construct -//! canonical `QueryExpr` shapes directly during their own `interpret` step -//! (issue #179) — heavy-hitter `topk` recognition, the window-over-aggregate -//! fold, the `PerEntity`/`Reduce` reduction choice, and every other -//! *structural* decision happen right there, since a front end already knows -//! the answer at parse time. What's left for [`resolve_root`] is exactly the -//! "mechanical, schema-dependent substitution" #179 describes: a single -//! generic, shape-preserving walk — every [`UnresolvedQueryExpr`] variant maps to the -//! identical [`ResolvedQueryExpr`] variant — that resolves every [`ColumnRef`] to -//! the [`SchemaResolver`](super::schema_resolver::SchemaResolver)-computed positional [`ColumnId`]. -//! -//! ## Why positional `ColumnId`, not just carrying names all the way through (issue #216) -//! -//! A mature query engine can legitimately choose either design — DataFusion's -//! own logical plan (what `asap-frontend-sql` walks to build its `QueryExpr`) -//! and Calcite both keep names, with an optional table qualifier, all the way -//! through logical optimization, only going positional once they lower to a -//! physical plan. Resolving once, immediately after each front end's own -//! `interpret` step, is the better trade for *this* codebase's shape — one -//! front-end-facing DAG feeding several independent downstream passes -//! (`canonicalize`, the cost model, `dag_export`, schema/type inference, -//! `asap-aware-mapping`'s summary binding) — for three concrete reasons: -//! -//! 1. **Names collide across joins.** Not hypothetical: `join_predicate_disambiguates_shared_column_name` -//! (`crates/frontend-sql/tests/sql_lowering.rs`) exists specifically because -//! `metrics.service` and `hosts.service` are both just `"service"` once their -//! schemas are concatenated. A bare name is ambiguous the moment two sources -//! share one; `ColumnId` is what makes "the second `service`, position 4, not -//! the first" a fact recorded once, instead of a lookup redone at every use site. -//! 2. **A name's meaning changes going up the DAG.** `Project` renames/aliases, -//! `Aggregate` collapses columns and introduces synthetic ones, `Join` -//! concatenates two schemas — a name valid at a `Scan` leaf isn't -//! automatically the right binding three nodes up; it has to be reinterpreted -//! against whatever schema is in scope at that node. Resolving bottom-up -//! pins each reference to "this exact column of this exact node's -//! already-derived output schema," so nothing downstream re-derives that scope. -//! 3. **It concentrates scoping logic in one place instead of ~6.** Every -//! downstream pass just compares/indexes `ColumnId`s — O(1), unambiguous. If -//! they worked on names instead, each would need its own qualifier-aware, -//! join-collision-aware name resolver, or risk silently binding to the wrong -//! `"service"`. -//! -//! Removing this resolution step and carrying `ColumnRef` everywhere would -//! therefore be a real regression for this repo's shape, not just a rename — -//! every one of those downstream passes would have to reimplement the scoping -//! this module already centralizes. - -use std::rc::Rc; - -use thiserror::Error; - -use super::agg_intent::AggIntent; -use super::column_resolution::{ - resolve_column_ref, resolve_column_refs, resolve_expr, resolve_group_keys_promql, ResolveError, -}; -use super::expr_ir::ColumnRef; -use super::query_expr::{ - aggregate_output_schema, any_measure_filtered, ConcatDiscriminatorKey, GroupKeys, Predicate, - ProjectItem, QueryExprError, Reduction, ResolvedQueryExpr, SortKey, UnresolvedQueryExpr, -}; -use super::schema::{ColumnId, Schema}; -use super::schema_resolver::SchemaResolver; - -/// Errors from resolving a canonical, unresolved [`UnresolvedQueryExpr`] DAG. -#[derive(Debug, Error)] -pub enum ResolveDAGError { - /// A column reference did not resolve against its in-scope schema. - #[error("column resolution failed: {0}")] - Resolve(#[from] ResolveError), - /// Deriving the schema of an already-resolved child failed (needed to - /// resolve positional column references against it). - #[error("schema derivation failed: {0}")] - Schema(#[from] QueryExprError), -} - -/// Resolve a whole [`UnresolvedQueryExpr`] DAG rooted at `dag` into canonical -/// [`ResolvedQueryExpr`]: binds every `ColumnRef` to a `ColumnId` via the -/// [`SchemaResolver`], then [`canonicalize`](super::canonicalize::canonicalize)s the -/// result. -pub fn resolve_root(dag: &UnresolvedQueryExpr) -> Result { - resolve_root_with_inherited(dag, &[]) -} - -/// [`resolve_root`] with label names inherited from an enclosing scope seeded -/// into the leaf schema, used when re-binding a `BinaryOp` side (issue #52). -fn resolve_root_with_inherited( - dag: &UnresolvedQueryExpr, - inherited: &[String], -) -> Result { - let fallback = SchemaResolver::new().resolve_schema_with_inherited(dag, inherited); - let l3 = resolve(dag, &fallback)?; - Ok(super::canonicalize::canonicalize(l3)) -} - -/// The generic substitution walk: converts children first (bottom-up), then -/// resolves this node's own `ColumnRef`s against the *converted child's* -/// derived output schema — so a `JOIN`'s concatenated schema and a cross- -/// series aggregate's frozen-closed output bind to the right positions. -fn resolve( - dag: &UnresolvedQueryExpr, - fallback: &Schema, -) -> Result { - use super::query_expr::QueryExpr as QE; - Ok(match dag { - QE::Scan { - source, - predicates, - schema, - } => { - let schema = schema.clone().unwrap_or_else(|| fallback.clone()); - let predicates = predicates - .iter() - .map(|Predicate(e)| Ok(Predicate(Rc::new(resolve_expr(e, &schema)?)))) - .collect::, ResolveError>>()?; - QE::Scan { - source: source.clone(), - predicates, - schema, - } - } - - // `PromqlScalarBridge`'s child is a scalar-sub-language node (issue - // #220) sitting at this operator-DAG position — resolved through - // `resolve_expr`, same as every other scalar position (`Predicate`, - // `ProjectItem.expr`, …), not the operator walk. In practice it's - // always a `Literal`, which has no `ColumnRef` to resolve, so - // `fallback` is never actually consulted here. - QE::PromqlScalarBridge(inner) => { - QE::PromqlScalarBridge(Rc::new(resolve_expr(inner, fallback)?)) - } - QE::EvalTimestamp => QE::EvalTimestamp, - QE::CurrentTimestamp => QE::CurrentTimestamp, - - QE::PromqlVectorFromScalar(child) => { - QE::PromqlVectorFromScalar(Rc::new(resolve(child, fallback)?)) - } - QE::PromqlScalarFromVector(child) => { - QE::PromqlScalarFromVector(Rc::new(resolve(child, fallback)?)) - } - - QE::PromqlRelabel { dst, value, child } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - QE::PromqlRelabel { - dst: dst.clone(), - value: Rc::new(resolve_expr(value, &child_schema)?), - child: Rc::new(child), - } - } - - QE::PromqlInfoEnrich { selector, child } => QE::PromqlInfoEnrich { - selector: selector.clone(), - child: Rc::new(resolve(child, fallback)?), - }, - - QE::PromqlSeriesSample { by, kind, child } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - QE::PromqlSeriesSample { - by: resolve_group_keys(by, &child_schema)?, - kind: *kind, - child: Rc::new(child), - } - } - - QE::Filter { pred, child } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - QE::Filter { - pred: Predicate(Rc::new(resolve_expr(&pred.0, &child_schema)?)), - child: Rc::new(child), - } - } - - QE::Project { - cols, - qualifier, - child, - } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - let cols = cols - .iter() - .map(|item| -> Result { - Ok(ProjectItem { - alias: item.alias.clone(), - expr: resolve_expr(&item.expr, &child_schema)?, - }) - }) - .collect::, _>>()?; - QE::Project { - cols, - qualifier: qualifier.clone(), - child: Rc::new(child), - } - } - - QE::Aggregate { - reduction, - measures, - output_names, - filters, - having, - child, - } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - let reduction = resolve_reduction(reduction, &child_schema)?; - let measures = measures - .iter() - .map(|m| resolve_agg_intent(m, &child_schema)) - .collect::, ResolveError>>()?; - // A measure filter reads the rows being aggregated, so it binds - // against the child's schema, not the aggregate's output. - let filters = filters - .iter() - .map(|f| { - f.as_ref() - .map(|Predicate(p)| Ok(Predicate(Rc::new(resolve_expr(p, &child_schema)?)))) - .transpose() - }) - .collect::, ResolveError>>()?; - // One canonical spelling of "unfiltered" (empty), so structural - // equality and CSE never split on `[]` versus `[None, None]`. - let filters = if any_measure_filtered(&filters) { - filters - } else { - Vec::new() - }; - let having = having - .as_ref() - .map(|Predicate(h)| -> Result { - let out_schema = aggregate_output_schema( - &child_schema, - &reduction, - &measures, - output_names, - )?; - Ok(Predicate(Rc::new(resolve_expr(h, &out_schema)?))) - }) - .transpose()?; - QE::Aggregate { - reduction, - measures, - output_names: output_names.clone(), - filters, - having, - child: Rc::new(child), - } - } - - QE::Dedup { cols, child } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - QE::Dedup { - cols: resolve_column_refs(cols, &child_schema)?, - child: Rc::new(child), - } - } - - QE::Concat { - children, - discriminator_unique_key, - } => { - let children: Vec<_> = children - .iter() - .map(|c| resolve(c, fallback)) - .collect::, _>>()?; - // No front end asserts this today (issue #228 shipped the - // extension point ahead of a wired call site) — resolved here - // regardless, against the first resolved branch's own output - // schema, exactly the schema `output_schema`'s `Concat` arm - // derives the merged schema from, so a future direct - // `concat_with_discriminator` caller upstream of `resolve_root` - // gets a correctly positional `ConcatDiscriminatorKey` out the - // other side. - let discriminator_unique_key = discriminator_unique_key - .as_ref() - .map(|key| -> Result<_, ResolveDAGError> { - let schema = children - .first() - .ok_or(QueryExprError::EmptyConcat)? - .output_schema()?; - Ok(ConcatDiscriminatorKey::new( - resolve_column_ref(key.discriminator(), &schema)?, - resolve_column_refs(key.inner_key(), &schema)?, - )) - }) - .transpose()?; - QE::Concat { - children, - discriminator_unique_key, - } - } - - QE::Join { - kind, - pred, - left, - right, - } => { - // Each branch is bound independently, same reasoning as `BinaryOp` - // below — different leaves / label sets. - let left = resolve_root_with_inherited(left, &[])?; - let right = resolve_root_with_inherited(right, &[])?; - let mut concat = left.output_schema()?; - concat.fields.extend(right.output_schema()?.fields); - let pred = Predicate(Rc::new(resolve_expr(&pred.0, &concat)?)); - QE::Join { - kind: kind.clone(), - pred, - left: Rc::new(left), - right: Rc::new(right), - } - } - - QE::SetOp { - kind, - all, - left, - right, - } => QE::SetOp { - kind: kind.clone(), - all: *all, - left: Rc::new(resolve_root_with_inherited(left, &[])?), - right: Rc::new(resolve_root_with_inherited(right, &[])?), - }, - - QE::Sort { - keys, - partition_by, - child, - } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - let keys = keys - .iter() - .map(|k| -> Result { - Ok(SortKey { - expr: resolve_expr(&k.expr, &child_schema)?, - ascending: k.ascending, - nulls_first: k.nulls_first, - }) - }) - .collect::, _>>()?; - let partition_by = resolve_group_keys(partition_by, &child_schema)?; - QE::Sort { - keys, - partition_by, - child: Rc::new(child), - } - } - - QE::Limit { n, offset, child } => QE::Limit { - n: *n, - offset: *offset, - child: Rc::new(resolve(child, fallback)?), - }, - - QE::PromqlSubquery { - range, - resolution, - child, - } => QE::PromqlSubquery { - range: *range, - resolution: *resolution, - child: Rc::new(resolve(child, fallback)?), - }, - - QE::TimeRange { range, child } => QE::TimeRange { - range: *range, - child: Rc::new(resolve(child, fallback)?), - }, - - QE::TimeShift { shift, child } => QE::TimeShift { - shift: *shift, - child: Rc::new(resolve(child, fallback)?), - }, - - QE::SQLWindowFunc { - func, - args, - partition_by, - order_by, - frame, - output_name, - child, - } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - let args = args - .iter() - .map(|a| resolve_expr(a, &child_schema)) - .collect::, _>>()?; - let partition_by = resolve_group_keys(partition_by, &child_schema)?; - let order_by = order_by - .iter() - .map(|k| -> Result { - Ok(SortKey { - expr: resolve_expr(&k.expr, &child_schema)?, - ascending: k.ascending, - nulls_first: k.nulls_first, - }) - }) - .collect::, _>>()?; - QE::SQLWindowFunc { - func: func.clone(), - args, - partition_by, - order_by, - frame: frame.clone(), - output_name: output_name.clone(), - child: Rc::new(child), - } - } - - QE::BinaryOp { - op, - lhs, - rhs, - vector_match, - } => { - // A binary op's two sides may scan different metrics with - // different label sets, so each branch resolves against its OWN - // bound schema; but an independently-bound side still has to see - // label names an *enclosing* node references (issue #52). - let own = super::schema_resolver::collect_referenced_columns(dag); - let inherited: Vec = inherited_names(fallback) - .into_iter() - .filter(|n| !own.contains(n)) - .collect(); - QE::BinaryOp { - op: op.clone(), - lhs: Rc::new(resolve_root_with_inherited(lhs, &inherited)?), - rhs: Rc::new(resolve_root_with_inherited(rhs, &inherited)?), - vector_match: vector_match.clone(), - } - } - - // The scalar variants (issue #205) are never reached here directly — - // `resolve` only ever recurses into `child`/operator positions; - // every scalar position (`Predicate`, `ProjectItem.expr`, …) goes - // through `resolve_expr` instead, at the operator arm that owns it. - other @ (QE::Column(_) - | QE::Literal(_) - | QE::Compare { .. } - | QE::BoolAnd(_) - | QE::BoolOr(_) - | QE::Not(_) - | QE::IsNull(_) - | QE::IsNotNull(_) - | QE::Cast { .. } - | QE::InList { .. } - | QE::FunctionCall { .. } - | QE::Arithmetic { .. } - | QE::Case { .. }) => { - unreachable!("resolve reached a scalar QueryExpr variant directly: {other:?}") - } - }) -} - -/// The label names an enclosing scope's schema carries beyond the `(ts, -/// value)` floor. -fn inherited_names(schema: &Schema) -> Vec { - schema - .fields - .iter() - .filter(|c| c.name != "ts" && c.name != "value") - .map(|c| c.name.clone()) - .collect() -} - -/// Resolve a name-based [`GroupKeys`] into positional -/// [`GroupKeys`], preserving its `by`/`without` mode. -fn resolve_group_keys( - keys: &GroupKeys, - schema: &Schema, -) -> Result, ResolveError> { - let ids = resolve_column_refs(keys.keys(), schema)?; - Ok(if keys.is_without() { - GroupKeys::without(ids) - } else { - GroupKeys::by(ids) - }) -} - -/// Resolve a name-based [`Reduction`] into positional -/// [`Reduction`]. -/// -/// Uses [`resolve_group_keys_promql`] rather than the strict -/// [`resolve_group_keys`], unlike every other group-key site in `resolve` -/// (`PromqlSeriesSample.by`, `Sort.partition_by`, `SQLWindowFunc.partition_by`): a key -/// absent from a **closed** schema (e.g. the output of a nested cross-series -/// aggregate that collapsed the label) is provably absent from every row, so -/// PromQL drops it from the grouping rather than rejecting the query (issue -/// #53) — `sum(sum by (group) (m)) by (job)` is the canonical case, `job` -/// absent from the inner aggregate's closed `[group, sum]` output. Applied -/// uniformly to every `Aggregate`, not just PromQL's: SQL's `GROUP BY` keys -/// are always genuinely present (DataFusion validates the plan), so the -/// "drop instead of reject" branch is simply never exercised there — the -/// lenient resolver is a no-op difference for a SQL DAG, not a behavior -/// change. -fn resolve_reduction( - reduction: &Reduction, - schema: &Schema, -) -> Result, ResolveError> { - Ok(match reduction { - Reduction::Reduce(by) => { - let ids = resolve_group_keys_promql(by.keys(), schema)?; - Reduction::Reduce(if by.is_without() { - GroupKeys::without(ids) - } else { - GroupKeys::by(ids) - }) - } - Reduction::PerEntity => Reduction::PerEntity, - }) -} - -/// Resolve a name-based [`AggIntent`] into positional -/// [`AggIntent`] — every `col: Option` resolves to -/// `Option` (`None` stays `None`, the sample-value convention); -/// every other field carries straight through unchanged. -fn resolve_agg_intent( - intent: &AggIntent, - schema: &Schema, -) -> Result, ResolveError> { - let col = |c: &Option| -> Result, ResolveError> { - c.as_ref() - .map(|r| resolve_column_ref(r, schema)) - .transpose() - }; - Ok(match intent { - AggIntent::Count { accuracy } => AggIntent::Count { - accuracy: accuracy.clone(), - }, - AggIntent::PearsonCorr { left, right } => AggIntent::PearsonCorr { - left: resolve_column_ref(left, schema)?, - right: resolve_column_ref(right, schema)?, - }, - AggIntent::Sum { col: c } => AggIntent::Sum { col: col(c)? }, - AggIntent::Min { col: c } => AggIntent::Min { col: col(c)? }, - AggIntent::Max { col: c } => AggIntent::Max { col: col(c)? }, - AggIntent::Avg { col: c } => AggIntent::Avg { col: col(c)? }, - AggIntent::StdDev { col: c, population } => AggIntent::StdDev { - col: col(c)?, - population: *population, - }, - AggIntent::Variance { col: c, population } => AggIntent::Variance { - col: col(c)?, - population: *population, - }, - AggIntent::Quantile { - col: c, - q, - accuracy, - } => AggIntent::Quantile { - col: col(c)?, - q: *q, - accuracy: accuracy.clone(), - }, - AggIntent::TopK { k, accuracy } => AggIntent::TopK { - k: *k, - accuracy: accuracy.clone(), - }, - AggIntent::Cardinality { cols, accuracy } => AggIntent::Cardinality { - cols: cols - .iter() - .map(|c| resolve_column_ref(c, schema)) - .collect::>()?, - accuracy: accuracy.clone(), - }, - AggIntent::FrequencyL2 { col: c, accuracy } => AggIntent::FrequencyL2 { - col: col(c)?, - accuracy: accuracy.clone(), - }, - AggIntent::FrequencyEntropy { col: c, accuracy } => AggIntent::FrequencyEntropy { - col: col(c)?, - accuracy: accuracy.clone(), - }, - AggIntent::Rate => AggIntent::Rate, - AggIntent::IRate => AggIntent::IRate, - AggIntent::Increase => AggIntent::Increase, - AggIntent::Changes => AggIntent::Changes, - AggIntent::Delta => AggIntent::Delta, - AggIntent::IDelta => AggIntent::IDelta, - AggIntent::Deriv => AggIntent::Deriv, - AggIntent::Resets => AggIntent::Resets, - AggIntent::PredictLinear { seconds } => AggIntent::PredictLinear { seconds: *seconds }, - AggIntent::DoubleExpSmoothing { smoothing, trend } => AggIntent::DoubleExpSmoothing { - smoothing: *smoothing, - trend: *trend, - }, - AggIntent::HistogramCount => AggIntent::HistogramCount, - AggIntent::HistogramSum => AggIntent::HistogramSum, - AggIntent::HistogramAvg => AggIntent::HistogramAvg, - AggIntent::HistogramStdDev => AggIntent::HistogramStdDev, - AggIntent::HistogramStdVar => AggIntent::HistogramStdVar, - AggIntent::HistogramFraction { lower, upper } => AggIntent::HistogramFraction { - lower: *lower, - upper: *upper, - }, - AggIntent::HistogramQuantile { q, le } => AggIntent::HistogramQuantile { - q: *q, - le: resolve_column_ref(le, schema)?, - }, - AggIntent::Math(f) => AggIntent::Math(f.clone()), - AggIntent::Absent => AggIntent::Absent, - AggIntent::AbsentOverTime => AggIntent::AbsentOverTime, - AggIntent::PresentOverTime => AggIntent::PresentOverTime, - AggIntent::TimeFn(f) => AggIntent::TimeFn(*f), - AggIntent::Group => AggIntent::Group, - AggIntent::CountValues { label } => AggIntent::CountValues { - label: label.clone(), - }, - AggIntent::LastOverTime => AggIntent::LastOverTime, - AggIntent::FirstOverTime => AggIntent::FirstOverTime, - AggIntent::MadOverTime => AggIntent::MadOverTime, - AggIntent::TsOfMinOverTime => AggIntent::TsOfMinOverTime, - AggIntent::TsOfMaxOverTime => AggIntent::TsOfMaxOverTime, - AggIntent::TsOfFirstOverTime => AggIntent::TsOfFirstOverTime, - AggIntent::TsOfLastOverTime => AggIntent::TsOfLastOverTime, - AggIntent::Extension { ext_kind, payload } => AggIntent::Extension { - ext_kind: ext_kind.clone(), - payload: payload.clone(), - }, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pre_asap::expr_ir::CompareOpKind; - use crate::pre_asap::query_expr::{ - BinaryOpKind, QueryExpr, Source, VectorMatch, VectorMatchKind, - }; - - // A measure filter (#466) binds positionally against the aggregate's - // input, and a vector with no set entry collapses to the empty spelling. - #[test] - fn resolve_measure_filters_against_the_child_schema() { - use crate::pre_asap::expr_ir::ScalarValue; - use crate::pre_asap::query_expr::Predicate; - use crate::pre_asap::{DataType, Field, GroupKeys}; - use crate::types::AccuracyTarget; - let scan = || UnresolvedQueryExpr::Scan { - source: Source::Table { - table_ref: "metrics".into(), - }, - predicates: vec![], - schema: Some(Schema::new(vec![ - Field::plain("service", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - Field::plain("bytes", DataType::Int64, false), - ])), - }; - let aggregate = |filters| UnresolvedQueryExpr::Aggregate { - reduction: Reduction::Reduce(GroupKeys::by(vec![ColumnRef::Named("service".into())])), - measures: vec![ - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - AggIntent::Sum { - col: Some(ColumnRef::Named("bytes".into())), - }, - ], - output_names: vec![], - filters, - having: None, - child: Rc::new(scan()), - }; - let latency_gt_one = Predicate(Rc::new(UnresolvedQueryExpr::Compare { - left: Rc::new(UnresolvedQueryExpr::Column(ColumnRef::Named( - "latency".into(), - ))), - op: CompareOpKind::Gt, - right: Rc::new(UnresolvedQueryExpr::Literal(ScalarValue::Float64(1.0))), - })); - - let resolved = resolve_root(&aggregate(vec![Some(latency_gt_one), None])).unwrap(); - let QueryExpr::Aggregate { filters, .. } = &resolved else { - unreachable!() - }; - let [Some(Predicate(first)), None] = filters.as_slice() else { - panic!("expected one filtered and one unfiltered measure, got {filters:?}"); - }; - assert!( - matches!(first.as_ref(), QueryExpr::Compare { left, .. } - if matches!(left.as_ref(), QueryExpr::Column(1))), - "latency is input column 1, got {first:?}" - ); - - let resolved = resolve_root(&aggregate(vec![None, None])).unwrap(); - let QueryExpr::Aggregate { filters, .. } = &resolved else { - unreachable!() - }; - assert!(filters.is_empty()); - } - - // Both sides resolve with qualifiers; an unknown right input is an error. - #[test] - fn resolve_pearson_corr_inputs() { - use crate::pre_asap::{DataType, Field}; - let schema = Schema::new(vec![ - Field::plain("x", DataType::Float64, true).with_table("a"), - Field::plain("x", DataType::Float64, true).with_table("b"), - ]); - let intent = AggIntent::PearsonCorr { - left: ColumnRef::Qualified { - table: "a".into(), - name: "x".into(), - }, - right: ColumnRef::Qualified { - table: "b".into(), - name: "x".into(), - }, - }; - assert_eq!( - resolve_agg_intent(&intent, &schema).unwrap(), - AggIntent::PearsonCorr { left: 0, right: 1 } - ); - let missing = AggIntent::PearsonCorr { - left: ColumnRef::Qualified { - table: "a".into(), - name: "x".into(), - }, - right: ColumnRef::Named("missing".into()), - }; - assert!(resolve_agg_intent(&missing, &schema).is_err()); - } - - // Every leg resolves independently, qualifiers included; one unknown leg - // fails rather than silently shortening the tuple. - #[test] - fn resolve_distinct_tuple_columns() { - use crate::pre_asap::{DataType, Field}; - use crate::types::AccuracyTarget; - let schema = Schema::new(vec![ - Field::plain("k", DataType::Int64, true).with_table("a"), - Field::plain("k", DataType::Int64, true).with_table("b"), - ]); - let qualified = |table: &str| ColumnRef::Qualified { - table: table.into(), - name: "k".into(), - }; - let intent = AggIntent::Cardinality { - cols: vec![qualified("b"), qualified("a")], - accuracy: AccuracyTarget::Exact, - }; - assert_eq!( - resolve_agg_intent(&intent, &schema).unwrap(), - AggIntent::Cardinality { - cols: vec![1, 0], - accuracy: AccuracyTarget::Exact, - } - ); - let missing = AggIntent::Cardinality { - cols: vec![qualified("a"), ColumnRef::Named("missing".into())], - accuracy: AccuracyTarget::Exact, - }; - assert!(resolve_agg_intent(&missing, &schema).is_err()); - } - - /// `resolve_root` over a `BinaryOp { , PromqlScalarBridge, vector_match }` - /// (issue #220): the bridged scalar operand resolves through the same - /// generic walk as every other node (its `Literal` child has no - /// `ColumnRef` to resolve, so it comes through unchanged), the vector - /// side's `ColumnRef`s resolve positionally, and the `VectorMatch` - /// modifier on the relational binary-op path survives resolution - /// untouched — Instance 2 of #220 (`BinaryOp` vs `Compare`/`Arithmetic`) - /// is out of scope for this change, so this pins that its behavior is - /// unaffected by the Instance-1 collapse. - #[test] - fn resolve_root_threads_a_scalar_bridge_operand_and_preserves_vector_match() { - let vm = VectorMatch { - kind: VectorMatchKind::Ignoring, - labels: vec!["job".into()], - grouping: None, - }; - let unresolved: UnresolvedQueryExpr = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOpKind::Gt), - lhs: Rc::new(UnresolvedQueryExpr::Scan { - source: Source::TimeSeries { - metric: "up".into(), - }, - predicates: vec![], - schema: None, - }), - rhs: Rc::new(UnresolvedQueryExpr::promql_scalar(1.0)), - vector_match: Some(vm.clone()), - }; - - let resolved = resolve_root(&unresolved).expect("resolves"); - let QueryExpr::BinaryOp { - lhs, - rhs, - vector_match, - .. - } = &resolved - else { - panic!("expected a resolved BinaryOp, got {resolved:?}"); - }; - assert!(matches!(lhs.as_ref(), QueryExpr::Scan { .. })); - assert_eq!(rhs.as_promql_scalar(), Some(1.0)); - assert_eq!(vector_match.as_ref(), Some(&vm)); - - // Schema derivation still follows the vector side post-resolution. - assert_eq!( - resolved.output_schema().unwrap(), - lhs.output_schema().unwrap() - ); - } - - /// Issue #228 review, end-to-end: `resolve_root` over a `Concat` whose - /// discriminator column is referenced *nowhere else* in the DAG, with a - /// schema-less (usage-derived) leaf `Scan` in the first branch — exactly - /// the scenario the review flagged. Before the `schema_resolver.rs` fix, the - /// SchemaResolver's fallback schema wouldn't contain `phi` at all, and this - /// `resolve_column_ref` call would fail `NotFound` for a column the - /// caller correctly named. It must resolve cleanly, and the resolved - /// `ConcatDiscriminatorKey` must carry the *positional* `ColumnId`s of - /// the branch's own (usage-derived) schema. - #[test] - fn resolve_root_seeds_and_resolves_an_otherwise_unreferenced_discriminator_column() { - let branch = || UnresolvedQueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: None, - }; - let unresolved = UnresolvedQueryExpr::concat_with_discriminator( - vec![branch(), branch()], - ColumnRef::Named("phi".into()), - vec![ColumnRef::Named("host".into())], - ); - - let resolved = resolve_root(&unresolved).expect("resolves"); - let QueryExpr::Concat { - children, - discriminator_unique_key, - } = &resolved - else { - panic!("expected a resolved Concat, got {resolved:?}"); - }; - let schema = children[0].output_schema().unwrap(); - let key = discriminator_unique_key - .as_ref() - .expect("discriminator key survives resolution"); - assert_eq!(*key.discriminator(), schema.column_id("phi").unwrap()); - assert_eq!( - key.inner_key().to_vec(), - vec![schema.column_id("host").unwrap()] - ); - } -} diff --git a/crates/types/src/pre_asap/scalar_type_rules.rs b/crates/types/src/pre_asap/scalar_type_rules.rs index 44eaa250c..3efad3f7e 100644 --- a/crates/types/src/pre_asap/scalar_type_rules.rs +++ b/crates/types/src/pre_asap/scalar_type_rules.rs @@ -1,5 +1,5 @@ -//! Shared type rules for structural map scalar expressions. -//! Execution must separately implement the documented ordering/default semantics. +//! Shared type and nullability rules used to validate scalar expressions. +//! These rules do not evaluate expressions or define physical representations. use super::schema::DataType; /// Names are resolved once against this closed builtin set; unknown functions @@ -123,6 +123,21 @@ fn common_type(left: &DataType, right: &DataType) -> Result { )) } +/// Closed, namespaced contracts for PromQL pointwise float functions. +/// Date functions consume Unix seconds; `timestamp` remains a sample-selection +/// operation because its operand is a sample timestamp rather than its value. +pub fn promql_function_arity(name: &str) -> Option { + Some(match name.strip_prefix("promql_")? { + "abs" | "ceil" | "floor" | "exp" | "ln" | "log2" | "log10" | "sqrt" | "sgn" | "sin" + | "cos" | "tan" | "asin" | "acos" | "atan" | "sinh" | "cosh" | "tanh" | "asinh" + | "acosh" | "atanh" | "deg" | "rad" | "minute" | "hour" | "day_of_week" + | "day_of_month" | "day_of_year" | "month" | "year" | "days_in_month" => 1, + "round" | "clamp_min" | "clamp_max" => 2, + "clamp" => 3, + _ => return None, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -196,324 +211,3 @@ mod tests { .is_err()); } } - -#[cfg(test)] -mod projection_tests { - use super::*; - use crate::pre_asap::{Field, ProjectItem, QueryExpr, ScalarValue, Schema, Source}; - use std::rc::Rc; - fn project(expr: QueryExpr) -> QueryExpr { - QueryExpr::Project { - cols: vec![ProjectItem { - alias: Some("result".into()), - expr, - }], - qualifier: None, - child: Rc::new(QueryExpr::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema::new(vec![ - Field::plain("k", DataType::Utf8, false), - Field::plain("v", DataType::Int64, true), - ]), - }), - } - } - #[test] - fn canonical_projection_uses_map_signature_and_rejects_invalid_arity() { - let map = QueryExpr::FunctionCall { - name: "map".into(), - args: vec![QueryExpr::Column(0), QueryExpr::Column(1)], - }; - let schema = project(map.clone()).output_schema().unwrap(); - assert_eq!( - schema.fields[0].dtype, - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Int64), - value_nullable: true - } - ); - assert!(!schema.fields[0].nullable); - let lookup = QueryExpr::FunctionCall { - name: "asap_map_access".into(), - args: vec![map, QueryExpr::Literal(ScalarValue::Utf8("missing".into()))], - }; - assert_eq!( - project(lookup).output_schema().unwrap().fields[0], - Field::plain("result", DataType::Int64, true) - ); - assert!(project(QueryExpr::FunctionCall { - name: "map".into(), - args: vec![QueryExpr::Column(0)] - }) - .output_schema() - .is_err()); - } -} - -/// Resolve the bounded canonical `asap_struct_field(struct, selector)` operation. -/// Selectors are positive 1-based literal ordinals or exact literal field names. -/// The existing Struct fields remain the sole authority for type/nullability. -/// Dynamic/negative/defaulted selectors and nullable containers are intentionally -/// unsupported here; this is not a claim of complete native tupleElement support. -pub fn struct_field_type( - args: &[super::QueryExpr], - schema: &super::Schema, -) -> Result<(DataType, bool), String> { - use super::{QueryExpr, ScalarValue}; - let [input, selector] = args else { - return Err("struct field access requires a struct and constant selector".into()); - }; - let (dtype, nullable) = input - .scalar_type(schema) - .map_err(|error| error.to_string())?; - if nullable { - return Err("nullable struct container access is unsupported".into()); - } - let DataType::Struct { fields } = dtype else { - return Err("struct field access requires a Struct input".into()); - }; - let field = match selector { - QueryExpr::Literal(ScalarValue::Int64(index)) if *index > 0 => usize::try_from(*index - 1) - .ok() - .and_then(|index| fields.get(index)) - .ok_or("struct field ordinal is out of bounds")?, - QueryExpr::Literal(ScalarValue::Utf8(name)) => { - let mut matches = fields.iter().filter(|field| field.name == *name); - let field = matches.next().ok_or("struct field name does not exist")?; - if matches.next().is_some() { - return Err("struct field name is ambiguous".into()); - } - field - } - _ => { - return Err( - "struct field selector must be a positive ordinal or field-name literal".into(), - ) - } - }; - Ok((field.dtype.clone(), field.nullable)) -} - -#[cfg(test)] -mod struct_field_tests { - use super::*; - use crate::pre_asap::{Field, FieldDataType, QueryExpr, ScalarValue, Schema}; - fn schema() -> Schema { - Schema::new(vec![Field::plain( - "record", - DataType::Struct { - fields: vec![ - Field::new("ts", DataType::Int64, false), - Field::new( - "values", - DataType::List { - element: Box::new(Field::new("item", DataType::Float64, true)), - }, - true, - ), - ], - }, - false, - )]) - } - fn access(selector: QueryExpr) -> QueryExpr { - QueryExpr::FunctionCall { - name: "asap_struct_field".into(), - args: vec![QueryExpr::Column(0), selector], - } - } - #[test] - fn field_access_reuses_nested_field_type_and_nullability() { - let schema = schema(); - assert_eq!( - access(QueryExpr::Literal(ScalarValue::Int64(1))) - .scalar_type(&schema) - .unwrap(), - (DataType::Int64, false) - ); - let named = access(QueryExpr::Literal(ScalarValue::Utf8("values".into()))); - let ordinal = access(QueryExpr::Literal(ScalarValue::Int64(2))); - assert_eq!( - named.scalar_type(&schema).unwrap(), - ordinal.scalar_type(&schema).unwrap() - ); - assert_eq!( - named.scalar_type(&schema).unwrap(), - ( - DataType::List { - element: Box::new(Field::new("item", DataType::Float64, true)) - }, - true - ) - ); - let roundtrip: QueryExpr = - serde_json::from_str(&serde_json::to_string(&named).unwrap()).unwrap(); - assert_eq!(roundtrip, named); - } - #[test] - fn unsupported_field_access_is_an_error_not_placeholder_typing() { - for selector in [ - QueryExpr::Column(0), - QueryExpr::Literal(ScalarValue::Int64(0)), - QueryExpr::Literal(ScalarValue::Int64(-1)), - QueryExpr::Literal(ScalarValue::Int64(3)), - QueryExpr::Literal(ScalarValue::Utf8("missing".into())), - ] { - assert!(access(selector).scalar_type(&schema()).is_err()); - } - let mut ambiguous = schema(); - if let FieldDataType::Plain(DataType::Struct { fields }) = &mut ambiguous.fields[0].dtype { - fields.push(Field::new("ts", DataType::Utf8, false)); - } - assert!(access(QueryExpr::Literal(ScalarValue::Utf8("ts".into()))) - .scalar_type(&ambiguous) - .is_err()); - let mut nullable = schema(); - nullable.fields[0].nullable = true; - assert!(access(QueryExpr::Literal(ScalarValue::Int64(1))) - .scalar_type(&nullable) - .is_err()); - } -} - -/// Canonical element lookup over a declared Map or List. Map lookup retains its -/// existing key/default contract. List lookup is one-based, supports negative -/// indices, and returns the declared element default when a dynamic index is -/// out of range. Literal zero is conservatively rejected because native array -/// behavior depends on whether the input array is constant. Nullable containers -/// are unsupported; nullable indices produce nullable results. -pub fn element_access_type( - args: &[super::QueryExpr], - schema: &super::Schema, -) -> Result<(DataType, bool), String> { - use super::{QueryExpr, ScalarValue}; - let [input, index] = args else { - return Err("element access requires a collection and index".into()); - }; - let source = input.scalar_type(schema).map_err(|e| e.to_string())?; - let key = index.scalar_type(schema).map_err(|e| e.to_string())?; - match &source.0 { - DataType::Map { .. } => MapScalarFunction::Access.output_type(&[source, key]), - DataType::List { element } => { - if source.1 { - return Err("nullable List container access is unsupported".into()); - } - if !matches!(key.0, DataType::Int64 | DataType::Null) { - return Err("List index must have integer type".into()); - } - if matches!(index, QueryExpr::Literal(ScalarValue::Int64(0))) { - return Err( - "literal zero List index is unsupported without constant-array proof".into(), - ); - } - Ok(( - element.dtype.clone(), - element.nullable || key.1 || key.0 == DataType::Null, - )) - } - _ => Err("element access requires a Map or List".into()), - } -} - -#[cfg(test)] -mod element_access_tests { - use super::*; - use crate::pre_asap::{Field, QueryExpr, ScalarValue, Schema}; - fn access(index: QueryExpr) -> QueryExpr { - QueryExpr::FunctionCall { - name: "asap_element_access".into(), - args: vec![QueryExpr::Column(0), index], - } - } - #[test] - fn list_index_preserves_nested_element_metadata() { - let element = DataType::Struct { - fields: vec![ - Field::new("ts", DataType::Int64, false), - Field::new("value", DataType::Float64, true), - ], - }; - let schema = Schema::new(vec![ - Field::plain( - "samples", - DataType::List { - element: Box::new(Field::new("item", element.clone(), false)), - }, - false, - ), - Field::plain("i", DataType::Int64, true), - ]); - for index in [1, -1, 100] { - assert_eq!( - access(QueryExpr::Literal(ScalarValue::Int64(index))) - .scalar_type(&schema) - .unwrap(), - (element.clone(), false) - ); - } - assert_eq!( - access(QueryExpr::Column(1)).scalar_type(&schema).unwrap(), - (element.clone(), true) - ); - assert!(access(QueryExpr::Literal(ScalarValue::Int64(0))) - .scalar_type(&schema) - .is_err()); - assert!(access(QueryExpr::Literal(ScalarValue::Float64(1.0))) - .scalar_type(&schema) - .is_err()); - let nested = QueryExpr::FunctionCall { - name: "asap_struct_field".into(), - args: vec![ - access(QueryExpr::Literal(ScalarValue::Int64(1))), - QueryExpr::Literal(ScalarValue::Int64(2)), - ], - }; - assert_eq!( - nested.scalar_type(&schema).unwrap(), - (DataType::Float64, true) - ); - let roundtrip: QueryExpr = - serde_json::from_value(serde_json::to_value(&nested).unwrap()).unwrap(); - assert_eq!(roundtrip, nested); - } - #[test] - fn generic_map_lookup_reuses_legacy_signature() { - let schema = Schema::new(vec![Field::plain( - "m", - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Int64), - value_nullable: false, - }, - false, - )]); - let key = QueryExpr::Literal(ScalarValue::Utf8("k".into())); - let legacy = QueryExpr::FunctionCall { - name: "asap_map_access".into(), - args: vec![QueryExpr::Column(0), key.clone()], - }; - assert_eq!( - access(key).scalar_type(&schema).unwrap(), - legacy.scalar_type(&schema).unwrap() - ); - } -} - -/// Closed, namespaced contracts for PromQL pointwise float functions. -/// Date functions consume Unix seconds; `timestamp` remains a sample-selection -/// operation because its operand is a sample timestamp rather than its value. -pub fn promql_function_arity(name: &str) -> Option { - Some(match name.strip_prefix("promql_")? { - "abs" | "ceil" | "floor" | "exp" | "ln" | "log2" | "log10" | "sqrt" | "sgn" | "sin" - | "cos" | "tan" | "asin" | "acos" | "atan" | "sinh" | "cosh" | "tanh" | "asinh" - | "acosh" | "atanh" | "deg" | "rad" | "minute" | "hour" | "day_of_week" - | "day_of_month" | "day_of_year" | "month" | "year" | "days_in_month" => 1, - "round" | "clamp_min" | "clamp_max" => 2, - "clamp" => 3, - _ => return None, - }) -} diff --git a/crates/types/src/pre_asap/schema.rs b/crates/types/src/pre_asap/schema.rs index 77c8ebe92..7eec788b7 100644 --- a/crates/types/src/pre_asap/schema.rs +++ b/crates/types/src/pre_asap/schema.rs @@ -325,82 +325,6 @@ impl TryFrom for Schema { /// label map. `$` cannot occur in a user PromQL label name. pub const PROMQL_SERIES_IDENTITY: &str = "$promql_series_identity"; -/// Resolve a PromQL root to rows carrying [`PROMQL_SERIES_IDENTITY`] before -/// candidate search. `closed` describes physical columns here: the final -/// column contains every dynamic source label. It does not assert that the -/// query's projected labels are the full label set. -/// -/// This realization supports explicit `by` grouping and per-series computation. -/// Operators that rewrite or implicitly match dynamic label sets require their -/// own realization; they must not accidentally treat the opaque identity as a -/// user label or silently discard it. -pub fn with_promql_series_identity(root: &super::QueryExpr) -> Result { - use super::{QueryExpr, Source}; - use std::rc::Rc; - let mut root = root.clone(); - fn visit(node: &mut QueryExpr) -> Result<(), String> { - match node { - QueryExpr::Scan { - source: Source::TimeSeries { .. }, - schema, - .. - } => { - if schema - .fields - .iter() - .any(|column| column.name == PROMQL_SERIES_IDENTITY) - { - return Err("source already contains a physical series identity".into()); - } - if schema.closed { - return Err("dynamic series identity requires an open PromQL source".into()); - } - schema - .fields - .push(Field::plain(PROMQL_SERIES_IDENTITY, DataType::Utf8, false)); - schema.closed = true; - Ok(()) - } - QueryExpr::TimeRange { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::PromqlScalarFromVector(child) - | QueryExpr::PromqlRelabel { child, .. } => visit(Rc::make_mut(child)), - // Constants read no series. - QueryExpr::PromqlScalarBridge(_) - | QueryExpr::EvalTimestamp - | QueryExpr::Literal(super::ScalarValue::Float64(_)) => Ok(()), - QueryExpr::PromqlVectorFromScalar(child) => visit(Rc::make_mut(child)), - QueryExpr::BinaryOp { lhs, rhs, .. } => { - visit(Rc::make_mut(lhs))?; - visit(Rc::make_mut(rhs)) - } - QueryExpr::Concat { children, .. } => { - for child in children { - visit(child)?; - } - Ok(()) - } - QueryExpr::Aggregate { child, .. } => visit(Rc::make_mut(child)), - QueryExpr::Sort { - child, - partition_by, - .. - } => { - if partition_by.is_without() { - return Err("dynamic without ranking requires label-set projection".into()); - } - visit(Rc::make_mut(child)) - } - _ => Err("operator has no dynamic series-identity realization".into()), - } - } - visit(&mut root)?; - root.output_schema().map_err(|error| error.to_string())?; - Ok(root) -} - impl Schema { pub fn has_promql_series_identity(&self) -> bool { self.closed @@ -605,12 +529,4 @@ mod tests { assert_eq!(back, c); assert_eq!(back.table.as_deref(), Some("hosts")); } - // Direct scalar literals remain valid vector inputs when series typing runs. - #[test] - fn series_identity_accepts_direct_vector_literal() { - let root = super::super::QueryExpr::PromqlVectorFromScalar(std::rc::Rc::new( - super::super::QueryExpr::Literal(super::super::ScalarValue::Float64(1.0)), - )); - assert!(with_promql_series_identity(&root).is_ok()); - } } diff --git a/crates/types/src/pre_asap/schema_resolver.rs b/crates/types/src/pre_asap/schema_resolver.rs deleted file mode 100644 index a9afff2cb..000000000 --- a/crates/types/src/pre_asap/schema_resolver.rs +++ /dev/null @@ -1,492 +0,0 @@ -//! The **SchemaResolver** — name resolution as an explicit pass. -//! -//! [`SchemaResolver::resolve_schema`] produces the complete, self-contained [`Schema`] every -//! `ColumnId` in the canonical DAG indexes into. [`resolve`](super::resolve) -//! then becomes purely structural: it threads the SchemaResolver's schema and -//! positional resolution downstream is **total**. -//! -//! The default [`UsageDerivedCatalog`] knows nothing — every schema is derived -//! purely from the query's own usage. That is the honest state for the -//! observability domain (metric label sets are open-ended). A registry-backed -//! `SchemaCatalog` is future work; the `SchemaResolver` pass does not change when it -//! lands, only the catalog impl swaps. - -use super::expr_ir::ColumnRef; -use super::query_expr::UnresolvedQueryExpr; -use super::schema::{DataType, Field, Schema}; - -/// The DB / source-schema metadata source — resolves a source (metric / -/// table) name to its known columns. -/// Source of truth for a source's columns — the "catalog". `SqlCatalog` backs -/// it for SQL; PromQL uses [`UsageDerivedCatalog`] (returns `None`) until a -/// registry-backed impl (returning a metric's known label set) drops in here. -/// Distinct from `Scan.schema`, which is the *resolved* binding schema this -/// feeds — the catalog is the input, the schema is the result. Even a -/// registry-backed PromQL catalog yields an **open** schema -/// ([`Schema::closed`] `= false`): a metric's -/// labels are per-series and time-varying, so the registry is a superset hint, -/// not a per-row contract. -pub trait SchemaCatalog { - /// Columns known for `source`. `None` when unknown — the [`SchemaResolver`] then - /// falls back to a usage-derived column set. - fn columns_for(&self, source: &str) -> Option>; -} - -/// The default catalog: knows nothing. Every schema the [`SchemaResolver`] produces -/// is derived purely from the query's own usage. -pub struct UsageDerivedCatalog; - -impl SchemaCatalog for UsageDerivedCatalog { - fn columns_for(&self, _source: &str) -> Option> { - None - } -} - -/// The explicit name-resolution pass. -pub struct SchemaResolver { - catalog: C, -} - -impl Default for SchemaResolver { - fn default() -> Self { - Self::new() - } -} - -impl SchemaResolver { - pub fn new() -> Self { - Self { - catalog: UsageDerivedCatalog, - } - } -} - -impl SchemaResolver { - pub fn with_catalog(catalog: C) -> Self { - Self { catalog } - } - - /// Resolve the complete [`Schema`] in scope for a query rooted at `dag`. - /// - /// Contains the time axis, the synthetic `value` column, and one column - /// per distinct name referenced anywhere in the DAG — so positional - /// `ColumnId` resolution downstream is total. - pub fn resolve_schema(&self, dag: &UnresolvedQueryExpr) -> Schema { - self.resolve_schema_with_inherited(dag, &[]) - } - - /// Like [`resolve_schema`](Self::resolve_schema), but also seeds `inherited` label names that are - /// referenced by an **enclosing** scope rather than by `dag` itself. This is - /// how an independently-bound `BinaryOp` side (each side re-binds against its - /// own sub-DAG) still sees an outer aggregate's group keys — e.g. the - /// `__name__` / `job` in `sum by (__name__)(a or b)`, which appear in neither - /// side's own matchers (issue #52). - pub fn resolve_schema_with_inherited( - &self, - dag: &UnresolvedQueryExpr, - inherited: &[String], - ) -> Schema { - let mut columns: Vec = leftmost_scan_name(dag) - .and_then(|name| self.catalog.columns_for(name)) - .unwrap_or_else(default_leaf_columns); - - // Ensure the (ts, value) floor is present. - for floor in default_leaf_columns() { - if !columns.iter().any(|c| c.name == floor.name) { - columns.push(floor); - } - } - - // Append one column per referenced-but-unknown name (group keys etc.), - // plus any inherited-from-enclosing-scope names. - let referenced = collect_referenced_columns(dag); - for name in referenced.iter().chain(inherited) { - if !columns.iter().any(|c| c.name == *name) { - columns.push(Field::plain(name.clone(), DataType::Utf8, true)); - } - } - - let time_index = columns.iter().position(|c| c.name == "ts"); - Schema { - fields: columns, - time_index, - unique_keys: Vec::new(), - // Usage-derived (schemaless PromQL): the metric's full label set is - // open and runtime-only, so this lists only what the query references. - closed: false, - } - } -} - -/// The conventional PromQL leaf shape: `(ts: Timestamp, value: Float64)`. -fn default_leaf_columns() -> Vec { - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ] -} - -/// Push a `ColumnRef`'s bare name (the schema-seedable identifier). `Qualified` -/// collapses to its `name`; `SampleValue`/`Wildcard` carry no name. -fn push_ref_name(c: &ColumnRef, out: &mut Vec) { - match c { - ColumnRef::Named(n) => out.push(n.clone()), - ColumnRef::Qualified { name, .. } => out.push(name.clone()), - ColumnRef::SampleValue | ColumnRef::Wildcard => {} - } -} - -/// The leftmost `Scan`'s source name in a canonical (`UnresolvedQueryExpr`) DAG — -/// the [`collect_referenced_columns`] counterpart to what a dedicated -/// `Source` leaf type would carry as a method; the canonical DAG's `Scan` -/// leaf needs this walk written out instead. -fn leftmost_scan_name(dag: &UnresolvedQueryExpr) -> Option<&str> { - use UnresolvedQueryExpr as QE; - match dag { - QE::Scan { source, .. } => Some(match source { - super::query_expr::Source::TimeSeries { metric } => metric.as_str(), - super::query_expr::Source::Table { table_ref } => table_ref.as_str(), - }), - // A scalar bridge's child is a scalar-sub-language leaf (in practice - // always a `Literal`, issue #220) — never a `Scan`, same as - // `EvalTimestamp`. - QE::PromqlScalarBridge(_) | QE::EvalTimestamp | QE::CurrentTimestamp => None, - QE::PromqlVectorFromScalar(child) | QE::PromqlScalarFromVector(child) => { - leftmost_scan_name(child) - } - QE::PromqlRelabel { child, .. } - | QE::PromqlInfoEnrich { child, .. } - | QE::PromqlSeriesSample { child, .. } - | QE::Filter { child, .. } - | QE::Project { child, .. } - | QE::Aggregate { child, .. } - | QE::Dedup { child, .. } - | QE::Sort { child, .. } - | QE::Limit { child, .. } - | QE::PromqlSubquery { child, .. } - | QE::TimeRange { child, .. } - | QE::TimeShift { child, .. } - | QE::SQLWindowFunc { child, .. } => leftmost_scan_name(child), - QE::Concat { children, .. } => children.first().and_then(leftmost_scan_name), - QE::Join { left, .. } | QE::SetOp { left, .. } | QE::BinaryOp { lhs: left, .. } => { - leftmost_scan_name(left) - } - // The scalar variants (issue #205) never appear as a direct - // `leftmost_scan_name` target — every reachable one sits behind a - // wrapper field (`Predicate`, `ProjectItem`, …) this walk never - // descends into; it only follows the relational skeleton. - QE::Column(_) - | QE::Literal(_) - | QE::Compare { .. } - | QE::BoolAnd(_) - | QE::BoolOr(_) - | QE::Not(_) - | QE::IsNull(_) - | QE::IsNotNull(_) - | QE::Cast { .. } - | QE::InList { .. } - | QE::FunctionCall { .. } - | QE::Arithmetic { .. } - | QE::Case { .. } => None, - } -} - -/// Collect every distinct column name referenced anywhere in `dag` that -/// resolves positionally — every place a front end constructing -/// [`QueryExpr`](super::query_expr::QueryExpr) directly (issue -/// #179) puts a name-based reference: `Scan.predicates`, `Aggregate`'s -/// `reduction`/`having`/per-measure `col`, `Dedup.cols`, `PromqlSeriesSample.by`, -/// `Filter.pred`, `Project.cols`, `Sort.keys`/`partition_by`, -/// `SQLWindowFunc.args`/`partition_by`/`order_by`, `Join.pred`, `PromqlRelabel.value`. -/// The SchemaResolver seeds these into the usage-derived leaf so positional -/// resolution downstream is total. -pub(crate) fn collect_referenced_columns(dag: &UnresolvedQueryExpr) -> Vec { - use UnresolvedQueryExpr as QE; - fn named(expr: &UnresolvedQueryExpr, out: &mut Vec) { - for c in expr.columns_referenced() { - push_ref_name(c, out); - } - } - fn group_keys(g: &super::query_expr::GroupKeys, out: &mut Vec) { - g.keys().iter().for_each(|k| push_ref_name(k, out)); - } - fn measure_cols(measures: &[super::agg_intent::AggIntent], out: &mut Vec) { - for m in measures { - for c in m.input_cols() { - push_ref_name(&c, out); - } - } - } - fn walk(node: &UnresolvedQueryExpr, out: &mut Vec) { - match node { - QE::Scan { predicates, .. } => { - for super::query_expr::Predicate(p) in predicates { - named(p, out); - } - } - QE::Aggregate { - reduction, - measures, - filters, - having, - child, - .. - } => { - if let super::query_expr::Reduction::Reduce(by) = reduction { - group_keys(by, out); - } - measure_cols(measures, out); - for super::query_expr::Predicate(f) in filters.iter().flatten() { - named(f, out); - } - if let Some(super::query_expr::Predicate(h)) = having { - named(h, out); - } - walk(child, out); - } - QE::Dedup { cols, child } => { - cols.iter().for_each(|c| push_ref_name(c, out)); - walk(child, out); - } - QE::PromqlSeriesSample { by, child, .. } => { - group_keys(by, out); - walk(child, out); - } - QE::Filter { pred, child } => { - named(&pred.0, out); - walk(child, out); - } - QE::Project { cols, child, .. } => { - for item in cols { - named(&item.expr, out); - } - walk(child, out); - } - QE::Sort { - keys, - partition_by, - child, - } => { - for k in keys { - named(&k.expr, out); - } - group_keys(partition_by, out); - walk(child, out); - } - QE::SQLWindowFunc { - args, - partition_by, - order_by, - child, - .. - } => { - for a in args { - named(a, out); - } - group_keys(partition_by, out); - for k in order_by { - named(&k.expr, out); - } - walk(child, out); - } - QE::PromqlRelabel { value, child, .. } => { - named(value, out); - walk(child, out); - } - QE::Join { - pred, left, right, .. - } => { - named(&pred.0, out); - walk(left, out); - walk(right, out); - } - QE::EvalTimestamp | QE::CurrentTimestamp => {} - // The bridged child is a genuine scalar-sub-language position now - // (issue #220) — peel its column refs off with `named`, same as - // every other scalar-typed field (`Scan.predicates`, - // `Filter.pred`, …). In practice it's always a `Literal`, which - // references no columns, so this is a no-op today. - QE::PromqlScalarBridge(inner) => named(inner, out), - QE::PromqlVectorFromScalar(child) | QE::PromqlScalarFromVector(child) => { - walk(child, out) - } - QE::PromqlInfoEnrich { child, .. } - | QE::Limit { child, .. } - | QE::PromqlSubquery { child, .. } - | QE::TimeRange { child, .. } - | QE::TimeShift { child, .. } => walk(child, out), - QE::Concat { - children, - discriminator_unique_key, - } => { - // Same treatment as `Dedup.cols` above: an own-field - // `ColumnRef` must be seeded here too, or a discriminator - // column that isn't otherwise referenced anywhere else in - // the DAG (plausible — a raw usage-derived label, not one a - // `Project`/relabel freshly created) is absent from the - // SchemaResolver's usage-derived fallback schema, and - // `resolve.rs`'s later `resolve_column_ref` call fails with - // `NotFound` for a column the caller correctly named. - if let Some(key) = discriminator_unique_key { - push_ref_name(key.discriminator(), out); - key.inner_key().iter().for_each(|c| push_ref_name(c, out)); - } - children.iter().for_each(|c| walk(c, out)); - } - QE::SetOp { left, right, .. } => { - walk(left, out); - walk(right, out); - } - QE::BinaryOp { lhs, rhs, .. } => { - walk(lhs, out); - walk(rhs, out); - } - // The scalar variants (issue #205) never appear as a direct - // `walk` target — every reachable one is peeled off first by - // `named` at whichever operator field holds it (`Scan.predicates`, - // `Filter.pred`, `Project.cols`, …). - QE::Column(_) - | QE::Literal(_) - | QE::Compare { .. } - | QE::BoolAnd(_) - | QE::BoolOr(_) - | QE::Not(_) - | QE::IsNull(_) - | QE::IsNotNull(_) - | QE::Cast { .. } - | QE::InList { .. } - | QE::FunctionCall { .. } - | QE::Arithmetic { .. } - | QE::Case { .. } => { - unreachable!("walk reached a scalar QueryExpr variant directly: {node:?}") - } - } - } - let mut out: Vec = Vec::new(); - walk(dag, &mut out); - out.sort(); - out.dedup(); - out -} - -#[cfg(test)] -mod tests { - use std::rc::Rc; - - use super::super::query_expr::{GroupKeys, Source}; - use super::*; - - fn src(name: &str) -> UnresolvedQueryExpr { - UnresolvedQueryExpr::Scan { - source: Source::TimeSeries { - metric: name.into(), - }, - predicates: vec![], - schema: None, - } - } - - // Both correlation inputs must seed a usage-derived schema before positional resolution. - #[test] - fn pearson_corr_inputs_seed_usage_derived_schema() { - use crate::pre_asap::{AggIntent, Reduction}; - let dag = UnresolvedQueryExpr::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::PearsonCorr { - left: ColumnRef::Named("x".into()), - right: ColumnRef::Named("y".into()), - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(src("m")), - }; - assert_eq!(collect_referenced_columns(&dag), vec!["x", "y"]); - let schema = SchemaResolver::new().resolve_schema(&dag); - assert!(schema.column_id("x").is_some()); - assert!(schema.column_id("y").is_some()); - } - - #[test] - fn bare_source_yields_ts_value_floor() { - let schema = SchemaResolver::new().resolve_schema(&src("m")); - assert_eq!(schema.fields.len(), 2); - assert_eq!(schema.fields[0].name, "ts"); - assert_eq!(schema.fields[1].name, "value"); - assert_eq!(schema.time_index, Some(0)); - } - - #[test] - fn sort_partition_keys_land_in_schema() { - // Per-group ranking keys (`topk by (host)` → `Sort.partition_by`) must be - // seeded into the usage-derived leaf so they resolve positionally. - let dag = UnresolvedQueryExpr::Sort { - keys: vec![super::super::query_expr::SortKey { - expr: UnresolvedQueryExpr::Column(ColumnRef::SampleValue), - ascending: false, - nulls_first: false, - }], - partition_by: GroupKeys::by(vec![ColumnRef::Named("host".into())]), - child: Rc::new(src("hits")), - }; - let schema = SchemaResolver::new().resolve_schema(&dag); - assert!(schema.column_id("host").is_some()); - } - - /// Issue #228 review: a `Concat`'s `discriminator_unique_key` columns — - /// even one referenced nowhere else in the DAG — must be seeded into - /// the usage-derived fallback schema, exactly like `Dedup.cols`, or - /// `resolve.rs`'s later `resolve_column_ref` fails `NotFound` for a - /// column the caller correctly named. - #[test] - fn concat_discriminator_key_is_seeded_into_the_resolver_schema() { - let dag = UnresolvedQueryExpr::concat_with_discriminator( - vec![src("m")], - ColumnRef::Named("phi".into()), - vec![ColumnRef::Named("host".into())], - ); - let schema = SchemaResolver::new().resolve_schema(&dag); - assert!( - schema.column_id("phi").is_some(), - "discriminator column must be seeded" - ); - assert!( - schema.column_id("host").is_some(), - "inner_key column must be seeded" - ); - } - - #[test] - fn inherited_names_are_seeded_alongside_referenced() { - // A `BinaryOp` side re-binds against its own sub-DAG, but must still see - // an enclosing aggregate's group key (`__name__` / `job`) that appears in - // neither side's own matchers (issue #52). `resolve_schema_with_inherited` seeds it. - let schema = - SchemaResolver::new().resolve_schema_with_inherited(&src("m"), &["__name__".into()]); - assert!(schema.column_id("__name__").is_some()); - // `resolve_schema` (no inheritance) does not conjure it. - let plain = SchemaResolver::new().resolve_schema(&src("m")); - assert!(plain.column_id("__name__").is_none()); - } - - #[test] - fn custom_catalog_supplies_base_columns() { - struct FixedCatalog; - impl SchemaCatalog for FixedCatalog { - fn columns_for(&self, source: &str) -> Option> { - (source == "known").then(|| { - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("datacenter", DataType::Utf8, false), - ] - }) - } - } - let schema = SchemaResolver::with_catalog(FixedCatalog).resolve_schema(&src("known")); - let dc = schema - .column_id("datacenter") - .and_then(|id| schema.fields.get(id)); - assert!(matches!(dc, Some(c) if !c.nullable)); - } -} diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index eaafb7dd2..f69cc5c9b 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -4,8 +4,8 @@ use asap_types::ir::export::{ }; use asap_types::ir::summary_coverage::{CoverageRegion, SummaryCoverage}; use asap_types::ir::{ - apply_lifecycle_timings, ASAPOp, LifecycleAssignment, NonASAPOp, Operator, OperatorNode, - TimingMemo, + apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, + OperatorNode, TimingMemo, }; use asap_types::post_asap::{ExactKind, ExactParams, ExecutionTiming, SummaryUpdate}; use asap_types::pre_asap::{ColumnRef, DataType, Field, FieldDataType, Reduction, Schema, Source}; @@ -48,12 +48,12 @@ fn plan() -> Rc { .unwrap() } -/// Default lifecycle: the summary is maintained at ingestion time and read at query time. +/// A summary maintained at ingestion time is read at query time. #[test] fn timed_plan_exports_with_timing_and_coverage() { - let timed = apply_lifecycle_timings( + let timed = apply_materialization_timings( &plan(), - &LifecycleAssignment::default_maintained(), + &MaterializationAssignment::all_ingestion_time(), &mut TimingMemo::new(), ) .unwrap(); @@ -80,9 +80,9 @@ fn timed_plan_exports_with_timing_and_coverage() { /// A query-time producer cannot feed an ingestion-time consumer. #[test] fn query_time_input_to_ingestion_is_rejected() { - let timed = apply_lifecycle_timings( + let timed = apply_materialization_timings( &plan(), - &LifecycleAssignment::default_maintained(), + &MaterializationAssignment::all_ingestion_time(), &mut TimingMemo::new(), ) .unwrap(); @@ -111,11 +111,11 @@ fn batch_shares_the_summary_and_keeps_one_root_per_query() { child: state, })) .unwrap(); - let assignment = LifecycleAssignment::default_maintained(); + let assignment = MaterializationAssignment::all_query_time(); let mut memo = TimingMemo::new(); let timed: Vec<_> = [first, second] .iter() - .map(|root| apply_lifecycle_timings(root, &assignment, &mut memo).unwrap()) + .map(|root| apply_materialization_timings(root, &assignment, &mut memo).unwrap()) .collect(); let dag = compile_physical_asap_workload(&timed).unwrap(); dag.validate().unwrap(); diff --git a/crates/types/tests/planner_vocabulary.rs b/crates/types/tests/planner_vocabulary.rs index ae8a72711..2ba817861 100644 --- a/crates/types/tests/planner_vocabulary.rs +++ b/crates/types/tests/planner_vocabulary.rs @@ -1,6 +1,5 @@ use asap_types::ir::export::WindowEdgeCompatibility; use asap_types::post_asap::{validate_pane_coverage, PaneLayout, WindowEdgeCoverage}; -use asap_types::pre_asap::{SchemaResolver, Source, UnresolvedQueryExpr}; use asap_types::resources::{PhysicalHandoffBytes, PhysicalHandoffKind}; // Renamed pane APIs still read and emit the deployed wire contract. @@ -29,18 +28,11 @@ fn window_edge_names_preserve_wire_values() { ); } -// External consumers can use the new resolver and resource names without changing behavior. +// External consumers can use the new resource names without changing behavior. +// (The schema-resolver half moved with the resolver to `asap-frontend-common`; +// `schema_resolver::tests::bare_source_yields_ts_value_floor` covers it.) #[test] -fn renamed_schema_and_handoff_apis_are_public() { - let dag = UnresolvedQueryExpr::Scan { - source: Source::TimeSeries { - metric: "requests".into(), - }, - predicates: vec![], - schema: None, - }; - let schema = SchemaResolver::new().resolve_schema(&dag); - assert!(schema.column_id("value").is_some()); +fn renamed_handoff_apis_are_public() { let bytes = PhysicalHandoffBytes { network_bytes: 12, materialization_bytes: 4, diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index d7c144f4e..bebdadb18 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -65,7 +65,7 @@ fn values_contract_is_checked() { } /// Scalar typing validates every branch and never assigns placeholder types. #[test] -fn scalar_type_ruless_fail_closed() { +fn scalar_type_rules_fail_closed() { for expr in [ ScalarExpr::Column(99), ScalarExpr::FunctionCall { diff --git a/docs/README.md b/docs/README.md index 9f802e094..c9abe1177 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,8 +19,7 @@ Start with [ASAPPlanner input, output, and workflows](design_docs/architecture/i for the integration boundary, nested inputs, and choice of planning workflow. Use [Public library functions and examples](develop_docs/library-api.md) for -frontend lowering, workload search, ranking, optional selection and lifecycle -integration. +frontend lowering, workload search, ranking, selection and DAG assembly. ## Extend the planner diff --git a/docs/design_docs/architecture/README.md b/docs/design_docs/architecture/README.md index 8b936eafe..d0cc61026 100644 --- a/docs/design_docs/architecture/README.md +++ b/docs/design_docs/architecture/README.md @@ -8,7 +8,7 @@ deployment-level decision, and run the selected contract. For the integration workflow, start with [ASAPPlanner input, output, and workflows](input-output-workflow.md). It defines inputs, `CandidateLogicalASAPDAGs`, selection -and summary-maintenance lifecycle workflows, and future replanning support. +and assembly workflows, and future replanning support. ## Planner component flow @@ -17,16 +17,12 @@ flowchart TD W["PlanningWorkload: query demand + optional data facts"] F["Frontend dependencies: SQL catalog or PromQL time"] E["Strategy, accuracy model, and applicable evidence"] - PRE["Frontend lowering → canonical Pre-ASAP QueryExpr roots"] + PRE["Frontend lowering → canonical Pre-ASAP OperatorNode roots"] SEARCH["Whole-workload candidate search: sharing, legality, accuracy"] SPACE["CandidateLogicalASAPDAGs: compact logical candidate DAG space"] RANK["Optional cost_sorted: ranked inspection view"] SELECT["Optional global_selection + assemble_selected_dag"] DAG["Selected logical Post-ASAP DAG"] - LINPUT["Optional lifecycle inputs: horizon, rates, capabilities, costs"] - LIFE["global_selection_with_summary_maintenance_lifecycles"] - LMAT["assemble_selected_dag_with_summary_maintenance_lifecycles"] - LPLAN["SummaryMaintenanceLifecyclePlan: DAG root + lifecycle decisions"] BACKEND["Downstream: bind physical alternatives, decide deployment, compile and execute"] W --> PRE F --> PRE @@ -35,19 +31,15 @@ flowchart TD SEARCH --> SPACE SPACE --> RANK --> BACKEND SPACE --> SELECT --> DAG --> BACKEND - SPACE --> LIFE - LINPUT --> LIFE --> LMAT --> LPLAN --> BACKEND ``` `CandidateLogicalASAPDAGs` is the output of logical candidate search. Each target's candidate set holds -alternatives and rejection reasons, but no selected maintenance lifecycle. -Choose among the three branches: inspect candidates (optionally ranked), select -and assemble logical DAGs, or select and assemble with summary-maintenance -lifecycle decisions. Use the last branch when Planner owns the maintenance -decision; otherwise the backend owns it. Its first -call returns a `GlobalSelection`; the second returns a -`SummaryMaintenanceLifecyclePlan` with an assembled DAG root and lifecycle -decisions. No branch by itself deploys or executes a physical plan. +alternatives and rejection reasons, but no materialization decision. +Choose between two branches: inspect candidates (optionally ranked), or select +and assemble logical DAGs. Stage 2 materialization (#509) will decide per +sub-DAG whether to materialize and whether at ingestion or query time; until +then every summary runs at query time. No branch by itself deploys or executes +a physical plan. Known-invalid evidence rejects a logical candidate. Missing accuracy evidence leaves a constructible candidate visible in `CandidateLogicalASAPDAGs` but uncertified; default selection does not commit it without the required guarantee. Cost evidence can @@ -58,9 +50,10 @@ an unsupported physical alternative into a deployable plan. | Area | Main crate or module | Responsibility | |---|---|---| -| Shared IR | `asap-types` | Pre-ASAP and Post-ASAP expressions, schemas, workloads, guarantees, and exported plan data | +| Shared IR | `asap-types` | The unified operator IR (`ir`: one `OperatorNode` before and after ASAP optimization), schemas, workloads, guarantees, and exported plan data | +| Front-end common | `frontend-common` | Name-based `UnresolvedOp` tree shared by the front ends, and `resolve_root` into the operator IR | | Query frontends | `frontend-sql`, `frontend-promql`, `frontend-metricsql` | Parse source languages and produce canonical Pre-ASAP queries | -| ASAP-aware mapping | `asap-aware-mapping` | Candidate generation, CSE, legality, accuracy propagation, lifecycle expansion, costing, and ranking | +| ASAP-aware mapping | `asap-aware-mapping` | Candidate generation, CSE, legality, accuracy propagation, costing, and ranking | | Developer inspection | `devtools` | Expose planner DAGs, alternatives, decisions, and explanations for inspection | | End-to-end validation | `integration-tests` | Verify behavior across frontends, mapping, and output IR | @@ -76,8 +69,7 @@ The primary output is `CandidateLogicalASAPDAGs`; `cost_sorted` derives an optio view with index-aligned costs. Downstream may inspect compatible choices across targets rather than assuming the first candidate is a feasible physical workload plan. Candidates carry logical summary algorithms, -parameters, and guarantees; selected maintenance lifecycle decisions appear -only after a summary-maintenance-lifecycle-aware helper runs. Rejection reasons +parameters, and guarantees, but no materialization decision. Rejection reasons are retained in the candidate space. ASAPQuery-backend and other downstream applications translate the candidates @@ -89,10 +81,8 @@ must not silently change Planner-owned semantics. `CandidateLogicalASAPDAGs::global_selection` optionally coordinates structural choices across targets; `GlobalSelection::assemble_selected_dag` constructs a selected semantic DAG. -Those plain APIs do not establish physical feasibility or a -maintenance-versus-recompute decision. The lifecycle-aware selection call uses -additional workload and evidence inputs; its DAG assembly call returns a -plan with both a root and lifecycle decisions. See the [library guide](../../develop_docs/library-api.md#optional-whole-plan-selection-and-dag-assembly) +Those APIs do not establish physical feasibility or a +materialization decision. See the [library guide](../../develop_docs/library-api.md#optional-whole-plan-selection-and-dag-assembly) for the distinction. Downstream may consume candidates directly and retains responsibility for physical commitment. @@ -103,7 +93,6 @@ responsibility for physical commitment. - [Post-ASAP IR](../concepts/post-asap-ir.md) - [ASAP-aware mapping](asap-aware-mapping.md) - [Accuracy guarantees](../proposals/asap-aware-mapping/end-to-end-accuracy-guarantees.md) -- [Workload demand and summary lifecycle](../proposals/asap-aware-mapping/workload-demand-and-summary-lifecycle.md) - [Physical-plan integration](physical-plan-integration.md) - [Analytical resource cost](../proposals/asap-aware-mapping/analytical-resource-cost.md) - [Searching over plans](asap-aware-plan-search.md) diff --git a/docs/design_docs/architecture/asap-aware-mapping.md b/docs/design_docs/architecture/asap-aware-mapping.md index 2600fc42d..c04fa530a 100644 --- a/docs/design_docs/architecture/asap-aware-mapping.md +++ b/docs/design_docs/architecture/asap-aware-mapping.md @@ -8,7 +8,7 @@ Given a logical query plan, the mapping layer explores alternative plans that ma Candidate search takes canonical **Pre-ASAP query roots** and produces `CandidateLogicalASAPDAGs`, a compact set of **candidate Post-ASAP DAGs**. Ranking, selection, -and summary-maintenance lifecycle decisions are subsequent operations over it; +and DAG assembly are subsequent operations over it; see [input, output, and workflows](input-output-workflow.md). For example, a percentile query might be answered by: @@ -44,7 +44,7 @@ budgets; deployment belongs to a later stage. - **Replacement Sub-DAG**: A candidate post-ASAP sub-DAG to replace a target sub-DAG. For example, a quantile aggregation may have KLL, DDSketch, and exact aggregation as alternatives. - **ReplacementStrategy**: A rule to recognize a target Sub-DAG and produces one or more valid replacement Sub-DAGs. - **Candidate Plan**: A complete post-ASAP plan formed by choosing compatible replacement alternatives across the plan. -- **Maintained population**: A multiset of qualifying records retained across evaluations and updated as members enter, change, leave or expire; multiple readouts can share this state. +- **Maintained population**: A multiset of qualifying records retained across evaluations and updated as members enter, change, leave or expire; multiple evaluations can share this state. - **Cost Model**: A model used to compare valid candidate plans according to criteria such as storage, update cost, query latency, and accuracy. The distinction between **ReplacementStrategy** and **Candidate Plan** is important. A ReplacementStrategy generates alternatives at a decision point, while a candidate plan is a complete plan that combines choices across all relevant decision points. @@ -75,7 +75,6 @@ CandidateLogicalASAPDAGs: compact candidate Post-ASAP DAGs | +--> inspect / rank +--> select and assemble logical DAGs - +--> select and assemble with summary-maintenance lifecycle decisions ``` --- @@ -104,9 +103,6 @@ The design is split into focused documents: - [ASAPPlanner planner-runtime contract](planner-runtime-contract.md) separates planner-owned search and selection from downstream physical implementation, deployment, and execution. -- [Query workloads, data workloads, and summary lifecycle maintenance](../proposals/asap-aware-mapping/workload-demand-and-summary-lifecycle.md) separates - query-workload properties from data-workload properties and defines ephemeral, prepared, - shared, and continuously maintained summary-state alternatives. - [Explainability](../../develop_docs/replacement-explanations.md) describes how the planner reports available replacements using the same candidate space it optimizes. diff --git a/docs/design_docs/architecture/asap-aware-plan-search.md b/docs/design_docs/architecture/asap-aware-plan-search.md index fbc5982bc..e612740b6 100644 --- a/docs/design_docs/architecture/asap-aware-plan-search.md +++ b/docs/design_docs/architecture/asap-aware-plan-search.md @@ -84,10 +84,10 @@ sets. This avoids copying every full plan when most structure is shared. Cartesian product. `cost_sorted` returns a `RankedTargetSubDAGCandidates` view for each target. `global_selection` coordinates supported sharing and composition choices; `assemble_selected_dag(root)` assembles one selected DAG -per query root. This does not prove global physical optimality or select a -summary-maintenance lifecycle. The -[workflow design](input-output-workflow.md#workflows) explains when to use the -ordinary or summary-maintenance-lifecycle-aware path. +per query root. This does not prove global physical optimality or decide +which summaries are materialized; Stage 2 materialization (#509) will own that. +The [workflow design](input-output-workflow.md#workflows) describes the call +order. The [code architecture](../../develop_docs/asap-aware-mapping-architecture.md) describes current discovery and registry behavior; the diff --git a/docs/design_docs/architecture/evidence-dependent-candidates.md b/docs/design_docs/architecture/evidence-dependent-candidates.md index abdfbdb90..437cfe4b0 100644 --- a/docs/design_docs/architecture/evidence-dependent-candidates.md +++ b/docs/design_docs/architecture/evidence-dependent-candidates.md @@ -15,7 +15,7 @@ target)` returns `true`. **Uncertified** means Planner cannot make that claim: the guarantee is absent, contains unknown terms, or is known not to meet the target. An uncertified summary may still be a well-formed logical candidate; this label says nothing about whether the backend can physically execute it. -The exact `KeepPreAsap` path has an exact guarantee. +The exact path (the pre-ASAP sub-DAG kept by `retain_exact`) has an exact guarantee. | State | Planner representation | Consequence / next step | |---|---|---| @@ -58,12 +58,10 @@ not emit a `RejectedCandidate` for that case. | HLL confidence | Symbolic failure probability | Reject a fully known unmet root target. | | Relative-value composition | Symbolic bound when input sign is unknown | Reject known signed input for this rule. | | Exact sum/average/extremum | Symbolic row-count probability term | Reject unsupported metric combinations. | -| Cost/rate/physical evidence | `None` cost or missing workload rate; candidate remains in `CandidateLogicalASAPDAGs` | Physical/lifecycle evaluation reports unavailable or rejected evidence. | +| Cost/rate/physical evidence | `None` cost or missing workload rate; candidate remains in `CandidateLogicalASAPDAGs` | Physical evaluation reports unavailable or rejected evidence. | | Mixed exact/summary operator | Unknown runtime support; candidate remains in `CandidateLogicalASAPDAGs` | `Some(false)` prevents construction. | -Lifecycle deployment choices are a separate output from `CandidateLogicalASAPDAGs`; their -capability/cost rejections do not erase the logical summary candidate. The -backend must still check ordinary summary family, window, and state-operation +The backend must still check summary family, window, and state-operation capabilities before deployment. - Accuracy/domain: `AccuracyEvidenceProvider` supplies quantile domains and @@ -88,7 +86,7 @@ The default `global_selection()` skips summaries that `has_missing_accuracy_evidence()` identifies as uncertified. Its `GlobalSelection::assemble_selected_dag()` result is a selected logical plan, not an instruction to deploy every candidate in `CandidateLogicalASAPDAGs`. If no alternative -is chosen at a site, DAG assembly retains the exact `KeepPreAsap` path. The +is chosen at a site, DAG assembly retains the exact pre-ASAP sub-DAG. The backend can inspect alternatives, apply its own evidence and policy, then choose a physically supported one; it must not equate candidate presence with approval. Models may explicitly opt into qualitative candidate ranking when no @@ -109,7 +107,7 @@ backend. | PromQL input | Before this PR | After this PR | |---|---|---| | `count by(job)(up)` with an ε/δ target | Hydra's shared CMS/CountSketch alternatives are absent: missing shared-grid bounds make the strategy decline the target. | Both Hydra alternatives remain in `CandidateLogicalASAPDAGs` with symbolic unknown bound/probability terms. `has_missing_accuracy_evidence()` is true; default `global_selection()` does not choose either as a certified answer. | -| `entropy_over_time(m[5m])` with an ε target | The uncalibrated frequency readout has no `SummaryEstimate` candidate. | Its `SummaryEstimate` remains inspectable with `guarantee: None`. Default selection still skips it, so candidate visibility is not an accuracy certificate. | +| `entropy_over_time(m[5m])` with an ε target | The uncalibrated frequency evaluation has no `SummaryEstimate` candidate. | Its `SummaryEstimate` remains inspectable with `guarantee: None`. Default selection still skips it, so candidate visibility is not an accuracy certificate. | | `quantile_over_time(0.9,data[5m]) / quantile_over_time(0.5,data[5m])` with an ε target | The uncertified direct DDSketch ratio is **already** visible because of #449. | Still visible with `guarantee: None`, and still skipped by default selection. This is a regression/control example, not a new candidate introduced by this PR. | For the first two rows, the observable change is the alternative set delivered @@ -121,7 +119,7 @@ evidence (for example a failure probability of `1.5`) instead produces a The corresponding reproducible checks are `cargo test -p asap-frontend-promql grouped_count_keeps_uncertified_hydra_candidates_for_backend_review`, -`cargo test -p asap-frontend-promql uncalibrated_frequency_readouts_do_not_bypass_accuracy_targets`, +`cargo test -p asap-frontend-promql uncalibrated_frequency_evaluations_do_not_bypass_accuracy_targets`, and `cargo test -p asap-integration-tests ddsketch_ratio_without_domain_proof_is_uncertified`. All three start from PromQL text and exercise frontend lowering and planning. None runs a deployed query. diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index 65ee43141..18221035e 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -18,7 +18,7 @@ Post-ASAP alternatives for the workload. | `PlanningWorkload.data_workload` | Data arrival and optional evidence about ingestion, cardinality, and distribution | No implicit default. Set `None` when unavailable for non-PromQL workloads; PromQL requires `Some(DataWorkload)` with a nonzero ingestion interval. | | Frontend-specific dependencies (outside `PlanningWorkload`) | `SqlCatalog` for SQL; `now_ms` and, when needed, `HistogramCatalog` for PromQL | `SqlCatalog` is required for SQL lowering; `now_ms` is required for PromQL lowering | | Planning models | Candidate cost/ranking and accuracy composition/checking | Used by the relevant APIs; built-in `DefaultCostModel` and `DefaultAccuracyModel` are available | -| External evidence and capabilities | Domain facts, measured costs, workload statistics, and runtime support | Supply when available and when the chosen optimization or lifecycle decision depends on them; absence is not proof | +| External evidence and capabilities | Domain facts, measured costs, workload statistics, and runtime support | Supply when available and when the chosen optimization depends on them; absence is not proof | Frontend lowering and candidate search are stages within this workflow, not additional end-to-end inputs. See [Inputs](#inputs) for the nested workload @@ -30,13 +30,14 @@ fields and [frontend dependencies](#frontend-specific-dependencies). |---|---|---| | `CandidateLogicalASAPDAGs` | The legal candidate Post-ASAP DAGs for the workload, represented compactly as canonical roots, one candidate set per target sub-DAG, and cross-target composition information | The ASAPPlanner output | -[Ranking](#ranked-view), [selection and -DAG assembly](#selection-and-dag-assembly), and -[summary-maintenance lifecycle](#summary-maintenance-lifecycle-aware-helper) APIs operate on this `CandidateLogicalASAPDAGs`. +[Ranking](#ranked-view) and [selection and +DAG assembly](#selection-and-dag-assembly) APIs operate on this `CandidateLogicalASAPDAGs`. These are alternative uses of the candidate space, not mandatory sequential -stages. `CandidateLogicalASAPDAGs` itself has no selected summary-maintenance lifecycle, and -its candidates do not choose precompute versus query-time placement: a chosen -lifecycle assignment sets each node's execution timing. +stages. Its candidates do not choose ingestion-time versus query-time +placement: a `MaterializationAssignment` sets each node's execution timing. +Stage 2 materialization (#509) will decide per sub-DAG whether to materialize +and whether at ingestion or query time; until then every summary runs at query +time. The candidate DAGs are logical planning artifacts. ASAPPlanner does **not** produce a deployed executable plan; downstream systems bind physical operators, @@ -58,7 +59,7 @@ PlanningWorkload + frontend dependencies + planning models/evidence Suppose a dashboard evaluates `count_over_time(up[5m])` once a minute, and `up` receives a sample every 15 seconds. This diagram traces the concrete -inputs and the three possible uses of the same candidate space: +inputs and the two possible uses of the same candidate space: ```mermaid flowchart TD @@ -66,16 +67,12 @@ flowchart TD D["data_workload: continuous arrival; declared ingestion interval 15 s"] T["Frontend argument: now_ms"] F["PromQL lowering"] - R["One canonical QueryExpr root"] + R["One canonical OperatorNode root"] S["Candidate search"] P["CandidateLogicalASAPDAGs: logical choices for this root"] I["cost_sorted: inspect choices"] G["global_selection + assemble_selected_dag(root)"] - L["One selected Post-ASAP DAG; exact KeepPreAsap if no optimization is selected"] - X["Extra lifecycle inputs: horizon; update rate; capabilities; comparable summary/raw costs"] - H["Summary-maintenance-lifecycle-aware selection"] - HM["Assemble one selected DAG and decide summary maintenance"] - O["SummaryMaintenanceLifecyclePlan: assembled DAG root + maintenance/recompute decision"] + L["One selected Post-ASAP DAG; the exact pre-ASAP sub-DAG if no optimization is selected"] B["Backend: bind physical operators, deploy, and execute"] Q --> F D --> F @@ -83,16 +80,13 @@ flowchart TD F --> R --> S --> P P --> I P --> G --> L --> B - P --> H - X --> H --> HM --> O --> B ``` “Predictable” says the query is known in advance; it is independent of its one-minute recurrence. The `CandidateLogicalASAPDAGs` may contain an exact count-summary -realization, but it is not a deployed query. Without the extra lifecycle -inputs, the caller can still inspect candidates or obtain a logical DAG; it -cannot conclude that maintaining a summary is cheaper than recomputing raw -results. +realization, but it is not a deployed query. The caller can inspect candidates +or obtain a logical DAG; deciding whether maintaining a summary is cheaper than +recomputing raw results belongs to Stage 2 materialization (#509). For contrast, a one-time SQL query needs a catalog but need not supply data arrival evidence merely to inspect logical alternatives: @@ -102,7 +96,7 @@ flowchart LR Q["query_batch: SELECT COUNT(*) FROM metrics; invocations 1; AdHoc"] C["SqlCatalog: resolves metrics and its columns"] F["SQL lowering"] - R["One QueryExpr root"] + R["One OperatorNode root"] P["Candidate search → CandidateLogicalASAPDAGs"] Q --> F C --> F @@ -110,8 +104,7 @@ flowchart LR ``` In this SQL example, `data_workload` can be `None` if the chosen lowering and -search rules do not consume it. The lifecycle helper is not needed merely to -inspect the `CandidateLogicalASAPDAGs`. +search rules do not consume it. --- @@ -175,9 +168,9 @@ struct BatchEntry { |---|---:|---|---| | `query` | Yes | Raw query text in `QueryWorkload.language`. | `count(up)` determines the expression to lower and plan. | | `requirements` | Yes | Accuracy and response-latency requirements. Defaults mean exact accuracy and unspecified latency. | An explicit ε target permits approximate candidates; the exact default does not. | -| `predictability` | Yes as a field; `Unknown` is allowed | Whether the query is ad hoc, known in advance, or unknown. `known_at` records when a predictable query became known. | A report known at 10:00 and scheduled for 11:00 may use a `Prepared` summary before execution. `AdHoc` or `Unknown` does not establish that eligibility. | +| `predictability` | Yes as a field; `Unknown` is allowed | Whether the query is ad hoc, known in advance, or unknown. `known_at` records when a predictable query became known. | A report known at 10:00 and scheduled for 11:00 could have its summary prepared before execution. `AdHoc` or `Unknown` does not establish that eligibility. | | `invocations` | Yes, nonzero | Number of executions in this finite batch. | Ten executions can amortize one summary build differently from one execution. | -| `execute_at` | Optional | Known execution time. | The `Prepared` case above also needs an execution time; without it Planner cannot establish a preparation window. | +| `execute_at` | Optional | Known execution time. | The prepared case above also needs an execution time; without it no preparation window can be established. | | `time_selection` | Yes | Whether the query follows current data or a historical interval, its lookback, and any fixed upper bound. | A moving five-minute window can require deletion/window support that a fixed historical interval does not. | ##### `repeating_queries: Option>` @@ -197,7 +190,7 @@ struct RepeatingEntry { | `query` | Yes | Raw query text in `QueryWorkload.language`. | `rate(up[5m])` determines the expression to lower and plan. | | `demand` | Yes | A nonzero fixed interval, fixed interval with evaluation phase, nonempty explicit schedule, or evidence-backed estimated rate. | A query every minute produces more expected reads over a horizon than one every hour. | | `requirements` | Yes | Accuracy and response-latency requirements. | An exact dashboard query cannot use an approximate summary solely because it is cheaper. | -| `predictability` | Yes as a field; `Unknown` is allowed | Records whether future executions are known in advance; independent of recurrence. | Current lifecycle code does not use this field for repeating entries; set `Unknown` if no predictability claim is available. | +| `predictability` | Yes as a field; `Unknown` is allowed | Records whether future executions are known in advance; independent of recurrence. | Current planner code does not use this field for repeating entries; set `Unknown` if no predictability claim is available. | | `time_selection` | Yes | Event-time scope, optional lookback, and optional fixed `as_of` time. | A live five-minute lookback differs from a fixed historical range when checking maintenance capabilities. | ##### Shared entry fields @@ -214,9 +207,9 @@ fields expand as follows: | `TimeSelection` | `lookback` | Optional event-time duration selected before the upper bound. | | `TimeSelection` | `as_of` | Optional fixed upper-bound timestamp; `None` means planning/evaluation time. | -Frontend lowering produces one Pre-ASAP `QueryExpr` root for each normalized +Frontend lowering produces one Pre-ASAP `Rc` root for each normalized query entry. The caller must retain each root's association with its workload -entry for later recurrence and lifecycle planning. +entry for later recurrence and materialization planning. #### `data_workload: Option` @@ -285,8 +278,8 @@ latter cannot be fabricated by one. | Accuracy model | Target-aware search takes an `AccuracyModel`; `DefaultAccuracyModel` is available. Default strategies also use it for candidate construction. | Composes candidate guarantees and checks them against requested accuracy. The model does not itself provide missing data-domain facts. | | Cost model | Candidate strategies and `cost_sorted`/`global_selection` use a `CostModel`; `DefaultCostModel` is available. | Ranks or selects candidates. The built-in model is not a measured deployment cost for every physical implementation. | | Accuracy/domain evidence | `AccuracyEvidenceProvider`; default strategies use `NoAccuracyEvidence` when no provider is supplied. | Input ranges, nonempty populations, Top-K intervals, and similar facts can certify or rule out particular approximations. Missing facts remain unknown. | -| Measured cost evidence | Supplied through a deployment-specific cost model or physical-evidence provider when cost-based physical/lifecycle comparison is needed. | CPU, memory, and I/O estimates must be comparable before claiming a summary beats raw recomputation. | -| Runtime/lifecycle capabilities | Passed to lifecycle APIs or checked by deployment-specific providers; `SummaryMaintenanceLifecycleCapabilities::default()` enables all four lifecycle shapes, so it is not proof of actual backend support. | Prevents choosing a maintenance/window operation the intended executor cannot implement. | +| Measured cost evidence | Supplied through a deployment-specific cost model or physical-evidence provider when cost-based physical comparison is needed. | CPU, memory, and I/O estimates must be comparable before claiming a summary beats raw recomputation. | +| Runtime capabilities | Checked by deployment-specific providers. | Prevents choosing a maintenance/window operation the intended executor cannot implement. | For example, the query `quantile_over_time(0.9, data[5m]) / quantile_over_time(0.5, data[5m])` does not tell Planner whether the windows @@ -305,9 +298,6 @@ domain evidence is missing; automatic `global_selection` does not choose it. See the [candidate-search reference](../../develop_docs/library-api.md#generate-and-rank-candidates) for this backend-selection path. -Additional inputs for a Planner-owned maintenance decision are listed with the -[summary-maintenance-lifecycle-aware helper](#summary-maintenance-lifecycle-aware-helper). - --- ## Output @@ -334,7 +324,7 @@ below. A future higher-level API could hide `CandidateLogicalASAPDAGs` behind th the current interface lets an integrator own them. DAG assembly connects choices after selection and does not replace this candidate interface. -Here, a **root** is the top-level `Rc` for a workload query. A +Here, a **root** is the top-level `Rc` for a workload query. A **target** is any discovered sub-DAG that may be replaced, including roots. For `count(up) + 1`, the addition is a root and `count(up)` can be an inner target. `TargetSubDAGCandidates` holds the alternatives for one such target. @@ -361,7 +351,7 @@ All paths start by lowering the workload and searching for candidates: ```text PlanningWorkload + frontend dependencies + planning models/evidence - -> frontend lowering: one QueryExpr root per normalized query entry + -> frontend lowering: one OperatorNode root per normalized query entry -> search_workload_with_targets -> CandidateLogicalASAPDAGs ``` @@ -375,8 +365,7 @@ Then choose the operation matching the caller's responsibility: | Purpose | Operation | Result | |---|---|---| | Inspect candidates or let the backend choose | [Ranked view](#ranked-view), if ranking is useful | Per-target candidate lists and costs | -| Ask Planner to choose logical computations; backend owns summary maintenance | [Selection and DAG assembly](#selection-and-dag-assembly) | One selected Post-ASAP DAG root per query | -| Ask Planner to also decide summary maintenance versus raw recomputation | [Summary-maintenance-lifecycle-aware helper](#summary-maintenance-lifecycle-aware-helper) | One plan containing a DAG root and maintenance decisions per query | +| Ask Planner to choose logical computations | [Selection and DAG assembly](#selection-and-dag-assembly) | One selected Post-ASAP DAG root per query | ### Ranked view @@ -390,7 +379,7 @@ The return type is `Vec>`; each element has thi ```rust struct RankedTargetSubDAGCandidates<'a> { - target: &'a Rc, + target: &'a Rc, consumer_count: usize, candidates: Vec<&'a ReplacementSubDAG>, costs: Vec, // costs[i] describes candidates[i] @@ -430,71 +419,13 @@ the result for one query root. | **Output:** one selected logical [Post-ASAP DAG](../concepts/post-asap-ir.md) per query root | Each output DAG specifies the chosen operators, parameters, and accuracy -guarantees. Its root is represented by `Rc`; the +guarantees. Its root is an `Rc` (the same IR as the input, +with some nodes now ASAP operators) and carries no execution timing yet; the [API reference](../../develop_docs/library-api.md#api-definition-and-example) describes the function signatures and return handling. -This path selects how to compute the query, not how to maintain summary state. - -### Summary-maintenance-lifecycle-aware helper - -This workflow performs both candidate selection and DAG assembly, incorporating -summary-maintenance lifecycle costs. Use it when ASAPPlanner owns the decision -to maintain summaries versus recompute raw data. It is not needed for candidate -inspection or when the downstream backend owns that decision. - -Starting from an existing `CandidateLogicalASAPDAGs`, call these two public helpers in order; -there is no need to run the ordinary selection/assembly workflow first: - -1. `global_selection_with_summary_maintenance_lifecycles` uses the workload - binding, lifecycle capabilities, and comparable costs to choose compatible - candidates across target sub-DAGs. It returns `GlobalSelection`, not a DAG or a - deployment plan. -2. For each wanted query root, `assemble_selected_dag_with_summary_maintenance_lifecycles` - takes that selection and root, constructs a Post-ASAP DAG, compares the - selected summary's maintenance cost with raw recomputation, and returns - `Result, SummaryMaintenanceLifecycleAssemblyError>`. - When a summary does not beat a - known raw cost, or a required comparable cost is unavailable, the result - retains the exact `KeepPreAsap` root and no summary deployments. - -As in ordinary selection, one selection call serves the workload and assembly -is per root. The second helper calls `assemble_selected_dag` internally; callers -do not need a separate assembly call. Neither helper creates a materialized view -or deploys runtime state. -The output is a selected logical DAG with lifecycle decisions, not an executable -deployment plan. Any claim of optimization is relative to the supplied cost -model, evidence, and available candidates. -See the [library guide's lifecycle and capabilities section](../../develop_docs/library-api.md#lifecycle-and-capabilities) -for an API example and the capability contract. - -Across the two calls, the caller supplies these parameters: - -| Helper parameter | Source | Required | -|---|---|---:| -| `CandidateLogicalASAPDAGs` | Canonical ASAPPlanner output; passed to selection | Yes | -| `GlobalSelection` and one root | Selection result and a root in that `CandidateLogicalASAPDAGs`; passed to DAG assembly | Yes for each assembled root | -| Workload binding | `QueryWorkload` plus the workload-entry indices associated with each root | Yes | -| Planning time (`now_ms`) | Caller clock in Unix milliseconds | Yes | -| Planning horizon | Caller policy | Conditional: required for finite totals over recurring demand | -| Data arrival and update rate | `DataWorkload` evidence | Conditional: required to cost continuous maintenance | -| Lifecycle capabilities | Deployment/runtime provider | Yes for checking deployable lifecycle alternatives | -| Summary and raw cost information | Cost model and physical-evidence provider | Yes for a cost-based maintenance-versus-recompute decision | - -Recurrence and time selection are already fields of the bound `QueryWorkload`; -they are not duplicated as separate top-level inputs. Similarly, data arrival -and update rate are read from the optional `DataWorkload`. Missing required -facts remain unknown rather than being treated as zero. - -The per-query output, `SummaryMaintenanceLifecyclePlan`, **contains** the -Post-ASAP DAG rather than being a parallel representation. It records: - -* the assembled Post-ASAP DAG root (`Rc`); -* lifecycle choices for summary state; -* planning horizon and expected reads/updates; -* selected window implementation and guarantees; -* comparable summary and raw-recomputation costs; and -* whether raw recomputation was selected. +This path selects how to compute the query, not whether summary state is +materialized. --- diff --git a/docs/design_docs/architecture/metricsql-frontend.md b/docs/design_docs/architecture/metricsql-frontend.md index ddc463239..ffb45b728 100644 --- a/docs/design_docs/architecture/metricsql-frontend.md +++ b/docs/design_docs/architecture/metricsql-frontend.md @@ -9,16 +9,17 @@ on VictoriaMetrics and models MetricsQL syntax directly, including `WITH`, rollup expressions, step-relative durations, MetricsQL binary operators, aggregate limits, or-delimited matchers, and `keep_metric_names`. -The frontend walks that AST directly and emits the existing canonical -`QueryExpr`. It does not add MetricsQL fields to `QueryExpr`, SDS descriptors, -or the physical summary DAG. +The frontend walks that AST directly into the shared name-based `UnresolvedOp` +tree (`asap-frontend-common`) and calls `resolve_root`, which returns the +canonical `Rc` DAG. It does not add MetricsQL fields to the +operator IR, SDS descriptors, or the physical summary DAG. ```text MetricsQL source | MetricsqlExpr (extension semantics retained) | -canonical QueryExpr +UnresolvedOp tree --resolve_root--> canonical OperatorNode DAG | existing ASAP-aware mapping and physical Summary DAG ``` @@ -33,8 +34,8 @@ existing ASAP-aware mapping and physical Summary DAG | Common rollups: rate/increase/derivatives and statistical `*_over_time` | Existing per-entity canonical intents over the lowered range. | | PromQL arithmetic, comparison, and set binary operators without modifiers | Existing canonical `BinaryOp`. | | `default_rollup(selector[range])` | Lower to `Aggregate(LastOverTime)` over the explicit `TimeRange`. | -| `default_rollup(selector)` | Reject for exact fallback because the implicit lookbehind window depends on the runtime evaluation step, which is not a property of canonical `QueryExpr`. | -| `expr keep_metric_names` | Parsed natively, then rejected for exact fallback because canonical `QueryExpr` does not carry metric-name lineage. | +| `default_rollup(selector)` | Reject for exact fallback because the implicit lookbehind window depends on the runtime evaluation step, which is not a property of the canonical operator IR. | +| `expr keep_metric_names` | Parsed natively, then rejected for exact fallback because the canonical operator IR does not carry metric-name lineage. | | `if`, `ifnot`, `default`, aggregate `limit`, or-delimited matchers, binary match modifiers | Parsed natively and rejected until the canonical executor has the exact semantics. | | `WITH` | Expanded by the native parser; the expanded expression lowers when every resulting node is supported. | diff --git a/docs/design_docs/architecture/physical-plan-integration.md b/docs/design_docs/architecture/physical-plan-integration.md index 057ba7c17..2fb056412 100644 --- a/docs/design_docs/architecture/physical-plan-integration.md +++ b/docs/design_docs/architecture/physical-plan-integration.md @@ -5,14 +5,13 @@ This document defines the boundary between ASAPPlanner's logical plans, physical lowering, statistics resolution, and analytical resource estimation. It answers which representation is authoritative at each stage and prevents -the cost model from being coupled directly to either logical IR. +the cost model from being coupled directly to the logical IR. The integration pipeline is: ```text -pre-ASAP QueryExpr ─┐ - ├─ physical lowering ─> PhysicalOperator DAG -post-ASAP SummaryExpr┘ │ +logical OperatorNode DAG ─ physical lowering ─> PhysicalOperator DAG +(NonASAPOp + ASAPOp nodes) │ v OperatorStatistics │ @@ -30,8 +29,8 @@ Each representation is authoritative for a different concern: | Representation | Authoritative concern | |---|---| -| `QueryExpr` | Original exact query semantics: sources, predicates, relational and PromQL operations, and output shape. | -| `SummaryExpr` | Logical summary semantics: selected family, grouping strategy, summary composition, and summary readout. | +| `NonASAPOp` nodes | Exact query semantics: sources, predicates, relational and PromQL operations, and output shape. | +| `ASAPOp` nodes | Logical summary semantics: selected family, grouping strategy, summary composition, and summary evaluation. | | `PhysicalOperator` DAG | Selected executable algorithms, their configuration, physical identity, edges, and execution multiplicity. | | `OperatorStatistics` | Workload-dependent evidence required by each selected physical operator's resource formula. | | `ResourceEstimate` | Estimated CPU operations, peak live memory, and physical source/disk reads over one comparison scope. | @@ -39,7 +38,7 @@ Each representation is authoritative for a different concern: `PhysicalOperator` is therefore the source of truth for the operator vocabulary consumed by analytical costing. `OperatorStatistics` corresponds one-to-one with that vocabulary. It must not independently invent operator kinds or copy -all variants from either logical IR. +all variants from the logical IR. The canonical physical-plan types should live at a neutral boundary shared by lowering, costing, explanation, and downstream compilation. Their conceptual @@ -51,7 +50,7 @@ being established. One logical operation may choose between algorithms or expand into a physical sub-DAG. Conversely, one physical operator may implement nodes originating -from either logical IR. +from either operator category (`NonASAPOp` or `ASAPOp`). Examples include: @@ -61,14 +60,14 @@ Examples include: supported join algorithm. - `SummaryAgg` may lower to an exact accumulator build, CMS build, KLL build, or another physical summary algorithm selected by the candidate. -- `SummaryEstimate` must lower to a readout operator compatible with the +- `SummaryEstimate` must lower to a evaluation operator compatible with the concrete summary state it consumes. - shared logical sub-DAGs become shared physical nodes only when they refer to the same physical identity and compatible evidence. -For this reason, aligning `OperatorStatistics` directly with `QueryExpr` would -lose post-ASAP summary implementations, while aligning it directly with -`SummaryExpr` would lose raw query operators and physical algorithm choices. +For this reason, aligning `OperatorStatistics` directly with the logical +operators would lose physical algorithm choices, and with only one category +would lose either summary implementations or raw query operators. ## Lowering obligations @@ -91,14 +90,16 @@ its modeled descendants is invalid because it undercounts the candidate. ### Pre-ASAP lowering -`KeepPreAsap` recursively lowers its contained `QueryExpr`. Typical physical +Every `NonASAPOp` node lowers recursively, whether it is in a raw query or +kept exact inside a post-ASAP plan. Typical physical operators include scans, filters, projections, hash aggregates, joins, ordering, bounded Top-K, limits, and PromQL-specific operators. The selected physical algorithm, rather than the logical spelling, determines the formula. ### Post-ASAP lowering -Every `SummaryExpr` operation also needs explicit physical realization: +Every `ASAPOp` node, and every exact operator composed with one, also needs +explicit physical realization: | Logical summary operation | Required physical realization | |---|---| @@ -107,38 +108,24 @@ Every `SummaryExpr` operation also needs explicit physical realization: | `SummaryMerge` | merge operator over compatible concrete summary states | | `SummarySubtract` | subtract operator supported by the selected state representation | | `SummaryDelete` | physical deletion/update operator supported by the selected representation | -| `SummaryEstimate` | family- and query-specific readout operator | -| `KeepPreAsap` | recursive lowering of the contained `QueryExpr` | -| `BinaryOp` | binary evaluation preserving operand order, execution timing and any typed finite/relative-division guard | -| `ValueOperation` | concrete realization of the value operation with its required execution timing and data state | -| `RelationalJoin` | concrete row-join algorithm preserving join kind and predicate | -| `RelationalJoin` with `JoinKind::Semi` | retain left rows matching explicit right-side keys; candidate pruning carries completeness evidence and ordinary TopK ranks the result | +| `SummaryEstimate` | family- and query-specific evaluation operator | +| `FinalizeExactAccumulator` | exact-state finalization before value consumers | +| `MaintainPopulation` / `EvaluatePopulation` | maintained-population update and its aggregate or TopK-prefix evaluation | +| retained `NonASAPOp` sub-DAG | recursive lowering of the exact operators (see above) | +| `BinaryOp` | binary evaluation preserving operand order, the node's execution timing and any typed finite/relative-division guard | +| `Project` / `Filter` / `Sort` / `Limit` / `Aggregate` over a evaluation | concrete realization at the node's execution timing and data state | +| `Join` | concrete row-join algorithm preserving join kind and predicate | +| `Join` with `JoinKind::Semi` | retain left rows matching explicit right-side keys; candidate pruning carries completeness evidence and ordinary TopK ranks the result | This table is a completeness requirement, not a claim that every realization already exists. Until lowering introduces an explicit physical operator, statistics contract, validation rule, and resource formula for an operation, a candidate containing it is unavailable. -The streaming integration can consume a complete binding through -`SummaryNodeEvidence`. That binding is keyed to exact `SummaryNode` -identities and uses structured evidence for aggregate state, join, merge, -subtract, delete, readout, and retained pre-ASAP work. It is a physical -evidence boundary, not automatic physical lowering: a deployment must still -select each concrete implementation and provide all edges, resource facts, -multiplicities, source ownership, and stable physical identities. The planner -fails closed when any reachable `SummaryExpr` node lacks that binding. - -The raw/query portion of a streaming comparison remains a `PhysicalDAG` using -the canonical `PhysicalOperator` and `OperatorStatistics` pairing. Summary -evidence is kept separate only where lifecycle-driven update, retention, and -expiration multiplicities require facts beyond the query-DAG -`Once`/`PerEvaluation` schedule. It must not redefine workload, lifecycle, or -summary-family semantics. - -Lifecycle choice affects the physical DAG but does not replace it. Ephemeral, -prepared, shared, and continuously maintained alternatives determine when -build, update, readout, merge, subtract, or delete nodes execute. The physical -operators still determine how each execution consumes CPU, memory, and I/O. +Materialization choice (Stage 2, #509) affects the physical DAG but does not +replace it: it determines when build, update, evaluation, merge, subtract, or +delete nodes execute. The physical operators still determine how each execution +consumes CPU, memory, and I/O. ## Statistics contract @@ -427,7 +414,7 @@ recovering average semantics from query text. ### Candidate pruning is a sub-DAG -Candidate-based TopK uses a summary key readout, a general semi-join over +Candidate-based TopK uses a summary key evaluation, a general semi-join over explicit matching key columns, grouped Sort by the authoritative score, and grouped Limit. Sort and Limit carry the same partition keys. The join preserves authoritative left-side values and does not rank or limit @@ -440,10 +427,12 @@ fields. The phase assignment API updates producer edge states and rejects an ingestion computation that depends on query-time work. Deployment capability, storage readiness, schemas and approximation guarantees remain separate checks. -Post-ASAP DAG wire version 4 removes the special membership operator, its edge +Post-ASAP DAG wire version 4 removed the special membership operator, its edge roles and the duplicate operator phase fields without compatibility aliases. +Version 6 (current) exports one node per operator: retained exact operators are +`Relational` nodes, not embedded sub-DAGs. -Post-ASAP DAG wire version 6 adds a per-measure row predicate to the aggregate +Post-ASAP DAG wire version 7 adds a per-measure row predicate to the aggregate operators (#466): `filters` on the exact aggregate value operation, parallel to its measures, and `filter` on `SummaryAgg`, gating which rows update the summary state. The version bump makes an older reader fail loudly instead of @@ -465,38 +454,8 @@ a numeric entity key is not a score. Exact accumulator inputs are explicitly finalized before row operators consume them. None of these operations proves candidate completeness; that evidence belongs to the semi-join's pruning step. -The semantic `SummaryExpr` constructors still propose an initial execution -layout. Uniform phase assignment applies to the exported post-ASAP DAG; +The logical DAG carries no execution layout: `apply_materialization_timings` +writes each node's timing from a `MaterializationAssignment` before export (all +query time by default). Uniform phase +assignment applies to the exported post-ASAP DAG; it is not a claim that every deployment has implemented every placement. - - -## Summary cost evidence across data-arrival modes - -`SummaryMaintenanceCostModel` binds `SummaryNodeEvidence` and -`SummaryOperatorEvidence` independently of data-arrival mode. `ComparisonScope` -and the canonical `DataWorkload` determine arrival semantics; individual operator -resource records do not define another workload model. - -`SummaryMaintenanceInputs::from_workload` requires fresh snapshot cardinality. -For `AtRest`, it derives zero arrivals without requiring ingestion-rate evidence; -a fresh nonzero or invalid rate contradicts that declaration and is rejected. -For `ContinuouslyIngesting`, fresh, finite, nonnegative rate evidence remains -mandatory. Missing continuous rate evidence is never treated as zero. -Raw and summary evidence supplied directly by a provider obey the same arrival -invariant. Their source lineage, horizon, evaluation count, and snapshot dimensions -must still match. The existing lifecycle planner selects direct builds for a fixed -snapshot and charges bootstrap work, result evaluation, and retention; it charges -no arrival updates. This does not add computation-placement policy. - -`Mixed` and `Unknown` remain unsupported for analytical comparisons: the current -workload schema cannot identify separate backlog and arrival populations. The -adapter fails explicitly rather than guessing a split. The estimator version is -`summary-maintenance-resource-v2`; evidence type names drop the `Streaming` prefix -(`SummaryMaintenanceInputs`, `SummaryPhysicalInputEvidence`, `SummaryAggregateEvidence`, -`RetainedSubDAGEvidence`, `RawInputEvidence`, and the summary window/alternative -types). Update source imports; no legacy-name aliases are provided. - -Regressions cover a fixed snapshot with no rate evidence, contradictory arrival -rates, scope mismatches, missing continuous-rate/cardinality evidence, and actual -lifecycle selection of a completely costed at-rest summary against its raw scan. -The existing continuous-ingestion and mixed-arrival rejection tests remain. diff --git a/docs/design_docs/architecture/planner-runtime-contract.md b/docs/design_docs/architecture/planner-runtime-contract.md index 58e3f06b6..0077252ca 100644 --- a/docs/design_docs/architecture/planner-runtime-contract.md +++ b/docs/design_docs/architecture/planner-runtime-contract.md @@ -4,33 +4,27 @@ ASAPPlanner produces `CandidateLogicalASAPDAGs`, a compact logical candidate space. Integrators may select candidates downstream or ask Planner's helpers to select and assemble -DAGs. Summary-maintenance lifecycle decisions belong to Planner only when the -integration uses its lifecycle-aware workflow; physical deployment and execution -remain downstream. The [input/output/workflow design](input-output-workflow.md) -defines this boundary. - -A downstream provider can report implementation alternatives and their cost and -accuracy evidence for a Planner-owned comparison. The resulting -`SummaryMaintenanceLifecyclePlan` contains a Post-ASAP DAG root and maintenance -decisions; it is not an executable plan. Repeated provider calls do not constitute -an implemented end-to-end replanning or deployment-transition protocol. +DAGs. Stage 2 materialization (#509) will decide per sub-DAG whether to +materialize and whether at ingestion or query time; until then every summary +runs at query time. Physical deployment and execution remain downstream. The +[input/output/workflow design](input-output-workflow.md) defines this boundary. ## Three decision layers | Layer | Owner | Examples | |---|---|---| | Logical candidate semantics | ASAPPlanner | Query rewrite; summary family and parameters; grouping; accuracy guarantees when established. | -| Summary maintenance and realization selection | Planner helpers when delegated to Planner; otherwise downstream | `Ephemeral`, `Prepared`, `Shared`, `ContinuouslyMaintained`; `DirectBuild` or `Incremental`; window implementations compared using provider evidence. | +| Summary materialization and realization selection | Stage 2 materialization (#509); downstream until then | Ingestion-time maintenance or query-time computation per summary state; direct build or incremental update; window implementations compared using provider evidence. | | Concrete implementation and deployment | ASAPQuery-backend and its workload optimizer | Library and data-structure implementation, exact pane layout, placement, sharding, storage, transmission, materialization IDs, executor configuration, and workload-wide assignment. | ASAPCollector and the ASAPQuery data plane execute the compiled downstream plans. They validate capabilities and plan identities, maintain or read the specified state, and report runtime observations. They do not silently choose -a different summary, lifecycle, or realization framework. +a different summary, materialization, or realization framework. ## Incremental-maintenance example -When the integration delegates summary-maintenance decisions to Planner, +Once Stage 2 materialization (#509) owns summary-maintenance decisions, ASAPPlanner may decide that a logical summary should be incrementally maintained: new data updates existing summary state. It may also select the planner-visible window realization—such as tumbling, sliding/panes, or an @@ -43,7 +37,7 @@ pane representation, runtime operator implementation, placement, sharding, watermark behavior, and materialization identifiers. ASAPCollector maintains the compiled panes and summary state. -Thus `Incremental` describes the state-update lifecycle, while tumbling, +Thus incremental update describes how state is maintained, while tumbling, sliding, and exponential-histogram describe realization algorithms. They are distinct axes, but both can participate in ASAPPlanner's candidate space. The backend still owns how the selected algorithms are physically realized. @@ -53,7 +47,7 @@ backend still owns how the selected algorithms are physically realized. The same contract applies when ASAPPlanner selects a summary algorithm. Planner can choose KLL rather than DDSketch, while downstream chooses the concrete KLL implementation and runtime configuration that satisfies the selected parameter -and accuracy contract. Empirical KLL error, update work, state size, and readout +and accuracy contract. Empirical KLL error, update work, state size, and evaluation work observed on a particular workload can be fed back as evidence for later Planner comparisons. @@ -64,11 +58,10 @@ selected algorithm's semantics or guarantees. ## Iterative planning protocol (future integration) The sequence below is an intended integration design, not one shipped public -API or a required path for every caller. Current provider and lifecycle helpers -support a bounded planning decision; cross-run identity, migration, activation, -and rollback are not an end-to-end Planner protocol. +API or a required path for every caller. Cross-run identity, migration, +activation, and rollback are not an end-to-end Planner protocol. -1. ASAPPlanner enumerates semantically valid logical summaries, lifecycle +1. ASAPPlanner enumerates semantically valid logical summaries, materialization alternatives, and registered realization strategies. 2. A physical-plan provider maps those candidates to executor-feasible complete alternatives. Unsupported candidates are omitted or explicitly rejected. @@ -76,12 +69,12 @@ and rollback are not an end-to-end Planner protocol. scan selection, input/output edges, operation counts, update and bootstrap fanout, retained state, CPU, memory, I/O, and accuracy facts. 4. ASAPPlanner keeps constructible candidates with missing evidence visible - in `CandidateLogicalASAPDAGs` but does not certify unknown accuracy. The - summary-maintenance-lifecycle-aware workflow compares supported alternatives - over the same workload horizon. Missing or incomparable costs do not establish + in `CandidateLogicalASAPDAGs` but does not certify unknown accuracy. + Materialization compares supported alternatives over the same workload + horizon. Missing or incomparable costs do not establish that maintaining a summary beats raw recomputation; structural scores and optimistic zeroes are not substitutes. -5. ASAPPlanner outputs the selected Post-ASAP semantics, lifecycle guarantees, +5. ASAPPlanner outputs the selected Post-ASAP semantics, materialization choices, realization contract, and chosen provider identity. 6. ASAPQuery-backend compiles that result into consistent `CollectorPlan`, `BackendPlan`, and `QueryPlan` projections and performs deployment-level and @@ -106,11 +99,8 @@ such as cache behavior, serialization overhead, compression, spill I/O, or data-distribution-dependent sketch error. Provenance and version information must accompany those facts so stale observations fail closed. -`SummaryPhysicalPlanAlternative` is the current integration point for a -complete provider-enumerated implementation. Its identity is returned with the -winning lifecycle combination. More structured planner-owned realization -contracts can refine the candidate space without moving executor -implementation into ASAPPlanner. +More structured planner-owned realization contracts can refine the candidate +space without moving executor implementation into ASAPPlanner. ## Workload-wide optimization @@ -124,7 +114,7 @@ The ASAPQuery configuration and MIP formulations can supply physical alternatives and coefficients. Their general principles also inform Planner costing: arrival rate scales ingestion work, overlapping active windows multiply update work and live state, retained windows consume memory, and -merge/subtract/readout work scales with query recurrence. Disagreement between +merge/subtract/evaluation work scales with query recurrence. Disagreement between formulations must become distinct explicit alternatives, not hidden assumptions in one cost formula. @@ -146,7 +136,7 @@ in one cost formula. automatically selected. - Shared logical nodes remain shared across the planner-runtime contract; physical sharing additionally requires compatible filters, grouping, windows, parameters, - lifecycle, and guarantees. + materialization, and guarantees. - Collector, backend, and query plans are projections of one compiled decision and cannot be optimized independently into inconsistent semantics. @@ -155,7 +145,6 @@ in one cost formula. - [Post-ASAP IR](../concepts/post-asap-ir.md) - [Physical plan integration](physical-plan-integration.md) - [Analytical resource cost](../proposals/asap-aware-mapping/analytical-resource-cost.md) -- [Workload demand and summary lifecycle](../proposals/asap-aware-mapping/workload-demand-and-summary-lifecycle.md) - [ASAPCollector physical compilation](https://github.com/ProjectASAP/ASAPCollector/blob/87684f4b61514382d8b087724694f93187bfc19c/docs/design_docs/control-plane/post-asap-physical-compilation.md) - [ASAPQuery configuration formulation](https://github.com/ProjectASAP/ASAPQuery/blob/main/.design_docs/sketch-config-optimization-formulation.md) - [ASAPQuery optimizer MIP formulation](https://github.com/ProjectASAP/ASAPQuery/blob/main/.design_docs/optimizer-mip-formulation.md) diff --git a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md index 72e626a2e..682055b0a 100644 --- a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md +++ b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md @@ -9,8 +9,8 @@ What that buys: * One call in place of six across three stages. `CandidateLogicalASAPDAGs` and `GlobalSelection` no longer appear in user code. -* The root-to-entry bindings a caller used to build by hand are derived, and - their ordering contract is checked rather than assumed. +* The root-to-entry binding a caller used to build by hand is derived, and + its ordering contract is checked rather than assumed. * A new optimization algorithm can be freely implemented as a trait implementation, rather than a rule disguised to fit a two-phase pipeline it does not share. @@ -20,7 +20,6 @@ Unchanged: `CandidateLogicalASAPDAGs`, `cost_sorted`, `global_selection`, and th ```text PlanningWorkload ──lowering──▶ ParsedWorkload ──OptimizationPass──▶ PlanOutput + frontend deps + models - + lifecycle input ``` --- @@ -64,7 +63,6 @@ Details of these types are provided below. | `workload` | `&PlanningWorkload` | | `frontend_specific` | `Sql { catalog }` / `Promql { now_ms, histograms }` / `Metricsql`; fixed by `query_workload.language` | | `models` | Cost model, accuracy model, evidence provider; `PlanningModels::builtin()` for the defaults | -| `lifecycle` | Planning clock and runtime capabilities for the maintenance-versus-recompute decision every plan carries | | `pass` | `None` uses `MajorPass` | ### `OptimizationInput` @@ -73,7 +71,6 @@ Details of these types are provided below. pub struct OptimizationInput<'a> { pub workload: &'a ParsedWorkload, pub models: PlanningModels<'a>, // same type UserInput uses - pub lifecycle: LifecycleInput, // same type UserInput uses } ``` @@ -83,19 +80,18 @@ pub struct OptimizationInput<'a> { ```rust pub struct PlanOutput { - pub plans: Vec, // one per workload entry, in entries() order + pub plans: Vec, // one per workload entry, in entries() order } -pub struct QueryLifecyclePlan { - pub entry_index: usize, // index into QueryWorkload::entries() - pub plan: SummaryMaintenanceLifecyclePlan, // its `root` is the DAG +pub struct QueryPlan { + pub entry_index: usize, // index into QueryWorkload::entries() + pub root: Rc, // selected post-ASAP DAG; shared nodes are the same Rc } ``` -Every plan carries the maintenance decisions, so the pass always runs -lifecycle-aware selection. A cost model that cannot price lifecycles -(`DefaultCostModel` today) makes that selection fall back to raw recompute for -every summary target; supply a model with the lifecycle cost hooks. +Plans carry no materialization decision. `PlanOutput::execution_timed_dag()` +times every summary at query time until Stage 2 materialization (#509) decides +per sub-DAG whether to materialize and whether at ingestion or query time. --- @@ -112,8 +108,9 @@ The `MajorPass` described below will be used by default, which corresponds to th |---|---| | Build roots | `Id` is the entry's position in `entries()`; the accuracy target comes from its `requirements` | | Candidate search | `search_workload_with_targets` with `default_strategies_with_evidence` | -| Select | `global_selection`, or `global_selection_with_summary_maintenance_lifecycles` with a `WorkloadDemand` derived from the `ParsedWorkload` | -| Assemble, per root | `assemble_selected_dag`, or its lifecycle-aware counterpart | +| Select | `CandidateLogicalASAPDAGs::global_selection` | +| Assemble, per root | `GlobalSelection::assemble_selected_dag` | +| Share | `asap_types::ir::cse::share_common_sub_dags` across the assembled roots | Moving it behind the trait changes one thing for existing developers: **`ReplacementStrategy` is now a concept of `MajorPass`, not of the optimization @@ -159,70 +156,60 @@ for name in registry.names() { `PassRegistry` is caller-owned, not a link-time global, so two tests in one binary cannot see each other's registrations. -### 3.3 The three existing workflows, in this shape +### 3.3 The existing workflows, in this shape -[Input, output, and workflows](input-output-workflow.md) describes three ways to -use the candidate space. Only the last is what a pass produces; the other two -stay on the old interfaces. +[Input, output, and workflows](input-output-workflow.md) describes two ways to +use the candidate space. The second is what a pass produces; the first stays +on the old interfaces. | Workflow there | Here | |---|---| | Ranked view (`cost_sorted`) | Not covered by this design, you should handle it with old interfaces | -| Selection and DAG assembly | Not covered either: `search_workload_with_targets` + `global_selection` + `assemble_selected_dag` | -| Summary-maintenance-lifecycle-aware helper | `PlanOutput` | +| Selection and DAG assembly | `PlanOutput` | -The third is no longer a call sequence the caller drives. +Selection and assembly are no longer a call sequence the caller drives. Following is an example of how the old workflow maps to the new interface. ```rust -// Before — from a PlanningWorkload and a catalog, with lifecycle decisions. +// Before — from a PlanningWorkload and a catalog. // 1. Lower every normalized entry, and record which entry each root came from. // Not lower_sql_batch: it walks query_batch alone and drops repeating entries. let mut roots = Vec::new(); -let mut entry_indices = Vec::new(); for (index, entry) in workload.query_workload.entries().enumerate() { let accuracy = entry.requirements.accuracy.target(); let expr = lower_sql_dialect(&entry.query.0, &catalog, dialect.clone(), accuracy.clone()) .await?; - roots.push((index, Rc::new(expr), Some(accuracy))); - entry_indices.push(index); + roots.push((index, expr, Some(accuracy))); } // 2. Search for candidates. let strategies = default_strategies_with_evidence(&cost_model, &evidence); let space = search_workload_with_targets(roots, &strategies, &accuracy_model); -// 3. Select once for the whole workload, re-binding roots to workload entries. -let demand = WorkloadDemand { - workload: &workload.query_workload, - data_workload: workload.data_workload.as_ref(), - entry_indices: &entry_indices, -}; -let selection = global_selection_with_summary_maintenance_lifecycles( - &space, demand, now_ms, horizon, capabilities, &cost_model)?; - -// 4. Assemble once per root. -let mut plans = Vec::new(); +// 3. Select once for the whole workload. +let selection = space.global_selection(&cost_model); + +// 4. Assemble once per root, then share common sub-DAGs across roots. +let mut assembled = Vec::new(); for (index, root) in &space.roots { - let plan = assemble_selected_dag_with_summary_maintenance_lifecycles( - &selection, root, demand, now_ms, horizon, capabilities, &cost_model)?; - plans.push((*index, plan)); + if let Some(dag) = selection.assemble_selected_dag(root)? { + assembled.push((*index, dag)); + } } +let plans = share_common_sub_dags(assembled); ``` ```rust // After. let output = e2e_plan( UserInput::new(&workload, FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin(), - LifecycleInput::new(now_ms, capabilities).with_horizon(horizon)) + PlanningModels::builtin()) ).await?; ``` -Steps 1 and 3 are where the two bindings lived: the `Id` carried through the -roots tuple, and the `&[usize]` rebuilt for `WorkloadDemand`. Both had to agree -with `entries()` order, and nothing checked that they did. `MajorPass` still +Step 1 is where the binding lived: the `Id` carried through the roots tuple had +to agree with `entries()` order, and nothing checked that it did. `MajorPass` still runs all four steps; another pass need not run any of them. ## 4. Code layout @@ -230,7 +217,7 @@ runs all four steps; another pass need not run any of them. | Crate | What it holds | |---|---| | `asap-types` | `ParsedWorkload` | -| `asap-aware-mapping` | `OptimizationPass`, `OptimizationInput`, `PlanOutput`, `PlanningModels`, `LifecycleInput`, `optimize`, `PassRegistry`, `MajorPass` | +| `asap-aware-mapping` | `OptimizationPass`, `OptimizationInput`, `PlanOutput`, `PlanningModels`, `optimize`, `PassRegistry`, `MajorPass` | | `asap-planner` *(new)* | `e2e_plan`, `UserInput`, `FrontendInput`, lowering dispatch | ```text @@ -242,9 +229,9 @@ asap-planner ──┬──> asap-frontend-{sql, promql, metricsql} `asap-planner` is separate because it is the only crate depending on every frontend; before it, the sole facade re-exporting more than one was -`asap-devtools`, a developer-tools crate. `PlanningModels` and `LifecycleInput` -live in `asap-aware-mapping` because both inputs use them, and `asap-planner` -re-exports them. +`asap-devtools`, a developer-tools crate. `PlanningModels` lives in +`asap-aware-mapping` because both inputs use it, and `asap-planner` re-exports +it. --- diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index 5353ef9ef..baeadbe9e 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -69,7 +69,7 @@ query text or cost estimates. flowchart TD Request[Query semantics and accuracy target] --> Generate[Generate candidates and size parameters] Evidence[Scoped source contracts and evidence] --> Generate - Generate --> Local[Derive local readout guarantees] + Generate --> Local[Derive local evaluation guarantees] Evidence --> Local Local --> Compose[Propagate guarantees through the DAG] Evidence --> Compose @@ -100,7 +100,7 @@ is ready, or that a complete deployment cost is available. ## Local estimator models and parameter sizing -A local model describes a specific readout of a specific estimator with +A local model describes a specific evaluation of a specific estimator with committed parameters and applicable assumptions. A family name or a parameter such as HLL precision is not, by itself, a confidence certificate. @@ -121,8 +121,8 @@ The built-in models currently include: | CMS | L1-normalized frequency bound from width and depth; does not by itself certify TopK membership | | CountSketch | L2-normalized frequency bound and median concentration bound, requiring valid odd depth | | KMV / Theta | Parameter-derived cardinality bounds using the registered variance/Chebyshev model at 99% confidence | -| UnivMon | Exact unit-update total for the supported readout; no universal guarantee for all its statistics | -| Other families/readouts | No default certificate where no accuracy model is registered | +| UnivMon | Exact unit-update total for the supported evaluation; no universal guarantee for all its statistics | +| Other families/evaluations | No default certificate where no accuracy model is registered | This table describes Planner's registered contracts, not independent mathematical verification of every estimator or permission to substitute @@ -214,7 +214,7 @@ an observation into a guarantee. A deployment supplies `EstimatorContract::ClassicHll` for the complete aggregate expression. It asserts the classic estimator, independent uniform bucket -hashing and an enforced maximum distinct population per readout, including +hashing and an enforced maximum distinct population per evaluation, including all merged panes. Planner combines this contract with the query or allocated local target, selects a supported precision, derives the guarantee and uses the normal propagation and selection checks. @@ -290,7 +290,7 @@ accuracy/ ├── allocation.rs # End-to-end budget allocation ├── reconciliation.rs # Accuracy coordination across consumers └── estimators/ - ├── mod.rs # Family/readout dispatch and source-contract integration + ├── mod.rs # Family/evaluation dispatch and source-contract integration ├── kll.rs ├── ddsketch.rs ├── hll.rs # Generic HLL and bounded Classic HLL @@ -308,7 +308,7 @@ share the same contract. Adding an estimator or composition requires: -1. A precisely defined error metric, estimator/readout semantics and assumptions. +1. A precisely defined error metric, estimator/evaluation semantics and assumptions. 2. Sizing behavior and a guarantee derived from the committed parameters, including unsupported parameter domains. 3. Explicit evidence requirements, population scope and provenance. 4. Propagation rules where supported; rejection or retained unknowns elsewhere. diff --git a/docs/design_docs/concepts/planner-pipeline.md b/docs/design_docs/concepts/planner-pipeline.md index 7401ae0da..c07d17775 100644 --- a/docs/design_docs/concepts/planner-pipeline.md +++ b/docs/design_docs/concepts/planner-pipeline.md @@ -14,12 +14,11 @@ over that output, not mandatory stages of candidate search. | +--> inspect candidates, optionally using cost_sorted +--> select and assemble logical DAGs - +--> select and assemble with summary-maintenance lifecycle decisions -The last two branches are alternatives: use the summary-maintenance-lifecycle-aware -workflow when Planner owns maintenance-versus-recomputation decisions; otherwise -the backend owns them. All physical binding, deployment, and execution remain -downstream responsibilities. +Stage 2 materialization (#509) will decide per sub-DAG whether to materialize +and whether at ingestion or query time; until then every summary runs at query +time. All physical binding, deployment, and execution remain downstream +responsibilities. The [input, output, and workflows](../architecture/input-output-workflow.md) document defines the public boundary and helper call order. diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index 6c9aa1461..207ad227e 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -1,22 +1,53 @@ # Post-ASAP IR The goal of the post-ASAP IR is to represent operations using ASAP primitives -such as sketches, exact summaries, samples and wavelets. Post-ASAP IR also -retains exact Pre-ASAP sub-DAGs and supports operations over summary readouts, -since only some query operations can be satisfied using summaries. - -The lists below cover every current variant of -[`SummaryExpr`](../../../crates/types/src/post_asap/expr.rs). A node's presence -in the IR does not imply that every summary family, cost model or downstream -runtime supports it. - -## ASAP-specific nodes operated over a summary structure, not raw data - -- `SummaryAgg`: produce summary state from input data using the selected family, - parameters, update input, reduction and grouping layout. -- `SummaryEstimate`: read the requested statistic from summary state and return - query values. Exact accumulators can expose results without a separate sketch - readout. +such as sketches, exact summaries, samples and wavelets, while retaining the +exact query operators that no summary replaces, and supporting operations over +summary evaluations. + +ASAPPlanner has one operator IR before and after ASAP optimization +([`crates/types/src/ir/`](../../../crates/types/src/ir/)). A post-ASAP plan is +the same `Rc` DAG a front end produced, in which some nodes now +carry `Operator::ASAP(ASAPOp)` instead of `Operator::NonASAP(NonASAPOp)`. There +is no wrapper around retained exact work: an unreplaced `Filter`, `Join` or +`Aggregate` is the same node it was before, and either category can consume +the other's output. The node structure, the `Schema`, scalar expressions and +the catalog of non-ASAP operators are described once in the +[Pre-ASAP IR reference](../../develop_docs/pre-asap-ir.md); this document covers +what optimization adds: the ASAP operators, the accuracy guarantee, execution +timing, and the exported DAG. + +A node's presence in the IR does not imply that every summary family, cost +model or downstream runtime supports it. + +## ASAP operators + +Every variant of [`ASAPOp`](../../../crates/types/src/ir/asap.rs) operates +over summary state rather than raw data. The summary family, kind/algorithm and +parameters are committed in the node; the state itself is typed by the +`FieldDataType` of the output field that carries it (`ExactAggregate`, +`Sketch`, `Sample`, `Wavelet`, `StatModel`). + +Implemented: + +- `SummaryAgg { child, family, input, reduction, grouping }`: produce summary + state from input rows using the selected family, parameters, update input, + reduction and grouping layout. Output: the grouping columns plus one `state` + field typed `family`; result kind `State`. +- `SummaryEstimate { summary_input, query }`: read the requested statistic + (`SketchStatistic`) from summary state and return query values in a row-shaped + schema. +- `FinalizeExactAccumulator { child }`: read an exact accumulator's state as + its finalized value — the maintenance-to-read boundary before query-time + operators consume it. +- `MaintainPopulation { child, population }`: maintain the full declared + population, including membership changes. +- `EvaluatePopulation { child, evaluation }`: read an aggregate or TopK prefix from a + maintained population. + +Reserved (migrated but unimplemented; schema derivation, timing and export +reject them with `UNIMPLEMENTED_ASAP_OP`): + - `SummaryMerge`: merge compatible summary states when the family supports merging. - `SummarySubtract`: subtract one summary state from another when supported by the selected representation. @@ -24,62 +55,114 @@ runtime supports it. deletion. - `SummaryJoin`: combine summary states for join estimation; this is distinct from joining ordinary rows. +- `Extension`: a deployment-defined operator. The earlier draft listed `SummaryCreate` and `SummaryInsert`. These are not -separate variants in the current IR. `SummaryAgg` describes the state-producing -computation and its update input. The -[summary-maintenance lifecycle](../proposals/asap-aware-mapping/workload-demand-and-summary-lifecycle.md) -separately describes when state is created, retained, shared, updated and retired. -Physical binding and runtime execution implement the actual build and update -operations. This is not a one-to-one rename of the old nodes, and not every -summary family supports incremental maintenance. - -## Exact work and composition nodes - -- `KeepPreAsap`: retain an exact Pre-ASAP sub-DAG when it is not rewritten. -- `BinaryOp`: combine independently planned operands with the specified binary - semantics and execution timing. -- `ValueOperation`: apply aggregate, exact-function, population, projection, - filter, sort, limit or extension semantics with explicit execution timing. -- `RelationalJoin`: join row-producing children using the specified join kind - and predicate. -- Candidate pruning uses `RelationalJoin` with `JoinKind::Semi` and an explicit +separate variants. `SummaryAgg` describes the state-producing +computation and its update input. Stage 2 materialization (#509) will decide +whether and when that state is maintained. Physical binding and runtime +execution implement the actual build and update operations. Not every summary family supports incremental maintenance. + +## Exact work and composition + +Exact work is represented by the ordinary operators, unchanged: + +- A sub-DAG the planner does not rewrite keeps its `NonASAPOp` nodes. Plan + assembly marks such a sub-DAG with an exact `ResultGuarantee` + (`asap_aware_mapping::replacement::retain_exact`); a sub-DAG with no ASAP + operator and no guarantee is a logical rewrite candidate that has not been + assessed yet (`is_logical_rewrite`). +- `BinaryOp` combines independently planned operands. Summary planning may set + its typed division guards (`checked_finite_division`, + `checked_relative_division`); the operator's timing comes from the + materialization assignment, not from the operator. +- Aggregate, projection, filter, sort and limit over a evaluation are the ordinary + `Aggregate`, `Project`, `Filter`, `Sort` and `Limit` operators reading an ASAP + node. Exact-accumulator state may pass through the projection-like + operators unchanged; a value consumer needs a `FinalizeExactAccumulator` + boundary first. +- Candidate pruning uses `Join` with `JoinKind::Semi` and an explicit equality predicate on key columns. The left input supplies authoritative - values; the right input supplies keys. Grouped Sort followed by grouped Limit ranks - and selects the joined rows. Completeness evidence belongs to pruning, not ranking. + values; the right input supplies keys. Grouped `Sort` followed by grouped + `Limit` (both with the same `partition_by`) ranks and selects the joined + rows. Completeness evidence belongs to pruning, not ranking. -A `SummaryNode` carries its expression, schema and optional result guarantee. +Every `OperatorNode` carries its schema and an optional `ResultGuarantee`. State and query values have different contracts. Exact operations over -approximate readouts still require composed accuracy guarantees. See the +approximate evaluations still require composed accuracy guarantees. See the [accuracy implementation companion](../../develop_docs/end-to-end-accuracy-guarantees.md) and [physical-plan integration](../architecture/physical-plan-integration.md) for the corresponding correctness and realization requirements. -## In-memory and exported DAG forms - -The Pre-ASAP DAG and the Post-ASAP DAG are both logical: they describe what is -computed, not which physical operators execute it. The Post-ASAP DAG has two -forms of the same content. Planning builds and shares `SummaryNode` DAGs. -`compile_post_asap_dag` converts a selected DAG into a -[`PostAsapDAG`](../../../crates/types/src/post_asap/post_asap_dag.rs) with -stable node IDs and typed edges; `PostAsapDAGDocument` is its versioned wire -envelope. Physical compilation consumes `PostAsapDAG` and produces a separate -physical DAG. - -## Execution phase +## Execution timing An operator defines what computation happens. The plan decides when it happens: **ingestion time** or **query time**. Operator identity must not imply one of these phases. Backend capability restrictions are implementation gaps, not definitions of the operator. -Every post-ASAP operator payload supports both phase assignments. Phase is -stored on the `PostAsapDAG` node, independently of its operator payload. -`PostAsapDAG::with_execution_phases` assigns a phase to every node and updates -its edges. Ingestion work cannot depend on a future query result. Default -semantic realization still proposes an initial layout; it does not restrict -which phase an operator may use. Deployments must separately check that -they have an implementation and a valid data source for the chosen placement. +The logical DAG carries no timing: `OperatorNode::timing` is `None` on every +front-end node and every candidate, and `map_children` clears it. Summary +materialization chooses a timing per summary state and records it in a +[`MaterializationAssignment`](../../../crates/types/src/ir/timing.rs) (ingestion-time +maintenance or query-time computation per `SummaryAgg`). The default is +`all_query_time()`; until Stage 2 materialization (#509) decides otherwise, the +planner times every `SummaryAgg` at query time. +`apply_materialization_timings(root, &assignment, &mut TimingMemo)` then writes a +timing into every node, top-down: + +- a node of fixed kind takes its kind's timing — `SummaryEstimate` and + `EvaluatePopulation` run at query time, `MaintainPopulation` at ingestion time; +- a `SummaryAgg` takes the assignment's timing, unless something below it can + only exist at query time (a evaluation); +- every other node runs when its consumer runs: everything that feeds a + maintained state runs at ingestion time, everything above a evaluation at + query time. + +The pass then validates every edge (rows or exact-accumulator state into a +`SummaryAgg`, state into a evaluation, an ingestion-time `MaintainPopulation` under +a `EvaluatePopulation`, no ingestion work reading a query-time value) and rejects a +node reached from two consumers that need different timings; +`split_shared_by_phase` copies such a sub-DAG for one side before the +assignment is applied. `validate_maintained` and `planned_data_state` answer the +same questions for a candidate at planning time, assuming every summary is +maintained at ingestion time, without keeping anything. + +## Exported DAG + +The pre-ASAP DAG and the post-ASAP DAG are both logical: they describe what is +computed, not which physical operators execute it. Planning builds and shares +`OperatorNode` trees; +[`asap_types::ir::export::compile_post_asap_dag`](../../../crates/types/src/ir/export.rs) +converts a selected, timed tree into a `PostAsapDAG` with stable node IDs and +typed edges, and `PostAsapDAGDocument` is its versioned wire envelope +(`schema_version` = `POST_ASAP_DAG_WIRE_VERSION`, currently 6). Physical +compilation consumes `PostAsapDAG` and produces a separate physical DAG. + +Wire version 7 emits **one node per operator** — relational operators +included — with children as edges and no embedded sub-DAGs: + +- A non-ASAP node is a `Relational { operator: NonASAPOpKind }` payload: + the operator's own fields with scalar expressions mirrored as + `WireScalarExpr`, children removed. An ASAP node's payload is its variant + (`SummaryAgg`, `SummaryEstimate`, `FinalizeExactAccumulator`, + `MaintainPopulation`, `EvaluatePopulation`, …). +- Edges carry a role: `Input`, `Left`/`Right` for the two sides of a `Join`, + `SetOp`, `BinaryOp`, `SummarySubtract` or `SummaryJoin`, and `ScalarRef` + when the consumer reads the producer from inside one of its scalar + expressions (`scalar(v)`). Every edge records the intermediate schema, the + producer's data state and grouping/window compatibility. +- Each node records `output_state` (timing plus `Raw` or `SummaryState`), + `output_schema` and `guarantee`. Export reads the timing written by + `apply_materialization_timings` and rejects an untimed node + (`ExecutionDataStateError::UntimedNode`); it does not re-run data-state + validation. + +Phase is stored on the `PostAsapDAG` node, independently of its payload. +`PostAsapDAG::with_execution_phases` reassigns a phase to every node and +updates its edges; ingestion work cannot depend on a future query result. +Deployments must separately check that they have an implementation and a +valid data source for the chosen placement. ## Weighted grouped TopK @@ -90,19 +173,19 @@ the update weight is the series rate. Summing updates for one item implements the logical grouped sum without first constructing all exact grouped sums. The DAG is per-series rate → finalized values → partitioned summary construction -→ typed candidate/score readout → output projection → grouped Sort → grouped +→ typed candidate/score evaluation → output projection → grouped Sort → grouped Limit. The output count is two per job. The candidate capacity is a separate parameter, provisionally `max(k, ceil(1 / epsilon))`; this sizing choice is not a membership theorem. Missing evidence retains a logical candidate with symbolic unknown guarantees; default selection does not certify or choose it. -The row readout restores job and service identities and returns estimated sums. +The row evaluation restores job and service identities and returns estimated sums. There is no mandatory exact scoring branch or candidate semi-join in this path. The old raw counter-delta update expression is removed rather than retained as a compatibility option: counter increments are not complete windowed rate results. -The direct readout represents both score error and membership. A source provider +The direct evaluation represents both score error and membership. A source provider supplies an enforced upper bound on distinct partition/item identities for the -complete readout. Planner uses this bound to size confidence and union-bound +complete evaluation. Planner uses this bound to size confidence and union-bound score errors over adaptively selected items. Membership evidence is evaluated for the query's output count, not the candidate capacity. Score and membership failure probabilities are combined, and the score guarantee remains in the @@ -110,8 +193,8 @@ membership guarantee's child provenance. An exact request does not accept this approximate output path merely because its selected identities are certified. Deployment chooses ingestion time or query time for these operators. The -semantic constructor proposes a layout; `with_execution_phases` assigns the -placement. Either deployment must give each evaluation a complete +materialization assignment writes the placement; `with_execution_phases` can +reassign it on the exported DAG. Either deployment must give each evaluation a complete rate window and an isolated summary state, or maintain an equivalent replacement strategy. Appending successive rate snapshots to one cumulative state is invalid. An ingestion execution can compute a window before the query and store its state; diff --git a/docs/design_docs/concepts/pre-asap-ir.md b/docs/design_docs/concepts/pre-asap-ir.md index 735f522f3..0b12766b1 100644 --- a/docs/design_docs/concepts/pre-asap-ir.md +++ b/docs/design_docs/concepts/pre-asap-ir.md @@ -12,16 +12,18 @@ Only semantics that affect correctness, summary applicability, or cost become fi ### Time -- TimeRange — a PromQL range-vector lookback such as [5m]. +- TimeRange — PromQL sample selection: an instant selector's lookback, or a range selector such as [5m]. - TimeShift — moves when a selector is evaluated (offset or @). - PromqlSubquery — re-evaluates an instant-vector expression over a range. ### Relational - Scan — identifies a logical data source. +- Values — literal rows; one empty row is the input of a `SELECT` without `FROM`. +- ScalarBridge — a scalar expression at an operator position: a bare scalar query, or the scalar operand of ` op `. - Filter — restricts rows using a predicate. - Project — selects or derives output columns. -- BinaryOp — composes two inputs with arithmetic, comparison, or boolean logic. +- BinaryOp — composes two inputs with arithmetic, comparison, or boolean logic. A PromQL `bool` comparison returns 0/1 instead of filtering. - Sort — orders rows without expressing a heavy-hitter intent. - Limit — caps a row count, optionally after an offset. - Dedup — removes duplicate rows. @@ -31,10 +33,7 @@ Only semantics that affect correctness, summary applicability, or cost become fi ### PromQL-specific -- PromqlScalarBridge — holds a scalar sub-expression at an operator-DAG position. -- EvalTimestamp — provides the evaluation timestamp as a scalar. -- PromqlVectorFromScalar — promotes a scalar to a label-less instant vector. -- PromqlScalarFromVector — collapses a single-series vector to a scalar. +- PromqlVectorFromScalar — promotes a scalar to a label-less instant vector. Its inverse, PromQL `scalar(v)`, is a scalar expression that reads `v`. - PromqlRelabel — rewrites labels on each series. - PromqlInfoEnrich — enriches labels from an info metric. - PromqlSeriesSample — selects whole series without reducing them. diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index 274e4974c..6f6d7f207 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -7,11 +7,11 @@ A Post-ASAP computation is progressively realized through four layers: ```mermaid flowchart LR L["Logical Post-ASAP DAG
What computation?"] - M["Summary Maintenance Lifecycle
How is state maintained?"] + M["Materialization
How is state maintained?"] P["Physical DAG(s)
How is it executed?"] D["Deployment Plan / DAG
How is it instantiated?"] - L -->|"Summary Maintenance
Candidate Generation"| M + L -->|"Stage 2
Materialization (#509)"| M M -->|"Physical Plan
Compiler"| P P -->|"Deployment Plan
Compiler"| D ``` @@ -19,47 +19,44 @@ flowchart LR | Layer | Defines | | --- | --- | | **Logical Post-ASAP DAG** | Computation semantics | -| **Summary Maintenance Lifecycle** | Build, retention, reuse, and window strategy | +| **Materialization** | Build, retention, reuse, and window strategy | | **Physical DAG(s)** | Supported physical candidates, executable operators and typed input boundaries | | **Deployment Plan / DAG** | Selected candidate, concrete data/state bindings and operational lifecycle | ASAPPlanner owns the first three layers and the shared physical operator implementation library. Deployment systems such as ASAPQuery and asap-fusion -own deployment compilation and operation. The lifecycle is a planning contract +own deployment compilation and operation. Materialization is a planning contract associated with the logical DAG, not a separate computation IR. -The Logical Post-ASAP DAG is preceded by the Pre-ASAP DAG (`QueryExpr`), the +> **Status:** Stage 2 materialization (#509) will decide per sub-DAG whether to +> materialize and whether at ingestion or query time. It is not implemented yet; +> until then the planner times every summary at query time. Sections 2 and 3 +> describe the intended contract. + +The Logical Post-ASAP DAG is preceded by the Pre-ASAP DAG, the language-independent query semantics before summary selection. Both are -logical. Planning builds Post-ASAP `SummaryNode` DAGs; `compile_post_asap_dag` +logical `OperatorNode` DAGs; `compile_post_asap_dag` exports the selected DAG as a `PostAsapDAG`, which is the Physical Plan Compiler's input. Its per-node execution phase (ingestion or query time) is -decided by the selected summary maintenance lifecycle, as the layer contract -below states. +decided by a `MaterializationAssignment`, as the layer contract below states. ### Layer contract 1. **Logical Post-ASAP** (`CandidateLogicalASAPDAGs`) decides what to compute: summary families, readouts and sharing. It does not decide placement; timing that a realization strategy writes while building a candidate is provisional. -2. **Summary maintenance lifecycle** (Planner) lists the lifecycle choices for - each unique retained state: every summary state (`SummaryAgg`) and every - maintained population that does not feed a summary state. - A chosen assignment determines every node's - `ExecutionTiming`, plus window framework and retention. - `SummaryMaintenanceLifecyclePlan::execution_timed_dag` applies it: a retained - (non-`Ephemeral`) state and all of its inputs run at ingestion time; - readouts, other consumers, and `Ephemeral` states not consumed by retained - state run at query time. A population that feeds a summary state is one of - that state's inputs and follows its timing. +2. **Materialization** (Stage 2, #509) chooses ingestion or query time for each + summary state (`SummaryAgg`) and records it in a `MaterializationAssignment`. + `apply_materialization_timings` writes every node's `ExecutionTiming`: an + ingestion-time state and all of its inputs run at ingestion time; readouts, + other consumers, and query-time states run at query time. + `MaintainPopulation` always runs at ingestion time. The default assignment + is all query time, which `PlanOutput::execution_timed_dag` applies. 3. **Physical compile** (Planner) reads timing: ingestion-time nodes form the precompute DAG and the rest form the query DAG, joined by typed outputs. It does not see raw ingestion, panes, storage or stored-state readout. -4. **Backend** chooses the lifecycle assignment with its own `CostModel`: - precompute CPU (`maintenance_cost_per_update`), sketch/summary store cost - (`retention_cost_rate`), query reads (`summary_read_cost`) and per-query - builds (`build_cost`, for `Ephemeral`), counting shared state once. - `Ephemeral` requires the deployment to supply the state's raw input as a - query-time source. +4. **Backend** binds and executes the timed DAG. A query-time summary requires + the deployment to supply the state's raw input as a query-time source. ### Candidate generation and deployment selection @@ -69,7 +66,7 @@ because a deployment-independent cost estimate prefers another candidate. Logical candidates are an internal search stage, not the deployment handoff. ```text -Query semantics + accuracy and lifecycle requirements +Query semantics + accuracy and freshness requirements ↓ Planner Supported Physical DAG candidates + typed inputs/outputs + requirements ↓ backend @@ -93,9 +90,9 @@ and a feasible candidate that loses on cost. Absence is not a cost comparison. For `sum by(job)(rate(m[1m]))`, Rate remains per series before grouped Sum. `CandidateLogicalASAPDAGs` offers one such candidate, with a per-series Rate state and a grouped -Sum state. Its lifecycle assignment places it: a retained Sum state finalizes -Rate and builds Sum within a bounded precompute run; an `Ephemeral` Sum over a -retained Rate state leaves the Rate readout and Sum in the query DAG. Storing a +Sum state. Its materialization assignment places it: an ingestion-time Sum state +finalizes Rate and builds Sum within a bounded precompute run; a query-time Sum +over an ingestion-time Rate state leaves the Rate readout and Sum in the query DAG. Storing a value requires its exact evaluation window, revision, readiness and serving cadence to match the query contract. @@ -109,7 +106,7 @@ Planner's candidate space decides what to compute, not placement. For an instant-vector PromQL TopK, Planner resolves rows that carry the complete series identity and lists the current-series heap realizations per root with the other candidates, unranked. Precompute or query-time placement of Rate and grouped Sum -is not a separate Planner candidate: the summary maintenance lifecycle assigns +is not a separate Planner candidate: the materialization assignment sets each node's timing, and the physical compiler reads it. This is the target ownership contract. A backend path that still reconstructs @@ -182,10 +179,10 @@ KLLMerge p50 p99 │ - │ Summary Maintenance Candidate Generation + │ Stage 2 Materialization (#509) ▼ -2. Summary Maintenance Lifecycle +2. Materialization KLLBuild(k=200) strategy = continuously maintain @@ -239,7 +236,7 @@ Each stage adds a different class of decision while preserving the preceding contracts. Here, continuous maintenance means recurring production of pane state; the bounded build DAG does not itself implement an unbounded streaming window. -## 2. Logical Post-ASAP DAG → Summary Maintenance Lifecycle +## 2. Logical Post-ASAP DAG → Materialization The **Logical Post-ASAP DAG** defines computation semantics: @@ -257,33 +254,13 @@ Quantile(.5) Quantile(.99) It establishes that KLL with `k=200` is used and that the merge is shared by the two readouts. It does not determine when KLL states are built or retained. -**Summary Maintenance Candidate Generation** enumerates legal lifecycle choices -using workload demand, window/freshness requirements and supported physical -implementations. Backend selection uses runtime feasibility and cost after -physical compilation. The following example follows one candidate. - -Candidate generation and selection are separate steps. For every unique retained -state, enumeration reports each lifecycle (ephemeral, prepared, shared, -continuously maintained) as legal, with a Planner cost or explicitly unknown -cost, or as rejected with a reason. Planner does not remove a legal alternative -because its own estimate prefers another. A deployment prices the legal -alternatives over the whole workload, counting shared state once, and binds one -lifecycle per state. Binding checks that the choice is legal and that states on -one maintenance path share an evaluation schedule. An alternative whose cost is -unknown can be bound only when the deployment's cost model is authoritative for -complete-candidate cost; unknown cost is never treated as zero. It then yields the same -lifecycle guarantee and window framework the physical compiler consumes when -Planner selects. Planner's own cheapest-alternative selection remains available -for callers without deployment pricing. The window framework is decided for the -complete combination, not for one alternative in isolation. - -A maintained population (for example, the current series of `topk by(job)(1, m)`) -is retained state like a summary. Retaining it maintains the latest sample per -series at ingestion and leaves only the readout at query time. Choosing -`Ephemeral` rebuilds that snapshot from raw samples for each query, so the -deployment must supply the raw source at query time. The caller's `CostModel` -prices both through the same lifecycle hooks; a model without population -evidence leaves them unknown, and they are not selected. +**Stage 2 Materialization (#509)** will choose when states are built and how +long they are retained, using workload demand, window/freshness requirements +and supported physical implementations. Backend selection uses runtime +feasibility and cost after physical compilation. Retained state includes +maintained populations (for example, the current series of +`topk by(job)(1, m)`), which keep the latest sample per series at ingestion and +leave only the readout at query time. For the running example, assume it selects: @@ -303,24 +280,24 @@ reuse: one merged state serves p50 and p99 ``` -This produces the **Summary Maintenance Lifecycle**. +This is the running example's **Materialization**. -The lifecycle specifies how the selected logical summary should be maintained, +It specifies how the selected logical summary should be maintained, but not its concrete operator implementation or storage location. Physical feasibility may feed back into selection. For example, if the required -pane-based maintenance cannot be implemented, this lifecycle candidate cannot be +pane-based maintenance cannot be implemented, this materialization cannot be selected. One-minute panes alone also cannot cover an arbitrarily phased query window; that requires supported boundary handling or a different candidate. -## 3. Summary Maintenance Lifecycle → Physical DAG +## 3. Materialization → Physical DAG The **Physical Plan Compiler** consumes both computation semantics and maintenance requirements: ```text Logical Post-ASAP DAG (PostAsapDAG) -+ Summary Maintenance Lifecycle ++ Materialization + physical capabilities ↓ Physical Plan Compiler @@ -328,14 +305,13 @@ Physical Plan Compiler Physical DAG(s) ``` -For the running example, the lifecycle creates two execution boundaries. +For the running example, materialization creates two execution boundaries. These two halves are named as `PhysicalASAPDAG` names them, `precompute` -and `query`. *Maintenance* stays the lifecycle's word (section 2): it covers +and `query`. *Maintenance* stays materialization's word (section 2): it covers how state is built, retained, reused and scheduled. A precompute DAG is the -physical object that a maintenance lifecycle compiles to, so reusing -*maintenance* for it collapses two layers that the crates keep apart: -`asap-aware-mapping::summary_maintenance_*` owns the lifecycle, and +physical object that maintenance compiles to, so reusing *maintenance* for it +collapses two layers: Stage 2 materialization (#509) owns maintenance, and `asap-physical-operators::physical_planner` owns the DAGs. ### Precompute Physical DAG @@ -397,17 +373,17 @@ DAG. If the required behavior cannot be realized, physical compilation fails. Materialization frontiers are Planner decisions. A candidate records both the precompute Physical DAG and the query Physical DAG, with typed outputs connecting them. The deployment compiler binds those outputs; it does not move operators. -Lifecycle timing gives the frontier: ingestion-time nodes read by query-time -nodes. For `sum by(job)(rate(m[1m]))`, the two lifecycle choices of the single +Execution timing gives the frontier: ingestion-time nodes read by query-time +nodes. For `sum by(job)(rate(m[1m]))`, two materialization choices for the single logical candidate give: ```text -Candidate A (Rate state retained, Sum Ephemeral): +Candidate A (Rate state at ingestion time, Sum at query time): precompute: counter samples → per-series Rate state materialized output: per-series Rate states for window/evaluation/revision query: stored Rate states → Rate readout → grouped Sum → result -Candidate B (Rate and Sum states retained): +Candidate B (Rate and Sum states at ingestion time): precompute: counter samples → per-series Rate → grouped Sum state materialized output: grouped Sum states for window/evaluation/revision query: stored grouped Sum states → Sum readout → result @@ -430,9 +406,9 @@ feasibility is rejected before pricing. The optimizer supplies candidate frontiers and cost evidence, including updates, retention, recurrence and sharing. `enumerate_frontiers` constructs bounded, reachable antichain frontiers above explicit input boundaries, including query-only and fully precomputed results. It fails explicitly when the candidate budget is exceeded. Maintenance selection must still reject frontiers that violate window, freshness, or reuse requirements; deployment feasibility is checked before pricing. -The lifecycle layer decides timing; physical compilation reads it. Lowering a +Materialization decides timing; physical compilation reads it. Lowering a node does not depend on the frontier, so each query DAG is lowered once and -different lifecycle assignments are different cuts of that lowering. +different materialization assignments are different cuts of that lowering. `compile(dag, inputs, roots)` yields the complete `CompiledPhysicalDAG`. `frontier_from_timing(&timed_dag)` reads an assignment's timed DAG (from `execution_timed_dag`) and returns its frontier: ingestion-time nodes read by @@ -448,7 +424,7 @@ pane candidates remain a separate lowering. Physical compilation opens no readers. Bounded precompute outputs become typed query inputs. Their source, filters, grouping, build window, evaluation time, readiness and -revision contracts must accompany the selected lifecycle and be checked during +revision contracts must accompany the selected materialization and be checked during deployment binding. Type compatibility alone does not establish reuse legality. The Planner integration test executes both candidates through the shared runtime @@ -463,7 +439,7 @@ The **Deployment Plan Compiler** binds the Physical DAGs to the concrete deploym ```text Physical DAGs -+ Summary Maintenance Lifecycle ++ Materialization + deployment catalog/state + sources/materializations + operational policy @@ -507,7 +483,7 @@ InputSlot[5 panes] ``` The Deployment Plan Compiler establishes bindings and checks that their contracts -satisfy the physical inputs and selected lifecycle, including KLL parameters, +satisfy the physical inputs and selected materialization, including KLL parameters, source, filters, grouping, window coverage and revision scope. The deployment engine resolves request-specific states and checks their actual coverage, revisions and readiness at execution time. A compiled plan cannot establish future readiness. @@ -523,8 +499,8 @@ The complete example makes the ownership boundary explicit: | Stage | KLL example decision | | --- | --- | | **Logical Post-ASAP DAG** | Use `KLL(k=200)` with shared merge for p50/p99 | -| **Summary Maintenance Candidate Generation** | Maintain 1-minute panes and reuse them for aligned five-minute queries | -| **Summary Maintenance Lifecycle** | Record pane/window/freshness/reuse requirements and each node's execution timing | +| **Stage 2 Materialization (#509)** | Maintain 1-minute panes and reuse them for aligned five-minute queries | +| **Materialization** | Record pane/window/freshness/reuse requirements and each node's execution timing | | **Physical Plan Compiler** | Lower to native KLL build, merge, and readout operators | | **Physical DAG** | Define precompute and query DAGs with typed input/output boundaries | | **Deployment Plan Compiler** | Bind raw input and KLL state slots to concrete sources/materializations | @@ -534,7 +510,7 @@ The complete example makes the ownership boundary explicit: Logical: "Use KLL for p50/p99." -Lifecycle: +Materialization: "Maintain reusable 1-minute KLL panes." Physical: @@ -564,17 +540,11 @@ snapshots as separate inputs. ## 6. Executable acceptance coverage -The tests cover optimizer-selected lifecycle execution alongside independent +The tests cover physical candidate execution alongside independent operator/runtime fixtures: | Test | Contract exercised | | --- | --- | -| `summary_maintenance_lifecycle_e2e::continuous_lifecycle_compiles_and_executes_spatial_kll` | PromQL workload → selected continuous lifecycle → logical DAG → compiled precompute/query candidate → results in independent revisions; an unbounded candidate fails before pricing, and a bounded request candidate summarizes the same input samples | -| `summary_maintenance_lifecycle_e2e::chosen_lifecycle_timing_decides_precompute_contents` | PromQL workload → enumerated lifecycles → explicit choice → timed DAG → compiled candidate; ContinuouslyMaintained stores the state in precompute, Ephemeral leaves precompute empty and reads the raw source at query time; both return the same p99 | -| `summary_maintenance_lifecycle_e2e::lifecycle_timing_cuts_one_compilation` | KLL quantile and grouped Rate→Sum: one compilation cut by the ContinuouslyMaintained and Ephemeral timed DAGs equals `compile_candidate` for each; the frontier is the retained state or empty | - -| `summary_maintenance_lifecycle_e2e::chosen_population_lifecycle_decides_precompute_contents` | PromQL `topk by(job)` over a maintained population → explicit choice → timed DAG → compiled candidate; ContinuouslyMaintained stores the population in precompute, Ephemeral rebuilds it from raw samples at query time; both rank alike | -| `summary_maintenance_lifecycle_e2e::planner_lifecycle_selection_reproduces_strategy_timing` | For PromQL summary fixtures, the timed DAG from Planner's retained selection equals the DAG realization strategies produce | | `kll_pane_execution::five_panes_roundtrip_and_shared_merge_runs_once` | Explicit one-minute precompute DAGs → real MessagePack state bytes → five required query inputs → shared native merge → p50/p99; counts every sample once, checks adjacent aligned windows and instruments one merge start per run | | `kll_pane_execution::restored_panes_reject_corruption_parameters_schema_and_missing_binding` | Corrupt bytes, parameter relabelling, incompatible schemas and absent bindings fail explicitly | | `precompute_candidates::grouped_rate_can_be_materialized_before_or_after_grouped_sum` | Cost changes select different legal precompute frontiers; both selected candidates execute with the same reset-sensitive result; uncompilable candidates are not priced | diff --git a/docs/design_docs/proposals/asap-aware-mapping/README.md b/docs/design_docs/proposals/asap-aware-mapping/README.md index a58d3b5dd..814d50134 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/README.md +++ b/docs/design_docs/proposals/asap-aware-mapping/README.md @@ -9,4 +9,3 @@ extensions. Each status note identifies the implemented scope and remaining work - [Shared maintained populations](maintained-populations.md) - [Optimization dimensions](optimizations.md) - [Summary properties](summary-properties.md) -- [Workload demand and summary lifecycle](workload-demand-and-summary-lifecycle.md) diff --git a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md index b742a8970..6e270159e 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md @@ -2,7 +2,7 @@ > Status: implemented model with explicit support limits. The > [analytical estimator](../../../../crates/asap-aware-mapping/src/analytical_cost.rs) -> and physical/streaming adapters implement supported evidenced comparisons. +> and physical-plan adapter implement supported evidenced comparisons. > Unsupported operators, arrival modes and missing evidence remain unavailable; > proposed extensions are not implied by the implemented formulas. @@ -17,23 +17,19 @@ plans. These are separate concerns: evidence to estimate CPU work, peak memory, and source/disk I/O. The physical-resource estimator itself is independent of the arrival mode. -Two planner adapters currently lower work into it. `PhysicalPlanCostModel` -compares complete at-rest plans. `SummaryMaintenanceCostModel` resolves -`DataArrival::ContinuouslyIngesting` over a finite horizon, including -bootstrap, arriving updates, retained state, and query readout. Evidence from -one arrival mode must not be reused for the other. `Mixed` and `Unknown` -remain unavailable until their distinct data regions are modeled. - -Both entry points replace dimensionless plan-node counts with estimates +One planner adapter currently lowers work into it: `PhysicalPlanCostModel` +compares complete at-rest plans. Continuously-ingesting, `Mixed` and `Unknown` +comparisons are unavailable; costing summary maintenance over arriving data +belongs to Stage 2 materialization (#509). Evidence from one arrival mode must +not be reused for another. + +The adapter replaces dimensionless plan-node counts with estimates derived from operator complexity, cardinality, row width, and concrete summary parameters. The estimates are predictions; they are not measurements reported by a physical executor. The model does not decide semantic legality. Ordinary summary guarantees are -composed before costing. A window framework that itself introduces error must, -however, carry a typed composed guarantee in the same complete evidence bundle; -the streaming adapter checks that guarantee against every bound workload -accuracy target before the candidate can be ranked. Missing evidence produces +composed before costing. Missing evidence produces an unavailable estimate, never an assumed zero or a structural-cost fallback. The implementation keeps five layers distinct: @@ -64,21 +60,12 @@ physical_operator_statistics.rs ──────┤ physical evidence contract ▼ analytical_cost.rs ─────────── operator formulas and CPU/memory/I/O composition │ - ├──────────────► physical_plan_cost_model.rs - │ at-rest raw/rewrite/summary comparison adapter - │ - └──────────────► summary_maintenance_cost/ - evidence.rs authoritative summary evidence - estimator.rs complete maintenance-DAG resources - window.rs window assignment and accuracy - model.rs lifecycle/alternative ranking adapter + └──────────────► physical_plan_cost_model.rs + at-rest raw/rewrite/summary comparison adapter ``` -Raw query plans and incrementally maintained summary plans share -`EvidenceBackedPhysicalDAG`; there is no streaming-only duplicate of the -physical DAG or operator-statistics contract. Summary-maintenance modules add -only the evidence and scheduling semantics that do not exist for an ordinary -query plan. +Raw query plans and summary plans share `EvidenceBackedPhysicalDAG`; there is +no separate physical DAG or operator-statistics contract for summaries. An estimate has physical dimensions: @@ -236,14 +223,14 @@ normalized workload, lowered query IR, and freshness-aware statistics: `DataWorkload` does define whether input is streaming: its `arrival` field is `AtRest`, `ContinuouslyIngesting`, `Mixed`, or `Unknown`, and a continuous arrival rate comes from fresh `ingestion_rate` evidence. These facts describe -how source data arrives. They do not choose a lifecycle or a window framework: +how source data arrives. They do not choose materialization or a window framework: `Incremental` describes how a selected summary state is updated, while tumbling, sliding, and exponential histogram describe how that state is organized over time. Evidence is read through `Evidence::value_at(planning_time)`. Stale, future, or improperly time-bounded evidence remains unknown. Costing follows -the same freshness rule as accuracy and lifecycle planning. +the same freshness rule as accuracy checking. The current workload schema does not yet contain every physical statistic. The missing facts have explicit ownership: @@ -306,7 +293,7 @@ physical operators and matching statistics variants. Until then, a candidate containing such an unlowered operation is unavailable rather than partially costed. -## Workload horizon and lifecycle +## Workload horizon Every alternative must cover the same source data and query horizon. The `DataArrival::AtRest` physical-DAG comparison is build-once, read-many: @@ -327,9 +314,7 @@ incremental updates are a one-time snapshot build. The at-rest summary alternative scans the selected source snapshot once and retains state. Its raw alternative recomputes from that snapshot for every -query read. The continuously-ingesting entry point separately charges -bootstrap, updates, summary operations, retained state, and raw evaluations; -its lifecycle rules are defined below. +query read. ### Comparable source and workload scope @@ -733,157 +718,14 @@ input. ## Summary operator formulas -### Incremental single-summary foundation - -For `DataArrival::ContinuouslyIngesting`, the incremental estimator accepts -one selected lifecycle and one unique logical `SummaryAgg`. This deliberately -narrow contract prevents one flat evidence record from being reused across -several summary nodes with different input cardinalities, algorithms, or state -sizes. Complete multi-node streaming alternatives require per-node physical -evidence. - -The canonical workload supplies fresh bootstrap cardinality, ingestion rate, -query recurrence, planning time, and a finite horizon. Physical evidence adds -logical/bootstrap bytes, physical bootstrap scan bytes, active and retained -window counts, the number of concrete summary-state instances per window, and -bytes per state instance. Names use `summary`, not `sketch`, because an exact -aggregate or another non-sketch state is equally valid. - -For bootstrap rows `B`, arrivals `U`, simultaneously updated windows `A`, -query evaluations `Q`, physical summary instances `P`, and state bytes `S`: - -```text -insert invocations = (B + U) × A -retained memory = (A + retained_windows) × P × S -``` - -Each input row is routed to its matching summary instance; it is not inserted -into every group. Merge, subtract, and readout work may operate over all `P` -instances. Delete work follows the same routed window updates rather than -multiplying every update by every possible group. - -An empty bootstrap is valid and has zero logical bytes and zero source reads. -A non-empty bootstrap requires positive logical and physical source bytes. -Active window count, summary-instance count, state width, horizon, and query -evaluation count must be positive; retained-window count may be zero for a new -stream. Required per-operation CPU evidence must be finite and positive. - -Lifecycle retention and the planning horizon are different quantities. A -short retained window may be maintained throughout a much longer planning -horizon, so the estimator does not require `retention >= horizon`. Lifecycle -legality and query time-coverage checks establish whether the retained window -can answer the query. - -### Comparing single-summary lifecycle alternatives - -For one logical `SummaryAgg`, the analytical lifecycle adapter converts the -same physical evidence into the existing lifecycle planner's five cost terms: - -| Lifecycle term | Resource basis | -|---|---| -| Initial build | Bootstrap rows routed to every bootstrap-active window, plus the bootstrap source read. | -| Maintenance per update | One arriving row routed to every currently active window. | -| Summary read | Readout of every physical summary instance needed by one query evaluation. | -| Retention rate | All active and retained state bytes calibrated over the finite comparison horizon. | -| Retirement | Zero only for releasing modeled memory; an actual delete, expiration, or rebuild requires explicit operation evidence. | - -The existing lifecycle model—not this adapter—enumerates `Ephemeral`, -`Prepared`, `Shared`, and `ContinuouslyMaintained`, checks workload and runtime -legality, and multiplies per-update and per-read terms by the normalized -workload rates. Missing any required term leaves that alternative unavailable. - -`Ephemeral` is a direct build, not incremental maintenance. For every query -evaluation, it rebuilds from the snapshot visible at that evaluation, charges -that evaluation's complete source read, and releases its state afterward. -Its state contributes to peak transient memory but not persistent retention. - -The raw side is supplied as a complete `ResourceEstimate` for one execution of -the raw physical DAG. The lifecycle planner applies the same recurrence and -horizon. This deliberately avoids reconstructing raw work with a special-case -`input_rows × cpu_per_row` formula that would omit joins, windows, sorts, or -other operators. - -Flat single-summary evidence is bound to the exact `SummaryNode` and raw -`QueryExpr` identities for which it was produced. It cannot be reused for a -structurally similar node or for multiple summary states. A complete -multi-summary `SummaryExpr` DAG requires per-node physical evidence and -physical-identity deduplication. - -### Complete bound streaming summary DAGs - -The multi-node streaming path accepts a complete, already-bound -`SummaryExpr` DAG. It does not guess physical implementations. The provider must -provide evidence for every reachable node: - -| Logical node | Required physical evidence | -|---|---| -| `KeepPreAsap` | One retained preprocessing operator with output edge, horizon CPU, workspace, and output buffer. | -| `SummaryAgg` | Input/output edges, insert CPU, concrete state count and width, bootstrap/update window fanout, and explicit source-read ownership. | -| `SummaryMerge` | Typed merge evidence with total CPU, workspace, output buffer, I/O, and execution multiplicity. | -| `SummarySubtract` | Typed subtract evidence with the same resource dimensions. | -| `SummaryDelete` | Typed delete evidence plus expiration/retraction rate, routing fanout, and the exact state owner. | -| `SummaryEstimate` | Typed readout evidence with total resource use per execution. | -| `SummaryJoin` | Ordered input/output edges and total physical join CPU, workspace, output buffer, I/O, and multiplicity. | - -The merge/subtract/delete/readout evidence is an enum structured by operation -kind. Delete-only rate and routing fields therefore cannot be attached to a -merge or readout. Join CPU is the total build, probe, match-production, and -output work of the selected algorithm; matched output pairs alone are not a -valid join cost. - -Every parent input edge must equal the corresponding child output edge. -Provider-owned `physical_id` values deduplicate a shared operator only when -its complete evidence and physical child identities also agree. The cost model -holds owning `Rc` references for bound target and summary roots, so pointer -keys cannot become stale and alias a later allocation. - -A `SummaryAgg` that reads storage declares `scan_selection_index = Some(i)`, -a non-empty bootstrap-read identity, and positive physical source bytes. An -aggregate over an already-materialized summary edge declares `None`, an empty -read identity, and zero source bytes. Its logical input rows and bytes remain -positive when the intermediate is non-empty. This prevents nested aggregates -from charging the original source scan repeatedly. - -For streaming raw recomputation, `planning_time_input_rows`, -`planning_time_input_bytes`, and `planning_time_source_scan_bytes` describe the -initial snapshot. Logical bytes per arriving row and physical source bytes per -arriving row are separate. The recurrence determines every evaluation offset; -the provider supplies one once-counted physical DAG whose statistics aggregate -those evolving evaluations over the complete horizon. Marking its nodes -`PerEvaluation` would multiply the already-aggregated evidence again and is -rejected. Validation follows only nodes reachable from the physical root. If -the raw algorithm intentionally reads the same semantic source more than once, -each reachable scan carries the same evolved source statistics and is charged -separately; equal scan selection does not deduplicate physical I/O. - -This raw-evolution contract currently supports exactly one distinct source -coverage. A multi-source streaming target is unavailable until per-source -arrival rates and widths are supplied. Target lineage includes ordinary -predicates and PromQL info selectors; extra, missing, or mismatched source -coverage makes both sides incomparable. - -Lifecycle enumeration considers only alternatives legal for the canonical -workload and runtime. A `Prepared` state must cover every scheduled evaluation -it serves. `Shared.retention` describes data/window coverage, not the planning -horizon, so a shorter retention value is not rejected merely because the -optimizer horizon is longer. Missing node evidence, zero required CPU, -unknown I/O, inconsistent edges, or an unsupported lifecycle combination -makes the complete candidate unavailable; partial per-state costs are never -used as a fallback. +Costing incremental maintenance of continuously-ingested summaries was removed +with the summary maintenance lifecycle; Stage 2 materialization (#509) will +define it. The formulas below give per-operation work and state size. ### Ranking complete physical implementations A logical summary candidate can be bound to more than one complete physical -implementation. Each alternative has a non-empty, provider-owned identity and -a complete `StreamingNodeEvidence` bundle. The planner evaluates every legal -lifecycle combination against every bound physical implementation over the -same `ComparisonScope`, excludes alternatives whose evidence is incomplete or -invalid, and returns both the least calibrated cost and its physical-plan -identity. Duplicate identities are rejected because they would make the -selection result ambiguous. If no explicit alternatives are registered, the -candidate's single canonical evidence bundle is used. - -Physical evidence is alternative-specific: window fanout, retained state, +implementation. Physical evidence is alternative-specific: window fanout, retained state, operation costs, and source reads must describe that implementation as a whole. The planner does not mix individual nodes from different alternatives. @@ -903,7 +745,7 @@ cpu_ops = bootstrap_rows × bootstrap_window_count × insert_ops(params) scan_bytes = source_read_bytes for the build ``` -Merge, subtract, and delete add their own invocation counts described above; +Merge, subtract, and delete add their own invocation counts; they are never folded into the simple formula implicitly. Concrete accuracy-sized parameters determine state and work: @@ -927,7 +769,7 @@ from logical group count alone. Summary merge, subtract, delete, and readout are separate physical operators. Their CPU and memory use the concrete summary state size and number of input states. A plan using one of these operations is unavailable until the -corresponding formula and required lifecycle evidence are present. +corresponding formula and required evidence are present. Summary construction uses physical-input realization rules before it emits a `SummaryAgg`. The default rule consumes the logical aggregate's immediate @@ -988,9 +830,7 @@ mismatches and arithmetic overflow also fail closed. ### Downstream physical-planning boundary This cost model consumes resource evidence for a physical implementation, but -ASAPPlanner does not own or select that implementation. It does select the -abstract per-summary `SummaryWindowFramework` assignment by comparing complete -`StreamingWindowFrameworkCandidate` evidence bundles. Component ownership, +ASAPPlanner does not own or select that implementation. Component ownership, including the distinction between a window primitive and its concrete runtime implementation, is defined in [ASAPPlanner planner-runtime contract](../../architecture/planner-runtime-contract.md). @@ -1064,7 +904,7 @@ candidate. The intended end-to-end selection pipeline is: 1. enumerates semantically valid alternatives; -2. checks end-to-end accuracy and lifecycle legality; +2. checks end-to-end accuracy; 3. derives fresh workload and operator statistics; 4. sizes physical summary parameters; 5. estimates the complete candidate DAG; @@ -1075,31 +915,8 @@ listed above. `PhysicalPlanCostModel` executes this pipeline for every candidate supplied to `CandidateLogicalASAPDAGs::global_selection`. Logical rewrites are lowered recursively. Summary candidates participate only after the deployment has bound their complete `SummaryExpr` DAG; there is no optimistic generic -summary fallback. The streaming adapter connects raw recomputation and -primitive summary lifecycle costs to the existing global lifecycle-selection -hooks. -The lifecycle planner enumerates compatible lifecycle combinations for the -unique `SummaryAgg` deployments and invokes -`complete_summary_candidate_estimate` -for each combination before selecting the minimum. The hook receives explicit -node-to-guarantee bindings plus the horizon and expected reads. Each logical -occurrence is looked up by exact `Rc` identity, while every -evidence record also carries a provider-owned physical identity. Equal physical -identities deduplicate work and retained state only when their logical summary, -selected window framework, operator facts, edge statistics, lifecycle -guarantee, and physical child identities agree; -conflicts make the candidate unavailable. Thus heterogeneous states are costed -independently and genuinely shared deployments once. Merge, subtract, delete, -readout, and join participate in automatic -candidate ranking. Exhaustive whole-root scoring is capped at 4,096 lifecycle -combinations because an arbitrary whole-candidate hook cannot be soundly -pruned by primitive costs; a larger space is unavailable rather than consuming -exponential planner time. If the root needs unavailable operation evidence, the -hook returns unavailable. Global selection then excludes that summary and -materialization retains the raw expression. A missing raw estimate also forces -raw fallback, because no public selection/materialization path may publish an -uncompared summary. The planner never falls back to the partial `SummaryAgg` -sum. +summary fallback. Choosing between a maintained summary and raw recomputation +belongs to Stage 2 materialization (#509). Before applying the following arithmetic, callers validate exact equality of the raw and selected alternative's `ComparisonScope`, and use the same diff --git a/docs/design_docs/proposals/asap-aware-mapping/ddsketch-quantile-ratios.md b/docs/design_docs/proposals/asap-aware-mapping/ddsketch-quantile-ratios.md index f71dde2ea..8bb452783 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/ddsketch-quantile-ratios.md +++ b/docs/design_docs/proposals/asap-aware-mapping/ddsketch-quantile-ratios.md @@ -14,7 +14,7 @@ The final guarantee records both input ranges and their contract identifiers. Th ## Candidate generation without evidence -The default `SketchAlgorithmStrategy` permits a direct DDSketch quantile-ratio +The default `ASAPStrategies` permits a direct DDSketch quantile-ratio candidate when domain evidence is absent, but leaves the root guarantee unset. This is useful for the v1 integration path; it does not turn missing evidence into evidence. Other approximate divisions still require their own composition diff --git a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md index 005670db8..c2b05588e 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md +++ b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md @@ -70,7 +70,7 @@ columns and multi-measure aggregates need additional rules. ```text KeepPreAsap(input) - -> MaintainPopulation { input, max_k, quantiles } [lifecycle-timed] + -> MaintainPopulation { input, max_k, quantiles } [ingestion time] -> ReadPopulation { Quantile(q1) } [read] -> ReadPopulation { Quantile(q2) } [read] -> ReadPopulation { TopK(k1) } [read] @@ -116,8 +116,7 @@ because their source names or numeric values happen to agree. ## Validation, selection and execution responsibilities Planner validates the declared input, the query-time readout and readout -compatibility; the population's lifecycle decides whether it is maintained at -ingestion or rebuilt per query. Its intended guarantee is exact membership and exact readout; +compatibility; `MaintainPopulation` always runs at ingestion time. Its intended guarantee is exact membership and exact readout; a physical implementation still must preserve the language's numeric and empty-input semantics. In particular, SQL global COUNT over an empty population returns a row with zero, while PromQL COUNT over an empty vector returns an empty vector. diff --git a/docs/design_docs/proposals/asap-aware-mapping/workload-demand-and-summary-lifecycle.md b/docs/design_docs/proposals/asap-aware-mapping/workload-demand-and-summary-lifecycle.md deleted file mode 100644 index 483e04503..000000000 --- a/docs/design_docs/proposals/asap-aware-mapping/workload-demand-and-summary-lifecycle.md +++ /dev/null @@ -1,758 +0,0 @@ -# Design: Query Workloads, Data Workloads, and Summary Lifecycle Maintenance - -> Status: partially implemented design. Workload types and -> [lifecycle planning APIs](../../../../crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs) -> implement the bounded planning path described below. The current-support and -> future-work sections distinguish available behavior from broader search, -> forecast integration and runtime deployment work. - -## Audience and context - -This document is for ASAPPlanner designers, architects, researchers, and -developers working on workload-aware plan selection. It defines how the -planner should describe query workload, data workload, and the lifecycle of -summary state. It is a design contract, not a description of the current -public Rust API. - -The terminology follows the ProjectASAP -[glossary](https://github.com/ProjectASAP/internal-docs/blob/03e1c70f5af3ae9221471898541067eee7f86338/glossary.md). -That glossary is authoritative for the meanings of data workload, query -workload, ad-hoc and predictable queries, one-time and repeated queries, -real-time and longitudinal queries, output cardinality, and lookback window. -This document maps those concepts into planner responsibilities and records -where the current model is incomplete. - -This design is orthogonal to -[end-to-end accuracy guarantees](end-to-end-accuracy-guarantees.md). Accuracy -decides whether a candidate is correct enough. Workload demand and state -lifecycle decide whether building, maintaining, sharing, or recomputing that -candidate is worthwhile. Neither decision may override the other. - -## Problem and why now - -A summary operator does not imply one execution lifecycle. The same exact or -approximate summary can be: - -- built once from data at rest and discarded after one query; -- prepared before a known future query and retired afterward; -- shared across a bounded set of requests; or -- maintained incrementally as data continues to arrive. - -Likewise, an exact stateless operator may run once over a batch, once per -update in an incremental pipeline, or once per readout. Operator statefulness, -execution schedule, and output representation are separate properties. - -The query expression alone cannot determine those properties. The same query -may arrive unexpectedly during exploration, run once at a scheduled time, or -repeat every ten seconds on a dashboard. Planning summary state from syntax -alone either misses reuse or invents reuse that the workload does not justify. - -The current `PlanningWorkload` separates query demand from data arrival. -Query entries include predictability, recurrence or invocation count, accuracy, -and time selection; `DataWorkload` contains arrival and empirical facts. -These fields do not themselves select a summary-maintenance lifecycle. -The [input/output/workflow design](../../architecture/input-output-workflow.md) -is authoritative for current fields, defaults, and public call sequences. - -## Inputs, outputs, and end-to-end behavior - -For the broader lifecycle design, four categories of information matter -(these are not four current top-level Rust fields): - -1. logical queries, which define query semantics; -2. query workload, including per-query accuracy and latency requirements, - predictability, recurrence, and queried time scope; -3. data-workload characteristics, including arrival, volume, cardinality, and - distribution; -4. existing summaries and the lifecycle actions available to the deployment. - -Candidate search outputs `CandidateLogicalASAPDAGs`. The implemented lifecycle-aware workflow -then returns a `SummaryMaintenanceLifecyclePlan` per query root, containing the -Post-ASAP DAG and maintenance decisions. It can choose exact raw recomputation -when summary maintenance does not beat raw cost or comparable costs are missing. A -state deployment states whether a summary is ephemeral, prepared, shared for a -bounded period, or continuously maintained. It retains costs, assumptions, and -structured rejection reasons. Exporting full input provenance remains a later -integration. - -```text - logical queries ---+ - query workload -----+ - data workload ---+--> candidate plans - available summaries ---+ -> semantic and accuracy legality - -> lifecycle alternatives - -> horizon-normalized cost - -> selected plan + deployments -``` - -For an unpredictable one-time query, the planner may read an existing summary, -build an ephemeral summary, or recompute from raw data. It must not assume -future reuse. For a predictable one-time query, it may additionally compare -preparing state in advance with building or recomputing at execution time. For -repeated queries, it may amortize build and maintenance cost across reads over -an explicit horizon. - -### End-to-end decision order - -```text -normalize query and data workloads - -> derive recurrence, time-scope, and data evidence - -> enumerate semantic plan alternatives - -> enumerate legal execution contracts and state lifecycles - -> validate summary capabilities and phase constraints - -> derive and check accuracy guarantees - -> normalize one-time and rate costs over an explicit horizon - -> rank legal alternatives and compare the selected summary deployment - with raw recomputation - -> emit plan, deployments, assumptions, and rejected alternatives -``` - -## Goals and non-goals - -### Goals - -- Represent glossary-defined query-workload and data-workload concepts without - collapsing independent axes into one enum. -- Separate an operator's statefulness from its execution schedule and the - lifecycle of the state it produces. -- Make unknown demand explicit and fail closed rather than treating it as zero - or infinite reuse. -- Compare one-time and rate-valued costs only through an explicit horizon. -- Explain why a selected plan builds, reuses, maintains, or avoids summary - state. -- Preserve a minimal path from the current batch/repeating workload and - recurrence profile to the proposed model. - -### Non-goals - -- Scheduling jobs, assigning machines, admission control, or executing queries. -- Predicting future query text inside ASAPPlanner. -- Defining a sketch runtime or state-storage protocol. -- Choosing a concrete forecasting algorithm for uncertain demand. -- Changing accuracy targets or guarantee algebra. - -## Heilmeier questions - -- **What are we trying to do?** Choose whether summary state should be built, - maintained, shared, reused, or avoided for different query workloads - and data workload. -- **How is it done today, and what are the limits?** The planner distinguishes - one-shot counts, fixed repeating intervals, and an ingest-rate proxy. It - cannot distinguish an unexpected exploratory query from a scheduled one-time - report, or data at rest from continuous ingestion as an explicit mode. -- **What is new, and why will it succeed?** Orthogonal workload axes and an - explicit state lifecycle let the existing recurrence formulas compare the - same summary under different deployment choices without changing query - semantics. -- **Who cares?** Users need predictable latency and cost; operators need to - know what state will exist and for how long; planner developers need demand - assumptions to be auditable. -- **What are the risks and costs?** More inputs can make planning harder to - configure, forecasts may be stale, and a large lifecycle search space can - increase planning cost. -- **What are the checks for success?** The acceptance cases below must produce - different lifecycle alternatives and cost terms for identical query syntax - under different workload contracts. - -## Proposed design - -### Authoritative concepts and ownership - -| Concept | Authoritative layer | Reason | -| --- | --- | --- | -| Query meaning | Pre-ASAP query IR | Workload metadata must not change semantics | -| Accuracy requirement | Query workload (per-query) | The required result fidelity may be explicit or supplied by the normalization default | -| Response-latency requirement | Query workload (per-query) | The optional end-to-end response-time bound belongs to one query execution | -| Query workload | Workload input | Arrival and recurrence are not inferable from syntax | -| Data workload | Workload input | Ingestion and distribution describe the data, not query workload | -| Summary capability | Summary properties | Merge, delete, and update support constrain legal lifecycles | -| State lifecycle | Physical planning decision | Lifecycle is selected, not declared by `SummaryAgg` | -| Cost | Cost model and explanation | Cost consumes all inputs but does not define their meaning | - -### Query workload - -Accuracy and latency are separate per-query requirements within the query -workload. They constrain different planner decisions and must not be collapsed -into one SLA value: - -```rust -enum AccuracyRequirement { - /// The caller supplied the required result fidelity. - Explicit(AccuracyTarget), - /// The source omitted accuracy; normalization applies the exact default. - ImplicitExact, -} - -enum LatencyRequirement { - /// Maximum permitted end-to-end latency for one query execution. - ExplicitMax(Duration), - /// The caller supplied no latency bound. - Unspecified, -} - -struct QueryRequirements { - accuracy: AccuracyRequirement, - response_latency: LatencyRequirement, -} -``` - -An omitted accuracy field is not an unknown accuracy target and does not permit -arbitrary approximation: the current normalization policy makes it -`ImplicitExact`. Keeping that variant distinct from `Explicit(Exact)` preserves -whether the caller chose exactness or inherited the default. An unspecified -response-latency requirement imposes no response-time constraint; it is not a -zero-duration bound or evidence that every latency is acceptable. Accuracy is -checked as a legality constraint. The normalized model preserves response -latency, but the current planner does not yet reject plans against that bound. - -#### Classification axes - -The glossary classifications must be modeled independently. - -##### Predictability - -```rust -enum Predictability { - /// The query shape is not known before arrival. - AdHoc, - /// The query or parameterized template is known before execution. - Predictable { - known_at: Option, - }, - /// The caller supplied no reliable classification. - Unknown, -} -``` - -`AdHoc` does not mean repeated or one-time. It means the query shape was not -known in advance. The glossary currently places exploratory/ad-hoc queries in -the one-time category, so the MVP should accept `AdHoc + OneTime` and reserve -other combinations until a concrete use case establishes their semantics. - -##### Recurrence - -```rust -enum QueryRecurrence { - OneTime { - invocations: u64, - execute_at: Option, - }, - Repeated { - demand: RepeatedDemand, - }, - Unknown, -} - -enum RepeatedDemand { - FixedInterval(Duration), - Scheduled(Vec), - EstimatedRate(DemandEstimate), -} - -struct DemandEstimate { - /// Time range over which the demand was measured or forecast. - observation_window: ObservationWindow, - /// Expected demand, expressed in exactly one form. - expected: ExpectedDemand, - /// Highest expected invocation rate within the observation window. - peak_rate: Option, - /// Highest expected number of simultaneously executing invocations. - max_concurrency: Option, - /// Confidence in this estimate, in the inclusive range [0.0, 1.0]. - confidence: Confidence, - source: EvidenceSource, - observed_at: Option, - valid_for: Option, -} - -enum ExpectedDemand { - /// Expected total invocations over `observation_window`. - InvocationCount(u64), - /// Expected average invocations per second over `observation_window`. - AverageRate(Rate), -} - -struct ObservationWindow { - start: Timestamp, - end: Timestamp, -} - -struct Confidence(f64); -``` - -One-time means no recurrence is expected for that workload entry. Several -one-time consumers may still share a subplan within a submitted workload. -Repeated means the same query expression over its selected data is evaluated -over time, matching the glossary. Parameterized templates require an explicit -equivalence policy before their executions count as the same query. - -Query-workload volume is more than an average rate. Cost and latency can differ -for the same total request count when requests arrive in bursts or concurrently. -`ExpectedDemand` makes invocation count and average rate alternative -representations, preventing conflicting values in one estimate. The observation -window must be non-empty, rates must be finite and non-negative, and -`Confidence` must be between zero and one. Fixed intervals and explicit -schedules are declarations rather than estimates and do not need fabricated -confidence. The MVP may cost only invocation count and evaluation rate, but it -must preserve unsupported volume characteristics for explanation rather than -silently discarding them. - -##### Queried time scope - -```rust -enum QueryTimeScope { - RealTime, - Longitudinal, - Mixed, - Unknown, -} -``` - -`QueryTimeScope` is not a response-latency requirement. It classifies the event -time of the data selected by the query; `LatencyRequirement` constrains the -wall-clock time allowed to produce the result. They are independent: a -longitudinal query over archived data may require a 100 ms response, while a -real-time query over the latest data may permit a 30 second response. - -This classification is not derived only from a numeric lookback. A five-minute -lookback over recent data is real-time; the same duration over archived data is -not. Planning input should therefore carry the classification and the concrete -time selection separately: - -```rust -struct TimeSelection { - scope: QueryTimeScope, - lookback: Option, - as_of: Option, -} -``` - -For example, the same five-minute lookback has a different scope depending on -whether it is anchored at the current planning time or at a historical time: - -```rust -// The last five minutes: real-time. -TimeSelection { - scope: QueryTimeScope::RealTime, - lookback: Some(Duration::minutes(5)), - as_of: None, -} - -// A five-minute interval from archived data: longitudinal. -TimeSelection { - scope: QueryTimeScope::Longitudinal, - lookback: Some(Duration::minutes(5)), - as_of: Some(timestamp!("2024-01-01T12:05:00Z")), -} -``` - -`lookback` is a query property already represented by temporal query nodes in -some frontends. The normalized workload should reference or derive it rather -than introduce a second conflicting value. - -### Data workload is separate from query workload - -```rust -enum DataArrival { - AtRest, - ContinuouslyIngesting, - Mixed, - Unknown, -} - -/// Statistical distribution of keys in the input data. -enum DataDistribution { - /// A small number of keys account for most observations. - Zipf, - /// Keys are approximately equally likely. - Uniform, - /// Observations arrive in bursts with a temporarily concentrated key set. - Bursty, -} - -struct DataWorkload { - arrival: DataArrival, - ingestion_volume: Evidence, - ingestion_rate: Evidence, - input_cardinality: Evidence, - distribution: Evidence, -} -``` - -`DataDistribution` reuses the existing ASAPPlanner classification. It describes -the key-frequency shape used by summary accuracy and cost models, not whether -data arrives continuously. An unavailable or unsupported distribution is -represented by `Evidence.value = None` rather than by assuming the default -distribution. - -The former `DataCharacteristics` was a stale, continuous-ingestion-specific -case built around series count and samples per second. `DataWorkload` replaces -it as the normalized input rather than embedding that special case in the -general model. Data at rest may have row count and scan statistics without a -nonzero ingestion rate. Unknown arrival must not be interpreted as continuously -ingesting or at rest. - -Every empirical value uses an evidence wrapper conceptually containing: - -```rust -struct Evidence { - value: Option, - source: EvidenceSource, - observed_at: Option, - valid_for: Option, -} -``` - -This reuses the provenance and freshness principles from empirical summary -parameter configuration. Missing, stale, or future-dated evidence remains -unknown. - -### Output cardinality is a derived or evidenced cost input - -Output cardinality depends on input cardinality and grouping columns. The -planner may derive it analytically, accept a catalog estimate, or leave it -unknown. The source and freshness metadata must be preserved because output -cardinality affects summary size, read cost, post-processing cost, and network -cost. It is not a query correctness requirement. - -### Separate operator state, schedule, and output - -The physical design must not use `SummaryAgg` as shorthand for incremental -maintenance. - -```rust -enum OperatorState { - Stateless, - Stateful { - mergeable: bool, - deletable: bool, - }, -} - -enum EvaluationSchedule { - OneShot, - PerUpdate, - OnRead, -} - -enum OutputRepresentation { - PlainRows, - SummaryState, - FinalizedValue, -} -``` - -A one-shot sketch builder is stateful while it consumes its input, but it does -not imply long-lived incremental maintenance. A stateless transform can run -`PerUpdate` before a downstream maintained summary. These types describe an -execution contract; they do not replace semantic operators in the post-ASAP IR. - -### State lifecycle is a plan alternative - -```rust -enum StateLifecycle { - Ephemeral, - Prepared { - activate_at: Timestamp, - retire_at: Timestamp, - }, - Shared { - retention: Duration, - }, - ContinuouslyMaintained, -} -``` - -- `Ephemeral` builds state for one submitted workload and discards it afterward. -- `Prepared` builds or begins maintaining state before a predictable query and - retires it after the known need ends. -- `Shared` retains state for multiple consumers over a bounded lifetime. -- `ContinuouslyMaintained` applies data updates until an explicit later - deployment decision retires the state. - -The summary family and its properties constrain which lifecycles are legal. -For example, an append-only sketch may support continuous inserts but not a -sliding-window lifecycle requiring deletion. Lifecycle legality is checked -before cost ranking, like accuracy legality. Deployments provide these -per-summary properties through `summary_lifecycle_capabilities`; moving -real-time windows require deletion support as well as incremental updates. - -### Existing summaries are planning input - -An ad-hoc query cannot justify creating permanent state from unknown future -demand, but it may use compatible state that already exists. The planning -problem therefore needs a state catalog describing identity, parameters, -coverage, freshness, accuracy guarantee, lifecycle, and ownership. Catalog -integration is a separate implementation increment; this design only requires -that "reuse existing" and "create new" remain distinguishable alternatives. - -### Cost over a horizon - -`H` is the optimization horizon: the future wall-clock duration over which the -planner compares one-time and recurring costs. The existing cost model -represents it in seconds: - -```rust -/// A finite, strictly positive optimization duration, in seconds. -struct Horizon(f64); -``` - -The horizon is not the query lookback, the queried time scope, or the response -latency bound. It answers only "over how much future execution time should -these alternatives be costed?" All alternatives in one decision must use the -same `H`. `reads(H)` is the number of query evaluations expected or scheduled -within that horizon; for a fixed evaluation rate it is -`H * evaluation_rate`, plus any separately modeled one-time invocations. -Who supplies `H`, and whether a deployment may default it, remains an explicit -architecture decision below. If no horizon is available, the planner must not -compare a one-time cost with a rate-valued cost. - -For a stateful incremental alternative over horizon `H`: - -```text -total(H) = build_cost - + H * update_rate * maintenance_cost_per_update - + reads(H) * summary_read_cost - + H * retention_cost_rate - + retirement_cost -``` - -For repeated raw recomputation: - -```text -total(H) = reads(H) * raw_recompute_cost -``` - -The current lifecycle-aware materialization sums the selected summary -deployments and can replace that plan with raw recomputation when the raw cost -is lower or the summary lifecycle is uncostable. Jointly reconsidering every -sibling semantic candidate under lifecycle costs remains a later optimizer -integration; this document does not claim that broader search is implemented. - -For an ephemeral summary: - -```text -total = invocations * (build_cost + summary_read_cost + retirement_cost) -``` - -`retirement_cost` consistently means the one-time cost of ending a summary -state lifecycle, including deallocation or other cleanup. For ephemeral state, -retirement happens immediately after each invocation; for prepared, shared, or -continuously maintained state, it happens when that deployment is retired. - -For prepared state, update and retention terms apply only between activation -and retirement. Existing state does not pay a new build cost, but its catalog -provenance must establish that assumption. - -The existing `Cost`, `CostRate`, `EvaluationRate`, `UpdateRate`, `Horizon`, and -`total_cost` types are the minimum viable foundation. The implementation should -extend their explanations and lifecycle coverage instead of creating a second -recurrence cost system. - -### Unknown and uncertain demand - -Unknown demand is not zero demand and is not evidence of future reuse. The MVP -policy is: - -- do not select newly created long-lived state solely on unknown future reuse; -- allow raw recomputation, ephemeral build, and reuse of already available - compatible state; -- retain an explicit explanation of the missing demand evidence; and -- require an explicit planning objective before using an estimated demand - distribution. - -Future uncertain-demand support may add expected-cost, percentile-cost, -worst-case, or regret objectives. Those policies must consume a typed estimate -with confidence and provenance; they are not implicit behavior of -`Predictability::Unknown`. - -## Review against the ProjectASAP glossary - -The glossary review found the following required coverage and current gaps. - -| Glossary concept | Current ASAPPlanner representation | Missing design support | -| --- | --- | --- | -| Data at rest vs continuously ingesting | `DataArrival` is explicit | Runtime/catalog-specific arrival discovery remains external | -| Ingestion volume | `DataWorkload::ingestion_volume` carries evidence | A concrete time basis for volume remains deployment-specific | -| Ingestion rate | Evidenced independently from query evaluation rate | Preserve richer unit/provenance metadata when integrations require it | -| Input cardinality | Evidenced workload-level cardinality feeds accuracy | Per-dataset/metric/column scoping remains future work | -| Data distribution | Evidenced built-in enum | Permit deployment-specific distributions later | -| Ad-hoc vs predictable | `Predictability` is independent from recurrence | Parameterized-template equivalence remains open | -| One-time vs repeated | One-time, fixed, scheduled, estimated, and unknown recurrence | Forecast-policy integration remains future work | -| Query volume and characteristics | Estimates preserve average/count, peak, concurrency, confidence, and freshness | Peak and concurrency are not yet consumed by cost or latency models | -| Real-time vs longitudinal | `TimeSelection` carries scope, lookback, and `as_of` | Conflict policy with temporal IR remains open | -| Output cardinality | May be inferred locally; no common evidenced input | Add derived/evidenced value and provenance for costing | -| Lookback window | Represented in temporal query shapes/frontends | Establish query IR as authority and expose it to workload costing | -| CTSA pipeline | Not explicitly modeled | Keep as architectural context; planner consumes collect/store/analyze facts but does not model transmission topology in the MVP | -| CSP(F) | Cost and fidelity partly modeled | Treat scale/performance/fidelity as objectives and constraints; do not collapse fidelity into cost | - -Two terminology constraints apply: - -1. A repeated query is not inherently a streaming-data workload. It may - repeatedly query data at rest. -2. A one-time query is not inherently stateless. A predictable one-time query - may justify prepared state, while an ephemeral summary is stateful during - its one execution. - -## Minimal complexity - -The minimum input model is determined by the downstream applications selected -for integration, not by a context-free notion of the fewest possible fields. -Each supported use case must contribute the workload facts that can change -plan legality, accuracy, lifecycle, or cost: - -- Time-series metric queries require queried time scope and lookback. -- Repeated dashboard queries, including an ASAPQuery integration, require - recurrence and evaluation frequency so the planner can cost reuse and - maintenance across executions. -- Batch queries over data at rest require an explicit at-rest arrival mode and - must not be assigned a fabricated ingestion rate. -- Summary techniques whose accuracy depends on the input distribution require - evidenced distribution characteristics; omitting them must produce unknown - accuracy or a conservative fallback rather than a favorable assumption. - -The initial implementation should include the union of fields required by its -committed integrations. Additional workload dimensions should be added when a -new downstream use case demonstrates that they affect a planning decision. - -The simplest alternative is to extend `BatchEntry` with optional schedule and -classification fields and extend `RepeatingEntry` with time scope. That is a -reasonable serialization migration, but it is not a sufficient conceptual -model: it continues to make predictability and recurrence mutually exclusive -container choices, and it has no place for data arrival or state lifecycle. - -The minimum new conceptual layers are therefore: - -1. orthogonal query-demand metadata, required because glossary categories are - not one taxonomy; -2. data-workload metadata, required because ingestion does not describe query - recurrence; -3. state lifecycle as a physical alternative, required because one summary - operator can be deployed ephemerally or incrementally. - -No separate scheduler, forecasting framework, or replacement cost model is -introduced. Existing query IR, summary properties, accuracy model, and -recurrence cost types remain authoritative in their domains. - -## Alternatives and decisions - -### Encode workload class as one enum - -Rejected. Variants such as `AdHoc`, `OneShot`, and `Repeated` overlap: -predictability and recurrence are different facts, and time scope is a third. - -### Infer demand from query syntax or submitted root count - -Rejected. Syntax contains no evidence of future arrival, and several roots in -one request establish only current structural sharing. - -### Treat every summary as continuously maintained - -Rejected. It excludes ephemeral construction over data at rest and overcharges -one-time plans. It also hides deployment lifetime from explanations. - -### Treat every one-time query as raw recomputation - -Rejected. An ephemeral summary may reduce memory or network cost during one -execution, an existing summary may already answer the query, and a predictable -future query may justify preparation. - -### Fold fidelity into a scalar cost - -Rejected. Accuracy and semantic correctness are constraints checked before -ranking. A cheaper plan cannot purchase permission to violate fidelity. - -### Extend the existing recurrence profile only - -Partially accepted for implementation reuse, rejected as the whole model. -`RecurrenceProfile` is an aggregated cost context for a target. It should remain -the derived input to cost decisions, while normalized workload metadata retains -predictability, time scope, provenance, and lifecycle information needed before -and after aggregation. - -## Quality attributes and evidence - -- **Understandability:** explanations use glossary terms and show each axis - separately. Proxy: reviewers can distinguish repeated queries from continuous - ingestion in exported plan evidence. -- **Debuggability:** selected and rejected lifecycle alternatives record costs, - horizon-derived decisions, assumptions, and typed rejection reasons. Full - demand/data provenance in exported explanations remains future work. -- **Maintainability:** current recurrence types remain the cost authority; - normalized workload types remain the source authority. No duplicate formula - system is introduced. -- **Extensibility:** scheduled and estimated recurrence fit without changing - query semantics. Forecasting policies remain pluggable planning objectives. -- **Performance:** lifecycle enumeration expands the candidate space. The MVP - should generate only capability-compatible alternatives and deduplicate - equivalent deployments before ranking. -- **Operability:** every long-lived state has activation, retention or retirement - semantics and ownership in output. Concrete runtime APIs are future work. -- **Security and privacy:** query logs and empirical distributions may be - sensitive. Provenance must identify a source without requiring raw query-log - contents to be embedded in exported plans. - -## Acceptance and test design - -Realization acceptance is defined by identical logical queries producing -different legal lifecycle choices under different workload contracts: - -1. **Unpredictable one-time query:** offers raw recomputation, compatible - existing state, and ephemeral build; does not justify new continuous state. -2. **Predictable scheduled one-time query:** may offer prepared state with a - bounded activation and retirement period. -3. **Repeated query over continuously ingesting data:** compares incremental - maintenance and repeated recomputation using distinct update and evaluation - rates over an explicit horizon. -4. **Repeated query over data at rest:** uses evaluation rate without inventing - maintenance updates. -5. **Real-time and longitudinal queries with the same expression:** preserve - different time selections and may receive different scan, retention, and - summary alternatives. -6. **Unknown demand:** remains unknown in explanation and cannot make a newly - created long-lived state win through assumed reuse. -7. **Mixed one-time and repeated consumers:** requires an explicit horizon and - accounts for shared build cost once. -8. **Accuracy failure:** rejects a lifecycle regardless of favorable workload - cost. - -Focused unit tests should cover normalization, invalid combinations, evidence -freshness, lifecycle capability checks, and dimensional cost arithmetic. -End-to-end tests should cover cases 1–8 through candidate selection and exported -explanations. A reviewer who did not implement the workload types should design -or review at least the unknown-demand and mixed-consumer cases; that independent -review has not occurred for this design document. - -## Risks, rollout, and exit criteria - -The implementation should roll out additively: - -1. add normalized metadata and explanations while preserving current - batch/repeating behavior; -2. derive the existing `RecurrenceProfile` from the richer model; -3. add ephemeral and existing-state alternatives; -4. add prepared and continuously maintained lifecycle selection; -5. integrate empirical demand and state catalogs only when provenance and - freshness contracts are available. - -Compatibility requires old workloads to normalize without changing their -current decisions when no new metadata is supplied. Unknown new fields must -take the documented conservative path rather than acquire optimistic defaults. - -Open decisions requiring architecture or product input: - -- whether predictable parameterized query templates count as the same repeated - query and under which equivalence relation; -- who supplies the optimization horizon and whether a deployment may define a - default for purely repeated workloads; -- which planning objective governs uncertain demand; -- how state ownership, quota, and retirement requests cross the planner/runtime - boundary; -- whether real-time versus longitudinal is supplied by the caller, derived by a - policy using `as_of` and lookback, or both with conflict diagnostics; and -- the minimum evidence freshness required before empirical workload data may - affect selection. - -The design exits draft status when these decisions have owners, the normalized -input has a compatibility plan, and acceptance cases 1–8 can be expressed in -fixtures without runtime-specific assumptions. diff --git a/docs/design_docs/proposals/asapquery-rule-coverage.md b/docs/design_docs/proposals/asapquery-rule-coverage.md index e496c9fe9..9ba999292 100644 --- a/docs/design_docs/proposals/asapquery-rule-coverage.md +++ b/docs/design_docs/proposals/asapquery-rule-coverage.md @@ -21,11 +21,11 @@ cost, and selection rules under `optimizer/`. The reviewed source is | Temporal aggregate functions | Lowering covered; realization varies | `Aggregate(PerEntity)` over `TimeRange` represents the full family. Sum, count, min, max, quantile, rate, and increase have summary realizations; `avg_over_time` is currently exact `PassThrough`, matching ASAPQuery's exact-only multi-stat fallback rather than claiming a maintained summary. | | Spatial aggregate functions | Lowering covered; realization varies | `Aggregate(Reduce(GroupKeys))` is shared by SQL and PromQL. Supported single accumulators and ordinary `by(...)` avg rewrites generate candidates; shapes such as `avg without(...)` retain the same exact raw fallback that ASAPQuery uses for multi-stat AQEs. | | Collapsible temporal + spatial aggregates | Semantic-equivalent rewriting | The existing rewrite strategy uses accumulator algebra: sum∘sum, sum∘count, min∘min, and max∘max. It rejects all other pairs and requires identical output schemas. | -| Sketch alternatives and exact fallback | Covered more generally | `SketchAlgorithmStrategy` enumerates legal summary realizations. The enclosing memo group always retains the original raw expression as the exact fallback; the strategy does not falsely label an approximate sketch as exact. | +| Sketch alternatives and exact fallback | Covered more generally | `ASAPStrategies` enumerates legal summary realizations. The enclosing memo group always retains the original raw expression as the exact fallback; the strategy does not falsely label an approximate sketch as exact. | | Subpopulation label placement | Covered more generally | `HydraGroupingStrategy` and `GroupingStrategy` express per-subpopulation and shared multi-subpopulation realizations. | | Shared computation | Covered more generally | workload-wide CSE and `SharedSubDAGStrategy` operate on physical DAG identity rather than AQE names. | | Average decomposition | Semantic-equivalent rewriting | The same rewrite strategy exposes independently optimizable sum/count accumulators when null semantics and schema permit it. | -| Merge/delete legality | Covered | Summary-family capabilities and lifecycle validation determine which maintenance operations are legal. | +| Merge/delete legality | Covered | Summary-family capabilities determine which maintenance operations are legal. | | Window-framework selection | Separate physical-planning work | Window selection must compare an extensible set of implementations, including tumbling, sliding, PromSketch-style exponential-histogram windows, and other window frameworks. This audit does not introduce a closed window enum or choose among them. | | Retention/cleanup scheduling | Outside planner scope | The audit deliberately does not import ASAPQuery's Arroyo-specific cleanup thresholds, timers, or failure workarounds. The planner may declare a selected summary's required retention horizon and cost it, but the runtime/storage layer owns when and how expired physical state is reclaimed. | | Empirical per-sketch atomic costs | Covered through evidence | Analytical statistics and deployment profiles provide cost evidence; benchmark tables should be ingested as calibrated evidence rather than compiled into matching rules. | @@ -38,22 +38,22 @@ does not create a new strategy category. | Decision | Existing owner | |---|---| -| Which summary algorithm can implement one aggregate intent | `SketchAlgorithmStrategy` | +| Which summary algorithm can implement one aggregate intent | `ASAPStrategies` | | How grouping/subpopulation state is laid out | `HydraGroupingStrategy` | | Whether an equivalent logical expression exposes better accumulators | `SemanticEquivalentRewriteStrategy` (the broadened existing avg rewrite; `AvgToSumOverCountStrategy` remains a compatibility name) | | Whether identical physical work is shared | `SharedSubDAGStrategy` | | Whether a finer grouping can answer a coarser grouping | `RollupStrategy` | | Whether tighter accuracy can answer a looser request | `AccuracyReconciliationStrategy` | | Whether a larger Top-K result can answer a smaller limit | `TopKLimitReuseStrategy` | -| Which maintenance lifecycle is legal | the summary-maintenance lifecycle planner | +| Whether and when a summary is materialized | Stage 2 materialization (#509) | | Which window framework implements a range | the physical deployment/window-selection planner | | How expired physical state is cleaned up | runtime/storage lifecycle management, not a planner strategy | Accordingly, ASAPQuery's four collapsible temporal/spatial patterns extend the existing semantic-rewrite owner. Temporal and spatial function recognition is already front-end lowering into `AggIntent`; sketch compatibility remains in -`SketchAlgorithmStrategy`; labels remain in `HydraGroupingStrategy`; and -maintenance lifecycle legality remains in the lifecycle planner. Window +`ASAPStrategies`; labels remain in `HydraGroupingStrategy`; and +materialization decisions belong to Stage 2 materialization (#509). Window framework selection is separate physical-planning work. None of these become a parallel syntax-oriented `PatternStrategy`. @@ -69,7 +69,7 @@ planner. Rules match typed operators and declared capabilities, never parser spellings. A rule that composes operators states the algebraic law it relies on and preserves the original output schema. Unknown pairs, missing statistics, or -unsupported lifecycle operations produce no candidate; they never silently +unsupported maintenance operations produce no candidate; they never silently fall back to an optimistic estimate. Window choices should follow the same principle without assuming that every diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 5273bfb3f..aad13359a 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -254,7 +254,7 @@ A summary-based candidate uses three kinds of summary nodes: summaries into a coarser one. * A **summary estimation node** computes an answer from a summary, for example the p99 estimate from a KLL, or the entropy estimate from a UnivMon. -* **summary subtract node** and **summary delete node** design is TODO. +* **summary subtract node** and **summary delete node** design is TODO. One summary build node can feed several estimation nodes, which is what Pass 2 exploits. diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index aee8fe53a..eaf3d879a 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -55,18 +55,20 @@ The diagram below follows a workload of one or more query roots through target d Terminology used in the diagram: - A **workload** is the set of named queries planned together. A **query root** - is the top-level `QueryExpr` (the logical query-expression type) for one of - those queries. **Pre-ASAP** means this logical input form, before the planner - realizes an operation as a concrete ASAP realization; **post-ASAP** means - the resulting realization form. + is the top-level `Rc` (the unified operator IR) for one of + those queries. **Pre-ASAP** means a DAG that contains only ordinary + `NonASAPOp` operators, before the planner realizes an operation with ASAP + primitives; **post-ASAP** means the same IR after some nodes became `ASAPOp` + summary operators. - A **DAG** (directed acyclic graph) represents query operators whose sub-DAGs may be shared. See [sub-DAG sharing and ASAP-aware CSE](../design_docs/proposals/planner-layering.md#pass-2-asap-aware-common-subexpression-elimination) for the sharing rules. Rust's `Rc` (reference-counted pointer) records shared node identity. - A **target** is one replaceable site. A **candidate** is one valid alternative - for it. `Replacement::Summary` is a constructed post-ASAP summary—maintained state - such as an exact accumulator or an approximate sketch—while - `Replacement::Rewrite` is another pre-ASAP logical expression. + for it. `Replacement::SubDAG` is a replacement sub-DAG: either a constructed + post-ASAP summary (it contains an `ASAPOp`, e.g. an exact accumulator or an + approximate sketch) or a logical rewrite with no ASAP operator + (`is_logical_rewrite` tells them apart). `Replacement::ExactComposition` refers to a child target whose realization must remain undecided until compatible selection. A **sketch** is a compact data structure that trades exactness for bounded error. A @@ -86,15 +88,15 @@ flowchart TB classDef report fill:#f2eafe,stroke:#7950b3,color:#34204f subgraph DISCOVERY[1. Discover every replaceable site] - WL["Input workload
one or more named pre-ASAP QueryExpr roots"]:::input + WL["Input workload
one or more named pre-ASAP OperatorNode roots"]:::input SEARCH["search_workload_with
run CSE once, then visit every node in every root DAG"]:::generate - TARGET["TargetSubDAG
one candidate site plus the number of workload locations
that reference the same Rc<QueryExpr>"]:::generate + TARGET["TargetSubDAG
one candidate site plus the number of workload locations
that reference the same Rc<OperatorNode>"]:::generate WL -->|"roots"| SEARCH -->|"one target per distinct node"| TARGET end subgraph GENERATION[2. Generate all legal alternatives at each site] STRATEGY["ReplacementStrategy
when a target matches, enumerate every legal replacement;
implementations generate but do not choose"]:::generate - CAND["ReplacementSubDAG candidates
each contains a Summary, Rewrite or ExactComposition
plus typed provenance and rationale;
no alternative is removed solely on cost"]:::store + CAND["ReplacementSubDAG candidates
each contains a Subtree (summary or logical rewrite) or ExactComposition
plus typed provenance and rationale;
no alternative is removed solely on cost"]:::store TARGET -->|"try every registered strategy"| STRATEGY --> CAND CM(["CostModel
orders candidates and supplies
deployment-specific parameters"]):::choose CM -. "rank and parameterize; accuracy checks remain required" .-> STRATEGY @@ -157,7 +159,7 @@ flowchart LR classDef workload fill:#e7f7ef,stroke:#31835e,color:#173f2d classDef common fill:#fff6dd,stroke:#b78922,color:#513d0c - ROOTS["Input
one or more named QueryExpr roots"]:::workload + ROOTS["Input
one or more named OperatorNode roots"]:::workload ROOTS --> CSE["Canonicalize sharing
merge structurally identical, legally shareable sub-DAGs"]:::workload CSE --> WALK["Discover sites
walk the complete DAG, including nodes below unshared parents"]:::workload WALK --> T["Build TargetSubDAG
retain the sub-DAG's Rc identity and measured consumer_count"]:::workload @@ -199,8 +201,8 @@ cost. The default context-free registry contains five `ReplacementStrategy` implementations: -- `SketchAlgorithmStrategy` matches supported aggregate and binary shapes. Its - `replacements(target)` method constructs every legal post-ASAP `SummaryNode`, +- `ASAPStrategies` matches supported aggregate and binary shapes. Its + `replacements(target)` method constructs every legal post-ASAP summary sub-DAG, including applicable sketch, exact-accumulator, and pass-through realizations. Candidates are sized and ordered for the target's accuracy requirement; candidates without a sufficient guarantee are rejected before @@ -255,8 +257,8 @@ contract consistent and preserves the full choice set for other callers. `CandidateLogicalASAPDAGs::global_selection` optionally coordinates cross-target sharing and composition choices. `GlobalSelection::assemble_selected_dag` constructs the selected -semantic DAG. These plain APIs do not establish lifecycle or physical deployment -feasibility. Recurrence and lifecycle-aware variants require the corresponding +semantic DAG. These APIs do not decide materialization or establish physical +deployment feasibility. Recurrence-aware variants require the corresponding workload and evidence inputs; downstream owns physical commitment and execution. See the [library workflow](library-api.md#optional-whole-plan-selection-and-dag-assembly). diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 447cb3823..1980dafd0 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -10,25 +10,25 @@ first; use the [extension guide](extend-asap-aware-mapping.md) when changing one ### `TargetSubDAG` -A pre-ASAP `QueryExpr` node that a strategy may replace. +A pre-ASAP `OperatorNode` that a strategy may replace. ```rust pub struct TargetSubDAG<'a> { - pub root: &'a Rc, + pub root: &'a Rc, pub consumer_count: usize, } ``` -`root` is the actual `Rc` from the workload. +`root` is the actual `Rc` from the workload. -`consumer_count` counts structural references, not runtime executions. It is the number of places in the workload DAG that point to this exact `Rc` node. +`consumer_count` counts structural references, not runtime executions. It is the number of places in the workload DAG that point to this exact `Rc` node. For example, consider two top-level queries: - `sum by (service) (rate(m[5m]))` - `avg by (service) (rate(m[5m]))` -After `share_common_sub_dags` merges their identical `rate(m[5m])` sub-DAGs, both query DAGs point to the same `Rc`. That node's `consumer_count` is `2`, regardless of how often either query executes. +After `share_common_sub_dags` merges their identical `rate(m[5m])` sub-DAGs, both query trees point to the same `Rc`. That node's `consumer_count` is `2`, regardless of how often either query executes. Use: @@ -54,19 +54,24 @@ when the caller already knows the real number of consumers. The actual object that substitutes the target. -There are currently three forms: +There are currently two forms: ```rust pub enum Replacement { - Summary(Rc), - Rewrite(Rc), + SubDAG(Rc), ExactComposition(ExactComposition), } ``` -Use `Replacement::Summary` when the alternative is a constructed post-ASAP summary plan. +Use `Replacement::SubDAG` for a replacement sub-DAG. It is one of: -Use `Replacement::Rewrite` when the alternative is still a logical pre-ASAP `QueryExpr`. +- a constructed post-ASAP summary plan: the sub-DAG contains an `ASAPOp` + (`SummaryAgg`, `SummaryEstimate`, ...); +- a logical rewrite: only `NonASAPOp` nodes and no guarantee yet. + +`is_logical_rewrite(&node)` tells the two apart. A kept pre-ASAP sub-DAG +(`retain_exact`) has no ASAP operator but carries an exact guarantee, so it +counts as a bound decision, not a rewrite. Use `Replacement::ExactComposition` when an exact operation refers to a child target whose realization must remain undecided. Selection coordinates the @@ -77,10 +82,10 @@ Examples: ```text Quantile(...) - -> KLL SummaryNode + -> SummaryEstimate(SummaryAgg(KLL)) ``` -is a `Summary`; KLL (Karnin–Lang–Liberty) is a quantile-sketch algorithm. +is a summary `Subtree`; KLL (Karnin–Lang–Liberty) is a quantile-sketch algorithm. ```text compute independently @@ -88,7 +93,7 @@ compute independently reuse an already shared logical sub-DAG ``` -is represented as a `Rewrite`. +is represented as two logical-rewrite `Subtree`s. --- @@ -157,7 +162,7 @@ aggregation must compute without committing to a physical summary algorithm. A realization may be an approximate sketch, an exact mergeable accumulator, or a pass-through that keeps the original operation instead of building a summary. `realizations_for_intent` enumerates these concrete -realizations; `SketchAlgorithmStrategy::replacements()` constructs each one as +realizations; `ASAPStrategies::replacements()` constructs each one as a `ReplacementSubDAG`. It returns all candidates in preferred order without selecting a winner. At workload scale, `search_workload`/`search_workload_with` preserve all supported legal alternatives @@ -170,8 +175,8 @@ This guide uses the Cascades/Volcano terminology: realization. For example, a quantile `AggIntent` may have KLL and DDSketch `Realization` values. - A **transformation rule** maps a logical operation to another logical - operation. In this crate, that kind of candidate is represented by - `Replacement::Rewrite`. + operation. In this crate, that kind of candidate is a logical-rewrite + `Replacement::SubDAG`. - A **replacement candidate** packages either kind of result as a `ReplacementSubDAG` for search. `CandidateLogicalASAPDAGs` stores and ranks these candidates. - **Physical commitment and placement** happen downstream. An `Realization` @@ -183,7 +188,7 @@ The concrete flow is: ```text AggIntent -> realizations_for_intent(): enumerate Realization values - -> SketchAlgorithmStrategy: construct ReplacementSubDAG candidates + -> ASAPStrategies: construct ReplacementSubDAG candidates -> CandidateLogicalASAPDAGs: store and rank candidates -> downstream deployment: select and place a final choice ``` @@ -206,7 +211,7 @@ bounds, but does not execute workloads or own deployment measurements. Most hook | `rank_candidates` | Order valid sketch algorithms | No | | `size_params` | Convert an accuracy target into sketch parameters | Yes | | `realize_extension` | Map a custom intent to a realization | Yes | -| `readout_extension` | Query a custom extension summary | Panics until paired with a custom realization | +| `evaluation_extension` | Query a custom extension summary | Panics until paired with a custom realization | | `cse_recompute_cost` | Estimate independent recomputation | Yes | | `cse_shared_maintenance_cost` | Estimate shared maintenance | Yes | | `cse_share_decision` | Choose sharing or recomputation | Yes | @@ -247,10 +252,10 @@ bounds, but does not execute workloads or own deployment measurements. Most hook fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Realization; ``` -- **`readout_extension`** — define how queries read an extension summary that `realize_extension` mapped to a `Sketch`. The two hooks are a pair: realization defines what is maintained; readout defines how it is queried. Override both for the same `ext_kind`. The default readout panics to prevent a silent wrong answer. +- **`evaluation_extension`** — define how queries read an extension summary that `realize_extension` mapped to a `Sketch`. The two hooks are a pair: realization defines what is maintained; evaluation defines how it is queried. Override both for the same `ext_kind`. The default evaluation panics to prevent a silent wrong answer. ```rust - fn readout_extension(&self, ext_kind: &str, payload: &serde_json::Value, col: &ColumnRef) -> SketchStatistic; + fn evaluation_extension(&self, ext_kind: &str, payload: &serde_json::Value, col: &ColumnRef) -> SketchStatistic; ``` - **`cse_recompute_cost`** — estimate the one-time cost of recomputing a CSE candidate's sub-DAG independently at a single consumer. Default: `default_cse_recompute_cost`, a structural-size proxy. @@ -297,14 +302,14 @@ A custom cost model does not necessarily need to override every hook. The curren // One TargetSubDAGCandidates per distinct TargetSubDAG in the whole workload — // never a flat list of fully assembled plans. pub struct TargetSubDAGCandidates { - pub target: Rc, + pub target: Rc, pub consumer_count: usize, pub candidates: Vec, // accepted alternatives, unranked pub rejected: Vec, // failed accuracy checks } pub struct RankedTargetSubDAGCandidates<'a> { - pub target: &'a Rc, + pub target: &'a Rc, pub consumer_count: usize, pub candidates: Vec<&'a ReplacementSubDAG>, // same candidates, ranked pub costs: Vec, // costs[i] <-> candidates[i] @@ -313,7 +318,7 @@ pub struct RankedTargetSubDAGCandidates<'a> { `search_workload(roots)` runs the shared-sub-DAG pass once, discovers every target across every root's whole DAG (not just root-level sharing — a `SharedSubDAGStrategy` candidate three levels under an unshared `Filter` is exactly as real a site as a shared whole root), and asks every registered strategy to a fixpoint. Two logically different candidates at two different targets are never copied into two separate plans — they're two entries in two different `TargetSubDAGCandidates`s, sharing every other node in the workload by construction. -`CandidateLogicalASAPDAGs::cost_sorted(cost_model)` is the one ranking step: for each candidate set, it dispatches by candidate shape — a same-shape `Rewrite` pair (a `SharedSubDAGStrategy` share/recompute choice) goes through `CostModel::cse_share_decision`; a same-shape run of `Summary` candidates realizing sketches (a `SketchAlgorithmStrategy` choice) goes through `CostModel::rank_candidates`; and a mixed candidate set is ordered by each candidate's `CostModel::estimate_cost`. Every candidate gets a numeric cost aligned index-for-index in `costs`. Count in, count out—nothing is dropped to produce a ranking. Legality checks +`CandidateLogicalASAPDAGs::cost_sorted(cost_model)` is the one ranking step: for each candidate set, it dispatches by candidate shape — the `SharedSubDAGStrategy` share/recompute pair (recognized by `ReplacementProvenance::CseShare`/`CseRecompute`) goes through `CostModel::cse_share_decision`; a set with a Hydra shared-grid alternative goes through `CostModel::grouping_state_cost`; a set whose candidates all realize sketches (a `ASAPStrategies` choice) goes through `CostModel::rank_candidates`; and any other mixed set is ordered by `CostModel::candidate_cost`. Every candidate gets a numeric cost aligned index-for-index in `costs`. Count in, count out—nothing is dropped to produce a ranking. Legality checks may already have removed proposals before this boundary. In particular, `search_workload_with_targets` checks explicit per-root targets, while retaining direct DDSketch ratios with missing domain evidence and no root guarantee for @@ -329,7 +334,7 @@ Sketches separate their query category from the concrete algorithm and its param | Level | Type | Example | | --- | --- | --- | -| **family** | `SummaryFamilyType` | `Sketch`, `Sample`, `Wavelet`, `StatModel`, `ExactAggregate` | +| **family** | `FieldDataType` (non-`Plain` variants) | `Sketch`, `Sample`, `Wavelet`, `StatModel`, `ExactAggregate` | | **category** | `SketchCategory` | `Quantile`, `Cardinality`, `Frequency`, `TopK` | | **algorithm** | `SketchAlgorithm` | `Kll` / `DDSketch` (both quantile); `Hll` (HyperLogLog) / `Theta` / `Kmv` (K-Minimum Values), all cardinality | | **committed choice** | `SketchKind` | one validated category + algorithm + parameter combination | @@ -340,7 +345,7 @@ to the selected algorithm and classifies the pair into its category. The public `.category()`, `.algorithm()`, and `.params()` accessors expose the committed values without permitting an invalid combination. -Where this matters in practice: `CostModel::rank_candidates`, `CostModel::size_params`, and `SketchAlgorithmStrategy::replacements` operate at the **algorithm** level. `summary_candidates(intent)` returns a list of `SketchAlgorithm`s (`[Kll, DDSketch]` for a `Quantile` intent), never a bare `SketchKind` with nothing chosen underneath it. `SketchKind` appears after an algorithm has been selected and sized—on `Realization::Sketch(SketchKind)` and `SummaryFamilyType::Sketch(SketchKind)`. +Where this matters in practice: `CostModel::rank_candidates`, `CostModel::size_params`, and `ASAPStrategies::replacements` operate at the **algorithm** level. `summary_candidates(intent)` returns a list of `SketchAlgorithm`s (`[Kll, DDSketch]` for a `Quantile` intent), never a bare `SketchKind` with nothing chosen underneath it. `SketchKind` appears after an algorithm has been selected and sized—on `Realization::Sketch(SketchKind)` and `FieldDataType::Sketch(SketchKind, GroupingStrategy)`. `Sample`, `Wavelet`, and `StatModel` each use a flat `(Kind, Params)` pair. `Sketch` needs the additional algorithm level because multiple algorithms can serve the same purpose—for example, KLL and DDSketch both answer quantile queries. @@ -378,15 +383,15 @@ The crate provides no default `Matcher` implementation because the answer depend Concretely, `explanation.rs` reports three candidate kinds from each `TargetSubDAGCandidates`: -- `ExplanationKind::SketchApproximation` — the set contains a `Replacement::Summary` that realizes `SummaryFamilyType::Sketch(..)`, not just an exact/pass-through candidate. -- `ExplanationKind::CommonSubexpressionReuse` — `consumer_count >= 2` and the set contains `SharedSubDAGStrategy`'s "build once and share" candidate (the `Replacement::Rewrite` whose `Rc` is the set's `target`). +- `ExplanationKind::SketchApproximation` — the set contains a summary `Replacement::SubDAG` that realizes `FieldDataType::Sketch(..)`, not just an exact/pass-through candidate. +- `ExplanationKind::CommonSubexpressionReuse` — `consumer_count >= 2` and the set contains `SharedSubDAGStrategy`'s "build once and share" candidate (the `Replacement::SubDAG` whose `Rc` is the set's `target`). - `ExplanationKind::ExactComposition` — the candidate set contains an exact operation composed with a child target whose realization remains a coordinated choice. Each `ReplacementExplanation::reason` is copied verbatim from the matching candidate's own `ReplacementSubDAG::rationale`. Nothing in `explanation.rs` re-explains why a candidate is valid; that explanation already exists exactly once, on the candidate itself. -`ReplacementExplanation` carries both `node_hash` and `target`. A downstream consumer first compares `node_hash` with an exported `DAGNode::hash` to narrow the search, then compares the exact target expression with the node's in-process source expression. This preserves the hash's role as a fast filter while making the final association collision-safe; `location` remains human-readable presentation text rather than a machine identifier. +`ReplacementExplanation` carries both `node_hash` and `target`. A downstream consumer first compares `node_hash` with an exported `DAGNode::hash` to narrow the search, then compares the exact `target` node with the exported node's in-process `DAGNode::source_node`. This preserves the hash's role as a fast filter while making the final association collision-safe; `location` remains human-readable presentation text rather than a machine identifier. ### Why there is no `ExplanationRule` trait @@ -394,6 +399,6 @@ Explanations are derived from candidates already present in `CandidateLogicalASA ### How it derives `location` text -`CandidateLogicalASAPDAGs`/`TargetSubDAGCandidates` track `Rc` pointer identity, not human-readable breadcrumbs. `ReplacementExplanation::location` provides prose such as `root "dash_a" > lhs` so reporting consumers can identify the relevant part of the query without interpreting pointer identity. Location derivation does not make replacement or costing decisions. +`CandidateLogicalASAPDAGs`/`TargetSubDAGCandidates` track `Rc` pointer identity, not human-readable breadcrumbs. `ReplacementExplanation::location` provides prose such as `root "dash_a" > lhs` so reporting consumers can identify the relevant part of the query without interpreting pointer identity. Location derivation does not make replacement or costing decisions. --- diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index 5a5e069a8..b9a830140 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -44,7 +44,7 @@ There are four decisions to make. `matches` should contain the minimum structural and semantic checks needed to determine whether the strategy applies. -For example, the aggregate path in `SketchAlgorithmStrategy` requires a +For example, the aggregate path in `ASAPStrategies` requires a supported shape: - the node is an `Aggregate`, @@ -112,23 +112,20 @@ and let costing decide later. --- -### Choose `Summary` vs. `Rewrite` +### Summary sub-DAG vs. logical rewrite -Return: +Both are returned as: ```rust -Replacement::Summary(...) +Replacement::SubDAG(node) ``` -when the candidate is a fully constructed post-ASAP summary. +- A fully constructed post-ASAP summary: `node` contains an `ASAPOp`. +- A logical pre-ASAP rewrite: `node` has only `NonASAPOp` nodes and no + guarantee. `is_logical_rewrite(&node)` checks this. -Return: - -```rust -Replacement::Rewrite(...) -``` - -when the candidate is a logical pre-ASAP rewrite. +Set `provenance` to say which one it is (`ReplacementProvenance::SummaryRealization`, +`LogicalRewrite`, ...); selection reads provenance, not the sub-DAG's shape. Use `Replacement::ExactComposition` when a candidate depends on a child target whose implementation must be selected compatibly later. Do not bind it to the @@ -204,7 +201,7 @@ how to realize it, wrap that logic. Do not create a second implementation of the same semantics inside the strategy. -The existing `SketchAlgorithmStrategy` is the model to follow: it reuses +The existing `ASAPStrategies` is the model to follow: it reuses `replacement.rs`'s existing candidate list and summary-construction path. --- @@ -229,23 +226,23 @@ If your transformation requires context not currently represented in `TargetSubD --- -### Example: current `SketchAlgorithmStrategy` +### Example: current `ASAPStrategies` -`SketchAlgorithmStrategy` is the reference implementation for a strategy that +`ASAPStrategies` is the reference implementation for a strategy that produces constructed post-ASAP summaries. Construction: ```rust let strategy = - SketchAlgorithmStrategy::default_cost_model(); + ASAPStrategies::default_cost_model(); ``` or with a custom cost model: ```rust let model = MyCostModel; // illustrative -let strategy = SketchAlgorithmStrategy::new(&model); +let strategy = ASAPStrategies::new(&model); ``` The strategy matches supported aggregate nodes. @@ -254,10 +251,10 @@ At a high level: ```mermaid flowchart LR - A["Input TargetSubDAG
root is a supported Aggregate"] --> B["SketchAlgorithmStrategy::matches
check whether the target shape can produce summaries"] - B -->|"true"| C["SketchAlgorithmStrategy::replacements
use CostModel preferences and sizing while preserving
every semantically valid realization"] + A["Input TargetSubDAG
root is a supported Aggregate"] --> B["ASAPStrategies::matches
check whether the target shape can produce summaries"] + B -->|"true"| C["ASAPStrategies::replacements
use CostModel preferences and sizing while preserving
every semantically valid realization"] B -->|"false"| NONE["Empty candidate list"] - C --> F["Output Vec<ReplacementSubDAG>
each entry contains a constructed SummaryNode and rationale;
all candidates retained in preferred order"] + C --> F["Output Vec<ReplacementSubDAG>
each entry contains a constructed summary sub-DAG and rationale;
all candidates retained in preferred order"] ``` For an approximate quantile, both KLL and DDSketch remain candidates when @@ -271,16 +268,17 @@ even if the cost model prefers one. When only one realization is legal, such as Call the public strategy interface and inspect every returned candidate: ```rust -let strategy = SketchAlgorithmStrategy::new(&cost_model); +let strategy = ASAPStrategies::new(&cost_model); let candidates = strategy.replacements(&target); for candidate in candidates { match candidate.replacement { - Replacement::Summary(summary) => { - // Inspect or execute this constructed SummaryNode. + Replacement::SubDAG(node) => { + // A constructed summary sub-DAG (`node.is_asap()`), or a kept + // pre-ASAP sub-DAG with an exact guarantee for pass-through. } - Replacement::Rewrite(_) => unreachable!( - "SketchAlgorithmStrategy produces summary candidates" + Replacement::ExactComposition(_) => unreachable!( + "ASAPStrategies produces sub-DAG candidates" ), } } @@ -310,16 +308,16 @@ and returns two alternatives: 2. Build independently for each consumer. ``` -The shared candidate reuses the same `Rc`: +The shared candidate reuses the same `Rc`: ```rust -Replacement::Rewrite(Rc::clone(target.root)) +Replacement::SubDAG(Rc::clone(target.root)) ``` The independent candidate creates a structurally equal but separately allocated node: ```rust -Replacement::Rewrite( +Replacement::SubDAG( Rc::new((**target.root).clone()) ) ``` @@ -353,7 +351,7 @@ The basic calling pattern is: ```rust let target = TargetSubDAG::new(&root); let strategy = - SketchAlgorithmStrategy::default_cost_model(); + ASAPStrategies::default_cost_model(); if strategy.matches(&target) { let candidates = @@ -546,13 +544,13 @@ Then inject it into code that accepts a `&dyn CostModel`: let model = PreferDDSketch; let strategy = - SketchAlgorithmStrategy::new(&model); + ASAPStrategies::new(&model); let replacements = strategy.replacements(&target); ``` -Important: changing `rank_candidates` changes the preferred ordering, but `SketchAlgorithmStrategy` still enumerates every valid sketch candidate. +Important: changing `rank_candidates` changes the preferred ordering, but `ASAPStrategies` still enumerates every valid sketch candidate. A custom cost model should not change which alternatives are semantically legal. @@ -641,12 +639,12 @@ Use it for implementation families that are intentionally outside the built-in e --- -#### `readout_extension` +#### `evaluation_extension` -Use when an extension-defined summary also needs custom query/readout behavior. +Use when an extension-defined summary also needs custom query/evaluation behavior. ```rust -fn readout_extension( +fn evaluation_extension( &self, ext_kind: &str, payload: &serde_json::Value, @@ -654,7 +652,7 @@ fn readout_extension( ) -> SketchStatistic; ``` -This complements `realize_extension`: realization defines what gets maintained; readout defines how it is queried (see the [CostModel reference](asap-aware-mapping-contracts.md#costmodel)). +This complements `realize_extension`: realization defines what gets maintained; evaluation defines how it is queried (see the [CostModel reference](asap-aware-mapping-contracts.md#costmodel)). --- @@ -741,7 +739,7 @@ For example: ```rust let strategy = - SketchAlgorithmStrategy::new(&model); + ASAPStrategies::new(&model); let replacements = strategy.replacements(&target); @@ -770,7 +768,7 @@ Declare built-in sketch applicability through the public candidate registry: summary_candidates(intent) ``` -`SketchAlgorithmStrategy` consumes this registry through its public `replacements` method. +`ASAPStrategies` consumes this registry through its public `replacements` method. Therefore, when adding a new built-in sketch algorithm, the intended flow is: @@ -778,9 +776,9 @@ Therefore, when adding a new built-in sketch algorithm, the intended flow is: flowchart LR MAP["1. Declare legality
add the algorithm to summary_candidates
for each AggIntent it can answer"] MAP --> MODEL["2. Define costing
rank it, derive its SketchParams,
and provide a comparable numeric cost"] - MODEL --> BUILD["3. Define realization behavior
ensure the public strategy output contains a valid SummaryNode
with the correct maintained state and readout"] + MODEL --> BUILD["3. Define realization behavior
ensure the public strategy output contains a valid summary sub-DAG
with the correct maintained state and evaluation"] BUILD --> ACC["4. Certify accuracy
derive from committed parameters;
propagate and check the final target"] - ACC --> ENUM["5. Verify integration
SketchAlgorithmStrategy includes it automatically;
tests confirm enumeration, ordering, sizing, and cost"] + ACC --> ENUM["5. Verify integration
ASAPStrategies includes it automatically;
tests confirm enumeration, ordering, sizing, and cost"] ``` This keeps one source of truth for sketch applicability. Applicability alone @@ -793,9 +791,9 @@ ranking; preserve exact fallback and structured rejection information. See the [accuracy implementation companion](end-to-end-accuracy-guarantees.md) for formulas and evidence requirements. For a new algorithm, also update its -parameter, readout, schema and serialization definitions in `asap-types`. +parameter, evaluation, schema and serialization definitions in `asap-types`. -Do not special-case the new sketch inside `SketchAlgorithmStrategy` unless the strategy itself needs fundamentally new behavior. +Do not special-case the new sketch inside `ASAPStrategies` unless the strategy itself needs fundamentally new behavior. ### Verifying a new sketch algorithm @@ -804,7 +802,7 @@ or malformed evidence, incompatible metrics and unsupported composition. Test root-target checking before cost ranking, exact fallback, and exported rejection or guarantee data. A cheaper estimate must never admit an accuracy-illegal plan. -After wiring the new algorithm into `summary_candidates` and giving the cost model a real `rank_candidates`/`size_params` opinion about it, check two things. First, that `SketchAlgorithmStrategy::replacements()` for a matching `TargetSubDAG` actually includes a candidate realizing the new algorithm — extend a test shaped like `replacement.rs`'s own test-module coverage-matrix tests (e.g. `agg_intent_to_summary_kind_coverage_matrix`) to cover the new algorithm's `AggIntent`. Second, that `cost_sorted`/`estimate_cost` produce sane, comparable numbers for the new candidate rather than a `NaN` placeholder or an outlier that swamps every other candidate. +After wiring the new algorithm into `summary_candidates` and giving the cost model a real `rank_candidates`/`size_params` opinion about it, check two things. First, that `ASAPStrategies::replacements()` for a matching `TargetSubDAG` actually includes a candidate realizing the new algorithm — extend a test shaped like `replacement.rs`'s own test-module coverage-matrix tests (e.g. `agg_intent_to_summary_kind_coverage_matrix`) to cover the new algorithm's `AggIntent`. Second, that `cost_sorted`/`estimate_cost` produce sane, comparable numbers for the new candidate rather than a `NaN` placeholder or an outlier that swamps every other candidate. --- @@ -895,10 +893,11 @@ silently disagree. ### Mistake: reimplementing summary construction inside a strategy -If the candidate should produce a normal `SummaryNode`, use the existing +If the candidate should produce a normal summary sub-DAG (`SummaryAgg` / +`SummaryEstimate`), use the existing summary-construction path. -A strategy should steer or wrap that path when necessary, not recreate schema derivation, column resolution, readout construction, or parameter sizing. +A strategy should steer or wrap that path when necessary, not recreate schema derivation, column resolution, evaluation construction, or parameter sizing. --- @@ -922,7 +921,7 @@ Workload-wide target discovery, deduplication, and consumer counting are separat For CSE-style decisions, pointer identity can encode actual sharing. -Two `Rc` values can be structurally equal but deliberately represent independent computation. +Two `Rc` values can be structurally equal but deliberately represent independent computation. Use the distinction intentionally. @@ -937,8 +936,8 @@ When adding a new strategy: - [ ] Implement `ReplacementStrategy::replacements`. - [ ] Return every semantically valid replacement. - [ ] Return an empty vector for non-matching targets. -- [ ] Use `Replacement::Summary` for constructed post-ASAP output. -- [ ] Use `Replacement::Rewrite` for logical pre-ASAP alternatives. +- [ ] Return `Replacement::SubDAG` for both constructed post-ASAP output and + logical pre-ASAP alternatives, with the matching `provenance`. - [ ] Add a useful rationale to every candidate. - [ ] Reuse existing legality and implementation logic instead of duplicating it. - [ ] Keep ranking and cost-based pruning out of the strategy. @@ -953,11 +952,11 @@ When adding a new cost model: - [ ] Keep semantic applicability outside the cost model. - [ ] Use `rank_candidates` for algorithm preference; return every input candidate exactly once. - [ ] Use `size_params` for accuracy-to-parameter mapping. -- [ ] Use extension hooks for extension-defined implementations/readouts. +- [ ] Use extension hooks for extension-defined implementations/evaluations. - [ ] Use CSE hooks for recompute-vs.-sharing costs. - [ ] Override `estimate_cost` if consumers require numeric costs instead of `NaN`. - [ ] Test the hook directly. -- [ ] Test integration through a consumer such as `SketchAlgorithmStrategy`. +- [ ] Test integration through a consumer such as `ASAPStrategies`. - [ ] Verify that changing cost preferences does not silently remove valid replacement candidates. --- @@ -975,12 +974,12 @@ Use this table to find the right place for a change. | Prefer one sketch algorithm over another | `CostModel::rank_candidates` | | Change sketch sizing for an accuracy target | `CostModel::size_params` | | Add extension-defined implementation behavior | `CostModel::realize_extension` | -| Add extension-defined readout behavior | `CostModel::readout_extension` | +| Add extension-defined evaluation behavior | `CostModel::evaluation_extension` | | Change CSE recomputation cost | `CostModel::cse_recompute_cost` | | Change shared-maintenance cost | `CostModel::cse_shared_maintenance_cost` | | Change current share/recompute choice | `CostModel::cse_share_decision` | | Decide whether an available implementation satisfies a required one | `impl Matcher` | -| Produce a normal (ranked-first) post-ASAP summary for one target | `SketchAlgorithmStrategy::replacements(...).into_iter().next()` | +| Produce a normal (ranked-first) post-ASAP summary for one target | `ASAPStrategies::replacements(...).into_iter().next()` | | Search a whole workload for supported legal candidates | `search_workload`/`search_workload_with` | | Enforce per-root result accuracy requirements | `search_workload_with_targets` | | Coordinate compatible choices across groups | `CandidateLogicalASAPDAGs::global_selection` | diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 719de6ba2..7e8b2768d 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -15,7 +15,6 @@ do not deploy a plan, and a serializable DAG is not evidence of runtime readines | Pre-ASAP IR | Frontend `lower_*` | [Lower a query](#lower-a-query-into-pre-asap-ir) | | All ranked candidates | `search_workload_with_targets` -> `cost_sorted` | [Generate and rank](#generate-and-rank-candidates) | | Custom optimization set | Construct `Vec>`, then search | [Strategies and models](#choose-strategies-and-models) | -| Summary-maintenance lifecycle comparison | Lifecycle-aware selection -> DAG assembly with maintenance decisions | [Lifecycle recipe](#lifecycle-and-capabilities) | | Selected semantic DAG / export | `global_selection` -> `assemble_selected_dag` -> export | [Selection example](#optional-whole-plan-selection-and-dag-assembly) | Each recipe ends at a different artifact. Use only the stages needed for that @@ -38,14 +37,16 @@ asap-types = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e7fdb2 | Public function | Required input | Output | | --- | --- | --- | -| `asap_frontend_promql::lower_promql_workload` | PromQL `PlanningWorkload` with a nonzero `data_ingestion_interval` | All-or-nothing `Result, PromqlError>` for normalized batch and repeating entries | -| `asap_frontend_metricsql::lower_metricsql` | Query string, `AccuracyTarget` | `Result` | -| `asap_frontend_sql::lower_sql` | Query string, `SqlCatalog`, accuracy | Async `Result`; default SQL dialect is DataFusionSQL | +| `asap_frontend_promql::lower_promql_workload` | PromQL `PlanningWorkload` with a nonzero `data_ingestion_interval` | All-or-nothing `Result>, PromqlError>` for normalized batch and repeating entries | +| `asap_frontend_metricsql::lower_metricsql` | Query string, `AccuracyTarget` | `Result, MetricsqlError>` | +| `asap_frontend_sql::lower_sql` | Query string, `SqlCatalog`, accuracy | Async `Result, SqlError>`; default SQL dialect is DataFusionSQL | | `asap_frontend_sql::lower_sql_dialect` | Same inputs plus `SqlDialect` | Async resolved Pre-ASAP query or error | | `asap_frontend_sql::lower_sql_batch` | `QueryWorkload` and catalog | Per-query results for `query_batch`; does not iterate `repeating_queries` | Lowering resolves the supported source language into the canonical query -representation. It does not enumerate Post-ASAP alternatives. A frontend may +representation: an `asap_types::ir::OperatorNode` DAG containing only +`NonASAPOp` operators, with no timing (see the +[Pre-ASAP IR reference](pre-asap-ir.md)). It does not enumerate Post-ASAP alternatives. A frontend may reject unsupported syntax or semantics; a declared language/dialect enum does not imply complete support. PromQL workload lowering uses normalized `PlanningWorkload::query_workload.entries()` order, preserving entry-to-root associations for later @@ -58,12 +59,12 @@ PromQL's public signature (types are imported from their respective crates): ```text lower_promql_workload(workload: &PlanningWorkload, now_ms: u64) - -> Result, PromqlError> + -> Result>, PromqlError> ``` `DataWorkload.data_ingestion_interval` must contain a nonzero `Evidence`. Pass the actual planning time as `now_ms` (Unix milliseconds), consistently with -downstream lifecycle planning. Expired or future cadence evidence is rejected, +downstream planning. Expired or future cadence evidence is rejected, as is expiring evidence without an observation timestamp. The histogram variant takes the same timestamp after its histogram catalog argument. The examples use `0` only because their explicitly supplied cadence is timeless. @@ -125,9 +126,9 @@ For SQL, the corresponding signatures are: ```text async lower_sql(query: &str, catalog: &SqlCatalog, accuracy: AccuracyTarget) - -> Result + -> Result, SqlError> async lower_sql_dialect(query: &str, catalog: &SqlCatalog, - dialect: SqlDialect, accuracy: AccuracyTarget) -> Result + dialect: SqlDialect, accuracy: AccuracyTarget) -> Result, SqlError> ``` | `SqlDialect` value | Current behavior | @@ -162,7 +163,7 @@ It keeps the alternatives available; it does not select an entire workload plan. ```text search_workload_with_targets<'s, Id>( - roots: Vec<(Id, Rc, Option)>, + roots: Vec<(Id, Rc, Option)>, strategies: &[Box], accuracy_model: &dyn AccuracyModel, ) -> CandidateLogicalASAPDAGs @@ -197,7 +198,6 @@ accuracy target, and prints every ranked candidate instead of selecting a winner The default cost model is suitable for inspection, not deployment calibration. ```rust -use std::rc::Rc; use asap_frontend_promql::lower_promql_workload; use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, Query, @@ -235,7 +235,7 @@ fn main() -> Result<(), Box> { ..Default::default() }), }; - let root = Rc::new(lower_promql_workload(&workload, 0)?.remove(0)); + let root = lower_promql_workload(&workload, 0)?.remove(0); let cost_model = DefaultCostModel; let strategies = default_strategies_with(&cost_model); let space = search_workload_with_targets( @@ -254,12 +254,12 @@ fn main() -> Result<(), Box> { | API (`asap_aware_mapping`, unless qualified) | Inputs | Output and limits | | --- | --- | --- | -| `search_workload` | `(query_id, Rc)` roots | `CandidateLogicalASAPDAGs` with built-in strategies/model; no explicit per-root target argument | +| `search_workload` | `(query_id, Rc)` roots | `CandidateLogicalASAPDAGs` with built-in strategies/model; no explicit per-root target argument | | `search_workload_with` | Roots, strategy slice | `CandidateLogicalASAPDAGs`; callers choose context-free replacement strategies | | `search_workload_with_targets` | Roots with optional end-to-end targets, strategies, accuracy model | Candidate space with supplied root-target checks; `None` does not supply a root-level requirement; uncertified direct DDSketch ratios remain available for backend selection | | `CandidateLogicalASAPDAGs::cost_sorted` | Cost model | `Vec`; retains alternatives and pairs `candidates[i]` with `costs[i]` | | `CandidateLogicalASAPDAGs::cost_sorted_with_recurrence` | Cost model, recurrence profiles, optional horizon | Ranked per-target candidate sets or `RecurrenceError`; uses recurrence for applicable share/recompute comparisons | -| `SketchAlgorithmStrategy::replacements` through `ReplacementStrategy` | One `TargetSubDAG` | Alternatives at that target; not whole-workload search | +| `ASAPStrategies::replacements` through `ReplacementStrategy` | One `TargetSubDAG` | Alternatives at that target; not whole-workload search | `cost_sorted` is a ranking view, not a request to discard all but the first candidate. Display costs follow model hooks and may be unavailable/non-finite; @@ -279,15 +279,16 @@ choices are not multiplied in. Exceeding `expansion_limit` is an error, never a partial inventory. For PromQL roots that carry a target, `search_workload_with_targets` also asks -each strategy's `ReplacementStrategy::propose_for_root`. `SketchAlgorithmStrategy` +each strategy's `ReplacementStrategy::propose_for_root`. `ASAPStrategies` answers an instant-vector TopK with current-series heap realizations over rows carrying the complete series identity (`$promql_series_identity`). They are finalized, deduplicated, and marked `ReplacementProvenance::RootPhysicalRealization`. Callers do not apply `with_series_identity` themselves. Compile each with -`promql_rows::compile_current_series_readout`; other queries keep their previous +`promql_rows::compile_current_series_evaluation`; other queries keep their previous inventory. `global_selection` never commits these candidates; the backend compiles and prices them. CandidateLogicalASAPDAGs lists no placement variants: node timing -comes from the summary maintenance lifecycle. +comes from a `MaterializationAssignment` (all query time until Stage 2 +materialization, #509, decides otherwise). ## Choose strategies and models @@ -300,11 +301,11 @@ pass. An omitted strategy contributes no proposals of its own. | Value to put inside `Box::new(...)` | Meaning | In default factories? | | --- | --- | --- | -| `SketchAlgorithmStrategy::new(&model)` | Enumerates supported exact/sketch implementations and parameter choices for aggregate targets | Yes | +| `ASAPStrategies::new(&model)` | Enumerates supported exact/sketch implementations and parameter choices for aggregate targets | Yes | | `HydraGroupingStrategy::new(&model)` | Considers a shared multi-subpopulation structure for supported grouped sketch families, subject to accuracy evidence | Yes | | `SharedSubDAGStrategy` | Proposes sharing versus independent recomputation at reused sub-DAGs | Yes | | `SemanticEquivalentRewriteStrategy` | Proposes supported equivalent aggregate rewrites, including decomposing average into sum/count | Yes | -| `ExactCompositionStrategy::new(&model)` | Proposes supported exact operations around summary readouts or in maintenance | Yes | +| `ExactCompositionStrategy::new(&model)` | Proposes supported exact operations around summary evaluations or in maintenance | Yes | | Your `ReplacementStrategy` implementation | Adds domain-specific legal replacement proposals | No | `AvgToSumOverCountStrategy` is an alias for `SemanticEquivalentRewriteStrategy` @@ -345,7 +346,6 @@ replacement::default_strategies_with_evidence<'a>( ### Example: supply two strategies and run search ```rust -use std::rc::Rc; use asap_frontend_promql::lower_promql_workload; use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, Query, @@ -353,7 +353,7 @@ use asap_types::workload::{ }; use asap_aware_mapping::{ search_workload_with_targets, DefaultAccuracyModel, DefaultCostModel, - ReplacementStrategy, SketchAlgorithmStrategy, SharedSubDAGStrategy, + ReplacementStrategy, ASAPStrategies, SharedSubDAGStrategy, }; use asap_types::types::AccuracyTarget; @@ -383,10 +383,10 @@ fn main() -> Result<(), Box> { ..Default::default() }), }; - let root = Rc::new(lower_promql_workload(&workload, 0)?.remove(0)); + let root = lower_promql_workload(&workload, 0)?.remove(0); let model = DefaultCostModel; let strategies: Vec> = vec![ - Box::new(SketchAlgorithmStrategy::new(&model)), + Box::new(ASAPStrategies::new(&model)), Box::new(SharedSubDAGStrategy), ]; let space = search_workload_with_targets( @@ -409,7 +409,7 @@ Module-qualified paths below are relative to `asap_aware_mapping`. | Parameter | Available value / constructor | Meaning | | --- | --- | --- | | `&dyn CostModel` | `DefaultCostModel` | Built-in ordering/sizing and structural estimates; no measured deployment guarantee | -| `&dyn CostModel` | `empirical_cost::EmpiricalCostModel::new(provider)` | Offline sketch-benchmark model: ranks algorithms using matching offline measurements and supplies partial lifecycle costs | +| `&dyn CostModel` | `empirical_cost::EmpiricalCostModel::new(provider)` | Offline sketch-benchmark model: ranks algorithms using matching offline measurements | | `&dyn CostModel` | `physical_plan_cost_model::PhysicalPlanCostModel::new(&provider, calibration)?` | Deployment-specific physical-plan model: compares complete physical alternatives using provider evidence and resource calibration; evidence may be offline or online | | `&dyn AccuracyModel` | `DefaultAccuracyModel` | Built-in guarantee rules and satisfaction checks | | `&dyn AccuracyBudgetAllocator` | `EqualSplitAllocator` | Built-in allocation of composition accuracy budgets | @@ -423,7 +423,7 @@ These models differ in scope, not simply in whether they are offline or online. | Model | Evidence and comparison | Missing evidence / limits | | --- | --- | --- | -| `EmpiricalCostModel` | Offline sketch benchmarks matched to exact parameters, distribution, environment and validity interval; current algorithm ranking uses measured update CPU nanoseconds | If the measurements required for ranking are incomplete, preserves the incoming algorithm order. Supplies partial build/update lifecycle costs; `estimate_cost()` still uses `DefaultCostModel` structural scores | +| `EmpiricalCostModel` | Offline sketch benchmarks matched to exact parameters, distribution, environment and validity interval; current algorithm ranking uses measured update CPU nanoseconds | If the measurements required for ranking are incomplete, preserves the incoming algorithm order. `estimate_cost()` still uses `DefaultCostModel` structural scores | | `PhysicalPlanCostModel` | A downstream provider supplies a consistent evidence snapshot and complete physical alternatives; calibration converts modeled resource quantities into comparable costs | A candidate with incomplete evidence is unavailable, without structural-cost fallback. Current candidate admission also requires it to cost less than the raw alternative | `PhysicalPlanCostModel` does not collect online telemetry itself. Its provider @@ -442,7 +442,7 @@ accuracy guarantees. ```rust use asap_aware_mapping::{ DefaultAccuracyModel, DefaultCostModel, EqualSplitAllocator, - NoAccuracyEvidence, ReplacementStrategy, SketchAlgorithmStrategy, + NoAccuracyEvidence, ReplacementStrategy, ASAPStrategies, }; fn main() { @@ -451,7 +451,7 @@ fn main() { let allocation = EqualSplitAllocator; let evidence = NoAccuracyEvidence; let strategies: Vec> = vec![Box::new( - SketchAlgorithmStrategy::new_with_planning_inputs_and_evidence( + ASAPStrategies::new_with_planning_inputs_and_evidence( &cost, &accuracy, &allocation, &evidence, ), )]; @@ -463,22 +463,22 @@ fn main() { Constructor definition: ```text -SketchAlgorithmStrategy::new_with_planning_inputs_and_evidence( +ASAPStrategies::new_with_planning_inputs_and_evidence( cost_model: &dyn CostModel, accuracy_model: &dyn AccuracyModel, allocator: &dyn AccuracyBudgetAllocator, evidence: &dyn AccuracyEvidenceProvider, -) -> SketchAlgorithmStrategy +) -> ASAPStrategies ``` All provider arguments are required for this constructor. They must outlive the -strategy vector. `SketchAlgorithmStrategy::new(&cost_model)` is the shorter +strategy vector. `ASAPStrategies::new(&cost_model)` is the shorter constructor using default accuracy/allocation and no extra evidence. | Extension point | What it controls | What it cannot establish alone | | --- | --- | --- | | `ReplacementStrategy` | Proposed semantic alternatives | Permission to violate query semantics or downstream support | -| `CostModel` | Candidate ordering/sizing hooks, recurrence/lifecycle and complete-cost evidence hooks | Correctness, measured costs without evidence, or installed runtime support | +| `CostModel` | Candidate ordering/sizing hooks and recurrence cost hooks | Correctness, measured costs without evidence, or installed runtime support | | `AccuracyModel` | Derivation, propagation and satisfaction of guarantees | A meaningful guarantee without its required assumptions/evidence | | `AccuracyBudgetAllocator` | Local accuracy requirements proposed within composition | End-to-end correctness without subsequent validation | | `AccuracyEvidenceProvider` | Planning-time statistics used by supported strategies | Authority to change query requirements | @@ -488,7 +488,7 @@ with the intended model/evidence; replacing only the final sorting model does no regenerate parameter choices. For evidence-aware defaults, use `asap_aware_mapping::replacement::default_strategies_with_evidence`. For custom accuracy/allocation/evidence on sketches, -`SketchAlgorithmStrategy::new_with_planning_inputs_and_evidence` exposes these providers. +`ASAPStrategies::new_with_planning_inputs_and_evidence` exposes these providers. Keep each provider's evidence scope and freshness valid for the query population. ## Workload inputs and defaults @@ -497,12 +497,9 @@ Keep each provider's evidence scope and freshness valid for the query population inputs. `QueryWorkload` contains the language and optional batch/repeating entries. Entries carry requirements, predictability, recurrence and time selection. These facts are separate: repeated queries can read data at rest. -`WorkloadDemand` associates a target with the relevant workload entry indices -and explicitly includes or omits the parallel data evidence. -Both recurrence and lifecycle planning validate this independent data evidence: -ingestion rates must be finite and nonnegative, and data at rest cannot have a -positive ingestion rate. `DataWorkload::validate()` shares these checks with -`PlanningWorkload::validate()`. +`DataWorkload::validate()` checks the independent data evidence: ingestion +rates must be finite and nonnegative, and data at rest cannot have a positive +ingestion rate. `PlanningWorkload::validate()` shares these checks. | Type/input | Current behavior | Caller responsibility | | --- | --- | --- | @@ -510,188 +507,11 @@ positive ingestion rate. `DataWorkload::validate()` shares these checks with | `DataWorkload::default()` | Unknown arrival, unknown evidence | Supply facts needed for the requested comparisons | | `Evidence::default()` | No value, unknown source | Unknown/stale evidence is not zero; provide scoped valid observations | | `DefaultCostModel` | Built-in ordering/sizing and structural cost hooks | Supply deployment evidence for calibrated comparisons | -| `SummaryMaintenanceLifecycleCostInputs::default()` | All primitive costs unknown | Implement the required lifecycle cost hooks; structural defaults are insufficient | -| `horizon: None` in lifecycle planning | Horizon-dependent alternatives are unselectable | Supply a positive horizon when comparing rates/amortized reuse | -| Lifecycle capabilities default | All four modes enabled | Override with the actual runtime support | -| Per-summary maintenance capabilities default | Incremental update, merge, delete all false | Advertise supported operations for the concrete state representation | `Default` is a Rust constructor contract, not a general serde omission rule. Several workload fields require explicit serialized values. A struct field being optional also does not guarantee every planning operation can succeed without it. -## Lifecycle and capabilities - -Use this workflow when Planner owns summary-maintenance lifecycle decisions; -otherwise the backend may make them from logical candidates. It includes both -selection and DAG assembly, so callers do not first run the ordinary workflow. -The first helper returns one `GlobalSelection`; the second is called per root -and returns a plan containing `root: Rc` plus maintenance decisions. -See the [workflow design](../design_docs/architecture/input-output-workflow.md#summary-maintenance-lifecycle-aware-helper). - -Two capabilities are distinct: the runtime can orchestrate a lifecycle, and the -chosen summary representation supports the required state operations. Both must -hold. Workload legality and known cost evidence can further restrict alternatives. - -### API definition and options - -```text -global_selection_with_summary_maintenance_lifecycles<'a, Id>( - space: &'a CandidateLogicalASAPDAGs, demand: WorkloadDemand<'_>, - now_ms: u64, horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, cost_model: &dyn CostModel, -) -> Result, SummaryMaintenanceLifecycleSelectionError> - -assemble_selected_dag_with_summary_maintenance_lifecycles( - selection: &GlobalSelection<'_>, target: &Rc, - demand: WorkloadDemand<'_>, now_ms: u64, horizon: Option, - capabilities: SummaryMaintenanceLifecycleCapabilities, cost_model: &dyn CostModel, -) -> Result, SummaryMaintenanceLifecycleAssemblyError> -``` - -| Argument | Values / requirements | -| --- | --- | -| `space`, `demand` | Actual candidate space plus query demand, optional data evidence, and one normalized workload entry index for each `space.roots` entry | -| `target` | A root from `space.roots`, after canonical sharing | -| `demand` | `WorkloadDemand::new_with_data(...)` when data evidence is available; use `new_without_data(...)` only when its absence is intentional | -| `now_ms` | Actual planning time in Unix milliseconds for evidence freshness | -| `horizon` | `Some(Horizon(seconds))` with positive finite seconds, or `None` when horizon-dependent comparisons are unavailable | -| `capabilities` | Explicit Boolean fields below; several may be true | -| `cost_model` | A model supplying required lifecycle and raw-comparison evidence; default structural estimates are not enough | - -| Capability field | `true` permits consideration of… | `false` means… | -| --- | --- | --- | -| `supports_ephemeral` | Fresh build per invocation, retired afterward | Exclude that lifecycle | -| `supports_prepared` | Build before a predictable execution and retain until it | Exclude that lifecycle | -| `supports_shared` | Retain state for multiple reads | Exclude that lifecycle | -| `supports_continuously_maintained` | Keep state current as updates arrive | Exclude that lifecycle | - -All flags default to true; integrations should pass real support. Enabling a -flag does not override workload, algorithm-operation or evidence checks. - -### Example: lifecycle-aware planning for a batch-only runtime - -This helper takes the real workload and cost provider from your application. -It supports one searched root mapped to one workload entry, and returns a typed -plan/error rather than making up costs. For a shared root consumed by several -entries, construct demand using all applicable indices. - -```rust -use asap_aware_mapping::{ - global_selection_with_summary_maintenance_lifecycles, - assemble_selected_dag_with_summary_maintenance_lifecycles, CostModel, Horizon, CandidateLogicalASAPDAGs, - SummaryMaintenanceLifecycleCapabilities, SummaryMaintenanceLifecyclePlan, - WorkloadDemand, -}; -use asap_types::workload::PlanningWorkload; - -fn plan_batch_root( - space: &CandidateLogicalASAPDAGs<&str>, - workload: &PlanningWorkload, - entry_index: usize, - now_ms: u64, - horizon: Option, - model: &dyn CostModel, -) -> Result, Box> { - if space.roots.len() != 1 { - return Err("this example requires exactly one root".into()); - } - let capabilities = SummaryMaintenanceLifecycleCapabilities { - supports_ephemeral: true, - supports_prepared: false, - supports_shared: false, - supports_continuously_maintained: false, - }; - let indices = [entry_index]; - let demand = WorkloadDemand { - workload: &workload.query_workload, - data_workload: workload.data_workload.as_ref(), - entry_indices: &indices, - }; - let selection = global_selection_with_summary_maintenance_lifecycles( - space, demand, now_ms, horizon, capabilities, model, - )?; - let plan = assemble_selected_dag_with_summary_maintenance_lifecycles( - &selection, &space.roots[0].1, demand, - now_ms, horizon, capabilities, model, - )?; - if let Some(plan) = &plan { - println!("raw_recompute={}, deployments={:#?}", - plan.selected_raw_recompute, plan.deployments); - } - Ok(plan) -} -``` - -Use this helper with the `space` built by the search example and the corresponding -workload/provider. No incremental lifecycle is permitted, but unknown evidence -can still prevent choosing summary state. If only one legal alternative remains, -recording it is a complete lifecycle decision. Data-at-rest alone does not imply -that prepared or retained shared state is supported. - -| Function | Inputs | Output / promise | -| --- | --- | --- | -| `plan_summary_maintenance_lifecycles` | Assembled logical DAG root, `WorkloadDemand`, `now_ms`, optional horizon, runtime capabilities, cost model | `Result` for that fixed root; does not revisit all semantic candidates | -| `global_selection_with_summary_maintenance_lifecycles` | `CandidateLogicalASAPDAGs`, workload/root-entry associations, time, horizon, capabilities, cost model | Lifecycle-aware compatible selection/error, using eligible cost evidence | -| `assemble_selected_dag_with_summary_maintenance_lifecycles` | Selection, target root and lifecycle context | Optional lifecycle plan/error; attaches state deployment decisions | -| `enumerate_summary_maintenance_lifecycles` | Same inputs as `plan_summary_maintenance_lifecycles` | `SummaryMaintenanceLifecycleCandidates`: per unique retained state, every alternative with its cost or rejection; nothing selected. `guarantee(&lifecycle)` gives the mode/schedule that alternative would carry | -| `SummaryMaintenanceLifecycleCandidates::select(choices)` | One `(PostAsapNodeId, SummaryMaintenanceLifecycle)` per state, copied from `deployments()` | The same `SummaryMaintenanceLifecyclePlan` Planner selection would produce for that combination, or `SummaryMaintenanceLifecycleChoiceError` when a choice is unknown, missing, duplicated, rejected, schedule-incompatible, or not completely estimable | - -Inspect `deployments`, their selected lifecycle/alternatives/rejections, -`selected_raw_recompute`, and optional summary/raw costs. Success of a function -call alone is not a certificate that every desired summary was selected or fully -costed. A raw alternative remains a downstream execution obligation. - -Lifecycle feasibility and costs must affect final deployment comparison. Running -lifecycle analysis after structural selection can evaluate the selected root, -but does not make the earlier selection lifecycle-optimal. An application may -consume ranked candidates and perform this comparison downstream instead. - -A deployment that prices lifecycles itself calls -`enumerate_summary_maintenance_lifecycles`, prices the alternatives, and binds -its choice with `select`. A choice is accepted only if Planner could select it: -an alternative with `MissingCostEvidence` is accepted only when the cost model's -complete-candidate hook covers lifecycle costs. Window frameworks and totals come -from that hook, as in Planner selection. - -A lifecycle choice then fixes each physical placement through timing: a -continuously maintained state and its inputs run at ingestion time, while an -ephemeral one stays at query time. Compile each query's `PostAsapDAG` once and -cut every chosen assignment from that result: - -```rust -use asap_physical_operators::physical_planner::{ - compile, cut_candidate, frontier_from_timing, -}; - -let compiled = compile(&dag, inputs, &roots)?; // each node lowered once -for plan in lifecycle_plans { - let frontier = frontier_from_timing(&plan.execution_timed_dag()?)?; - // Precompute/query DAGs split at `frontier`; no logical lowering. - let candidate = cut_candidate(&compiled, &frontier)?; - // Check feasibility and price `candidate`; bind the selected one as is. -} -``` - -The frontier is the set of ingestion-time nodes read by query-time nodes (or an -ingestion-time root). `frontier_from_timing` rejects a query-time node feeding -an ingestion-time node. `cut_candidate` returns exactly what -`compile_candidate(&dag, inputs, &roots, &frontier)` returns and rejects the -same invalid frontiers. If the DAG has an ingestion-time `Binary`, compile with -the same timing for that node, because it lowers differently. Temporal pane -candidates are a different lowering and still use -`compile_temporal_pane_candidate`. - -Retained states are `SummaryAgg` nodes and `MaintainPopulation` nodes that do -not feed a `SummaryAgg`; a population that does feed one is part of that -state's input. The lifecycle cost hooks (`summary_maintenance_capabilities`, -`summary_maintenance_lifecycle_cost_inputs_for_horizon`) and the complete-candidate -hook therefore also receive `MaintainPopulation` nodes. A model that does not -recognize one should return unknown costs, which keep its alternatives -unselected; a model that prices every node uniformly now also prices -populations, so population candidates can win lifecycle-aware selection. `SummaryMaintenanceLifecyclePlan::execution_timed_dag` times a -population as it times a summary state: retained at ingestion, `Ephemeral` at -query time from the raw source. - ## Optional whole-plan selection and DAG assembly ### What does global selection mean? @@ -727,15 +547,15 @@ constructs the selected semantic DAG while preserving shared nodes. | `cost_sorted()` | How are the alternatives ranked for each subexpression? | Ranked alternatives per target | | `global_selection()` | Which compatible choices should be used together, accounting for sharing and dependencies? | A coordinated selection across targets under the supplied model | -Plain `global_selection()` does not automatically perform lifecycle planning or -establish physical deployment feasibility. Use the corresponding evidence-aware -workflow for those decisions. Downstream still owns physical commitment. +Plain `global_selection()` does not decide materialization or establish +physical deployment feasibility. Stage 2 materialization (#509) will own +materialization; downstream still owns physical commitment. | Method on `CandidateLogicalASAPDAGs` / `GlobalSelection` | Behavior | | --- | --- | -| `CandidateLogicalASAPDAGs::global_selection(&model)` | Compatible structural selection across targets; no recurrence or lifecycle planning implied | -| `CandidateLogicalASAPDAGs::global_selection_with_recurrence(...)` | Compatible selection using supplied recurrence profiles/horizon; no lifecycle commitments implied | -| `GlobalSelection::assemble_selected_dag(&target)` | `Result>, RealizationError>`; constructs semantic IR, not stored summary data | +| `CandidateLogicalASAPDAGs::global_selection(&model)` | Compatible structural selection across targets; no recurrence or materialization planning implied | +| `CandidateLogicalASAPDAGs::global_selection_with_recurrence(...)` | Compatible selection using supplied recurrence profiles/horizon; no materialization commitments implied | +| `GlobalSelection::assemble_selected_dag(&target)` | `Result>, RealizationError>`; constructs untimed semantic IR, not stored summary data | Use a target associated with the searched space; DAG assembly can return `None` when that target is absent. A downstream integration can use these convenience @@ -747,16 +567,15 @@ for checking complete physical alternatives and deployment constraints. ```text CandidateLogicalASAPDAGs::global_selection(&self, cost_model: &dyn CostModel) -> GlobalSelection<'_> -GlobalSelection::assemble_selected_dag(&self, target: &Rc) - -> Result>, RealizationError> +GlobalSelection::assemble_selected_dag(&self, target: &Rc) + -> Result>, RealizationError> ``` For structural inspection only, this complete example selects a semantic root -and exports its inspection DAG. It performs no lifecycle or deployment planning. -Use lifecycle-aware selection above when the comparison needs those decisions. +and exports its inspection DAG. It performs no materialization or deployment +planning. ```rust -use std::rc::Rc; use asap_frontend_promql::lower_promql_workload; use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, Query, @@ -790,7 +609,7 @@ fn main() -> Result<(), Box> { ..Default::default() }), }; - let root = Rc::new(lower_promql_workload(&workload, 0)?.remove(0)); + let root = lower_promql_workload(&workload, 0)?.remove(0); let space = search_workload(vec![("q1", root)]); let selection = space.global_selection(&DefaultCostModel); // Search may canonicalize roots; use the root returned by CandidateLogicalASAPDAGs. @@ -806,15 +625,15 @@ fn main() -> Result<(), Box> { | Function/type | Purpose | | --- | --- | -| `asap_types::dag_export::export(&query)` | Pre-ASAP inspection DAG | -| `asap_types::dag_export::export_summary(&summary)` | Post-ASAP inspection DAG | -| `asap_types::post_asap::compile_post_asap_dag(&root)` | Compile a semantic DAG with execution-data-state validation; not a physical plan | +| `asap_types::dag_export::export(&query)` | Pre-ASAP inspection dag | +| `asap_types::dag_export::export_summary(&summary)` | Post-ASAP inspection dag | +| `asap_types::ir::apply_materialization_timings(&root, &assignment, &mut TimingMemo::new())` | Write execution timing into every node from a `MaterializationAssignment` (default: all query time) and validate the data-state edges; `PlanOutput::execution_timed_dag()` applies the default to a planned workload | +| `asap_types::ir::export::compile_post_asap_dag(&timed_root)` | Export a timed DAG as a `PostAsapDAG` (wire version 7); rejects an untimed node; not a physical plan | | `PostAsapDAGDocument::new(dag)` and `.validate()` | Versioned semantic envelope and explicit validation; constructing it alone does not validate | -| `asap_aware_mapping::export_summary_maintenance_plan(&plan)` | DAG plus lifecycle deployments, alternatives and available cost/guarantee information | | `explain_replacements` / `explain_replacements_with` | Findings from default/custom-strategy search; not a complete physical feasibility report | Choose the export matching your intended handoff: an inspection DAG is not -interchangeable with a versioned execution contract. Preserve lifecycle and +interchangeable with a versioned execution contract. Preserve cost/guarantee evidence needed downstream instead of exporting only a bare DAG. For public symbol details, build local API documentation with: @@ -827,6 +646,5 @@ cargo doc -p asap-aware-mapping -p asap-types --no-deps - [Frontend PromQL](../../crates/frontend-promql/src/lib.rs), [SQL](../../crates/frontend-sql/src/lib.rs), [MetricsQL](../../crates/frontend-metricsql/src/lib.rs) - [Search, ranking and selection](../../crates/asap-aware-mapping/src/replacement.rs) - [Cost models](../../crates/asap-aware-mapping/src/cost_model.rs) -- [Lifecycle APIs](../../crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs) - [Workload types](../../crates/types/src/workload.rs) - [Planner-runtime contract](../design_docs/architecture/planner-runtime-contract.md) diff --git a/docs/develop_docs/metrics-observability-corpora.md b/docs/develop_docs/metrics-observability-corpora.md index d127e92ce..129ea0af1 100644 --- a/docs/develop_docs/metrics-observability-corpora.md +++ b/docs/develop_docs/metrics-observability-corpora.md @@ -49,19 +49,19 @@ o11y-bench, and awesome-prometheus-alerts. They are not duplicated here. The test prints totals, parse errors, lowering errors, pre-ASAP successes, post-ASAP candidates, unchanged queries, and post-ASAP errors. `Pre-ASAP` means -that parsing and lowering produced a `QueryExpr`. `Post-ASAP candidate` means -the isolated `SketchAlgorithmStrategy` produced a non-`KeepPreAsap` summary -candidate. `Unchanged` is a successful pre-ASAP query for which that strategy -returned only the pre-ASAP fallback. +that parsing and lowering produced an `OperatorNode` DAG. `Post-ASAP candidate` +means the isolated `ASAPStrategies` produced a candidate that contains +an ASAP operator (`contains_asap()`). `Unchanged` is a successful pre-ASAP query +for which that strategy returned only the kept pre-ASAP sub-DAG (`retain_exact`). ## Strategies The corpus measurement deliberately uses only -`SketchAlgorithmStrategy::default_cost_model().replacements(...)` on each +`ASAPStrategies::default_cost_model().replacements(...)` on each query root. It does not measure workload-wide search or the other default strategies. -The default workload search currently registers `SketchAlgorithmStrategy`, +The default workload search currently registers `ASAPStrategies`, `HydraGroupingStrategy`, `SharedSubDAGStrategy`, and `AvgToSumOverCountStrategy`. Workload context can additionally contribute `RollupStrategy` and `AccuracyReconciliationStrategy`. This baseline is diff --git a/docs/develop_docs/native-promql-inputs.md b/docs/develop_docs/native-promql-inputs.md index d263b7825..d2c10f535 100644 --- a/docs/develop_docs/native-promql-inputs.md +++ b/docs/develop_docs/native-promql-inputs.md @@ -23,7 +23,7 @@ operator's metric-name/result-label rules. Source selection, complete window coverage and revision admission remain deployment responsibilities. Planner's maintained-population candidate recognizes this explicit identity -representation. Its TopK readout compiles automatically to `CurrentSeries`, +representation. Its TopK evaluation compiles automatically to `CurrentSeries`, `Sort`, and `Limit`; deployment supplies the raw boundary or an already maintained population boundary. Compilation does not open either source. diff --git a/docs/develop_docs/offline-sketch-evidence.md b/docs/develop_docs/offline-sketch-evidence.md index cd331e57d..4a6d0c2ca 100644 --- a/docs/develop_docs/offline-sketch-evidence.md +++ b/docs/develop_docs/offline-sketch-evidence.md @@ -60,8 +60,7 @@ distribution or machine; the provider does not interpolate between datasets. Each measured resource is an optional `Measurement` with `value`, optional `stddev`, `samples`, and optional `method`. CPU fields are process CPU nanoseconds per operation; `build_cpu_ns` measures empty construction. Building an ingested -snapshot additionally requires `sample_count × update_cpu_ns`; the lifecycle -helper returns that sum only when both measurements exist. Memory and disk +snapshot additionally requires `sample_count × update_cpu_ns`. Memory and disk fields are bytes; `scan_bytes` records bytes read by scans, not storage occupancy. Producer methods must state what was measured and how normalization was performed. `retained_bytes` is distinct @@ -87,12 +86,10 @@ scores as CPU or measured savings. Deployment cost models can own the provider and call `lookup` with their own parameter sizing. This preserves the deployment's other cost and capability -hooks. The provider's lifecycle helper returns available build/update CPU costs -for a single independently instantiated state. It deliberately leaves retention, -retirement and read costs unknown. In particular, a point-frequency benchmark -read does not price a total-count read, even when both use CMS. A deployment must -match readout semantics and supply the missing lifecycle and raw-query evidence -before selecting and pricing a complete physical plan. Never combine these +hooks. A point-frequency benchmark read does not price a total-count read, even +when both use CMS. A deployment must match evaluation semantics and supply +retention, retirement, read and raw-query evidence before selecting and pricing +a complete physical plan. Never combine these nanosecond costs with CPU operation counts without explicit calibration. `error` contains offline observed statistics and a query descriptor. Its metric @@ -116,7 +113,7 @@ not be passed as these disjoint phase measurements. `MeasurementQueryBinding` is the producer's explicit assertion identifying the read/error probe population. The consumer checks that binding and the error -record's readout kind/value type; it cannot recover or certify the original +record's evaluation kind/value type; it cannot recover or certify the original probe set from an aggregate error number alone. The supported workload is an immutable i64 point-frequency snapshot, fully @@ -131,7 +128,7 @@ post-merge error and an exact merge baseline exist. The caller supplies an `EmpiricalAccuracyRequirement`: the exact observed error metric, maximum accepted mean, and minimum number of offline trials. This is -separate from `AccuracyTarget`. Every candidate must match the readout descriptor, +separate from `AccuracyTarget`. Every candidate must match the evaluation descriptor, error metric, trial count and all ordinary distribution/configuration/environment checks. A zero observed error is neither proof of exactness nor a per-key bound. diff --git a/docs/develop_docs/physical-compile-coverage.md b/docs/develop_docs/physical-compile-coverage.md index 5d048d7f7..720767696 100644 --- a/docs/develop_docs/physical-compile-coverage.md +++ b/docs/develop_docs/physical-compile-coverage.md @@ -5,10 +5,11 @@ Audience: developers moving computation from ASAPQuery-backend into ## Contract -Logical selection decides what to compute. The maintenance lifecycle sets node -timing. `physical_planner::compile` turns a timed `PostAsapDAG` into physical +Logical selection decides what to compute. A `MaterializationAssignment` sets +node timing (all query time until Stage 2 materialization, #509, decides +otherwise). `physical_planner::compile` turns a timed `PostAsapDAG` into physical operator DAGs. The backend owns ingestion, panes, storage, stored-state -readout, external exact engines, pricing/selection, and execution scheduling. +evaluation, external exact engines, pricing/selection, and execution scheduling. A backend lowering is *covered* when `compile` accepts the corresponding `PostAsapDAG` node and produces operators with the same result. The backend @@ -29,7 +30,7 @@ Status values: | # | Backend site | Computation | Planner node | Status at #475 | Notes | |---|---|---|---|---|---| -| 1 | `query_time.rs` `Lower::lower`, `compile_logical` | PromQL AST → `QueryTimeOperator` DAG for a native query | `Fallback { QueryExpr }` sub-DAGs plus value payloads | Missing | `compile` lowers `Fallback` only as a raw `Scan` source. | +| 1 | `query_time.rs` `Lower::lower`, `compile_logical` | PromQL AST → `QueryTimeOperator` dag for a native query | `Fallback { QueryExpr }` sub-DAGs plus value payloads | Missing | `compile` lowers `Fallback` only as a raw `Scan` source. | | 2 | `QueryTimeOperator::Aggregate` (sum/min/max/avg/count) | Grouped value aggregation | `Value::Exact(Aggregate)`; `SummaryAgg{ExactAggregate, Reduce}` over finalized values | Supported | Also `promql_values::compile_aggregate`. | | 3 | `QueryTimeOperator::Sort`, `Limit` (topk, sort, sort_desc) | Ordering and per-group limits | `Value::Sort`, `Value::Limit` | Supported | | | 4 | `QueryTimeOperator::Binary`, `QueryPlanNode::Binary` (vector ⊗ scalar) | Arithmetic with a scalar operand | `Binary` whose operand is `Fallback{PromqlScalarBridge(Literal)}` | Missing | Query-time `Binary` accepts only label-map vector schemas. The literal node has no native binding. | @@ -43,15 +44,15 @@ Status values: | 12 | `logical_dag.rs` `Subquery`, `subquery_grid`, `expanded_inputs` | Re-evaluate the child on a step grid and assemble a matrix | `Fallback{PromqlSubquery}` | Missing | No Planner operator. | | 13 | `QueryPlanNode::Scalar`, `DAGCompiler::lower` scalar literal | Scalar constant | `Fallback{PromqlScalarBridge(Literal)}` | Missing | Only `promql_values::compile_scalar`. | | 14 | `DAGCompiler::lower` `ReduceSum`; `physical_values.rs` PerEntity projection | Sum over finalized values; per-entity identity | `SummaryAgg{ExactAggregate(Sum)}` | Supported | The backend builds an identity `Operator::project` itself for PerEntity. | -| 15 | `DAGCompiler::lower` `ExactReadout`; `post_asap_readout.rs` ExactReadout | Finalize exact state (sum/count/min/max/rate/increase) | `Value::FinalizeExactAccumulator` | Partial | Count yields Int64 against a declared Float64 PromQL value. `compile` rejects it. | -| 16 | `post_asap_readout.rs` SummaryEstimate (`readout_bound`, `expand_item_rows`) | Sketch estimate per group; TopK item expansion | `SummaryEstimate` | Partial | The backend's label-map state layout and MetricsQL `__name__` rules have no Planner equivalent. `compile_exact_readout` has no sketch counterpart. | -| 17 | `post_asap_readout.rs` SummaryMerge (`merge_bound_states`) | Merge states by group | `SummaryMerge` | Supported | Union plus `summary_merge`. | -| 18 | `post_asap_readout.rs` counter range parameters | Counter lookback for rate/increase | `TimeRange` ancestor of finalization | Supported | Applied through `with_counter_lookback`. | -| 19 | `post_asap_readout.rs` `execute_value_fragment` | Per-timestamp binding of a value fragment | n/a | Backend | Evaluation scheduling. | +| 15 | `DAGCompiler::lower` `ExactEvaluation`; `post_asap_evaluation.rs` ExactEvaluation | Finalize exact state (sum/count/min/max/rate/increase) | `Value::FinalizeExactAccumulator` | Partial | Count yields Int64 against a declared Float64 PromQL value. `compile` rejects it. | +| 16 | `post_asap_evaluation.rs` SummaryEstimate (`evaluation_bound`, `expand_item_rows`) | Sketch estimate per group; TopK item expansion | `SummaryEstimate` | Partial | The backend's label-map state layout and MetricsQL `__name__` rules have no Planner equivalent. `compile_exact_evaluation` has no sketch counterpart. | +| 17 | `post_asap_evaluation.rs` SummaryMerge (`merge_bound_states`) | Merge states by group | `SummaryMerge` | Supported | Union plus `summary_merge`. | +| 18 | `post_asap_evaluation.rs` counter range parameters | Counter lookback for rate/increase | `TimeRange` ancestor of finalization | Supported | Applied through `with_counter_lookback`. | +| 19 | `post_asap_evaluation.rs` `execute_value_fragment` | Per-timestamp binding of a value fragment | n/a | Backend | Evaluation scheduling. | | 20 | `DAGCompiler::lower` SummaryJoin / Subtract / Delete | Summary algebra | `SummaryJoin`, `SummarySubtract`, `SummaryDelete` | Missing | The backend also rejects these (`ExactFallback`). | -| 21 | `current_series.rs` Snapshot + TopK | Current-series ranking | `ReadPopulation{TopK}` | Supported | | -| 22 | `current_series.rs` Sum / Count / Average | Current-series aggregates | `ReadPopulation{Sum,Count,Average}` | Missing | `compile` accepts only TopK. | -| 23 | `current_series.rs` Quantile | Current-series quantile | `ReadPopulation{Quantile}` | Missing | No exact quantile reduction. | +| 21 | `current_series.rs` Snapshot + TopK | Current-series ranking | `EvaluatePopulation{TopK}` | Supported | | +| 22 | `current_series.rs` Sum / Count / Average | Current-series aggregates | `EvaluatePopulation{Sum,Count,Average}` | Missing | `compile` accepts only TopK. | +| 23 | `current_series.rs` Quantile | Current-series quantile | `EvaluatePopulation{Quantile}` | Missing | No exact quantile reduction. | | 24 | `raw_dag.rs` weight `Column` | Summary update from a sample/projected value | `SummaryAgg` | Supported | | | 25 | `raw_dag.rs` weight `Constant` | Unit/constant-weight update | `SummaryAgg` | Missing | `compile_node` requires a column weight. | | 26 | `raw_dag.rs` item `Column` / `Tuple` | Keyed update item | `SummaryAgg{item}` | Supported | `keyed_summary_build`. | @@ -70,7 +71,7 @@ Totals at #475: 11 Supported, 4 Partial, 14 Missing, 2 Backend. | 4, 8, 13 | Query-time `Binary` folds a scalar-literal operand into a projection over grouped value rows. | | 5 | Query-time `Binary` over grouped value rows performs an inner equi-join on equal label columns, then applies the operator. Per-series rows remain Partial. | | 15 | Count finalization converts exactly to the declared Float64 value. | -| 22, 23 | `ReadPopulation` Sum/Count/Average/Quantile compile to grouped aggregation. `Reduction::Quantile` implements PromQL interpolation. | +| 22, 23 | `EvaluatePopulation` Sum/Count/Average/Quantile compile to grouped aggregation. `Reduction::Quantile` implements PromQL interpolation. | Totals after this change: 17 Supported, 4 Partial, 8 Missing, 2 Backend. @@ -124,10 +125,10 @@ Totals are unchanged: 19 Supported, 5 Partial, 5 Missing, 2 Backend. | Row | Change | |---|---| -| 5 | Query-time `Binary` over rows with a series identity, such as per-series readouts of stored state, uses the Fallback's `series_labels` and `series_binary`. Examples: `avg_over_time` as stored sum/count, and `rate(a) / rate(b)`. Matching drops `__name__` and honors `on`/`ignoring` when the payload carries them. Only one-to-one arithmetic is covered; `group_left`/`group_right` stay rejected and comparisons are row 7. Now Supported. | +| 5 | Query-time `Binary` over rows with a series identity, such as per-series evaluations of stored state, uses the Fallback's `series_labels` and `series_binary`. Examples: `avg_over_time` as stored sum/count, and `rate(a) / rate(b)`. Matching drops `__name__` and honors `on`/`ignoring` when the payload carries them. Only one-to-one arithmetic is covered; `group_left`/`group_right` stay rejected and comparisons are row 7. Now Supported. | | 4, 8 | A literal operand also applies to per-series rows and drops `__name__`, in the Fallback too. Series whose label sets become equal are an error, as in Prometheus. | -Grouped `sum`/`avg`, current-series `Sum`/`Average` readouts, and +Grouped `sum`/`avg`, current-series `Sum`/`Average` evaluations, and `sum_over_time`/`avg_over_time` use Prometheus' Kahan-Neumaier summation. An average switches to an incremental mean once the running sum would overflow. The grouped path also serves SQL `SUM`/`AVG` over Float64, which are now @@ -175,7 +176,7 @@ and `group_left`, including a right-side series identity when needed. Thus |---|---| | 1 | Comparisons, `bool`, set operators, `group_left`/`group_right`, `scalar()` operands, and literals over aggregates whose value has another name, such as `sum by (job) (a) * 2`. Still Partial. | | 5 | Grouped `Binary` rows use the same operator instead of a relational join. A duplicate match group is now an error instead of a cross product. | -| 7 | Fallback, grouped `Binary`, and per-series comparisons and sets. Temporal stored readouts drop `__name__` before matching, including exact Count conversion, and reject duplicate output identities. | +| 7 | Fallback, grouped `Binary`, and per-series comparisons and sets. Temporal stored evaluations drop `__name__` before matching, including exact Count conversion, and reject duplicate output identities. | Totals after this change: 20 Supported, 5 Partial, 4 Missing, 2 Backend. @@ -188,7 +189,7 @@ Totals after this change: 20 Supported, 5 Partial, 4 Missing, 2 Backend. An argument whose output provably lacks `le`, such as `sum by (job) (rate(x_bucket[5m]))`, is rejected at lowering. Prometheus returns an empty vector for it. Candidate search keeps the classic form as one -exact `KeepPreAsap` sub-DAG for every accuracy target; it has no sketch +retained exact ordinary sub-DAG for every accuracy target; it has no sketch candidate. `histogram_quantiles` lowers each branch the same way; the Fallback compiler accepts its `Concat` of relabeled branches and rejects duplicate output label sets. Nested aggregation, such as @@ -208,8 +209,8 @@ In order of backend usage: After these shapes are covered, the backend can delete rows 28 and 30. 2. Rows 25 and 27: constant weights and `EntityIdentity` items for precompute `SummaryAgg`. -3. Row 16: a label-map sketch-state readout, the counterpart of - `compile_exact_readout`, and MetricsQL `__name__` retention rules. +3. Row 16: a label-map sketch-state evaluation, the counterpart of + `compile_exact_evaluation`, and MetricsQL `__name__` retention rules. 4. Row 20: summary join, subtract, and delete. `fill`, `fill_left`, and `fill_right` matching modifiers are rejected by the diff --git a/docs/develop_docs/physical-handoff-costs.md b/docs/develop_docs/physical-handoff-costs.md index 29525f2c5..dcd95e747 100644 --- a/docs/develop_docs/physical-handoff-costs.md +++ b/docs/develop_docs/physical-handoff-costs.md @@ -88,8 +88,8 @@ traffic is inferred from logical edges, operator buffers, or scan bytes. Unknown endpoints, mismatched payloads, absent node evidence, duplicate IDs, stale evidence, invalid coefficients, and integer overflow return typed errors; ranking/export report the comparison as unavailable. This extends the physical -plan adapter; lifecycle-specific summary-maintenance costing and caching are -separate follow-up integration points. +plan adapter; summary-maintenance costing for Stage 2 materialization (#509) is +a separate follow-up integration point. Verification: diff --git a/docs/develop_docs/planner-vocabulary-migration.md b/docs/develop_docs/planner-vocabulary-migration.md index ea46e992c..d4b29fecd 100644 --- a/docs/develop_docs/planner-vocabulary-migration.md +++ b/docs/develop_docs/planner-vocabulary-migration.md @@ -32,8 +32,8 @@ names. | Physical evidence/comparison `boundaries` fields | `handoffs` | | `BoundaryEstimate::per_boundary` | `PhysicalHandoffEstimate::per_handoff` | | Internal `Models` | `CandidatePlanningInputs` | -| `SketchAlgorithmStrategy::with_models` | `SketchAlgorithmStrategy::new_with_planning_inputs` | -| `SketchAlgorithmStrategy::with_models_and_evidence` | `SketchAlgorithmStrategy::new_with_planning_inputs_and_evidence` | +| `ASAPStrategies::with_models` | `ASAPStrategies::new_with_planning_inputs` | +| `ASAPStrategies::with_models_and_evidence` | `ASAPStrategies::new_with_planning_inputs_and_evidence` | | `HydraGroupingStrategy::with_models_and_evidence` | `HydraGroupingStrategy::new_with_planning_inputs_and_evidence` | For example, `Binder::new().bind(&dag)` becomes diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index abf5dc50b..04f71b15a 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -2,7 +2,15 @@ This is the detailed node reference. Start with the [Pre-ASAP IR concept](../design_docs/concepts/pre-asap-ir.md) for purpose and the compact catalog. -The goal of the pre-ASAP IR is represent operations from different query languages in a single representation, and make it easier to analyze how/where ASAP primitives can be used. +ASAPPlanner has **one operator IR before and after ASAP optimization**, defined in +`crates/types/src/ir/`. "Pre-ASAP" is not a separate type: it is this IR as a front end +emits it, before any ASAP operator has been introduced. This document covers what every +plan shares — the node, the schema, scalar expressions, how front ends produce the DAG, and +the catalog of ordinary (`NonASAPOp`) operators. The ASAP operators, execution timing and +the exported wire form are described in the [Post-ASAP IR](../design_docs/concepts/post-asap-ir.md) +document; the two do not repeat each other. + +The goal of the pre-ASAP form is to represent operations from different query languages in a single representation, and make it easier to analyze how/where ASAP primitives can be used. Only operations that are semantically relevant to answering the query and selecting an ASAP primitive need to become first-class nodes here. ## Design principles @@ -13,7 +21,135 @@ Only operations that are semantically relevant to answering the query and select > Notes: **SQL and PromQL use different schema models**. SQL typically uses a closed schema, where tables, columns, and types are predefined, while PromQL uses an open (schemaless) schema, where metrics and labels can evolve without a fixed table schema. Closed schemas provide stronger structure and validation; open schemas provide greater flexibility and makes it easier to evolve or ingest diverse data, but can require more care around naming conventions, label cardinality, and query consistency. -The pre-ASAP IR is defined using the `QueryExpr` enum. We discuss some of important enum types below. +## The node + +A plan is a DAG of `Rc` (`crates/types/src/ir/node.rs`). Nodes are immutable +and shared through `Rc`: a structurally identical sub-DAG referenced from several parents is +one node, and that pointer identity is what CSE, target discovery and plan assembly key on. + +```rust +pub struct OperatorNode { + pub operator: Operator, // NonASAP(NonASAPOp) | ASAP(ASAPOp) + pub result_kind: OperatorResultKind, // Relation | InstantVector | RangeVector | State | Scalar + pub schema: Schema, // output schema, derived at construction + pub guarantee: Option, // None until accuracy assessment establishes one + pub timing: Option, // None until a materialization assignment is applied +} +``` + +- `operator` is the operation. A front-end DAG contains only `Operator::NonASAP` nodes; + `OperatorNode::expect_non_asap()` relies on that. +- `result_kind` is the output category, derived from the operator and its inputs. Matching + column schemas do not make categories interchangeable (a range vector is not an instant + vector). +- `schema` is derived by `OperatorNode::new(operator)`; it fails when the schema cannot be + derived (a column reference out of range, a reserved ASAP operator). ASAP planning may + retain a more specific schema through `OperatorNode::with_schema`. +- `guarantee` is `None` until accuracy assessment establishes one; `None` never means exact. +- `timing` is `None` in every front-end DAG and every candidate. It is written by + `ir::timing::apply_materialization_timings` (see the Post-ASAP IR document); export rejects an + untimed node. + +`OperatorNode::children()` returns the operator's inputs in field order followed by the +operator nodes its scalar expressions read (see "Scalar expressions"). Every DAG traversal — +`map_children`, `reachable`, `contains_asap`, CSE, export — follows that same list. +`OperatorNode::validate_structure()` checks every operator's input contract, scalar typing +against the owning operator's input schema, and that each retained schema agrees with the +derived one. + +## Schema + +One `Schema` type (`crates/types/src/pre_asap/schema.rs`) describes every edge, whether it +carries rows or summary state: + +```rust +pub struct Schema { + pub fields: Vec, // positional; every ColumnId indexes into this + pub time_index: Option, // the time axis, if any (PromQL leaves always have one) + pub unique_keys: Vec>, + pub closed: bool, // true: these are all the columns; false: open (schemaless) superset +} + +pub struct Field { + pub name: String, + pub dtype: FieldDataType, // Plain(DataType) | ExactAggregate(..) | Sketch(..) | Sample(..) | Wavelet(..) | StatModel(..) + pub nullable: bool, + pub table: Option, // SQL table/alias qualifier; None for PromQL labels +} +``` + +A pre-ASAP field is always `FieldDataType::Plain(DataType)`. The other variants carry summary +state and only appear below an ASAP operator; a scalar expression that reads such a field is a +typing error (`ScalarExpr::scalar_type`), because state must be read out before a value can use +it. Column references are positional `ColumnId`s (indexes into the input schema), never names. + +`Schema::has_unique_key()` is the legality gate CSE uses: a non-ASAP producer is only shared +across consumers when its row identity is provable. + +## Scalar expressions + +Value computation lives in `ScalarExpr` (`crates/types/src/ir/scalar.rs`), owned **by value** +by an operator field: `Scan.predicates`, `Filter.pred`, `Join.pred`, `Project.cols[i].expr`, +`Aggregate.having`, `Sort.keys[i].expr`, `SQLWindowFunc.args`/`order_by`, `PromqlRelabel.value`, +`Values.rows`, and `QueryRoot::Scalar` and `PromqlVectorFromScalar`. A scalar expression never +produces a table and is never a node of the DAG; it is evaluated against the input schema of +the operator that owns it. + +Variants: `Column(ColumnId)`, `Literal(ScalarValue)`, `Negative` (unary minus), `Compare`, +`BoolAnd` / `BoolOr` (flat conjunction/disjunction), `Not`, `IsNull` / `IsNotNull`, `Cast` +(with `try_cast`), `InList`, `FunctionCall { name, args }`, `Arithmetic`, `Case`, +`CurrentTimestamp` (SQL `NOW()`), `EvalTimestamp` (PromQL `time()`), and four +**plan-reading** variants that reference an operator node: + +| Variant | Meaning | +|---|---| +| `PromqlScalarFromVector(Rc)` | PromQL `scalar(v)`: the single sample of an instant vector, NaN otherwise | +| `ScalarSubquery(Rc)` | Uncorrelated SQL scalar subquery: one column; zero rows is NULL, more than one row is an error | +| `Exists { subquery, negated }` | SQL `[NOT] EXISTS (subquery)` | +| `InSubquery { expr, subquery, negated }` | SQL `expr [NOT] IN (subquery)` over a one-column relation | + +These are the **only** operator references inside a scalar tree. `ScalarExpr::operator_refs()` +lists them, `NonASAPOp::children()` appends them after the operator's own inputs, and +canonicalization lowers the three SQL subquery forms to joins (see below), so a canonical SQL +DAG contains none of them. `PromqlScalarFromVector` survives canonicalization: its referenced +vector is a real plan dependency, exported as a `ScalarRef` edge. + +`Compare`, `Arithmetic` and `Negative` carry an `ExprSemantics` (`Sql` or `Promql`): both +languages use `Float64`, so the result type alone does not preserve NaN, ordering or error +rules, and the executing engine needs to know which language's rules apply. + +Wrapper types: `Predicate(ScalarExpr)`, `ProjectItem { alias, expr }`, +`SortKey { expr, ascending, nulls_first }`. + +## How a front end produces the DAG + +A front end never constructs `OperatorNode`s directly. It builds a name-based tree in +`crates/frontend-common` — `UnresolvedOp` / `UnresolvedScalar`, a mirror of `NonASAPOp` / +`ScalarExpr` in which every column reference is a `ColumnRef` and a PromQL `Scan` has no schema +yet — and calls `asap_frontend_common::resolve_root`, which does three things in order: + +1. **Resolution** — a bottom-up walk that binds every `ColumnRef` to a positional `ColumnId` + against the derived schema of the already-resolved child. A schemaless (PromQL) leaf gets + its binding schema from `SchemaResolver`, built from the names the query references. + `Join` / `SetOp` sides and the operators referenced from scalar positions are each bound as + a root in their own scope; a `BinaryOp` side additionally inherits the label names its + enclosing scope references. +2. **Schema derivation** — each `OperatorNode::new` derives the node's output schema and + result kind from the operator and its children. +3. **Canonicalization** — `asap_types::ir::canonicalize::canonicalize` erases structural + differences between semantically identical queries: it promotes an additive + `Limit { Sort { Aggregate } }` ranking to the `AggIntent::TopK` heavy-hitter shape, and + lowers `EXISTS` / `NOT EXISTS` / `IN (subquery)` predicates to `Join { Semi | Anti }` and a + scalar subquery to a `Join { Cross }` plus column reference. The pass is idempotent and + keeps the pointer identity of every untouched sub-DAG. + +The result is `Rc`. `lower_promql_workload`, `lower_sql` / `lower_sql_dialect` / +`lower_sql_batch` and `lower_metricsql` all return it. + +Workload search then runs structural CSE (`asap_types::ir::cse::share_common_sub_dags`) once +across every root: bottom-up hash-consing where the structural hash is only a filter and the +typed `PartialEq` decides sharing, following scalar references like any other input, and +gated by `Schema::has_unique_key()` for non-ASAP producers. ## Fields and column references @@ -47,27 +183,28 @@ to one source language. - [`Aggregate`](#aggregate) — collapses input rows into fewer output rows via a reduction and aggregate intents. **[Time-related nodes](#time-related-nodes)** -- [`TimeRange`](#timerange) — a range-vector lookback over the time axis (PromQL `[5m]`). +- [`TimeRange`](#timerange) — temporal selection over a time-series input (PromQL instant lookback or `[5m]` range selector). - [`TimeShift`](#timeshift) — shifts *when* a selector is evaluated (PromQL `offset`/`@`). - [`PromqlSubquery`](#promqlsubquery) — re-evaluates an instant-vector expression over a range at a given step. **[Relational nodes](#relational-nodes)** — common to both SQL and PromQL - [`Scan`](#scan) — identifies the logical data source. +- [`Values`](#values) — SQL `VALUES` rows, or the one empty row of a `SELECT` without `FROM`. - [`Filter`](#filter) — restricts rows using a predicate. - [`Project`](#project) — column projection (SQL `SELECT` list). -- [`BinaryOp`](#binaryop) — arithmetic / comparison / boolean composition of two inputs. +- [`BinaryOp`](#binaryop) — arithmetic / comparison / set composition of two inputs. - [`Sort`](#sort) — generic (non-heavy-hitter) order-by, optionally per-group. -- [`Limit`](#limit) — caps the row count, with an offset. +- [`Limit`](#limit) — caps the row count, with an offset, optionally per-group. - [`Dedup`](#dedup) — row-level deduplication. - [`Join`](#join) — logical join of two inputs. - [`SetOp`](#setop) — SQL's typed set operations (`UNION`/`INTERSECT`/`EXCEPT`). - [`Concat`](#concat) — exact, untyped `UNION ALL` of union-compatible branches. -**[PromQL-specific nodes](#promql-specific-nodes)** -- [`PromqlScalarBridge`](#promqlscalarbridge) — a scalar sub-expression at an operator-DAG position. -- [`EvalTimestamp`](#evaltimestamp) — the query evaluation time as a scalar (PromQL `time()`). +**[Scalar-position nodes](#scalar-position-nodes)** +- `QueryRoot::Scalar` — a standalone scalar expression, without an operator node. - [`PromqlVectorFromScalar`](#promqlvectorfromscalar) — promotes a scalar to a label-less instant vector. -- [`PromqlScalarFromVector`](#promqlscalarfromvector) — collapses a single-series vector to a scalar. + +**[PromQL-specific nodes](#promql-specific-nodes)** - [`PromqlRelabel`](#promqlrelabel) — per-series label rewrite (PromQL `label_replace`/`label_join`). - [`PromqlInfoEnrich`](#promqlinfoenrich) — left-join label enrichment from an info metric. - [`PromqlSeriesSample`](#promqlseriessample) — keeps a subset of whole series, not a reduction. @@ -75,6 +212,9 @@ to one source language. **[SQL-specific nodes](#sql-specific-nodes)** - [`SQLWindowFunc`](#sqlwindowfunc) — SQL analytic window function (`OVER (...)`). +PromQL `time()` and `scalar(v)` are scalar expressions (`ScalarExpr::EvalTimestamp`, +`ScalarExpr::PromqlScalarFromVector`), not nodes. + ## Aggregation-related nodes ### Aggregate @@ -109,7 +249,7 @@ list of aggregate intents (`measures`). value is still recomputed by the agg intent, e.g. `Rate`), for a computation with no `by(...)` clause to attach to. `PerEntity` is different from `by` for all columns, because in PromQL, it is schemaless and you don't know all columns beforehand. E.g. PromQL `rate(http_requests_total[5m])`, which has one rate value - per input series: + per input series: ```text Aggregate( @@ -117,7 +257,7 @@ list of aggregate intents (`measures`). measures = [Rate], output_names = [], having = None, - child = TimeRange(range = 5m, child = Scan("http_requests_total")) + child = TimeRange(range = 5m, kind = Range, child = Scan("http_requests_total")) ) ``` @@ -224,7 +364,7 @@ Example for `filters`: `count(CASE WHEN p THEN x END)` (`p`, plus `x IS NOT NULL` when `x` is nullable), and from `count(expr)` over any other nullable `expr` (`expr IS NOT NULL`), because canonical `Count` counts rows and never consults its argument. A filtered measure has no summary binding yet: - `asap-aware-mapping` keeps such an `Aggregate` as `KeepPreAsap`, and canonicalization does + `asap-aware-mapping` retains such an `Aggregate` as an ordinary exact sub-DAG, and canonicalization does not promote a filtered count ranking to a heavy-hitter `TopK`. Example for `having`: @@ -248,8 +388,8 @@ Example for `having`: **Rules/Invariants**: A filtering predicate will be passed to at the lowest node (closer to the leaves) in the AST/DAG that can express it — `Scan.predicates`, then `Aggregate.having`, then `Filter` as the fallback — so its constraint is visible at - the node it actually applies to, not behind an opaque wrapper, once pre-ASAP IR translates - to post-ASAP IR with summary binding. The upper nodes (closer to the root) in the AST/DAG can still have a `Filter` node with the same condition. This intentional duplication is for Summary related translation and optimizations. + the node it actually applies to, not behind an opaque wrapper, once summary binding reads it. + The upper nodes (closer to the root) in the AST/DAG can still have a `Filter` node with the same condition. This intentional duplication is for Summary related translation and optimizations. For example, `SELECT srcip, COUNT(*) AS cnt FROM packets GROUP BY srcip HAVING COUNT(*) > 10` pins `cnt > 10` to the lowest node that can express it, `Aggregate.having`: @@ -282,7 +422,7 @@ Example for `having`: Both are valid at once, and neither is derived from the other: `having` is the canonical spot a summary-aware pass reads to decide whether `Aggregate` can bind to a summary, while the outer `Filter` is what a plain logical evaluator runs without knowing `having` exists. The duplication is forward-looking groundwork for - once HAVING-aware summary binding (pre-ASAP-IR to post-ASAP-IR translation) lands. + once HAVING-aware summary binding lands. Neither direction of that push-down is enforced yet: the SQL front end doesn't populate `having` from a real `HAVING` clause (#201), and canonicalization doesn't fold an existing @@ -294,14 +434,21 @@ Example for `having`: ### TimeRange -Represents a range of time. Kept different from `Filter` to treat time as an explicit concern. +Temporal selection over a time-series input. Kept different from `Filter` to treat time as an +explicit concern. `kind` records which samples a PromQL selector reads: + +- `TimeRangeKind::Instant` — an instant selector: `range` is the lookback horizon and the + latest eligible sample per series is selected (the planner injects the declared + `data_ingestion_interval` around a bare selector). +- `TimeRangeKind::Range` — a range selector (`m[5m]`): every sample in the window. ```promql rate(http_requests_total[5m]) ``` **Fields:** -- `range` — how far back to look (the PromQL `[5m]` duration). +- `range` — how far back to look (the PromQL `[5m]` duration, or the instant lookback). +- `kind` — `Instant` or `Range`. - `child` — the input the range applies to. ### TimeShift @@ -350,9 +497,19 @@ the same logical data domain. **Fields:** - `source` — the logical data source (a table name or PromQL metric selector). - `predicates` — row-level filters pushed all the way down to this scan (Rules/Invariants - rule 1); enforced structurally at lowering time — a `Filter` directly over a `Scan` never - survives. -- `schema` — the binding schema every positional column reference in the DAG resolves against. + rule 1): PromQL label matchers and pushed-down `WHERE` conjuncts. +- `schema` — the binding schema every positional column reference in the tree resolves against. + A catalog-backed SQL leaf carries its catalog schema; a PromQL leaf carries the usage-derived + schema `SchemaResolver` built from the labels the query references. + +### Values + +SQL `VALUES` rows, or the one empty row of a `SELECT` without `FROM` +(`SELECT 1 + 1`). Row expressions have no input-column scope. + +**Fields:** +- `rows` — one `Vec` per row. +- `schema` — the output schema of the rows. ### Filter @@ -386,6 +543,10 @@ that's neither a base scan column nor an aggregate output: SELECT * FROM (SELECT srcip, bytes_in + bytes_out AS total FROM packets) t WHERE total > 500 ``` +A `Filter` whose predicate contains `EXISTS` / `NOT EXISTS` / `IN (subquery)` does not +survive canonicalization: the conjunct becomes a `Join { Semi | Anti }` under the remaining +predicate. + **Fields:** - `pred` — the row-level predicate to apply. - `child` — the input being filtered. @@ -406,18 +567,21 @@ SELECT srcip, dstip FROM packets ### BinaryOp -Arithmetic / comparison / boolean composition. PromQL binary operators between two vectors, -a vector and a scalar, or two scalars. +Arithmetic / comparison / set composition of two operands. PromQL binary operators between two vectors, +two vectors. Mixed vector/scalar arithmetic uses `Project`; non-bool comparison uses `Filter`. Standalone scalar expressions are `QueryRoot::Scalar`. ```promql up > 1 ``` **Fields:** -- `op` — the arithmetic/comparison/boolean operator. +- `operator` — a `BinaryOperator { kind, vector_match, checked_relative_division, checked_finite_division }`: + - `kind` — `BinaryOpKind::Arithmetic(..)`, `Compare(..)` or `Set(..)` (PromQL `and`/`or`/`unless`). + - `vector_match` — PromQL vector-matching modifiers (`on`/`ignoring`, `group_left`/`group_right`); `None` outside PromQL and the only supported value today. + - `checked_relative_division` / `checked_finite_division` — typed division guards set by summary planning, never by a front end (see [physical-plan integration](../design_docs/architecture/physical-plan-integration.md#conditional-temporal-average-lowering)). +- `return_bool` — the PromQL `bool` modifier: a comparison returns `0`/`1` instead of filtering. Valid only for comparison operators. - `lhs` — the left operand. - `rhs` — the right operand. -- `vector_match` — PromQL vector-matching modifiers (`on`/`ignoring`, `group_left`/`group_right`); `None` outside PromQL. ### Sort @@ -429,7 +593,7 @@ sort_desc(up) ``` **Fields:** -- `keys` — the ordering columns/expressions and direction. +- `keys` — the ordering expressions and direction (`SortKey`). - `partition_by` — grouping keys that make the ordering per-group instead of global; empty = a single global order. - `child` — the input being ordered. @@ -443,8 +607,9 @@ topk(3, up) ``` **Fields:** -- `n` — the maximum number of rows to keep. +- `n` — the maximum number of rows to keep; `None` is offset-only. - `offset` — how many leading rows to skip first. +- `partition_by` — applies the limit per group (PromQL `topk by (..)`); empty = global. - `child` — the input being capped. ### Dedup @@ -463,14 +628,16 @@ SELECT DISTINCT srcip, dstip FROM packets ### Join -Logical join; the physical strategy (hash/merge/broadcast) is picked in the post-ASAP IR. SQL `JOIN`. +Logical join; the physical strategy (hash/merge/broadcast) is picked downstream of the planner. SQL `JOIN`, +and the shape canonicalization lowers subqueries to. ```sql SELECT u.prefix FROM bgp_updates u JOIN bgp_rib_state r ON u.prefix = r.prefix ``` **Fields:** -- `kind` — the join type (inner/left/right/full/semi/anti). +- `kind` — the join type (`Inner`/`Left`/`Right`/`Full`/`Cross`/`Semi`/`Anti`). A semi/anti join + outputs the left input's columns alone, but its predicate resolves against `left ++ right`. - `pred` — the join condition. - `left` — the left input. - `right` — the right input. @@ -496,7 +663,7 @@ SELECT srcip FROM packets UNION ALL SELECT dstip FROM packets never dedup. Used when a single `Aggregate` can't express the shape — the canonical case is PromQL `histogram_quantiles` (one branch per φ, each its own `HistogramQuantile` reduction relabeled with its `le` value) — and SQL `ROLLUP`/`CUBE`/`GROUPING SETS` (one branch per -grouping level). +grouping level). The output schema is the first child's. ```promql histogram_quantiles(rate(http_request_duration_seconds_bucket[5m]), "le", 0.5, 0.9) @@ -504,39 +671,32 @@ histogram_quantiles(rate(http_request_duration_seconds_bucket[5m]), "le", 0.5, 0 **Fields:** - `children` — the union-compatible branches to concatenate; must be non-empty. +- `discriminator_unique_key` — an optional caller-proven compound unique key + `(discriminator, inner_key)` over the output; nothing verifies the claim. -## PromQL-specific nodes - -### PromqlScalarBridge +## Scalar-position nodes -A scalar sub-expression (issue #220: in practice always `Literal(ScalarValue::Float64(_))` — -a PromQL number literal, or a folded constant scalar expression) sitting at an **operator-DAG -position** — a `BinaryOp` operand for ` op ` thresholds and unit conversions, -a `PromqlVectorFromScalar` child, or a whole query's root. This wrapper is what marks the -position; it no longer duplicates `Literal`'s value the way the old `PromqlScalar(f64)` variant -did. - -```promql -up > 1 -``` +### Scalar query roots -**Fields:** a single unnamed child `QueryExpr` — the wrapped scalar sub-expression. +`QueryRoot` distinguishes an operator result from an owned `ScalarExpr`. It is +an API root discriminator, not an operator. `2`, `time()`, and +`scalar(sum(up)) + 1` therefore introduce no constant-wrapper nodes. -### EvalTimestamp +Use `lower_promql_query_workload` for mixed scalar/vector workloads. The +operator-only convenience API rejects standalone scalar roots. `ParsedWorkload` +retains each scalar's workload index; `PlanOutput::roots()` returns all results +in workload order. Scalar plan reads remain exact and retain their operator +references; summary selection currently operates on operator roots. -The query **evaluation timestamp** as Unix seconds — PromQL `time()` — and the implicit -input of the no-argument calendar functions (`hour()`, `day_of_week()`, ...). It is the -instant or range-step at which the expression is evaluated, not inherently the current -wall-clock time. The Prometheus instant-query HTTP API separately defaults an omitted -`time` request parameter to the server's current time. - -```promql -time() -``` +`up * 2` projects the sample expression while retaining time and full series +identity, removing the metric name. `up > 0` and `0 < up` filter the vector and +retain its sample and name. `up > bool 0` projects a zero-or-one `Case`. +Open label schemas acquire a full runtime series-identity field before this +lowering. The runtime must populate that field with all labels. ### PromqlVectorFromScalar -The scalar→instant-vector bridge — PromQL `vector(s)`. Promotes a scalar-typed child to a +The scalar→instant-vector bridge — PromQL `vector(s)`. Promotes a scalar expression to a single label-less series carrying that value at every step, e.g. for dead-man's-switch patterns (`up or vector(0)`). @@ -544,18 +704,9 @@ patterns (`up or vector(0)`). vector(1) ``` -**Fields:** a single unnamed child `QueryExpr` — the scalar-typed expression being promoted to a vector. - -### PromqlScalarFromVector +**Fields:** a single unnamed `ScalarExpr` — the scalar being promoted to a vector. -The instant-vector→scalar bridge — PromQL `scalar(v)`. Collapses a single-element vector to -its value (NaN at runtime if the input isn't exactly one series). - -```promql -scalar(up) -``` - -**Fields:** a single unnamed child `QueryExpr` — the single-series vector being collapsed to a scalar. +## PromQL-specific nodes ### PromqlRelabel @@ -616,5 +767,6 @@ SELECT srcip, LAG(time) OVER (PARTITION BY srcip ORDER BY time) FROM packets - `args` — the function's operand expressions; empty for rank-only functions. - `partition_by` — grouping keys the window is computed within. - `order_by` — the ordering the window function reads. +- `frame` — the optional window frame. - `output_name` — the name of the new output column. - `child` — the input the window function is computed over. diff --git a/docs/develop_docs/storage-operation-costs.md b/docs/develop_docs/storage-operation-costs.md index 1a8dacdbd..999b79103 100644 --- a/docs/develop_docs/storage-operation-costs.md +++ b/docs/develop_docs/storage-operation-costs.md @@ -71,8 +71,7 @@ estimate and storage request estimate remain independently inspectable. Missing entries, expired/future evidence, incompatible node snapshots, zero request sizes, invalid calibration, and overflow return typed analytical errors. When used by plan ranking/export they make that comparison unavailable. -This profile extends the physical-plan adapter; the separate summary-maintenance -lifecycle estimator retains its existing dimensions. Combined physical-plan +This profile extends the physical-plan adapter. Combined physical-plan ranking currently supports storage profiles only with an explicit `NoCache` profile. `CacheProfile::Evidence` together with storage evidence makes the comparison unavailable: aggregate cache hit ratios cannot identify which diff --git a/docs/develop_docs/target-candidate-api-migration.md b/docs/develop_docs/target-candidate-api-migration.md index a5b081a61..4619be2a8 100644 --- a/docs/develop_docs/target-candidate-api-migration.md +++ b/docs/develop_docs/target-candidate-api-migration.md @@ -13,7 +13,7 @@ are unchanged. #453 separately defines the integration API surface. | `MaterializeSummaryMaintenanceLifecycleError` | `SummaryMaintenanceLifecycleAssemblyError` | Failure assembling a DAG or deriving maintenance decisions | | Error variant `Materialize` | `AssembleDAG` | Wrap an underlying `RealizationError` from DAG assembly | | Internal `materialize_inner` / `materialize_residual` | `assemble_target` / `assemble_residual` | Assemble selected nodes, not runtime materialized views | -| Internal assembly cache `materialized` | `assembled_nodes` | Preserve shared `Rc` identity | +| Internal assembly cache `materialized` | `assembled_nodes` | Preserve shared node identity (now `Rc`, see below) | Update imports and calls together; old public names are not retained as aliases. Downstream Rust integrations using these symbols must migrate. No serialized @@ -24,5 +24,28 @@ The earlier #445 renames (`TargetSubDAGCandidates`, counterpart) are prerequisites, not additional changes here. The workflow remains one selection call per workload followed by one assembly -call per query root. `SummaryMaintenanceLifecyclePlan` contains the assembled -Post-ASAP DAG root plus maintenance decisions; it is not an executable plan. +call per query root. The summary-maintenance lifecycle API named above was later +removed; Stage 2 materialization (#509) will own maintenance decisions. + +## Later: unified operator IR (operator flattening) + +The pre-ASAP and post-ASAP trees became one IR in `asap_types::ir`. Every +node is an `Rc` whose `operator` is `Operator::NonASAP(NonASAPOp)` +or `Operator::ASAP(ASAPOp)`. Old public names are not kept as aliases. + +| Old | New | +|---|---| +| `Rc` (pre-ASAP) | `Rc` holding `Operator::NonASAP(NonASAPOp)` | +| `Rc` / `SummaryExpr` (post-ASAP) | The same `Rc`; summary steps are `Operator::ASAP(ASAPOp)` | +| `SummaryExpr::KeepPreAsap(q)` | The non-ASAP sub-DAG itself; `retain_exact` only adds an exact `guarantee` | +| `SummaryExpr::ValueOperation { .. }` over a evaluation | An ordinary `NonASAPOp` (`Project`, `Filter`, `Sort`, `Limit`, `Aggregate`) reading an ASAP node; `FinalizeExactAccumulator`, `MaintainPopulation`, `EvaluatePopulation` are `ASAPOp` variants | +| `Replacement::Summary(..)` / `Replacement::Rewrite(..)` | `Replacement::SubDAG(Rc)`; `is_logical_rewrite` tells them apart | +| `SummaryFamilyType` | `FieldDataType` (its non-`Plain` variants) | +| Timing stored on post-ASAP nodes | `OperatorNode::timing`, `None` until `ir::timing::apply_materialization_timings` writes it from a `MaterializationAssignment` (default: all query time) | +| `UnresolvedQueryExpr` + `asap_types::pre_asap::resolve_root` | `UnresolvedOp` / `UnresolvedScalar` + `asap_frontend_common::resolve_root` | +| `pre_asap::canonicalize`, `pre_asap::cse::share_common_sub_dags` | `ir::canonicalize::canonicalize`, `ir::cse::share_common_sub_dags` | +| `asap_types::post_asap::compile_post_asap_dag` (wire version 5, `Fallback`/`Binary`/`Value` payloads) | `asap_types::ir::export::compile_post_asap_dag` (wire version 7: one node per operator, `Relational` payloads, `ScalarRef` edges); input must be timed | +| Exported schema JSON `columns` | `fields` | + +Field and schema details: [Pre-ASAP IR](pre-asap-ir.md) and +[Post-ASAP IR](../design_docs/concepts/post-asap-ir.md). diff --git a/docs/user_guide_docs/run-a-query.md b/docs/user_guide_docs/run-a-query.md index 3f5825c29..c6f52aa61 100644 --- a/docs/user_guide_docs/run-a-query.md +++ b/docs/user_guide_docs/run-a-query.md @@ -6,7 +6,7 @@ corpus coverage. These commands do not deploy or execute a physical plan. To develop an application using the Rust library, start with [Library API: definitions, options, and examples](../develop_docs/library-api.md). That guide explains how to choose strategies and models, rank candidates, and -work with lifecycle capabilities. +assemble selected DAGs. ## Choose a command @@ -96,7 +96,7 @@ cargo run -p asap-devtools --bin show_post_asap_ir -- --data-ingestion-interval- available binding from the sketch strategy for each query, numbered in cost-model order. If no candidate is available, it prints the pre-ASAP fallback as candidate 1. It does not show the complete ranked workload candidate set or choose a -deployment lifecycle. Its SQL examples use a fixed demonstration catalog, not +deployment. Its SQL examples use a fixed demonstration catalog, not your database schema. Use the [library workflow](../develop_docs/library-api.md) to retain workload alternatives and provide your own models. @@ -109,8 +109,10 @@ whether to select them using its own evidence. Planner's automatic `global_selection` skips them; their presence alone does not show that they meet the requested target. -Each input line is followed by its debug IR or an `ERR:` message. Post-ASAP -output may contain summary state, readouts or exact `KeepPreAsap` work. An +Each input line is followed by its debug IR or an `ERR:` message. Pre-ASAP and +Post-ASAP output use the same node format: Post-ASAP output adds summary nodes +(state, readouts) and keeps the original exact operators wherever no summary +replaces them. An approximate target permits approximation; it does not guarantee a legal or certified sketch. The tool prints plans, not query results. diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index b90aae272..182c0f94c 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -69,7 +69,8 @@ cargo run -p asap-devtools --bin dag_export -- \ Load the JSON with the page's file picker. `--planner-cost-json` is a complete physical-evidence document: an immutable `evidence_version`, calibration, and -target records containing the exact target `QueryExpr` and comparison scope. +target records containing the exact target node (a serialized pre-ASAP +`OperatorNode`) and comparison scope. Each exact replacement candidate owns its complete logical-node `PhysicalNodeEvidence`; summary candidates additionally own their bound `PhysicalDAG`. Candidate-local evidence prevents statistics for one physical @@ -101,15 +102,6 @@ to calibrate against, and `--planner-cost-json` once there is. Without either flag, `--post-asap` exports the raw DAG only. -The viewer also accepts the JSON produced by -`export_summary_maintenance_plan`. It renders the materialized summary DAG as -a single lifecycle-plan lane. Selecting a `SummaryAgg` shows the chosen -lifecycle and maintenance mode together with every alternative's cost, -assumptions, and rejection reason. The selected-node panel also shows the -plan-level summary-versus-raw decision, costs, horizon, expected reads, and -evaluation/update rates. Raw-recomputation plans retain that decision summary -even though they have no deployed `SummaryAgg` to annotate. - ## Standalone HTML ```sh @@ -129,7 +121,7 @@ a selected replacement directly contains: { "decision": { "id": 7, - "strategy": "SketchAlgorithmStrategy", + "strategy": "ASAPStrategies", "rationale": "count realizes as a Cms sketch", "rank": 0, "cost": 1.14001088, @@ -147,8 +139,13 @@ The exporter assigns `workload_node_id`; union rendering reads that mapping directly. Node boxes use concrete IR fields: aggregate measures/grouping, sort keys, -filter predicates, projections, sources, summary families, and readout -queries. Category icons are deliberately omitted so they cannot be confused +filter predicates, projections, sources, summary families, and evaluation +queries. A node's `kind` is the operator variant name (`Operator::kind_name`): +a `NonASAPOp` such as `Aggregate` or `Values`, or an `ASAPOp` such as +`SummaryAgg` or `EvaluatePopulation`. `node-style.js` maps each kind to a color +category. Scalar expressions are not nodes; an operator a scalar expression +reads (`scalar(v)`, `EXISTS (subquery)`) is a child node, shown in `detail` +as `{"scalar_ref": }`. Schemas list their entries under `fields`. Category icons are deliberately omitted so they cannot be confused with IR text. ### Cost/benefit annotations (issue #286) diff --git a/tools/dag-viewer/dag.example.json b/tools/dag-viewer/dag.example.json index 85c0e97b6..5249bb412 100644 --- a/tools/dag-viewer/dag.example.json +++ b/tools/dag-viewer/dag.example.json @@ -210,7 +210,7 @@ { "decision_id": 0, "target_pre_id": 1, - "strategy": "SketchAlgorithmStrategy", + "strategy": "ASAPStrategies", "rationale": "count realizes as a Cms sketch", "rank": 0, "cost": 1.14001088, @@ -363,108 +363,7 @@ "kind": "Summary", "dag": { "nodes": [ - { - "id": 0, - "kind": "KeepPreAsap", - "label": "KeepPreAsap(Scan)", - "detail": { - "pre_asap_sub_dag": { - "nodes": [ - { - "children": [], - "detail": { - "predicates": [], - "schema": { - "closed": true, - "columns": [ - { - "dtype": "timestamp", - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "utf8", - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "utf8", - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "float64", - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "int64", - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "Table": { - "table_ref": "metrics" - } - } - }, - "hash": 2606922452740434172, - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "schema": { - "closed": true, - "columns": [ - { - "dtype": "timestamp", - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "utf8", - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "utf8", - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "float64", - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "int64", - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - } - } - ], - "root": 0 - } - }, - "children": [] - }, + {"id": 0, "kind": "Scan", "label": "Scan(metrics)", "detail": {"predicates": [], "schema": {"closed": true, "columns": [{"dtype": "timestamp", "name": "ts", "nullable": false, "table": "metrics"}, {"dtype": "utf8", "name": "service", "nullable": false, "table": "metrics"}, {"dtype": "utf8", "name": "region", "nullable": false, "table": "metrics"}, {"dtype": "float64", "name": "latency", "nullable": false, "table": "metrics"}, {"dtype": "int64", "name": "bytes", "nullable": false, "table": "metrics"}], "time_index": 0, "unique_keys": []}, "source": {"Table": {"table_ref": "metrics"}}}, "children": []}, { "id": 1, "kind": "SummaryAgg", @@ -657,7 +556,7 @@ "hash": 2606922452740434172, "decision": { "id": 0, - "strategy": "SketchAlgorithmStrategy", + "strategy": "ASAPStrategies", "rationale": "count realizes as a Cms sketch", "rank": 0, "cost": 1.14001088, @@ -763,7 +662,7 @@ "workload_node_id": 1, "decision": { "id": 0, - "strategy": "SketchAlgorithmStrategy", + "strategy": "ASAPStrategies", "rationale": "count realizes as a Cms sketch", "rank": 0, "cost": 1.14001088, @@ -862,7 +761,7 @@ "workload_node_id": 2, "decision": { "id": 0, - "strategy": "SketchAlgorithmStrategy", + "strategy": "ASAPStrategies", "rationale": "count realizes as a Cms sketch", "rank": 0, "cost": 1.14001088, diff --git a/tools/dag-viewer/generate-sample.sh b/tools/dag-viewer/generate-sample.sh index e916cba5f..66921d8c7 100755 --- a/tools/dag-viewer/generate-sample.sh +++ b/tools/dag-viewer/generate-sample.sh @@ -6,7 +6,7 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." # --epsilon asks for an approximate accuracy target instead of the default -# Exact, so SketchAlgorithmStrategy actually has a sketch alternative to +# Exact, so ASAPStrategies actually has a sketch alternative to # report — without it, no query below would ever pick up a `notes` badge # (see crates/devtools/src/bin/dag_export.rs's own `--epsilon` doc comment). cargo run -p asap-devtools --bin dag_export -- \ diff --git a/tools/dag-viewer/lifecycle-summary-maintenance.png b/tools/dag-viewer/lifecycle-summary-maintenance.png deleted file mode 100644 index e873ffd9f..000000000 Binary files a/tools/dag-viewer/lifecycle-summary-maintenance.png and /dev/null differ diff --git a/tools/dag-viewer/node-style.js b/tools/dag-viewer/node-style.js index a957817c2..fee49586e 100644 --- a/tools/dag-viewer/node-style.js +++ b/tools/dag-viewer/node-style.js @@ -1,20 +1,17 @@ -// Logical QueryExpr/SummaryExpr kinds exported by +// Operator kinds (`Operator::kind_name`) exported by // crates/types/src/dag_export.rs. Categories describe the visible logical DAG // shape. They do not model hidden physical inputs: for example, // PromqlInfoEnrich is a one-child enrichment here even if physical costing // later accounts for an auxiliary source scan. const KIND_CATEGORY_JSON = `{ "Scan": "data", - "PromqlScalarBridge": "data", - "EvalTimestamp": "data", - "CurrentTimestamp": "data", + "Values": "data", "Filter": "filter", "PromqlSeriesSample": "sample", "Project": "derive", "PromqlRelabel": "derive", "PromqlInfoEnrich": "derive", "PromqlVectorFromScalar": "derive", - "PromqlScalarFromVector": "derive", "BinaryOp": "derive", "Aggregate": "aggregate", "TimeRange": "window", @@ -22,21 +19,21 @@ const KIND_CATEGORY_JSON = `{ "TimeShift": "window", "SQLWindowFunc": "window", "Join": "join", - "RelationalJoin": "join", "Dedup": "set", "SetOp": "set", "Concat": "combine", "Sort": "sort", "Limit": "sort", - "KeepPreAsap": "summary", "SummaryAgg": "summary", "SummaryJoin": "summary", "SummarySubtract": "summary", - "SummaryBinaryOp": "summary", - "ValueOperation": "summary", "SummaryDelete": "summary", "SummaryEstimate": "summary", - "SummaryMerge": "summary" + "SummaryMerge": "summary", + "FinalizeExactAccumulator": "summary", + "MaintainPopulation": "summary", + "EvaluatePopulation": "summary", + "Extension": "summary" }`; const KIND_CATEGORY = Object.freeze(JSON.parse(KIND_CATEGORY_JSON)); @@ -44,7 +41,7 @@ const KIND_CATEGORY = Object.freeze(JSON.parse(KIND_CATEGORY_JSON)); const CATEGORIES = { data: { label: 'Data', - description: 'Scan, PromqlScalarBridge, EvalTimestamp, CurrentTimestamp — leaves that introduce a value', + description: 'Scan and Values — data sources', light: { bg: '#eef5fd', border: '#0369a1' }, dark: { bg: '#0c2438', border: '#38bdf8' }, }, @@ -62,7 +59,7 @@ const CATEGORIES = { }, derive: { label: 'Derive', - description: 'Project, PromqlRelabel, PromqlInfoEnrich, PromqlVectorFromScalar, PromqlScalarFromVector, BinaryOp — transforms or enriches columns on otherwise-unchanged rows', + description: 'Project, PromqlRelabel, PromqlInfoEnrich, PromqlVectorFromScalar, BinaryOp — transforms or enriches columns on otherwise-unchanged rows', light: { bg: '#f5f0fd', border: '#6d28d9' }, dark: { bg: '#241a3d', border: '#a78bfa' }, }, @@ -102,10 +99,10 @@ const CATEGORIES = { light: { bg: '#eef4fd', border: '#1d4ed8' }, dark: { bg: '#12233d', border: '#60a5fa' }, }, - // Post-ASAP nodes use a neutral palette; KeepPreAsap has a muted override. + // ASAP operators use a neutral palette. summary: { label: 'Summary', - description: 'KeepPreAsap, SummaryBinaryOp, ValueOperation, SummaryAgg, SummaryJoin, SummarySubtract, SummaryDelete, SummaryEstimate, SummaryMerge — post-ASAP materialized structures', + description: 'SummaryAgg, SummaryEstimate, FinalizeExactAccumulator, MaintainPopulation, EvaluatePopulation, SummaryJoin, SummarySubtract, SummaryDelete, SummaryMerge, Extension — summary state and its evaluations', light: { bg: '#f1f2f4', border: '#4b5563' }, dark: { bg: '#20242b', border: '#9ca3af' }, }, diff --git a/tools/dag-viewer/post_asap_fixture.json b/tools/dag-viewer/post_asap_fixture.json index ea2de185f..e616e80e2 100644 --- a/tools/dag-viewer/post_asap_fixture.json +++ b/tools/dag-viewer/post_asap_fixture.json @@ -14,9 +14,9 @@ }, "post_dag": { "nodes": [ - { "id": 0, "kind": "KeepPreAsap", "label": "KeepPreAsap(Scan)", "detail": {"pre_asap_sub_dag": {"nodes": [{"id": 0, "kind": "Scan", "label": "Scan(netflow_table)", "detail": {}, "children": []}], "root": 0}}, "children": [] }, + {"id": 0, "kind": "Scan", "label": "Scan(netflow_table)", "detail": {}, "children": []}, { "id": 1, "kind": "SummaryAgg", "label": "SummaryAgg(Kll)", "detail": {"family": {"Sketch": ["Kll", {"k": 200}]}, "col": {"Column": 6}, "reduction": {"Reduce": [1]}, "grouping": "PerSubpopulationInstance"}, "children": [0], "origin_pre_id": 1 }, - { "id": 2, "kind": "KeepPreAsap", "label": "KeepPreAsap(Project)", "detail": {"pre_asap_sub_dag": {"nodes": [{"id": 0, "kind": "Project", "label": "Project(2 cols)", "detail": {}, "children": []}], "root": 0}}, "children": [1] } + {"id": 2, "kind": "Project", "label": "Project(2 cols)", "detail": {}, "children": [1]} ], "root": 2 }, @@ -39,7 +39,7 @@ "kind": "Summary", "dag": { "nodes": [ - { "id": 0, "kind": "KeepPreAsap", "label": "KeepPreAsap(Scan)", "detail": {"pre_asap_sub_dag": {"nodes": [{"id": 0, "kind": "Scan", "label": "Scan(netflow_table)", "detail": {}, "children": [], "hash": 111}], "root": 0}}, "children": [] }, + {"id": 0, "kind": "Scan", "label": "Scan(netflow_table)", "detail": {}, "children": []}, { "id": 1, "kind": "SummaryAgg", "label": "SummaryAgg(Kll)", "detail": {"family": {"Sketch": ["Kll", {"k": 200}]}, "col": {"Column": 6}, "reduction": {"Reduce": [1]}, "grouping": "PerSubpopulationInstance"}, "children": [0], "origin_pre_id": 1 } ], "root": 1 @@ -64,7 +64,7 @@ "kind": "Summary", "dag": { "nodes": [ - { "id": 0, "kind": "KeepPreAsap", "label": "KeepPreAsap(Scan)", "detail": {}, "children": [] }, + {"id": 0, "kind": "Scan", "label": "Scan", "detail": {}, "children": []}, { "id": 1, "kind": "SummaryAgg", "label": "SummaryAgg(HydraKll)", "detail": {"family": {"Sketch": ["Kll", {"k": 200}]}, "grouping": {"SharedMultiSubpopulation": {"params": {}}}}, "children": [0], "origin_pre_id": 1 } ], "root": 1 diff --git a/tools/dag-viewer/render.py b/tools/dag-viewer/render.py index fbdaca97a..7b16796da 100755 --- a/tools/dag-viewer/render.py +++ b/tools/dag-viewer/render.py @@ -17,7 +17,7 @@ This does not add anything index.html doesn't already do — it shares viewer.js and node-style.js with it verbatim (see viewer.js's header comment) and only differs in packaging: one query's worth of exported -`QueryExpr` detail *is* its plan (see the side panel on node click), and +`OperatorNode` detail *is* its plan (see the side panel on node click), and shared-hash highlighting *is* what this repo has for CSE today — both a hash-based proxy, not real CSE output; see README.md's "Shared-sub-DAG highlighting is a proxy" section. Structured cost/benefit annotations @@ -72,7 +72,7 @@ def _compact(value: object) -> str: if not isinstance(value, dict): return str(value) - # Common serde enum/newtype shapes in QueryExpr detail. + # Common serde enum/newtype shapes in OperatorNode detail. if set(value) == {"Column"}: return f"col[{_compact(value['Column'])}]" if set(value) == {"Table"} and isinstance(value["Table"], dict): @@ -269,23 +269,6 @@ def load_workload(paths: list[Path]) -> dict: for path in paths: data = json.loads(path.read_text()) incoming = data.get("queries", []) - if not incoming and isinstance(data.get("dag"), dict) and isinstance(data.get("deployments"), list): - incoming = [{ - "name": path.stem or "Summary maintenance plan", - "dag": data["dag"], - "post_dag": data["dag"], - "lifecycle_plan": True, - "lifecycle_summary": { - "selected_raw_recompute": data.get("selected_raw_recompute", False), - "summary_total_cost": data.get("summary_total_cost"), - "raw_recompute_total_cost": data.get("raw_recompute_total_cost"), - "horizon_seconds": data.get("horizon_seconds"), - "evaluation_rate_per_second": data.get("evaluation_rate_per_second"), - "update_rate_per_second": data.get("update_rate_per_second"), - "expected_reads": data.get("expected_reads"), - "deployment_count": len(data["deployments"]), - }, - }] for q in incoming: name = q["name"] if name in seen_names: diff --git a/tools/dag-viewer/test_render.py b/tools/dag-viewer/test_render.py index 20ccb64cd..e78e978f0 100644 --- a/tools/dag-viewer/test_render.py +++ b/tools/dag-viewer/test_render.py @@ -79,43 +79,6 @@ def test_boundary_terms_and_provenance_survive_standalone_export(self): self.assertIn(annotation["model_version"], html) self.assertIn(annotation["evidence_version"], html) - def test_loads_summary_maintenance_export_as_a_lifecycle_plan(self): - dag = named_dag("unused")["dag"] - summary = { - "selected_raw_recompute": True, - "summary_total_cost": None, - "raw_recompute_total_cost": 7.5, - "horizon_seconds": 60.0, - "evaluation_rate_per_second": 2.0, - "update_rate_per_second": 3.0, - "expected_reads": 120.0, - } - with tempfile.TemporaryDirectory() as d: - path = Path(d) / "lifecycle.json" - path.write_text(json.dumps({"dag": dag, "deployments": [], **summary})) - workload = load_workload([path]) - - query = workload["queries"][0] - self.assertEqual(query["name"], "lifecycle") - self.assertTrue(query["lifecycle_plan"]) - self.assertEqual(query["post_dag"], dag) - self.assertEqual( - query["lifecycle_summary"], - {**summary, "deployment_count": 0}, - ) - - def test_preserves_summary_plan_deployment_count(self): - dag = named_dag("unused")["dag"] - with tempfile.TemporaryDirectory() as d: - path = Path(d) / "lifecycle.json" - path.write_text(json.dumps({"dag": dag, "deployments": [{}, {}]})) - workload = load_workload([path]) - - self.assertEqual( - workload["queries"][0]["lifecycle_summary"]["deployment_count"], - 2, - ) - def test_merges_queries_across_files_in_order(self): with tempfile.TemporaryDirectory() as d: f1 = Path(d) / "a.json" diff --git a/tools/dag-viewer/viewer.js b/tools/dag-viewer/viewer.js index 6e4c696ee..d8191dc1a 100644 --- a/tools/dag-viewer/viewer.js +++ b/tools/dag-viewer/viewer.js @@ -18,7 +18,7 @@ cytoscape.use(window.cytoscapeDagre); // --post-asap whole-query merged post-ASAP DAG (same flattened // `{nodes, root}` shape as `dag`, but nodes may be post-ASAP-only kinds // like "SummaryAgg" mixed in, and any such node has no `hash` — there's no -// corresponding QueryExpr to hash) — left `undefined` when absent (omitted +// corresponding OperatorNode to hash) — left `undefined` when absent (omitted // whenever --post-asap wasn't set, or this query had zero replacements), // unlike `replacements` which always defaults to an array. `workload_cost` // is the optional per-query `NamedDAG.workload_cost` (issue #286), also @@ -122,13 +122,7 @@ function loadFiles(fileList) { reader.onload = () => { try { const parsed = JSON.parse(reader.result); - const incoming = parsed.queries || (parsed.dag && parsed.deployments ? [{ - name: file.name.replace(/\.json$/i, '') || 'Summary maintenance plan', - dag: parsed.dag, - post_dag: parsed.dag, - lifecycle_plan: true, - lifecycle_summary: lifecyclePlanSummary(parsed), - }] : []); + const incoming = parsed.queries || []; const existingNames = new Set(queries.map((q) => q.name)); // One batch id per *file* — every query this one dag_export // invocation produced shares its decision.id numbering. @@ -137,7 +131,7 @@ function loadFiles(fileList) { let name = q.name; if (existingNames.has(name)) name = `${q.name} (${file.name})`; existingNames.add(name); - queries.push({ name, dag: q.dag, source: q.source, replacements: q.replacements || [], post_dag: q.post_dag, workload_cost: q.workload_cost, lifecycle_plan: q.lifecycle_plan, lifecycle_summary: q.lifecycle_summary, sourceBatch }); + queries.push({ name, dag: q.dag, source: q.source, replacements: q.replacements || [], post_dag: q.post_dag, workload_cost: q.workload_cost, sourceBatch }); }); } catch (err) { alert(`Failed to parse ${file.name}: ${err.message}`); @@ -154,19 +148,6 @@ function loadFiles(fileList) { fileInput.value = ''; } -function lifecyclePlanSummary(plan) { - return { - selected_raw_recompute: Boolean(plan.selected_raw_recompute), - summary_total_cost: plan.summary_total_cost ?? null, - raw_recompute_total_cost: plan.raw_recompute_total_cost ?? null, - horizon_seconds: plan.horizon_seconds ?? null, - evaluation_rate_per_second: plan.evaluation_rate_per_second ?? null, - update_rate_per_second: plan.update_rate_per_second ?? null, - expected_reads: plan.expected_reads ?? null, - deployment_count: Array.isArray(plan.deployments) ? plan.deployments.length : 0, - }; -} - function getParticipants() { return Array.from(participants) .filter((i) => i >= 0 && i < queries.length) @@ -290,22 +271,6 @@ function buildCyStyle() { selector: 'node[category = "unknown"]', style: { 'border-style': 'dashed', 'border-width': 3 }, }, - { - // KeepPreAsap (post-ASAP lane only) is post-ASAP-only - // as a *kind*, but represents literally unchanged pre-ASAP content — - // override the 'summary' category's color/icon with the same neutral - // panel/muted/dashed treatment the rest of the chrome uses for "nothing - // to see here", so a glance at the After lane separates "the planner - // did something" (solid, colored) from "left alone" (dashed, muted). - // See node-style.js's CATEGORIES.summary comment for the category-level - // color choice this overrides. - selector: 'node[kind = "KeepPreAsap"]', - style: { - 'background-color': panelColor, - 'border-color': borderColor, - 'border-style': 'dashed', - }, - }, { selector: 'node.root', style: { 'border-width': 2.5 }, @@ -497,28 +462,6 @@ function renderPrePostAsap() { return; } hideModeHint(); - if (selected.length === 1 && selected[0].lifecycle_plan) { - viewTitleEl.textContent = `Summary maintenance: ${selected[0].name}`; - const elements = laneElements( - 'summary-maintenance', - `${selected[0].name} · lifecycle plan`, - selected[0].post_dag, - selected[0], - 'post', - ); - buildCy(elements); - finalizeDAGInteractions(); - applyHighlighting(); - const initial = cy.nodes().filter((node) => !node.data('isLane') && node.data('root')).first(); - if (initial && initial.length) { - initial.select(); - showPrePostDetail(initial.data()); - } else { - clearDetail(); - } - fitAndSyncZoom(); - return; - } viewTitleEl.textContent = selected.length === 1 ? `Pre/Post-ASAP: ${selected[0].name}` : `Pre/Post-ASAP workload union: ${selected.length} queries`; @@ -663,10 +606,7 @@ function laneElements(laneId, laneLabel, dag, query, stage, laneCost) { // exactly the plain IR label. label: node.label + nodeCostBadgeSuffix(node), node, - // Flat (not nested under `node`) so buildCyStyle's - // `node[kind = "KeepPreAsap"]` selector can actually match it — - // cytoscape selectors can't reach into a data field that's itself an - // object. + // Cytoscape selectors read flat data fields. kind: node.kind, category: categoryOf(node.kind), root: node.id === dag.root, @@ -674,7 +614,6 @@ function laneElements(laneId, laneLabel, dag, query, stage, laneCost) { laneId, stage, queryName: query.name, - lifecycleSummary: query.lifecycle_summary, translations: translationsForNode(query, node, stage), }, classes: node.decision && typeof node.decision.benefit?.value === 'number' @@ -1028,33 +967,6 @@ function showPrePostDetail(data) { : ''; const decisions = data.translations || []; - const planSummary = data.lifecycleSummary; - let planSummaryHtml = ''; - if (planSummary) { - const value = (item) => item === null || item === undefined ? 'unknown' : String(item); - const selected = planSummary.selected_raw_recompute - ? 'Raw recomputation' - : 'Summary maintenance'; - planSummaryHtml = `

Lifecycle plan decision

-
Selected: ${escapeHtml(selected)}
-
summary cost: ${escapeHtml(value(planSummary.summary_total_cost))} · raw recompute cost: ${escapeHtml(value(planSummary.raw_recompute_total_cost))} · deployments: ${escapeHtml(value(planSummary.deployment_count))}
-
horizon: ${escapeHtml(value(planSummary.horizon_seconds))} s · expected reads: ${escapeHtml(value(planSummary.expected_reads))} · evaluation rate: ${escapeHtml(value(planSummary.evaluation_rate_per_second))}/s · update rate: ${escapeHtml(value(planSummary.update_rate_per_second))}/s
-
`; - } - const lifecycle = node.detail && node.detail.summary_maintenance; - let lifecycleHtml = ''; - if (lifecycle) { - const selected = lifecycle.selected; - const selectedText = selected - ? `${selected.lifecycle.kind} · ${selected.maintenance_mode} · ${selected.evaluation_schedule} · ${selected.output_representation}` - : 'No lifecycle selected'; - const alternatives = (lifecycle.alternatives || []).map((alternative) => { - const status = alternative.rejection ? `rejected: ${alternative.rejection}` : `cost: ${alternative.total_cost}`; - const assumptions = (alternative.assumptions || []).join('; ') || 'none'; - return `
${escapeHtml(alternative.lifecycle.kind)}
${escapeHtml(status)}
assumptions: ${escapeHtml(assumptions)}
`; - }).join(''); - lifecycleHtml = `

Summary maintenance lifecycle

Selected: ${escapeHtml(selectedText)}
${alternatives}
`; - } let translationHtml = ''; if (decisions.length > 0) { const cards = decisions.map((entry) => ` @@ -1076,9 +988,7 @@ function showPrePostDetail(data) { ${escapeHtml(chipLabel)}
${escapeHtml(node.label)}
${rootHtml} - ${planSummaryHtml} ${translationHtml} - ${lifecycleHtml}

IR node content

${escapeHtml(JSON.stringify(node.detail, null, 2))}
`; @@ -1172,10 +1082,6 @@ function renderLegend() { Shared workload nodeExplicitly identified by the exporter as shared across selected queries`); rows.push(`
Query root${escapeHtml(ROOT_BADGE.description)}
`); - const panelBg = getComputedStyle(document.documentElement).getPropertyValue('--panel2').trim() || '#f0f2f5'; - const mutedColor = getComputedStyle(document.documentElement).getPropertyValue('--muted').trim() || '#6b7280'; - rows.push(`
- Pass-through (KeepPreAsap)Unchanged pre-ASAP sub-DAG carried into the Summary DAG as-is
`); legendList.innerHTML = rows.join(''); } @@ -1204,7 +1110,7 @@ function loadWorkload(parsed) { const incoming = (parsed && parsed.queries) || []; // One batch id for this whole document — see `sourceBatch`'s own doc above. const sourceBatch = nextSourceBatch++; - incoming.forEach((q) => queries.push({ name: q.name, dag: q.dag, source: q.source, replacements: q.replacements || [], post_dag: q.post_dag, workload_cost: q.workload_cost, lifecycle_plan: q.lifecycle_plan, lifecycle_summary: q.lifecycle_summary, sourceBatch })); + incoming.forEach((q) => queries.push({ name: q.name, dag: q.dag, source: q.source, replacements: q.replacements || [], post_dag: q.post_dag, workload_cost: q.workload_cost, sourceBatch })); if (activeIndex === -1 && queries.length > 0) activeIndex = 0; if (participants.size === 0 && activeIndex >= 0) participants.add(activeIndex); }