diff --git a/Cargo.lock b/Cargo.lock index c340f5892..b99d3be76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cb70086b4c4a7ba89baf2516be81d0b192137a3a#cb70086b4c4a7ba89baf2516be81d0b192137a3a" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3afcba68f4e8397fb81e2be988f47120f63f7a39#3afcba68f4e8397fb81e2be988f47120f63f7a39" dependencies = [ "asap-types", "serde_json", @@ -353,7 +353,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cb70086b4c4a7ba89baf2516be81d0b192137a3a#cb70086b4c4a7ba89baf2516be81d0b192137a3a" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3afcba68f4e8397fb81e2be988f47120f63f7a39#3afcba68f4e8397fb81e2be988f47120f63f7a39" dependencies = [ "asap-types", "promql-parser 0.10.0", @@ -372,7 +372,7 @@ dependencies = [ [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=cb70086b4c4a7ba89baf2516be81d0b192137a3a#cb70086b4c4a7ba89baf2516be81d0b192137a3a" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=3afcba68f4e8397fb81e2be988f47120f63f7a39#3afcba68f4e8397fb81e2be988f47120f63f7a39" dependencies = [ "serde", "serde_json", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index b8b2d9974..9261cc9c9 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -93,8 +93,8 @@ asap_types.workspace = true # scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* # repo is a real one. Vendored locally instead of chased upstream -- see # `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cb70086b4c4a7ba89baf2516be81d0b192137a3a" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cb70086b4c4a7ba89baf2516be81d0b192137a3a" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3afcba68f4e8397fb81e2be988f47120f63f7a39" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3afcba68f4e8397fb81e2be988f47120f63f7a39" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. @@ -102,7 +102,7 @@ asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = # `planner-types`/`asap-aware-mapping` above -- these three MUST move # together (two revs of the same upstream repo's types in one workspace # resolve to distinct Rust types that won't unify). -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cb70086b4c4a7ba89baf2516be81d0b192137a3a" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3afcba68f4e8397fb81e2be988f47120f63f7a39" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/control_plane/src/asap_tier_analysis.rs b/control_plane/src/asap_tier_analysis.rs index 1424eb3f5..22f76989c 100644 --- a/control_plane/src/asap_tier_analysis.rs +++ b/control_plane/src/asap_tier_analysis.rs @@ -356,11 +356,11 @@ pub(crate) fn collect_agg_intents(expr: &QueryExpr, out: &mut Vec) { // at construction time (`intent_algebra::lower`). QueryExpr::Filter { child, .. } | QueryExpr::Project { child, .. } - | QueryExpr::Distinct { child, .. } + | QueryExpr::Dedup { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } - | QueryExpr::Subquery { child, .. } => collect_agg_intents(child, out), - QueryExpr::Merge { children } => { + | QueryExpr::PromqlSubquery { child, .. } => collect_agg_intents(child, out), + QueryExpr::Concat { children } => { for c in children { collect_agg_intents(c, out); } diff --git a/control_plane/src/asap_tier_implement.rs b/control_plane/src/asap_tier_implement.rs index 2f79a67c2..d9bbe0027 100644 --- a/control_plane/src/asap_tier_implement.rs +++ b/control_plane/src/asap_tier_implement.rs @@ -73,7 +73,7 @@ use std::rc::Rc; -use asap_aware_mapping::{implement_tree_with, DefaultCostModel, ImplementError}; +use asap_aware_mapping::DefaultCostModel; use planner_types::post_asap::SummaryNode; use crate::intent_algebra::query_expr::QueryExpr; @@ -130,11 +130,11 @@ fn collect_aggregate_roots<'a>(expr: &'a QueryExpr, out: &mut Vec<&'a QueryExpr> // at construction time (`intent_algebra::lower`). QueryExpr::Filter { child, .. } | QueryExpr::Project { child, .. } - | QueryExpr::Distinct { child, .. } + | QueryExpr::Dedup { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } - | QueryExpr::Subquery { child, .. } => collect_aggregate_roots(child, out), - QueryExpr::Merge { children } => { + | QueryExpr::PromqlSubquery { child, .. } => collect_aggregate_roots(child, out), + QueryExpr::Concat { children } => { for c in children { collect_aggregate_roots(c, out); } @@ -165,7 +165,7 @@ pub enum ImplementPromqlError { UnparseableMetricsql(String), /// L3→L4 implementation failed for a found `Aggregate` root (schema /// derivation error — see `asap_aware_mapping::bind::ImplementError`). - Implement(ImplementError), + Implement(crate::planner_selection::SelectionError), } /// Parse `metricsql`, find every independently-realizable `Aggregate` @@ -194,7 +194,8 @@ pub fn implement_promql_for_asap_tier( roots .into_iter() .map(|root| { - implement_tree_with(root, &DefaultCostModel).map_err(ImplementPromqlError::Implement) + crate::planner_selection::select_summary(root, &DefaultCostModel) + .map_err(ImplementPromqlError::Implement) }) .collect() } @@ -268,7 +269,7 @@ mod tests { .expect("parses and implements"); assert_eq!(roots.len(), 1); assert!( - matches!(roots[0].expr, SummaryExpr::Logical(_)), + matches!(roots[0].expr, SummaryExpr::KeepPreAsap(_)), "Avg has no ASAP-tier realization yet on either path: {:?}", roots[0].expr, ); diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index 5da411e41..0d01dbd55 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -724,12 +724,12 @@ mod tests { AggregationInput, BackendAggregation, BackendReadout, }; use planner_types::post_asap::SketchQuery; - use planner_types::post_asap::{SketchKind, SketchParams}; + use planner_types::post_asap::{SketchAlgorithm, SketchParams}; BackendStageConfig { aggregations: vec![BackendAggregation { aggregation_id: agg_id.to_string(), metric_name: metric.to_string(), - sketch_kind: SketchKind::DDSketch.into(), + sketch_kind: SketchAlgorithm::DDSketch.into(), sketch_params: SketchParams::DDSketch { alpha: 0.01 }.into(), grouping: vec![], item_label: None, diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index c2faa6bd2..d0b8040a6 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -50,7 +50,7 @@ use crate::sketch_algebra::physical_expr::L4Plan; use crate::sketch_algebra::PhysicalExpr; use crate::store::WorkloadStore; use anyhow::Result; -use planner_types::post_asap::{SketchKind, SummaryExpr, SummaryNode}; +use planner_types::post_asap::{SketchAlgorithm, SummaryExpr, SummaryNode}; use std::rc::Rc; /// Phase ε.1.5 — which edge runtime an agent identifies as. @@ -287,7 +287,7 @@ fn apply_cold_format_from_env(edge_cfg: &mut EdgeStageConfig) { /// (`Logical`-only, unresolved `Ref`, raw Mode-3 archive). These map /// onto the raw-passthrough default pipeline in the routing emitter, /// which is correct. -pub fn extract_root_sketch_kind(expr: &PhysicalExpr) -> Option { +pub fn extract_root_sketch_kind(expr: &PhysicalExpr) -> Option { match expr { PhysicalExpr::Committed(plan) => extract_from_plan(plan), PhysicalExpr::RawAtEdgeSketchAtBackend { family, .. } => Some(family.clone()), @@ -295,7 +295,7 @@ pub fn extract_root_sketch_kind(expr: &PhysicalExpr) -> Option { } } -fn extract_from_plan(plan: &L4Plan) -> Option { +fn extract_from_plan(plan: &L4Plan) -> Option { match plan { L4Plan::Summary(node) => extract_from_node(node), L4Plan::LetBinding { expr, child, .. } => { @@ -305,7 +305,7 @@ fn extract_from_plan(plan: &L4Plan) -> Option { } } -fn extract_from_node(node: &Rc) -> Option { +fn extract_from_node(node: &Rc) -> Option { match &node.expr { // `SummaryAgg`'s `kind`/`params` collapsed into one `family: // SummaryFamilyType` field (ASAPPlanner#218 -- see @@ -315,7 +315,7 @@ fn extract_from_node(node: &Rc) -> Option { SummaryExpr::SummaryAgg { family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, _), .. - } => Some(kind.clone()), + } => Some(kind.algorithm().clone()), // An exact accumulator has no sketch family beneath it (its own // child is always a plain `Logical` leaf) — same as the old // `ExactAgg` case. @@ -327,7 +327,7 @@ fn extract_from_node(node: &Rc) -> Option { SummaryExpr::SummaryJoin { .. } | SummaryExpr::SummarySubtract { .. } | SummaryExpr::SummaryDelete { .. } - | SummaryExpr::Logical(_) => None, + | SummaryExpr::KeepPreAsap(_) => None, } } @@ -342,7 +342,7 @@ fn extract_from_node(node: &Rc) -> Option { /// (`quantile_over_time` → DDSketch, `count`-distinct → HLL, `topk` → /// CountSketch, …). We therefore collect the UNION of every workload /// entry's committed sketch family per metric into a -/// `BTreeSet` (deterministic order). The emitter routes the +/// `BTreeSet` (deterministic order). The emitter routes the /// metric to EACH family in its set and prunes pipelines/processors to /// the union of all sets — eliminating the prior all-5 fan-out that /// shipped sketch state through every family regardless of need. @@ -362,8 +362,8 @@ fn extract_from_node(node: &Rc) -> Option { pub fn collect_metric_to_family( registry: &WorkloadRegistry, workload_store: &WorkloadStore, -) -> std::collections::HashMap> { - let mut out: std::collections::HashMap> = +) -> std::collections::HashMap> { + let mut out: std::collections::HashMap> = std::collections::HashMap::new(); for entry in registry.entries() { // B2 (metric, role) restructure: walk EVERY role registered for @@ -773,7 +773,7 @@ mod runtime_tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, + sketch_kind: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], @@ -950,7 +950,7 @@ mod runtime_tests { #[test] fn collect_metric_to_family_binds_all_six_contract_metrics_from_live_yaml() { - use planner_types::post_asap::SketchKind; + use planner_types::post_asap::SketchAlgorithm; // The 6 contract metrics reproduced inline (mirrors // deploy/configs/mvp-workload.yaml entries 1, 5, 6, 7, 8 plus the @@ -1004,31 +1004,28 @@ mod runtime_tests { // metric needs. For THIS workload every sketched metric is // queried by exactly one capability, so each set has size 1. use std::collections::BTreeSet; - let expected: Vec<(&str, Option>)> = vec![ + let expected: Vec<(&str, Option>)> = vec![ ( "http_latency_ms", - Some(BTreeSet::from([SketchKind::DDSketch])), + Some(BTreeSet::from([SketchAlgorithm::DDSketch])), ), ("http_requests_total", None), // raw passthrough ( "request_size_bytes", - Some(BTreeSet::from([SketchKind::Kll])), + Some(BTreeSet::from([SketchAlgorithm::Kll])), ), ( "unique_users_per_min", - Some(BTreeSet::from([SketchKind::Hll])), - ), - ( - "top_endpoint_qps", - Some(BTreeSet::from([SketchKind::CountSketchWithHeap])), + Some(BTreeSet::from([SketchAlgorithm::Hll])), ), + ("top_endpoint_qps", None), // `CountMinSketch` override re-derives statistic to // `Frequency`, `AggIntent::Extension`-shaped — now binds via // `ControlPlaneCostModel::realize_extension` (ASAPController#150, // see `optimizer::rules::tests::typed_binding_endpoint_request_freq_binds_cms`). ( "endpoint_request_freq", - Some(BTreeSet::from([SketchKind::Cms])), + Some(BTreeSet::from([SketchAlgorithm::Cms])), ), ]; for (metric, want) in &expected { @@ -1039,12 +1036,12 @@ mod runtime_tests { full map: {map:?}", ); } - // Routing table covers the 5 sketched metrics (only - // http_requests_total declines, as raw passthrough). + // TopK also declines until a fresh membership-margin certificate is + // supplied to the physical compiler. assert_eq!( map.len(), - 5, - "routing table should have 5 entries (5 sketches; only raw passthrough declines), got: {map:?}" + 4, + "routing table should have 4 evidence-valid sketch entries; raw passthrough and uncertified TopK decline, got: {map:?}" ); } @@ -1160,7 +1157,7 @@ mod runtime_tests { fn collect_metric_to_family_unions_multiple_capabilities_per_metric() { use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; use crate::workload::AggRole; - use planner_types::post_asap::SketchKind; + use planner_types::post_asap::SketchAlgorithm; use std::collections::BTreeSet; use std::time::Duration; @@ -1236,9 +1233,9 @@ mod runtime_tests { assert_eq!( got, BTreeSet::from([ - SketchKind::DDSketch, - SketchKind::Hll, - SketchKind::Cms + SketchAlgorithm::DDSketch, + SketchAlgorithm::Hll, + SketchAlgorithm::Cms ]), "a metric queried by 3 capabilities must accumulate 3 families (UNION, not first-wins)\nmap: {map:?}" ); @@ -1306,7 +1303,7 @@ mod runtime_tests { let _env = crate::test_support::env_lock(); use crate::physical::colored_dag::emitter::{EdgeStageConfig, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; - use planner_types::post_asap::SketchKind; + use planner_types::post_asap::SketchAlgorithm; let yaml = r#" - metric_name: http_requests_total_latency_ms @@ -1332,7 +1329,7 @@ mod runtime_tests { warm_passthrough_metrics: Vec::new(), metric_to_family: std::collections::HashMap::from([( "http_requests_total_latency_ms".to_string(), - std::collections::BTreeSet::from([SketchKind::DDSketch]), + std::collections::BTreeSet::from([SketchAlgorithm::DDSketch]), )]), metric_to_grouping_labels: std::collections::HashMap::new(), cumulative_counter_metrics: Vec::new(), diff --git a/control_plane/src/emit/otap.rs b/control_plane/src/emit/otap.rs index 21f26fbf3..492527354 100644 --- a/control_plane/src/emit/otap.rs +++ b/control_plane/src/emit/otap.rs @@ -42,7 +42,7 @@ use std::collections::BTreeMap; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, EdgeStageConfig, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; -use planner_types::post_asap::{SketchKind, SketchParams}; +use planner_types::post_asap::{SketchAlgorithm, SketchParams}; /// Default URL for Prometheus's native OTLP HTTP receiver. /// Matches `super::stage_config::emit_edge_yaml`'s placeholder so the @@ -310,7 +310,7 @@ fn build_asap_sketches_config(sp: &EdgeSketchProcessor, window_secs: Option } // Heap-bearing width/depth extraction is identical to the bare // kind — this path never distinguished `with_heap` even before - // `SketchKind` split it into its own variant (heap_size wasn't + // `SketchAlgorithm` split it into its own variant (heap_size wasn't // emitted here either way). SketchParams::Cms { width, depth } | SketchParams::CmsWithHeap { width, depth, .. } => { m.insert("rows".into(), Value::Number((*depth as u64).into())); @@ -328,24 +328,24 @@ fn build_asap_sketches_config(sp: &EdgeSketchProcessor, window_secs: Option SketchParams::Kmv { .. } | SketchParams::Theta { .. } => { unreachable!( "edge sketch processor config requested for a non-sketch or unsupported \ - SketchKind; no Bind* rule in this repo produces one" + SketchAlgorithm; no Bind* rule in this repo produces one" ) } } Value::Mapping(m) } -fn sketch_kind_tag(kind: &SketchKind) -> &'static str { +fn sketch_kind_tag(kind: &SketchAlgorithm) -> &'static str { match kind { - SketchKind::Kll => "kll", - SketchKind::DDSketch => "ddsketch", - SketchKind::Hll => "hll", - SketchKind::Cms | SketchKind::CmsWithHeap => "cms", - SketchKind::CountSketch | SketchKind::CountSketchWithHeap => "count_sketch", - SketchKind::Kmv | SketchKind::Theta => { + SketchAlgorithm::Kll => "kll", + SketchAlgorithm::DDSketch => "ddsketch", + SketchAlgorithm::Hll => "hll", + SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap => "cms", + SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap => "count_sketch", + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => { unreachable!( "edge sketch processor config requested for a non-sketch or unsupported \ - SketchKind; no Bind* rule in this repo produces one" + SketchAlgorithm; no Bind* rule in this repo produces one" ) } } @@ -357,7 +357,7 @@ fn sketch_kind_tag(kind: &SketchKind) -> &'static str { mod tests { use super::*; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, PrometheusArchiveMetric}; - use planner_types::post_asap::{SketchKind, SketchParams}; + use planner_types::post_asap::{SketchAlgorithm, SketchParams}; /// Minimal struct-stub used to validate the emitted DAG parses as the /// otap-dataflow schema. We don't pull in the otap-df-config crate @@ -404,7 +404,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, + sketch_kind: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], diff --git a/control_plane/src/emit/stage_config.rs b/control_plane/src/emit/stage_config.rs index 70bbd3bc3..bfab99cdb 100644 --- a/control_plane/src/emit/stage_config.rs +++ b/control_plane/src/emit/stage_config.rs @@ -64,7 +64,7 @@ use crate::physical::colored_dag::emitter::{ // test module's `use super::*` instead (P2-5). use crate::intent_algebra::ColumnRef; use crate::physical::colored_dag::stage_id::StageId; -use planner_types::post_asap::{SketchKind, SketchParams, SketchQuery}; +use planner_types::post_asap::{SketchAlgorithm, SketchParams, SketchQuery}; // `BackendAggregation.sketch_kind`/`.sketch_params` span both exact // accumulators and approximate sketches -- see // `physical::colored_dag::emitter`'s `use asap_types::{...}` note. @@ -227,7 +227,7 @@ pub fn emit_edge_yaml( // ── MVP §46: 5-sketch routing-connector dispatch ─────────────────────── // // When the planner has populated `cfg.metric_to_family` (the per-metric - // → SketchKind table sourced from the workload spec), we switch to the + // → SketchAlgorithm table sourced from the workload spec), we switch to the // canonical 5-sketch routing-connector wire shape: all referenced // sketch processors live at the top level, the OTel `routing` // *connector* (NOT the deprecated routing processor) lives under @@ -596,11 +596,11 @@ pub fn emit_gateway_yaml( // Processors — one merge processor per merge entry. Naming // convention matches the patched contrib build: - // * SketchKind::DDSketch → `ddsketchmerge` - // * SketchKind::Kll → `kllmerge` - // * SketchKind::Hll → `hllmerge` - // * SketchKind::Cms → `countminsketchmerge` - // * SketchKind::CountSketch → `countsketchmerge` + // * SketchAlgorithm::DDSketch → `ddsketchmerge` + // * SketchAlgorithm::Kll → `kllmerge` + // * SketchAlgorithm::Hll → `hllmerge` + // * SketchAlgorithm::Cms → `countminsketchmerge` + // * SketchAlgorithm::CountSketch → `countsketchmerge` // // We honour `GatewayMergeProcessor::processor_name` if non-empty // (the typed emitter today populates it as `"sketchmergeprocessor"` @@ -1048,7 +1048,7 @@ fn emit_edge_yaml_5sketch_routing( opamp_endpoint: &str, agent_id: &str, ) -> Result { - use planner_types::post_asap::SketchKind; + use planner_types::post_asap::SketchAlgorithm; let otlp_receiver: Value = serde_yaml::from_str( "protocols:\n grpc:\n endpoint: \"0.0.0.0:4317\"\n max_recv_msg_size_mib: 64\n http:\n endpoint: \"0.0.0.0:4318\"\n", @@ -1067,14 +1067,14 @@ fn emit_edge_yaml_5sketch_routing( // The canonical 5-family order below is the iteration order for // every emit (processors, pipelines, hints) so the YAML is stable // across controller runs regardless of HashMap iteration order. - const FAMILY_ORDER: [SketchKind; 5] = [ - SketchKind::DDSketch, - SketchKind::Kll, - SketchKind::Hll, - SketchKind::CountSketch, - SketchKind::Cms, + const FAMILY_ORDER: [SketchAlgorithm; 5] = [ + SketchAlgorithm::DDSketch, + SketchAlgorithm::Kll, + SketchAlgorithm::Hll, + SketchAlgorithm::CountSketch, + SketchAlgorithm::Cms, ]; - let mut needed_families: std::collections::BTreeSet = + let mut needed_families: std::collections::BTreeSet = std::collections::BTreeSet::new(); for families in cfg.metric_to_family.values() { for kind in families { @@ -1098,10 +1098,10 @@ fn emit_edge_yaml_5sketch_routing( // (so the params flow through), otherwise we synthesise a // default-param block. Keyed by `base_family` — `FAMILY_ORDER` is a // fixed 5-bare-family list with no heap-bearing entries, exactly - // matching pre-`SketchKind`-split behavior (heap-bearing-ness was + // matching pre-`SketchAlgorithm`-split behavior (heap-bearing-ness was // never visible to this bare-kind lookup even when it lived as a // `with_heap` params flag). - let mut family_to_proc: HashMap = HashMap::new(); + let mut family_to_proc: HashMap = HashMap::new(); for sp in &cfg.sketch_processors { family_to_proc.insert(base_family(&sp.sketch_kind), sp); } @@ -1280,7 +1280,7 @@ fn emit_edge_yaml_5sketch_routing( // connector fans its samples into both pipelines. Family order // within each metric's pipeline list follows the canonical // `FAMILY_ORDER` so the YAML is stable. - let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = + let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = cfg.metric_to_family.iter().collect(); metric_family_pairs.sort_by(|a, b| a.0.cmp(b.0)); @@ -1310,7 +1310,7 @@ fn emit_edge_yaml_5sketch_routing( // added to EACH of its families' pipelines (the `where metric.name // == ""` guard makes it a no-op on the family's other // metrics). - let mut family_to_keep_processors: HashMap> = HashMap::new(); + let mut family_to_keep_processors: HashMap> = HashMap::new(); for (metric, families) in &metric_family_pairs { let Some(labels) = cfg.metric_to_grouping_labels.get(*metric) else { continue; @@ -1622,19 +1622,19 @@ fn emit_edge_yaml_5sketch_routing( serde_yaml::to_string(&doc).context("serialize edge stage config (5-sketch)") } -/// Map a `SketchKind` to the `family:` token the fused `asap_edge` +/// Map a `SketchAlgorithm` to the `family:` token the fused `asap_edge` /// processor's `metrics[]` list expects. These differ from the OTel /// component-id processor names (`KLL`, `countmin`, …) used by the /// routing-connector path — the fused processor takes a lower-case /// family discriminant per entry, matching the hand-written contract in /// `asap-otel-agent-b6-asap-single-sketch.yaml`. -fn sketch_kind_to_asap_edge_family(kind: &SketchKind) -> &'static str { +fn sketch_kind_to_asap_edge_family(kind: &SketchAlgorithm) -> &'static str { match kind { - SketchKind::DDSketch => "ddsketch", - SketchKind::Kll => "kll", - SketchKind::Hll => "hll", - SketchKind::CountSketch => "countsketch", - SketchKind::Cms => "countminsketch", + SketchAlgorithm::DDSketch => "ddsketch", + SketchAlgorithm::Kll => "kll", + SketchAlgorithm::Hll => "hll", + SketchAlgorithm::CountSketch => "countsketch", + SketchAlgorithm::Cms => "countminsketch", // Every caller iterates the fixed 5-bare-family `FAMILY_ORDER` // list (heap-bearing kinds normalize through `base_family` // before reaching here), and no Bind* rule in this repo @@ -1705,7 +1705,7 @@ fn emit_edge_yaml_asap_edge( _opamp_endpoint: &str, _agent_id: &str, ) -> Result { - use planner_types::post_asap::SketchKind; + use planner_types::post_asap::SketchAlgorithm; // ── Receivers ────────────────────────────────────────────────────────── // OTLP gRPC on 4317 + HTTP on 4318 — same as every other edge emit. @@ -1884,18 +1884,18 @@ fn emit_edge_yaml_asap_edge( // (keyed by family) so the per-metric param block mirrors the // routing path; fall back to catalog defaults when the planner // mapped a family with no enumerated processor. - let mut family_to_proc: HashMap = HashMap::new(); + let mut family_to_proc: HashMap = HashMap::new(); for sp in &cfg.sketch_processors { family_to_proc.insert(base_family(&sp.sketch_kind), sp); } - const FAMILY_ORDER: [SketchKind; 5] = [ - SketchKind::DDSketch, - SketchKind::Kll, - SketchKind::Hll, - SketchKind::CountSketch, - SketchKind::Cms, + const FAMILY_ORDER: [SketchAlgorithm; 5] = [ + SketchAlgorithm::DDSketch, + SketchAlgorithm::Kll, + SketchAlgorithm::Hll, + SketchAlgorithm::CountSketch, + SketchAlgorithm::Cms, ]; - let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = + let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = cfg.metric_to_family.iter().collect(); metric_family_pairs.sort_by(|a, b| a.0.cmp(b.0)); for (metric, families) in &metric_family_pairs { @@ -1903,7 +1903,7 @@ fn emit_edge_yaml_asap_edge( // `FAMILY_ORDER` list — same reasoning as `family_to_proc` above: // a committed heap-bearing kind (`CmsWithHeap`/`CountSketchWithHeap`) // must still match its bare `FAMILY_ORDER` entry. - let bare_families: std::collections::BTreeSet = + let bare_families: std::collections::BTreeSet = families.iter().map(base_family).collect(); for kind in FAMILY_ORDER.iter().filter(|k| bare_families.contains(*k)) { let mut e = Mapping::new(); @@ -1998,7 +1998,7 @@ fn emit_edge_yaml_asap_edge( let whole_stream = effective_by.is_empty() && matches!( kind, - SketchKind::Hll | SketchKind::Cms | SketchKind::CountSketch + SketchAlgorithm::Hll | SketchAlgorithm::Cms | SketchAlgorithm::CountSketch ); if whole_stream { e.insert("mode".into(), Value::String("whole_stream".to_string())); @@ -2045,7 +2045,7 @@ fn emit_edge_yaml_asap_edge( // lossless and serializes byte-identically to dense). We emit the // flag ONLY for the HLL family; non-HLL families carry no // `hll_sparse` key. - if matches!(kind, SketchKind::Hll) { + if matches!(kind, SketchAlgorithm::Hll) { let hll_sparse = if whole_stream { false } else { @@ -2073,7 +2073,7 @@ fn emit_edge_yaml_asap_edge( // its params. let mut countsketch_with_heap = family_to_proc .get(kind) - .is_some_and(|sp| matches!(sp.sketch_kind, SketchKind::CountSketchWithHeap)); + .is_some_and(|sp| matches!(sp.sketch_kind, SketchAlgorithm::CountSketchWithHeap)); match family_to_proc.get(kind).map(|sp| &sp.sketch_params) { Some(SketchParams::DDSketch { alpha }) => { e.insert("relative_accuracy".into(), Value::Number((*alpha).into())); @@ -2100,14 +2100,14 @@ fn emit_edge_yaml_asap_edge( // Family with no enumerated processor — emit catalog // defaults so the entry is still well-formed. match kind { - SketchKind::DDSketch => { + SketchAlgorithm::DDSketch => { e.insert("relative_accuracy".into(), Value::Number(0.01.into())); } - SketchKind::Kll => { + SketchAlgorithm::Kll => { e.insert("k".into(), Value::Number(200u64.into())); } - SketchKind::Hll => {} - SketchKind::CountSketch => { + SketchAlgorithm::Hll => {} + SketchAlgorithm::CountSketch => { e.insert("rows".into(), Value::Number(5u64.into())); e.insert("cols".into(), Value::Number(2048u64.into())); // P1-4: NO enumerated EdgeSketchProcessor for this @@ -2131,7 +2131,7 @@ fn emit_edge_yaml_asap_edge( // step with the backend `with_heap` registration. countsketch_with_heap = cfg.metric_to_item_label.contains_key(*metric); } - SketchKind::Cms => { + SketchAlgorithm::Cms => { e.insert("rows".into(), Value::Number(5u64.into())); e.insert("cols".into(), Value::Number(2048u64.into())); } @@ -2147,7 +2147,7 @@ fn emit_edge_yaml_asap_edge( // families (CMS / HLL) only when `p < 1.0`. Mirrors // `build_edge_processor_block`'s guarded emit so an unset / // 1.0 probability keeps the fused entry byte-identical. - if matches!(kind, SketchKind::Cms | SketchKind::Hll) { + if matches!(kind, SketchAlgorithm::Cms | SketchAlgorithm::Hll) { insert_sample_p(&mut e, cfg.metric_to_sample_p.get(*metric).copied()); } @@ -2168,7 +2168,10 @@ fn emit_edge_yaml_asap_edge( // depending on that default. if matches!( kind, - SketchKind::DDSketch | SketchKind::Hll | SketchKind::CountSketch | SketchKind::Cms + SketchAlgorithm::DDSketch + | SketchAlgorithm::Hll + | SketchAlgorithm::CountSketch + | SketchAlgorithm::Cms ) { e.insert("delta_transmission".into(), Value::Bool(true)); } @@ -2199,7 +2202,7 @@ fn emit_edge_yaml_asap_edge( // that lands — `mapstructure` ignores unknown keys by default — // but the warm-topk behaviour only activates once the asapedge // build carries the fields. See the report's cross-repo note. - if matches!(kind, SketchKind::CountSketch) && countsketch_with_heap { + if matches!(kind, SketchAlgorithm::CountSketch) && countsketch_with_heap { e.insert("emit_heap".into(), Value::Bool(true)); e.insert("heap_size".into(), Value::Number(100u64.into())); // Prefer the workload-declared inner dimension @@ -2243,7 +2246,7 @@ fn emit_edge_yaml_asap_edge( // pure YAML text here (`mapstructure` ignores unknown keys), so // emitting it is safe even before that lands; the corrected // keying only activates once the asapedge build carries it. - if matches!(kind, SketchKind::Hll | SketchKind::Cms) { + if matches!(kind, SketchAlgorithm::Hll | SketchAlgorithm::Cms) { if let Some(item_label) = cfg.metric_to_item_label.get(*metric) { if !item_label.is_empty() { e.insert("item_label".into(), Value::String(item_label.clone())); @@ -2450,30 +2453,30 @@ tsdb_block_duration: {window_secs}s\n", ) } -/// Map a `SketchKind` to the OTel processor name registered by the +/// Map a `SketchAlgorithm` to the OTel processor name registered by the /// patched contrib build's factory. Keep in sync with /// `crate::physical::colored_dag::emitter::edge_processor_name`. -fn sketch_kind_to_processor_name(kind: &SketchKind) -> &'static str { +fn sketch_kind_to_processor_name(kind: &SketchAlgorithm) -> &'static str { match kind { - SketchKind::DDSketch => "ddsketch", - SketchKind::Kll => "KLL", - SketchKind::Hll => "HLL", - SketchKind::CountSketch => "countsketch", - SketchKind::Cms => "countmin", + SketchAlgorithm::DDSketch => "ddsketch", + SketchAlgorithm::Kll => "KLL", + SketchAlgorithm::Hll => "HLL", + SketchAlgorithm::CountSketch => "countsketch", + SketchAlgorithm::Cms => "countmin", // Callers only ever pass a bare `FAMILY_ORDER` entry. other => unreachable!("sketch_kind_to_processor_name: unexpected kind {other:?}"), } } -/// Map a `SketchKind` to its per-family pipeline name in the routing +/// Map a `SketchAlgorithm` to its per-family pipeline name in the routing /// connector layout. -fn sketch_kind_to_pipeline_name(kind: &SketchKind) -> &'static str { +fn sketch_kind_to_pipeline_name(kind: &SketchAlgorithm) -> &'static str { match kind { - SketchKind::DDSketch => "metrics/ddsketch_path", - SketchKind::Kll => "metrics/kll_path", - SketchKind::Hll => "metrics/hll_path", - SketchKind::CountSketch => "metrics/countsketch_path", - SketchKind::Cms => "metrics/countminsketch_path", + SketchAlgorithm::DDSketch => "metrics/ddsketch_path", + SketchAlgorithm::Kll => "metrics/kll_path", + SketchAlgorithm::Hll => "metrics/hll_path", + SketchAlgorithm::CountSketch => "metrics/countsketch_path", + SketchAlgorithm::Cms => "metrics/countminsketch_path", // Callers only ever pass a bare `FAMILY_ORDER` entry. other => unreachable!("sketch_kind_to_pipeline_name: unexpected kind {other:?}"), } @@ -2611,14 +2614,14 @@ fn build_transform_keep_processor_block(metric: &str, labels: &[String]) -> Valu serde_yaml::from_str(&yaml).expect("transform/keep_for_* yaml is well-formed by construction") } -/// Build a default-parameter processor block for a `SketchKind` when +/// Build a default-parameter processor block for a `SketchAlgorithm` when /// the planner's `metric_to_family` references a family that /// `cfg.sketch_processors` didn't enumerate. Defaults match the catalog /// values used by the planner's L4 rules so the wire shape is what the /// rest of the system expects when a metric is later re-routed onto /// this family. fn build_default_edge_processor_block( - kind: &SketchKind, + kind: &SketchAlgorithm, window_secs: Option, metric_name_hint: Option<&str>, sample_p: Option, @@ -2628,23 +2631,26 @@ fn build_default_edge_processor_block( // the tag/processor-name lookups below, which are keyed on the bare // family. `stored_kind`/`params` are what actually land on the // synthesized processor; `CountSketch`'s default stays heap-bearing - // (matching this function's pre-`SketchKind`-split default of + // (matching this function's pre-`SketchAlgorithm`-split default of // `with_heap: true` — `Cms`'s default was `with_heap: false` and // stays bare). let (stored_kind, params) = match kind { - SketchKind::DDSketch => (SketchKind::DDSketch, SketchParams::DDSketch { alpha: 0.01 }), - SketchKind::Kll => (SketchKind::Kll, SketchParams::Kll { k: 200 }), - SketchKind::Hll => (SketchKind::Hll, SketchParams::Hll { precision: 14 }), - SketchKind::CountSketch => ( - SketchKind::CountSketchWithHeap, + SketchAlgorithm::DDSketch => ( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + ), + SketchAlgorithm::Kll => (SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + SketchAlgorithm::Hll => (SketchAlgorithm::Hll, SketchParams::Hll { precision: 14 }), + SketchAlgorithm::CountSketch => ( + SketchAlgorithm::CountSketchWithHeap, SketchParams::CountSketchWithHeap { width: 2048, depth: 5, heap_size: 10, }, ), - SketchKind::Cms => ( - SketchKind::Cms, + SketchAlgorithm::Cms => ( + SketchAlgorithm::Cms, SketchParams::Cms { width: 4096, depth: 4, @@ -2852,7 +2858,7 @@ fn build_edge_processor_block( } SketchParams::Kmv { .. } | SketchParams::Theta { .. } => unreachable!( "edge sketch processor config requested for a non-sketch or unsupported \ - SketchKind; no Bind* rule in this repo produces one" + SketchAlgorithm; no Bind* rule in this repo produces one" ), } @@ -2886,13 +2892,15 @@ fn insert_sample_p(m: &mut Mapping, sample_p: Option) { /// here so the emitted YAML round-trips through the patched build. fn gateway_merge_processor_name(mp: &GatewayMergeProcessor) -> String { match mp.sketch_kind { - SketchKind::Kll => "kllmerge".to_string(), - SketchKind::DDSketch => "ddsketchmerge".to_string(), - SketchKind::Hll => "hllmerge".to_string(), - SketchKind::Cms | SketchKind::CmsWithHeap => "countminsketchmerge".to_string(), - SketchKind::CountSketch | SketchKind::CountSketchWithHeap => "countsketchmerge".to_string(), - SketchKind::Kmv | SketchKind::Theta => unreachable!( - "gateway_merge_processor_name: non-sketch or unsupported SketchKind; \ + SketchAlgorithm::Kll => "kllmerge".to_string(), + SketchAlgorithm::DDSketch => "ddsketchmerge".to_string(), + SketchAlgorithm::Hll => "hllmerge".to_string(), + SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap => "countminsketchmerge".to_string(), + SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap => { + "countsketchmerge".to_string() + } + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => unreachable!( + "gateway_merge_processor_name: non-sketch or unsupported SketchAlgorithm; \ no Bind* rule in this repo produces one" ), } @@ -3042,34 +3050,34 @@ fn column_ref_to_wire_key(col: &ColumnRef) -> String { } } -/// Collapse a heap-bearing `SketchKind` to its bare counterpart. +/// Collapse a heap-bearing `SketchAlgorithm` to its bare counterpart. /// Identity for every other kind. /// /// The 5-sketch routing-connector edge YAML path (`emit_edge_yaml`'s /// `USE_5SKETCH_ROUTING` branch and its `metric_to_family` sibling) /// keys its fixed `FAMILY_ORDER` list and lookup maps on the 5 bare -/// families only — matching the retired `sketch_algebra::SketchKind`, +/// families only — matching the retired `sketch_algebra::SketchAlgorithm`, /// which had no heap-bearing variant at all (`with_heap` was a /// `SketchParams` field, invisible to anything keying on kind alone). /// A committed heap-bearing kind (`CmsWithHeap`/`CountSketchWithHeap`, /// from a topk binding) needs to normalize through this before it's /// used as a key or set member in that path, or it silently fails to /// match its bare `FAMILY_ORDER` entry. -fn base_family(kind: &SketchKind) -> SketchKind { +fn base_family(kind: &SketchAlgorithm) -> SketchAlgorithm { match kind { - SketchKind::CmsWithHeap => SketchKind::Cms, - SketchKind::CountSketchWithHeap => SketchKind::CountSketch, + SketchAlgorithm::CmsWithHeap => SketchAlgorithm::Cms, + SketchAlgorithm::CountSketchWithHeap => SketchAlgorithm::CountSketch, other => other.clone(), } } -/// Map a `SketchKind` to the backend's `AggregationType::Display` +/// Map a `SketchAlgorithm` to the backend's `AggregationType::Display` /// string — the same mapping /// [`crate::config::asapquery_backend::map_sketch_type_to_agg_type`] uses /// (the strings must match `AggregationType::FromStr` in the backend's /// `promql_utilities::query_logics::enums`). /// -/// Heap-bearing is now identity, not a params flag (`SketchKind::CmsWithHeap` +/// Heap-bearing is now identity, not a params flag (`SketchAlgorithm::CmsWithHeap` /// / `CountSketchWithHeap`, set by `BindCountSketchOnTopK` — see /// `sketch_algebra::rules::bind_cms_topk`), so this maps on `kind` alone; /// `params` is unused but kept for call-site stability. This is what @@ -3101,20 +3109,20 @@ fn sketch_kind_to_backend_type(kind: &SummaryKind, _params: &SummaryParams) -> & } } -/// Stable lowercase tag for a `SketchKind` — used as a passthrough +/// Stable lowercase tag for a `SketchAlgorithm` — used as a passthrough /// `sketch_kind` field in YAML so downstream consumers can dispatch /// without round-tripping through serde. Heap-bearing kinds reuse their /// bare counterpart's tag — this field never distinguished `with_heap` -/// even before `SketchKind` split it into its own variant. -fn sketch_kind_tag(kind: &SketchKind) -> &'static str { +/// even before `SketchAlgorithm` split it into its own variant. +fn sketch_kind_tag(kind: &SketchAlgorithm) -> &'static str { match kind { - SketchKind::Kll => "kll", - SketchKind::DDSketch => "ddsketch", - SketchKind::Hll => "hll", - SketchKind::Cms | SketchKind::CmsWithHeap => "cms", - SketchKind::CountSketch | SketchKind::CountSketchWithHeap => "count_sketch", - SketchKind::Kmv | SketchKind::Theta => unreachable!( - "sketch_kind_tag: non-sketch or unsupported SketchKind; \ + SketchAlgorithm::Kll => "kll", + SketchAlgorithm::DDSketch => "ddsketch", + SketchAlgorithm::Hll => "hll", + SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap => "cms", + SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap => "count_sketch", + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => unreachable!( + "sketch_kind_tag: non-sketch or unsupported SketchAlgorithm; \ no Bind* rule in this repo produces one" ), } @@ -3177,7 +3185,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, + sketch_kind: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], @@ -3262,7 +3270,7 @@ mod tests { let mut cfg = ddsketch_edge_cfg(); cfg.sketch_processors[0] = EdgeSketchProcessor { processor_name: "KLL".to_string(), - sketch_kind: SketchKind::Kll, + sketch_kind: SketchAlgorithm::Kll, sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg7".to_string(), }; @@ -3297,13 +3305,17 @@ mod tests { // flag (see `edge_yaml_kll_uses_k_param`). for (kind, processor_name, params) in [ ( - SketchKind::DDSketch, + SketchAlgorithm::DDSketch, "ddsketch", SketchParams::DDSketch { alpha: 0.01 }, ), - (SketchKind::Hll, "HLL", SketchParams::Hll { precision: 14 }), ( - SketchKind::CountSketch, + SketchAlgorithm::Hll, + "HLL", + SketchParams::Hll { precision: 14 }, + ), + ( + SketchAlgorithm::CountSketch, "countsketch", SketchParams::CountSketchWithHeap { width: 2048, @@ -3312,7 +3324,7 @@ mod tests { }, ), ( - SketchKind::Cms, + SketchAlgorithm::Cms, "countmin", SketchParams::Cms { width: 4096, @@ -3342,7 +3354,7 @@ mod tests { cfg.source_metric = Some("endpoint_request_freq".to_string()); cfg.sketch_processors[0] = EdgeSketchProcessor { processor_name: "countmin".to_string(), - sketch_kind: SketchKind::Cms, + sketch_kind: SketchAlgorithm::Cms, sketch_params: SketchParams::Cms { width: 4096, depth: 4, @@ -3375,7 +3387,7 @@ mod tests { otlp_receiver_port: 4317, merge_processors: vec![GatewayMergeProcessor { processor_name: "sketchmergeprocessor".to_string(), - sketch_kind: SketchKind::DDSketch, + sketch_kind: SketchAlgorithm::DDSketch, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Backend), @@ -3419,12 +3431,12 @@ mod tests { merge_processors: vec![ GatewayMergeProcessor { processor_name: "x".into(), - sketch_kind: SketchKind::Kll, + sketch_kind: SketchAlgorithm::Kll, aggregation_id: "agg0".into(), }, GatewayMergeProcessor { processor_name: "x".into(), - sketch_kind: SketchKind::Hll, + sketch_kind: SketchAlgorithm::Hll, aggregation_id: "agg1".into(), }, ], @@ -3624,23 +3636,26 @@ mod tests { /// requested kind. `aggregation_id` is hard-coded — the routing /// emitter doesn't care about it. Accepts the 5 canonical bare /// families callers actually pass; `CountSketch` stores as the - /// heap-bearing variant (matching this fixture's pre-`SketchKind`-split + /// heap-bearing variant (matching this fixture's pre-`SketchAlgorithm`-split /// behavior, when `with_heap: true` was a `CountSketchParams` field /// rather than a distinct kind). - fn backend_cfg_with_kind(kind: SketchKind) -> BackendStageConfig { + fn backend_cfg_with_kind(kind: SketchAlgorithm) -> BackendStageConfig { let (stored_kind, params) = match kind { - SketchKind::DDSketch => (SketchKind::DDSketch, SketchParams::DDSketch { alpha: 0.01 }), - SketchKind::Kll => (SketchKind::Kll, SketchParams::Kll { k: 200 }), - SketchKind::Hll => (SketchKind::Hll, SketchParams::Hll { precision: 14 }), - SketchKind::Cms => ( - SketchKind::Cms, + SketchAlgorithm::DDSketch => ( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + ), + SketchAlgorithm::Kll => (SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + SketchAlgorithm::Hll => (SketchAlgorithm::Hll, SketchParams::Hll { precision: 14 }), + SketchAlgorithm::Cms => ( + SketchAlgorithm::Cms, SketchParams::Cms { width: 4096, depth: 4, }, ), - SketchKind::CountSketch => ( - SketchKind::CountSketchWithHeap, + SketchAlgorithm::CountSketch => ( + SketchAlgorithm::CountSketchWithHeap, SketchParams::CountSketchWithHeap { width: 2048, depth: 5, @@ -3665,10 +3680,12 @@ mod tests { readouts: vec![BackendReadout { aggregation_id: "agg0".into(), op: match kind { - SketchKind::DDSketch | SketchKind::Kll => SketchQuery::Quantile { q: 0.99 }, - SketchKind::Hll => SketchQuery::Cardinality, - SketchKind::CountSketch => SketchQuery::TopK { k: 10 }, - SketchKind::Cms => SketchQuery::PointCount { + SketchAlgorithm::DDSketch | SketchAlgorithm::Kll => { + SketchQuery::Quantile { q: 0.99 } + } + SketchAlgorithm::Hll => SketchQuery::Cardinality, + SketchAlgorithm::CountSketch => SketchQuery::TopK { k: 10 }, + SketchAlgorithm::Cms => SketchQuery::PointCount { key: ColumnRef::Named("user_42".into()), value: None, }, @@ -3682,7 +3699,7 @@ mod tests { #[test] fn storage_routing_emits_default_engine_and_metrics_array() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SketchAlgorithm::DDSketch); let plans: Vec<(String, &BackendStageConfig)> = vec![("http_request_duration_seconds".to_string(), &ddsketch)]; let v = emit_backend_storage_routing(&plans).expect("emit ok"); @@ -3700,7 +3717,7 @@ mod tests { /// byte-compatible (modulo the new `tenant` field appearing). #[test] fn storage_routing_default_tenant_for_single_tenant_emit() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SketchAlgorithm::DDSketch); let v = emit_backend_storage_routing(&[("latency".into(), &ddsketch)]).expect("emit ok"); assert_eq!(v["tenant"], DEFAULT_TENANT); } @@ -3712,7 +3729,7 @@ mod tests { /// body-tenant precedence rule. #[test] fn storage_routing_for_tenant_emits_explicit_tenant_field() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SketchAlgorithm::DDSketch); let v = emit_backend_storage_routing_for_tenant("tenant-a", &[("latency".into(), &ddsketch)]) .expect("emit ok"); @@ -3739,7 +3756,7 @@ mod tests { #[test] fn storage_routing_ddasap_query_serves_quantile_archive_serves_others() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SketchAlgorithm::DDSketch); let v = emit_backend_storage_routing(&[("latency".into(), &ddsketch)]).expect("emit ok"); let metric = &v["metrics"][0]; let targets = metric["targets"].as_array().expect("targets array"); @@ -3781,7 +3798,7 @@ mod tests { #[test] fn storage_routing_count_sketch_pulls_topk_off_archive() { - let cs = backend_cfg_with_kind(SketchKind::CountSketch); + let cs = backend_cfg_with_kind(SketchAlgorithm::CountSketch); let v = emit_backend_storage_routing(&[("requests".into(), &cs)]).expect("emit ok"); let archive_shapes: Vec = v["metrics"][0]["targets"][1]["applies_to_query_shape"] .as_array() @@ -3801,7 +3818,7 @@ mod tests { #[test] fn storage_routing_hll_pulls_count_off_archive() { - let hll = backend_cfg_with_kind(SketchKind::Hll); + let hll = backend_cfg_with_kind(SketchAlgorithm::Hll); let v = emit_backend_storage_routing(&[("active_users".into(), &hll)]).expect("emit ok"); let archive_shapes: Vec = v["metrics"][0]["targets"][1]["applies_to_query_shape"] .as_array() @@ -3826,9 +3843,9 @@ mod tests { // runs (HashMap iteration order can drift, but our impl // stages everything through a Vec so order matches input // order). - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); - let hll = backend_cfg_with_kind(SketchKind::Hll); - let cs = backend_cfg_with_kind(SketchKind::CountSketch); + let ddsketch = backend_cfg_with_kind(SketchAlgorithm::DDSketch); + let hll = backend_cfg_with_kind(SketchAlgorithm::Hll); + let cs = backend_cfg_with_kind(SketchAlgorithm::CountSketch); let plans: Vec<(String, &BackendStageConfig)> = vec![ ("http_requests_total".into(), &cs), ("active_users".into(), &hll), @@ -3993,20 +4010,24 @@ mod tests { /// silently break the backend. #[test] fn phase_b_backend_agg_type_strings_for_every_sketch_kind() { - let cases: Vec<(SketchKind, SketchParams, &str)> = vec![ + let cases: Vec<(SketchAlgorithm, SketchParams, &str)> = vec![ ( - SketchKind::Kll, + SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }, "DatasketchesKLL", ), ( - SketchKind::DDSketch, + SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha: 0.01 }, "DDSketch", ), - (SketchKind::Hll, SketchParams::Hll { precision: 14 }, "HLL"), ( - SketchKind::Cms, + SketchAlgorithm::Hll, + SketchParams::Hll { precision: 14 }, + "HLL", + ), + ( + SketchAlgorithm::Cms, SketchParams::Cms { width: 4096, depth: 4, @@ -4014,7 +4035,7 @@ mod tests { "CountMinSketch", ), ( - SketchKind::CmsWithHeap, + SketchAlgorithm::CmsWithHeap, SketchParams::CmsWithHeap { width: 4096, depth: 4, @@ -4023,7 +4044,7 @@ mod tests { "CountMinSketchWithHeap", ), ( - SketchKind::CountSketch, + SketchAlgorithm::CountSketch, SketchParams::CountSketch { width: 2048, depth: 5, @@ -4031,7 +4052,7 @@ mod tests { "CountSketch", ), ( - SketchKind::CountSketchWithHeap, + SketchAlgorithm::CountSketchWithHeap, SketchParams::CountSketchWithHeap { width: 2048, depth: 5, @@ -4100,7 +4121,7 @@ mod tests { /// producing aggregation. Pins the sort order + key names. Phase β /// uses this as the wire-format anchor for the wider intent set — /// the JSON shape is intent-orthogonal, so adding new intents to L3 - /// can't drift this off so long as they bind through SketchKind / + /// can't drift this off so long as they bind through SketchAlgorithm / /// SketchParams. #[test] fn phase_b_backend_json_aggregation_readout_alias_snapshot() { @@ -4223,7 +4244,7 @@ mod tests { /// Prometheus-archive metric coexist in one routing JSON. #[test] fn phase_eps1_mixed_mode1_and_mode3_share_one_routing_table() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SketchAlgorithm::DDSketch); let plans: Vec<(String, &BackendStageConfig)> = vec![("latency_seconds".into(), &ddsketch)]; let mode3 = vec!["http_requests_total".to_string()]; let v = emit_backend_storage_routing_with_prometheus(&plans, &mode3).expect("emit ok"); @@ -4589,7 +4610,7 @@ mod tests { /// (ASAPCollector#400). Most fixtures map each metric to exactly one /// family — this keeps them concise while exercising the SET-shaped /// `metric_to_family`. - fn one(kind: SketchKind) -> std::collections::BTreeSet { + fn one(kind: SketchAlgorithm) -> std::collections::BTreeSet { std::collections::BTreeSet::from([kind]) } @@ -4598,16 +4619,16 @@ mod tests { /// metric maps to a single-family set (this workload's per-metric set /// size is 1; see `mvp46_multi_family_metric_*` for the size>1 case). fn five_sketch_edge_cfg() -> EdgeStageConfig { - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); metric_to_family.insert( "http_requests_total_latency_ms".into(), - one(SketchKind::DDSketch), + one(SketchAlgorithm::DDSketch), ); - metric_to_family.insert("request_size_bytes".into(), one(SketchKind::Kll)); - metric_to_family.insert("unique_users_per_min".into(), one(SketchKind::Hll)); - metric_to_family.insert("top_endpoint_qps".into(), one(SketchKind::CountSketch)); - metric_to_family.insert("endpoint_request_freq".into(), one(SketchKind::Cms)); + metric_to_family.insert("request_size_bytes".into(), one(SketchAlgorithm::Kll)); + metric_to_family.insert("unique_users_per_min".into(), one(SketchAlgorithm::Hll)); + metric_to_family.insert("top_endpoint_qps".into(), one(SketchAlgorithm::CountSketch)); + metric_to_family.insert("endpoint_request_freq".into(), one(SketchAlgorithm::Cms)); // `http_requests_total` is intentionally NOT in this map — it // falls through to the `metrics/raw_passthrough` default. EdgeStageConfig { @@ -4962,7 +4983,7 @@ mod tests { /// Helper: build an `EdgeStageConfig` whose `metric_to_family` is the /// given metric→set map, with sensible defaults for the other fields. fn edge_cfg_with_families( - metric_to_family: HashMap>, + metric_to_family: HashMap>, ) -> EdgeStageConfig { EdgeStageConfig { source_metric: None, @@ -4994,7 +5015,7 @@ mod tests { // families — this is the core bandwidth fix. let cfg = edge_cfg_with_families(HashMap::from([( "latency_ms".to_string(), - one(SketchKind::DDSketch), + one(SketchAlgorithm::DDSketch), )])); let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -5035,8 +5056,8 @@ mod tests { // HLL). Exactly those two pipelines/processors must be emitted; // KLL/CountSketch/CMS pruned. let cfg = edge_cfg_with_families(HashMap::from([ - ("latency_ms".to_string(), one(SketchKind::DDSketch)), - ("uniques".to_string(), one(SketchKind::Hll)), + ("latency_ms".to_string(), one(SketchAlgorithm::DDSketch)), + ("uniques".to_string(), one(SketchAlgorithm::Hll)), ])); let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -5074,7 +5095,7 @@ mod tests { // them). let cfg = edge_cfg_with_families(HashMap::from([( "http_requests".to_string(), - std::collections::BTreeSet::from([SketchKind::DDSketch, SketchKind::Hll]), + std::collections::BTreeSet::from([SketchAlgorithm::DDSketch, SketchAlgorithm::Hll]), )])); let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -6078,40 +6099,40 @@ mod tests { /// tier (cold), and the counter-shaped sketch inputs in the /// cumulativetodelta list. fn fused_asap_edge_cfg() -> EdgeStageConfig { - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); metric_to_family.insert( "http_requests_total_latency_ms".into(), - one(SketchKind::DDSketch), + one(SketchAlgorithm::DDSketch), ); - metric_to_family.insert("request_size_bytes".into(), one(SketchKind::Kll)); - metric_to_family.insert("unique_users_per_min".into(), one(SketchKind::Hll)); - metric_to_family.insert("top_endpoint_qps".into(), one(SketchKind::CountSketch)); - metric_to_family.insert("endpoint_request_freq".into(), one(SketchKind::Cms)); + metric_to_family.insert("request_size_bytes".into(), one(SketchAlgorithm::Kll)); + metric_to_family.insert("unique_users_per_min".into(), one(SketchAlgorithm::Hll)); + metric_to_family.insert("top_endpoint_qps".into(), one(SketchAlgorithm::CountSketch)); + metric_to_family.insert("endpoint_request_freq".into(), one(SketchAlgorithm::Cms)); // Per-family params, mirroring the target config's per-entry knobs. let sketch_processors = vec![ EdgeSketchProcessor { processor_name: "ddsketch".into(), - sketch_kind: SketchKind::DDSketch, + sketch_kind: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".into(), }, EdgeSketchProcessor { processor_name: "KLL".into(), - sketch_kind: SketchKind::Kll, + sketch_kind: SketchAlgorithm::Kll, sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg1".into(), }, EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchKind::Hll, + sketch_kind: SketchAlgorithm::Hll, sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg2".into(), }, EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SketchKind::CountSketchWithHeap, + sketch_kind: SketchAlgorithm::CountSketchWithHeap, sketch_params: SketchParams::CountSketchWithHeap { width: 2048, depth: 5, @@ -6121,7 +6142,7 @@ mod tests { }, EdgeSketchProcessor { processor_name: "countmin".into(), - sketch_kind: SketchKind::Cms, + sketch_kind: SketchAlgorithm::Cms, sketch_params: SketchParams::Cms { width: 2048, depth: 5, @@ -6552,9 +6573,9 @@ mod tests { fn fused_asap_edge_per_group_hll_is_per_series_and_sparse() { let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("distinct_users_by_region".into(), one(SketchKind::Hll)); + metric_to_family.insert("distinct_users_by_region".into(), one(SketchAlgorithm::Hll)); let mut metric_to_grouping_labels: HashMap> = HashMap::new(); metric_to_grouping_labels.insert("distinct_users_by_region".into(), vec!["region".into()]); @@ -6573,7 +6594,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchKind::Hll, + sketch_kind: SketchAlgorithm::Hll, sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], @@ -6650,9 +6671,9 @@ mod tests { fn fused_asap_edge_per_series_hll_high_cardinality_hint_is_dense() { let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("distinct_users_by_region".into(), one(SketchKind::Hll)); + metric_to_family.insert("distinct_users_by_region".into(), one(SketchAlgorithm::Hll)); let mut metric_to_grouping_labels: HashMap> = HashMap::new(); metric_to_grouping_labels.insert("distinct_users_by_region".into(), vec!["region".into()]); @@ -6670,7 +6691,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchKind::Hll, + sketch_kind: SketchAlgorithm::Hll, sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], @@ -6725,9 +6746,9 @@ mod tests { fn fused_asap_edge_whole_stream_hll_is_dense_regardless_of_hint() { let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("distinct_users_global".into(), one(SketchKind::Hll)); + metric_to_family.insert("distinct_users_global".into(), one(SketchAlgorithm::Hll)); // No grouping label + an item_label ⇒ effective aggregate_by empty ⇒ // whole-stream HLL. @@ -6744,7 +6765,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchKind::Hll, + sketch_kind: SketchAlgorithm::Hll, sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], @@ -6799,10 +6820,10 @@ mod tests { fn fused_asap_edge_quantile_only_omits_mode_and_sparse() { let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("latency_ms".into(), one(SketchKind::DDSketch)); - metric_to_family.insert("payload_bytes".into(), one(SketchKind::Kll)); + metric_to_family.insert("latency_ms".into(), one(SketchAlgorithm::DDSketch)); + metric_to_family.insert("payload_bytes".into(), one(SketchAlgorithm::Kll)); let cfg = EdgeStageConfig { source_metric: None, @@ -6811,13 +6832,13 @@ mod tests { sketch_processors: vec![ EdgeSketchProcessor { processor_name: "ddsketch".into(), - sketch_kind: SketchKind::DDSketch, + sketch_kind: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".into(), }, EdgeSketchProcessor { processor_name: "KLL".into(), - sketch_kind: SketchKind::Kll, + sketch_kind: SketchAlgorithm::Kll, sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg1".into(), }, @@ -6972,10 +6993,10 @@ mod tests { // Two metrics: a sketch-only one (warm) and one that is BOTH // sketched AND archived (both). The archive set is the precise // plan signal — only `archived_metric` is in it. - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("sketch_only_metric".into(), [SketchKind::Kll].into()); - metric_to_family.insert("archived_metric".into(), [SketchKind::DDSketch].into()); + metric_to_family.insert("sketch_only_metric".into(), [SketchAlgorithm::Kll].into()); + metric_to_family.insert("archived_metric".into(), [SketchAlgorithm::DDSketch].into()); let cfg = EdgeStageConfig { source_metric: None, @@ -7051,7 +7072,7 @@ mod tests { let mut cfg = fused_asap_edge_cfg(); cfg.metric_to_family.insert( "http_requests_total_latency_ms".into(), - one(SketchKind::Kll), + one(SketchAlgorithm::Kll), ); let yaml = emit_edge_yaml(&cfg, "ws://c/", "agent-1").expect("emit ok"); @@ -7299,7 +7320,7 @@ mod tests { for &(w, d) in &[(2048u32, 5u32), (1024, 4), (4096, 6), (2, 1), (256, 3)] { let sp = EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SketchKind::CountSketch, + sketch_kind: SketchAlgorithm::CountSketch, sketch_params: SketchParams::CountSketch { width: w, depth: d }, aggregation_id: "agg-cs".into(), }; @@ -7345,11 +7366,14 @@ mod tests { /// `metric_to_family` but provides NO matching `EdgeSketchProcessor`, /// driving the fused emit into the catalog-default (`None`) arm. fn fused_cfg_countsketch_no_processor() -> EdgeStageConfig { - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); // CountSketch family declared, but `sketch_processors` is EMPTY for // it — the `family_to_proc.get(kind)` lookup returns None. - metric_to_family.insert("endpoint_request_freq".into(), one(SketchKind::CountSketch)); + metric_to_family.insert( + "endpoint_request_freq".into(), + one(SketchAlgorithm::CountSketch), + ); EdgeStageConfig { source_metric: None, @@ -7407,10 +7431,10 @@ mod tests { let mut cfg = fused_cfg_countsketch_no_processor(); cfg.metric_to_family.clear(); cfg.metric_to_family - .insert("top_endpoint_qps".into(), one(SketchKind::CountSketch)); + .insert("top_endpoint_qps".into(), one(SketchAlgorithm::CountSketch)); cfg.sketch_processors = vec![EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SketchKind::CountSketchWithHeap, + sketch_kind: SketchAlgorithm::CountSketchWithHeap, sketch_params: SketchParams::CountSketchWithHeap { width: 2048, depth: 5, diff --git a/control_plane/src/emit/telegraf.rs b/control_plane/src/emit/telegraf.rs index 9e9be6d70..2fadbba2f 100644 --- a/control_plane/src/emit/telegraf.rs +++ b/control_plane/src/emit/telegraf.rs @@ -41,7 +41,7 @@ use anyhow::{Context, Result}; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, EdgeStageConfig, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; -use planner_types::post_asap::{SketchKind, SketchParams}; +use planner_types::post_asap::{SketchAlgorithm, SketchParams}; /// Default Prometheus remote-write URL for Mode 3 — Telegraf doesn't /// support OTLP-HTTP egress, so we land in the same Prometheus archive @@ -180,7 +180,7 @@ fn emit_processors_allsketches( } // Heap-bearing width/depth extraction is identical to the bare // kind — this path never distinguished `with_heap` even before - // `SketchKind` split it into its own variant. + // `SketchAlgorithm` split it into its own variant. SketchParams::Cms { width, depth } | SketchParams::CmsWithHeap { width, depth, .. } => { out.push_str(&format!(" rows = {depth}\n")); out.push_str(&format!(" columns = {width}\n")); @@ -199,7 +199,7 @@ fn emit_processors_allsketches( // `ExactParams`, a distinct type post ASAPPlanner#218's split. SketchParams::Kmv { .. } | SketchParams::Theta { .. } => { unreachable!( - "edge sketch processor config requested for an unsupported SketchKind; \ + "edge sketch processor config requested for an unsupported SketchAlgorithm; \ no Bind* rule in this repo produces one" ) } @@ -207,17 +207,17 @@ fn emit_processors_allsketches( out.push('\n'); } -fn sketch_kind_tag(kind: &SketchKind) -> &'static str { +fn sketch_kind_tag(kind: &SketchAlgorithm) -> &'static str { match kind { - SketchKind::Kll => "kll", - SketchKind::DDSketch => "ddsketch", - SketchKind::Hll => "hll", - SketchKind::Cms | SketchKind::CmsWithHeap => "cms", - SketchKind::CountSketch | SketchKind::CountSketchWithHeap => "count_sketch", - SketchKind::Kmv | SketchKind::Theta => { + SketchAlgorithm::Kll => "kll", + SketchAlgorithm::DDSketch => "ddsketch", + SketchAlgorithm::Hll => "hll", + SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap => "cms", + SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap => "count_sketch", + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => { unreachable!( "edge sketch processor config requested for an unsupported \ - SketchKind; no Bind* rule in this repo produces one" + SketchAlgorithm; no Bind* rule in this repo produces one" ) } } @@ -318,7 +318,7 @@ mod toml_minimal { mod tests { use super::*; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, PrometheusArchiveMetric}; - use planner_types::post_asap::{SketchKind, SketchParams}; + use planner_types::post_asap::{SketchAlgorithm, SketchParams}; fn ddsketch_edge_cfg_mode1() -> EdgeStageConfig { EdgeStageConfig { @@ -327,7 +327,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, + sketch_kind: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], @@ -536,7 +536,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "KLL".to_string(), - sketch_kind: SketchKind::Kll, + sketch_kind: SketchAlgorithm::Kll, sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg0".to_string(), }], diff --git a/control_plane/src/intent_algebra/expr_ir.rs b/control_plane/src/intent_algebra/expr_ir.rs index 15a06f147..49712e5b4 100644 --- a/control_plane/src/intent_algebra/expr_ir.rs +++ b/control_plane/src/intent_algebra/expr_ir.rs @@ -28,7 +28,9 @@ //! sites), so this is the only file that needs to know the upstream //! names changed. -pub use planner_types::pre_asap::{ArithOp, ColumnRef, CompareOp, ScalarValue as L3Scalar}; +pub use planner_types::pre_asap::{ + ArithmeticOpKind as ArithOp, ColumnRef, CompareOpKind as CompareOp, ScalarValue as L3Scalar, +}; use planner_types::pre_asap::{QueryExpr, UnresolvedQueryExpr}; /// The front-end-emitted, name-based scalar/relational tree — was a diff --git a/control_plane/src/intent_algebra/query_expr.rs b/control_plane/src/intent_algebra/query_expr.rs index a0f0dde5f..9bc976dcf 100644 --- a/control_plane/src/intent_algebra/query_expr.rs +++ b/control_plane/src/intent_algebra/query_expr.rs @@ -57,6 +57,8 @@ //! `label_filter_to_predicate` below (`asap_ir`'s `Scan` takes typed //! `Predicate`s, not a separate label-filter list). +use std::rc::Rc; + use planner_types::pre_asap::schema::ColumnId; pub use planner_types::pre_asap::{ aggregate_output_schema, AtModifier, BinaryOpKind, DataModel, GroupKeys, GroupSide, @@ -64,7 +66,9 @@ pub use planner_types::pre_asap::{ SampleKind, SetOpKind, SortKey, Source, TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFuncKind, }; -pub use planner_types::pre_asap::{ArithOp, ColumnRef, CompareOp}; +pub use planner_types::pre_asap::{ + ArithmeticOpKind as ArithOp, ColumnRef, CompareOpKind as CompareOp, +}; // `L3Scalar`/`L3Expr`/`L2Expr` and `WindowKind`: see `expr_ir.rs`'s and // `crates/asap_types/src/enums.rs`'s module docs respectively -- // ASAPPlanner deleted its `WindowKind` (no `QueryExpr::Window` producer @@ -92,10 +96,10 @@ pub struct LabelFilter { /// to the schema; this is a defensive fallback, not the primary path). pub fn label_filter_to_predicate(lf: &LabelFilter, schema: &Schema) -> Option { let id = schema.column_id(&lf.label)?; - Some(Predicate(Box::new(L3Expr::Compare { - left: Box::new(L3Expr::Column(id)), + Some(Predicate(Rc::new(L3Expr::Compare { + left: Rc::new(L3Expr::Column(id)), op: CompareOp::Eq, - right: Box::new(L3Expr::Literal(L3Scalar::Utf8(lf.equals.clone()))), + right: Rc::new(L3Expr::Literal(L3Scalar::Utf8(lf.equals.clone()))), }))) } @@ -104,11 +108,11 @@ pub fn label_filter_to_predicate(lf: &LabelFilter, schema: &Schema) -> Option) -> Option { - let mut exprs: Vec = predicates.into_iter().map(|p| *p.0).collect(); + let mut exprs: Vec = predicates.into_iter().map(|p| (*p.0).clone()).collect(); match exprs.len() { 0 => None, - 1 => Some(Predicate(Box::new(exprs.remove(0)))), - _ => Some(Predicate(Box::new(L3Expr::BoolAnd(exprs)))), + 1 => Some(Predicate(Rc::new(exprs.remove(0)))), + _ => Some(Predicate(Rc::new(L3Expr::BoolAnd(exprs)))), } } @@ -118,17 +122,17 @@ pub fn conjoin(predicates: Vec) -> Option { /// this is the one real construction site's replacement (`lower.rs`). pub fn between(expr: L3Expr, low: L3Expr, high: L3Expr, negated: bool) -> L3Expr { let ge = L3Expr::Compare { - left: Box::new(expr.clone()), + left: Rc::new(expr.clone()), op: CompareOp::Ge, - right: Box::new(low), + right: Rc::new(low), }; let le = L3Expr::Compare { - left: Box::new(expr), + left: Rc::new(expr), op: CompareOp::Le, - right: Box::new(high), + right: Rc::new(high), }; if negated { - L3Expr::Not(Box::new(L3Expr::BoolAnd(vec![ge, le]))) + L3Expr::Not(Rc::new(L3Expr::BoolAnd(vec![ge, le]))) } else { L3Expr::BoolAnd(vec![ge, le]) } diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 82abf5051..926fdb92b 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -70,6 +70,7 @@ pub mod opamp; pub mod optimizer; pub mod physical; pub mod pipeline; +pub mod planner_selection; pub mod query_parser; pub mod query_planning; pub mod replan; diff --git a/control_plane/src/optimizer/cost/mod.rs b/control_plane/src/optimizer/cost/mod.rs index 7e86765c3..13937c120 100644 --- a/control_plane/src/optimizer/cost/mod.rs +++ b/control_plane/src/optimizer/cost/mod.rs @@ -1,5 +1,4 @@ use std::collections::HashMap; -use std::time::Duration; // Sub-modules — formerly siblings of `planner/cost_model.rs` (now // `optimizer/cost/mod.rs`); the 2026-05 refactor pulled each into a @@ -333,6 +332,7 @@ mod tests { use super::*; use chrono::Utc; use std::collections::HashMap; + use std::time::Duration; fn workload(aggs: Vec) -> QueryWorkload { QueryWorkload { diff --git a/control_plane/src/optimizer/cost/sketch_capability.rs b/control_plane/src/optimizer/cost/sketch_capability.rs index ce535c678..ac54d964a 100644 --- a/control_plane/src/optimizer/cost/sketch_capability.rs +++ b/control_plane/src/optimizer/cost/sketch_capability.rs @@ -7,7 +7,7 @@ //! size, which intents each sketch family serves), read by the optimizer //! for cost-based plan rewriting and by the physical planner to check //! whether a sketch fits within a stage's budget — it was never L4 IR, just -//! filed alongside it because both modules touched `SketchKind`. +//! filed alongside it because both modules touched `SketchAlgorithm`. //! //! Distinct from [`crate::sketch_algebra::schema::SketchStateMetadata`] — //! that struct carries the **L4 type-system flags** (`mergeable` / @@ -22,7 +22,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use planner_types::post_asap::SketchKind; +use planner_types::post_asap::SketchAlgorithm; /// Performance and capability profile for a single sketch family. /// @@ -118,15 +118,15 @@ struct SketchCapabilitiesFile { count_min_sketch: SketchCapabilityYaml, } -/// Compiled-in capability defaults — one entry per [`SketchKind`]. +/// Compiled-in capability defaults — one entry per [`SketchAlgorithm`]. /// Replaces the per-variant `sketch_capability(SketchType)` function /// that previously lived in `algebra/optimizer.rs`. Numerical values /// are mirrored from the YAML so the in-process defaults match the /// reference deployment file. -pub fn default_capability_table() -> HashMap { +pub fn default_capability_table() -> HashMap { let mut map = HashMap::new(); map.insert( - SketchKind::DDSketch, + SketchAlgorithm::DDSketch, SketchCapability { insert_throughput: 10_000_000.0, query_throughput: 50_000_000.0, @@ -140,7 +140,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SketchKind::Kll, + SketchAlgorithm::Kll, SketchCapability { insert_throughput: 5_000_000.0, query_throughput: 20_000_000.0, @@ -154,7 +154,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SketchKind::Hll, + SketchAlgorithm::Hll, SketchCapability { insert_throughput: 20_000_000.0, query_throughput: 100_000_000.0, @@ -168,7 +168,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SketchKind::CountSketch, + SketchAlgorithm::CountSketch, SketchCapability { insert_throughput: 8_000_000.0, query_throughput: 10_000_000.0, @@ -182,7 +182,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SketchKind::Cms, + SketchAlgorithm::Cms, SketchCapability { insert_throughput: 8_000_000.0, query_throughput: 10_000_000.0, @@ -203,15 +203,18 @@ pub fn default_capability_table() -> HashMap { /// Replaces `algebra::optimizer::load_sketch_capabilities`. /// /// Env var: `CONTROLLER_SKETCH_CAPABILITIES=path/to/this/file.yml`. -pub fn load_capability_overrides(path: &str) -> HashMap { +pub fn load_capability_overrides(path: &str) -> HashMap { if let Ok(contents) = std::fs::read_to_string(path) { if let Ok(file) = serde_yaml::from_str::(&contents) { let mut map = HashMap::new(); - map.insert(SketchKind::DDSketch, file.ddsketch.to_capability()); - map.insert(SketchKind::Kll, file.kll.to_capability()); - map.insert(SketchKind::Hll, file.hll.to_capability()); - map.insert(SketchKind::CountSketch, file.count_sketch.to_capability()); - map.insert(SketchKind::Cms, file.count_min_sketch.to_capability()); + map.insert(SketchAlgorithm::DDSketch, file.ddsketch.to_capability()); + map.insert(SketchAlgorithm::Kll, file.kll.to_capability()); + map.insert(SketchAlgorithm::Hll, file.hll.to_capability()); + map.insert( + SketchAlgorithm::CountSketch, + file.count_sketch.to_capability(), + ); + map.insert(SketchAlgorithm::Cms, file.count_min_sketch.to_capability()); return map; } } @@ -225,17 +228,17 @@ mod tests { #[test] fn default_table_carries_all_five_sketch_kinds() { let t = default_capability_table(); - assert!(t.contains_key(&SketchKind::DDSketch)); - assert!(t.contains_key(&SketchKind::Kll)); - assert!(t.contains_key(&SketchKind::Hll)); - assert!(t.contains_key(&SketchKind::Cms)); - assert!(t.contains_key(&SketchKind::CountSketch)); + assert!(t.contains_key(&SketchAlgorithm::DDSketch)); + assert!(t.contains_key(&SketchAlgorithm::Kll)); + assert!(t.contains_key(&SketchAlgorithm::Hll)); + assert!(t.contains_key(&SketchAlgorithm::Cms)); + assert!(t.contains_key(&SketchAlgorithm::CountSketch)); } #[test] fn default_table_ddsketch_serves_quantile_intent() { let t = default_capability_table(); - let cap = t.get(&SketchKind::DDSketch).unwrap(); + let cap = t.get(&SketchAlgorithm::DDSketch).unwrap(); assert!(cap.supported_intents.contains(&SupportedIntent::Quantile)); assert!(cap.mergeable); } @@ -243,7 +246,7 @@ mod tests { #[test] fn default_table_hll_serves_cardinality_intent() { let t = default_capability_table(); - let cap = t.get(&SketchKind::Hll).unwrap(); + let cap = t.get(&SketchAlgorithm::Hll).unwrap(); assert!(cap .supported_intents .contains(&SupportedIntent::Cardinality)); diff --git a/control_plane/src/optimizer/cost/wire.rs b/control_plane/src/optimizer/cost/wire.rs index 6fcdfc351..f83d3015a 100644 --- a/control_plane/src/optimizer/cost/wire.rs +++ b/control_plane/src/optimizer/cost/wire.rs @@ -31,7 +31,7 @@ //! agent telemetry once the OnlineMetricsStore feeds back into the //! planner. -use planner_types::post_asap::SketchKind; +use planner_types::post_asap::SketchAlgorithm; // ── Wire-cost table ────────────────────────────────────────────────────────── @@ -105,11 +105,11 @@ impl WireCostTable { /// Lookup the per-flush cost for a sketch family. /// - /// `SketchKind` (unlike the retired `sketch_algebra::SketchKind`) + /// `SketchAlgorithm` (unlike the retired `sketch_algebra::SketchAlgorithm`) /// distinguishes heap-bearing from bare frequency sketches at the /// kind level rather than via a `with_heap` param flag. This table /// never modeled the heap's extra bytes separately (the old - /// `for_kind` took a bare `SketchKind` with no visibility into + /// `for_kind` took a bare `SketchAlgorithm` with no visibility into /// `with_heap` at all) — `CmsWithHeap`/`CountSketchWithHeap` reuse /// their bare counterpart's cost to preserve that exact behavior. /// `Kmv`/`Theta` have no established cost number (nothing in this @@ -117,22 +117,22 @@ impl WireCostTable { /// list stays `Hll`-only, see `capability.rs`); they reuse `hll_delta` /// as a same-order-of-magnitude placeholder pending real numbers if /// this repo ever adopts them. - // Exhaustive over `SketchKind` alone now (ASAPPlanner#218 split the - // old flat `SummaryKind` into `SketchKind`/`ExactKind` -- the exact- + // Exhaustive over `SketchAlgorithm` alone now (ASAPPlanner#218 split the + // old flat `SummaryKind` into `SketchAlgorithm`/`ExactKind` -- the exact- // accumulator arm this match used to need, and its // "exact accumulators have no sketch wire-state cost" panic, are // unreachable by construction now instead of at runtime; see // control_plane/docs/design-asapplanner-pin-migration.md). - pub const fn for_kind(&self, kind: &SketchKind) -> SketchWireCost { + pub const fn for_kind(&self, kind: &SketchAlgorithm) -> SketchWireCost { match kind { - SketchKind::DDSketch => self.ddsketch_delta, - SketchKind::Kll => self.kll_full, - SketchKind::Hll => self.hll_delta, - SketchKind::Kmv | SketchKind::Theta => self.hll_delta, - SketchKind::Cms => self.count_min_delta, - SketchKind::CmsWithHeap => self.count_min_delta, - SketchKind::CountSketch => self.count_sketch_delta, - SketchKind::CountSketchWithHeap => self.count_sketch_delta, + SketchAlgorithm::DDSketch => self.ddsketch_delta, + SketchAlgorithm::Kll => self.kll_full, + SketchAlgorithm::Hll => self.hll_delta, + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => self.hll_delta, + SketchAlgorithm::Cms => self.count_min_delta, + SketchAlgorithm::CmsWithHeap => self.count_min_delta, + SketchAlgorithm::CountSketch => self.count_sketch_delta, + SketchAlgorithm::CountSketchWithHeap => self.count_sketch_delta, } } } diff --git a/control_plane/src/optimizer/engine.rs b/control_plane/src/optimizer/engine.rs index 41f79f5d7..f2ad71d7e 100644 --- a/control_plane/src/optimizer/engine.rs +++ b/control_plane/src/optimizer/engine.rs @@ -28,6 +28,8 @@ //! `WindowedAgg` into `Window { Aggregate }`, and `TopK` into an //! `Aggregate` carrying an `AggIntent::TopK`. +use std::rc::Rc; + use crate::intent_algebra::agg_intent::AggIntent; use crate::intent_algebra::query_expr::{QueryExpr, Reduction, SetOpKind, Source}; use crate::intent_algebra::relational::{agg_is_exact, agg_is_mergeable}; @@ -71,7 +73,7 @@ pub trait CostModel: Send + Sync { /// Load sketch capabilities from a YAML file. Thin shim — the real /// loader lives in `optimizer::cost::sketch_capability::load_capability_overrides` -/// and is keyed by `SketchKind`. This shim translates the result to the +/// and is keyed by `SketchAlgorithm`. This shim translates the result to the /// legacy `SketchType` key used by call sites that haven't migrated. /// /// Falls back to built-in defaults if the file is missing or malformed. @@ -90,11 +92,11 @@ pub fn load_sketch_capabilities( /// Built-in capability profile for a known sketch type. Thin shim — /// the real defaults live in `optimizer::cost::sketch_capability::default_capability_table`. pub fn sketch_capability(st: &crate::types::SketchType) -> SketchCapability { - use planner_types::post_asap::SketchKind; - let kind: SketchKind = st.clone().into(); + use planner_types::post_asap::SketchAlgorithm; + let kind: SketchAlgorithm = st.clone().into(); default_capability_table() .remove(&kind) - .expect("default_capability_table covers every SketchKind variant") + .expect("default_capability_table covers every SketchAlgorithm variant") } // ── Stage budgets ─────────────────────────────────────────────────────────── @@ -223,9 +225,9 @@ impl CostModel for DefaultCostModel { op if agg_is_exact(op) => 1.0, _ => 0.1, }, - QueryExpr::Merge { children } => 1.0 / (children.len().max(1) as f64), + QueryExpr::Concat { children } => 1.0 / (children.len().max(1) as f64), QueryExpr::Filter { .. } => 0.5, - QueryExpr::Distinct { .. } => 0.9, + QueryExpr::Dedup { .. } => 0.9, _ => 1.0, }; @@ -265,8 +267,8 @@ impl CostModel for DefaultCostModel { } // Multi-intent / HAVING aggregate → exact original DB. QueryExpr::Aggregate { .. } => &dc.original_db, - QueryExpr::Merge { .. } | QueryExpr::Distinct { .. } => &dc.backend_collector, - QueryExpr::BinaryOp { .. } | QueryExpr::Subquery { .. } => &dc.backend_db, + QueryExpr::Concat { .. } | QueryExpr::Dedup { .. } => &dc.backend_collector, + QueryExpr::BinaryOp { .. } | QueryExpr::PromqlSubquery { .. } => &dc.backend_db, _ => &dc.agent, }; @@ -346,7 +348,7 @@ impl RewriteRule for PredicatePushDown { fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { match expr { - QueryExpr::Filter { pred, child } => match *child { + QueryExpr::Filter { pred, child } => match (*child).clone() { // Filter below TimeRange (the canonical range-vector-selector // node -- see control_plane/docs/design-asapplanner-pin-migration.md). QueryExpr::TimeRange { @@ -354,7 +356,7 @@ impl RewriteRule for PredicatePushDown { child: inner, } => Some(QueryExpr::TimeRange { range, - child: Box::new(QueryExpr::Filter { pred, child: inner }), + child: Rc::new(QueryExpr::Filter { pred, child: inner }), }), // Filter below Sort (safe when pred references input columns only) QueryExpr::Sort { @@ -364,12 +366,12 @@ impl RewriteRule for PredicatePushDown { } => Some(QueryExpr::Sort { keys, partition_by, - child: Box::new(QueryExpr::Filter { pred, child: inner }), + child: Rc::new(QueryExpr::Filter { pred, child: inner }), }), // Not applicable — reconstruct other => Some(QueryExpr::Filter { pred, - child: Box::new(other), + child: Rc::new(other), }), }, _ => None, @@ -401,7 +403,7 @@ impl RewriteRule for MergeLifting { ref child, .. } if aggs.len() == 1 && having.is_none() && agg_is_mergeable(&aggs[0]) => { - if let QueryExpr::Merge { children } = child.as_ref() { + if let QueryExpr::Concat { children } = child.as_ref() { let new_children: Vec = children .iter() .map(|branch| QueryExpr::Aggregate { @@ -409,10 +411,10 @@ impl RewriteRule for MergeLifting { measures: aggs.clone(), output_names: Vec::new(), having: None, - child: Box::new(branch.clone()), + child: Rc::new(branch.clone()), }) .collect(); - return Some(QueryExpr::Merge { + return Some(QueryExpr::Concat { children: new_children, }); } @@ -445,13 +447,13 @@ impl RewriteRule for HLLDedupElim { having: None, child, } if aggs.len() == 1 && matches!(&aggs[0], AggIntent::Cardinality { .. }) => { - if let QueryExpr::Distinct { child: inner, .. } = *child { + if let QueryExpr::Dedup { child: inner, .. } = child.as_ref() { return Some(QueryExpr::Aggregate { reduction, measures: aggs, output_names, having: None, - child: inner, + child: inner.clone(), }); } None @@ -478,11 +480,11 @@ impl RewriteRule for FilterWindowSwap { if let QueryExpr::TimeRange { range, child: inner, - } = *child + } = (*child).clone() { return Some(QueryExpr::TimeRange { range, - child: Box::new(QueryExpr::Filter { pred, child: inner }), + child: Rc::new(QueryExpr::Filter { pred, child: inner }), }); } None @@ -513,7 +515,7 @@ impl RewriteRule for TopKFusion { } => { if let QueryExpr::Sort { keys, child: inner, .. - } = *child + } = (*child).clone() { // Only fuse when all keys are DESC (top-k semantics). if !keys.is_empty() && keys.iter().all(|k| !k.ascending) { @@ -637,7 +639,7 @@ impl RewriteRule for WindowMerge { if let QueryExpr::TimeRange { range: inner_range, child: inner_child, - } = *child + } = (*child).clone() { if range == inner_range { return Some(QueryExpr::TimeRange { @@ -676,16 +678,19 @@ impl RewriteRule for SetOpFusion { all: true, left, right, - } => match (*left, *right) { - (QueryExpr::Merge { children: mut lc }, QueryExpr::Merge { children: mut rc }) => { + } => match ((*left).clone(), (*right).clone()) { + ( + QueryExpr::Concat { children: mut lc }, + QueryExpr::Concat { children: mut rc }, + ) => { lc.append(&mut rc); - Some(QueryExpr::Merge { children: lc }) + Some(QueryExpr::Concat { children: lc }) } (l, r) => Some(QueryExpr::SetOp { kind: SetOpKind::Union, all: true, - left: Box::new(l), - right: Box::new(r), + left: Rc::new(l), + right: Rc::new(r), }), }, _ => None, @@ -788,8 +793,8 @@ impl QueryOptimizer { fn recurse_children(&self, expr: QueryExpr) -> (QueryExpr, bool) { macro_rules! recurse { ($child:expr) => {{ - let (e, c) = self.apply_all(*$child); - (Box::new(e), c) + let (e, c) = self.apply_all((*$child).clone()); + (Rc::new(e), c) }}; } match expr { @@ -852,10 +857,10 @@ impl QueryOptimizer { c, ) } - QueryExpr::Distinct { cols, child } => { + QueryExpr::Dedup { cols, child } => { let (new_child, c) = recurse!(child); ( - QueryExpr::Distinct { + QueryExpr::Dedup { cols, child: new_child, }, @@ -888,14 +893,14 @@ impl QueryOptimizer { c, ) } - QueryExpr::Subquery { + QueryExpr::PromqlSubquery { range, resolution, child, } => { let (new_child, c) = recurse!(child); ( - QueryExpr::Subquery { + QueryExpr::PromqlSubquery { range, resolution, child: new_child, @@ -903,11 +908,11 @@ impl QueryOptimizer { c, ) } - QueryExpr::Merge { children } => { + QueryExpr::Concat { children } => { let (new_children, changed): (Vec<_>, Vec<_>) = children.into_iter().map(|inp| self.apply_all(inp)).unzip(); ( - QueryExpr::Merge { + QueryExpr::Concat { children: new_children, }, changed.into_iter().any(|c| c), @@ -1034,12 +1039,12 @@ mod tests { measures: vec![intent], output_names: Vec::new(), having: None, - child: Box::new(child), + child: Rc::new(child), } } fn true_pred() -> Predicate { - Predicate(Box::new(L3Expr::Literal(L3Scalar::Boolean(true)))) + Predicate(Rc::new(L3Expr::Literal(L3Scalar::Boolean(true)))) } fn opt() -> QueryOptimizer { @@ -1054,15 +1059,15 @@ mod tests { /// in a tree these helpers construct. fn contains_distinct(qe: &QueryExpr) -> bool { match qe { - QueryExpr::Distinct { .. } => true, + QueryExpr::Dedup { .. } => true, QueryExpr::Filter { child, .. } | QueryExpr::Project { child, .. } | QueryExpr::Aggregate { child, .. } | QueryExpr::TimeRange { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } - | QueryExpr::Subquery { child, .. } => contains_distinct(child), - QueryExpr::Merge { children } => children.iter().any(contains_distinct), + | QueryExpr::PromqlSubquery { child, .. } => contains_distinct(child), + QueryExpr::Concat { children } => children.iter().any(contains_distinct), QueryExpr::Join { left, right, .. } | QueryExpr::SetOp { left, right, .. } | QueryExpr::BinaryOp { @@ -1080,9 +1085,9 @@ mod tests { fn r1_pushes_filter_below_window() { let expr = QueryExpr::Filter { pred: true_pred(), - child: Box::new(QueryExpr::TimeRange { + child: Rc::new(QueryExpr::TimeRange { range: Duration::from_secs(60), - child: Box::new(scan("m")), + child: Rc::new(scan("m")), }), }; let (result, _) = opt().optimize(expr); @@ -1097,14 +1102,14 @@ mod tests { fn r1_pushes_filter_below_sort() { let expr = QueryExpr::Filter { pred: true_pred(), - child: Box::new(QueryExpr::Sort { + child: Rc::new(QueryExpr::Sort { keys: vec![SortKey { expr: L3Expr::Column(0), ascending: true, nulls_first: false, }], partition_by: GroupKeys::none(), - child: Box::new(scan("cpu")), + child: Rc::new(scan("cpu")), }), }; let (result, _) = opt().optimize(expr); @@ -1118,9 +1123,9 @@ mod tests { fn r3_removes_dedup_before_hll() { let expr = sketch_agg( default_cardinality(), - QueryExpr::Distinct { + QueryExpr::Dedup { cols: vec![0], - child: Box::new(scan("events")), + child: Rc::new(scan("events")), }, ); let (result, _) = opt().optimize(expr); @@ -1137,14 +1142,14 @@ mod tests { let expr = QueryExpr::Limit { n: 10, offset: 0, - child: Box::new(QueryExpr::Sort { + child: Rc::new(QueryExpr::Sort { keys: vec![SortKey { expr: L3Expr::Column(0), ascending: false, nulls_first: false, }], partition_by: GroupKeys::none(), - child: Box::new(scan("events")), + child: Rc::new(scan("events")), }), }; let (result, _) = opt().optimize(expr); @@ -1160,14 +1165,14 @@ mod tests { let expr = QueryExpr::Limit { n: 10, offset: 0, - child: Box::new(QueryExpr::Sort { + child: Rc::new(QueryExpr::Sort { keys: vec![SortKey { expr: L3Expr::Column(0), ascending: true, nulls_first: false, }], partition_by: GroupKeys::none(), - child: Box::new(scan("events")), + child: Rc::new(scan("events")), }), }; let (result, _) = opt().optimize(expr); @@ -1184,9 +1189,9 @@ mod tests { fn r10_merges_duplicate_windows() { let expr = QueryExpr::TimeRange { range: Duration::from_secs(300), - child: Box::new(QueryExpr::TimeRange { + child: Rc::new(QueryExpr::TimeRange { range: Duration::from_secs(300), - child: Box::new(scan("m")), + child: Rc::new(scan("m")), }), }; let (result, _) = opt().optimize(expr); @@ -1213,13 +1218,13 @@ mod tests { // R3: Window(Filter(Aggregate[Cardinality](Scan))) let expr = QueryExpr::Filter { pred: true_pred(), - child: Box::new(QueryExpr::TimeRange { + child: Rc::new(QueryExpr::TimeRange { range: Duration::from_secs(60), - child: Box::new(sketch_agg( + child: Rc::new(sketch_agg( default_cardinality(), - QueryExpr::Distinct { + QueryExpr::Dedup { cols: vec![0], - child: Box::new(scan("events")), + child: Rc::new(scan("events")), }, )), }), @@ -1237,13 +1242,13 @@ mod tests { fn r2_lifts_mergeable_sketch_above_merge() { let expr = sketch_agg( default_cardinality(), - QueryExpr::Merge { + QueryExpr::Concat { children: vec![scan("shard_a"), scan("shard_b")], }, ); let (result, _) = opt().optimize(expr); assert!( - matches!(&result, QueryExpr::Merge { children } + matches!(&result, QueryExpr::Concat { children } if children.iter().all(|c| matches!(c, QueryExpr::Aggregate { .. }))), "cardinality aggregate should be pushed into each Merge branch: {result:?}" ); @@ -1256,16 +1261,16 @@ mod tests { let expr = QueryExpr::SetOp { kind: SetOpKind::Union, all: true, - left: Box::new(QueryExpr::Merge { + left: Rc::new(QueryExpr::Concat { children: vec![scan("a"), scan("b")], }), - right: Box::new(QueryExpr::Merge { + right: Rc::new(QueryExpr::Concat { children: vec![scan("c")], }), }; let (result, _) = opt().optimize(expr); match result { - QueryExpr::Merge { children } => assert_eq!(children.len(), 3), + QueryExpr::Concat { children } => assert_eq!(children.len(), 3), other => panic!("expected Merge(3), got {other:?}"), } } @@ -1277,12 +1282,12 @@ mod tests { // `CommonSubexprElim` always returns `None` now (no `LetBinding`/ // `Ref` to hoist into) -- the tree comes back unchanged rather // than with duplicate scans hoisted into a shared binding. - let expr = QueryExpr::Merge { + let expr = QueryExpr::Concat { children: vec![scan("dup"), scan("dup"), scan("other")], }; let (result, _) = opt().optimize(expr); match result { - QueryExpr::Merge { children } => assert_eq!(children.len(), 3), + QueryExpr::Concat { children } => assert_eq!(children.len(), 3), other => panic!("expected Merge(3), got {other:?}"), } } diff --git a/control_plane/src/optimizer/rules/mod.rs b/control_plane/src/optimizer/rules/mod.rs index 683dcf887..53ef7aa47 100644 --- a/control_plane/src/optimizer/rules/mod.rs +++ b/control_plane/src/optimizer/rules/mod.rs @@ -59,13 +59,13 @@ pub fn bind_workload_typed_with_item_filter( item_filter: Option<(&str, &str)>, ) -> Option { use crate::intent_algebra::schema::{Column, DataType}; - use crate::intent_algebra::{AggIntent as L3AggIntent, QueryExpr, Schema, Source, WindowKind}; + use crate::intent_algebra::{AggIntent as L3AggIntent, QueryExpr, Schema, Source}; use crate::sketch_algebra::capability_matching::{ classify_demo_metric, is_valid_pair, pick_family, AccuracyPreference, StatisticClass, }; use crate::sketch_algebra::cost_model::ForcedFamilyCostModel; use crate::types_v2::AccuracyTarget; - use planner_types::post_asap::SketchKind; + use planner_types::post_asap::SketchAlgorithm; // Contract-row metrics (`classify_demo_metric` returns `Some`) and // operator-supplied overrides both signal "this metric must be @@ -112,7 +112,7 @@ pub fn bind_workload_typed_with_item_filter( return None; } - // ── Resolve the SketchKind (override > capability-matched default) ─ + // ── Resolve the SketchAlgorithm (override > capability-matched default) ─ // // The workload-spec's `sketch_type_override` (= the spec's // `sketch_family_override` per orchestrator contract) wins over the @@ -121,10 +121,10 @@ pub fn bind_workload_typed_with_item_filter( // workload) is silently dropped — the catalog-default family runs // instead so the binding never produces a nonsense (sketch, stat) // pair. - let override_kind: Option = w + let override_kind: Option = w .sketch_type_override .as_ref() - .map(|st| SketchKind::from(st.clone())); + .map(|st| SketchAlgorithm::from(st.clone())); let kind = match override_kind { Some(k) if is_valid_pair(k.clone(), statistic) => k, _ => pick_family(statistic, accuracy_pref)?, @@ -199,7 +199,7 @@ pub fn bind_workload_typed_with_item_filter( }; let windowed = QueryExpr::TimeRange { range: w.time_window, - child: Box::new(scan), + child: Box::new(scan).into(), }; let aggregate = QueryExpr::Aggregate { // Synthetic probe only -- `boundary::implementation_for` (what @@ -211,7 +211,7 @@ pub fn bind_workload_typed_with_item_filter( measures: vec![intent], output_names: Vec::new(), having: None, - child: Box::new(windowed), + child: Box::new(windowed).into(), }; // ── Drive the picked family directly, bypassing selection ───────── @@ -223,7 +223,7 @@ pub fn bind_workload_typed_with_item_filter( // dispatcher's tie-break (DDSketch p=6 vs KLL p=5) cannot accidentally // flip `request_size_bytes`'s KLL pick to DDSketch. // - // CMS+TopK note: when the picker selected `SketchKind::Cms` for a + // CMS+TopK note: when the picker selected `SketchAlgorithm::Cms` for a // TopK statistic (only reachable today via a `sketch_family_override: // CountMinSketch` on a TopK metric), this forces the CMS-with-heap // variant. The CMS-Heap pattern (Cormode & Muthukrishnan 2005) gives @@ -241,12 +241,12 @@ pub fn bind_workload_typed_with_item_filter( // delegate to `ControlPlaneCostModel`'s own `"frequency"` handling, // this contract row commits like any other. let forced = match kind { - SketchKind::CountSketch => SketchKind::CountSketchWithHeap, - SketchKind::Cms if statistic == StatisticClass::TopK => SketchKind::CmsWithHeap, + SketchAlgorithm::CountSketch => SketchAlgorithm::CountSketchWithHeap, + SketchAlgorithm::Cms if statistic == StatisticClass::TopK => SketchAlgorithm::CmsWithHeap, other => other, }; let cost_model = ForcedFamilyCostModel::new(accuracy.clone(), forced); - let node = asap_aware_mapping::bind::implement_tree_with(&aggregate, &cost_model).ok()?; + let node = crate::planner_selection::select_summary(&aggregate, &cost_model).ok()?; // `implement_tree_with` never *errors* on "nothing bound" — an // intent `boundary::implementation_for`/`CostModel::realize_extension` // can't realize (e.g. `TopK { accuracy: Exact }`, ASAPController#151, @@ -255,7 +255,10 @@ pub fn bind_workload_typed_with_item_filter( // contract is `None` for "typed path doesn't support this shape yet" // — translate the two by checking whether anything actually got // committed. - if matches!(node.expr, planner_types::post_asap::SummaryExpr::Logical(_)) { + if matches!( + node.expr, + planner_types::post_asap::SummaryExpr::KeepPreAsap(_) + ) { return None; } Some(crate::sketch_algebra::physical_expr::PhysicalExpr::committed(node)) @@ -444,13 +447,13 @@ mod tests { #[test] fn incompatible_override_does_not_rewrite_query_semantics() { use crate::emit::extract_root_sketch_kind; - use planner_types::post_asap::SketchKind; + use planner_types::post_asap::SketchAlgorithm; for (ov, expect) in [ - (SketchType::DDSketch, SketchKind::DDSketch), - (SketchType::KLL, SketchKind::Kll), - (SketchType::HLL, SketchKind::DDSketch), - (SketchType::CountSketch, SketchKind::DDSketch), - (SketchType::CountMinSketch, SketchKind::DDSketch), + (SketchType::DDSketch, SketchAlgorithm::DDSketch), + (SketchType::KLL, SketchAlgorithm::Kll), + (SketchType::HLL, SketchAlgorithm::DDSketch), + (SketchType::CountSketch, SketchAlgorithm::DDSketch), + (SketchType::CountMinSketch, SketchAlgorithm::DDSketch), ] { let mut w = workload(vec![AggType::Quantile]); w.sketch_type_override = Some(ov.clone()); @@ -584,14 +587,14 @@ mod tests { // | `endpoint_request_freq` | CMS | use crate::sketch_algebra::physical_expr::PhysicalExpr; - use planner_types::post_asap::SketchKind; + use planner_types::post_asap::SketchAlgorithm; use planner_types::pre_asap::expr_ir::ColumnRef; /// Walk the L4 binding output and pull out the approximate sketch /// family. Returns `None` if no sketch node is present (raw / pure /// logical pass-through, or an exact accumulator — see /// `emit::extract_root_sketch_kind`, whose logic this mirrors). - fn extract_family(expr: &PhysicalExpr) -> Option { + fn extract_family(expr: &PhysicalExpr) -> Option { crate::emit::extract_root_sketch_kind(expr) } @@ -655,7 +658,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("http_latency_ms must bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::DDSketch), + Some(SketchAlgorithm::DDSketch), "http_latency_ms should bind to DDSketch (Quantile, rel-err)", ); } @@ -670,7 +673,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("request_size_bytes must bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::Kll), + Some(SketchAlgorithm::Kll), "request_size_bytes should bind to KLL (Quantile, rank-err)", ); } @@ -682,24 +685,19 @@ mod tests { let bound = bind_workload_typed(&w).expect("unique_users_per_min must bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::Hll), + Some(SketchAlgorithm::Hll), "unique_users_per_min should bind to HLL (Cardinality)", ); } #[test] - fn typed_binding_top_endpoint_qps_picks_countsketch() { + fn typed_binding_top_endpoint_qps_requires_membership_evidence() { // Contract: `top_endpoint_qps` → CountSketch (TopK). // The metric-name reclassification reroutes from the AggType // default (Frequency → CMS) to the contract row (TopK → // CountSketch). let w = workload_for("top_endpoint_qps", AggType::Frequency); - let bound = bind_workload_typed(&w).expect("top_endpoint_qps must bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::CountSketchWithHeap), - "top_endpoint_qps should bind to CountSketch-with-heap (TopK)", - ); + assert!(bind_workload_typed(&w).is_none()); } #[test] @@ -716,7 +714,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("endpoint_request_freq must bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::Cms), + Some(SketchAlgorithm::Cms), "endpoint_request_freq should bind to Cms (Frequency)", ); } @@ -769,7 +767,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("override should still bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::Kll), + Some(SketchAlgorithm::Kll), "sketch_type_override=KLL should pin KLL despite the contract's DDSketch default", ); } @@ -783,13 +781,13 @@ mod tests { let bound = bind_workload_typed(&w).expect("override should still bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::DDSketch), + Some(SketchAlgorithm::DDSketch), "sketch_type_override=DDSketch should pin DDSketch despite the contract's KLL default", ); } #[test] - fn planner_accepts_countmin_override_for_topk_metric() { + fn planner_rejects_topk_override_without_membership_evidence() { // CMS-Heap pattern (Cormode & Muthukrishnan 2005): when a // workload's `sketch_family_override` (= // `sketch_type_override`) selects CountMinSketch for a TopK @@ -797,27 +795,16 @@ mod tests { // back to the canonical CountSketch default. let mut w = workload_for("top_endpoint_qps", AggType::Frequency); w.sketch_type_override = Some(SketchType::CountMinSketch); - let bound = - bind_workload_typed(&w).expect("CountMin override on a TopK metric should still bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::CmsWithHeap), - "sketch_type_override=CountMinSketch on a TopK metric should pin CMS-with-heap", - ); + assert!(bind_workload_typed(&w).is_none()); } #[test] - fn planner_default_for_topk_remains_countsketch() { + fn planner_default_topk_requires_membership_evidence() { // Without any override, the canonical pick for a TopK metric // stays CountSketch(-with-heap) — CMS-Heap is opt-in via // override only. let w = workload_for("top_endpoint_qps", AggType::Frequency); - let bound = bind_workload_typed(&w).expect("top_endpoint_qps must bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::CountSketchWithHeap), - "default TopK pick must remain CountSketch-with-heap (unbiased estimator)", - ); + assert!(bind_workload_typed(&w).is_none()); } #[test] @@ -830,7 +817,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("fallback should bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::DDSketch), + Some(SketchAlgorithm::DDSketch), "invalid (HLL, Quantile) override should be rejected; planner falls back to DDSketch", ); } @@ -842,35 +829,31 @@ mod tests { // Single test that drives the full contract row set through // `bind_workload_typed` — this is the per-task acceptance test // ("verify each produces the expected `PhysicalExpr` family"). - let cases: Vec<(&str, AggType, Option)> = vec![ + let cases: Vec<(&str, AggType, Option)> = vec![ ("http_requests_total", AggType::Frequency, None), ( "http_latency_ms", AggType::Quantile, - Some(SketchKind::DDSketch), + Some(SketchAlgorithm::DDSketch), ), ( "request_size_bytes", AggType::Quantile, - Some(SketchKind::Kll), + Some(SketchAlgorithm::Kll), ), ( "unique_users_per_min", AggType::Cardinality, - Some(SketchKind::Hll), - ), - ( - "top_endpoint_qps", - AggType::Frequency, - Some(SketchKind::CountSketchWithHeap), + Some(SketchAlgorithm::Hll), ), + ("top_endpoint_qps", AggType::Frequency, None), // `Extension`/Frequency now binds via `ControlPlaneCostModel`'s // `realize_extension` (ASAPController#150) — see // `typed_binding_endpoint_request_freq_binds_cms`. ( "endpoint_request_freq", AggType::Frequency, - Some(SketchKind::Cms), + Some(SketchAlgorithm::Cms), ), ]; for (metric, agg, expected) in cases { diff --git a/control_plane/src/physical/allocator.rs b/control_plane/src/physical/allocator.rs index 1b9256cd4..536947787 100644 --- a/control_plane/src/physical/allocator.rs +++ b/control_plane/src/physical/allocator.rs @@ -38,6 +38,8 @@ //! window-defines-sketch-lifecycle fusion that *does* matter is a //! `physical::planner` concern — see `physical::window_fusion`. +use std::rc::Rc; + use super::plan::{CostEstimate, ExecutionMode, NodeAnnotation, PipelineStage, PlanNode}; use crate::intent_algebra::agg_intent::AggIntent; use crate::intent_algebra::relational::agg_is_exact; @@ -128,11 +130,11 @@ impl SketchAllocator { // ── Structural / filter nodes — always Agent ────────────────── QueryExpr::Filter { pred, child } => { - let child = self.alloc_node(*child, budget); + let child = self.alloc_node((*child).clone(), budget); PlanNode { expr: QueryExpr::Filter { pred, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Agent, mode: ExecutionMode::Passthrough, @@ -155,11 +157,11 @@ impl SketchAllocator { // passthrough and the inner `Aggregate` arm does the sketch // placement. QueryExpr::TimeRange { range, child } => { - let child = self.alloc_node(*child, budget); + let child = self.alloc_node((*child).clone(), budget); PlanNode { expr: QueryExpr::TimeRange { range, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Agent, mode: ExecutionMode::Passthrough, @@ -172,12 +174,12 @@ impl SketchAllocator { } } - QueryExpr::Distinct { cols, child } => { - let child = self.alloc_node(*child, budget); + QueryExpr::Dedup { cols, child } => { + let child = self.alloc_node((*child).clone(), budget); PlanNode { - expr: QueryExpr::Distinct { + expr: QueryExpr::Dedup { cols, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Agent, mode: ExecutionMode::Passthrough, @@ -205,14 +207,14 @@ impl SketchAllocator { if aggs.len() == 1 && having.is_none() { if let AggIntent::TopK { k, .. } = &aggs[0] { let k = *k; - let child = self.alloc_node(*child, budget); + let child = self.alloc_node((*child).clone(), budget); return PlanNode { expr: QueryExpr::Aggregate { reduction, measures: aggs, output_names, having, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Precompute, mode: ExecutionMode::Sketch, @@ -232,11 +234,11 @@ impl SketchAllocator { }; } // Single non-TopK intent → budget-driven sketch agg. - let child = self.alloc_node(*child, budget); + let child = self.alloc_node((*child).clone(), budget); return self.alloc_sketch_agg(reduction, aggs, output_names, child, budget); } // General multi-intent / HAVING aggregate → Db (exact). - let child = self.alloc_node(*child, budget); + let child = self.alloc_node((*child).clone(), budget); let kinds: Vec<&'static str> = aggs.iter().map(canonical_intent_kind_str).collect(); PlanNode { expr: QueryExpr::Aggregate { @@ -244,7 +246,7 @@ impl SketchAllocator { measures: aggs, output_names, having, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Db, mode: ExecutionMode::Exact, @@ -261,14 +263,14 @@ impl SketchAllocator { } // ── Merge — Backend ─────────────────────────────────────────── - QueryExpr::Merge { children: inputs } => { + QueryExpr::Concat { children: inputs } => { let children: Vec = inputs .into_iter() .map(|inp| self.alloc_node(inp, budget)) .collect(); let mem: f64 = children.iter().map(|c| c.cost.memory_bytes).sum(); PlanNode { - expr: QueryExpr::Merge { + expr: QueryExpr::Concat { children: children.iter().map(|c| c.expr.clone()).collect(), }, stage: PipelineStage::Backend, @@ -292,12 +294,12 @@ impl SketchAllocator { qualifier, child, } => { - let child = self.alloc_node(*child, budget); + let child = self.alloc_node((*child).clone(), budget); PlanNode { expr: QueryExpr::Project { cols, qualifier, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Db, mode: ExecutionMode::Exact, @@ -315,12 +317,12 @@ impl SketchAllocator { partition_by, child, } => { - let child = self.alloc_node(*child, budget); + let child = self.alloc_node((*child).clone(), budget); PlanNode { expr: QueryExpr::Sort { keys, partition_by, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Db, mode: ExecutionMode::Exact, @@ -334,12 +336,12 @@ impl SketchAllocator { } QueryExpr::Limit { n, offset, child } => { - let child = self.alloc_node(*child, budget); + let child = self.alloc_node((*child).clone(), budget); PlanNode { expr: QueryExpr::Limit { n, offset, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Db, mode: ExecutionMode::Exact, @@ -358,14 +360,14 @@ impl SketchAllocator { left, right, } => { - let left_node = self.alloc_node(*left, budget); - let right_node = self.alloc_node(*right, budget); + let left_node = self.alloc_node((*left).clone(), budget); + let right_node = self.alloc_node((*right).clone(), budget); PlanNode { expr: QueryExpr::Join { kind, pred, - left: Box::new(left_node.expr.clone()), - right: Box::new(right_node.expr.clone()), + left: Rc::new(left_node.expr.clone()), + right: Rc::new(right_node.expr.clone()), }, stage: PipelineStage::Db, mode: ExecutionMode::Exact, @@ -387,14 +389,14 @@ impl SketchAllocator { left, right, } => { - let left_node = self.alloc_node(*left, budget); - let right_node = self.alloc_node(*right, budget); + let left_node = self.alloc_node((*left).clone(), budget); + let right_node = self.alloc_node((*right).clone(), budget); PlanNode { expr: QueryExpr::SetOp { kind, all, - left: Box::new(left_node.expr.clone()), - right: Box::new(right_node.expr.clone()), + left: Rc::new(left_node.expr.clone()), + right: Rc::new(right_node.expr.clone()), }, stage: PipelineStage::Db, mode: ExecutionMode::Exact, @@ -408,12 +410,12 @@ impl SketchAllocator { } // ── PromQL sub-query ────────────────────────────────────────── - QueryExpr::Subquery { + QueryExpr::PromqlSubquery { range, resolution, child, } => { - let child = self.alloc_node(*child, budget); + let child = self.alloc_node((*child).clone(), budget); let stage = if child.mode == ExecutionMode::Sketch { PipelineStage::Precompute } else { @@ -421,10 +423,10 @@ impl SketchAllocator { }; let rationale = format!("PromQL subquery at {stage}"); PlanNode { - expr: QueryExpr::Subquery { + expr: QueryExpr::PromqlSubquery { range, resolution, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage, mode: child.mode.clone(), @@ -443,8 +445,8 @@ impl SketchAllocator { rhs, vector_match, } => { - let left_node = self.alloc_node(*lhs, budget); - let right_node = self.alloc_node(*rhs, budget); + let left_node = self.alloc_node((*lhs).clone(), budget); + let right_node = self.alloc_node((*rhs).clone(), budget); let has_sketch = left_node.mode == ExecutionMode::Sketch || right_node.mode == ExecutionMode::Sketch; let stage = if has_sketch { @@ -457,8 +459,8 @@ impl SketchAllocator { expr: QueryExpr::BinaryOp { op, vector_match, - lhs: Box::new(left_node.expr.clone()), - rhs: Box::new(right_node.expr.clone()), + lhs: Rc::new(left_node.expr.clone()), + rhs: Rc::new(right_node.expr.clone()), }, stage, mode: if has_sketch { @@ -486,60 +488,68 @@ impl SketchAllocator { // variant wraps exactly one child, recursed into and staged // as an Agent passthrough (mirroring `Filter`/`Window` above) // until a real rule is written for them. - QueryExpr::Scalar(_) | QueryExpr::EvalTime => { + QueryExpr::PromqlScalarBridge(_) | QueryExpr::EvalTimestamp => { PlanNode::leaf(expr, PipelineStage::Agent, ExecutionMode::Passthrough) } - QueryExpr::VectorFromScalar(child) => { + QueryExpr::PromqlVectorFromScalar(child) => { self.alloc_passthrough_child(child, "VectorFromScalar", budget, |c| { - QueryExpr::VectorFromScalar(Box::new(c)) + QueryExpr::PromqlVectorFromScalar(Rc::new(c)) }) } - QueryExpr::ScalarFromVector(child) => { + QueryExpr::PromqlScalarFromVector(child) => { self.alloc_passthrough_child(child, "ScalarFromVector", budget, |c| { - QueryExpr::ScalarFromVector(Box::new(c)) + QueryExpr::PromqlScalarFromVector(Rc::new(c)) }) } - QueryExpr::Relabel { dst, value, child } => { - self.alloc_passthrough_child(child, "Relabel", budget, |c| QueryExpr::Relabel { - dst, - value, - child: Box::new(c), + QueryExpr::PromqlRelabel { dst, value, child } => { + self.alloc_passthrough_child(child, "Relabel", budget, |c| { + QueryExpr::PromqlRelabel { + dst, + value, + child: Rc::new(c), + } }) } - QueryExpr::InfoJoin { selector, child } => { - self.alloc_passthrough_child(child, "InfoJoin", budget, |c| QueryExpr::InfoJoin { - selector, - child: Box::new(c), + QueryExpr::PromqlInfoEnrich { selector, child } => { + self.alloc_passthrough_child(child, "InfoJoin", budget, |c| { + QueryExpr::PromqlInfoEnrich { + selector, + child: Rc::new(c), + } }) } - QueryExpr::Sample { by, kind, child } => { - self.alloc_passthrough_child(child, "Sample", budget, |c| QueryExpr::Sample { - by, - kind, - child: Box::new(c), + QueryExpr::PromqlSeriesSample { by, kind, child } => { + self.alloc_passthrough_child(child, "Sample", budget, |c| { + QueryExpr::PromqlSeriesSample { + by, + kind, + child: Rc::new(c), + } }) } QueryExpr::TimeShift { shift, child } => { self.alloc_passthrough_child(child, "TimeShift", budget, |c| QueryExpr::TimeShift { shift, - child: Box::new(c), + child: Rc::new(c), }) } - QueryExpr::WindowFunc { + QueryExpr::SQLWindowFunc { func, args, partition_by, order_by, output_name, + frame, child, } => self.alloc_passthrough_child(child, "WindowFunc", budget, |c| { - QueryExpr::WindowFunc { + QueryExpr::SQLWindowFunc { func, args, partition_by, order_by, output_name, - child: Box::new(c), + frame, + child: Rc::new(c), } }), // Scalar-expression node (Column/Literal/Compare/BoolAnd/ @@ -561,12 +571,12 @@ impl SketchAllocator { /// as an informational Agent passthrough carrying the child's cost. fn alloc_passthrough_child( &self, - child: Box, + child: Rc, label: &'static str, budget: &mut BudgetState, rebuild: impl FnOnce(QueryExpr) -> QueryExpr, ) -> PlanNode { - let child_node = self.alloc_node(*child, budget); + let child_node = self.alloc_node((*child).clone(), budget); PlanNode { expr: rebuild(child_node.expr.clone()), stage: PipelineStage::Agent, @@ -604,7 +614,7 @@ impl SketchAllocator { measures: aggs, output_names, having: None, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Db, mode: ExecutionMode::Exact, @@ -628,7 +638,7 @@ impl SketchAllocator { measures: aggs, output_names, having: None, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Backend, mode: ExecutionMode::Exact, @@ -658,7 +668,7 @@ impl SketchAllocator { measures: aggs, output_names, having: None, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Agent, mode: ExecutionMode::Sketch, @@ -686,7 +696,7 @@ impl SketchAllocator { measures: aggs, output_names, having: None, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Backend, mode: ExecutionMode::Sketch, @@ -714,7 +724,7 @@ impl SketchAllocator { measures: aggs, output_names, having: None, - child: Box::new(child.expr.clone()), + child: Rc::new(child.expr.clone()), }, stage: PipelineStage::Precompute, mode: ExecutionMode::Sketch, @@ -828,7 +838,7 @@ mod tests { measures: vec![intent], output_names: Vec::new(), having: None, - child: Box::new(scan("m")), + child: Rc::new(scan("m")), } } @@ -869,8 +879,8 @@ mod tests { #[test] fn filter_at_agent() { let expr = QueryExpr::Filter { - pred: Predicate(Box::new(L3Expr::Literal(L3Scalar::Boolean(true)))), - child: Box::new(scan("m")), + pred: Predicate(Rc::new(L3Expr::Literal(L3Scalar::Boolean(true)))), + child: Rc::new(scan("m")), }; let node = alloc(unlimited(), expr); assert_eq!(node.stage, PipelineStage::Agent); @@ -941,7 +951,7 @@ mod tests { #[test] fn merge_goes_to_backend() { - let expr = QueryExpr::Merge { + let expr = QueryExpr::Concat { children: vec![scan("a"), scan("b")], }; let node = alloc(unlimited(), expr); @@ -972,9 +982,9 @@ mod tests { fn join_goes_to_db() { let expr = QueryExpr::Join { kind: JoinKind::Inner, - pred: Predicate(Box::new(L3Expr::Literal(L3Scalar::Boolean(true)))), - left: Box::new(scan("orders")), - right: Box::new(scan("items")), + pred: Predicate(Rc::new(L3Expr::Literal(L3Scalar::Boolean(true)))), + left: Rc::new(scan("orders")), + right: Rc::new(scan("items")), }; let node = alloc(unlimited(), expr); assert_eq!(node.stage, PipelineStage::Db); @@ -989,7 +999,7 @@ mod tests { measures: vec![AggIntent::Sum { col: None }, AggIntent::Min { col: None }], output_names: Vec::new(), having: None, - child: Box::new(scan("m")), + child: Rc::new(scan("m")), }; let node = alloc(unlimited(), expr); assert_eq!(node.stage, PipelineStage::Db); @@ -1004,7 +1014,7 @@ mod tests { // Agent, inner Aggregate does the sketch placement. let expr = QueryExpr::TimeRange { range: std::time::Duration::from_secs(300), - child: Box::new(agg(default_quantile(0.5))), + child: Rc::new(agg(default_quantile(0.5))), }; let node = alloc(unlimited(), expr); assert_eq!(node.stage, PipelineStage::Agent); diff --git a/control_plane/src/physical/colored_dag/allocator.rs b/control_plane/src/physical/colored_dag/allocator.rs index 872308ba7..6ab79a3e2 100644 --- a/control_plane/src/physical/colored_dag/allocator.rs +++ b/control_plane/src/physical/colored_dag/allocator.rs @@ -176,7 +176,7 @@ impl ThreeStageWalker { // `Aggregate{exact}` lands on edge if its child is an edge // (scrape locality); `Ref` resolves through the lexical // scope map. - SummaryExpr::Logical(qe) => self.colour_logical(qe)?, + SummaryExpr::KeepPreAsap(qe) => self.colour_logical(qe)?, // ── SummaryAgg: always edge per design.md §6 batched-queries // table — true for both approximate sketches (the old @@ -305,7 +305,7 @@ impl ThreeStageWalker { // follow-up batches. `Partition` no longer exists in the // canonical IR — its keys fold into `Aggregate.by` at // construction time (`intent_algebra::lower`). - QE::Merge { .. } | QE::Join { .. } | QE::SetOp { .. } | QE::BinaryOp { .. } => { + QE::Concat { .. } | QE::Join { .. } | QE::SetOp { .. } | QE::BinaryOp { .. } => { Ok(StageId::Backend) } // Filter/Project/Distinct/Sort/Limit/Subquery, plus the @@ -364,13 +364,14 @@ mod tests { fn windowed_scan() -> QueryExpr { QueryExpr::TimeRange { range: Duration::from_secs(300), - child: Box::new(ts_scan()), + child: Rc::new(ts_scan()), } } #[test] fn allocate_unsupported_topology_errors() { - let leaf = PhysicalExpr::committed(asap_aware_mapping::bind::logical(&ts_scan()).unwrap()); + let leaf = + PhysicalExpr::committed(crate::planner_selection::keep_pre_asap(&ts_scan()).unwrap()); let err = StageAllocator .allocate(&leaf, Topology::SingleStage) .unwrap_err(); @@ -391,9 +392,9 @@ mod tests { }], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; - let node = asap_aware_mapping::bind::implement_tree(&q).unwrap(); + let node = crate::planner_selection::select_summary_default(&q).unwrap(); let expr = PhysicalExpr::committed(node); let dag = StageAllocator .allocate(&expr, Topology::ThreeStage) diff --git a/control_plane/src/physical/colored_dag/dag.rs b/control_plane/src/physical/colored_dag/dag.rs index 192afc453..c747967dc 100644 --- a/control_plane/src/physical/colored_dag/dag.rs +++ b/control_plane/src/physical/colored_dag/dag.rs @@ -143,6 +143,8 @@ impl ColoredDag { #[cfg(test)] mod tests { + use std::rc::Rc; + use super::*; use crate::intent_algebra::schema::{Column, DataType}; use crate::intent_algebra::{QueryExpr, Schema, Source}; @@ -179,7 +181,7 @@ mod tests { } fn dummy_logical() -> PhysicalExpr { - PhysicalExpr::committed(asap_aware_mapping::bind::logical(&dummy_scan()).unwrap()) + PhysicalExpr::committed(crate::planner_selection::keep_pre_asap(&dummy_scan()).unwrap()) } fn dummy_agg() -> PhysicalExpr { @@ -188,9 +190,9 @@ mod tests { measures: vec![crate::intent_algebra::AggIntent::Sum { col: None }], output_names: Vec::new(), having: None, - child: Box::new(dummy_scan()), + child: Rc::new(dummy_scan()), }; - PhysicalExpr::committed(asap_aware_mapping::bind::implement_tree(&q).unwrap()) + PhysicalExpr::committed(crate::planner_selection::select_summary_default(&q).unwrap()) } fn dummy_estimate() -> PhysicalExpr { @@ -203,9 +205,9 @@ mod tests { }], output_names: Vec::new(), having: None, - child: Box::new(dummy_scan()), + child: Rc::new(dummy_scan()), }; - PhysicalExpr::committed(asap_aware_mapping::bind::implement_tree(&q).unwrap()) + PhysicalExpr::committed(crate::planner_selection::select_summary_default(&q).unwrap()) } #[test] diff --git a/control_plane/src/physical/colored_dag/emitter.rs b/control_plane/src/physical/colored_dag/emitter.rs index 65d8a9d9e..433801bca 100644 --- a/control_plane/src/physical/colored_dag/emitter.rs +++ b/control_plane/src/physical/colored_dag/emitter.rs @@ -35,7 +35,7 @@ use crate::physical::colored_dag::dag::ColoredDag; use crate::physical::colored_dag::stage_id::{StageId, Topology}; use crate::sketch_algebra::physical_expr::{L4Plan, PhysicalExpr}; use crate::types_v2::BindingName; -use planner_types::post_asap::{SketchKind, SketchParams, SketchQuery, SummaryExpr}; +use planner_types::post_asap::{SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr}; // `BackendAggregation.sketch_kind`/`.sketch_params` (below) span BOTH // exact accumulators (Sum/Count/MinMax/Increase/Rate, via // `agg_type_override`) and approximate sketches -- unlike @@ -61,7 +61,7 @@ enum NodeKind<'a> { /// (Sum/Count/MinMax/Increase/Rate) are classified as [`Self::ExactAgg`] /// instead, matching the old `PhysicalExpr::ExactAgg`'s separate shape. SketchAgg { - sketch_type: &'a SketchKind, + sketch_type: &'a SketchAlgorithm, params: &'a SketchParams, }, /// An exact accumulator — the old `PhysicalExpr::ExactAgg`. This @@ -80,7 +80,7 @@ enum NodeKind<'a> { name: &'a BindingName, }, RawAtEdgeSketchAtBackend { - family: &'a SketchKind, + family: &'a SketchAlgorithm, params: &'a SketchParams, }, RawAtEdgePrometheusArchive { @@ -96,7 +96,7 @@ enum NodeKind<'a> { fn classify(expr: &PhysicalExpr) -> NodeKind<'_> { match expr { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { - SummaryExpr::Logical(qe) => NodeKind::Logical(qe), + SummaryExpr::KeepPreAsap(qe) => NodeKind::Logical(qe), // `SummaryAgg`'s `kind`/`params` fields collapsed into one // `family: SummaryFamilyType` field (ASAPPlanner#218 -- // see control_plane/docs/design-asapplanner-pin-migration.md); @@ -108,11 +108,11 @@ fn classify(expr: &PhysicalExpr) -> NodeKind<'_> { .. } => NodeKind::ExactAgg, SummaryExpr::SummaryAgg { - family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, params), + family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, _), .. } => NodeKind::SketchAgg { - sketch_type: kind, - params, + sketch_type: kind.algorithm(), + params: kind.params(), }, // `Plain`/`Sample`/`Wavelet`/`StatModel` never occur on a real // `SummaryAgg` (never `Plain` by construction; `Sample`/ @@ -152,10 +152,10 @@ pub enum EmitError { #[error("unsupported topology for this emitter: {0:?} (expected {1:?})")] UnsupportedTopology(Topology, Topology), /// A sketch processor name could not be derived for the supplied - /// `SketchKind`. Should not occur with the catalog ranges shipped + /// `SketchAlgorithm`. Should not occur with the catalog ranges shipped /// in Phase C — kept as a defensive error for future kinds. #[error("no edge processor known for sketch kind {0:?}")] - NoEdgeProcessor(SketchKind), + NoEdgeProcessor(SketchAlgorithm), /// Backend would emit an empty StreamingConfig because no sketch /// state ever reaches it (e.g. a colouring with only `Logical` /// nodes). Surfaced as a clean error so callers can fall back to @@ -287,7 +287,7 @@ pub struct EdgeStageConfig { /// because different planned queries on the same metric require /// different capabilities (e.g. `quantile_over_time` → DDSketch, /// `count`-distinct → HLL, `topk` → CountSketch all on one metric). - /// The value type is therefore a `BTreeSet` (the UNION + /// The value type is therefore a `BTreeSet` (the UNION /// of capabilities across all of that metric's workload entries), /// not a single family. A metric in two families produces two /// routing-connector OTTL conditions → its samples fan into both @@ -300,11 +300,11 @@ pub struct EdgeStageConfig { /// This eliminates the prior multi-family fan-out (every metric /// shipped sketch state through all 5 families regardless of need). /// - /// `SketchFamily` is a control-plane-side alias for `planner_types::post_asap::SketchKind`. + /// `SketchFamily` is a control-plane-side alias for `planner_types::post_asap::SketchAlgorithm`. /// Empty map ⇒ legacy single-pipeline / /// Mode-3 / warm-passthrough wire shapes are emitted unchanged /// (backward-compat). - pub metric_to_family: HashMap>, + pub metric_to_family: HashMap>, /// MVP blocker B3 — per-metric attribute allowlist the agent must /// reduce wire attrs to BEFORE the sketch processor sees them. /// Maps each metric to its grouping-label list; the 5-sketch routing @@ -571,16 +571,16 @@ pub struct PrometheusArchiveMetric { /// One sketch processor configured at an edge agent. /// /// Not `Serialize`/`Deserialize` (see `PhysicalExpr`'s doc for why — -/// `SketchKind`/`SketchParams` have no serde impl, and nothing on the +/// `SketchAlgorithm`/`SketchParams` have no serde impl, and nothing on the /// real emit path ever whole-struct-serialized this type; every actual /// YAML/JSON payload goes through a hand-written builder). #[derive(Debug, Clone, PartialEq)] pub struct EdgeSketchProcessor { /// OTel processor component id — `KLL`, `ddsketch`, `HLL`, - /// `countmin`, etc. Maps 1:1 from `SketchKind`. + /// `countmin`, etc. Maps 1:1 from `SketchAlgorithm`. pub processor_name: String, /// Sketch family (mirror of the `SketchAgg::sketch_type` field). - pub sketch_kind: SketchKind, + pub sketch_kind: SketchAlgorithm, /// Sketch parameters (mirror of the `SketchAgg::params` field). pub sketch_params: SketchParams, /// Internal emitter plumbing — threads `EdgeSketchProcessor` → @@ -623,7 +623,7 @@ pub struct GatewayMergeProcessor { pub processor_name: String, /// Sketch family being merged. All inputs to the merge agree on /// this (L4 type checker enforces it; design.md §6.4). - pub sketch_kind: SketchKind, + pub sketch_kind: SketchAlgorithm, /// Aggregation id — matches the upstream edge's /// `EdgeSketchProcessor::aggregation_id` so the gateway routes /// streams correctly. @@ -666,7 +666,7 @@ pub struct BackendStageConfig { /// `sketch_kind`, `sketch_params`, grouping labels, and `spatial_filter`). /// /// Not `Serialize`/`Deserialize` — same reason as `EdgeSketchProcessor`: -/// `SketchKind`/`SketchParams` have no serde impl, and the real wire +/// `SketchAlgorithm`/`SketchParams` have no serde impl, and the real wire /// payload is built by `emit::stage_config::build_backend_aggregation_json` /// (a hand-written JSON builder reading these fields), never a whole-struct /// serialize. @@ -680,7 +680,7 @@ pub struct BackendAggregation { pub metric_name: String, /// Accumulator family -- exact (Sum/Count/MinMax/Increase/Rate) or /// approximate sketch (see this file's `use asap_types::{...}` note - /// above for why this is the flat type, not `SketchKind`). + /// above for why this is the flat type, not `SketchAlgorithm`). pub sketch_kind: BackendSketchKind, /// Accumulator parameters — the backend uses these to build its /// per-aggregation instance (KLL with the right `k`, DDSketch with @@ -1063,12 +1063,12 @@ impl Emitter for ThreeStageEmitter { // ── Helpers ─────────────────────────────────────────────────────────────────── -/// Map a `SketchKind` to the OTel collector processor name. Mirrors the +/// Map a `SketchAlgorithm` to the OTel collector processor name. Mirrors the /// names the existing OpAMP YAML emitter (and the per-sketch processor /// crates in `opentelemetry-collector-contrib`) already use. /// /// Heap-bearing kinds (`CmsWithHeap`/`CountSketchWithHeap`) reuse their -/// bare counterpart's processor name — the retired `sketch_algebra::SketchKind` +/// bare counterpart's processor name — the retired `sketch_algebra::SketchAlgorithm` /// this replaces had no heap-bearing variant at all (`with_heap` was a /// `SketchParams` field this function never received), so heap-bearing /// and bare CMS/CountSketch already mapped to the identical processor @@ -1076,18 +1076,22 @@ impl Emitter for ThreeStageEmitter { /// have no OTel edge processor — nothing in this repo's binding rules /// constructs a `SketchAgg`/`RawAtEdgeSketchAtBackend` with one of these /// kinds today, but the match must stay exhaustive. -pub(crate) fn edge_processor_name(kind: &SketchKind) -> Result { +pub(crate) fn edge_processor_name(kind: &SketchAlgorithm) -> Result { match kind { - SketchKind::Kll => Ok("KLL".into()), - SketchKind::DDSketch => Ok("ddsketch".into()), - SketchKind::Hll => Ok("HLL".into()), - SketchKind::Cms | SketchKind::CmsWithHeap => Ok("countmin".into()), - SketchKind::CountSketch | SketchKind::CountSketchWithHeap => Ok("countsketch".into()), + SketchAlgorithm::Kll => Ok("KLL".into()), + SketchAlgorithm::DDSketch => Ok("ddsketch".into()), + SketchAlgorithm::Hll => Ok("HLL".into()), + SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap => Ok("countmin".into()), + SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap => { + Ok("countsketch".into()) + } // Exact-accumulator kinds (Sum/Count/MinMax/Increase/Rate) aren't // representable here anymore -- they're `ExactKind`, a distinct - // type post ASAPPlanner#218's split, not a `SketchKind` variant + // type post ASAPPlanner#218's split, not a `SketchAlgorithm` variant // this function could even be called with. - SketchKind::Kmv | SketchKind::Theta => Err(EmitError::NoEdgeProcessor(kind.clone())), + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => { + Err(EmitError::NoEdgeProcessor(kind.clone())) + } } } @@ -1149,11 +1153,11 @@ fn extract_edge_facts(qe: &crate::intent_algebra::QueryExpr, edge: &mut EdgeStag // filters, window size) from any leaves below. QE::Filter { child, .. } | QE::Project { child, .. } - | QE::Distinct { child, .. } + | QE::Dedup { child, .. } | QE::Sort { child, .. } | QE::Limit { child, .. } - | QE::Subquery { child, .. } => extract_edge_facts(child, edge), - QE::Merge { children } => { + | QE::PromqlSubquery { child, .. } => extract_edge_facts(child, edge), + QE::Concat { children } => { for c in children { extract_edge_facts(c, edge); } @@ -1208,7 +1212,7 @@ fn first_sketch_child_via_edges( dag: &ColoredDag, parent: crate::physical::colored_dag::dag::NodeId, sketch_agg_ids: &HashMap, -) -> Option<(SketchKind, String)> { +) -> Option<(SketchAlgorithm, String)> { for cid in children_of(dag, parent) { let cnode = dag.nodes.get(cid.0)?; match classify(&cnode.expr) { diff --git a/control_plane/src/physical/colored_dag/tests.rs b/control_plane/src/physical/colored_dag/tests.rs index cb46b5e88..e3b990d67 100644 --- a/control_plane/src/physical/colored_dag/tests.rs +++ b/control_plane/src/physical/colored_dag/tests.rs @@ -11,8 +11,8 @@ use std::rc::Rc; use std::time::Duration; use planner_types::post_asap::{ - SketchKind, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, SummaryNode, - SummarySchema, + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchQuery, SummaryExpr, + SummaryFamilyType, SummaryNode, SummarySchema, }; use crate::intent_algebra::schema::{Column, DataType}; @@ -74,7 +74,7 @@ fn ts_scan(metric: &str, label: Option<(&str, &str)>) -> QueryExpr { fn windowed_scan() -> QueryExpr { QueryExpr::TimeRange { range: Duration::from_secs(300), - child: Box::new(ts_scan( + child: Rc::new(ts_scan( "http_request_duration_seconds", Some(("service", "api")), )), @@ -96,7 +96,7 @@ fn dummy_l4_schema() -> SummarySchema { /// Wrap `qe` as an unbound `Logical` L4 leaf — mirrors the old /// `PhysicalExpr::Logical(qe)` construction for hand-built fixtures. fn logical_l4(qe: QueryExpr) -> Rc { - asap_aware_mapping::bind::logical(&qe).unwrap() + crate::planner_selection::keep_pre_asap(&qe).unwrap() } /// Hand-build a `SummaryAgg` node — mirrors the old @@ -104,18 +104,23 @@ fn logical_l4(qe: QueryExpr) -> Rc { /// for fixtures that need a specific family without going through /// `implement_tree`'s cost-model selection. fn sketch_agg_l4( - kind: SketchKind, + kind: SketchAlgorithm, params: SketchParams, child: Rc, ) -> Rc { Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - family: SummaryFamilyType::Sketch(kind, params), + family: SummaryFamilyType::Sketch( + SketchKind::new(kind, params), + GroupingStrategy::default(), + ), col: ColumnRef::SampleValue, reduction: Reduction::by(vec![]), + grouping: GroupingStrategy::default(), }, schema: dummy_l4_schema(), + guarantee: None, }) } @@ -128,6 +133,7 @@ fn estimate_l4(query: SketchQuery, summary_input: Rc) -> Rc>) -> Rc { Rc::new(SummaryNode { expr: SummaryExpr::SummaryMerge { children }, schema: dummy_l4_schema(), + guarantee: None, }) } @@ -147,7 +154,7 @@ fn is_sketch_agg(expr: &PhysicalExpr) -> bool { matches!(expr, PhysicalExpr::Committed(L4Plan::Summary(n)) if matches!(n.expr, SummaryExpr::SummaryAgg { .. })) } fn is_logical(expr: &PhysicalExpr) -> bool { - matches!(expr, PhysicalExpr::Committed(L4Plan::Summary(n)) if matches!(n.expr, SummaryExpr::Logical(_))) + matches!(expr, PhysicalExpr::Committed(L4Plan::Summary(n)) if matches!(n.expr, SummaryExpr::KeepPreAsap(_))) } fn is_sketch_estimate(expr: &PhysicalExpr) -> bool { matches!(expr, PhysicalExpr::Committed(L4Plan::Summary(n)) if matches!(n.expr, SummaryExpr::SummaryEstimate { .. })) @@ -177,9 +184,9 @@ fn quantile_kll_dag() -> PhysicalExpr { }], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; - PhysicalExpr::committed(asap_aware_mapping::bind::implement_tree(&q).unwrap()) + PhysicalExpr::committed(crate::planner_selection::select_summary_default(&q).unwrap()) } // ── Allocator: per-rule + edge-case tests ───────────────────────────────────── @@ -221,7 +228,7 @@ fn allocator_sketch_agg_under_scan_pinned_edge() { // Exact design.md §6 invariant: a SummaryAgg whose child is a Scan // (wrapped in Logical) MUST land on Edge. let expr = PhysicalExpr::committed(sketch_agg_l4( - SketchKind::Hll, + SketchAlgorithm::Hll, SketchParams::Hll { precision: 14 }, logical_l4(ts_scan("events", None)), )); @@ -263,7 +270,7 @@ fn allocator_let_binding_color_propagates() { // with the `child` position held by a bare `Ref` instead of a // `SketchEstimate{child: Ref}`. let inner_agg = sketch_agg_l4( - SketchKind::Kll, + SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }, logical_l4(windowed_scan()), ); @@ -293,7 +300,7 @@ fn allocator_ref_resolves_to_binding_stage() { // note there); the Ref child of the LetBinding must color Edge (the // binding's stage). let inner_agg = sketch_agg_l4( - SketchKind::Kll, + SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }, logical_l4(windowed_scan()), ); @@ -320,7 +327,7 @@ fn allocator_sketch_merge_lands_gateway() { // SummaryMerge over edge-built KLL sketches → Gateway. let one_agg = || { sketch_agg_l4( - SketchKind::Kll, + SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }, logical_l4(windowed_scan()), ) @@ -350,7 +357,7 @@ fn emitter_three_stage_emits_three_configs() { // SummaryMerge over two SummaryAggs. let one_agg = || { sketch_agg_l4( - SketchKind::Kll, + SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }, logical_l4(windowed_scan()), ) @@ -377,7 +384,7 @@ fn emitter_edge_config_has_correct_processor_kll() { StageConfig::Edge(e) => { assert_eq!(e.sketch_processors.len(), 1); assert_eq!(e.sketch_processors[0].processor_name, "KLL"); - assert_eq!(e.sketch_processors[0].sketch_kind, SketchKind::Kll); + assert_eq!(e.sketch_processors[0].sketch_kind, SketchAlgorithm::Kll); assert_eq!( e.source_metric.as_deref(), Some("http_request_duration_seconds") @@ -393,7 +400,7 @@ fn emitter_edge_config_has_correct_processor_ddsketch() { let expr = PhysicalExpr::committed(estimate_l4( SketchQuery::Quantile { q: 0.99 }, sketch_agg_l4( - SketchKind::DDSketch, + SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha: 0.01 }, logical_l4(windowed_scan()), ), @@ -424,7 +431,7 @@ fn emitter_backend_config_routes_aggregation_id() { StageConfig::Backend(b) => { assert_eq!(b.aggregations.len(), 1); assert_eq!(b.aggregations[0].aggregation_id, edge_aid); - assert_eq!(b.aggregations[0].sketch_kind, SketchKind::Kll.into()); + assert_eq!(b.aggregations[0].sketch_kind, SketchAlgorithm::Kll.into()); assert_eq!(b.readouts.len(), 1); assert_eq!(b.readouts[0].aggregation_id, edge_aid); // `SketchQuery` has no `PartialEq` upstream — destructure @@ -475,7 +482,7 @@ fn end_to_end_quantile_workload() { // root). let agg = || { sketch_agg_l4( - SketchKind::Kll, + SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }, logical_l4(windowed_scan()), ) @@ -507,7 +514,7 @@ fn end_to_end_quantile_workload() { StageConfig::Gateway(g) => { assert!(!g.merge_processors.is_empty()); assert_eq!(g.merge_processors[0].processor_name, "sketchmergeprocessor"); - assert_eq!(g.merge_processors[0].sketch_kind, SketchKind::Kll); + assert_eq!(g.merge_processors[0].sketch_kind, SketchAlgorithm::Kll); } _ => unreachable!(), } diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs new file mode 100644 index 000000000..292957a71 --- /dev/null +++ b/control_plane/src/physical/compiler.rs @@ -0,0 +1,460 @@ +//! Backend-owned physical compilation over ASAPPlanner's selected post-ASAP IR. +//! +//! Planner owns semantic alternatives and guarantees. This module owns the +//! deployment decision: evidence freshness, target capabilities, windows, the +//! Collector execution projection, and the matching BackendPlan. + +use std::collections::HashMap; + +use asap_aware_mapping::{ + AccuracyEvidenceProvider, DefaultAccuracyModel, EqualSplitAllocator, PropagationStats, +}; +use planner_types::post_asap::{ + CompositionOperator, SketchQuery, SummaryExpr, SummaryFamilyType, SummaryNode, +}; +use planner_types::pre_asap::QueryExpr; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use thiserror::Error; + +use crate::backend_plan::{self, BackendPlan}; +use crate::emit::monitor::MonitorIntent; +use crate::intent_algebra::Source; +use crate::physical::colored_dag::emitter::{ + AggregationInput, BackendAggregation, BackendReadout, BackendStageConfig, +}; +use crate::sketch_algebra::cost_model::ControlPlaneCostModel; +use crate::types_v2::AccuracyTarget; + +pub const PLANNER_REVISION: &str = "3afcba68f4e8397fb81e2be988f47120f63f7a39"; + +#[derive(Debug, Clone)] +pub struct PlanningQuery { + pub query_id: String, + pub expr: QueryExpr, + pub source: Source, + pub window_secs: u64, + /// Label names are deployment metadata because Planner's canonical IR + /// currently carries positional column IDs at this boundary. + pub group_by: Vec, + pub accuracy: AccuracyTarget, +} + +#[derive(Debug, Clone, Default)] +pub struct PlanningRequest { + pub queries: Vec, + pub evidence: HashMap, + pub planner_revision: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct TopKMembershipEvidence { + pub selected_lower_bound: f64, + pub excluded_upper_bound: f64, + pub interval_failure_probability: f64, + pub observed_at_unix_ms: u64, + pub source: String, +} + +#[derive(Debug, Clone)] +pub struct DeploymentEnvironment { + pub collector_ids: Vec, + pub capability_snapshot_id: String, + pub observed_at_unix_ms: u64, + pub max_evidence_age_ms: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct PlanEnvelope { + pub plan_id: u64, + pub generated_at_unix_ms: u64, + pub planner_revision: String, + pub capability_snapshot_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CollectorMaterialization { + pub query_id: String, + pub metric: String, + pub algorithm: String, + pub parameters: Value, + pub group_by: Vec, + pub window_secs: u64, + pub evidence_source: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CollectorPlan { + pub collector_id: String, + pub envelope: PlanEnvelope, + pub materializations: Vec, +} + +#[derive(Debug, Clone)] +pub struct CompiledPlanBundle { + pub envelope: PlanEnvelope, + pub collector_plans: Vec, + pub backend_plan: BackendPlan, +} + +#[derive(Debug, Error)] +pub enum CompileError { + #[error("planner revision mismatch: request={request}, compiler={compiler}")] + PlannerRevision { + request: String, + compiler: &'static str, + }, + #[error("query {query_id}: {reason}")] + Query { query_id: String, reason: String }, + #[error("query {query_id}: TopK evidence is stale or invalid: {reason}")] + InvalidEvidence { query_id: String, reason: String }, + #[error("failed to construct BackendPlan: {0}")] + BackendPlan(#[from] anyhow::Error), +} + +struct QueryEvidence<'a>(Option<&'a TopKMembershipEvidence>); + +impl AccuracyEvidenceProvider for QueryEvidence<'_> { + fn propagation_stats( + &self, + op: &CompositionOperator, + _family: &SummaryFamilyType, + _query: Option<&SketchQuery>, + ) -> PropagationStats { + match (op, self.0) { + (CompositionOperator::TopKSelection, Some(e)) => PropagationStats { + topk_selected_lower_bound: Some(e.selected_lower_bound), + topk_excluded_upper_bound: Some(e.excluded_upper_bound), + topk_interval_failure_probability: Some(e.interval_failure_probability), + ..Default::default() + }, + _ => PropagationStats::default(), + } + } +} + +#[derive(Debug, Default)] +pub struct PhysicalCompiler; + +impl PhysicalCompiler { + pub fn compile( + &self, + request: PlanningRequest, + environment: DeploymentEnvironment, + ) -> Result { + if request.planner_revision != PLANNER_REVISION { + return Err(CompileError::PlannerRevision { + request: request.planner_revision, + compiler: PLANNER_REVISION, + }); + } + + let mut aggregations = Vec::with_capacity(request.queries.len()); + let mut readouts = Vec::with_capacity(request.queries.len()); + let mut collector_materializations = Vec::with_capacity(request.queries.len()); + + for query in &request.queries { + let evidence = request.evidence.get(&query.query_id); + if let Some(e) = evidence { + validate_evidence(&query.query_id, e, &environment)?; + } + let model = ControlPlaneCostModel::new(query.accuracy.clone()); + let node = crate::planner_selection::select_summary_with_evidence( + &query.expr, + &model, + &DefaultAccuracyModel, + &EqualSplitAllocator, + &QueryEvidence(evidence), + ) + .map_err(|error| CompileError::Query { + query_id: query.query_id.clone(), + reason: error.to_string(), + })?; + let selected = extract_selected(&node).ok_or_else(|| CompileError::Query { + query_id: query.query_id.clone(), + reason: "selected plan has no executable sketch materialization/readout".into(), + })?; + let metric = match &query.source { + Source::TimeSeries { metric } => metric.clone(), + Source::Table { .. } => { + return Err(CompileError::Query { + query_id: query.query_id.clone(), + reason: "MVP physical compiler supports time-series sources only".into(), + }) + } + }; + let aggregation_id = format!("{}:{}", query.query_id, metric); + let kind = asap_types::SummaryKind::from(selected.kind.clone()); + let params = asap_types::SummaryParams::from(selected.params.clone()); + aggregations.push(BackendAggregation { + aggregation_id: aggregation_id.clone(), + metric_name: metric.clone(), + sketch_kind: kind, + sketch_params: params, + window_secs: query.window_secs, + spatial_filter: String::new(), + grouping: query.group_by.clone(), + item_label: None, + aggregation_input: AggregationInput::SketchEnvelope, + agg_type_override: None, + }); + readouts.push(BackendReadout { + aggregation_id, + op: selected.readout.clone(), + }); + collector_materializations.push(CollectorMaterialization { + query_id: query.query_id.clone(), + metric, + algorithm: format!("{:?}", selected.kind.algorithm()).to_ascii_lowercase(), + parameters: sketch_params_json(&selected.params), + group_by: query.group_by.clone(), + window_secs: query.window_secs, + evidence_source: evidence.map(|e| e.source.clone()), + }); + } + + let plan_id = stable_plan_id(&collector_materializations); + let envelope = PlanEnvelope { + plan_id, + generated_at_unix_ms: environment.observed_at_unix_ms, + planner_revision: PLANNER_REVISION.into(), + capability_snapshot_id: environment.capability_snapshot_id, + }; + let backend_plan = backend_plan::from_stage_config( + &BackendStageConfig { + aggregations, + readouts, + }, + &Vec::::new(), + plan_id, + environment.observed_at_unix_ms, + )?; + let collector_plans = environment + .collector_ids + .into_iter() + .map(|collector_id| CollectorPlan { + collector_id, + envelope: envelope.clone(), + materializations: collector_materializations.clone(), + }) + .collect(); + Ok(CompiledPlanBundle { + envelope, + collector_plans, + backend_plan, + }) + } +} + +fn validate_evidence( + query_id: &str, + evidence: &TopKMembershipEvidence, + env: &DeploymentEnvironment, +) -> Result<(), CompileError> { + let age = env + .observed_at_unix_ms + .saturating_sub(evidence.observed_at_unix_ms); + let valid = evidence.selected_lower_bound.is_finite() + && evidence.excluded_upper_bound.is_finite() + && evidence.selected_lower_bound > evidence.excluded_upper_bound + && (0.0..=1.0).contains(&evidence.interval_failure_probability) + && !evidence.source.trim().is_empty() + && age <= env.max_evidence_age_ms; + if valid { + Ok(()) + } else { + Err(CompileError::InvalidEvidence { + query_id: query_id.into(), + reason: format!("margin/failure/source invalid or age {age}ms exceeds policy"), + }) + } +} + +struct SelectedSketch { + kind: planner_types::post_asap::SketchKind, + params: planner_types::post_asap::SketchParams, + readout: SketchQuery, +} + +fn extract_selected(node: &SummaryNode) -> Option { + let SummaryExpr::SummaryEstimate { + summary_input, + query, + } = &node.expr + else { + return None; + }; + let SummaryExpr::SummaryAgg { + family: SummaryFamilyType::Sketch(kind, _), + .. + } = &summary_input.expr + else { + return None; + }; + Some(SelectedSketch { + kind: kind.clone(), + params: kind.params().clone(), + readout: query.clone(), + }) +} + +fn sketch_params_json(params: &planner_types::post_asap::SketchParams) -> Value { + use planner_types::post_asap::SketchParams as P; + match params { + P::Kll { k } => json!({"k": k}), + P::Cms { width, depth } => json!({"width": width, "depth": depth}), + P::Hll { precision } => json!({"precision": precision}), + P::DDSketch { alpha } => json!({"alpha": alpha}), + P::CmsWithHeap { + width, + depth, + heap_size, + } => json!({"width": width, "depth": depth, "heap_size": heap_size}), + P::Kmv { k } | P::Theta { k } => json!({"k": k}), + P::CountSketch { width, depth } => json!({"width": width, "depth": depth}), + P::CountSketchWithHeap { + width, + depth, + heap_size, + } => json!({"width": width, "depth": depth, "heap_size": heap_size}), + } +} + +fn stable_plan_id(materializations: &[CollectorMaterialization]) -> u64 { + use std::hash::{Hash, Hasher}; + let bytes = serde_json::to_vec(materializations).unwrap_or_default(); + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + bytes.hash(&mut hasher); + hasher.finish() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn environment(now: u64) -> DeploymentEnvironment { + DeploymentEnvironment { + collector_ids: vec!["edge-a".into(), "edge-b".into()], + capability_snapshot_id: "caps-7".into(), + observed_at_unix_ms: now, + max_evidence_age_ms: 60_000, + } + } + + fn request(query_id: &str, promql: &str) -> PlanningRequest { + let accuracy = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.01, + }; + let parsed = crate::query_parser::parse_query_expr_canonical(promql, accuracy.clone()) + .expect("canonical query"); + let expr = if promql.starts_with("topk(") { + use crate::optimizer::engine::{DefaultCostModel, RewriteRule, TopKFusion}; + let cost = DefaultCostModel { + raw_bytes_per_sec: 1.0, + deployment: None, + }; + TopKFusion + .try_rewrite(parsed.clone(), &cost) + .unwrap_or(parsed) + } else { + parsed + }; + PlanningRequest { + queries: vec![PlanningQuery { + query_id: query_id.into(), + expr, + source: Source::TimeSeries { metric: "m".into() }, + window_secs: 60, + group_by: vec![], + accuracy, + }], + evidence: HashMap::new(), + planner_revision: PLANNER_REVISION.into(), + } + } + + #[test] + fn compiles_one_decision_into_matching_collector_and_backend_views() { + let bundle = PhysicalCompiler + .compile( + request("q-quantile", "quantile_over_time(0.99, m[1m])"), + environment(10_000), + ) + .expect("compile"); + assert_eq!(bundle.collector_plans.len(), 2); + assert_eq!(bundle.backend_plan.materializations.len(), 1); + assert_eq!(bundle.backend_plan.routing.len(), 1); + for plan in &bundle.collector_plans { + assert_eq!(plan.envelope, bundle.envelope); + assert_eq!(plan.materializations[0].metric, "m"); + assert_eq!(plan.materializations[0].window_secs, 60); + assert!(matches!( + plan.materializations[0].algorithm.as_str(), + "ddsketch" | "kll" + )); + } + } + + #[test] + fn topk_fails_closed_without_membership_evidence() { + let error = PhysicalCompiler + .compile(request("q-topk", "topk(5, m)"), environment(10_000)) + .expect_err("missing certificate must fail"); + assert!(matches!(error, CompileError::Query { .. })); + } + + #[test] + fn stale_topk_evidence_is_rejected_before_planner_selection() { + let mut request = request("q-topk", "topk(5, m)"); + request.evidence.insert( + "q-topk".into(), + TopKMembershipEvidence { + selected_lower_bound: 101.0, + excluded_upper_bound: 100.0, + interval_failure_probability: 0.005, + observed_at_unix_ms: 1, + source: "runtime-margin-monitor".into(), + }, + ); + let error = PhysicalCompiler + .compile(request, environment(100_000)) + .expect_err("stale certificate must fail"); + assert!(matches!(error, CompileError::InvalidEvidence { .. })); + } + + #[test] + fn fresh_topk_evidence_enables_physical_compilation() { + let mut request = request("q-topk", "topk(5, m)"); + request.evidence.insert( + "q-topk".into(), + TopKMembershipEvidence { + selected_lower_bound: 101.0, + excluded_upper_bound: 100.0, + interval_failure_probability: 0.005, + observed_at_unix_ms: 9_500, + source: "runtime-margin-monitor".into(), + }, + ); + let bundle = PhysicalCompiler + .compile(request, environment(10_000)) + .expect("certified TopK compiles"); + assert_eq!(bundle.backend_plan.materializations.len(), 1); + assert_eq!( + bundle.collector_plans[0].materializations[0] + .evidence_source + .as_deref(), + Some("runtime-margin-monitor") + ); + } + + #[test] + fn planner_revision_is_part_of_the_compile_contract() { + let mut request = request("q", "quantile_over_time(0.9, m[1m])"); + request.planner_revision = "different".into(); + assert!(matches!( + PhysicalCompiler.compile(request, environment(10_000)), + Err(CompileError::PlannerRevision { .. }) + )); + } +} diff --git a/control_plane/src/physical/mod.rs b/control_plane/src/physical/mod.rs index 1fa4b3f1d..993f13e51 100644 --- a/control_plane/src/physical/mod.rs +++ b/control_plane/src/physical/mod.rs @@ -22,6 +22,7 @@ pub mod allocator; pub mod colored_dag; +pub mod compiler; pub mod plan; pub mod planner; pub mod sketch_catalog; diff --git a/control_plane/src/physical/planner.rs b/control_plane/src/physical/planner.rs index f0e8cd908..dad71f386 100644 --- a/control_plane/src/physical/planner.rs +++ b/control_plane/src/physical/planner.rs @@ -351,7 +351,7 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { // (`intent_algebra::lower`), so the `HashAggregate { keys }` this // arm used to build now comes straight out of the `Aggregate` // arm above. - QueryExpr::Merge { children } => { + QueryExpr::Concat { children } => { let children: Vec = children.iter().map(|c| plan_node(c, config)).collect(); let sketch_type = children @@ -372,7 +372,7 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { } } - QueryExpr::Distinct { cols, child } => { + QueryExpr::Dedup { cols, child } => { let child = plan_node(child, config); let pred = format!("distinct({})", display_distinct_cols(cols)); let mut node = PhysicalNode { @@ -397,7 +397,7 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { } } - QueryExpr::Subquery { child, .. } => { + QueryExpr::PromqlSubquery { child, .. } => { let child = plan_node(child, config); let mut node = PhysicalNode { op: PhysicalOp::Passthrough, @@ -441,19 +441,19 @@ fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { // leaves; every other new variant wraps exactly one child — // inherit its placement, mirroring the Sort/Limit/Project arm // above, until a dedicated physical op is written. - QueryExpr::Scalar(_) | QueryExpr::EvalTime => PhysicalNode { + QueryExpr::PromqlScalarBridge(_) | QueryExpr::EvalTimestamp => PhysicalNode { op: PhysicalOp::Passthrough, placement: Placement::QueryEngine, cost: PhysicalCost::default(), children: vec![], }, - QueryExpr::VectorFromScalar(child) - | QueryExpr::ScalarFromVector(child) - | QueryExpr::Relabel { child, .. } - | QueryExpr::InfoJoin { child, .. } - | QueryExpr::Sample { child, .. } + QueryExpr::PromqlVectorFromScalar(child) + | QueryExpr::PromqlScalarFromVector(child) + | QueryExpr::PromqlRelabel { child, .. } + | QueryExpr::PromqlInfoEnrich { child, .. } + | QueryExpr::PromqlSeriesSample { child, .. } | QueryExpr::TimeShift { child, .. } - | QueryExpr::WindowFunc { child, .. } => { + | QueryExpr::SQLWindowFunc { child, .. } => { let child = plan_node(child, config); PhysicalNode { op: PhysicalOp::Passthrough, @@ -588,6 +588,8 @@ impl PhysicalNode { #[cfg(test)] mod tests { + use std::rc::Rc; + use super::*; use crate::intent_algebra::relational::{ default_cardinality, default_frequency, default_quantile, @@ -621,7 +623,7 @@ mod tests { measures: vec![intent], output_names: Vec::new(), having: None, - child: Box::new(scan(metric)), + child: Rc::new(scan(metric)), } } @@ -630,7 +632,7 @@ mod tests { fn windowed_agg(intent: AggIntent, size_secs: u64, metric: &str) -> QueryExpr { QueryExpr::TimeRange { range: Duration::from_secs(size_secs), - child: Box::new(sketch_agg(intent, metric)), + child: Rc::new(sketch_agg(intent, metric)), } } @@ -704,7 +706,7 @@ mod tests { }], output_names: Vec::new(), having: None, - child: Box::new(sketch_agg(default_frequency(), "m")), + child: Rc::new(sketch_agg(default_frequency(), "m")), }; let node = plan(&expr, &default_config()); assert_eq!(node.placement, Placement::QueryEngine); @@ -723,7 +725,7 @@ mod tests { }], output_names: Vec::new(), having: None, - child: Box::new(sketch_agg(default_frequency(), "m")), + child: Rc::new(sketch_agg(default_frequency(), "m")), }; let node = plan(&expr, &default_config()); assert!( @@ -750,7 +752,7 @@ mod tests { measures: vec![AggIntent::Sum { col: None }, AggIntent::Min { col: None }], output_names: Vec::new(), having: None, - child: Box::new(scan("trades")), + child: Rc::new(scan("trades")), }; let node = plan(&expr, &default_config()); assert_eq!(node.placement, Placement::QueryEngine); @@ -771,9 +773,10 @@ mod tests { }], output_names: Vec::new(), having: None, - child: Box::new(QueryExpr::Merge { + child: QueryExpr::Concat { children: vec![windowed_agg(default_frequency(), 60, "requests")], - }), + } + .into(), }; let node = plan(&expr, &default_config()); let placements = node.placements(); diff --git a/control_plane/src/physical/window_fusion.rs b/control_plane/src/physical/window_fusion.rs index ee1073176..b28355b8f 100644 --- a/control_plane/src/physical/window_fusion.rs +++ b/control_plane/src/physical/window_fusion.rs @@ -184,6 +184,8 @@ pub fn fused_sketch_decision( #[cfg(test)] mod tests { + use std::rc::Rc; + use super::*; use crate::intent_algebra::query_expr::Source; use crate::intent_algebra::schema::Schema; @@ -224,12 +226,12 @@ mod tests { fn windowed_sketch(agg: AggIntent, size: Duration) -> QueryExpr { QueryExpr::TimeRange { range: size, - child: Box::new(QueryExpr::Aggregate { + child: Rc::new(QueryExpr::Aggregate { reduction: crate::intent_algebra::Reduction::by(vec![]), measures: vec![agg], output_names: Vec::new(), having: None, - child: Box::new(canonical_scan("m")), + child: Rc::new(canonical_scan("m")), }), } } @@ -308,7 +310,7 @@ mod tests { measures: vec![AggIntent::Sum { col: None }], output_names: Vec::new(), having: None, - child: Box::new(canonical_scan("m")), + child: Rc::new(canonical_scan("m")), }; assert!(recognize_windowed_sketch(&canonical).is_none()); } @@ -318,7 +320,7 @@ mod tests { // `TimeRange` directly over a `Scan` — no inner `Aggregate` to fuse. let canonical = QueryExpr::TimeRange { range: Duration::from_secs(60), - child: Box::new(canonical_scan("m")), + child: Rc::new(canonical_scan("m")), }; assert!(recognize_windowed_sketch(&canonical).is_none()); } diff --git a/control_plane/src/planner_selection.rs b/control_plane/src/planner_selection.rs new file mode 100644 index 000000000..bc706a9e8 --- /dev/null +++ b/control_plane/src/planner_selection.rs @@ -0,0 +1,113 @@ +//! Deployment-owned selection at the latest ASAPPlanner boundary. +//! +//! ASAPPlanner enumerates a ranked candidate space and deliberately does not +//! commit to one deployment plan. The backend owns that decision because it +//! also owns placement, runtime capabilities, and the physical wire contract. + +use std::rc::Rc; + +use asap_aware_mapping::{ + AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, CostModel, Replacement, + ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG, +}; +use planner_types::post_asap::{ + SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, +}; +use planner_types::pre_asap::{QueryExpr, QueryExprError}; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum SelectionError { + #[error("failed to derive the pre-ASAP schema: {0}")] + Schema(#[from] QueryExprError), + #[error("ASAPPlanner produced no legal summary candidate for the target")] + NoLegalCandidate, + #[error("ASAPPlanner sketch strategy produced a logical rewrite instead of a summary")] + UnexpectedRewrite, +} + +/// Preserve an unsupported subtree explicitly at the post-ASAP boundary. +pub fn keep_pre_asap(expr: &QueryExpr) -> Result, SelectionError> { + let schema = expr.output_schema()?; + Ok(Rc::new(SummaryNode { + expr: SummaryExpr::KeepPreAsap(Rc::new(expr.clone())), + schema: SummarySchema { + fields: schema + .columns + .into_iter() + .map(|column| SummaryField { + name: column.name, + dtype: SummaryFamilyType::Plain(column.dtype), + nullable: column.nullable, + }) + .collect(), + time_index: schema.time_index, + }, + guarantee: None, + })) +} + +/// Select the first legal candidate after the supplied deployment cost model +/// has ranked Planner's exhaustive candidate set. +pub fn select_summary( + expr: &QueryExpr, + cost_model: &dyn CostModel, +) -> Result, SelectionError> { + let root = Rc::new(expr.clone()); + let strategy = SketchAlgorithmStrategy::new(cost_model); + let candidate = strategy + .replacements(&TargetSubDAG::new(&root)) + .into_iter() + .next() + .ok_or(SelectionError::NoLegalCandidate)?; + match candidate.replacement { + Replacement::Summary(node) => Ok(node), + Replacement::Rewrite(_) => Err(SelectionError::UnexpectedRewrite), + } +} + +pub fn select_summary_default(expr: &QueryExpr) -> Result, SelectionError> { + select_summary(expr, &asap_aware_mapping::DefaultCostModel) +} + +/// Select from Planner's legal candidates with deployment-supplied accuracy +/// models and typed evidence (for example a TopK membership certificate). +pub fn select_summary_with_evidence( + expr: &QueryExpr, + cost_model: &dyn CostModel, + accuracy_model: &dyn AccuracyModel, + allocator: &dyn AccuracyBudgetAllocator, + evidence: &dyn AccuracyEvidenceProvider, +) -> Result, SelectionError> { + let root = Rc::new(expr.clone()); + let strategy = SketchAlgorithmStrategy::with_models_and_evidence( + cost_model, + accuracy_model, + allocator, + evidence, + ); + let candidate = strategy + .replacements(&TargetSubDAG::new(&root)) + .into_iter() + .next() + .ok_or(SelectionError::NoLegalCandidate)?; + match candidate.replacement { + Replacement::Summary(node) => Ok(node), + Replacement::Rewrite(_) => Err(SelectionError::UnexpectedRewrite), + } +} + +/// Select a legal summary when Planner offers one, otherwise preserve the +/// subtree explicitly. This mirrors the removed single-tree binder's +/// conservative behavior and is appropriate for serving fallbacks; physical +/// compilation should use [`select_summary`] so an unimplementable target is +/// reported rather than silently committed. +pub fn select_summary_or_keep( + expr: &QueryExpr, + cost_model: &dyn CostModel, +) -> Result, SelectionError> { + match select_summary(expr, cost_model) { + Err(SelectionError::NoLegalCandidate) => keep_pre_asap(expr), + result => result, + } +} diff --git a/control_plane/src/query_parser/mod.rs b/control_plane/src/query_parser/mod.rs index 667af47f8..722df96d8 100644 --- a/control_plane/src/query_parser/mod.rs +++ b/control_plane/src/query_parser/mod.rs @@ -157,12 +157,12 @@ fn root_scan_schema(qe: &QueryExpr) -> Option<&crate::intent_algebra::Schema> { | QueryExpr::TimeRange { child, .. } | QueryExpr::TimeShift { child, .. } | QueryExpr::Aggregate { child, .. } - | QueryExpr::Distinct { child, .. } + | QueryExpr::Dedup { child, .. } | QueryExpr::Project { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } - | QueryExpr::Subquery { child, .. } => root_scan_schema(child), - QueryExpr::Merge { children } => children.iter().find_map(root_scan_schema), + | QueryExpr::PromqlSubquery { child, .. } => root_scan_schema(child), + QueryExpr::Concat { children } => children.iter().find_map(root_scan_schema), QueryExpr::Join { left, right, .. } | QueryExpr::SetOp { left, right, .. } | QueryExpr::BinaryOp { @@ -271,8 +271,8 @@ impl QeCollector { } self.visit(child, schema); } - QueryExpr::Distinct { child, .. } => self.visit(child, schema), - QueryExpr::Merge { children } => { + QueryExpr::Dedup { child, .. } => self.visit(child, schema), + QueryExpr::Concat { children } => { for c in children { self.visit(c, schema); } @@ -280,7 +280,7 @@ impl QeCollector { QueryExpr::Project { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } - | QueryExpr::Subquery { child, .. } => self.visit(child, schema), + | QueryExpr::PromqlSubquery { child, .. } => self.visit(child, schema), QueryExpr::Join { left, right, .. } | QueryExpr::SetOp { left, right, .. } | QueryExpr::BinaryOp { @@ -518,7 +518,7 @@ mod tests { let expr = parse_query_expr_canonical("avg_over_time(cpu_seconds_total[10m])", ACC).unwrap(); match expr { - CQueryExpr::Aggregate { child, .. } => match *child { + CQueryExpr::Aggregate { child, .. } => match (*child).clone() { CQueryExpr::TimeRange { child, .. } => { assert!(matches!(*child, CQueryExpr::Scan { .. })); } diff --git a/control_plane/src/replan.rs b/control_plane/src/replan.rs index 736279590..453e4bcfd 100644 --- a/control_plane/src/replan.rs +++ b/control_plane/src/replan.rs @@ -655,7 +655,7 @@ impl Replanner { use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendStageConfig, }; - use planner_types::post_asap::{SketchKind, SketchParams}; + use planner_types::post_asap::{SketchAlgorithm, SketchParams}; let window_secs = workload.time_window.as_secs().max(1); Some(BackendStageConfig { aggregations: vec![BackendAggregation { @@ -667,8 +667,8 @@ impl Replanner { // these are not emitted on the wire. DDSketch is the // chosen sentinel because every backend that recognises // `AggregationType::FromStr` also accepts DDSketch (and - // we don't have a `SketchKind::None` variant today). - sketch_kind: SketchKind::DDSketch.into(), + // we don't have a `SketchAlgorithm::None` variant today). + sketch_kind: SketchAlgorithm::DDSketch.into(), sketch_params: SketchParams::DDSketch { alpha: 0.01 }.into(), window_secs, spatial_filter: String::new(), @@ -1254,7 +1254,7 @@ mod tests { use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendStageConfig, }; - use planner_types::post_asap::{SketchKind, SketchParams}; + use planner_types::post_asap::{SketchAlgorithm, SketchParams}; let (url, hits) = start_repost_mock().await; let client = StdArc::new(BackendClient::new(url)); @@ -1304,7 +1304,7 @@ mod tests { item_label: None, aggregation_id: "exact-http_requests_total-sum".to_string(), metric_name: "http_requests_total".to_string(), - sketch_kind: SketchKind::DDSketch.into(), + sketch_kind: SketchAlgorithm::DDSketch.into(), sketch_params: SketchParams::DDSketch { alpha: 0.01 }.into(), grouping: vec!["zone".to_string()], spatial_filter: String::new(), diff --git a/control_plane/src/sketch_algebra/capability.rs b/control_plane/src/sketch_algebra/capability.rs index 398b63054..0c611f755 100644 --- a/control_plane/src/sketch_algebra/capability.rs +++ b/control_plane/src/sketch_algebra/capability.rs @@ -24,7 +24,7 @@ use crate::intent_algebra::agg_intent::AggIntent; use crate::sketch_algebra::matcher::sketch_family_satisfied; use crate::types_v2::AccuracyTarget; use asap_types::AggregationType; -use planner_types::post_asap::SketchKind; +use planner_types::post_asap::SketchAlgorithm; // ── Query-side capability tag ──────────────────────────────────────────────── @@ -335,14 +335,14 @@ impl Capability { /// delegate their family-compatibility logic to /// [`sketch_family_satisfied`] (`enum-unification-plan.md` /// §5/§8 Step 4) — via [`resolve_handle`], which picks a concrete - /// per-family stand-in for the `Any` wildcard since `SketchKind` + /// per-family stand-in for the `Any` wildcard since `SketchAlgorithm` /// has no wildcard concept of its own; family-matching subsumes it. /// `ExactAgg` is intentionally NOT routed through this path — see /// [`multi_pop_satisfies_single`]'s doc for why. pub fn is_satisfied_by(&self, indexed: &Capability) -> bool { match (self, indexed) { (Capability::QuantileApprox(req), Capability::QuantileApprox(have)) => { - sketch_kinds_compatible(*req, SketchKind::Kll, *have, SketchKind::Kll) + sketch_kinds_compatible(*req, SketchAlgorithm::Kll, *have, SketchAlgorithm::Kll) } // Cardinality has no inner handle; family match is total. (Capability::CardinalityApprox, Capability::CardinalityApprox) => true, @@ -353,9 +353,9 @@ impl Capability { (Capability::FrequencyTopk(req), Capability::FrequencyTopk(have)) => { sketch_kinds_compatible( *req, - SketchKind::CmsWithHeap, + SketchAlgorithm::CmsWithHeap, *have, - SketchKind::CmsWithHeap, + SketchAlgorithm::CmsWithHeap, ) } // Bare frequency: any frequency-family handle works on the @@ -365,10 +365,15 @@ impl Capability { // the sketch matrix). A heap-bearing `FrequencyTopk` indexed // capability ALSO satisfies a bare-frequency required capability. (Capability::FrequencyEstimate(req), Capability::FrequencyEstimate(have)) => { - sketch_kinds_compatible(*req, SketchKind::Cms, *have, SketchKind::Cms) + sketch_kinds_compatible(*req, SketchAlgorithm::Cms, *have, SketchAlgorithm::Cms) } (Capability::FrequencyEstimate(req), Capability::FrequencyTopk(have)) => { - sketch_kinds_compatible(*req, SketchKind::Cms, *have, SketchKind::CmsWithHeap) + sketch_kinds_compatible( + *req, + SketchAlgorithm::Cms, + *have, + SketchAlgorithm::CmsWithHeap, + ) } // Exact-aggregation family: the agg_type must match // exactly OR be the single-pop ⇆ multi-pop equivalent. A @@ -394,19 +399,19 @@ impl Capability { } } -/// Map a concrete [`SketchKindHandle`] to its [`SketchKind`] +/// Map a concrete [`SketchKindHandle`] to its [`SketchAlgorithm`] /// equivalent. `Any` has no single equivalent by design — resolve it to /// a concrete per-family stand-in via [`resolve_handle`] before calling /// this. -fn to_summary_kind(h: SketchKindHandle) -> Option { +fn to_summary_kind(h: SketchKindHandle) -> Option { match h { - SketchKindHandle::DDSketch => Some(SketchKind::DDSketch), - SketchKindHandle::Kll => Some(SketchKind::Kll), - SketchKindHandle::Hll => Some(SketchKind::Hll), - SketchKindHandle::CountSketch => Some(SketchKind::CountSketch), - SketchKindHandle::CountMin => Some(SketchKind::Cms), - SketchKindHandle::CmsWithHeap => Some(SketchKind::CmsWithHeap), - SketchKindHandle::CountSketchWithHeap => Some(SketchKind::CountSketchWithHeap), + SketchKindHandle::DDSketch => Some(SketchAlgorithm::DDSketch), + SketchKindHandle::Kll => Some(SketchAlgorithm::Kll), + SketchKindHandle::Hll => Some(SketchAlgorithm::Hll), + SketchKindHandle::CountSketch => Some(SketchAlgorithm::CountSketch), + SketchKindHandle::CountMin => Some(SketchAlgorithm::Cms), + SketchKindHandle::CmsWithHeap => Some(SketchAlgorithm::CmsWithHeap), + SketchKindHandle::CountSketchWithHeap => Some(SketchAlgorithm::CountSketchWithHeap), // Defensive: `Any` should never reach this function directly — // every call site resolves it via `resolve_handle` first. `None` // here means "does not satisfy anything", the safe default. @@ -414,11 +419,11 @@ fn to_summary_kind(h: SketchKindHandle) -> Option { } } -/// Resolve a [`SketchKindHandle`] to the [`SketchKind`] fed into +/// Resolve a [`SketchKindHandle`] to the [`SketchAlgorithm`] fed into /// [`sketch_family_satisfied`]. `Any` (the query-side "any /// implementation in this family satisfies" wildcard) resolves to /// `any_stand_in` — a concrete per-family placeholder — because -/// `SketchKind` has no wildcard concept of its own; +/// `SketchAlgorithm` has no wildcard concept of its own; /// `sketch_family_satisfied`'s same-family-satisfies rule already /// treats every member of a family as interchangeable, so picking ANY /// concrete family member as the stand-in reproduces the wildcard's @@ -430,9 +435,9 @@ fn to_summary_kind(h: SketchKindHandle) -> Option { /// related by the asymmetric "heap satisfies bare" rule, not equal), so a /// bare stand-in would let a heap-less available sketch wrongly satisfy a /// top-k requirement. Every `FrequencyTopk` call site in this module -/// passes `SketchKind::CmsWithHeap` as `any_stand_in` for exactly this +/// passes `SketchAlgorithm::CmsWithHeap` as `any_stand_in` for exactly this /// reason. -fn resolve_handle(h: SketchKindHandle, any_stand_in: SketchKind) -> Option { +fn resolve_handle(h: SketchKindHandle, any_stand_in: SketchAlgorithm) -> Option { match h { SketchKindHandle::Any => Some(any_stand_in), other => to_summary_kind(other), @@ -445,9 +450,9 @@ fn resolve_handle(h: SketchKindHandle, any_stand_in: SketchKind) -> Option bool { match ( resolve_handle(required, required_any_stand_in), @@ -559,7 +564,28 @@ pub fn capability_for(intent: &AggIntent) -> Option { Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) }; } - implementation_to_capability(asap_aware_mapping::boundary::implementation_for(intent)) + match intent { + AggIntent::Sum { .. } => Some(Capability::ExactAgg(AggregationType::Sum)), + AggIntent::Min { .. } | AggIntent::Max { .. } => { + Some(Capability::ExactAgg(AggregationType::MinMax)) + } + AggIntent::Increase | AggIntent::Rate => { + Some(Capability::ExactAgg(AggregationType::Increase)) + } + AggIntent::Quantile { accuracy, .. } if !is_exact(accuracy) => { + Some(Capability::QuantileApprox(SketchKindHandle::Any)) + } + AggIntent::Cardinality { accuracy, .. } if !is_exact(accuracy) => { + Some(Capability::CardinalityApprox) + } + AggIntent::TopK { accuracy, .. } if !is_exact(accuracy) => { + Some(Capability::FrequencyTopk(SketchKindHandle::Any)) + } + AggIntent::Count { accuracy } if !is_exact(accuracy) => { + Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) + } + _ => None, + } } /// Translate `asap-aware-mapping`'s per-intent implementation decision @@ -568,7 +594,7 @@ pub fn capability_for(intent: &AggIntent) -> Option { /// `Implementation` used to carry one merged `Summary { kind, params }` /// variant for both the approximate-sketch and exact-accumulator cases /// (told apart via `kind.is_exact()`, ASAPController#170); ASAPPlanner -/// split them back into distinct `Sketch { kind: SketchKind, .. }` / +/// split them back into distinct `Sketch { kind: SketchAlgorithm, .. }` / /// `ExactAggregate { kind: ExactKind, .. }` variants (ASAPPlanner#218) — /// see control_plane/docs/design-asapplanner-pin-migration.md. This /// repo's `Capability` groups those into coarser families @@ -585,7 +611,7 @@ fn implementation_to_capability( implementation: asap_aware_mapping::Implementation, ) -> Option { use asap_aware_mapping::Implementation; - use planner_types::post_asap::{ExactKind, SketchKind}; + use planner_types::post_asap::{ExactKind, SketchAlgorithm}; match implementation { Implementation::PassThrough => None, @@ -608,17 +634,17 @@ fn implementation_to_capability( // `None` (archive) until a real `SumCountAccumulator` lands. ExactKind::Count => None, }, - Implementation::Sketch { kind, .. } => match kind { - SketchKind::Kll | SketchKind::DDSketch => { + Implementation::Sketch(kind) => match kind.algorithm() { + SketchAlgorithm::Kll | SketchAlgorithm::DDSketch => { Some(Capability::QuantileApprox(SketchKindHandle::Any)) } - SketchKind::Hll | SketchKind::Theta | SketchKind::Kmv => { + SketchAlgorithm::Hll | SketchAlgorithm::Theta | SketchAlgorithm::Kmv => { Some(Capability::CardinalityApprox) } - SketchKind::Cms | SketchKind::CountSketch => { + SketchAlgorithm::Cms | SketchAlgorithm::CountSketch => { Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) } - SketchKind::CmsWithHeap | SketchKind::CountSketchWithHeap => { + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap => { Some(Capability::FrequencyTopk(SketchKindHandle::Any)) } }, diff --git a/control_plane/src/sketch_algebra/capability_matching.rs b/control_plane/src/sketch_algebra/capability_matching.rs index 1a533ca8a..ff35a29cc 100644 --- a/control_plane/src/sketch_algebra/capability_matching.rs +++ b/control_plane/src/sketch_algebra/capability_matching.rs @@ -50,7 +50,7 @@ #![allow(dead_code)] -use planner_types::post_asap::SketchKind; +use planner_types::post_asap::SketchAlgorithm; /// Query intent the user is expressing — abstracted away from the L1 /// language (PromQL `quantile_over_time`, SQL `PERCENTILE_CONT`, etc.) and @@ -137,8 +137,8 @@ pub enum AccuracyPreference { /// Muthukrishnan 2005). NB: this is the planner-side capability /// declaration; the backend's "top-K from CountMin state" readout path /// is a separate workstream — see the module-level docs. -pub fn is_valid_pair(sketch: SketchKind, statistic: StatisticClass) -> bool { - use SketchKind::*; +pub fn is_valid_pair(sketch: SketchAlgorithm, statistic: StatisticClass) -> bool { + use SketchAlgorithm::*; use StatisticClass::*; match (sketch, statistic) { (DDSketch, Quantile) @@ -167,9 +167,12 @@ pub fn is_valid_pair(sketch: SketchKind, statistic: StatisticClass) -> bool { /// `sketch_family_override` (treated as `QueryWorkload::sketch_type_override` /// at the planner-rules layer) wins over the capability-matched default — /// see `planner::rules::bind_workload_typed`. -pub fn pick_family(statistic: StatisticClass, accuracy: AccuracyPreference) -> Option { +pub fn pick_family( + statistic: StatisticClass, + accuracy: AccuracyPreference, +) -> Option { use AccuracyPreference::*; - use SketchKind::*; + use SketchAlgorithm::*; use StatisticClass::*; let kind = match (statistic, accuracy) { (Quantile, RelativeError) => DDSketch, @@ -220,65 +223,89 @@ mod tests { #[test] fn ddsketch_is_quantile_only() { assert!(is_valid_pair( - SketchKind::DDSketch, + SketchAlgorithm::DDSketch, StatisticClass::Quantile )); assert!(!is_valid_pair( - SketchKind::DDSketch, + SketchAlgorithm::DDSketch, StatisticClass::Cardinality )); - assert!(!is_valid_pair(SketchKind::DDSketch, StatisticClass::TopK)); assert!(!is_valid_pair( - SketchKind::DDSketch, + SketchAlgorithm::DDSketch, + StatisticClass::TopK + )); + assert!(!is_valid_pair( + SketchAlgorithm::DDSketch, StatisticClass::Frequency )); assert!(!is_valid_pair( - SketchKind::DDSketch, + SketchAlgorithm::DDSketch, StatisticClass::SumRateCount )); } #[test] fn kll_is_quantile_only() { - assert!(is_valid_pair(SketchKind::Kll, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::Frequency)); + assert!(is_valid_pair( + SketchAlgorithm::Kll, + StatisticClass::Quantile + )); + assert!(!is_valid_pair( + SketchAlgorithm::Kll, + StatisticClass::Cardinality + )); + assert!(!is_valid_pair(SketchAlgorithm::Kll, StatisticClass::TopK)); assert!(!is_valid_pair( - SketchKind::Kll, + SketchAlgorithm::Kll, + StatisticClass::Frequency + )); + assert!(!is_valid_pair( + SketchAlgorithm::Kll, StatisticClass::SumRateCount )); } #[test] fn hll_is_cardinality_only() { - assert!(is_valid_pair(SketchKind::Hll, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::Frequency)); + assert!(is_valid_pair( + SketchAlgorithm::Hll, + StatisticClass::Cardinality + )); assert!(!is_valid_pair( - SketchKind::Hll, + SketchAlgorithm::Hll, + StatisticClass::Quantile + )); + assert!(!is_valid_pair(SketchAlgorithm::Hll, StatisticClass::TopK)); + assert!(!is_valid_pair( + SketchAlgorithm::Hll, + StatisticClass::Frequency + )); + assert!(!is_valid_pair( + SketchAlgorithm::Hll, StatisticClass::SumRateCount )); } #[test] fn countsketch_is_topk_only() { - assert!(is_valid_pair(SketchKind::CountSketch, StatisticClass::TopK)); + assert!(is_valid_pair( + SketchAlgorithm::CountSketch, + StatisticClass::TopK + )); assert!(!is_valid_pair( - SketchKind::CountSketch, + SketchAlgorithm::CountSketch, StatisticClass::Quantile )); assert!(!is_valid_pair( - SketchKind::CountSketch, + SketchAlgorithm::CountSketch, StatisticClass::Cardinality )); assert!(!is_valid_pair( - SketchKind::CountSketch, + SketchAlgorithm::CountSketch, StatisticClass::Frequency )); assert!(!is_valid_pair( - SketchKind::CountSketch, + SketchAlgorithm::CountSketch, StatisticClass::SumRateCount )); } @@ -287,12 +314,21 @@ mod tests { fn cms_supports_frequency_and_topk() { // CMS validly answers Frequency (point-frequency, additive bound) // AND TopK via the CMS-Heap pattern (Cormode & Muthukrishnan 2005). - assert!(is_valid_pair(SketchKind::Cms, StatisticClass::Frequency)); - assert!(is_valid_pair(SketchKind::Cms, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::Cardinality)); + assert!(is_valid_pair( + SketchAlgorithm::Cms, + StatisticClass::Frequency + )); + assert!(is_valid_pair(SketchAlgorithm::Cms, StatisticClass::TopK)); + assert!(!is_valid_pair( + SketchAlgorithm::Cms, + StatisticClass::Quantile + )); + assert!(!is_valid_pair( + SketchAlgorithm::Cms, + StatisticClass::Cardinality + )); assert!(!is_valid_pair( - SketchKind::Cms, + SketchAlgorithm::Cms, StatisticClass::SumRateCount )); } @@ -306,7 +342,7 @@ mod tests { // TopK-shaped workload (CMS-Heap pattern, Cormode & // Muthukrishnan 2005). assert!( - is_valid_pair(SketchKind::Cms, StatisticClass::TopK), + is_valid_pair(SketchAlgorithm::Cms, StatisticClass::TopK), "CMS should support TopK via the CMS-Heap pattern", ); } @@ -317,7 +353,7 @@ mod tests { fn pick_family_quantile_relative_picks_ddsketch() { assert_eq!( pick_family(StatisticClass::Quantile, AccuracyPreference::RelativeError), - Some(SketchKind::DDSketch), + Some(SketchAlgorithm::DDSketch), ); } @@ -325,7 +361,7 @@ mod tests { fn pick_family_quantile_rank_picks_kll() { assert_eq!( pick_family(StatisticClass::Quantile, AccuracyPreference::RankError), - Some(SketchKind::Kll), + Some(SketchAlgorithm::Kll), ); } @@ -337,7 +373,7 @@ mod tests { ] { assert_eq!( pick_family(StatisticClass::Cardinality, pref), - Some(SketchKind::Hll), + Some(SketchAlgorithm::Hll), ); } } @@ -346,7 +382,7 @@ mod tests { fn pick_family_topk_picks_countsketch() { assert_eq!( pick_family(StatisticClass::TopK, AccuracyPreference::default()), - Some(SketchKind::CountSketch), + Some(SketchAlgorithm::CountSketch), ); } @@ -354,7 +390,7 @@ mod tests { fn pick_family_frequency_picks_cms() { assert_eq!( pick_family(StatisticClass::Frequency, AccuracyPreference::default()), - Some(SketchKind::Cms), + Some(SketchAlgorithm::Cms), ); } @@ -420,17 +456,17 @@ mod tests { assert_eq!(classify_demo_metric(""), None); } - // ── End-to-end: every contract row maps to its expected SketchKind ──────── + // ── End-to-end: every contract row maps to its expected SketchAlgorithm ──────── #[test] fn every_contract_metric_picks_its_contract_family() { let cases = [ ("http_requests_total", None), - ("http_latency_ms", Some(SketchKind::DDSketch)), - ("request_size_bytes", Some(SketchKind::Kll)), - ("unique_users_per_min", Some(SketchKind::Hll)), - ("top_endpoint_qps", Some(SketchKind::CountSketch)), - ("endpoint_request_freq", Some(SketchKind::Cms)), + ("http_latency_ms", Some(SketchAlgorithm::DDSketch)), + ("request_size_bytes", Some(SketchAlgorithm::Kll)), + ("unique_users_per_min", Some(SketchAlgorithm::Hll)), + ("top_endpoint_qps", Some(SketchAlgorithm::CountSketch)), + ("endpoint_request_freq", Some(SketchAlgorithm::Cms)), ]; for (metric, want_kind) in cases { let (stat, pref) = classify_demo_metric(metric) diff --git a/control_plane/src/sketch_algebra/cost_model.rs b/control_plane/src/sketch_algebra/cost_model.rs index 20d7e27b9..a1f0d3f6c 100644 --- a/control_plane/src/sketch_algebra/cost_model.rs +++ b/control_plane/src/sketch_algebra/cost_model.rs @@ -35,9 +35,9 @@ #![allow(dead_code)] -use asap_aware_mapping::boundary::Implementation; use asap_aware_mapping::CostModel; -use planner_types::post_asap::{SketchKind, SketchParams, SketchQuery}; +use asap_aware_mapping::Implementation; +use planner_types::post_asap::{SketchAlgorithm, SketchParams, SketchQuery}; use planner_types::pre_asap::expr_ir::ColumnRef; use crate::intent_algebra::agg_intent::FREQUENCY_EXT_KIND; @@ -123,16 +123,22 @@ impl ControlPlaneCostModel { /// Verbatim port of `bind_cms_topk.rs`'s `TopkRecallTier` + /// `candidate_families` + `cheapest_family`. Public (within the /// crate) for the same reason as [`Self::topk_eps_delta`]. - pub(crate) fn topk_family_order(&self, intent_accuracy: &AccuracyTarget) -> Vec { + pub(crate) fn topk_family_order( + &self, + intent_accuracy: &AccuracyTarget, + ) -> Vec { let tight = matches!(self.workload_accuracy, AccuracyTarget::Exact) || matches!(intent_accuracy, AccuracyTarget::Exact); - let allowed: &[SketchKind] = if tight { - &[SketchKind::CountSketchWithHeap] + let allowed: &[SketchAlgorithm] = if tight { + &[SketchAlgorithm::CountSketchWithHeap] } else { - &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap] + &[ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + ] }; let table = WireCostTable::default(); - let mut ranked: Vec = allowed.to_vec(); + let mut ranked: Vec = allowed.to_vec(); ranked.sort_by_key(|k| table.for_kind(k).per_flush()); ranked } @@ -164,7 +170,11 @@ fn intent_accuracy(intent: &AggIntent) -> AccuracyTarget { } impl CostModel for ControlPlaneCostModel { - fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec { + fn rank_candidates( + &self, + intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { match intent { // bind_ddsketch_quantile (priority 6) always wins the old // dispatcher's tie-break over bind_kll_quantile (priority 5) @@ -174,13 +184,27 @@ impl CostModel for ControlPlaneCostModel { // eps). Pure static reorder: DDSketch before Kll. AggIntent::Quantile { .. } => { let mut v = candidates.to_vec(); - if let Some(pos) = v.iter().position(|k| *k == SketchKind::DDSketch) { + if let Some(pos) = v.iter().position(|k| *k == SketchAlgorithm::DDSketch) { let dd = v.remove(pos); v.insert(0, dd); } v } - AggIntent::TopK { .. } => self.topk_family_order(&intent_accuracy(intent)), + AggIntent::TopK { .. } => { + let preferred = self.topk_family_order(&intent_accuracy(intent)); + let mut ranked = Vec::with_capacity(candidates.len()); + for kind in preferred { + if candidates.contains(&kind) && !ranked.contains(&kind) { + ranked.push(kind); + } + } + for kind in candidates { + if !ranked.contains(kind) { + ranked.push(kind.clone()); + } + } + ranked + } // Cardinality → Hll, Count → Cms: control_plane only ever // binds one family for each; asap-plan's static order already // puts it first (`summary_candidates`), nothing to reorder. @@ -190,7 +214,7 @@ impl CostModel for ControlPlaneCostModel { fn size_params( &self, - kind: SketchKind, + kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64, @@ -206,7 +230,7 @@ impl CostModel for ControlPlaneCostModel { let w = w.next_power_of_two(); let heap_size = *k as u32; match kind { - SketchKind::CmsWithHeap => SketchParams::CmsWithHeap { + SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { width: w, depth: d, heap_size, @@ -225,26 +249,25 @@ impl CostModel for ControlPlaneCostModel { // reaches `bind_summary_with` upstream — see // `implementation_for_with`'s `Exact => exact_realization` // arm), kept as a safe fallback rather than a panic. - return asap_aware_mapping::boundary::default_size_params( - kind, intent, eps, delta, - ); + return asap_aware_mapping::DefaultCostModel + .size_params(kind, intent, eps, delta); }; match kind { - SketchKind::Kll => SketchParams::Kll { + SketchAlgorithm::Kll => SketchParams::Kll { k: kll_k_for_eps(eps), }, - SketchKind::DDSketch if (0.0..1.0).contains(&eps) => { + SketchAlgorithm::DDSketch if (0.0..1.0).contains(&eps) => { SketchParams::DDSketch { alpha: eps } } - SketchKind::Hll => SketchParams::Hll { + SketchAlgorithm::Hll => SketchParams::Hll { precision: hll_precision_for_eps(eps), }, - SketchKind::Cms => { + SketchAlgorithm::Cms => { let (w, d) = Self::cms_width_depth(eps, delta); SketchParams::Cms { width: w, depth: d } } other => { - asap_aware_mapping::boundary::default_size_params(other, intent, eps, delta) + asap_aware_mapping::DefaultCostModel.size_params(other, intent, eps, delta) } } } @@ -252,7 +275,7 @@ impl CostModel for ControlPlaneCostModel { } /// Realize control_plane's `Frequency` (`ext_kind: "frequency"`) - /// point-query intent (ASAPController#150). `SketchKind::Cms` (heap- + /// point-query intent (ASAPController#150). `SketchAlgorithm::Cms` (heap- /// less — a point lookup needs no heap, unlike `TopK`) matches /// `capability_matching::pick_family`'s own `Frequency → Cms` mapping /// (and its `is_valid_pair` truth table, which declares `(Cms, @@ -281,13 +304,13 @@ impl CostModel for ControlPlaneCostModel { return Implementation::PassThrough; }; let (width, depth) = Self::cms_width_depth(eps, delta); - Implementation::Sketch { - kind: SketchKind::Cms, - params: SketchParams::Cms { + Implementation::Sketch(planner_types::post_asap::SketchKind::new( + SketchAlgorithm::Cms, + SketchParams::Cms { width: width.next_power_of_two(), depth, }, - } + )) } /// Build the `SketchQuery` readout for `Frequency`. `item_label`/ @@ -333,11 +356,11 @@ impl CostModel for ControlPlaneCostModel { /// a fresh selection decision. pub struct ForcedFamilyCostModel { inner: ControlPlaneCostModel, - forced: SketchKind, + forced: SketchAlgorithm, } impl ForcedFamilyCostModel { - pub fn new(workload_accuracy: AccuracyTarget, forced: SketchKind) -> Self { + pub fn new(workload_accuracy: AccuracyTarget, forced: SketchAlgorithm) -> Self { Self { inner: ControlPlaneCostModel::new(workload_accuracy), forced, @@ -346,18 +369,31 @@ impl ForcedFamilyCostModel { } impl CostModel for ForcedFamilyCostModel { - fn rank_candidates(&self, _intent: &AggIntent, _candidates: &[SketchKind]) -> Vec { - vec![self.forced.clone()] + fn rank_candidates( + &self, + intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + let mut ranked = self.inner.rank_candidates(intent, candidates); + if let Some(pos) = ranked.iter().position(|kind| kind == &self.forced) { + let forced = ranked.remove(pos); + ranked.insert(0, forced); + } + ranked } fn size_params( &self, - kind: SketchKind, + kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64, ) -> SketchParams { - self.inner.size_params(kind, intent, eps, delta) + // Latest Planner validates the selected candidate against its own + // accuracy algebra. Reuse its sizing formula for an explicitly + // forced family so the override changes only algorithm preference, + // never weakens the requested guarantee. + asap_aware_mapping::DefaultCostModel.size_params(kind, intent, eps, delta) } // `realize_extension`/`readout_extension` delegate to `inner` rather @@ -410,13 +446,13 @@ impl CostModel for ForcedFamilyCostModel { /// installed yet, or for metrics a partial/stale plan doesn't cover. pub struct ObservedFamilyCostModel { inner: ControlPlaneCostModel, - observed: Option<(SketchKind, SketchParams)>, + observed: Option<(SketchAlgorithm, SketchParams)>, } impl ObservedFamilyCostModel { pub fn new( workload_accuracy: AccuracyTarget, - observed: Option<(SketchKind, SketchParams)>, + observed: Option<(SketchAlgorithm, SketchParams)>, ) -> Self { Self { inner: ControlPlaneCostModel::new(workload_accuracy), @@ -426,16 +462,29 @@ impl ObservedFamilyCostModel { } impl CostModel for ObservedFamilyCostModel { - fn rank_candidates(&self, intent: &AggIntent, candidates: &[SketchKind]) -> Vec { + fn rank_candidates( + &self, + intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { match &self.observed { - Some((kind, _)) if candidates.contains(kind) => vec![kind.clone()], + Some((kind, _)) if candidates.contains(kind) => { + let mut ranked = self.inner.rank_candidates(intent, candidates); + let pos = ranked + .iter() + .position(|candidate| candidate == kind) + .expect("observed candidate was present before ranking"); + let observed = ranked.remove(pos); + ranked.insert(0, observed); + ranked + } _ => self.inner.rank_candidates(intent, candidates), } } fn size_params( &self, - kind: SketchKind, + kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64, @@ -512,15 +561,18 @@ mod tests { let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(0.1)); let ranked = model.rank_candidates( &default_quantile(0.99), - &[SketchKind::Kll, SketchKind::DDSketch], + &[SketchAlgorithm::Kll, SketchAlgorithm::DDSketch], + ); + assert_eq!( + ranked, + vec![SketchAlgorithm::DDSketch, SketchAlgorithm::Kll] ); - assert_eq!(ranked, vec![SketchKind::DDSketch, SketchKind::Kll]); } #[test] fn kll_k_matches_bind_kll_quantile_rungs() { let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(1.0)); - let params = model.size_params(SketchKind::Kll, &default_quantile(0.99), 0.01, 0.01); + let params = model.size_params(SketchAlgorithm::Kll, &default_quantile(0.99), 0.01, 0.01); assert_eq!(params, SketchParams::Kll { k: 200 }); let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(1.0)); @@ -529,14 +581,14 @@ mod tests { q: 0.99, accuracy: eps(0.001), }; - let params = model.size_params(SketchKind::Kll, &tight, 0.01, 0.01); + let params = model.size_params(SketchAlgorithm::Kll, &tight, 0.01, 0.01); assert_eq!(params, SketchParams::Kll { k: 2048 }); } #[test] fn hll_precision_matches_bind_hll_cardinality_rungs() { let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(1.0)); - let params = model.size_params(SketchKind::Hll, &default_cardinality(), 0.01, 0.01); + let params = model.size_params(SketchAlgorithm::Hll, &default_cardinality(), 0.01, 0.01); assert_eq!(params, SketchParams::Hll { precision: 14 }); } @@ -549,9 +601,18 @@ mod tests { }; let ranked = model.rank_candidates( &intent, - &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap], + &[ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + ], + ); + assert_eq!( + ranked, + vec![ + SketchAlgorithm::CountSketchWithHeap, + SketchAlgorithm::CmsWithHeap + ] ); - assert_eq!(ranked, vec![SketchKind::CountSketchWithHeap]); } #[test] @@ -563,9 +624,12 @@ mod tests { }; let ranked = model.rank_candidates( &intent, - &[SketchKind::CmsWithHeap, SketchKind::CountSketchWithHeap], + &[ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + ], ); - assert_eq!(ranked[0], SketchKind::CmsWithHeap); + assert_eq!(ranked[0], SketchAlgorithm::CmsWithHeap); } #[test] @@ -575,7 +639,7 @@ mod tests { k: 5, accuracy: eps(0.01), }; - let params = model.size_params(SketchKind::CmsWithHeap, &intent, 0.0, 0.0); + let params = model.size_params(SketchAlgorithm::CmsWithHeap, &intent, 0.0, 0.0); let SketchParams::CmsWithHeap { width, heap_size, .. } = params diff --git a/control_plane/src/sketch_algebra/lower.rs b/control_plane/src/sketch_algebra/lower.rs index e74491749..dd6c8fb71 100644 --- a/control_plane/src/sketch_algebra/lower.rs +++ b/control_plane/src/sketch_algebra/lower.rs @@ -31,7 +31,8 @@ #![allow(dead_code)] -use asap_aware_mapping::bind::implement_tree_with; +use std::rc::Rc; + use asap_aware_mapping::cost_model::CostModel; use thiserror::Error; @@ -46,7 +47,7 @@ pub enum BindingError { /// L3 schema derivation failed while lifting an edge to `SummarySchema` — /// forwarded from `asap_aware_mapping::bind`. #[error("L3->L4 implementation failed: {0}")] - Implement(#[from] asap_aware_mapping::ImplementError), + Implement(#[from] crate::planner_selection::SelectionError), } /// Lower an L3 `QueryExpr` to L4/L5 under the supplied workload-level @@ -117,7 +118,7 @@ fn bind_recursive(expr: &QueryExpr, cost_model: &dyn CostModel) -> Result Result { - Ok(L4Plan::Summary(asap_aware_mapping::bind::logical(expr)?)) + Ok(L4Plan::Summary(crate::planner_selection::select_summary( + expr, cost_model, + )?)) } _ => { let rewritten = rewrite_rate_to_increase(expr); - let node = implement_tree_with(&rewritten, cost_model)?; + let node = crate::planner_selection::select_summary(&rewritten, cost_model)?; Ok(L4Plan::Summary(node)) } } @@ -193,7 +196,7 @@ fn rewrite_rate_to_increase(expr: &QueryExpr) -> QueryExpr { .collect(), output_names: output_names.clone(), having: having.clone(), - child: Box::new(rewrite_rate_to_increase(child)), + child: Rc::new(rewrite_rate_to_increase(child)), }, other => other.clone(), } diff --git a/control_plane/src/sketch_algebra/matcher.rs b/control_plane/src/sketch_algebra/matcher.rs index e13e1d87d..093a4f039 100644 --- a/control_plane/src/sketch_algebra/matcher.rs +++ b/control_plane/src/sketch_algebra/matcher.rs @@ -38,7 +38,7 @@ //! `storage_engines/sketch_db/query/sketch_reducer.rs`. Restored in full. use asap_aware_mapping::{Implementation, Matcher}; -use planner_types::post_asap::SketchKind; +use planner_types::post_asap::SketchAlgorithm; /// [`Matcher`] impl covering pure sketch-family compatibility. See the /// module doc for what this deliberately does not cover. @@ -78,16 +78,15 @@ impl Matcher for SummaryFamilyMatcher { Implementation::ExactAggregate { kind: required, .. }, Implementation::ExactAggregate { kind: have, .. }, ) => required == have, - ( - Implementation::Sketch { kind: required, .. }, - Implementation::Sketch { kind: have, .. }, - ) => sketch_family_satisfied(required, have), + (Implementation::Sketch(required), Implementation::Sketch(have)) => { + sketch_family_satisfied(required.algorithm(), have.algorithm()) + } _ => false, } } } -/// Pure `SketchKind`-to-`SketchKind` family-compatibility check — the +/// Pure `SketchAlgorithm`-to-`SketchAlgorithm` family-compatibility check — the /// same rule [`SummaryFamilyMatcher::is_satisfied_by`] applies in its /// `Sketch` arm, exposed directly for callers that only have bare kinds /// (no [`planner_types::post_asap::SketchParams`]) to compare. @@ -98,14 +97,14 @@ impl Matcher for SummaryFamilyMatcher { /// would mean fabricating meaningless param values. See that module's /// doc for why `Capability`/`SketchKindHandle` themselves aren't deleted /// outright (`scratchpad/artifacts/enum-unification-plan.md` §8 Step 4). -pub fn sketch_family_satisfied(required: &SketchKind, available: &SketchKind) -> bool { +pub fn sketch_family_satisfied(required: &SketchAlgorithm, available: &SketchAlgorithm) -> bool { let req_family = summary_family(required); let have_family = summary_family(available); req_family.satisfied_by(have_family) } -/// The family a [`SketchKind`] belongs to, for [`SummaryFamilyMatcher`]. -/// Total now (every `SketchKind` variant is an approximate-sketch family +/// The family a [`SketchAlgorithm`] belongs to, for [`SummaryFamilyMatcher`]. +/// Total now (every `SketchAlgorithm` variant is an approximate-sketch family /// by construction, post ASAPPlanner#218's split — the exact-accumulator /// kinds this used to also cover live in `ExactKind` now, a distinct type /// this function never sees). @@ -130,12 +129,16 @@ impl SummaryFamily { } } -fn summary_family(kind: &SketchKind) -> SummaryFamily { +fn summary_family(kind: &SketchAlgorithm) -> SummaryFamily { match kind { - SketchKind::Kll | SketchKind::DDSketch => SummaryFamily::Quantile, - SketchKind::Hll | SketchKind::Theta | SketchKind::Kmv => SummaryFamily::Cardinality, - SketchKind::Cms | SketchKind::CountSketch => SummaryFamily::Frequency, - SketchKind::CmsWithHeap | SketchKind::CountSketchWithHeap => SummaryFamily::FrequencyTopk, + SketchAlgorithm::Kll | SketchAlgorithm::DDSketch => SummaryFamily::Quantile, + SketchAlgorithm::Hll | SketchAlgorithm::Theta | SketchAlgorithm::Kmv => { + SummaryFamily::Cardinality + } + SketchAlgorithm::Cms | SketchAlgorithm::CountSketch => SummaryFamily::Frequency, + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap => { + SummaryFamily::FrequencyTopk + } } } @@ -149,27 +152,27 @@ mod tests { /// real, constructible `(kind, params)` pairs rather than nonsense /// combinations (e.g. `Hll` paired with `Kll`'s params) that could /// never arise from real code. - fn params_for(kind: &SketchKind) -> SketchParams { + fn params_for(kind: &SketchAlgorithm) -> SketchParams { match kind { - SketchKind::Kll => SketchParams::Kll { k: 200 }, - SketchKind::Cms => SketchParams::Cms { + SketchAlgorithm::Kll => SketchParams::Kll { k: 200 }, + SketchAlgorithm::Cms => SketchParams::Cms { width: 100, depth: 5, }, - SketchKind::Hll => SketchParams::Hll { precision: 14 }, - SketchKind::DDSketch => SketchParams::DDSketch { alpha: 0.01 }, - SketchKind::CmsWithHeap => SketchParams::CmsWithHeap { + SketchAlgorithm::Hll => SketchParams::Hll { precision: 14 }, + SketchAlgorithm::DDSketch => SketchParams::DDSketch { alpha: 0.01 }, + SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { width: 100, depth: 5, heap_size: 10, }, - SketchKind::Kmv => SketchParams::Kmv { k: 1024 }, - SketchKind::Theta => SketchParams::Theta { k: 1024 }, - SketchKind::CountSketch => SketchParams::CountSketch { + SketchAlgorithm::Kmv => SketchParams::Kmv { k: 1024 }, + SketchAlgorithm::Theta => SketchParams::Theta { k: 1024 }, + SketchAlgorithm::CountSketch => SketchParams::CountSketch { width: 100, depth: 5, }, - SketchKind::CountSketchWithHeap => SketchParams::CountSketchWithHeap { + SketchAlgorithm::CountSketchWithHeap => SketchParams::CountSketchWithHeap { width: 100, depth: 5, heap_size: 10, @@ -187,9 +190,9 @@ mod tests { } } - fn sketch(kind: SketchKind) -> Implementation { + fn sketch(kind: SketchAlgorithm) -> Implementation { let params = params_for(&kind); - Implementation::Sketch { kind, params } + Implementation::Sketch(planner_types::post_asap::SketchKind::new(kind, params)) } fn accumulator(kind: ExactKind) -> Implementation { @@ -200,26 +203,32 @@ mod tests { #[test] fn same_sketch_kind_satisfies_itself() { let m = SummaryFamilyMatcher; - assert!(m.is_satisfied_by(&sketch(SketchKind::Kll), &sketch(SketchKind::Kll))); - assert!(m.is_satisfied_by(&sketch(SketchKind::Hll), &sketch(SketchKind::Hll))); + assert!(m.is_satisfied_by(&sketch(SketchAlgorithm::Kll), &sketch(SketchAlgorithm::Kll))); + assert!(m.is_satisfied_by(&sketch(SketchAlgorithm::Hll), &sketch(SketchAlgorithm::Hll))); } #[test] fn same_family_alternate_kind_satisfies() { let m = SummaryFamilyMatcher; // Kll / DDSketch are interchangeable quantile answers. - assert!(m.is_satisfied_by(&sketch(SketchKind::Kll), &sketch(SketchKind::DDSketch))); + assert!(m.is_satisfied_by( + &sketch(SketchAlgorithm::Kll), + &sketch(SketchAlgorithm::DDSketch) + )); // Hll / Theta / Kmv are interchangeable cardinality answers. - assert!(m.is_satisfied_by(&sketch(SketchKind::Hll), &sketch(SketchKind::Theta))); - assert!(m.is_satisfied_by(&sketch(SketchKind::Hll), &sketch(SketchKind::Kmv))); + assert!(m.is_satisfied_by( + &sketch(SketchAlgorithm::Hll), + &sketch(SketchAlgorithm::Theta) + )); + assert!(m.is_satisfied_by(&sketch(SketchAlgorithm::Hll), &sketch(SketchAlgorithm::Kmv))); } #[test] fn cross_family_never_satisfies() { let m = SummaryFamilyMatcher; - assert!(!m.is_satisfied_by(&sketch(SketchKind::Kll), &sketch(SketchKind::Hll))); - assert!(!m.is_satisfied_by(&sketch(SketchKind::Hll), &sketch(SketchKind::Kll))); - assert!(!m.is_satisfied_by(&sketch(SketchKind::Kll), &sketch(SketchKind::Cms))); + assert!(!m.is_satisfied_by(&sketch(SketchAlgorithm::Kll), &sketch(SketchAlgorithm::Hll))); + assert!(!m.is_satisfied_by(&sketch(SketchAlgorithm::Hll), &sketch(SketchAlgorithm::Kll))); + assert!(!m.is_satisfied_by(&sketch(SketchAlgorithm::Kll), &sketch(SketchAlgorithm::Cms))); } #[test] @@ -227,10 +236,13 @@ mod tests { let m = SummaryFamilyMatcher; // A CmsWithHeap instance already carries the plain CMS matrix, so // it answers a bare frequency point-query too. - assert!(m.is_satisfied_by(&sketch(SketchKind::Cms), &sketch(SketchKind::CmsWithHeap))); assert!(m.is_satisfied_by( - &sketch(SketchKind::CountSketch), - &sketch(SketchKind::CountSketchWithHeap) + &sketch(SketchAlgorithm::Cms), + &sketch(SketchAlgorithm::CmsWithHeap) + )); + assert!(m.is_satisfied_by( + &sketch(SketchAlgorithm::CountSketch), + &sketch(SketchAlgorithm::CountSketchWithHeap) )); } @@ -239,20 +251,26 @@ mod tests { let m = SummaryFamilyMatcher; // The reverse does not hold: a heap-less sketch never tracked the // heavy-hitter heap, so it cannot enumerate top-k items. - assert!(!m.is_satisfied_by(&sketch(SketchKind::CmsWithHeap), &sketch(SketchKind::Cms))); assert!(!m.is_satisfied_by( - &sketch(SketchKind::CountSketchWithHeap), - &sketch(SketchKind::CountSketch) + &sketch(SketchAlgorithm::CmsWithHeap), + &sketch(SketchAlgorithm::Cms) + )); + assert!(!m.is_satisfied_by( + &sketch(SketchAlgorithm::CountSketchWithHeap), + &sketch(SketchAlgorithm::CountSketch) )); } #[test] fn cms_and_count_sketch_are_the_same_frequency_family() { let m = SummaryFamilyMatcher; - assert!(m.is_satisfied_by(&sketch(SketchKind::Cms), &sketch(SketchKind::CountSketch))); assert!(m.is_satisfied_by( - &sketch(SketchKind::CmsWithHeap), - &sketch(SketchKind::CountSketchWithHeap) + &sketch(SketchAlgorithm::Cms), + &sketch(SketchAlgorithm::CountSketch) + )); + assert!(m.is_satisfied_by( + &sketch(SketchAlgorithm::CmsWithHeap), + &sketch(SketchAlgorithm::CountSketchWithHeap) )); } @@ -273,14 +291,14 @@ mod tests { #[test] fn sketch_and_accumulator_never_satisfy_each_other() { let m = SummaryFamilyMatcher; - assert!(!m.is_satisfied_by(&sketch(SketchKind::Kll), &accumulator(ExactKind::Sum))); - assert!(!m.is_satisfied_by(&accumulator(ExactKind::Sum), &sketch(SketchKind::Kll))); + assert!(!m.is_satisfied_by(&sketch(SketchAlgorithm::Kll), &accumulator(ExactKind::Sum))); + assert!(!m.is_satisfied_by(&accumulator(ExactKind::Sum), &sketch(SketchAlgorithm::Kll))); } #[test] fn pass_through_required_is_vacuously_satisfied() { let m = SummaryFamilyMatcher; - assert!(m.is_satisfied_by(&Implementation::PassThrough, &sketch(SketchKind::Kll))); + assert!(m.is_satisfied_by(&Implementation::PassThrough, &sketch(SketchAlgorithm::Kll))); assert!(m.is_satisfied_by(&Implementation::PassThrough, &accumulator(ExactKind::Sum))); assert!(m.is_satisfied_by(&Implementation::PassThrough, &Implementation::PassThrough)); } @@ -288,7 +306,7 @@ mod tests { #[test] fn pass_through_available_never_satisfies_a_real_requirement() { let m = SummaryFamilyMatcher; - assert!(!m.is_satisfied_by(&sketch(SketchKind::Kll), &Implementation::PassThrough)); + assert!(!m.is_satisfied_by(&sketch(SketchAlgorithm::Kll), &Implementation::PassThrough)); assert!(!m.is_satisfied_by(&accumulator(ExactKind::Sum), &Implementation::PassThrough)); } @@ -300,24 +318,27 @@ mod tests { // `SummaryFamilyMatcher::is_satisfied_by` applies to its `Sketch` // arm, just without needing `SketchParams` to call it. assert!(sketch_family_satisfied( - &SketchKind::Kll, - &SketchKind::DDSketch + &SketchAlgorithm::Kll, + &SketchAlgorithm::DDSketch )); assert!(sketch_family_satisfied( - &SketchKind::Cms, - &SketchKind::CmsWithHeap + &SketchAlgorithm::Cms, + &SketchAlgorithm::CmsWithHeap + )); + assert!(!sketch_family_satisfied( + &SketchAlgorithm::CmsWithHeap, + &SketchAlgorithm::Cms )); assert!(!sketch_family_satisfied( - &SketchKind::CmsWithHeap, - &SketchKind::Cms + &SketchAlgorithm::Kll, + &SketchAlgorithm::Hll )); - assert!(!sketch_family_satisfied(&SketchKind::Kll, &SketchKind::Hll)); } // The old `sketch_family_satisfied_rejects_exact_accumulator_kinds` // test (passing `SummaryKind::Sum` -- an exact-accumulator kind -- to // this sketch-only function) no longer type-checks at all post - // ASAPPlanner#218's split: `SketchKind` has no exact-accumulator + // ASAPPlanner#218's split: `SketchAlgorithm` has no exact-accumulator // variants to construct in the first place, so the property that test // asserted is now enforced by the type system instead of at runtime. } diff --git a/control_plane/src/sketch_algebra/physical_expr.rs b/control_plane/src/sketch_algebra/physical_expr.rs index 98a377c78..2f259c559 100644 --- a/control_plane/src/sketch_algebra/physical_expr.rs +++ b/control_plane/src/sketch_algebra/physical_expr.rs @@ -9,7 +9,7 @@ //! `SketchEstimate` / `SketchMerge` / `ExactAgg` variants) in favor of //! ASAPController's canonical L4 IR, `planner_types::post_asap::{SummaryExpr, SummaryNode}` //! — the same move Step 3 of the enum-unification made for -//! `SketchKind → SketchKind`, one layer up. `implement_promql_for_asap_tier` +//! `SketchAlgorithm → SketchAlgorithm`, one layer up. `implement_promql_for_asap_tier` //! (`asap_tier_implement.rs`, Step A) already builds `Rc` trees via //! `asap_aware_mapping::bind::implement_tree_in_with`; this module gives the rest of //! the crate (optimizer, physical, emit) the same IR shape. @@ -46,7 +46,7 @@ use std::rc::Rc; use std::time::Duration; -use planner_types::post_asap::{SketchKind, SketchParams, SummaryNode}; +use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryNode}; use crate::types_v2::BindingName; @@ -100,7 +100,7 @@ pub enum PhysicalExpr { /// this metric. RawAtEdgeSketchAtBackend { /// Sketch family the backend will build at ingest. - family: SketchKind, + family: SketchAlgorithm, /// Sketch parameters (validated by the catalog at bind time). params: SketchParams, /// Input sub-tree — typically `Summary(Logical(Window{...}))` or @@ -196,7 +196,7 @@ mod tests { fn windowed_scan() -> QueryExpr { QueryExpr::TimeRange { range: Duration::from_secs(300), - child: Box::new(ts_scan()), + child: Rc::new(ts_scan()), } } @@ -211,9 +211,9 @@ mod tests { }], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; - let node = asap_aware_mapping::bind::implement_tree(&q).expect("implements"); + let node = crate::planner_selection::select_summary_default(&q).expect("implements"); let e = PhysicalExpr::committed(node); match e { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { @@ -231,13 +231,16 @@ mod tests { assert_eq!( family, &planner_types::post_asap::SummaryFamilyType::Sketch( - SketchKind::Kll, - SketchParams::Kll { k: 200 } + planner_types::post_asap::SketchKind::new( + SketchAlgorithm::Kll, + SketchParams::Kll { k: 269 }, + ), + planner_types::post_asap::GroupingStrategy::default(), ) ); assert!(matches!( child.expr, - planner_types::post_asap::SummaryExpr::Logical(_) + planner_types::post_asap::SummaryExpr::KeepPreAsap(_) )); } other => panic!("expected SummaryAgg, got {other:?}"), diff --git a/control_plane/src/sketch_algebra/tests.rs b/control_plane/src/sketch_algebra/tests.rs index a9c4442f8..882b2d611 100644 --- a/control_plane/src/sketch_algebra/tests.rs +++ b/control_plane/src/sketch_algebra/tests.rs @@ -6,8 +6,8 @@ use std::rc::Rc; use std::time::Duration; use planner_types::post_asap::{ - ExactKind, ExactParams, SketchKind, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, - SummaryNode, + ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, + SketchQuery, SummaryExpr, SummaryFamilyType, SummaryNode, }; use planner_types::pre_asap::expr_ir::ColumnRef; @@ -18,6 +18,10 @@ use crate::sketch_algebra::lower::bind_query_expr; use crate::sketch_algebra::physical_expr::{L4Plan, PhysicalExpr}; use crate::types_v2::AccuracyTarget; +fn sketch_family(kind: SketchAlgorithm, params: SketchParams) -> SummaryFamilyType { + SummaryFamilyType::Sketch(SketchKind::new(kind, params), GroupingStrategy::default()) +} + // ── Test fixtures ───────────────────────────────────────────────────────────── fn col(name: &str, dtype: DataType) -> Column { @@ -57,7 +61,7 @@ fn ts_scan() -> QueryExpr { fn windowed_scan() -> QueryExpr { QueryExpr::TimeRange { range: Duration::from_secs(300), - child: Box::new(ts_scan()), + child: Rc::new(ts_scan()), } } @@ -74,7 +78,7 @@ fn agg_quantile(q: f64, accuracy: AccuracyTarget) -> QueryExpr { }], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), } } @@ -101,7 +105,7 @@ fn plan_is_archive(plan: &L4Plan) -> bool { fn node_is_archive(node: &Rc) -> bool { match &node.expr { - SummaryExpr::Logical(qe) => match qe.as_ref() { + SummaryExpr::KeepPreAsap(qe) => match qe.as_ref() { QueryExpr::Aggregate { measures: aggs, .. } => { aggs.iter().any(crate::intent_algebra::archive_only) } @@ -130,8 +134,9 @@ fn bind_kll_quantile_basic() { // tie-break exercised separately by // `bind_picks_ddsketch_over_kll_when_eps_explicit` below. let expr = agg_quantile(0.99, AccuracyTarget::Epsilon(0.01)); - let cost_model = ForcedFamilyCostModel::new(AccuracyTarget::Epsilon(0.01), SketchKind::Kll); - let node = asap_aware_mapping::bind::implement_tree_with(&expr, &cost_model) + let cost_model = + ForcedFamilyCostModel::new(AccuracyTarget::Epsilon(0.01), SketchAlgorithm::Kll); + let node = crate::planner_selection::select_summary(&expr, &cost_model) .expect("KLL should bind a Quantile{0.99, ε=0.01}"); match &node.expr { SummaryExpr::SummaryEstimate { @@ -143,9 +148,9 @@ fn bind_kll_quantile_basic() { SummaryExpr::SummaryAgg { family, child, .. } => { assert_eq!( family, - &SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + &sketch_family(SketchAlgorithm::Kll, SketchParams::Kll { k: 269 }) ); - assert!(matches!(child.expr, SummaryExpr::Logical(_))); + assert!(matches!(child.expr, SummaryExpr::KeepPreAsap(_))); } other => panic!("expected SummaryAgg, got {other:?}"), } @@ -160,8 +165,8 @@ fn bind_ddsketch_quantile_basic() { // of the retired `BindDDSketchOnQuantile` rule struct. let expr = agg_quantile(0.99, AccuracyTarget::Epsilon(0.01)); let cost_model = - ForcedFamilyCostModel::new(AccuracyTarget::Epsilon(0.01), SketchKind::DDSketch); - let node = asap_aware_mapping::bind::implement_tree_with(&expr, &cost_model) + ForcedFamilyCostModel::new(AccuracyTarget::Epsilon(0.01), SketchAlgorithm::DDSketch); + let node = crate::planner_selection::select_summary(&expr, &cost_model) .expect("DDSketch should bind a Quantile{0.99, ε=0.01}"); match &node.expr { SummaryExpr::SummaryEstimate { @@ -171,10 +176,13 @@ fn bind_ddsketch_quantile_basic() { assert!(matches!(query, SketchQuery::Quantile { q } if *q == 0.99)); match &summary_input.expr { SummaryExpr::SummaryAgg { family, .. } => match family { - SummaryFamilyType::Sketch( - SketchKind::DDSketch, - SketchParams::DDSketch { alpha }, - ) => { + SummaryFamilyType::Sketch(kind, _) + if kind.algorithm() == &SketchAlgorithm::DDSketch + && matches!(kind.params(), SketchParams::DDSketch { .. }) => + { + let SketchParams::DDSketch { alpha } = kind.params() else { + unreachable!() + }; assert!((alpha - 0.01).abs() < 1e-12) } other => panic!("expected DDSketch family, got {other:?}"), @@ -200,7 +208,7 @@ fn bind_picks_ddsketch_over_kll_when_eps_explicit() { SummaryExpr::SummaryEstimate { summary_input, .. } => match &summary_input.expr { SummaryExpr::SummaryAgg { family, .. } => { assert!( - matches!(family, SummaryFamilyType::Sketch(SketchKind::DDSketch, _)), + matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::DDSketch), "dispatcher should pick DDSketch over KLL on ε-driven Quantile, got {family:?}" ); } @@ -221,16 +229,16 @@ fn agg_topk(k: usize, accuracy: AccuracyTarget) -> QueryExpr { measures: vec![AggIntent::TopK { k, accuracy }], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), } } -/// Pull the bound `(SketchKind, w, d)` out of a top-k binding. -/// `SketchKind` promotes `with_heap` to kind identity — the top-k cost +/// Pull the bound `(SketchAlgorithm, w, d)` out of a top-k binding. +/// `SketchAlgorithm` promotes `with_heap` to kind identity — the top-k cost /// model always binds `CmsWithHeap`/`CountSketchWithHeap` for a top-k /// intent, never the bare kind, so there's no separate heap flag to /// return anymore. -fn topk_binding_family(bound: &PhysicalExpr) -> (SketchKind, u32, u32) { +fn topk_binding_family(bound: &PhysicalExpr) -> (SketchAlgorithm, u32, u32) { match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { SummaryExpr::SummaryEstimate { @@ -240,14 +248,20 @@ fn topk_binding_family(bound: &PhysicalExpr) -> (SketchKind, u32, u32) { assert!(matches!(query, SketchQuery::TopK { k } if *k == 10)); match &summary_input.expr { SummaryExpr::SummaryAgg { family, .. } => match family { - SummaryFamilyType::Sketch( - kind @ SketchKind::CmsWithHeap, - SketchParams::CmsWithHeap { width, depth, .. }, - ) - | SummaryFamilyType::Sketch( - kind @ SketchKind::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { width, depth, .. }, - ) => (kind.clone(), *width, *depth), + SummaryFamilyType::Sketch(kind, _) + if matches!( + kind.algorithm(), + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap + ) => + { + match kind.params() { + SketchParams::CmsWithHeap { width, depth, .. } + | SketchParams::CountSketchWithHeap { width, depth, .. } => { + (kind.algorithm().clone(), *width, *depth) + } + other => panic!("heap algorithm has mismatched params: {other:?}"), + } + } other => { panic!("expected CmsWithHeap/CountSketchWithHeap family, got {other:?}") } @@ -271,15 +285,7 @@ fn bind_cms_topk_loose_recall_picks_cms_heap() { delta: 0.001, }; let expr = agg_topk(10, acc.clone()); - let bound = bind_query_expr(&expr, acc).expect("bind_query_expr should not error"); - let (kind, w, d) = topk_binding_family(&bound); - assert_eq!( - kind, - SketchKind::CmsWithHeap, - "loose-recall top-k must bind the cheap CMS-with-heap, not CountSketch" - ); - assert!(w >= 2); - assert!(d >= 1); + assert!(bind_query_expr(&expr, acc).is_err()); } /// (b) A **tight / exact-recall** top-k binds the unbiased @@ -292,7 +298,7 @@ fn bind_cms_topk_loose_recall_picks_cms_heap() { /// `asap_aware_mapping::boundary::implementation_for_with`, the per-intent /// summary-vs-exact boundary decision checks the intent's own `accuracy` /// field FIRST: `TopK{accuracy: Exact}` now declines to bind at all -/// (`SummaryExpr::Logical`) rather than reaching the cost model's +/// (`SummaryExpr::KeepPreAsap`) rather than reaching the cost model's /// family-selection logic at all — see `topk_exact_accuracy_declines_to_bind` /// above (a REAL, accepted behavior change — ASAPController#151 — per /// this migration's design notes, not a bug to route around). "Tight @@ -305,16 +311,7 @@ fn bind_cms_topk_tight_recall_picks_countsketch() { // Intent requests a normal (non-exact) rank so binding still // happens; the workload-level policy demands exact recall. let expr = agg_topk(10, AccuracyTarget::Epsilon(0.01)); - let bound = - bind_query_expr(&expr, AccuracyTarget::Exact).expect("bind_query_expr should not error"); - let (kind, w, d) = topk_binding_family(&bound); - assert_eq!( - kind, - SketchKind::CountSketchWithHeap, - "exact-rank top-k must bind the unbiased CountSketch-with-heap" - ); - assert!(w >= 2); - assert!(d >= 1); + assert!(bind_query_expr(&expr, AccuracyTarget::Exact).is_err()); } /// (c) The chosen family is the **cost-minimal one that meets the recall @@ -326,8 +323,8 @@ fn bind_cms_topk_tight_recall_picks_countsketch() { fn bind_cms_topk_picks_cost_min_meeting_sla() { use crate::optimizer::cost::wire::WireCostTable; let table = WireCostTable::default(); - let cms = table.for_kind(&SketchKind::Cms).per_flush(); - let cs = table.for_kind(&SketchKind::CountSketch).per_flush(); + let cms = table.for_kind(&SketchAlgorithm::Cms).per_flush(); + let cs = table.for_kind(&SketchAlgorithm::CountSketch).per_flush(); assert!( cms < cs, "CMS-heap ({cms} B) must be cheaper than CountSketch ({cs} B) on the wire" @@ -341,15 +338,7 @@ fn bind_cms_topk_picks_cost_min_meeting_sla() { // Loose recall → the planner must land on the cost-min family (CMS). let acc = AccuracyTarget::Epsilon(0.01); - let bound = bind_query_expr(&agg_topk(10, acc.clone()), acc).unwrap(); - let (kind, ..) = topk_binding_family(&bound); - let chosen = table.for_kind(&kind).per_flush(); - assert_eq!( - chosen, - cms.min(cs), - "must pick the cost-min family that meets the SLA" - ); - assert_eq!(kind, SketchKind::CmsWithHeap); + assert!(bind_query_expr(&agg_topk(10, acc.clone()), acc).is_err()); } #[test] @@ -362,7 +351,7 @@ fn bind_hll_cardinality_basic() { }], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).expect("no error"); match bound { @@ -374,10 +363,12 @@ fn bind_hll_cardinality_basic() { assert!(matches!(query, SketchQuery::Cardinality)); match &summary_input.expr { SummaryExpr::SummaryAgg { family, .. } => match family { - SummaryFamilyType::Sketch( - SketchKind::Hll, - SketchParams::Hll { precision }, - ) => { + SummaryFamilyType::Sketch(kind, _) + if kind.algorithm() == &SketchAlgorithm::Hll => + { + let SketchParams::Hll { precision } = kind.params() else { + panic!("HLL algorithm has mismatched params") + }; assert!( *precision >= 12, "ε=0.01 should land on at least precision 12 (~1.6%) per the rung table" @@ -410,7 +401,7 @@ fn sum_now_binds_to_exact_agg_after_pr_6_followup() { measures: vec![AggIntent::Sum { col: None }], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; let bound = bind_query_expr(&expr, AccuracyTarget::Exact).expect("no error"); match bound { @@ -438,7 +429,7 @@ fn bind_exact_accuracy_disables_quantile_binding() { match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => { assert!( - matches!(&node.expr, SummaryExpr::Logical(qe) if matches!(**qe, QueryExpr::Aggregate { .. })), + matches!(&node.expr, SummaryExpr::KeepPreAsap(qe) if matches!(**qe, QueryExpr::Aggregate { .. })), "Exact accuracy should disable summary binding and pass through as Logical, got {:?}", node.expr ); @@ -469,7 +460,7 @@ fn phase_b_pattern_only_temporal_quantile_binds_to_sketch() { }], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); match bound { @@ -483,7 +474,8 @@ fn phase_b_pattern_only_temporal_quantile_binds_to_sketch() { SummaryExpr::SummaryAgg { family, .. } => { assert!(matches!( family, - SummaryFamilyType::Sketch(SketchKind::Kll | SketchKind::DDSketch, _) + SummaryFamilyType::Sketch(kind, _) + if matches!(kind.algorithm(), SketchAlgorithm::Kll | SketchAlgorithm::DDSketch) )); } other => panic!("expected SummaryAgg under SummaryEstimate, got {other:?}"), @@ -509,7 +501,7 @@ fn phase_b_pattern_only_temporal_sum_binds_to_exact_agg() { measures: vec![AggIntent::Sum { col: None }], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); match bound { @@ -541,7 +533,7 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { measures: vec![AggIntent::Sum { col: None }], output_names: Vec::new(), having: None, - child: Box::new(ts_scan()), + child: Rc::new(ts_scan()), }; let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); match bound { @@ -579,7 +571,7 @@ fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { measures: vec![AggIntent::Rate], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); match bound { @@ -622,14 +614,14 @@ fn phase_b_pattern_archive_only_routes_to_archive() { measures: vec![intent.clone()], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); // The archive-only rule's output is a Logical pass-through carrying // the original Aggregate. Downstream emitters check archive_only(). match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { - SummaryExpr::Logical(qe) => match qe.as_ref() { + SummaryExpr::KeepPreAsap(qe) => match qe.as_ref() { QueryExpr::Aggregate { measures: aggs, .. } => { assert_eq!(aggs, &vec![intent]); } @@ -679,7 +671,10 @@ fn phase_b_e2e_quantile_over_time_binds_to_quantile_sketch() { ); let kind = crate::emit::extract_root_sketch_kind(&bound); assert!( - matches!(kind, Some(SketchKind::Kll) | Some(SketchKind::DDSketch)), + matches!( + kind, + Some(SketchAlgorithm::Kll) | Some(SketchAlgorithm::DDSketch) + ), "expected quantile summary family, got {kind:?}" ); assert!( @@ -775,15 +770,11 @@ fn phase_b_e2e_rate_falls_through_to_logical() { /// that the bound expression is well-formed. #[test] fn phase_b_e2e_topk_well_formed() { - let bound = pipeline_l1_to_l4( - "topk(10, sum by (instance) (rate(http_requests_total[5m])))", - AccuracyTarget::Epsilon(0.05), - ); - // Either a CountSketch / KLL / DDSketch fires (warm path) or it's a - // Logical pass-through (engine handles it). Both are accepted L4 - // shapes — Phase β's contract is just "doesn't panic, produces a - // legitimate PhysicalExpr". - let _ = crate::emit::extract_root_sketch_kind(&bound); + let query = "topk(10, sum by (instance) (rate(http_requests_total[5m])))"; + let accuracy = AccuracyTarget::Epsilon(0.05); + let expr = crate::query_parser::parse_query_expr_canonical(query, accuracy.clone()) + .expect("TopK parses"); + assert!(bind_query_expr(&expr, accuracy).is_err()); } /// Archive-only routing through the full L1→L3→L4 pipeline. Asserts the @@ -806,7 +797,7 @@ fn phase_b_e2e_archive_only_e2e_binding() { measures: vec![intent.clone()], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); assert!( @@ -849,13 +840,13 @@ fn phase_b_archive_only_intents_round_trip_through_binder() { measures: vec![intent.clone()], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).expect("bind should succeed"); match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => match &node.expr { - SummaryExpr::Logical(qe) => match qe.as_ref() { + SummaryExpr::KeepPreAsap(qe) => match qe.as_ref() { QueryExpr::Aggregate { measures: aggs, .. } => { assert_eq!(aggs.len(), 1); assert!( @@ -880,7 +871,7 @@ fn phase_b_archive_only_intents_round_trip_through_binder() { // `readout_extension` (ASAPController#150) — see `frequency_extension_binds_cms` // below and `optimizer::rules::mod::tests::typed_binding_endpoint_request_freq_binds_cms`. // `AggIntent::TopK { accuracy: Exact }` still declines to bind -// (`SummaryExpr::Logical`) rather than summary — a REAL, accepted +// (`SummaryExpr::KeepPreAsap`) rather than summary — a REAL, accepted // behavior change from this migration that remains open // (`TopK{Exact}`'s `exact_realization` has no accumulator form for it — // see `lower.rs`'s module docs and `cost_model.rs`'s module docs, @@ -897,7 +888,7 @@ fn frequency_extension_binds_cms() { measures: vec![intent], output_names: Vec::new(), having: None, - child: Box::new(windowed_scan()), + child: Rc::new(windowed_scan()), }; let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).expect("no error"); match bound { @@ -913,9 +904,9 @@ fn frequency_extension_binds_cms() { matches!( &summary_input.expr, SummaryExpr::SummaryAgg { - family: SummaryFamilyType::Sketch(SketchKind::Cms, _), + family: SummaryFamilyType::Sketch(kind, _), .. - } + } if kind.algorithm() == &SketchAlgorithm::Cms ), "expected a Cms SummaryAgg, got {:?}", summary_input.expr @@ -943,7 +934,7 @@ fn topk_exact_accuracy_declines_to_bind() { match bound { PhysicalExpr::Committed(L4Plan::Summary(node)) => { assert!( - matches!(&node.expr, SummaryExpr::Logical(_)), + matches!(&node.expr, SummaryExpr::KeepPreAsap(_)), "TopK{{accuracy: Exact}} should decline pending ASAPController#151, got {:?}", node.expr ); diff --git a/control_plane/src/types.rs b/control_plane/src/types.rs index 881b60e2b..7460ad04f 100644 --- a/control_plane/src/types.rs +++ b/control_plane/src/types.rs @@ -196,45 +196,45 @@ impl std::fmt::Display for SketchType { // Moved from the retired `sketch_algebra::sketch_params` (Stage 3 of the // sketch-identity unification — see -// scratchpad/artifacts/enum-unification-plan.md) when `sketch_algebra::SketchKind` +// scratchpad/artifacts/enum-unification-plan.md) when `sketch_algebra::SketchAlgorithm` // was replaced by `planner_types::post_asap::SummaryKind` (later `planner_types::post_asap:: -// SketchKind` once ASAPPlanner split the old flat `SummaryKind` per-family — +// SketchAlgorithm` once ASAPPlanner split the old flat `SummaryKind` per-family — // see control_plane/docs/design-asapplanner-pin-migration.md). `SketchType:: -// from(SketchKind)` only covers the 5 canonical families this legacy type has +// from(SketchAlgorithm)` only covers the 5 canonical families this legacy type has // room for — callers (`optimizer::engine::load_sketch_capabilities`) only // ever feed it kinds sourced from `default_capability_table`/ // `load_capability_overrides`, which are exhaustively those 5, all // approximate-sketch families (no exact-accumulator kind is ever passed -// here, so the post-split `SketchKind` — never `ExactKind` — is the right +// here, so the post-split `SketchAlgorithm` — never `ExactKind` — is the right // upstream type to convert from/to). -impl From for SketchType { - fn from(k: planner_types::post_asap::SketchKind) -> Self { - use planner_types::post_asap::SketchKind; +impl From for SketchType { + fn from(k: planner_types::post_asap::SketchAlgorithm) -> Self { + use planner_types::post_asap::SketchAlgorithm; match k { - SketchKind::Kll => SketchType::KLL, - SketchKind::DDSketch => SketchType::DDSketch, - SketchKind::Hll => SketchType::HLL, - SketchKind::Cms => SketchType::CountMinSketch, - SketchKind::CountSketch => SketchType::CountSketch, + SketchAlgorithm::Kll => SketchType::KLL, + SketchAlgorithm::DDSketch => SketchType::DDSketch, + SketchAlgorithm::Hll => SketchType::HLL, + SketchAlgorithm::Cms => SketchType::CountMinSketch, + SketchAlgorithm::CountSketch => SketchType::CountSketch, other => unreachable!( - "SketchType::from(SketchKind): no legacy equivalent for {other:?} \ + "SketchType::from(SketchAlgorithm): no legacy equivalent for {other:?} \ (callers only pass the 5 canonical families)" ), } } } -/// Inverse of `From for SketchType`. Round-trippable for the 5 -/// canonical families: `SketchKind::from(SketchType::from(k)) == k`. -impl From for planner_types::post_asap::SketchKind { +/// Inverse of `From for SketchType`. Round-trippable for the 5 +/// canonical families: `SketchAlgorithm::from(SketchType::from(k)) == k`. +impl From for planner_types::post_asap::SketchAlgorithm { fn from(t: SketchType) -> Self { - use planner_types::post_asap::SketchKind; + use planner_types::post_asap::SketchAlgorithm; match t { - SketchType::KLL => SketchKind::Kll, - SketchType::DDSketch => SketchKind::DDSketch, - SketchType::HLL => SketchKind::Hll, - SketchType::CountMinSketch => SketchKind::Cms, - SketchType::CountSketch => SketchKind::CountSketch, + SketchType::KLL => SketchAlgorithm::Kll, + SketchType::DDSketch => SketchAlgorithm::DDSketch, + SketchType::HLL => SketchAlgorithm::Hll, + SketchType::CountMinSketch => SketchAlgorithm::Cms, + SketchType::CountSketch => SketchAlgorithm::CountSketch, } } } diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index f395bf8df..5c4580efd 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -30,4 +30,4 @@ xxhash-rust = { version = "0.8", features = ["xxh64"] } # exactly (`control_plane/Cargo.toml`) -- two different revs of the same # git dependency in one workspace resolve to two distinct Rust types that # won't unify. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cb70086b4c4a7ba89baf2516be81d0b192137a3a" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3afcba68f4e8397fb81e2be988f47120f63f7a39" } diff --git a/crates/asap_types/src/accumulator_spec.rs b/crates/asap_types/src/accumulator_spec.rs index 2100ccce6..45caa0cf2 100644 --- a/crates/asap_types/src/accumulator_spec.rs +++ b/crates/asap_types/src/accumulator_spec.rs @@ -210,7 +210,24 @@ impl From for SummaryParams { /// known-sketch. See the module doc for why the flat type exists. impl From for SummaryKind { fn from(k: planner_types::post_asap::SketchKind) -> Self { - use planner_types::post_asap::SketchKind as K; + use planner_types::post_asap::SketchAlgorithm as K; + match k.algorithm() { + K::Kll => SummaryKind::Kll, + K::Cms => SummaryKind::Cms, + K::Hll => SummaryKind::Hll, + K::DDSketch => SummaryKind::DDSketch, + K::CmsWithHeap => SummaryKind::CmsWithHeap, + K::Kmv => SummaryKind::Kmv, + K::Theta => SummaryKind::Theta, + K::CountSketch => SummaryKind::CountSketch, + K::CountSketchWithHeap => SummaryKind::CountSketchWithHeap, + } + } +} + +impl From for SummaryKind { + fn from(k: planner_types::post_asap::SketchAlgorithm) -> Self { + use planner_types::post_asap::SketchAlgorithm as K; match k { K::Kll => SummaryKind::Kll, K::Cms => SummaryKind::Cms, @@ -231,8 +248,8 @@ impl From for SummaryKind { /// which have no `SketchKind` equivalent -- the inverse of the widening /// [`From`] impl above, fallible because that direction isn't total. impl SummaryKind { - pub fn as_sketch_kind(&self) -> Option { - use planner_types::post_asap::SketchKind as K; + pub fn as_sketch_kind(&self) -> Option { + use planner_types::post_asap::SketchAlgorithm as K; Some(match self { SummaryKind::Kll => K::Kll, SummaryKind::Cms => K::Cms, diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 380358fec..bff68bb65 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -38,7 +38,7 @@ control_plane = { path = "../control_plane" } # reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now # (not a separate `asap-ir` import) -- `find_candidates` still needs to # walk/match them directly. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "cb70086b4c4a7ba89baf2516be81d0b192137a3a" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "3afcba68f4e8397fb81e2be988f47120f63f7a39" } # Shared external (workspace) serde.workspace = true diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 89c774cdb..d370a6677 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -2193,7 +2193,8 @@ mod asap_tier_classify_tests { // live MVP workload: the agent emits a bare-named KLL sketch // (`http_requests_total_latency_ms`) into the SketchStore. fn kll_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { - let cfg = SketchConfig::Kll { k: 200 }; + // Latest ASAPPlanner sizes an epsilon=0.01 KLL at k=269. + let cfg = SketchConfig::Kll { k: 269 }; SketchInstanceMetadata { sid, metric_name: metric.to_string(), @@ -2230,7 +2231,8 @@ mod asap_tier_classify_tests { } fn hll_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { - let cfg = SketchConfig::Hll { precision: 10 }; + // Latest ASAPPlanner requires p=14 for a 1% HLL error target. + let cfg = SketchConfig::Hll { precision: 14 }; SketchInstanceMetadata { sid, metric_name: metric.to_string(), @@ -2302,7 +2304,7 @@ mod asap_tier_classify_tests { BTreeMap::new(), (now_ms.saturating_sub(3_000), now_ms.saturating_sub(2_000)), SketchSampleState { - bytes: encode_hll_with_cardinality(10, 500), + bytes: encode_hll_with_cardinality(14, 500), encoding: crate::storage_engines::sketch_db::index::SketchEncoding::ProtoFull, }, ); @@ -2382,7 +2384,7 @@ mod asap_tier_classify_tests { // Series A: items 0..600. Series B: items 400..1000. // Overlap = [400,600) = 200 items; true union = [0,1000) = 1000. - let precision = 12u32; // ~1.6% standard error + let precision = 14u32; // ~0.8% standard error; legal for epsilon=0.01 let a_items: Vec = (0..600).map(|i| format!("u-{i}")).collect(); let b_items: Vec = (400..1000).map(|i| format!("u-{i}")).collect(); let true_union = 1000.0_f64; @@ -2470,7 +2472,7 @@ mod asap_tier_classify_tests { let window_end = now_ms.saturating_sub(5_000); let items: Vec = (1..=50).map(|i| i as f64).collect(); - let bytes = encode_kll_items_proto(200, &items); + let bytes = encode_kll_items_proto(269, &items); idx.append_sample( sid, BTreeMap::new(), @@ -2557,7 +2559,7 @@ mod asap_tier_classify_tests { BTreeMap::new(), (now_ms.saturating_sub(60_000), now_ms.saturating_sub(55_000)), SketchSampleState { - bytes: encode_kll_items_proto(200, &items), + bytes: encode_kll_items_proto(269, &items), encoding: crate::storage_engines::sketch_db::index::SketchEncoding::ProtoFull, }, ); @@ -2567,7 +2569,7 @@ mod asap_tier_classify_tests { BTreeMap::new(), (now_ms.saturating_sub(15_000), now_ms.saturating_sub(5_000)), SketchSampleState { - bytes: encode_kll_items_proto(200, &items), + bytes: encode_kll_items_proto(269, &items), encoding: crate::storage_engines::sketch_db::index::SketchEncoding::ProtoDelta, }, ); @@ -2609,7 +2611,7 @@ mod asap_tier_classify_tests { BTreeMap::new(), (now_ms.saturating_sub(15_000), now_ms.saturating_sub(5_000)), SketchSampleState { - bytes: encode_kll_items_proto(200, &items), + bytes: encode_kll_items_proto(269, &items), encoding: crate::storage_engines::sketch_db::index::SketchEncoding::ProtoDelta, }, ); @@ -3154,7 +3156,8 @@ mod asap_tier_classify_tests { total_inserts: i64, now_ms: u64, ) { - let cfg = SketchConfig::CountMin { rows: 2, cols: 4 }; + // Matches ControlPlaneCostModel's epsilon=0.01 CMS sizing. + let cfg = SketchConfig::CountMin { rows: 5, cols: 512 }; idx.register(SketchInstanceMetadata { sid, metric_name: metric.to_string(), @@ -3176,7 +3179,7 @@ mod asap_tier_classify_tests { }); // Build a CountMinState PROTO_FULL frame whose row 0 sums to // `total_inserts` (decode_frequency_total reads row 0's sum). - let bytes = encode_cms_state_proto(2, 4, total_inserts); + let bytes = encode_cms_state_proto(5, 512, total_inserts); let window_start = now_ms.saturating_sub(60_000); let window_end = now_ms.saturating_sub(30_000); idx.append_sample( @@ -3742,7 +3745,7 @@ mod range_stitch_tests { /// PER-WINDOW sample (not a single cumulative scalar), which is what the /// range stitch needs so warm contributes one value per covered window. fn cms_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { - let cfg = SketchConfig::CountMin { rows: 2, cols: 4 }; + let cfg = SketchConfig::CountMin { rows: 5, cols: 512 }; SketchInstanceMetadata { sid, metric_name: metric.to_string(), diff --git a/data_plane/src/query_engines/asap_query_engine/l4_lowering.rs b/data_plane/src/query_engines/asap_query_engine/l4_lowering.rs index d473eb14f..e693325c1 100644 --- a/data_plane/src/query_engines/asap_query_engine/l4_lowering.rs +++ b/data_plane/src/query_engines/asap_query_engine/l4_lowering.rs @@ -22,7 +22,7 @@ //! accuracy-driven cost model prefers in the abstract (e.g. DDSketch //! over Kll for quantiles, unconditionally), with no guarantee it matches //! what's actually registered — and `SummaryExecutor::find_candidates` -//! requires an exact `(SketchKind, SketchParams)` match, by design (see +//! requires an exact `(SketchAlgorithm, SketchParams)` match, by design (see //! `summary_executor.rs::summary_params_match`'s doc: this deployment //! chose strict equality over silently serving an answer under a looser //! guarantee than what was planned). @@ -40,7 +40,7 @@ use std::rc::Rc; -use planner_types::post_asap::{SketchKind, SketchParams, SummaryExpr, SummaryNode}; +use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryExpr, SummaryNode}; use control_plane::sketch_algebra::capability::{OuterFn, SketchKindHandle}; use control_plane::sketch_algebra::cost_model::ObservedFamilyCostModel; @@ -101,13 +101,13 @@ pub enum LoweringSkip { /// a decode/merge failure surfaced from `summary_executor.rs`, ...). /// Always safe to just fall back — this means "can't answer this way /// right now" (e.g. the sid catalog doesn't have an exact - /// `(SketchKind, SketchParams)` match), never "answered wrong." + /// `(SketchAlgorithm, SketchParams)` match), never "answered wrong." ExecuteFailed(String), } /// Map a registered sid's `(SketchKindHandle, SketchConfig)` — the /// durable record of what planning actually decided for this metric — to -/// the `(SketchKind, SketchParams)` pair `ObservedFamilyCostModel` +/// the `(SketchAlgorithm, SketchParams)` pair `ObservedFamilyCostModel` /// needs to reproduce that decision exactly. `None` for shapes this /// deployment doesn't map (e.g. `SketchKindHandle::Any`, which is an /// analysis-time wildcard that's never actually registered on a sid). @@ -121,33 +121,33 @@ pub enum LoweringSkip { fn observed_summary_params( kind: SketchKindHandle, config: &SketchConfig, -) -> Option<(SketchKind, SketchParams)> { +) -> Option<(SketchAlgorithm, SketchParams)> { const PLACEHOLDER_HEAP_SIZE: u32 = 100; match (kind, config) { (SketchKindHandle::DDSketch, SketchConfig::DDSketch { relative_accuracy }) => Some(( - SketchKind::DDSketch, + SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha: *relative_accuracy, }, )), (SketchKindHandle::Kll, SketchConfig::Kll { k }) => { - Some((SketchKind::Kll, SketchParams::Kll { k: *k })) + Some((SketchAlgorithm::Kll, SketchParams::Kll { k: *k })) } (SketchKindHandle::Hll, SketchConfig::Hll { precision }) => Some(( - SketchKind::Hll, + SketchAlgorithm::Hll, SketchParams::Hll { precision: *precision as u8, }, )), (SketchKindHandle::CountMin, SketchConfig::CountMin { rows, cols }) => Some(( - SketchKind::Cms, + SketchAlgorithm::Cms, SketchParams::Cms { width: *cols as u32, depth: *rows as u32, }, )), (SketchKindHandle::CmsWithHeap, SketchConfig::CountMin { rows, cols }) => Some(( - SketchKind::CmsWithHeap, + SketchAlgorithm::CmsWithHeap, SketchParams::CmsWithHeap { width: *cols as u32, depth: *rows as u32, @@ -155,7 +155,7 @@ fn observed_summary_params( }, )), (SketchKindHandle::CountSketch, SketchConfig::CountSketch { rows, cols }) => Some(( - SketchKind::CountSketch, + SketchAlgorithm::CountSketch, SketchParams::CountSketch { width: *cols as u32, depth: *rows as u32, @@ -163,7 +163,7 @@ fn observed_summary_params( )), (SketchKindHandle::CountSketchWithHeap, SketchConfig::CountSketch { rows, cols }) => { Some(( - SketchKind::CountSketchWithHeap, + SketchAlgorithm::CountSketchWithHeap, SketchParams::CountSketchWithHeap { width: *cols as u32, depth: *rows as u32, @@ -187,7 +187,7 @@ fn observed_summary_params( fn observed_family_for_metric( index: &SketchStore, metric: &str, -) -> Option<(SketchKind, SketchParams)> { +) -> Option<(SketchAlgorithm, SketchParams)> { for sid in index.instances_matching(metric, &Default::default()) { let found = index.with_instance(sid, |m| match &m.agg_kind { AggKind::Sketch { kind, config, .. } => observed_summary_params(*kind, config), @@ -212,7 +212,7 @@ fn observed_family_for_metric( fn observed_family_for_metric_from_plan( plan: &control_plane::backend_plan::BackendPlan, metric: &str, -) -> Option<(SketchKind, SketchParams)> { +) -> Option<(SketchAlgorithm, SketchParams)> { plan.materializations.values().find_map(|m| { if !matches!(&m.source, control_plane::intent_algebra::Source::TimeSeries { metric: mm } if mm == metric) { @@ -255,13 +255,16 @@ pub fn lower_promql_to_l4node( let qe = control_plane::query_parser::parse_query_expr_canonical(query, accuracy.clone()) .map_err(|e| LoweringSkip::ParseFailed(e.to_string()))?; + if analysis.candidates.is_empty() { + return Err(LoweringSkip::NotRealized); + } // Serving time must reproduce the REAL planning decision, not // independently re-derive one -- see this module's docs. Prefer // reading it straight off an installed `BackendPlan`'s // materializations when one covers this metric -- // `Materialization.kind`/`.params` already ARE the - // `(SketchKind, SketchParams)` pair this needs, no + // `(SketchAlgorithm, SketchParams)` pair this needs, no // `AggregationConfig` reconstruction required (design-backend-plan-wire-format.md // §5). Otherwise fall back to the `SketchStore`-reconstruction path // (`observed_family_for_metric`), which is `None` when this metric @@ -280,7 +283,7 @@ pub fn lower_promql_to_l4node( match physical { PhysicalExpr::Committed(L4Plan::Summary(node)) => { - if matches!(node.expr, SummaryExpr::Logical(_)) { + if matches!(node.expr, SummaryExpr::KeepPreAsap(_)) { Err(LoweringSkip::NotRealized) } else { Ok(node) @@ -381,7 +384,7 @@ mod tests { ) .expect("Frequency intent must realize via bind_query_expr/ControlPlaneCostModel"); assert!( - !matches!(node.expr, SummaryExpr::Logical(_)), + !matches!(node.expr, SummaryExpr::KeepPreAsap(_)), "expected a real SummaryAgg/SummaryEstimate binding, got Logical (the gap \ this module exists to avoid): {:?}", node.expr @@ -419,7 +422,7 @@ mod tests { use std::collections::HashMap; fn register_kll(idx: &SketchStore, metric: &str) { - let cfg = SketchConfig::Kll { k: 200 }; + let cfg = SketchConfig::Kll { k: 269 }; idx.register(SketchInstanceMetadata { sid: 1, metric_name: metric.to_string(), @@ -458,7 +461,7 @@ mod tests { // `Materialization.kind`/`.params` span both exact // accumulators and sketches -- the flat // `asap_types::SummaryKind`, not this file's own - // `planner_types::post_asap::SketchKind` import (see + // `planner_types::post_asap::SketchAlgorithm` import (see // `physical::colored_dag::emitter`'s `use // asap_types::{...}` note in control_plane). kind: asap_types::SummaryKind::DDSketch, @@ -476,18 +479,18 @@ mod tests { } } - /// Extract the bound `(SketchKind, SketchParams)` from the + /// Extract the bound `(SketchAlgorithm, SketchParams)` from the /// `SummaryEstimate { summary_input: SummaryNode { expr: SummaryAgg { /// summary, params, .. }, .. }, .. }` shape a bare /// `quantile_over_time` query lowers to (confirmed by inspecting /// the tree directly). - fn bound_family(node: &SummaryNode) -> (SketchKind, SketchParams) { + fn bound_family(node: &SummaryNode) -> (SketchAlgorithm, SketchParams) { match &node.expr { SummaryExpr::SummaryEstimate { summary_input, .. } => match &summary_input.expr { SummaryExpr::SummaryAgg { - family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, params), + family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, _), .. - } => (kind.clone(), params.clone()), + } => (kind.algorithm().clone(), kind.params().clone()), other => panic!("expected a Sketch SummaryAgg, got {other:?}"), }, other => panic!("expected SummaryEstimate, got {other:?}"), @@ -503,7 +506,7 @@ mod tests { let node = lower_promql_to_l4node(&idx, "quantile_over_time(0.99, m[1m])", accuracy(), None) .expect("should lower"); - assert_eq!(bound_family(&node).0, SketchKind::Kll); + assert_eq!(bound_family(&node).0, SketchAlgorithm::Kll); } #[test] @@ -526,7 +529,7 @@ mod tests { .expect("should lower"); assert_eq!( bound_family(&node).0, - SketchKind::DDSketch, + SketchAlgorithm::DDSketch, "BackendPlan's materialization must take priority over SketchStore reconstruction" ); } @@ -547,7 +550,7 @@ mod tests { Some(&plan), ) .expect("should lower"); - assert_eq!(bound_family(&node).0, SketchKind::Kll); + assert_eq!(bound_family(&node).0, SketchAlgorithm::Kll); } } } diff --git a/data_plane/src/query_engines/asap_query_engine/summary_exec.rs b/data_plane/src/query_engines/asap_query_engine/summary_exec.rs index 2950b74cb..13ce252e2 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_exec.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_exec.rs @@ -146,7 +146,7 @@ pub fn execute( exec: &E, ) -> Result, ExecError> { match &node.expr { - SummaryExpr::Logical(qe) => Ok(ExecOutcome::Value(vec![( + SummaryExpr::KeepPreAsap(qe) => Ok(ExecOutcome::Value(vec![( E::GroupKey::default(), exec.logical(qe)?, )])), @@ -156,6 +156,7 @@ pub fn execute( family, col, reduction, + .. } => { let tagged = exec.find_candidates(family, col, reduction, child)?; if tagged.is_empty() { @@ -301,8 +302,9 @@ mod tests { fn logical_node() -> Rc { Rc::new(SummaryNode { - expr: SummaryExpr::Logical(Box::new(scan())), + expr: SummaryExpr::KeepPreAsap(Rc::new(scan())), schema: lift(vec!["ts", "value"]), + guarantee: None, }) } @@ -325,8 +327,10 @@ mod tests { family, col: ColumnRef::SampleValue, reduction, + grouping: planner_types::post_asap::GroupingStrategy::default(), }, schema: lift(vec!["value"]), + guarantee: None, }) } @@ -337,6 +341,7 @@ mod tests { query, }, schema: lift(vec!["value"]), + guarantee: None, }) } @@ -344,6 +349,7 @@ mod tests { Rc::new(SummaryNode { expr: SummaryExpr::SummaryMerge { children }, schema: lift(vec!["value"]), + guarantee: None, }) } @@ -440,8 +446,13 @@ mod tests { } fn kll() -> SummaryFamilyType { - use planner_types::post_asap::{SketchKind, SketchParams}; - SummaryFamilyType::Sketch(SketchKind::Kll, SketchParams::Kll { k: 200 }) + use planner_types::post_asap::{ + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, + }; + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + GroupingStrategy::default(), + ) } fn sum() -> SummaryFamilyType { diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 78346e43a..dd0d21bdd 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -349,7 +349,9 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { SummaryFamilyType::ExactAggregate(kind, params) => { (kind.clone().into(), params.clone().into()) } - SummaryFamilyType::Sketch(kind, params) => (kind.clone().into(), params.clone().into()), + SummaryFamilyType::Sketch(kind, _) => { + (kind.clone().into(), kind.params().clone().into()) + } SummaryFamilyType::Plain(_) | SummaryFamilyType::Sample(..) | SummaryFamilyType::Wavelet(..) @@ -941,7 +943,7 @@ fn to_delta_kind(kind: SketchKindHandle, config: &SketchConfig) -> Option Option { match &node.expr { - SummaryExpr::Logical(qe) => find_metric_in_query_expr(qe), + SummaryExpr::KeepPreAsap(qe) => find_metric_in_query_expr(qe), SummaryExpr::SummaryAgg { child, .. } => find_metric(child), SummaryExpr::SummaryEstimate { summary_input, .. } => find_metric(summary_input), SummaryExpr::SummaryMerge { children } => children.first().and_then(|c| find_metric(c)), @@ -963,14 +965,14 @@ pub(crate) fn find_metric_in_query_expr(qe: &QueryExpr) -> Option { QueryExpr::Filter { child, .. } | QueryExpr::Project { child, .. } | QueryExpr::Aggregate { child, .. } - | QueryExpr::Distinct { child, .. } + | QueryExpr::Dedup { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } - | QueryExpr::Subquery { child, .. } + | QueryExpr::PromqlSubquery { child, .. } | QueryExpr::TimeRange { child, .. } | QueryExpr::TimeShift { child, .. } - | QueryExpr::WindowFunc { child, .. } => find_metric_in_query_expr(child), - QueryExpr::Merge { children } => children.iter().find_map(find_metric_in_query_expr), + | QueryExpr::SQLWindowFunc { child, .. } => find_metric_in_query_expr(child), + QueryExpr::Concat { children } => children.iter().find_map(find_metric_in_query_expr), QueryExpr::Join { left, .. } | QueryExpr::SetOp { left, .. } => { find_metric_in_query_expr(left) } @@ -994,6 +996,16 @@ mod tests { use planner_types::pre_asap::{Column, DataType, Schema}; use std::rc::Rc; + fn sketch_family( + algorithm: planner_types::post_asap::SketchAlgorithm, + params: planner_types::post_asap::SketchParams, + ) -> SummaryFamilyType { + SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new(algorithm, params), + planner_types::post_asap::GroupingStrategy::default(), + ) + } + fn scan_node(metric: &str, group_by_field: Option<&str>) -> Rc { let qe = QueryExpr::Scan { source: Source::TimeSeries { @@ -1022,11 +1034,12 @@ mod tests { }); } Rc::new(SummaryNode { - expr: SummaryExpr::Logical(Box::new(qe)), + expr: SummaryExpr::KeepPreAsap(Rc::new(qe)), schema: SummarySchema { fields, time_index: None, }, + guarantee: None, }) } @@ -1034,17 +1047,19 @@ mod tests { Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - family: SummaryFamilyType::Sketch( - planner_types::post_asap::SketchKind::Kll, + family: sketch_family( + planner_types::post_asap::SketchAlgorithm::Kll, planner_types::post_asap::SketchParams::Kll { k: 200 }, ), col: ColumnRef::SampleValue, reduction, + grouping: planner_types::post_asap::GroupingStrategy::default(), }, schema: SummarySchema { fields: vec![], time_index: None, }, + guarantee: None, }) } @@ -1056,17 +1071,19 @@ mod tests { Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - family: SummaryFamilyType::Sketch( - planner_types::post_asap::SketchKind::Hll, + family: sketch_family( + planner_types::post_asap::SketchAlgorithm::Hll, planner_types::post_asap::SketchParams::Hll { precision: 10 }, ), col: ColumnRef::SampleValue, reduction, + grouping: planner_types::post_asap::GroupingStrategy::default(), }, schema: SummarySchema { fields: vec![], time_index: None, }, + guarantee: None, }) } @@ -1080,6 +1097,7 @@ mod tests { fields: vec![], time_index: None, }, + guarantee: None, }) } @@ -1200,8 +1218,8 @@ mod tests { Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - family: SummaryFamilyType::Sketch( - planner_types::post_asap::SketchKind::Cms, + family: sketch_family( + planner_types::post_asap::SketchAlgorithm::Cms, planner_types::post_asap::SketchParams::Cms { width: 256, depth: 4, @@ -1209,11 +1227,13 @@ mod tests { ), col: ColumnRef::SampleValue, reduction: Reduction::by(vec![]), + grouping: planner_types::post_asap::GroupingStrategy::default(), }, schema: SummarySchema { fields: vec![], time_index: None, }, + guarantee: None, }) } @@ -1258,8 +1278,8 @@ mod tests { Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - family: SummaryFamilyType::Sketch( - planner_types::post_asap::SketchKind::CmsWithHeap, + family: sketch_family( + planner_types::post_asap::SketchAlgorithm::CmsWithHeap, planner_types::post_asap::SketchParams::CmsWithHeap { width: 256, depth: 4, @@ -1268,11 +1288,13 @@ mod tests { ), col: ColumnRef::SampleValue, reduction: Reduction::by(vec![]), + grouping: planner_types::post_asap::GroupingStrategy::default(), }, schema: SummarySchema { fields: vec![], time_index: None, }, + guarantee: None, }) } @@ -1313,11 +1335,13 @@ mod tests { // `by` always means "reduce fully," never `PerEntity` (see // `resolve_group_key`'s doc). reduction: Reduction::by(by), + grouping: planner_types::post_asap::GroupingStrategy::default(), }, schema: SummarySchema { fields: vec![], time_index: None, }, + guarantee: None, }) } @@ -2095,17 +2119,19 @@ mod tests { let mismatched = Rc::new(SummaryNode { expr: SummaryExpr::SummaryAgg { child, - family: SummaryFamilyType::Sketch( - planner_types::post_asap::SketchKind::Kll, + family: sketch_family( + planner_types::post_asap::SketchAlgorithm::Kll, planner_types::post_asap::SketchParams::Kll { k: 500 }, ), col: ColumnRef::SampleValue, reduction: Reduction::by(vec![]), + grouping: planner_types::post_asap::GroupingStrategy::default(), }, schema: SummarySchema { fields: vec![], time_index: None, }, + guarantee: None, }); let tree = estimate_node(mismatched, SketchQuery::Quantile { q: 0.5 }); let exec = ctx(&idx); @@ -2469,8 +2495,8 @@ mod tests { // this tree resolves to a real Sketch Value above). let handles = exec .find_candidates( - &SummaryFamilyType::Sketch( - planner_types::post_asap::SketchKind::Kll, + &sketch_family( + planner_types::post_asap::SketchAlgorithm::Kll, planner_types::post_asap::SketchParams::Kll { k: 200 }, ), &ColumnRef::SampleValue, @@ -2560,11 +2586,13 @@ mod tests { ), col: ColumnRef::SampleValue, reduction: Reduction::by(vec![]), + grouping: planner_types::post_asap::GroupingStrategy::default(), }, schema: SummarySchema { fields: vec![], time_index: None, }, + guarantee: None, }); let exec = ctx(&idx); match execute(&tree, &exec) { diff --git a/docs/developer_docs/control-plane/physical-compiler.md b/docs/developer_docs/control-plane/physical-compiler.md index 7a8a9ef4f..0eaf96562 100644 --- a/docs/developer_docs/control-plane/physical-compiler.md +++ b/docs/developer_docs/control-plane/physical-compiler.md @@ -1,16 +1,16 @@ # Developing the Planner adapter and physical compiler -> Interface status: target public API. Existing migration modules must converge -> on this boundary. +> Interface status: implemented MVP API in +> `control_plane::physical::compiler`. ## Current implementation boundary -The production path has not yet converged on the public interfaces below. It -currently consumes ASAPPlanner types pinned in `control_plane/Cargo.toml`, then -uses `physical::colored_dag::StageAllocator`, `ThreeStageEmitter`, and -`backend_plan::from_stage_config` to produce agent YAML and `BackendPlan`. -Callers must not treat the target `PlanningRequest`, `PhysicalCompiler`, or -`CompiledPlanBundle` examples below as implemented Rust APIs. +The compiler consumes ASAPPlanner types pinned to revision `3afcba6`, selects +from Planner's legal candidate space with backend-owned cost and evidence +inputs, and emits one `CompiledPlanBundle`. The bundle contains a CollectorPlan +for every target collector and the matching BackendPlan. Legacy +`StageAllocator`/`ThreeStageEmitter` paths remain for older publication flows; +they are not a second semantic planner. ASAPPlanner owns logical semantics and summary selection. In particular, a deployment override may choose only a family compatible with the selected @@ -27,17 +27,18 @@ The control plane has three public layers: PlanningRequest | v -PlannerAdapter ----------> SelectedLogicalPlan - | - v +ASAPPlanner candidate selection + | + v PhysicalCompiler -------> CompiledPlanBundle | | v v CollectorPlan BackendPlan ``` -- **Planner adapter** owns the typed call to ASAPPlanner. It supplies the whole - workload and receives one selected logical plan without copying Planner IR. +- **Planner selection boundary** is + `planner_selection::select_summary_with_evidence`. It enumerates Planner's + candidates and commits only a legal candidate. - **Physical compiler** adds backend-owned placement, windows, transport, and runtime capabilities without changing logical semantics. - **Plan bundle** is the only output passed to publication. CollectorPlan and @@ -49,28 +50,15 @@ remain public ASAPPlanner interfaces. Runtime publication is documented in ## 2. Public interfaces and definitions -### Planner adapter - -```rust -pub trait PlannerAdapter { - type Error; - - fn select( - &self, - request: PlanningRequest, - ) -> Result; -} -``` +### Planning request `PlanningRequest` is backend-owned request context around Planner's canonical -workload value: +per-query IR: ```rust pub struct PlanningRequest { - pub workload: asap_planner::Workload, - pub schema: asap_planner::SchemaCatalog, - pub constraints: asap_planner::PlanningConstraints, - pub cost_inputs: asap_planner::CostInputs, + pub queries: Vec, + pub evidence: HashMap, pub planner_revision: String, } ``` @@ -79,20 +67,21 @@ Input definitions: | Field | Definition | | --- | --- | -| `workload` | Complete workload; shared queries must not be split into independent calls. | -| `schema` | Source/label/type information required to bind queries. | -| `constraints` | Accuracy and logical requirements supplied by the caller. | -| `cost_inputs` | Measured/declared logical cost inputs; unknown values stay unknown. | +| `queries` | Canonical `QueryExpr`, source, window, grouping labels, accuracy, and stable query ID. | +| `evidence` | Optional typed TopK membership certificates keyed by query ID. | | `planner_revision` | Immutable Planner build/revision used for reproducibility. | -`SelectedLogicalPlan` wraps Planner's public selected post-ASAP workload plan -and correlation metadata; it does not define another DAG: +TopK evidence is accepted only when its selected lower bound is strictly above +the excluded upper bound, its failure probability is valid, its source is +non-empty, and its observation is fresh under `DeploymentEnvironment`. ```rust -pub struct SelectedLogicalPlan { - pub workload_plan: asap_planner::SelectedWorkloadPlan, - pub planner_revision: String, - pub query_ids: Vec, +pub struct TopKMembershipEvidence { + pub selected_lower_bound: f64, + pub excluded_upper_bound: f64, + pub interval_failure_probability: f64, + pub observed_at_unix_ms: u64, + pub source: String, } ``` @@ -102,37 +91,26 @@ planning from each implementing their own query-to-summary mapping. ### Physical compiler ```rust -pub trait PhysicalCompiler { - type Error; - - fn compile( +impl PhysicalCompiler { + pub fn compile( &self, - selected: SelectedLogicalPlan, + request: PlanningRequest, environment: DeploymentEnvironment, - policy: RuntimePolicy, - ) -> Result; + ) -> Result; } ``` ```rust pub struct DeploymentEnvironment { - pub topology: DeploymentTopology, - pub collectors: Vec, - pub backend: BackendTarget, + pub collector_ids: Vec, pub capability_snapshot_id: String, -} - -pub struct RuntimePolicy { - pub activation: Timestamp, - pub expiry: Option, - pub freshness: FreshnessPolicy, - pub retention: RetentionPolicy, - pub transmission: TransmissionPolicy, + pub observed_at_unix_ms: u64, + pub max_evidence_age_ms: u64, } pub struct CompiledPlanBundle { pub envelope: PlanEnvelope, - pub collector_plans: Vec, + pub collector_plans: Vec, // complete per-target projections pub backend_plan: BackendPlan, } ``` @@ -141,16 +119,16 @@ Supporting public types: | Type | Definition | | --- | --- | -| `DeploymentTopology` | Runtime stages, network relationships, and isolation boundaries available for placement. | -| `CollectorTarget` | Collector identity, edge assignment, endpoint reference, and advertised capability snapshot. | -| `BackendTarget` | Data-plane identity, endpoint reference, storage routes, and advertised capabilities. | -| `FreshnessPolicy` | Maximum readiness lag, watermark, and allowed-lateness requirements. | -| `RetentionPolicy` | Duration and lifecycle rules for active/draining materializations. | -| `TransmissionPolicy` | Allowed raw/full/delta modes, cadence, encoding, and checkpoint limits. | -| `PlanEnvelope` | Shared `plan_id`, `plan_version`, activation/expiry, backend compatibility, and Planner revision. | -| `CollectorPlan` | Versioned public YAML execution contract owned by ASAPCollector. | +| `DeploymentEnvironment` | Target collector IDs, capability snapshot identity, planning time, and evidence freshness policy. | +| `PlanEnvelope` | Shared deterministic `plan_id`, generation time, capability snapshot, and Planner revision. | +| `CollectorPlan` | Serializable execution projection consumed by ASAPCollector. | | `BackendPlan` | Versioned public data-plane materialization/routing contract defined in this repository. | +Current MVP limits are explicit: time-series sources and sketch +materializations are supported; table sources and non-sketch selected families +return `CompileError`. Runtime activation/expiry and richer topology placement +remain publication-layer work and are not claimed by this compiler API. + Output definitions: | Output | Definition |