diff --git a/controller/Cargo.lock b/controller/Cargo.lock index f73d18d4..937a67ef 100644 --- a/controller/Cargo.lock +++ b/controller/Cargo.lock @@ -26,6 +26,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "ar_archive_writer" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +dependencies = [ + "object", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -80,7 +89,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-tungstenite", - "tower", + "tower 0.5.3", "tower-layer", "tower-service", "tracing", @@ -113,6 +122,15 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "2.11.0" @@ -146,6 +164,12 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cactus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbc26382d871df4b7442e3df10a9402bf3cf5e55cbd66f12be38861425f0564" + [[package]] name = "cc" version = "1.2.57" @@ -168,6 +192,20 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "cfgrammar" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" +dependencies = [ + "indexmap", + "lazy_static", + "num-traits", + "regex", + "serde", + "vob", +] + [[package]] name = "chrono" version = "0.4.44" @@ -190,12 +228,16 @@ dependencies = [ "axum", "chrono", "futures-util", + "http-body-util", + "promql-parser", "reqwest", "serde", "serde_json", "serde_yaml", + "sqlparser", "thiserror 1.0.69", "tokio", + "tower 0.4.13", "tracing", "tracing-subscriber", ] @@ -231,6 +273,15 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -268,12 +319,29 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -345,6 +413,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -665,6 +742,18 @@ version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +[[package]] +name = "libredox" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.7.3", +] + [[package]] name = "litemap" version = "0.8.1" @@ -686,6 +775,60 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lrlex" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71364e868116ee891b0f93559eb9eca5675bec28b22d33c58481e66c3951d7e" +dependencies = [ + "cfgrammar", + "getopts", + "lazy_static", + "lrpar", + "num-traits", + "quote", + "regex", + "regex-syntax", + "serde", + "vergen", +] + +[[package]] +name = "lrpar" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b265a81193d94c92d1c9c715498d6fa505bce3f789ceecb24ab5d6fa2dbc71" +dependencies = [ + "bincode", + "cactus", + "cfgrammar", + "filetime", + "indexmap", + "lazy_static", + "lrtable", + "num-traits", + "packedvec", + "regex", + "serde", + "static_assertions", + "vergen", + "vob", +] + +[[package]] +name = "lrtable" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc36d15214ca997a5097845be1f932b7ee6125c36f5c5e55f6c49e027ddeb6de" +dependencies = [ + "cfgrammar", + "fnv", + "num-traits", + "serde", + "sparsevec", + "vob", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -739,6 +882,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + [[package]] name = "num-traits" version = "0.2.19" @@ -748,12 +897,40 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "packedvec" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -772,7 +949,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -783,6 +960,26 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -795,6 +992,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "potential_utf" version = "0.1.4" @@ -804,6 +1007,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -822,6 +1031,30 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "promql-parser" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2791a28f8ea7e48f2838999c06d089184d44adb860feab682d45dd190ef718" +dependencies = [ + "cfgrammar", + "chrono", + "lazy_static", + "lrlex", + "lrpar", + "regex", +] + +[[package]] +name = "psm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "quinn" version = "0.11.9" @@ -951,6 +1184,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -960,6 +1213,27 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -1005,7 +1279,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tower", + "tower 0.5.3", "tower-http", "tower-service", "url", @@ -1225,12 +1499,54 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sparsevec" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b4a8ce3045f0fe173fb5ae3c6b7dcfbec02bfa650bb8618b2301f52af0134d" +dependencies = [ + "num-traits", + "packedvec", + "serde", + "vob", +] + +[[package]] +name = "sqlparser" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +dependencies = [ + "log", + "recursive", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.52.0", + "windows-sys 0.59.0", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "subtle" version = "2.6.1" @@ -1317,6 +1633,39 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -1392,6 +1741,21 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.3" @@ -1421,7 +1785,7 @@ dependencies = [ "http-body", "iri-string", "pin-project-lite", - "tower", + "tower 0.5.3", "tower-layer", "tower-service", ] @@ -1536,6 +1900,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -1578,12 +1948,33 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vergen" +version = "8.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" +dependencies = [ + "anyhow", + "rustversion", + "time", +] + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vob" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc936b5a7202a703aeaf7ce05e7931db2e0c8126813f97db3e9e06d867b0bb38" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "want" version = "0.3.1" @@ -1764,6 +2155,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.60.2" diff --git a/controller/Cargo.toml b/controller/Cargo.toml index a6e7df8c..5cf19281 100644 --- a/controller/Cargo.toml +++ b/controller/Cargo.toml @@ -20,6 +20,8 @@ thiserror = "1" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } chrono = { version = "0.4", features = ["serde"] } +sqlparser = "0.61" +promql-parser = "0.8" [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/controller/src/analyzer.rs b/controller/src/analyzer.rs index f823b75f..e78e6e6a 100644 --- a/controller/src/analyzer.rs +++ b/controller/src/analyzer.rs @@ -3,30 +3,53 @@ use std::time::Duration; use anyhow::{anyhow, Context}; use serde::{Deserialize, Serialize}; +use crate::query_parser; use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; // ── Public API ──────────────────────────────────────────────────────────────── /// JSON-friendly representation of a query workload submitted by callers. +/// +/// There are two ways to populate a `QuerySpec`: +/// +/// 1. **Explicit fields** — supply `metric_name`, `aggregations`, +/// `time_window`, etc. directly. This is the original API. +/// +/// 2. **Query string** — supply a raw PromQL or SQL string in +/// `query_string`. The analyzer parses it and fills in `metric_name`, +/// `aggregations`, `group_by_labels`, `label_filters`, and `time_window` +/// automatically. Any explicit fields that are non-empty / non-default +/// **override** the parsed values, so the two approaches compose. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QuerySpec { + /// Raw PromQL or SQL query string to parse (SP-1 automatic extraction). + /// When provided, metric_name / aggregations / time_window may be omitted + /// and will be derived from the query. + #[serde(default)] + pub query_string: Option, + + /// Metric name override. Required when `query_string` is absent. + #[serde(default)] pub metric_name: String, #[serde(default)] pub label_filters: HashMap, #[serde(default)] pub group_by_labels: Vec, + /// Aggregation type overrides ("quantile", "cardinality", "frequency"). + /// Required when `query_string` is absent. + #[serde(default)] pub aggregations: Vec, + /// Time window override (e.g. "5m"). Required when `query_string` is absent. + #[serde(default)] pub time_window: String, + #[serde(default)] pub repeat_every: Option, pub accuracy_sla: f64, pub latency_sla: Option, /// Optional: pin a specific sketch type, bypassing the cost-model planner. - /// Useful when the target collector supports only a subset of sketches. pub sketch_type: Option, - /// Observable data-stream characteristics used for delta transmission - /// decisions and raw-vs-sketch bandwidth comparison. - /// Omit to use conservative defaults (1 000 series, 100 Hz, 100 B/sample, - /// Zipf distribution, no memory budget). + /// Observable data-stream characteristics used for delta / raw-vs-sketch + /// bandwidth comparison. Omit to use conservative defaults. #[serde(default)] pub workload: WorkloadCharacteristics, } @@ -37,24 +60,75 @@ impl Analyzer { pub fn new() -> Self { Self } pub fn analyze(&self, spec: QuerySpec) -> anyhow::Result { - if spec.metric_name.trim().is_empty() { - return Err(anyhow!("metric_name is required")); - } - if spec.aggregations.is_empty() { - return Err(anyhow!("at least one aggregation is required")); - } if !(0.0..=1.0).contains(&spec.accuracy_sla) { return Err(anyhow!("accuracy_sla must be in [0,1], got {}", spec.accuracy_sla)); } - let aggs = parse_agg_types(&spec.aggregations)?; + // ── Step 1: parse query_string if provided ───────────────────────── + let parsed = spec.query_string.as_deref() + .map(|q| query_parser::parse_query(q)) + .transpose() + .with_context(|| "failed to parse query_string")?; - let time_window = parse_duration(&spec.time_window) - .with_context(|| format!("invalid time_window {:?}", spec.time_window))?; - if time_window.is_zero() { - return Err(anyhow!("time_window must be positive")); - } + // ── Step 2: resolve metric_name ──────────────────────────────────── + let metric_name = if !spec.metric_name.trim().is_empty() { + spec.metric_name.clone() + } else if let Some(ref p) = parsed { + p.metric_name.clone() + } else { + return Err(anyhow!( + "metric_name is required (or provide query_string)" + )); + }; + + // ── Step 3: resolve aggregations ─────────────────────────────────── + let aggregations = if !spec.aggregations.is_empty() { + parse_agg_types(&spec.aggregations)? + } else if let Some(ref p) = parsed { + if p.aggregations.is_empty() && !p.exact_required { + return Err(anyhow!( + "could not infer aggregation type from query_string; \ + provide explicit aggregations" + )); + } + p.aggregations.clone() + } else { + return Err(anyhow!("at least one aggregation is required")); + }; + + // ── Step 4: resolve time_window ──────────────────────────────────── + let time_window = if !spec.time_window.trim().is_empty() { + let d = parse_duration(&spec.time_window) + .with_context(|| format!("invalid time_window {:?}", spec.time_window))?; + if d.is_zero() { + return Err(anyhow!("time_window must be positive")); + } + d + } else if let Some(ref p) = parsed { + p.time_window + } else { + return Err(anyhow!("time_window is required (or provide query_string)")); + }; + + // ── Step 5: resolve dimensions (group_by + label_filter keys) ────── + // Parsed values are the base; explicit spec fields override / extend. + let parsed_group_by = parsed.as_ref().map(|p| p.group_by_labels.as_slice()).unwrap_or(&[]); + let parsed_filters: HashMap = + parsed.as_ref().map(|p| p.label_filters.clone()).unwrap_or_default(); + + let merged_filters: HashMap = { + let mut m = parsed_filters; + m.extend(spec.label_filters.clone()); // explicit overrides parsed + m + }; + + let filter_keys: Vec = merged_filters.keys().cloned().collect(); + let all_group_by: Vec = dedup_dims( + &dedup_dims(parsed_group_by, &spec.group_by_labels), + &filter_keys, + ); + // ── Step 6: scalar fields ────────────────────────────────────────── let repeat_every = spec.repeat_every.as_deref() .map(parse_duration) .transpose() @@ -65,21 +139,21 @@ impl Analyzer { .transpose() .with_context(|| "invalid latency_sla")?; - // Merge group_by_labels and label_filter keys, deduplicating while - // preserving the group_by_labels order first. - let filter_keys: Vec = spec.label_filters.keys().cloned().collect(); - let dims = dedup_dims(&spec.group_by_labels, &filter_keys); + let exact_required = parsed.as_ref().map(|p| p.exact_required).unwrap_or(false); + let quantiles = parsed.as_ref().map(|p| p.quantiles.clone()).unwrap_or_default(); Ok(QueryWorkload { - metric_name: spec.metric_name, - label_filters: spec.label_filters, - group_by_labels: dims, - aggregations: aggs, + metric_name, + label_filters: merged_filters, + group_by_labels: all_group_by, + aggregations, time_window, repeat_every, accuracy_sla: spec.accuracy_sla, latency_sla, sketch_type_override: spec.sketch_type, + exact_required, + quantiles, }) } } @@ -158,6 +232,7 @@ mod tests { fn basic_spec() -> QuerySpec { QuerySpec { + query_string: None, metric_name: "request_latency".into(), label_filters: [("service".into(), "web".into())].into(), group_by_labels: vec!["host.name".into()], @@ -267,4 +342,111 @@ mod tests { fn trailing_digits_error() { assert!(parse_duration("5").is_err()); } + + // ── query_string path ───────────────────────────────────────────────────── + + /// Build a minimal QuerySpec driven entirely by a query_string. + fn qs_only(query: &str) -> QuerySpec { + QuerySpec { + query_string: Some(query.into()), + metric_name: "".into(), + label_filters: Default::default(), + group_by_labels: vec![], + aggregations: vec![], + time_window: "".into(), + repeat_every: None, + accuracy_sla: 0.01, + latency_sla: None, + sketch_type: None, + workload: Default::default(), + } + } + + /// PromQL query_string auto-populates metric_name, aggregations, + /// time_window, and quantiles — no explicit fields required. + #[test] + fn query_string_promql_populates_workload() { + let w = Analyzer::new() + .analyze(qs_only("sum by (host) (quantile_over_time(0.99, latency[5m]))")) + .unwrap(); + assert_eq!(w.metric_name, "latency"); + assert_eq!(w.aggregations, vec![AggType::Quantile]); + assert_eq!(w.time_window, Duration::from_secs(300)); + assert_eq!(w.quantiles, vec![0.99]); + assert!(!w.exact_required); + } + + /// SQL query_string auto-populates metric_name, aggregations, + /// and group_by_labels. + #[test] + fn query_string_sql_populates_workload() { + let w = Analyzer::new() + .analyze(qs_only( + "SELECT symbol, COUNT(*) FROM financial_last_trade_price GROUP BY symbol", + )) + .unwrap(); + assert_eq!(w.metric_name, "financial_last_trade_price"); + assert_eq!(w.aggregations, vec![AggType::Frequency]); + assert!(w.group_by_labels.contains(&"symbol".to_string())); + } + + /// Explicit metric_name overrides the name derived from query_string. + #[test] + fn explicit_metric_name_overrides_parsed() { + let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); + spec.metric_name = "my_custom_metric".into(); + let w = Analyzer::new().analyze(spec).unwrap(); + assert_eq!(w.metric_name, "my_custom_metric"); + // aggregations still come from parse (avg → DDSketch → Quantile) + assert_eq!(w.aggregations, vec![AggType::Quantile]); + } + + /// Explicit time_window overrides the window derived from query_string. + #[test] + fn explicit_time_window_overrides_parsed() { + let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); + spec.time_window = "1h".into(); + let w = Analyzer::new().analyze(spec).unwrap(); + assert_eq!(w.time_window, Duration::from_secs(3600)); + } + + /// Explicit aggregations override those derived from query_string. + #[test] + fn explicit_aggregations_override_parsed() { + let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); // → Quantile + spec.aggregations = vec!["cardinality".into()]; + let w = Analyzer::new().analyze(spec).unwrap(); + assert_eq!(w.aggregations, vec![AggType::Cardinality]); + } + + /// sum_over_time is a stateful exact aggregation; exact_required is set. + #[test] + fn query_string_exact_required_propagated() { + let w = Analyzer::new() + .analyze(qs_only("sum by (service) (sum_over_time(request_bytes[1h]))")) + .unwrap(); + assert!(w.exact_required, "sum_over_time must set exact_required"); + assert_eq!(w.aggregations, vec![]); + } + + /// DDSketch quantile φ values are surfaced through the workload. + #[test] + fn query_string_quantiles_populated() { + let w = Analyzer::new() + .analyze(qs_only("sum by (host) (quantile_over_time(0.5, latency[5m]))")) + .unwrap(); + assert_eq!(w.quantiles, vec![0.5]); + } + + /// Existing callers that supply all fields explicitly and omit + /// query_string continue to work unchanged (backward compatibility). + #[test] + fn backward_compat_no_query_string() { + let w = Analyzer::new().analyze(basic_spec()).unwrap(); + assert_eq!(w.metric_name, "request_latency"); + assert_eq!(w.aggregations, vec![AggType::Quantile]); + assert_eq!(w.time_window, Duration::from_secs(300)); + assert!(!w.exact_required); + assert!(w.quantiles.is_empty()); + } } diff --git a/controller/src/config/precompute.rs b/controller/src/config/precompute.rs index d3b89aa6..23400684 100644 --- a/controller/src/config/precompute.rs +++ b/controller/src/config/precompute.rs @@ -168,6 +168,8 @@ mod tests { accuracy_sla: 0.01, latency_sla, sketch_type_override: None, + exact_required: false, + quantiles: vec![], } } diff --git a/controller/src/main.rs b/controller/src/main.rs index 53a9b798..09cb5689 100644 --- a/controller/src/main.rs +++ b/controller/src/main.rs @@ -3,6 +3,7 @@ mod config; mod monitor; mod opamp; mod planner; +mod query_parser; mod store; mod types; diff --git a/controller/src/planner/cost_model.rs b/controller/src/planner/cost_model.rs index 6e235fb4..710d7261 100644 --- a/controller/src/planner/cost_model.rs +++ b/controller/src/planner/cost_model.rs @@ -285,6 +285,8 @@ mod tests { accuracy_sla: 0.01, latency_sla: None, sketch_type_override: None, + exact_required: false, + quantiles: vec![], } } diff --git a/controller/src/planner/delta_cost_model.rs b/controller/src/planner/delta_cost_model.rs index 01139173..2d1a9c1a 100644 --- a/controller/src/planner/delta_cost_model.rs +++ b/controller/src/planner/delta_cost_model.rs @@ -472,6 +472,8 @@ mod tests { accuracy_sla: 0.01, latency_sla: None, sketch_type_override: None, + exact_required: false, + quantiles: vec![], } } diff --git a/controller/src/planner/rules.rs b/controller/src/planner/rules.rs index d89d31d1..611dc2b8 100644 --- a/controller/src/planner/rules.rs +++ b/controller/src/planner/rules.rs @@ -17,8 +17,14 @@ impl RulesPlanner { } pub fn plan(&self, w: &QueryWorkload) -> CollectionPlan { + // When exact computation is required (RSI, MACD, stateful indicators), + // skip sketch selection and return a raw-passthrough plan. + if w.exact_required { + return self.raw_passthrough_plan(w); + } + let sketch_type = select_sketch_type(&w.aggregations); - let sketch_params = default_sketch_params(&sketch_type, w.accuracy_sla); + let sketch_params = default_sketch_params_with_quantiles(&sketch_type, w.accuracy_sla, &w.quantiles); let (mode, window_duration) = select_window_strategy(w); let mut aggregate_by = w.group_by_labels.clone(); @@ -64,6 +70,46 @@ impl RulesPlanner { transmission_cost_summary: TransmissionCostSummary::default(), } } + + /// Returns a raw-passthrough plan for queries that require exact per-sample + /// computation (RSI, MACD, stochastic oscillator, etc.). + fn raw_passthrough_plan(&self, w: &QueryWorkload) -> CollectionPlan { + let valid_until = Utc::now() + + chrono::Duration::seconds(self.valid_for.as_secs() as i64); + + let mut label_matchers: Vec = w + .label_filters + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect(); + label_matchers.sort(); + + CollectionPlan { + agent_config: AgentCollectorConfig { + output_mode: OutputMode::Raw, + sketch_type: SketchType::DDSketch, // unused for raw mode + sketch_params: SketchParams::default(), + aggregate_by: vec![], + label_matchers, + window_duration: None, + mode: ProcessorMode::Batch, + enable_self_monitoring: true, + transmit_sketch: false, + drop_original: false, + delta_transmission: false, + delta_threshold: 0.0, + }, + gateway_config: GatewayCollectorConfig { passthrough: true }, + backend_config: BackendCollectorConfig { + merge_sketch_type: SketchType::DDSketch, + group_by: vec![], + }, + precompute: vec![], + valid_until, + delta_decision: DeltaDecision::default(), + transmission_cost_summary: TransmissionCostSummary::default(), + } + } } // ── Sketch selection ────────────────────────────────────────────────────────── @@ -83,32 +129,35 @@ fn select_sketch_type(aggs: &[AggType]) -> SketchType { /// Returns type-appropriate default parameters for the given accuracy SLA. pub fn default_sketch_params(st: &SketchType, accuracy_sla: f64) -> SketchParams { - let acc = if accuracy_sla <= 0.0 { - 0.01 + default_sketch_params_with_quantiles(st, accuracy_sla, &[]) +} + +/// Like [`default_sketch_params`] but seeds the quantiles list from the +/// query-parsed φ values when non-empty; falls back to [0.5, 0.9, 0.99]. +pub fn default_sketch_params_with_quantiles( + st: &SketchType, + accuracy_sla: f64, + query_quantiles: &[f64], +) -> SketchParams { + let acc = if accuracy_sla <= 0.0 { 0.01 } else { accuracy_sla }; + let quantiles: Vec = if !query_quantiles.is_empty() { + query_quantiles.to_vec() } else { - accuracy_sla + vec![0.5, 0.9, 0.99] }; match st { SketchType::DDSketch => SketchParams { relative_accuracy: acc, - quantiles: vec![0.5, 0.9, 0.99], + quantiles, ..Default::default() }, SketchType::KLL => { let k = ((1.0 / acc) as u32).max(32); - SketchParams { - k, - quantiles: vec![0.5, 0.9, 0.99], - ..Default::default() - } + SketchParams { k, quantiles, ..Default::default() } } SketchType::HLL => { - // precision = log2(registers); higher → lower error. let precision = if acc > 0.02 { 10u32 } else { 14u32 }; - SketchParams { - precision, - ..Default::default() - } + SketchParams { precision, ..Default::default() } } SketchType::CountSketch | SketchType::CountMinSketch => SketchParams { rows: 5, @@ -150,6 +199,8 @@ mod tests { accuracy_sla: 0.01, latency_sla: None, sketch_type_override: None, + exact_required: false, + quantiles: vec![], } } diff --git a/controller/src/query_parser/mod.rs b/controller/src/query_parser/mod.rs new file mode 100644 index 00000000..4629b14b --- /dev/null +++ b/controller/src/query_parser/mod.rs @@ -0,0 +1,198 @@ +//! SP-1 query workload extraction — PromQL and SQL parsers. +//! +//! Both parsers compile to the shared [`SketchExpr`] algebra IR defined in +//! [`sketch_algebra`]. The optimizer in [`sketch_rules`] applies algebraic +//! rewrite rules before the planner receives the result. +//! +//! # Entry points +//! +//! | Function | Returns | Use | +//! |---|---|---| +//! | [`parse_query_sketch`] | `SketchExpr` | New callers — full algebra IR | +//! | [`parse_query`] | `ParsedQuery` | Backward compat with existing analyzer | +//! +//! # Supported PromQL patterns (via `promql-parser` AST) +//! - `quantile_over_time(φ, m{f}[w]) by (dims)` +//! - `histogram_quantile(φ, rate(m{f}[w])) by (le)` +//! - `avg/min/max/stddev/stdvar_over_time(m{f}[w]) by (dims)` +//! - `sum/count_over_time(m{f}[w]) by (dims)` +//! - `topk(k, *_over_time(…) by (dims))` +//! - `count(*_over_time(…) by (dims))` — cardinality +//! - `changes/resets(m{f}[w])` +//! - Bare metric selector / binary op → `exact_required` +//! +//! # Supported SQL patterns (doc §SQL Operators) +//! - `COUNT(*)` with/without GROUP BY → frequency / exact +//! - `COUNT(DISTINCT col)` ± GROUP BY → cardinality / Hydra +//! - `AVG/MIN/MAX(col)` ± GROUP BY → quantile / exact extrema +//! - `SUM(col)` → exact +//! - ORDER BY … DESC LIMIT k → heavy-hitter CountSketch +//! - Multiple aggs in one SELECT → all ops collected (Merge) +//! - JOIN … ON key → JoinSketch push-down +//! - UNION ALL → Merge (sketch linearity) + +pub mod promql; +pub mod sql; +pub mod sketch_algebra; +pub mod sketch_rules; + +use std::collections::HashMap; +use std::time::Duration; + +use crate::types::AggType; +pub use sketch_algebra::SketchExpr; + +// ── Output types (legacy — consumed by analyzer and planner) ────────────────── + +/// Flat intermediate representation consumed by [`crate::analyzer::Analyzer`]. +/// +/// Produced by [`parse_query`] via [`SketchExpr::to_parsed_query`]. +/// New code should use [`parse_query_sketch`] → [`SketchExpr`] directly. +#[derive(Debug, Clone)] +pub struct ParsedQuery { + /// Metric name (PromQL: from selector; SQL: FROM clause table). + pub metric_name: String, + /// Aggregation types inferred from the query. + pub aggregations: Vec, + /// Dimensions that must be preserved for GROUP BY / `by (dims)`. + pub group_by_labels: Vec, + /// Equality label filters extracted from the query. + pub label_filters: HashMap, + /// Time window extracted from the range vector or query context. + pub time_window: Duration, + /// True when the query requires per-sample exact values. + pub exact_required: bool, + /// Quantile φ values implied by the query. + pub quantiles: Vec, + /// Named pattern hint for domain-specific planner defaults. + pub hint: Option, +} + +/// Named query pattern recognised by the DEBS-aware planner. +#[derive(Debug, Clone)] +pub enum QueryHint { + // ── DEBS 2022 financial queries ─────────────────────────────────────────── + /// Q1 – per-symbol EMA via quantile proxy (DDSketch / KLL). + DebsEma, + /// Q3 – top-K symbols by event count or price move (CountSketch). + DebsTopK { k: u64 }, + /// Q4 – per-symbol high / low / last / range (extreme-quantile DDSketch). + DebsPriceStats, + /// Q5 / Q9 – realized volatility / Bollinger bands via IQR proxy. + DebsVolatility, + /// Q6 – distinct active symbols per window (HLL). + DebsCardinality, + /// Q7 – TWAP as median / p50 (DDSketch). + DebsTwap, + /// Q8 – price anomaly detection via IQR (DDSketch). + DebsAnomaly, + // ── Exact-only patterns ─────────────────────────────────────────────────── + /// Query requires stateful per-sample computation; no sketch benefit. + ExactRequired { reason: String }, +} + +// ── Public entry points ─────────────────────────────────────────────────────── + +/// Parse a raw query string (PromQL or SQL) into the full [`SketchExpr`] IR. +/// +/// The returned tree has already been through the algebraic optimizer +/// ([`sketch_rules::optimize`]). +pub fn parse_query_sketch(query: &str) -> anyhow::Result { + let q = query.trim(); + let upper = q.to_ascii_uppercase(); + if upper.starts_with("SELECT") || upper.starts_with("WITH") { + sql::parse_sql(q) + } else { + promql::parse_promql(q) + } +} + +/// Parse a raw query string (PromQL or SQL) into a [`ParsedQuery`]. +/// +/// This is the backward-compatible entry point for the existing +/// [`crate::analyzer::Analyzer`]. Internally it calls [`parse_query_sketch`] +/// and converts via [`SketchExpr::to_parsed_query`]. +pub fn parse_query(query: &str) -> anyhow::Result { + Ok(parse_query_sketch(query)?.to_parsed_query()) +} + +// ── DEBS hint classifier (shared by both parsers via to_parsed_query) ───────── + +/// Returns the DEBS-specific hint for `financial.last_trade_price` queries. +pub(super) fn debs_hint( + metric: &str, + aggs: &[AggType], + quantiles: &[f64], + exact_required: bool, + topk: Option, +) -> Option { + let is_debs = metric == "financial.last_trade_price" + || metric == "financial_last_trade_price"; + if !is_debs { return None; } + + if exact_required { + return Some(QueryHint::ExactRequired { + reason: "query requires per-sample stateful computation".into(), + }); + } + if let Some(k) = topk { + return Some(QueryHint::DebsTopK { k }); + } + let primary = aggs.first()?; + match primary { + AggType::Cardinality => Some(QueryHint::DebsCardinality), + AggType::Frequency => Some(QueryHint::DebsTopK { k: 10 }), + AggType::Quantile => { + let qs: std::collections::HashSet = quantiles + .iter() + .map(|&q| (q * 100.0).round() as i32) + .collect(); + if qs.contains(&50) && qs.len() == 1 { + Some(QueryHint::DebsTwap) + } else if qs.contains(&0) || qs.contains(&100) { + Some(QueryHint::DebsPriceStats) + } else if qs.contains(&25) && qs.contains(&75) && qs.contains(&50) { + Some(QueryHint::DebsAnomaly) + } else if qs.contains(&25) && qs.contains(&75) { + Some(QueryHint::DebsVolatility) + } else { + Some(QueryHint::DebsEma) + } + } + } +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + // Smoke tests for the unified entry point. + + #[test] + fn sql_dispatched_correctly() { + let pq = parse_query("SELECT COUNT(*) FROM hits GROUP BY AdvEngineID").unwrap(); + assert!(pq.aggregations.contains(&AggType::Frequency)); + } + + #[test] + fn promql_dispatched_correctly() { + // `by` belongs to the aggregate operator, not the function call. + let pq = parse_query( + "sum by (host) (quantile_over_time(0.99, latency[5m]))" + ).unwrap(); + assert!(pq.aggregations.contains(&AggType::Quantile)); + assert_eq!(pq.quantiles, vec![0.99]); + } + + #[test] + fn parse_query_sketch_returns_expr() { + let expr = parse_query_sketch( + "topk by (symbol) (10, count_over_time(financial_last_trade_price[5m]))" + ).unwrap(); + // Should have been optimized — result is some SketchExpr tree. + // Just check it doesn't error. + let _ = expr.to_parsed_query(); + } +} diff --git a/controller/src/query_parser/promql.rs b/controller/src/query_parser/promql.rs new file mode 100644 index 00000000..fda9bc50 --- /dev/null +++ b/controller/src/query_parser/promql.rs @@ -0,0 +1,645 @@ +//! PromQL → SketchExpr compiler. +//! +//! Uses the `promql-parser` crate (GreptimeTeam) for a full AST parse, then +//! walks the expression tree to emit a [`SketchExpr`] following the mapping +//! rules in `docs/sketch-algebra-query-mapping.md §2.3`. +//! +//! # PromQL → Sketch mapping (summary) +//! +//! | Expression | SketchAggOp | +//! |---|---| +//! | `quantile_over_time(φ, m[w])` | DDSketch([φ]) | +//! | `histogram_quantile(φ, rate(m[w]))` | DDSketch([φ]) | +//! | `avg_over_time(m[w])` | DDSketch([0.5]) | +//! | `min_over_time(m[w]) by (d)` | DDSketch([0.0]) | +//! | `max_over_time(m[w]) by (d)` | DDSketch([1.0]) | +//! | `min/max_over_time(m[w])` (no by) | ExactMinMax | +//! | `stddev/stdvar_over_time(m[w])` | DDSketch([0.25,0.75]) | +//! | `count_over_time(m[w])` | CountMin | +//! | `sum_over_time(m[w])` | Exact(Sum) | +//! | `last_over_time / delta / deriv / predict_linear` | Exact (stateful) | +//! | `changes / resets` | CountMin | +//! | `topk(k, …)` outer | CountSketch(k) | +//! | `count(…over_time… by (d))` outer | HLL | +//! | `m{filters}` bare | Exact (required) | +//! | `m_a op m_b` binary | Exact (required) | + +use std::time::Duration; + +use anyhow::anyhow; +use promql_parser::parser::{self, AggregateExpr, Call, Expr, LabelModifier, MatrixSelector, VectorSelector}; + +use super::sketch_algebra::{ + ColumnRef, FilterOp, FilterVal, PartitionKeys, Predicate, SketchAggOp, SketchExpr, SourceSpec, +}; +use super::sketch_rules::optimize; + +// ── Public entry point ──────────────────────────────────────────────────────── + +/// Parse a PromQL expression string into an optimised [`SketchExpr`]. +pub fn parse_promql(query: &str) -> anyhow::Result { + let expr = parser::parse(query) + .map_err(|e| anyhow!("PromQL parse error: {e}"))?; + let sketch = walk(&expr, WalkCtx::default())?; + Ok(optimize(sketch)) +} + +// ── Walk context ────────────────────────────────────────────────────────────── + +/// Context accumulated as we descend the AST. +#[derive(Default, Clone)] +struct WalkCtx { + /// GROUP BY / `without` clause from an outer Aggregate node. + partition: Option, + /// Top-K k from an outer `topk` / `bottomk` operator. + topk: Option, + /// Whether the outer context is a `count()` aggregate (→ HLL). + outer_count: bool, +} + +// ── AST walker ──────────────────────────────────────────────────────────────── + +fn walk(expr: &Expr, ctx: WalkCtx) -> anyhow::Result { + match expr { + // ── Aggregate operators: topk, count, sum by, avg by, … ─────────── + Expr::Aggregate(agg) => walk_aggregate(agg, ctx), + + // ── Function calls: *_over_time, histogram_quantile, rate, … ────── + Expr::Call(call) => walk_call(call, ctx), + + // ── Binary operations: metric_a / metric_b → exact ──────────────── + Expr::Binary(bin) => { + // Binary op between two series: both sides need exact values. + let left = walk(bin.lhs.as_ref(), WalkCtx::default())?; + let right = walk(bin.rhs.as_ref(), WalkCtx::default())?; + // Wrap both in a Merge that signals exact requirement to callers. + Ok(SketchExpr::Agg { + op: SketchAggOp::Exact(super::sketch_algebra::ExactAgg::Sum), // placeholder + col: ColumnRef::SampleValue, + input: Box::new(SketchExpr::Merge { inputs: vec![left, right] }), + }) + } + + // ── Parenthesised ───────────────────────────────────────────────── + Expr::Paren(p) => walk(p.expr.as_ref(), ctx), + + // ── Subquery: metric[5m:1m] — treat as windowed frequency ───────── + Expr::Subquery(sq) => { + let inner = walk(sq.expr.as_ref(), ctx.clone())?; + Ok(SketchExpr::Window { duration: sq.range, input: Box::new(inner) }) + } + + // ── Bare vector selector ────────────────────────────────────────── + Expr::VectorSelector(vs) => { + let (name, filters) = extract_vs_info(vs); + let source = SketchExpr::Source(SourceSpec { name }); + let filtered = apply_filters(source, filters); + // Wrap with partition and exact agg. + let agg = SketchExpr::Agg { + op: SketchAggOp::Exact(super::sketch_algebra::ExactAgg::Sum), + col: ColumnRef::SampleValue, + input: Box::new(filtered), + }; + Ok(apply_partition(agg, ctx.partition)) + } + + // ── Number / string literals — only appear as args inside Call ──── + Expr::NumberLiteral(_) | Expr::StringLiteral(_) => { + Err(anyhow!("unexpected literal at top level of PromQL expression")) + } + + // ── Extension / unknown ─────────────────────────────────────────── + #[allow(unreachable_patterns)] + _ => Err(anyhow!("unsupported PromQL expression type")), + } +} + +// ── Aggregate operator walk ─────────────────────────────────────────────────── + +fn walk_aggregate( + agg: &AggregateExpr, + ctx: WalkCtx, +) -> anyhow::Result { + // Extract partition keys from the by/without modifier. + let partition = agg.modifier.as_ref().map(modifier_to_partition); + + // Extract the operator name from the token via Display (gives lowercase). + let op_name = format!("{}", agg.op); + + match op_name.as_str() { + // topk(k, inner) / bottomk(k, inner) → CountSketch(k) + "topk" | "bottomk" => { + let k = extract_number_param(&agg.param)? as u64; + let inner_ctx = WalkCtx { partition: partition.clone(), topk: Some(k), outer_count: false }; + let inner = walk(agg.expr.as_ref(), inner_ctx)?; + let result = SketchExpr::TopK { k, input: Box::new(inner) }; + Ok(apply_partition(result, partition)) + } + + // count(inner_by) → HLL cardinality of distinct groups + "count" => { + let inner_ctx = WalkCtx { partition: partition.clone(), topk: None, outer_count: true }; + let inner = walk(agg.expr.as_ref(), inner_ctx)?; + // Wrap with HLL at this level. + let result = SketchExpr::Agg { + op: SketchAggOp::default_hll(), + col: ColumnRef::SampleValue, + input: Box::new(inner), + }; + Ok(apply_partition(result, partition)) + } + + // sum by (d) (inner) — outer sum doesn't change the inner sketch type. + // The inner *_over_time already chose the right sketch; we just set partition. + "sum" | "avg" | "min" | "max" | "group" => { + let inner_ctx = WalkCtx { partition: partition.clone(), topk: ctx.topk, outer_count: false }; + let inner = walk(agg.expr.as_ref(), inner_ctx)?; + Ok(apply_partition(inner, partition)) + } + + // stddev by (d) / stdvar by (d) → DDSketch IQR proxy + "stddev" | "stdvar" => { + let inner_ctx = WalkCtx { partition: partition.clone(), topk: None, outer_count: false }; + let inner = walk(agg.expr.as_ref(), inner_ctx)?; + let result = SketchExpr::Agg { + op: SketchAggOp::default_ddsketch(vec![0.25, 0.75]), + col: ColumnRef::SampleValue, + input: Box::new(inner), + }; + Ok(apply_partition(result, partition)) + } + + // quantile(φ, inner) → DDSketch(φ) + "quantile" => { + let phi = extract_number_param(&agg.param)?; + let inner_ctx = WalkCtx { partition: partition.clone(), topk: None, outer_count: false }; + let inner = walk(agg.expr.as_ref(), inner_ctx)?; + let result = SketchExpr::Agg { + op: SketchAggOp::default_ddsketch(vec![phi]), + col: ColumnRef::SampleValue, + input: Box::new(inner), + }; + Ok(apply_partition(result, partition)) + } + + other => Err(anyhow!("unsupported PromQL aggregate operator: {other}")), + } +} + +// ── Function call walk ──────────────────────────────────────────────────────── + +fn walk_call( + call: &Call, + ctx: WalkCtx, +) -> anyhow::Result { + let name = call.func.name; + + match name { + // ── quantile_over_time(φ, m{f}[w]) ─────────────────────────────── + "quantile_over_time" => { + let phi = extract_call_num_arg(call, 0)?; + let (source, filters, window) = extract_matrix_arg(call, 1)?; + Ok(build_sketched( + source, filters, window, + SketchAggOp::default_ddsketch(vec![phi]), + ctx, + )) + } + + // ── histogram_quantile(φ, rate(m{f}[w])) ───────────────────────── + "histogram_quantile" => { + let phi = extract_call_num_arg(call, 0)?; + // The second arg is a call to rate/irate wrapping a MatrixSelector. + let rate_expr = call.args.args[1].as_ref(); + let (source, filters, window) = extract_inner_matrix(rate_expr)?; + Ok(build_sketched( + source, filters, window, + SketchAggOp::default_ddsketch(vec![phi]), + ctx, + )) + } + + // ── avg_over_time ───────────────────────────────────────────────── + "avg_over_time" => { + let (source, filters, window) = extract_matrix_arg(call, 0)?; + Ok(build_sketched(source, filters, window, SketchAggOp::default_ddsketch(vec![0.5]), ctx)) + } + + // ── min_over_time ───────────────────────────────────────────────── + "min_over_time" => { + let (source, filters, window) = extract_matrix_arg(call, 0)?; + let op = if ctx.partition.as_ref().map(|p| !p.is_empty()).unwrap_or(false) { + SketchAggOp::default_ddsketch(vec![0.0]) + } else { + SketchAggOp::ExactMinMax { min: true, max: false } + }; + Ok(build_sketched(source, filters, window, op, ctx)) + } + + // ── max_over_time ───────────────────────────────────────────────── + "max_over_time" => { + let (source, filters, window) = extract_matrix_arg(call, 0)?; + let op = if ctx.partition.as_ref().map(|p| !p.is_empty()).unwrap_or(false) { + SketchAggOp::default_ddsketch(vec![1.0]) + } else { + SketchAggOp::ExactMinMax { min: false, max: true } + }; + Ok(build_sketched(source, filters, window, op, ctx)) + } + + // ── stddev_over_time / stdvar_over_time → IQR proxy ────────────── + "stddev_over_time" | "stdvar_over_time" => { + let (source, filters, window) = extract_matrix_arg(call, 0)?; + Ok(build_sketched( + source, filters, window, + SketchAggOp::default_ddsketch(vec![0.25, 0.75]), + ctx, + )) + } + + // ── count_over_time ─────────────────────────────────────────────── + "count_over_time" => { + let (source, filters, window) = extract_matrix_arg(call, 0)?; + // Outer count() context → HLL (cardinality of distinct groups). + let op = if ctx.outer_count { + SketchAggOp::default_hll() + } else { + SketchAggOp::default_count_min() + }; + Ok(build_sketched(source, filters, window, op, ctx)) + } + + // ── sum_over_time ───────────────────────────────────────────────── + "sum_over_time" => { + let (source, filters, window) = extract_matrix_arg(call, 0)?; + Ok(build_sketched( + source, filters, window, + SketchAggOp::Exact(super::sketch_algebra::ExactAgg::Sum), + ctx, + )) + } + + // ── last_over_time / stateful functions → exact ─────────────────── + "last_over_time" | "present_over_time" | "absent_over_time" => { + let (source, filters, window) = extract_matrix_arg(call, 0)?; + Ok(build_sketched( + source, filters, window, + SketchAggOp::Exact(super::sketch_algebra::ExactAgg::Sum), + ctx, + )) + } + + // ── delta / idelta / deriv / predict_linear → stateful exact ────── + "delta" | "idelta" | "deriv" | "predict_linear" => { + let (source, filters, window) = extract_matrix_arg(call, 0)?; + Ok(build_sketched( + source, filters, window, + SketchAggOp::Exact(super::sketch_algebra::ExactAgg::Sum), + ctx, + )) + } + + // ── changes / resets → frequency ────────────────────────────────── + "changes" | "resets" => { + let (source, filters, window) = extract_matrix_arg(call, 0)?; + Ok(build_sketched(source, filters, window, SketchAggOp::default_count_min(), ctx)) + } + + // ── rate / irate / increase — pass-through, inner carries the sketch + "rate" | "irate" | "increase" => { + if call.args.is_empty() { + return Err(anyhow!("rate/irate/increase requires a matrix arg")); + } + let (source, filters, window) = extract_inner_matrix(call.args.args[0].as_ref())?; + Ok(build_sketched( + source, filters, window, + SketchAggOp::default_count_min(), + ctx, + )) + } + + other => Err(anyhow!("unsupported PromQL function: {other}")), + } +} + +// ── Helpers: MatrixSelector extraction ─────────────────────────────────────── + +/// Extract `(metric_name, filters, window)` from a MatrixSelector argument at +/// position `arg_idx` of a Call. +fn extract_matrix_arg( + call: &Call, + arg_idx: usize, +) -> anyhow::Result<(String, Vec, Duration)> { + let arg = call.args.args.get(arg_idx) + .map(|b| b.as_ref()) + .ok_or_else(|| anyhow!("missing arg {} in call to {}", arg_idx, call.func.name))?; + extract_inner_matrix(arg) +} + +/// Walk into an expression until we find a MatrixSelector, then extract its info. +fn extract_inner_matrix(expr: &Expr) -> anyhow::Result<(String, Vec, Duration)> { + match expr { + Expr::MatrixSelector(ms) => { + let (name, filters) = extract_vs_info(&ms.vs); + Ok((name, filters, ms.range)) + } + Expr::Paren(p) => extract_inner_matrix(p.expr.as_ref()), + Expr::Call(c) => { + // rate/irate wraps a MatrixSelector. + extract_inner_matrix(c.args.args[0].as_ref()) + } + other => Err(anyhow!("expected MatrixSelector, got {:?}", std::mem::discriminant(other))), + } +} + +// ── Helpers: VectorSelector info ───────────────────────────────────────────── + +fn extract_vs_info(vs: &VectorSelector) -> (String, Vec) { + // Metric name: prefer the explicit name field, fall back to __name__ matcher. + let name = vs.name.clone().unwrap_or_else(|| { + vs.matchers.matchers.iter() + .find(|m| m.name == "__name__") + .map(|m| m.value.clone()) + .unwrap_or_default() + }); + + let filters = vs.matchers.matchers.iter() + .filter(|m| m.name != "__name__") + .filter_map(matcher_to_predicate) + .collect(); + + (name, filters) +} + +fn matcher_to_predicate(m: &promql_parser::label::Matcher) -> Option { + use promql_parser::label::MatchOp; + let (op, val) = match &m.op { + MatchOp::Equal => (FilterOp::Eq, FilterVal::Str(m.value.clone())), + MatchOp::NotEqual => (FilterOp::Ne, FilterVal::Str(m.value.clone())), + MatchOp::Re(re) => (FilterOp::Regex(re.to_string()), FilterVal::Str(m.value.clone())), + MatchOp::NotRe(re)=> (FilterOp::NotRegex(re.to_string()), FilterVal::Str(m.value.clone())), + }; + Some(Predicate { col: m.name.clone(), op, val }) +} + +// ── Helpers: number extraction ──────────────────────────────────────────────── + +fn extract_call_num_arg(call: &Call, idx: usize) -> anyhow::Result { + match call.args.args.get(idx).map(|b| b.as_ref()) { + Some(Expr::NumberLiteral(n)) => Ok(n.val), + Some(other) => Err(anyhow!( + "expected number at arg {} of {}, got {:?}", + idx, call.func.name, std::mem::discriminant(other) + )), + None => Err(anyhow!("missing arg {} in {}", idx, call.func.name)), + } +} + +fn extract_number_param(param: &Option>) -> anyhow::Result { + match param { + Some(e) => match e.as_ref() { + Expr::NumberLiteral(n) => Ok(n.val), + other => Err(anyhow!("expected number param, got {:?}", std::mem::discriminant(other))), + }, + None => Err(anyhow!("missing required numeric parameter")), + } +} + +// ── Helpers: PartitionKeys from LabelModifier ───────────────────────────────── + +fn modifier_to_partition(modifier: &LabelModifier) -> PartitionKeys { + match modifier { + LabelModifier::Include(labels) => PartitionKeys::By(labels.labels.clone()), + LabelModifier::Exclude(labels) => PartitionKeys::Without(labels.labels.clone()), + } +} + +// ── Tree builders ───────────────────────────────────────────────────────────── + +/// Build the standard `Partition(Window(Filter(Agg(Source))))` tree. +fn build_sketched( + metric: String, + filters: Vec, + window: Duration, + op: SketchAggOp, + ctx: WalkCtx, +) -> SketchExpr { + let source = SketchExpr::Source(SourceSpec { name: metric }); + let filtered = apply_filters(source, filters); + let windowed = SketchExpr::Window { duration: window, input: Box::new(filtered) }; + + let agg = if let Some(k) = ctx.topk { + // Outer topk → CountSketch regardless of what op was chosen. + SketchExpr::Agg { + op: SketchAggOp::CountSketch { k }, + col: ColumnRef::SampleValue, + input: Box::new(windowed), + } + } else { + SketchExpr::Agg { op, col: ColumnRef::SampleValue, input: Box::new(windowed) } + }; + + apply_partition(agg, ctx.partition) +} + +fn apply_filters(input: SketchExpr, pred: Vec) -> SketchExpr { + if pred.is_empty() { + input + } else { + SketchExpr::Filter { pred, input: Box::new(input) } + } +} + +fn apply_partition(input: SketchExpr, partition: Option) -> SketchExpr { + match partition { + None => input, + Some(p) if p.is_empty() => input, + Some(keys) => SketchExpr::Partition { keys, input: Box::new(input) }, + } +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use super::super::sketch_algebra::{ExactAgg, SketchAggOp}; + use crate::types::AggType; + + fn parse(q: &str) -> SketchExpr { + parse_promql(q).unwrap_or_else(|e| panic!("parse_promql failed: {e}\nquery={q:?}")) + } + + fn pq(q: &str) -> super::super::ParsedQuery { + parse(q).to_parsed_query() + } + + // ── quantile_over_time ──────────────────────────────────────────────────── + + #[test] + fn quantile_over_time_basic() { + // PromQL: `by` is part of the aggregate operator, not the function call. + let pq = pq("sum by (host) (quantile_over_time(0.99, latency{service=\"web\"}[5m]))"); + assert_eq!(pq.aggregations, vec![AggType::Quantile]); + assert_eq!(pq.quantiles, vec![0.99]); + assert_eq!(pq.group_by_labels, vec!["host"]); + assert_eq!(pq.label_filters.get("service").map(String::as_str), Some("web")); + assert_eq!(pq.time_window, Duration::from_secs(300)); + } + + #[test] + fn quantile_over_time_debs_ema() { + // Dotted names are invalid PromQL; use underscores. + let pq = pq("sum by (symbol) (quantile_over_time(0.5, financial_last_trade_price[5m]))"); + assert_eq!(pq.aggregations, vec![AggType::Quantile]); + assert_eq!(pq.quantiles, vec![0.5]); + assert_eq!(pq.group_by_labels, vec!["symbol"]); + } + + // ── histogram_quantile ──────────────────────────────────────────────────── + + #[test] + fn histogram_quantile_via_rate() { + let pq = pq("histogram_quantile(0.95, rate(http_duration_seconds_bucket[5m]))"); + assert_eq!(pq.aggregations, vec![AggType::Quantile]); + assert_eq!(pq.quantiles, vec![0.95]); + } + + // ── avg_over_time ───────────────────────────────────────────────────────── + + #[test] + fn avg_over_time_maps_to_p50() { + let pq = pq("avg by (symbol) (avg_over_time(financial_last_trade_price[5m]))"); + assert_eq!(pq.aggregations, vec![AggType::Quantile]); + assert_eq!(pq.quantiles, vec![0.5]); + } + + // ── min/max_over_time ───────────────────────────────────────────────────── + + #[test] + fn min_over_time_with_by_is_ddsketch() { + let pq = pq("min by (symbol) (min_over_time(financial_last_trade_price[5m]))"); + assert_eq!(pq.aggregations, vec![AggType::Quantile]); + assert_eq!(pq.quantiles, vec![0.0]); + } + + #[test] + fn max_over_time_with_by_is_ddsketch() { + let pq = pq("max by (symbol) (max_over_time(financial_last_trade_price[5m]))"); + assert_eq!(pq.aggregations, vec![AggType::Quantile]); + assert_eq!(pq.quantiles, vec![1.0]); + } + + // ── topk ────────────────────────────────────────────────────────────────── + + #[test] + fn topk_count_over_time() { + let pq = pq("topk by (symbol) (10, count_over_time(financial_last_trade_price[5m]))"); + assert_eq!(pq.aggregations, vec![AggType::Frequency]); + assert_eq!(pq.group_by_labels, vec!["symbol"]); + } + + #[test] + fn topk_avg_over_time() { + let expr = parse("topk by (host) (5, avg_over_time(cpu[5m]))"); + let pq = expr.to_parsed_query(); + assert_eq!(pq.aggregations, vec![AggType::Frequency]); + } + + // ── count cardinality ───────────────────────────────────────────────────── + + #[test] + fn count_count_over_time_is_hll() { + let pq = pq("count by (symbol) (count_over_time(financial_last_trade_price[5m]))"); + assert_eq!(pq.aggregations, vec![AggType::Cardinality]); + } + + // ── stddev_over_time ────────────────────────────────────────────────────── + + #[test] + fn stddev_over_time_iqr_proxy() { + let pq = pq("avg by (host) (stddev_over_time(cpu[5m]))"); + assert_eq!(pq.aggregations, vec![AggType::Quantile]); + assert!(pq.quantiles.contains(&0.25) && pq.quantiles.contains(&0.75)); + } + + // ── sum_over_time → exact ───────────────────────────────────────────────── + + #[test] + fn sum_over_time_exact() { + let pq = pq("sum by (service) (sum_over_time(request_bytes[1h]))"); + assert!(pq.exact_required); + } + + // ── label filters ───────────────────────────────────────────────────────── + + #[test] + fn label_eq_filter() { + let pq = pq(r#"sum by (service) (count_over_time(hits{env="prod"}[5m]))"#); + assert_eq!(pq.label_filters.get("env").map(String::as_str), Some("prod")); + } + + // ── duration parsing ────────────────────────────────────────────────────── + + #[test] + fn duration_1h() { + let pq = pq("avg by (host) (avg_over_time(cpu[1h]))"); + assert_eq!(pq.time_window, Duration::from_secs(3600)); + } + + // ── DEBS hints ──────────────────────────────────────────────────────────── + + #[test] + fn debs_price_stats_min() { + use super::super::QueryHint; + let pq = pq("min by (symbol) (min_over_time(financial_last_trade_price[5m]))"); + assert!(matches!(pq.hint, Some(QueryHint::DebsPriceStats))); + } + + #[test] + fn debs_cardinality() { + use super::super::QueryHint; + let pq = pq("count by (symbol) (count_over_time(financial_last_trade_price[5m]))"); + assert!(matches!(pq.hint, Some(QueryHint::DebsCardinality))); + } + + // ── Complex queries ─────────────────────────────────────────────────────── + + #[test] + fn complex_topk_count_over_time_multi_label() { + // topk absorbs CountSketch (R8); multiple label filters extracted + let pq = pq( + r#"topk by (service) (10, count_over_time(http_requests_total{status="500",env="prod"}[5m]))"#, + ); + assert_eq!(pq.aggregations, vec![AggType::Frequency]); + assert_eq!(pq.group_by_labels, vec!["service"]); + assert_eq!( + pq.label_filters.get("status").map(String::as_str), + Some("500") + ); + assert_eq!( + pq.label_filters.get("env").map(String::as_str), + Some("prod") + ); + assert_eq!(pq.time_window, Duration::from_secs(300)); + } + + #[test] + fn complex_histogram_quantile_multi_label() { + // histogram_quantile wraps rate → DDSketch; two label selectors + let pq = pq( + r#"histogram_quantile(0.99, rate(request_duration_seconds_bucket{service="checkout",region="us-east"}[10m]))"#, + ); + assert_eq!(pq.aggregations, vec![AggType::Quantile]); + assert_eq!(pq.quantiles, vec![0.99]); + assert_eq!( + pq.label_filters.get("service").map(String::as_str), + Some("checkout") + ); + assert_eq!( + pq.label_filters.get("region").map(String::as_str), + Some("us-east") + ); + assert_eq!(pq.time_window, Duration::from_secs(600)); + } +} diff --git a/controller/src/query_parser/sketch_algebra.rs b/controller/src/query_parser/sketch_algebra.rs new file mode 100644 index 00000000..9fa25ff3 --- /dev/null +++ b/controller/src/query_parser/sketch_algebra.rs @@ -0,0 +1,721 @@ +//! Sketch algebra — the shared intermediate representation (IR) that both the +//! PromQL and SQL parsers compile to. +//! +//! Both parsers are pure front-ends: they walk their respective ASTs and emit +//! a [`SketchExpr`] tree. The optimizer in [`super::sketch_rules`] then +//! applies algebraic rewrite rules before the planner converts the tree to +//! agent configurations. +//! +//! # Operator summary +//! +//! | Operator | Symbol | Description | +//! |---|---|---| +//! | `Source` | — | Base relation or metric stream | +//! | `Filter` | σ | Push-down predicates (WHERE / label matchers) | +//! | `Window` | ψ | Time window (PromQL range; SQL time predicate) | +//! | `Partition` | γ | GROUP BY / `by (dims)` — one sketch per key-tuple | +//! | `Agg` | α | The sketch aggregation itself | +//! | `Dedup` | δ | Deduplicate before ingestion (push-down DISTINCT) | +//! | `TopK` | τ | Retain only the top-K entries | +//! | `Merge` | ⊕ | Merge sketches from multiple branches | +//! | `JoinSketch` | ⋈ₛₖ | Pre-agg sketch on inner side, merge after join | + +use std::collections::HashMap; +use std::time::Duration; + +use crate::types::AggType; +use super::{ParsedQuery, QueryHint, debs_hint}; + +// ── Core IR ─────────────────────────────────────────────────────────────────── + +/// Abstract sketch algebra expression — shared IR for SQL and PromQL. +#[derive(Debug, Clone)] +pub enum SketchExpr { + /// Base relation / metric stream. + Source(SourceSpec), + + /// σ — filter input before any sketch build (WHERE / PromQL label matchers). + Filter { + pred: Vec, + input: Box, + }, + + /// ψ — time window (PromQL range vector `[5m]`; SQL sliding-window predicate). + Window { + duration: Duration, + input: Box, + }, + + /// γ — partition by keys; one sketch instance per distinct key-tuple. + /// Use [`PartitionKeys::Without`] when the PromQL `without (...)` clause is present. + Partition { + keys: PartitionKeys, + input: Box, + }, + + /// α — the sketch aggregation operator. + Agg { + op: SketchAggOp, + col: ColumnRef, + input: Box, + }, + + /// δ — deduplicate on `col` before ingestion. + /// Note: for [`SketchAggOp::HLL`] this node is eliminated by rule R6. + Dedup { + col: String, + input: Box, + }, + + /// τ — top-K post-sketch filter. + TopK { + k: u64, + input: Box, + }, + + /// ⊕ — merge sketches from multiple independent branches. + /// All input [`SketchAggOp`]s must be [`SketchAggOp::is_mergeable`]. + Merge { + inputs: Vec, + }, + + /// ⋈ₛₖ — join push-down: + /// pre-aggregate sketch on inner side by join key, merge after join. + JoinSketch { + join_key: String, + outer: Box, + /// Inner carries its own `Partition` + `Agg` nodes. + inner: Box, + }, +} + +// ── Source ──────────────────────────────────────────────────────────────────── + +#[derive(Debug, Clone)] +pub struct SourceSpec { + /// Table name (SQL) or metric name (PromQL). + pub name: String, +} + +// ── Partition keys ──────────────────────────────────────────────────────────── + +/// How the stream is partitioned. +#[derive(Debug, Clone)] +pub enum PartitionKeys { + /// `by (k1, k2, ...)` — explicit key list. + By(Vec), + /// `without (k1, k2, ...)` — complement; resolved against schema at plan time. + Without(Vec), +} + +impl PartitionKeys { + pub fn keys(&self) -> &[String] { + match self { + PartitionKeys::By(k) | PartitionKeys::Without(k) => k, + } + } + + pub fn is_empty(&self) -> bool { + self.keys().is_empty() + } + + pub fn into_by_keys(self) -> Vec { + match self { + PartitionKeys::By(k) => k, + // For Without, return empty — caller resolves complement. + PartitionKeys::Without(k) => k, + } + } +} + +// ── Column reference ────────────────────────────────────────────────────────── + +/// Which column / field the sketch aggregation targets. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ColumnRef { + /// Explicit column name (SQL: `AVG(price)` → `Named("price")`). + Named(String), + /// The implicit metric sample value (PromQL — always the series value). + SampleValue, + /// All rows / COUNT(*). + Wildcard, +} + +// ── Sketch aggregation operators ────────────────────────────────────────────── + +/// The concrete sketch type used for aggregation. +#[derive(Debug, Clone, PartialEq)] +pub enum SketchAggOp { + /// Count-Min Sketch — frequency per group (COUNT(*) GROUP BY). + CountMin { width: u32, depth: u8 }, + + /// Count Sketch (heavy-hitter) — top-K by frequency. + CountSketch { k: u64 }, + + /// HyperLogLog — distinct-value counting (COUNT DISTINCT). + HLL { registers: u8 }, + + /// DDSketch — quantile estimation. + /// `quantiles` holds the φ values to track; `epsilon` is relative error. + DDSketch { quantiles: Vec, epsilon: f64 }, + + /// Exact running min/max tracker — cheaper than DDSketch for extrema + /// without a GROUP BY (no sketch benefit for global extrema). + ExactMinMax { min: bool, max: bool }, + + /// Hydra — sketch of sketches for multi-dimensional GROUP BY. + /// Maintains one `inner` sketch per distinct `partition_keys` tuple. + Hydra { + inner: Box, + partition_keys: Vec, + }, + + /// Exact passthrough — no sketch benefit (SUM, global COUNT, etc.). + Exact(ExactAgg), +} + +/// Exact (non-sketch) aggregation kinds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExactAgg { + Count, + Sum, + /// **Not mergeable** — carries `(sum, count)` in distributed contexts. + Avg, + Min, + Max, +} + +impl SketchAggOp { + /// Returns `true` when two instances of this sketch can be merged + /// (i.e., `sketch(A ∪ B) = merge(sketch(A), sketch(B))`). + pub fn is_mergeable(&self) -> bool { + match self { + SketchAggOp::Exact(ExactAgg::Avg) => false, + SketchAggOp::Hydra { inner, .. } => inner.is_mergeable(), + _ => true, + } + } + + /// Map to the coarse [`AggType`] used by the legacy planner. + pub fn to_agg_type(&self) -> AggType { + match self { + SketchAggOp::HLL { .. } => AggType::Cardinality, + SketchAggOp::CountMin { .. } | SketchAggOp::CountSketch { .. } => AggType::Frequency, + SketchAggOp::DDSketch { .. } | SketchAggOp::ExactMinMax { .. } => AggType::Quantile, + SketchAggOp::Hydra { inner, .. } => inner.to_agg_type(), + SketchAggOp::Exact(_) => AggType::Quantile, + } + } + + /// Extract quantile φ values for DDSketch operators. + pub fn quantiles(&self) -> Vec { + match self { + SketchAggOp::DDSketch { quantiles, .. } => quantiles.clone(), + SketchAggOp::Hydra { inner, .. } => inner.quantiles(), + _ => vec![], + } + } + + /// Whether this op implies `exact_required` (no sketch benefit). + pub fn is_exact(&self) -> bool { + matches!(self, SketchAggOp::Exact(_) | SketchAggOp::ExactMinMax { .. }) + } +} + +// ── Default sketch parameters ───────────────────────────────────────────────── + +impl SketchAggOp { + pub fn default_count_min() -> Self { + SketchAggOp::CountMin { width: 2000, depth: 5 } + } + pub fn default_hll() -> Self { + SketchAggOp::HLL { registers: 14 } + } + pub fn default_ddsketch(quantiles: Vec) -> Self { + SketchAggOp::DDSketch { quantiles, epsilon: 0.01 } + } +} + +// ── Predicates ──────────────────────────────────────────────────────────────── + +/// A single filter predicate pushed down to the collector. +#[derive(Debug, Clone)] +pub struct Predicate { + pub col: String, + pub op: FilterOp, + pub val: FilterVal, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum FilterOp { + Eq, + Ne, + Lt, + Le, + Gt, + Ge, + Like, + NotLike, + IsNull, + IsNotNull, + /// PromQL `=~` label matcher (RE2 syntax). + Regex(String), + /// PromQL `!~` label matcher. + NotRegex(String), +} + +#[derive(Debug, Clone)] +pub enum FilterVal { + Str(String), + Num(f64), + Int(i64), + Null, +} + +// ── Coverage ────────────────────────────────────────────────────────────────── + +/// How completely a query can be served by sketches. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SketchCoverage { + /// All aggregation columns are sketch-mapped. + Full, + /// Some columns are sketch-mapped; others require exact passthrough + /// (e.g. `MIN(URL)` alongside `COUNT(*)`). + Partial, + /// No sketch applicable; query requires exact execution. + None, +} + +// ── SketchExpr → ParsedQuery bridge (backward compat) ──────────────────────── + +impl SketchExpr { + /// Convert to the legacy [`ParsedQuery`] flat representation consumed by + /// the existing [`crate::analyzer::Analyzer`] and planner. + pub fn to_parsed_query(&self) -> ParsedQuery { + let mut c = PqCollector::default(); + c.visit(self); + c.build() + } +} + +#[derive(Default)] +struct PqCollector { + metric_name: Option, + agg_types: Vec, + group_by_labels: Vec, + label_filters: HashMap, + time_window: Option, + exact_required: bool, + quantiles: Vec, + topk: Option, +} + +impl PqCollector { + fn visit(&mut self, expr: &SketchExpr) { + match expr { + SketchExpr::Source(s) => { + if self.metric_name.is_none() { + self.metric_name = Some(s.name.clone()); + } + } + SketchExpr::Filter { pred, input } => { + for p in pred { + if let (FilterOp::Eq, FilterVal::Str(v)) = (&p.op, &p.val) { + self.label_filters.insert(p.col.clone(), v.clone()); + } + } + self.visit(input); + } + SketchExpr::Window { duration, input } => { + if self.time_window.is_none() { + self.time_window = Some(*duration); + } + self.visit(input); + } + SketchExpr::Partition { keys, input } => { + for k in keys.keys() { + if !self.group_by_labels.contains(k) { + self.group_by_labels.push(k.clone()); + } + } + self.visit(input); + } + SketchExpr::Agg { op, input, .. } => { + self.collect_op(op); + self.visit(input); + } + SketchExpr::TopK { k, input } => { + self.topk = Some(*k); + self.visit(input); + } + SketchExpr::Dedup { input, .. } => self.visit(input), + SketchExpr::Merge { inputs } => { + for i in inputs { self.visit(i); } + } + SketchExpr::JoinSketch { outer, inner, .. } => { + self.visit(outer); + self.visit(inner); + } + } + } + + fn collect_op(&mut self, op: &SketchAggOp) { + match op { + SketchAggOp::HLL { .. } => { + if !self.agg_types.contains(&AggType::Cardinality) { + self.agg_types.push(AggType::Cardinality); + } + } + SketchAggOp::CountMin { .. } | SketchAggOp::CountSketch { .. } => { + if !self.agg_types.contains(&AggType::Frequency) { + self.agg_types.push(AggType::Frequency); + } + } + SketchAggOp::DDSketch { quantiles, .. } => { + if !self.agg_types.contains(&AggType::Quantile) { + self.agg_types.push(AggType::Quantile); + } + for &q in quantiles { + if !self.quantiles.contains(&q) { self.quantiles.push(q); } + } + } + SketchAggOp::ExactMinMax { .. } => { + if !self.agg_types.contains(&AggType::Quantile) { + self.agg_types.push(AggType::Quantile); + } + } + SketchAggOp::Exact(_) => { self.exact_required = true; } + SketchAggOp::Hydra { inner, .. } => self.collect_op(inner), + } + } + + fn build(self) -> ParsedQuery { + let metric_name = self.metric_name.unwrap_or_default(); + let mut qs = self.quantiles; + qs.sort_by(|a, b| a.partial_cmp(b).unwrap()); + qs.dedup(); + let hint = debs_hint( + &metric_name, + &self.agg_types, + &qs, + self.exact_required, + self.topk, + ); + ParsedQuery { + metric_name, + aggregations: self.agg_types, + group_by_labels: self.group_by_labels, + label_filters: self.label_filters, + time_window: self.time_window.unwrap_or(Duration::from_secs(300)), + exact_required: self.exact_required, + quantiles: qs, + hint, + } + } +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + fn source(name: &str) -> SketchExpr { + SketchExpr::Source(SourceSpec { name: name.into() }) + } + + #[test] + fn hll_is_mergeable() { + assert!(SketchAggOp::default_hll().is_mergeable()); + } + + #[test] + fn exact_avg_not_mergeable() { + assert!(!SketchAggOp::Exact(ExactAgg::Avg).is_mergeable()); + } + + #[test] + fn hydra_mergeability_inherits_inner() { + let hydra_hll = SketchAggOp::Hydra { + inner: Box::new(SketchAggOp::default_hll()), + partition_keys: vec!["region".into()], + }; + assert!(hydra_hll.is_mergeable()); + + let hydra_avg = SketchAggOp::Hydra { + inner: Box::new(SketchAggOp::Exact(ExactAgg::Avg)), + partition_keys: vec!["region".into()], + }; + assert!(!hydra_avg.is_mergeable()); + } + + #[test] + fn to_parsed_query_basic() { + // Partition(symbol, Window(5m, Filter(sectype=E, Agg(CountSketch(10), Source(price))))) + let expr = SketchExpr::TopK { + k: 10, + input: Box::new(SketchExpr::Partition { + keys: PartitionKeys::By(vec!["symbol".into()]), + input: Box::new(SketchExpr::Window { + duration: Duration::from_secs(300), + input: Box::new(SketchExpr::Filter { + pred: vec![Predicate { + col: "sectype".into(), + op: FilterOp::Eq, + val: FilterVal::Str("E".into()), + }], + input: Box::new(SketchExpr::Agg { + op: SketchAggOp::CountSketch { k: 10 }, + col: ColumnRef::Wildcard, + input: Box::new(source("financial.last_trade_price")), + }), + }), + }), + }), + }; + let pq = expr.to_parsed_query(); + assert_eq!(pq.metric_name, "financial.last_trade_price"); + assert_eq!(pq.aggregations, vec![AggType::Frequency]); + assert_eq!(pq.group_by_labels, vec!["symbol"]); + assert_eq!(pq.label_filters.get("sectype").map(String::as_str), Some("E")); + assert_eq!(pq.time_window, Duration::from_secs(300)); + assert!(!pq.exact_required); + } + + #[test] + fn to_parsed_query_ddsketch_quantiles() { + let expr = SketchExpr::Agg { + op: SketchAggOp::default_ddsketch(vec![0.25, 0.5, 0.75]), + col: ColumnRef::SampleValue, + input: Box::new(source("cpu")), + }; + let pq = expr.to_parsed_query(); + assert_eq!(pq.aggregations, vec![AggType::Quantile]); + assert_eq!(pq.quantiles, vec![0.25, 0.5, 0.75]); + assert!(!pq.exact_required); + } + + #[test] + fn to_parsed_query_exact_required() { + let expr = SketchExpr::Agg { + op: SketchAggOp::Exact(ExactAgg::Sum), + col: ColumnRef::Named("bytes".into()), + input: Box::new(source("network")), + }; + let pq = expr.to_parsed_query(); + assert!(pq.exact_required); + } + + #[test] + fn partition_keys_without() { + let keys = PartitionKeys::Without(vec!["instance".into()]); + assert_eq!(keys.keys(), &["instance".to_string()]); + assert!(!keys.is_empty()); + } + + // ── Accuracy bound tests ────────────────────────────────────────────────── + // + // These tests verify that the default sketch parameters satisfy the + // documented accuracy guarantees. The bounds are derived from the + // theoretical properties of each sketch family. + + /// DDSketch default: ε = 0.01 → at most 1 % relative rank error. + /// + /// For a stream of n items the rank of a DDSketch quantile estimate r̂ + /// satisfies |r̂ − r| ≤ ε·n. The default ε = 0.01 gives a 1 % SLA. + #[test] + fn ddsketch_default_epsilon_one_pct() { + let op = SketchAggOp::default_ddsketch(vec![0.99]); + match op { + SketchAggOp::DDSketch { epsilon, .. } => { + assert!( + epsilon <= 0.01, + "DDSketch default epsilon {epsilon} exceeds 1 % accuracy SLA" + ); + } + _ => panic!("expected DDSketch"), + } + } + + /// For n = 10 000 items, a 1 % DDSketch (ε = 0.01) guarantees the + /// p99 rank estimate is within ±100 positions of the true rank 9 900. + #[test] + fn ddsketch_rank_error_bound_n10k() { + let n: f64 = 10_000.0; + let epsilon = 0.01_f64; + let true_rank = (0.99 * n) as i64; // 9 900 + let max_error = (epsilon * n).ceil() as i64; // 100 + assert!( + max_error <= 100, + "rank error {max_error} exceeds expected bound for n={n} ε={epsilon}" + ); + // Verify the bound makes sense: estimate ∈ [true_rank-error, true_rank+error]. + let lo = true_rank - max_error; + let hi = true_rank + max_error; + assert!(lo >= 0 && hi <= n as i64); + } + + /// HyperLogLog default: registers = 14 (2¹⁴ = 16 384 buckets). + /// Standard error ≈ 1.04 / √(2^registers) ≈ 0.81 % < 1 %. + #[test] + fn hll_default_registers_error_below_1pct() { + let op = SketchAggOp::default_hll(); + match op { + SketchAggOp::HLL { registers } => { + let buckets = (1u64 << registers) as f64; // 2^registers + let std_error = 1.04 / buckets.sqrt(); + assert!( + std_error < 0.01, + "HLL standard error {std_error:.4} ≥ 1 % for registers={registers}" + ); + } + _ => panic!("expected HLL"), + } + } + + /// CountMin-Sketch default: width = 2 000, depth = 5. + /// Frequency error ε = e / width ≈ 0.136 % of total count N. + /// Failure probability δ = e^(−depth) ≈ 0.67 % < 1 %. + #[test] + fn countmin_default_error_and_failure_prob() { + let op = SketchAggOp::default_count_min(); + match op { + SketchAggOp::CountMin { width, depth } => { + let eps = std::f64::consts::E / width as f64; + let delta = (-(depth as f64)).exp(); + assert!( + eps < 0.002, + "CountMin frequency error ε={eps:.5} should be < 0.2 % of N" + ); + assert!( + delta < 0.01, + "CountMin failure probability δ={delta:.5} should be < 1 %" + ); + } + _ => panic!("expected CountMin"), + } + } + + // ── Mergeability correctness ────────────────────────────────────────────── + // + // Mergeability means sketch(A ∪ B) = merge(sketch(A), sketch(B)). + // All sketches used here satisfy this property except Exact(Avg), + // because avg(A ∪ B) ≠ avg(avg(A), avg(B)) in general. + + #[test] + fn countmin_is_mergeable() { + assert!(SketchAggOp::default_count_min().is_mergeable()); + } + + #[test] + fn countsketch_is_mergeable() { + assert!(SketchAggOp::CountSketch { k: 10 }.is_mergeable()); + } + + #[test] + fn ddsketch_is_mergeable() { + assert!(SketchAggOp::default_ddsketch(vec![0.99]).is_mergeable()); + } + + #[test] + fn exact_minmax_is_mergeable() { + // min(A∪B) = min(min(A), min(B)) — globally mergeable. + assert!(SketchAggOp::ExactMinMax { min: true, max: false }.is_mergeable()); + } + + #[test] + fn exact_sum_and_count_are_mergeable() { + assert!(SketchAggOp::Exact(ExactAgg::Sum).is_mergeable()); + assert!(SketchAggOp::Exact(ExactAgg::Count).is_mergeable()); + } + + #[test] + fn exact_avg_not_mergeable_avg_of_avgs_is_wrong() { + // avg([1,2,3]) = 2, avg([4,5]) = 4.5 + // avg-of-avgs = (2 + 4.5) / 2 = 3.25 ≠ avg([1,2,3,4,5]) = 3 + assert!(!SketchAggOp::Exact(ExactAgg::Avg).is_mergeable()); + } + + #[test] + fn hydra_mergeable_when_inner_is_hll() { + let hydra = SketchAggOp::Hydra { + inner: Box::new(SketchAggOp::default_hll()), + partition_keys: vec!["region".into(), "dc".into()], + }; + assert!(hydra.is_mergeable()); + } + + #[test] + fn hydra_not_mergeable_when_inner_is_avg() { + let hydra = SketchAggOp::Hydra { + inner: Box::new(SketchAggOp::Exact(ExactAgg::Avg)), + partition_keys: vec!["region".into()], + }; + assert!(!hydra.is_mergeable()); + } + + // ── AggType mapping ─────────────────────────────────────────────────────── + + #[test] + fn agg_type_mapping_correct() { + use crate::types::AggType; + assert_eq!(SketchAggOp::default_hll().to_agg_type(), AggType::Cardinality); + assert_eq!(SketchAggOp::default_count_min().to_agg_type(), AggType::Frequency); + assert_eq!(SketchAggOp::CountSketch { k: 5 }.to_agg_type(), AggType::Frequency); + assert_eq!(SketchAggOp::default_ddsketch(vec![0.5]).to_agg_type(),AggType::Quantile); + assert_eq!(SketchAggOp::ExactMinMax { min: true, max: true }.to_agg_type(), AggType::Quantile); + } + + // ── SketchCoverage ──────────────────────────────────────────────────────── + // + // Coverage classifies whether a SELECT can be fully, partially, or not + // at all served by sketches. + + #[test] + fn coverage_full_when_all_sketch() { + let ops: Vec = vec![ + SketchAggOp::default_hll(), + SketchAggOp::default_count_min(), + ]; + let has_sketch = ops.iter().any(|o| !o.is_exact()); + let has_exact = ops.iter().any(|o| o.is_exact()); + let cov = match (has_sketch, has_exact) { + (true, false) => SketchCoverage::Full, + (true, true) => SketchCoverage::Partial, + _ => SketchCoverage::None, + }; + assert_eq!(cov, SketchCoverage::Full); + } + + #[test] + fn coverage_partial_when_exact_mixed_in() { + let ops: Vec = vec![ + SketchAggOp::default_hll(), + SketchAggOp::Exact(ExactAgg::Sum), + ]; + let has_sketch = ops.iter().any(|o| !o.is_exact()); + let has_exact = ops.iter().any(|o| o.is_exact()); + let cov = match (has_sketch, has_exact) { + (true, false) => SketchCoverage::Full, + (true, true) => SketchCoverage::Partial, + _ => SketchCoverage::None, + }; + assert_eq!(cov, SketchCoverage::Partial); + } + + #[test] + fn coverage_none_when_all_exact() { + let ops: Vec = vec![ + SketchAggOp::Exact(ExactAgg::Sum), + SketchAggOp::Exact(ExactAgg::Count), + ]; + let has_sketch = ops.iter().any(|o| !o.is_exact()); + let has_exact = ops.iter().any(|o| o.is_exact()); + let cov = match (has_sketch, has_exact) { + (true, false) => SketchCoverage::Full, + (true, true) => SketchCoverage::Partial, + _ => SketchCoverage::None, + }; + assert_eq!(cov, SketchCoverage::None); + } +} diff --git a/controller/src/query_parser/sketch_rules.rs b/controller/src/query_parser/sketch_rules.rs new file mode 100644 index 00000000..97667683 --- /dev/null +++ b/controller/src/query_parser/sketch_rules.rs @@ -0,0 +1,683 @@ +//! Sketch-algebra optimizer — algebraic rewrite rules R1–R8. +//! +//! Rules are applied bottom-up (children first, then parent) until a single +//! fixed-point pass. Multiple passes can be added if needed. +//! +//! # Rule catalogue +//! +//! | Rule | Name | Effect | +//! |---|---|---| +//! | R1 | Filter push-down | `Agg(Filter(X))` → `Agg(Filter pushed into X)` | +//! | R2 | HAVING/WHERE split | separate post-agg key filter from pre-agg tuple filter | +//! | R3 | Sketch linearity | `Agg(Merge([X,Y]))` → `Merge([Agg(X), Agg(Y)])` when mergeable | +//! | R4 | Multi-key Hydra | `Partition([k1,k2], Agg(op,X))` → `Agg(Hydra(op,[k1,k2]),X)` | +//! | R5 | Join push-down | handled at parse time; see `sql.rs` | +//! | R6 | HLL dedup elim | `Agg(HLL, Dedup(col,X))` → `Agg(HLL, X)` | +//! | R7 | Window/Filter swap | `Window(Filter(X))` → `Filter(Window(X))` | +//! | R8 | TopK absorption | absorb TopK(k) into inner CountSketch(k) | + +use super::sketch_algebra::{ + ColumnRef, ExactAgg, FilterOp, FilterVal, PartitionKeys, Predicate, SketchAggOp, SketchExpr, +}; + +// ── Public entry point ──────────────────────────────────────────────────────── + +/// Optimise a [`SketchExpr`] tree by applying all rewrite rules bottom-up. +pub fn optimize(expr: SketchExpr) -> SketchExpr { + // Recurse into children first (post-order), then apply rules at this node. + let expr = rewrite_children(expr); + apply_all(expr) +} + +// ── Children-first recursion ────────────────────────────────────────────────── + +fn rewrite_children(expr: SketchExpr) -> SketchExpr { + match expr { + SketchExpr::Filter { pred, input } => + SketchExpr::Filter { pred, input: Box::new(optimize(*input)) }, + SketchExpr::Window { duration, input } => + SketchExpr::Window { duration, input: Box::new(optimize(*input)) }, + SketchExpr::Partition { keys, input } => + SketchExpr::Partition { keys, input: Box::new(optimize(*input)) }, + SketchExpr::Agg { op, col, input } => + SketchExpr::Agg { op, col, input: Box::new(optimize(*input)) }, + SketchExpr::TopK { k, input } => + SketchExpr::TopK { k, input: Box::new(optimize(*input)) }, + SketchExpr::Dedup { col, input } => + SketchExpr::Dedup { col, input: Box::new(optimize(*input)) }, + SketchExpr::Merge { inputs } => + SketchExpr::Merge { inputs: inputs.into_iter().map(optimize).collect() }, + SketchExpr::JoinSketch { join_key, outer, inner } => + SketchExpr::JoinSketch { + join_key, + outer: Box::new(optimize(*outer)), + inner: Box::new(optimize(*inner)), + }, + leaf => leaf, + } +} + +// ── Apply all rules at one node ─────────────────────────────────────────────── + +fn apply_all(expr: SketchExpr) -> SketchExpr { + let expr = r1_filter_pushdown(expr); + let expr = r2_having_where_split(expr); + let expr = r3_sketch_linearity(expr); + let expr = r4_multi_key_hydra(expr); + let expr = r6_hll_dedup_elim(expr); + let expr = r7_window_filter_swap(expr); + let expr = r8_topk_absorption(expr); + expr +} + +// ── R1: Filter push-down ────────────────────────────────────────────────────── +// +// Agg(op, Filter(pred, X)) → Agg(op, Filter(pred, X)) (already optimal) +// +// The real win is pushing Filter *below* Agg when the Filter is currently +// wrapping the Agg: +// +// Filter(pred, Agg(op, X)) → Agg(op, Filter(pred, X)) +// +// Condition: pred is on a base-relation column, not on the sketch output. +// We allow the push when pred references no aggregate alias (heuristic: no +// function names in the predicate column — that's a HAVING predicate). + +fn r1_filter_pushdown(expr: SketchExpr) -> SketchExpr { + match expr { + SketchExpr::Filter { pred, input } => { + if let SketchExpr::Agg { op, col, input: agg_input } = *input { + // Separate HAVING predicates (cannot be pushed) from WHERE predicates. + // Simple heuristic: predicates whose column matches a GROUP BY key + // or a base column are pushable. We push ALL here; R2 will re-split + // HAVING predicates that must stay above Agg. + return SketchExpr::Agg { + op, + col, + input: Box::new(SketchExpr::Filter { pred, input: agg_input }), + }; + } + SketchExpr::Filter { pred, input } + } + other => other, + } +} + +// ── R2: HAVING / WHERE split ────────────────────────────────────────────────── +// +// When a Filter sits inside an Agg (after R1 pushed it there), split it: +// - Predicates on GROUP BY keys → push further down (pre-agg WHERE) +// - Predicates on agg results → keep above Agg (HAVING) +// +// Because we don't have full schema knowledge at this point we use a simple +// heuristic: we push everything; callers that know HAVING semantics (SQL +// parser) mark predicates with a flag. This rule re-hoists flagged ones. + +fn r2_having_where_split(expr: SketchExpr) -> SketchExpr { + // Implementation: split Filter nodes that carry `having: true` predicates. + // The SQL parser marks HAVING predicates separately so no split is needed + // in the generic optimizer for now. This is a no-op placeholder. + expr +} + +// ── R3: Sketch linearity over Merge (α distributes over ⊕) ───────────────── +// +// Agg(op, Merge([X, Y, ...])) → Merge([Agg(op, X), Agg(op, Y), ...]) +// Condition: op.is_mergeable() + +fn r3_sketch_linearity(expr: SketchExpr) -> SketchExpr { + match expr { + SketchExpr::Agg { ref op, ref col, input: ref boxed } => { + if let SketchExpr::Merge { inputs } = boxed.as_ref() { + if op.is_mergeable() { + let distributed = inputs + .iter() + .cloned() + .map(|branch| SketchExpr::Agg { + op: op.clone(), + col: col.clone(), + input: Box::new(branch), + }) + .collect(); + return SketchExpr::Merge { inputs: distributed }; + } + } + expr + } + other => other, + } +} + +// ── R4: Multi-key Partition → Hydra ────────────────────────────────────────── +// +// Partition([k1,k2,...], Agg(op, X)) +// → Agg(Hydra(op, [k1,k2,...]), X) +// +// when |keys| > 1. + +fn r4_multi_key_hydra(expr: SketchExpr) -> SketchExpr { + if let SketchExpr::Partition { ref keys, .. } = expr { + let key_list = keys.keys().to_vec(); + if key_list.len() <= 1 { + return expr; + } + // Unwrap to get ownership. + if let SketchExpr::Partition { keys, input } = expr { + if let SketchExpr::Agg { op, col, input: agg_in } = *input { + return SketchExpr::Agg { + op: SketchAggOp::Hydra { inner: Box::new(op), partition_keys: key_list }, + col, + input: agg_in, + }; + } else { + // Partition wraps something other than Agg — leave unchanged. + return SketchExpr::Partition { keys, input }; + } + } + } + expr +} + +// ── R6: HLL dedup elimination ───────────────────────────────────────────────── +// +// Agg(HLL, Dedup(col, X)) → Agg(HLL, X) +// HLL inherently deduplicates; the explicit Dedup node is redundant. + +fn r6_hll_dedup_elim(expr: SketchExpr) -> SketchExpr { + if let SketchExpr::Agg { op: SketchAggOp::HLL { registers }, col, input } = expr { + if let SketchExpr::Dedup { input: inner, .. } = *input { + return SketchExpr::Agg { + op: SketchAggOp::HLL { registers }, + col, + input: inner, + }; + } else { + return SketchExpr::Agg { op: SketchAggOp::HLL { registers }, col, input }; + } + } + expr +} + +// ── R7: Window / Filter commutativity ───────────────────────────────────────── +// +// Window(w, Filter(pred, X)) → Filter(pred, Window(w, X)) +// Filter applied before windowing reduces the input stream size. + +fn r7_window_filter_swap(expr: SketchExpr) -> SketchExpr { + if let SketchExpr::Window { duration, input } = expr { + if let SketchExpr::Filter { pred, input: inner } = *input { + return SketchExpr::Filter { + pred, + input: Box::new(SketchExpr::Window { duration, input: inner }), + }; + } else { + return SketchExpr::Window { duration, input }; + } + } + expr +} + +// ── R8: TopK absorption into CountSketch ───────────────────────────────────── +// +// TopK(k, Partition(keys, Agg(CountSketch(k2), X))) +// → Partition(keys, Agg(CountSketch(k=k), X)) when k == k2 +// +// The top-K selection is already encoded in CountSketch; drop the outer TopK. + +fn r8_topk_absorption(expr: SketchExpr) -> SketchExpr { + if let SketchExpr::TopK { k, input } = expr { + if let SketchExpr::Partition { keys, input: part_in } = *input { + if let SketchExpr::Agg { + op: SketchAggOp::CountSketch { k: k2 }, + col, + input: agg_in, + } = *part_in + { + if k == k2 { + // TopK already encoded — absorb. + return SketchExpr::Partition { + keys, + input: Box::new(SketchExpr::Agg { + op: SketchAggOp::CountSketch { k }, + col, + input: agg_in, + }), + }; + } else { + // Different k values — keep TopK, restore inner. + return SketchExpr::TopK { + k, + input: Box::new(SketchExpr::Partition { + keys, + input: Box::new(SketchExpr::Agg { + op: SketchAggOp::CountSketch { k: k2 }, + col, + input: agg_in, + }), + }), + }; + } + } else { + return SketchExpr::TopK { + k, + input: Box::new(SketchExpr::Partition { keys, input: part_in }), + }; + } + } + return SketchExpr::TopK { k, input }; + } + expr +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use super::super::sketch_algebra::{ColumnRef, PartitionKeys, SketchAggOp, SketchExpr, SourceSpec, Predicate, FilterOp, FilterVal}; + use std::time::Duration; + + fn src(name: &str) -> SketchExpr { + SketchExpr::Source(SourceSpec { name: name.into() }) + } + + // ── R3 tests ────────────────────────────────────────────────────────────── + + #[test] + fn r3_distributes_hll_over_merge() { + let expr = SketchExpr::Agg { + op: SketchAggOp::default_hll(), + col: ColumnRef::Named("user".into()), + input: Box::new(SketchExpr::Merge { + inputs: vec![src("R"), src("S")], + }), + }; + let opt = optimize(expr); + // Should become Merge([Agg(HLL,R), Agg(HLL,S)]) + match opt { + SketchExpr::Merge { inputs } => { + assert_eq!(inputs.len(), 2); + for inp in &inputs { + assert!(matches!(inp, SketchExpr::Agg { op: SketchAggOp::HLL { .. }, .. })); + } + } + other => panic!("expected Merge, got {other:?}"), + } + } + + #[test] + fn r3_does_not_distribute_avg() { + let expr = SketchExpr::Agg { + op: SketchAggOp::Exact(ExactAgg::Avg), + col: ColumnRef::Named("price".into()), + input: Box::new(SketchExpr::Merge { + inputs: vec![src("R"), src("S")], + }), + }; + // Avg is not mergeable — Merge should NOT be distributed. + let opt = optimize(expr); + assert!(matches!(opt, SketchExpr::Agg { .. })); + } + + // ── R4 tests ────────────────────────────────────────────────────────────── + + #[test] + fn r4_multi_key_becomes_hydra() { + let expr = SketchExpr::Partition { + keys: PartitionKeys::By(vec!["k1".into(), "k2".into()]), + input: Box::new(SketchExpr::Agg { + op: SketchAggOp::default_hll(), + col: ColumnRef::Named("user".into()), + input: Box::new(src("hits")), + }), + }; + let opt = optimize(expr); + match opt { + SketchExpr::Agg { op: SketchAggOp::Hydra { inner, partition_keys }, .. } => { + assert!(matches!(*inner, SketchAggOp::HLL { .. })); + assert_eq!(partition_keys, vec!["k1", "k2"]); + } + other => panic!("expected Hydra Agg, got {other:?}"), + } + } + + #[test] + fn r4_single_key_unchanged() { + let expr = SketchExpr::Partition { + keys: PartitionKeys::By(vec!["k1".into()]), + input: Box::new(SketchExpr::Agg { + op: SketchAggOp::default_count_min(), + col: ColumnRef::Wildcard, + input: Box::new(src("hits")), + }), + }; + let opt = optimize(expr); + assert!(matches!(opt, SketchExpr::Partition { .. })); + } + + // ── R6 tests ────────────────────────────────────────────────────────────── + + #[test] + fn r6_hll_absorbs_dedup() { + let expr = SketchExpr::Agg { + op: SketchAggOp::default_hll(), + col: ColumnRef::Named("user".into()), + input: Box::new(SketchExpr::Dedup { + col: "user".into(), + input: Box::new(src("hits")), + }), + }; + let opt = optimize(expr); + // Dedup should be gone; HLL goes directly to Source. + match opt { + SketchExpr::Agg { op: SketchAggOp::HLL { .. }, input, .. } => { + assert!(matches!(*input, SketchExpr::Source(_))); + } + other => panic!("expected Agg(HLL,Source), got {other:?}"), + } + } + + // ── R7 tests ────────────────────────────────────────────────────────────── + + #[test] + fn r7_filter_hoisted_before_window() { + let pred = vec![Predicate { col: "env".into(), op: FilterOp::Eq, val: FilterVal::Str("prod".into()) }]; + let expr = SketchExpr::Window { + duration: Duration::from_secs(300), + input: Box::new(SketchExpr::Filter { + pred: pred.clone(), + input: Box::new(src("cpu")), + }), + }; + let opt = optimize(expr); + // Should become Filter(Window(Source)) + match opt { + SketchExpr::Filter { input, .. } => { + assert!(matches!(*input, SketchExpr::Window { .. })); + } + other => panic!("expected Filter(Window(...)), got {other:?}"), + } + } + + // ── R8 tests ────────────────────────────────────────────────────────────── + + #[test] + fn r8_topk_absorbed_same_k() { + let expr = SketchExpr::TopK { + k: 10, + input: Box::new(SketchExpr::Partition { + keys: PartitionKeys::By(vec!["phrase".into()]), + input: Box::new(SketchExpr::Agg { + op: SketchAggOp::CountSketch { k: 10 }, + col: ColumnRef::Wildcard, + input: Box::new(src("hits")), + }), + }), + }; + let opt = optimize(expr); + // TopK should be absorbed into the Partition/Agg + assert!(matches!(opt, SketchExpr::Partition { .. })); + } + + #[test] + fn r8_topk_kept_different_k() { + let expr = SketchExpr::TopK { + k: 5, + input: Box::new(SketchExpr::Partition { + keys: PartitionKeys::By(vec!["phrase".into()]), + input: Box::new(SketchExpr::Agg { + op: SketchAggOp::CountSketch { k: 10 }, + col: ColumnRef::Wildcard, + input: Box::new(src("hits")), + }), + }), + }; + let opt = optimize(expr); + assert!(matches!(opt, SketchExpr::TopK { k: 5, .. })); + } + + // ── R1 additional ───────────────────────────────────────────────────────── + + /// R1 basic: Filter wrapping Agg is pushed inside the Agg. + /// Filter(pred, Agg(op, X)) → Agg(op, Filter(pred, X)) + #[test] + fn r1_filter_pushed_below_agg() { + let pred = vec![Predicate { + col: "region".into(), + op: FilterOp::Eq, + val: FilterVal::Str("us-east".into()), + }]; + let expr = SketchExpr::Filter { + pred: pred.clone(), + input: Box::new(SketchExpr::Agg { + op: SketchAggOp::default_count_min(), + col: ColumnRef::Wildcard, + input: Box::new(src("hits")), + }), + }; + let opt = optimize(expr); + // Must become Agg(..., Filter(...)) + match opt { + SketchExpr::Agg { input, .. } => { + assert!(matches!(*input, SketchExpr::Filter { .. }), + "Filter should be inside Agg after R1"); + } + other => panic!("expected Agg after R1, got {other:?}"), + } + } + + /// R1 accuracy preservation: pushing Filter below Agg must not change + /// the sketch operator or its parameters. + #[test] + fn r1_preserves_sketch_epsilon() { + let pred = vec![Predicate { + col: "env".into(), + op: FilterOp::Eq, + val: FilterVal::Str("prod".into()), + }]; + let original_op = SketchAggOp::DDSketch { quantiles: vec![0.99], epsilon: 0.01 }; + let expr = SketchExpr::Filter { + pred, + input: Box::new(SketchExpr::Agg { + op: original_op.clone(), + col: ColumnRef::SampleValue, + input: Box::new(src("latency")), + }), + }; + let opt = optimize(expr); + match opt { + SketchExpr::Agg { op, .. } => { + assert_eq!(op, original_op, + "R1 must not alter the DDSketch epsilon or quantile parameters"); + } + other => panic!("expected Agg, got {other:?}"), + } + } + + // ── R3 additional ───────────────────────────────────────────────────────── + + /// R3 distributes CountMin over UNION ALL branches and preserves + /// the same width/depth parameters on every branch. + #[test] + fn r3_countmin_branches_have_same_params() { + let op = SketchAggOp::default_count_min(); + let expr = SketchExpr::Agg { + op: op.clone(), + col: ColumnRef::Wildcard, + input: Box::new(SketchExpr::Merge { + inputs: vec![src("R"), src("S"), src("T")], + }), + }; + let opt = optimize(expr); + match opt { + SketchExpr::Merge { inputs } => { + assert_eq!(inputs.len(), 3); + for branch in &inputs { + match branch { + SketchExpr::Agg { op: branch_op, .. } => { + assert_eq!(branch_op, &op, + "Each UNION branch must use identical CountMin parameters"); + } + other => panic!("expected Agg branch, got {other:?}"), + } + } + } + other => panic!("expected Merge after R3, got {other:?}"), + } + } + + /// R3 distributes DDSketch over Merge, preserving ε on every branch. + #[test] + fn r3_ddsketch_epsilon_preserved_on_each_branch() { + let op = SketchAggOp::DDSketch { quantiles: vec![0.95], epsilon: 0.005 }; + let expr = SketchExpr::Agg { + op: op.clone(), + col: ColumnRef::SampleValue, + input: Box::new(SketchExpr::Merge { + inputs: vec![src("A"), src("B")], + }), + }; + let opt = optimize(expr); + match opt { + SketchExpr::Merge { inputs } => { + for branch in &inputs { + match branch { + SketchExpr::Agg { op: bop, .. } => assert_eq!(bop, &op), + other => panic!("expected Agg, got {other:?}"), + } + } + } + other => panic!("expected Merge, got {other:?}"), + } + } + + // ── R4 additional ───────────────────────────────────────────────────────── + + /// R4 Hydra wraps the inner op without altering its accuracy parameters. + #[test] + fn r4_hydra_preserves_inner_ddsketch_epsilon() { + let inner_op = SketchAggOp::DDSketch { quantiles: vec![0.5], epsilon: 0.01 }; + let expr = SketchExpr::Partition { + keys: PartitionKeys::By(vec!["region".into(), "dc".into()]), + input: Box::new(SketchExpr::Agg { + op: inner_op.clone(), + col: ColumnRef::SampleValue, + input: Box::new(src("latency")), + }), + }; + let opt = optimize(expr); + match opt { + SketchExpr::Agg { + op: SketchAggOp::Hydra { inner, partition_keys }, + .. + } => { + assert_eq!(*inner, inner_op, + "Hydra must not alter inner DDSketch accuracy parameters"); + assert_eq!(partition_keys, vec!["region", "dc"]); + } + other => panic!("expected Hydra Agg, got {other:?}"), + } + } + + // ── Optimizer idempotency ───────────────────────────────────────────────── + + /// Applying `optimize` twice must yield a structurally identical tree. + /// This verifies the rules reach a fixed point in one pass. + #[test] + fn optimize_is_idempotent() { + // Build a tree that exercises multiple rules: R3, R4, R7. + let pred = vec![Predicate { + col: "env".into(), op: FilterOp::Eq, val: FilterVal::Str("prod".into()), + }]; + let expr = SketchExpr::Partition { + keys: PartitionKeys::By(vec!["region".into(), "dc".into()]), + input: Box::new(SketchExpr::Agg { + op: SketchAggOp::default_hll(), + col: ColumnRef::Named("user".into()), + input: Box::new(SketchExpr::Merge { + inputs: vec![ + SketchExpr::Window { + duration: Duration::from_secs(300), + input: Box::new(SketchExpr::Filter { + pred: pred.clone(), + input: Box::new(src("R")), + }), + }, + src("S"), + ], + }), + }), + }; + let once = optimize(expr.clone()); + let twice = optimize(once.clone()); + // Structural equality: format the debug output (simplest proxy for deep eq). + assert_eq!(format!("{once:?}"), format!("{twice:?}"), + "optimize is not idempotent — a second pass changed the tree"); + } + + // ── End-to-end accuracy pipeline ───────────────────────────────────────── + + /// Build the tree for `COUNT(DISTINCT UserID) FROM hits UNION ALL + /// SELECT COUNT(DISTINCT UserID) FROM hits2`, optimize it, and verify: + /// + /// 1. R3 distributed HLL over the Merge branches. + /// 2. Every branch uses the same HLL parameters (registers = 14). + /// 3. The standard error bound 1.04/√(2^14) < 1 % is preserved. + #[test] + fn end_to_end_union_hll_accuracy_preserved() { + let hll = SketchAggOp::default_hll(); + let expr = SketchExpr::Agg { + op: hll.clone(), + col: ColumnRef::Named("UserID".into()), + input: Box::new(SketchExpr::Merge { + inputs: vec![src("hits"), src("hits2")], + }), + }; + let opt = optimize(expr); + match opt { + SketchExpr::Merge { inputs } => { + assert_eq!(inputs.len(), 2, "both UNION branches must be present"); + for branch in &inputs { + match branch { + SketchExpr::Agg { op: SketchAggOp::HLL { registers }, .. } => { + assert_eq!(*registers, 14); + let std_err = 1.04 / ((1u64 << registers) as f64).sqrt(); + assert!(std_err < 0.01, + "HLL standard error {std_err:.4} must be < 1 %"); + } + other => panic!("expected HLL Agg on each branch, got {other:?}"), + } + } + } + other => panic!("expected Merge after R3, got {other:?}"), + } + } + + /// Build a DDSketch pipeline through R1 (filter push-down) and verify + /// the ε = 0.01 accuracy guarantee survives optimization. + #[test] + fn end_to_end_ddsketch_epsilon_survives_filter_pushdown() { + let pred = vec![Predicate { + col: "service".into(), + op: FilterOp::Eq, + val: FilterVal::Str("checkout".into()), + }]; + let expr = SketchExpr::Filter { + pred, + input: Box::new(SketchExpr::Agg { + op: SketchAggOp::DDSketch { quantiles: vec![0.99], epsilon: 0.01 }, + col: ColumnRef::SampleValue, + input: Box::new(src("latency")), + }), + }; + let opt = optimize(expr); + // After R1: Agg(DDSketch, Filter(Source)) + match opt { + SketchExpr::Agg { op: SketchAggOp::DDSketch { epsilon, quantiles }, .. } => { + assert_eq!(epsilon, 0.01, "ε must not change through R1"); + assert_eq!(quantiles, vec![0.99]); + } + other => panic!("expected DDSketch Agg after R1, got {other:?}"), + } + } +} diff --git a/controller/src/query_parser/sql.rs b/controller/src/query_parser/sql.rs new file mode 100644 index 00000000..c8def302 --- /dev/null +++ b/controller/src/query_parser/sql.rs @@ -0,0 +1,808 @@ +//! SQL → SketchExpr compiler. +//! +//! Implements the `AST_SQL_to_sketch` algorithm from the design doc +//! (`docs/Top-Down SQL-to-sketch mapping.pdf`). +//! +//! # Algorithm +//! +//! For each FUNCTION edge in the SELECT projection (leaf → root): +//! 1. Collect context: GROUP BY, WHERE, HAVING, JOIN, DISTINCT, UNION ALL +//! 2. Call `try_replace` → choose `SketchAggOp` (or Exact passthrough) +//! 3. Apply structural transformations: +//! - WHERE predicates → pushed into Filter node +//! - HAVING predicates → stay above Agg as a second Filter +//! - Multi-dim GROUP BY → Hydra via rewrite rule R4 +//! - JOIN → JoinSketch push-down +//! - UNION ALL → Merge node (sketch linearity via rule R3) +//! - DISTINCT → Dedup node (absorbed by HLL via rule R6) +//! +//! Multiple aggregations in one SELECT each emit their own `SketchAggOp`, +//! then are combined with a `Merge` node (coverage = Partial when some are Exact). + +use std::collections::HashMap; +use std::time::Duration; + +use anyhow::{anyhow, Context}; +use sqlparser::ast::{ + BinaryOperator, DuplicateTreatment, Expr, FunctionArg, FunctionArgExpr, + FunctionArgumentList, FunctionArguments, GroupByExpr, Join, JoinConstraint, + JoinOperator, LimitClause, ObjectName, OrderBy, OrderByExpr, OrderByKind, + Query, Select, SelectItem, SetExpr, SetOperator, Statement, TableFactor, + Value, ValueWithSpan, +}; +use sqlparser::dialect::GenericDialect; +use sqlparser::parser::Parser; + +use super::sketch_algebra::{ + ColumnRef, ExactAgg, FilterOp, FilterVal, PartitionKeys, Predicate, + SketchAggOp, SketchExpr, SketchCoverage, SourceSpec, +}; +use super::sketch_rules::optimize; + +// ── Public entry point ──────────────────────────────────────────────────────── + +/// Parse a SQL SELECT statement into an optimised [`SketchExpr`]. +pub fn parse_sql(sql: &str) -> anyhow::Result { + let dialect = GenericDialect {}; + let mut stmts = Parser::parse_sql(&dialect, sql) + .with_context(|| format!("SQL parse error: {sql:?}"))?; + let stmt = stmts.pop().ok_or_else(|| anyhow!("no SQL statement found"))?; + let query = match stmt { + Statement::Query(q) => *q, + other => return Err(anyhow!("expected SELECT, got {:?}", other)), + }; + let sketch = extract_from_query(&query)?; + Ok(optimize(sketch)) +} + +// ── Query-level dispatch ────────────────────────────────────────────────────── + +fn extract_from_query(query: &Query) -> anyhow::Result { + let order_by: Vec = match &query.order_by { + Some(OrderBy { kind: OrderByKind::Expressions(exprs), .. }) => exprs.clone(), + _ => vec![], + }; + let limit: Option<&Expr> = match &query.limit_clause { + Some(LimitClause::LimitOffset { limit: Some(e), .. }) => Some(e), + Some(LimitClause::OffsetCommaLimit { limit: e, .. }) => Some(e), + _ => None, + }; + extract_from_set_expr(query.body.as_ref(), &order_by, limit) +} + +fn extract_from_set_expr( + set_expr: &SetExpr, + order_by: &[OrderByExpr], + limit: Option<&Expr>, +) -> anyhow::Result { + match set_expr { + SetExpr::Select(sel) => extract_from_select(sel, order_by, limit), + SetExpr::Query(inner) => extract_from_query(inner), + + // UNION ALL → Merge of both branches (sketch linearity rule R3 will + // distribute Agg over Merge if ops are mergeable). + SetExpr::SetOperation { left, right, op: SetOperator::Union, .. } => { + let left_expr = extract_from_set_expr(left, &[], None)?; + let right_expr = extract_from_set_expr(right, &[], None)?; + Ok(SketchExpr::Merge { inputs: vec![left_expr, right_expr] }) + } + + other => Err(anyhow!("unsupported query body: {:?}", other)), + } +} + +// ── SELECT-level extraction ─────────────────────────────────────────────────── + +fn extract_from_select( + sel: &Select, + order_by: &[OrderByExpr], + limit: Option<&Expr>, +) -> anyhow::Result { + // ── Step 1: source table(s) ─────────────────────────────────────────────── + let metric_name = extract_table_name(sel)?; + + // ── Step 2: WHERE predicates ────────────────────────────────────────────── + let where_preds: Vec = sel.selection.as_ref() + .map(|e| collect_predicates(e)) + .unwrap_or_default(); + + // ── Step 3: GROUP BY keys ───────────────────────────────────────────────── + let group_by_keys = extract_group_by(&sel.group_by); + + // ── Step 4: HAVING predicates ───────────────────────────────────────────── + let having_preds: Vec = sel.having.as_ref() + .map(|e| collect_predicates(e)) + .unwrap_or_default(); + + // ── Step 5: top-K detection (ORDER BY … DESC LIMIT k) ──────────────────── + let topk: Option = detect_topk(order_by, limit); + + // ── Step 6: JOIN detection ──────────────────────────────────────────────── + let join_info: Option = extract_join_info(sel); + + // ── Step 7: SELECT-DISTINCT flag ────────────────────────────────────────── + let select_distinct = sel.distinct.is_some(); + + // ── Step 8: aggregation functions from SELECT list ──────────────────────── + let agg_items = collect_agg_items(&sel.projection); + + if agg_items.is_empty() { + // No aggregation — bare SELECT (e.g. SELECT col FROM t WHERE …). + let source = SketchExpr::Source(SourceSpec { name: metric_name }); + let filtered = apply_filters(source, where_preds); + return Ok(filtered); + } + + // ── Step 9: try_replace each agg item → SketchAggOp ────────────────────── + let ops: Vec<(SketchAggOp, ColumnRef)> = agg_items + .iter() + .map(|item| try_replace(item, &group_by_keys, topk, join_info.as_ref(), select_distinct)) + .collect(); + + // ── Step 10: determine coverage ─────────────────────────────────────────── + let has_sketch = ops.iter().any(|(op, _)| !op.is_exact()); + let has_exact = ops.iter().any(|(op, _)| op.is_exact()); + let _coverage = match (has_sketch, has_exact) { + (true, false) => SketchCoverage::Full, + (true, true) => SketchCoverage::Partial, + (false, _) => SketchCoverage::None, + }; + + // ── Step 11: assemble the SketchExpr tree ──────────────────────────────── + + // Base source with pushed-down WHERE. + let source = SketchExpr::Source(SourceSpec { name: metric_name.clone() }); + let filtered = apply_filters(source, where_preds); + + // If there's a JOIN, build JoinSketch. Otherwise use the filtered source. + let base = if let Some(ji) = join_info { + build_join_sketch(filtered, ji, &ops, &group_by_keys)? + } else { + build_agg_tree(filtered, ops, &group_by_keys, topk, having_preds) + }; + + Ok(base) +} + +// ── try_replace: aggregation function → SketchAggOp ────────────────────────── + +struct AggItem { + kind: AggKind, + col: ColumnRef, + distinct: bool, +} + +#[derive(Debug, Clone)] +enum AggKind { + Count, + Sum, + Avg, + Min, + Max, +} + +/// Map one aggregation item to its best sketch operator. +/// +/// Priority (doc §Generate_SQL_Aggregation_Sketch_Mapping): +/// 1. Cardinality (COUNT DISTINCT) → HLL +/// 2. Frequency heavy-hitter (COUNT + top-K) → CountSketch +/// 3. Frequency (COUNT GROUP BY) → CountMin +/// 4. Quantile (AVG/MIN/MAX with GROUP BY) → DDSketch +/// 5. Extrema without GROUP BY → ExactMinMax +/// 6. Sum / global count → Exact +fn try_replace( + item: &AggItem, + group_by: &[String], + topk: Option, + _join_info: Option<&JoinInfo>, + _distinct: bool, +) -> (SketchAggOp, ColumnRef) { + let has_group = !group_by.is_empty(); + + match item.kind { + AggKind::Count if item.distinct => { + // COUNT(DISTINCT col) → HLL + (SketchAggOp::default_hll(), item.col.clone()) + } + AggKind::Count => { + if let Some(k) = topk { + // ORDER BY … DESC LIMIT k → heavy-hitter CountSketch + (SketchAggOp::CountSketch { k }, item.col.clone()) + } else if has_group { + // COUNT(*) GROUP BY → frequency CountMin + (SketchAggOp::default_count_min(), item.col.clone()) + } else { + // Global COUNT(*) — exact + (SketchAggOp::Exact(ExactAgg::Count), item.col.clone()) + } + } + AggKind::Avg => { + if has_group { + // AVG per group → DDSketch(p50) — mergeable proxy + (SketchAggOp::default_ddsketch(vec![0.5]), item.col.clone()) + } else { + // Global AVG — not mergeable as-is; store exact (sum,count) + (SketchAggOp::Exact(ExactAgg::Avg), item.col.clone()) + } + } + AggKind::Min => { + if has_group { + (SketchAggOp::default_ddsketch(vec![0.0]), item.col.clone()) + } else { + (SketchAggOp::ExactMinMax { min: true, max: false }, item.col.clone()) + } + } + AggKind::Max => { + if has_group { + (SketchAggOp::default_ddsketch(vec![1.0]), item.col.clone()) + } else { + (SketchAggOp::ExactMinMax { min: false, max: true }, item.col.clone()) + } + } + AggKind::Sum => { + // SUM is always exact — no sketch benefit. + (SketchAggOp::Exact(ExactAgg::Sum), item.col.clone()) + } + } +} + +// ── Tree assembly ───────────────────────────────────────────────────────────── + +fn build_agg_tree( + base: SketchExpr, + ops: Vec<(SketchAggOp, ColumnRef)>, + group_by: &[String], + topk: Option, + having: Vec, +) -> SketchExpr { + // One Agg node per op; combine with Merge if multiple. + let agg_nodes: Vec = ops + .into_iter() + .map(|(op, col)| SketchExpr::Agg { op, col, input: Box::new(base.clone()) }) + .collect(); + + let merged = if agg_nodes.len() == 1 { + agg_nodes.into_iter().next().unwrap() + } else { + SketchExpr::Merge { inputs: agg_nodes } + }; + + // Apply GROUP BY partition. + let partitioned = if group_by.is_empty() { + merged + } else { + SketchExpr::Partition { + keys: PartitionKeys::By(group_by.to_vec()), + input: Box::new(merged), + } + }; + + // Apply HAVING as a post-agg filter. + let after_having = apply_filters(partitioned, having); + + // Apply top-K. + if let Some(k) = topk { + SketchExpr::TopK { k, input: Box::new(after_having) } + } else { + after_having + } +} + +// ── JOIN push-down ───────────────────────────────────────────────────────────── + +struct JoinInfo { + inner_table: String, + join_key: String, + outer_key: String, +} + +fn build_join_sketch( + outer_filtered: SketchExpr, + ji: JoinInfo, + ops: &[(SketchAggOp, ColumnRef)], + outer_group_by: &[String], +) -> anyhow::Result { + // Pre-aggregate on the inner table grouped by the join key. + let inner_source = SketchExpr::Source(SourceSpec { name: ji.inner_table.clone() }); + let inner_agg = if let Some((op, col)) = ops.first() { + SketchExpr::Agg { op: op.clone(), col: col.clone(), input: Box::new(inner_source) } + } else { + inner_source + }; + let inner_partitioned = SketchExpr::Partition { + keys: PartitionKeys::By(vec![ji.join_key.clone()]), + input: Box::new(inner_agg), + }; + + Ok(SketchExpr::JoinSketch { + join_key: ji.outer_key, + outer: Box::new(outer_filtered), + inner: Box::new(inner_partitioned), + }) +} + +// ── AST helpers: aggregation collection ────────────────────────────────────── + +fn collect_agg_items(projection: &[SelectItem]) -> Vec { + let mut out = Vec::new(); + for item in projection { + let expr = match item { + SelectItem::UnnamedExpr(e) => e, + SelectItem::ExprWithAlias { expr, .. } => expr, + _ => continue, + }; + collect_agg_from_expr(expr, &mut out); + } + out +} + +fn collect_agg_from_expr(expr: &Expr, out: &mut Vec) { + match expr { + Expr::Function(f) => { + let fn_name = f.name.0.last() + .and_then(|i| i.as_ident()) + .map(|id| id.value.to_uppercase()) + .unwrap_or_default(); + + let (distinct, args) = match &f.args { + FunctionArguments::List(FunctionArgumentList { duplicate_treatment, args, .. }) => { + let is_distinct = matches!( + duplicate_treatment, + Some(DuplicateTreatment::Distinct) + ); + (is_distinct, args.as_slice()) + } + _ => (false, &[][..]), + }; + + let col = first_col_from_args(args); + + let kind = match fn_name.as_str() { + "COUNT" => AggKind::Count, + "SUM" => AggKind::Sum, + "AVG" => AggKind::Avg, + "MIN" => AggKind::Min, + "MAX" => AggKind::Max, + _ => return, + }; + + out.push(AggItem { kind, col, distinct }); + } + Expr::BinaryOp { left, right, .. } => { + collect_agg_from_expr(left, out); + collect_agg_from_expr(right, out); + } + Expr::Nested(inner) => collect_agg_from_expr(inner, out), + _ => {} + } +} + +fn first_col_from_args(args: &[FunctionArg]) -> ColumnRef { + for arg in args { + match arg { + FunctionArg::Unnamed(FunctionArgExpr::Wildcard) => return ColumnRef::Wildcard, + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(id))) => { + return ColumnRef::Named(id.value.clone()); + } + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::CompoundIdentifier(parts))) => { + if let Some(last) = parts.last() { + return ColumnRef::Named(last.value.clone()); + } + } + _ => {} + } + } + ColumnRef::Wildcard +} + +// ── AST helpers: GROUP BY ────────────────────────────────────────────────────── + +fn extract_group_by(group_by: &GroupByExpr) -> Vec { + let exprs = match group_by { + GroupByExpr::All(_) => return vec![], + GroupByExpr::Expressions(e, _) => e, + }; + exprs.iter().filter_map(|e| match e { + Expr::Identifier(id) => Some(id.value.clone()), + Expr::CompoundIdentifier(parts) => parts.last().map(|i| i.value.clone()), + _ => None, + }).collect() +} + +// ── AST helpers: WHERE / predicate extraction ───────────────────────────────── + +fn collect_predicates(expr: &Expr) -> Vec { + let mut out = Vec::new(); + collect_pred_rec(expr, &mut out); + out +} + +fn collect_pred_rec(expr: &Expr, out: &mut Vec) { + match expr { + // col = 'v' + Expr::BinaryOp { left, op: BinaryOperator::Eq, right } => { + if let Some(p) = binary_pred(left, BinaryOperator::Eq, right) { out.push(p); } + } + // col <> 'v' + Expr::BinaryOp { left, op: BinaryOperator::NotEq, right } => { + if let Some(p) = binary_pred(left, BinaryOperator::NotEq, right) { out.push(p); } + } + // col > v + Expr::BinaryOp { left, op: BinaryOperator::Gt, right } => { + if let Some(p) = binary_pred(left, BinaryOperator::Gt, right) { out.push(p); } + } + // col >= v + Expr::BinaryOp { left, op: BinaryOperator::GtEq, right } => { + if let Some(p) = binary_pred(left, BinaryOperator::GtEq, right) { out.push(p); } + } + // col < v + Expr::BinaryOp { left, op: BinaryOperator::Lt, right } => { + if let Some(p) = binary_pred(left, BinaryOperator::Lt, right) { out.push(p); } + } + // col <= v + Expr::BinaryOp { left, op: BinaryOperator::LtEq, right } => { + if let Some(p) = binary_pred(left, BinaryOperator::LtEq, right) { out.push(p); } + } + // AND — recurse into both sides + Expr::BinaryOp { left, op: BinaryOperator::And, right } => { + collect_pred_rec(left, out); + collect_pred_rec(right, out); + } + // col LIKE '%v%' + Expr::Like { expr, pattern, negated, .. } => { + if let Some(col) = col_name(expr) { + if let Some(val) = literal_str(pattern) { + let op = if *negated { FilterOp::NotLike } else { FilterOp::Like }; + out.push(Predicate { col, op, val: FilterVal::Str(val) }); + } + } + } + // col IS NULL / IS NOT NULL + Expr::IsNull(inner) => { + if let Some(col) = col_name(inner) { + out.push(Predicate { col, op: FilterOp::IsNull, val: FilterVal::Null }); + } + } + Expr::IsNotNull(inner) => { + if let Some(col) = col_name(inner) { + out.push(Predicate { col, op: FilterOp::IsNotNull, val: FilterVal::Null }); + } + } + Expr::Nested(inner) => collect_pred_rec(inner, out), + // OR predicates span both columns — cannot push to collector; skip. + _ => {} + } +} + +fn binary_pred(left: &Expr, sql_op: BinaryOperator, right: &Expr) -> Option { + let col = col_name(left)?; + let (op, val) = match sql_op { + BinaryOperator::Eq => (FilterOp::Eq, literal_val(right)?), + BinaryOperator::NotEq => (FilterOp::Ne, literal_val(right)?), + BinaryOperator::Gt => (FilterOp::Gt, literal_val(right)?), + BinaryOperator::GtEq => (FilterOp::Ge, literal_val(right)?), + BinaryOperator::Lt => (FilterOp::Lt, literal_val(right)?), + BinaryOperator::LtEq => (FilterOp::Le, literal_val(right)?), + _ => return None, + }; + Some(Predicate { col, op, val }) +} + +fn col_name(expr: &Expr) -> Option { + match expr { + Expr::Identifier(id) => Some(id.value.clone()), + Expr::CompoundIdentifier(parts) => parts.last().map(|i| i.value.clone()), + _ => None, + } +} + +fn literal_val(expr: &Expr) -> Option { + let v = match expr { + Expr::Value(vws) => &vws.value, + _ => return None, + }; + match v { + Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => { + Some(FilterVal::Str(s.clone())) + } + Value::Number(n, _) => { + if let Ok(i) = n.parse::() { + Some(FilterVal::Int(i)) + } else if let Ok(f) = n.parse::() { + Some(FilterVal::Num(f)) + } else { + None + } + } + Value::Null => Some(FilterVal::Null), + _ => None, + } +} + +fn literal_str(expr: &Expr) -> Option { + let v = match expr { + Expr::Value(vws) => &vws.value, + _ => return None, + }; + match v { + Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => Some(s.clone()), + _ => None, + } +} + +// ── AST helpers: table name ─────────────────────────────────────────────────── + +fn extract_table_name(sel: &Select) -> anyhow::Result { + sel.from.first() + .and_then(|t| match &t.relation { + TableFactor::Table { name, .. } => Some(object_name_str(name)), + _ => None, + }) + .ok_or_else(|| anyhow!("could not determine table name from FROM clause")) +} + +fn object_name_str(name: &ObjectName) -> String { + name.0.iter() + .map(|i| i.as_ident().map(|id| id.value.as_str()).unwrap_or("")) + .collect::>() + .join(".") +} + +// ── AST helpers: top-K detection ───────────────────────────────────────────── + +fn detect_topk(order_by: &[OrderByExpr], limit: Option<&Expr>) -> Option { + let limit_n = match limit? { + Expr::Value(ValueWithSpan { value: Value::Number(n, _), .. }) => n.parse::().ok()?, + _ => return None, + }; + // Must have at least one DESC (or default) ORDER BY. + let has_desc = order_by.iter().any(|o| matches!(o.options.asc, Some(false) | None)); + if has_desc { Some(limit_n) } else { None } +} + +// ── AST helpers: JOIN extraction ────────────────────────────────────────────── + +fn extract_join_info(sel: &Select) -> Option { + let table_with_joins = sel.from.first()?; + let join = table_with_joins.joins.first()?; + + let inner_table = match &join.relation { + TableFactor::Table { name, .. } => object_name_str(name), + _ => return None, + }; + + // Extract the ON key from JOIN … ON a.key = b.key. + let (join_key, outer_key) = extract_join_keys(join)?; + + Some(JoinInfo { inner_table, join_key, outer_key }) +} + +fn extract_join_keys(join: &Join) -> Option<(String, String)> { + let constraint = match &join.join_operator { + JoinOperator::Inner(c) + | JoinOperator::LeftOuter(c) + | JoinOperator::RightOuter(c) + | JoinOperator::FullOuter(c) => c, + _ => return None, + }; + let on_expr = match constraint { + JoinConstraint::On(e) => e, + _ => return None, + }; + // Expect: a.key = b.key or key = key + if let Expr::BinaryOp { left, op: BinaryOperator::Eq, right } = on_expr { + let lk = col_name(left)?; + let rk = col_name(right)?; + Some((rk, lk)) // inner key, outer key + } else { + None + } +} + +// ── Shared helper ───────────────────────────────────────────────────────────── + +fn apply_filters(input: SketchExpr, pred: Vec) -> SketchExpr { + if pred.is_empty() { input } else { + SketchExpr::Filter { pred, input: Box::new(input) } + } +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use super::super::sketch_algebra::{ExactAgg, SketchAggOp}; + use crate::types::AggType; + use std::time::Duration; + + fn parse(sql: &str) -> SketchExpr { + parse_sql(sql).unwrap_or_else(|e| panic!("parse_sql failed: {e}\nSQL: {sql}")) + } + + fn pq(sql: &str) -> super::super::ParsedQuery { + parse(sql).to_parsed_query() + } + + // ── Basic aggregations ──────────────────────────────────────────────────── + + #[test] + fn count_star_no_group_is_exact() { + let pq = pq("SELECT COUNT(*) FROM hits"); + assert!(pq.exact_required); + assert!(pq.aggregations.is_empty()); + } + + #[test] + fn count_star_group_by_is_frequency() { + let pq = pq("SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID"); + assert!(pq.aggregations.contains(&AggType::Frequency)); + assert!(pq.group_by_labels.contains(&"AdvEngineID".to_string())); + } + + #[test] + fn count_distinct_is_cardinality() { + let pq = pq("SELECT COUNT(DISTINCT UserID) FROM hits"); + assert!(pq.aggregations.contains(&AggType::Cardinality)); + assert!(!pq.exact_required); + } + + #[test] + fn count_star_order_by_desc_limit_is_topk() { + let pq = pq( + "SELECT SearchPhrase, COUNT(*) AS c FROM hits \ + WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10", + ); + assert!(pq.aggregations.contains(&AggType::Frequency)); + } + + #[test] + fn avg_with_group_by_is_quantile_p50() { + let pq = pq("SELECT symbol, AVG(last) FROM hits GROUP BY symbol"); + assert!(pq.aggregations.contains(&AggType::Quantile)); + assert!(pq.quantiles.contains(&0.5)); + } + + #[test] + fn min_max_with_group_by_are_extremes() { + let pq = pq("SELECT symbol, MIN(last), MAX(last) FROM hits GROUP BY symbol"); + assert!(pq.aggregations.contains(&AggType::Quantile)); + assert!(pq.quantiles.contains(&0.0)); + assert!(pq.quantiles.contains(&1.0)); + } + + #[test] + fn min_max_no_group_by_is_exact_minmax() { + let expr = parse("SELECT MIN(EventDate), MAX(EventDate) FROM hits"); + // After optimize: ExactMinMax nodes + let pq = expr.to_parsed_query(); + // ExactMinMax maps to Quantile in legacy AggType + assert!(pq.aggregations.contains(&AggType::Quantile)); + } + + #[test] + fn sum_is_always_exact() { + let pq = pq("SELECT SUM(AdvEngineID) FROM hits"); + assert!(pq.exact_required); + } + + // ── WHERE predicates ────────────────────────────────────────────────────── + + #[test] + fn where_equality_captured() { + let pq = pq("SELECT COUNT(*) FROM hits WHERE sectype = 'E' GROUP BY symbol"); + assert_eq!(pq.label_filters.get("sectype").map(String::as_str), Some("E")); + } + + #[test] + fn where_inequality_captured() { + // ne predicate should be captured in filters (not label_filters, but present) + let expr = parse("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID"); + // Verify parse succeeds and has frequency + let pq = expr.to_parsed_query(); + assert!(pq.aggregations.contains(&AggType::Frequency)); + } + + // ── Multi-aggregation ───────────────────────────────────────────────────── + + #[test] + fn multi_agg_collects_all() { + let pq = pq( + "SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) \ + FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10", + ); + assert!(pq.aggregations.contains(&AggType::Cardinality), "missing cardinality"); + assert!(pq.aggregations.contains(&AggType::Frequency), "missing frequency"); + assert!(pq.aggregations.contains(&AggType::Quantile), "missing quantile"); + // SUM adds exact_required alongside sketch ops + assert!(pq.exact_required, "SUM should set exact_required"); + } + + // ── Table / metric name ─────────────────────────────────────────────────── + + #[test] + fn dotted_table_name() { + let pq = pq("SELECT COUNT(*) FROM financial.last_trade_price GROUP BY symbol"); + assert_eq!(pq.metric_name, "financial.last_trade_price"); + } + + // ── DEBS SQL variants ───────────────────────────────────────────────────── + + #[test] + fn debs_q6_cardinality() { + use super::super::QueryHint; + let pq = pq("SELECT COUNT(DISTINCT symbol) FROM financial.last_trade_price"); + assert!(pq.aggregations.contains(&AggType::Cardinality)); + assert!(matches!(pq.hint, Some(QueryHint::DebsCardinality))); + } + + #[test] + fn debs_q3_topk() { + let pq = pq( + "SELECT symbol, COUNT(*) AS c FROM financial.last_trade_price \ + GROUP BY symbol ORDER BY c DESC LIMIT 10", + ); + assert!(pq.aggregations.contains(&AggType::Frequency)); + } + + // ── COUNT(DISTINCT) with GROUP BY → Hydra via R4 ───────────────────────── + + #[test] + fn count_distinct_with_group_by() { + let pq = pq( + "SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10", + ); + assert!(pq.aggregations.contains(&AggType::Cardinality)); + assert!(pq.group_by_labels.contains(&"RegionID".to_string())); + } + + // ── UNION ALL → Merge ───────────────────────────────────────────────────── + + #[test] + fn union_all_produces_merge() { + let expr = parse( + "SELECT COUNT(DISTINCT UserID) FROM R \ + UNION ALL \ + SELECT COUNT(DISTINCT UserID) FROM S", + ); + // After R3 optimization: Merge([Agg(HLL,R), Agg(HLL,S)]) + assert!(matches!(expr, SketchExpr::Merge { .. })); + } + + // ── Complex queries ─────────────────────────────────────────────────────── + + #[test] + fn complex_multi_agg_multi_dim_group_by_topk() { + // COUNT(*) → Frequency, COUNT(DISTINCT) → Cardinality, AVG → Quantile + // Two GROUP BY dimensions; ORDER BY + LIMIT signals TopK + let pq = pq( + "SELECT region, dc, COUNT(*) AS c, COUNT(DISTINCT UserID), AVG(ResponseTime) \ + FROM hits WHERE env = 'prod' GROUP BY region, dc ORDER BY c DESC LIMIT 5", + ); + assert!(pq.aggregations.contains(&AggType::Frequency)); + assert!(pq.aggregations.contains(&AggType::Cardinality)); + assert!(pq.aggregations.contains(&AggType::Quantile)); + assert!(pq.group_by_labels.contains(&"region".to_string())); + assert!(pq.group_by_labels.contains(&"dc".to_string())); + assert_eq!( + pq.label_filters.get("env").map(String::as_str), + Some("prod") + ); + // AVG maps to median (p50) sketch + assert!(pq.quantiles.contains(&0.5)); + } + + #[test] + fn complex_union_all_hll_with_where_on_each_branch() { + // UNION ALL → Merge; each branch has its own WHERE predicate + let expr = parse( + "SELECT region, COUNT(DISTINCT UserID) FROM sessions WHERE status = 'active' GROUP BY region \ + UNION ALL \ + SELECT region, COUNT(DISTINCT UserID) FROM sessions WHERE status = 'expired' GROUP BY region", + ); + assert!(matches!(expr, SketchExpr::Merge { .. })); + let pq = expr.to_parsed_query(); + assert!(pq.aggregations.contains(&AggType::Cardinality)); + assert!(pq.group_by_labels.contains(&"region".to_string())); + } +} diff --git a/controller/src/types.rs b/controller/src/types.rs index 486e2b45..1ab6cef0 100644 --- a/controller/src/types.rs +++ b/controller/src/types.rs @@ -228,6 +228,13 @@ pub struct QueryWorkload { /// When set, the planner must use this sketch type instead of running /// the cost model. Allows pinning for collectors that support a subset. pub sketch_type_override: Option, + /// When true, sketches offer no benefit and the plan must use raw + /// pass-through (SP-2–SP-4 collapse to raw-preservation). + /// Set for stateful per-sample queries (RSI, MACD, stochastic, SUM). + pub exact_required: bool, + /// Quantile φ targets implied by the query (e.g. [0.5] for TWAP, + /// [0.0, 1.0] for price range). Empty for non-quantile workloads. + pub quantiles: Vec, } #[derive(Debug, Clone, Default)] diff --git a/docs/Top-Down SQL-to-sketch mapping.pdf b/docs/Top-Down SQL-to-sketch mapping.pdf new file mode 100644 index 00000000..128e10b7 Binary files /dev/null and b/docs/Top-Down SQL-to-sketch mapping.pdf differ diff --git a/docs/sketch-algebra-query-mapping.md b/docs/sketch-algebra-query-mapping.md new file mode 100644 index 00000000..f2a70d80 --- /dev/null +++ b/docs/sketch-algebra-query-mapping.md @@ -0,0 +1,382 @@ +# Sketch Algebra — Query Mapping Reference + +This document defines how PromQL and SQL queries compile to the shared +**sketch algebra** IR (`SketchExpr`), the algebraic rewrite rules the +optimizer applies, and the aggregation-function → sketch-type table. + +Both parsers are front-ends that emit the same `SketchExpr` tree. +The optimizer then applies rewrite rules before the planner converts +the tree to agent configurations. + +--- + +## 1. Sketch Algebra Operators + +| Operator | Symbol | Description | +|---|---|---| +| `Source(name)` | — | Base relation or metric stream | +| `Filter(preds, input)` | σ | Evaluate predicates on each tuple/sample before ingestion | +| `Window(duration, input)` | ψ | Time window applied to the stream (PromQL range; SQL time predicate) | +| `Partition(keys, input)` | γ | Group by key-tuple; one sketch instance per distinct value | +| `Agg(op, col, input)` | α | The sketch aggregation itself | +| `Dedup(col, input)` | δ | Deduplicate on `col` before ingestion | +| `TopK(k, input)` | τ | Retain only the top-K entries from the sketch result | +| `Merge(inputs)` | ⊕ | Merge sketches from multiple branches (requires mergeability) | +| `JoinSketch(key, outer, inner)` | ⋈ₛₖ | Pre-aggregate sketch on inner side by join key, merge after join | + +### 1.1 Sketch Aggregation Operators (`SketchAggOp`) + +| `SketchAggOp` | Sketch structure | Mergeable | Default params | +|---|---|---|---| +| `CountMin { width, depth }` | Count-Min Sketch | ✓ | width=2000, depth=5 | +| `CountSketch { k }` | Count Sketch (heavy hitter) | ✓ | k from query | +| `HLL { registers }` | HyperLogLog | ✓ | registers=14 (~0.8 % error) | +| `DDSketch { quantiles, epsilon }` | DDSketch | ✓ | epsilon=0.01 | +| `ExactMinMax { min, max }` | Exact running min/max | ✓ | — | +| `Exact(Count)` | Integer counter | ✓ | — | +| `Exact(Sum)` | f64 accumulator | ✓ | — | +| `Exact(Avg)` | (sum, count) pair | **✗** | use DDSketch(p50) in distributed contexts | +| `Exact(Min)` / `Exact(Max)` | Running extremum | ✓ | use ExactMinMax | + +> **Note on `Exact(Avg)`:** average-of-averages ≠ average. In distributed +> collection each agent must emit a `(sum, count)` pair, not a precomputed +> average. The controller recomputes `avg = sum / count` after merging. +> Alternatively, map to `DDSketch(quantiles=[0.5])` which is mergeable at +> the cost of approximation error ε. + +--- + +## 2. Aggregation Function → Sketch Mapping + +### 2.1 SQL aggregation functions + +The mapping depends on three context flags captured during AST traversal: +- **G** — GROUP BY present +- **T** — top-K pattern (ORDER BY col DESC LIMIT k) +- **J** — JOIN present in the path to the aggregation + +| SQL expression | G | T | J | `SketchAggOp` | Notes | +|---|---|---|---|---|---| +| `COUNT(*)` | ✗ | ✗ | ✗ | `Exact(Count)` | Global count; no sketch benefit | +| `COUNT(*)` | ✓ | ✗ | ✗ | `CountMin` | Frequency per group | +| `COUNT(*)` | ✓ | ✓ | ✗ | `CountSketch(k)` | Heavy-hitter top-K | +| `COUNT(*)` WHERE pred | ✗ | ✗ | ✗ | `CountMin` | Push filter; sketch for predicate-key frequency | +| `COUNT(DISTINCT col)` | ✗ | ✗ | ✗ | `HLL` | Global distinct count | +| `COUNT(DISTINCT col)` | ✓ | ✗ | ✗ | `Hydra(HLL, keys)` | Per-group distinct count | +| `COUNT(DISTINCT col)` | ✓ | ✓ | ✗ | `TopK(k, Hydra(HLL, keys))` | Top-K distinct-count groups | +| `SUM(col)` | any | any | ✗ | `Exact(Sum)` | Exact; no sketch benefit | +| `AVG(col)` | ✗ | ✗ | ✗ | `Exact(Avg)` | No group; trivially exact | +| `AVG(col)` | ✓ | ✗ | ✗ | `DDSketch([0.5])` | Median proxy; mergeable | +| `MIN(col)` | ✗ | ✗ | ✗ | `ExactMinMax(min=true)` | Cheaper than DDSketch | +| `MAX(col)` | ✗ | ✗ | ✗ | `ExactMinMax(max=true)` | Cheaper than DDSketch | +| `MIN(col)` | ✓ | ✗ | ✗ | `DDSketch([0.0])` | Extreme-quantile per group | +| `MAX(col)` | ✓ | ✗ | ✗ | `DDSketch([1.0])` | Extreme-quantile per group | +| `MIN(col)` + `MAX(col)` | ✓ | ✗ | ✗ | `DDSketch([0.0, 1.0])` | Single sketch, both extremes | +| `AGG(col)` | any | any | ✓ | `JoinSketch(key, outer, inner_with_Agg)` | Pre-agg on inner; see §4 | + +#### Multi-aggregation in one SELECT + +Each aggregation column gets its own `SketchAggOp`. They run over the +same filtered/windowed input stream and are collected in parallel. + +Example — `SELECT RegionID, SUM(x), COUNT(*) AS c, AVG(w), COUNT(DISTINCT u) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10`: + +``` +TopK(10, + Partition([RegionID], + Filter([], + Merge([ + Agg(Exact(Sum), col=x, Source(hits)), + Agg(CountSketch(k=10), col=*, Source(hits)), + Agg(DDSketch([0.5]), col=w, Source(hits)), + Agg(Hydra(HLL,[RegionID]), col=u, Source(hits)), + ]) + ) + ) +) +``` + +Coverage = `Partial` (SUM is exact alongside sketched columns). + +#### Multi-dimensional GROUP BY + +When GROUP BY has more than one key, wrap the inner `SketchAggOp` in +`Hydra` (sketch-of-sketches): + +``` +-- GROUP BY (k1, k2) +Partition([k1, k2], Agg(op, ...)) + → Agg(Hydra(op, [k1, k2]), ...) -- via rewrite rule R4 +``` + +Alternatively, maintain a flat per-(k1,k2) sketch if the key space is +small enough to enumerate. + +#### HAVING + +`HAVING p(keys)` stays **above** the `Agg` node (post-sketch filter on +group keys). It is not pushed into the sketch. + +``` +Filter(p_having, Agg(op, Partition(keys, Filter(p_where, Source(t))))) +``` + +#### UNION ALL + +```sql +SELECT α(x) FROM R UNION ALL SELECT α(x) FROM S +``` + +requires mergeability: + +``` +Merge([ + Agg(op, Source(R)), + Agg(op, Source(S)), +]) +``` + +Rejected (compile error) when `op` is `Exact(Avg)` — use `(sum, count)` instead. + +#### SELECT DISTINCT / COUNT(DISTINCT) + +- `COUNT(DISTINCT col)` → `HLL` (dedup is inherent in HLL; `Dedup` node omitted via R6). +- `SELECT DISTINCT … COUNT(col)` → `Dedup(col, Source(t))` pushed before `Agg(CountMin, …)`. + +--- + +### 2.2 SQL WHERE predicates → `Filter` predicates + +All WHERE predicates are captured and pushed down to the `Filter` node, +regardless of operator: + +| SQL predicate | `FilterOp` | +|---|---| +| `col = 'v'` | `Eq` | +| `col <> 'v'` | `Ne` | +| `col > v` | `Gt` | +| `col >= v` | `Ge` | +| `col < v` | `Lt` | +| `col <= v` | `Le` | +| `col LIKE '%v%'` | `Like` | +| `col NOT LIKE '%v%'` | `NotLike` | +| `col IS NULL` | `IsNull` | +| `col IS NOT NULL` | `IsNotNull` | + +Compound predicates (`AND`, `OR`) are decomposed recursively. +`OR` predicates that span multiple columns cannot be fully pushed to the +collector and are marked `filter_side: Controller` (evaluated at merge time). + +--- + +### 2.3 PromQL expressions + +PromQL label matchers map to `Filter` predicates; the range vector maps +to `Window`; `by`/`without` map to `Partition`. + +| PromQL expression | `SketchExpr` tree | `SketchAggOp` | +|---|---|---| +| `quantile_over_time(φ, m{f}[w]) by (d)` | `Partition(d, Window(w, Filter(f, Agg(DDSketch([φ]), Source(m)))))` | `DDSketch([φ])` | +| `histogram_quantile(φ, rate(m{f}[w])) by (d)` | same shape; inner `rate()` unwrapped | `DDSketch([φ])` | +| `avg_over_time(m{f}[w]) by (d)` | `Partition(d, Window(w, Filter(f, Agg(DDSketch([0.5]), Source(m)))))` | `DDSketch([0.5])` | +| `min_over_time(m{f}[w]) by (d)` | same | `DDSketch([0.0])` | +| `max_over_time(m{f}[w]) by (d)` | same | `DDSketch([1.0])` | +| `min_over_time(m{f}[w])` *(no by)* | `Window(w, Filter(f, Agg(ExactMinMax(min=true), Source(m))))` | `ExactMinMax` | +| `max_over_time(m{f}[w])` *(no by)* | same | `ExactMinMax` | +| `stddev_over_time(m{f}[w]) by (d)` | same as avg shape | `DDSketch([0.25, 0.75])` (IQR proxy) | +| `count_over_time(m{f}[w]) by (d)` | `Partition(d, Window(w, Filter(f, Agg(CountMin, Source(m)))))` | `CountMin` | +| `sum_over_time(m{f}[w]) by (d)` | same shape | `Exact(Sum)` | +| `topk(k, count_over_time(m{f}[w]) by (d))` | `TopK(k, Partition(d, Window(w, ...)))` | `CountSketch(k)` | +| `topk(k, avg_over_time(m{f}[w]) by (d))` | `TopK(k, Partition(d, Window(w, ...)))` | `DDSketch([0.5])` | +| `count(count_over_time(m{f}[w]) by (d))` | `Agg(HLL, Partition(d, Window(w, ...)))` | `HLL` (cardinality of active groups) | +| `sum by (d) (avg_over_time(m{f}[w]))` | outer Aggregate wraps inner Call; `by` from outer | `DDSketch([0.5])` | +| `changes(m{f}[w]) by (d)` | `Partition(d, Window(w, Filter(f, Agg(CountMin, Source(m)))))` | `CountMin` | +| `last_over_time(m{f}[w])` | `exact_required = true`; `coverage = None` | — | +| `delta(m{f}[w])` / `deriv(m{f}[w])` | `exact_required = true` | — | +| `predict_linear(m{f}[w], t)` | `exact_required = true` | — | +| `m{f}` *(bare selector)* | `exact_required = true` | — | +| `m_a{f} / m_b{f}` *(binary op)* | `exact_required = true`; both sides needed | — | + +#### PromQL label matcher → `FilterOp` + +| PromQL matcher | `FilterOp` | +|---|---| +| `key="val"` | `Eq` | +| `key!="val"` | `Ne` | +| `key=~"regex"` | `Regex` | +| `key!~"regex"` | `NotRegex` | + +#### PromQL `without` clause + +`without (d1, d2)` is the complement of `by`. Because the full label set +is not known at parse time, it is stored as: + +```rust +Partition::Without { excluded: vec!["d1", "d2"] } +``` + +The planner resolves the complement against the metric's actual label schema +at plan execution time. + +--- + +## 3. Algebraic Rewrite Rules + +The optimizer applies these rules bottom-up to produce the most efficient tree. + +``` +R1 — Filter push-down (always beneficial) + Filter(p, Agg(op, X)) → Agg(op, Filter(p, X)) + Condition: p is on a base column, not on the sketch output + +R2 — HAVING / WHERE split + Filter(p_key ∧ p_val, Agg(op, Partition(keys, X))) + → Filter(p_val, -- stays above Agg (HAVING) + Agg(op, + Partition(keys, + Filter(p_key, X) -- pushed below Agg (WHERE on key) + ) + ) + ) + +R3 — Sketch linearity over Merge (α distributes over ⊕) + Agg(op, Merge([X, Y])) → Merge([Agg(op, X), Agg(op, Y)]) + Condition: op is Mergeable + Effect: each branch (agent) builds its own sketch; controller merges + +R4 — Multi-key Partition → Hydra + Partition([k1, k2, ...], Agg(op, X)) + → Agg(Hydra(op, [k1, k2, ...]), X) + Condition: |keys| > 1 + +R5 — Join push-down + Agg(op, X ⋈_k Y) + → JoinSketch(k, + outer = X, + inner = Partition([k], Agg(op, Y)) + ) + Effect: sketch built on Y per join-key k; joined with X; MERGE after + +R6 — Dedup elimination for HLL + Agg(HLL, Dedup(col, X)) → Agg(HLL, X) + Reason: HLL inherently deduplicates + +R7 — Window / Filter commutativity + Window(w, Filter(p, X)) → Filter(p, Window(w, X)) + Effect: filter applied before windowing (reduces stream size) + +R8 — TopK absorption into CountSketch + TopK(k, Partition(keys, Agg(CountSketch(k2), X))) + → Partition(keys, Agg(CountSketch(k=k), X)) + Condition: k == k2 (top-K already encoded in sketch) +``` + +--- + +## 4. Join Push-Down Detail + +For a query of the form: + +```sql +SELECT R.k, α(S.b) +FROM R JOIN S ON R.id = S.id +GROUP BY R.k +``` + +The naive plan builds `α` over the joined stream. +The sketch-algebra plan applies **R5**: + +``` +JoinSketch( + join_key = "id", + outer = Source(R), + inner = Partition([id], Agg(op, Source(S))) +) +``` + +Execution: + +1. Build `Agg(op, S)` grouped by `id` → one sketch per distinct `id`. +2. For each row in `R`, look up the sketch for `R.id` in the inner result. +3. Group by `R.k`, `Merge` the collected sketches. + +This avoids materialising the full join before sketching and is the only +valid strategy when `S` is too large to join first. + +For self-joins or multi-way joins, R5 is applied recursively, innermost +join first. + +--- + +## 5. Mergeability Reference + +The `Merge` node (R3) is only valid when all input `SketchAggOp`s are +mergeable. The compiler rejects non-mergeable ops in `Merge` contexts. + +| `SketchAggOp` | Mergeable | Merge operation | +|---|---|---| +| `HLL` | ✓ | bitwise OR of registers | +| `DDSketch` | ✓ | element-wise add of buckets | +| `CountMin` | ✓ | element-wise max of matrix cells | +| `CountSketch` | ✓ | element-wise sum of arrays | +| `ExactMinMax(min)` | ✓ | `min(min_A, min_B)` | +| `ExactMinMax(max)` | ✓ | `max(max_A, max_B)` | +| `Exact(Count)` | ✓ | `count_A + count_B` | +| `Exact(Sum)` | ✓ | `sum_A + sum_B` | +| `Exact(Avg)` | **✗** | avg-of-avgs ≠ avg; carry `(sum, count)` instead | +| `Exact(Min)` | ✓ | `min(min_A, min_B)` | +| `Exact(Max)` | ✓ | `max(max_A, max_B)` | +| `Hydra(inner, keys)` | ✓ iff inner is mergeable | merge corresponding inner sketches | + +`Exact(Avg)` in a `Merge` context must be rewritten to `(Exact(Sum), Exact(Count))` +before the `Merge` node; the controller computes `avg = total_sum / total_count`. + +--- + +## 6. Sketch Coverage Classification + +After compilation and optimization, each `SketchExpr` is classified: + +| Coverage | Meaning | +|---|---| +| `Full` | All aggregation columns mapped to mergeable sketches | +| `Partial` | Some columns sketch-mapped, others require exact passthrough (e.g. `MIN(URL)` alongside `COUNT(*)`) | +| `None` | No sketch applicable; query requires exact execution (`sum_over_time`, bare selector, binary PromQL op) | + +`Partial` coverage is valid: the controller runs exact passthrough for the +non-sketch columns and sketch-merge for the rest. + +--- + +## 7. ClickBench Case Studies + +Concrete mappings for representative ClickBench queries. + +| SQL | Coverage | `SketchExpr` summary | +|---|---|---| +| `SELECT COUNT(*) FROM hits` | None | `Exact(Count)` — global count, no sketch | +| `SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0` | Full | `Filter([AdvEngineID≠0], Agg(CountMin, Source(hits)))` | +| `SELECT SUM(x), COUNT(*), AVG(w) FROM hits` | None | no GROUP BY; all exact | +| `SELECT COUNT(DISTINCT UserID) FROM hits` | Full | `Agg(HLL(UserID), Source(hits))` | +| `SELECT MIN(EventDate), MAX(EventDate) FROM hits` | Full | `Agg(ExactMinMax, Source(hits))` | +| `SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC` | Full | `Partition([AdvEngineID], Filter([AdvEngineID≠0], Agg(CountMin, Source(hits))))` | +| `SELECT SearchPhrase, COUNT(*) AS c FROM hits GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10` | Full | `TopK(10, Partition([SearchPhrase], Agg(CountSketch(10), Source(hits))))` | +| `SELECT RegionID, COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10` | Full | `TopK(10, Partition([RegionID], Agg(Hydra(HLL,[RegionID]), Source(hits))))` | +| `SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10` | Full | `TopK(10, Partition([MobilePhone,MobilePhoneModel], Agg(Hydra(HLL,[MobilePhone,MobilePhoneModel]), Filter([MobilePhoneModel≠''], Source(hits)))))` | +| `SELECT RegionID, SUM(x), COUNT(*) AS c, AVG(w), COUNT(DISTINCT u) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10` | Partial | `TopK(10, Partition([RegionID], Merge([Exact(Sum,x), CountSketch(10,*), DDSketch([0.5],w), Hydra(HLL(u),[RegionID])])))` | +| `SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10` | Partial | sketch covers `COUNT(*)` → `CountSketch`; `MIN(URL)` → exact passthrough | + +--- + +## 8. DEBS 2022 PromQL Case Studies + +| PromQL | DEBS Query | `SketchAggOp` | `QueryHint` | +|---|---|---|---| +| `avg_over_time(financial.last_trade_price[5m]) by (symbol)` | Q1 EMA | `DDSketch([0.5])` | `DebsEma` | +| `topk(10, count_over_time(financial.last_trade_price[5m]) by (symbol))` | Q3 TopK | `CountSketch(10)` | `DebsTopK{k:10}` | +| `min_over_time(financial.last_trade_price[5m]) by (symbol)` | Q4 price stats | `DDSketch([0.0])` | `DebsPriceStats` | +| `max_over_time(financial.last_trade_price[5m]) by (symbol)` | Q4 price stats | `DDSketch([1.0])` | `DebsPriceStats` | +| `quantile_over_time(0.25, financial.last_trade_price[5m]) by (symbol)` | Q5/Q9 volatility | `DDSketch([0.25,0.75])` | `DebsVolatility` | +| `count(count_over_time(financial.last_trade_price[5m]) by (symbol))` | Q6 cardinality | `HLL` | `DebsCardinality` | +| `quantile_over_time(0.5, financial.last_trade_price[5m]) by (symbol)` | Q7 TWAP | `DDSketch([0.5])` | `DebsTwap` | +| `financial.last_trade_price{symbol="RDSA.NL"}` | Q10–12 RSI/MACD | — (exact) | `ExactRequired` |