From c2c329bf67f48f547e92fcf1ed56cf9e9a40ab6e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:20:01 +0000 Subject: [PATCH] feat: lower SQL through unified operators and scalars --- Cargo.lock | 10 + Cargo.toml | 1 + crates/frontend-common/Cargo.toml | 11 + crates/frontend-common/src/lib.rs | 23 + crates/frontend-common/src/resolve.rs | 1188 +++++++ crates/frontend-common/src/schema_resolver.rs | 443 +++ crates/frontend-common/src/unresolved.rs | 374 +++ crates/frontend-sql/Cargo.toml | 1 + crates/frontend-sql/src/lib.rs | 3 + crates/frontend-sql/src/unified/error.rs | 63 + crates/frontend-sql/src/unified/mod.rs | 103 + .../src/unified/sql/clickhouse_ast.rs | 139 + .../src/unified/sql/collection_planning.rs | 189 ++ .../frontend-sql/src/unified/sql/dialect.rs | 112 + crates/frontend-sql/src/unified/sql/expr.rs | 354 ++ crates/frontend-sql/src/unified/sql/mod.rs | 2525 +++++++++++++++ crates/frontend-sql/src/unified/sql/types.rs | 378 +++ .../tests/unified_sql_lowering.rs | 2874 +++++++++++++++++ crates/types/src/ir/mod.rs | 5 +- crates/types/src/ir/schema_support.rs | 85 + crates/types/src/pre_asap/mod.rs | 2 + 21 files changed, 8881 insertions(+), 2 deletions(-) create mode 100644 crates/frontend-common/Cargo.toml create mode 100644 crates/frontend-common/src/lib.rs create mode 100644 crates/frontend-common/src/resolve.rs create mode 100644 crates/frontend-common/src/schema_resolver.rs create mode 100644 crates/frontend-common/src/unresolved.rs create mode 100644 crates/frontend-sql/src/unified/error.rs create mode 100644 crates/frontend-sql/src/unified/mod.rs create mode 100644 crates/frontend-sql/src/unified/sql/clickhouse_ast.rs create mode 100644 crates/frontend-sql/src/unified/sql/collection_planning.rs create mode 100644 crates/frontend-sql/src/unified/sql/dialect.rs create mode 100644 crates/frontend-sql/src/unified/sql/expr.rs create mode 100644 crates/frontend-sql/src/unified/sql/mod.rs create mode 100644 crates/frontend-sql/src/unified/sql/types.rs create mode 100644 crates/frontend-sql/tests/unified_sql_lowering.rs create mode 100644 crates/types/src/ir/schema_support.rs diff --git a/Cargo.lock b/Cargo.lock index b0dd1db9..ab43962c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -329,6 +329,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "asap-frontend-common" +version = "0.1.0" +dependencies = [ + "asap-types", + "serde", + "thiserror 2.0.18", +] + [[package]] name = "asap-frontend-metricsql" version = "0.1.0" @@ -352,6 +361,7 @@ name = "asap-frontend-sql" version = "0.1.0" dependencies = [ "asap-aware-mapping", + "asap-frontend-common", "asap-sql-function-catalog", "asap-types", "datafusion", diff --git a/Cargo.toml b/Cargo.toml index a2b019af..b3302644 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/asap-physical-operators", "crates/types", + "crates/frontend-common", "crates/sql-function-catalog", "crates/asap-aware-mapping", "crates/frontend-promql", diff --git a/crates/frontend-common/Cargo.toml b/crates/frontend-common/Cargo.toml new file mode 100644 index 00000000..18f351e6 --- /dev/null +++ b/crates/frontend-common/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "asap-frontend-common" +version = "0.1.0" +edition = "2021" + +# Shared front-end layer: the name-based `UnresolvedOp` tree every front end +# emits, and the resolver that binds it into the unified `OperatorNode` IR. +[dependencies] +asap-types = { path = "../types" } +serde = { version = "1", features = ["derive", "rc"] } +thiserror = "2" diff --git a/crates/frontend-common/src/lib.rs b/crates/frontend-common/src/lib.rs new file mode 100644 index 00000000..f756bc46 --- /dev/null +++ b/crates/frontend-common/src/lib.rs @@ -0,0 +1,23 @@ +//! `asap-frontend-common` — the front-end-facing, name-based operator tree +//! and its resolver into the unified IR. +//! +//! A front end builds an [`UnresolvedOp`] tree (column references are +//! name-based [`ColumnRef`](asap_types::pre_asap::ColumnRef)s) during its +//! own `interpret` step and calls [`resolve_root`], which binds every +//! reference to a positional `ColumnId` and returns the +//! [`OperatorNode`](asap_types::ir::OperatorNode) DAG. +//! +//! - [`unresolved`] — [`UnresolvedOp`] / [`UnresolvedScalar`]: the tree. +//! - [`schema_resolver`] — [`SchemaResolver`]: builds the binding schema of a +//! schemaless (PromQL) leaf from the names the query references. +//! - [`resolve`] — [`resolve_root`]: the bottom-up binding walk. + +pub mod resolve; +pub mod schema_resolver; +pub mod unresolved; + +pub use resolve::{resolve_expr, resolve_root, resolve_scalar_root, ResolveDAGError}; +pub use schema_resolver::{SchemaCatalog, SchemaResolver, UsageDerivedCatalog}; +pub use unresolved::{ + UnresolvedOp, UnresolvedPredicate, UnresolvedProjectItem, UnresolvedScalar, UnresolvedSortKey, +}; diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs new file mode 100644 index 00000000..dc204527 --- /dev/null +++ b/crates/frontend-common/src/resolve.rs @@ -0,0 +1,1188 @@ +//! Resolve a front-end-emitted [`UnresolvedOp`] tree into the unified IR +//! ([`Rc`]): a single, shape-preserving, bottom-up walk that +//! binds every [`ColumnRef`] to a positional `ColumnId`. +//! +//! Every structural decision (reduction choice, window folds, heavy-hitter +//! recognition, ...) is the front end's; what is left here is the mechanical, +//! schema-dependent substitution. Children are resolved first; each child +//! becomes an `OperatorNode` whose derived `.schema` is the scope the parent's +//! own references resolve against, so a `JOIN`'s concatenated schema and a +//! cross-series aggregate's frozen-closed output bind to the right positions. +//! +//! Scope boundaries: `Join` / `SetOp` sides and the operators referenced from +//! scalar positions (`scalar(v)`, subqueries) are each bound as a root in +//! their own scope. A `BinaryOp` side is too, but additionally inherits the +//! label names its enclosing scope references (issue #52): the `job` in +//! `sum by (job)(a or b)` appears in neither side's own matchers. + +use asap_types::ir::aggregate_schema::aggregate_output_schema; +use std::rc::Rc; + +use thiserror::Error; + +use asap_types::ir::operator_properties::ConcatDiscriminatorKey; +use asap_types::ir::{NonASAPOp, OperatorNode, Predicate, ProjectItem, ScalarExpr, SortKey}; +use asap_types::pre_asap::column_resolution::resolve_group_keys_promql; +use asap_types::pre_asap::{ + resolve_column_ref, resolve_column_refs, AggIntent, ColumnId, ColumnRef, GroupKeys, Reduction, + ResolveError, Schema, SchemaDerivationError, +}; + +use crate::schema_resolver::{collect_referenced_columns, SchemaResolver}; +use crate::unresolved::{UnresolvedOp, UnresolvedScalar, UnresolvedSortKey}; + +/// Errors from resolving an [`UnresolvedOp`] tree. +#[derive(Debug, Error)] +pub enum ResolveDAGError { + /// A column reference did not resolve against its in-scope schema. + #[error("column resolution failed: {0}")] + Resolve(#[from] ResolveError), + /// Deriving the schema of an already-resolved child failed (needed to + /// resolve positional column references against it). + #[error("schema derivation failed: {0}")] + Schema(#[from] SchemaDerivationError), +} + +use asap_types::ir::canonicalize::canonicalize; + +/// Resolve the whole tree rooted at `tree`: bind every `ColumnRef` to a +/// `ColumnId` via the [`SchemaResolver`], then canonicalize the result. +pub fn resolve_root(tree: &UnresolvedOp) -> Result, ResolveDAGError> { + resolve_root_with_inherited(tree, &[]) +} + +/// [`resolve_root`] with label names inherited from an enclosing scope seeded +/// into the leaf schema (a `BinaryOp` side, a scalar operand's operator). +fn resolve_root_with_inherited( + tree: &UnresolvedOp, + inherited: &[String], +) -> Result, ResolveDAGError> { + let fallback = SchemaResolver::new().resolve_schema_with_inherited(tree, inherited); + let root = resolve(tree, &fallback)?; + let root = canonicalize(root)?; + root.validate_structure()?; + Ok(root) +} + +/// Bind `tree` as a root in its own scope, inheriting from `enclosing` the +/// label names `tree` does not reference itself (issue #52). +fn resolve_nested_root( + tree: &UnresolvedOp, + enclosing: &Schema, +) -> Result, ResolveDAGError> { + let own = collect_referenced_columns(tree); + let inherited: Vec = inherited_names(enclosing) + .into_iter() + .filter(|n| !own.contains(n)) + .collect(); + resolve_root_with_inherited(tree, &inherited) +} + +fn node(op: NonASAPOp) -> Result, ResolveDAGError> { + Ok(OperatorNode::new_shared( + asap_types::ir::Operator::NonASAP(op), + )?) +} + +/// The generic substitution walk. `fallback` is the usage-derived schema a +/// schemaless `Scan` in this scope binds to. +fn resolve(tree: &UnresolvedOp, fallback: &Schema) -> Result, ResolveDAGError> { + use UnresolvedOp as U; + let expr = |e: &UnresolvedScalar, schema: &Schema| resolve_expr_in(e, schema, fallback); + let pred = |p: &UnresolvedScalar, schema: &Schema| { + Ok::<_, ResolveDAGError>(Predicate(expr(p, schema)?)) + }; + let sort_keys = |keys: &[UnresolvedSortKey], schema: &Schema| { + keys.iter() + .map(|k| { + Ok::<_, ResolveDAGError>(SortKey { + expr: expr(&k.expr, schema)?, + ascending: k.ascending, + nulls_first: k.nulls_first, + }) + }) + .collect::, _>>() + }; + match tree { + U::Scan { + source, + predicates, + schema, + } => { + let schema = schema.clone().unwrap_or_else(|| fallback.clone()); + let predicates = predicates + .iter() + .map(|p| pred(&p.0, &schema)) + .collect::, _>>()?; + node(NonASAPOp::Scan { + source: source.clone(), + predicates, + schema, + }) + } + + // Row expressions have no input-column scope. + U::Values { rows, schema } => { + let empty = Schema::new(Vec::new()); + let rows = rows + .iter() + .map(|row| { + row.iter() + .map(|e| expr(e, &empty)) + .collect::, _>>() + }) + .collect::, _>>()?; + node(NonASAPOp::Values { + rows, + schema: schema.clone(), + }) + } + + // A scalar at an operator position has no child scope; in practice a + // literal, so `fallback` is never consulted for a column here. + U::PromqlScalarOp { + child, + scalar, + op, + scalar_left, + return_bool, + } => { + let child = resolve(child, fallback)?; + let child = if child.schema.closed { + child + } else { + asap_types::ir::schema_support::with_promql_series_identity(&child) + .map_err(SchemaDerivationError::InvalidScalarSignature)? + }; + let scalar = resolve_expr(scalar, &Schema::default())?; + lower_scalar_vector(child, scalar, op, *scalar_left, *return_bool) + } + U::PromqlMap { + child, + sample, + drop_metric_name, + } => { + let child = resolve(child, fallback)?; + let child = if child.schema.closed { + child + } else { + asap_types::ir::schema_support::with_promql_series_identity(&child) + .map_err(SchemaDerivationError::InvalidScalarSignature)? + }; + let sample = resolve_expr(sample, &child.schema)?; + project_sample(child, sample, *drop_metric_name) + } + U::PromqlVectorFromScalar(inner) => { + node(NonASAPOp::PromqlVectorFromScalar(expr(inner, fallback)?)) + } + + U::PromqlRelabel { dst, value, child } => { + let child = resolve(child, fallback)?; + let value = expr(value, &child.schema)?; + node(NonASAPOp::PromqlRelabel { + dst: dst.clone(), + value, + child, + }) + } + + U::PromqlInfoEnrich { selector, child } => node(NonASAPOp::PromqlInfoEnrich { + selector: selector.clone(), + child: resolve(child, fallback)?, + }), + + U::PromqlSeriesSample { by, kind, child } => { + let child = resolve(child, fallback)?; + let by = resolve_group_keys(by, &child.schema)?; + node(NonASAPOp::PromqlSeriesSample { + by, + kind: *kind, + child, + }) + } + + U::Filter { pred: p, child } => { + let child = resolve(child, fallback)?; + let pred = pred(&p.0, &child.schema)?; + node(NonASAPOp::Filter { pred, child }) + } + + U::Project { + cols, + qualifier, + child, + } => { + let child = resolve(child, fallback)?; + let cols = cols + .iter() + .map(|item| { + Ok::<_, ResolveDAGError>(ProjectItem { + alias: item.alias.clone(), + expr: expr(&item.expr, &child.schema)?, + }) + }) + .collect::, _>>()?; + node(NonASAPOp::Project { + cols, + qualifier: qualifier.clone(), + child, + }) + } + + U::Aggregate { + reduction, + measures, + output_names, + filters, + having, + child, + } => { + let child = resolve(child, fallback)?; + let reduction = resolve_reduction(reduction, &child.schema)?; + let measures = measures + .iter() + .map(|m| resolve_agg_intent(m, &child.schema)) + .collect::, ResolveError>>()?; + let filters = filters + .iter() + .map(|p| p.as_ref().map(|p| pred(&p.0, &child.schema)).transpose()) + .collect::, _>>()?; + // HAVING is evaluated over the aggregate's own output. + let having = having + .as_ref() + .map(|h| { + let out_schema = aggregate_output_schema( + &child.schema, + &reduction, + &measures, + output_names, + )?; + pred(&h.0, &out_schema) + }) + .transpose()?; + node(NonASAPOp::Aggregate { + reduction, + measures, + output_names: output_names.clone(), + filters, + having, + child, + }) + } + + U::Dedup { cols, child } => { + let child = resolve(child, fallback)?; + let cols = resolve_column_refs(cols, &child.schema)?; + node(NonASAPOp::Dedup { cols, child }) + } + + U::Concat { + children, + discriminator_unique_key, + } => { + let children = children + .iter() + .map(|c| resolve(c, fallback)) + .collect::, _>>()?; + // Resolved against the first branch's own output schema — the one + // `output_schema`'s `Concat` arm derives the merged schema from. + let discriminator_unique_key = discriminator_unique_key + .as_ref() + .map(|key| { + let schema = &children + .first() + .ok_or(SchemaDerivationError::EmptyConcat)? + .schema; + Ok::<_, ResolveDAGError>(ConcatDiscriminatorKey::new( + resolve_column_ref(key.discriminator(), schema)?, + resolve_column_refs(key.inner_key(), schema)?, + )) + }) + .transpose()?; + node(NonASAPOp::Concat { + children, + discriminator_unique_key, + }) + } + + U::Join { + kind, + pred: p, + left, + right, + } => { + // Each branch is bound independently (different leaves / label + // sets); the predicate sees left ++ right. + let left = resolve_root_with_inherited(left, &[])?; + let right = resolve_root_with_inherited(right, &[])?; + let mut concat = left.schema.clone(); + concat.fields.extend(right.schema.fields.iter().cloned()); + let pred = pred(&p.0, &concat)?; + node(NonASAPOp::Join { + kind: kind.clone(), + pred, + left, + right, + }) + } + + U::SetOp { + kind, + all, + left, + right, + } => node(NonASAPOp::SetOp { + kind: kind.clone(), + all: *all, + left: resolve_root_with_inherited(left, &[])?, + right: resolve_root_with_inherited(right, &[])?, + }), + + U::Sort { + keys, + partition_by, + child, + } => { + let child = resolve(child, fallback)?; + let keys = sort_keys(keys, &child.schema)?; + let partition_by = resolve_group_keys(partition_by, &child.schema)?; + node(NonASAPOp::Sort { + keys, + partition_by, + child, + }) + } + + U::Limit { + n, + offset, + partition_by, + child, + } => { + let child = resolve(child, fallback)?; + let partition_by = resolve_group_keys(partition_by, &child.schema)?; + node(NonASAPOp::Limit { + n: *n, + offset: *offset, + partition_by, + child, + }) + } + + U::PromqlSubquery { + range, + resolution, + child, + } => node(NonASAPOp::PromqlSubquery { + range: *range, + resolution: *resolution, + child: resolve(child, fallback)?, + }), + + U::TimeRange { range, kind, child } => node(NonASAPOp::TimeRange { + range: *range, + kind: *kind, + child: resolve(child, fallback)?, + }), + + U::TimeShift { shift, child } => node(NonASAPOp::TimeShift { + shift: *shift, + child: resolve(child, fallback)?, + }), + + U::SQLWindowFunc { + func, + args, + partition_by, + order_by, + frame, + output_name, + child, + } => { + let child = resolve(child, fallback)?; + let args = args + .iter() + .map(|a| expr(a, &child.schema)) + .collect::, _>>()?; + let partition_by = resolve_group_keys(partition_by, &child.schema)?; + let order_by = sort_keys(order_by, &child.schema)?; + node(NonASAPOp::SQLWindowFunc { + func: func.clone(), + args, + partition_by, + order_by, + frame: frame.clone(), + output_name: output_name.clone(), + child, + }) + } + + U::BinaryOp { + operator, + return_bool, + lhs, + rhs, + } => { + // The two sides may scan different metrics with different label + // sets, so each resolves against its OWN bound schema — but still + // sees the label names the enclosing scope references (issue #52). + // The inherited set is computed over the whole `BinaryOp`, so one + // side's own labels are not conjured into the other. + let own = collect_referenced_columns(tree); + let inherited: Vec = inherited_names(fallback) + .into_iter() + .filter(|n| !own.contains(n)) + .collect(); + node(NonASAPOp::BinaryOp { + operator: operator.clone(), + return_bool: *return_bool, + lhs: resolve_root_with_inherited(lhs, &inherited)?, + rhs: resolve_root_with_inherited(rhs, &inherited)?, + }) + } + } +} + +/// The label names an enclosing scope's schema carries beyond the `(ts, +/// value)` floor. +fn inherited_names(schema: &Schema) -> Vec { + schema + .fields + .iter() + .filter(|c| c.name != "ts" && c.name != "value") + .map(|c| c.name.clone()) + .collect() +} + +/// Resolve a name-based scalar expression against `schema`. Operators it +/// reads (`scalar(v)`, subqueries) are bound as roots in their own scope, +/// inheriting `schema`'s label names. +pub fn resolve_expr( + expr: &UnresolvedScalar, + schema: &Schema, +) -> Result { + resolve_expr_in(expr, schema, schema) +} + +/// [`resolve_expr`] where the operators the expression reads inherit from +/// `enclosing` (the owning root's fallback schema) rather than from `schema`. +fn resolve_expr_in( + expr: &UnresolvedScalar, + schema: &Schema, + enclosing: &Schema, +) -> Result { + use UnresolvedScalar as S; + let bx = |e: &UnresolvedScalar| -> Result, ResolveDAGError> { + Ok(Box::new(resolve_expr_in(e, schema, enclosing)?)) + }; + let each = |es: &[UnresolvedScalar]| -> Result, ResolveDAGError> { + es.iter() + .map(|e| resolve_expr_in(e, schema, enclosing)) + .collect() + }; + let op = |o: &UnresolvedOp| resolve_nested_root(o, enclosing); + Ok(match expr { + S::Column(c) => ScalarExpr::Column(resolve_column_ref(c, schema)?), + S::Literal(s) => ScalarExpr::Literal(s.clone()), + S::EvalTimestamp => ScalarExpr::EvalTimestamp, + S::CurrentTimestamp => ScalarExpr::CurrentTimestamp, + S::Negative { expr, semantics } => ScalarExpr::Negative { + expr: bx(expr)?, + semantics: *semantics, + }, + S::Compare { + left, + op, + right, + semantics, + } => ScalarExpr::Compare { + left: bx(left)?, + op: op.clone(), + right: bx(right)?, + semantics: *semantics, + }, + S::BoolAnd(v) => ScalarExpr::BoolAnd(each(v)?), + S::BoolOr(v) => ScalarExpr::BoolOr(each(v)?), + S::Not(e) => ScalarExpr::Not(bx(e)?), + S::IsNull(e) => ScalarExpr::IsNull(bx(e)?), + S::IsNotNull(e) => ScalarExpr::IsNotNull(bx(e)?), + S::Cast { expr, to, try_cast } => ScalarExpr::Cast { + expr: bx(expr)?, + to: to.clone(), + try_cast: *try_cast, + }, + S::InList { + expr, + list, + negated, + } => ScalarExpr::InList { + expr: bx(expr)?, + list: each(list)?, + negated: *negated, + }, + S::FunctionCall { name, args } => ScalarExpr::FunctionCall { + name: name.clone(), + args: each(args)?, + }, + S::Arithmetic { + op, + left, + right, + semantics, + } => ScalarExpr::Arithmetic { + op: op.clone(), + left: bx(left)?, + right: bx(right)?, + semantics: *semantics, + }, + S::Case { + operand, + branches, + else_expr, + } => ScalarExpr::Case { + operand: operand.as_deref().map(bx).transpose()?, + branches: branches + .iter() + .map(|(w, t)| { + Ok(( + resolve_expr_in(w, schema, enclosing)?, + resolve_expr_in(t, schema, enclosing)?, + )) + }) + .collect::, ResolveDAGError>>()?, + else_expr: else_expr.as_deref().map(bx).transpose()?, + }, + S::PromqlScalarFromVector(o) => ScalarExpr::PromqlScalarFromVector(op(o)?), + S::ScalarSubquery(o) => ScalarExpr::ScalarSubquery(op(o)?), + S::Exists { subquery, negated } => ScalarExpr::Exists { + subquery: op(subquery)?, + negated: *negated, + }, + S::InSubquery { + expr, + subquery, + negated, + } => ScalarExpr::InSubquery { + expr: bx(expr)?, + subquery: op(subquery)?, + negated: *negated, + }, + }) +} + +/// Resolve name-based group keys positionally, preserving `by`/`without`. +fn resolve_group_keys( + keys: &GroupKeys, + schema: &Schema, +) -> Result, ResolveError> { + let ids = resolve_column_refs(keys.keys(), schema)?; + Ok(if keys.is_without() { + GroupKeys::without(ids) + } else { + GroupKeys::by(ids) + }) +} + +/// Resolve a name-based reduction. Uses [`resolve_group_keys_promql`] rather +/// than the strict [`resolve_group_keys`]: a key absent from a **closed** +/// schema (the output of a nested cross-series aggregate that collapsed the +/// label) is provably absent from every row, so PromQL drops it from the +/// grouping rather than rejecting the query (issue #53) — `sum(sum by (group) +/// (m)) by (job)`. SQL `GROUP BY` keys are always present, so the lenient +/// path is a no-op difference there. +fn resolve_reduction( + reduction: &Reduction, + schema: &Schema, +) -> Result, ResolveError> { + Ok(match reduction { + Reduction::Reduce(by) => { + let ids = resolve_group_keys_promql(by.keys(), schema)?; + Reduction::Reduce(if by.is_without() { + GroupKeys::without(ids) + } else { + GroupKeys::by(ids) + }) + } + Reduction::PerEntity => Reduction::PerEntity, + }) +} + +/// Resolve a name-based aggregate intent: every `col: Option` +/// resolves to `Option` (`None` stays `None`, the sample-value +/// convention); every other field carries through unchanged. +fn resolve_agg_intent( + intent: &AggIntent, + schema: &Schema, +) -> Result, ResolveError> { + let col = |c: &Option| -> Result, ResolveError> { + c.as_ref() + .map(|r| resolve_column_ref(r, schema)) + .transpose() + }; + Ok(match intent { + AggIntent::Count { accuracy } => AggIntent::Count { + accuracy: accuracy.clone(), + }, + AggIntent::PearsonCorr { left, right } => AggIntent::PearsonCorr { + left: resolve_column_ref(left, schema)?, + right: resolve_column_ref(right, schema)?, + }, + AggIntent::Sum { col: c } => AggIntent::Sum { col: col(c)? }, + AggIntent::Min { col: c } => AggIntent::Min { col: col(c)? }, + AggIntent::Max { col: c } => AggIntent::Max { col: col(c)? }, + AggIntent::Avg { col: c } => AggIntent::Avg { col: col(c)? }, + AggIntent::StdDev { col: c, population } => AggIntent::StdDev { + col: col(c)?, + population: *population, + }, + AggIntent::Variance { col: c, population } => AggIntent::Variance { + col: col(c)?, + population: *population, + }, + AggIntent::Quantile { + col: c, + q, + accuracy, + } => AggIntent::Quantile { + col: col(c)?, + q: *q, + accuracy: accuracy.clone(), + }, + AggIntent::TopK { k, accuracy } => AggIntent::TopK { + k: *k, + accuracy: accuracy.clone(), + }, + AggIntent::Cardinality { cols, accuracy } => AggIntent::Cardinality { + cols: cols + .iter() + .map(|c| resolve_column_ref(c, schema)) + .collect::>()?, + accuracy: accuracy.clone(), + }, + AggIntent::FrequencyL2 { col: c, accuracy } => AggIntent::FrequencyL2 { + col: col(c)?, + accuracy: accuracy.clone(), + }, + AggIntent::FrequencyEntropy { col: c, accuracy } => AggIntent::FrequencyEntropy { + col: col(c)?, + accuracy: accuracy.clone(), + }, + AggIntent::Rate => AggIntent::Rate, + AggIntent::IRate => AggIntent::IRate, + AggIntent::Increase => AggIntent::Increase, + AggIntent::Changes => AggIntent::Changes, + AggIntent::Delta => AggIntent::Delta, + AggIntent::IDelta => AggIntent::IDelta, + AggIntent::Deriv => AggIntent::Deriv, + AggIntent::Resets => AggIntent::Resets, + AggIntent::PredictLinear { seconds } => AggIntent::PredictLinear { seconds: *seconds }, + AggIntent::DoubleExpSmoothing { smoothing, trend } => AggIntent::DoubleExpSmoothing { + smoothing: *smoothing, + trend: *trend, + }, + AggIntent::HistogramCount => AggIntent::HistogramCount, + AggIntent::HistogramSum => AggIntent::HistogramSum, + AggIntent::HistogramAvg => AggIntent::HistogramAvg, + AggIntent::HistogramStdDev => AggIntent::HistogramStdDev, + AggIntent::HistogramStdVar => AggIntent::HistogramStdVar, + AggIntent::HistogramFraction { lower, upper } => AggIntent::HistogramFraction { + lower: *lower, + upper: *upper, + }, + AggIntent::HistogramQuantile { q, le } => AggIntent::HistogramQuantile { + q: *q, + le: resolve_column_ref(le, schema)?, + }, + AggIntent::Math(f) => AggIntent::Math(f.clone()), + AggIntent::Absent => AggIntent::Absent, + AggIntent::AbsentOverTime => AggIntent::AbsentOverTime, + AggIntent::PresentOverTime => AggIntent::PresentOverTime, + AggIntent::TimeFn(f) => AggIntent::TimeFn(*f), + AggIntent::Group => AggIntent::Group, + AggIntent::CountValues { label } => AggIntent::CountValues { + label: label.clone(), + }, + AggIntent::LastOverTime => AggIntent::LastOverTime, + AggIntent::FirstOverTime => AggIntent::FirstOverTime, + AggIntent::MadOverTime => AggIntent::MadOverTime, + AggIntent::TsOfMinOverTime => AggIntent::TsOfMinOverTime, + AggIntent::TsOfMaxOverTime => AggIntent::TsOfMaxOverTime, + AggIntent::TsOfFirstOverTime => AggIntent::TsOfFirstOverTime, + AggIntent::TsOfLastOverTime => AggIntent::TsOfLastOverTime, + AggIntent::Extension { ext_kind, payload } => AggIntent::Extension { + ext_kind: ext_kind.clone(), + payload: payload.clone(), + }, + }) +} + +/// Resolve a standalone scalar in an empty column scope; plan reads retain their own scope. +pub fn resolve_scalar_root(tree: &UnresolvedScalar) -> Result { + let resolved = resolve_expr(tree, &Schema::default())?; + resolved.scalar_type(&Schema::default())?; + Ok(resolved) +} + +fn lower_scalar_vector( + child: Rc, + scalar: ScalarExpr, + op: &asap_types::pre_asap::BinaryOpKind, + scalar_left: bool, + return_bool: bool, +) -> Result, ResolveDAGError> { + use asap_types::ir::ExprSemantics; + use asap_types::pre_asap::{BinaryOpKind, DataType, ScalarValue}; + let value = child + .schema + .column_id("value") + .or_else(|| { + child + .schema + .fields + .iter() + .enumerate() + .filter(|(i, f)| { + Some(*i) != child.schema.time_index + && matches!(f.plain_dtype(), Some(DataType::Float64 | DataType::Int64)) + }) + .map(|(i, _)| i) + .next_back() + }) + .ok_or_else(|| { + SchemaDerivationError::InvalidScalarSignature("vector has no numeric sample".into()) + })?; + let sample = ScalarExpr::Column(value); + let (left, right) = if scalar_left { + (scalar, sample) + } else { + (sample, scalar) + }; + let semantics = ExprSemantics::Promql; + let return_bool = return_bool || matches!(op, BinaryOpKind::CompareBool(_)); + let computed = match op { + BinaryOpKind::Arithmetic(op) => ScalarExpr::Arithmetic { + op: op.clone(), + left: Box::new(left), + right: Box::new(right), + semantics, + }, + BinaryOpKind::Compare(op) | BinaryOpKind::CompareBool(op) => { + let predicate = ScalarExpr::Compare { + op: op.clone(), + left: Box::new(left), + right: Box::new(right), + semantics, + }; + if !return_bool { + return node(NonASAPOp::Filter { + child, + pred: Predicate(predicate), + }); + } + ScalarExpr::Case { + operand: None, + branches: vec![(predicate, ScalarExpr::Literal(ScalarValue::Float64(1.0)))], + else_expr: Some(Box::new(ScalarExpr::Literal(ScalarValue::Float64(0.0)))), + } + } + BinaryOpKind::Set(_) => { + return Err(SchemaDerivationError::InvalidScalarSignature( + "set operators require two vectors".into(), + ) + .into()) + } + }; + project_sample(child, computed, true) +} + +fn project_sample( + child: Rc, + computed: ScalarExpr, + drop_metric_name: bool, +) -> Result, ResolveDAGError> { + let value = asap_types::pre_asap::column_resolution::resolve_column_ref( + &ColumnRef::SampleValue, + &child.schema, + )?; + let cols = child + .schema + .fields + .iter() + .enumerate() + .filter(|(_, f)| !drop_metric_name || f.name != "__name__") + .map(|(i, f)| { + let expr = if i == value { + computed.clone() + } else if drop_metric_name + && f.name == asap_types::pre_asap::schema::PROMQL_SERIES_IDENTITY + { + ScalarExpr::FunctionCall { + name: "promql_drop_metric_name".into(), + args: vec![ScalarExpr::Column(i)], + } + } else { + ScalarExpr::Column(i) + }; + ProjectItem { + alias: Some(f.name.clone()), + expr, + } + }) + .collect(); + node(NonASAPOp::Project { + child, + cols, + qualifier: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::unresolved::UnresolvedPredicate; + use asap_types::ir::BinaryOperator; + use asap_types::ir::ExprSemantics; + use asap_types::pre_asap::{ + BinaryOpKind, CompareOpKind, DataType, Field, JoinKind, PromQLVectorSetOpKind, ScalarValue, + Source, VectorMatch, + }; + use asap_types::types::AccuracyTarget; + + fn scan(metric: &str) -> UnresolvedOp { + UnresolvedOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: None, + } + } + + fn named(n: &str) -> UnresolvedScalar { + UnresolvedScalar::Column(ColumnRef::Named(n.into())) + } + + fn eq_lit(col: UnresolvedScalar, v: &str) -> UnresolvedScalar { + UnresolvedScalar::Compare { + left: Box::new(col), + op: CompareOpKind::Eq, + right: Box::new(UnresolvedScalar::Literal(ScalarValue::Utf8(v.into()))), + semantics: ExprSemantics::Promql, + } + } + + fn binary(kind: BinaryOpKind, vector_match: Option) -> BinaryOperator { + BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind, + vector_match, + } + } + + // Both sides resolve with qualifiers; an unknown right input is an error. + #[test] + fn resolve_pearson_corr_inputs() { + let schema = Schema::new(vec![ + Field::plain("x", DataType::Float64, true).with_table("a"), + Field::plain("x", DataType::Float64, true).with_table("b"), + ]); + let intent = AggIntent::PearsonCorr { + left: ColumnRef::Qualified { + table: "a".into(), + name: "x".into(), + }, + right: ColumnRef::Qualified { + table: "b".into(), + name: "x".into(), + }, + }; + assert_eq!( + resolve_agg_intent(&intent, &schema).unwrap(), + AggIntent::PearsonCorr { left: 0, right: 1 } + ); + let missing = AggIntent::PearsonCorr { + left: ColumnRef::Qualified { + table: "a".into(), + name: "x".into(), + }, + right: ColumnRef::Named("missing".into()), + }; + assert!(resolve_agg_intent(&missing, &schema).is_err()); + } + + // Every leg resolves independently, qualifiers included; one unknown leg + // fails rather than silently shortening the tuple. + #[test] + fn resolve_distinct_tuple_columns() { + let schema = Schema::new(vec![ + Field::plain("k", DataType::Int64, true).with_table("a"), + Field::plain("k", DataType::Int64, true).with_table("b"), + ]); + let qualified = |table: &str| ColumnRef::Qualified { + table: table.into(), + name: "k".into(), + }; + let intent = AggIntent::Cardinality { + cols: vec![qualified("b"), qualified("a")], + accuracy: AccuracyTarget::Exact, + }; + assert_eq!( + resolve_agg_intent(&intent, &schema).unwrap(), + AggIntent::Cardinality { + cols: vec![1, 0], + accuracy: AccuracyTarget::Exact, + } + ); + let missing = AggIntent::Cardinality { + cols: vec![qualified("a"), ColumnRef::Named("missing".into())], + accuracy: AccuracyTarget::Exact, + }; + assert!(resolve_agg_intent(&missing, &schema).is_err()); + } + + // ` > `: the bridged literal comes through unchanged, the + // vector side binds positionally, the `VectorMatch` survives untouched, and + // the node's schema follows the vector side. + #[test] + fn scalar_comparison_preserves_vector_values_and_labels() { + let unresolved = UnresolvedOp::PromqlScalarOp { + child: Rc::new(scan("up")), + scalar: UnresolvedScalar::Literal(ScalarValue::Float64(1.0)), + op: BinaryOpKind::Compare(CompareOpKind::Gt), + scalar_left: true, + return_bool: false, + }; + let resolved = resolve_root(&unresolved).unwrap(); + let NonASAPOp::Filter { + child, + pred: Predicate(ScalarExpr::Compare { left, right, .. }), + } = resolved.expect_non_asap() + else { + panic!("expected Filter") + }; + assert_eq!(**left, ScalarExpr::literal_f64(1.0)); + assert_eq!( + **right, + ScalarExpr::Column(child.schema.column_id("value").unwrap()) + ); + assert_eq!(resolved.schema, child.schema); + assert!(resolved.schema.has_promql_series_identity()); + } + + // A `Concat` discriminator column referenced nowhere else, over a + // schemaless first branch, resolves to the branch's own positional ids. + #[test] + fn resolve_root_seeds_and_resolves_an_otherwise_unreferenced_discriminator_column() { + let unresolved = UnresolvedOp::concat_with_discriminator( + vec![scan("m"), scan("m")], + ColumnRef::Named("phi".into()), + vec![ColumnRef::Named("host".into())], + ); + + let resolved = resolve_root(&unresolved).expect("resolves"); + let NonASAPOp::Concat { + children, + discriminator_unique_key, + } = resolved.expect_non_asap() + else { + panic!("expected a resolved Concat, got {resolved:?}"); + }; + let schema = &children[0].schema; + let key = discriminator_unique_key + .as_ref() + .expect("discriminator key survives resolution"); + assert_eq!(*key.discriminator(), schema.column_id("phi").unwrap()); + assert_eq!( + key.inner_key().to_vec(), + vec![schema.column_id("host").unwrap()] + ); + } + + // `sum by (job)(a or b)`: each `BinaryOp` side binds in its own scope but + // inherits the enclosing aggregate's group key (issue #52). + #[test] + fn binary_op_sides_inherit_enclosing_group_keys() { + let unresolved = UnresolvedOp::Aggregate { + reduction: Reduction::by(vec![ColumnRef::Named("job".into())]), + measures: vec![AggIntent::Sum { col: None }], + output_names: vec![], + filters: vec![], + having: None, + child: Rc::new(UnresolvedOp::BinaryOp { + operator: binary(BinaryOpKind::Set(PromQLVectorSetOpKind::Or), None), + return_bool: false, + lhs: Rc::new(scan("a")), + rhs: Rc::new(scan("b")), + }), + }; + let resolved = resolve_root(&unresolved).expect("resolves"); + let NonASAPOp::Aggregate { + reduction, child, .. + } = resolved.expect_non_asap() + else { + panic!("expected Aggregate"); + }; + let NonASAPOp::BinaryOp { lhs, rhs, .. } = child.expect_non_asap() else { + panic!("expected BinaryOp"); + }; + let job = lhs.schema.column_id("job").expect("lhs sees job"); + assert_eq!(rhs.schema.column_id("job"), Some(job)); + assert_eq!(reduction.expect_reduce().keys(), &[job]); + assert_eq!(resolved.schema.fields[0].name, "job"); + } + + // HAVING binds against the aggregate's output, not its input. + #[test] + fn having_resolves_against_aggregate_output() { + let input = Schema::new(vec![ + Field::plain("k", DataType::Utf8, false), + Field::plain("v", DataType::Float64, false), + ]); + let unresolved = UnresolvedOp::Aggregate { + reduction: Reduction::by(vec![ColumnRef::Named("k".into())]), + measures: vec![AggIntent::Sum { + col: Some(ColumnRef::Named("v".into())), + }], + output_names: vec!["total".into()], + filters: vec![], + having: Some(UnresolvedPredicate(UnresolvedScalar::Compare { + left: Box::new(named("total")), + op: CompareOpKind::Gt, + right: Box::new(UnresolvedScalar::Literal(ScalarValue::Float64(1.0))), + semantics: ExprSemantics::Sql, + })), + child: Rc::new(UnresolvedOp::Scan { + source: Source::Table { + table_ref: "t".into(), + }, + predicates: vec![], + schema: Some(input), + }), + }; + let resolved = resolve_root(&unresolved).expect("resolves"); + let NonASAPOp::Aggregate { + having: Some(Predicate(ScalarExpr::Compare { left, .. })), + .. + } = resolved.expect_non_asap() + else { + panic!("expected Aggregate with HAVING"); + }; + assert_eq!(**left, ScalarExpr::Column(1)); + assert_eq!(resolved.schema.fields[1].name, "total"); + } + + // A join predicate binds against left ++ right; a qualified reference + // picks the right side even when both inputs share the column name. + #[test] + fn join_predicate_resolves_against_left_then_right() { + let side = |table: &str| UnresolvedOp::Scan { + source: Source::Table { + table_ref: table.into(), + }, + predicates: vec![], + schema: Some(Schema::new(vec![ + Field::plain("k", DataType::Int64, false).with_table(table) + ])), + }; + let qualified = |table: &str| { + UnresolvedScalar::Column(ColumnRef::Qualified { + table: table.into(), + name: "k".into(), + }) + }; + let unresolved = UnresolvedOp::Join { + kind: JoinKind::Inner, + pred: UnresolvedPredicate(UnresolvedScalar::Compare { + left: Box::new(qualified("b")), + op: CompareOpKind::Eq, + right: Box::new(qualified("a")), + semantics: ExprSemantics::Sql, + }), + left: Rc::new(side("a")), + right: Rc::new(side("b")), + }; + let resolved = resolve_root(&unresolved).expect("resolves"); + let NonASAPOp::Join { + pred: Predicate(ScalarExpr::Compare { left, right, .. }), + .. + } = resolved.expect_non_asap() + else { + panic!("expected Join"); + }; + assert_eq!(**left, ScalarExpr::Column(1)); + assert_eq!(**right, ScalarExpr::Column(0)); + } + + // `m * scalar(x{a="1"})`: the operator inside the scalar operand is bound + // as a root in its own scope — its matcher label seeds its own leaf, not + // the vector side's. + #[test] + fn scalar_from_vector_operand_binds_in_its_own_scope() { + let x = UnresolvedOp::Scan { + source: Source::TimeSeries { metric: "x".into() }, + predicates: vec![UnresolvedPredicate(eq_lit(named("a"), "1"))], + schema: None, + }; + let unresolved = UnresolvedOp::PromqlScalarOp { + child: Rc::new(scan("m")), + scalar: UnresolvedScalar::PromqlScalarFromVector(Rc::new(x)), + op: BinaryOpKind::Arithmetic(asap_types::pre_asap::ArithmeticOpKind::Mul), + scalar_left: false, + return_bool: false, + }; + let resolved = resolve_root(&unresolved).unwrap(); + let NonASAPOp::Project { + child: lhs, cols, .. + } = resolved.expect_non_asap() + else { + panic!("expected Project") + }; + assert!(lhs.schema.column_id("a").is_none()); + let ScalarExpr::Arithmetic { right, .. } = &cols[1].expr else { + panic!("expected arithmetic") + }; + let ScalarExpr::PromqlScalarFromVector(inner) = right.as_ref() else { + panic!("expected scalar(v)") + }; + let a = inner + .schema + .column_id("a") + .expect("own matcher label seeded"); + let NonASAPOp::Scan { predicates, .. } = inner.expect_non_asap() else { + panic!("expected Scan"); + }; + let Predicate(ScalarExpr::Compare { left, .. }) = &predicates[0] else { + panic!("expected Compare"); + }; + assert_eq!(**left, ScalarExpr::Column(a)); + assert_eq!(resolved.schema.fields.len(), lhs.schema.fields.len()); + } + + // PromQL grouping drops a key provably absent from a closed input (#53): + // `sum(sum by (group)(m)) by (job)`. + #[test] + fn nested_aggregate_drops_absent_promql_group_key() { + let inner = UnresolvedOp::Aggregate { + reduction: Reduction::by(vec![ColumnRef::Named("group".into())]), + measures: vec![AggIntent::Sum { col: None }], + output_names: vec![], + filters: vec![], + having: None, + child: Rc::new(scan("m")), + }; + let outer = UnresolvedOp::Aggregate { + reduction: Reduction::by(vec![ColumnRef::Named("job".into())]), + measures: vec![AggIntent::Sum { col: None }], + output_names: vec![], + filters: vec![], + having: None, + child: Rc::new(inner), + }; + let resolved = resolve_root(&outer).expect("resolves"); + let NonASAPOp::Aggregate { reduction, .. } = resolved.expect_non_asap() else { + panic!("expected Aggregate"); + }; + assert!(reduction.expect_reduce().keys().is_empty()); + } +} diff --git a/crates/frontend-common/src/schema_resolver.rs b/crates/frontend-common/src/schema_resolver.rs new file mode 100644 index 00000000..ad7605fe --- /dev/null +++ b/crates/frontend-common/src/schema_resolver.rs @@ -0,0 +1,443 @@ +//! The **SchemaResolver** — name resolution as an explicit pass. +//! +//! [`SchemaResolver::resolve_schema`] produces the complete, self-contained +//! [`Schema`] every `ColumnId` in a schemaless leaf's scope indexes into, so +//! positional resolution in [`resolve`](crate::resolve) is total. +//! +//! The default [`UsageDerivedCatalog`] knows nothing — every schema is derived +//! purely from the query's own usage. That is the honest state for the +//! observability domain (metric label sets are open-ended). A registry-backed +//! `SchemaCatalog` is future work; only the catalog impl swaps when it lands. + +use asap_types::pre_asap::{AggIntent, ColumnRef, DataType, Field, GroupKeys, Reduction, Schema}; + +use crate::unresolved::{UnresolvedOp, UnresolvedScalar}; + +/// The DB / source-schema metadata source — resolves a source (metric / +/// table) name to its known columns. Distinct from `Scan.schema`, which is +/// the *resolved* binding schema this feeds. Even a registry-backed PromQL +/// catalog yields an **open** schema: a metric's labels are per-series and +/// time-varying, so the registry is a superset hint, not a per-row contract. +pub trait SchemaCatalog { + /// Columns known for `source`. `None` when unknown — the resolver then + /// falls back to a usage-derived column set. + fn columns_for(&self, source: &str) -> Option>; +} + +/// The default catalog: knows nothing. +pub struct UsageDerivedCatalog; + +impl SchemaCatalog for UsageDerivedCatalog { + fn columns_for(&self, _source: &str) -> Option> { + None + } +} + +/// The explicit name-resolution pass. +pub struct SchemaResolver { + catalog: C, +} + +impl Default for SchemaResolver { + fn default() -> Self { + Self::new() + } +} + +impl SchemaResolver { + pub fn new() -> Self { + Self { + catalog: UsageDerivedCatalog, + } + } +} + +impl SchemaResolver { + pub fn with_catalog(catalog: C) -> Self { + Self { catalog } + } + + /// The complete [`Schema`] in scope for a query rooted at `tree`: the + /// time axis, the synthetic `value` column, and one column per distinct + /// name referenced anywhere in the tree. + pub fn resolve_schema(&self, tree: &UnresolvedOp) -> Schema { + self.resolve_schema_with_inherited(tree, &[]) + } + + /// Like [`resolve_schema`](Self::resolve_schema), but also seeds + /// `inherited` label names referenced by an **enclosing** scope rather + /// than by `tree` itself. This is how an independently-bound `BinaryOp` + /// side still sees an outer aggregate's group keys — the `__name__` / + /// `job` in `sum by (__name__)(a or b)`, which appear in neither side's + /// own matchers (issue #52). + pub fn resolve_schema_with_inherited( + &self, + tree: &UnresolvedOp, + inherited: &[String], + ) -> Schema { + let mut columns: Vec = leftmost_scan_name(tree) + .and_then(|name| self.catalog.columns_for(name)) + .unwrap_or_else(default_leaf_columns); + + // Ensure the (ts, value) floor is present. + for floor in default_leaf_columns() { + if !columns.iter().any(|c| c.name == floor.name) { + columns.push(floor); + } + } + + // One column per referenced-but-unknown name, plus the inherited ones. + let referenced = collect_referenced_columns(tree); + for name in referenced.iter().chain(inherited) { + if !columns.iter().any(|c| c.name == *name) { + columns.push(Field::plain(name.clone(), DataType::Utf8, true)); + } + } + + let time_index = columns.iter().position(|c| c.name == "ts"); + Schema { + fields: columns, + time_index, + unique_keys: Vec::new(), + // Usage-derived (schemaless PromQL): the metric's full label set is + // open and runtime-only, so this lists only what the query references. + closed: false, + } + } +} + +/// The conventional PromQL leaf shape: `(ts: Timestamp, value: Float64)`. +fn default_leaf_columns() -> Vec { + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ] +} + +/// Push a `ColumnRef`'s bare name (the schema-seedable identifier). `Qualified` +/// collapses to its `name`; `SampleValue`/`Wildcard` carry no name. +fn push_ref_name(c: &ColumnRef, out: &mut Vec) { + match c { + ColumnRef::Named(n) => out.push(n.clone()), + ColumnRef::Qualified { name, .. } => out.push(name.clone()), + ColumnRef::SampleValue | ColumnRef::Wildcard => {} + } +} + +/// The leftmost `Scan`'s source name, following the relational skeleton only +/// (never the operators referenced from scalar positions: those are bound in +/// their own scope). +fn leftmost_scan_name(tree: &UnresolvedOp) -> Option<&str> { + use asap_types::pre_asap::Source; + use UnresolvedOp as U; + match tree { + U::Scan { source, .. } => Some(match source { + Source::TimeSeries { metric } => metric.as_str(), + Source::Table { table_ref } => table_ref.as_str(), + }), + U::Values { .. } | U::PromqlVectorFromScalar(_) => None, + U::PromqlMap { child, .. } + | U::PromqlScalarOp { child, .. } + | U::PromqlRelabel { child, .. } + | U::PromqlInfoEnrich { child, .. } + | U::PromqlSeriesSample { child, .. } + | U::Filter { child, .. } + | U::Project { child, .. } + | U::Aggregate { child, .. } + | U::Dedup { child, .. } + | U::Sort { child, .. } + | U::Limit { child, .. } + | U::PromqlSubquery { child, .. } + | U::TimeRange { child, .. } + | U::TimeShift { child, .. } + | U::SQLWindowFunc { child, .. } => leftmost_scan_name(child), + U::Concat { children, .. } => children.first().and_then(|c| leftmost_scan_name(c)), + U::Join { left, .. } | U::SetOp { left, .. } | U::BinaryOp { lhs: left, .. } => { + leftmost_scan_name(left) + } + } +} + +/// Every distinct column name referenced anywhere in `tree` that resolves +/// positionally — every place a front end puts a name-based reference: +/// `Scan.predicates`, `Aggregate`'s `reduction`/`having`/per-measure `col`, +/// `Dedup.cols`, `PromqlSeriesSample.by`, `Filter.pred`, `Project.cols`, +/// `Sort`/`Limit`/`SQLWindowFunc` keys, `Join.pred`, `PromqlRelabel.value`, +/// `Concat.discriminator_unique_key`. Operators referenced from scalar +/// positions (`scalar(v)`, subqueries) are walked too, as the old +/// `PromqlScalarFromVector` operator child was. Sorted and deduplicated. +pub fn collect_referenced_columns(tree: &UnresolvedOp) -> Vec { + use UnresolvedOp as U; + fn named(expr: &UnresolvedScalar, out: &mut Vec) { + for c in expr.columns_referenced() { + push_ref_name(c, out); + } + for op in expr.operator_refs() { + walk(op, out); + } + } + fn group_keys(g: &GroupKeys, out: &mut Vec) { + g.keys().iter().for_each(|k| push_ref_name(k, out)); + } + fn measure_cols(measures: &[AggIntent], out: &mut Vec) { + for m in measures { + for c in m.input_cols() { + push_ref_name(&c, out); + } + } + } + fn walk(node: &UnresolvedOp, out: &mut Vec) { + match node { + U::Scan { predicates, .. } => { + for p in predicates { + named(&p.0, out); + } + } + U::Values { rows, .. } => { + for e in rows.iter().flatten() { + named(e, out); + } + } + U::Aggregate { + reduction, + measures, + having, + child, + .. + } => { + if let Reduction::Reduce(by) = reduction { + group_keys(by, out); + } + measure_cols(measures, out); + if let Some(h) = having { + named(&h.0, out); + } + walk(child, out); + } + U::Dedup { cols, child } => { + cols.iter().for_each(|c| push_ref_name(c, out)); + walk(child, out); + } + U::PromqlSeriesSample { by, child, .. } => { + group_keys(by, out); + walk(child, out); + } + U::Filter { pred, child } => { + named(&pred.0, out); + walk(child, out); + } + U::Project { cols, child, .. } => { + for item in cols { + named(&item.expr, out); + } + walk(child, out); + } + U::Sort { + keys, + partition_by, + child, + } => { + for k in keys { + named(&k.expr, out); + } + group_keys(partition_by, out); + walk(child, out); + } + U::Limit { + partition_by, + child, + .. + } => { + group_keys(partition_by, out); + walk(child, out); + } + U::SQLWindowFunc { + args, + partition_by, + order_by, + child, + .. + } => { + for a in args { + named(a, out); + } + group_keys(partition_by, out); + for k in order_by { + named(&k.expr, out); + } + walk(child, out); + } + U::PromqlRelabel { value, child, .. } => { + named(value, out); + walk(child, out); + } + U::Join { + pred, left, right, .. + } => { + named(&pred.0, out); + walk(left, out); + walk(right, out); + } + U::PromqlVectorFromScalar(inner) => named(inner, out), + U::PromqlMap { child, .. } + | U::PromqlScalarOp { child, .. } + | U::PromqlInfoEnrich { child, .. } + | U::PromqlSubquery { child, .. } + | U::TimeRange { child, .. } + | U::TimeShift { child, .. } => walk(child, out), + U::Concat { + children, + discriminator_unique_key, + } => { + // An own-field `ColumnRef` must be seeded like `Dedup.cols`, or + // a discriminator column referenced nowhere else in the tree is + // absent from the fallback schema and fails `NotFound` later. + if let Some(key) = discriminator_unique_key { + push_ref_name(key.discriminator(), out); + key.inner_key().iter().for_each(|c| push_ref_name(c, out)); + } + children.iter().for_each(|c| walk(c, out)); + } + U::SetOp { left, right, .. } => { + walk(left, out); + walk(right, out); + } + U::BinaryOp { lhs, rhs, .. } => { + walk(lhs, out); + walk(rhs, out); + } + } + } + let mut out: Vec = Vec::new(); + walk(tree, &mut out); + out.sort(); + out.dedup(); + out +} + +#[cfg(test)] +mod tests { + use std::rc::Rc; + + use asap_types::pre_asap::{AggIntent, Reduction, Source}; + + use super::*; + use crate::unresolved::UnresolvedSortKey; + + fn src(name: &str) -> UnresolvedOp { + UnresolvedOp::Scan { + source: Source::TimeSeries { + metric: name.into(), + }, + predicates: vec![], + schema: None, + } + } + + // Both correlation inputs seed the usage-derived schema. + #[test] + fn pearson_corr_inputs_seed_usage_derived_schema() { + let tree = UnresolvedOp::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::PearsonCorr { + left: ColumnRef::Named("x".into()), + right: ColumnRef::Named("y".into()), + }], + output_names: vec![], + filters: vec![], + having: None, + child: Rc::new(src("m")), + }; + assert_eq!(collect_referenced_columns(&tree), vec!["x", "y"]); + let schema = SchemaResolver::new().resolve_schema(&tree); + assert!(schema.column_id("x").is_some()); + assert!(schema.column_id("y").is_some()); + } + + // A bare source gets exactly the (ts, value) floor. + #[test] + fn bare_source_yields_ts_value_floor() { + let schema = SchemaResolver::new().resolve_schema(&src("m")); + assert_eq!(schema.fields.len(), 2); + assert_eq!(schema.fields[0].name, "ts"); + assert_eq!(schema.fields[1].name, "value"); + assert_eq!(schema.time_index, Some(0)); + } + + // Per-group ranking keys (`topk by (host)` → `Sort.partition_by`) are + // seeded into the usage-derived leaf so they resolve positionally. + #[test] + fn sort_partition_keys_land_in_schema() { + let tree = UnresolvedOp::Sort { + keys: vec![UnresolvedSortKey { + expr: UnresolvedScalar::Column(ColumnRef::SampleValue), + ascending: false, + nulls_first: false, + }], + partition_by: GroupKeys::by(vec![ColumnRef::Named("host".into())]), + child: Rc::new(src("hits")), + }; + let schema = SchemaResolver::new().resolve_schema(&tree); + assert!(schema.column_id("host").is_some()); + } + + // `Limit.partition_by` (PromQL `topk by (..)`) is seeded like `Sort`'s. + #[test] + fn limit_partition_keys_land_in_schema() { + let tree = UnresolvedOp::Limit { + n: Some(3), + offset: 0, + partition_by: GroupKeys::by(vec![ColumnRef::Named("host".into())]), + child: Rc::new(src("hits")), + }; + let schema = SchemaResolver::new().resolve_schema(&tree); + assert!(schema.column_id("host").is_some()); + } + + // A `Concat`'s discriminator key columns, even ones referenced nowhere + // else, are seeded like `Dedup.cols` (issue #228 review). + #[test] + fn concat_discriminator_key_is_seeded_into_the_resolver_schema() { + let tree = UnresolvedOp::concat_with_discriminator( + vec![src("m")], + ColumnRef::Named("phi".into()), + vec![ColumnRef::Named("host".into())], + ); + let schema = SchemaResolver::new().resolve_schema(&tree); + assert!(schema.column_id("phi").is_some(), "discriminator seeded"); + assert!(schema.column_id("host").is_some(), "inner_key seeded"); + } + + // Inherited names are seeded alongside the tree's own references; plain + // `resolve_schema` does not conjure them (issue #52). + #[test] + fn inherited_names_are_seeded_alongside_referenced() { + let schema = + SchemaResolver::new().resolve_schema_with_inherited(&src("m"), &["__name__".into()]); + assert!(schema.column_id("__name__").is_some()); + let plain = SchemaResolver::new().resolve_schema(&src("m")); + assert!(plain.column_id("__name__").is_none()); + } + + // A catalog-known source supplies its base columns, typed as the catalog says. + #[test] + fn custom_catalog_supplies_base_columns() { + struct FixedCatalog; + impl SchemaCatalog for FixedCatalog { + fn columns_for(&self, source: &str) -> Option> { + (source == "known").then(|| { + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("datacenter", DataType::Utf8, false), + ] + }) + } + } + let schema = SchemaResolver::with_catalog(FixedCatalog).resolve_schema(&src("known")); + let dc = schema + .column_id("datacenter") + .and_then(|id| schema.fields.get(id)); + assert!(matches!(dc, Some(c) if !c.nullable)); + } +} diff --git a/crates/frontend-common/src/unresolved.rs b/crates/frontend-common/src/unresolved.rs new file mode 100644 index 00000000..0bca427c --- /dev/null +++ b/crates/frontend-common/src/unresolved.rs @@ -0,0 +1,374 @@ +//! The front-end-emitted, name-based operator tree: a mirror of the unified +//! IR ([`NonASAPOp`](asap_types::ir::NonASAPOp) / [`ScalarExpr`](asap_types::ir::ScalarExpr)) +//! before name resolution. +//! +//! Differences from the resolved IR, and nothing else: +//! - every `ColumnId` is a name-based [`ColumnRef`]; +//! - `Scan.schema` is `Option` — a front end knows the schema only for +//! a catalog-backed SQL leaf; `None` (PromQL) defers to the +//! [`SchemaResolver`](crate::schema_resolver::SchemaResolver); +//! - children are `Rc` rather than `Rc` — no +//! derived schema exists yet. + +use std::rc::Rc; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +use asap_types::ir::operator_properties::ConcatDiscriminatorKey; +use asap_types::ir::BinaryOperator; +use asap_types::ir::{ExprSemantics, TimeRangeKind}; +use asap_types::pre_asap::{ + AggIntent, ArithmeticOpKind, ColumnRef, CompareOpKind, DataType, GroupKeys, InfoMatcher, + JoinKind, Reduction, RelationalSetOpKind, SampleKind, ScalarValue, Schema, Source, TimeShift, + WindowFrame, WindowFuncKind, +}; + +/// A row-level filter predicate (WHERE clause / PromQL label matcher). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct UnresolvedPredicate(pub UnresolvedScalar); + +/// One item in a SELECT projection list. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct UnresolvedProjectItem { + pub alias: Option, + pub expr: UnresolvedScalar, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct UnresolvedSortKey { + pub expr: UnresolvedScalar, + pub ascending: bool, + pub nulls_first: bool, +} + +/// A name-based scalar expression; see +/// [`ScalarExpr`](asap_types::ir::ScalarExpr) for the meaning of each variant. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum UnresolvedScalar { + Column(ColumnRef), + Literal(ScalarValue), + Negative { + expr: Box, + semantics: ExprSemantics, + }, + Compare { + left: Box, + op: CompareOpKind, + right: Box, + semantics: ExprSemantics, + }, + BoolAnd(Vec), + BoolOr(Vec), + Not(Box), + IsNull(Box), + IsNotNull(Box), + Cast { + expr: Box, + to: DataType, + try_cast: bool, + }, + InList { + expr: Box, + list: Vec, + negated: bool, + }, + FunctionCall { + name: String, + args: Vec, + }, + Arithmetic { + op: ArithmeticOpKind, + left: Box, + right: Box, + semantics: ExprSemantics, + }, + Case { + operand: Option>, + branches: Vec<(UnresolvedScalar, UnresolvedScalar)>, + else_expr: Option>, + }, + CurrentTimestamp, + EvalTimestamp, + /// PromQL `scalar(v)`. The operator is resolved as a root in its own scope. + PromqlScalarFromVector(Rc), + ScalarSubquery(Rc), + Exists { + subquery: Rc, + negated: bool, + }, + InSubquery { + expr: Box, + subquery: Rc, + negated: bool, + }, +} + +/// The name-based operator tree; see [`NonASAPOp`](asap_types::ir::NonASAPOp) +/// for the meaning of each variant. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum UnresolvedOp { + Scan { + source: Source, + predicates: Vec, + /// `Some` for a catalog-backed (SQL) leaf; `None` defers to the + /// usage-derived schema resolver. + schema: Option, + }, + Values { + rows: Vec>, + schema: Schema, + }, + Filter { + pred: UnresolvedPredicate, + child: Rc, + }, + Project { + cols: Vec, + qualifier: Option, + child: Rc, + }, + Aggregate { + reduction: Reduction, + measures: Vec>, + output_names: Vec, + filters: Vec>, + having: Option, + child: Rc, + }, + Join { + kind: JoinKind, + pred: UnresolvedPredicate, + left: Rc, + right: Rc, + }, + SetOp { + kind: RelationalSetOpKind, + all: bool, + left: Rc, + right: Rc, + }, + Concat { + children: Vec>, + discriminator_unique_key: Option>, + }, + Dedup { + cols: Vec, + child: Rc, + }, + Sort { + keys: Vec, + partition_by: GroupKeys, + child: Rc, + }, + Limit { + n: Option, + offset: usize, + partition_by: GroupKeys, + child: Rc, + }, + BinaryOp { + operator: BinaryOperator, + return_bool: bool, + lhs: Rc, + rhs: Rc, + }, + SQLWindowFunc { + func: WindowFuncKind, + args: Vec, + partition_by: GroupKeys, + order_by: Vec, + frame: Option, + output_name: String, + child: Rc, + }, + TimeRange { + range: Duration, + kind: TimeRangeKind, + child: Rc, + }, + TimeShift { + shift: TimeShift, + child: Rc, + }, + PromqlVectorFromScalar(UnresolvedScalar), + PromqlRelabel { + dst: String, + value: UnresolvedScalar, + child: Rc, + }, + PromqlInfoEnrich { + selector: Vec, + child: Rc, + }, + PromqlSeriesSample { + by: GroupKeys, + kind: SampleKind, + child: Rc, + }, + PromqlSubquery { + range: Duration, + resolution: Option, + child: Rc, + }, + /// Bind the complete vector schema before lowering to Project or Filter. + /// Frontend-only expansion to a projection preserving the complete series identity. + PromqlMap { + child: Rc, + sample: UnresolvedScalar, + drop_metric_name: bool, + }, + PromqlScalarOp { + child: Rc, + scalar: UnresolvedScalar, + op: asap_types::pre_asap::BinaryOpKind, + scalar_left: bool, + return_bool: bool, + }, +} + +impl UnresolvedScalar { + /// The direct scalar sub-expressions (not the operators this expression + /// reads — see [`operator_refs`](Self::operator_refs)). + pub fn children(&self) -> Vec<&UnresolvedScalar> { + use UnresolvedScalar::*; + match self { + Column(_) + | Literal(_) + | CurrentTimestamp + | EvalTimestamp + | PromqlScalarFromVector(_) + | ScalarSubquery(_) + | Exists { .. } => vec![], + Negative { expr, .. } + | Not(expr) + | IsNull(expr) + | IsNotNull(expr) + | Cast { expr, .. } + | InSubquery { expr, .. } => vec![expr], + Compare { left, right, .. } | Arithmetic { left, right, .. } => vec![left, right], + BoolAnd(parts) | BoolOr(parts) => parts.iter().collect(), + InList { expr, list, .. } => { + let mut v = vec![expr.as_ref()]; + v.extend(list.iter()); + v + } + FunctionCall { args, .. } => args.iter().collect(), + Case { + operand, + branches, + else_expr, + } => { + let mut v = Vec::new(); + if let Some(op) = operand { + v.push(op.as_ref()); + } + for (when, then) in branches { + v.push(when); + v.push(then); + } + if let Some(e) = else_expr { + v.push(e.as_ref()); + } + v + } + } + } + + /// Every column referenced in this expression, not inside the operators + /// it reads (those have their own scope). + pub fn columns_referenced(&self) -> Vec<&ColumnRef> { + let mut out = Vec::new(); + self.collect_columns(&mut out); + out + } + + fn collect_columns<'a>(&'a self, out: &mut Vec<&'a ColumnRef>) { + if let UnresolvedScalar::Column(c) = self { + out.push(c); + } + for child in self.children() { + child.collect_columns(out); + } + } + + /// The operators this expression (transitively) reads. + pub fn operator_refs(&self) -> Vec<&Rc> { + let mut out = Vec::new(); + self.collect_operator_refs(&mut out); + out + } + + fn collect_operator_refs<'a>(&'a self, out: &mut Vec<&'a Rc>) { + use UnresolvedScalar::*; + match self { + PromqlScalarFromVector(op) | ScalarSubquery(op) => out.push(op), + Exists { subquery, .. } | InSubquery { subquery, .. } => out.push(subquery), + _ => {} + } + for child in self.children() { + child.collect_operator_refs(out); + } + } +} + +impl UnresolvedOp { + /// An ordinary `Concat` (no unique-key claim). + pub fn concat(children: Vec) -> Self { + UnresolvedOp::Concat { + children: children.into_iter().map(Rc::new).collect(), + discriminator_unique_key: None, + } + } + + /// A `Concat` whose output carries the caller-proven compound unique key + /// `(discriminator, inner_key)`. Nothing verifies the claim. + pub fn concat_with_discriminator( + children: Vec, + discriminator: ColumnRef, + inner_key: Vec, + ) -> Self { + UnresolvedOp::Concat { + children: children.into_iter().map(Rc::new).collect(), + discriminator_unique_key: Some(ConcatDiscriminatorKey::new(discriminator, inner_key)), + } + } + + /// Every scalar expression this operator owns. + pub fn scalar_exprs(&self) -> Vec<&UnresolvedScalar> { + use UnresolvedOp::*; + match self { + Scan { predicates, .. } => predicates.iter().map(|p| &p.0).collect(), + Values { rows, .. } => rows.iter().flatten().collect(), + Filter { pred, .. } | Join { pred, .. } => vec![&pred.0], + Project { cols, .. } => cols.iter().map(|c| &c.expr).collect(), + Aggregate { + filters, having, .. + } => filters + .iter() + .flatten() + .chain(having.iter()) + .map(|p| &p.0) + .collect(), + Sort { keys, .. } => keys.iter().map(|k| &k.expr).collect(), + SQLWindowFunc { args, order_by, .. } => args + .iter() + .chain(order_by.iter().map(|k| &k.expr)) + .collect(), + PromqlVectorFromScalar(e) => vec![e], + PromqlScalarOp { scalar, .. } => vec![scalar], + PromqlMap { sample, .. } => vec![sample], + PromqlRelabel { value, .. } => vec![value], + SetOp { .. } + | Concat { .. } + | Dedup { .. } + | Limit { .. } + | BinaryOp { .. } + | TimeRange { .. } + | TimeShift { .. } + | PromqlInfoEnrich { .. } + | PromqlSeriesSample { .. } + | PromqlSubquery { .. } => vec![], + } + } +} diff --git a/crates/frontend-sql/Cargo.toml b/crates/frontend-sql/Cargo.toml index ac6bd2f5..a6841830 100644 --- a/crates/frontend-sql/Cargo.toml +++ b/crates/frontend-sql/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" # #225) it consults when lowering an aggregate call — never promql-parser. [dependencies] asap-types = { path = "../types" } +asap-frontend-common = { path = "../frontend-common" } asap-sql-function-catalog = { path = "../sql-function-catalog" } datafusion = "43" # `AggIntent::Extension.payload` for ClickHouse's argMax/argMin (issue #232) diff --git a/crates/frontend-sql/src/lib.rs b/crates/frontend-sql/src/lib.rs index 4ec61e69..aa1449e5 100644 --- a/crates/frontend-sql/src/lib.rs +++ b/crates/frontend-sql/src/lib.rs @@ -102,3 +102,6 @@ pub async fn lower_sql_batch( } results } + +/// Unified SQL lowering; promoted to the root API at the planner cutover. +pub mod unified; diff --git a/crates/frontend-sql/src/unified/error.rs b/crates/frontend-sql/src/unified/error.rs new file mode 100644 index 00000000..f819cfd4 --- /dev/null +++ b/crates/frontend-sql/src/unified/error.rs @@ -0,0 +1,63 @@ +use std::fmt; + +use asap_frontend_common::ResolveDAGError; + +/// Errors from lowering a SQL query (parse + plan via DataFusion → the +/// name-based [`UnresolvedOp`](asap_frontend_common::UnresolvedOp) tree → +/// [`resolve_root`](asap_frontend_common::resolve_root) binds it into the +/// unified IR). +/// +/// Carries no PromQL type — the SQL front end never depends on the PromQL +/// parser. The language-neutral variants (`UnsupportedFeature` / `WrongLanguage` +/// / `Convert`) are mirrored by [`asap_frontend_promql::PromqlError`] rather +/// than shared, so neither front end pulls the other's parser. +#[derive(Debug)] +pub enum SqlError { + /// DataFusion failed to parse / plan the SQL query. + DataFusion(datafusion::error::DataFusionError), + /// A table referenced by the query is absent from the catalog. + TableNotFound(String), + /// A SQL aggregate function not supported in this version. + UnsupportedAggregate(String), + /// A SQL scalar expression that could not be lowered. + InvalidExpression(String), + /// The SQL dialect is not supported (only DataFusionSQL is implemented). + UnsupportedDialect(String), + /// A structural feature (JOIN type / subquery / derived table) not + /// supported in this version. + UnsupportedFeature(String), + /// The workload's query language is not SQL. + WrongLanguage(String), + /// Resolving the name-based tree failed (name resolution against the + /// bound schema, or schema derivation). + Convert(ResolveDAGError), +} + +impl fmt::Display for SqlError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DataFusion(e) => write!(f, "DataFusion error: {e}"), + Self::TableNotFound(t) => write!(f, "table not found in catalog: {t}"), + Self::UnsupportedAggregate(n) => write!(f, "unsupported aggregate: {n}"), + Self::InvalidExpression(m) => write!(f, "invalid expression: {m}"), + Self::UnsupportedDialect(d) => write!(f, "unsupported SQL dialect: {d}"), + Self::UnsupportedFeature(m) => write!(f, "unsupported feature: {m}"), + Self::WrongLanguage(l) => write!(f, "unsupported query language: {l}"), + Self::Convert(e) => write!(f, "column resolution failed: {e}"), + } + } +} + +impl std::error::Error for SqlError {} + +impl From for SqlError { + fn from(e: ResolveDAGError) -> Self { + Self::Convert(e) + } +} + +impl From for SqlError { + fn from(e: datafusion::error::DataFusionError) -> Self { + Self::DataFusion(e) + } +} diff --git a/crates/frontend-sql/src/unified/mod.rs b/crates/frontend-sql/src/unified/mod.rs new file mode 100644 index 00000000..279585fb --- /dev/null +++ b/crates/frontend-sql/src/unified/mod.rs @@ -0,0 +1,103 @@ +//! SQL front end: parse + plan (via DataFusion) → the name-based +//! [`UnresolvedOp`](asap_frontend_common::UnresolvedOp) tree, built directly +//! (issue #179) → [`resolve_root`]. +//! +//! Emits the shared front-end tree (`UnresolvedOp` / `UnresolvedScalar`, +//! name-based [`ColumnRef`](asap_types::pre_asap::ColumnRef)s) directly, rather +//! than a separate per-language relational tree; `resolve_root` binds it into +//! the unified [`OperatorNode`] IR, deriving every schema on the way. +//! Depends on DataFusion only — never on the PromQL parser. + +pub mod error; +pub mod sql; + +use std::rc::Rc; + +use asap_frontend_common::resolve_root; +use asap_types::ir::OperatorNode; +use asap_types::types::AccuracyTarget; +use asap_types::workload::{QueryLanguage, QueryWorkload, SqlDialect}; + +pub use error::SqlError; +pub use sql::{SqlCatalog, SqlLowerer}; + +/// Lower a single SQL query string to the resolved, canonical operator DAG, +/// parsed as `SqlDialect::DataFusionSQL`. +/// +/// The `catalog` supplies table schemas (used both to plan the SQL with +/// DataFusion and to carry positional column identity into the resolved +/// tree). `accuracy` is threaded onto every approximate intent as it's built. +pub async fn lower_sql( + query: &str, + catalog: &SqlCatalog, + accuracy: AccuracyTarget, +) -> Result, SqlError> { + lower_sql_dialect(query, catalog, SqlDialect::DataFusionSQL, accuracy).await +} + +/// Lower a single SQL query string under an explicit [`SqlDialect`]. +/// +/// `ClickhouseSQL` parses via sqlparser's vendored `ClickHouseDialect` +/// (array-lambda syntax, `arr[-1]` indexing). It also teaches DataFusion's +/// planner the ClickHouse-only builtin functions listed in +/// `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` (`uniqExact`, `countIf`) +/// — every other ClickHouse-only builtin still fails to plan. +/// `ElasticSQL` has no vendored parser and always returns `UnsupportedDialect`. +pub async fn lower_sql_dialect( + query: &str, + catalog: &SqlCatalog, + dialect: SqlDialect, + accuracy: AccuracyTarget, +) -> Result, SqlError> { + let unresolved = SqlLowerer::with_dialect(catalog, dialect) + .lower(query, &accuracy) + .await?; + // Binding resolves names and derives every node's schema; result-type + // checks (such as temporal subtraction, whose duration unit the IR cannot + // represent) surface here as `ResolveDAGError::Schema`. + Ok(resolve_root(&unresolved)?) +} + +/// Lower every SQL batch entry in `workload` to an operator DAG. +/// +/// One `Result` per entry — errors are per-query, not fatal for the batch. +/// Returns `WrongLanguage` for every entry if the workload is not SQL, and +/// `UnsupportedDialect` for `ElasticSQL` (no vendored parser). +pub async fn lower_sql_batch( + workload: &QueryWorkload, + catalog: &SqlCatalog, +) -> Vec, SqlError>> { + let entries = match &workload.query_batch { + Some(e) if !e.is_empty() => e, + _ => return vec![], + }; + + // `DataFusion` is a legacy alias for `SQL(DataFusionSQL)`; accept both. + if !matches!( + workload.language, + QueryLanguage::SQL(_) | QueryLanguage::DataFusion + ) { + let lang = format!("{:?}", workload.language); + return entries + .iter() + .map(|_| Err(SqlError::WrongLanguage(lang.clone()))) + .collect(); + } + let dialect = match &workload.language { + QueryLanguage::SQL(d) => d.clone(), + _ => SqlDialect::DataFusionSQL, + }; + if matches!(dialect, SqlDialect::ElasticSQL) { + return entries + .iter() + .map(|_| Err(SqlError::UnsupportedDialect("ElasticSQL".into()))) + .collect(); + } + + let mut results = Vec::with_capacity(entries.len()); + for entry in entries { + let accuracy = entry.requirements.accuracy.target(); + results.push(lower_sql_dialect(&entry.query.0, catalog, dialect.clone(), accuracy).await); + } + results +} diff --git a/crates/frontend-sql/src/unified/sql/clickhouse_ast.rs b/crates/frontend-sql/src/unified/sql/clickhouse_ast.rs new file mode 100644 index 00000000..c68a803b --- /dev/null +++ b/crates/frontend-sql/src/unified/sql/clickhouse_ast.rs @@ -0,0 +1,139 @@ +//! Structural ClickHouse syntax normalization before DataFusion type inference. +use datafusion::sql::sqlparser::ast::{ + visit_expressions, visit_expressions_mut, BinaryOperator, Expr, Function, FunctionArg, + FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, MapAccessSyntax, ObjectName, + Query, SelectItem, SetExpr, Statement, VisitMut, VisitorMut, +}; +use std::ops::ControlFlow; + +pub(super) fn normalize(statement: &mut Statement) { + struct PreserveNames; + impl VisitorMut for PreserveNames { + type Break = (); + fn pre_visit_query(&mut self, query: &mut Query) -> ControlFlow<()> { + fn preserve(body: &mut SetExpr) { + match body { + SetExpr::Select(select) => { + for item in &mut select.projection { + if let SelectItem::UnnamedExpr(expr) = item { + let mut changed = false; + let _: ControlFlow<()> = visit_expressions_mut(expr, |node| { + changed |= normalize_map_access(node); + ControlFlow::Continue(()) + }); + let _: ControlFlow<()> = visit_expressions(expr, |candidate| { + if let Expr::Function(function) = candidate { + changed |= function.name.0.len() == 1 + && function.name.0[0].quote_style.is_none() + && matches!( + function.name.0[0] + .value + .to_ascii_lowercase() + .as_str(), + "modulo" + | "map" + | "mapconcat" + | "arrayelement" + | "tupleelement" + ); + } + ControlFlow::Continue(()) + }); + if changed { + let alias = Ident::with_quote('"', expr.to_string()); + let value = std::mem::replace( + expr, + Expr::Value(datafusion::sql::sqlparser::ast::Value::Null), + ); + *item = SelectItem::ExprWithAlias { expr: value, alias }; + } + } + } + } + SetExpr::SetOperation { left, right, .. } => { + preserve(left); + preserve(right); + } + _ => {} + } + } + preserve(&mut query.body); + ControlFlow::Continue(()) + } + } + let _: ControlFlow<()> = statement.visit(&mut PreserveNames); + let _: ControlFlow<()> = visit_expressions_mut(statement, |expr| { + normalize_map_access(expr); + let Expr::Function(function) = expr else { + return ControlFlow::Continue(()); + }; + if function.name.0.len() != 1 + || function.name.0[0].quote_style.is_some() + || !function.name.0[0].value.eq_ignore_ascii_case("modulo") + || !matches!(function.parameters, FunctionArguments::None) + || function.filter.is_some() + || function.over.is_some() + || function.null_treatment.is_some() + || !function.within_group.is_empty() + { + return ControlFlow::Continue(()); + } + let FunctionArguments::List(arguments) = &function.args else { + return ControlFlow::Continue(()); + }; + if arguments.duplicate_treatment.is_some() || !arguments.clauses.is_empty() { + return ControlFlow::Continue(()); + } + let [FunctionArg::Unnamed(FunctionArgExpr::Expr(left)), FunctionArg::Unnamed(FunctionArgExpr::Expr(right))] = + arguments.args.as_slice() + else { + return ControlFlow::Continue(()); + }; + *expr = Expr::BinaryOp { + left: Box::new(left.clone()), + op: BinaryOperator::Modulo, + right: Box::new(right.clone()), + }; + ControlFlow::Continue(()) + }); +} + +fn normalize_map_access(expression: &mut Expr) -> bool { + let Expr::MapAccess { keys, .. } = expression else { + return false; + }; + if keys.is_empty() + || keys + .iter() + .any(|key| key.syntax != MapAccessSyntax::Bracket) + { + return false; + } + let Expr::MapAccess { column, keys } = std::mem::replace( + expression, + Expr::Value(datafusion::sql::sqlparser::ast::Value::Null), + ) else { + unreachable!() + }; + let mut input = *column; + for key in keys { + input = Expr::Function(Function { + name: ObjectName(vec![Ident::new("arrayElement")]), + parameters: FunctionArguments::None, + args: FunctionArguments::List(FunctionArgumentList { + duplicate_treatment: None, + clauses: vec![], + args: vec![ + FunctionArg::Unnamed(FunctionArgExpr::Expr(input)), + FunctionArg::Unnamed(FunctionArgExpr::Expr(key.key)), + ], + }), + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }); + } + *expression = input; + true +} diff --git a/crates/frontend-sql/src/unified/sql/collection_planning.rs b/crates/frontend-sql/src/unified/sql/collection_planning.rs new file mode 100644 index 00000000..75d0450c --- /dev/null +++ b/crates/frontend-sql/src/unified/sql/collection_planning.rs @@ -0,0 +1,189 @@ +//! DataFusion planning adapters. Types come from the canonical signature rules; +//! physical evaluation deliberately remains the query engine's responsibility. +use super::types::{arrow_to_dtype, dtype_to_arrow, scalar_value_to_asap}; +use asap_types::ir::scalar::{element_access_type, struct_field_type}; +use asap_types::ir::ScalarExpr; +use asap_types::pre_asap::scalar_type_rules::MapScalarFunction; +use asap_types::pre_asap::{Field, Schema}; +use datafusion::arrow::datatypes::DataType; +use datafusion::common::{DataFusionError, ExprSchema, Result}; +use datafusion::logical_expr::{ + ColumnarValue, Expr, ExprSchemable, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, + Volatility, +}; +use datafusion::prelude::SessionContext; + +#[derive(Debug, Clone, Copy)] +enum PlanningFunction { + Map(MapScalarFunction), + Element, + StructField, +} + +pub(super) fn register(context: &SessionContext) { + for (name, function) in [ + ("map", PlanningFunction::Map(MapScalarFunction::Construct)), + ( + "mapconcat", + PlanningFunction::Map(MapScalarFunction::Concat), + ), + ("arrayelement", PlanningFunction::Element), + ("tupleelement", PlanningFunction::StructField), + ] { + context.register_udf(ScalarUDF::from(CollectionPlanningFunction { + name, + function, + signature: match function { + PlanningFunction::Map(MapScalarFunction::Construct) => Signature::one_of( + vec![TypeSignature::Exact(vec![]), TypeSignature::VariadicAny], + Volatility::Immutable, + ), + PlanningFunction::Map(MapScalarFunction::Access) + | PlanningFunction::Element + | PlanningFunction::StructField => Signature::any(2, Volatility::Immutable), + PlanningFunction::Map(MapScalarFunction::Concat) => { + Signature::variadic_any(Volatility::Immutable) + } + }, + })); + } +} +#[derive(Debug)] +struct CollectionPlanningFunction { + name: &'static str, + function: PlanningFunction, + signature: Signature, +} +impl CollectionPlanningFunction { + fn output( + &self, + args: &[DataType], + nullable: &[bool], + expressions: Option<&[Expr]>, + ) -> Result<(DataType, bool)> { + let inputs = args + .iter() + .zip(nullable) + .map(|(dtype, null)| { + arrow_to_dtype(dtype) + .map(|dtype| (dtype, *null)) + .map_err(|e| DataFusionError::Plan(e.to_string())) + }) + .collect::>>()?; + let (dtype, nullable) = if matches!( + self.function, + PlanningFunction::Element | PlanningFunction::StructField + ) { + // DataFusion asks for argument-dependent types before canonical + // expression binding. Reuse the shared resolver over typed argument + // slots; final canonical binding also validates literal selectors. + let schema = Schema::new( + inputs + .into_iter() + .enumerate() + .map(|(index, (dtype, nullable))| { + Field::plain(format!("argument_{index}"), dtype, nullable) + }) + .collect(), + ); + let args = (0..schema.fields.len()) + .map(|index| { + if let Some(Expr::Literal(value)) = expressions.and_then(|args| args.get(index)) + { + scalar_value_to_asap(value) + .map(ScalarExpr::Literal) + .map_err(|error| DataFusionError::Plan(error.to_string())) + } else { + Ok(ScalarExpr::Column(index)) + } + }) + .collect::>>()?; + match self.function { + PlanningFunction::Element => element_access_type(&args, &schema), + PlanningFunction::StructField => struct_field_type(&args, &schema), + PlanningFunction::Map(_) => unreachable!(), + } + } else if let PlanningFunction::Map(function) = self.function { + function.output_type(&inputs) + } else { + unreachable!() + } + .map_err(DataFusionError::Plan)?; + Ok((dtype_to_arrow(&dtype), nullable)) + } +} +impl ScalarUDFImpl for CollectionPlanningFunction { + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn name(&self) -> &str { + self.name + } + fn signature(&self) -> &Signature { + &self.signature + } + fn return_type(&self, args: &[DataType]) -> Result { + self.output( + args, + &args + .iter() + .map(|dtype| *dtype == DataType::Null) + .collect::>(), + None, + ) + .map(|output| output.0) + } + fn return_type_from_exprs( + &self, + args: &[Expr], + schema: &dyn ExprSchema, + types: &[DataType], + ) -> Result { + let nullable = args + .iter() + .map(|arg| arg.nullable(schema)) + .collect::>>()?; + self.output(types, &nullable, Some(args)) + .map(|output| output.0) + } + fn is_nullable(&self, args: &[Expr], schema: &dyn ExprSchema) -> bool { + let types = args + .iter() + .map(|arg| arg.get_type(schema)) + .collect::>>(); + let nullable = args + .iter() + .map(|arg| arg.nullable(schema)) + .collect::>>(); + match (types, nullable) { + (Ok(types), Ok(nullable)) => self + .output(&types, &nullable, Some(args)) + .map(|out| out.1) + .unwrap_or(true), + _ => true, + } + } + fn invoke_batch(&self, _args: &[ColumnarValue], _number_rows: usize) -> Result { + Err(DataFusionError::NotImplemented("collection planning adapter cannot execute; use a capable query engine or external exact sub_dag".into())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn planning_adapter_explicitly_refuses_physical_execution() { + let adapter = CollectionPlanningFunction { + name: "map", + function: PlanningFunction::Map(MapScalarFunction::Construct), + signature: Signature::any(0, Volatility::Immutable), + }; + assert!(matches!( + adapter.invoke_batch(&[], 1), + Err(DataFusionError::NotImplemented(_)) + )); + let result = adapter.return_type(&[]).unwrap(); + let (expected, _) = MapScalarFunction::Construct.output_type(&[]).unwrap(); + assert_eq!(result, dtype_to_arrow(&expected)); + } +} diff --git a/crates/frontend-sql/src/unified/sql/dialect.rs b/crates/frontend-sql/src/unified/sql/dialect.rs new file mode 100644 index 00000000..03d9253e --- /dev/null +++ b/crates/frontend-sql/src/unified/sql/dialect.rs @@ -0,0 +1,112 @@ +//! The parser dialect for `SqlDialect::DataFusionSQL`. +//! +//! sqlparser's `GenericDialect` leaves `FILTER (WHERE …)` on aggregate calls +//! off (`supports_filter_during_aggregation`), and DataFusion only selects a +//! dialect by name — so `count(x) FILTER (WHERE p)` cannot reach the planner +//! through `SessionContext::sql`. This wrapper is `GenericDialect` with that +//! one switch flipped (issue #466); `lower` parses through +//! `DFParser::parse_sql_with_dialect` with it and plans the statement itself, +//! exactly as the ClickHouse path already does. + +use std::any::TypeId; + +use datafusion::sql::sqlparser::dialect::{Dialect, GenericDialect}; + +#[derive(Debug, Default)] +pub(crate) struct GenericWithAggregateFilter; + +/// Forward every boolean switch `GenericDialect` overrides, so the only +/// behavioural difference is `supports_filter_during_aggregation`. +macro_rules! forward_to_generic { + ($($method:ident),* $(,)?) => { + $(fn $method(&self) -> bool { + GenericDialect.$method() + })* + }; +} + +impl Dialect for GenericWithAggregateFilter { + /// The parser's own `dialect_of!(… is GenericDialect)` checks keep + /// matching, so generic-only syntax paths stay enabled. + fn dialect(&self) -> TypeId { + GenericDialect.dialect() + } + + fn is_delimited_identifier_start(&self, ch: char) -> bool { + GenericDialect.is_delimited_identifier_start(ch) + } + + fn is_identifier_start(&self, ch: char) -> bool { + GenericDialect.is_identifier_start(ch) + } + + fn is_identifier_part(&self, ch: char) -> bool { + GenericDialect.is_identifier_part(ch) + } + + fn supports_filter_during_aggregation(&self) -> bool { + true + } + + forward_to_generic!( + supports_unicode_string_literal, + supports_group_by_expr, + supports_connect_by, + supports_match_recognize, + supports_start_transaction_modifier, + supports_window_function_null_treatment_arg, + supports_dictionary_syntax, + supports_window_clause_named_window_reference, + supports_parenthesized_set_variables, + supports_select_wildcard_except, + support_map_literal_syntax, + allow_extract_custom, + allow_extract_single_quotes, + supports_create_index_with_clause, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::sql::parser::DFParser; + + // Every switch `GenericDialect` sets is mirrored, and only the aggregate + // FILTER switch differs. + #[test] + fn mirrors_generic_except_for_aggregate_filter() { + let ours = GenericWithAggregateFilter; + let generic = GenericDialect; + assert_eq!(ours.dialect(), generic.dialect()); + for ch in ['"', '`', '_', '#', '@', '$', 'a', '1', ' '] { + assert_eq!( + ours.is_delimited_identifier_start(ch), + generic.is_delimited_identifier_start(ch) + ); + assert_eq!( + ours.is_identifier_start(ch), + generic.is_identifier_start(ch) + ); + assert_eq!(ours.is_identifier_part(ch), generic.is_identifier_part(ch)); + } + assert_eq!( + ours.supports_group_by_expr(), + generic.supports_group_by_expr() + ); + assert!(!generic.supports_filter_during_aggregation()); + assert!(ours.supports_filter_during_aggregation()); + } + + // The generic dialect rejects an aggregate FILTER clause; ours parses it. + #[test] + fn parses_aggregate_filter_clause() { + let sql = "SELECT count(*) FILTER (WHERE a > 1) FROM t"; + assert!(DFParser::parse_sql_with_dialect(sql, &GenericDialect).is_err()); + assert_eq!( + DFParser::parse_sql_with_dialect(sql, &GenericWithAggregateFilter) + .unwrap() + .len(), + 1 + ); + } +} diff --git a/crates/frontend-sql/src/unified/sql/expr.rs b/crates/frontend-sql/src/unified/sql/expr.rs new file mode 100644 index 00000000..b726bfc1 --- /dev/null +++ b/crates/frontend-sql/src/unified/sql/expr.rs @@ -0,0 +1,354 @@ +use std::rc::Rc; + +use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; + +use asap_frontend_common::UnresolvedScalar as Unresolved; +use asap_types::ir::ExprSemantics; +use asap_types::pre_asap::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; + +use crate::unified::error::SqlError as LoweringError; + +use super::types::{arrow_to_dtype, scalar_value_to_asap}; +use super::SqlLowerer; + +pub(super) fn split_conjuncts(expr: &Expr) -> Vec<&Expr> { + match expr { + Expr::BinaryExpr(BinaryExpr { + left, + op: Operator::And, + right, + }) => { + let mut v = split_conjuncts(left); + v.extend(split_conjuncts(right)); + v + } + _ => vec![expr], + } +} + +impl SqlLowerer<'_> { + /// Translate a DataFusion `Expr` to the name-based scalar tree. Every + /// `Compare` / `Arithmetic` / `Negative` carries `ExprSemantics::Sql`. + /// Subquery-valued expressions lower their plan as a root of its own + /// (which is why this is a method: the plan walk needs the catalog). + /// Returns `UnsupportedFeature` for anything not needed in v1. + pub(super) fn lower_expr(&self, expr: &Expr) -> Result { + let bx = |e: &Expr| self.lower_expr(e).map(Box::new); + match expr { + // Preserve DataFusion's relation qualifier so a column name shared + // across a join (`a.k` vs `b.k`) resolves to the correct side. + Expr::Column(col) => Ok(Unresolved::Column(match &col.relation { + Some(rel) => ColumnRef::Qualified { + table: rel.to_string(), + name: col.name.clone(), + }, + None => ColumnRef::Named(col.name.clone()), + })), + + // Keep Arrow date literals equivalent to SQL CAST('YYYY-MM-DD' AS DATE), + // including typed nulls, without adding another canonical scalar variant. + Expr::Literal( + sv @ (datafusion::common::ScalarValue::Date32(_) + | datafusion::common::ScalarValue::Date64(_)), + ) => { + let text = sv.cast_to(&datafusion::arrow::datatypes::DataType::Utf8)?; + // Arrow formats Date64 with a time suffix; the canonical Date has + // no time-of-day, just like Date64 catalog registration as Date32. + let text = match text { + datafusion::common::ScalarValue::Utf8(Some(value)) => { + ScalarValue::Utf8(value.split('T').next().unwrap().to_owned()) + } + other => scalar_value_to_asap(&other)?, + }; + Ok(Unresolved::Cast { + expr: Box::new(Unresolved::Literal(text)), + to: asap_types::pre_asap::schema::DataType::Date, + try_cast: false, + }) + } + Expr::Literal(sv) => scalar_value_to_asap(sv).map(Unresolved::Literal), + + Expr::Alias(a) => self.lower_expr(&a.expr), + + Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op { + Operator::And => { + let parts = split_conjuncts(expr); + let lowered: Result, _> = + parts.iter().map(|e| self.lower_expr(e)).collect(); + Ok(Unresolved::BoolAnd(lowered?)) + } + Operator::Or => { + let parts = split_disjuncts(expr); + let lowered: Result, _> = + parts.iter().map(|e| self.lower_expr(e)).collect(); + Ok(Unresolved::BoolOr(lowered?)) + } + Operator::Eq => self.compare(left, CompareOpKind::Eq, right), + Operator::NotEq => self.compare(left, CompareOpKind::Ne, right), + Operator::Lt => self.compare(left, CompareOpKind::Lt, right), + Operator::LtEq => self.compare(left, CompareOpKind::Le, right), + Operator::Gt => self.compare(left, CompareOpKind::Gt, right), + Operator::GtEq => self.compare(left, CompareOpKind::Ge, right), + // BinaryExpr LIKE/ILIKE operators (from optimizer rewrites) + Operator::LikeMatch => self.compare(left, CompareOpKind::Like, right), + Operator::ILikeMatch => self.compare(left, CompareOpKind::ILike, right), + Operator::NotLikeMatch => self.compare(left, CompareOpKind::NotLike, right), + Operator::NotILikeMatch => self.compare(left, CompareOpKind::NotILike, right), + // Arithmetic + Operator::Plus => self.arith(left, ArithmeticOpKind::Add, right), + Operator::Minus => self.arith(left, ArithmeticOpKind::Sub, right), + Operator::Multiply => self.arith(left, ArithmeticOpKind::Mul, right), + Operator::Divide => self.arith(left, ArithmeticOpKind::Div, right), + Operator::Modulo => self.arith(left, ArithmeticOpKind::Mod, right), + other => Err(LoweringError::UnsupportedFeature(format!( + "operator: {other:?}" + ))), + }, + + // SQL LIKE / ILIKE (dedicated expr node from the SQL parser) + Expr::Like(like) => { + let op = match (like.negated, like.case_insensitive) { + (false, false) => CompareOpKind::Like, + (true, false) => CompareOpKind::NotLike, + (false, true) => CompareOpKind::ILike, + (true, true) => CompareOpKind::NotILike, + }; + self.compare(&like.expr, op, &like.pattern) + } + + // Unary minus. (DataFusion's planner already folds `-` + // into a negative literal, so this is a non-literal operand.) + Expr::Negative(inner) => Ok(Unresolved::Negative { + expr: bx(inner)?, + semantics: ExprSemantics::Sql, + }), + + // SQL CASE expression + Expr::Case(c) => { + let operand = c.expr.as_deref().map(bx).transpose()?; + let branches = c + .when_then_expr + .iter() + .map(|(when, then)| Ok((self.lower_expr(when)?, self.lower_expr(then)?))) + .collect::, LoweringError>>()?; + let else_expr = c.else_expr.as_deref().map(bx).transpose()?; + Ok(Unresolved::Case { + operand, + branches, + else_expr, + }) + } + + Expr::Not(inner) => Ok(Unresolved::Not(bx(inner)?)), + + Expr::IsNull(inner) => Ok(Unresolved::IsNull(bx(inner)?)), + + Expr::IsNotNull(inner) => Ok(Unresolved::IsNotNull(bx(inner)?)), + + Expr::Cast(c) => Ok(Unresolved::Cast { + expr: bx(&c.expr)?, + to: arrow_to_dtype(&c.data_type)?, + try_cast: false, + }), + + // TRY_CAST returns NULL on conversion failure; preserve that semantic. + Expr::TryCast(c) => Ok(Unresolved::Cast { + expr: bx(&c.expr)?, + to: arrow_to_dtype(&c.data_type)?, + try_cast: true, + }), + + Expr::InList(il) => { + let list: Result, _> = il.list.iter().map(|e| self.lower_expr(e)).collect(); + Ok(Unresolved::InList { + expr: bx(&il.expr)?, + list: list?, + negated: il.negated, + }) + } + + Expr::Between(b) => { + // Normalize: `x BETWEEN low AND high` → `x >= low AND x <= high`. + // `x NOT BETWEEN low AND high` → `x < low OR x > high`. + if b.negated { + let lt = self.compare(&b.expr, CompareOpKind::Lt, &b.low)?; + let gt = self.compare(&b.expr, CompareOpKind::Gt, &b.high)?; + Ok(Unresolved::BoolOr(vec![lt, gt])) + } else { + let x_low = self.compare(&b.expr, CompareOpKind::Ge, &b.low)?; + let x_high = self.compare(&b.expr, CompareOpKind::Le, &b.high)?; + Ok(Unresolved::BoolAnd(vec![x_low, x_high])) + } + } + + // `NOW()` / `CURRENT_TIMESTAMP` read the SQL statement evaluation + // time. Keep this timestamp-typed leaf distinct from PromQL's + // Float64 Unix-seconds `EvalTimestamp`. Issue #184. + Expr::ScalarFunction(sf) + if sf.args.is_empty() + && matches!( + sf.func.name().to_ascii_lowercase().as_str(), + "now" | "current_timestamp" + ) => + { + Ok(Unresolved::CurrentTimestamp) + } + + Expr::ScalarFunction(sf) => { + let args: Result, _> = sf.args.iter().map(|e| self.lower_expr(e)).collect(); + Ok(Unresolved::FunctionCall { + name: if sf.func.name().eq_ignore_ascii_case("arrayelement") { + "asap_element_access".into() + } else if sf.func.name().eq_ignore_ascii_case("tupleelement") { + "asap_struct_field".into() + } else { + sf.func.name().to_string() + }, + args: args?, + }) + } + + // Subquery-valued expressions. Each subquery plan is lowered as a + // root of its own; `resolve_root` binds it in its own scope, so an + // outer reference inside it has nothing to resolve against — a + // correlated subquery is rejected rather than mislowered. + Expr::ScalarSubquery(sq) => Ok(Unresolved::ScalarSubquery(Rc::new( + self.lower_uncorrelated_subquery(sq, "scalar subquery")?, + ))), + Expr::Exists(ex) => Ok(Unresolved::Exists { + subquery: Rc::new(self.lower_uncorrelated_subquery(&ex.subquery, "EXISTS")?), + negated: ex.negated, + }), + Expr::InSubquery(is) => { + let fields = is.subquery.subquery.schema().fields().len(); + if fields != 1 { + return Err(LoweringError::InvalidExpression(format!( + "IN (subquery) must select exactly one column, got {fields}" + ))); + } + Ok(Unresolved::InSubquery { + expr: bx(&is.expr)?, + subquery: Rc::new( + self.lower_uncorrelated_subquery(&is.subquery, "IN (subquery)")?, + ), + negated: is.negated, + }) + } + + other => Err(LoweringError::UnsupportedFeature(format!( + "expression: {}", + other + ))), + } + } + + fn lower_uncorrelated_subquery( + &self, + sq: &datafusion::logical_expr::Subquery, + what: &str, + ) -> Result { + if !sq.outer_ref_columns.is_empty() { + return Err(LoweringError::UnsupportedFeature(format!( + "correlated {what}" + ))); + } + self.lower_plan(&sq.subquery) + } + + pub(super) fn compare( + &self, + left: &Expr, + op: CompareOpKind, + right: &Expr, + ) -> Result { + Ok(Unresolved::Compare { + left: Box::new(self.lower_expr(left)?), + op, + right: Box::new(self.lower_expr(right)?), + semantics: ExprSemantics::Sql, + }) + } + + fn arith( + &self, + left: &Expr, + op: ArithmeticOpKind, + right: &Expr, + ) -> Result { + Ok(Unresolved::Arithmetic { + op, + left: Box::new(self.lower_expr(left)?), + right: Box::new(self.lower_expr(right)?), + semantics: ExprSemantics::Sql, + }) + } +} + +pub(super) fn split_disjuncts(expr: &Expr) -> Vec<&Expr> { + match expr { + Expr::BinaryExpr(BinaryExpr { + left, + op: Operator::Or, + right, + }) => { + let mut v = split_disjuncts(left); + v.extend(split_disjuncts(right)); + v + } + _ => vec![expr], + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::unified::sql::SqlCatalog; + use asap_types::pre_asap::schema::DataType; + use datafusion::common::ScalarValue as DfScalarValue; + + // Typed Arrow dates normalize to the same typed form as SQL date casts. + #[test] + fn arrow_date_literals_preserve_value_and_type() { + let catalog = SqlCatalog::new(); + let lowerer = SqlLowerer::new(&catalog); + for (value, expected) in [ + ( + DfScalarValue::Date32(Some(0)), + ScalarValue::Utf8("1970-01-01".into()), + ), + ( + DfScalarValue::Date64(Some(-86_400_000)), + ScalarValue::Utf8("1969-12-31".into()), + ), + (DfScalarValue::Date32(None), ScalarValue::Null), + (DfScalarValue::Date64(None), ScalarValue::Null), + ] { + let actual = lowerer.lower_expr(&Expr::Literal(value)).unwrap(); + assert_eq!( + actual, + Unresolved::Cast { + expr: Box::new(Unresolved::Literal(expected)), + to: DataType::Date, + try_cast: false, + } + ); + } + } + + // Unary minus over a non-literal is the `Negative` scalar, SQL-flavoured. + #[test] + fn unary_minus_lowers_to_negative_with_sql_semantics() { + let catalog = SqlCatalog::new(); + let lowerer = SqlLowerer::new(&catalog); + let expr = Expr::Negative(Box::new(Expr::Column( + datafusion::common::Column::new_unqualified("x"), + ))); + assert_eq!( + lowerer.lower_expr(&expr).unwrap(), + Unresolved::Negative { + expr: Box::new(Unresolved::Column(ColumnRef::Named("x".into()))), + semantics: ExprSemantics::Sql, + } + ); + } +} diff --git a/crates/frontend-sql/src/unified/sql/mod.rs b/crates/frontend-sql/src/unified/sql/mod.rs new file mode 100644 index 00000000..1f58f92c --- /dev/null +++ b/crates/frontend-sql/src/unified/sql/mod.rs @@ -0,0 +1,2525 @@ +//! SQL → the name-based front-end tree +//! ([`UnresolvedOp`](asap_frontend_common::UnresolvedOp) / +//! [`UnresolvedScalar`](asap_frontend_common::UnresolvedScalar)). +//! +//! Parses SQL via DataFusion (over the catalog's registered tables), then +//! walks the unoptimized `LogicalPlan` and emits `UnresolvedOp` nodes with +//! unresolved `ColumnRef`s directly (issue #179) — the same tree shape +//! [`resolve_root`](asap_frontend_common::resolve_root) binds into the +//! positional, unified `OperatorNode` IR. Unlike PromQL's front end, SQL's +//! Ordinary SQL `Aggregate` nodes are `Reduction::Reduce`. The explicit +//! `asap_rate`/`asap_increase` bridge is the narrow exception: it +//! spells a time-series range reducer with an explicit value, time-index, and +//! window and therefore lowers to the same `TimeRange` + `PerEntity` shape as +//! its PromQL counterpart. The front end also has to fold a `WHERE` directly +//! over a bare table scan onto +//! `Scan.predicates` itself (`filter_or_fold`) — canonical's invariant that a +//! `Filter` never sits directly over a `Scan` — since front ends producing +//! this shape are responsible for it now, not a converter. +//! +//! Heavy-hitter `topk` recognition (`ORDER BY count(...) DESC LIMIT k`) is +//! *not* done here: SQL emits a plain `Sort`/`Limit`, and the shared +//! `canonicalize` pass (issue #34, run by `resolve_root`) recognises the +//! count-ranked shape positionally, so a SQL `ORDER BY`/`LIMIT` and a PromQL +//! `topk(...)` converge without either front end special-casing the other's +//! syntax. + +use std::rc::Rc; +use std::sync::Arc; +use std::time::Duration; + +use datafusion::arrow::compute::kernels::cast_utils::parse_interval_month_day_nano; +use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field}; +use datafusion::catalog_common::MemorySchemaProvider; +use datafusion::common::config::ConfigOptions; +use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; +use datafusion::common::{Column as DfColumn, DFSchema, ScalarValue as DfScalarValue}; +use datafusion::datasource::MemTable; +use datafusion::functions_aggregate::count::count_udaf; +use datafusion::functions_aggregate::sum::sum_udaf; +use datafusion::logical_expr::expr::AggregateFunction; +use datafusion::logical_expr::expr_rewriter::FunctionRewrite; +use datafusion::logical_expr::function::{PartitionEvaluatorArgs, WindowUDFFieldArgs}; +use datafusion::logical_expr::{ + self, lit, AggregateUDF, Case, Distinct, Expr, ExprSchemable, JoinType, LogicalPlan, + PartitionEvaluator, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF, TypeSignature, + Volatility, WindowFrameBound as DfWindowFrameBound, WindowFrameUnits as DfWindowFrameUnits, + WindowFunctionDefinition, WindowUDF, WindowUDFImpl, +}; +use datafusion::optimizer::analyzer::function_rewrite::ApplyFunctionRewrites; +use datafusion::optimizer::{AnalyzerRule, OptimizerConfig}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion::sql::parser::DFParser; + +use asap_frontend_common::{ + resolve_root, UnresolvedOp as Unresolved, UnresolvedPredicate as Predicate, + UnresolvedProjectItem as ProjectItem, UnresolvedScalar as Scalar, UnresolvedSortKey as SortKey, +}; +use asap_sql_function_catalog::{AggSemantic, Arity, RewriteKind}; +use asap_types::ir::operator_properties::{ + GroupKeys, Reduction, Source, WindowFrame, WindowFrameBound, WindowFrameOffset, + WindowFrameUnits, +}; +use asap_types::ir::TimeRangeKind; +use asap_types::pre_asap::agg_intent::AggIntent; +use asap_types::pre_asap::schema::{DataType, FieldDataType, Schema}; + +use asap_types::pre_asap::{ + resolve_column_ref, ColumnRef, CompareOpKind, JoinKind, RelationalSetOpKind, ScalarValue, + WindowFuncKind, +}; +use asap_types::types::AccuracyTarget; +use asap_types::workload::SqlDialect; + +use crate::unified::error::SqlError as LoweringError; + +mod clickhouse_ast; +mod collection_planning; +mod dialect; +mod expr; +mod types; + +pub use types::SqlCatalog; + +use self::dialect::GenericWithAggregateFilter; +use self::types::{arrow_to_dtype, scalar_value_to_asap, schema_to_arrow}; + +std::thread_local! { + static ACCURACY: std::cell::RefCell = + const { std::cell::RefCell::new(AccuracyTarget::Exact) }; +} + +/// RAII guard installing `accuracy` as the ambient accuracy target for the +/// current thread's lowering, restoring the prior value on drop — same +/// ambient-thread-local shape as `asap_frontend_promql::promql`'s +/// `AccuracyGuard`, for the same reason: it injects `accuracy` into the deep +/// `lower_plan` recursion without a parameter on every one of its +/// signatures, consulted only at the couple of sites that build an +/// accuracy-bearing `AggIntent`. +struct AccuracyGuard(AccuracyTarget); + +impl AccuracyGuard { + fn install(accuracy: AccuracyTarget) -> Self { + let prev = ACCURACY.with(|a| a.replace(accuracy)); + AccuracyGuard(prev) + } +} + +impl Drop for AccuracyGuard { + fn drop(&mut self) { + ACCURACY.with(|a| *a.borrow_mut() = std::mem::replace(&mut self.0, AccuracyTarget::Exact)); + } +} + +fn current_accuracy() -> AccuracyTarget { + ACCURACY.with(|a| a.borrow().clone()) +} + +/// Lowers SQL strings to the name-based [`UnresolvedOp`](asap_frontend_common::UnresolvedOp) +/// tree over a table [`SqlCatalog`]. Call +/// [`resolve_root`](asap_frontend_common::resolve_root) on the result for +/// the resolved operator DAG. +pub struct SqlLowerer<'a> { + catalog: &'a SqlCatalog, + dialect: SqlDialect, +} + +impl<'a> SqlLowerer<'a> { + pub fn new(catalog: &'a SqlCatalog) -> Self { + Self { + catalog, + dialect: SqlDialect::DataFusionSQL, + } + } + + /// Parse under a specific SQL dialect (e.g. `ClickhouseSQL`, which maps to + /// sqlparser's vendored `ClickHouseDialect` — array-lambda syntax and + /// `arr[-1]` indexing parse under it that don't parse generically). This + /// only changes *parsing*: a ClickHouse-only builtin function not listed + /// in `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` (`uniqExact` and + /// `countIf` are; most of ClickHouse's builtin surface isn't yet) is + /// still unknown to DataFusion's planner and still fails there, and + /// `ElasticSQL` has no vendored parser at all. + pub fn with_dialect(catalog: &'a SqlCatalog, dialect: SqlDialect) -> Self { + Self { catalog, dialect } + } + + /// Parse + lower a SQL query to the name-based tree, threading + /// `accuracy` onto every approximate intent (`Count`, `Quantile`, + /// `Cardinality`) as it is built. + /// + /// The `AccuracyGuard` installs *after* the only `.await` point + /// (DataFusion statement planning) — `lower_plan` itself is synchronous, so once it starts + /// there is no further suspension point that could move this task to a + /// different OS thread out from under a thread-local set beforehand. + /// + /// Runs `ApplyFunctionRewrites` — the single `AnalyzerRule` DataFusion's + /// own `Analyzer` uses internally to apply `FunctionRewrite`s, called + /// directly rather than through `Analyzer::execute_and_check` — over the + /// raw parsed plan before lowering, carrying only + /// `ClickHouseBuiltinRewrite` (catalog-driven, see its own doc — it + /// covers every `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` entry, + /// not just one). `ctx.sql(...).into_unoptimized_plan()` alone returns + /// `SqlToRel`'s output untouched, and a `FunctionRewrite` only ever runs + /// as part of this rule, so calling it directly is unavoidable to make + /// the rewrite fire. Its `analyze()` already does a full + /// `transform_up_with_subqueries` over the whole plan, so it needs no + /// wrapping `Analyzer` at all — deliberately not + /// `Analyzer::execute_and_check` (whether with the default 5-rule + /// analyzer or an empty one carrying just this rewrite): that method + /// runs an unconditional post-check (`check_plan`, hardcoded, not itself + /// a rule) that isn't wanted here — e.g. it independently rejects a + /// multi-column `IN (subquery)` before `lower_in_subquery`'s own arity + /// check would. Going straight to `ApplyFunctionRewrites` avoids that + /// entirely. TypeCoercion then records implicit conversions explicitly, + /// including timestamp literals in predicates, before IR validation. + pub async fn lower( + &self, + sql: &str, + accuracy: &AccuracyTarget, + ) -> Result { + let ctx = self.build_context()?; + let state = ctx.state(); + let statement = if matches!(self.dialect, SqlDialect::ClickhouseSQL) { + let mut statement = state.sql_to_statement(sql, "ClickHouse")?; + if let datafusion::sql::parser::Statement::Statement(ast) = &mut statement { + clickhouse_ast::normalize(ast); + } + statement + } else { + // Not `ctx.sql(sql)`: that parses under the by-name `generic` + // dialect, which cannot see an aggregate `FILTER (WHERE …)`. + let mut statements = DFParser::parse_sql_with_dialect(sql, &GenericWithAggregateFilter) + .map_err(|e| datafusion::error::DataFusionError::SQL(e, None))?; + let (Some(statement), true) = (statements.pop_front(), statements.is_empty()) else { + return Err(LoweringError::UnsupportedFeature( + "exactly one SQL statement per query".into(), + )); + }; + statement + }; + let plan = state.statement_to_plan(statement).await?; + let rewriter = ApplyFunctionRewrites::new(vec![Arc::new(ClickHouseBuiltinRewrite)]); + let plan = rewriter.analyze(plan, ctx.state().options())?; + let plan = datafusion::optimizer::analyzer::type_coercion::TypeCoercion::new() + .analyze(plan, ctx.state().options())?; + // Output schemas omit predicate and nested-expression types. Check the + // typed SQL plan before lowering erases fixed-duration units. + plan.apply_with_subqueries(|node| { + let mut schema = DFSchema::empty(); + for input in node.inputs() { + schema.merge(input.schema()); + } + schema.merge(node.schema()); + node.apply_expressions(|expr| { + expr.apply(|nested| { + if let Expr::BinaryExpr(binary) = nested { + if binary.op == logical_expr::Operator::Minus + && matches!(nested.get_type(&schema)?, ArrowDataType::Duration(_)) + { + return Err(datafusion::common::DataFusionError::Plan( + "temporal subtraction produces an unsupported duration type".into(), + )); + } + } + Ok(TreeNodeRecursion::Continue) + }) + }) + })?; + let _guard = AccuracyGuard::install(accuracy.clone()); + self.lower_plan(&plan) + } + + /// Register the catalog tables (empty Arrow `MemTable`s) so DataFusion can + /// resolve table/column references during planning. + fn build_context(&self) -> Result { + let dialect_name = match &self.dialect { + SqlDialect::DataFusionSQL => "generic", + SqlDialect::ClickhouseSQL => "ClickHouse", + SqlDialect::ElasticSQL => { + return Err(LoweringError::UnsupportedDialect("ElasticSQL".into())) + } + }; + let config = SessionConfig::new().set_str("datafusion.sql_parser.dialect", dialect_name); + let ctx = SessionContext::new_with_config(config); + if matches!(self.dialect, SqlDialect::ClickhouseSQL) { + collection_planning::register(&ctx); + } + // A catalog key like "bgp.bgp_updates" schema-qualifies the table + // (e.g. a ClickHouse database name). DataFusion requires the parent + // schema to be registered before a qualified table can be, so create + // it on demand. + let catalog_provider = ctx.catalog("datafusion").ok_or_else(|| { + LoweringError::InvalidExpression("default \"datafusion\" catalog missing".into()) + })?; + for (name, schema) in &self.catalog.tables { + if let Some((schema_name, _)) = name.split_once('.') { + if catalog_provider.schema(schema_name).is_none() { + catalog_provider + .register_schema(schema_name, Arc::new(MemorySchemaProvider::new()))?; + } + } + let arrow_schema = Arc::new(schema_to_arrow(schema)); + let mem_table = MemTable::try_new(arrow_schema, vec![])?; + ctx.register_table(name.as_str(), Arc::new(mem_table))?; + } + // Register a stub `AggregateUDF` for every catalog-listed + // ClickHouse-only builtin, purely so DataFusion's planner can + // resolve its name during parsing — `lower()` rewrites every call + // site to a native DataFusion aggregate via `ClickHouseBuiltinRewrite` + // before `lower_plan` sees it. + for builtin in asap_sql_function_catalog::CLICKHOUSE_BUILTINS { + ctx.register_udaf(clickhouse_builtin_stub_udaf(builtin.name, builtin.arity)); + } + // Register a stub `ScalarUDF` for every catalog-listed ClickHouse-only + // *scalar* builtin — same reason as the `AggregateUDF` loop above + // (DataFusion otherwise rejects the call as an unknown function + // during `SqlToRel` conversion), but with no rewrite step to follow: + // `lower_expr`'s `Expr::ScalarFunction` arm already lowers any + // scalar call generically to `UnresolvedScalar::FunctionCall { name, + // args }`, so registering the stub is the entire fix (issue #230). + for builtin in asap_sql_function_catalog::CLICKHOUSE_SCALAR_BUILTINS { + ctx.register_udf(clickhouse_scalar_builtin_stub_udf( + builtin.name, + builtin.arity, + )); + } + // Planning-only relation markers. They let a workload author state + // the PromQL temporal/classic-histogram semantics of an equivalent SQL + // rewrite without teaching the canonical IR a second, SQL-specific + // spelling of either operation. `lower_projection` consumes these + // calls; they can never survive as executable scalar functions. + for (name, arity) in [ + ("asap_promql_subquery", Arity::Exact(2)), + ("asap_histogram_quantile", Arity::Exact(1)), + ] { + ctx.register_udf(clickhouse_scalar_builtin_stub_udf(name, arity)); + } + // Register a stub `WindowUDF` for every catalog-listed ClickHouse-only + // *window* builtin — same reason as the two loops above, but with no + // rewrite step to follow: `lower_window_func_kind` already maps each + // name directly to its own `WindowFuncKind` variant (issue #267). + for builtin in asap_sql_function_catalog::CLICKHOUSE_WINDOW_BUILTINS { + ctx.register_udwf(clickhouse_window_builtin_stub_udwf( + builtin.name, + builtin.arity, + )); + } + Ok(ctx) + } + + pub(super) fn lower_plan(&self, plan: &LogicalPlan) -> Result { + match plan { + LogicalPlan::TableScan(scan) => self.lower_table_scan(scan), + // The one empty input row of a `SELECT` without `FROM`. + LogicalPlan::EmptyRelation(empty) => Ok(Unresolved::Values { + rows: if empty.produce_one_row { + vec![vec![]] + } else { + vec![] + }, + schema: Schema { + fields: vec![], + time_index: None, + unique_keys: vec![], + closed: true, + }, + }), + LogicalPlan::Values(values) => self.lower_values(values), + LogicalPlan::Filter(filter) => self.lower_filter(filter), + LogicalPlan::Projection(proj) => self.lower_projection(proj), + LogicalPlan::Aggregate(agg) => self.lower_aggregate(agg), + LogicalPlan::Sort(sort) => self.lower_sort(sort), + LogicalPlan::Limit(limit) => self.lower_limit(limit), + LogicalPlan::Distinct(d) => match d { + Distinct::On(_) => Err(LoweringError::UnsupportedFeature("DISTINCT ON".into())), + Distinct::All(input) => Ok(Unresolved::Dedup { + cols: vec![], + child: Rc::new(self.lower_plan(input)?), + }), + }, + LogicalPlan::Union(u) => { + // Fold n inputs left-associatively into SetOp { Union, all: true }. + let mut iter = u.inputs.iter(); + let first = iter + .next() + .ok_or_else(|| LoweringError::InvalidExpression("empty union".into()))?; + let first_expr = self.lower_plan(first)?; + iter.try_fold(first_expr, |left, right_plan| { + Ok(Unresolved::SetOp { + kind: RelationalSetOpKind::Union, + all: true, + left: Rc::new(left), + right: Rc::new(self.lower_plan(right_plan)?), + }) + }) + } + LogicalPlan::Window(window) => self.lower_window(window), + LogicalPlan::Join(join) => self.lower_join(join), + LogicalPlan::Subquery(_) => Err(LoweringError::UnsupportedFeature("subquery".into())), + LogicalPlan::SubqueryAlias(alias) => { + // An alias over a table re-qualifies the scan's columns with the + // alias (so `a.col` / `b.col` in a self-join disambiguate). + match alias.input.as_ref() { + LogicalPlan::TableScan(scan) => { + self.scan_source(&scan.table_name.to_string(), &alias.alias.to_string()) + } + // A *derived table* / inline view — `FROM (SELECT …) t`, the + // SQL counterpart of PromQL function nesting (an aggregate + // over an aggregate, a filter over a derived aggregate, …). + // Lower the inner plan, then re-qualify its output columns + // with the alias so `t.col` resolves to *this* relation — and, + // critically, so a join over two derived tables disambiguates + // its keys instead of both binding to the first bare-name + // match (issue #66). The inner column *names* are unchanged; + // only the qualifier is stamped. + other => { + let alias_name = alias.alias.to_string(); + match self.lower_plan(other)? { + // The derived SELECT list already lowered to a + // Projection — stamp the alias onto it, no extra node. + Unresolved::Project { cols, child, .. } => Ok(Unresolved::Project { + cols, + qualifier: Some(alias_name), + child, + }), + // Otherwise (e.g. `SELECT *` unwrapped to a scan) wrap + // in an identity projection that re-qualifies each + // output column. Names come from the sub-plan's schema. + inner => { + let cols = alias + .input + .schema() + .fields() + .iter() + .map(|f| ProjectItem { + alias: Some(f.name().clone()), + expr: Scalar::Column(ColumnRef::Named(f.name().clone())), + }) + .collect(); + Ok(Unresolved::Project { + cols, + qualifier: Some(alias_name), + child: Rc::new(inner), + }) + } + } + } + } + } + other => Err(LoweringError::UnsupportedFeature(format!( + "plan node: {}", + other.display() + ))), + } + } + + /// `WHERE` — a conjunction of ordinary predicates plus, possibly, subquery + /// predicates (issue #111). + /// + /// The ordinary conjuncts stay one predicate, folded onto a bare `Scan` + /// (`filter_or_fold`). A subquery conjunct — `c IN (SELECT …)`, `EXISTS + /// (…)`, `x > (SELECT …)` — is a row filter whose predicate reads another + /// operator (`UnresolvedScalar::InSubquery` / `Exists` / + /// `ScalarSubquery`); each one becomes its own `Filter` **above** the + /// ordinary predicate, so the shared `canonicalize` pass can turn it into + /// the join it is without having to peel it out of a conjunction or off + /// a `Scan` (it only lifts subqueries out of `Filter` / `Project`). A + /// semi-join only ever drops left rows, so the two orders agree. + /// + /// The one subquery shape still lowered to a join here is a *correlated* + /// `EXISTS`: its correlation references both sides, which only a join + /// predicate can bind (a subquery referenced from a scalar position is + /// resolved as a root in its own scope). + fn lower_filter(&self, filter: &logical_expr::Filter) -> Result { + let mut conjuncts = Vec::new(); + split_conjunction(&filter.predicate, &mut conjuncts); + let (subqueries, residual): (Vec<_>, Vec<_>) = + conjuncts.into_iter().partition(|e| reads_subquery(e)); + + let input = self.lower_plan(&filter.input)?; + let mut node = match rebuild_conjunction(&residual) { + Some(pred) => filter_or_fold(self.lower_expr(&pred)?, input), + None => input, + }; + for sq in subqueries { + node = match sq { + Expr::Exists(ex) if !ex.subquery.outer_ref_columns.is_empty() => { + self.lower_correlated_exists(ex, node)? + } + other => Unresolved::Filter { + pred: Predicate(self.lower_expr(other)?), + child: Rc::new(node), + }, + }; + } + Ok(node) + } + + /// `[NOT] EXISTS (SELECT … WHERE inner.k = outer.k)` → a semi- / anti-join + /// on the correlation predicate (issue #111). + fn lower_correlated_exists( + &self, + ex: &logical_expr::expr::Exists, + left: Unresolved, + ) -> Result { + let kind = if ex.negated { + JoinKind::Anti + } else { + JoinKind::Semi + }; + // A semi-join discards the right side's columns, and `SELECT 1` projects + // the correlation columns away — so drop the subquery's projections and + // join against what they sit on. + let mut inner = ex.subquery.subquery.as_ref(); + while let LogicalPlan::Projection(p) = inner { + inner = &p.input; + } + // Lift the correlated conjuncts out of the subquery's filter; they are + // the join predicate. Whatever is left stays an ordinary inner filter. + let (inner, correlation) = split_correlation(inner)?; + let right = self.lower_plan(&inner)?; + let pred = match correlation { + Some(e) => Predicate(self.lower_expr(&e)?), + None => Predicate(Scalar::Literal(ScalarValue::Boolean(true))), + }; + Ok(Unresolved::Join { + kind, + pred, + left: Rc::new(left), + right: Rc::new(right), + }) + } + + /// `VALUES (…), (…)` — one row per values row, typed by DataFusion's + /// declared schema. Row expressions have no input-column scope. + fn lower_values(&self, values: &logical_expr::Values) -> Result { + let rows = values + .values + .iter() + .map(|row| row.iter().map(|e| self.lower_expr(e)).collect()) + .collect::>, LoweringError>>()?; + let fields = values + .schema + .fields() + .iter() + .map(|f| { + Ok(asap_types::pre_asap::Field::plain( + f.name().clone(), + arrow_to_dtype(f.data_type())?, + f.is_nullable(), + )) + }) + .collect::, LoweringError>>()?; + Ok(Unresolved::Values { + rows, + schema: Schema { + fields, + time_index: None, + unique_keys: vec![], + closed: true, + }, + }) + } + + /// Table leaf — carries the catalog's resolved schema directly on `Scan` + /// (`schema: Some(_)`), so `resolve_root`'s SchemaResolver doesn't need to + /// usage-derive it (SQL is never schemaless). Projection pushdown is left + /// to the enclosing `Project` (DataFusion's unoptimized plan sets no + /// projection). + fn lower_table_scan( + &self, + scan: &logical_expr::TableScan, + ) -> Result { + let table = scan.table_name.to_string(); + self.scan_source(&table, &table) + } + + /// A `Scan` over catalog table `table`, with its columns qualified by + /// `qualifier` (the table name, or an alias from a `SubqueryAlias`) so + /// `Qualified` column refs resolve to the right side across a join. + fn scan_source(&self, table: &str, qualifier: &str) -> Result { + let schema = self + .catalog + .tables + .get(table) + .ok_or_else(|| LoweringError::TableNotFound(table.to_string()))?; + let qualified = Schema { + fields: schema + .fields + .iter() + .cloned() + .map(|c| c.with_table(qualifier)) + .collect(), + time_index: schema.time_index, + unique_keys: schema.unique_keys.clone(), + // Catalog-backed: the table's columns are fully declared → closed. + closed: true, + }; + Ok(Unresolved::Scan { + source: Source::Table { + table_ref: table.to_string(), + }, + predicates: vec![], + schema: Some(qualified), + }) + } + + /// ⋈ — equijoin. The `on` key pairs become `left = right` comparisons, + /// AND-ed with any non-equi `filter`, into the join predicate — still + /// name-based here (like a `WHERE`); `resolve_root` derives the + /// concatenated output schema downstream. Semi/anti/mark joins have no + /// canonical counterpart yet and are rejected. + fn lower_join(&self, join: &logical_expr::Join) -> Result { + let kind = match join.join_type { + JoinType::Inner => JoinKind::Inner, + JoinType::Left => JoinKind::Left, + JoinType::Right => JoinKind::Right, + JoinType::Full => JoinKind::Full, + other => { + return Err(LoweringError::UnsupportedFeature(format!( + "join type: {other:?}" + ))) + } + }; + let mut conjuncts = join + .on + .iter() + .map(|(l, r)| self.compare(l, CompareOpKind::Eq, r)) + .collect::, LoweringError>>()?; + if let Some(filter) = &join.filter { + conjuncts.push(self.lower_expr(filter)?); + } + let pred = Predicate(match conjuncts.len() { + // No condition (a CROSS JOIN) is unconditionally true. + 0 => Scalar::Literal(ScalarValue::Boolean(true)), + 1 => conjuncts.pop().unwrap(), + _ => Scalar::BoolAnd(conjuncts), + }); + Ok(Unresolved::Join { + kind, + pred, + left: Rc::new(self.lower_plan(&join.left)?), + right: Rc::new(self.lower_plan(&join.right)?), + }) + } + + /// `func(args) OVER (PARTITION BY … ORDER BY … ROWS/RANGE BETWEEN …)`. One + /// window function per plan node. + fn lower_window(&self, window: &logical_expr::Window) -> Result { + if window.window_expr.len() > 1 { + return Err(LoweringError::UnsupportedFeature(format!( + "multiple window functions in one plan node (got {}); split them", + window.window_expr.len() + ))); + } + let child = Rc::new(self.lower_plan(&window.input)?); + let first = window + .window_expr + .first() + .ok_or_else(|| LoweringError::InvalidExpression("empty window expression".into()))?; + let first = match first { + Expr::Alias(alias) => alias.expr.as_ref(), + other => other, + }; + let Expr::WindowFunction(wf) = first else { + return Err(LoweringError::InvalidExpression( + "expected a window function in Window plan node".into(), + )); + }; + let func = lower_window_func_kind(&wf.fun)?; + let mut args = wf + .args + .iter() + .map(|e| self.lower_expr(e)) + .collect::, _>>()?; + // Nth_value: lift N from the (literal) 2nd arg, keep only the column. + let func = if matches!(func, WindowFuncKind::NthValue(None)) { + let n = match args.get(1) { + Some(Scalar::Literal(ScalarValue::Int64(n))) if *n > 0 => *n as u64, + other => { + return Err(LoweringError::InvalidExpression(format!( + "NTH_VALUE requires a positive integer literal 2nd arg, got {other:?}" + ))) + } + }; + args.truncate(1); + WindowFuncKind::NthValue(Some(n)) + } else { + func + }; + let partition_by = wf + .partition_by + .iter() + .map(expr_to_group_ref) + .collect::, _>>()?; + let order_by = wf + .order_by + .iter() + .map(|s| { + self.lower_expr(&s.expr).map(|expr| SortKey { + expr, + ascending: s.asc, + nulls_first: s.nulls_first, + }) + }) + .collect::, _>>()?; + let frame = lower_window_frame(&wf.window_frame)?; + // The window plan's schema is `[input fields …, window output]`; the last + // field is the window column's name (what an enclosing Project references). + let output_name = window + .schema + .fields() + .last() + .map(|f| f.name().clone()) + .unwrap_or_else(|| "window".into()); + Ok(Unresolved::SQLWindowFunc { + func, + args, + partition_by: partition_by.into(), + order_by, + frame: Some(frame), + output_name, + child, + }) + } + + fn lower_projection( + &self, + proj: &logical_expr::Projection, + ) -> Result { + if let Some(bridge) = planning_bridge(proj)? { + let input = self.lower_plan(&proj.input)?; + return Ok(match bridge { + PlanningBridge::PromqlSubquery { range, resolution } => { + let child = Rc::new(self.temporal_bridge_projection(proj, input)?); + Unresolved::PromqlSubquery { + range, + resolution: Some(resolution), + child, + } + } + PlanningBridge::HistogramQuantile { q } => Unresolved::Aggregate { + // The marker is the projection's only column: one histogram. + reduction: Reduction::Reduce(GroupKeys::none()), + measures: vec![AggIntent::HistogramQuantile { + q, + le: ColumnRef::Named("le".into()), + }], + output_names: vec!["value".into()], + filters: vec![], + having: None, + child: Rc::new(input), + }, + }); + } + // SELECT * — no column constraint; pass through without a Project. + if proj.expr.iter().any(|e| matches!(e, Expr::Wildcard { .. })) { + return self.lower_plan(&proj.input); + } + let child = Rc::new(self.lower_plan(&proj.input)?); + let temporal_input = plan_has_temporal_aggregate(&proj.input); + let cols = proj + .expr + .iter() + .map(|e| match e { + Expr::Alias(a) => { + let expr = if temporal_input && is_temporal_output_column(&a.expr) { + Scalar::Column(ColumnRef::Named("value".into())) + } else { + self.lower_expr(&a.expr)? + }; + Ok::(ProjectItem { + expr, + alias: Some(a.name.clone()), + }) + } + _ => { + let expr = if temporal_input && is_temporal_output_column(e) { + Scalar::Column(ColumnRef::Named("value".into())) + } else { + self.lower_expr(e)? + }; + Ok::(ProjectItem { expr, alias: None }) + } + }) + .collect::, _>>()?; + Ok(Unresolved::Project { + cols, + qualifier: None, + child, + }) + } + + fn lower_aggregate(&self, agg: &logical_expr::Aggregate) -> Result { + let input = self.lower_plan(&agg.input)?; + // Each measure's row predicate (`FILTER (WHERE …)`, or the NULL-skip + // a `count(expr)` implies), read off the original typed expression + // before derived-column rewriting erases the argument's nullability. + let measure_filters = agg + .aggr_expr + .iter() + .map(|e| measure_filter(e, agg.input.schema())) + .collect::, LoweringError>>()?; + + if agg.aggr_expr.iter().any(is_temporal_aggregate) { + if measure_filters.iter().any(Option::is_some) { + return Err(LoweringError::UnsupportedFeature( + "FILTER on an ASAP temporal aggregate".into(), + )); + } + return self.lower_temporal_aggregate(agg, input); + } + + // `GROUPING SETS`/`ROLLUP`/`CUBE` emit several grouping levels from one + // scan. `Aggregate.by` is a single key set, so each level becomes its own + // `Aggregate` and they are merged (issue #118). + if let Some(gs) = agg.group_expr.iter().find_map(as_grouping_set) { + if measure_filters.iter().any(Option::is_some) { + return Err(LoweringError::UnsupportedFeature( + "FILTER on a measure inside a multi-level grouping".into(), + )); + } + return self.lower_grouping_sets(agg, gs, input); + } + + // `Aggregate.by` and the reducers index *columns*, so a grouping or + // reducer expression (`GROUP BY date_trunc(…)`, `SUM(a * 8)`) has no + // slot. Materialize each one as a derived column in a `Project` beneath + // the aggregate, then group/reduce over that column (issue #110). + let mut derived = DerivedCols::new(self); + + // DataFusion strips `AS m` from a grouping expression, so the aggregate + // schema's field name is what the enclosing Projection references — + // the derived column has to carry exactly that name. + let group_names: Vec = agg + .schema + .fields() + .iter() + .take(agg.group_expr.len()) + .map(|f| f.name().to_string()) + .collect(); + + let mut keys = Vec::with_capacity(agg.group_expr.len()); + for (i, e) in agg.group_expr.iter().enumerate() { + match unalias(e) { + Expr::Column(_) => { + derived.passthrough(e)?; + keys.push(expr_to_group_ref(e)?); + } + other => { + let name = group_names + .get(i) + .cloned() + .unwrap_or_else(|| other.to_string()); + derived.materialize(name.clone(), self.lower_expr(other)?)?; + keys.push(ColumnRef::Named(name)); + } + } + } + + // Reducer arguments get the same treatment; `rewrite_agg` returns the + // aggregate with its argument repointed at the derived column. + let aggr_expr = agg + .aggr_expr + .iter() + .map(|e| derived.rewrite_agg(e)) + .collect::, LoweringError>>()?; + // A measure filter reads the aggregate's input rows, so the columns + // it names must survive any derived-column `Project` inserted below. + for column in measure_filters.iter().flatten().flat_map(Expr::column_refs) { + derived.passthrough(&Expr::Column(column.clone()))?; + } + + let child = Rc::new(derived.wrap(input)?); + // DataFusion names the aggregate outputs in its own schema (e.g. + // "sum(metrics.bytes)") — the same names the enclosing Projection + // references. The schema is [group fields …, aggregate fields …], so + // skip the group fields and thread the rest straight through as + // `Aggregate.output_names`, letting that Projection resolve them. + let output_names: Vec = agg + .schema + .fields() + .iter() + .skip(agg.group_expr.len()) + .map(|f| f.name().to_string()) + .collect(); + let measures = aggr_expr + .iter() + .map(lower_agg_intent) + .collect::, LoweringError>>()?; + // Empty when nothing is filtered — the one canonical unfiltered shape. + let filters = if measure_filters.iter().any(Option::is_some) { + measure_filters + .iter() + .map(|f| { + f.as_ref() + .map(|f| Ok(Predicate(self.lower_expr(f)?))) + .transpose() + }) + .collect::, LoweringError>>()? + } else { + Vec::new() + }; + Ok(Unresolved::Aggregate { + // SQL `GROUP BY` is always an inclusion list, never PromQL's + // `without(...)` exclusion form — and always a genuine reduction, + // never `PerEntity` (there's no windowed/subquery-child concept + // in SQL for that to apply to). + reduction: Reduction::Reduce(GroupKeys::by(keys)), + measures, + output_names, + filters, + having: None, + child, + }) + } + + fn lower_temporal_aggregate( + &self, + agg: &logical_expr::Aggregate, + input: Unresolved, + ) -> Result { + if agg.aggr_expr.len() != 1 { + return Err(LoweringError::UnsupportedFeature( + "an ASAP temporal aggregate cannot share an Aggregate node with another reducer" + .into(), + )); + } + let Expr::AggregateFunction(call) = unalias(&agg.aggr_expr[0]) else { + unreachable!("is_temporal_aggregate accepted a non-aggregate expression") + }; + let name = call.func.name().to_lowercase(); + let [value, timestamp, window] = call.args.as_slice() else { + unreachable!("ASAP temporal UDAF signatures require exactly three arguments") + }; + + let value_ref = reducer_col(&name, std::slice::from_ref(value))?; + let timestamp_ref = reducer_col(&name, std::slice::from_ref(timestamp))?; + let Expr::Literal(window) = unalias(window) else { + return Err(LoweringError::InvalidExpression(format!( + "{name} window_ms must be a positive integer literal" + ))); + }; + let window_ms = scalar_positive_u64(window).ok_or_else(|| { + LoweringError::InvalidExpression(format!( + "{name} window_ms must be a positive integer literal" + )) + })?; + + let input_schema = resolve_root(&input)?.schema.clone(); + let timestamp_id = resolve_column_ref(×tamp_ref, &input_schema).map_err(|error| { + LoweringError::InvalidExpression(format!("{name} timestamp argument: {error}")) + })?; + if input_schema.time_index != Some(timestamp_id) { + return Err(LoweringError::InvalidExpression(format!( + "{name} timestamp argument must name the input schema's time-index column" + ))); + } + let value_id = resolve_column_ref(&value_ref, &input_schema).map_err(|error| { + LoweringError::InvalidExpression(format!("{name} value argument: {error}")) + })?; + if value_id == timestamp_id + || !matches!( + input_schema.fields[value_id].dtype, + FieldDataType::Plain(DataType::Int64 | DataType::Float64) + ) + { + return Err(LoweringError::InvalidExpression(format!( + "{name} value argument must name a numeric non-time column" + ))); + } + + let mut group_ids = Vec::with_capacity(agg.group_expr.len()); + let mut group_refs = Vec::with_capacity(agg.group_expr.len()); + for group in &agg.group_expr { + let group_ref = expr_to_group_ref(group)?; + let group_id = resolve_column_ref(&group_ref, &input_schema).map_err(|error| { + LoweringError::InvalidExpression(format!("{name} GROUP BY column: {error}")) + })?; + if group_id == timestamp_id || group_id == value_id { + return Err(LoweringError::InvalidExpression(format!( + "{name} GROUP BY cannot contain its timestamp or value column" + ))); + } + if group_ids.contains(&group_id) { + return Err(LoweringError::InvalidExpression(format!( + "{name} GROUP BY contains the same resolved column more than once" + ))); + } + group_ids.push(group_id); + group_refs.push(group_ref); + } + // Minimal series-identity contract without adding SQL-only metadata to + // the shared Schema: a declared row-unique key must contain the time + // index, and removing that index yields the complete series key. The + // GROUP BY must match that key exactly. A unique key that omits time is + // only row identity and proves nothing about time-series continuity. + let identifies_one_series = input_schema + .unique_keys + .iter() + .filter(|key| key.contains(×tamp_id)) + .any(|key| { + let mut series_key: Vec<_> = key + .iter() + .copied() + .filter(|id| *id != timestamp_id) + .collect(); + series_key.sort_unstable(); + series_key.dedup(); + let mut grouped = group_ids.clone(); + grouped.sort_unstable(); + series_key == grouped + }); + if !identifies_one_series { + return Err(LoweringError::InvalidExpression(format!( + "{name} GROUP BY must exactly match a declared series identity (a unique key without the time index)" + ))); + } + + let mut cols = vec![ + ProjectItem { + alias: Some("ts".into()), + expr: Scalar::Column(timestamp_ref.clone()), + }, + ProjectItem { + alias: Some("value".into()), + expr: Scalar::Column(value_ref.clone()), + }, + ]; + for group_ref in group_refs { + let group_name = named_ref(&group_ref).to_string(); + cols.push(ProjectItem { + alias: Some(group_name), + expr: Scalar::Column(group_ref), + }); + } + let child = Unresolved::Project { + cols, + qualifier: None, + child: Rc::new(input), + }; + // The explicit window is a range selector over the series, the same + // shape PromQL's `rate(m[5m])` lowers to. + let child = Unresolved::TimeRange { + range: Duration::from_millis(window_ms), + kind: TimeRangeKind::Range, + child: Rc::new(child), + }; + let intent = match name.as_str() { + "asap_rate" => AggIntent::Rate, + "asap_increase" => AggIntent::Increase, + + _ => unreachable!("is_temporal_aggregate admitted {name}"), + }; + Ok(Unresolved::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![intent], + output_names: vec![], + filters: vec![], + having: None, + child: Rc::new(child), + }) + } + + /// `GROUP BY ROLLUP/CUBE/GROUPING SETS` — multi-level grouping (issue #118). + /// + /// One scan produces several grouping levels; `Aggregate.by` holds a single + /// key set. So each level becomes its own `Aggregate`, and the levels are + /// `Concat`ed. A level that omits a key still has to *emit* it — as `NULL`, per + /// SQL — so each branch is wrapped in a `Project` that reinstates the missing + /// keys as typed nulls and restores the canonical column order. That keeps + /// the branches union-compatible, which `Concat` requires (it derives its + /// schema from the first child). + /// + /// `Aggregate.child` is duplicated per level — the same trade + /// `histogram_quantiles` makes (#109); a future workload-level reuse pass + /// could hoist it back into a single producer. + /// + /// DataFusion's `__grouping_id` discriminator is dropped: it only exists to + /// tell a subtotal's `NULL` apart from a data `NULL`, which is observable + /// solely through `GROUPING(col)` — an aggregate this front end rejects. + fn lower_grouping_sets( + &self, + agg: &logical_expr::Aggregate, + gs: &logical_expr::GroupingSet, + input: Unresolved, + ) -> Result { + // DataFusion normalizes every mixed form (`GROUP BY g, ROLLUP(d)`) into a + // single `GroupingSets`, so one grouping expression is the only shape. + if agg.group_expr.len() != 1 { + return Err(LoweringError::UnsupportedFeature( + "a grouping set alongside plain GROUP BY keys".into(), + )); + } + + // `distinct_expr()` is ordered exactly like the aggregate's leading + // schema fields, which is the column order the enclosing Projection + // expects. The field after them is `__grouping_id`. + let distinct = gs.distinct_expr(); + for e in &distinct { + if !matches!(unalias(e), Expr::Column(_)) { + return Err(LoweringError::UnsupportedFeature(format!( + "non-column key inside a multi-level grouping: {e}" + ))); + } + } + let keys: Vec<(String, DataType)> = agg + .schema + .fields() + .iter() + .take(distinct.len()) + .map(|f| Ok((f.name().to_string(), arrow_to_dtype(f.data_type())?))) + .collect::>()?; + + let output_names: Vec = agg + .schema + .fields() + .iter() + .skip(distinct.len() + 1) // + `__grouping_id` + .map(|f| f.name().to_string()) + .collect(); + + // Reducer arguments still materialize as derived columns (#110); the + // grouping keys are plain columns, so they only need carrying through. + let mut derived = DerivedCols::new(self); + for e in &distinct { + derived.passthrough(e)?; + } + let aggr_expr = agg + .aggr_expr + .iter() + .map(|e| derived.rewrite_agg(e)) + .collect::, LoweringError>>()?; + let measures = aggr_expr + .iter() + .map(lower_agg_intent) + .collect::, LoweringError>>()?; + let input = derived.wrap(input)?; + + let branches = expand_grouping_set(gs) + .iter() + .map(|level| { + let level_keys = distinct + .iter() + .filter(|e| level.contains(e)) + .map(|e| expr_to_group_ref(e)) + .collect::, LoweringError>>()?; + let aggregate = Unresolved::Aggregate { + reduction: Reduction::Reduce(GroupKeys::by(level_keys)), + measures: measures.clone(), + output_names: output_names.clone(), + filters: vec![], + having: None, + child: Rc::new(input.clone()), + }; + // Reinstate omitted keys as typed nulls, in canonical order. + let cols = keys + .iter() + .zip(&distinct) + .map(|((name, dtype), e)| ProjectItem { + alias: Some(name.clone()), + expr: if level.contains(e) { + Scalar::Column(ColumnRef::Named(name.clone())) + } else { + Scalar::Cast { + expr: Box::new(Scalar::Literal(ScalarValue::Null)), + to: dtype.clone(), + try_cast: false, + } + }, + }) + .chain(output_names.iter().map(|n| ProjectItem { + alias: Some(n.clone()), + expr: Scalar::Column(ColumnRef::Named(n.clone())), + })) + .collect(); + Ok(Unresolved::Project { + cols, + qualifier: None, + child: Rc::new(aggregate), + }) + }) + .collect::, LoweringError>>()?; + + // No discriminator asserted here today (issue #228): DataFusion's own + // `__grouping_id` would be the natural one, but this front end + // already discards it (see above — `GROUPING()` itself is rejected), + // so there is no distinct-per-branch column available to name yet. + // `Unresolved::concat` keeps `output_schema`'s default (drop + // `unique_keys` entirely). See + // `docs/design_docs/concat-unique-keys-decision.md`. + Ok(Unresolved::concat(branches)) + } + + fn lower_sort(&self, sort: &logical_expr::Sort) -> Result { + // A count-ranked `ORDER BY … LIMIT k` is the frequency heavy-hitter the + // `TopK` intent represents, but that promotion now happens in the shared + // `canonicalize` pass (issue #34) — the same one both front ends run — + // so SQL emits a plain `Sort` (+ `Limit`) here and lets canonicalization + // recognise the count-ranked shape positionally. This removes the gate's + // alias blind spot (#20). + let keys = sort + .expr + .iter() + .map(|s| { + self.lower_expr(&s.expr).map(|expr| SortKey { + expr, + ascending: s.asc, + nulls_first: s.nulls_first, + }) + }) + .collect::, _>>()?; + Ok(Unresolved::Sort { + keys, + // SQL `ORDER BY` is a global sort; per-group ranking would come from a + // window function (`SQLWindowFunc`), not a bare Sort. + partition_by: GroupKeys::none(), + child: Rc::new(self.lower_plan(&sort.input)?), + }) + } + + fn lower_limit(&self, limit: &logical_expr::Limit) -> Result { + // Count-ranked `LIMIT k` over a `Sort` is promoted to the heavy-hitter + // `TopK` by the shared `canonicalize` pass (issue #34), not here. + Ok(Unresolved::Limit { + // No (literal) fetch is offset-only. + n: eval_fetch(&limit.fetch), + offset: eval_fetch(&limit.skip).unwrap_or(0), + partition_by: GroupKeys::none(), + child: Rc::new(self.lower_plan(&limit.input)?), + }) + } +} + +/// A deliberately explicit marker accepted only in a projection of planning +/// SQL. The marker describes a relation operator, so it is removed rather than +/// lowered to the ordinary scalar `FunctionCall` variant. +enum PlanningBridge { + PromqlSubquery { + range: Duration, + resolution: Duration, + }, + HistogramQuantile { + q: f64, + }, +} + +fn planning_bridge( + projection: &logical_expr::Projection, +) -> Result, LoweringError> { + let mut found = None; + for expr in &projection.expr { + let Expr::ScalarFunction(call) = unalias(expr) else { + continue; + }; + let name = call.func.name().to_ascii_lowercase(); + let bridge = match name.as_str() { + "asap_promql_subquery" => { + let [range, resolution] = call.args.as_slice() else { + return Err(LoweringError::InvalidExpression( + "asap_promql_subquery requires (range_ms, resolution_ms)".into(), + )); + }; + let range = positive_millis_literal(range, "range_ms")?; + let resolution = positive_millis_literal(resolution, "resolution_ms")?; + PlanningBridge::PromqlSubquery { range, resolution } + } + "asap_histogram_quantile" => { + let [q] = call.args.as_slice() else { + return Err(LoweringError::InvalidExpression( + "asap_histogram_quantile requires one literal quantile".into(), + )); + }; + let q = float_literal(q).ok_or_else(|| { + LoweringError::InvalidExpression( + "asap_histogram_quantile quantile must be a numeric literal".into(), + ) + })?; + if !q.is_finite() || !(0.0..=1.0).contains(&q) { + return Err(LoweringError::InvalidExpression(format!( + "asap_histogram_quantile quantile must be finite and in [0,1], got {q}" + ))); + } + PlanningBridge::HistogramQuantile { q } + } + _ => continue, + }; + if found.is_some() { + return Err(LoweringError::InvalidExpression( + "a planning projection may contain only one asap_* relation marker".into(), + )); + } + found = Some(bridge); + } + if matches!(found, Some(PlanningBridge::HistogramQuantile { .. })) && projection.expr.len() != 1 + { + return Err(LoweringError::InvalidExpression( + "asap_histogram_quantile must be the projection's only expression".into(), + )); + } + Ok(found) +} + +/// Rebuild the SQL projection around the relation sampled by the temporal +/// marker. The marker's alias names the existing child column that occupies +/// its output slot (`... asap_promql_subquery(...) AS value ...`). This makes +/// the bridge schema-preserving without silently retaining columns that SQL +/// projected away. +impl SqlLowerer<'_> { + fn temporal_bridge_projection( + &self, + projection: &logical_expr::Projection, + child: Unresolved, + ) -> Result { + let cols = projection + .expr + .iter() + .map(|expr| { + if let Expr::ScalarFunction(call) = unalias(expr) { + if call + .func + .name() + .eq_ignore_ascii_case("asap_promql_subquery") + { + let Expr::Alias(alias) = expr else { + return Err(LoweringError::InvalidExpression( + "asap_promql_subquery must have an alias naming its child value column" + .into(), + )); + }; + return Ok(ProjectItem { + expr: Scalar::Column(ColumnRef::Named(alias.name.clone())), + alias: Some(alias.name.clone()), + }); + } + } + match expr { + Expr::Alias(alias) => Ok(ProjectItem { + expr: self.lower_expr(&alias.expr)?, + alias: Some(alias.name.clone()), + }), + other => Ok(ProjectItem { + expr: self.lower_expr(other)?, + alias: None, + }), + } + }) + .collect::, LoweringError>>()?; + Ok(Unresolved::Project { + cols, + qualifier: None, + child: Rc::new(child), + }) + } +} + +fn positive_millis_literal(expr: &Expr, argument: &str) -> Result { + let millis = match unalias(expr) { + Expr::Literal(DfScalarValue::Int64(Some(value))) if *value > 0 => *value as u64, + Expr::Literal(DfScalarValue::UInt64(Some(value))) if *value > 0 => *value, + Expr::Literal(DfScalarValue::Int32(Some(value))) if *value > 0 => *value as u64, + other => { + return Err(LoweringError::InvalidExpression(format!( + "{argument} must be a positive integer millisecond literal, got {other}" + ))) + } + }; + Ok(Duration::from_millis(millis)) +} + +fn float_literal(expr: &Expr) -> Option { + match unalias(expr) { + Expr::Literal(DfScalarValue::Float64(Some(value))) => Some(*value), + Expr::Literal(DfScalarValue::Float32(Some(value))) => Some(*value as f64), + Expr::Literal(DfScalarValue::Int64(Some(value))) => Some(*value as f64), + Expr::Literal(DfScalarValue::UInt64(Some(value))) => Some(*value as f64), + Expr::Literal(DfScalarValue::Int32(Some(value))) => Some(*value as f64), + _ => None, + } +} + +// ── ClickHouse-builtin compatibility, taught to DataFusion itself ────────────── +// +// Generalized over `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` (issue +// #225): adding support for one more ClickHouse-only builtin DataFusion +// doesn't know at all is a catalog data entry (name, arity, `RewriteKind`) +// plus, only if its rewrite target is a genuinely new shape, one match arm +// in `ClickHouseBuiltinRewrite::rewrite` below — never a new stub-UDAF +// constructor or a new `FunctionRewrite`-implementing type. `uniqExact` +// (issue #221) and `countIf` both go through this one mechanism. + +/// A stub `AggregateUDF` for one `CLICKHOUSE_BUILTINS` entry, registered +/// purely so DataFusion's planner can resolve the function name during +/// `SqlToRel` conversion (it errors on an unknown function before a rewrite +/// ever gets a chance to run). Every call site is replaced by +/// `ClickHouseBuiltinRewrite` — via the `Analyzer` `lower()` runs after +/// parsing — before physical planning could ever ask this UDAF for an +/// `Accumulator`, so `accumulator` is unreachable for every catalog entry. +fn clickhouse_builtin_stub_udaf(name: &'static str, arity: Arity) -> AggregateUDF { + AggregateUDF::from(SimpleAggregateUDF::new_with_signature( + name, + arity_to_signature(arity), + ArrowDataType::Int64, + Arc::new(move |_| { + // ponytail: dead code by construction (see doc comment above) — + // a real accumulator would just reimplement whatever native + // shape `ClickHouseBuiltinRewrite` rewrites this call to. + unimplemented!( + "{name} has no accumulator: every call site is rewritten to a native \ + DataFusion aggregate before physical planning" + ) + }), + vec![], + )) +} + +/// A catalog [`Arity`] as the DataFusion `Signature` a stub UDAF/UDF is +/// registered with — shared by the aggregate stub above and the scalar stub +/// below, since neither wants to model per-argument types, only how many +/// arguments a call may take. +fn arity_to_signature(arity: Arity) -> Signature { + match arity { + Arity::Exact(n) => Signature::any(n, Volatility::Immutable), + Arity::Range { min, max } => Signature::one_of( + (min..=max).map(TypeSignature::Any).collect(), + Volatility::Immutable, + ), + } +} + +// ── ClickHouse scalar-builtin compatibility ───────────────────────────────── +// +// The scalar counterpart of the aggregate mechanism above, but simpler: +// `asap_sql_function_catalog::CLICKHOUSE_SCALAR_BUILTINS` carries no +// `RewriteKind`, because a scalar call needs none. Unlike an aggregate call +// (which must become a real `AggIntent`, hence the rewrite to a native +// DataFusion aggregate shape `lower_agg_intent` can classify), a scalar +// function call in this IR is already deliberately opaque — +// `SqlLowerer::lower_expr`'s `Expr::ScalarFunction` arm lowers *any* +// scalar call generically to `UnresolvedScalar::FunctionCall { name, args }`, with +// zero name-specific logic. So teaching DataFusion's planner to accept a +// ClickHouse scalar builtin's name — a stub `ScalarUDF`, registered below — +// is the entire fix; the existing generic lowering already does the rest. + +/// A stub `ScalarUDF` for one `CLICKHOUSE_SCALAR_BUILTINS` entry, registered +/// purely so DataFusion's planner can resolve the function name during +/// `SqlToRel` conversion (it errors on an unknown function otherwise), and so +/// it can keep building the surrounding expression's type from a plausible +/// return type. Unlike `clickhouse_builtin_stub_udaf`, no `FunctionRewrite` +/// ever fires for these — the call survives to `lower_plan` as-is and lowers +/// through the generic `Expr::ScalarFunction` arm — so `invoke`/`invoke_batch` +/// (left at their default, which returns a `NotImplemented` `DataFusionError`) +/// are unreachable for every catalog entry: this front end only ever uses +/// DataFusion for planning/type-checking, never physical execution. +fn clickhouse_scalar_builtin_stub_udf(name: &'static str, arity: Arity) -> ScalarUDF { + ScalarUDF::from(ClickHouseScalarBuiltinStub { + name, + signature: arity_to_signature(arity), + return_type: clickhouse_scalar_builtin_return_type(name), + }) +} + +/// A plausible Arrow return type for one `CLICKHOUSE_SCALAR_BUILTINS` entry — +/// just precise enough that DataFusion's planner can keep building the type +/// of whatever expression the call sits inside (e.g. a `WHERE` predicate +/// wants `Boolean`), not a claim about ClickHouse's actual return type. +/// Real function typing happens downstream, at post-ASAP binding. +fn clickhouse_scalar_builtin_return_type(name: &str) -> ArrowDataType { + match name { + // Array(String) in ClickHouse; a plain `Utf8` element list is close + // enough for planning purposes here. + "splitbychar" => ArrowDataType::List(Arc::new(datafusion::arrow::datatypes::Field::new( + "item", + ArrowDataType::Utf8, + true, + ))), + "todate" => ArrowDataType::Date32, + // ClickHouse returns UInt8 (0/1), but every corpus use is a boolean + // predicate — `Boolean` keeps that context type-checking. + "match" | "startswith" => ArrowDataType::Boolean, + "tostartofhour" + | "tostartofweek" + | "tostartofminute" + | "tostartoffiveminutes" + | "tostartofinterval" => { + ArrowDataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None) + } + // 1-based match position, 0 if not found. + "positioncaseinsensitive" => ArrowDataType::UInt64, + // Relation markers are removed by `lower_projection`; Float64 merely + // lets DataFusion type the temporary SELECT list. + "asap_promql_subquery" | "asap_histogram_quantile" => ArrowDataType::Float64, + other => unreachable!( + "{other}: every CLICKHOUSE_SCALAR_BUILTINS entry must have a return type listed here" + ), + } +} + +/// A stub `ScalarUDFImpl` carrying only what DataFusion's planner needs: +/// name, arity-only [`Signature`], and a fixed return type. `invoke`/ +/// `invoke_batch` are left at their trait defaults (a `NotImplemented` +/// `DataFusionError`) — see [`clickhouse_scalar_builtin_stub_udf`]'s doc for +/// why that is unreachable in practice. +#[derive(Debug)] +struct ClickHouseScalarBuiltinStub { + name: &'static str, + signature: Signature, + return_type: ArrowDataType, +} + +impl ScalarUDFImpl for ClickHouseScalarBuiltinStub { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn name(&self) -> &str { + self.name + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type( + &self, + _arg_types: &[ArrowDataType], + ) -> datafusion::common::Result { + Ok(self.return_type.clone()) + } +} + +// ── ClickHouse window-builtin compatibility ───────────────────────────────── +// +// The window counterpart of the scalar mechanism above: a stub `WindowUDF` +// registered purely so DataFusion's planner accepts the call name during +// `SqlToRel` conversion. No rewrite step follows — `lower_window_func_kind` +// already maps each `asap_sql_function_catalog::CLICKHOUSE_WINDOW_BUILTINS` +// name directly to its own `WindowFuncKind` variant (issue #267). + +/// A stub `WindowUDF` for one `CLICKHOUSE_WINDOW_BUILTINS` entry, registered +/// purely so DataFusion's planner can resolve the function name inside an +/// `OVER (...)` clause. This front end only ever uses DataFusion for +/// planning/type-checking, never physical execution, so +/// `partition_evaluator` (which physical execution alone would call) is +/// unreachable in practice. +fn clickhouse_window_builtin_stub_udwf(name: &'static str, arity: Arity) -> WindowUDF { + WindowUDF::from(ClickHouseWindowBuiltinStub { + name, + signature: arity_to_signature(arity), + }) +} + +/// A stub `WindowUDFImpl` carrying only what DataFusion's planner needs: +/// name, arity-only [`Signature`], and a field type derived from the first +/// argument (matching `lag`/`lead`'s own "output type = input type" +/// behavior). `partition_evaluator` is left `unimplemented!()` — see +/// [`clickhouse_window_builtin_stub_udwf`]'s doc for why that is unreachable. +#[derive(Debug)] +struct ClickHouseWindowBuiltinStub { + name: &'static str, + signature: Signature, +} + +impl WindowUDFImpl for ClickHouseWindowBuiltinStub { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn name(&self) -> &str { + self.name + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn field(&self, field_args: WindowUDFFieldArgs) -> datafusion::common::Result { + let dtype = field_args.get_input_type(0).unwrap_or(ArrowDataType::Null); + Ok(Field::new(field_args.name(), dtype, true)) + } + + fn partition_evaluator( + &self, + _partition_evaluator_args: PartitionEvaluatorArgs, + ) -> datafusion::common::Result> { + let name = self.name; + unimplemented!( + "{name} has no partition evaluator: this front end never runs DataFusion's \ + physical planner, only SqlToRel + the unoptimized LogicalPlan" + ) + } +} + +/// Rewrites every `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` call to +/// the native DataFusion aggregate shape its entry's `RewriteKind` names — +/// so a ClickHouse-only builtin DataFusion doesn't know at all becomes an +/// ordinary DataFusion aggregate before the plan ever reaches +/// `lower_agg_intent`, which needs no ClickHouse-specific name of its own. +#[derive(Debug)] +struct ClickHouseBuiltinRewrite; + +impl FunctionRewrite for ClickHouseBuiltinRewrite { + fn name(&self) -> &str { + "clickhouse builtin -> native DataFusion aggregate" + } + + fn rewrite( + &self, + expr: Expr, + _schema: &DFSchema, + _config: &ConfigOptions, + ) -> datafusion::common::Result> { + let Expr::AggregateFunction(f) = expr else { + return Ok(Transformed::no(expr)); + }; + let Some(builtin) = asap_sql_function_catalog::lookup_clickhouse_builtin(f.func.name()) + else { + return Ok(Transformed::no(Expr::AggregateFunction(f))); + }; + let rewritten = match builtin.rewrite { + // No native DataFusion shape to become — leave the call exactly + // as DataFusion's planner parsed it. `lower_agg_intent` handles + // the ClickHouse name (`argMax`/`argMin`) directly (issue #232). + RewriteKind::PassThrough => return Ok(Transformed::no(Expr::AggregateFunction(f))), + // `f(args...)` -> `count(args...) DISTINCT` — `lower_agg_intent` + // already maps `count` + `DISTINCT` to `AggIntent::Cardinality`, + // at whatever arity the call carries. + RewriteKind::CountDistinct => AggregateFunction::new_udf( + count_udaf(), + f.args, + true, + f.filter, + f.order_by, + f.null_treatment, + ), + // `f(cond)` -> `sum(CASE WHEN cond THEN 1 ELSE 0 END)` — see + // `RewriteKind::CountIfToSum`'s doc; moving the `-If` family onto + // `Aggregate.filters` (issue #466) is a follow-up. + RewriteKind::CountIfToSum => { + let cond = f.args.into_iter().next().expect( + "countif's stub signature fixes its arity at 1 -- the planner \ + already rejected any other argument count before this rewrite runs", + ); + let indicator = Expr::Case(Case::new( + None, + vec![(Box::new(cond), Box::new(lit(1i64)))], + Some(Box::new(lit(0i64))), + )); + AggregateFunction::new_udf( + sum_udaf(), + vec![indicator], + false, + f.filter, + f.order_by, + f.null_treatment, + ) + } + }; + Ok(Transformed::yes(Expr::AggregateFunction(rewritten))) + } +} + +// ── Aggregate / group-key helpers ─────────────────────────────────────────────── + +/// The row predicate one aggregate call carries (issue #466): its explicit +/// `FILTER (WHERE p)`, plus — for a plain `count(expr)`, which canonical +/// `AggIntent::Count` lowers to a row count that never looks at `expr` — the +/// NULL-skipping SQL gives it. `count(CASE WHEN p THEN x END)` is the +/// conditional-count idiom, so it becomes `p [AND x IS NOT NULL]` rather +/// than the opaque `CASE … IS NOT NULL`; any other nullable argument becomes +/// `expr IS NOT NULL`. `None` when the call updates on every row. +fn measure_filter(expr: &Expr, input: &DFSchema) -> Result, LoweringError> { + let Expr::AggregateFunction(agg_fn) = unalias(expr) else { + return Ok(None); + }; + let mut conjuncts: Vec = agg_fn.filter.iter().map(|f| (**f).clone()).collect(); + let counts_rows = agg_fn.func.name().eq_ignore_ascii_case("count") && !agg_fn.distinct; + if counts_rows { + for argument in &agg_fn.args { + let nullable = argument + .nullable(input) + .map_err(|error| LoweringError::UnsupportedFeature(error.to_string()))?; + if !nullable { + continue; + } + match conditional_count_arm(argument) { + Some((when, then)) => { + conjuncts.push(when.clone()); + if then + .nullable(input) + .map_err(|error| LoweringError::UnsupportedFeature(error.to_string()))? + { + conjuncts.push(then.clone().is_not_null()); + } + } + None => conjuncts.push(argument.clone().is_not_null()), + } + } + } + Ok(conjuncts.into_iter().reduce(Expr::and)) +} + +/// `CASE WHEN p THEN x END` (searched, one arm, no `ELSE` or `ELSE NULL`) +/// as `(p, x)`. +fn conditional_count_arm(expr: &Expr) -> Option<(&Expr, &Expr)> { + let Expr::Case(case) = unalias(expr) else { + return None; + }; + if case.expr.is_some() { + return None; + } + let else_is_null = match case.else_expr.as_deref() { + None => true, + Some(Expr::Literal(value)) => value.is_null(), + Some(_) => false, + }; + if !else_is_null { + return None; + } + let [(when, then)] = case.when_then_expr.as_slice() else { + return None; + }; + Some((when, then)) +} + +/// Map a DataFusion aggregate expression directly to the canonical +/// [`AggIntent`] — issue #179's "dedicated function → canonical +/// intent directly" front-end construction, no `AggFunc` intermediate. The +/// name → semantic mapping itself lives in `asap_sql_function_catalog` +/// (issue #225) as flat data (`NATIVE_FUNCTIONS`); what stays here is +/// call-site logic that isn't a function of the name alone — the DISTINCT +/// modifier rule, the "reducer argument must be a bare column" rule +/// (`reducer_col`), φ extraction from a literal argument, and the ambient +/// `AccuracyTarget`. `resolve_root` resolves `col` to a positional +/// `ColumnId`; the output name (DataFusion's own, e.g. +/// `"sum(metrics.bytes)"`) is threaded separately as `Aggregate.output_names`, +/// not carried here. +fn lower_agg_intent(expr: &Expr) -> Result, LoweringError> { + match expr { + Expr::Alias(a) => lower_agg_intent(&a.expr), + Expr::AggregateFunction(agg_fn) => { + let name = agg_fn.func.name().to_lowercase(); + // ClickHouse's row-selecting `argMax`/`argMin` — `RewriteKind:: + // PassThrough` in the catalog, so the call reaches here under its + // own name rather than a native DataFusion aggregate. Handled + // before the `NATIVE_FUNCTIONS` lookup below since neither name + // is in that table (issue #232). + if let Some(intent) = lower_arg_selector(&name, &agg_fn.args)? { + return Ok(intent); + } + let semantic = asap_sql_function_catalog::lookup_native(&name) + .ok_or_else(|| LoweringError::UnsupportedAggregate(name.clone()))?; + // The canonical intent algebra has no DISTINCT modifier for the + // value reducers; only + // COUNT(DISTINCT) maps (to Cardinality). Reject DISTINCT elsewhere + // rather than silently lowering `SUM(DISTINCT x)` as `SUM(x)`. + if agg_fn.distinct && !matches!(semantic, AggSemantic::Count) { + return Err(LoweringError::UnsupportedAggregate(format!( + "DISTINCT {name}" + ))); + } + // Value reducers (`reducer_col`) require a real column — `SUM(a*b)` + // is rejected, not silently reduced over a probe column. Quantile + // and CountDistinct reduce a column too, so they take the same path: + // `col` is `Option` once resolved, where `None` means "the + // PromQL sample value", which a SQL query never has. Taking an + // expression here would set `col: None` and silently drop it (#115). + let col = |args: &[Expr]| -> Result, LoweringError> { + reducer_col(&name, args).map(Some) + }; + Ok(match semantic { + AggSemantic::Correlation => { + if agg_fn.order_by.is_some() || agg_fn.null_treatment.is_some() { + return Err(LoweringError::UnsupportedAggregate( + "corr with ORDER BY or explicit null treatment".into(), + )); + } + let [left, right] = agg_fn.args.as_slice() else { + return Err(LoweringError::UnsupportedAggregate( + "corr requires two arguments".into(), + )); + }; + AggIntent::PearsonCorr { + left: expr_to_group_ref(left)?, + right: expr_to_group_ref(right)?, + } + } + // Every argument reaches the intent: `COUNT(DISTINCT a, b)` + // counts distinct *tuples*, which is a different quantity from + // the distinct count of either column. + AggSemantic::Count if agg_fn.distinct => match agg_fn.args.as_slice() { + // DataFusion's planner rejects a bare `COUNT(DISTINCT)` + // before lowering. Guarded anyway: an empty `cols` is the + // PromQL sample-value convention, which SQL never has. + [] => { + return Err(LoweringError::UnsupportedAggregate( + "COUNT(DISTINCT) without an argument".into(), + )) + } + args => AggIntent::Cardinality { + cols: args.iter().map(distinct_col).collect::>()?, + accuracy: current_accuracy(), + }, + }, + AggSemantic::Count => AggIntent::Count { + accuracy: current_accuracy(), + }, + AggSemantic::Sum => AggIntent::Sum { + col: col(&agg_fn.args)?, + }, + AggSemantic::Min => AggIntent::Min { + col: col(&agg_fn.args)?, + }, + AggSemantic::Max => AggIntent::Max { + col: col(&agg_fn.args)?, + }, + AggSemantic::Avg => AggIntent::Avg { + col: col(&agg_fn.args)?, + }, + AggSemantic::StdDev { population } => AggIntent::StdDev { + col: col(&agg_fn.args)?, + population, + }, + AggSemantic::Variance { population } => AggIntent::Variance { + col: col(&agg_fn.args)?, + population, + }, + // `fixed_q = Some(0.5)` is `median`/`approx_median`. As with + // `approx_distinct` and `approx_percentile_cont`, the + // `approx_` prefix does not force an approximation: the + // sketch-vs-exact choice is the AccuracyTarget's (see + // `plan::boundary`), so both spellings share one intent + // (#111). + AggSemantic::Quantile { fixed_q } => AggIntent::Quantile { + col: col(&agg_fn.args)?, + q: match fixed_q { + Some(q) => q, + None => extract_percentile_q(&agg_fn.args)?, + }, + accuracy: current_accuracy(), + }, + AggSemantic::Cardinality => AggIntent::Cardinality { + cols: vec![reducer_col(&name, &agg_fn.args)?], + accuracy: current_accuracy(), + }, + }) + } + _ => Err(LoweringError::UnsupportedAggregate(format!( + "measure is not an aggregate function call: {expr}" + ))), + } +} + +fn temporal_aggregate_name(expr: &Expr) -> Option { + let Expr::AggregateFunction(call) = unalias(expr) else { + return None; + }; + let name = call.func.name().to_lowercase(); + matches!(name.as_str(), "asap_rate" | "asap_increase").then_some(name) +} + +fn is_temporal_aggregate(expr: &Expr) -> bool { + temporal_aggregate_name(expr).is_some() +} + +fn is_temporal_output_column(expr: &Expr) -> bool { + let Expr::Column(col) = unalias(expr) else { + return false; + }; + let name = col.name.to_lowercase(); + ["asap_rate(", "asap_increase("] + .iter() + .any(|prefix| name.starts_with(prefix)) +} + +fn plan_has_temporal_aggregate(plan: &LogicalPlan) -> bool { + match plan { + LogicalPlan::Aggregate(agg) => agg.aggr_expr.iter().any(is_temporal_aggregate), + LogicalPlan::Filter(filter) => plan_has_temporal_aggregate(&filter.input), + LogicalPlan::SubqueryAlias(alias) => plan_has_temporal_aggregate(&alias.input), + _ => false, + } +} + +fn named_ref(col: &ColumnRef) -> &str { + match col { + ColumnRef::Named(name) | ColumnRef::Qualified { name, .. } => name, + ColumnRef::SampleValue | ColumnRef::Wildcard => { + unreachable!("reducer_col only returns named column references") + } + } +} + +fn scalar_positive_u64(value: &DfScalarValue) -> Option { + match value { + DfScalarValue::Int64(Some(v)) if *v > 0 => Some(*v as u64), + DfScalarValue::Int32(Some(v)) if *v > 0 => Some(*v as u64), + DfScalarValue::UInt64(Some(v)) if *v > 0 => Some(*v), + DfScalarValue::UInt32(Some(v)) if *v > 0 => Some(*v as u64), + _ => None, + } +} + +/// ClickHouse's row-selecting `argMax(arg, val)` / `argMin(arg, val)` — +/// "return `arg`'s value from the row where `val` is maximal/minimal". +/// `Some(name)` for `"argmax"`/`"argmin"`, `None` for every other name (the +/// caller falls through to the ordinary `NATIVE_FUNCTIONS` path). +/// +/// Unlike every existing `AggIntent` reducer (`Sum`/`Min`/`Max`/`Avg`/…), +/// which folds *one* column to a value derived from itself, this is a +/// two-column, row-selecting aggregate: it returns a *different* column's +/// value, selected by which row maximizes/minimizes a second column. No +/// existing `AggIntent` shape fits, and — per its own doc comment's +/// "core only grows for intents ≥2 deployment models actually use" bar — +/// a repo-wide search (PromQL front end, the other SQL dialects, docs) found +/// no second deployment model wanting this shape, so this lowers to +/// `AggIntent::Extension` rather than earning a first-class `ArgMax`/`ArgMin` +/// core variant (issue #232). Core treats `Extension` opaquely: both columns +/// are kept only as validated bare-column names in `payload` (`reducer_col`'s +/// same "no expression arguments" rule, issue #115) — they are **not** run +/// through `resolve_agg_intent`'s positional `ColumnRef` -> `ColumnId` +/// binding the way a real reducer's `col` is, since `Extension` carries no +/// typed column field for core to resolve. Shared `arg_selector_columns` validates +/// and resolves those names during aggregate schema derivation, preserving the +/// selected argument's type and nullability for downstream exact execution. +/// +/// DerivedCols preserves both bare-column arguments when grouping expressions +/// introduce an intermediate Project. Shared aggregate schema derivation resolves +/// the payload and preserves the selected argument's type and nullability. +fn lower_arg_selector( + name: &str, + args: &[Expr], +) -> Result>, LoweringError> { + let ext_kind = match name { + "argmax" => "arg_max", + "argmin" => "arg_min", + _ => return Ok(None), + }; + let [arg, val] = args else { + unreachable!( + "{name}'s stub signature (asap_sql_function_catalog::CLICKHOUSE_BUILTINS) fixes \ + its arity at 2 -- the planner already rejected any other argument count before \ + lower_agg_intent runs" + ); + }; + let arg_col = reducer_col(name, std::slice::from_ref(arg))?; + let val_col = reducer_col(name, std::slice::from_ref(val))?; + Ok(Some(AggIntent::Extension { + ext_kind: ext_kind.to_string(), + payload: serde_json::json!({ "arg_col": arg_col, "val_col": val_col }), + })) +} + +/// Fold `pred` directly onto `child.predicates` when `child` is a bare `Scan` +/// (a `WHERE` directly over a table), otherwise wrap it in an ordinary +/// `Filter` — canonical's invariant that a `Filter` never sits directly over a +/// `Scan`. A front end emitting the canonical shape directly is responsible +/// for maintaining that invariant itself (issue #179). +fn filter_or_fold(pred: Scalar, child: Unresolved) -> Unresolved { + match child { + Unresolved::Scan { + source, + mut predicates, + schema, + } => { + predicates.push(Predicate(pred)); + Unresolved::Scan { + source, + predicates, + schema, + } + } + other => Unresolved::Filter { + pred: Predicate(pred), + child: Rc::new(other), + }, + } +} + +/// Flatten a top-level `AND` chain into its conjuncts. +fn split_conjunction<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) { + match expr { + Expr::BinaryExpr(b) if b.op == logical_expr::Operator::And => { + split_conjunction(&b.left, out); + split_conjunction(&b.right, out); + } + other => out.push(other), + } +} + +/// Whether `expr` reads another operator anywhere inside it (`EXISTS`, +/// `IN (…)`, a scalar subquery). +fn reads_subquery(expr: &Expr) -> bool { + expr.exists(|e| { + Ok(matches!( + e, + Expr::ScalarSubquery(_) | Expr::InSubquery(_) | Expr::Exists(_) + )) + }) + .expect("the predicate never fails") +} + +/// Re-`AND` the conjuncts, or `None` when there are none left. +fn rebuild_conjunction(conjuncts: &[&Expr]) -> Option { + conjuncts + .iter() + .map(|e| (*e).clone()) + .reduce(|acc, e| acc.and(e)) +} + +/// Split a correlated subquery's plan into `(uncorrelated plan, correlation)`. +/// +/// The correlation is the conjunction of the filter conjuncts that mention an +/// outer column, rewritten so `outer_ref(t.c)` becomes a plain `t.c` — it then +/// resolves against the join's concatenated `left ++ right` schema, like any +/// other join predicate. Everything else stays an ordinary inner `Filter`. +/// +/// An outer reference anywhere but a top-level filter conjunct is rejected: it +/// would need real decorrelation, not a predicate lift. +fn split_correlation(plan: &LogicalPlan) -> Result<(LogicalPlan, Option), LoweringError> { + let LogicalPlan::Filter(filter) = plan else { + return if plan_has_outer_ref(plan) { + Err(LoweringError::UnsupportedFeature( + "correlated subquery whose outer reference is not a filter conjunct".into(), + )) + } else { + Ok((plan.clone(), None)) + }; + }; + + let mut conjuncts = Vec::new(); + split_conjunction(&filter.predicate, &mut conjuncts); + let (correlated, inner): (Vec<_>, Vec<_>) = + conjuncts.into_iter().partition(|e| expr_has_outer_ref(e)); + + let input = filter.input.as_ref(); + if plan_has_outer_ref(input) { + return Err(LoweringError::UnsupportedFeature( + "correlated subquery whose outer reference is below its filter".into(), + )); + } + + let correlation = rebuild_conjunction(&correlated) + .map(|e| strip_outer_refs(&e)) + .transpose()?; + let plan = match rebuild_conjunction(&inner) { + Some(pred) => LogicalPlan::Filter( + logical_expr::Filter::try_new(pred, filter.input.clone()) + .map_err(LoweringError::DataFusion)?, + ), + None => input.clone(), + }; + Ok((plan, correlation)) +} + +/// Rewrite `outer_ref(t.c)` to `t.c` so the expression resolves against the +/// join's concatenated schema. +fn strip_outer_refs(expr: &Expr) -> Result { + expr.clone() + .transform(|e| { + Ok(match e { + Expr::OuterReferenceColumn(_, col) => Transformed::yes(Expr::Column(col)), + other => Transformed::no(other), + }) + }) + .map(|t| t.data) + .map_err(LoweringError::DataFusion) +} + +fn expr_has_outer_ref(expr: &Expr) -> bool { + let mut found = false; + expr.apply(|e| { + if matches!(e, Expr::OuterReferenceColumn(..)) { + found = true; + return Ok(TreeNodeRecursion::Stop); + } + Ok(TreeNodeRecursion::Continue) + }) + .expect("infallible visitor"); + found +} + +fn plan_has_outer_ref(plan: &LogicalPlan) -> bool { + let mut found = false; + plan.apply(|p| { + if p.expressions().iter().any(expr_has_outer_ref) { + found = true; + return Ok(TreeNodeRecursion::Stop); + } + Ok(TreeNodeRecursion::Continue) + }) + .expect("infallible visitor"); + found +} + +/// Strip `AS alias` wrappers. +fn unalias(expr: &Expr) -> &Expr { + match expr { + Expr::Alias(a) => unalias(&a.expr), + other => other, + } +} + +/// The `GroupingSet` inside a grouping expression, if any. +fn as_grouping_set(expr: &Expr) -> Option<&logical_expr::GroupingSet> { + match unalias(expr) { + Expr::GroupingSet(gs) => Some(gs), + _ => None, + } +} + +/// The grouping levels a `GroupingSet` stands for, widest first (issue #118). +/// +/// `ROLLUP(a, b)` → `(a,b), (a), ()` — the prefixes. +/// `CUBE(a, b)` → `(a,b), (a), (b), ()` — the power set. +/// `GROUPING SETS` is already the explicit list. +fn expand_grouping_set(gs: &logical_expr::GroupingSet) -> Vec> { + match gs { + logical_expr::GroupingSet::Rollup(exprs) => (0..=exprs.len()) + .rev() + .map(|n| exprs[..n].to_vec()) + .collect(), + logical_expr::GroupingSet::Cube(exprs) => { + // Bitmask descending, so the full set leads and `()` trails. + (0..(1u32 << exprs.len())) + .rev() + .map(|mask| { + exprs + .iter() + .enumerate() + .filter(|(i, _)| mask & (1 << i) != 0) + .map(|(_, e)| e.clone()) + .collect() + }) + .collect() + } + logical_expr::GroupingSet::GroupingSets(sets) => sets.clone(), + } +} + +/// Derived columns materialized in a `Project` beneath an `Aggregate` (#110). +/// +/// `Aggregate.by` holds positional `ColumnId`s and each reducer holds one input +/// column, so neither can hold an expression. `GROUP BY date_trunc('minute', t)` +/// and `SUM(bytes * 8)` are therefore rewritten to group/reduce over a projected +/// column that carries the expression's value. +/// +/// The projection also has to carry through the plain columns the aggregate +/// still references, since a `Project` replaces its child's schema rather than +/// extending it. +struct DerivedCols<'l> { + lowerer: &'l SqlLowerer<'l>, + cols: Vec, + /// Whether any column is genuinely derived. Without one the aggregate keeps + /// its original child, so trees that lower today keep their exact shape. + any: bool, + /// First same-name-different-value collision, reported only if the + /// projection is actually inserted (see [`Self::wrap`]). + collision: Option, +} + +impl<'l> DerivedCols<'l> { + fn new(lowerer: &'l SqlLowerer<'l>) -> Self { + Self { + lowerer, + cols: Vec::new(), + any: false, + collision: None, + } + } + + /// Add `alias := expr`, or note a collision if `alias` already means + /// something else. `Project` carries one relation qualifier for all its + /// columns, so `a.k` and `b.k` cannot both survive it — but that only + /// matters when a projection gets inserted at all. + fn push(&mut self, alias: String, expr: Scalar) { + let existing = self + .cols + .iter() + .find(|c| c.alias.as_deref() == Some(&alias)); + match existing { + // Same name, same value — one projected column serves both uses. + Some(e) if e.expr == expr => {} + Some(_) => { + self.collision.get_or_insert(alias); + } + None => self.cols.push(ProjectItem { + alias: Some(alias), + expr, + }), + } + } + + /// A plain column the aggregate references — carried through unchanged. + fn passthrough(&mut self, expr: &Expr) -> Result<(), LoweringError> { + let Expr::Column(c) = unalias(expr) else { + return Ok(()); + }; + self.push(c.name.clone(), self.lowerer.lower_expr(expr)?); + Ok(()) + } + + /// A genuinely derived column: `alias` now names `expr`'s value. + fn materialize(&mut self, alias: String, expr: Scalar) -> Result<(), LoweringError> { + self.any = true; + self.push(alias, expr); + Ok(()) + } + + /// Repoint a reducer's argument at a derived column when it is an + /// expression; otherwise carry its plain input column through. + fn rewrite_agg(&mut self, expr: &Expr) -> Result { + let Expr::AggregateFunction(agg_fn) = unalias(expr) else { + return Ok(expr.clone()); + }; + if matches!( + asap_sql_function_catalog::lookup_native(&agg_fn.func.name().to_lowercase()), + Some(AggSemantic::Correlation) + ) { + // Give each value argument its own projected name, including casts + // and qualified columns. This retains both inputs and avoids losing + // relation qualifiers when the projection becomes an unqualified schema. + let mut rewritten = agg_fn.clone(); + for arg in &mut rewritten.args { + let alias = unalias(arg).to_string(); + self.materialize(alias.clone(), self.lowerer.lower_expr(arg)?)?; + *arg = Expr::Column(DfColumn::new_unqualified(alias)); + } + return Ok(Expr::AggregateFunction(rewritten)); + } + // `COUNT(*)` reduces no column; `agg_col_name` covers bare/aliased/cast + // columns, so `None` here means the argument really is an expression. + let counts_rows = agg_fn.func.name().eq_ignore_ascii_case("count") && !agg_fn.distinct; + let Some(arg) = agg_fn.args.first() else { + return Ok(expr.clone()); + }; + if counts_rows { + return Ok(expr.clone()); + } + // Preserve every additional column dependency (e.g. argMax's ordering + // column) when an unrelated grouping expression creates a Project. + // Literal parameters need no source column and remain untouched. + for argument in agg_fn.args.iter().skip(1) { + self.passthrough(argument)?; + } + match agg_col_name(&agg_fn.args) { + Some(name) => { + self.push(name, self.lowerer.lower_expr(arg)?); + Ok(expr.clone()) + } + None => { + let alias = unalias(arg).to_string(); + self.materialize(alias.clone(), self.lowerer.lower_expr(arg)?)?; + let mut agg_fn = agg_fn.clone(); + agg_fn.args[0] = Expr::Column(DfColumn::new_unqualified(alias)); + Ok(Expr::AggregateFunction(agg_fn)) + } + } + } + + /// Wrap `input` in the materializing `Project`, or return it untouched when + /// nothing needed deriving — so a query that lowers today keeps its exact + /// tree, and a name collision that the projection would have flattened only + /// matters once the projection exists. + fn wrap(self, input: Unresolved) -> Result { + if !self.any { + return Ok(input); + } + if let Some(alias) = self.collision { + return Err(LoweringError::UnsupportedFeature(format!( + "ambiguous column `{alias}` beneath an expression GROUP BY / \ + aggregate — alias the relations apart" + ))); + } + Ok(Unresolved::Project { + cols: self.cols, + qualifier: None, + child: Rc::new(input), + }) + } +} + +/// The first aggregate argument's column name (bare / aliased / cast column), +/// or `None` for `*` / a non-column expression. +fn agg_col_name(args: &[Expr]) -> Option { + fn col_name(e: &Expr) -> Option { + match e { + Expr::Column(c) => Some(c.name.clone()), + Expr::Alias(a) => col_name(&a.expr), + Expr::Cast(c) => col_name(&c.expr), + _ => None, + } + } + args.first().and_then(col_name) +} + +/// The single input column of a value reducer (`SUM`/`MIN`/`MAX`/`AVG`/stddev/ +/// variance/quantile/count-distinct). Errors if the argument is not a column: +/// the canonical `AggIntent` reduces a column, not an arbitrary expression +/// (`SUM(a*b)`), so silently picking a probe column would compute the wrong +/// result. +fn reducer_col(name: &str, args: &[Expr]) -> Result { + agg_col_name(args).map(ColumnRef::Named).ok_or_else(|| { + LoweringError::UnsupportedAggregate(format!("{name} over a non-column expression")) + }) +} + +/// One argument of a `COUNT(DISTINCT ...)`. Resolved the way a grouping key is +/// — what is being counted is an identity, and its qualifier has to survive a +/// join (`a.k` vs `b.k`) — but reported as an aggregate restriction, since an +/// aggregate call is what the user wrote. +fn distinct_col(expr: &Expr) -> Result { + expr_to_group_ref(expr).map_err(|_| { + LoweringError::UnsupportedAggregate( + "COUNT(DISTINCT ...) over a non-column expression".into(), + ) + }) +} + +fn expr_to_group_ref(expr: &Expr) -> Result { + match expr { + // Preserve the relation qualifier so a GROUP BY / PARTITION BY key over a + // join (`b.k` vs `a.k`) resolves to the correct side — the same rule the + // scalar predicate path uses (`lower_expr`). + Expr::Column(col) => Ok(match &col.relation { + Some(rel) => ColumnRef::Qualified { + table: rel.to_string(), + name: col.name.clone(), + }, + None => ColumnRef::Named(col.name.clone()), + }), + Expr::Alias(a) => expr_to_group_ref(&a.expr), + other => Err(LoweringError::UnsupportedFeature(format!( + "non-column GROUP BY expression: {other}" + ))), + } +} + +fn extract_percentile_q(args: &[Expr]) -> Result { + let q = match args.get(1) { + Some(Expr::Literal(DfScalarValue::Float64(Some(q)))) => *q, + Some(Expr::Literal(DfScalarValue::Float32(Some(q)))) => *q as f64, + _ => { + return Err(LoweringError::InvalidExpression( + "percentile value must be a float literal (2nd arg)".into(), + )) + } + }; + if q.is_finite() && (0.0..=1.0).contains(&q) { + Ok(q) + } else { + Err(LoweringError::InvalidExpression(format!( + "percentile must be in [0, 1], got {q}" + ))) + } +} + +// ── LogicalPlan navigation helpers ────────────────────────────────────────────── + +fn eval_fetch(expr_opt: &Option>) -> Option { + expr_opt.as_ref().and_then(|e| match e.as_ref() { + Expr::Literal(DfScalarValue::Int64(Some(v))) if *v >= 0 => Some(*v as usize), + Expr::Literal(DfScalarValue::UInt64(Some(v))) => Some(*v as usize), + Expr::Literal(DfScalarValue::Int32(Some(v))) if *v >= 0 => Some(*v as usize), + _ => None, + }) +} + +/// Map a DataFusion window-function definition to the canonical +/// [`WindowFuncKind`]. +/// `NthValue` is returned with `None`; `lower_window` fills in `n` from args. +fn lower_window_func_kind(fun: &WindowFunctionDefinition) -> Result { + let unsupported = |what: &str, name: &str| { + LoweringError::UnsupportedFeature(format!("window {what}: {name}")) + }; + match fun { + WindowFunctionDefinition::WindowUDF(udf) => match udf.name().to_lowercase().as_str() { + "row_number" => Ok(WindowFuncKind::RowNumber), + "rank" => Ok(WindowFuncKind::Rank), + "dense_rank" => Ok(WindowFuncKind::DenseRank), + "lag" => Ok(WindowFuncKind::Lag), + "lead" => Ok(WindowFuncKind::Lead), + // ClickHouse: frame-respecting variants, not plain Lag/Lead (#267). + "laginframe" => Ok(WindowFuncKind::LagInFrame), + "leadinframe" => Ok(WindowFuncKind::LeadInFrame), + "first_value" => Ok(WindowFuncKind::FirstValue), + "last_value" => Ok(WindowFuncKind::LastValue), + "nth_value" => Ok(WindowFuncKind::NthValue(None)), + other => Err(unsupported("function", other)), + }, + WindowFunctionDefinition::AggregateUDF(udf) => match udf.name().to_lowercase().as_str() { + "sum" => Ok(WindowFuncKind::Sum), + "avg" | "mean" => Ok(WindowFuncKind::Avg), + "count" => Ok(WindowFuncKind::Count), + "min" => Ok(WindowFuncKind::Min), + "max" => Ok(WindowFuncKind::Max), + other => Err(unsupported("aggregate", other)), + }, + WindowFunctionDefinition::BuiltInWindowFunction(biwf) => { + use datafusion::logical_expr::BuiltInWindowFunction; + match biwf { + BuiltInWindowFunction::FirstValue => Ok(WindowFuncKind::FirstValue), + BuiltInWindowFunction::LastValue => Ok(WindowFuncKind::LastValue), + BuiltInWindowFunction::NthValue => Ok(WindowFuncKind::NthValue(None)), + } + } + } +} + +/// Map DataFusion's resolved `WindowFrame` (issue #268) to the canonical +/// [`WindowFrame`]. DataFusion's planner always fills in the SQL-standard +/// default frame before the logical plan is built, so this never sees an +/// "absent" frame — only `ROWS`/`RANGE`/`GROUPS` with concrete bounds. +/// `GROUPS` is rejected: no query in this repo's SQL corpora uses it, and +/// nothing downstream interprets frame semantics yet, so it isn't worth +/// modelling untested. +fn lower_window_frame( + frame: &datafusion::logical_expr::WindowFrame, +) -> Result { + let units = match frame.units { + DfWindowFrameUnits::Rows => WindowFrameUnits::Rows, + DfWindowFrameUnits::Range => WindowFrameUnits::Range, + DfWindowFrameUnits::Groups => { + return Err(LoweringError::UnsupportedFeature( + "window frame unit: GROUPS".into(), + )) + } + }; + let offset = |v: &DfScalarValue| -> Result { + Ok(match v { + DfScalarValue::IntervalYearMonth(Some(months)) => WindowFrameOffset::Interval { + months: *months, + days: 0, + nanoseconds: 0, + }, + DfScalarValue::IntervalDayTime(Some(value)) => WindowFrameOffset::Interval { + months: 0, + days: value.days, + nanoseconds: i64::from(value.milliseconds) * 1_000_000, + }, + DfScalarValue::IntervalMonthDayNano(Some(value)) => WindowFrameOffset::Interval { + months: value.months, + days: value.days, + nanoseconds: value.nanoseconds, + }, + // DataFusion 43 keeps every RANGE offset as text: both numeric + // bounds such as `1.5` and normalized interval literals such as + // `"1 HOUR"`. Arrow's interval parser accepts bare numbers and + // interprets them as months, so classify numeric text first. + DfScalarValue::Utf8(Some(value)) | DfScalarValue::LargeUtf8(Some(value)) => { + if let Ok(value) = value.parse::() { + WindowFrameOffset::Scalar(ScalarValue::Int64(value)) + } else if let Ok(value) = value.parse::() { + WindowFrameOffset::Scalar(ScalarValue::Float64(value)) + } else { + match parse_interval_month_day_nano(value) { + Ok(interval) => WindowFrameOffset::Interval { + months: interval.months, + days: interval.days, + nanoseconds: interval.nanoseconds, + }, + Err(_) => WindowFrameOffset::Scalar(scalar_value_to_asap(v)?), + } + } + } + _ => WindowFrameOffset::Scalar(scalar_value_to_asap(v)?), + }) + }; + let bound = |b: &DfWindowFrameBound| -> Result { + Ok(match b { + DfWindowFrameBound::Preceding(v) => WindowFrameBound::Preceding(offset(v)?), + DfWindowFrameBound::CurrentRow => WindowFrameBound::CurrentRow, + DfWindowFrameBound::Following(v) => WindowFrameBound::Following(offset(v)?), + }) + }; + Ok(WindowFrame { + units, + start_bound: bound(&frame.start_bound)?, + end_bound: bound(&frame.end_bound)?, + }) +} + +// ── Issue #225, item 3: DataFusion registry drift detection ──────────────── +// +// `asap_sql_function_catalog::NATIVE_FUNCTIONS` is hand-maintained data +// mirroring what DataFusion's own aggregate-function registry resolves. That +// mirror can only silently drift out of sync — a DataFusion version bump +// that adds, renames, or removes a builtin aggregate leaves the catalog +// looking fine while `lower_agg_intent` quietly gains or loses coverage. Of +// the two introspectable sources the issue names, DataFusion's own registry +// is the one with no external dependency: `SessionContext` already lists its +// aggregate UDFs in-process, so the check below builds a real context the +// same way `build_context` does and walks it directly — no live database, +// no new CI infra, just `cargo test`. (ClickHouse's `system.functions` is +// the other source; it needs a live ClickHouse instance, which is handled +// separately by the dev-only `tools/clickhouse/extract_functions.py` script, +// deliberately not wired into this test or into CI.) +#[cfg(test)] +mod catalog_drift { + use super::*; + + /// Every aggregate function name DataFusion's planner resolves inside a + /// context built the same way `build_context` builds one must be + /// *accounted for* by the catalog: either `lookup_native` maps it to a + /// canonical semantic, it is one of our own `CLICKHOUSE_BUILTINS` stub + /// registrations (`build_context` registers those into the very same + /// context, so they show up here too), or it is explicitly listed in + /// `KNOWN_UNMAPPED_NATIVE_FUNCTIONS` with a reason. + /// + /// This does *not* assert the reverse (that every `NATIVE_FUNCTIONS` + /// entry is resolvable) — a name that stops resolving after a DataFusion + /// bump just becomes permanently unreachable dead data, not a lowering + /// hazard, so it's out of scope for a regression gate. It also does not + /// try to derive `AggSemantic` from anything DataFusion reports — that + /// judgment call stays with whoever adds the catalog entry. + #[test] + fn every_datafusion_aggregate_name_is_covered_by_the_catalog() { + let catalog = SqlCatalog::new(); + let ctx = SqlLowerer::new(&catalog) + .build_context() + .expect("build_context with an empty table catalog cannot fail"); + let state = ctx.state(); + let mut uncovered: Vec<&str> = state + .aggregate_functions() + .keys() + .map(String::as_str) + .filter(|name| { + asap_sql_function_catalog::lookup_native(name).is_none() + && asap_sql_function_catalog::lookup_clickhouse_builtin(name).is_none() + && !asap_sql_function_catalog::KNOWN_UNMAPPED_NATIVE_FUNCTIONS.contains(name) + }) + .collect(); + uncovered.sort_unstable(); + assert!( + uncovered.is_empty(), + "DataFusion resolves these aggregate names but the catalog doesn't know about them \ + (crates/sql-function-catalog/src/lib.rs): {uncovered:?}\n\ + Either add a `NativeFunction` entry mapping each to its `AggSemantic`, or -- if it's \ + a deliberate non-goal (no `AggIntent` shape for it, or it's rejected elsewhere) -- \ + add it to `KNOWN_UNMAPPED_NATIVE_FUNCTIONS` with a reason. This usually means a \ + DataFusion version bump added or renamed a builtin aggregate." + ); + } + + /// Every `KNOWN_UNMAPPED_NATIVE_FUNCTIONS` entry earns its place by + /// actually being a name DataFusion resolves today — otherwise it is + /// stale documentation for a name that no longer exists (e.g. a prior + /// DataFusion version renamed it), not a real "deliberately not mapped" + /// decision, and should be removed. + #[test] + fn known_unmapped_entries_are_all_real_datafusion_names() { + let catalog = SqlCatalog::new(); + let ctx = SqlLowerer::new(&catalog) + .build_context() + .expect("build_context with an empty table catalog cannot fail"); + let resolved = ctx.state().aggregate_functions().clone(); + for name in asap_sql_function_catalog::KNOWN_UNMAPPED_NATIVE_FUNCTIONS { + assert!( + resolved.contains_key(*name), + "`{name}` is listed in KNOWN_UNMAPPED_NATIVE_FUNCTIONS but DataFusion no longer \ + resolves it -- remove the stale entry" + ); + } + } +} diff --git a/crates/frontend-sql/src/unified/sql/types.rs b/crates/frontend-sql/src/unified/sql/types.rs new file mode 100644 index 00000000..05d46f93 --- /dev/null +++ b/crates/frontend-sql/src/unified/sql/types.rs @@ -0,0 +1,378 @@ +//! Type bridges between DataFusion's Arrow types and the canonical `DataType`, plus +//! the SQL table catalog used to register tables with DataFusion and to carry +//! resolved leaf schemas into the canonical, unresolved tree. + +use std::collections::HashMap; + +use datafusion::arrow::datatypes::{ + DataType as ArrowDataType, Field as ArrowField, Fields, Schema as ArrowSchema, +}; +use datafusion::common::ScalarValue as DfScalarValue; + +use asap_types::pre_asap::schema::{DataType, Field, Schema}; +use asap_types::pre_asap::ScalarValue; + +use crate::unified::error::SqlError as LoweringError; + +/// Table catalog for SQL lowering: table name → resolved canonical [`Schema`]. +/// +/// Used twice: to register Arrow-backed `MemTable`s so DataFusion can resolve +/// `SELECT … FROM t`, and to attach each table's schema directly onto the +/// canonical `Scan` (`schema: Some(_)`) so the SchemaResolver doesn't need to +/// usage-derive it. +#[derive(Debug, Clone, Default)] +pub struct SqlCatalog { + pub tables: HashMap, +} + +impl SqlCatalog { + pub fn new() -> Self { + Self::default() + } + + /// Builder: register `name` with its resolved canonical schema. + pub fn with_table(mut self, name: impl Into, schema: Schema) -> Self { + self.tables.insert(name.into(), schema); + self + } +} + +pub(super) fn scalar_value_to_asap(sv: &DfScalarValue) -> Result { + match sv { + DfScalarValue::Int64(Some(v)) => Ok(ScalarValue::Int64(*v)), + DfScalarValue::Int32(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), + DfScalarValue::Int16(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), + DfScalarValue::Int8(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), + DfScalarValue::UInt64(Some(v)) => i64::try_from(*v).map(ScalarValue::Int64).map_err(|_| { + LoweringError::InvalidExpression(format!("UInt64 value {v} overflows i64")) + }), + DfScalarValue::UInt32(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), + DfScalarValue::Float64(Some(v)) => Ok(ScalarValue::Float64(*v)), + DfScalarValue::Float32(Some(v)) => Ok(ScalarValue::Float64(*v as f64)), + DfScalarValue::Utf8(Some(s)) | DfScalarValue::LargeUtf8(Some(s)) => { + Ok(ScalarValue::Utf8(s.clone())) + } + DfScalarValue::Boolean(Some(b)) => Ok(ScalarValue::Boolean(*b)), + // All three of DataFusion's interval scalars land on one canonical + // shape; the narrower two simply leave the fields they do not carry + // at zero. + DfScalarValue::IntervalYearMonth(Some(months)) => Ok(ScalarValue::Interval { + months: *months, + days: 0, + nanos: 0, + }), + DfScalarValue::IntervalDayTime(Some(v)) => Ok(ScalarValue::Interval { + months: 0, + days: v.days, + nanos: i64::from(v.milliseconds) * 1_000_000, + }), + DfScalarValue::IntervalMonthDayNano(Some(v)) => Ok(ScalarValue::Interval { + months: v.months, + days: v.days, + nanos: v.nanoseconds, + }), + _ if sv.is_null() => Ok(ScalarValue::Null), + _ => Err(LoweringError::InvalidExpression(format!( + "unsupported scalar: {sv:?}" + ))), + } +} + +/// Arrow → the canonical `DataType` (used for `CAST` targets). Deliberately narrow. +pub(super) fn arrow_to_dtype(dt: &ArrowDataType) -> Result { + match dt { + ArrowDataType::Null => Ok(DataType::Null), + ArrowDataType::Int64 + | ArrowDataType::Int32 + | ArrowDataType::Int16 + | ArrowDataType::Int8 => Ok(DataType::Int64), + ArrowDataType::Float64 | ArrowDataType::Float32 => Ok(DataType::Float64), + ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => Ok(DataType::Utf8), + ArrowDataType::Boolean => Ok(DataType::Bool), + ArrowDataType::Timestamp(_, _) => Ok(DataType::Timestamp), + ArrowDataType::Date32 | ArrowDataType::Date64 => Ok(DataType::Date), + ArrowDataType::Interval(_) => Ok(DataType::Interval), + ArrowDataType::List(element) => Ok(DataType::List { + element: Box::new(Field::new( + element.name(), + arrow_to_dtype(element.data_type())?, + element.is_nullable(), + )), + }), + ArrowDataType::Struct(fields) => Ok(DataType::Struct { + fields: fields + .iter() + .map(|field| { + Ok(Field::new( + field.name(), + arrow_to_dtype(field.data_type())?, + field.is_nullable(), + )) + }) + .collect::, LoweringError>>()?, + }), + ArrowDataType::Map(entries, _) => { + let ArrowDataType::Struct(fields) = entries.data_type() else { + return Err(LoweringError::UnsupportedFeature( + "map entries must be a struct".into(), + )); + }; + if fields.len() != 2 || fields[0].is_nullable() { + return Err(LoweringError::UnsupportedFeature( + "map entries require a non-null key and a value".into(), + )); + } + Ok(DataType::Map { + key: Box::new(arrow_to_dtype(fields[0].data_type())?), + value: Box::new(arrow_to_dtype(fields[1].data_type())?), + value_nullable: fields[1].is_nullable(), + }) + } + other => Err(LoweringError::UnsupportedFeature(format!( + "Arrow type: {other:?}" + ))), + } +} + +/// The canonical `DataType` → Arrow (for registering catalog tables with DataFusion). +pub(super) fn dtype_to_arrow(dt: &DataType) -> ArrowDataType { + match dt { + DataType::Null => ArrowDataType::Null, + DataType::Int64 => ArrowDataType::Int64, + DataType::Float64 => ArrowDataType::Float64, + DataType::Utf8 => ArrowDataType::Utf8, + DataType::Bool => ArrowDataType::Boolean, + DataType::List { element } => ArrowDataType::List(std::sync::Arc::new(ArrowField::new( + &element.name, + dtype_to_arrow(&element.dtype), + element.nullable, + ))), + DataType::Struct { fields } => ArrowDataType::Struct( + fields + .iter() + .map(|field| { + ArrowField::new(&field.name, dtype_to_arrow(&field.dtype), field.nullable) + }) + .collect::>() + .into(), + ), + DataType::Map { + key, + value, + value_nullable, + } => ArrowDataType::Map( + std::sync::Arc::new(ArrowField::new( + "entries", + ArrowDataType::Struct( + vec![ + ArrowField::new("key", dtype_to_arrow(key), false), + ArrowField::new("value", dtype_to_arrow(value), *value_nullable), + ] + .into(), + ), + false, + )), + false, + ), + DataType::Timestamp => { + ArrowDataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None) + } + // Deliberately narrowing: `Date64` lowers to `DataType::Date` and comes + // back as `Date32`. Both spell the same calendar date and nothing in + // the planner reads the width; a catalog that wants `Date64` back would + // need a second variant carrying no planning information. + DataType::Date => ArrowDataType::Date32, + // Only reachable through a hand-built schema: `Interval` types a + // literal, and no catalog declares a column with it. Mapped to the + // same three-field shape `ScalarValue::Interval` carries rather than + // left to panic. + DataType::Interval => { + ArrowDataType::Interval(datafusion::arrow::datatypes::IntervalUnit::MonthDayNano) + } + } +} + +/// Build an Arrow schema from a canonical [`Schema`] (column name + type + nullability). +pub(super) fn schema_to_arrow(schema: &Schema) -> ArrowSchema { + let fields: Fields = schema + .fields + .iter() + .map(|c: &Field| { + ArrowField::new(&c.name, dtype_to_arrow(c.expect_plain_dtype()), c.nullable) + }) + .collect(); + ArrowSchema::new(fields) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Both Arrow date widths bridge to the one canonical `Date`, and it + /// registers back as `Date32` — the documented narrowing. + #[test] + fn both_arrow_date_widths_bridge_to_date() { + assert_eq!( + arrow_to_dtype(&ArrowDataType::Date32).unwrap(), + DataType::Date + ); + assert_eq!( + arrow_to_dtype(&ArrowDataType::Date64).unwrap(), + DataType::Date + ); + assert_eq!(dtype_to_arrow(&DataType::Date), ArrowDataType::Date32); + } + + /// Every Arrow interval width shares the canonical calendar interval type. + #[test] + fn interval_types_round_trip_through_the_catalog_bridge() { + use datafusion::arrow::datatypes::IntervalUnit; + for unit in [ + IntervalUnit::YearMonth, + IntervalUnit::DayTime, + IntervalUnit::MonthDayNano, + ] { + assert_eq!( + arrow_to_dtype(&ArrowDataType::Interval(unit)).unwrap(), + DataType::Interval + ); + } + assert_eq!( + arrow_to_dtype(&dtype_to_arrow(&DataType::Interval)).unwrap(), + DataType::Interval + ); + } + + /// All three of DataFusion's interval scalars carry into the one canonical + /// three-field shape, with the fields they do not spell left at zero. + #[test] + fn every_datafusion_interval_scalar_carries_across() { + use datafusion::arrow::datatypes::{IntervalDayTime, IntervalMonthDayNano}; + + assert_eq!( + scalar_value_to_asap(&DfScalarValue::IntervalYearMonth(Some(14))).unwrap(), + ScalarValue::Interval { + months: 14, + days: 0, + nanos: 0 + } + ); + assert_eq!( + scalar_value_to_asap(&DfScalarValue::IntervalDayTime(Some(IntervalDayTime::new( + 30, 500 + )))) + .unwrap(), + ScalarValue::Interval { + months: 0, + days: 30, + nanos: 500_000_000 + } + ); + assert_eq!( + scalar_value_to_asap(&DfScalarValue::IntervalMonthDayNano(Some( + IntervalMonthDayNano::new(1, 2, 3) + ))) + .unwrap(), + ScalarValue::Interval { + months: 1, + days: 2, + nanos: 3 + } + ); + } + + /// Nested map values and value nullability survive catalog registration. + #[test] + fn nested_map_schema_round_trip() { + let map = DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::Map { + key: Box::new(DataType::Int64), + value: Box::new(DataType::Float64), + value_nullable: true, + }), + value_nullable: false, + }; + assert_eq!(arrow_to_dtype(&dtype_to_arrow(&map)).unwrap(), map); + let encoded = serde_json::to_string(&map).unwrap(); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), map); + } +} + +#[cfg(test)] +mod collection_tests { + use super::*; + #[test] + fn nested_collections_preserve_field_names_order_and_nullability() { + let dtype = DataType::Struct { + fields: vec![ + Field::new( + "samples", + DataType::List { + element: Box::new(Field::new( + "sample", + DataType::Struct { + fields: vec![ + Field::new("timestamp", DataType::Timestamp, false), + Field::new("value", DataType::Float64, true), + Field::new( + "labels", + DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::List { + element: Box::new(Field::new( + "label_value", + DataType::Utf8, + false, + )), + }), + value_nullable: true, + }, + true, + ), + ], + }, + true, + )), + }, + false, + ), + Field::new("optional", DataType::Int64, true), + ], + }; + let arrow = dtype_to_arrow(&dtype); + assert_eq!(arrow_to_dtype(&arrow).unwrap(), dtype); + assert_eq!(dtype_to_arrow(&arrow_to_dtype(&arrow).unwrap()), arrow); + let encoded = serde_json::to_string(&dtype).unwrap(); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), dtype); + } + #[test] + fn empty_struct_and_nonnullable_list_element_roundtrip() { + let dtype = DataType::List { + element: Box::new(Field::new( + "empty", + DataType::Struct { fields: vec![] }, + false, + )), + }; + assert_eq!(arrow_to_dtype(&dtype_to_arrow(&dtype)).unwrap(), dtype); + } +} + +#[cfg(test)] +mod bottom_map_tests { + use super::*; + #[test] + fn empty_map_bottom_types_roundtrip_without_string_defaults() { + let (map, nullable) = asap_types::pre_asap::scalar_type_rules::MapScalarFunction::Construct + .output_type(&[]) + .unwrap(); + assert!(!nullable); + let arrow = dtype_to_arrow(&map); + assert_eq!(arrow_to_dtype(&arrow).unwrap(), map); + assert_eq!( + arrow_to_dtype(&ArrowDataType::Null).unwrap(), + DataType::Null + ); + } +} diff --git a/crates/frontend-sql/tests/unified_sql_lowering.rs b/crates/frontend-sql/tests/unified_sql_lowering.rs new file mode 100644 index 00000000..2cc9e9ee --- /dev/null +++ b/crates/frontend-sql/tests/unified_sql_lowering.rs @@ -0,0 +1,2874 @@ +//! End-to-end SQL → unresolved → resolved operator DAG lowering tests. +//! +//! Validates the DataFusion front end: SQL parses + plans, lowers directly to +//! the name-based `UnresolvedOp` tree (issue #179), and the shared +//! `resolve_root` produces the positional, canonical `OperatorNode` DAG (the +//! same resolver the PromQL path uses). Every node's schema is derived during +//! resolution, so a successful `lower` already proves schema derivation is +//! total over the tree. + +use ::asap_frontend_sql::unified as asap_frontend_sql; +use asap_types::ir::Predicate; +use std::rc::Rc; + +use asap_frontend_common::{UnresolvedOp, UnresolvedScalar}; +use asap_frontend_sql::{ + lower_sql, lower_sql_dialect, SqlCatalog, SqlError as LoweringError, SqlLowerer, +}; +use asap_types::ir::{ExprSemantics, NonASAPOp, OperatorNode, ScalarExpr}; +use asap_types::pre_asap::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::pre_asap::{ + AggIntent, CompareOpKind, GroupKeys, JoinKind, Reduction, ScalarValue, Source, + WindowFrameBound, WindowFrameOffset, WindowFrameUnits, WindowFuncKind, +}; +use asap_types::types::AccuracyTarget; +use asap_types::workload::SqlDialect; + +fn col(name: &str, dtype: DataType) -> Field { + Field::plain(name, dtype, false) +} + +/// `metrics(ts, service, latency, bytes)` + `hosts(service, region)`. +fn catalog() -> SqlCatalog { + SqlCatalog::new() + .with_table( + "metrics", + Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("latency", DataType::Float64), + col("bytes", DataType::Int64), + ], + 0, + vec![vec![0, 1]], + ), + ) + .with_table( + "hosts", + Schema::new(vec![ + col("service", DataType::Utf8), + col("region", DataType::Utf8), + ]), + ) +} + +async fn lower(sql: &str) -> Rc { + lower_sql(sql, &catalog(), AccuracyTarget::Exact) + .await + .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) +} + +/// The operator of a front-end node: a front-end DAG never holds an ASAP node. +fn op(node: &OperatorNode) -> &NonASAPOp { + node.expect_non_asap() +} + +#[tokio::test] +async fn planning_subquery_bridge_rejects_a_relation_without_vector_conversion() { + let result = lower_sql("SELECT max(value) FROM (SELECT asap_promql_subquery(21600000, 60000) AS value FROM (SELECT sum(bytes) AS value FROM metrics))", &catalog(), AccuracyTarget::Exact).await; + assert!(result.is_err()); +} + +#[tokio::test] +async fn planning_histogram_bridge_reuses_classic_bucket_intent() { + let query = lower( + "SELECT asap_histogram_quantile(0.95) AS value FROM (\ + SELECT service AS le, sum(bytes) AS value FROM metrics GROUP BY service)", + ) + .await; + let NonASAPOp::Aggregate { + reduction, + measures, + child, + .. + } = op(&query) + else { + panic!("expected canonical histogram aggregate"); + }; + // One histogram over all rows; the bucket bound is the child's column 0. + assert!(reduction.expect_reduce().keys().is_empty()); + assert!(matches!( + measures.as_slice(), + [AggIntent::HistogramQuantile { q, le: 0 }] if (*q - 0.95).abs() < 1e-12 + )); + assert!(matches!(op(child), NonASAPOp::Project { .. })); +} + +#[tokio::test] +async fn planning_relation_bridges_reject_ambiguous_shapes() { + let missing_alias = lower_sql( + "SELECT asap_promql_subquery(300000, 60000) FROM metrics", + &catalog(), + AccuracyTarget::Exact, + ) + .await + .unwrap_err(); + assert!(missing_alias.to_string().contains("must have an alias")); + + let histogram_with_extra_column = lower_sql( + "SELECT service, asap_histogram_quantile(0.95) AS value FROM metrics", + &catalog(), + AccuracyTarget::Exact, + ) + .await + .unwrap_err(); + assert!(histogram_with_extra_column + .to_string() + .contains("only expression")); + + let invalid_q = lower_sql( + "SELECT asap_histogram_quantile(1.5) AS value FROM metrics", + &catalog(), + AccuracyTarget::Exact, + ) + .await + .unwrap_err(); + assert!(invalid_q.to_string().contains("finite and in [0,1]")); +} + +/// Find the first `Aggregate` node along the single-child spine. +fn find_aggregate(node: &OperatorNode) -> Option<(&GroupKeys, &Vec)> { + match op(node) { + NonASAPOp::Aggregate { + reduction, + measures, + .. + } => Some((reduction.expect_reduce(), measures)), + NonASAPOp::Project { child, .. } + | NonASAPOp::Filter { child, .. } + | NonASAPOp::Dedup { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } + | NonASAPOp::PromqlSubquery { child, .. } => find_aggregate(child), + _ => None, + } +} + +/// The first `Aggregate` node itself, for tests that need its child. +fn find_aggregate_node(node: &OperatorNode) -> Option<&OperatorNode> { + match op(node) { + NonASAPOp::Aggregate { .. } => Some(node), + NonASAPOp::Project { child, .. } + | NonASAPOp::Filter { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } => find_aggregate_node(child), + _ => None, + } +} + +/// The names of the columns the first `Aggregate`'s reducers read, resolved +/// against its child's schema, plus whether that child is a materializing +/// `Project` (issue #110). +fn reducer_input_names(node: &OperatorNode) -> (Vec, bool) { + let NonASAPOp::Aggregate { + measures, child, .. + } = op(find_aggregate_node(node).expect("expected an Aggregate")) + else { + unreachable!() + }; + let schema = &child.schema; + let names = measures + .iter() + .flat_map(|a| a.input_cols()) + .map(|id| schema.fields[id].name.clone()) + .collect(); + (names, matches!(op(child), NonASAPOp::Project { .. })) +} + +/// Find the first `Join` node along the single-child spine. +fn find_join(node: &OperatorNode) -> Option<&OperatorNode> { + match op(node) { + NonASAPOp::Join { .. } => Some(node), + NonASAPOp::Project { child, .. } + | NonASAPOp::Filter { child, .. } + | NonASAPOp::Aggregate { child, .. } + | NonASAPOp::Dedup { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } + | NonASAPOp::PromqlSubquery { child, .. } => find_join(child), + _ => None, + } +} + +/// The first `Filter` node along the single-child spine. +fn find_filter(node: &OperatorNode) -> Option<&OperatorNode> { + match op(node) { + NonASAPOp::Filter { .. } => Some(node), + NonASAPOp::Project { child, .. } + | NonASAPOp::Aggregate { child, .. } + | NonASAPOp::Dedup { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } + | NonASAPOp::PromqlSubquery { child, .. } => find_filter(child), + _ => None, + } +} + +#[tokio::test] +async fn select_star_with_where_folds_predicate_onto_scan() { + // SELECT * elides the projection; WHERE folds onto the Scan predicates. + let qe = lower("SELECT * FROM metrics WHERE service = 'api'").await; + let NonASAPOp::Scan { + source, + predicates, + schema, + } = op(&qe) + else { + panic!("expected Scan at root, got {qe:?}"); + }; + assert!(matches!(source, Source::Table { table_ref } if table_ref == "metrics")); + assert_eq!(predicates.len(), 1, "WHERE clause folded onto the scan"); + assert!( + schema.closed, + "a catalog-backed SQL scan has a closed schema" + ); +} + +#[tokio::test] +async fn multi_aggregate_group_by_binds_columns_positionally() { + // SUM(bytes)=col 3, AVG(latency)=col 2, GROUP BY service=col 1. + let qe = lower("SELECT service, SUM(bytes), AVG(latency) FROM metrics GROUP BY service").await; + let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate in the tree"); + assert_eq!(by, &vec![1], "GROUP BY service → column 1"); + assert!( + measures.contains(&AggIntent::Sum { col: Some(3) }), + "SUM(bytes) → Sum{{col:3}}, got {measures:?}" + ); + assert!( + measures.contains(&AggIntent::Avg { col: Some(2) }), + "AVG(latency) → Avg{{col:2}}, got {measures:?}" + ); +} + +#[tokio::test] +async fn projection_over_aggregate_resolves_output_types_via_output_names() { + // The enclosing Projection references the aggregates by DataFusion's + // generated names (e.g. "sum(metrics.bytes)"); output_names threads those + // onto the canonical Aggregate so the Project resolves real types — not + // the Utf8 fallback that an unresolved column would get. + let qe = lower("SELECT SUM(bytes), AVG(latency) FROM metrics").await; + let schema = &qe.schema; + assert_eq!(schema.fields.len(), 2); + assert_eq!( + schema.fields[0].dtype, + DataType::Int64, + "SUM(bytes:Int64) resolves to Int64, not the Utf8 fallback" + ); + assert_eq!( + schema.fields[1].dtype, + DataType::Float64, + "AVG(latency) resolves to Float64" + ); +} + +#[tokio::test] +async fn single_agg_group_by_keeps_key_in_output_schema() { + // A tabular single-aggregate GROUP BY routes through the positional + // Aggregate.by path (not the PromQL fused-Partition shape), so the group + // key is a real output column the enclosing SELECT projection resolves. + let qe = lower("SELECT service, SUM(bytes) FROM metrics GROUP BY service").await; + let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate (not a Partition)"); + assert_eq!(by, &vec![1], "GROUP BY service → Aggregate.by column 1"); + assert!(matches!( + measures.as_slice(), + [AggIntent::Sum { col: Some(3) }] + )); + + // Both the group key and the aggregate resolve in the root projection schema. + let schema = &qe.schema; + assert_eq!(schema.fields.len(), 2); + assert_eq!( + schema.fields[0].dtype, + DataType::Utf8, + "service is in the output" + ); + assert_eq!(schema.fields[1].dtype, DataType::Int64, "SUM(bytes)"); +} + +#[tokio::test] +async fn count_ranked_topk_is_heavy_hitter() { + // `ORDER BY COUNT(*) DESC LIMIT k` over a single COUNT aggregate is the one + // case the heavy-hitter (frequency) sketch is correct for. The shared + // `canonicalize` pass (issue #34) promotes it to the canonical two-level + // form: an outer global `TopK` (by: []) over the explicit inner `Count` + // grouped by `service`. + let qe = lower( + "SELECT service, COUNT(*) FROM metrics GROUP BY service ORDER BY COUNT(*) DESC LIMIT 10", + ) + .await; + let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!( + by.is_empty(), + "outer TopK is a global ranking (by: []), got {by:?}" + ); + assert!( + matches!(measures.as_slice(), [AggIntent::TopK { k: 10, .. }]), + "count-ranked topk → heavy-hitter TopK, got {measures:?}" + ); + // The inner child is the explicit Count, grouped by service (col 1). + let NonASAPOp::Aggregate { child, .. } = op(&qe) else { + panic!("expected outer Aggregate, got {qe:?}"); + }; + let (inner_by, inner_measures) = find_aggregate(child).expect("expected inner Count aggregate"); + assert_eq!(inner_by, &vec![1], "inner Count grouped by service → col 1"); + assert!( + matches!(inner_measures.as_slice(), [AggIntent::Count { .. }]), + "inner aggregate is the explicit Count, got {inner_measures:?}" + ); +} + +#[tokio::test] +async fn count_ranked_topk_via_alias_is_also_heavy_hitter() { + // Regression for #20: aliasing `COUNT(*)` in the ORDER BY used to defeat the + // SQL front-end gate. The positional `canonicalize` pass now promotes it too, + // so the aliased and inline forms produce an identical canonical tree. + let inline = lower( + "SELECT service, COUNT(*) FROM metrics GROUP BY service ORDER BY COUNT(*) DESC LIMIT 10", + ) + .await; + let aliased = lower( + "SELECT service, COUNT(*) AS cnt FROM metrics GROUP BY service ORDER BY cnt DESC LIMIT 10", + ) + .await; + assert_eq!( + inline, aliased, + "aliased count-ranked topk must match the inline form" + ); + let (_, measures) = find_aggregate(&aliased).expect("expected an Aggregate"); + assert!( + matches!(measures.as_slice(), [AggIntent::TopK { k: 10, .. }]), + "aliased count-ranked topk → heavy-hitter TopK, got {measures:?}" + ); +} + +#[tokio::test] +async fn non_count_ranked_limit_keeps_the_aggregate() { + // Ranking by AVG (not a count) must NOT become a frequency heavy-hitter — + // the AVG aggregate has to survive as a generic Sort+Limit. + let qe = lower( + "SELECT service, AVG(latency) AS a FROM metrics GROUP BY service ORDER BY a DESC LIMIT 10", + ) + .await; + let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!( + measures.iter().any(|a| matches!(a, AggIntent::Avg { .. })), + "AVG must be preserved, got {measures:?}" + ); + assert!( + !measures.iter().any(|a| matches!(a, AggIntent::TopK { .. })), + "AVG ranking must not become a frequency heavy-hitter, got {measures:?}" + ); +} + +#[tokio::test] +async fn distinct_value_reducer_is_rejected_not_dropped() { + // The canonical intent algebra has no distinct-Sum; SUM(DISTINCT x) must + // be rejected, not silently lowered as SUM(x). + let res = lower_sql( + "SELECT SUM(DISTINCT bytes) FROM metrics", + &catalog(), + AccuracyTarget::Exact, + ) + .await; + assert!(res.is_err(), "SUM(DISTINCT ...) should be rejected"); +} + +#[tokio::test] +async fn aggregate_over_an_expression_reduces_a_derived_column() { + // The canonical `AggIntent` reduces a column, not an arbitrary expression. + // `SUM(bytes + 1)` used to be rejected for that reason; since #110 the + // expression is materialized as a derived column in a `Project` beneath + // the aggregate, and reduced there. + let qe = lower("SELECT SUM(bytes + 1) FROM metrics").await; + let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!( + matches!(measures.as_slice(), [AggIntent::Sum { col: Some(_) }]), + "expected Sum bound to the derived column, got {measures:?}" + ); + let (names, materialized) = reducer_input_names(&qe); + assert!(materialized, "expected a materializing Project"); + assert!( + names[0].contains("bytes") && names[0].contains('1'), + "the reduced column should be the projected `bytes + 1`, got {names:?}" + ); +} + +#[tokio::test] +async fn count_star_is_count_intent() { + let qe = lower("SELECT COUNT(*) FROM metrics").await; + let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!(by.is_empty()); + assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); +} + +#[tokio::test] +async fn count_distinct_is_cardinality() { + let qe = lower("SELECT COUNT(DISTINCT service) FROM metrics").await; + let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!(matches!( + measures.as_slice(), + [AggIntent::Cardinality { .. }] + )); +} + +#[tokio::test] +async fn select_distinct_lowers_to_distinct_with_positional_cols() { + // SELECT DISTINCT → a `Dedup` node whose `cols` are positional ColumnIds + // (not name-based ColumnRefs). DataFusion's `Distinct::All` dedups on every + // column, so `cols` is empty here — but the field type is now `Vec`. + let qe = lower("SELECT DISTINCT service FROM metrics").await; + let NonASAPOp::Dedup { cols, .. } = op(&qe) else { + panic!("expected a Dedup at the root, got {qe:?}"); + }; + let _: &Vec = cols; // compile-time: positional ids, not ColumnRefs + assert!(cols.is_empty(), "DISTINCT * dedups on all columns"); +} + +#[tokio::test] +async fn inner_join_lowers_to_join_over_two_scans() { + // INNER JOIN over two distinct tables → a canonical Join with both leaves as Scans. + let qe = lower( + "SELECT metrics.bytes, hosts.region \ + FROM metrics JOIN hosts ON metrics.service = hosts.service", + ) + .await; + let join = find_join(&qe).expect("expected a Join in the tree"); + let NonASAPOp::Join { + kind, left, right, .. + } = op(join) + else { + unreachable!("find_join only returns Join"); + }; + assert_eq!(*kind, JoinKind::Inner); + assert!(matches!(op(left), NonASAPOp::Scan { .. })); + assert!(matches!(op(right), NonASAPOp::Scan { .. })); +} + +/// The two `ColumnId`s an equijoin predicate `Column(l) = Column(r)` binds to, +/// returned sorted so the assertion is independent of left/right ordering. +fn join_eq_columns(join: &OperatorNode) -> [usize; 2] { + let NonASAPOp::Join { pred, .. } = op(join) else { + unreachable!("expected a Join"); + }; + let ScalarExpr::Compare { + left, + op: CompareOpKind::Eq, + right, + .. + } = &pred.0 + else { + panic!("expected an equijoin Compare, got {:?}", pred.0); + }; + match (left.as_ref(), right.as_ref()) { + (ScalarExpr::Column(l), ScalarExpr::Column(r)) => { + let mut cols = [*l, *r]; + cols.sort_unstable(); + cols + } + other => panic!("expected Field = Field, got {other:?}"), + } +} + +#[tokio::test] +async fn join_predicate_disambiguates_shared_column_name() { + // Issue #7: `metrics.service = hosts.service` shares a column name across the + // join. The qualified refs must bind to two *distinct* positions in the + // concatenated schema, not collapse onto the first `service`. + // metrics(ts,service,latency,bytes) ++ hosts(service,region) + // → metrics.service = col 1, hosts.service = col 4. + let qe = lower( + "SELECT metrics.bytes, hosts.region \ + FROM metrics JOIN hosts ON metrics.service = hosts.service", + ) + .await; + let join = find_join(&qe).expect("expected a Join in the tree"); + assert_eq!( + join_eq_columns(join), + [1, 4], + "join key must bind to distinct positions, not the same `service`" + ); +} + +#[tokio::test] +async fn derived_table_join_disambiguates_via_alias() { + // Issue #66: a join over two *derived tables* must bind its keys to distinct + // positions. Before the fix the derived output columns lost their qualifier, + // so `a.service` and `b.service` both fell back to the first bare `service` + // (col 0) — `service = service`, always true → a silent cross product. + // Concatenated: a[service,region] ++ b[service,region] → a.service=0, b.service=2. + let qe = lower( + "SELECT a.region, b.region \ + FROM (SELECT service, region FROM hosts) a \ + JOIN (SELECT service, region FROM hosts) b ON a.service = b.service", + ) + .await; + let join = find_join(&qe).expect("expected a Join in the tree"); + assert_eq!( + join_eq_columns(join), + [0, 2], + "derived-table join keys must bind to distinct positions, not both to the first `service`" + ); +} + +#[tokio::test] +async fn derived_table_select_star_join_disambiguates_via_alias() { + // Same as above but `SELECT *` derived tables (the non-Projection path that + // wraps the inner plan in an identity re-qualifying projection). + let qe = lower( + "SELECT a.region, b.region \ + FROM (SELECT * FROM hosts) a JOIN (SELECT * FROM hosts) b \ + ON a.service = b.service", + ) + .await; + let join = find_join(&qe).expect("expected a Join in the tree"); + let [l, r] = join_eq_columns(join); + assert_ne!( + l, r, + "SELECT * derived-table join keys must not collapse to one column" + ); +} + +#[tokio::test] +async fn self_join_disambiguates_via_aliases() { + // A self-join shares *every* column name; the alias qualifiers (`a`/`b`) are + // the only way to tell the two `service` columns apart. + // metrics ++ metrics → a.service = col 1, b.service = col 5 (4 cols/side). + let qe = lower( + "SELECT a.bytes, b.latency \ + FROM metrics a JOIN metrics b ON a.service = b.service", + ) + .await; + let join = find_join(&qe).expect("expected a self-Join in the tree"); + assert_eq!( + join_eq_columns(join), + [1, 5], + "self-join keys must bind to distinct sides" + ); +} + +#[tokio::test] +async fn qualified_where_over_join_resolves_to_right_side() { + // Issue #7 beyond the join key: a WHERE on the *duplicated* column name + // (`service` exists on both sides) must bind to the qualified side, not the + // first match. metrics.service = col 1, hosts.service = col 4 → `hosts.service` + // must resolve to 4. (Unoptimized plan keeps the Filter above the Join — no + // predicate pushdown — so it binds against the concatenated schema.) + let qe = lower( + "SELECT metrics.bytes FROM metrics JOIN hosts ON metrics.service = hosts.service \ + WHERE hosts.service = 'api'", + ) + .await; + let filter = find_filter(&qe).expect("expected a Filter over the join"); + let NonASAPOp::Filter { pred, .. } = op(filter) else { + unreachable!("find_filter only returns Filter"); + }; + assert!( + matches!(&pred.0, ScalarExpr::Compare { left, op: CompareOpKind::Eq, .. } + if matches!(left.as_ref(), ScalarExpr::Column(4))), + "hosts.service must bind to concatenated position 4 (not the first `service`), got {:?}", + pred.0 + ); +} + +#[tokio::test] +async fn self_join_group_by_disambiguates_via_qualifier() { + // Group-key qualifier fix: GROUP BY on the *duplicated* column over a + // self-join must bind to the qualified side, not first-match. metrics ⋈ + // metrics → a.service = col 1, b.service = col 5. (Without qualified keys, + // both `GROUP BY a.service` and `GROUP BY b.service` collapsed to col 1.) + let qe_b = lower( + "SELECT b.service, COUNT(*) FROM metrics a JOIN metrics b \ + ON a.service = b.service GROUP BY b.service", + ) + .await; + let (by, _) = find_aggregate(&qe_b).expect("expected an Aggregate over the self-join"); + assert_eq!( + by, + &vec![5], + "GROUP BY b.service binds to the b side (col 5)" + ); + + let qe_a = lower( + "SELECT a.service, COUNT(*) FROM metrics a JOIN metrics b \ + ON a.service = b.service GROUP BY a.service", + ) + .await; + let (by, _) = find_aggregate(&qe_a).expect("expected an Aggregate over the self-join"); + assert_eq!( + by, + &vec![1], + "GROUP BY a.service binds to the a side (col 1)" + ); +} + +#[tokio::test] +async fn aggregate_over_join_binds_against_concatenated_schema() { + // GROUP BY a right-table column over a join: the key must resolve against + // the concatenated schema, exercising the bottom-up converter end to end. + // Two aggregates → the multi-agg path, which carries GROUP BY keys as + // positional `Aggregate.by` (as does every reducing GROUP BY). + let qe = lower( + "SELECT hosts.region, SUM(metrics.bytes), COUNT(*) \ + FROM metrics JOIN hosts ON metrics.service = hosts.service \ + GROUP BY hosts.region", + ) + .await; + let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate over the join"); + // metrics(ts,service,latency,bytes) ++ hosts(service,region) → + // region is column 5, bytes is column 3 of the concatenated schema. + assert_eq!( + by, + &vec![5], + "GROUP BY hosts.region → concatenated column 5" + ); + assert!( + measures.contains(&AggIntent::Sum { col: Some(3) }), + "SUM(metrics.bytes) → Sum{{col:3}}, got {measures:?}" + ); +} + +// ── Issue #111: IN / EXISTS subquery predicates become semi / anti joins ──── +// +// The front end now leaves them as `UnresolvedScalar::{InSubquery, Exists}` +// filter conjuncts; the shared `canonicalize` pass (run by `resolve_root`) +// lowers each to the semi-/anti-join, so the resolved DAG a test sees is the +// same join shape the front end used to emit directly. + +/// The first `Join` node's `(kind, predicate, left column count)`. +fn join_parts(node: &OperatorNode) -> (&JoinKind, &ScalarExpr, usize) { + let NonASAPOp::Join { + kind, + pred, + left, + right: _, + } = op(find_join(node).expect("expected a Join")) + else { + unreachable!() + }; + (kind, &pred.0, left.schema.fields.len()) +} + +#[tokio::test] +async fn in_subquery_lowers_to_a_semi_join() { + // `metrics(ts, service, latency, bytes)` — service is column 1. + let qe = + lower("SELECT service FROM metrics WHERE service IN (SELECT service FROM hosts)").await; + let (kind, pred, left_len) = join_parts(&qe); + assert_eq!(kind, &JoinKind::Semi); + assert_eq!(left_len, 4); + + // The predicate resolves against `left ++ right`. Both relations have a + // `service` column, so a name-based lookup would bind *both* sides to the + // left's — silently making this `service = service`, always true. The key is + // bound positionally to the subquery's column (right after the left's), + // which makes that impossible. + let ScalarExpr::Compare { left, right, .. } = pred else { + panic!("expected a comparison, got {pred:?}"); + }; + assert_eq!(**left, ScalarExpr::Column(1), "outer service"); + assert_eq!( + **right, + ScalarExpr::Column(left_len), + "the subquery key, not the outer column again" + ); +} + +#[tokio::test] +async fn a_semi_join_outputs_only_the_left_schema() { + // The right side is a filter, not a source of columns. + let qe = + lower("SELECT service FROM metrics WHERE service IN (SELECT service FROM hosts)").await; + let join = find_join(&qe).expect("expected a Join"); + let names: Vec<_> = join.schema.fields.iter().map(|c| c.name.clone()).collect(); + assert_eq!(names, ["ts", "service", "latency", "bytes"]); +} + +#[tokio::test] +async fn a_subquery_key_that_is_an_expression_still_binds() { + // `SELECT bytes + 1 …` has no column name of its own; the join key binds + // to it positionally rather than through an unreferenceable `col_0`. + let qe = + lower("SELECT service FROM metrics WHERE bytes IN (SELECT bytes + 1 FROM metrics)").await; + assert_eq!(join_parts(&qe).0, &JoinKind::Semi); +} + +#[tokio::test] +async fn a_multi_column_in_subquery_is_rejected() { + let err = lower_sql( + "SELECT service FROM metrics WHERE service IN (SELECT service, region FROM hosts)", + &catalog(), + AccuracyTarget::Exact, + ) + .await + .expect_err("IN must select one column"); + assert!(format!("{err}").contains("exactly one column"), "got {err}"); +} + +#[tokio::test] +async fn an_ordinary_conjunct_still_folds_onto_the_scan() { + // The residual filter stays *below* the semi-join, where the converter can + // still fold it onto the Scan. A semi-join only drops left rows, so the + // orders agree. + let qe = lower( + "SELECT service FROM metrics WHERE bytes > 10 \ + AND service IN (SELECT service FROM hosts)", + ) + .await; + fn scan_has_predicate(node: &OperatorNode) -> bool { + match op(node) { + NonASAPOp::Scan { predicates, .. } => !predicates.is_empty(), + NonASAPOp::Project { child, .. } + | NonASAPOp::Filter { child, .. } + | NonASAPOp::Aggregate { child, .. } => scan_has_predicate(child), + NonASAPOp::Join { left, right, .. } => { + scan_has_predicate(left) || scan_has_predicate(right) + } + _ => false, + } + } + assert_eq!(join_parts(&qe).0, &JoinKind::Semi); + assert!( + scan_has_predicate(&qe), + "WHERE bytes > 10 should reach the Scan" + ); +} + +/// Find the first `SQLWindowFunc` node along the single-child spine. +fn find_windowfunc(node: &OperatorNode) -> Option<&OperatorNode> { + match op(node) { + NonASAPOp::SQLWindowFunc { .. } => Some(node), + NonASAPOp::Project { child, .. } + | NonASAPOp::Filter { child, .. } + | NonASAPOp::Aggregate { child, .. } + | NonASAPOp::Dedup { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } + | NonASAPOp::PromqlSubquery { child, .. } => find_windowfunc(child), + _ => None, + } +} + +#[tokio::test] +async fn window_function_lowers_to_positional_windowfunc() { + // ROW_NUMBER() OVER (PARTITION BY service ORDER BY bytes DESC). + let qe = lower( + "SELECT service, ROW_NUMBER() OVER (PARTITION BY service ORDER BY bytes DESC) \ + FROM metrics", + ) + .await; + let win = find_windowfunc(&qe).expect("expected a SQLWindowFunc node"); + let NonASAPOp::SQLWindowFunc { + func, + partition_by, + order_by, + .. + } = op(win) + else { + unreachable!("find_windowfunc only returns SQLWindowFunc"); + }; + assert_eq!(*func, WindowFuncKind::RowNumber); + assert_eq!(partition_by, &vec![1], "PARTITION BY service → col 1"); + assert_eq!(order_by.len(), 1); + assert_eq!( + order_by[0].expr, + ScalarExpr::Column(3), + "ORDER BY bytes → col 3" + ); + assert!(!order_by[0].ascending, "DESC"); + + // The window output column is appended to the schema (Int64 for ROW_NUMBER), + // and the enclosing projection resolves it (output_name threading). + let schema = &qe.schema; + assert!( + schema.fields.iter().any(|c| c.dtype == DataType::Int64), + "row_number output column present, got {:?}", + schema.fields + ); +} + +#[tokio::test] +async fn window_aggregate_lowers_to_windowfunc() { + let qe = lower("SELECT service, SUM(bytes) OVER (PARTITION BY service) FROM metrics").await; + let win = find_windowfunc(&qe).expect("expected a SQLWindowFunc node"); + let NonASAPOp::SQLWindowFunc { func, args, .. } = op(win) else { + unreachable!(); + }; + assert_eq!(*func, WindowFuncKind::Sum); + assert_eq!(args, &vec![ScalarExpr::Column(3)], "SUM(bytes) → arg col 3"); +} + +// ── Window frames (issue #268) ─────────────────────────────────────────────── + +/// The frame clause must actually reach the IR, not just the display string: +/// three window frames that differ semantically must lower to different +/// `SQLWindowFunc.frame` values. +#[tokio::test] +async fn window_frame_is_captured_not_dropped() { + let default_frame = + lower("SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts) FROM metrics") + .await; + let two_preceding = lower( + "SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts \ + ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) FROM metrics", + ) + .await; + let unbounded_following = lower( + "SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts \ + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM metrics", + ) + .await; + + let frame_of = |node: &OperatorNode| { + let NonASAPOp::SQLWindowFunc { frame, .. } = op(find_windowfunc(node).unwrap()) else { + unreachable!(); + }; + frame + .clone() + .expect("newly lowered SQL always records a frame") + }; + let (a, b, c) = ( + frame_of(&default_frame), + frame_of(&two_preceding), + frame_of(&unbounded_following), + ); + assert_ne!(a, b, "default frame vs ROWS 2 PRECEDING must differ"); + assert_ne!( + a, c, + "default frame vs ROWS CURRENT..UNBOUNDED FOLLOWING must differ" + ); + assert_ne!(b, c); + + assert_eq!(b.units, WindowFrameUnits::Rows); + assert_eq!( + b.start_bound, + WindowFrameBound::Preceding(WindowFrameOffset::Scalar(ScalarValue::Int64(2))) + ); + assert_eq!(b.end_bound, WindowFrameBound::CurrentRow); + + assert_eq!(c.start_bound, WindowFrameBound::CurrentRow); + assert_eq!( + c.end_bound, + WindowFrameBound::Following(WindowFrameOffset::Scalar(ScalarValue::Null)) + ); +} + +#[tokio::test] +async fn range_interval_frame_is_preserved() { + let qe = lower( + "SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts \ + RANGE BETWEEN INTERVAL '1' HOUR PRECEDING AND CURRENT ROW) FROM metrics", + ) + .await; + let NonASAPOp::SQLWindowFunc { + frame: Some(frame), .. + } = op(find_windowfunc(&qe).unwrap()) + else { + panic!("expected a window function with a concrete frame"); + }; + + assert_eq!(frame.units, WindowFrameUnits::Range); + assert_eq!( + frame.start_bound, + WindowFrameBound::Preceding(WindowFrameOffset::Interval { + months: 0, + days: 0, + nanoseconds: 3_600_000_000_000, + }) + ); + assert_eq!(frame.end_bound, WindowFrameBound::CurrentRow); +} + +#[tokio::test] +async fn range_numeric_frames_remain_scalar_offsets() { + let integer = lower( + "SELECT SUM(bytes) OVER (ORDER BY bytes \ + RANGE BETWEEN 2 PRECEDING AND CURRENT ROW) FROM metrics", + ) + .await; + let fractional = lower( + "SELECT SUM(latency) OVER (ORDER BY latency \ + RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW) FROM metrics", + ) + .await; + + let start_bound = |node: &OperatorNode| { + let NonASAPOp::SQLWindowFunc { + frame: Some(frame), .. + } = op(find_windowfunc(node).unwrap()) + else { + panic!("expected a window function with a concrete frame"); + }; + frame.start_bound.clone() + }; + + assert_eq!( + start_bound(&integer), + WindowFrameBound::Preceding(WindowFrameOffset::Scalar(ScalarValue::Int64(2))) + ); + assert_eq!( + start_bound(&fractional), + WindowFrameBound::Preceding(WindowFrameOffset::Scalar(ScalarValue::Float64(1.5))) + ); +} + +/// `GROUPS` frames aren't in this repo's SQL corpora and nothing downstream +/// interprets frame semantics yet — rejected explicitly rather than silently +/// mis-lowered. +#[tokio::test] +async fn groups_frame_is_rejected() { + let err = lower_sql( + "SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts \ + GROUPS BETWEEN 2 PRECEDING AND CURRENT ROW) FROM metrics", + &catalog(), + AccuracyTarget::Exact, + ) + .await + .expect_err("GROUPS frame unit must be rejected"); + assert!(format!("{err}").contains("GROUPS"), "got {err}"); +} + +// ── Nested query functions: derived tables / inline views (issue #27) ─────────── + +/// Collect every `AggIntent` in the DAG, root-to-leaf (every reachable node, +/// including operators referenced from scalar positions). +fn all_intents(root: &Rc) -> Vec { + OperatorNode::reachable(root) + .iter() + .filter_map(|node| match op(node) { + NonASAPOp::Aggregate { measures, .. } => Some(measures.clone()), + _ => None, + }) + .flatten() + .collect() +} + +#[tokio::test] +async fn derived_table_aggregate_over_aggregate_nests() { + // `MAX(s)` over a derived table `(SELECT service, SUM(bytes) AS s … GROUP BY + // service)` — the SQL counterpart of PromQL function nesting (issue #27). + // Both reductions survive into the canonical tree: an outer `Max` over + // the inner `Sum`. + let qe = lower( + "SELECT MAX(s) FROM \ + (SELECT service, SUM(bytes) AS s FROM metrics GROUP BY service) t", + ) + .await; + let intents = all_intents(&qe); + assert!( + intents.iter().any(|i| matches!(i, AggIntent::Max { .. })), + "outer MAX survives, got {intents:?}" + ); + assert!( + intents.iter().any(|i| matches!(i, AggIntent::Sum { .. })), + "inner SUM survives, got {intents:?}" + ); + // The whole nested tree's output schema derives (positional resolution + // is total across the derived-table boundary). + assert_eq!(qe.schema.fields.len(), 1); +} + +#[tokio::test] +async fn derived_table_outer_avg_over_inner_percentile() { + // Outer exact `AVG` over an inner approximate `Quantile` — each layer keeps + // its own intent (the per-node sketch-vs-exact choice is a post-ASAP decision). + let qe = lower( + "SELECT AVG(p) FROM \ + (SELECT service, approx_percentile_cont(latency, 0.9) AS p \ + FROM metrics GROUP BY service) t", + ) + .await; + let intents = all_intents(&qe); + assert!(intents.iter().any(|i| matches!(i, AggIntent::Avg { .. }))); + assert!(intents + .iter() + .any(|i| matches!(i, AggIntent::Quantile { q, .. } if (*q - 0.9).abs() < 1e-9))); +} + +#[tokio::test] +async fn filter_over_derived_aggregate_resolves_alias_column() { + // `WHERE t.s > 100` over a derived aggregate — the qualified ref `t.s` + // resolves by bare name against the derived output schema, and the Filter + // sits above the inner Aggregate. + let qe = lower( + "SELECT t.service, t.s FROM \ + (SELECT service, SUM(bytes) AS s FROM metrics GROUP BY service) t \ + WHERE t.s > 100", + ) + .await; + assert!( + find_filter(&qe).is_some(), + "the outer WHERE lowers to a Filter, got {qe:?}" + ); + assert!(all_intents(&qe) + .iter() + .any(|i| matches!(i, AggIntent::Sum { .. }))); + // Schema derivation is total across the boundary: the root carries one. + assert_eq!(qe.schema.fields.len(), 2); +} + +#[tokio::test] +async fn scalar_subquery_in_predicate_lowers_through_a_cross_join() { + let qe = + lower("SELECT service FROM metrics WHERE bytes > (SELECT AVG(bytes) FROM metrics)").await; + let filter = find_filter(&qe).unwrap(); + let NonASAPOp::Filter { pred, child } = op(filter) else { + panic!() + }; + assert!(matches!(op(child), NonASAPOp::Scan { .. })); + assert!( + matches!(&pred.0,ScalarExpr::Compare { right,.. } if matches!(right.as_ref(),ScalarExpr::ScalarSubquery(_))) + ); + qe.validate_structure().unwrap(); +} + +#[tokio::test] +async fn correlated_exists_lifts_its_correlation_into_the_join() { + // `EXISTS (SELECT 1 FROM hosts h WHERE h.service = m.service)` → a semi-join + // on `h.service = m.service`. The `SELECT 1` projection is dropped: a + // semi-join keeps no right columns, and it would have projected away the + // very column the correlation needs. + let qe = lower( + "SELECT service FROM metrics m WHERE EXISTS \ + (SELECT 1 FROM hosts h WHERE h.service = m.service)", + ) + .await; + let (kind, pred, left_len) = join_parts(&qe); + assert_eq!(kind, &JoinKind::Semi); + let ScalarExpr::Compare { left, right, .. } = pred else { + panic!("expected the correlation as a comparison, got {pred:?}"); + }; + assert_eq!( + **left, + ScalarExpr::Column(left_len), + "h.service (right side)" + ); + assert_eq!(**right, ScalarExpr::Column(1), "m.service (left side)"); +} + +#[tokio::test] +async fn not_exists_lowers_to_an_anti_join() { + let qe = lower( + "SELECT service FROM metrics m WHERE NOT EXISTS \ + (SELECT 1 FROM hosts h WHERE h.service = m.service)", + ) + .await; + assert_eq!(join_parts(&qe).0, &JoinKind::Anti); +} + +#[tokio::test] +async fn an_uncorrelated_exists_is_an_unconditional_semi_join() { + // No correlation → keep every left row iff the right side has any row. + let qe = lower("SELECT service FROM metrics WHERE EXISTS (SELECT 1 FROM hosts)").await; + let (kind, pred, _) = join_parts(&qe); + assert_eq!(kind, &JoinKind::Semi); + assert_eq!(*pred, ScalarExpr::Literal(ScalarValue::Boolean(true))); +} + +#[tokio::test] +async fn where_exists_resolves_to_a_semi_join_over_the_subquery() { + // The front end emits `Filter { Exists(s) }`; the resolved DAG is the + // `Semi` join with the subquery (a filtered `hosts` scan) on the right. + let qe = lower( + "SELECT service FROM metrics WHERE EXISTS (SELECT service FROM hosts WHERE region = 'eu')", + ) + .await; + let NonASAPOp::Project { child, .. } = op(&qe) else { + panic!("expected the SELECT list as a Project, got {qe:?}"); + }; + let NonASAPOp::Join { + kind, + pred, + left, + right, + } = op(child) + else { + panic!("expected the Semi join directly under the Project, got {child:?}"); + }; + assert_eq!(*kind, JoinKind::Semi); + assert_eq!(pred.0, ScalarExpr::Literal(ScalarValue::Boolean(true))); + assert!( + matches!(op(left), NonASAPOp::Scan { .. }), + "left is metrics" + ); + let NonASAPOp::Project { child: scan, .. } = op(right) else { + panic!("expected the subquery's projection on the right, got {right:?}"); + }; + assert!( + matches!(op(scan), NonASAPOp::Scan { predicates, .. } if predicates.len() == 1), + "the subquery's WHERE stays on its own Scan, got {scan:?}" + ); + assert_eq!( + child.schema.fields.len(), + 4, + "a semi join outputs the left's columns alone" + ); +} + +#[tokio::test] +async fn not_in_subquery_is_rejected_rather_than_mislowered_as_an_anti_join() { + let qe = + lower("SELECT service FROM metrics WHERE service NOT IN (SELECT service FROM hosts)").await; + let filter = find_filter(&qe).unwrap(); + let NonASAPOp::Filter { pred, .. } = op(filter) else { + panic!() + }; + assert!(matches!( + pred.0, + ScalarExpr::InSubquery { negated: true, .. } + )); + qe.validate_structure().unwrap(); +} + +#[tokio::test] +async fn a_correlated_in_subquery_is_rejected() { + let err = lower_sql( + "SELECT service FROM metrics m WHERE service IN \ + (SELECT h.service FROM hosts h WHERE h.region = m.service)", + &catalog(), + AccuracyTarget::Exact, + ) + .await + .expect_err("correlated IN needs both a key match and a correlation"); + assert!(format!("{err}").contains("correlated IN"), "got {err}"); +} + +// ── Subquery-valued expressions at the `UnresolvedOp` level ───────────────── + +/// `SqlLowerer::lower` output, before `resolve_root`. +async fn lower_unresolved(sql: &str) -> UnresolvedOp { + let catalog = catalog(); + SqlLowerer::new(&catalog) + .lower(sql, &AccuracyTarget::Exact) + .await + .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) +} + +#[tokio::test] +async fn scalar_subquery_in_projection_lowers_to_a_scalar_subquery_item() { + // An uncorrelated `(SELECT max(v) FROM t2)` in the SELECT list is a + // `ScalarSubquery` projection item reading its own lowered plan; the + // cross-join rewrite is `canonicalize`'s job, not the front end's. + let tree = lower_unresolved("SELECT (SELECT max(latency) FROM metrics) FROM hosts").await; + let UnresolvedOp::Project { cols, child, .. } = &tree else { + panic!("expected the SELECT list as a Project, got {tree:?}"); + }; + assert!( + matches!(child.as_ref(), UnresolvedOp::Scan { source: Source::Table { table_ref }, .. } + if table_ref == "hosts"), + "the outer relation stays the projection's child, got {child:?}" + ); + assert_eq!(cols.len(), 1); + let UnresolvedScalar::ScalarSubquery(sub) = &cols[0].expr else { + panic!("expected a ScalarSubquery item, got {:?}", cols[0].expr); + }; + let UnresolvedOp::Project { child: inner, .. } = sub.as_ref() else { + panic!("expected the subquery's own SELECT list, got {sub:?}"); + }; + assert!( + matches!(inner.as_ref(), UnresolvedOp::Aggregate { measures, .. } + if matches!(measures.as_slice(), [AggIntent::Max { .. }])), + "the subquery plan is lowered as a root of its own, got {inner:?}" + ); +} + +#[tokio::test] +async fn exists_and_in_subqueries_lower_to_scalar_filter_conjuncts() { + // The front end no longer builds the semi join itself: `EXISTS` / `IN + // (…)` are `Filter` predicates reading the subquery operator. + let tree = + lower_unresolved("SELECT service FROM metrics WHERE EXISTS (SELECT 1 FROM hosts)").await; + let UnresolvedOp::Project { child, .. } = &tree else { + panic!("expected a Project, got {tree:?}"); + }; + assert!( + matches!(child.as_ref(), UnresolvedOp::Filter { pred, .. } + if matches!(pred.0, UnresolvedScalar::Exists { negated: false, .. })), + "expected Filter {{ Exists }}, got {child:?}" + ); + + let tree = lower_unresolved( + "SELECT service FROM metrics WHERE service IN (SELECT service FROM hosts)", + ) + .await; + let UnresolvedOp::Project { child, .. } = &tree else { + panic!("expected a Project, got {tree:?}"); + }; + assert!( + matches!(child.as_ref(), UnresolvedOp::Filter { pred, .. } + if matches!(pred.0, UnresolvedScalar::InSubquery { negated: false, .. })), + "expected Filter {{ InSubquery }}, got {child:?}" + ); +} + +// ── `SELECT` without `FROM`, unary minus, SQL expression semantics ────────── + +#[tokio::test] +async fn select_without_from_projects_over_one_empty_row() { + // `SELECT 1` has no table: DataFusion's `EmptyRelation` is one empty + // input row, which the SELECT list projects a literal over. + let qe = lower("SELECT 1").await; + let NonASAPOp::Project { cols, child, .. } = op(&qe) else { + panic!("expected Project at root, got {qe:?}"); + }; + assert_eq!(cols.len(), 1); + assert_eq!(cols[0].expr, ScalarExpr::Literal(ScalarValue::Int64(1))); + let NonASAPOp::Values { rows, schema } = op(child) else { + panic!("expected Values under the Project, got {child:?}"); + }; + assert_eq!(rows, &vec![Vec::::new()], "one empty row"); + assert!(schema.fields.is_empty() && schema.closed); + assert_eq!(qe.schema.fields.len(), 1); + assert_eq!(qe.schema.fields[0].dtype, DataType::Int64); +} + +#[tokio::test] +async fn values_lowers_to_one_row_per_values_row() { + let qe = lower("SELECT * FROM (VALUES (1, 'a'), (2, 'b')) AS v(n, s)").await; + let values = OperatorNode::reachable(&qe) + .into_iter() + .find(|n| matches!(op(n), NonASAPOp::Values { .. })) + .expect("expected a Values node"); + let NonASAPOp::Values { rows, schema } = op(&values) else { + unreachable!() + }; + assert_eq!(rows.len(), 2); + assert_eq!( + rows[1], + vec![ + ScalarExpr::Literal(ScalarValue::Int64(2)), + ScalarExpr::Literal(ScalarValue::Utf8("b".into())), + ] + ); + assert_eq!(schema.fields.len(), 2); + assert_eq!(schema.fields[0].dtype, DataType::Int64); + assert_eq!(schema.fields[1].dtype, DataType::Utf8); + assert_eq!( + qe.schema + .fields + .iter() + .map(|f| f.name.as_str()) + .collect::>(), + ["n", "s"] + ); +} + +#[tokio::test] +async fn unary_minus_lowers_to_negative() { + // `-x` over a column is the `Negative` scalar (a negative *literal* is + // folded by DataFusion's planner before lowering). + let qe = lower("SELECT -latency FROM metrics").await; + let NonASAPOp::Project { cols, .. } = op(&qe) else { + panic!("expected Project at root, got {qe:?}"); + }; + assert_eq!( + cols[0].expr, + ScalarExpr::Negative { + expr: Box::new(ScalarExpr::Column(2)), + semantics: ExprSemantics::Sql, + } + ); + assert_eq!(qe.schema.fields[0].dtype, DataType::Float64); +} + +#[tokio::test] +async fn sql_comparisons_and_arithmetic_carry_sql_semantics() { + let qe = lower("SELECT bytes * 8 FROM metrics WHERE latency > 1.5").await; + let NonASAPOp::Project { cols, child, .. } = op(&qe) else { + panic!("expected Project at root, got {qe:?}"); + }; + assert!( + matches!( + &cols[0].expr, + ScalarExpr::Arithmetic { + semantics: ExprSemantics::Sql, + .. + } + ), + "got {:?}", + cols[0].expr + ); + let NonASAPOp::Scan { predicates, .. } = op(child) else { + panic!("expected the WHERE folded onto the Scan, got {child:?}"); + }; + assert!( + matches!( + &predicates[0].0, + ScalarExpr::Compare { + semantics: ExprSemantics::Sql, + .. + } + ), + "got {:?}", + predicates[0].0 + ); +} + +// ── Issue #115: Quantile / Cardinality carry their input column ───────────── + +#[tokio::test] +async fn quantile_carries_its_input_column() { + // `metrics(ts=0, service=1, latency=2, bytes=3)`. Two quantiles over + // different columns must not compare equal — a workload-level dedupe pass + // would compare on `AggIntent` equality, so a col-less intent would + // collapse them. + let qe = lower( + "SELECT approx_percentile_cont(latency, 0.5), \ + approx_percentile_cont(bytes, 0.5) FROM metrics", + ) + .await; + let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!( + matches!( + measures.as_slice(), + [ + AggIntent::Quantile { col: Some(2), .. }, + AggIntent::Quantile { col: Some(3), .. } + ] + ), + "quantiles must bind their own column, got {measures:?}" + ); + assert_ne!( + measures[0], measures[1], + "distinct-column quantiles must not compare equal" + ); +} + +#[tokio::test] +async fn count_distinct_carries_its_input_column() { + let qe = lower("SELECT COUNT(DISTINCT service), COUNT(DISTINCT bytes) FROM metrics").await; + let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!( + matches!( + measures.as_slice(), + [ + AggIntent::Cardinality { cols: c1, .. }, + AggIntent::Cardinality { cols: c2, .. } + ] if c1 == &[1] && c2 == &[3] + ), + "cardinalities must bind their own column, got {measures:?}" + ); + assert_ne!( + measures[0], measures[1], + "distinct-column cardinalities must not compare equal" + ); +} + +#[tokio::test] +async fn quantile_and_count_distinct_over_an_expression_bind_the_derived_column() { + // A SQL aggregate has no "sample value" to fall back on, so an expression + // argument must never reach the canonical tree as `col: None` (#115). + // Since #110 it reaches the canonical tree as `col: Some(derived)` + // instead of being rejected. + for q in [ + "SELECT approx_percentile_cont(bytes * 8, 0.95) FROM metrics", + "SELECT COUNT(DISTINCT bytes * 8) FROM metrics", + "SELECT approx_distinct(bytes * 8) FROM metrics", + ] { + let qe = lower(q).await; + let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!( + !measures[0].input_cols().is_empty(), + "{q} must bind a column, never the implicit input, got {measures:?}" + ); + let (names, materialized) = reducer_input_names(&qe); + assert!(materialized, "{q} expected a materializing Project"); + assert!( + names[0].contains("bytes"), + "{q} should reduce the projected `bytes * 8`, got {names:?}" + ); + } +} + +// ── Issue #111: median / approx_median → the φ=0.5 quantile ───────────────── + +#[tokio::test] +async fn median_lowers_to_the_half_quantile() { + // `metrics(ts=0, service=1, latency=2, bytes=3)`. + for sql in [ + "SELECT median(latency) FROM metrics", + "SELECT approx_median(latency) FROM metrics", + ] { + let qe = lower(sql).await; + let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!( + matches!( + measures.as_slice(), + [AggIntent::Quantile { col: Some(2), q, .. }] if (*q - 0.5).abs() < 1e-9 + ), + "{sql} should lower to Quantile(0.5) over latency, got {measures:?}" + ); + } +} + +#[tokio::test] +async fn median_is_the_same_intent_as_an_explicit_half_percentile() { + // Two spellings of one intent: CSE should be able to merge them. + let m = lower("SELECT median(latency) FROM metrics").await; + let p = lower("SELECT approx_percentile_cont(latency, 0.5) FROM metrics").await; + let (_, m_measures) = find_aggregate(&m).expect("expected an Aggregate"); + let (_, p_measures) = find_aggregate(&p).expect("expected an Aggregate"); + assert_eq!(m_measures, p_measures); +} + +#[tokio::test] +async fn median_threads_the_accuracy_target() { + // The `approx_` prefix does not decide: the AccuracyTarget does. + let qe = lower_sql( + "SELECT approx_median(latency) FROM metrics", + &catalog(), + AccuracyTarget::Epsilon(0.01), + ) + .await + .expect("approx_median should lower"); + let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!( + matches!( + measures.as_slice(), + [AggIntent::Quantile { accuracy: AccuracyTarget::Epsilon(e), .. }] + if (*e - 0.01).abs() < 1e-12 + ), + "median must carry the workload's accuracy target, got {measures:?}" + ); +} + +#[tokio::test] +async fn median_over_an_expression_binds_the_derived_column() { + // Was rejected when filed (#111); supported since #110 materialized the + // expression. What must still hold is the #115 rule: never `col: None`. + let qe = lower("SELECT median(bytes * 8) FROM metrics").await; + let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!( + matches!(measures.as_slice(), [AggIntent::Quantile { col: Some(_), q, .. }] if (*q - 0.5).abs() < 1e-9), + "expected Quantile(0.5) bound to the derived column, got {measures:?}" + ); +} + +// ── Issue #110: expression GROUP BY (time bucketing) ──────────────────────── + +#[tokio::test] +async fn time_bucketing_group_by_lowers_to_a_derived_key() { + // The canonical time-series shape: `GROUP BY date_trunc(...)`. The bucket + // expression is materialized beneath the aggregate and grouped on. + let qe = + lower("SELECT date_trunc('minute', ts) AS m, SUM(bytes) FROM metrics GROUP BY m").await; + let node = find_aggregate_node(&qe).expect("expected an Aggregate"); + let NonASAPOp::Aggregate { + reduction, + measures, + child, + .. + } = op(node) + else { + unreachable!() + }; + assert!( + matches!(op(child), NonASAPOp::Project { .. }), + "expected a materializing Project beneath the Aggregate" + ); + let schema = &child.schema; + assert_eq!(reduction, &Reduction::by(vec![0])); + assert!( + schema.fields[0].name.contains("date_trunc"), + "group key should be the projected bucket, got {:?}", + schema.fields[0].name + ); + // The reducer still binds its own column, not the bucket. + assert!(matches!( + measures.as_slice(), + [AggIntent::Sum { col: Some(1) }] + )); +} + +#[tokio::test] +async fn time_bucketing_keeps_the_scan_predicate() { + // The projection is inserted above the scan, so a WHERE clause still folds + // onto the Scan rather than being stranded. + let qe = lower( + "SELECT date_trunc('minute', ts) AS m, SUM(bytes) FROM metrics \ + WHERE bytes > 10 GROUP BY m", + ) + .await; + fn scan_has_predicate(node: &OperatorNode) -> bool { + match op(node) { + NonASAPOp::Scan { predicates, .. } => !predicates.is_empty(), + NonASAPOp::Project { child, .. } + | NonASAPOp::Filter { child, .. } + | NonASAPOp::Aggregate { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } => scan_has_predicate(child), + _ => false, + } + } + assert!(scan_has_predicate(&qe), "WHERE should stay on the Scan"); +} + +#[tokio::test] +async fn a_plain_group_by_inserts_no_projection() { + // Queries that lowered before #110 must keep their exact tree shape — the + // projection appears only when something actually needs materializing. + for q in [ + "SELECT service, SUM(bytes) FROM metrics GROUP BY service", + "SELECT SUM(bytes) FROM metrics", + "SELECT COUNT(*) FROM metrics", + ] { + let qe = lower(q).await; + let NonASAPOp::Aggregate { child, .. } = + op(find_aggregate_node(&qe).expect("expected an Aggregate")) + else { + unreachable!() + }; + assert!( + !matches!(op(child), NonASAPOp::Project { .. }), + "{q} should not gain a projection" + ); + } +} + +#[tokio::test] +async fn a_shared_expression_is_materialized_once() { + let qe = lower("SELECT SUM(bytes * 2), MIN(bytes * 2) FROM metrics").await; + let NonASAPOp::Aggregate { + measures, child, .. + } = op(find_aggregate_node(&qe).expect("expected an Aggregate")) + else { + unreachable!() + }; + assert_eq!( + child.schema.fields.len(), + 1, + "the two reducers should share one derived column" + ); + assert_eq!(measures[0].input_cols(), measures[1].input_cols()); +} + +// ── Issue #118: multi-level grouping expands into one Aggregate per level ─── + +/// The branches of the first `Concat` along the single-child spine. +fn merge_branches(node: &OperatorNode) -> &Vec> { + fn find(node: &OperatorNode) -> Option<&Vec>> { + match op(node) { + NonASAPOp::Concat { children, .. } => Some(children), + NonASAPOp::Project { child, .. } + | NonASAPOp::Filter { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } => find(child), + _ => None, + } + } + find(node).expect("expected a Concat") +} + +/// `(group keys, column names)` of each merged grouping level. +fn grouping_levels(node: &OperatorNode) -> Vec<(GroupKeys, Vec)> { + merge_branches(node) + .iter() + .map(|b| { + let NonASAPOp::Project { child, .. } = op(b) else { + panic!("expected a Project per level, got {b:?}"); + }; + let NonASAPOp::Aggregate { reduction, .. } = op(child) else { + panic!("expected an Aggregate under the Project, got {child:?}"); + }; + let names = b.schema.fields.iter().map(|c| c.name.clone()).collect(); + (reduction.expect_reduce().clone(), names) + }) + .collect() +} + +#[tokio::test] +async fn rollup_expands_to_one_aggregate_per_prefix() { + // ROLLUP(a, b) → (a,b), (a), () — three levels, widest first. + let qe = + lower("SELECT service, bytes, SUM(latency) FROM metrics GROUP BY ROLLUP(service, bytes)") + .await; + let levels = grouping_levels(&qe); + let keys: Vec<_> = levels.iter().map(|(by, _)| by.clone()).collect(); + assert_eq!( + keys, + vec![ + GroupKeys::by(vec![1, 3]), + GroupKeys::by(vec![1]), + GroupKeys::none(), + ] + ); +} + +#[tokio::test] +async fn cube_expands_to_the_power_set() { + // CUBE(a, b) → (a,b), (a), (b), () — four levels. + let qe = + lower("SELECT service, bytes, SUM(latency) FROM metrics GROUP BY CUBE(service, bytes)") + .await; + assert_eq!(grouping_levels(&qe).len(), 4); +} + +#[tokio::test] +async fn a_mixed_grouping_set_is_normalized_by_datafusion() { + // `GROUP BY g, ROLLUP(d)` arrives as one GroupingSets, not a plain key + // alongside a grouping set — so there is only one shape to handle. + let qe = + lower("SELECT service, bytes, SUM(latency) FROM metrics GROUP BY service, ROLLUP(bytes)") + .await; + assert_eq!(grouping_levels(&qe).len(), 2); +} + +#[tokio::test] +async fn omitted_grouping_keys_become_typed_nulls() { + // Every level must emit every key — as NULL where the level omits it — or + // `Concat` (which takes the first child's schema) would misdescribe the rest. + // The null is *cast*: a bare Null literal infers as Float64. + let qe = lower("SELECT service, SUM(bytes) FROM metrics GROUP BY ROLLUP(service)").await; + let levels = grouping_levels(&qe); + assert_eq!(levels.len(), 2); + for (_, names) in &levels { + assert_eq!( + names, + &["service".to_string(), "sum(metrics.bytes)".to_string()] + ); + } + + // The `()` level projects `service` as a Utf8 null, not a Float64 one. + let schema = &merge_branches(&qe)[1].schema; + assert_eq!(schema.fields[0].name, "service"); + assert_eq!( + schema.fields[0].dtype, + DataType::Utf8, + "the omitted key must keep its declared type" + ); +} + +#[tokio::test] +async fn grouping_levels_are_union_compatible() { + let qe = lower( + "SELECT service, bytes, SUM(latency) FROM metrics GROUP BY GROUPING SETS ((service),(bytes),())", + ) + .await; + let shapes: Vec<_> = merge_branches(&qe) + .iter() + .map(|b| { + b.schema + .fields + .iter() + .map(|c| (c.name.clone(), c.dtype.clone())) + .collect::>() + }) + .collect(); + assert!( + shapes.windows(2).all(|w| w[0] == w[1]), + "levels disagree: {shapes:?}" + ); +} + +#[tokio::test] +async fn grouping_function_is_rejected() { + // `__grouping_id` is dropped when the levels are expanded. It is observable + // only through `GROUPING(col)`, so dropping it loses nothing representable — + // this test is what makes that true. + let err = lower_sql( + "SELECT service, SUM(bytes), GROUPING(service) FROM metrics GROUP BY ROLLUP(service)", + &catalog(), + AccuracyTarget::Exact, + ) + .await + .expect_err("GROUPING() must be rejected while __grouping_id is dropped"); + assert!(format!("{err}").contains("grouping"), "got {err}"); +} + +#[tokio::test] +async fn a_non_column_key_inside_a_grouping_set_is_rejected() { + // The #110 derived-column machinery covers plain `GROUP BY `; inside a + // grouping set the key also has to be reinstatable as a typed null. + let err = lower_sql( + "SELECT date_trunc('minute', ts) AS m, SUM(bytes) FROM metrics GROUP BY ROLLUP(m)", + &catalog(), + AccuracyTarget::Exact, + ) + .await + .expect_err("expression key inside ROLLUP must be rejected"); + assert!( + format!("{err}").contains("non-column key inside a multi-level grouping"), + "got {err}" + ); +} + +#[tokio::test] +async fn multi_level_grouping_composes_with_a_derived_reducer_argument() { + // #110's materializing Project sits beneath every level's Aggregate. + let qe = lower("SELECT service, SUM(bytes * 8) FROM metrics GROUP BY ROLLUP(service)").await; + for b in merge_branches(&qe) { + let NonASAPOp::Project { child, .. } = op(b) else { + panic!("expected a Project per level"); + }; + let NonASAPOp::Aggregate { + measures, child, .. + } = op(child) + else { + panic!("expected an Aggregate"); + }; + assert!(matches!( + measures.as_slice(), + [AggIntent::Sum { col: Some(_) }] + )); + assert!( + matches!(op(child), NonASAPOp::Project { .. }), + "the derived-column projection should sit under each level" + ); + } +} + +#[tokio::test] +async fn an_ambiguous_passthrough_column_is_rejected_only_when_projecting() { + // A `Project` carries one relation qualifier for all its columns, so `a.k` + // and `b.k` cannot both survive it. That only matters once a projection is + // inserted: without a derived column the join keys resolve as before. + let ok = lower_sql( + "SELECT m.service, h.service, SUM(m.bytes) FROM metrics m \ + JOIN hosts h ON m.service = h.service GROUP BY m.service, h.service", + &catalog(), + AccuracyTarget::Exact, + ) + .await; + assert!( + ok.is_ok(), + "no derived column ⇒ no projection ⇒ no ambiguity" + ); + + let err = lower_sql( + "SELECT m.service, h.service, SUM(m.bytes * 2) FROM metrics m \ + JOIN hosts h ON m.service = h.service GROUP BY m.service, h.service", + &catalog(), + AccuracyTarget::Exact, + ) + .await + .expect_err("ambiguous passthrough must be rejected, not silently resolved"); + assert!(format!("{err}").contains("ambiguous column"), "got {err}"); +} + +// ── Issue #111: array_agg is deliberately not an intent (WONTFIX) ─────────── + +#[tokio::test] +async fn array_agg_is_deliberately_rejected() { + // Not a coverage gap. `AggIntent` exists so the planner can bind a sketch or + // a mergeable accumulator per node; `array_agg` pre-aggregates nothing (its + // output is O(input rows)), has no bounded-memory approximate form, and its + // partial state *is* the data. An `AggIntent::ArrayAgg` would force every + // arm of `plan::boundary::realize` — an exhaustive match — to answer + // `PassThrough`. Contrast `median`, which is `Quantile { q: 0.5 }` and does + // feed the sketch path. + // + // This test exists so the rejection reads as a decision rather than a gap. + let err = lower_sql( + "SELECT array_agg(service) FROM metrics", + &catalog(), + AccuracyTarget::Exact, + ) + .await + .expect_err("array_agg must not lower to an intent"); + assert!( + format!("{err}").contains("unsupported aggregate: array_agg"), + "expected a clean UnsupportedAggregate, got {err}" + ); +} + +// ── Issue #225: catalog-driven ClickHouse builtins (countIf, generalizing +// uniqExact from #221) ─────────────────────────────────────────────────── + +async fn lower_clickhouse(sql: &str) -> Rc { + lower_sql_dialect( + sql, + &catalog(), + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) +} + +fn temporal_aggregate(node: &OperatorNode) -> (&AggIntent, std::time::Duration, &OperatorNode) { + match op(node) { + NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + measures, + child, + .. + } => { + let NonASAPOp::TimeRange { range, child, .. } = op(child) else { + panic!("temporal Aggregate must directly wrap TimeRange, got {child:?}"); + }; + (&measures[0], *range, child) + } + NonASAPOp::Project { child, .. } | NonASAPOp::Filter { child, .. } => { + temporal_aggregate(child) + } + other => panic!("expected temporal Aggregate, got {other:?}"), + } +} + +#[tokio::test] +async fn explicit_temporal_aggregates_share_promql_intents_and_timerange() { + for (function, expected) in [ + ("asap_rate", AggIntent::Rate), + ("asap_increase", AggIntent::Increase), + ] { + let sql = format!( + "SELECT service, {function}(latency, ts, 300000) AS v \ + FROM metrics WHERE service = 'api' GROUP BY service" + ); + let qe = lower_clickhouse(&sql).await; + let (intent, range, child) = temporal_aggregate(&qe); + assert_eq!(intent, &expected); + assert_eq!(range, std::time::Duration::from_secs(300)); + assert!(matches!(op(child), NonASAPOp::Project { child, .. } + if matches!(op(child), NonASAPOp::Scan { predicates, .. } if predicates.len() == 1))); + + let NonASAPOp::Project { cols, .. } = op(&qe) else { + panic!("SELECT list must remain a Project, got {qe:?}"); + }; + assert!(matches!(cols[0].expr, ScalarExpr::Column(2))); + assert_eq!(cols[1].alias.as_deref(), Some("v")); + assert!(matches!(cols[1].expr, ScalarExpr::Column(1))); + } +} + +#[tokio::test] +async fn temporal_aggregate_rejects_non_timestamp_and_non_positive_window() { + for sql in [ + "SELECT asap_rate(latency, bytes, 300000) FROM metrics", + "SELECT asap_rate(latency, ts, 0) FROM metrics", + "SELECT asap_rate(latency, ts, bytes) FROM metrics", + ] { + let err = lower_sql_dialect( + sql, + &catalog(), + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .expect_err("invalid temporal arguments must fail closed"); + assert!( + format!("{err}").contains("timestamp argument") + || format!("{err}").contains("window_ms"), + "unexpected error for {sql}: {err}" + ); + } +} + +#[tokio::test] +async fn temporal_aggregate_rejects_mixed_reducers() { + let err = lower_sql_dialect( + "SELECT asap_rate(latency, ts, 300000), sum(bytes) FROM metrics", + &catalog(), + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .expect_err("one child cannot carry temporal and ordinary aggregate semantics"); + assert!(format!("{err}").contains("cannot share an Aggregate node")); +} + +#[tokio::test] +async fn last_fails_closed_until_an_executable_summary_exists() { + let err = lower_sql_dialect( + "SELECT service, asap_last(latency, ts, 300000) FROM metrics GROUP BY service", + &catalog(), + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .expect_err("last must not be advertised without an executable physical summary"); + assert!(format!("{err}").contains("Invalid function 'asap_last'")); +} + +#[tokio::test] +async fn temporal_grouping_requires_the_complete_declared_series_identity() { + let multi_series = SqlCatalog::new().with_table( + "samples", + Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("instance", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![0, 1, 2]], + ), + ); + for sql in [ + "SELECT asap_rate(value, ts, 300000) FROM samples", + "SELECT service, asap_rate(value, ts, 300000) FROM samples GROUP BY service", + ] { + let err = lower_sql_dialect( + sql, + &multi_series, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .expect_err("partial identity must not merge counter series"); + assert!(format!("{err}").contains("declared series identity")); + } + + lower_sql_dialect( + "SELECT service, instance, asap_rate(value, ts, 300000) \ + FROM samples GROUP BY service, instance", + &multi_series, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .expect("the complete declared series identity is safe"); + + let row_id_only = SqlCatalog::new().with_table( + "samples", + Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![1]], + ), + ); + lower_sql_dialect( + "SELECT service, asap_rate(value, ts, 300000) FROM samples GROUP BY service", + &row_id_only, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .expect_err("a row key without time does not prove a series identity"); +} + +#[tokio::test] +async fn temporal_grouping_rejects_value_time_and_duplicate_resolved_columns() { + for sql in [ + "SELECT asap_rate(latency, ts, 300000) FROM metrics GROUP BY ts", + "SELECT asap_rate(latency, ts, 300000) FROM metrics GROUP BY latency", + "SELECT m.service, asap_rate(m.latency, m.ts, 300000) \ + FROM metrics m GROUP BY m.service, service", + ] { + let err = lower_sql_dialect( + sql, + &catalog(), + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .expect_err("unsafe or duplicate resolved grouping must fail closed"); + let message = format!("{err}"); + assert!( + message.contains("timestamp or value") + || message.contains("same resolved column more than once"), + "unexpected error for {sql}: {message}" + ); + } +} + +#[tokio::test] +async fn qualified_columns_are_validated_by_resolved_identity() { + let qe = lower_clickhouse( + "SELECT m.service, asap_increase(m.latency, m.ts, 300000) AS v \ + FROM metrics AS m GROUP BY m.service", + ) + .await; + let (intent, range, _) = temporal_aggregate(&qe); + assert_eq!(intent, &AggIntent::Increase); + assert_eq!(range, std::time::Duration::from_secs(300)); +} + +#[tokio::test] +async fn project_filter_and_outer_aggregate_preserve_temporal_child() { + let qe = lower_clickhouse( + "SELECT max(v) FROM (\ + SELECT service, asap_rate(latency, ts, 300000) AS v \ + FROM metrics WHERE bytes > 0 GROUP BY service\ + ) r WHERE v >= 0", + ) + .await; + let NonASAPOp::Project { child, .. } = op(&qe) else { + panic!("expected outer SELECT Project, got {qe:?}"); + }; + let NonASAPOp::Aggregate { + reduction: Reduction::Reduce(_), + measures, + child, + .. + } = op(child) + else { + panic!("expected outer Aggregate, got {child:?}"); + }; + assert!(matches!(measures.as_slice(), [AggIntent::Max { .. }])); + let NonASAPOp::Filter { child, .. } = op(child) else { + panic!("derived-table WHERE must remain above the inner query, got {child:?}"); + }; + let (intent, range, _) = temporal_aggregate(child); + assert_eq!(intent, &AggIntent::Rate); + assert_eq!(range, std::time::Duration::from_secs(300)); +} + +#[tokio::test] +async fn count_if_lowers_to_a_sum_over_a_derived_indicator_column() { + // ClickHouse's `countIf(cond)` has no DataFusion equivalent at all, so it + // goes through the same stub-UDAF + catalog-driven `FunctionRewrite` + // mechanism `uniqExact` (#221) does — rewritten, before `lower_agg_intent` + // ever runs, to `sum(CASE WHEN cond THEN 1 ELSE 0 END)`. A per-measure + // filter (#466) could express it as a filtered `Count` now; that move is + // a follow-up, so the indicator sum is still the shape to expect. + let qe = lower_clickhouse("SELECT countIf(bytes > 100) AS big FROM metrics").await; + let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!(by.is_empty()); + assert!( + matches!(measures.as_slice(), [AggIntent::Sum { col: Some(_) }]), + "expected a Sum bound to the derived indicator column, got {measures:?}" + ); + let (_, materialized) = reducer_input_names(&qe); + assert!( + materialized, + "the indicator expression must be materialized in a Project beneath the Aggregate" + ); +} + +#[tokio::test] +async fn two_count_ifs_with_different_conditions_stay_distinct_reducers() { + // The corpus pattern (`countIf(operation = 'A'), countIf(operation = 'W')` + // in one GROUP BY) needs each call's own condition to survive as its own + // derived column, not collapse onto a shared one. + let qe = lower_clickhouse( + "SELECT service, countIf(bytes > 100) AS big, countIf(bytes <= 100) AS small \ + FROM metrics GROUP BY service", + ) + .await; + let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert_eq!(*by, GroupKeys::by(vec![0])); + assert!( + matches!( + measures.as_slice(), + [ + AggIntent::Sum { col: Some(a) }, + AggIntent::Sum { col: Some(b) } + ] if a != b + ), + "expected two distinct Sum reducers, got {measures:?}" + ); +} + +#[tokio::test] +async fn count_if_composes_with_group_by() { + let qe = lower_clickhouse( + "SELECT service, countIf(bytes > 100) AS big FROM metrics GROUP BY service", + ) + .await; + let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert_eq!(*by, GroupKeys::by(vec![0])); + assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); +} + +// ── Issue #232: argMax/argMin -- AggIntent::Extension, not a first-class +// core variant. A repo-wide search (PromQL front end, other SQL dialects, +// docs) turned up no second deployment model wanting this two-column, +// row-selecting shape, so per `AggIntent::Extension`'s own "core only grows +// for intents ≥2 deployment models actually use" bar, it stays an opaque +// `Extension` rather than a new `ArgMax`/`ArgMin` core variant. Unlike +// `countIf`/`uniqExact`, there is no native DataFusion aggregate shape to +// rewrite to (`RewriteKind::PassThrough`) -- `lower_agg_intent` builds the +// `AggIntent` directly from the ClickHouse name. ───────────────────────── + +#[tokio::test] +async fn arg_max_lowers_to_an_extension_intent() { + // No existing `AggIntent` reducer fits: every one folds one column to a + // value derived from itself, while `argMax(arg, val)` returns a + // *different* column's value, selected by which row maximizes a second. + let qe = lower_clickhouse( + "SELECT service, argMax(service, latency) AS busiest FROM metrics GROUP BY service", + ) + .await; + let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert_eq!( + *by, + GroupKeys::by(vec![1]), + "grouped by `service` (schema index 1)" + ); + assert!( + matches!( + measures.as_slice(), + [AggIntent::Extension { ext_kind, .. }] if ext_kind == "arg_max" + ), + "expected Extension {{ ext_kind: \"arg_max\", .. }}, got {measures:?}" + ); +} + +#[tokio::test] +async fn arg_min_lowers_to_its_own_extension_kind() { + let qe = lower_clickhouse("SELECT argMin(service, latency) FROM metrics").await; + let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!(by.is_empty()); + assert!( + matches!( + measures.as_slice(), + [AggIntent::Extension { ext_kind, .. }] if ext_kind == "arg_min" + ), + "expected Extension {{ ext_kind: \"arg_min\", .. }}, got {measures:?}" + ); +} + +#[tokio::test] +async fn arg_max_payload_preserves_both_column_names() { + // Core never resolves an `Extension`'s payload, so both columns are kept + // as validated bare-column `ColumnRef`s in `payload`, not run through + // positional `ColumnId` binding -- see `lower_arg_selector`'s doc. + let qe = lower_clickhouse("SELECT argMax(service, latency) AS m FROM metrics").await; + let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); + let AggIntent::Extension { payload, .. } = &measures[0] else { + panic!("expected an Extension intent, got {:?}", measures[0]); + }; + let named = |key: &str| { + payload + .get(key) + .and_then(|c| c.get("Named")) + .and_then(|n| n.as_str()) + .map(str::to_string) + }; + assert_eq!(named("arg_col"), Some("service".to_string())); + assert_eq!(named("val_col"), Some("latency".to_string())); +} + +#[tokio::test] +async fn arg_max_rejects_a_non_column_argument() { + // Same "bare column only" rule as every other reducer (`reducer_col`, + // issue #115) -- an expression argument is rejected, not silently + // dropped or materialized into the wrong column. + let err = lower_sql_dialect( + "SELECT argMax(service, latency * 2) FROM metrics", + &catalog(), + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .expect_err("argMax over a non-column expression must be rejected"); + assert!( + format!("{err}").contains("non-column expression"), + "got {err}" + ); +} + +// ── Issue #267: lagInFrame/leadInFrame get distinct WindowFuncKind variants, +// not conflated with ANSI Lag/Lead ────────────────────────────────────────── + +#[tokio::test] +async fn lag_in_frame_lowers_to_its_own_kind_not_lag() { + let qe = lower_clickhouse( + "SELECT service, lagInFrame(bytes) OVER (PARTITION BY service ORDER BY ts) \ + FROM metrics", + ) + .await; + let win = find_windowfunc(&qe).expect("expected a SQLWindowFunc node"); + let NonASAPOp::SQLWindowFunc { func, args, .. } = op(win) else { + unreachable!(); + }; + assert_eq!(*func, WindowFuncKind::LagInFrame); + assert_eq!( + args, + &vec![ScalarExpr::Column(3)], + "lagInFrame(bytes) → arg col 3" + ); +} + +#[tokio::test] +async fn lead_in_frame_lowers_to_its_own_kind_not_lead() { + let qe = lower_clickhouse( + "SELECT service, leadInFrame(bytes) OVER (PARTITION BY service ORDER BY ts) \ + FROM metrics", + ) + .await; + let win = find_windowfunc(&qe).expect("expected a SQLWindowFunc node"); + let NonASAPOp::SQLWindowFunc { func, .. } = op(win) else { + unreachable!(); + }; + assert_eq!(*func, WindowFuncKind::LeadInFrame); +} + +/// Issue #184: `NOW()` in a predicate must lower to the timestamp-typed +/// `CurrentTimestamp` leaf, not the semantically-opaque function catch-all or +/// PromQL's Float64 Unix-seconds `EvalTimestamp`. +#[tokio::test] +async fn now_in_predicate_lowers_to_current_timestamp() { + // SELECT * folds WHERE onto Scan.predicates (no explicit Filter node). + let qe = lower("SELECT * FROM metrics WHERE ts < NOW()").await; + let NonASAPOp::Scan { predicates, .. } = op(&qe) else { + panic!("expected Scan at root, got {qe:?}"); + }; + assert_eq!(predicates.len(), 1); + assert!( + matches!(&predicates[0].0, ScalarExpr::Compare { right, .. } + if matches!(right.as_ref(), ScalarExpr::Cast { expr, to: DataType::Timestamp, .. } if matches!(expr.as_ref(), ScalarExpr::CurrentTimestamp))), + "NOW() must lower to CurrentTimestamp, got {:?}", + predicates[0].0 + ); +} + +/// Same for ClickHouse's `now()`, since #184 was raised specifically against +/// the ClickHouse dialect. +#[tokio::test] +async fn clickhouse_now_in_predicate_lowers_to_current_timestamp() { + let qe = lower_clickhouse("SELECT * FROM metrics WHERE ts < now()").await; + let NonASAPOp::Scan { predicates, .. } = op(&qe) else { + panic!("expected Scan at root, got {qe:?}"); + }; + assert_eq!(predicates.len(), 1); + assert!( + matches!(&predicates[0].0, ScalarExpr::Compare { right, .. } + if matches!(right.as_ref(), ScalarExpr::Cast { expr, to: DataType::Timestamp, .. } if matches!(expr.as_ref(), ScalarExpr::CurrentTimestamp))), + "now() must lower to CurrentTimestamp, got {:?}", + predicates[0].0 + ); +} + +#[tokio::test] +async fn current_timestamp_lowers_to_typed_current_timestamp_leaf() { + let qe = lower("SELECT CURRENT_TIMESTAMP FROM metrics").await; + let NonASAPOp::Project { cols, child, .. } = op(&qe) else { + panic!("expected Project at root, got {qe:?}"); + }; + assert!(matches!(&cols[0].expr, ScalarExpr::CurrentTimestamp)); + let (dtype, _) = cols[0] + .expr + .scalar_type(&child.schema) + .expect("timestamp type"); + assert_eq!(dtype, DataType::Timestamp); + assert_eq!(qe.schema.fields[0].dtype, DataType::Timestamp); +} + +// A `count` over a non-null input is a plain row count; over a nullable +// input it keeps SQL's NULL-skipping as the measure's own filter (#466), and +// only the multi-level grouping path, which cannot carry one, still rejects it. +#[tokio::test] +async fn count_null_semantics_become_a_measure_filter() { + let catalog = SqlCatalog::new().with_table( + "samples", + Schema::new(vec![ + Field::plain("nullable_value", DataType::Float64, true), + Field::plain("value", DataType::Float64, false), + ]), + ); + for sql in [ + "SELECT count(*) FROM samples", + "SELECT count(1) FROM samples", + "SELECT count(value) FROM samples", + "SELECT count(value + 1) FROM samples", + ] { + let qe = lower_sql(sql, &catalog, AccuracyTarget::Exact) + .await + .unwrap_or_else(|error| panic!("{sql}: {error}")); + assert!( + aggregate_filters(&qe).is_empty(), + "{sql}: unfiltered row count" + ); + } + for sql in [ + "SELECT count(nullable_value) FROM samples", + "SELECT count(NULL) FROM samples", + "SELECT count(nullable_value + 1) FROM samples", + ] { + let qe = lower_sql(sql, &catalog, AccuracyTarget::Exact) + .await + .unwrap_or_else(|error| panic!("{sql}: {error}")); + let [Some(Predicate(cond))] = aggregate_filters(&qe) else { + panic!( + "{sql}: expected one filtered Count, got {:?}", + aggregate_filters(&qe) + ); + }; + assert!(matches!(cond, ScalarExpr::IsNotNull(_)), "{sql}: {cond:?}"); + } + // Only the second measure is filtered. + let qe = lower_sql( + "SELECT count(*), count(nullable_value) FROM samples", + &catalog, + AccuracyTarget::Exact, + ) + .await + .unwrap(); + assert!(matches!(aggregate_filters(&qe), [None, Some(_)])); + let error = lower_sql( + "SELECT count(nullable_value) FROM samples GROUP BY ROLLUP(value)", + &catalog, + AccuracyTarget::Exact, + ) + .await + .unwrap_err(); + assert!( + matches!(error, LoweringError::UnsupportedFeature(_)), + "{error}" + ); +} + +/// A native SQL map grouping key retains its typed key/value schema. +#[tokio::test] +async fn grouped_map_column_preserves_map_type() { + let map = DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::Utf8), + value_nullable: false, + }; + let catalog = SqlCatalog::new().with_table( + "raw_samples", + Schema::new(vec![ + col("labels", map.clone()), + col("value", DataType::Float64), + ]), + ); + let query = lower_sql_dialect( + "SELECT labels, max(value) AS value FROM raw_samples GROUP BY labels ORDER BY labels", + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .unwrap(); + assert_eq!(query.schema.fields[0].dtype, map); +} + +#[tokio::test] +async fn clickhouse_modulo_uses_native_arithmetic_types_and_nullability() { + let catalog = SqlCatalog::new().with_table( + "numbers", + Schema::new(vec![ + Field::plain("i", DataType::Int64, false), + Field::plain("n", DataType::Int64, true), + Field::plain("f", DataType::Float64, false), + ]), + ); + for (call, native) in [ + ("modulo(i, 3)", "i % 3"), + ("modulo(n, -3)", "n % -3"), + ("modulo(f, 2.5)", "f % 2.5"), + ("modulo(-7, 3)", "-7 % 3"), + ("modulo(i, 0)", "i % 0"), + ("modulo(modulo(i, 5), 2)", "(i % 5) % 2"), + ] { + let function = lower_sql_dialect( + &format!("SELECT {call} AS value FROM numbers"), + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .unwrap(); + let operator = lower_sql_dialect( + &format!("SELECT {native} AS value FROM numbers"), + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .unwrap(); + assert_eq!(function, operator, "{call}"); + assert_eq!(function.schema, operator.schema); + } + let nullable = lower_sql_dialect( + "SELECT modulo(n, 3) AS value FROM numbers", + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .unwrap() + .schema + .clone(); + assert_eq!(nullable.fields[0].dtype, DataType::Int64); + assert!(nullable.fields[0].nullable); +} + +#[tokio::test] +async fn original_o11y_map_queries_lower_with_typed_results() { + let catalog = SqlCatalog::new().with_table( + "raw_samples", + Schema::new(vec![ + Field::plain("metric", DataType::Utf8, false), + Field::plain("ts_ms", DataType::Int64, false), + Field::plain("value", DataType::Float64, false), + Field::plain( + "labels", + DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::Utf8), + value_nullable: false, + }, + false, + ), + ]), + ); + for sql in [ + include_str!("data/o11y_q10.sql"), + include_str!("data/o11y_q27.sql"), + include_str!("data/o11y_q07.sql"), + include_str!("data/o11y_q09.sql"), + include_str!("data/o11y_q12.sql"), + ] { + let query = lower_sql_dialect( + sql, + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + let schema = &query.schema; + assert!( + schema + .fields + .iter() + .any(|column| matches!(column.dtype, FieldDataType::Plain(DataType::Map { .. }))), + "{schema:?}" + ); + } +} + +#[tokio::test] +async fn clickhouse_modulo_preserves_projection_names_and_outer_references() { + for (sql, name) in [ + ("SELECT modulo(bytes, 3) FROM metrics", "modulo(bytes, 3)"), + ( + "SELECT modulo(bytes, 3) AS remainder FROM metrics", + "remainder", + ), + ( + "SELECT \"modulo(bytes, 3)\" FROM (SELECT modulo(bytes, 3) FROM metrics) t", + "modulo(bytes, 3)", + ), + ] { + let query = lower_sql_dialect( + sql, + &catalog(), + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .unwrap(); + assert_eq!(query.schema.fields[0].name, name); + } +} + +#[tokio::test] +async fn clickhouse_map_access_keeps_generated_names_and_rejects_variant_coercion() { + let catalog = SqlCatalog::new().with_table( + "t", + Schema::new(vec![ + Field::plain( + "labels", + DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::Utf8), + value_nullable: false, + }, + false, + ), + Field::plain("integer", DataType::Int64, false), + Field::plain("floating", DataType::Float64, false), + ]), + ); + let query = lower_sql_dialect( + "SELECT labels['job'] FROM t", + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .unwrap(); + let output = &query.schema; + assert_eq!(output.fields[0].name, "arrayElement(labels, 'job')"); + assert_eq!(output.fields[0].dtype, DataType::Utf8); + assert!(!output.fields[0].nullable); + assert!(lower_sql_dialect( + "SELECT map()['a'] FROM t", + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .is_err()); + assert!(lower_sql_dialect( + "SELECT map('a', integer, 'b', floating) FROM t", + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact + ) + .await + .is_err()); +} + +#[tokio::test] +async fn arg_selector_result_schema_tracks_selected_argument() { + let catalog = SqlCatalog::new().with_table( + "t", + Schema::new(vec![ + Field::plain("v", DataType::Float64, false), + Field::plain("text", DataType::Utf8, true), + Field::plain("ts", DataType::Int64, true), + ]), + ); + for (sql, dtype, nullable) in [ + ( + "SELECT argMax(v, ts) AS value FROM t", + DataType::Float64, + false, + ), + ( + "SELECT argMin(text, ts) AS value FROM t", + DataType::Utf8, + true, + ), + ] { + let query = lower_sql_dialect( + sql, + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .unwrap(); + let schema = &query.schema; + assert_eq!(schema.fields[0].dtype, dtype); + assert_eq!(schema.fields[0].nullable, nullable); + } +} + +#[tokio::test] +async fn clickhouse_list_element_uses_canonical_typed_access() { + let catalog = SqlCatalog::new().with_table( + "t", + Schema::new(vec![ + Field::plain( + "samples", + DataType::List { + element: Box::new(Field::new("item", DataType::Int64, false)), + }, + false, + ), + Field::plain("index", DataType::Int64, true), + ]), + ); + for (sql, nullable) in [ + ("SELECT samples[1] AS selected FROM t", false), + ("SELECT arrayElement(samples, -1) AS selected FROM t", false), + ("SELECT samples[index] AS selected FROM t", true), + ] { + let query = lower_sql_dialect( + sql, + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .unwrap(); + let output = &query.schema; + assert_eq!(output.fields[0].dtype, DataType::Int64); + assert_eq!(output.fields[0].nullable, nullable); + let serialized = serde_json::to_string(&query).unwrap(); + assert!(serialized.contains("asap_element_access"), "{serialized}"); + } + for sql in ["SELECT samples[0] FROM t", "SELECT samples['bad'] FROM t"] { + assert!( + lower_sql_dialect( + sql, + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact + ) + .await + .is_err(), + "{sql}" + ); + } +} + +#[tokio::test] +async fn clickhouse_tuple_element_preserves_declared_field_metadata() { + let catalog = SqlCatalog::new().with_table( + "t", + Schema::new(vec![ + Field::plain( + "sample", + DataType::Struct { + fields: vec![ + Field::new("time", DataType::Int64, false), + Field::new("value", DataType::Float64, true), + ], + }, + false, + ), + Field::plain("index", DataType::Int64, false), + ]), + ); + for (sql, dtype, nullable) in [ + ( + "SELECT tupleElement(sample, 1) AS chosen FROM t", + DataType::Int64, + false, + ), + ( + "SELECT tupleElement(sample, 'value') AS chosen FROM t", + DataType::Float64, + true, + ), + ] { + let query = lower_sql_dialect( + sql, + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact, + ) + .await + .unwrap(); + let output = &query.schema; + assert_eq!(output.fields[0].dtype, dtype); + assert_eq!(output.fields[0].nullable, nullable); + assert!(serde_json::to_string(&query) + .unwrap() + .contains("asap_struct_field")); + } + for selector in ["0", "-1", "3", "'missing'", "index"] { + let sql = format!("SELECT tupleElement(sample, {selector}) FROM t"); + assert!( + lower_sql_dialect( + &sql, + &catalog, + SqlDialect::ClickhouseSQL, + AccuracyTarget::Exact + ) + .await + .is_err(), + "{sql}" + ); + } +} + +/// Correlation lowers to a nullable numeric result instead of UnsupportedAggregate. +#[tokio::test] +async fn corr_result_is_nullable_float() { + let query = lower("SELECT corr(latency, bytes) AS correlation FROM metrics").await; + let schema = &query.schema; + assert_eq!(schema.fields[0].name, "correlation"); + assert_eq!(schema.fields[0].dtype, DataType::Float64); + assert!(schema.fields[0].nullable); +} + +// A multi-column DISTINCT counts tuples; one column stays the single-column +// intent, so neither form can be mistaken for the other downstream. +#[tokio::test] +async fn composite_distinct_counts_tuples() { + let cat = SqlCatalog::new().with_table( + "t", + Schema::new(vec![ + Field::plain("a", DataType::Int64, false), + Field::plain("b", DataType::Int64, false), + ]), + ); + let composite = lower_sql( + "SELECT COUNT(DISTINCT a, b) FROM t", + &cat, + AccuracyTarget::Exact, + ) + .await + .unwrap(); + let NonASAPOp::Aggregate { measures, .. } = + op(find_aggregate_node(&composite).expect("expected an Aggregate")) + else { + unreachable!() + }; + assert!( + matches!(measures.as_slice(), [AggIntent::Cardinality { cols, .. }] if cols == &[0, 1]), + "{measures:?}" + ); + + let single = lower_sql( + "SELECT COUNT(DISTINCT a) FROM t", + &cat, + AccuracyTarget::Exact, + ) + .await + .unwrap(); + let NonASAPOp::Aggregate { measures, .. } = + op(find_aggregate_node(&single).expect("expected an Aggregate")) + else { + unreachable!() + }; + assert!( + matches!(measures.as_slice(), [AggIntent::Cardinality { cols, .. }] if cols == &[0]), + "{measures:?}" + ); +} + +// An expression argument has no column identity to hash, so it is rejected +// rather than silently reduced over a probe column. +#[tokio::test] +async fn composite_distinct_rejects_expression_arguments() { + let cat = SqlCatalog::new().with_table( + "t", + Schema::new(vec![ + Field::plain("a", DataType::Int64, false), + Field::plain("b", DataType::Int64, false), + ]), + ); + let error = lower_sql( + "SELECT COUNT(DISTINCT a, b + 1) FROM t", + &cat, + AccuracyTarget::Exact, + ) + .await + .unwrap_err(); + assert!( + matches!(&error, LoweringError::UnsupportedAggregate(reason) + if reason.contains("non-column expression")), + "{error}" + ); +} + +// DISTINCT inputs survive projections introduced by sibling aggregates. +#[tokio::test] +async fn distinct_with_derived_sibling() { + let catalog = SqlCatalog::new().with_table( + "t", + Schema::new(vec![ + Field::plain("a", DataType::Int64, false), + Field::plain("b", DataType::Int64, false), + ]), + ); + for sql in [ + "SELECT count(DISTINCT a), sum(b + 1) FROM t", + "SELECT count(DISTINCT a, b), sum(b + 1) FROM t", + "SELECT count(DISTINCT a, b), corr(a,b) FROM t", + ] { + let result = lower_sql(sql, &catalog, AccuracyTarget::Exact).await; + assert!(result.is_ok(), "{sql}: {result:?}"); + } +} + +// ── Issue #466: per-measure FILTER predicates ───────────────────────────────── + +/// The first `Aggregate`'s `filters`, positional against its child. +fn aggregate_filters(qe: &OperatorNode) -> &[Option] { + let Some(NonASAPOp::Aggregate { filters, .. }) = + find_aggregate_node(qe).map(|n| n.expect_non_asap()) + else { + panic!("expected an Aggregate, got {qe:?}"); + }; + filters +} + +// The motivating query: one scan, one grouping, one conditional count next to +// a plain sum — a single `Aggregate` whose Count carries the condition, with no +// `Join` and no derived column for the `CASE`. +#[tokio::test] +async fn conditional_count_lowers_to_a_filtered_measure() { + let qe = lower( + "SELECT service, count(CASE WHEN latency > 1.0 THEN 1 END), sum(bytes) \ + FROM metrics GROUP BY service", + ) + .await; + assert!(find_join(&qe).is_none(), "no join: {qe:?}"); + let (by, measures) = find_aggregate(&qe).unwrap(); + assert_eq!(by.keys(), &[1]); + assert!( + matches!( + measures.as_slice(), + [AggIntent::Count { .. }, AggIntent::Sum { col: Some(3) }] + ), + "{measures:?}" + ); + let [Some(Predicate(cond)), None] = aggregate_filters(&qe) else { + panic!("expected [Some, None], got {:?}", aggregate_filters(&qe)); + }; + assert!( + matches!(cond, ScalarExpr::Compare { left, op: CompareOpKind::Gt, .. } + if matches!(left.as_ref(), ScalarExpr::Column(2))), + "latency > 1.0 against the scan, got {cond:?}" + ); + let Some(NonASAPOp::Aggregate { child, .. }) = + find_aggregate_node(&qe).map(|n| n.expect_non_asap()) + else { + unreachable!() + }; + assert!( + matches!(child.expect_non_asap(), NonASAPOp::Scan { .. }), + "{child:?}" + ); +} + +// `FILTER (WHERE …)` parses under the DataFusion dialect and lands on exactly +// the measure it annotates. +#[tokio::test] +async fn filter_clause_lowers_to_a_measure_filter() { + let qe = lower("SELECT sum(bytes) FILTER (WHERE service = 'a'), count(*) FROM metrics").await; + let [Some(Predicate(cond)), None] = aggregate_filters(&qe) else { + panic!("expected [Some, None], got {:?}", aggregate_filters(&qe)); + }; + assert!( + matches!(cond, ScalarExpr::Compare { left, op: CompareOpKind::Eq, right, .. } + if matches!(left.as_ref(), ScalarExpr::Column(1)) + && matches!(right.as_ref(), ScalarExpr::Literal(ScalarValue::Utf8(s)) if s == "a")), + "{cond:?}" + ); +} + +// SQL `count(expr)` skips NULLs; canonical `Count` counts rows and never sees +// `expr`, so a nullable argument becomes the measure filter `expr IS NOT NULL` +// instead of being rejected (the pre-#466 behavior) or silently over-counted. +#[tokio::test] +async fn count_of_a_nullable_expression_filters_nulls() { + let qe = lower("SELECT count(nullif(bytes, 0)) FROM metrics").await; + let [Some(Predicate(cond))] = aggregate_filters(&qe) else { + panic!("expected [Some], got {:?}", aggregate_filters(&qe)); + }; + assert!(matches!(cond, ScalarExpr::IsNotNull(_)), "{cond:?}"); + assert!( + matches!( + find_aggregate(&qe).unwrap().1.as_slice(), + [AggIntent::Count { .. }] + ), + "still a row count" + ); +} + +// The columns a measure filter reads must survive the derived-column +// `Project` a reducer expression inserts beneath the aggregate. +#[tokio::test] +async fn measure_filter_columns_survive_a_derived_column_projection() { + let qe = lower("SELECT sum(bytes * 2) FILTER (WHERE latency > 1.0) FROM metrics").await; + let Some(NonASAPOp::Aggregate { child, .. }) = + find_aggregate_node(&qe).map(|n| n.expect_non_asap()) + else { + unreachable!() + }; + assert!( + matches!(child.expect_non_asap(), NonASAPOp::Project { .. }), + "{child:?}" + ); + let [Some(Predicate(cond))] = aggregate_filters(&qe) else { + panic!("expected [Some], got {:?}", aggregate_filters(&qe)); + }; + let ScalarExpr::Compare { left, .. } = cond else { + panic!("{cond:?}"); + }; + let ScalarExpr::Column(id) = left.as_ref() else { + panic!("{left:?}"); + }; + assert_eq!(child.schema.fields[*id].name, "latency"); +} + +// `GROUP BY ROLLUP` fans one measure list out into one `Aggregate` per level; +// a filtered measure there is rejected rather than silently unfiltered. +#[tokio::test] +async fn measure_filter_inside_a_rollup_is_rejected() { + let err = lower_sql( + "SELECT service, count(*) FILTER (WHERE latency > 1.0) FROM metrics GROUP BY ROLLUP(service)", + &catalog(), + AccuracyTarget::Exact, + ) + .await + .unwrap_err(); + assert!(matches!(err, LoweringError::UnsupportedFeature(_)), "{err}"); +} diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index 04b9f33e..d4885bff 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -1,6 +1,5 @@ //! Unified operator and scalar representation from #511. -//! Graph algorithms are added in the next stack layer; legacy consumers -//! remain on their existing representation until the planner cutover. +//! Legacy consumers remain on their existing representation until the planner cutover. pub mod aggregate_schema; pub mod asap; pub mod error; @@ -22,3 +21,5 @@ pub mod export; /// Semantic observation coverage, separate from field layout and physical timing. pub mod summary_coverage; mod wire; + +pub mod schema_support; diff --git a/crates/types/src/ir/schema_support.rs b/crates/types/src/ir/schema_support.rs new file mode 100644 index 00000000..c710c865 --- /dev/null +++ b/crates/types/src/ir/schema_support.rs @@ -0,0 +1,85 @@ +//! Series-identity realization for the unified dag. +use crate::pre_asap::schema::*; +pub fn with_promql_series_identity( + root: &std::rc::Rc, +) -> Result, String> { + use crate::ir::{NonASAPOp, Operator, OperatorNode}; + use crate::pre_asap::Source; + use std::{collections::HashMap, rc::Rc}; + fn visit( + node: &Rc, + memo: &mut HashMap<*const OperatorNode, Rc>, + ) -> Result, String> { + if let Some(found) = memo.get(&Rc::as_ptr(node)) { + return Ok(Rc::clone(found)); + } + let mut error = None; + let mut operator = node + .operator + .map_children(|child| match visit(child, memo) { + Ok(child) => child, + Err(e) => { + error = Some(e); + Rc::clone(child) + } + }); + if let Some(error) = error { + return Err(error); + } + match &mut operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { .. }, + schema, + .. + }) => { + if schema + .fields + .iter() + .any(|field| field.name == PROMQL_SERIES_IDENTITY) + { + if !schema.has_promql_series_identity() { + return Err("invalid physical series identity".into()); + } + memo.insert(Rc::as_ptr(node), Rc::clone(node)); + return Ok(Rc::clone(node)); + } + if schema.closed { + return Err("dynamic series identity requires an open PromQL source".into()); + } + schema.fields.push(Field::new( + PROMQL_SERIES_IDENTITY, + FieldDataType::Plain(DataType::Utf8), + false, + )); + schema.closed = true; + } + Operator::NonASAP(NonASAPOp::Sort { partition_by, .. }) + if partition_by.is_without() => + { + return Err("dynamic without ranking requires label-set projection".into()); + } + Operator::NonASAP( + NonASAPOp::TimeRange { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlSubquery { .. } + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlVectorFromScalar(_) + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Sort { .. }, + ) => {} + _ => return Err("operator has no dynamic series-identity realization".into()), + } + let mut rebuilt = OperatorNode::new(operator).map_err(|e| e.to_string())?; + rebuilt.guarantee = node.guarantee.clone(); + rebuilt.timing = node.timing; + let rebuilt = Rc::new(rebuilt); + memo.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); + Ok(rebuilt) + } + visit(root, &mut HashMap::new()) +} diff --git a/crates/types/src/pre_asap/mod.rs b/crates/types/src/pre_asap/mod.rs index f434eb15..1a017ec2 100644 --- a/crates/types/src/pre_asap/mod.rs +++ b/crates/types/src/pre_asap/mod.rs @@ -62,3 +62,5 @@ pub use query_expr::{ pub use resolve::{resolve_root, ResolveDAGError}; pub use schema::{ColumnId, DataType, Field, FieldDataType, Schema}; pub use schema_resolver::{SchemaCatalog, SchemaResolver, UsageDerivedCatalog}; + +pub use crate::ir::SchemaDerivationError;