diff --git a/Cargo.lock b/Cargo.lock index de76c170..575ab8bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -318,8 +318,8 @@ name = "asap-aware-mapping" version = "0.1.0" dependencies = [ "asap-frontend-promql", + "asap-logical-optimizer", "asap-types", - "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", "serde", "serde_json", "thiserror 2.0.18", @@ -332,6 +332,7 @@ dependencies = [ "asap-aware-mapping", "asap-frontend-promql", "asap-frontend-sql", + "asap-logical-optimizer", "asap-types", "serde", "serde_json", @@ -364,6 +365,7 @@ version = "0.1.0" dependencies = [ "asap-aware-mapping", "asap-frontend-common", + "asap-logical-optimizer", "asap-types", "promql-parser", ] @@ -374,6 +376,7 @@ version = "0.1.0" dependencies = [ "asap-aware-mapping", "asap-frontend-common", + "asap-logical-optimizer", "asap-sql-function-catalog", "asap-types", "datafusion", @@ -390,6 +393,7 @@ dependencies = [ "asap-aware-mapping", "asap-frontend-promql", "asap-frontend-sql", + "asap-logical-optimizer", "asap-physical-operators", "asap-planner", "asap-types", @@ -399,12 +403,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "asap-logical-optimizer" +version = "0.1.0" +dependencies = [ + "asap-frontend-promql", + "asap-types", + "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "asap-physical-operators" version = "0.1.0" dependencies = [ "asap-aware-mapping", "asap-frontend-promql", + "asap-logical-optimizer", "asap-types", "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", "chrono", @@ -425,6 +441,7 @@ dependencies = [ "asap-frontend-metricsql", "asap-frontend-promql", "asap-frontend-sql", + "asap-logical-optimizer", "asap-types", "thiserror 2.0.18", "tokio", diff --git a/Cargo.toml b/Cargo.toml index b3302644..1da0fa04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/frontend-common", "crates/sql-function-catalog", "crates/asap-aware-mapping", + "crates/logical-optimizer", "crates/frontend-promql", "crates/frontend-metricsql", "crates/metricsql-common-parser-support", diff --git a/crates/asap-aware-mapping/Cargo.toml b/crates/asap-aware-mapping/Cargo.toml index b2c03ed1..f96a5311 100644 --- a/crates/asap-aware-mapping/Cargo.toml +++ b/crates/asap-aware-mapping/Cargo.toml @@ -3,13 +3,11 @@ name = "asap-aware-mapping" version = "0.1.0" edition = "2021" -# The cost-aware optimizer layer (L4 decisions) over the L3 intent algebra: -# selects and sizes the summary family for each intent. Consumes and produces -# asap-types (L3 intent algebra + L4 sketch-bound IR) — never depends on a -# front end. +# #509 Stage 2, Stage 3 and the planner facade over the Stage 1 candidates of +# asap-logical-optimizer. Never depends on a front end. [dependencies] -asap_sketchlib = { workspace = true } asap-types = { path = "../types" } +asap-logical-optimizer = { path = "../logical-optimizer" } thiserror = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 8735cb8d..c9a7c88b 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -31,25 +31,25 @@ //! `docs/design_docs/cse-cost-model-decision.md` for the full design discussion (why //! cost-based, why not a full plan-search engine, the layering constraint //! that forces detection to stay cost-agnostic). -//! [`CandidateLogicalASAPDAGs::cost_sorted`](crate::replacement::CandidateLogicalASAPDAGs::cost_sorted) -//! (via [`crate::replacement`]'s own `cse_preference`) and +//! [`cost_sorted`](crate::plan_selection::candidate_selection::cost_sorted) +//! (via [`asap_logical_optimizer::pass1::replacement`]'s own `cse_preference`) and //! [`DefaultCostModel::estimate_cost`] are this crate's own callers. use std::rc::Rc; -use crate::exact_composition::ExactOperation; +use asap_logical_optimizer::pass1::exact_composition::ExactOperation; use asap_types::ir::operator::agg_intent::AggIntent; use asap_types::ir::schema::{ FieldDataType, GroupingStrategy, HydraParams, SketchAlgorithm, SketchParams, }; use asap_types::ir::{ASAPOp, Operator, OperatorNode}; -use crate::exact_composition::{ExactComposition, OperationPlacement}; use crate::recurrence::{ self, CostRate, EvaluationRate, Horizon, RecurrenceCostExplanation, RecurrenceError, RecurrenceProfile, }; -use crate::replacement::{ +use asap_logical_optimizer::pass1::exact_composition::{ExactComposition, OperationPlacement}; +use asap_logical_optimizer::pass1::replacement::{ realize_child, Replacement, ReplacementProvenance, ReplacementSubDAG, TargetSubDAG, }; @@ -70,7 +70,7 @@ pub struct CostProvenance { } /// Which mixed-execution shapes the downstream runtime can actually -/// execute (issue #171). [`crate::exact_composition::ExactCompositionStrategy`] +/// execute (issue #171). [`asap_logical_optimizer::pass1::exact_composition::ExactCompositionStrategy`] /// proposes an `ValueOperationAtQueryTime` candidate only when /// `query_time` is set, and an `ValueOperationAtIngestionTime` candidate only /// when `ingestion_time` is — a runtime that cannot run an exact @@ -122,7 +122,7 @@ pub struct ExactCompositionCostRequest<'a> { /// formula charges. pub summary: &'a OperatorNode, /// How many times this site actually runs once ancestors' own choices - /// are accounted for (see `CandidateLogicalASAPDAGs::global_selection`). + /// are accounted for (see `candidate_selection::global_selection`). pub effective_consumer_count: usize, } @@ -251,8 +251,8 @@ fn finite_rate(units_per_second: f64) -> Option { /// A CSE-detected, legality-gated shared sub-DAG with two or more consumers /// — the unit [`CostModel::cse_share_decision`] decides over. Built by -/// [`CandidateLogicalASAPDAGs::cost_sorted`](crate::replacement::CandidateLogicalASAPDAGs::cost_sorted) -/// (via [`crate::replacement`]'s own `cse_preference`) the first time it +/// [`cost_sorted`](crate::plan_selection::candidate_selection::cost_sorted) +/// (via [`asap_logical_optimizer::pass1::replacement`]'s own `cse_preference`) the first time it /// needs a representative bound node for a sub-DAG that /// [`asap_types::ir::cse::share_common_sub_dags`] already collapsed /// onto one `Rc` for two or more workload roots. See @@ -403,7 +403,7 @@ pub trait CostModel { } /// Rank `candidates` (as returned by - /// [`summary_candidates`](crate::replacement::summary_candidates)) for + /// [`summary_candidates`](asap_logical_optimizer::pass1::replacement::summary_candidates)) for /// `intent`, best choice first. /// /// Implementations MAY reorder freely, but MUST return exactly the input @@ -413,7 +413,7 @@ pub trait CostModel { /// [`ReplacementStrategy`]'s exhaustive, never-prune contract. This /// invariant is checked at every production call site. /// - /// [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy + /// [`ReplacementStrategy`]: asap_logical_optimizer::pass1::replacement::ReplacementStrategy fn rank_candidates( &self, intent: &AggIntent, @@ -619,12 +619,12 @@ pub trait CostModel { /// [`ReplacementSubDAG`] candidate at `target` — a real `f64`, not just a /// relative rank, meant for a caller that wants to *display* "candidate A /// costs ≈ X, candidate B costs ≈ Y" (e.g. a DAG-visualization view built - /// on [`CandidateLogicalASAPDAGs::cost_sorted`](crate::replacement::CandidateLogicalASAPDAGs::cost_sorted)), + /// on [`cost_sorted`](crate::plan_selection::candidate_selection::cost_sorted)), /// not just order candidates against each other — that ordering job /// already belongs to [`rank_candidates`](Self::rank_candidates) (for a - /// [`ASAPStrategies`](crate::replacement::ASAPStrategies) + /// [`ASAPStrategies`](asap_logical_optimizer::pass1::replacement::ASAPStrategies) /// group) and [`cse_share_decision`](Self::cse_share_decision) (for a - /// [`SharedSubDAGStrategy`](crate::replacement::SharedSubDAGStrategy) + /// [`SharedSubDAGStrategy`](asap_logical_optimizer::pass1::replacement::SharedSubDAGStrategy) /// group). /// /// One method covers both candidate shapes this crate ships: @@ -663,7 +663,7 @@ pub trait CostModel { /// Which mixed exact/summary execution shapes the downstream runtime /// advertises (issue #171). Gates candidate *generation* in - /// [`crate::exact_composition::ExactCompositionStrategy`]: a shape the + /// [`asap_logical_optimizer::pass1::exact_composition::ExactCompositionStrategy`]: a shape the /// runtime can't execute is never proposed, so it can't be selected /// either. /// @@ -710,7 +710,7 @@ pub trait CostModel { /// /// Default: every input unknown ([`ExactCompositionCostInputs::unknown`]) /// — unknown is never zero, and with no rate derivable - /// `CandidateLogicalASAPDAGs::global_selection` keeps the conservative keep-as-is + /// `candidate_selection::global_selection` keeps the conservative keep-as-is /// behavior for the site. A deployment that wants defaults must supply /// them here explicitly. fn exact_composition_cost_inputs( @@ -790,7 +790,7 @@ pub(crate) fn validated_candidate_ranking( /// The default cost model: preserves [`summary_candidates`]'s built-in static /// order. /// -/// [`summary_candidates`]: crate::replacement::summary_candidates +/// [`summary_candidates`]: asap_logical_optimizer::pass1::replacement::summary_candidates pub struct DefaultCostModel; impl CostModel for DefaultCostModel { @@ -887,7 +887,7 @@ impl CostModel for DefaultCostModel { } } // A composed candidate is costed in cost-units-per-second by - // `CandidateLogicalASAPDAGs::global_selection` against the child decision it + // `candidate_selection::global_selection` against the child decision it // is committed with — a different unit from this structural // estimate, and unknowable here without that child. `NaN` // keeps it from ever out-ranking a real estimate by accident. @@ -899,7 +899,7 @@ impl CostModel for DefaultCostModel { #[cfg(test)] mod tests { use super::*; - use crate::replacement::summary_candidates; + use asap_logical_optimizer::pass1::replacement::summary_candidates; use asap_types::ir::operator::agg_intent::default_cardinality; #[test] @@ -1289,7 +1289,7 @@ mod tests { replacement: Replacement::SubDAG(summary_node(FieldDataType::Plain( asap_types::ir::schema::DataType::Float64, ))), - provenance: crate::replacement::ReplacementProvenance::SummaryRealization, + provenance: asap_logical_optimizer::pass1::replacement::ReplacementProvenance::SummaryRealization, rationale: "whatever".into(), }; assert!(RankOnly.estimate_cost(&candidate, &target).is_nan()); @@ -1314,7 +1314,7 @@ mod tests { ExactKind::Sum, ExactParams::Sum, ))), - provenance: crate::replacement::ReplacementProvenance::SummaryRealization, + provenance: asap_logical_optimizer::pass1::replacement::ReplacementProvenance::SummaryRealization, rationale: "exact accumulator".into(), }; let pricey = ReplacementSubDAG { @@ -1325,7 +1325,7 @@ mod tests { family: "gaussian_mixture".into(), }, ))), - provenance: crate::replacement::ReplacementProvenance::SummaryRealization, + provenance: asap_logical_optimizer::pass1::replacement::ReplacementProvenance::SummaryRealization, rationale: "fitted statistical model".into(), }; @@ -1359,13 +1359,14 @@ mod tests { let share = ReplacementSubDAG { strategy: "TestStrategy", replacement: Replacement::SubDAG(Rc::clone(&target_root)), - provenance: crate::replacement::ReplacementProvenance::CseShare, + provenance: asap_logical_optimizer::pass1::replacement::ReplacementProvenance::CseShare, rationale: "build once and share".into(), }; let recompute = ReplacementSubDAG { strategy: "TestStrategy", replacement: Replacement::SubDAG(Rc::new((*target_root).clone())), - provenance: crate::replacement::ReplacementProvenance::CseRecompute, + provenance: + asap_logical_optimizer::pass1::replacement::ReplacementProvenance::CseRecompute, rationale: "build independently".into(), }; diff --git a/crates/asap-aware-mapping/src/empirical_cost.rs b/crates/asap-aware-mapping/src/empirical_cost.rs index 96822bcc..eda0ee97 100644 --- a/crates/asap-aware-mapping/src/empirical_cost.rs +++ b/crates/asap-aware-mapping/src/empirical_cost.rs @@ -7,7 +7,7 @@ use asap_types::ir::schema::{SketchAlgorithm, SketchParams}; use serde::{Deserialize, Serialize}; use crate::cost_model::{CostModel, DefaultCostModel}; -use crate::replacement::{ +use asap_logical_optimizer::pass1::replacement::{ accuracy_budget, accuracy_target, default_size_params, ReplacementSubDAG, TargetSubDAG, }; diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index aaeec08b..95e6cb8b 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -1,165 +1,46 @@ -//! `asap-plan` — the cost-aware optimizer layer over the pre-ASAP intent algebra. +//! `asap-aware-mapping` — #509 Stage 2, Stage 3 and the planner facade, over +//! the Stage 1 candidates of [`asap_logical_optimizer`]. //! -//! This crate sits between the language-agnostic IR ([`asap_ir`]) and -//! any runtime: it consumes pre-ASAP [`OperatorNode`](asap_types::ir::OperatorNode) -//! DAGs and makes the cost-aware decisions the pre-ASAP IR deliberately -//! leaves open — which sketch (if any) realises each approximate intent. -//! -//! **Common sub-expression elimination (CSE) is not this crate's job.** -//! Detection is a primary pass over the pre-ASAP operator IR itself -//! (`asap_types::ir::cse`, design tracked in issue #223), run before a -//! tree ever reaches [`replacement::ASAPStrategies`] — see issue #222 -//! for why (batch query optimization needs to see shared work across a -//! `QueryWorkload` before summary binding, not after). This crate may -//! eventually run a second, narrower CSE pass of its own over an -//! already-bound post-ASAP `OperatorNode` DAG, recognizing sharing that's invisible -//! at the pre-ASAP level by construction — e.g. `Quantile(x, 0.99)` and -//! `Quantile(x, 0.95)` are structurally distinct `AggIntent`s but can -//! still share one built sketch, read out twice. That post-ASAP pass is -//! secondary to, and downstream of, the primary pre-ASAP pass, not a -//! replacement for it. -//! -//! It depends only on the IR crate, never on a front end — the layering -//! invariant (arrows point up) holds here too. -//! -//! Post-lowering **canonicalization** is *not* here: it lives in -//! `asap_types::ir::canonicalize`, run inside `asap_frontend_common`'s shared -//! `resolve_root` so every front end normalizes before the IR leaves resolution -//! (issue #34, closed). +//! It is being split into one crate per stage (#572): Stage 1 already lives in +//! `asap-logical-optimizer`; Stage 2 (`physical_candidates`), Stage 3 +//! (`plan_selection`, the cost model and its inputs) and the facade (`pass`) +//! remain here for now. //! //! ## Planning workflows //! -//! Candidate search returns [`CandidateLogicalASAPDAGs`](replacement::CandidateLogicalASAPDAGs), a compact -//! logical choice space with one [`TargetSubDAGCandidates`] per target sub-DAG. -//! [`ReplacementStrategy`] implementations propose local alternatives; search -//! applies the applicable semantic and accuracy checks. Candidate presence does -//! not certify physical deployability or an unknown accuracy guarantee. -//! -//! Integrators choose among these workflows: -//! -//! - Inspect the candidate space, optionally using [`CandidateLogicalASAPDAGs::cost_sorted`] -//! to obtain ranked views, and perform selection downstream. -//! - Call [`CandidateLogicalASAPDAGs::global_selection`] once for the workload, then -//! [`GlobalSelection::assemble_selected_dag`] for each query root. This -//! coordinates logical choices and preserves shared nodes. Whether and when -//! a summary state is materialized is not decided here: every summary runs -//! at query time until Stage 2 materialization (#509) owns that choice. //! - Run the #509 stage pipeline through [`optimize`] with [`StagePipeline`]: //! Stage 1 local alternatives, Stage 2 physical candidates, and Stage 3 -//! selection, the only stage that prices plans. It does not use the -//! candidate search above. +//! selection, the only stage that prices plans. +//! - Legacy: search the workload with +//! [`asap_logical_optimizer::search_workload`], then rank with +//! [`cost_sorted`](plan_selection::candidate_selection::cost_sorted) or +//! select with +//! [`global_selection`](plan_selection::candidate_selection::global_selection) +//! and assemble each query root with +//! [`GlobalSelection::assemble_selected_dag`](asap_logical_optimizer::GlobalSelection::assemble_selected_dag). +//! Every summary runs at query time until Stage 2 materialization (#509) +//! owns that choice. This path is deleted under #580. //! -//! Models and evidence determine which choices the helpers can justify. //! Physical operator binding, placement, storage, deployment, and execution -//! remain downstream responsibilities. Neither taking the first candidate nor -//! assembling a logical DAG creates an executable deployment plan. +//! remain downstream responsibilities. //! //! ## Supporting components //! //! - [`cost_model`] — the [`CostModel`](cost_model::CostModel) trait every -//! deployment's cost-based sketch selection plugs into (issues #6, #33). -//! `asap-plan` itself only ships [`DefaultCostModel`](cost_model::DefaultCostModel), -//! which preserves [`replacement`]'s built-in static preference order and -//! — via [`CostModel::estimate_cost`](cost_model::CostModel::estimate_cost) -//! — exposes an actual numeric cost per candidate, not just a relative -//! rank, for a caller (e.g. a DAG-visualization view) that wants to show -//! "candidate A costs ≈ X" next to "candidate B costs ≈ Y". -//! - [`explanation`] — this crate's explanation of a replacement: a -//! reporting *view* over [`replacement`]'s candidate-plan space (issue -//! #257, part of #33) that translates every discovered `TargetSubDAG` with -//! a non-trivial candidate list into an -//! [`explanation::ReplacementExplanation`] (why a replacement exists, -//! where, reusing the candidate's own rationale rather than inventing new -//! prose), meant for the same downstream consumer (e.g. a -//! DAG-visualization view) the crate doc's planning workflows section above -//! already names for [`replacement::CandidateLogicalASAPDAGs`] itself. Superseded PR -//! #247's own rule-based traversal, which re-walked the DAG once per -//! optimization before [`replacement::search_workload`] existed to read -//! from instead — see that module's docs for the full reframing. -//! - [`rollup`] — [`rollup::RollupStrategy`] wraps group-by-lattice roll-up -//! reuse (issue #254, part of #33) as a [`ReplacementStrategy`]: given a -//! coarser `Aggregate` target and a caller-supplied sibling set, proposes -//! re-deriving it from an already-computed, strictly finer sibling -//! `Aggregate` over identical child IR instead of an independent pass -//! over the raw source — the cross-aggregate sibling of -//! `ir::cse::share_common_sub_dags`'s identical-sub-DAG sharing. -//! [`rollup::is_legal_rollup_source`] is the standalone legality predicate -//! other axes (e.g. issue #256's `GroupingStrategy`) are expected to -//! consult directly, so it and this module's `RollupStrategy` can never -//! disagree about which siblings qualify. -//! - [`grouping`] — [`grouping::HydraGroupingStrategy`] (issue #256, part of -//! #33) is an additional `ReplacementStrategy`: the orthogonal -//! `GroupingStrategy` axis (one summary instance per `by` subpopulation -//! versus one shared Hydra-family structure serving all of them), offered -//! alongside the candidates [`replacement::ASAPStrategies`] -//! enumerates for the same target. -//! - [`rewrite`] — the "semantic-equivalent rewriting (e.g. `avg` → -//! `sum`/`count`) to increase how often the [sharing/sketch] optimizations -//! above apply" degree of freedom `docs/design_docs/asap_aware_mapping.md` -//! names (issue #253, part of #33): [`rewrite::AvgToSumOverCountStrategy`] -//! is a [`replacement::ReplacementStrategy`] that reshapes a bare `avg` -//! node — which [`replacement::realizations_for_intent`] can only -//! dispatch to `Realization::PassThrough`, so it can never be a -//! [`replacement::SharedSubDAGStrategy`] target — into a `sum`/`count` -//! pair under the same grouping, re-divided back by a wrapping `Project`, -//! so those *are* ordinary mergeable accumulators sharing/sketching can -//! reach. It only reshapes; [`replacement::search_workload`]'s cost-based -//! ranking (or a downstream consumer reading [`replacement::CandidateLogicalASAPDAGs`]) -//! is what decides whether the reshaped form is actually worth picking, -//! the same propose-don't-decide split every other strategy here keeps. -//! -//! ## Terminology -//! -//! Schema resolution, candidate realization, and runtime placement are distinct stages. -//! -//! | Term | Meaning | Entry point | -//! |---|---|---| -//! | Schema resolution | Derive input schemas and resolve column names to positions | `asap_frontend_common::schema_resolver::SchemaResolver::resolve_schema`, `asap_frontend_common::resolve::resolve_root` | -//! | Realization | Enumerate ranked physical forms for one aggregate intent | `replacement::realizations_for_intent` | -//! | Replacement | Construct each candidate summary sub-DAG | [`replacement::ASAPStrategies`] | -//! | Search | Enumerate and compare alternatives across a workload | [`replacement::search_workload`] | -//! | Runtime placement | Choose deployment locations and concrete executors | Downstream physical plan providers | -//! -//! A related question (tracked alongside issues #6/#33): whether this -//! crate should also own a **matching** predicate — "does an already -//! *available* `Realization` satisfy a *required* one" — the way a -//! database's materialized-view matching / "answering queries using -//! views" layer does. It owns the *question*, not an *answer*: -//! [`replacement::Matcher`] is a trait with no default implementation and -//! no shipped instance, the same shape as [`cost_model::CostModel`] and for -//! the same reason — which `Realization`s are actually *available* -//! anywhere is entirely a downstream deployment's concern (an inventory -//! this crate has no way to see), and even the pure sketch-algebra -//! compatibility rules (e.g. a heap-bearing top-k sketch also satisfying a -//! bare frequency point-query) turned out to have deployment-specific -//! competitors (e.g. single-vs-multi-population re-aggregation) that -//! don't reduce to a fact about a summary family's kind alone. `control_plane`'s own -//! `sketch_algebra::capability::Capability`/`is_satisfied_by` is the -//! reference downstream implementation. -//! -//! - [`accuracy`] — the [`AccuracyModel`](accuracy::AccuracyModel) / -//! [`AccuracyBudgetAllocator`](accuracy::AccuracyBudgetAllocator) -//! extension points (issue #172): the planning-time algebra that derives -//! a machine-readable [`ResultGuarantee`](asap_types::ir::properties::ResultGuarantee) -//! for every finalized post-ASAP value, propagates it through -//! approximate-over-approximate compositions under conservative rules -//! (no independence assumptions, unknown statistics stay unknown), and -//! rejects — before any `CostModel` ranks anything — every candidate with -//! no sound rule or one that misses the applicable `AccuracyTarget`. -//! Legality and cost are separate responsibilities; see that module's -//! docs for the pipeline order and the root-vs-per-node precedence rules. +//! deployment's cost-based selection plugs into (issues #6, #33). This crate +//! ships [`DefaultCostModel`](cost_model::DefaultCostModel), which keeps the +//! built-in static preference order and exposes a numeric cost per +//! candidate through [`CostModel::estimate_cost`](cost_model::CostModel::estimate_cost). +//! - [`recurrence`] — recurring and one-shot cost rates over a horizon. +//! - [`analytical_cost`], [`physical_plan_cost_model`], [`empirical_cost`] — +//! analytical and evidence-based pricing for Stage 3. -pub mod accuracy; pub mod analytical_cost; pub mod cost_model; pub mod empirical_comparison; pub mod empirical_cost; pub mod empirical_resources; pub mod erp; -pub mod exact_composition; -pub mod explanation; -mod function_rules; -pub mod grouping; pub mod pane_sharing; pub mod pass; pub mod physical_handoff_cost; @@ -167,58 +48,27 @@ pub mod physical_operator_statistics; pub mod physical_plan_cost_model; pub mod query_physical_lowering; pub mod recurrence; -pub mod replacement; -pub mod rewrite; -pub mod rollup; pub mod storage_io; #[cfg(test)] mod test_support; -pub mod topk_reuse; -pub use accuracy::reconciliation::AccuracyReconciliationStrategy; -pub use accuracy::{ - AccuracyAllocation, AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, - CompositionShape, DefaultAccuracyModel, EqualSplitAllocator, NoAccuracyEvidence, - PropagationStats, WorkloadAccuracyEvidence, -}; pub use cost_model::{ maintenance_operation_plan_cost_rate, raw_recompute_cost_rate, read_operation_plan_cost_rate, CostModel, CostProvenance, CostUnit, DefaultCostModel, ExactCompositionCostInputs, ExactCompositionCostRequest, ValueOperationCapabilities, }; -pub use exact_composition::{ExactComposition, ExactCompositionStrategy, OperationPlacement}; -pub use explanation::{ - explain_replacements, explain_replacements_with, ExplanationKind, ReplacementExplanation, -}; -pub use grouping::{has_subpopulations, HydraGroupingStrategy}; pub use pass::{ optimize, OptimizationInput, OptimizationInputError, OptimizationPass, OptimizeError, PassNameConflict, PassRegistry, PlanOutput, PlanningModels, QueryPlan, StagePipeline, }; pub use plan_selection::candidate_selection::{ - CompositionDecision, GlobalSelection, RankedTargetSubDAGCandidates, RecurrenceProfileMap, - TargetSubDAGSelection, + CompositionDecision, CostedGlobalSelection, RankedTargetSubDAGCandidates, RecurrenceProfileMap, }; pub use recurrence::{ evaluation_rate_of, total_cost, update_rate_from_data_workload, CostRate, EvaluationRate, Horizon, RecurrenceCostExplanation, RecurrenceError, RecurrenceProfile, RootRecurrence, UpdateRate, }; -pub use replacement::{ - default_strategies, is_logical_rewrite, search_workload, search_workload_with, - search_workload_with_targets, summary_candidates, ASAPStrategies, CandidateLogicalASAPDAGs, - Matcher, Proposals, Realization, RealizationError, RejectedCandidate, Replacement, - ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, SharedSubDAGStrategy, - TargetSubDAG, TargetSubDAGCandidates, MAX_SEARCH_ITERATIONS, -}; -pub use rewrite::{AvgToSumOverCountStrategy, SemanticEquivalentRewriteStrategy}; -pub use topk_reuse::TopKLimitReuseStrategy; - -pub mod maintained_population; - -/// Local candidate generation over the unified IR. No execution timing is -/// assigned: that is a Stage 2 materialization decision. -pub mod logical_candidates; /// #509 Stage 2 MVP: physical operator implementation, all at query time. pub mod physical_candidates; diff --git a/crates/asap-aware-mapping/src/pass/mod.rs b/crates/asap-aware-mapping/src/pass/mod.rs index 9a8cd661..8cb91af9 100644 --- a/crates/asap-aware-mapping/src/pass/mod.rs +++ b/crates/asap-aware-mapping/src/pass/mod.rs @@ -26,8 +26,8 @@ use asap_types::ir::OperatorNode; use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; -use crate::logical_candidates::LogicalCandidateError; use crate::plan_selection::{Selection, SelectionError}; +use asap_logical_optimizer::pass1::logical_candidates::LogicalCandidateError; pub use crate::plan_selection::PlanningModels; pub use stage_pipeline::StagePipeline; diff --git a/crates/asap-aware-mapping/src/pass/stage_pipeline.rs b/crates/asap-aware-mapping/src/pass/stage_pipeline.rs index e3e1b802..d974b30c 100644 --- a/crates/asap-aware-mapping/src/pass/stage_pipeline.rs +++ b/crates/asap-aware-mapping/src/pass/stage_pipeline.rs @@ -14,8 +14,8 @@ use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::workload::QueryLanguage; use super::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, QueryPlan}; -use crate::logical_candidates::enumerate_local_logical_candidates; use crate::plan_selection::select_plan; +use asap_logical_optimizer::pass1::logical_candidates::enumerate_local_logical_candidates; #[derive(Debug, Default, Clone, Copy)] pub struct StagePipeline; diff --git a/crates/asap-aware-mapping/src/physical_candidates.rs b/crates/asap-aware-mapping/src/physical_candidates.rs index 21d61859..25507b78 100644 --- a/crates/asap-aware-mapping/src/physical_candidates.rs +++ b/crates/asap-aware-mapping/src/physical_candidates.rs @@ -244,11 +244,11 @@ mod tests { delta: 0.001, }, ); - let inventory = crate::logical_candidates::enumerate_local_logical_candidates(vec![( - 0, - QueryRoot::Operator(root), - )]) - .unwrap(); + let inventory = + asap_logical_optimizer::pass1::logical_candidates::enumerate_local_logical_candidates( + vec![(0, QueryRoot::Operator(root))], + ) + .unwrap(); // A summary (the last alternative) for every target. let choice: Vec<_> = inventory .targets @@ -256,14 +256,16 @@ mod tests { .map(|t| t.alternatives.len() - 1) .collect(); let roots: Vec<_> = - crate::logical_candidates::compose_logical_candidate(&inventory, &choice) - .unwrap() - .into_iter() - .map(|(_, root)| match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(); + asap_logical_optimizer::pass1::logical_candidates::compose_logical_candidate( + &inventory, &choice, + ) + .unwrap() + .into_iter() + .map(|(_, root)| match root { + QueryRoot::Operator(node) => node, + QueryRoot::Scalar(_) => panic!("operator root"), + }) + .collect(); let candidate = stage2_physical("L1", &roots).unwrap(); assert!(candidate .dag diff --git a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs index b787267d..a3e33749 100644 --- a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs +++ b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs @@ -17,7 +17,7 @@ use crate::physical_operator_statistics::ComparisonScope; use crate::query_physical_lowering::{ lower_query_physical_dag, PhysicalNodeEvidenceProvider, PhysicalNodeRequest, }; -use crate::replacement::{Replacement, ReplacementSubDAG, TargetSubDAG}; +use asap_logical_optimizer::pass1::replacement::{Replacement, ReplacementSubDAG, TargetSubDAG}; /// One immutable generation of deployment evidence for a planner target. /// @@ -356,6 +356,7 @@ impl CostModel for PhysicalPlanCostModel<'_> { #[cfg(test)] mod tests { use super::*; + use crate::plan_selection::candidate_selection::global_selection; use std::cell::Cell; use std::collections::HashMap; @@ -371,7 +372,7 @@ mod tests { use crate::physical_operator_statistics::{ EdgeStatistics, OperatorStatistics, ScanSelection, UnaryEdgeStatistics, }; - use crate::replacement::ReplacementStrategy; + use asap_logical_optimizer::pass1::replacement::ReplacementStrategy; fn edge(rows: u64, bytes: u64) -> EdgeStatistics { EdgeStatistics { rows, bytes } @@ -691,7 +692,8 @@ mod tests { cost_per_retained_byte: 0.0, version: "unused-base-v1".into(), }; - let candidates = crate::replacement::ASAPStrategies::default().replacements(&target); + let candidates = asap_logical_optimizer::pass1::replacement::ASAPStrategies::default() + .replacements(&target); provider.storage_io = Some(profile.clone()); let model = PhysicalPlanCostModel::new(&provider, base.clone()).unwrap(); let estimate = model.estimate_candidate(&candidates[0], &target).unwrap(); @@ -721,7 +723,8 @@ mod tests { fn missing_storage_profile_remains_unestimated() { let root = query(); let target = TargetSubDAG::new(&root); - let candidates = crate::replacement::ASAPStrategies::default().replacements(&target); + let candidates = asap_logical_optimizer::pass1::replacement::ASAPStrategies::default() + .replacements(&target); let provider = TestProvider::new(true, 800); let model = PhysicalPlanCostModel::new(&provider, calibration()).unwrap(); let estimate = model.estimate_candidate(&candidates[0], &target).unwrap(); @@ -816,12 +819,12 @@ mod tests { cost_per_retained_byte: 0.0, version: "handoff-only-v1".into(), }; - let space = crate::replacement::search_workload_with( + let space = asap_logical_optimizer::pass1::replacement::search_workload_with( vec![("q", Rc::clone(&root))], - &crate::replacement::default_strategies(), + &asap_logical_optimizer::pass1::replacement::default_strategies(), ); let model = PhysicalPlanCostModel::new(&provider, zero_base.clone()).unwrap(); - let selected = space.global_selection(&model); + let selected = global_selection(&space, &model); assert!(selected .for_target(&space.roots[0].1) .unwrap() @@ -836,8 +839,7 @@ mod tests { .calibration .cost_per_network_byte = coefficient; let model = PhysicalPlanCostModel::new(&provider, zero_base.clone()).unwrap(); - assert!(space - .global_selection(&model) + assert!(global_selection(&space, &model) .for_target(&space.roots[0].1) .unwrap() .chosen @@ -845,8 +847,7 @@ mod tests { } provider.handoffs = None; let model = PhysicalPlanCostModel::new(&provider, zero_base).unwrap(); - assert!(space - .global_selection(&model) + assert!(global_selection(&space, &model) .for_target(&space.roots[0].1) .unwrap() .chosen @@ -856,15 +857,15 @@ mod tests { #[test] fn global_selection_uses_complete_physical_comparison() { let root = query(); - let space = crate::replacement::search_workload_with( + let space = asap_logical_optimizer::pass1::replacement::search_workload_with( vec![("q", Rc::clone(&root))], - &crate::replacement::default_strategies(), + &asap_logical_optimizer::pass1::replacement::default_strategies(), ); let planned_root = Rc::clone(&space.roots[0].1); let provider = TestProvider::new(true, 800); let model = PhysicalPlanCostModel::new(&provider, calibration()).unwrap(); - let selected = space.global_selection(&model); + let selected = global_selection(&space, &model); assert!( selected.for_target(&planned_root).unwrap().chosen.is_some(), "a fully bound build-once summary cheaper than ten raw scans must be selected" @@ -922,9 +923,9 @@ mod tests { self.0.summary_physical_dag(snapshot, summary, target) } } - let space = crate::replacement::search_workload_with( + let space = asap_logical_optimizer::pass1::replacement::search_workload_with( vec![("q", query())], - &crate::replacement::default_strategies(), + &asap_logical_optimizer::pass1::replacement::default_strategies(), ); let provider = AlmostResident(TestProvider::new(true, WORKING_SET)); let model = PhysicalPlanCostModel::new( @@ -937,7 +938,7 @@ mod tests { }, ) .unwrap(); - let selected = space.global_selection(&model); + let selected = global_selection(&space, &model); assert!( selected .for_target(&space.roots[0].1) @@ -967,15 +968,15 @@ mod tests { #[test] fn missing_summary_evidence_keeps_the_raw_target() { let root = query(); - let space = crate::replacement::search_workload_with( + let space = asap_logical_optimizer::pass1::replacement::search_workload_with( vec![("q", Rc::clone(&root))], - &crate::replacement::default_strategies(), + &asap_logical_optimizer::pass1::replacement::default_strategies(), ); let planned_root = Rc::clone(&space.roots[0].1); let provider = TestProvider::new(false, 800); let model = PhysicalPlanCostModel::new(&provider, calibration()).unwrap(); - let selected = space.global_selection(&model); + let selected = global_selection(&space, &model); assert!( selected.for_target(&planned_root).unwrap().chosen.is_none(), "missing physical summary evidence must not fall back to a structural estimate" @@ -1018,8 +1019,8 @@ mod tests { } let root = query(); - let candidates = - crate::replacement::ASAPStrategies::default().replacements(&TargetSubDAG::new(&root)); + let candidates = asap_logical_optimizer::pass1::replacement::ASAPStrategies::default() + .replacements(&TargetSubDAG::new(&root)); let provider = WrongScope(TestProvider::new(true, 800)); let model = PhysicalPlanCostModel::new(&provider, calibration()).unwrap(); assert_eq!( @@ -1064,8 +1065,8 @@ mod tests { } let root = query(); - let candidates = - crate::replacement::ASAPStrategies::default().replacements(&TargetSubDAG::new(&root)); + let candidates = asap_logical_optimizer::pass1::replacement::ASAPStrategies::default() + .replacements(&TargetSubDAG::new(&root)); let model = PhysicalPlanCostModel::new(&BlankVersionProvider, calibration()).unwrap(); assert_eq!( model.candidate_cost(&candidates[0], &TargetSubDAG::new(&root)), @@ -1076,23 +1077,23 @@ mod tests { #[test] fn complete_candidate_that_costs_more_than_raw_is_not_selected() { let root = query(); - let space = crate::replacement::search_workload_with( + let space = asap_logical_optimizer::pass1::replacement::search_workload_with( vec![("q", Rc::clone(&root))], - &crate::replacement::default_strategies(), + &asap_logical_optimizer::pass1::replacement::default_strategies(), ); let planned_root = Rc::clone(&space.roots[0].1); let provider = TestProvider::new(true, 100_000); let model = PhysicalPlanCostModel::new(&provider, calibration()).unwrap(); - let selected = space.global_selection(&model); + let selected = global_selection(&space, &model); assert!(selected.for_target(&planned_root).unwrap().chosen.is_none()); } #[test] fn sibling_candidates_share_one_scope_and_raw_baseline() { let root = query(); - let candidates = - crate::replacement::ASAPStrategies::default().replacements(&TargetSubDAG::new(&root)); + let candidates = asap_logical_optimizer::pass1::replacement::ASAPStrategies::default() + .replacements(&TargetSubDAG::new(&root)); assert!(candidates.len() >= 2); let provider = TestProvider::new(true, 800); let model = PhysicalPlanCostModel::new(&provider, calibration()).unwrap(); diff --git a/crates/asap-aware-mapping/src/plan_selection/candidate_selection.rs b/crates/asap-aware-mapping/src/plan_selection/candidate_selection.rs index a57fb039..72d3a06b 100644 --- a/crates/asap-aware-mapping/src/plan_selection/candidate_selection.rs +++ b/crates/asap-aware-mapping/src/plan_selection/candidate_selection.rs @@ -1,170 +1,167 @@ -//! Whole-workload selection over the legacy Stage 1 search space +//! Legacy whole-workload selection over the Stage 1 search space //! ([`CandidateLogicalASAPDAGs`]): per-target cost ranking, recurrence -//! profiles, global selection and assembly of the selected DAG. It lives here, -//! not in `replacement`, so that Stage 1 does not depend on the cost model. -//! The stage pipeline does not call it; its deletion is tracked in #580. +//! profiles and global selection. These are free functions over Stage 1 +//! types, so that Stage 1 does not depend on the cost model. Assembling the +//! selected DAG is Stage 1's [`GlobalSelection`]; [`CostedGlobalSelection`] +//! adds the cost comparison behind each chosen exact composition. +//! +//! The stage pipeline does not call this module. It is deleted under #580. -use std::cell::RefCell; use std::collections::{HashMap, HashSet, VecDeque}; use std::rc::Rc; -use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::operator::non_asap::any_measure_filtered; -use asap_types::ir::operator::operator_properties::JoinKind; -use asap_types::ir::properties::timing::validate_maintained; -use asap_types::ir::properties::{ExecutionTiming, ResultGuarantee}; -use asap_types::ir::schema::{ColumnId, FieldDataType, GroupingStrategy, SketchAlgorithm}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; +use asap_types::ir::schema::{FieldDataType, GroupingStrategy, SketchAlgorithm}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; use crate::cost_model::{ raw_recompute_cost_rate, CostModel, CseCandidate, ExactCompositionCostInputs, ExactCompositionCostRequest, ShareDecision, }; -use crate::exact_composition::OperationPlacement; use crate::recurrence::{ CostRate, Horizon, RecurrenceError, RecurrenceProfile, RootRecurrence, UpdateRate, }; -use crate::replacement::{ - bindable_intent, cse_candidate_pair, direct_child_counts, finalize_query_candidate, - is_logical_rewrite, realize_child, retain_exact, CandidateLogicalASAPDAGs, PreparedComposition, - RealizationError, Replacement, ReplacementProvenance, ReplacementSubDAG, TargetSubDAG, - TargetSubDAGCandidates, +use asap_logical_optimizer::pass1::exact_composition::OperationPlacement; +use asap_logical_optimizer::pass1::replacement::{ + bindable_intent, cse_candidate_pair, direct_child_counts, is_logical_rewrite, realize_child, + CandidateLogicalASAPDAGs, GlobalSelection, PreparedComposition, Replacement, + ReplacementProvenance, ReplacementSubDAG, TargetSubDAG, TargetSubDAGCandidates, + TargetSubDAGSelection, }; -impl ReplacementSubDAG { - /// Physical feasibility evidence for this candidate. A pure logical - /// rewrite needs no new operator. Unknown support is checked during - /// physical/deployment compilation; explicit rejection prevents selection. - pub fn runtime_support_evidence(&self, cost_model: &dyn CostModel) -> Option { - match &self.replacement { - Replacement::ExactComposition(composition) => { - cost_model.value_operation_support_evidence(&composition.op, composition.placement) - } - // Any summary decision, including one rooted in a relational - // operator above its evaluations, asks the deployment for support. - Replacement::SubDAG(node) if !is_logical_rewrite(node) => { - cost_model.summary_support_evidence(node) - } - Replacement::SubDAG(_) => Some(true), +/// Physical feasibility evidence for `candidate`. A pure logical +/// rewrite needs no new operator. Unknown support is checked during +/// physical/deployment compilation; explicit rejection prevents selection. +pub fn runtime_support_evidence( + candidate: &ReplacementSubDAG, + cost_model: &dyn CostModel, +) -> Option { + match &candidate.replacement { + Replacement::ExactComposition(composition) => { + cost_model.value_operation_support_evidence(&composition.op, composition.placement) + } + // Any summary decision, including one rooted in a relational + // operator above its evaluations, asks the deployment for support. + Replacement::SubDAG(node) if !is_logical_rewrite(node) => { + cost_model.summary_support_evidence(node) } + Replacement::SubDAG(_) => Some(true), } } -impl CandidateLogicalASAPDAGs { - /// The `sorted_by(cost_model)` step: every group, each with its own - /// candidates ranked best-first under `cost_model` where this module - /// knows how (see the module docs' "Cost-based final selection" - /// section) — groups themselves stay in discovery order, since targets - /// are independent decision points, not alternatives competing with - /// each other. - /// - /// Ranking itself is decided entirely by [`rank_group`] before - /// [`RankedTargetSubDAGCandidates::costs`] is ever computed — pairing each candidate with - /// [`CostModel::grouping_state_cost`] for grouping alternatives, or - /// [`CostModel::estimate_cost`] otherwise, is an additive annotation - /// for a caller that wants to *display* a cost (e.g. a - /// DAG-visualization view), not a second ranking signal, so plugging in - /// a `CostModel` whose `estimate_cost` disagrees with its own - /// `rank_candidates`/`cse_share_decision` (a deployment bug, not - /// something this method tries to protect against) would show a - /// `RankedTargetSubDAGCandidates` whose `costs` aren't monotonically non-decreasing — - /// `cost_sorted`'s own ordering guarantee is unaffected either way. - pub fn cost_sorted(&self, cost_model: &dyn CostModel) -> Vec> { - self.order - .iter() - .map(|ptr| { - let group = &self.groups[ptr]; - let target = TargetSubDAG::with_consumer_count(&group.target, group.consumer_count); - let mut candidates = rank_group(group, cost_model); - // Availability is candidate-specific and cannot be expressed - // by `rank_candidates`' exhaustive permutation contract. - // Keep unavailable alternatives for explanation, but place - // them after every selectable candidate. - candidates.sort_by_key(|candidate| { - cost_model.candidate_cost(candidate, &target).is_none() - }); - let costs = candidates - .iter() - .map(|c| { - cost_model - .grouping_state_cost(c, &target) - .map_or_else(|| cost_model.estimate_cost(c, &target), |cost| cost.0) - }) - .collect(); - RankedTargetSubDAGCandidates { - target: &group.target, - consumer_count: group.consumer_count, - candidates, - costs, - } - }) - .collect() - } +/// The `sorted_by(cost_model)` step: every group, each with its own +/// candidates ranked best-first under `cost_model` where this module +/// knows how (see the module docs' "Cost-based final selection" +/// section) — groups themselves stay in discovery order, since targets +/// are independent decision points, not alternatives competing with +/// each other. +/// +/// Ranking itself is decided entirely by [`rank_group`] before +/// [`RankedTargetSubDAGCandidates::costs`] is ever computed — pairing each candidate with +/// [`CostModel::grouping_state_cost`] for grouping alternatives, or +/// [`CostModel::estimate_cost`] otherwise, is an additive annotation +/// for a caller that wants to *display* a cost (e.g. a +/// DAG-visualization view), not a second ranking signal, so plugging in +/// a `CostModel` whose `estimate_cost` disagrees with its own +/// `rank_candidates`/`cse_share_decision` (a deployment bug, not +/// something this method tries to protect against) would show a +/// `RankedTargetSubDAGCandidates` whose `costs` aren't monotonically non-decreasing — +/// `cost_sorted`'s own ordering guarantee is unaffected either way. +pub fn cost_sorted<'a, Id>( + space: &'a CandidateLogicalASAPDAGs, + cost_model: &dyn CostModel, +) -> Vec> { + space + .order() + .iter() + .map(|ptr| { + let group = &space.groups()[ptr]; + let target = TargetSubDAG::with_consumer_count(&group.target, group.consumer_count); + let mut candidates = rank_group(group, cost_model); + // Availability is candidate-specific and cannot be expressed + // by `rank_candidates`' exhaustive permutation contract. + // Keep unavailable alternatives for explanation, but place + // them after every selectable candidate. + candidates + .sort_by_key(|candidate| cost_model.candidate_cost(candidate, &target).is_none()); + let costs = candidates + .iter() + .map(|c| { + cost_model + .grouping_state_cost(c, &target) + .map_or_else(|| cost_model.estimate_cost(c, &target), |cost| cost.0) + }) + .collect(); + RankedTargetSubDAGCandidates { + target: &group.target, + consumer_count: group.consumer_count, + candidates, + costs, + } + }) + .collect() +} - /// Recurrence-aware counterpart to [`Self::cost_sorted`]. CSE - /// share/recompute pairs are ordered with the target's recurrence - /// profile; all other candidate shapes retain their existing ranking. - pub fn cost_sorted_with_recurrence( - &self, - cost_model: &dyn CostModel, - profiles: &RecurrenceProfileMap, - horizon: Option, - ) -> Result>, RecurrenceError> { - self.order - .iter() - .map(|ptr| { - let group = &self.groups[ptr]; - let mut candidates = rank_group(group, cost_model); - if cse_candidate_pair(group).is_some() { - if let Some(decision) = decide_group_with_recurrence( - group, - group.consumer_count, - profiles.for_target(&group.target), - horizon, - cost_model, - )? { - candidates.sort_by_key(|candidate| match candidate.provenance { - ReplacementProvenance::CseShare if decision == ShareDecision::Share => { - 0 - } - ReplacementProvenance::CseRecompute - if decision == ShareDecision::RecomputeIndependently => - { - 0 - } - ReplacementProvenance::CseShare - | ReplacementProvenance::CseRecompute => 2, - _ => 1, - }); - } +/// Recurrence-aware counterpart to [`cost_sorted`]. CSE +/// share/recompute pairs are ordered with the target's recurrence +/// profile; all other candidate shapes retain their existing ranking. +pub fn cost_sorted_with_recurrence<'a, Id>( + space: &'a CandidateLogicalASAPDAGs, + cost_model: &dyn CostModel, + profiles: &RecurrenceProfileMap, + horizon: Option, +) -> Result>, RecurrenceError> { + space + .order() + .iter() + .map(|ptr| { + let group = &space.groups()[ptr]; + let mut candidates = rank_group(group, cost_model); + if cse_candidate_pair(group).is_some() { + if let Some(decision) = decide_group_with_recurrence( + group, + group.consumer_count, + profiles.for_target(&group.target), + horizon, + cost_model, + )? { + candidates.sort_by_key(|candidate| match candidate.provenance { + ReplacementProvenance::CseShare if decision == ShareDecision::Share => 0, + ReplacementProvenance::CseRecompute + if decision == ShareDecision::RecomputeIndependently => + { + 0 + } + ReplacementProvenance::CseShare | ReplacementProvenance::CseRecompute => 2, + _ => 1, + }); } - let target = TargetSubDAG::with_consumer_count(&group.target, group.consumer_count); - let costs = candidates - .iter() - .map(|candidate| { - cost_model - .grouping_state_cost(candidate, &target) - .map_or_else( - || cost_model.estimate_cost(candidate, &target), - |cost| cost.0, - ) - }) - .collect(); - Ok(RankedTargetSubDAGCandidates { - target: &group.target, - consumer_count: group.consumer_count, - candidates, - costs, + } + let target = TargetSubDAG::with_consumer_count(&group.target, group.consumer_count); + let costs = candidates + .iter() + .map(|candidate| { + cost_model + .grouping_state_cost(candidate, &target) + .map_or_else( + || cost_model.estimate_cost(candidate, &target), + |cost| cost.0, + ) }) + .collect(); + Ok(RankedTargetSubDAGCandidates { + target: &group.target, + consumer_count: group.consumer_count, + candidates, + costs, }) - .collect() - } + }) + .collect() } // ── Recurrence-aware cost context (issue #287) ────────────────────────── /// One [`RecurrenceProfile`] per discovered [`TargetSubDAGCandidates`] target, built by -/// [`CandidateLogicalASAPDAGs::recurrence_profiles`] — the "carry `RepeatingEntry.demand` +/// [`recurrence_profiles`] — the "carry `RepeatingEntry.demand` /// and relevant `DataWorkload` into ASAP-aware search/cost context" /// half of issue #287. Looked up by `Rc` pointer identity, the same /// currency [`CandidateLogicalASAPDAGs::candidates_for_target`]/[`GlobalSelection::for_target`] already @@ -196,172 +193,170 @@ impl RecurrenceProfileMap { } } -impl CandidateLogicalASAPDAGs { - /// Build one [`RecurrenceProfile`] per discovered site, by walking every - /// root's whole reachable sub-DAG (the same relational-skeleton - /// traversal [`discover_targets`] itself used to discover those sites) - /// and folding each root's own recurrence tag - /// (a normalized repeating rate or a one-time invocation count) into every - /// site reachable from it. - /// - /// `root_recurrence` is positional: `root_recurrence[i]` describes - /// `self.roots[i]` — the same order [`search_workload`]/ - /// [`search_workload_with`] were originally called with (post-CSE - /// dedup preserves both root count and order — see - /// `asap_types::ir::cse::share_common_sub_dags`'s own - /// `.map(...).collect()` body). This keeps `Id` fully opaque (no `Eq`/ - /// `Hash`/`Clone` bound needed on it at all — issue #287's "keep - /// caller/query identifiers opaque" requirement) at the cost of the - /// caller keeping the two slices in step; `root_recurrence.len()` must - /// equal `self.roots.len()`. - /// - /// A shared sub-DAG reachable from more than one root aggregates every - /// reaching root's contribution — repeating roots' rates are summed and - /// one-shot roots - /// increment [`RecurrenceProfile::one_shot_consumers`] — so a summary - /// consumed by queries with different intervals gets one profile - /// reflecting all of them, per issue #287's "support a shared sub-DAG - /// consumed by queries with different intervals". - /// - /// `update_rate` is applied uniformly to every discovered site *that - /// this walk actually reached from some root* (see the "unreachable - /// sites" note below): today's - /// [`asap_types::workload::DataWorkload`] is a single - /// workload-level value (applies to every query in a `QueryWorkload`), - /// not per-target, so there is no finer-grained source to attach - /// instead. `None` when no `DataWorkload` evidence was available — - /// preserves "missing metadata" behavior for the update-rate term alone - /// even when repeating/one-shot consumer information is present. - /// - /// A parent that structurally references the same child more than once - /// (e.g. `BinaryOp{lhs: X, rhs: X}`) credits that child with one - /// contribution per reference, not one contribution per distinct node — - /// matching how [`TargetSubDAGCandidates::consumer_count`] counts that occurrence. - /// Multiplicity is propagated through the full descendant path: if the - /// repeated parent is independently evaluated twice, its child is also - /// evaluated twice. This supplies recurrence-aware selection with the - /// effective structural execution rate rather than mere reachability. - /// - /// **Unreachable sites**: [`CandidateLogicalASAPDAGs`] can contain a site no root's own - /// structural DAG actually reaches — e.g. one only ever produced by a - /// [`Replacement::Rewrite`] candidate a [`ReplacementStrategy`] invented - /// (this walk only follows [`TargetSubDAGCandidates::target`]'s own structural - /// children, the same scope [`discover_targets`] uses for the original - /// roots, never a candidate's rewritten value). Such a site gets - /// [`RecurrenceProfile::EMPTY`] — in particular, `update_rate` is - /// **not** stamped onto it — so it falls back to the ordinary - /// structural decision instead of being charged an ingest-driven - /// maintenance cost against a real evaluation/one-shot signal of - /// exactly zero, which previously made `RecomputeIndependently` win - /// there unconditionally, regardless of the site's actual - /// `consumer_count` (issue #287 review, bug 2). - /// - /// Returns [`RecurrenceError::InvalidEvaluationRate`] if any repeating - /// rate is non-finite or negative, - /// [`RecurrenceError::InvalidUpdateRate`] if `update_rate` is non-finite - /// or negative, or [`RecurrenceError::RootCountMismatch`] if - /// `root_recurrence.len() != self.roots.len()`. - pub fn recurrence_profiles( - &self, - root_recurrence: &[RootRecurrence], - update_rate: Option, - ) -> Result { - if root_recurrence.len() != self.roots.len() { - return Err(crate::recurrence::RecurrenceError::RootCountMismatch { - expected: self.roots.len(), - got: root_recurrence.len(), - }); - } - if let Some(rate) = update_rate { - crate::recurrence::validate_update_rate(rate)?; - } - for recurrence in root_recurrence { - if let RootRecurrence::Repeating(rate) = recurrence { - if !rate.0.is_finite() || rate.0 < 0.0 { - return Err(crate::recurrence::RecurrenceError::InvalidEvaluationRate( - *rate, - )); - } +/// Build one [`RecurrenceProfile`] per discovered site, by walking every +/// root's whole reachable sub-DAG (the same relational-skeleton +/// traversal `discover_targets` itself used to discover those sites) +/// and folding each root's own recurrence tag +/// (a normalized repeating rate or a one-time invocation count) into every +/// site reachable from it. +/// +/// `root_recurrence` is positional: `root_recurrence[i]` describes +/// `space.roots[i]` — the same order [`search_workload`]/ +/// [`search_workload_with`] were originally called with (post-CSE +/// dedup preserves both root count and order — see +/// `asap_types::ir::cse::share_common_sub_dags`'s own +/// `.map(...).collect()` body). This keeps `Id` fully opaque (no `Eq`/ +/// `Hash`/`Clone` bound needed on it at all — issue #287's "keep +/// caller/query identifiers opaque" requirement) at the cost of the +/// caller keeping the two slices in step; `root_recurrence.len()` must +/// equal `space.roots.len()`. +/// +/// A shared sub-DAG reachable from more than one root aggregates every +/// reaching root's contribution — repeating roots' rates are summed and +/// one-shot roots +/// increment [`RecurrenceProfile::one_shot_consumers`] — so a summary +/// consumed by queries with different intervals gets one profile +/// reflecting all of them, per issue #287's "support a shared sub-DAG +/// consumed by queries with different intervals". +/// +/// `update_rate` is applied uniformly to every discovered site *that +/// this walk actually reached from some root* (see the "unreachable +/// sites" note below): today's +/// [`asap_types::workload::DataWorkload`] is a single +/// workload-level value (applies to every query in a `QueryWorkload`), +/// not per-target, so there is no finer-grained source to attach +/// instead. `None` when no `DataWorkload` evidence was available — +/// preserves "missing metadata" behavior for the update-rate term alone +/// even when repeating/one-shot consumer information is present. +/// +/// A parent that structurally references the same child more than once +/// (e.g. `BinaryOp{lhs: X, rhs: X}`) credits that child with one +/// contribution per reference, not one contribution per distinct node — +/// matching how [`TargetSubDAGCandidates::consumer_count`] counts that occurrence. +/// Multiplicity is propagated through the full descendant path: if the +/// repeated parent is independently evaluated twice, its child is also +/// evaluated twice. This supplies recurrence-aware selection with the +/// effective structural execution rate rather than mere reachability. +/// +/// **Unreachable sites**: [`CandidateLogicalASAPDAGs`] can contain a site no root's own +/// structural DAG actually reaches — e.g. one only ever produced by a +/// [`Replacement::Rewrite`] candidate a [`ReplacementStrategy`] invented +/// (this walk only follows [`TargetSubDAGCandidates::target`]'s own structural +/// children, the same scope `discover_targets` uses for the original +/// roots, never a candidate's rewritten value). Such a site gets +/// [`RecurrenceProfile::EMPTY`] — in particular, `update_rate` is +/// **not** stamped onto it — so it falls back to the ordinary +/// structural decision instead of being charged an ingest-driven +/// maintenance cost against a real evaluation/one-shot signal of +/// exactly zero, which previously made `RecomputeIndependently` win +/// there unconditionally, regardless of the site's actual +/// `consumer_count` (issue #287 review, bug 2). +/// +/// Returns [`RecurrenceError::InvalidEvaluationRate`] if any repeating +/// rate is non-finite or negative, +/// [`RecurrenceError::InvalidUpdateRate`] if `update_rate` is non-finite +/// or negative, or [`RecurrenceError::RootCountMismatch`] if +/// `root_recurrence.len() != space.roots.len()`. +pub fn recurrence_profiles( + space: &CandidateLogicalASAPDAGs, + root_recurrence: &[RootRecurrence], + update_rate: Option, +) -> Result { + if root_recurrence.len() != space.roots.len() { + return Err(crate::recurrence::RecurrenceError::RootCountMismatch { + expected: space.roots.len(), + got: root_recurrence.len(), + }); + } + if let Some(rate) = update_rate { + crate::recurrence::validate_update_rate(rate)?; + } + for recurrence in root_recurrence { + if let RootRecurrence::Repeating(rate) = recurrence { + if !rate.0.is_finite() || rate.0 < 0.0 { + return Err(crate::recurrence::RecurrenceError::InvalidEvaluationRate( + *rate, + )); } } + } - let mut rates: HashMap<*const OperatorNode, f64> = HashMap::new(); - let mut one_shot_counts: HashMap<*const OperatorNode, usize> = HashMap::new(); - // Sites actually reached by at least one root's own recurrence tag - // during the walk below — see this method's own "Unreachable - // sites" doc. - let mut reached: HashSet<*const OperatorNode> = HashSet::new(); - - for ((_, root), recurrence) in self.roots.iter().zip(root_recurrence) { - let recurrence = *recurrence; - let root_ptr = Rc::as_ptr(root); - // Carry path multiplicity transitively. If a shared ancestor is - // referenced twice, every descendant below an independently - // recomputed occurrence is evaluated twice as well; stopping - // expansion after the first pointer visit undercounts exactly - // the effective-consumer rate recurrence-aware costing needs. - let mut queue: VecDeque<(*const OperatorNode, usize)> = VecDeque::new(); - queue.push_back((root_ptr, 1)); - - while let Some((ptr, path_count)) = queue.pop_front() { - contribute( - ptr, - path_count, - recurrence, - &mut rates, - &mut one_shot_counts, - &mut reached, - ); - // Every reachable node was itself discovered as its own - // `TargetSubDAGCandidates` (`discover_targets` walks the identical - // relational-skeleton scope) — its own `target` is the - // canonical `Rc` to read children off. - if let Some(group) = self.groups.get(&ptr) { - for (child, edge_count) in direct_child_counts(&group.target) { - queue.push_back(( - child, - path_count - .checked_mul(edge_count) - .expect("query DAG path multiplicity overflowed usize"), - )); - } + let mut rates: HashMap<*const OperatorNode, f64> = HashMap::new(); + let mut one_shot_counts: HashMap<*const OperatorNode, usize> = HashMap::new(); + // Sites actually reached by at least one root's own recurrence tag + // during the walk below — see this method's own "Unreachable + // sites" doc. + let mut reached: HashSet<*const OperatorNode> = HashSet::new(); + + for ((_, root), recurrence) in space.roots.iter().zip(root_recurrence) { + let recurrence = *recurrence; + let root_ptr = Rc::as_ptr(root); + // Carry path multiplicity transitively. If a shared ancestor is + // referenced twice, every descendant below an independently + // recomputed occurrence is evaluated twice as well; stopping + // expansion after the first pointer visit undercounts exactly + // the effective-consumer rate recurrence-aware costing needs. + let mut queue: VecDeque<(*const OperatorNode, usize)> = VecDeque::new(); + queue.push_back((root_ptr, 1)); + + while let Some((ptr, path_count)) = queue.pop_front() { + contribute( + ptr, + path_count, + recurrence, + &mut rates, + &mut one_shot_counts, + &mut reached, + ); + // Every reachable node was itself discovered as its own + // `TargetSubDAGCandidates` (`discover_targets` walks the identical + // relational-skeleton scope) — its own `target` is the + // canonical `Rc` to read children off. + if let Some(group) = space.groups().get(&ptr) { + for (child, edge_count) in direct_child_counts(&group.target) { + queue.push_back(( + child, + path_count + .checked_mul(edge_count) + .expect("query DAG path multiplicity overflowed usize"), + )); } } } + } - let mut profiles = HashMap::with_capacity(self.order.len()); - for ptr in &self.order { - let rate = rates.get(ptr).copied().unwrap_or(0.0); - let evaluation_rate = (rate > 0.0).then_some(crate::recurrence::EvaluationRate(rate)); - let one_shot_consumers = one_shot_counts.get(ptr).copied().unwrap_or(0); - // Bug 2 fix (see "Unreachable sites" above): only a reached - // site carries the caller-supplied `update_rate`. - let site_update_rate = if reached.contains(ptr) { - update_rate - } else { - None - }; - let node = Rc::clone(&self.groups[ptr].target); - profiles.insert( - *ptr, - ( - node, - RecurrenceProfile { - evaluation_rate, - one_shot_consumers, - update_rate: site_update_rate, - }, - ), - ); - } - - Ok(RecurrenceProfileMap { profiles }) + let mut profiles = HashMap::with_capacity(space.order().len()); + for ptr in space.order() { + let rate = rates.get(ptr).copied().unwrap_or(0.0); + let evaluation_rate = (rate > 0.0).then_some(crate::recurrence::EvaluationRate(rate)); + let one_shot_consumers = one_shot_counts.get(ptr).copied().unwrap_or(0); + // Bug 2 fix (see "Unreachable sites" above): only a reached + // site carries the caller-supplied `update_rate`. + let site_update_rate = if reached.contains(ptr) { + update_rate + } else { + None + }; + let node = Rc::clone(&space.groups()[ptr].target); + profiles.insert( + *ptr, + ( + node, + RecurrenceProfile { + evaluation_rate, + one_shot_consumers, + update_rate: site_update_rate, + }, + ), + ); } + + Ok(RecurrenceProfileMap { profiles }) } /// Record `times` occurrences of `recurrence` against `ptr` — `times > 1` /// when a single parent structurally references `ptr` more than once (see -/// [`CandidateLogicalASAPDAGs::recurrence_profiles`]'s own doc on edge multiplicity). +/// [`recurrence_profiles`]'s own doc on edge multiplicity). /// A no-op for `times == 0` (an `Rc` returned as a `direct_child_counts` /// child always has `edge_count >= 1` in practice, but this keeps the /// helper correct regardless). @@ -389,7 +384,7 @@ fn contribute( } /// One [`TargetSubDAGCandidates`]'s candidates, ranked best-first by -/// [`CandidateLogicalASAPDAGs::cost_sorted`]. +/// [`cost_sorted`]. #[derive(Debug)] pub struct RankedTargetSubDAGCandidates<'a> { pub target: &'a Rc, @@ -568,7 +563,7 @@ fn realize_one(target: &Rc) -> Option> { /// copy is test-only, so this needs its own for real (non-test) ranking /// code — the same "duplicate a small, self-contained traversal rather than /// restructure a test helper" call this file's own top doc already makes -/// for [`discover_targets`]. +/// for `discover_targets`. pub(crate) fn sketch_kind_of(node: &OperatorNode) -> Option { match &node.operator { Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { @@ -596,50 +591,7 @@ fn summary_grouping(node: &OperatorNode) -> Option<&GroupingStrategy> { // ── global_selection ───────────────────────────────────────────────────── -/// One target sub-DAG's selected choice and usage information — the answer -/// [`CandidateLogicalASAPDAGs::global_selection`] commits to for one site, after folding in -/// every ancestor [`SharedSubDAGStrategy`] decision on the path from a -/// workload root to this site. See the module docs' "Whole-plan -/// (cross-group) selection" section for the full recurrence. -/// -/// Contrast with [`RankedTargetSubDAGCandidates`] ([`CandidateLogicalASAPDAGs::cost_sorted`]'s output): -/// that ranks every candidate for one target in isolation and never commits -/// to just one; this commits to exactly one (or none), and the count it -/// ranks against — [`Self::effective_consumer_count`] — can differ from the -/// target's own raw structural [`TargetSubDAGCandidates::consumer_count`] whenever an -/// ancestor's choice changes how many times this site truly runs. Use -/// `cost_sorted` to inspect every alternative for a site; use -/// `global_selection` when you need this module's best single answer, -/// accounting for cross-target interaction where it knows how to. -#[derive(Debug)] -pub struct TargetSubDAGSelection<'a> { - /// The target sub-DAG this selection is for. - pub target: &'a Rc, - /// [`TargetSubDAGCandidates::consumer_count`] — how many operator-child positions - /// directly reference `target`, ignoring every ancestor's own choice. - pub consumer_count: usize, - /// How many times `target`'s computation actually runs once every - /// ancestor's own selected candidate is accounted for — see - /// [`multiplier`]'s doc for the exact recurrence. Equal to - /// `consumer_count` unless some ancestor on a path from a root to this - /// site has a [`SharedSubDAGStrategy`] alternative that chose - /// [`ShareDecision::RecomputeIndependently`]. - pub effective_consumer_count: usize, - /// The candidate chosen for this target, or `None` when no replacement - /// is selected. The candidate set need not be empty: an unproven DDSketch - /// ratio can remain available for backend inspection but be excluded from - /// automatic selection, or costing can prefer raw recomputation. - /// DAG assembly then preserves exact computation at this target where - /// supported, while independently selected children may remain visible. - pub chosen: Option<&'a ReplacementSubDAG>, - /// When `chosen` is a [`Replacement::ExactComposition`]: the child - /// decision it was committed together with, and the cost comparison - /// that justified it — the explicit target-to-decision provenance - /// chain (issue #171). - pub composition: Option>, -} - -/// Why [`CandidateLogicalASAPDAGs::global_selection`] committed an exact composition at a +/// Why [`global_selection`] committed an exact composition at a /// site: which child candidate it composes with, and the /// cost-units-per-second comparison against the raw fallback that it won. #[derive(Debug)] @@ -662,387 +614,28 @@ pub struct CompositionDecision<'a> { pub inputs: ExactCompositionCostInputs, } -/// [`CandidateLogicalASAPDAGs::global_selection`]'s result: one [`TargetSubDAGSelection`] per -/// discovered site, in the same discovery order [`CandidateLogicalASAPDAGs::target_subdag_candidates`]/ -/// [`CandidateLogicalASAPDAGs::cost_sorted`] use. +/// [`global_selection`]'s result: the [`GlobalSelection`] it committed to, +/// plus the cost comparison behind each exact composition it chose. The +/// selection is a Stage 1 type and carries no cost data, so the comparison +/// is kept here. #[derive(Debug)] -pub struct GlobalSelection<'a> { - pub(crate) order: Vec<*const OperatorNode>, - pub(crate) groups: HashMap<*const OperatorNode, TargetSubDAGSelection<'a>>, - /// [`Self::assemble_selected_dag`]'s memo — one bound node per target for the - /// life of this selection, so two parents composing over one shared - /// child get the *same* `Rc` (a kept pre-ASAP sub-DAG - /// shared by two parents stays one `Rc` the same way). - pub(crate) assembled_nodes: RefCell>>, +pub struct CostedGlobalSelection<'a> { + selection: GlobalSelection<'a>, + compositions: HashMap<*const OperatorNode, CompositionDecision<'a>>, } -fn normalize_cross_input_equi_predicate( - pred: &Predicate, - left_width: usize, - total_width: usize, -) -> Option { - let ScalarExpr::Compare { - left, - op: asap_types::ir::scalar::CompareOpKind::Eq, - right, - semantics, - } = &pred.0 - else { - return None; - }; - let (ScalarExpr::Column(left_id), ScalarExpr::Column(right_id)) = - (left.as_ref(), right.as_ref()) - else { - return None; - }; - let is_left = |id: ColumnId| id < left_width; - let is_right = |id: ColumnId| left_width <= id && id < total_width; - let (left_id, right_id) = if is_left(*left_id) && is_right(*right_id) { - (*left_id, *right_id) - } else if is_right(*left_id) && is_left(*right_id) { - (*right_id, *left_id) - } else { - return None; - }; - Some(Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(left_id)), - op: asap_types::ir::scalar::CompareOpKind::Eq, - right: Box::new(ScalarExpr::Column(right_id)), - semantics: *semantics, - })) -} - -impl<'a> GlobalSelection<'a> { - /// One selection per discovered target sub-DAG, in discovery order. - pub fn target_selections(&self) -> impl Iterator> { - self.order.iter().map(move |ptr| &self.groups[ptr]) - } - - /// The selection for `target`, if `target`'s own `Rc` is a discovered - /// site (i.e. `Rc::ptr_eq` to some node reachable from the workload's - /// roots). - pub fn for_target(&self, target: &Rc) -> Option<&TargetSubDAGSelection<'a>> { - self.groups.get(&Rc::as_ptr(target)) - } - - /// Link this selection's per-site decisions into one data_state-validated - /// post-ASAP DAG rooted at `target` — the one place a committed - /// composition's child *reference* becomes an actual `Rc` - /// edge (issue #171). `None` if `target` is not a discovered site. - /// - /// Per site: a [`Replacement::ExactComposition`] uses its validated - /// operation/child plan, retaining the search model's guarantee; - /// a bound-summary [`Replacement::SubDAG`] is - /// re-linked so its `SummaryAgg` child is the child target's own - /// DAG assembly whenever that is phase-legal beneath maintenance - /// (so a child that chose an `ValueOperationAtIngestionTime` actually ends up under - /// the summary); a logical-rewrite [`Replacement::SubDAG`] is kept - /// as it is (exact); an unmatched site keeps its own operator with each - /// child assembled independently ([`Self::assemble_residual`]). - /// Memoized by target identity, so a shared inner summary is one `Rc` - /// no matter how many roots reach it. - pub fn assemble_selected_dag( - &self, - target: &Rc, - ) -> Result>, RealizationError> { - if !self.groups.contains_key(&Rc::as_ptr(target)) { - return Ok(None); - } - self.assemble_target(target).map(Some) - } - - /// Assemble a complete query result, including an exact-state evaluation when - /// needed. `assemble_selected_dag` also serves internal state frontiers; - /// callers exposing query results must use this boundary instead. - pub fn assemble_selected_query( - &self, - target: &Rc, - ) -> Result>, RealizationError> { - self.assemble_selected_dag(target)? - .map(|node| finalize_query_candidate(node, target)) - .transpose() - } - - pub(crate) fn assemble_target( - &self, - target: &Rc, - ) -> Result, RealizationError> { - let ptr = Rc::as_ptr(target); - if let Some(node) = self.assembled_nodes.borrow().get(&ptr) { - return Ok(Rc::clone(node)); - } - // A selected summary that realizes its inner aggregate, instead of - // hiding it in `KeepPreAsap`, is kept; materialization assignment decides - // whether it runs in precompute or at query time. - let selected_composed_summary = self - .groups - .get(&ptr) - .and_then(|sel| sel.chosen) - .is_some_and(|candidate| { - matches!(&candidate.replacement, - Replacement::SubDAG(node) if matches!(&node.operator, - Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) - if child.contains_asap() || !contains_aggregate(child))) - }); - let node = if query_time_nested_sum(target) && !selected_composed_summary { - self.assemble_residual(target)? - } else { - match self - .groups - .get(&ptr) - .and_then(|sel| sel.chosen) - .map(|c| &c.replacement) - { - None => self.assemble_residual(target)?, - Some(Replacement::SubDAG(node)) if node.contains_asap() => { - self.relink_summary(node, target)? - } - Some(Replacement::SubDAG(kept)) => retain_exact(kept)?, - Some(Replacement::ExactComposition(_)) => Rc::clone( - &self.groups[&ptr] - .composition - .as_ref() - .expect("selected compositions have a validated decision") - .plan, - ), - } - }; - self.assembled_nodes - .borrow_mut() - .insert(ptr, Rc::clone(&node)); - Ok(node) - } - - /// Keep `target`'s own operator and assemble each child independently, - /// so a selected summary remains visible beneath a relational operator - /// that has no summary realization of its own instead of being - /// swallowed by one opaque kept sub-DAG. Every child that is a - /// discovered target is assembled (and finalized to query-time values); - /// any other child is kept as it is. The guarantee is composed from the - /// assembled children: all exact → exact; exactly one child → that - /// child's guarantee; otherwise unknown. An inner `Join` first has its - /// cross-input equi-predicate normalized; any other join is kept whole. - fn assemble_residual( - &self, - target: &Rc, - ) -> Result, RealizationError> { - if target.children().is_empty() { - // A leaf has nothing to assemble beneath it: keep it as it is. - return retain_exact(target); - } - let mut operator = target.operator.clone(); - if let Operator::NonASAP(NonASAPOp::Join { - left, - right, - kind, - pred, - }) = &mut operator - { - let left_width = left.schema.fields.len(); - let total_width = left_width + right.schema.fields.len(); - let normalized_pred = matches!(kind, JoinKind::Inner) - .then(|| normalize_cross_input_equi_predicate(pred, left_width, total_width)) - .flatten(); - let Some(normalized) = normalized_pred else { - return retain_exact(target); - }; - *pred = normalized; - } - let mut failure = None; - let mut children = Vec::new(); - let operator = operator.map_children(|child| { - if failure.is_some() { - return Rc::clone(child); - } - let assembled = if self.groups.contains_key(&Rc::as_ptr(child)) { - self.assemble_target(child) - .and_then(|node| finalize_query_candidate(node, child)) - } else { - Ok(Rc::clone(child)) - }; - match assembled { - Ok(node) => { - children.push(Rc::clone(&node)); - node - } - Err(error) => { - failure = Some(error); - Rc::clone(child) - } - } - }); - if let Some(error) = failure { - return Err(error); - } - // An operator that computes new values from its input rows has no - // sound accuracy composition over an approximate input (e.g. `max` - // over a quantile evaluation's rank error). Without a selected - // composition such a node stays an exact pre-ASAP sub-DAG; only the - // read-time nested SUM keeps its assembled children. - let computes_values = matches!( - target.non_asap(), - Some( - NonASAPOp::Aggregate { .. } - | NonASAPOp::BinaryOp { .. } - | NonASAPOp::SQLWindowFunc { .. } - ) - ) && !query_time_nested_sum(target); - let approximate_input = children.iter().any(|child| { - !child - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact) - }); - if computes_values && approximate_input { - return retain_exact(target); - } - let guarantee = match children.as_slice() { - [child] => child.guarantee.clone(), - children - if children.iter().all(|child| { - child - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact) - }) => - { - Some(ResultGuarantee::exact(format!( - "{} over exact inputs", - target.operator.kind_name() - ))) - } - _ => None, - }; - let node = Rc::new( - OperatorNode::with_schema(operator, target.schema.clone()).with_guarantee(guarantee), - ); - validate_maintained(&node, ExecutionTiming::QueryTime)?; - Ok(node) - } - - /// Re-link a bound summary candidate's `SummaryAgg` child to the - /// child target's own DAG assembly when that is legal beneath - /// maintenance; otherwise keep the candidate exactly as constructed. - fn relink_summary( - &self, - node: &Rc, - target: &Rc, - ) -> Result, RealizationError> { - let Some(NonASAPOp::Aggregate { - child: pre_child, .. - }) = target.non_asap() - else { - return Ok(Rc::clone(node)); - }; - let has_maintenance_operation = self - .groups - .get(&Rc::as_ptr(pre_child)) - .and_then(|selection| selection.chosen) - .is_some_and(|candidate| { - matches!( - &candidate.replacement, - Replacement::ExactComposition(composition) - if composition.placement == OperationPlacement::Maintenance - ) - }); - if !has_maintenance_operation { - return Ok(Rc::clone(node)); - } - let new_child = self.assemble_target(pre_child)?; - Ok(relink_agg_child(node, &new_child)) +impl<'a> CostedGlobalSelection<'a> { + /// The decision behind `target`'s chosen exact composition, if it chose one. + pub fn composition(&self, target: &Rc) -> Option<&CompositionDecision<'a>> { + self.compositions.get(&Rc::as_ptr(target)) } } -/// A mergeable outer SUM over a relationally wrapped aggregate is a read-time -/// reduction of the inner summary values. Maintaining the outer SUM directly -/// would hide that inner temporal aggregate inside one kept sub-DAG and lose -/// its independently selected summary. -fn query_time_nested_sum(target: &OperatorNode) -> bool { - let Some(NonASAPOp::Aggregate { - measures, - filters, - having: None, - child, - .. - }) = target.non_asap() - else { - return false; - }; - !any_measure_filtered(filters) - && matches!(measures.as_slice(), [AggIntent::Sum { .. }]) - && contains_aggregate(child) -} +impl<'a> std::ops::Deref for CostedGlobalSelection<'a> { + type Target = GlobalSelection<'a>; -fn contains_aggregate(expr: &OperatorNode) -> bool { - match expr.non_asap() { - Some(NonASAPOp::Aggregate { .. }) => true, - Some( - NonASAPOp::Project { child, .. } - | NonASAPOp::Filter { child, .. } - | NonASAPOp::Sort { child, .. } - | NonASAPOp::Limit { child, .. }, - ) => contains_aggregate(child), - _ => false, - } -} - -/// Rebuild `node` (a `SummaryAgg`, possibly under a `SummaryEstimate`) with -/// `new_child` as the `SummaryAgg`'s child, if the result still validates -/// as maintained state; otherwise return `node` unchanged. -fn relink_agg_child(node: &Rc, new_child: &Rc) -> Rc { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) => { - let inner = relink_agg_child(summary_input, new_child); - if Rc::ptr_eq(&inner, summary_input) { - return Rc::clone(node); - } - std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: inner, - query: query.clone(), - }), - node.schema.clone(), - ) - .with_guarantee(node.guarantee.clone()), - ) - } - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - grouping, - filter, - }) => { - if Rc::ptr_eq(child, new_child) { - return Rc::clone(node); - } - // The same summary over a re-placed input keeps its coverage. - let rebuilt = std::rc::Rc::new(OperatorNode { - coverage: node.coverage.clone(), - ..OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(new_child), - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }), - node.schema.clone(), - ) - .with_guarantee(node.guarantee.clone()) - }); - match validate_maintained(&rebuilt, ExecutionTiming::IngestionTime) { - Ok(_) => rebuilt, - Err(_) => Rc::clone(node), - } - } - _ => Rc::clone(node), + fn deref(&self) -> &Self::Target { + &self.selection } } @@ -1063,7 +656,7 @@ fn is_composition_candidate(candidate: &ReplacementSubDAG) -> bool { matches!(candidate.replacement, Replacement::ExactComposition(_)) } -/// Everything [`CandidateLogicalASAPDAGs::global_selection`] threads between sites for +/// Everything [`global_selection`] threads between sites for /// exact compositions (issue #171): child candidates already committed by /// an earlier parent, and the maintained summary above each site. #[derive(Default)] @@ -1101,7 +694,7 @@ fn composition_options<'a>( let Replacement::ExactComposition(composition) = &candidate.replacement else { continue; }; - if candidate.runtime_support_evidence(cost_model) != Some(true) { + if runtime_support_evidence(candidate, cost_model) != Some(true) { continue; } let child_ptr = Rc::as_ptr(&composition.child_target); @@ -1208,306 +801,310 @@ fn composition_options<'a>( options } -impl CandidateLogicalASAPDAGs { - /// The whole-plan (cross-group) selection step the module docs' - /// "Whole-plan (cross-group) selection" section describes: one - /// [`TargetSubDAGSelection`] per discovered site, each ranked against an - /// `effective_consumer_count` that accounts for every ancestor - /// [`SharedSubDAGStrategy`] decision on the path to it — unlike - /// [`Self::cost_sorted`], whose per-group ranking only ever sees a - /// group's own raw [`TargetSubDAGCandidates::consumer_count`]. - /// Uncertified DDSketch ratios remain in [`CandidateLogicalASAPDAGs`] for downstream - /// inspection but are not chosen automatically by this selector. - pub fn global_selection(&self, cost_model: &dyn CostModel) -> GlobalSelection<'_> { - self.global_selection_impl(cost_model, None, None) - .expect("structural global selection cannot produce a recurrence error") - } +/// The whole-plan (cross-group) selection step the module docs' +/// "Whole-plan (cross-group) selection" section describes: one +/// [`TargetSubDAGSelection`] per discovered site, each ranked against an +/// `effective_consumer_count` that accounts for every ancestor +/// [`SharedSubDAGStrategy`] decision on the path to it — unlike +/// [`cost_sorted`], whose per-group ranking only ever sees a +/// group's own raw [`TargetSubDAGCandidates::consumer_count`]. +/// Uncertified DDSketch ratios remain in [`CandidateLogicalASAPDAGs`] for downstream +/// inspection but are not chosen automatically by this selector. +pub fn global_selection<'a, Id>( + space: &'a CandidateLogicalASAPDAGs, + cost_model: &dyn CostModel, +) -> CostedGlobalSelection<'a> { + global_selection_impl(space, cost_model, None, None) + .expect("structural global selection cannot produce a recurrence error") +} - /// Recurrence-aware counterpart to [`Self::global_selection`]. The same - /// whole-plan traversal and effective structural consumer counts are - /// retained, while every CSE share/recompute choice is made from the - /// corresponding recurrence profile. - pub fn global_selection_with_recurrence( - &self, - cost_model: &dyn CostModel, - profiles: &RecurrenceProfileMap, - horizon: Option, - ) -> Result, RecurrenceError> { - self.global_selection_impl(cost_model, Some(profiles), horizon) - } +/// Recurrence-aware counterpart to [`global_selection`]. The same +/// whole-plan traversal and effective structural consumer counts are +/// retained, while every CSE share/recompute choice is made from the +/// corresponding recurrence profile. +pub fn global_selection_with_recurrence<'a, Id>( + space: &'a CandidateLogicalASAPDAGs, + cost_model: &dyn CostModel, + profiles: &RecurrenceProfileMap, + horizon: Option, +) -> Result, RecurrenceError> { + global_selection_impl(space, cost_model, Some(profiles), horizon) +} - fn global_selection_impl( - &self, - cost_model: &dyn CostModel, - profiles: Option<&RecurrenceProfileMap>, - horizon: Option, - ) -> Result, RecurrenceError> { - let dag = reference_dag(self); - let topo = topological_order(&self.order, &dag); - - let mut effective_uses = dag.external_root_uses.clone(); - let mut chosen_share: HashMap<*const OperatorNode, ShareDecision> = HashMap::new(); - let mut groups: HashMap<*const OperatorNode, TargetSubDAGSelection<'_>> = HashMap::new(); - let mut context = CompositionContext::default(); - - for ptr in &topo { - let group = &self.groups[ptr]; - - let effective = effective_uses.get(ptr).copied().unwrap_or(0); - effective_uses.insert(*ptr, effective); - - // ── Exact compositions (issue #171) ───────────────────────── - // A child an earlier parent's composition committed to is - // forced to exactly that candidate — the parent/child pair is - // one decision. Otherwise, a composition here wins only when - // its cost-units-per-second rate is *known* and beats the raw - // recompute baseline; missing statistics keep the conservative - // path below. - let mut composition_decision = None; - let forced = context - .committed_child - .get(ptr) - .and_then(|&cptr| group.candidates.iter().find(|c| std::ptr::eq(*c, cptr))); - let composed = if forced.is_some() { - None - } else { - composition_options( - group, - &self.groups, - effective, - cost_model, - &context, - &self.composition_plans, - ) - .into_iter() - .min_by(|a, b| a.decision.cost_rate.0.total_cmp(&b.decision.cost_rate.0)) - }; - if let Some(option) = &composed { - if let Some(child_candidate) = option.decision.child_candidate { - context.committed_child.insert( - Rc::as_ptr(option.decision.child_target), - child_candidate as *const ReplacementSubDAG, - ); - } - if let Replacement::ExactComposition(composition) = &option.candidate.replacement { - if composition.placement == OperationPlacement::Maintenance { - // A chain of functions feeds the same summary. - if let Some(parent) = context.maintaining_parent.get(ptr).cloned() { - context - .maintaining_parent - .insert(Rc::as_ptr(&composition.child_target), parent); - } +fn global_selection_impl<'a, Id>( + space: &'a CandidateLogicalASAPDAGs, + cost_model: &dyn CostModel, + profiles: Option<&RecurrenceProfileMap>, + horizon: Option, +) -> Result, RecurrenceError> { + let dag = reference_dag(space); + let topo = topological_order(space.order(), &dag); + + let mut effective_uses = dag.external_root_uses.clone(); + let mut chosen_share: HashMap<*const OperatorNode, ShareDecision> = HashMap::new(); + let mut groups: HashMap<*const OperatorNode, TargetSubDAGSelection<'a>> = HashMap::new(); + let mut compositions: HashMap<*const OperatorNode, CompositionDecision<'a>> = HashMap::new(); + let mut context = CompositionContext::default(); + + for ptr in &topo { + let group = &space.groups()[ptr]; + + let effective = effective_uses.get(ptr).copied().unwrap_or(0); + effective_uses.insert(*ptr, effective); + + // ── Exact compositions (issue #171) ───────────────────────── + // A child an earlier parent's composition committed to is + // forced to exactly that candidate — the parent/child pair is + // one decision. Otherwise, a composition here wins only when + // its cost-units-per-second rate is *known* and beats the raw + // recompute baseline; missing statistics keep the conservative + // path below. + let mut composition_decision = None; + let forced = context + .committed_child + .get(ptr) + .and_then(|&cptr| group.candidates.iter().find(|c| std::ptr::eq(*c, cptr))); + let composed = if forced.is_some() { + None + } else { + composition_options( + group, + space.groups(), + effective, + cost_model, + &context, + space.composition_plans(), + ) + .into_iter() + .min_by(|a, b| a.decision.cost_rate.0.total_cmp(&b.decision.cost_rate.0)) + }; + if let Some(option) = &composed { + if let Some(child_candidate) = option.decision.child_candidate { + context.committed_child.insert( + Rc::as_ptr(option.decision.child_target), + child_candidate as *const ReplacementSubDAG, + ); + } + if let Replacement::ExactComposition(composition) = &option.candidate.replacement { + if composition.placement == OperationPlacement::Maintenance { + // A chain of functions feeds the same summary. + if let Some(parent) = context.maintaining_parent.get(ptr).cloned() { + context + .maintaining_parent + .insert(Rc::as_ptr(&composition.child_target), parent); } } } + } - let complete_plan_choice = (!forced.is_some() - && composed.is_none() - && cost_model.candidate_cost_covers_complete_plan()) - .then(|| { - let effective_target = TargetSubDAG::with_consumer_count(&group.target, effective); - let bound = group - .candidates - .iter() - .filter(|candidate| { - !is_cse_candidate(candidate) - && !is_composition_candidate(candidate) - && is_automatically_selectable(candidate, cost_model) - }) - .filter_map(|candidate| { - cost_model - .candidate_cost(candidate, &effective_target) - .map(|cost| (candidate, cost)) - }) - .min_by(|(_, left), (_, right)| left.0.total_cmp(&right.0)) - .map(|(candidate, _)| candidate); - bound.or_else(|| { - (cost_model.allow_uncosted_legacy_selection() && effective >= 2) - .then(|| { - decide_with_effective_count(group, effective, cost_model).and_then( - |decision| { - let candidate = pick_shared_sub_dag_candidate(group, decision)?; - chosen_share.insert(*ptr, decision); - Some(candidate) - }, - ) - }) - .flatten() + let complete_plan_choice = (!forced.is_some() + && composed.is_none() + && cost_model.candidate_cost_covers_complete_plan()) + .then(|| { + let effective_target = TargetSubDAG::with_consumer_count(&group.target, effective); + let bound = group + .candidates + .iter() + .filter(|candidate| { + !is_cse_candidate(candidate) + && !is_composition_candidate(candidate) + && is_automatically_selectable(candidate, cost_model) + }) + .filter_map(|candidate| { + cost_model + .candidate_cost(candidate, &effective_target) + .map(|cost| (candidate, cost)) }) + .min_by(|(_, left), (_, right)| left.0.total_cmp(&right.0)) + .map(|(candidate, _)| candidate); + bound.or_else(|| { + (cost_model.allow_uncosted_legacy_selection() && effective >= 2) + .then(|| { + decide_with_effective_count(group, effective, cost_model).and_then( + |decision| { + let candidate = pick_shared_sub_dag_candidate(group, decision)?; + chosen_share.insert(*ptr, decision); + Some(candidate) + }, + ) + }) + .flatten() }) - .flatten(); - - let chosen = if let Some(forced) = forced { - Some(forced) - } else if let Some(option) = composed { - composition_decision = Some(option.decision); - Some(option.candidate) - } else if cost_model.candidate_cost_covers_complete_plan() { - complete_plan_choice - } else if effective >= 2 && cse_candidate_pair(group).is_some() { - let decision = if let Some(profiles) = profiles { - decide_group_with_recurrence( - group, - effective, - profiles.for_target(&group.target), - horizon, - cost_model, - )? - } else { - decide_with_effective_count(group, effective, cost_model) - }; - match decision { - Some(decision) => { - let cse = pick_shared_sub_dag_candidate(group, decision); - let effective_target = - TargetSubDAG::with_consumer_count(&group.target, effective); - let logical = group - .candidates - .iter() - .filter(|candidate| { - !is_cse_candidate(candidate) - && !is_composition_candidate(candidate) - && is_automatically_selectable(candidate, cost_model) - }) - .filter_map(|candidate| { - cost_model - .candidate_cost(candidate, &effective_target) - .map(|cost| (candidate, cost)) - }) - .min_by(|(_, a), (_, b)| a.0.total_cmp(&b.0)) - .map(|(candidate, _)| candidate); - let cse = cse.filter(|candidate| { + }) + .flatten(); + + let chosen = if let Some(forced) = forced { + Some(forced) + } else if let Some(option) = composed { + composition_decision = Some(option.decision); + Some(option.candidate) + } else if cost_model.candidate_cost_covers_complete_plan() { + complete_plan_choice + } else if effective >= 2 && cse_candidate_pair(group).is_some() { + let decision = if let Some(profiles) = profiles { + decide_group_with_recurrence( + group, + effective, + profiles.for_target(&group.target), + horizon, + cost_model, + )? + } else { + decide_with_effective_count(group, effective, cost_model) + }; + match decision { + Some(decision) => { + let cse = pick_shared_sub_dag_candidate(group, decision); + let effective_target = + TargetSubDAG::with_consumer_count(&group.target, effective); + let logical = group + .candidates + .iter() + .filter(|candidate| { + !is_cse_candidate(candidate) + && !is_composition_candidate(candidate) + && is_automatically_selectable(candidate, cost_model) + }) + .filter_map(|candidate| { cost_model .candidate_cost(candidate, &effective_target) - .is_some() - || cost_model.allow_uncosted_legacy_selection() - }); - match (cse, logical) { - (Some(cse), Some(logical)) - if cost_model - .candidate_cost(cse, &effective_target) - .is_none_or(|cse_cost| { - cost_model - .candidate_cost(logical, &effective_target) - .is_some_and(|logical_cost| logical_cost.0 < cse_cost.0) - }) => - { - Some(logical) - } - (cse, _) => { - if cse.is_some() { - chosen_share.insert(*ptr, decision); - } - cse + .map(|cost| (candidate, cost)) + }) + .min_by(|(_, a), (_, b)| a.0.total_cmp(&b.0)) + .map(|(candidate, _)| candidate); + let cse = cse.filter(|candidate| { + cost_model + .candidate_cost(candidate, &effective_target) + .is_some() + || cost_model.allow_uncosted_legacy_selection() + }); + match (cse, logical) { + (Some(cse), Some(logical)) + if cost_model + .candidate_cost(cse, &effective_target) + .is_none_or(|cse_cost| { + cost_model + .candidate_cost(logical, &effective_target) + .is_some_and(|logical_cost| logical_cost.0 < cse_cost.0) + }) => + { + Some(logical) + } + (cse, _) => { + if cse.is_some() { + chosen_share.insert(*ptr, decision); } + cse } } - // `realize_child` couldn't produce even a logical fallback — - // not expected in practice for a target that's already - // part of a legitimate workload DAG (mirrors - // `cse_preference`'s own doc on this same degrade). - // Falling back to ordinary local ranking is still a - // valid answer, just not a cross-group-aware one; this - // group also contributes no Share collapse to its own - // children (see `multiplier`'s `_ => effective` arm). - None => rank_group(group, cost_model).into_iter().find(|candidate| { - !is_composition_candidate(candidate) - && is_automatically_selectable(candidate, cost_model) - && (cost_model - .candidate_cost( - candidate, - &TargetSubDAG::with_consumer_count(&group.target, effective), - ) - .is_some() - || cost_model.allow_uncosted_legacy_selection()) - }), } - } else { - let effective_target = TargetSubDAG::with_consumer_count(&group.target, effective); - rank_group(group, cost_model) - .into_iter() - .find(|candidate| { - !is_cse_candidate(candidate) - && !is_composition_candidate(candidate) - && is_automatically_selectable(candidate, cost_model) - && (cost_model + // `realize_child` couldn't produce even a logical fallback — + // not expected in practice for a target that's already + // part of a legitimate workload DAG (mirrors + // `cse_preference`'s own doc on this same degrade). + // Falling back to ordinary local ranking is still a + // valid answer, just not a cross-group-aware one; this + // group also contributes no Share collapse to its own + // children (see `multiplier`'s `_ => effective` arm). + None => rank_group(group, cost_model).into_iter().find(|candidate| { + !is_composition_candidate(candidate) + && is_automatically_selectable(candidate, cost_model) + && (cost_model + .candidate_cost( + candidate, + &TargetSubDAG::with_consumer_count(&group.target, effective), + ) + .is_some() + || cost_model.allow_uncosted_legacy_selection()) + }), + } + } else { + let effective_target = TargetSubDAG::with_consumer_count(&group.target, effective); + rank_group(group, cost_model) + .into_iter() + .find(|candidate| { + !is_cse_candidate(candidate) + && !is_composition_candidate(candidate) + && is_automatically_selectable(candidate, cost_model) + && (cost_model + .candidate_cost(candidate, &effective_target) + .is_some() + || cost_model.allow_uncosted_legacy_selection()) + }) + .or_else(|| { + cse_candidate_pair(group) + .map(|(share, _)| share) + .filter(|candidate| { + cost_model .candidate_cost(candidate, &effective_target) .is_some() - || cost_model.allow_uncosted_legacy_selection()) - }) - .or_else(|| { - cse_candidate_pair(group) - .map(|(share, _)| share) - .filter(|candidate| { - cost_model - .candidate_cost(candidate, &effective_target) - .is_some() - || cost_model.allow_uncosted_legacy_selection() - }) - }) - }; + || cost_model.allow_uncosted_legacy_selection() + }) + }) + }; - // Record the maintained summary this site's bound candidate - // builds, for a child that may compose an `ValueOperationAtIngestionTime` - // beneath it. - if let (Some(Replacement::SubDAG(node)), Some(NonASAPOp::Aggregate { child, .. })) = - (chosen.map(|c| &c.replacement), group.target.non_asap()) - { - if let Some(summary) = maintained_summary(node) { - context - .maintaining_parent - .insert(Rc::as_ptr(child), Rc::clone(summary)); - } + // Record the maintained summary this site's bound candidate + // builds, for a child that may compose an `ValueOperationAtIngestionTime` + // beneath it. + if let (Some(Replacement::SubDAG(node)), Some(NonASAPOp::Aggregate { child, .. })) = + (chosen.map(|c| &c.replacement), group.target.non_asap()) + { + if let Some(summary) = maintained_summary(node) { + context + .maintaining_parent + .insert(Rc::as_ptr(child), Rc::clone(summary)); } + } - let outgoing_multiplier = multiplier(*ptr, &effective_uses, &chosen_share); - match chosen { - Some(ReplacementSubDAG { - replacement: Replacement::SubDAG(source), - provenance: ReplacementProvenance::AccuracyReconciliation, - .. - }) => { - // Accuracy reconciliation reads another discovered memo - // group, rather than inlining that group's children. Let - // the source group receive the uses and propagate them - // through its own selected realization when its turn - // arrives in topological order. - *effective_uses.entry(Rc::as_ptr(source)).or_insert(0) += outgoing_multiplier; - } - _ => { - let selected_rewrite = match chosen.map(|candidate| &candidate.replacement) { - Some(Replacement::SubDAG(rewrite)) if is_logical_rewrite(rewrite) => { - rewrite - } - Some(Replacement::SubDAG(_) | Replacement::ExactComposition(_)) | None => { - &group.target - } - }; - for (child, edge_count) in direct_child_counts(selected_rewrite) { - *effective_uses.entry(child).or_insert(0) += - edge_count * outgoing_multiplier; + let outgoing_multiplier = multiplier(*ptr, &effective_uses, &chosen_share); + match chosen { + Some(ReplacementSubDAG { + replacement: Replacement::SubDAG(source), + provenance: ReplacementProvenance::AccuracyReconciliation, + .. + }) => { + // Accuracy reconciliation reads another discovered memo + // group, rather than inlining that group's children. Let + // the source group receive the uses and propagate them + // through its own selected realization when its turn + // arrives in topological order. + *effective_uses.entry(Rc::as_ptr(source)).or_insert(0) += outgoing_multiplier; + } + _ => { + let selected_rewrite = match chosen.map(|candidate| &candidate.replacement) { + Some(Replacement::SubDAG(rewrite)) if is_logical_rewrite(rewrite) => rewrite, + Some(Replacement::SubDAG(_) | Replacement::ExactComposition(_)) | None => { + &group.target } + }; + for (child, edge_count) in direct_child_counts(selected_rewrite) { + *effective_uses.entry(child).or_insert(0) += edge_count * outgoing_multiplier; } } - - groups.insert( - *ptr, - TargetSubDAGSelection { - target: &group.target, - consumer_count: group.consumer_count, - effective_consumer_count: effective, - chosen, - composition: composition_decision, - }, - ); } - Ok(GlobalSelection { - order: self.order.clone(), - groups, - assembled_nodes: RefCell::new(HashMap::new()), - }) + groups.insert( + *ptr, + TargetSubDAGSelection { + target: &group.target, + consumer_count: group.consumer_count, + effective_consumer_count: effective, + chosen, + }, + ); + if let Some(decision) = composition_decision { + compositions.insert(*ptr, decision); + } } + + let composition_plans = compositions + .iter() + .map(|(ptr, decision)| (*ptr, Rc::clone(&decision.plan))) + .collect(); + Ok(CostedGlobalSelection { + selection: GlobalSelection::new(space.order().to_vec(), groups, composition_plans), + compositions, + }) } fn is_cse_candidate(candidate: &ReplacementSubDAG) -> bool { @@ -1520,7 +1117,7 @@ fn is_cse_candidate(candidate: &ReplacementSubDAG) -> bool { fn is_automatically_selectable(candidate: &ReplacementSubDAG, cost_model: &dyn CostModel) -> bool { candidate.provenance != ReplacementProvenance::RootPhysicalRealization && !candidate.has_missing_accuracy_evidence() - && candidate.runtime_support_evidence(cost_model) != Some(false) + && runtime_support_evidence(candidate, cost_model) != Some(false) } /// How much one direct reference to `parent_ptr` actually costs, once @@ -1540,7 +1137,7 @@ fn is_automatically_selectable(candidate: &ReplacementSubDAG, cost_model: &dyn C /// /// Composing this recurrence transitively up the whole ancestor chain (not /// just the immediate parent) is exactly what makes -/// [`CandidateLogicalASAPDAGs::global_selection`]'s `effective_consumer_count` differ from +/// [`global_selection`]'s `effective_consumer_count` differ from /// [`TargetSubDAGCandidates::consumer_count`] whenever a `RecomputeIndependently` /// ancestor sits anywhere on the path from a root to a site — see the /// module docs' "Whole-plan (cross-group) selection" section. @@ -1618,8 +1215,8 @@ fn pick_shared_sub_dag_candidate( // ── reference DAG + topological order ───────────────────────────────── -/// The parent/child structure [`CandidateLogicalASAPDAGs::global_selection`]'s DP walks — -/// built separately from [`discover_targets`]'s own `order`/`nodes`/`counts` +/// The parent/child structure [`global_selection`]'s DP walks — +/// built separately from `discover_targets`'s own `order`/`nodes`/`counts` /// maps (which only track *aggregate* reference counts, not per-parent /// breakdown or direction). Selection needs per-parent edge counts to /// distinguish shared producers from repeated uses within one consumer. @@ -1638,7 +1235,7 @@ struct ReferenceDAG { /// a node's "external" use. Nothing inside the DAG decides this (it /// isn't a reference from another discovered site), so it's never /// subject to any ancestor's Share/Recompute choice — it's the base - /// case [`CandidateLogicalASAPDAGs::global_selection`]'s recurrence starts from. + /// case [`global_selection`]'s recurrence starts from. external_root_uses: HashMap<*const OperatorNode, usize>, } @@ -1658,8 +1255,8 @@ fn reference_dag(space: &CandidateLogicalASAPDAGs) -> ReferenceDAG { for (_, root) in &space.roots { *dag.external_root_uses.entry(Rc::as_ptr(root)).or_insert(0) += 1; } - for ptr in &space.order { - let group = &space.groups[ptr]; + for ptr in space.order() { + let group = &space.groups()[ptr]; record_possible_edges(*ptr, &group.target, &mut dag); for candidate in &group.candidates { if let Replacement::SubDAG(rewrite) = &candidate.replacement { @@ -1709,11 +1306,11 @@ fn record_possible_edges( /// A topological order over `order` (parent before every child) via Kahn's /// algorithm on `dag`'s reverse adjacency — needed because -/// [`discover_targets`]'s own `order` is only a valid *discovery* order +/// `discover_targets`'s own `order` is only a valid *discovery* order /// (first-seen-first), not a valid topological one: a node reached via two /// different root paths can have a parent that's discovered *after* it (see /// this function's own test for a worked diamond example), which is exactly -/// backwards for [`CandidateLogicalASAPDAGs::global_selection`]'s recurrence. +/// backwards for [`global_selection`]'s recurrence. fn topological_order( order: &[*const OperatorNode], dag: &ReferenceDAG, @@ -1758,16 +1355,24 @@ fn topological_order( #[cfg(test)] mod tests { use super::*; - use crate::accuracy::DefaultAccuracyModel; use crate::cost_model::{Cost, DefaultCostModel}; - use crate::replacement::{ - default_strategies, discover_targets, search_workload, search_workload_with, - search_workload_with_targets, ASAPStrategies, ReplacementStrategy, - }; use crate::test_support::{agg, lower_promql, metric_scan}; - use asap_types::ir::operator::agg_intent::default_quantile; + use asap_logical_optimizer::accuracy::{ + AccuracyModel, DefaultAccuracyModel, EqualSplitAllocator, PropagationStats, + }; + use asap_logical_optimizer::pass1::replacement::{ + default_strategies, search_workload, search_workload_with, search_workload_with_targets, + ASAPStrategies, ReplacementStrategy, + }; + use asap_logical_optimizer::pass2::reconciliation::AccuracyReconciliationStrategy; + use asap_types::ir::operator::agg_intent::{default_cardinality, default_quantile, AggIntent}; use asap_types::ir::operator::operator_properties::Reduction; - use asap_types::ir::ProjectItem; + use asap_types::ir::properties::{ + AccuracyError, CompositionOperator, ErrorMetric, ResultGuarantee, + }; + use asap_types::ir::schema::ColumnId; + use asap_types::ir::schema::SketchStatistic; + use asap_types::ir::{Predicate, ScalarExpr}; use asap_types::types::AccuracyTarget; fn equi_pred(left: ColumnId, right: ColumnId) -> Predicate { @@ -1787,21 +1392,6 @@ mod tests { } } - fn realize(expr: &OperatorNode) -> Result, RealizationError> { - realize_child(&Rc::new(expr.clone())) - } - - #[test] - fn relational_join_predicate_requires_and_normalizes_cross_input_columns() { - let forward = normalize_cross_input_equi_predicate(&equi_pred(1, 3), 2, 4) - .expect("left-to-right equality"); - let reverse = normalize_cross_input_equi_predicate(&equi_pred(3, 1), 2, 4) - .expect("right-to-left equality"); - assert_eq!(forward, reverse, "reverse equality must be canonicalized"); - assert!(normalize_cross_input_equi_predicate(&equi_pred(0, 1), 2, 4).is_none()); - assert!(normalize_cross_input_equi_predicate(&equi_pred(0, 4), 2, 4).is_none()); - } - #[test] fn relational_join_is_exact_only_when_both_inputs_are_exact() { // `relational_join_guarantee` folded into assembly's generic @@ -1838,8 +1428,7 @@ mod tests { ), ] { let space = search_workload(vec![(0usize, Rc::clone(&root))]); - let assembled = space - .global_selection(&DefaultCostModel) + let assembled = global_selection(&space, &DefaultCostModel) .assemble_selected_query(&space.roots[0].1) .unwrap() .unwrap(); @@ -1873,7 +1462,7 @@ mod tests { let group = space.candidates_for_target(&space.roots[0].1).unwrap(); assert_eq!(group.consumer_count, 20); - let ranked = space.cost_sorted(&DefaultCostModel); + let ranked = cost_sorted(&space, &DefaultCostModel); let ranked_group = ranked .iter() .find(|g| Rc::ptr_eq(g.target, &space.roots[0].1)) @@ -1919,7 +1508,7 @@ mod tests { let root = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); let space = search_workload(vec![("q", root)]); - let ranked = space.cost_sorted(&PreferDDSketch); + let ranked = cost_sorted(&space, &PreferDDSketch); let agg_group = ranked .iter() .find(|g| matches!(g.target.non_asap(), Some(NonASAPOp::Aggregate { .. }))) @@ -1960,7 +1549,7 @@ mod tests { }; let root = agg(vec![2, 3], intent, metric_scan(&["tenant_id", "endpoint"])); let space = search_workload(vec![("tenant_endpoint_count", root)]); - let ranked = space.cost_sorted(&model); + let ranked = cost_sorted(&space, &model); let aggregate = ranked .iter() .find(|group| matches!(group.target.non_asap(), Some(NonASAPOp::Aggregate { .. }))) @@ -1991,7 +1580,7 @@ mod tests { fn cost_sorted_pairs_each_candidate_with_its_own_estimate_cost() { let root = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); let space = search_workload(vec![("q", root)]); - let ranked = space.cost_sorted(&DefaultCostModel); + let ranked = cost_sorted(&space, &DefaultCostModel); let agg_group = ranked .iter() .find(|g| matches!(g.target.non_asap(), Some(NonASAPOp::Aggregate { .. }))) @@ -2077,7 +1666,7 @@ mod tests { let space = search_workload(vec![("left", Rc::clone(&aggregate)), ("right", aggregate)]); let root = &space.roots[0].1; assert!(cse_candidate_pair(space.candidates_for_target(root).unwrap()).is_some()); - let selected = space.global_selection(&MixedCost); + let selected = global_selection(&space, &MixedCost); let chosen = selected.for_target(root).unwrap().chosen.unwrap(); assert!(!is_cse_candidate(chosen)); } @@ -2091,8 +1680,8 @@ mod tests { let root = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); let space = search_workload(vec![("q", root)]); - let ranked = space.cost_sorted(&DefaultCostModel); - let selected = space.global_selection(&DefaultCostModel); + let ranked = cost_sorted(&space, &DefaultCostModel); + let selected = global_selection(&space, &DefaultCostModel); assert_eq!(ranked.len(), selected.target_selections().count()); for ranked_group in &ranked { @@ -2117,7 +1706,7 @@ mod tests { // doc) — global_selection must not invent a candidate for it. let root = metric_scan(&["job"]); let space = search_workload(vec![("q", root)]); - let selected = space.global_selection(&DefaultCostModel); + let selected = global_selection(&space, &DefaultCostModel); let scan_group = selected .target_selections() .find(|g| matches!(g.target.non_asap(), Some(NonASAPOp::Scan { .. }))) @@ -2155,7 +1744,7 @@ mod tests { let root = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); let space = search_workload(vec![("q", root)]); - let selected = space.global_selection(&PreferDDSketch); + let selected = global_selection(&space, &PreferDDSketch); let agg_group = selected .target_selections() .find(|g| matches!(g.target.non_asap(), Some(NonASAPOp::Aggregate { .. }))) @@ -2176,8 +1765,7 @@ mod tests { candidates.to_vec() } } - assert!(space - .global_selection(&Uncosted) + assert!(global_selection(&space, &Uncosted) .for_target(&space.roots[0].1) .unwrap() .chosen @@ -2187,32 +1775,36 @@ mod tests { #[test] fn mixed_rewrite_group_keeps_and_selects_its_explicit_cse_pair() { let target = metric_scan(&["job"]); - let mut group = TargetSubDAGCandidates::new(Rc::clone(&target), 2); - group.candidates = vec![ - ReplacementSubDAG { - strategy: "TestStrategy", - replacement: Replacement::SubDAG(Rc::clone(&target)), - provenance: ReplacementProvenance::CseShare, - rationale: "share".into(), - }, - ReplacementSubDAG { - strategy: "TestStrategy", - replacement: Replacement::SubDAG(Rc::new(target.as_ref().clone())), - provenance: ReplacementProvenance::CseRecompute, - rationale: "recompute".into(), - }, - ReplacementSubDAG { - strategy: "TestStrategy", - replacement: Replacement::SubDAG( - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::PromqlVectorFromScalar(ScalarExpr::EvalTimestamp), - )) - .unwrap(), - ), - provenance: ReplacementProvenance::LogicalRewrite, - rationale: "different rewrite strategy".into(), - }, - ]; + let group = TargetSubDAGCandidates { + target: Rc::clone(&target), + consumer_count: 2, + rejected: Vec::new(), + candidates: vec![ + ReplacementSubDAG { + strategy: "TestStrategy", + replacement: Replacement::SubDAG(Rc::clone(&target)), + provenance: ReplacementProvenance::CseShare, + rationale: "share".into(), + }, + ReplacementSubDAG { + strategy: "TestStrategy", + replacement: Replacement::SubDAG(Rc::new(target.as_ref().clone())), + provenance: ReplacementProvenance::CseRecompute, + rationale: "recompute".into(), + }, + ReplacementSubDAG { + strategy: "TestStrategy", + replacement: Replacement::SubDAG( + OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( + NonASAPOp::PromqlVectorFromScalar(ScalarExpr::EvalTimestamp), + )) + .unwrap(), + ), + provenance: ReplacementProvenance::LogicalRewrite, + rationale: "different rewrite strategy".into(), + }, + ], + }; assert!(cse_candidate_pair(&group).is_some()); let ranked = rank_group(&group, &ConstantCseCost); @@ -2317,7 +1909,7 @@ mod tests { // The naive/local answer: cost_sorted ranks c using its raw count // (2) alone and prefers RecomputeIndependently. - let ranked = space.cost_sorted(&ConstantCseCost); + let ranked = cost_sorted(&space, &ConstantCseCost); let c_ranked = ranked .iter() .find(|g| Rc::ptr_eq(g.target, c_via_a)) @@ -2335,7 +1927,7 @@ mod tests { // The corrected, cross-group-aware answer: global_selection folds // a's own RecomputeIndependently choice into c's effective count // (2 from a + 1 from root3 = 3) and flips to Share. - let selected = space.global_selection(&ConstantCseCost); + let selected = global_selection(&space, &ConstantCseCost); let a_selected = selected.for_target(a_rc).unwrap(); let c_selected = selected.for_target(c_via_a).unwrap(); @@ -2408,7 +2000,7 @@ mod tests { ]); let planned = &space.roots[0].1; - let selected = space.global_selection(&CompletePlanCost); + let selected = global_selection(&space, &CompletePlanCost); assert!(selected.for_target(planned).unwrap().chosen.is_none()); } @@ -2440,7 +2032,7 @@ mod tests { assert_eq!(space.candidates_for_target(c_rc).unwrap().consumer_count, 1); assert!(cse_candidate_pair(space.candidates_for_target(c_rc).unwrap()).is_some()); - let selected = space.global_selection(&ConstantCseCost); + let selected = global_selection(&space, &ConstantCseCost); let child = selected.for_target(c_rc).unwrap(); assert_eq!(child.effective_consumer_count, 2); assert!(child.chosen.is_some()); @@ -2493,7 +2085,7 @@ mod tests { panic!("expected Filter root"); }; - let selected = space.global_selection(&AlwaysShare); + let selected = global_selection(&space, &AlwaysShare); assert_eq!( selected .for_target(parent_rc) @@ -2548,7 +2140,7 @@ mod tests { let strategies: Vec> = vec![Box::new(ReplaceFilterChild)]; let space = search_workload_with(vec![("q", root)], &strategies); let root = &space.roots[0].1; - let selected = space.global_selection(&DefaultCostModel); + let selected = global_selection(&space, &DefaultCostModel); let Replacement::SubDAG(rewrite) = &selected .for_target(root) .unwrap() @@ -2610,7 +2202,7 @@ mod tests { } let root = lower_promql("sum_over_time(a[1m])", AccuracyTarget::Exact); let space = search_workload(vec![("q", root)]); - let selected = space.global_selection(&Unsupported); + let selected = global_selection(&space, &Unsupported); assert!(selected .for_target(&space.roots[0].1) .unwrap() @@ -2657,8 +2249,11 @@ mod tests { } } let space = search_workload(vec![("q", root.clone())]); - let selected = space.global_selection(&PreferComposed); - let node = selected.assemble_target(&space.roots[0].1).unwrap(); + let selected = global_selection(&space, &PreferComposed); + let node = selected + .assemble_selected_dag(&space.roots[0].1) + .unwrap() + .unwrap(); assert!(matches!(&node.operator, Operator::ASAP(ASAPOp::SummaryAgg { reduction: Reduction::Reduce(_), child, .. }) if !child.contains_asap())); @@ -2692,7 +2287,7 @@ mod tests { } let root = lower_promql("sum by(job)(sum_over_time(a[1m]))", AccuracyTarget::Exact); let space = search_workload(vec![("q", root)]); - let selection = space.global_selection(&ExplicitCosts); + let selection = global_selection(&space, &ExplicitCosts); let selected = selection .for_target(&space.roots[0].1) .unwrap() @@ -2730,7 +2325,7 @@ mod tests { let b = agg(vec![2], AggIntent::Avg { col: None }, metric_scan(&["job"])); let space = search_workload(vec![("a", a), ("b", b)]); let root = &space.roots[0].1; - let selected = space.global_selection(&PreferLogicalRewrite); + let selected = global_selection(&space, &PreferLogicalRewrite); assert_eq!( selected @@ -2768,25 +2363,8 @@ mod tests { .unwrap(); let roots = vec![("a", root_a), ("b", root_b)]; - let mut order = Vec::new(); - let mut nodes = HashMap::new(); - let mut counts = HashMap::new(); - discover_targets(&roots, &mut order, &mut nodes, &mut counts); - let groups = order - .iter() - .map(|ptr| { - ( - *ptr, - TargetSubDAGCandidates::new(Rc::clone(&nodes[ptr]), counts[ptr]), - ) - }) - .collect(); - let space = CandidateLogicalASAPDAGs { - roots, - groups, - order: order.clone(), - composition_plans: Vec::new(), - }; + let space = search_workload(roots); + let order = space.order(); let dag = reference_dag(&space); // Discovery-order sanity: root_b comes after the shared child in @@ -2808,7 +2386,7 @@ mod tests { "fixture sanity: discover_targets's own order must NOT already be topological here" ); - let topo = topological_order(&order, &dag); + let topo = topological_order(order, &dag); let shared_topo_pos = topo.iter().position(|p| *p == shared_ptr).unwrap(); let root_b_topo_pos = topo.iter().position(|p| *p == root_b_ptr).unwrap(); assert!( @@ -2818,50 +2396,464 @@ mod tests { ); } - // A value projection cannot consume an opaque exact accumulator edge. + // ── Selection over Stage 1 candidates (split from `replacement` tests) ─ + + // Every exposed query result has a evaluation; internal accumulator frontiers stay states. + #[test] + fn selected_query_roots_do_not_leak_exact_accumulator_state() { + for query in [ + "sum by(job)(rate(m[1m]))", + "sum by(job)(m)", + "sum_over_time(m[1m])", + ] { + let root = lower_promql(query, AccuracyTarget::Exact); + let space = search_workload(vec![(0usize, root)]); + let selected = global_selection(&space, &DefaultCostModel) + .assemble_selected_query(&space.roots[0].1) + .unwrap() + .unwrap(); + assert!( + selected + .schema + .fields + .iter() + .all(|field| matches!(field.dtype, FieldDataType::Plain(_))), + "{query}: query root leaks state: {:?}", + selected.schema + ); + } + } + + /// Candidates kept with missing accuracy evidence are never chosen automatically. #[test] - fn residual_projection_finalizes_selected_exact_state() { - let inner = agg(vec![], AggIntent::Sum { col: None }, metric_scan(&[])); - let root = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - cols: vec![ProjectItem { - expr: ScalarExpr::Column(0), - alias: Some("result".into()), - }], - qualifier: None, - child: inner.clone(), - })) - .unwrap(); + fn selection_never_chooses_a_candidate_missing_accuracy_evidence() { + let never_unproven = |space: &CandidateLogicalASAPDAGs<&str>| { + let selected = global_selection(space, &DefaultCostModel); + assert!(!selected + .for_target(&space.roots[0].1) + .unwrap() + .chosen + .is_some_and(ReplacementSubDAG::has_missing_accuracy_evidence)); + assert!(selected + .assemble_selected_dag(&space.roots[0].1) + .unwrap() + .is_some()); + }; + let count = AggIntent::Count { + accuracy: AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.01, + }, + }; + never_unproven(&search_workload(vec![( + "q", + agg(vec![2], count, metric_scan(&["job"])), + )])); + never_unproven(&search_workload_with_targets( + vec![( + "q", + agg(vec![2], default_cardinality(), metric_scan(&["job"])), + Some(AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.01, + }), + )], + &default_strategies(), + &DefaultAccuracyModel, + )); + let inner = agg( + vec![2], + AggIntent::Count { + accuracy: AccuracyTarget::Epsilon(0.01), + }, + metric_scan(&["job"]), + ); + let topk = agg( + vec![], + AggIntent::TopK { + k: 10, + accuracy: AccuracyTarget::Epsilon(0.01), + }, + inner, + ); + never_unproven(&search_workload_with_targets( + vec![("q", topk, Some(AccuracyTarget::Epsilon(0.01)))], + &default_strategies(), + &DefaultAccuracyModel, + )); + } + + /// A root target that rejects every summary leaves nothing to select. + #[test] + fn nothing_is_selected_when_the_root_target_rejects_every_summary() { + let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); let space = search_workload_with_targets( - vec![("q", root.clone(), Some(AccuracyTarget::Exact))], + vec![("q", q, Some(AccuracyTarget::Epsilon(0.001)))], &default_strategies(), &DefaultAccuracyModel, ); - let selected = space.global_selection(&DefaultCostModel); - // CSE re-interns the workload, so the space's root/child `Rc`s are not - // the fixture's. Assembly only assembles children that are discovered - // targets, so seed the memo under the space's own child pointer. - let root = Rc::clone(&space.roots[0].1); - let Some(NonASAPOp::Project { child: inner, .. }) = root.non_asap() else { - unreachable!() - }; - assert!(space.candidates_for_target(inner).is_some()); - selected - .assembled_nodes - .borrow_mut() - .insert(Rc::as_ptr(inner), realize(inner.as_ref()).unwrap()); - let node = selected.assemble_target(&root).unwrap(); - let Operator::NonASAP(NonASAPOp::Project { child, .. }) = &node.operator else { - panic!("expected Project"); + let root = &space.roots[0].1; + assert!(global_selection(&space, &DefaultCostModel) + .for_target(root) + .unwrap() + .chosen + .is_none()); + } + + /// Admits rank-over-rank composition, which `DefaultAccuracyModel` has no + /// rule for, so a nested quantile summary can be selected. + struct RankAdditiveModel; + + impl AccuracyModel for RankAdditiveModel { + fn local_guarantee( + &self, + family: &FieldDataType, + query: &SketchStatistic, + ) -> Option { + DefaultAccuracyModel.local_guarantee(family, query) + } + + fn propagate( + &self, + op: &CompositionOperator, + inputs: &[ResultGuarantee], + local: Option<&ResultGuarantee>, + stats: &PropagationStats, + ) -> Result { + let rank = |g: &ResultGuarantee| g.is_exact() || g.metric == ErrorMetric::Rank; + if let (CompositionOperator::ApproximateAggregate, true, Some(local)) = + (op, inputs.iter().all(rank), local) + { + let relabel = |g: &ResultGuarantee| ResultGuarantee { + metric: ErrorMetric::AbsoluteValue, + ..g.clone() + }; + let inputs: Vec<_> = inputs.iter().map(relabel).collect(); + let mut out = + DefaultAccuracyModel.propagate(op, &inputs, Some(&relabel(local)), stats)?; + out.metric = local.metric; + return Ok(out); + } + DefaultAccuracyModel.propagate(op, inputs, local, stats) + } + + fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool { + DefaultAccuracyModel.satisfies(guarantee, target) + } + } + + #[test] + fn global_selection_can_choose_nested_summaries() { + // The same nested summary remains available through workload search + // and global cost ranking. + let inner = agg( + vec![2], + quantile_eps_intent(0.5, 0.1), + metric_scan(&["job"]), + ); + let outer = agg(vec![], quantile_eps_intent(0.99, 0.1), inner); + let strategies: Vec> = vec![Box::new( + ASAPStrategies::new_with_planning_inputs(&RankAdditiveModel, &EqualSplitAllocator), + )]; + let space = search_workload_with(vec![("q", Rc::clone(&outer))], &strategies); + let root = &space.roots[0].1; + let group = space.candidates_for_target(root).unwrap(); + assert!(!group.rejected.is_empty()); + assert!(group.candidates.iter().all(|c| match &c.replacement { + // A summary candidate (old `Replacement::Summary`) contains an + // ASAP node; a logical rewrite (old `Replacement::Rewrite`) does not. + Replacement::SubDAG(node) if node.contains_asap() => { + node.guarantee.as_ref().is_some_and(|g| { + DefaultAccuracyModel.satisfies(g, &AccuracyTarget::Epsilon(0.1)) + }) + } + Replacement::SubDAG(_) => false, + Replacement::ExactComposition(_) => false, + })); + let ranked = cost_sorted(&space, &DefaultCostModel); + let root_ranked = ranked.iter().find(|g| Rc::ptr_eq(g.target, root)).unwrap(); + assert_eq!(root_ranked.candidates.len(), group.candidates.len()); + + let selection = global_selection(&space, &DefaultCostModel); + let chosen = selection + .for_target(root) + .unwrap() + .chosen + .expect("a nested summary candidate wins"); + let Replacement::SubDAG(node) = &chosen.replacement else { + panic!() }; assert!(matches!( - child.operator, - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) + node.operator, + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) )); - assert!(child - .schema - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); + } + + // ── Accuracy reconciliation (moved from `accuracy::reconciliation` tests) ─ + + mod accuracy_reconciliation { + use super::*; + use asap_types::ir::operator::operator_properties::Source; + use asap_types::ir::schema::{DataType, Field, Schema}; + + /// `[ts(0), value(1), job(2)]`, uniquely keyed by `ts` so CSE can hoist it. + fn metric_scan() -> Rc { + OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("job", DataType::Utf8, true), + ], + 0, + vec![vec![0]], + ), + })) + .unwrap() + } + + fn quantile( + q: f64, + accuracy: AccuracyTarget, + child: &Rc, + ) -> Rc { + agg( + vec![2], + AggIntent::Quantile { + col: None, + q, + accuracy, + }, + Rc::clone(child), + ) + } + + // ── cost: reading the sibling must not be priced like recomputing + // `target` independently per consumer ──────────────────────────────── + + #[test] + fn estimate_cost_does_not_scale_with_the_readers_own_consumer_count() { + // Regression guard for the review-reported sign inversion: pricing + // this candidate like `CseRecompute` ("rebuild `target`, once per + // consumer") made it artificially *more* expensive exactly as more + // of `target`'s own consumers stood to benefit from reading the + // already-necessary tighter sibling instead — the literal opposite + // of the intended incentive. The real cost is "one more read against + // `rc`'s own build," which must not scale with `target`'s own + // `consumer_count`. + let scan = metric_scan(); + let tight = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); + let loose = quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); + let strategy = + AccuracyReconciliationStrategy::new(&[Rc::clone(&tight), Rc::clone(&loose)]); + let candidate = strategy + .replacements(&TargetSubDAG::new(&loose)) + .into_iter() + .next() + .expect("loose has a reconciliation candidate reading the tight sibling"); + + let cost_model = DefaultCostModel; + let single_consumer = TargetSubDAG::with_consumer_count(&loose, 1); + let many_consumers = TargetSubDAG::with_consumer_count(&loose, 5); + + let cost_single = cost_model.estimate_cost(&candidate, &single_consumer); + let cost_many = cost_model.estimate_cost(&candidate, &many_consumers); + + assert!( + cost_single.is_finite(), + "expected a real cost, not the NaN placeholder: {cost_single}" + ); + assert_eq!( + cost_single, cost_many, + "AccuracyReconciliation's estimate_cost must price 'read the sibling', not scale \ + with the reader's own consumer_count the way CseRecompute's 'rebuild independently \ + per consumer' formula does (single-consumer: {cost_single}, 5 consumers: \ + {cost_many})" + ); + } + + // ── cost_sorted / global_selection: single-consumer and shared-consumer ─ + + #[test] + fn cost_sorted_and_global_selection_handle_a_single_consumer_looser_target() { + let scan = metric_scan(); + let tight = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); + let loose = quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); + let space = search_workload(vec![("tight", tight), ("loose", loose)]); + let loose_root = &space.roots[1].1; + + let cost_model = DefaultCostModel; + let ranked = cost_sorted(&space, &cost_model); + let loose_ranked = ranked + .iter() + .find(|group| Rc::ptr_eq(group.target, loose_root)) + .expect("loose has its own ranked group"); + assert!( + loose_ranked.costs.iter().all(|cost| cost.is_finite()), + "no candidate should cost NaN under DefaultCostModel: {:?}", + loose_ranked.costs + ); + assert!( + loose_ranked + .candidates + .iter() + .any(|c| c.strategy == "AccuracyReconciliationStrategy"), + "the reconciliation candidate must still be present, ranked, not filtered" + ); + + let selected = global_selection(&space, &cost_model); + let chosen = selected + .for_target(loose_root) + .and_then(|group| group.chosen); + assert!( + chosen.is_some(), + "global_selection must commit to some candidate for a single-consumer looser target" + ); + // With no recompute term at all (it never rebuilds `target`), this + // candidate strictly undercuts every ASAPStrategies + // candidate (which each pay a recompute term on top of their own + // maintenance term) under DefaultCostModel's numbers — the sane + // direction: reading an already-necessary sibling should be able to + // win on its own merit, not just fail to lose as badly as before. + assert_eq!( + chosen.map(|c| c.provenance), + Some(asap_logical_optimizer::pass1::replacement::ReplacementProvenance::AccuracyReconciliation) + ); + } + + #[test] + fn cost_sorted_and_global_selection_handle_a_shared_looser_target() { + // The loose accuracy target itself has 2 direct consumers (two + // independently-built but structurally identical loose queries + // merge onto one Rc via ordinary CSE), *and* a separate, + // single-consumer tight sibling exists over the same input — the + // scenario the issue itself targets: `SharedSubDAGStrategy`'s own + // CseShare/CseRecompute pair is on the table for the loose target's + // own 2 consumers at the same time as this strategy's "read the + // tight sibling instead" candidate. + let scan = metric_scan(); + let loose_a = (*quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan)).clone(); + let loose_b = (*quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan)).clone(); + let tight = (*quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan)).clone(); + + let space = search_workload(vec![ + ("loose_a", Rc::new(loose_a)), + ("loose_b", Rc::new(loose_b)), + ("tight", Rc::new(tight)), + ]); + + // Fixture sanity: the two loose roots really did merge onto one Rc. + assert!(Rc::ptr_eq(&space.roots[0].1, &space.roots[1].1)); + let loose_group = space + .candidates_for_target(&space.roots[0].1) + .expect("the merged loose target has a group"); + assert_eq!(loose_group.consumer_count, 2); + assert!( + loose_group + .candidates + .iter() + .any(|c| c.strategy == "AccuracyReconciliationStrategy"), + "the reconciliation candidate must still be proposed alongside the CSE share/recompute \ + pair, not crowded out: {:?}", + loose_group + .candidates + .iter() + .map(|c| (c.strategy, c.provenance)) + .collect::>() + ); + + let cost_model = DefaultCostModel; + let ranked = cost_sorted(&space, &cost_model); + let loose_ranked = ranked + .iter() + .find(|group| Rc::ptr_eq(group.target, &space.roots[0].1)) + .expect("loose has its own ranked group"); + assert!( + loose_ranked.costs.iter().all(|cost| cost.is_finite()), + "no candidate should cost NaN under DefaultCostModel, shared or not: {:?}", + loose_ranked.costs + ); + + let selected = global_selection(&space, &cost_model); + let chosen = selected + .for_target(&space.roots[0].1) + .and_then(|group| group.chosen); + assert!( + chosen.is_some(), + "global_selection must commit to some candidate for the shared looser target" + ); + // Under `DefaultCostModel`'s numbers, `CseShare` (flat maintenance, + // no recompute term) and this strategy's own candidate (also a + // flat, non-scaling read cost after the fix) land tied, and + // `global_selection` breaks ties in `CseShare`'s favor (it only ever + // overrides the CSE choice on a *strict* `<`, not `<=`) — a sane, + // deliberate tie-break, not the "reconciliation always loses to + // CseShare regardless of its own real merit" bug this test guards + // against (see `estimate_cost_does_not_scale_with_the_readers_own_consumer_count` + // for the direct regression check that the old `* consumer_count` + // scaling — which made this an unfair, ever-widening loss instead + // of a tie — is gone). + assert_eq!( + chosen.map(|c| c.provenance), + Some(asap_logical_optimizer::pass1::replacement::ReplacementProvenance::CseShare) + ); + } + + #[test] + fn global_selection_propagates_reconciled_consumers_to_the_tighter_group() { + struct PreferReconciliation; + + impl CostModel for PreferReconciliation { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + + fn estimate_cost( + &self, + candidate: &ReplacementSubDAG, + _target: &TargetSubDAG<'_>, + ) -> f64 { + if candidate.provenance == ReplacementProvenance::AccuracyReconciliation { + 0.0 + } else { + 100.0 + } + } + } + + let scan = metric_scan(); + let tight = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); + let loose = quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); + let space = search_workload(vec![ + ("tight", Rc::clone(&tight)), + ("loose", Rc::clone(&loose)), + ]); + + let selected = global_selection(&space, &PreferReconciliation); + let tight_root = &space.roots[0].1; + let loose_root = &space.roots[1].1; + assert_eq!( + selected + .for_target(loose_root) + .and_then(|group| group.chosen) + .map(|candidate| candidate.provenance), + Some(ReplacementProvenance::AccuracyReconciliation), + "fixture must select the cross-sibling rewrite" + ); + assert_eq!( + selected + .for_target(tight_root) + .expect("the tighter sibling is a discovered memo group") + .effective_consumer_count, + 2, + "the tighter build serves its original root and the reconciled looser root" + ); + } } } diff --git a/crates/asap-aware-mapping/src/plan_selection/mod.rs b/crates/asap-aware-mapping/src/plan_selection/mod.rs index 090151cf..f2886bc6 100644 --- a/crates/asap-aware-mapping/src/plan_selection/mod.rs +++ b/crates/asap-aware-mapping/src/plan_selection/mod.rs @@ -38,21 +38,21 @@ use asap_types::types::AccuracyTarget; use asap_types::workload::DataWorkload; use thiserror::Error; -use crate::accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, -}; use crate::analytical_cost::{ estimate_operator, AnalyticalCostError, PhysicalOperator, ResourceCalibration, ResourceEstimate, }; use crate::cost_model::{CostModel, DefaultCostModel}; -use crate::logical_candidates::{ - choice_index, combination_count, compose_logical_candidate, enumerate_choices, nested_targets, - LocalLogicalCandidates, -}; use crate::physical_candidates::{stage2_physical, PhysicalCandidate}; use crate::physical_operator_statistics::{ EdgeStatistics, OperatorStatistics, PartitionStatistics, UnaryEdgeStatistics, }; +use asap_logical_optimizer::accuracy::{ + AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, +}; +use asap_logical_optimizer::pass1::logical_candidates::{ + choice_index, combination_count, compose_logical_candidate, enumerate_choices, nested_targets, + LocalLogicalCandidates, +}; pub const COST_UNIT: &str = "cpu_ms_per_workload_evaluation"; pub const COST_SOURCE: &str = "analytical-cost-v1 (illustrative statistics)"; @@ -1039,11 +1039,11 @@ mod tests { delta: 0.001, }, ); - let inventory = crate::logical_candidates::enumerate_local_logical_candidates(vec![( - 0, - QueryRoot::Operator(root), - )]) - .unwrap(); + let inventory = + asap_logical_optimizer::pass1::logical_candidates::enumerate_local_logical_candidates( + vec![(0, QueryRoot::Operator(root))], + ) + .unwrap(); let topk = inventory .targets .iter() @@ -1055,14 +1055,16 @@ mod tests { let mut choice = vec![0; inventory.targets.len()]; choice[topk] = alternative; let roots: Vec<_> = - crate::logical_candidates::compose_logical_candidate(&inventory, &choice) - .unwrap() - .into_iter() - .map(|(_, root)| match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(); + asap_logical_optimizer::pass1::logical_candidates::compose_logical_candidate( + &inventory, &choice, + ) + .unwrap() + .into_iter() + .map(|(_, root)| match root { + QueryRoot::Operator(node) => node, + QueryRoot::Scalar(_) => panic!("operator root"), + }) + .collect(); let mut candidate = stage2_physical("L", &roots).unwrap(); candidate.id = format!("P{}", alternative + 1); candidate @@ -1137,7 +1139,8 @@ mod tests { (i, QueryRoot::Operator(root)) }) .collect(); - crate::logical_candidates::enumerate_local_logical_candidates(roots).unwrap() + asap_logical_optimizer::pass1::logical_candidates::enumerate_local_logical_candidates(roots) + .unwrap() } fn no_targets(inventory: &LocalLogicalCandidates) -> Vec> { @@ -1262,10 +1265,10 @@ mod tests { "quantile_over_time(0.5, m[5m])", "quantile_over_time(0.99, m[5m])", ]); - let kll = |t: &crate::logical_candidates::LocalLogicalTarget| { + let kll = |t: &asap_logical_optimizer::pass1::logical_candidates::LocalLogicalTarget| { t.alternatives .iter() - .position(|a| matches!(a, crate::Realization::Sketch(kind) if *kind.algorithm() == SketchAlgorithm::Kll)) + .position(|a| matches!(a, asap_logical_optimizer::Realization::Sketch(kind) if *kind.algorithm() == SketchAlgorithm::Kll)) .unwrap() }; let choice: Vec<_> = inventory.targets.iter().map(kll).collect(); diff --git a/crates/asap-aware-mapping/src/recurrence.rs b/crates/asap-aware-mapping/src/recurrence.rs index 9a648f6d..15e86592 100644 --- a/crates/asap-aware-mapping/src/recurrence.rs +++ b/crates/asap-aware-mapping/src/recurrence.rs @@ -75,7 +75,7 @@ //! //! - [`EvaluationRate`]: derived from [`asap_types::workload::RepeatingEntry::demand`] //! values of every repeating consumer reaching a target (via -//! [`evaluation_rate_of`], or [`crate::replacement::CandidateLogicalASAPDAGs::recurrence_profiles`] +//! [`evaluation_rate_of`], or [`recurrence_profiles`](crate::plan_selection::candidate_selection::recurrence_profiles) //! for a whole workload). A one-shot ([`asap_types::workload::BatchEntry`]) //! consumer contributes to [`RecurrenceProfile::one_shot_consumers`] //! instead, never to this rate. @@ -216,7 +216,7 @@ pub enum RecurrenceError { CostRate with a one-shot Cost without distorting the comparison" )] InvalidHorizon(Horizon), - /// [`crate::replacement::CandidateLogicalASAPDAGs::recurrence_profiles`] was called + /// [`recurrence_profiles`](crate::plan_selection::candidate_selection::recurrence_profiles) was called /// with a `root_recurrence` slice whose length doesn't match the /// `CandidateLogicalASAPDAGs`'s own root count — a caller error, but recoverable /// (this method's whole signature promises a `Result`, so this is @@ -239,7 +239,7 @@ pub enum RecurrenceError { /// applied at every point an `UpdateRate` enters a [`RecurrenceProfile`] /// ([`RecurrenceProfile::with_update_rate`], /// [`update_rate_from_data_workload`], -/// [`crate::replacement::CandidateLogicalASAPDAGs::recurrence_profiles`]'s own parameter) +/// [`recurrence_profiles`](crate::plan_selection::candidate_selection::recurrence_profiles)'s own parameter) /// *and*, as a backstop that can't be bypassed by constructing a /// `RecurrenceProfile` via its public fields directly, inside [`decide`] /// itself before any comparison uses it. @@ -373,7 +373,7 @@ impl RecurrenceProfile { } /// How one workload root recurs — the opaque per-root tag -/// [`crate::replacement::CandidateLogicalASAPDAGs::recurrence_profiles`] threads down to +/// [`recurrence_profiles`](crate::plan_selection::candidate_selection::recurrence_profiles) threads down to /// every target reachable from that root. Mirrors /// [`asap_types::workload::QueryWorkload`]'s own `query_batch` (one-shot) /// vs. `repeating_queries` (an interval each) split, but at the @@ -633,6 +633,9 @@ pub(crate) fn decide( mod tests { use super::*; use crate::cost_model::DefaultCostModel; + use crate::plan_selection::candidate_selection::cost_sorted_with_recurrence; + use crate::plan_selection::candidate_selection::global_selection_with_recurrence; + use crate::plan_selection::candidate_selection::recurrence_profiles; fn interval(ms: u32) -> RepetitionInterval { RepetitionInterval(ms) @@ -1129,7 +1132,7 @@ mod tests { // ── multiple roots sharing a sub-DAG, via CandidateLogicalASAPDAGs ────────────────── - use crate::replacement::search_workload; + use asap_logical_optimizer::pass1::replacement::search_workload; use asap_types::ir::operator::agg_intent::AggIntent; use asap_types::ir::operator::operator_properties::Reduction as QueryReduction; use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; @@ -1196,7 +1199,7 @@ mod tests { /// Three workload roots share one underlying `sum_agg()` sub-DAG: two /// repeating consumers with different intervals, one one-shot batch - /// consumer. `CandidateLogicalASAPDAGs::recurrence_profiles` must aggregate all three + /// consumer. `candidate_selection::recurrence_profiles` must aggregate all three /// onto the shared sub-DAG's own profile: `evaluation_rate = 1/t1 + /// 1/t2`, `one_shot_consumers = 1` — issue #287's "support a shared /// sub-DAG consumed by queries with different intervals" and "multiple @@ -1233,9 +1236,8 @@ mod tests { repeating(10_000), // 0.1 Hz RootRecurrence::OneShotCount(1), ]; - let profiles = space - .recurrence_profiles(&root_recurrence, Some(UpdateRate(5.0))) - .unwrap(); + let profiles = + recurrence_profiles(&space, &root_recurrence, Some(UpdateRate(5.0))).unwrap(); let profile = profiles.for_target(&shared_group.target); let expected_rate = 1.0 / 1.0 + 1.0 / 10.0; // Hz @@ -1276,19 +1278,21 @@ mod tests { .expect("the aggregate is shared by both roots"); let update_rate = Some(UpdateRate(10.0)); - let frequent = space - .recurrence_profiles(&[repeating(10), repeating(10)], update_rate) - .unwrap(); - let infrequent = space - .recurrence_profiles(&[repeating(100_000), repeating(100_000)], update_rate) - .unwrap(); + let frequent = + recurrence_profiles(&space, &[repeating(10), repeating(10)], update_rate).unwrap(); + let infrequent = recurrence_profiles( + &space, + &[repeating(100_000), repeating(100_000)], + update_rate, + ) + .unwrap(); - let frequent_ranked = space - .cost_sorted_with_recurrence(&DeterministicUnitCostModel, &frequent, None) - .unwrap(); - let infrequent_ranked = space - .cost_sorted_with_recurrence(&DeterministicUnitCostModel, &infrequent, None) - .unwrap(); + let frequent_ranked = + cost_sorted_with_recurrence(&space, &DeterministicUnitCostModel, &frequent, None) + .unwrap(); + let infrequent_ranked = + cost_sorted_with_recurrence(&space, &DeterministicUnitCostModel, &infrequent, None) + .unwrap(); let first_provenance = |ranked: &[crate::plan_selection::candidate_selection::RankedTargetSubDAGCandidates<'_>]| { ranked @@ -1299,32 +1303,36 @@ mod tests { }; assert_eq!( first_provenance(&frequent_ranked), - Some(crate::replacement::ReplacementProvenance::CseShare) + Some(asap_logical_optimizer::pass1::replacement::ReplacementProvenance::CseShare) ); assert_eq!( first_provenance(&infrequent_ranked), - Some(crate::replacement::ReplacementProvenance::CseRecompute) + Some(asap_logical_optimizer::pass1::replacement::ReplacementProvenance::CseRecompute) ); - let frequent_selected = space - .global_selection_with_recurrence(&DeterministicUnitCostModel, &frequent, None) - .unwrap(); - let infrequent_selected = space - .global_selection_with_recurrence(&DeterministicUnitCostModel, &infrequent, None) - .unwrap(); + let frequent_selected = + global_selection_with_recurrence(&space, &DeterministicUnitCostModel, &frequent, None) + .unwrap(); + let infrequent_selected = global_selection_with_recurrence( + &space, + &DeterministicUnitCostModel, + &infrequent, + None, + ) + .unwrap(); assert_eq!( frequent_selected .for_target(&shared.target) .and_then(|group| group.chosen) .map(|candidate| candidate.provenance), - Some(crate::replacement::ReplacementProvenance::CseShare) + Some(asap_logical_optimizer::pass1::replacement::ReplacementProvenance::CseShare) ); assert_eq!( infrequent_selected .for_target(&shared.target) .and_then(|group| group.chosen) .map(|candidate| candidate.provenance), - Some(crate::replacement::ReplacementProvenance::CseRecompute) + Some(asap_logical_optimizer::pass1::replacement::ReplacementProvenance::CseRecompute) ); } @@ -1333,9 +1341,12 @@ mod tests { let root = scan(); let roots: Vec<(&str, Rc)> = vec![("only", root)]; let space = search_workload(roots); - let err = space - .recurrence_profiles(&[RootRecurrence::Repeating(EvaluationRate(f64::NAN))], None) - .unwrap_err(); + let err = recurrence_profiles( + &space, + &[RootRecurrence::Repeating(EvaluationRate(f64::NAN))], + None, + ) + .unwrap_err(); assert!(matches!(err, RecurrenceError::InvalidEvaluationRate(_))); } @@ -1347,7 +1358,7 @@ mod tests { let root = scan(); let roots: Vec<(&str, Rc)> = vec![("only", root)]; let space = search_workload(roots); - let err = space.recurrence_profiles(&[], None).unwrap_err(); + let err = recurrence_profiles(&space, &[], None).unwrap_err(); assert_eq!( err, RecurrenceError::RootCountMismatch { @@ -1362,12 +1373,12 @@ mod tests { let root = scan(); let roots: Vec<(&str, Rc)> = vec![("only", root)]; let space = search_workload(roots); - let err = space - .recurrence_profiles( - &[RootRecurrence::OneShotCount(1)], - Some(UpdateRate(f64::NAN)), - ) - .unwrap_err(); + let err = recurrence_profiles( + &space, + &[RootRecurrence::OneShotCount(1)], + Some(UpdateRate(f64::NAN)), + ) + .unwrap_err(); assert!(matches!(err, RecurrenceError::InvalidUpdateRate(_))); } @@ -1416,9 +1427,8 @@ mod tests { ); let root_recurrence = vec![repeating(1_000)]; - let profiles = space - .recurrence_profiles(&root_recurrence, Some(UpdateRate(5.0))) - .unwrap(); + let profiles = + recurrence_profiles(&space, &root_recurrence, Some(UpdateRate(5.0))).unwrap(); let count_profile = profiles.for_target(&count_group.target); assert_eq!( @@ -1472,7 +1482,7 @@ mod tests { ); let root_recurrence = vec![repeating(1_000)]; // 1 Hz - let profiles = space.recurrence_profiles(&root_recurrence, None).unwrap(); + let profiles = recurrence_profiles(&space, &root_recurrence, None).unwrap(); let profile = profiles.for_target(&shared_group.target); // Referenced twice from the one root: evaluation_rate should be diff --git a/crates/asap-aware-mapping/src/test_support.rs b/crates/asap-aware-mapping/src/test_support.rs index 368b5234..1c0cefb4 100644 --- a/crates/asap-aware-mapping/src/test_support.rs +++ b/crates/asap-aware-mapping/src/test_support.rs @@ -1,5 +1,5 @@ -// Shared fixture helpers; not every test module uses every helper. -#![allow(dead_code)] +// Fixture helpers for this crate's tests: the subset of +// `asap-logical-optimizer`'s `test_support` that Stage 2/3 tests use. use std::rc::Rc; @@ -45,18 +45,12 @@ pub(crate) fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc` whose schema is derived by // `OperatorNode::new_shared`, so a fixture is exactly what a front end -// would hand the planner. Added by the test migration; only add here, never -// rename or remove (several test modules share these). - -use std::time::Duration; +// would hand the planner. use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; -use asap_types::ir::properties::timing::{ - apply_materialization_timings, MaterializationAssignment, TimingMemo, -}; +use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; -use asap_types::ir::{NonASAPOp, Predicate, ScalarExpr, TimeRangeKind}; +use asap_types::ir::{NonASAPOp, Predicate}; /// A `TimeSeries("m")` scan over `[ts(0), value(1), labels...]`, time index 0, /// no unique key. @@ -128,86 +122,3 @@ pub(crate) fn agg( ) -> Rc { aggregate(Reduction::by(by), vec![intent], vec![], None, child) } - -/// `intent without (excluded)`. -pub(crate) fn without_agg( - excluded: Vec, - intent: AggIntent, - child: Rc, -) -> Rc { - aggregate( - Reduction::Reduce(GroupKeys::without(excluded)), - vec![intent], - vec![], - None, - child, - ) -} - -/// A per-entity (per-series) single-measure aggregate. -pub(crate) fn agg_per_entity(intent: AggIntent, child: Rc) -> Rc { - aggregate(Reduction::PerEntity, vec![intent], vec![], None, child) -} - -pub(crate) fn filter(pred: ScalarExpr, child: Rc) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(pred), - child, - })) - .unwrap() -} - -pub(crate) fn dedup(cols: Vec, child: Rc) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Dedup { - cols, - child, - })) - .unwrap() -} - -/// An explicit range selector `child[range]`. -pub(crate) fn time_range(range: Duration, child: Rc) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::TimeRange { - range, - kind: TimeRangeKind::Range, - child, - })) - .unwrap() -} - -/// `root` timed under the default (every summary at query time) -/// assignment — the shape export and the post-ASAP validators consume. -pub(crate) fn timed(root: &Rc) -> Rc { - apply_materialization_timings( - root, - &MaterializationAssignment::all_query_time(), - &mut TimingMemo::new(), - ) - .expect("default materialization timings apply") -} - -/// `root` timed with every summary maintained at ingestion time. -pub(crate) fn maintained(root: &Rc) -> Rc { - apply_materialization_timings( - root, - &MaterializationAssignment::all_ingestion_time(), - &mut TimingMemo::new(), - ) - .expect("maintained materialization timings apply") -} - -/// Time `root` under the default materialization assignment (which runs every -/// data-state / population-contract check) and export it as a physical ASAP DAG. -pub(crate) fn time_and_export( - root: &Rc, -) -> Result< - asap_types::ir::export::PhysicalASAPDAG, - asap_types::ir::properties::execution::ExecutionDataStateError, -> { - let timed = apply_materialization_timings( - root, - &MaterializationAssignment::all_query_time(), - &mut TimingMemo::new(), - )?; - asap_types::ir::export::compile_physical_asap_dag(&timed) -} diff --git a/crates/asap-aware-mapping/tests/stage1_cost_independence.rs b/crates/asap-aware-mapping/tests/stage1_cost_independence.rs deleted file mode 100644 index 0467563b..00000000 --- a/crates/asap-aware-mapping/tests/stage1_cost_independence.rs +++ /dev/null @@ -1,116 +0,0 @@ -//! Stage 1 (logical candidate generation) stays independent of the cost -//! model: only Stage 3 prices plans (#572, decision Q36(a)). - -use std::path::Path; - -/// The Stage 1 modules, relative to `src/`. -const STAGE1_MODULES: &[&str] = &[ - "replacement.rs", - "exact_composition.rs", - "grouping.rs", - "rollup.rs", - "topk_reuse.rs", - "rewrite.rs", - "maintained_population.rs", - "accuracy/reconciliation.rs", - "logical_candidates.rs", - "function_rules.rs", -]; - -const FORBIDDEN_MODULES: &[&str] = &["cost_model", "recurrence"]; - -/// The source lines outside `#[cfg(test)]` items and comments, numbered. -fn production_lines(source: &str) -> Vec<(usize, &str)> { - let mut lines = Vec::new(); - let mut skip_next_item = false; - let mut depth = 0i64; - for (index, line) in source.lines().enumerate() { - let trimmed = line.trim(); - if depth > 0 { - depth += brace_balance(line); - continue; - } - if trimmed == "#[cfg(test)]" { - skip_next_item = true; - continue; - } - if skip_next_item { - if trimmed.starts_with("#[") || trimmed.is_empty() { - continue; - } - skip_next_item = false; - depth = brace_balance(line); - continue; - } - if !trimmed.starts_with("//") { - lines.push((index + 1, line)); - } - } - lines -} - -fn brace_balance(line: &str) -> i64 { - line.chars() - .map(|c| match c { - '{' => 1, - '}' => -1, - _ => 0, - }) - .sum() -} - -/// Names `lib.rs` re-exports from `module` (`pub use module::{...};`). -fn reexports<'a>(lib: &'a str, module: &str) -> Vec<&'a str> { - let start = format!("pub use {module}::{{"); - let Some(begin) = lib.find(&start) else { - return Vec::new(); - }; - let rest = &lib[begin + start.len()..]; - rest[..rest.find('}').expect("re-export list is closed")] - .split(',') - .map(str::trim) - .filter(|name| !name.is_empty()) - .collect() -} - -/// Stage 1 production code imports neither `cost_model` nor `recurrence`. -#[test] -fn stage1_does_not_import_cost_model_or_recurrence() { - let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); - let lib = std::fs::read_to_string(src.join("lib.rs")).unwrap(); - let forbidden: Vec<&str> = FORBIDDEN_MODULES - .iter() - .copied() - .chain( - FORBIDDEN_MODULES - .iter() - .flat_map(|module| reexports(&lib, module)), - ) - .collect(); - let mut offenders = Vec::new(); - for module in STAGE1_MODULES { - let source = std::fs::read_to_string(src.join(module)).unwrap(); - let mut in_use = false; - for (number, line) in production_lines(&source) { - in_use |= line.trim_start().starts_with("use ") || line.contains(" use "); - let path_use = FORBIDDEN_MODULES - .iter() - .any(|module| line.contains(&format!("{module}::"))); - let imported = in_use - && line - .split(|c: char| !(c.is_alphanumeric() || c == '_')) - .any(|token| forbidden.contains(&token)); - if path_use || imported { - offenders.push(format!("{module}:{number}: {}", line.trim())); - } - if line.contains(';') { - in_use = false; - } - } - } - assert!( - offenders.is_empty(), - "Stage 1 must not depend on the cost model:\n{}", - offenders.join("\n") - ); -} diff --git a/crates/asap-physical-operators/Cargo.toml b/crates/asap-physical-operators/Cargo.toml index a7130158..b8e06312 100644 --- a/crates/asap-physical-operators/Cargo.toml +++ b/crates/asap-physical-operators/Cargo.toml @@ -18,4 +18,5 @@ regex = "1" [dev-dependencies] rmp-serde = "1" asap-aware-mapping = { path = "../asap-aware-mapping" } +asap-logical-optimizer = { path = "../logical-optimizer" } asap-frontend-promql = { path = "../frontend-promql" } diff --git a/crates/asap-physical-operators/src/physical_planner/candidates.rs b/crates/asap-physical-operators/src/physical_planner/candidates.rs index 286956ba..cbd56a13 100644 --- a/crates/asap-physical-operators/src/physical_planner/candidates.rs +++ b/crates/asap-physical-operators/src/physical_planner/candidates.rs @@ -364,12 +364,14 @@ mod tests { .unwrap() .remove(0); let root = promql_rows::with_series_identity(&root).unwrap(); - let space = asap_aware_mapping::search_workload(vec![("q", root)]); - let selected = space - .global_selection(&asap_aware_mapping::cost_model::DefaultCostModel) - .assemble_selected_dag(&space.roots[0].1) - .unwrap() - .unwrap(); + let space = asap_logical_optimizer::search_workload(vec![("q", root)]); + let selected = asap_aware_mapping::plan_selection::candidate_selection::global_selection( + &space, + &asap_aware_mapping::cost_model::DefaultCostModel, + ) + .assemble_selected_dag(&space.roots[0].1) + .unwrap() + .unwrap(); let selected = planner_types::ir::apply_materialization_timings( &selected, &planner_types::ir::MaterializationAssignment::all_ingestion_time(), diff --git a/crates/asap-physical-operators/tests/current_series_heap.rs b/crates/asap-physical-operators/tests/current_series_heap.rs index f1df7ce8..b414a2cb 100644 --- a/crates/asap-physical-operators/tests/current_series_heap.rs +++ b/crates/asap-physical-operators/tests/current_series_heap.rs @@ -327,7 +327,7 @@ fn planner_current_series_candidate_compiles_with_dynamic_identity() { .remove(0); let open_root = Rc::new(original.clone()); let open_selected = - asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( + asap_logical_optimizer::pass1::maintained_population::MaintainedPopulationStrategy::new( std::slice::from_ref(&open_root), ) .candidate(&open_root) @@ -345,11 +345,12 @@ fn planner_current_series_candidate_compiles_with_dynamic_identity() { assert!(encoded.contains("Sort") && encoded.contains("Limit")); assert_eq!(snapshot_program.input_contracts().count(), 1); let root = Rc::new(with_series_identity(&original).unwrap()); - let selected = asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( - std::slice::from_ref(&root), - ) - .candidate(&root) - .unwrap(); + let selected = + asap_logical_optimizer::pass1::maintained_population::MaintainedPopulationStrategy::new( + std::slice::from_ref(&root), + ) + .candidate(&root) + .unwrap(); let logical = compile_physical_asap_dag(&selected).unwrap(); let raw = logical .nodes diff --git a/crates/asap-physical-operators/tests/deployment_computation.rs b/crates/asap-physical-operators/tests/deployment_computation.rs index 623d65ca..1a7bdd0a 100644 --- a/crates/asap-physical-operators/tests/deployment_computation.rs +++ b/crates/asap-physical-operators/tests/deployment_computation.rs @@ -51,22 +51,25 @@ fn lower_with(query: &str, accuracy: AccuracyTarget) -> Rc PhysicalASAPDAG { let expression = lower(query); let root = promql_rows::with_series_identity(&expression).unwrap_or(expression); - let space = asap_aware_mapping::search_workload(vec![("q", root)]); - let selected = space - .global_selection(&asap_aware_mapping::DefaultCostModel) - .assemble_selected_dag(&space.roots[0].1) - .unwrap() - .unwrap(); + let space = asap_logical_optimizer::search_workload(vec![("q", root)]); + let selected = asap_aware_mapping::plan_selection::candidate_selection::global_selection( + &space, + &asap_aware_mapping::DefaultCostModel, + ) + .assemble_selected_dag(&space.roots[0].1) + .unwrap() + .unwrap(); compile_physical_asap_dag(&selected).unwrap() } fn population_dag(query: &str) -> PhysicalASAPDAG { let root = promql_rows::with_series_identity(&lower(query)).unwrap(); - let selected = asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( - std::slice::from_ref(&root), - ) - .candidate(&root) - .unwrap(); + let selected = + asap_logical_optimizer::pass1::maintained_population::MaintainedPopulationStrategy::new( + std::slice::from_ref(&root), + ) + .candidate(&root) + .unwrap(); compile_physical_asap_dag(&selected).unwrap() } @@ -625,10 +628,10 @@ fn population_sums_and_averages_are_compensated() { // returns the sketch's total update weight, including colliding items. #[test] fn stored_count_min_bare_count_compiles_to_a_evaluation() { - use asap_aware_mapping::{Replacement, ReplacementStrategy, TargetSubDAG}; + use asap_logical_optimizer::{Replacement, ReplacementStrategy, TargetSubDAG}; use asap_physical_operators::summary_kernels::CountMinSketchAccumulator; let root = lower_with("count(up)", AccuracyTarget::Epsilon(0.02)); - let dag = asap_aware_mapping::ASAPStrategies::default() + let dag = asap_logical_optimizer::ASAPStrategies::default() .replacements(&TargetSubDAG::new(&root)) .into_iter() .find_map(|candidate| match candidate.replacement { diff --git a/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs b/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs index a8f5728a..d1d4f72f 100644 --- a/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs +++ b/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs @@ -6,11 +6,13 @@ mod common; use common::compile_physical_asap_dag; use planner_types::ir::OperatorNode; -use asap_aware_mapping::{ - accuracy::{AccuracyEvidenceProvider, DefaultAccuracyModel, PropagationStats}, - cost_model::DefaultCostModel, - replacement::{default_strategies_with_evidence, ReplacementProvenance}, - search_workload_with_targets, Proposals, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, +use asap_aware_mapping::cost_model::DefaultCostModel; +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; +use asap_logical_optimizer::{ + accuracy::AccuracyEvidenceProvider, accuracy::DefaultAccuracyModel, accuracy::PropagationStats, + pass1::replacement::default_strategies_with_evidence, + pass1::replacement::ReplacementProvenance, search_workload_with_targets, Proposals, + ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; use asap_physical_operators::physical_planner::promql_rows::{ compile_current_series_evaluation, SERIES_IDENTITY_COLUMN, @@ -220,8 +222,7 @@ fn global_selection_never_commits_a_series_identity_heap() { &strategies, &DefaultAccuracyModel, ); - let selected = space - .global_selection(&DefaultCostModel) + let selected = global_selection(&space, &DefaultCostModel) .assemble_selected_dag(&space.roots[0].1) .unwrap() .unwrap(); diff --git a/crates/asap-physical-operators/tests/precompute_candidates.rs b/crates/asap-physical-operators/tests/precompute_candidates.rs index 2fa0ca77..6bee0c3e 100644 --- a/crates/asap-physical-operators/tests/precompute_candidates.rs +++ b/crates/asap-physical-operators/tests/precompute_candidates.rs @@ -1,6 +1,8 @@ //! Materialized frontiers are compiled by Planner, never rewritten by deployment. mod common; -use asap_aware_mapping::{cost_model::DefaultCostModel, search_workload}; +use asap_aware_mapping::cost_model::DefaultCostModel; +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; +use asap_logical_optimizer::search_workload; use asap_physical_operators::{ factory::create_planner_accumulator, operators::Operator, @@ -20,7 +22,7 @@ use planner_types::types::AccuracyTarget; use planner_types::workload::*; use std::{collections::BTreeMap, sync::Arc}; -fn grouped_rate_space() -> asap_aware_mapping::CandidateLogicalASAPDAGs<&'static str> { +fn grouped_rate_space() -> asap_logical_optimizer::CandidateLogicalASAPDAGs<&'static str> { let workload = PlanningWorkload { query_workload: QueryWorkload { language: QueryLanguage::PromQL, @@ -55,8 +57,7 @@ fn grouped_rate_space() -> asap_aware_mapping::CandidateLogicalASAPDAGs<&'static fn grouped_rate() -> PhysicalASAPDAG { let space = grouped_rate_space(); - let selected = space - .global_selection(&DefaultCostModel) + let selected = global_selection(&space, &DefaultCostModel) .assemble_selected_query(&space.roots[0].1) .unwrap() .unwrap(); @@ -657,11 +658,12 @@ fn population_topk_cuts_equal_per_frontier_compilation() { let root = asap_physical_operators::physical_planner::promql_rows::with_series_identity(&original) .unwrap(); - let selected = asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( - std::slice::from_ref(&root), - ) - .candidate(&root) - .unwrap(); + let selected = + asap_logical_optimizer::pass1::maintained_population::MaintainedPopulationStrategy::new( + std::slice::from_ref(&root), + ) + .candidate(&root) + .unwrap(); let dag = compile_physical_asap_dag(&selected).unwrap(); let raw = dag .nodes diff --git a/crates/asap-physical-operators/tests/promql_fallback.rs b/crates/asap-physical-operators/tests/promql_fallback.rs index 65272844..396e159f 100644 --- a/crates/asap-physical-operators/tests/promql_fallback.rs +++ b/crates/asap-physical-operators/tests/promql_fallback.rs @@ -1229,9 +1229,10 @@ fn histogram_quantile_rejects_equal_output_label_sets() { // for an approximate target, and the selected DAG compiles and executes. #[test] fn histogram_quantile_selection_keeps_the_exact_fallback() { - use asap_aware_mapping::{ - accuracy::DefaultAccuracyModel, cost_model::DefaultCostModel, default_strategies, - search_workload_with_targets, Replacement, + use asap_aware_mapping::cost_model::DefaultCostModel; + use asap_logical_optimizer::{ + accuracy::DefaultAccuracyModel, default_strategies, search_workload_with_targets, + Replacement, }; let samples = buckets(&[("job=a", HISTOGRAM)]); for target in [AccuracyTarget::Exact, AccuracyTarget::Epsilon(0.01)] { @@ -1253,8 +1254,11 @@ fn histogram_quantile_selection_keeps_the_exact_fallback() { Replacement::SubDAG(node) if !node.contains_asap() && node.operator == root.operator)), "{query}: {candidates:?}" ); - let selected = space - .global_selection(&DefaultCostModel) + let selected = + asap_aware_mapping::plan_selection::candidate_selection::global_selection( + &space, + &DefaultCostModel, + ) .assemble_selected_dag(planned) .unwrap() .unwrap(); diff --git a/crates/asap-physical-operators/tests/weighted_topk_binding.rs b/crates/asap-physical-operators/tests/weighted_topk_binding.rs index 115369c9..52f53084 100644 --- a/crates/asap-physical-operators/tests/weighted_topk_binding.rs +++ b/crates/asap-physical-operators/tests/weighted_topk_binding.rs @@ -1,10 +1,9 @@ //! Planner output binds directly to the shared runtime at a declared rate-value frontier. mod common; -use asap_aware_mapping::{ - accuracy::{ - AccuracyEvidenceProvider, DefaultAccuracyModel, EqualSplitAllocator, PropagationStats, - }, - ASAPStrategies, Replacement, ReplacementStrategy, TargetSubDAG, +use asap_logical_optimizer::{ + accuracy::AccuracyEvidenceProvider, accuracy::DefaultAccuracyModel, + accuracy::EqualSplitAllocator, accuracy::PropagationStats, ASAPStrategies, Replacement, + ReplacementStrategy, TargetSubDAG, }; use asap_physical_operators::dag::{ operators::Operator, @@ -53,11 +52,11 @@ fn planner_weighted_topk_binds_at_either_deployment_phase() { #[test] fn physical_binding_does_not_impose_an_accuracy_acceptance_policy() { assert_weighted_binding( - &asap_aware_mapping::accuracy::NoAccuracyEvidence, + &asap_logical_optimizer::accuracy::NoAccuracyEvidence, SketchAlgorithm::CmsWithHeap, ); assert_weighted_binding( - &asap_aware_mapping::accuracy::NoAccuracyEvidence, + &asap_logical_optimizer::accuracy::NoAccuracyEvidence, SketchAlgorithm::CountSketchWithHeap, ); } diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 0abc8ccd..d9b88a4b 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -11,6 +11,7 @@ edition = "2021" asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } asap-aware-mapping = { path = "../asap-aware-mapping" } +asap-logical-optimizer = { path = "../logical-optimizer" } # Used by the show_ir / dag_export / variant_coverage bins (catalog schemas, # async SQL path, JSON output) and by the topk_ir / canonical_examples diff --git a/crates/devtools/src/bin/dag_export.rs b/crates/devtools/src/bin/dag_export.rs index 43d107e6..65f86c97 100644 --- a/crates/devtools/src/bin/dag_export.rs +++ b/crates/devtools/src/bin/dag_export.rs @@ -12,7 +12,7 @@ // `--epsilon ` is optional and applies to every query in the run: it // lowers with `AccuracyTarget::Epsilon()` instead of the default // `AccuracyTarget::Exact`. Without it, every `AggIntent` lowers exact and -// `asap_aware_mapping::ASAPStrategies` never has a genuine sketch +// `asap_logical_optimizer::ASAPStrategies` never has a genuine sketch // alternative to report — so no node ever picks up a `SketchApproximation` // note. Pass it to actually exercise that path, e.g.: // cargo run -p asap-lower --bin dag_export -- \ @@ -23,10 +23,10 @@ // below). // // `--post-asap` is optional and off by default. When passed, this binary -// additionally runs `asap_aware_mapping::replacement::search_workload` (this +// additionally runs `asap_logical_optimizer::pass1::replacement::search_workload` (this // binary took no strategies of its own — `default_strategies()` already // includes `AvgToSumOverCountStrategy` as of #282) over every lowered query -// and ranks each discovered `TargetSubDAGCandidates` via `CandidateLogicalASAPDAGs::cost_sorted`. The +// and ranks each discovered `TargetSubDAGCandidates` via `candidate_selection::cost_sorted`. The // best-ranked // candidate per group feeds two additive outputs: // @@ -86,12 +86,13 @@ use asap_aware_mapping::physical_operator_statistics::ComparisonScope; use asap_aware_mapping::physical_plan_cost_model::{ PhysicalEvidenceSnapshot, PhysicalPlanCostModel, PlannerPhysicalPlanProvider, }; +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; use asap_aware_mapping::query_physical_lowering::PhysicalNodeRequest; -use asap_aware_mapping::replacement::{ +use asap_logical_optimizer::pass1::replacement::{ default_strategies_with_evidence, is_logical_rewrite, search_workload, search_workload_with, Replacement, ReplacementSubDAG, }; -use asap_aware_mapping::{AccuracyEvidenceProvider, PropagationStats}; +use asap_logical_optimizer::{AccuracyEvidenceProvider, PropagationStats}; use asap_types::cost::{BaselineRef, CostAnnotation, CostInput, CostSource, CostUnit}; use asap_types::dag_export::{ self, DAGDecision, DAGNote, ExportDAG, NamedDAG, PostAsapSubstitution, TargetRejection, @@ -318,7 +319,7 @@ impl ExportPhysicalProvider<'_> { impl PlannerPhysicalPlanProvider for ExportPhysicalProvider<'_> { fn capture_evidence_snapshot( &self, - _target: &asap_aware_mapping::replacement::TargetSubDAG<'_>, + _target: &asap_logical_optimizer::pass1::replacement::TargetSubDAG<'_>, ) -> Result { Ok(PhysicalEvidenceSnapshot { version: self.evidence_version.into(), @@ -369,7 +370,7 @@ impl PlannerPhysicalPlanProvider for ExportPhysicalProvider<'_> { &self, snapshot: &PhysicalEvidenceSnapshot, _summary: &Rc, - _target: &asap_aware_mapping::replacement::TargetSubDAG<'_>, + _target: &asap_logical_optimizer::pass1::replacement::TargetSubDAG<'_>, ) -> Result { if snapshot.scope != self.target.scope.resolve()? { return Err(AnalyticalCostError::ComparisonScopeMismatch( @@ -393,7 +394,7 @@ impl ExportPlannerCostModel<'_> { fn bound<'a>( &'a self, candidate: &ReplacementSubDAG, - target: &asap_aware_mapping::replacement::TargetSubDAG<'_>, + target: &asap_logical_optimizer::pass1::replacement::TargetSubDAG<'_>, ) -> Option<(ExportPhysicalProvider<'a>, &'a ResourceCalibration)> { let mut targets = self .document @@ -430,7 +431,7 @@ impl ExportPlannerCostModel<'_> { candidate: &ReplacementSubDAG, target: &Rc, ) -> (CostAnnotation, CostAnnotation, CostAnnotation) { - let target = asap_aware_mapping::replacement::TargetSubDAG::new(target); + let target = asap_logical_optimizer::pass1::replacement::TargetSubDAG::new(target); let Some((provider, calibration)) = self.bound(candidate, &target) else { return winner_cost_annotations(); }; @@ -649,7 +650,7 @@ impl CostModel for ExportPlannerCostModel<'_> { fn candidate_cost( &self, candidate: &ReplacementSubDAG, - target: &asap_aware_mapping::replacement::TargetSubDAG<'_>, + target: &asap_logical_optimizer::pass1::replacement::TargetSubDAG<'_>, ) -> Option { let (provider, calibration) = self.bound(candidate, target)?; let cost = PhysicalPlanCostModel::new(&provider, calibration.clone()) @@ -671,7 +672,7 @@ impl CostModel for ExportPlannerCostModel<'_> { fn estimate_cost( &self, candidate: &ReplacementSubDAG, - target: &asap_aware_mapping::replacement::TargetSubDAG<'_>, + target: &asap_logical_optimizer::pass1::replacement::TargetSubDAG<'_>, ) -> f64 { self.candidate_cost(candidate, target) .map_or(f64::NAN, |cost| cost.0) @@ -966,7 +967,7 @@ fn parse_args_from(argv: impl Iterator) -> ParsedArgs { /// defensive only). fn annotate_with_explanations( dag: &mut ExportDAG, - explanations: &[asap_aware_mapping::ReplacementExplanation], + explanations: &[asap_logical_optimizer::ReplacementExplanation], matched: &mut [bool], ) { for (i, explanation) in explanations.iter().enumerate() { @@ -1011,10 +1012,10 @@ fn decision_rationale(winner: &Winner<'_>) -> String { .to_string() } "SharedSubDAGStrategy" => match winner.candidate.provenance { - asap_aware_mapping::replacement::ReplacementProvenance::CseShare => { + asap_logical_optimizer::pass1::replacement::ReplacementProvenance::CseShare => { "Builds the repeated sub-DAG once and shares it across consumers.".to_string() } - asap_aware_mapping::replacement::ReplacementProvenance::CseRecompute => { + asap_logical_optimizer::pass1::replacement::ReplacementProvenance::CseRecompute => { "Recomputes the sub-DAG per consumer because that has the lower estimated cost." .to_string() } @@ -1210,11 +1211,11 @@ fn assign_workload_node_ids(dags: &mut [&mut ExportDAG]) { } } -/// Run `asap_aware_mapping::replacement::search_workload` (its own +/// Run `asap_logical_optimizer::pass1::replacement::search_workload` (its own /// `default_strategies()` — which includes `AvgToSumOverCountStrategy` as of /// #282 — is exactly the strategy set this binary wants; no custom list /// needed) over every lowered query, rank each discovered `TargetSubDAGCandidates` via -/// `CandidateLogicalASAPDAGs::global_selection`, and build both `--post-asap` outputs from the +/// `candidate_selection::global_selection`, and build both `--post-asap` outputs from the /// exact same set of winning candidates (see [`Winner`]), so the flat /// `replacements` list and the merged `post_dag` can never disagree about /// which candidate won for a given target. @@ -1241,7 +1242,7 @@ fn run_post_asap_with_progress( } else { search_workload(roots) }; - let selection = space.global_selection(cost_model); + let selection = global_selection(&space, cost_model); // A group's top candidate can be `retain_exact`'s own conservative // fallback — the *whole target* itself, unbound, carrying only an exact @@ -1438,7 +1439,7 @@ fn run_post_asap_with_progress( // winner's own `Replacement::Rewrite` — logged as an FYI rather than a // warning, since telling the two cases apart precisely would mean // reimplementing `search`'s own private descendant-discovery walk - // (`discover_new_descendant_targets` in `asap_aware_mapping::replacement`, + // (`discover_new_descendant_targets` in `asap_logical_optimizer::pass1::replacement`, // not exposed) a second time here just to double-check something // `post_dag`'s own construction already handled correctly. for (winner, matched) in winners.iter().zip(&matched) { @@ -1529,7 +1530,7 @@ async fn main() { if progress { eprintln!("[2/4] Pre-ASAP DAG generation is running…"); } - let explanations = asap_aware_mapping::explain_replacements( + let explanations = asap_logical_optimizer::explain_replacements( lowered_queries .iter() .map(|(name, _, qe)| (name.clone(), qe.clone())) @@ -1746,7 +1747,7 @@ mod tests { ) -> PhysicalDAG { let model = ExportPlannerCostModel { document }; let root = Rc::clone(query); - let target = asap_aware_mapping::replacement::TargetSubDAG::new(&root); + let target = asap_logical_optimizer::pass1::replacement::TargetSubDAG::new(&root); let (provider, _) = model.bound(candidate, &target).unwrap(); let snapshot = provider.capture_evidence_snapshot(&target).unwrap(); let evidence = @@ -1762,7 +1763,7 @@ mod tests { let raw = fixture_raw_dag(&query, &candidate, &document); let candidate_dag = cheap_candidate_dag(); let root = Rc::clone(&query); - let target = asap_aware_mapping::replacement::TargetSubDAG::new(&root); + let target = asap_logical_optimizer::pass1::replacement::TargetSubDAG::new(&root); assert!(ExportPlannerCostModel { document: &document } @@ -1925,7 +1926,7 @@ mod tests { let raw = fixture_raw_dag(&query, &candidate, &document); let candidate_dag = cheap_candidate_dag(); let root = Rc::clone(&query); - let target = asap_aware_mapping::replacement::TargetSubDAG::new(&root); + let target = asap_logical_optimizer::pass1::replacement::TargetSubDAG::new(&root); assert!(ExportPlannerCostModel { document: &document } @@ -2357,7 +2358,7 @@ mod tests { assert!(parsed.targets[0].candidates[0].matches(&candidate)); let model = ExportPlannerCostModel { document: &parsed }; let target_rc = Rc::clone(&query); - let target = asap_aware_mapping::replacement::TargetSubDAG::new(&target_rc); + let target = asap_logical_optimizer::pass1::replacement::TargetSubDAG::new(&target_rc); let (provider, calibration) = model.bound(&candidate, &target).expect("exact binding"); let estimate = PhysicalPlanCostModel::new(&provider, calibration.clone()) .unwrap() @@ -2418,7 +2419,7 @@ mod tests { // Identical repeats hit the result cache; distinct evaluations still execute. let (query, candidate, document) = cost_fixture(); let target_rc = query; - let target = asap_aware_mapping::replacement::TargetSubDAG::new(&target_rc); + let target = asap_logical_optimizer::pass1::replacement::TargetSubDAG::new(&target_rc); let no_cache = ExportPlannerCostModel { document: &document, } @@ -2505,7 +2506,7 @@ mod tests { fn duplicate_target_candidate_and_query_evidence_each_fail_closed() { let (query, candidate, document) = cost_fixture(); let target_rc = query; - let target = asap_aware_mapping::replacement::TargetSubDAG::new(&target_rc); + let target = asap_logical_optimizer::pass1::replacement::TargetSubDAG::new(&target_rc); let mut duplicate_target = document.clone(); duplicate_target @@ -2547,7 +2548,7 @@ mod tests { fn incomplete_or_unused_json_evidence_fails_closed() { let (query, candidate, document) = cost_fixture(); let target_rc = query; - let target = asap_aware_mapping::replacement::TargetSubDAG::new(&target_rc); + let target = asap_logical_optimizer::pass1::replacement::TargetSubDAG::new(&target_rc); let mut missing = document.clone(); match &mut missing.targets[0].candidates[0] { @@ -2588,7 +2589,7 @@ mod tests { let document = parse_planner_cost_document(&serde_json::to_string(&document).unwrap()) .expect("invalid physical semantics are checked by the estimator"); let target_rc = query; - let target = asap_aware_mapping::replacement::TargetSubDAG::new(&target_rc); + let target = asap_logical_optimizer::pass1::replacement::TargetSubDAG::new(&target_rc); assert!(ExportPlannerCostModel { document: &document } @@ -2664,7 +2665,7 @@ mod tests { let model = ExportPlannerCostModel { document: &document, }; - let selection = space.global_selection(&model); + let selection = global_selection(&space, &model); let chosen = selection .target_selections() .find(|selected| *selected.target == query) @@ -2771,13 +2772,15 @@ mod tests { let selected = ReplacementSubDAG { replacement: Replacement::SubDAG(Rc::clone(&selected_query)), strategy: "same-strategy", - provenance: asap_aware_mapping::replacement::ReplacementProvenance::LogicalRewrite, + provenance: + asap_logical_optimizer::pass1::replacement::ReplacementProvenance::LogicalRewrite, rationale: String::new(), }; let other = ReplacementSubDAG { replacement: Replacement::SubDAG(other_query), strategy: "same-strategy", - provenance: asap_aware_mapping::replacement::ReplacementProvenance::LogicalRewrite, + provenance: + asap_logical_optimizer::pass1::replacement::ReplacementProvenance::LogicalRewrite, rationale: String::new(), }; let selector = CandidatePhysicalEvidence::Rewrite { @@ -2834,10 +2837,10 @@ mod tests { let a = lower_promql(query, AccuracyTarget::Exact).unwrap(); let b = lower_promql(query, AccuracyTarget::Exact).unwrap(); let explanations = - asap_aware_mapping::explain_replacements(vec![("a", a.clone()), ("b", b.clone())]); - assert!(explanations - .iter() - .any(|e| { e.kind == asap_aware_mapping::ExplanationKind::CommonSubexpressionReuse })); + asap_logical_optimizer::explain_replacements(vec![("a", a.clone()), ("b", b.clone())]); + assert!(explanations.iter().any(|e| { + e.kind == asap_logical_optimizer::ExplanationKind::CommonSubexpressionReuse + })); let mut matched = vec![false; explanations.len()]; let mut dag_a = dag_export::export(&a); @@ -2855,10 +2858,10 @@ mod tests { AccuracyTarget::Epsilon(0.01), ) .unwrap(); - let explanations = asap_aware_mapping::explain_replacements(vec![("target", target)]); + let explanations = asap_logical_optimizer::explain_replacements(vec![("target", target)]); let explanation = explanations .iter() - .find(|e| e.kind == asap_aware_mapping::ExplanationKind::SketchApproximation) + .find(|e| e.kind == asap_logical_optimizer::ExplanationKind::SketchApproximation) .unwrap(); let unrelated = diff --git a/crates/devtools/src/bin/show_post_asap_ir.rs b/crates/devtools/src/bin/show_post_asap_ir.rs index 757809fc..fa43251d 100644 --- a/crates/devtools/src/bin/show_post_asap_ir.rs +++ b/crates/devtools/src/bin/show_post_asap_ir.rs @@ -21,11 +21,11 @@ // `metrics(ts, service, region, latency, bytes)` catalog — the same table // used in cross_language.rs and topk_ir.rs. -use asap_aware_mapping::replacement::retain_exact; -use asap_aware_mapping::{ +use asap_devtools::{lower_promql_with_data_ingestion_interval, lower_sql, SqlCatalog}; +use asap_logical_optimizer::pass1::replacement::retain_exact; +use asap_logical_optimizer::{ ASAPStrategies, Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; -use asap_devtools::{lower_promql_with_data_ingestion_interval, lower_sql, SqlCatalog}; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::OperatorNode; use asap_types::types::AccuracyTarget; diff --git a/crates/devtools/src/bin/sketch_coverage.rs b/crates/devtools/src/bin/sketch_coverage.rs index 9ac97e06..9d39c54a 100644 --- a/crates/devtools/src/bin/sketch_coverage.rs +++ b/crates/devtools/src/bin/sketch_coverage.rs @@ -2,7 +2,7 @@ // // Lowers every query in every corpus we have (mirrors `variant_coverage`'s // corpus list exactly, so the two reports are directly comparable) with an -// *approximate* `AccuracyTarget`, runs `asap_aware_mapping::explain_replacements` +// *approximate* `AccuracyTarget`, runs `asap_logical_optimizer::explain_replacements` // over each corpus as one workload, and reports the MVP demo's query-coverage // metric: of the queries that lowered successfully, what fraction got // @@ -25,9 +25,9 @@ // reuse inside one corpus shows up here the same way it would in the // dag-viewer's Union mode. -use asap_aware_mapping::{explain_replacements, ExplanationKind}; use asap_devtools::lower_promql_with_data_ingestion_interval; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; +use asap_logical_optimizer::{explain_replacements, ExplanationKind}; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::OperatorNode; use asap_types::types::AccuracyTarget; diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index ea2baacb..4b687dba 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -25,11 +25,12 @@ use std::rc::Rc; -use asap_aware_mapping::logical_candidates::{ +use asap_aware_mapping::plan_selection::{select_exhaustive, Selection, MAX_ENUMERATED_CANDIDATES}; +use asap_aware_mapping::PlanningModels; +use asap_logical_optimizer::pass1::logical_candidates::{ choice_index, enumerate_local_logical_candidates, LocalLogicalCandidates, }; -use asap_aware_mapping::plan_selection::{select_exhaustive, Selection, MAX_ENUMERATED_CANDIDATES}; -use asap_aware_mapping::{PlanningModels, Realization}; +use asap_logical_optimizer::Realization; use asap_types::ir::export::{ compile_logical_asap_workload, LogicalASAPDAG, LogicalASAPDAGDocument, }; diff --git a/crates/frontend-promql/Cargo.toml b/crates/frontend-promql/Cargo.toml index 569c87c9..985bbee5 100644 --- a/crates/frontend-promql/Cargo.toml +++ b/crates/frontend-promql/Cargo.toml @@ -17,6 +17,7 @@ promql-parser = { git = "https://github.com/ProjectASAP/promql-parser", rev = "9 [dev-dependencies] asap-aware-mapping = { path = "../asap-aware-mapping" } +asap-logical-optimizer = { path = "../logical-optimizer" } # Corpus tests live in domain folders (observability) but must still be declared # as test targets — Cargo only auto-discovers `.rs` files directly in `tests/`. diff --git a/crates/frontend-promql/tests/count_planning.rs b/crates/frontend-promql/tests/count_planning.rs index fd2b7128..c1d7db02 100644 --- a/crates/frontend-promql/tests/count_planning.rs +++ b/crates/frontend-promql/tests/count_planning.rs @@ -1,7 +1,8 @@ //! Query text through summary selection: counts use observations, never value weights. -use asap_aware_mapping::accuracy::DefaultAccuracyModel; use asap_aware_mapping::cost_model::DefaultCostModel; -use asap_aware_mapping::{ +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; +use asap_logical_optimizer::accuracy::DefaultAccuracyModel; +use asap_logical_optimizer::{ default_strategies, search_workload_with_targets, ASAPStrategies, Replacement, ReplacementStrategy, TargetSubDAG, }; @@ -39,8 +40,7 @@ fn grouped_count_keeps_uncertified_hydra_candidates_for_backend_review() { assert!(hydra .iter() .all(|candidate| candidate.has_missing_accuracy_evidence())); - assert!(!space - .global_selection(&DefaultCostModel) + assert!(!global_selection(&space, &DefaultCostModel) .for_target(planned) .unwrap() .chosen diff --git a/crates/frontend-promql/tests/maintained_population_horizon.rs b/crates/frontend-promql/tests/maintained_population_horizon.rs index cf98f2eb..6611ec78 100644 --- a/crates/frontend-promql/tests/maintained_population_horizon.rs +++ b/crates/frontend-promql/tests/maintained_population_horizon.rs @@ -1,5 +1,5 @@ mod support; -use asap_aware_mapping::maintained_population::MaintainedPopulationStrategy; +use asap_logical_optimizer::pass1::maintained_population::MaintainedPopulationStrategy; use asap_types::ir::operator::maintained_population::PopulationInput; use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; use asap_types::types::AccuracyTarget; diff --git a/crates/frontend-promql/tests/observability/metrics_observability.rs b/crates/frontend-promql/tests/observability/metrics_observability.rs index 5875dbc9..24178c20 100644 --- a/crates/frontend-promql/tests/observability/metrics_observability.rs +++ b/crates/frontend-promql/tests/observability/metrics_observability.rs @@ -6,11 +6,11 @@ use std::rc::Rc; -use asap_aware_mapping::replacement::{retain_exact, RealizationError}; -use asap_aware_mapping::{ +use asap_frontend_promql::PromqlError; +use asap_logical_optimizer::pass1::replacement::{retain_exact, RealizationError}; +use asap_logical_optimizer::{ ASAPStrategies, Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; -use asap_frontend_promql::PromqlError; #[path = "../support.rs"] mod support; use asap_types::ir::OperatorNode; diff --git a/crates/frontend-promql/tests/observability/promql_corpus.rs b/crates/frontend-promql/tests/observability/promql_corpus.rs index 3c6b923e..8af5af79 100644 --- a/crates/frontend-promql/tests/observability/promql_corpus.rs +++ b/crates/frontend-promql/tests/observability/promql_corpus.rs @@ -15,11 +15,11 @@ use std::rc::Rc; -use asap_aware_mapping::replacement::{retain_exact, RealizationError}; -use asap_aware_mapping::{ +use asap_frontend_promql::PromqlError as LoweringError; +use asap_logical_optimizer::pass1::replacement::{retain_exact, RealizationError}; +use asap_logical_optimizer::{ ASAPStrategies, Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; -use asap_frontend_promql::PromqlError as LoweringError; #[path = "../support.rs"] mod support; use asap_types::ir::OperatorNode; diff --git a/crates/frontend-promql/tests/univmon_candidates.rs b/crates/frontend-promql/tests/univmon_candidates.rs index b5817f4a..9917ff38 100644 --- a/crates/frontend-promql/tests/univmon_candidates.rs +++ b/crates/frontend-promql/tests/univmon_candidates.rs @@ -1,11 +1,14 @@ use std::rc::Rc; -use asap_aware_mapping::accuracy::{ +use asap_aware_mapping::cost_model::DefaultCostModel; +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; +use asap_logical_optimizer::accuracy::{ AccuracyModel, DefaultAccuracyModel, EqualSplitAllocator, PropagationStats, }; -use asap_aware_mapping::cost_model::DefaultCostModel; -use asap_aware_mapping::replacement::{default_strategies, search_workload_with_targets}; -use asap_aware_mapping::{ASAPStrategies, Replacement, ReplacementStrategy, TargetSubDAG}; +use asap_logical_optimizer::pass1::replacement::{ + default_strategies, search_workload_with_targets, +}; +use asap_logical_optimizer::{ASAPStrategies, Replacement, ReplacementStrategy, TargetSubDAG}; mod support; use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::properties::{ @@ -164,8 +167,7 @@ fn uncalibrated_frequency_evaluations_do_not_bypass_accuracy_targets() { .candidates .iter() .any(|candidate| candidate.has_missing_accuracy_evidence())); - assert!(!space - .global_selection(&DefaultCostModel) + assert!(!global_selection(&space, &DefaultCostModel) .for_target(&space.roots[0].1) .unwrap() .chosen diff --git a/crates/frontend-sql/Cargo.toml b/crates/frontend-sql/Cargo.toml index 3f81f110..03adb84d 100644 --- a/crates/frontend-sql/Cargo.toml +++ b/crates/frontend-sql/Cargo.toml @@ -18,6 +18,7 @@ serde_json = "1" [dev-dependencies] asap-aware-mapping = { path = "../asap-aware-mapping" } +asap-logical-optimizer = { path = "../logical-optimizer" } tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } # bgp_jan2024_workload corpus is sourced verbatim as YAML (ASAPQuery PR #561) # rather than transcribed into the flat .sql shape the other corpora use. diff --git a/crates/frontend-sql/tests/maintained_population.rs b/crates/frontend-sql/tests/maintained_population.rs index f156c480..9529fcd3 100644 --- a/crates/frontend-sql/tests/maintained_population.rs +++ b/crates/frontend-sql/tests/maintained_population.rs @@ -1,6 +1,6 @@ //! SQL and PromQL use the same shared-state rule without sharing membership semantics. -use asap_aware_mapping::maintained_population::MaintainedPopulationStrategy; use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_logical_optimizer::pass1::maintained_population::MaintainedPopulationStrategy; use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::export::compile_physical_asap_dag; use asap_types::ir::operator::maintained_population::{MaintainedPopulation, PopulationInput}; diff --git a/crates/frontend-sql/tests/pearson_corr.rs b/crates/frontend-sql/tests/pearson_corr.rs index 71a10793..ea648bec 100644 --- a/crates/frontend-sql/tests/pearson_corr.rs +++ b/crates/frontend-sql/tests/pearson_corr.rs @@ -148,7 +148,7 @@ async fn corr_filter_is_a_measure_filter() { #[tokio::test] async fn corr_survives_exact_plan_compilation() { let query = lower("SELECT corr(x, y) AS r FROM a").await; - let plan = asap_aware_mapping::replacement::retain_exact(&query).unwrap(); + let plan = asap_logical_optimizer::pass1::replacement::retain_exact(&query).unwrap(); assert!(plan.guarantee.as_ref().unwrap().is_exact()); // The exact fallback is the query's own operator DAG, no ASAP node added. assert!(!plan.contains_asap(), "expected exact fallback"); diff --git a/crates/integration-tests/Cargo.toml b/crates/integration-tests/Cargo.toml index afa7559b..db63edaa 100644 --- a/crates/integration-tests/Cargo.toml +++ b/crates/integration-tests/Cargo.toml @@ -8,6 +8,7 @@ asap-types = { path = "../types" } asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } asap-aware-mapping = { path = "../asap-aware-mapping" } +asap-logical-optimizer = { path = "../logical-optimizer" } [dev-dependencies] asap-planner = { path = "../planner" } diff --git a/crates/integration-tests/tests/cse.rs b/crates/integration-tests/tests/cse.rs index 56a0e657..6b5d5a82 100644 --- a/crates/integration-tests/tests/cse.rs +++ b/crates/integration-tests/tests/cse.rs @@ -24,8 +24,8 @@ use std::rc::Rc; -use asap_aware_mapping::{is_logical_rewrite, search_workload, Replacement}; use asap_integration_tests::fixtures::lower_promql; +use asap_logical_optimizer::{is_logical_rewrite, search_workload, Replacement}; use asap_types::ir::NonASAPOp; use asap_types::types::AccuracyTarget; @@ -33,7 +33,7 @@ use asap_types::types::AccuracyTarget; /// realistic case — two dashboards, or a query fired both standalone and as /// part of a larger batch) collapse onto one shared `Rc` after /// `search_workload`'s internal `share_common_sub_dags` pass, and onto one -/// genuinely-shared [`TargetSubDAGCandidates`](asap_aware_mapping::TargetSubDAGCandidates) — carrying +/// genuinely-shared [`TargetSubDAGCandidates`](asap_logical_optimizer::TargetSubDAGCandidates) — carrying /// every candidate discovered for it exactly once, not once per root — no /// second structural-equality pass at the post-ASAP layer needed for this /// kind of sharing. diff --git a/crates/integration-tests/tests/exact_composition.rs b/crates/integration-tests/tests/exact_composition.rs index b978f603..f3038a0e 100644 --- a/crates/integration-tests/tests/exact_composition.rs +++ b/crates/integration-tests/tests/exact_composition.rs @@ -1,6 +1,6 @@ //! Issue #171 — composing exact operators with summary plans across //! explicit update/evaluation boundaries, end to end through -//! `search_workload_with` → `CandidateLogicalASAPDAGs::global_selection` → +//! `search_workload_with` → `candidate_selection::global_selection` → //! `GlobalSelection::assemble_selected_dag` → `dag_export`. //! //! Covers the issue's integration matrix: both nesting directions, grouped @@ -15,16 +15,18 @@ use std::rc::Rc; use asap_aware_mapping::cost_model::{ CostProvenance, CostUnit, ExactCompositionCostInputs, ExactCompositionCostRequest, }; -use asap_aware_mapping::exact_composition::ExactOperation; -use asap_aware_mapping::replacement::{ - default_strategies, search_workload_with, ASAPStrategies, Replacement, ReplacementProvenance, - ReplacementStrategy, TargetSubDAG, -}; -use asap_aware_mapping::{ - CostModel, DefaultCostModel, EvaluationRate, ExplanationKind, OperationPlacement, +use asap_aware_mapping::plan_selection::candidate_selection::{ + global_selection, runtime_support_evidence, }; +use asap_aware_mapping::{CostModel, DefaultCostModel, EvaluationRate}; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::post_asap::{maintained, post_asap_dag, timed}; +use asap_logical_optimizer::pass1::exact_composition::ExactOperation; +use asap_logical_optimizer::pass1::replacement::{ + default_strategies, search_workload_with, ASAPStrategies, Replacement, ReplacementProvenance, + ReplacementStrategy, TargetSubDAG, +}; +use asap_logical_optimizer::{ExplanationKind, OperationPlacement}; use asap_types::dag_export; use asap_types::ir::export::{NonASAPOpKind, PhysicalASAPOperatorPayload}; use asap_types::ir::operator::agg_intent::{default_quantile, AggIntent}; @@ -102,7 +104,7 @@ struct StatsModel; /// Search, selection, and materialization must retain the caller's proven rule. #[test] fn custom_accuracy_rule_survives_root_target_and_materialization() { - use asap_aware_mapping::{AccuracyModel, DefaultAccuracyModel, PropagationStats}; + use asap_logical_optimizer::{AccuracyModel, DefaultAccuracyModel, PropagationStats}; use asap_types::ir::properties::{AccuracyError, CompositionOperator, ResultGuarantee}; use asap_types::ir::schema::SketchStatistic; struct Model; @@ -132,17 +134,13 @@ fn custom_accuracy_rule_survives_root_target_and_materialization() { } } let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); - let space = asap_aware_mapping::replacement::search_workload_with_targets( + let space = asap_logical_optimizer::pass1::replacement::search_workload_with_targets( vec![("q", root, Some(AccuracyTarget::Exact))], &default_strategies(), &Model, ); - let selection = space.global_selection(&StatsModel); - assert!(selection - .for_target(&space.roots[0].1) - .unwrap() - .composition - .is_some()); + let selection = global_selection(&space, &StatsModel); + assert!(selection.composition(&space.roots[0].1).is_some()); let node = selection .assemble_selected_dag(&space.roots[0].1) .unwrap() @@ -156,17 +154,13 @@ fn custom_accuracy_rule_survives_root_target_and_materialization() { #[test] fn root_target_rejects_unproven_composition() { let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); - let space = asap_aware_mapping::replacement::search_workload_with_targets( + let space = asap_logical_optimizer::pass1::replacement::search_workload_with_targets( vec![("q", root, Some(AccuracyTarget::Exact))], &default_strategies(), - &asap_aware_mapping::DefaultAccuracyModel, + &asap_logical_optimizer::DefaultAccuracyModel, ); - let selection = space.global_selection(&StatsModel); - assert!(selection - .for_target(&space.roots[0].1) - .unwrap() - .composition - .is_none()); + let selection = global_selection(&space, &StatsModel); + assert!(selection.composition(&space.roots[0].1).is_none()); } impl CostModel for StatsModel { @@ -236,22 +230,16 @@ fn unknown_runtime_capability_keeps_candidate_but_prevents_selection() { let group = space.candidates_for_target(&space.roots[0].1).unwrap(); assert!(group.candidates.iter().any(|candidate| { matches!(candidate.replacement, Replacement::ExactComposition(_)) - && candidate - .runtime_support_evidence(&UnknownCapabilityModel) - .is_none() + && runtime_support_evidence(candidate, &UnknownCapabilityModel).is_none() && UnknownCapabilityModel .candidate_cost( candidate, - &asap_aware_mapping::TargetSubDAG::new(&space.roots[0].1), + &asap_logical_optimizer::TargetSubDAG::new(&space.roots[0].1), ) .is_none() })); - let selection = space.global_selection(&UnknownCapabilityModel); - assert!(selection - .for_target(&space.roots[0].1) - .unwrap() - .composition - .is_none()); + let selection = global_selection(&space, &UnknownCapabilityModel); + assert!(selection.composition(&space.roots[0].1).is_none()); assert!(selection .assemble_selected_dag(&space.roots[0].1) .unwrap() @@ -260,7 +248,7 @@ fn unknown_runtime_capability_keeps_candidate_but_prevents_selection() { fn plan( roots: Vec<(&'static str, Rc)>, -) -> asap_aware_mapping::CandidateLogicalASAPDAGs<&'static str> { +) -> asap_logical_optimizer::CandidateLogicalASAPDAGs<&'static str> { search_workload_with(roots, &default_strategies()) } @@ -419,16 +407,15 @@ fn max_and_avg_over_quantile_compose_at_query_time_with_statistics() { "{intent:?}: the inner quantile keeps its own evaluation candidates" ); - let selection = space.global_selection(&StatsModel); + let selection = global_selection(&space, &StatsModel); let selected = selection.for_target(&root).unwrap(); let chosen = selected.chosen.expect("a decision"); assert_eq!( chosen.provenance, ReplacementProvenance::ValueOperationAtQueryTime ); - let decision = selected - .composition - .as_ref() + let decision = selection + .composition(&root) .expect("composition provenance"); assert!(Rc::ptr_eq(decision.child_target, inner)); assert!(decision.cost_rate < decision.baseline_rate); @@ -508,8 +495,7 @@ fn identity_and_genuine_multi_row_folds_both_compose() { let root = agg(vec![0], AggIntent::Max { col: None }, inner); let space = plan(vec![("q", root)]); let root = &space.roots[0].1; - let composed = space - .global_selection(&StatsModel) + let composed = global_selection(&space, &StatsModel) .assemble_selected_dag(root) .unwrap() .unwrap(); @@ -540,7 +526,7 @@ fn a_shared_inner_summary_is_materialized_once_for_several_outer_folds() { let max = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); let min = agg(vec![0], AggIntent::Min { col: None }, fine_quantile()); let space = plan(vec![("max", max), ("min", min)]); - let selection = space.global_selection(&StatsModel); + let selection = global_selection(&space, &StatsModel); let roots: Vec> = space.roots.iter().map(|(_, r)| Rc::clone(r)).collect(); let inner_of = |r: &Rc| match r.non_asap() { @@ -559,14 +545,7 @@ fn a_shared_inner_summary_is_materialized_once_for_several_outer_folds() { let decisions: Vec<_> = roots .iter() - .map(|r| { - selection - .for_target(r) - .unwrap() - .composition - .as_ref() - .expect("both roots compose") - }) + .map(|r| selection.composition(r).expect("both roots compose")) .collect(); assert!(std::ptr::eq( decisions[0].child_candidate.unwrap(), @@ -627,13 +606,13 @@ fn outer_summary_over_an_exact_function_composes_at_ingestion_time() { .iter() .any(|c| c.provenance == ReplacementProvenance::ValueOperationAtIngestionTime)); - let selection = space.global_selection(&StatsModel); + let selection = global_selection(&space, &StatsModel); let deriv_sel = selection.for_target(deriv).unwrap(); assert_eq!( deriv_sel.chosen.unwrap().provenance, ReplacementProvenance::ValueOperationAtIngestionTime ); - let decision = deriv_sel.composition.as_ref().unwrap(); + let decision = selection.composition(deriv).unwrap(); assert!(decision.child_candidate.is_none(), "function input is raw"); assert!(decision.cost_rate < decision.baseline_rate); @@ -667,8 +646,7 @@ fn outer_summary_over_an_exact_function_composes_at_ingestion_time() { fn summary_construction_follows_its_value_input_phase() { let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); let space = plan(vec![("q", Rc::clone(&root))]); - let post = space - .global_selection(&StatsModel) + let post = global_selection(&space, &StatsModel) .assemble_selected_dag(&space.roots[0].1) .unwrap() .unwrap(); @@ -711,9 +689,9 @@ fn missing_cost_statistics_preserve_the_conservative_retain_exact() { .candidates .iter() .any(|c| c.provenance == ReplacementProvenance::ValueOperationAtQueryTime)); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); let selected = selection.for_target(&root).unwrap(); - assert!(selected.composition.is_none()); + assert!(selection.composition(&root).is_none()); assert!(!matches!( selected.chosen.map(|c| &c.replacement), Some(Replacement::ExactComposition(_)) @@ -721,7 +699,7 @@ fn missing_cost_statistics_preserve_the_conservative_retain_exact() { let node = selection.assemble_selected_dag(&root).unwrap().unwrap(); assert!(!is_query_time_fold(&node)); - let explanations = asap_aware_mapping::explain_replacements(vec![("q", Rc::clone(&root))]); + let explanations = asap_logical_optimizer::explain_replacements(vec![("q", Rc::clone(&root))]); assert!(explanations .iter() .any(|e| e.kind == ExplanationKind::ExactComposition)); @@ -734,8 +712,7 @@ fn dag_export_carries_explicit_stage_and_plain_schema_for_a_composed_plan() { let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); let space = plan(vec![("q", root)]); let root = &space.roots[0].1; - let composed = space - .global_selection(&StatsModel) + let composed = global_selection(&space, &StatsModel) .assemble_selected_dag(root) .unwrap() .unwrap(); @@ -777,7 +754,7 @@ fn promql_max_by_zone_over_quantile_over_time_composes() { .unwrap(); let space = plan(vec![("q", expr)]); let root = &space.roots[0].1; - let selection = space.global_selection(&StatsModel); + let selection = global_selection(&space, &StatsModel); let selected = selection.for_target(root).unwrap(); assert_eq!( selected.chosen.map(|c| c.provenance), @@ -795,7 +772,7 @@ fn promql_max_by_zone_over_quantile_over_time_composes() { assert!(is_query_time_fold(&composed), "{:?}", composed.operator); assert_eq!(timed(&composed).timing, Some(ExecutionTiming::QueryTime)); assert_eq!( - selected.composition.as_ref().map(|d| d.inputs.unit), + selection.composition(root).map(|d| d.inputs.unit), Some(CostUnit::CostUnitsPerSecond) ); let _ = OperationPlacement::Read; diff --git a/crates/integration-tests/tests/operator_sharing.rs b/crates/integration-tests/tests/operator_sharing.rs index 5a102e4f..d4352088 100644 --- a/crates/integration-tests/tests/operator_sharing.rs +++ b/crates/integration-tests/tests/operator_sharing.rs @@ -9,9 +9,11 @@ use std::rc::Rc; -use asap_aware_mapping::{search_workload, DefaultCostModel}; +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; +use asap_aware_mapping::DefaultCostModel; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::post_asap::post_asap_dag; +use asap_logical_optimizer::search_workload; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; use asap_types::types::AccuracyTarget; @@ -53,7 +55,7 @@ async fn plan(sql: &str, accuracy: AccuracyTarget) -> Rc { .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); let space = search_workload(vec![("query", pre)]); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); selection .assemble_selected_dag(&space.roots[0].1) .expect("materialization failed") diff --git a/crates/integration-tests/tests/planner_layering_example1.rs b/crates/integration-tests/tests/planner_layering_example1.rs index 812161fa..07b520fa 100644 --- a/crates/integration-tests/tests/planner_layering_example1.rs +++ b/crates/integration-tests/tests/planner_layering_example1.rs @@ -33,7 +33,7 @@ mod stages { use std::rc::Rc; use super::*; - use asap_aware_mapping::logical_candidates::{ + use asap_logical_optimizer::pass1::logical_candidates::{ compose_logical_candidate, enumerate_local_logical_candidates, }; use asap_types::ir::export::{compile_logical_asap_workload, LogicalASAPQueryRoot}; diff --git a/crates/integration-tests/tests/precompute_raw_samples.rs b/crates/integration-tests/tests/precompute_raw_samples.rs index c0157511..929519df 100644 --- a/crates/integration-tests/tests/precompute_raw_samples.rs +++ b/crates/integration-tests/tests/precompute_raw_samples.rs @@ -7,11 +7,12 @@ use physical_common::compile_maintained_physical_asap_dag; use std::{collections::BTreeMap, collections::BTreeSet, rc::Rc, sync::Arc}; use asap_aware_mapping::cost_model::DefaultCostModel; -use asap_aware_mapping::{ +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; +use asap_integration_tests::fixtures::lower_promql; +use asap_logical_optimizer::{ search_workload, ASAPStrategies, Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; -use asap_integration_tests::fixtures::lower_promql; use asap_physical_operators::{ factory::create_planner_accumulator, operators::Operator, @@ -69,9 +70,8 @@ fn candidates(query: &str, accuracy: AccuracyTarget) -> Vec> { }) .collect::>(); let space = search_workload(vec![("query", root)]); - if let Ok(Some(selected)) = space - .global_selection(&DefaultCostModel) - .assemble_selected_dag(&space.roots[0].1) + if let Ok(Some(selected)) = + global_selection(&space, &DefaultCostModel).assemble_selected_dag(&space.roots[0].1) { result.push(selected); } diff --git a/crates/integration-tests/tests/promql_numeric_regressions.rs b/crates/integration-tests/tests/promql_numeric_regressions.rs index 292c9a0e..1d1aac8a 100644 --- a/crates/integration-tests/tests/promql_numeric_regressions.rs +++ b/crates/integration-tests/tests/promql_numeric_regressions.rs @@ -1,9 +1,9 @@ //! Numeric regression fixtures: actual PromQL lowering plus numeric update/evaluation checks. //! The count/sum interpreter below verifies planner update semantics, not a deployed backend. -use asap_aware_mapping::replacement::is_logical_rewrite; -use asap_aware_mapping::{ASAPStrategies, Replacement, ReplacementStrategy, TargetSubDAG}; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::post_asap::post_asap_dag; +use asap_logical_optimizer::pass1::replacement::is_logical_rewrite; +use asap_logical_optimizer::{ASAPStrategies, Replacement, ReplacementStrategy, TargetSubDAG}; use asap_types::ir::operator::Reduction; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{ExactKind, FieldDataType, SummaryInputExpr, SummaryUpdate}; @@ -21,7 +21,7 @@ fn plan(query: &str, accuracy: AccuracyTarget) -> Rc { Replacement::SubDAG(n) if !is_logical_rewrite(&n) => Some(n), _ => None, }) - .unwrap_or_else(|| asap_aware_mapping::replacement::retain_exact(&pre).unwrap()) + .unwrap_or_else(|| asap_logical_optimizer::pass1::replacement::retain_exact(&pre).unwrap()) } fn aggregate(node: &OperatorNode) -> (&FieldDataType, &SummaryUpdate, &Reduction) { match &node.operator { @@ -173,19 +173,19 @@ fn quantile_over_temporal_average_keeps_a_legal_candidate() { } struct OneKeyTopKEvidence; -impl asap_aware_mapping::accuracy::AccuracyEvidenceProvider for OneKeyTopKEvidence { +impl asap_logical_optimizer::accuracy::AccuracyEvidenceProvider for OneKeyTopKEvidence { fn propagation_stats( &self, op: &asap_types::ir::properties::CompositionOperator, _family: &FieldDataType, _query: Option<&asap_types::ir::schema::SketchStatistic>, - ) -> asap_aware_mapping::accuracy::PropagationStats { + ) -> asap_logical_optimizer::accuracy::PropagationStats { // Single-key fixture: no excluded keys; bounds cover every value below. if matches!( op, asap_types::ir::properties::CompositionOperator::TopKSelection ) { - asap_aware_mapping::accuracy::PropagationStats { + asap_logical_optimizer::accuracy::PropagationStats { topk_selected_lower_bound: Some(-1000.), topk_excluded_upper_bound: Some(-1001.), topk_interval_failure_probability: Some(0.001), @@ -199,7 +199,7 @@ impl asap_aware_mapping::accuracy::AccuracyEvidenceProvider for OneKeyTopKEviden #[test] fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { - use asap_aware_mapping::accuracy::{DefaultAccuracyModel, EqualSplitAllocator}; + use asap_logical_optimizer::accuracy::{DefaultAccuracyModel, EqualSplitAllocator}; use asap_types::ir::schema::{NonNegativeWeightProof, SketchAlgorithm, WeightDomain}; let strategy = ASAPStrategies::new_with_planning_inputs_and_evidence( &DefaultAccuracyModel, diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index 92632c0e..9d397ca3 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -8,20 +8,23 @@ use std::rc::Rc; -use asap_aware_mapping::accuracy::{ +use asap_aware_mapping::cost_model::DefaultCostModel; +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; +use asap_integration_tests::fixtures::lower_promql; +use asap_integration_tests::post_asap::{ + maintained, maintained_post_asap_dag, post_asap_dag, timed, +}; +use asap_logical_optimizer::accuracy::{ AccuracyEvidenceProvider, DefaultAccuracyModel, EqualSplitAllocator, PropagationStats, QuantileInputDomain, }; -use asap_aware_mapping::cost_model::DefaultCostModel; -use asap_aware_mapping::replacement::{is_logical_rewrite, retain_exact, RealizationError}; -use asap_aware_mapping::{ +use asap_logical_optimizer::pass1::replacement::{ + is_logical_rewrite, retain_exact, RealizationError, +}; +use asap_logical_optimizer::{ search_workload, search_workload_with_targets, ASAPStrategies, AccuracyModel, Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::post_asap::{ - maintained, maintained_post_asap_dag, post_asap_dag, timed, -}; use asap_types::ir::export::{NonASAPOpKind, PhysicalASAPOperatorPayload}; use asap_types::ir::operator::operator_properties::Reduction; use asap_types::ir::properties::CompositionOperator; @@ -87,7 +90,7 @@ fn distinct_over_time_offers_hll_cardinality_evaluation() { fn lower_search_and_materialize(query: &str) -> Rc { let pre = lower_promql(query, AccuracyTarget::Exact).expect("lowering failed"); let space = search_workload(vec![("query", pre)]); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); selection .assemble_selected_dag(&space.roots[0].1) .expect("materialization failed") @@ -573,11 +576,11 @@ fn ddsketch_quantile_ratio_meets_the_shared_relative_error_target() { }; let space = search_workload_with_targets( vec![("ratio", query, Some(target.clone()))], - &asap_aware_mapping::replacement::default_strategies_with_evidence(&evidence), + &asap_logical_optimizer::pass1::replacement::default_strategies_with_evidence(&evidence), &DefaultAccuracyModel, ); let root = &space.roots[0].1; - let selected = space.global_selection(&DefaultCostModel); + let selected = global_selection(&space, &DefaultCostModel); let chosen = selected .for_target(root) .and_then(|selection| selection.chosen.as_ref()) @@ -1051,7 +1054,7 @@ fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { ) .unwrap(); let space = search_workload(vec![("query", pre)]); - let selected = space.global_selection(&DefaultCostModel); + let selected = global_selection(&space, &DefaultCostModel); let plan = selected .assemble_selected_dag(&space.roots[0].1) .unwrap() @@ -1112,7 +1115,7 @@ fn physical_node_owns_phase_independently_of_binary_payload() { ] { let input = lower_promql(query, AccuracyTarget::Epsilon(0.05)).unwrap(); let search = search_workload(vec![("q", input)]); - let choice = search.global_selection(&DefaultCostModel); + let choice = global_selection(&search, &DefaultCostModel); let plan = choice .assemble_selected_dag(&search.roots[0].1) .unwrap() @@ -1154,7 +1157,7 @@ fn ddsketch_ratio_without_domain_proof_is_uncertified() { assert!(root.guarantee.is_none()); let space = search_workload_with_targets( vec![("unproven", pre, Some(AccuracyTarget::Epsilon(0.01)))], - &asap_aware_mapping::default_strategies(), + &asap_logical_optimizer::default_strategies(), &DefaultAccuracyModel, ); let root_group = space @@ -1173,7 +1176,7 @@ fn ddsketch_ratio_without_domain_proof_is_uncertified() { "backend must receive the uncertified ratio candidate for its own selection" ); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); assert!( selection .for_target(&space.roots[0].1) @@ -1382,7 +1385,7 @@ fn without_aggregation_candidates_export_valid_dags() { let root = lower_promql(query, accuracy.clone()).unwrap(); let space = search_workload_with_targets( vec![(0, root, Some(accuracy.clone()))], - &asap_aware_mapping::default_strategies(), + &asap_logical_optimizer::default_strategies(), &DefaultAccuracyModel, ); let inventory = space.enumerate_candidate_dags_for_root(&0, 65_536).unwrap(); diff --git a/crates/integration-tests/tests/sql_to_physical.rs b/crates/integration-tests/tests/sql_to_physical.rs index 722d375b..daaddc98 100644 --- a/crates/integration-tests/tests/sql_to_physical.rs +++ b/crates/integration-tests/tests/sql_to_physical.rs @@ -1,7 +1,9 @@ //! SQL frontend, candidate selection, physical compilation and fresh-run execution. mod physical_common; -use asap_aware_mapping::{search_workload, DefaultCostModel}; +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; +use asap_aware_mapping::DefaultCostModel; use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_logical_optimizer::search_workload; use asap_physical_operators::{ physical_planner::{compile, InputContract, Source}, runtime::{Limits, RunContext, Scope}, @@ -34,8 +36,7 @@ async fn sql_filter_grouped_sum_executes_and_rebinds() { .await .unwrap(); let space = search_workload(vec![("sql", logical)]); - let selected = space - .global_selection(&DefaultCostModel) + let selected = global_selection(&space, &DefaultCostModel) .assemble_selected_dag(&space.roots[0].1) .unwrap() .unwrap(); diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index 39f5741f..acbca0d0 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -19,13 +19,15 @@ use std::rc::Rc; -use asap_aware_mapping::replacement::{retain_exact, RealizationError}; -use asap_aware_mapping::{ - search_workload, ASAPStrategies, DefaultCostModel, Replacement, ReplacementStrategy, - ReplacementSubDAG, TargetSubDAG, -}; +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; +use asap_aware_mapping::DefaultCostModel; use asap_frontend_sql::{lower_sql, lower_sql_dialect, SqlCatalog}; use asap_integration_tests::post_asap::post_asap_dag; +use asap_logical_optimizer::pass1::replacement::{retain_exact, RealizationError}; +use asap_logical_optimizer::{ + search_workload, ASAPStrategies, Replacement, ReplacementStrategy, ReplacementSubDAG, + TargetSubDAG, +}; use asap_types::ir::export::{ EdgeRole, NonASAPOpKind, PhysicalASAPNodeId, PhysicalASAPOperatorPayload, WirePredicate, WireScalarExpr, @@ -201,7 +203,7 @@ async fn clickhouse_outer_sum_recursively_binds_inner_temporal_aggregate() { .await .expect("nested temporal SQL must lower"); let space = search_workload(vec![("nested", Rc::clone(&pre_asap))]); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); let root = selection .assemble_selected_dag(&space.roots[0].1) .expect("materialization failed") @@ -264,7 +266,7 @@ async fn sql_full_query_retains_project_and_binds_inner_aggregate() { panic!("sanity: a SQL root is a Project, unlike lower_promql's bare Aggregate"); }; let space = search_workload(vec![("query", Rc::clone(&pre_asap))]); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); let root = selection .assemble_selected_dag(&space.roots[0].1) .expect("materialization failed") @@ -310,7 +312,7 @@ async fn sql_join_recursively_binds_both_temporal_aggregate_children() { .await .expect("two-subquery rate ratio must lower"); let space = search_workload(vec![("ratio", Rc::clone(&pre_asap))]); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); let root = selection .assemble_selected_dag(&space.roots[0].1) .expect("materialization failed") @@ -428,7 +430,7 @@ async fn unsupported_sql_join_shapes_remain_fail_closed() { .await .unwrap_or_else(|error| panic!("join must lower before fail-closed mapping: {error}")); let space = search_workload(vec![("unsupported-join", Rc::clone(&pre_asap))]); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); let root = selection .assemble_selected_dag(&space.roots[0].1) .expect("materialization failed") @@ -457,7 +459,7 @@ async fn sql_relational_parents_retain_summary_bound_aggregate() { ) .await; let space = search_workload(vec![("query", Rc::clone(&pre_asap))]); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); let root = selection .assemble_selected_dag(&space.roots[0].1) .expect("materialization failed") @@ -548,7 +550,7 @@ async fn sql_filter_keeps_read_predicate_and_summary_population_selection() { assert_eq!(expected_source_predicates.len(), 1, "fixture source WHERE"); let space = search_workload(vec![("query", Rc::clone(&pre_asap))]); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); let root = selection .assemble_selected_dag(&space.roots[0].1) .expect("materialization failed") @@ -606,7 +608,7 @@ async fn sql_filter_preserves_local_fallback_boundary_for_unsupported_child() { ) .await; let space = search_workload(vec![("query", Rc::clone(&pre_asap))]); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); let root = selection .assemble_selected_dag(&space.roots[0].1) .expect("materialization failed") @@ -841,8 +843,7 @@ async fn map_projection_export_preserves_unsupported_child_boundary() { &catalog(), SqlDialect::ClickhouseSQL, AccuracyTarget::Exact, ).await.unwrap(); let space = search_workload(vec![("map_query", pre)]); - let root = space - .global_selection(&DefaultCostModel) + let root = global_selection(&space, &DefaultCostModel) .assemble_selected_dag(&space.roots[0].1) .unwrap() .unwrap(); diff --git a/crates/logical-optimizer/Cargo.toml b/crates/logical-optimizer/Cargo.toml new file mode 100644 index 00000000..06ff4377 --- /dev/null +++ b/crates/logical-optimizer/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "asap-logical-optimizer" +version = "0.1.0" +edition = "2021" + +# #509 Stage 1: logical candidate generation (Pass 1 rewrites and summary +# realization, Pass 2 ASAP-aware CSE) and the analytical accuracy model. +# Depends only on asap-types (the PromQL front end is a test-only +# dev-dependency); never on the cost model or a later stage. +# tests/stage1_cost_independence.rs checks this manifest. +[dependencies] +asap_sketchlib = { workspace = true } +asap-types = { path = "../types" } +thiserror = "2" +serde_json = "1" + +[dev-dependencies] +asap-frontend-promql = { path = "../frontend-promql" } diff --git a/crates/asap-aware-mapping/src/accuracy/allocation.rs b/crates/logical-optimizer/src/accuracy/allocation.rs similarity index 100% rename from crates/asap-aware-mapping/src/accuracy/allocation.rs rename to crates/logical-optimizer/src/accuracy/allocation.rs diff --git a/crates/asap-aware-mapping/src/accuracy/composition.rs b/crates/logical-optimizer/src/accuracy/composition.rs similarity index 99% rename from crates/asap-aware-mapping/src/accuracy/composition.rs rename to crates/logical-optimizer/src/accuracy/composition.rs index 55e01330..17efd657 100644 --- a/crates/asap-aware-mapping/src/accuracy/composition.rs +++ b/crates/logical-optimizer/src/accuracy/composition.rs @@ -450,7 +450,9 @@ fn composed_provenance( pub(super) fn exact_operation_rule(operation: &ExactOperation) -> Option { let ExactOperation::Aggregate { measures, .. } = operation; match measures.as_slice() { - [intent] => crate::function_rules::function_rules(intent).map(|rules| rules.accuracy), + [intent] => { + crate::pass1::function_rules::function_rules(intent).map(|rules| rules.accuracy) + } // The remaining functions are exact over exact samples, but have // no definition-backed rule over approximate values yet. _ => None, diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/cardinality.rs b/crates/logical-optimizer/src/accuracy/estimators/cardinality.rs similarity index 100% rename from crates/asap-aware-mapping/src/accuracy/estimators/cardinality.rs rename to crates/logical-optimizer/src/accuracy/estimators/cardinality.rs diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/cms.rs b/crates/logical-optimizer/src/accuracy/estimators/cms.rs similarity index 97% rename from crates/asap-aware-mapping/src/accuracy/estimators/cms.rs rename to crates/logical-optimizer/src/accuracy/estimators/cms.rs index 98073fc7..131dd1db 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/cms.rs +++ b/crates/logical-optimizer/src/accuracy/estimators/cms.rs @@ -41,7 +41,7 @@ mod tests { #[test] fn local_guarantee_inverts_frequency_sizing() { - use crate::replacement::default_size_params; + use crate::pass1::replacement::default_size_params; use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = asap_types::ir::operator::agg_intent::default_cardinality(); let params = default_size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/count_sketch.rs b/crates/logical-optimizer/src/accuracy/estimators/count_sketch.rs similarity index 97% rename from crates/asap-aware-mapping/src/accuracy/estimators/count_sketch.rs rename to crates/logical-optimizer/src/accuracy/estimators/count_sketch.rs index a1fbb621..9e9fc66c 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/count_sketch.rs +++ b/crates/logical-optimizer/src/accuracy/estimators/count_sketch.rs @@ -55,7 +55,7 @@ mod tests { #[test] fn count_sketch_uses_an_l2_guarantee() { - use crate::replacement::default_size_params; + use crate::pass1::replacement::default_size_params; use asap_types::ir::operator::agg_intent::default_cardinality; use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let intent = default_cardinality(); diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/ddsketch.rs b/crates/logical-optimizer/src/accuracy/estimators/ddsketch.rs similarity index 100% rename from crates/asap-aware-mapping/src/accuracy/estimators/ddsketch.rs rename to crates/logical-optimizer/src/accuracy/estimators/ddsketch.rs diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/hll.rs b/crates/logical-optimizer/src/accuracy/estimators/hll.rs similarity index 99% rename from crates/asap-aware-mapping/src/accuracy/estimators/hll.rs rename to crates/logical-optimizer/src/accuracy/estimators/hll.rs index 774691af..c7c26b43 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/hll.rs +++ b/crates/logical-optimizer/src/accuracy/estimators/hll.rs @@ -230,7 +230,7 @@ mod tests { } #[test] fn generic_rse_sizing_does_not_certify_confidence() { - use crate::replacement::default_size_params; + use crate::pass1::replacement::default_size_params; use asap_types::ir::operator::agg_intent::default_cardinality; use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = default_cardinality(); diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/kll.rs b/crates/logical-optimizer/src/accuracy/estimators/kll.rs similarity index 97% rename from crates/asap-aware-mapping/src/accuracy/estimators/kll.rs rename to crates/logical-optimizer/src/accuracy/estimators/kll.rs index 7d468cce..efa3feaa 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/kll.rs +++ b/crates/logical-optimizer/src/accuracy/estimators/kll.rs @@ -42,7 +42,7 @@ mod tests { #[test] fn local_guarantee_inverts_rank_sizing() { - use crate::replacement::default_size_params; + use crate::pass1::replacement::default_size_params; use asap_types::ir::operator::agg_intent::default_quantile; use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let q = default_quantile(0.99); diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs b/crates/logical-optimizer/src/accuracy/estimators/mod.rs similarity index 99% rename from crates/asap-aware-mapping/src/accuracy/estimators/mod.rs rename to crates/logical-optimizer/src/accuracy/estimators/mod.rs index 34e1eadb..6b094e30 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs +++ b/crates/logical-optimizer/src/accuracy/estimators/mod.rs @@ -157,7 +157,7 @@ impl<'a> EstimatorAccuracy<'a> { target: Option<&AccuracyTarget>, ) -> Self { let (epsilon, delta) = target - .map(crate::replacement::accuracy_budget) + .map(crate::pass1::replacement::accuracy_budget) .unwrap_or((0.0, 0.0)); Self { base, diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/univmon.rs b/crates/logical-optimizer/src/accuracy/estimators/univmon.rs similarity index 100% rename from crates/asap-aware-mapping/src/accuracy/estimators/univmon.rs rename to crates/logical-optimizer/src/accuracy/estimators/univmon.rs diff --git a/crates/asap-aware-mapping/src/accuracy/evidence.rs b/crates/logical-optimizer/src/accuracy/evidence.rs similarity index 100% rename from crates/asap-aware-mapping/src/accuracy/evidence.rs rename to crates/logical-optimizer/src/accuracy/evidence.rs diff --git a/crates/asap-aware-mapping/src/accuracy/mod.rs b/crates/logical-optimizer/src/accuracy/mod.rs similarity index 96% rename from crates/asap-aware-mapping/src/accuracy/mod.rs rename to crates/logical-optimizer/src/accuracy/mod.rs index 618393d6..2ea64d7f 100644 --- a/crates/asap-aware-mapping/src/accuracy/mod.rs +++ b/crates/logical-optimizer/src/accuracy/mod.rs @@ -9,7 +9,6 @@ pub mod allocation; pub mod composition; pub mod estimators; pub mod evidence; -pub mod reconciliation; pub use allocation::{ AccuracyAllocation, AccuracyBudgetAllocator, CompositionShape, EqualSplitAllocator, @@ -28,13 +27,13 @@ use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, Sketc use asap_types::ir::OperatorNode; use asap_types::types::AccuracyTarget; -use crate::exact_composition::ExactOperation; +use crate::pass1::exact_composition::ExactOperation; /// The deployment-extensible accuracy algebra. `asap-aware-mapping` ships /// [`DefaultAccuracyModel`]; a deployment with a proof for a composition the /// default rejects (a registered cross-metric conversion, say) implements /// this trait and passes it to -/// [`crate::replacement::ASAPStrategies::new_with_planning_inputs`]. +/// [`crate::pass1::replacement::ASAPStrategies::new_with_planning_inputs`]. pub trait AccuracyModel { /// The definition-registered rule for applying `operation` to an /// approximate input. `None` means the function is exact only over exact @@ -46,7 +45,7 @@ pub trait AccuracyModel { /// The guarantee of reading `query` out of a summary of family `family` /// built over an **exact** input — derived from the family's committed /// parameters by inverting the same sizing formulas - /// [`crate::replacement::default_size_params`] uses. `None` when this + /// [`crate::pass1::replacement::default_size_params`] uses. `None` when this /// model has no error model for the family (the default has none for /// `Sample`/`Wavelet`/`StatModel`). fn local_guarantee( diff --git a/crates/logical-optimizer/src/lib.rs b/crates/logical-optimizer/src/lib.rs new file mode 100644 index 00000000..2d6b7120 --- /dev/null +++ b/crates/logical-optimizer/src/lib.rs @@ -0,0 +1,78 @@ +//! `asap-logical-optimizer` — #509 Stage 1: logical candidate generation. +//! +//! It takes the pre-ASAP [`OperatorNode`](asap_types::ir::OperatorNode) DAGs a +//! front end produces and proposes the logical alternatives for each target +//! sub-DAG: which summary (if any) realizes each approximate intent, and which +//! semantic rewrites, roll-ups, groupings and exact compositions apply. It +//! never prices a plan: only Stage 3 uses the cost model (#572, decision +//! Q36(a)). Cargo enforces the stage order: this crate depends only on +//! `asap-types`, never on a front end, a later stage or the executor. +//! +//! - [`pass1`] — local alternatives per target sub-DAG. +//! - [`pass2`] — ASAP-aware sharing across targets. +//! - [`accuracy`] — the analytical accuracy model: per-family error bounds and +//! sizing ([`accuracy::estimators`]), propagation through compositions +//! ([`accuracy::composition`]) and error-budget allocation +//! ([`accuracy::allocation`]). Candidates no analytical rule can prove +//! invalid are kept. +//! +//! **Common sub-expression elimination (CSE) of identical sub-DAGs is not this +//! crate's job.** It runs over the pre-ASAP IR itself +//! (`asap_types::ir::cse`, issues #222/#223) before search. Pass 2 recognizes +//! sharing that is invisible at that level, such as `Quantile(x, 0.99)` and +//! `Quantile(x, 0.95)` reading one built sketch. +//! +//! ## Candidate search +//! +//! Search returns [`CandidateLogicalASAPDAGs`](pass1::replacement::CandidateLogicalASAPDAGs), +//! a compact logical choice space with one [`TargetSubDAGCandidates`] per +//! target sub-DAG. [`ReplacementStrategy`] implementations propose local +//! alternatives; search applies the applicable semantic and accuracy checks. +//! Candidate presence does not certify physical deployability or an unknown +//! accuracy guarantee. [`GlobalSelection`] assembles a DAG from given per-target +//! choices; choosing them is a later stage's job. +//! +//! | Term | Meaning | Entry point | +//! |---|---|---| +//! | Realization | Enumerate the physical forms for one aggregate intent | `pass1::replacement::realizations_for_intent` | +//! | Replacement | Construct each candidate summary sub-DAG | [`ASAPStrategies`] | +//! | Search | Enumerate alternatives across a workload | [`search_workload`] | +//! | Local candidates | The #509 stage pipeline's Stage 1 entry point | [`pass1::logical_candidates`] | +//! +//! [`Matcher`] asks whether an already available `Realization` satisfies a +//! required one. It has no shipped implementation: which realizations are +//! available is a deployment's concern. +//! +//! [`explanation`](pass1::explanation) reports, for each discovered target +//! with a non-trivial candidate list, why a replacement exists, reusing the +//! candidate's own rationale. + +pub mod accuracy; +pub mod pass1; +pub mod pass2; +#[cfg(test)] +mod test_support; + +pub use accuracy::{ + AccuracyAllocation, AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, + CompositionShape, DefaultAccuracyModel, EqualSplitAllocator, NoAccuracyEvidence, + PropagationStats, WorkloadAccuracyEvidence, +}; +pub use pass1::exact_composition::{ + ExactComposition, ExactCompositionStrategy, OperationPlacement, +}; +pub use pass1::explanation::{ + explain_replacements, explain_replacements_with, ExplanationKind, ReplacementExplanation, +}; +pub use pass1::grouping::{has_subpopulations, HydraGroupingStrategy}; +pub use pass1::replacement::{ + default_strategies, is_logical_rewrite, search_workload, search_workload_with, + search_workload_with_targets, summary_candidates, ASAPStrategies, CandidateLogicalASAPDAGs, + GlobalSelection, Matcher, Proposals, Realization, RealizationError, RejectedCandidate, + Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, + SharedSubDAGStrategy, TargetSubDAG, TargetSubDAGCandidates, TargetSubDAGSelection, + MAX_SEARCH_ITERATIONS, +}; +pub use pass1::rewrite::{AvgToSumOverCountStrategy, SemanticEquivalentRewriteStrategy}; +pub use pass2::reconciliation::AccuracyReconciliationStrategy; +pub use pass2::topk_reuse::TopKLimitReuseStrategy; diff --git a/crates/asap-aware-mapping/src/exact_composition.rs b/crates/logical-optimizer/src/pass1/exact_composition.rs similarity index 97% rename from crates/asap-aware-mapping/src/exact_composition.rs rename to crates/logical-optimizer/src/pass1/exact_composition.rs index c2c9cbdc..8b5fab90 100644 --- a/crates/asap-aware-mapping/src/exact_composition.rs +++ b/crates/logical-optimizer/src/pass1/exact_composition.rs @@ -22,7 +22,7 @@ //! [`Replacement::ExactComposition`] carries only the child *target* //! (`ExactComposition::child_target`, the same `Rc` whose //! `TargetSubDAGCandidates` in `CandidateLogicalASAPDAGs` already holds every candidate for it). It is -//! [`CandidateLogicalASAPDAGs::global_selection`](crate::replacement::CandidateLogicalASAPDAGs::global_selection) +//! `candidate_selection::global_selection` //! that commits the compatible parent/child pair — so the child's own //! cost-model ranking, workload-wide effective consumer count, and shared //! `Rc` identity (one inner summary serving two outer folds) all stay @@ -48,7 +48,7 @@ //! with explicit positive support evidence from its cost model. //! //! `avg` gets a read-time operation candidate *and* keeps -//! [`crate::rewrite::AvgToSumOverCountStrategy`]'s rewrite in the same +//! [`crate::pass1::rewrite::AvgToSumOverCountStrategy`]'s rewrite in the same //! group; the cost model picks between them, nothing here hard-codes one. //! //! ## What this strategy never does @@ -58,7 +58,7 @@ //! `RealizationError` regardless. //! - Decide whether a composition is *worth it*: that is //! `global_selection`'s job, using the issue's cost-units-per-second -//! formulas (see `crate::cost_model::read_operation_plan_cost_rate` and +//! formulas (see `cost_model::read_operation_plan_cost_rate` and //! siblings). Missing statistics keep the conservative kept sub-DAG. use asap_types::ir::operator::non_asap::any_measure_filtered; @@ -77,7 +77,7 @@ use asap_types::physical::execution_data_state::lift_plain; use asap_types::physical::ExactOperationSchemaError; use asap_types::types::AccuracyTarget; -use crate::replacement::{ +use crate::pass1::replacement::{ bindable_intent, describe_intent, realizations_for_intent, Realization, RealizationError, Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; @@ -266,7 +266,7 @@ impl ExactComposition { /// Structural identity for `TargetSubDAGCandidates` dedup: same placement, same /// operator, same child `Rc`. - pub(crate) fn same_as(&self, other: &Self) -> bool { + pub fn same_as(&self, other: &Self) -> bool { self.placement == other.placement && self.op == other.op && Rc::ptr_eq(&self.child_target, &other.child_target) @@ -426,7 +426,7 @@ impl ExactCompositionStrategy { accumulator cannot consume query-time values, so instead of keeping \ the whole tree pre-ASAP this applies the fold as an \ ExactRead over whichever summary evaluation global_selection \ - commits for the child target (asap_aware_mapping::exact_composition)", + commits for the child target (asap_logical_optimizer::pass1::exact_composition)", describe_intent(&intent), child_desc ), @@ -447,7 +447,7 @@ impl ExactCompositionStrategy { "{} is an exact per-entity function with no accumulator form; as an \ explicit ExactMaintenance on the update path its output can feed a \ maintained summary above it instead of being handed over as an opaque \ - raw kept sub_dag (asap_aware_mapping::exact_composition)", + raw kept sub_dag (asap_logical_optimizer::pass1::exact_composition)", describe_intent(&intent) ), }); @@ -469,7 +469,7 @@ impl ReplacementStrategy for ExactCompositionStrategy { #[cfg(test)] mod tests { use super::*; - use crate::replacement::retain_exact; + use crate::pass1::replacement::retain_exact; use crate::test_support::{agg, agg_per_entity as per_entity, metric_scan, timed}; use asap_types::ir::operator::agg_intent::default_quantile; use asap_types::ir::properties::ExecutionDataStateError; @@ -523,7 +523,7 @@ mod tests { let target = TargetSubDAG::new(&root); assert_eq!(ExactCompositionStrategy.replacements(&target).len(), 1); // `avg` competes with AvgToSumOverCountStrategy in the same group. - assert!(crate::rewrite::AvgToSumOverCountStrategy.matches(&target)); + assert!(crate::pass1::rewrite::AvgToSumOverCountStrategy.matches(&target)); } #[test] @@ -568,7 +568,7 @@ mod tests { }; // A bare SummaryAgg (state, no evaluation) is not a legal read-time operation // input — the operator would be consuming sketch state. - let state_child = crate::replacement::realize_child(&comp.child_target).unwrap(); + let state_child = crate::pass1::replacement::realize_child(&comp.child_target).unwrap(); let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &state_child.operator else { panic!("expected the child to realize to a evaluation"); @@ -610,7 +610,7 @@ mod tests { let Replacement::ExactComposition(comp) = &candidates[0].replacement else { unreachable!() }; - let evaluation = crate::replacement::realize_child(&comp.child_target).unwrap(); + let evaluation = crate::pass1::replacement::realize_child(&comp.child_target).unwrap(); assert!(!comp.accepts_child(&evaluation)); assert!(matches!( comp.compose(evaluation), diff --git a/crates/asap-aware-mapping/src/explanation.rs b/crates/logical-optimizer/src/pass1/explanation.rs similarity index 89% rename from crates/asap-aware-mapping/src/explanation.rs rename to crates/logical-optimizer/src/pass1/explanation.rs index 2f7f4178..7acaeebe 100644 --- a/crates/asap-aware-mapping/src/explanation.rs +++ b/crates/logical-optimizer/src/pass1/explanation.rs @@ -1,5 +1,5 @@ //! This crate's **explanation of a replacement**: for a `TargetSubDAG` that -//! [`crate::replacement::search_workload`] found something to say about, why +//! [`crate::pass1::replacement::search_workload`] found something to say about, why //! does that candidate exist? (issue #33: "Add logic to detect which //! optimizations are applicable to a query workload"; this module: issue //! #257.) @@ -7,7 +7,7 @@ //! This module does not answer "is optimization X applicable here, yes or //! no" — that framing implies a classifier deciding admissibility from //! scratch. What it actually does is narrower and more mechanical: reuse a -//! matching candidate's own [`crate::replacement::ReplacementSubDAG::rationale`] +//! matching candidate's own [`crate::pass1::replacement::ReplacementSubDAG::rationale`] //! to explain, in the candidate's own words, why a [`Replacement`] exists at //! a given target. No new prose is invented here; see "The reframing" below //! for exactly what's being reused and why. @@ -18,10 +18,10 @@ //! Earlier (PR #247, superseded by this module — see "What this replaces" //! below), "is optimization X applicable here?" was a yes/no fact each rule //! re-derived by walking the DAG itself. That made sense before there was -//! any other structure to consult. But [`crate::replacement::search_workload`] +//! any other structure to consult. But [`crate::pass1::replacement::search_workload`] //! (issue #252) now *already* computes, for every -//! [`TargetSubDAG`](crate::replacement::TargetSubDAG) in the workload, every -//! semantically valid [`crate::replacement::ReplacementSubDAG`] a registered +//! `TargetSubDAG` in the workload, every +//! semantically valid [`crate::pass1::replacement::ReplacementSubDAG`] a registered //! [`ReplacementStrategy`] can propose — a [`CandidateLogicalASAPDAGs`] of [`TargetSubDAGCandidates`]s. A //! rule re-deriving the same yes/no fact from scratch would be answering a //! question the search already answered, via a second, independently @@ -46,7 +46,7 @@ //! candidate list contains at least one summary-realization [`Replacement::SubDAG`] that //! actually realizes a sketch family (`FieldDataType::Sketch`), i.e. //! [`ASAPStrategies`] found something to offer beyond whatever -//! exact/pass-through candidate [`crate::replacement`]'s own +//! exact/pass-through candidate [`crate::pass1::replacement`]'s own //! `realizations_for_intent` would have committed to on its own. //! - [`ExplanationKind::CommonSubexpressionReuse`] — the `TargetSubDAG` //! has two or more consumers *and* its candidate list contains the @@ -56,7 +56,7 @@ //! just an accident of how the workload happened to be built. //! //! Each finding's `reason` is literally the matching candidate's own -//! [`crate::replacement::ReplacementSubDAG::rationale`] (joined, if more than one candidate +//! [`crate::pass1::replacement::ReplacementSubDAG::rationale`] (joined, if more than one candidate //! qualifies) — this module invents no new prose to explain *why* a //! candidate is valid; that explanation already exists on the candidate a //! [`ReplacementStrategy`] produced, and repeating it here (rather than @@ -73,7 +73,7 @@ //! unrepresented). So do the two top-level entry points, //! [`explain_replacements`] and [`explain_replacements_with`] //! — same "workload roots in, findings out" contract, mirroring -//! [`crate::replacement::search_workload`]/[`crate::replacement::search_workload_with`]'s +//! [`crate::pass1::replacement::search_workload`]/[`crate::pass1::replacement::search_workload_with`]'s //! own signature shape. Only the *data source* changed: this module now //! calls those two functions and translates the result, rather than running //! its own rules and their supporting traversal over the DAG a second time. @@ -86,15 +86,15 @@ //! //! PR #247 gave this module its own extension-point trait, `ApplicabilityRule` //! (`fn optimization(&self) -> ExplanationKind` + `fn evaluate(&self, roots) -//! -> Vec`), the same shape [`crate::cost_model::CostModel`] -//! and [`crate::replacement::Matcher`] use elsewhere in this crate. Once +//! -> Vec`), the same shape `cost_model::CostModel` +//! and [`crate::pass1::replacement::Matcher`] use elsewhere in this crate. Once //! findings are a *view* over [`CandidateLogicalASAPDAGs`] rather than an independent //! computation, that trait would be a second extension point answering a //! question [`ReplacementStrategy`] (issue #251) already answers: "does this //! `TargetSubDAG` have an alternative worth reporting, and why". A caller who //! wants a new optimization represented as a finding needs a new //! `impl ReplacementStrategy` wired into -//! [`crate::replacement::search_workload_with`]'s strategy set *regardless* +//! [`crate::pass1::replacement::search_workload_with`]'s strategy set *regardless* //! (that's the only way its candidates end up in the [`CandidateLogicalASAPDAGs`] this //! module reads) — adding an `ApplicabilityRule` too would mean maintaining //! two extension points for the same new capability, one of which (the rule) @@ -103,13 +103,13 @@ //! [`ReplacementStrategy`] already *is* that extension point, one layer //! down, and [`explain_replacements_with`]'s own `strategies` //! parameter is where a caller plugs in a custom one — the identical spot -//! [`crate::replacement::search_workload_with`] itself exposes. +//! [`crate::pass1::replacement::search_workload_with`] itself exposes. //! //! ## Two guarantees the old traversal made, re-verified against the new one //! //! 1. **A finding is reported at the maximal `TargetSubDAG`, never once more -//! per subsumed descendant.** [`crate::replacement`]'s own -//! `discover_targets` (used by [`crate::replacement::search_workload_with`], +//! per subsumed descendant.** [`crate::pass1::replacement`]'s own +//! `discover_targets` (used by [`crate::pass1::replacement::search_workload_with`], //! and so by this module) walks every workload root's whole DAG but only //! *recurses into a node's children the first time that node's `Rc` is //! seen*; every subsequent occurrence still counts towards @@ -119,7 +119,7 @@ //! references it — identical to PR #247's own discovery pass, which //! reported "the highest point sharing starts," not a finding at every //! subsumed level below it. Same guarantee, same mechanism, just living in -//! [`crate::replacement`] now instead of here. +//! [`crate::pass1::replacement`] now instead of here. //! 2. **A node reachable via more than one path is one finding, not one per //! path.** [`TargetSubDAGCandidates`]s are keyed by `Rc` pointer identity in //! [`CandidateLogicalASAPDAGs`]'s internal map — there is exactly one group per distinct @@ -143,9 +143,9 @@ //! the deleted rule traversal: it makes no applicability decision (it runs //! the same regardless of what any strategy found), and duplicating this //! small, self-contained shape rather than threading location strings -//! through [`crate::replacement`]'s own `discover_targets` matches the same +//! through [`crate::pass1::replacement`]'s own `discover_targets` matches the same //! call that module's own docs already make for its (test-only) -//! `count_consumers` counterpart — see [`crate::replacement`]'s "Where +//! `count_consumers` counterpart — see [`crate::pass1::replacement`]'s "Where //! `TargetSubDAG` discovery comes from" section. //! //! ## Catalog primitives deliberately left as future work @@ -156,12 +156,12 @@ //! this codebase today**. Faking a variant for one of them would report a //! finding this codebase cannot back with a real candidate, so none of the //! below get an [`ExplanationKind`] variant yet — each gets one once a real -//! strategy exists and is wired into [`crate::replacement::default_strategies`]: +//! strategy exists and is wired into [`crate::pass1::replacement::default_strategies`]: //! //! | Catalog entry | Status | Where a future `ExplanationKind` would come from | //! |---|---|---| -//! | Semantic-equivalent rewriting (e.g. `avg` → `sum`/`count`) | [`AvgToSumOverCountStrategy`](crate::rewrite::AvgToSumOverCountStrategy) exists and is wired into `default_strategies()` (issue #253) — but still no `ExplanationKind` of its own below, since this table is about *direct* findings for a catalog entry, and this strategy's whole point is indirect: its `Replacement::SubDAG` rewrite candidate exposes `sum`/`count` as independently bindable discovered targets, which can then earn `CommonSubexpressionReuse` findings when the workload actually reuses them | A dedicated variant would need `findings_from_candidate_logical_asap_dags` to recognize a `LogicalRewrite`-provenance candidate as a finding in its own right, not just rely on what it exposes downstream | -//! | Roll-ups (fine-to-coarse group-by reuse) | [`RollupStrategy`](crate::rollup::RollupStrategy), derived from workload siblings after CSE/target discovery (issue #254) | Any `Replacement::SubDAG` rewrite candidate that rolls a coarse aggregate up from a compatible finer aggregate | +//! | Semantic-equivalent rewriting (e.g. `avg` → `sum`/`count`) | `AvgToSumOverCountStrategy` exists and is wired into `default_strategies()` (issue #253) — but still no `ExplanationKind` of its own below, since this table is about *direct* findings for a catalog entry, and this strategy's whole point is indirect: its `Replacement::SubDAG` rewrite candidate exposes `sum`/`count` as independently bindable discovered targets, which can then earn `CommonSubexpressionReuse` findings when the workload actually reuses them | A dedicated variant would need `findings_from_candidate_logical_asap_dags` to recognize a `LogicalRewrite`-provenance candidate as a finding in its own right, not just rely on what it exposes downstream | +//! | Roll-ups (fine-to-coarse group-by reuse) | `RollupStrategy`, derived from workload siblings after CSE/target discovery (issue #254) | Any `Replacement::SubDAG` rewrite candidate that rolls a coarse aggregate up from a compatible finer aggregate | //! | Wavelets/OMP | Params type exists (`WaveletKind`/`WaveletParams`), not reachable (no core `AggIntent` dispatch picks it) | A `ReplacementStrategy` for it, once some intent shape actually maps to `Realization::Wavelet` | //! | Sampling | Same story as Wavelets: `SamplingKind`/`SamplingParams` exist, unreachable from core dispatch | Same hook as Wavelets, for `Realization::Sample` | //! | Deep generative compression | No representation at all — no `Realization`/`FieldDataType` variant | Needs a new summary family added to `asap_types::ir::schema::state_type` first | @@ -171,14 +171,14 @@ //! | Incremental computation across time | No representation — nothing carries state across repeated evaluations of a `RepeatingEntry` today | Would key off `RepeatingEntry` + `TimeShift`/`TimeRange` once incremental state-carry exists | //! | Delta encoding | No representation — `AggIntent::Delta`/`IDelta` are PromQL *value*-difference semantics, not a wire/storage delta-encoding optimization | Would plug into a future deployment-side wire/storage encoding decision (post-ASAP), not this crate's IR-level dispatch | //! -//! [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy -//! [`ReplacementSubDAG`]: crate::replacement::ReplacementSubDAG -//! [`Replacement`]: crate::replacement::Replacement -//! [`Replacement::SubDAG`]: crate::replacement::Replacement::SubDAG -//! [`ASAPStrategies`]: crate::replacement::ASAPStrategies -//! [`SharedSubDAGStrategy`]: crate::replacement::SharedSubDAGStrategy -//! [`CandidateLogicalASAPDAGs`]: crate::replacement::CandidateLogicalASAPDAGs -//! [`TargetSubDAGCandidates`]: crate::replacement::TargetSubDAGCandidates +//! [`ReplacementStrategy`]: crate::pass1::replacement::ReplacementStrategy +//! [`ReplacementSubDAG`]: crate::pass1::replacement::ReplacementSubDAG +//! [`Replacement`]: crate::pass1::replacement::Replacement +//! [`Replacement::SubDAG`]: crate::pass1::replacement::Replacement::SubDAG +//! [`ASAPStrategies`]: crate::pass1::replacement::ASAPStrategies +//! [`SharedSubDAGStrategy`]: crate::pass1::replacement::SharedSubDAGStrategy +//! [`CandidateLogicalASAPDAGs`]: crate::pass1::replacement::CandidateLogicalASAPDAGs +//! [`TargetSubDAGCandidates`]: crate::pass1::replacement::TargetSubDAGCandidates use std::collections::HashMap; use std::fmt::Display; @@ -188,7 +188,7 @@ use asap_types::ir::cse::{structural_hash, HashCache}; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator, OperatorNode}; -use crate::replacement::{ +use crate::pass1::replacement::{ self, CandidateLogicalASAPDAGs, Replacement, ReplacementStrategy, TargetSubDAGCandidates, }; @@ -199,26 +199,26 @@ use crate::replacement::{ /// deliberately left as future work" table for everything else in the /// catalog). /// -/// [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy +/// [`ReplacementStrategy`]: crate::pass1::replacement::ReplacementStrategy #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum ExplanationKind { /// A `TargetSubDAG`'s candidate list contains at least one /// [`Replacement::SubDAG`] that realizes a sketch family — - /// [`crate::replacement::ASAPStrategies`] found a genuine sketch + /// [`crate::pass1::replacement::ASAPStrategies`] found a genuine sketch /// alternative for this `Aggregate`, beyond whatever exact/pass-through - /// candidate `crate::replacement`'s own `realizations_for_intent` would + /// candidate `crate::pass1::replacement`'s own `realizations_for_intent` would /// have committed to on its own. SketchApproximation, /// A `TargetSubDAG` has two or more consumers *and* its candidate list - /// contains [`crate::replacement::SharedSubDAGStrategy`]'s "build once + /// contains [`crate::pass1::replacement::SharedSubDAGStrategy`]'s "build once /// and share" candidate — the catalog's cross-statistic / cross-metrics / /// cross-subpopulation reuse entries, all the same underlying structural /// fact. CommonSubexpressionReuse, /// A `TargetSubDAG`'s candidate list contains at least one /// [`Replacement::ExactComposition`] — - /// [`crate::exact_composition::ExactCompositionStrategy`] found an exact + /// [`crate::pass1::exact_composition::ExactCompositionStrategy`] found an exact /// operator that can be composed with a summary plan across an explicit /// update/evaluation boundary instead of keeping the whole tree as it is /// (issue #171). @@ -229,7 +229,7 @@ pub enum ExplanationKind { /// breadcrumb into the workload — e.g. `root "dashboard_p99"` or /// `root "ratio" > lhs`): `reason` (human-readable, meant for a report/log, /// not machine parsing — literally the matching candidate's own -/// [`crate::replacement::ReplacementSubDAG::rationale`]). +/// [`crate::pass1::replacement::ReplacementSubDAG::rationale`]). /// /// `node_hash` is [`structural_hash`](asap_types::ir::cse::structural_hash) /// of the `TargetSubDAG`'s own `target` sub-DAG — the same function, on the @@ -250,16 +250,16 @@ pub struct ReplacementExplanation { pub target: Rc, } -/// Explain every replacement [`crate::replacement::search_workload`] finds +/// Explain every replacement [`crate::pass1::replacement::search_workload`] finds /// across a workload's pre-ASAP query roots, using -/// [`crate::replacement::default_strategies`]. +/// [`crate::pass1::replacement::default_strategies`]. /// -/// `roots` — like [`crate::replacement::search_workload`]'s own `Id` type +/// `roots` — like [`crate::pass1::replacement::search_workload`]'s own `Id` type /// parameter — is caller-chosen: a `QueryWorkload` entry's own key, an index, /// a query name. It only needs [`Display`], since a finding's `location` is /// prose, not a structured key back to the caller. /// -/// Internally runs [`crate::replacement::search_workload`] to build the +/// Internally runs [`crate::pass1::replacement::search_workload`] to build the /// candidate-plan space, then reads findings off it — see the module docs' /// "The reframing" section for what that translation actually checks. pub fn explain_replacements( @@ -269,10 +269,10 @@ pub fn explain_replacements( } /// Like [`explain_replacements`], but searches with `strategies` -/// instead of [`crate::replacement::default_strategies`] — the extension +/// instead of [`crate::pass1::replacement::default_strategies`] — the extension /// point for a deployment-specific [`ReplacementStrategy`]. /// -/// [`ReplacementStrategy`]: crate::replacement::ReplacementStrategy +/// [`ReplacementStrategy`]: crate::pass1::replacement::ReplacementStrategy pub fn explain_replacements_with<'s, Id: Display>( roots: Vec<(Id, Rc)>, strategies: &[Box], @@ -292,7 +292,7 @@ pub fn explain_replacements_with<'s, Id: Display>( /// /// `space`'s own `Id` is always `String` here: [`explain_replacements_with`] /// already converted the caller's `Id: Display` into a `String` (via -/// `to_string()`) before calling [`crate::replacement::search_workload_with`], +/// `to_string()`) before calling [`crate::pass1::replacement::search_workload_with`], /// so this function (and [`collect_locations`], which formats `id` with /// [`std::fmt::Debug`] for the breadcrumb text) doesn't need its own generic /// `Id` bound. @@ -399,7 +399,7 @@ fn shared_subexpr_finding_reason(group: &TargetSubDAGCandidates) -> Option bool { } /// Wraps the `GroupingStrategy` axis (issue #256) as a -/// [`ReplacementStrategy`]: for a target [`ASAPStrategies`](crate::replacement::ASAPStrategies) +/// [`ReplacementStrategy`]: for a target `ASAPStrategies` /// already has an opinion on, offers an additional /// `GroupingStrategy::SharedMultiSubpopulation` candidate wherever the /// legality conditions in the module docs above hold — alongside, not @@ -129,7 +129,7 @@ pub struct HydraGroupingStrategy<'a> { impl Default for HydraGroupingStrategy<'static> { /// The built-in accuracy models, the same default - /// [`crate::replacement::ASAPStrategies`] uses. + /// [`crate::pass1::replacement::ASAPStrategies`] uses. fn default() -> Self { Self { planning_inputs: CandidatePlanningInputs::with_default_accuracy(), @@ -187,7 +187,7 @@ impl<'a> HydraGroupingStrategy<'a> { /// Find the already-ranked candidate [`Realization::Sketch`] matching /// `sketch_kind` among [`realizations_for_intent`]'s exhaustive list for /// `intent`, bind `root` to that exact, already-decided candidate via - /// [`crate::replacement::construct_summary_with`] (no steering/forcing — see + /// [`crate::pass1::replacement::construct_summary_with`] (no steering/forcing — see /// the module docs' "No `ForceSketchKind`-style steering"), then swap the /// resulting `SummaryAgg`'s `grouping` field from the default /// `PerSubpopulationInstance` to @@ -281,7 +281,7 @@ impl<'a> HydraGroupingStrategy<'a> { Some(ReplacementSubDAG { strategy: "HydraGroupingStrategy", replacement: Replacement::SubDAG(patched), - provenance: crate::replacement::ReplacementProvenance::SummaryRealization, + provenance: crate::pass1::replacement::ReplacementProvenance::SummaryRealization, rationale: format!( "{} realizes as a shared {hydra_kind:?} structure over {sketch_kind:?} \ serving every subpopulation of this grouped aggregate, instead of one \ @@ -692,7 +692,7 @@ mod tests { .iter() .all(|r| matches!(r.error, AccuracyError::UnsupportedComposition { .. }))); - let space = crate::replacement::search_workload_with( + let space = crate::pass1::replacement::search_workload_with( vec![("q", Rc::clone(&q))], &[Box::new(strategy)], ); diff --git a/crates/asap-aware-mapping/src/logical_candidates.rs b/crates/logical-optimizer/src/pass1/logical_candidates.rs similarity index 99% rename from crates/asap-aware-mapping/src/logical_candidates.rs rename to crates/logical-optimizer/src/pass1/logical_candidates.rs index c65021da..1f73b1e8 100644 --- a/crates/asap-aware-mapping/src/logical_candidates.rs +++ b/crates/logical-optimizer/src/pass1/logical_candidates.rs @@ -20,7 +20,7 @@ use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, Schem use asap_types::types::AccuracyTarget; use thiserror::Error; -use crate::replacement::{ +use crate::pass1::replacement::{ accuracy_budget, accuracy_target, default_size_params, summary_candidates, Realization, }; @@ -381,7 +381,7 @@ fn summary_update( FieldDataType::Sketch(kind, _) => Some(kind.algorithm()), _ => None, }; - let weight = crate::replacement::summarised_input(intent, child) + let weight = crate::pass1::replacement::summarised_input(intent, child) .map_err(|_| LogicalCandidateError::Unsupported("input column outside child schema"))?; Ok(match (intent, algorithm) { (AggIntent::TopK { .. }, Some(_)) => { @@ -402,7 +402,7 @@ fn summary_update( .into_iter() .flat_map(|keys| keys.iter()) .filter_map(|&index| child.fields.get(index)) - .map(crate::replacement::column_ref) + .map(crate::pass1::replacement::column_ref) .collect(); // Rows that carry the full series identity rank it as a column, // the item form the runtime builds keyed summaries from. diff --git a/crates/asap-aware-mapping/src/maintained_population.rs b/crates/logical-optimizer/src/pass1/maintained_population.rs similarity index 99% rename from crates/asap-aware-mapping/src/maintained_population.rs rename to crates/logical-optimizer/src/pass1/maintained_population.rs index db502a48..40f57d22 100644 --- a/crates/asap-aware-mapping/src/maintained_population.rs +++ b/crates/logical-optimizer/src/pass1/maintained_population.rs @@ -1,5 +1,5 @@ //! Shared maintained-population candidates over canonical relational IR. -use crate::replacement::{ +use crate::pass1::replacement::{ Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; use asap_types::ir::operator::maintained_population::*; diff --git a/crates/logical-optimizer/src/pass1/mod.rs b/crates/logical-optimizer/src/pass1/mod.rs new file mode 100644 index 00000000..bf32ceee --- /dev/null +++ b/crates/logical-optimizer/src/pass1/mod.rs @@ -0,0 +1,13 @@ +//! Pass 1: local logical alternatives for each target sub-DAG. Strategies +//! propose rewrites and summary realizations; a candidate is pruned only when +//! it is provably invalid. + +pub mod exact_composition; +pub mod explanation; +pub(crate) mod function_rules; +pub mod grouping; +pub mod logical_candidates; +pub mod maintained_population; +pub mod replacement; +pub mod rewrite; +pub mod rollup; diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/logical-optimizer/src/pass1/replacement.rs similarity index 92% rename from crates/asap-aware-mapping/src/replacement.rs rename to crates/logical-optimizer/src/pass1/replacement.rs index 657f18f2..e84019b5 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/logical-optimizer/src/pass1/replacement.rs @@ -41,12 +41,12 @@ //! (still logical, structurally different from the target but semantically //! equivalent) — see [`Replacement`] — plus a human-readable `rationale`. //! - [`ReplacementStrategy`] — `matches` + `replacements`, the same -//! extension-point shape [`CostModel`](crate::cost_model::CostModel) and [`Matcher`] already use in this +//! extension-point shape `CostModel` and [`Matcher`] already use in this //! crate: a new replacement source is a new `impl ReplacementStrategy`, not //! a restructuring of this trait or of any existing strategy. `replacements` //! is **exhaustive, not ranked, not filtered** — reporting "every valid //! candidate" is core's job; picking the best one is left to the caller. -//! [`crate::explanation`] (issue #257) is this trait's own downstream +//! [`crate::pass1::explanation`] (issue #257) is this trait's own downstream //! consumer, not a second extension point: it explains why a replacement //! exists as a pure view over the candidates strategies registered here //! already produced, rather than re-deriving that explanation with a rule @@ -54,7 +54,7 @@ //! //! A caller may inspect local replacements, but taking the first candidate //! does not establish a compatible workload plan or physical deployability. -//! For Planner-owned logical selection, call [`CandidateLogicalASAPDAGs::global_selection`] +//! For Planner-owned logical selection, call `candidate_selection::global_selection` //! once and [`GlobalSelection::assemble_selected_dag`] for each wanted query //! root. Physical binding, deployment, and execution remain downstream. //! @@ -84,7 +84,7 @@ //! fixtures), it reports the two-way candidate CSE's own detection pass //! deliberately declines to pick between on its own: build once and share //! the already-interned sub-DAG, or build it independently at each -//! consumer. [`crate::cost_model::CostModel::cse_share_decision`] is where +//! consumer. `cost_model::CostModel::cse_share_decision` is where //! that choice actually gets made *today* (a fixed comparison, not a //! search) — this strategy exposes the same two-way choice as an explicit, //! inspectable pair of candidates instead of a cost model's already-decided @@ -233,8 +233,8 @@ //! //! ### Cost-based final selection — reusing `CostModel`, not a second interface //! -//! [`CandidateLogicalASAPDAGs::cost_sorted`] is the `sorted_by(cost_model)` step, and it -//! reuses this crate's existing [`CostModel`](crate::cost_model::CostModel) trait rather than inventing a +//! `candidate_selection::cost_sorted` is the `sorted_by(cost_model)` step, and it +//! reuses this crate's existing `CostModel` trait rather than inventing a //! second cost interface (`docs/design_docs/cse-cost-model-decision.md`, //! issue #237, explicitly reasoned about *why* a narrow, direct cost //! comparison was enough for the CSE share/recompute decision alone, and @@ -244,25 +244,25 @@ //! //! - A group whose candidates are the [`SharedSubDAGStrategy`] //! share-vs-recompute pair is ranked by calling -//! [`CostModel::cse_share_decision`](crate::cost_model::CostModel::cse_share_decision) via this module's own +//! `CostModel::cse_share_decision` via this module's own //! [`cse_preference`] — rather than re-deriving a competing comparison. //! - A group whose candidates are [`ASAPStrategies`]'s sketch-family -//! candidates is ranked via [`CostModel::rank_candidates`](crate::cost_model::CostModel::rank_candidates) (the same hook +//! candidates is ranked via `CostModel::rank_candidates` (the same hook //! `realizations_for_intent` itself consults), applied to the //! candidates' own [`SketchAlgorithm`]s. //! - Any other shape (a single candidate, or a mix this module doesn't have //! a defined comparison for) keeps discovery order — there is nothing to -//! rank, or no [`CostModel`](crate::cost_model::CostModel) hook this module knows how to apply; it never +//! rank, or no `CostModel` hook this module knows how to apply; it never //! invents a comparison `CostModel` doesn't already define. //! //! ## Whole-plan (cross-group) selection — issue #271 //! -//! [`CandidateLogicalASAPDAGs::cost_sorted`] above ranks every group's candidates +//! `candidate_selection::cost_sorted` above ranks every group's candidates //! independently: it never lets one group's choice influence how another //! group is costed. That's the right behavior when groups genuinely don't //! interact — which both shipped strategies' one-round convergence (see //! "Termination" above) makes the common case — but it's the wrong answer -//! whenever they do. Concretely: [`CostModel::cse_share_decision`](crate::cost_model::CostModel::cse_share_decision) costs a +//! whenever they do. Concretely: `CostModel::cse_share_decision` costs a //! [`SharedSubDAGStrategy`] group by comparing a `consumer_count`-scaled //! recompute cost against a fixed maintenance cost — but a **nested** //! `SharedSubDAGStrategy` group's *true* recompute burden isn't its own @@ -275,7 +275,7 @@ //! per-group ranking has no way to see this — it only ever looks at one //! group's own `candidates`, in isolation. //! -//! [`CandidateLogicalASAPDAGs::global_selection`] is that missing step: a single +//! `candidate_selection::global_selection` is that missing step: a single //! **top-down dynamic-programming pass** over the discovered sites, //! processed in the topological order [`topological_order`] computes over a //! small [`ReferenceDAG`] built for exactly this purpose (parent before @@ -284,11 +284,11 @@ //! **effective consumer count** — how many times that site actually runs //! once every ancestor's own selected candidate is accounted for — and, for //! every [`SharedSubDAGStrategy`]-shaped group, re-decides -//! [`CostModel::cse_share_decision`](crate::cost_model::CostModel::cse_share_decision) against *that* corrected count instead +//! `CostModel::cse_share_decision` against *that* corrected count instead //! of the group's raw structural one. When that group also contains a //! non-CSE alternative such as a semantic rewrite, the chosen CSE candidate //! and the cheapest non-CSE candidate additionally compete through -//! [`CostModel::estimate_cost`](crate::cost_model::CostModel::estimate_cost); the CSE pair is no longer allowed to hide an +//! `CostModel::estimate_cost`; the CSE pair is no longer allowed to hide an //! otherwise valid logical alternative. See [`multiplier`]'s doc for the //! exact recurrence: a group that chooses `Share` collapses its own //! multiplicity to exactly `1` for everything beneath it (one shared @@ -305,7 +305,7 @@ //! into it) combined via a real recurrence — not just the MEMO-group //! sharing [`CandidateLogicalASAPDAGs`] itself already does for *storing* candidates. That //! distinction is exactly what issue #271 raised: this module already looks -//! like a Cascades/Volcano MEMO, but [`CandidateLogicalASAPDAGs::cost_sorted`] alone never +//! like a Cascades/Volcano MEMO, but `candidate_selection::cost_sorted` alone never //! actually performed this composition step; `global_selection` is that //! step, added alongside `cost_sorted` rather than replacing it (both stay //! available — see [`RankedTargetSubDAGCandidates`] vs. [`TargetSubDAGSelection`]'s own docs for when @@ -314,7 +314,7 @@ //! Two things this deliberately does **not** attempt, both left as //! documented follow-up rather than silently overclaimed: //! -//! - [`CostModel::rank_candidates`](crate::cost_model::CostModel::rank_candidates) — the hook +//! - `CostModel::rank_candidates` — the hook //! [`ASAPStrategies`] groups rank by — takes no `consumer_count` //! parameter at all today, so a `ASAPStrategies` group's selection //! here still falls back to [`rank_group`]'s ordinary (consumer-count- @@ -327,7 +327,7 @@ //! would need, and this module now computes it for every group, sketch //! groups included. //! - This is not an exhaustive search over combinations of choices for a -//! provably-global optimum in every case. [`CostModel::cse_share_decision`](crate::cost_model::CostModel::cse_share_decision) +//! provably-global optimum in every case. `CostModel::cse_share_decision` //! is still a *local*, pairwise comparison at each `SharedSubDAGStrategy` //! site (recompute-total vs. one fixed maintenance cost) — this module //! just now feeds it a *correct* input instead of an *incorrect* one. Two @@ -350,7 +350,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use asap_types::ir::cse::{share_common_sub_dags, structural_hash, HashCache}; use asap_types::ir::operator::agg_intent::{agg_is_mergeable, AggIntent}; -use asap_types::ir::operator::operator_properties::{BinaryOpKind, Reduction}; +use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction}; use asap_types::ir::properties::summary_coverage::{CoverageRegion, SummaryCoverage}; use asap_types::ir::properties::timing::validate_maintained; use asap_types::ir::properties::{ @@ -359,30 +359,32 @@ use asap_types::ir::properties::{ use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef}; use asap_types::ir::schema::{ - EntityIdentity, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, + ColumnId, EntityIdentity, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, NonNegativeWeightProof, SamplingKind, SamplingParams, Schema, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic as PostAsapSketchStatistic, StatModelKind, StatModelParams, SummaryInputExpr, SummaryUpdate, WaveletKind, WaveletParams, WeightDomain, }; use asap_types::ir::SchemaDerivationError; use asap_types::ir::{ - ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, ProjectItem, ScalarExpr, SortKey, + ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, Predicate, ProjectItem, ScalarExpr, + SortKey, }; use asap_types::physical::ExactOperationSchemaError; use asap_types::types::AccuracyTarget; use std::rc::{Rc, Weak}; use thiserror::Error; -use crate::accuracy::reconciliation::AccuracyReconciliationStrategy; use crate::accuracy::{ AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, CompositionShape, DefaultAccuracyModel, EqualSplitAllocator, NoAccuracyEvidence, }; -use crate::exact_composition::{ExactComposition, ExactCompositionStrategy, OperationPlacement}; -use crate::grouping::HydraGroupingStrategy; -use crate::plan_selection::candidate_selection::{GlobalSelection, TargetSubDAGSelection}; -use crate::rollup::RollupStrategy; -use crate::topk_reuse::TopKLimitReuseStrategy; +use crate::pass1::exact_composition::{ + ExactComposition, ExactCompositionStrategy, OperationPlacement, +}; +use crate::pass1::grouping::HydraGroupingStrategy; +use crate::pass1::rollup::RollupStrategy; +use crate::pass2::reconciliation::AccuracyReconciliationStrategy; +use crate::pass2::topk_reuse::TopKLimitReuseStrategy; /// Errors from the pre-ASAP → post-ASAP replacement/construction path /// ([`realize_child`] and [`retain_exact`]). Moved here from the former @@ -489,10 +491,10 @@ pub enum Replacement { /// decision across an explicit update/evaluation boundary (issue #171): /// `ValueOperationAtQueryTime` over a child's summary evaluation, or /// `ValueOperationAtIngestionTime` feeding a maintained summary above. Carries only a - /// reference to the child target — [`CandidateLogicalASAPDAGs::global_selection`] + /// reference to the child target — `candidate_selection::global_selection` /// commits the compatible parent/child pair and /// [`GlobalSelection::assemble_selected_dag`] links it into one validated - /// `OperatorNode` DAG. See [`crate::exact_composition`]. + /// `OperatorNode` DAG. See [`crate::pass1::exact_composition`]. ExactComposition(ExactComposition), } @@ -508,7 +510,7 @@ pub fn is_logical_rewrite(node: &OperatorNode) -> bool { /// One candidate replacement for a [`TargetSubDAG`], plus a human-readable /// `rationale` explaining why it's a valid candidate (meant for a /// report/log/debugging a search engine's choices, not machine parsing — -/// [`crate::explanation::ReplacementExplanation::reason`] literally reuses +/// [`crate::pass1::explanation::ReplacementExplanation::reason`] literally reuses /// this same string rather than inventing new prose of its own. #[derive(Debug, Clone)] pub struct ReplacementSubDAG { @@ -543,12 +545,12 @@ pub enum ReplacementProvenance { CseShare, CseRecompute, LogicalRewrite, - /// [`crate::accuracy::reconciliation::AccuracyReconciliationStrategy`]'s + /// [`crate::pass2::reconciliation::AccuracyReconciliationStrategy`]'s /// "read a strictly-tighter sibling instead of building an independent, /// looser copy" candidate (issue #273). Kept distinct from /// `LogicalRewrite` — even though both are structurally-different, /// semantically-equivalent rewrites — because - /// [`crate::cost_model::DefaultCostModel::estimate_cost`] needs to price + /// `cost_model::DefaultCostModel::estimate_cost` needs to price /// it differently: `LogicalRewrite` candidates (`RollupStrategy`, /// `TopKLimitReuseStrategy`) still rebuild `target` itself from a /// different source, so pricing them like an independent rebuild is @@ -575,7 +577,7 @@ pub enum ReplacementProvenance { /// accuracy-legality grounds (issue #172) — kept alongside the group's /// legal candidates in [`TargetSubDAGCandidates::rejected`] so a rejection is as /// inspectable (and exportable) as a selection. Never ranked: a -/// [`CostModel`](crate::cost_model::CostModel) only ever sees [`TargetSubDAGCandidates::candidates`]. +/// `CostModel` only ever sees [`TargetSubDAGCandidates::candidates`]. #[derive(Debug, Clone)] pub struct RejectedCandidate { /// Name of the [`ReplacementStrategy`] that considered it. @@ -602,7 +604,7 @@ pub struct Proposals { /// replacement (`replacements`)? /// /// The extension point this module exists for — the same shape -/// [`CostModel`](crate::cost_model::CostModel) and [`Matcher`] already use elsewhere in this crate: a new +/// `CostModel` and [`Matcher`] already use elsewhere in this crate: a new /// replacement source is a new `impl ReplacementStrategy`, no restructuring /// of this trait or any existing strategy required. /// @@ -627,7 +629,7 @@ pub trait ReplacementStrategy { /// Every valid replacement for `target` — not ranked, not filtered. /// Reporting "every valid candidate" is this method's whole job; picking - /// the best one is a [`CostModel`](crate::cost_model::CostModel)'s job, out of scope here. + /// the best one is a `CostModel`'s job, out of scope here. fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec; /// [`replacements`](Self::replacements) plus the accuracy-illegal @@ -864,7 +866,7 @@ pub(crate) fn realizations_for_intent(intent: &AggIntent) -> Vec { | AggIntent::Rate | AggIntent::IRate | AggIntent::Increase => { - let (kind, params) = crate::function_rules::function_rules(intent) + let (kind, params) = crate::pass1::function_rules::function_rules(intent) .and_then(|rules| rules.accumulator) .expect("exact accumulator intents have registered realizations"); vec![exact_accumulator(intent, kind, params)] @@ -1410,7 +1412,7 @@ impl<'a> ASAPStrategies<'a> { // during DAG assembly. Also expose its concrete summary realization // for selection. if intent_override.is_none() { - if let Some(rewritten) = crate::rewrite::composed_aggregate_rewrite(root) { + if let Some(rewritten) = crate::pass1::rewrite::composed_aggregate_rewrite(root) { if let Ok(node) = realize_child_with(&rewritten, self.planning_inputs, None) { if node.contains_asap() { proposals.candidates.push(ReplacementSubDAG { @@ -1714,7 +1716,7 @@ fn describe_realization(intent: &AggIntent, realization: &Realization) -> String match realization { Realization::Sketch(kind) => format!( "{} realizes as a {:?} sketch — one of summary_candidates' \ - candidates for this intent (asap_aware_mapping::replacement::realizations_for_intent)", + candidates for this intent (asap_logical_optimizer::pass1::replacement::realizations_for_intent)", describe_intent(intent), kind.algorithm() ), @@ -1752,7 +1754,7 @@ fn describe_realization(intent: &AggIntent, realization: &Realization) -> String /// this crate's other `AggIntent` matches, e.g. [`realizations_for_intent`]'s) /// — this is prose for a rationale string, not a decision, so an unlisted /// variant just falls back to its `Debug` tag rather than forcing every -/// future intent to be named here too. [`crate::explanation`] needs no +/// future intent to be named here too. [`crate::pass1::explanation`] needs no /// counterpart of its own: it reads a candidate's `rationale` — built from /// this text — straight off [`ReplacementSubDAG`], rather than re-describing /// the same intent a second time. @@ -1778,23 +1780,17 @@ pub(crate) fn describe_intent(intent: &AggIntent) -> String { /// every candidate via [`ASAPStrategies::replacements`], keep the /// `cost_model`-preferred (first) one, and fall back to [`retain_exact`] /// when there's no candidate at all — **not** a general single-answer API -/// for a whole workload. Use [`CandidateLogicalASAPDAGs::global_selection`] and DAG assembly +/// for a whole workload. Use `candidate_selection::global_selection` and DAG assembly /// for coordinated logical selection; physical deployment remains downstream. /// `root` must already be the caller's own /// `Rc`, never fabricated per call, so this never allocates beyond what the /// caller already held. /// -/// `pub(crate)`: reachable from this module's own construction helper -/// ([`construct_summary_agg`], so a nested aggregate gets its own -/// independent enumeration instead of inheriting the parent's forced -/// candidate), from this module's own [`realize_one`] (the representative -/// bound `OperatorNode` [`cse_preference`] needs for a -/// [`CostModel::cse_share_decision`](crate::cost_model::CostModel::cse_share_decision) comparison), and from -/// [`crate::cost_model::DefaultCostModel::estimate_cost`] (the same -/// representative-node need, for a [`Replacement::Rewrite`] candidate's own -/// cost estimate). Every other caller goes through +/// Public because Stage 3 cost models need one representative bound node for +/// a target: `DefaultCostModel::estimate_cost` and the legacy CSE ranking in +/// `plan_selection::candidate_selection`. Every other caller goes through /// [`ASAPStrategies::replacements`] directly and decides for itself. -pub(crate) fn realize_child(root: &Rc) -> Result, RealizationError> { +pub fn realize_child(root: &Rc) -> Result, RealizationError> { realize_child_with(root, CandidatePlanningInputs::with_default_accuracy(), None) } @@ -1893,7 +1889,7 @@ fn realize_temporal_average( planning_inputs: CandidatePlanningInputs<'_>, target: Option<&AccuracyTarget>, ) -> Result>, RealizationError> { - let Some(components) = crate::rewrite::temporal_average_components(root) else { + let Some(components) = crate::pass1::rewrite::temporal_average_components(root) else { return Ok(None); }; let mut node = realize_child_with(&components, planning_inputs, target)?; @@ -3028,7 +3024,7 @@ fn construct_summary_agg( // Explicit snapshot selection prevents historical observations from // becoming repeated weights in an instant-vector heap. let root = Rc::new(node.clone()); - let population = crate::maintained_population::MaintainedPopulationStrategy::new( + let population = crate::pass1::maintained_population::MaintainedPopulationStrategy::new( std::slice::from_ref(&root), ) .candidate(&root) @@ -3648,7 +3644,7 @@ fn compose_guarantee( // Lipschitz constant — over an approximate child this is a // deterministic transform with no registered rule. _ => { - crate::function_rules::function_rules(intent) + crate::pass1::function_rules::function_rules(intent) .expect("exact accumulator intents have registered accuracy rules") .accuracy } @@ -3816,7 +3812,7 @@ fn evaluation(intent: &AggIntent, input: &SummaryUpdate) -> PostAsapSketchStatis /// "Non-goals" on why that traversal isn't itself part of this strategy). /// This strategy only reframes "two or more consumers already share this /// `Rc`" as the two-way choice a downstream cost model (today, -/// [`CostModel::cse_share_decision`](crate::cost_model::CostModel::cse_share_decision)) picks between: build once and share, or +/// `CostModel::cse_share_decision`) picks between: build once and share, or /// build independently at each consumer. pub struct SharedSubDAGStrategy; @@ -3896,14 +3892,14 @@ pub struct TargetSubDAGCandidates { /// reference this exact `Rc` — see [`discover_targets`]. pub consumer_count: usize, /// Every distinct alternative discovered for `target`, in discovery - /// order (not ranked — see [`CandidateLogicalASAPDAGs::cost_sorted`] for the ranked + /// order (not ranked — see `candidate_selection::cost_sorted` for the ranked /// view). pub candidates: Vec, /// Every candidate a strategy considered for `target` but refused on /// accuracy-legality grounds (issue #172), plus any `candidates` entry /// the root-target check ([`search_workload_with_targets`]) moved here. - /// Never ranked — [`CandidateLogicalASAPDAGs::cost_sorted`]/[`CandidateLogicalASAPDAGs::global_selection`] - /// read only `candidates`, so a [`CostModel`](crate::cost_model::CostModel) cannot resurrect one. + /// Never ranked — `candidate_selection::cost_sorted`/`candidate_selection::global_selection` + /// read only `candidates`, so a `CostModel` cannot resurrect one. pub rejected: Vec, } @@ -4032,18 +4028,20 @@ pub struct CandidateLogicalASAPDAGs { pub roots: Vec<(Id, Rc)>, pub(crate) groups: HashMap<*const OperatorNode, TargetSubDAGCandidates>, /// Discovery order — stable iteration for [`CandidateLogicalASAPDAGs::target_subdag_candidates`]/ - /// [`CandidateLogicalASAPDAGs::cost_sorted`], since `HashMap` iteration order isn't. + /// `candidate_selection::cost_sorted`, since `HashMap` iteration order isn't. pub(crate) order: Vec<*const OperatorNode>, /// Composition proofs are computed with the search model, then retained /// through costing and DAG assembly so no later default can replace it. pub(crate) composition_plans: Vec, } -pub(crate) struct PreparedComposition { - pub(crate) target: *const OperatorNode, - pub(crate) operation: ExactComposition, - pub(crate) child: Rc, - pub(crate) plan: Rc, +/// One exact composition validated during search: `operation` at `target`, +/// over the child candidate `child`, giving `plan`. +pub struct PreparedComposition { + pub target: *const OperatorNode, + pub operation: ExactComposition, + pub child: Rc, + pub plan: Rc, } impl CandidateLogicalASAPDAGs { @@ -4242,13 +4240,13 @@ impl CandidateLogicalASAPDAGs { consumer_count: group.consumer_count, effective_consumer_count: group.consumer_count, chosen: *chosen, - composition: None, }, ); } let assembly = GlobalSelection { order: order.clone(), groups, + composition_plans: HashMap::new(), assembled_nodes: RefCell::new(assembled_nodes), }; let roots = roots @@ -4298,6 +4296,21 @@ impl CandidateLogicalASAPDAGs { self.order.iter().map(move |ptr| &self.groups[ptr]) } + /// Every discovered target's candidate set, keyed by target identity. + pub fn groups(&self) -> &HashMap<*const OperatorNode, TargetSubDAGCandidates> { + &self.groups + } + + /// Target identities in discovery order. + pub fn order(&self) -> &[*const OperatorNode] { + &self.order + } + + /// The exact compositions validated during search. + pub fn composition_plans(&self) -> &[PreparedComposition] { + &self.composition_plans + } + /// How many distinct targets were discovered. pub fn len(&self) -> usize { self.groups.len() @@ -4325,7 +4338,7 @@ impl CandidateLogicalASAPDAGs { /// when other strategies contributed additional alternatives to the same /// memo group. Provenance makes these two orthogonal choices identifiable /// without inferring semantics from pointer or expression shape. -pub(crate) fn cse_candidate_pair( +pub fn cse_candidate_pair( group: &TargetSubDAGCandidates, ) -> Option<(&ReplacementSubDAG, &ReplacementSubDAG)> { let mut share = None; @@ -4358,7 +4371,7 @@ pub(crate) fn cse_candidate_pair( } /// Direct relational-skeleton children and their edge multiplicities. /// `Concat` is transparent, matching [`walk_children`]'s site scope. -pub(crate) fn direct_child_counts(node: &OperatorNode) -> Vec<(*const OperatorNode, usize)> { +pub fn direct_child_counts(node: &OperatorNode) -> Vec<(*const OperatorNode, usize)> { fn push(children: &mut Vec<(*const OperatorNode, usize)>, child: &Rc) { let ptr = Rc::as_ptr(child); match children.iter_mut().find(|(existing, _)| *existing == ptr) { @@ -4387,6 +4400,434 @@ pub(crate) fn direct_child_counts(node: &OperatorNode) -> Vec<(*const OperatorNo collect(node, &mut children); children } +// ── GlobalSelection: assemble a DAG from given choices ─────────────────── + +/// One target sub-DAG's chosen candidate and usage counts: the input +/// [`GlobalSelection`] assembles a DAG from. Building a DAG from given +/// choices needs no cost model; whoever makes the choices (enumeration here, +/// or the legacy cost-based selection in plan selection) fills these in. +#[derive(Debug)] +pub struct TargetSubDAGSelection<'a> { + /// The target sub-DAG this selection is for. + pub target: &'a Rc, + /// [`TargetSubDAGCandidates::consumer_count`] — how many operator-child positions + /// directly reference `target`, ignoring every ancestor's own choice. + pub consumer_count: usize, + /// How many times `target`'s computation actually runs once every + /// ancestor's own selected candidate is accounted for. Equal to + /// `consumer_count` unless some ancestor on a path from a root to this + /// site has a [`SharedSubDAGStrategy`] alternative that chose to + /// recompute independently. + pub effective_consumer_count: usize, + /// The candidate chosen for this target, or `None` when no replacement + /// is selected. The candidate set need not be empty: an unproven DDSketch + /// ratio can remain available for backend inspection but be excluded from + /// automatic selection, or costing can prefer raw recomputation. + /// DAG assembly then preserves exact computation at this target where + /// supported, while independently selected children may remain visible. + pub chosen: Option<&'a ReplacementSubDAG>, +} + +/// One [`TargetSubDAGSelection`] per discovered site, in the same discovery +/// order [`CandidateLogicalASAPDAGs::target_subdag_candidates`] uses, plus the +/// DAG assembly over those choices. +#[derive(Debug)] +pub struct GlobalSelection<'a> { + order: Vec<*const OperatorNode>, + groups: HashMap<*const OperatorNode, TargetSubDAGSelection<'a>>, + /// The validated plan of each site whose chosen candidate is a + /// [`Replacement::ExactComposition`]. + composition_plans: HashMap<*const OperatorNode, Rc>, + /// [`Self::assemble_selected_dag`]'s memo — one bound node per target for the + /// life of this selection, so two parents composing over one shared + /// child get the *same* `Rc` (a kept pre-ASAP sub-DAG + /// shared by two parents stays one `Rc` the same way). + assembled_nodes: RefCell>>, +} + +fn normalize_cross_input_equi_predicate( + pred: &Predicate, + left_width: usize, + total_width: usize, +) -> Option { + let ScalarExpr::Compare { + left, + op: asap_types::ir::scalar::CompareOpKind::Eq, + right, + semantics, + } = &pred.0 + else { + return None; + }; + let (ScalarExpr::Column(left_id), ScalarExpr::Column(right_id)) = + (left.as_ref(), right.as_ref()) + else { + return None; + }; + let is_left = |id: ColumnId| id < left_width; + let is_right = |id: ColumnId| left_width <= id && id < total_width; + let (left_id, right_id) = if is_left(*left_id) && is_right(*right_id) { + (*left_id, *right_id) + } else if is_right(*left_id) && is_left(*right_id) { + (*right_id, *left_id) + } else { + return None; + }; + Some(Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(left_id)), + op: asap_types::ir::scalar::CompareOpKind::Eq, + right: Box::new(ScalarExpr::Column(right_id)), + semantics: *semantics, + })) +} + +impl<'a> GlobalSelection<'a> { + /// A selection over `groups`, listed in `order`. `composition_plans` + /// holds the validated plan of every site that chose an exact composition. + pub fn new( + order: Vec<*const OperatorNode>, + groups: HashMap<*const OperatorNode, TargetSubDAGSelection<'a>>, + composition_plans: HashMap<*const OperatorNode, Rc>, + ) -> Self { + Self { + order, + groups, + composition_plans, + assembled_nodes: RefCell::new(HashMap::new()), + } + } + + /// One selection per discovered target sub-DAG, in discovery order. + pub fn target_selections(&self) -> impl Iterator> { + self.order.iter().map(move |ptr| &self.groups[ptr]) + } + + /// The selection for `target`, if `target`'s own `Rc` is a discovered + /// site (i.e. `Rc::ptr_eq` to some node reachable from the workload's + /// roots). + pub fn for_target(&self, target: &Rc) -> Option<&TargetSubDAGSelection<'a>> { + self.groups.get(&Rc::as_ptr(target)) + } + + /// Link this selection's per-site decisions into one data_state-validated + /// post-ASAP DAG rooted at `target` — the one place a committed + /// composition's child *reference* becomes an actual `Rc` + /// edge (issue #171). `None` if `target` is not a discovered site. + /// + /// Per site: a [`Replacement::ExactComposition`] uses its validated + /// operation/child plan, retaining the search model's guarantee; + /// a bound-summary [`Replacement::SubDAG`] is + /// re-linked so its `SummaryAgg` child is the child target's own + /// DAG assembly whenever that is phase-legal beneath maintenance + /// (so a child that chose an `ValueOperationAtIngestionTime` actually ends up under + /// the summary); a logical-rewrite [`Replacement::SubDAG`] is kept + /// as it is (exact); an unmatched site keeps its own operator with each + /// child assembled independently ([`Self::assemble_residual`]). + /// Memoized by target identity, so a shared inner summary is one `Rc` + /// no matter how many roots reach it. + pub fn assemble_selected_dag( + &self, + target: &Rc, + ) -> Result>, RealizationError> { + if !self.groups.contains_key(&Rc::as_ptr(target)) { + return Ok(None); + } + self.assemble_target(target).map(Some) + } + + /// Assemble a complete query result, including an exact-state evaluation when + /// needed. `assemble_selected_dag` also serves internal state frontiers; + /// callers exposing query results must use this boundary instead. + pub fn assemble_selected_query( + &self, + target: &Rc, + ) -> Result>, RealizationError> { + self.assemble_selected_dag(target)? + .map(|node| finalize_query_candidate(node, target)) + .transpose() + } + + fn assemble_target( + &self, + target: &Rc, + ) -> Result, RealizationError> { + let ptr = Rc::as_ptr(target); + if let Some(node) = self.assembled_nodes.borrow().get(&ptr) { + return Ok(Rc::clone(node)); + } + // A selected summary that realizes its inner aggregate, instead of + // hiding it in `KeepPreAsap`, is kept; materialization assignment decides + // whether it runs in precompute or at query time. + let selected_composed_summary = self + .groups + .get(&ptr) + .and_then(|sel| sel.chosen) + .is_some_and(|candidate| { + matches!(&candidate.replacement, + Replacement::SubDAG(node) if matches!(&node.operator, + Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) + if child.contains_asap() || !contains_aggregate(child))) + }); + let node = if query_time_nested_sum(target) && !selected_composed_summary { + self.assemble_residual(target)? + } else { + match self + .groups + .get(&ptr) + .and_then(|sel| sel.chosen) + .map(|c| &c.replacement) + { + None => self.assemble_residual(target)?, + Some(Replacement::SubDAG(node)) if node.contains_asap() => { + self.relink_summary(node, target)? + } + Some(Replacement::SubDAG(kept)) => retain_exact(kept)?, + Some(Replacement::ExactComposition(_)) => Rc::clone( + self.composition_plans + .get(&ptr) + .expect("selected compositions have a validated plan"), + ), + } + }; + self.assembled_nodes + .borrow_mut() + .insert(ptr, Rc::clone(&node)); + Ok(node) + } + + /// Keep `target`'s own operator and assemble each child independently, + /// so a selected summary remains visible beneath a relational operator + /// that has no summary realization of its own instead of being + /// swallowed by one opaque kept sub-DAG. Every child that is a + /// discovered target is assembled (and finalized to query-time values); + /// any other child is kept as it is. The guarantee is composed from the + /// assembled children: all exact → exact; exactly one child → that + /// child's guarantee; otherwise unknown. An inner `Join` first has its + /// cross-input equi-predicate normalized; any other join is kept whole. + fn assemble_residual( + &self, + target: &Rc, + ) -> Result, RealizationError> { + if target.children().is_empty() { + // A leaf has nothing to assemble beneath it: keep it as it is. + return retain_exact(target); + } + let mut operator = target.operator.clone(); + if let Operator::NonASAP(NonASAPOp::Join { + left, + right, + kind, + pred, + }) = &mut operator + { + let left_width = left.schema.fields.len(); + let total_width = left_width + right.schema.fields.len(); + let normalized_pred = matches!(kind, JoinKind::Inner) + .then(|| normalize_cross_input_equi_predicate(pred, left_width, total_width)) + .flatten(); + let Some(normalized) = normalized_pred else { + return retain_exact(target); + }; + *pred = normalized; + } + let mut failure = None; + let mut children = Vec::new(); + let operator = operator.map_children(|child| { + if failure.is_some() { + return Rc::clone(child); + } + let assembled = if self.groups.contains_key(&Rc::as_ptr(child)) { + self.assemble_target(child) + .and_then(|node| finalize_query_candidate(node, child)) + } else { + Ok(Rc::clone(child)) + }; + match assembled { + Ok(node) => { + children.push(Rc::clone(&node)); + node + } + Err(error) => { + failure = Some(error); + Rc::clone(child) + } + } + }); + if let Some(error) = failure { + return Err(error); + } + // An operator that computes new values from its input rows has no + // sound accuracy composition over an approximate input (e.g. `max` + // over a quantile evaluation's rank error). Without a selected + // composition such a node stays an exact pre-ASAP sub-DAG; only the + // read-time nested SUM keeps its assembled children. + let computes_values = matches!( + target.non_asap(), + Some( + NonASAPOp::Aggregate { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + ) + ) && !query_time_nested_sum(target); + let approximate_input = children.iter().any(|child| { + !child + .guarantee + .as_ref() + .is_some_and(ResultGuarantee::is_exact) + }); + if computes_values && approximate_input { + return retain_exact(target); + } + let guarantee = match children.as_slice() { + [child] => child.guarantee.clone(), + children + if children.iter().all(|child| { + child + .guarantee + .as_ref() + .is_some_and(ResultGuarantee::is_exact) + }) => + { + Some(ResultGuarantee::exact(format!( + "{} over exact inputs", + target.operator.kind_name() + ))) + } + _ => None, + }; + let node = Rc::new( + OperatorNode::with_schema(operator, target.schema.clone()).with_guarantee(guarantee), + ); + validate_maintained(&node, ExecutionTiming::QueryTime)?; + Ok(node) + } + + /// Re-link a bound summary candidate's `SummaryAgg` child to the + /// child target's own DAG assembly when that is legal beneath + /// maintenance; otherwise keep the candidate exactly as constructed. + fn relink_summary( + &self, + node: &Rc, + target: &Rc, + ) -> Result, RealizationError> { + let Some(NonASAPOp::Aggregate { + child: pre_child, .. + }) = target.non_asap() + else { + return Ok(Rc::clone(node)); + }; + let has_maintenance_operation = self + .groups + .get(&Rc::as_ptr(pre_child)) + .and_then(|selection| selection.chosen) + .is_some_and(|candidate| { + matches!( + &candidate.replacement, + Replacement::ExactComposition(composition) + if composition.placement == OperationPlacement::Maintenance + ) + }); + if !has_maintenance_operation { + return Ok(Rc::clone(node)); + } + let new_child = self.assemble_target(pre_child)?; + Ok(relink_agg_child(node, &new_child)) + } +} + +/// A mergeable outer SUM over a relationally wrapped aggregate is a read-time +/// reduction of the inner summary values. Maintaining the outer SUM directly +/// would hide that inner temporal aggregate inside one kept sub-DAG and lose +/// its independently selected summary. +fn query_time_nested_sum(target: &OperatorNode) -> bool { + let Some(NonASAPOp::Aggregate { + measures, + filters, + having: None, + child, + .. + }) = target.non_asap() + else { + return false; + }; + !any_measure_filtered(filters) + && matches!(measures.as_slice(), [AggIntent::Sum { .. }]) + && contains_aggregate(child) +} + +fn contains_aggregate(expr: &OperatorNode) -> bool { + match expr.non_asap() { + Some(NonASAPOp::Aggregate { .. }) => true, + Some( + NonASAPOp::Project { child, .. } + | NonASAPOp::Filter { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. }, + ) => contains_aggregate(child), + _ => false, + } +} + +/// Rebuild `node` (a `SummaryAgg`, possibly under a `SummaryEstimate`) with +/// `new_child` as the `SummaryAgg`'s child, if the result still validates +/// as maintained state; otherwise return `node` unchanged. +fn relink_agg_child(node: &Rc, new_child: &Rc) -> Rc { + match &node.operator { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, + query, + }) => { + let inner = relink_agg_child(summary_input, new_child); + if Rc::ptr_eq(&inner, summary_input) { + return Rc::clone(node); + } + std::rc::Rc::new( + OperatorNode::with_schema( + asap_types::ir::Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: inner, + query: query.clone(), + }), + node.schema.clone(), + ) + .with_guarantee(node.guarantee.clone()), + ) + } + Operator::ASAP(ASAPOp::SummaryAgg { + child, + family, + input, + reduction, + grouping, + filter, + }) => { + if Rc::ptr_eq(child, new_child) { + return Rc::clone(node); + } + // The same summary over a re-placed input keeps its coverage. + let rebuilt = std::rc::Rc::new(OperatorNode { + coverage: node.coverage.clone(), + ..OperatorNode::with_schema( + asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { + child: Rc::clone(new_child), + family: family.clone(), + input: input.clone(), + reduction: reduction.clone(), + grouping: grouping.clone(), + filter: filter.clone(), + }), + node.schema.clone(), + ) + .with_guarantee(node.guarantee.clone()) + }); + match validate_maintained(&rebuilt, ExecutionTiming::IngestionTime) { + Ok(_) => rebuilt, + Err(_) => Rc::clone(node), + } + } + _ => Rc::clone(node), + } +} + // ── default_strategies ────────────────────────────────────────────────── /// The context-free strategies [`search_workload`] runs with the built-in @@ -4395,11 +4836,11 @@ pub(crate) fn direct_child_counts(node: &OperatorNode) -> Vec<(*const OperatorNo /// cross-consumer accuracy reconciliation for CSE sharing — see that /// module's own docs) are added by [`search_workload`] after CSE and target /// discovery, when their sibling context exists. -/// [`crate::explanation::explain_replacements`] (issue #257) uses +/// [`crate::pass1::explanation::explain_replacements`] (issue #257) uses /// this same set (via [`search_workload`]) rather than keeping a second, /// explanation-specific list to stay in sync with. /// -/// [`AvgToSumOverCountStrategy`](crate::rewrite::AvgToSumOverCountStrategy) is +/// `AvgToSumOverCountStrategy` is /// included here (issue #253) even though it's a /// [`Replacement::Rewrite`]-only strategy — it's context-free (`matches`/`replacements` need nothing beyond /// the target itself) exactly like [`SharedSubDAGStrategy`], so it belongs @@ -4414,7 +4855,7 @@ pub fn default_strategies() -> Vec> { Box::new(ASAPStrategies::default()), Box::new(HydraGroupingStrategy::default()), Box::new(SharedSubDAGStrategy), - Box::new(crate::rewrite::AvgToSumOverCountStrategy), + Box::new(crate::pass1::rewrite::AvgToSumOverCountStrategy), Box::new(ExactCompositionStrategy), ] } @@ -4440,7 +4881,7 @@ pub fn default_strategies_with_evidence<'a>( ), ), Box::new(SharedSubDAGStrategy), - Box::new(crate::rewrite::AvgToSumOverCountStrategy), + Box::new(crate::pass1::rewrite::AvgToSumOverCountStrategy), Box::new(ExactCompositionStrategy), ] } @@ -4462,7 +4903,7 @@ pub fn search_workload(roots: Vec<(Id, Rc)>) -> CandidateLogic /// /// Runs [`share_common_sub_dags`] once over `roots` first — so every /// strategy (and, transitively, every -/// [`crate::explanation::ReplacementExplanation`] a caller reads off the +/// [`crate::pass1::explanation::ReplacementExplanation`] a caller reads off the /// result) sees the same already-deduplicated DAG — then discovers every /// `TargetSubDAG` (see [`discover_targets`]) and runs the /// fixpoint loop the module docs describe, capped at @@ -4484,7 +4925,7 @@ pub fn search_workload_with<'s, Id>( /// `accuracy_model`'s [`AccuracyModel::satisfies`]: a candidate whose /// guarantee is fully known and misses the target is moved from /// [`TargetSubDAGCandidates::candidates`] to [`TargetSubDAGCandidates::rejected`] *before* -/// [`CandidateLogicalASAPDAGs::cost_sorted`]/[`CandidateLogicalASAPDAGs::global_selection`] ever rank the +/// `candidate_selection::cost_sorted`/`candidate_selection::global_selection` ever rank the /// group. A constructible candidate with unknown accuracy remains visible for /// downstream review under an approximate target, but default whole-plan /// selection does not commit it. An exact target cannot accept an unknown @@ -4983,8 +5424,6 @@ fn walk_children( mod tests { use super::*; use crate::accuracy::PropagationStats; - use crate::cost_model::DefaultCostModel; - use crate::plan_selection::candidate_selection::sketch_kind_of; use crate::test_support::{agg, agg_per_entity, lower_promql, maintained, metric_scan, timed}; use asap_types::ir::operator::agg_intent::{ agg_is_exact, default_cardinality, default_quantile, MathFunc, TimeFunc, @@ -5091,17 +5530,7 @@ mod tests { ); } } - let selected = space - .global_selection(&DefaultCostModel) - .assemble_selected_query(&space.roots[0].1) - .unwrap() - .unwrap(); - for node in inventory - .candidates - .iter() - .map(|forest| &forest[0].1) - .chain(std::iter::once(&selected)) - { + for node in inventory.candidates.iter().map(|forest| &forest[0].1) { assert!( node.schema .fields @@ -5235,7 +5664,7 @@ mod tests { .expect("maintained average candidate"); assert!(operator.checked_finite_division); assert!( - crate::rewrite::SemanticEquivalentRewriteStrategy + crate::pass1::rewrite::SemanticEquivalentRewriteStrategy .replacements(&TargetSubDAG::new(&root)) .is_empty(), "an unconditional pre-ASAP rewrite would bypass the runtime guard" @@ -6122,7 +6551,9 @@ mod tests { .candidates .iter() .filter_map(|c| match &c.replacement { - Replacement::SubDAG(node) => sketch_kind_of(node), + Replacement::SubDAG(node) if node.contains_asap() => { + Some(summary_family_algorithm(node)) + } _ => None, }) .collect(); @@ -6334,13 +6765,6 @@ mod tests { .count(), 2 ); - let selected = space.global_selection(&DefaultCostModel); - assert!(!selected - .for_target(&space.roots[0].1) - .unwrap() - .chosen - .is_some_and(ReplacementSubDAG::has_missing_accuracy_evidence)); - let scan_group = space .target_subdag_candidates() .find(|g| matches!(g.target.non_asap(), Some(NonASAPOp::Scan { .. }))) @@ -6391,12 +6815,6 @@ mod tests { Replacement::SubDAG(node) if node.guarantee.is_none() && candidate.has_missing_accuracy_evidence() ))); - assert!(!targeted - .global_selection(&DefaultCostModel) - .for_target(target) - .unwrap() - .chosen - .is_some_and(ReplacementSubDAG::has_missing_accuracy_evidence)); let exact_target = search_workload_with_targets( vec![( @@ -6704,7 +7122,7 @@ mod tests { // Moved from the former `bind.rs` (issue #251): `bind.rs`'s own // workload-wide orchestration (`implement_workload`/ // `implement_workload_with`) was deleted. Current whole-workload logical - // selection uses `CandidateLogicalASAPDAGs::global_selection`; these tests exercise + // selection uses `candidate_selection::global_selection`; these tests exercise // `construct_summary_agg`'s schema derivation end to end through // `realize_child` — production logic that still lives in this module — // so they move here rather than disappear. Unlike `bind.rs` (an @@ -7718,49 +8136,6 @@ mod tests { })); } - #[test] - fn global_selection_can_choose_nested_summaries() { - // The same nested summary remains available through workload search - // and global cost ranking. - let inner = agg(vec![2], quantile_eps(0.5, 0.1), metric_scan(&["job"])); - let outer = agg(vec![], quantile_eps(0.99, 0.1), inner); - let strategies: Vec> = vec![Box::new( - ASAPStrategies::new_with_planning_inputs(&RankAdditiveModel, &EqualSplitAllocator), - )]; - let space = search_workload_with(vec![("q", Rc::clone(&outer))], &strategies); - let root = &space.roots[0].1; - let group = space.candidates_for_target(root).unwrap(); - assert!(!group.rejected.is_empty()); - assert!(group.candidates.iter().all(|c| match &c.replacement { - // A summary candidate (old `Replacement::Summary`) contains an - // ASAP node; a logical rewrite (old `Replacement::Rewrite`) does not. - Replacement::SubDAG(node) if node.contains_asap() => { - node.guarantee.as_ref().is_some_and(|g| { - DefaultAccuracyModel.satisfies(g, &AccuracyTarget::Epsilon(0.1)) - }) - } - Replacement::SubDAG(_) => false, - Replacement::ExactComposition(_) => false, - })); - let ranked = space.cost_sorted(&DefaultCostModel); - let root_ranked = ranked.iter().find(|g| Rc::ptr_eq(g.target, root)).unwrap(); - assert_eq!(root_ranked.candidates.len(), group.candidates.len()); - - let selection = space.global_selection(&DefaultCostModel); - let chosen = selection - .for_target(root) - .unwrap() - .chosen - .expect("a nested summary candidate wins"); - let Replacement::SubDAG(node) = &chosen.replacement else { - panic!() - }; - assert!(matches!( - node.operator, - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - )); - } - #[test] fn root_target_check_removes_candidates_before_cost_ranking() { let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); @@ -7783,8 +8158,6 @@ mod tests { AccuracyError::TargetNotSatisfied { target: AccuracyTarget::Epsilon(e), .. } if e == 0.001 ))); assert!(group.rejected.len() >= 2); - let selection = space.global_selection(&DefaultCostModel); - assert!(selection.for_target(root).unwrap().chosen.is_none()); // A root target the node's own sizing meets keeps every candidate. let space = search_workload_with_targets( @@ -7856,16 +8229,6 @@ mod tests { .guarantee .as_ref() .is_some_and(ResultGuarantee::has_unknown)); - let selected = space.global_selection(&DefaultCostModel); - assert!(!selected - .for_target(&space.roots[0].1) - .unwrap() - .chosen - .is_some_and(ReplacementSubDAG::has_missing_accuracy_evidence)); - assert!(selected - .assemble_selected_dag(&space.roots[0].1) - .unwrap() - .is_some()); } // Source evidence alone must enable Planner-owned sizing and certification. #[test] @@ -8080,4 +8443,77 @@ mod tests { .unwrap(); assert_eq!(whole_source_coverage(&join), None); } + + #[test] + fn relational_join_predicate_requires_and_normalizes_cross_input_columns() { + let forward = normalize_cross_input_equi_predicate(&equi_pred(1, 3), 2, 4) + .expect("left-to-right equality"); + let reverse = normalize_cross_input_equi_predicate(&equi_pred(3, 1), 2, 4) + .expect("right-to-left equality"); + assert_eq!(forward, reverse, "reverse equality must be canonicalized"); + assert!(normalize_cross_input_equi_predicate(&equi_pred(0, 1), 2, 4).is_none()); + assert!(normalize_cross_input_equi_predicate(&equi_pred(0, 4), 2, 4).is_none()); + } + + // A value projection cannot consume an opaque exact accumulator edge. + #[test] + fn residual_projection_finalizes_selected_exact_state() { + let inner = agg(vec![], AggIntent::Sum { col: None }, metric_scan(&[])); + let root = + OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + expr: ScalarExpr::Column(0), + alias: Some("result".into()), + }], + qualifier: None, + child: inner.clone(), + })) + .unwrap(); + let space = search_workload_with_targets( + vec![("q", root.clone(), Some(AccuracyTarget::Exact))], + &default_strategies(), + &DefaultAccuracyModel, + ); + // No site has a chosen candidate, so the root is assembled as a residual. + let groups = space + .target_subdag_candidates() + .map(|group| { + ( + Rc::as_ptr(&group.target), + TargetSubDAGSelection { + target: &group.target, + consumer_count: group.consumer_count, + effective_consumer_count: group.consumer_count, + chosen: None, + }, + ) + }) + .collect(); + let selected = GlobalSelection::new(space.order.clone(), groups, HashMap::new()); + // CSE re-interns the workload, so the space's root/child `Rc`s are not + // the fixture's. Assembly only assembles children that are discovered + // targets, so seed the memo under the space's own child pointer. + let root = Rc::clone(&space.roots[0].1); + let Some(NonASAPOp::Project { child: inner, .. }) = root.non_asap() else { + unreachable!() + }; + assert!(space.candidates_for_target(inner).is_some()); + selected + .assembled_nodes + .borrow_mut() + .insert(Rc::as_ptr(inner), realize(inner.as_ref()).unwrap()); + let node = selected.assemble_target(&root).unwrap(); + let Operator::NonASAP(NonASAPOp::Project { child, .. }) = &node.operator else { + panic!("expected Project"); + }; + assert!(matches!( + child.operator, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) + )); + assert!(child + .schema + .fields + .iter() + .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); + } } diff --git a/crates/asap-aware-mapping/src/rewrite.rs b/crates/logical-optimizer/src/pass1/rewrite.rs similarity index 98% rename from crates/asap-aware-mapping/src/rewrite.rs rename to crates/logical-optimizer/src/pass1/rewrite.rs index 621a6a70..d8a1afaa 100644 --- a/crates/asap-aware-mapping/src/rewrite.rs +++ b/crates/logical-optimizer/src/pass1/rewrite.rs @@ -68,7 +68,9 @@ use asap_types::ir::{BinaryOperator, NonASAPOp, OperatorNode, ProjectItem, Scala use asap_types::types::AccuracyTarget; -use crate::replacement::{Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG}; +use crate::pass1::replacement::{ + Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, +}; /// The shape [`AvgToSumOverCountStrategy`] rewrites: a single `Avg{col}` /// measure, no `HAVING`, grouped with an ordinary `by(...)` reduction (see @@ -385,7 +387,7 @@ pub(crate) fn composed_aggregate_rewrite(root: &Rc) -> Option= 2` to propose //! sharing for. This crate would build two entirely independent sketches //! for what is conceptually one computation, even though a single sketch @@ -22,7 +22,7 @@ //! consumer that pattern-matches on a specific `AccuracyTarget` — depends on //! that). What this module adds is a *second*, narrower notion of "close //! enough to share" that sits entirely inside the [`ReplacementStrategy`] -//! extension point: one more candidate a [`crate::cost_model::CostModel`] +//! extension point: one more candidate a `cost_model::CostModel` //! may or may not prefer, never a forced rewrite and never a change to what //! `share_common_sub_dags` itself merges. //! @@ -31,17 +31,17 @@ //! Two `NonASAPOp::Aggregate` nodes are accuracy-near-duplicates here iff, //! **in this order**: //! -//! 1. Both are the same bindable shape [`crate::replacement::ASAPStrategies`] +//! 1. Both are the same bindable shape [`crate::pass1::replacement::ASAPStrategies`] //! itself targets — a single measure, no `HAVING` (`bindable_intent`'s own //! scope) — **and** that one measure is one of the four accuracy-bearing -//! [`AggIntent`] variants ([`crate::replacement::accuracy_target`]'s own +//! [`AggIntent`] variants ([`crate::pass1::replacement::accuracy_target`]'s own //! scope: `Count` / `Quantile` / `Cardinality` / `TopK`). Every other //! intent has no `AccuracyTarget` to reconcile in the first place. //! 2. Same `reduction` (grouping), same `output_names`, and the same shared //! `child` (`Rc::ptr_eq`, or value-equal for two independently-built but //! identical sub-DAGs CSE conservatively declined to alias) — the same -//! "identical everything else" bar [`crate::rollup::RollupStrategy`] and -//! [`crate::topk_reuse::TopKLimitReuseStrategy`] already hold their own +//! "identical everything else" bar [`crate::pass1::rollup::RollupStrategy`] and +//! [`crate::pass2::topk_reuse::TopKLimitReuseStrategy`] already hold their own //! sibling-reuse candidates to. //! 3. The one measure is identical **except** for `accuracy` — same variant, //! same `col`/`q`/`k` (see [`same_intent_except_accuracy`]). @@ -51,7 +51,7 @@ //! 5. The tighter candidate's own **output** schema carries a provable //! unique key (`Schema::has_unique_key`) — the exact legality gate //! `share_common_sub_dags` itself applies (see `cse.rs`'s "Legality" -//! section) and [`crate::rollup::RollupStrategy::is_legal_rollup_source`] +//! section) and [`crate::pass1::rollup::RollupStrategy::is_legal_rollup_source`] //! already reuses verbatim for the identical reason: a producer's output //! is only safely reusable across a second, independent consumer when //! its row identity is provably stable across reads. A global or @@ -61,7 +61,7 @@ //! //! ## Safety of tightening: why reading the tighter build is always sound //! -//! [`crate::replacement::accuracy_budget`] resolves *every* `AccuracyTarget` +//! [`crate::pass1::replacement::accuracy_budget`] resolves *every* `AccuracyTarget` //! (`Epsilon`/`EpsilonDelta`) to the literal `(eps, delta)` pair //! `realizations_for_intent`'s `sketch_realizations` feeds into the //! analytical sizing — the same numbers `default_size_params`' @@ -103,20 +103,20 @@ //! //! Like every [`ReplacementStrategy`], this only ever *proposes* — the //! looser-accuracy consumer's own independently-sized candidate (from -//! [`crate::replacement::ASAPStrategies`]) stays in its -//! [`crate::replacement::TargetSubDAGCandidates`] right alongside this strategy's +//! [`crate::pass1::replacement::ASAPStrategies`]) stays in its +//! [`crate::pass1::replacement::TargetSubDAGCandidates`] right alongside this strategy's //! "read the tighter sibling instead" [`Replacement::Rewrite`] candidate; -//! [`crate::cost_model::CostModel`]-driven ranking picks between them; +//! `cost_model::CostModel`-driven ranking picks between them; //! nothing here removes or filters the independent candidate. //! //! ## Costing this candidate shape: a dedicated arm, not a reused one //! //! This strategy's candidates carry their own -//! [`crate::replacement::ReplacementProvenance::AccuracyReconciliation`] +//! [`crate::pass1::replacement::ReplacementProvenance::AccuracyReconciliation`] //! rather than reusing `LogicalRewrite` -//! ([`crate::rollup::RollupStrategy`]/[`crate::topk_reuse::TopKLimitReuseStrategy`]'s +//! ([`crate::pass1::rollup::RollupStrategy`]/[`crate::pass2::topk_reuse::TopKLimitReuseStrategy`]'s //! tag), because it needs its own cost treatment in -//! [`crate::cost_model::DefaultCostModel::estimate_cost`], not just its own +//! `cost_model::DefaultCostModel::estimate_cost`, not just its own //! label. Every other `Replacement::Rewrite` shape that reaches //! `estimate_cost` (`SharedSubDAGStrategy`'s `CseRecompute`, `Rollup`'s and //! `TopKLimitReuse`'s `LogicalRewrite`) really does rebuild `target` from a @@ -133,13 +133,13 @@ //! the literal inversion this module's tests //! (`estimate_cost_does_not_scale_with_the_readers_own_consumer_count`) //! pin against. `estimate_cost` instead prices this shape as a -//! [`crate::cost_model::CostModel::cse_shared_maintenance_cost`] read +//! `cost_model::CostModel::cse_shared_maintenance_cost` read //! against `rc`'s **own** bound summary — the same order-of-magnitude, //! per-family cost `SharedSubDAGStrategy`'s own `CseShare` candidate is //! priced with, reflecting "one more reference into a structure that's //! already being maintained" rather than "build a whole new one." //! -//! `CandidateLogicalASAPDAGs::global_selection` treats this rewrite as a cross-group edge: +//! `candidate_selection::global_selection` treats this rewrite as a cross-group edge: //! selecting it increments `rc`'s own `effective_consumer_count`, then lets //! that sibling group propagate the uses through its selected implementation. //! Accuracy edges are directed strictly from looser to tighter budgets, so @@ -156,7 +156,7 @@ use asap_types::ir::operator::operator_properties::Reduction; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::types::AccuracyTarget; -use crate::replacement::{ +use crate::pass1::replacement::{ accuracy_budget, accuracy_target, Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; @@ -174,10 +174,10 @@ type BindableAccuracyAggregate<'a> = ( /// The `(reduction, intent, accuracy, output_names, child)` shape this /// module operates on: the same single-measure, no-`HAVING` bindable shape -/// [`crate::replacement::ASAPStrategies`] targets (see that +/// [`crate::pass1::replacement::ASAPStrategies`] targets (see that /// module's private `bindable_intent`), further narrowed to a measure whose /// intent actually carries an [`AccuracyTarget`] -/// ([`crate::replacement::accuracy_target`]'s own scope: `Count` / +/// ([`crate::pass1::replacement::accuracy_target`]'s own scope: `Count` / /// `Quantile` / `Cardinality` / `TopK`). `None` for anything else, including /// a multi-measure or `HAVING` aggregate, a non-`Aggregate` node, or an /// accuracy-free intent (`Sum`, `Avg`, …). @@ -207,7 +207,7 @@ fn bindable_accuracy_aggregate(node: &OperatorNode) -> Option bool { match (a, b) { @@ -268,13 +268,13 @@ fn strictly_tighter(a: &AccuracyTarget, b: &AccuracyTarget) -> bool { /// tighter accuracy, and proposes reading that sibling's own (to-be-built) /// result instead of building an independent, looser copy — "build once at /// the tightest of the group's accuracy requirements, all consumers read -/// from it," ranked by [`crate::cost_model::CostModel`] like any other +/// from it," ranked by `cost_model::CostModel` like any other /// candidate, never forced. See the module docs for the full design. /// /// `siblings` is **caller-supplied, not discovered here** — the identical /// "workload-wide discovery isn't this strategy's job" split -/// [`crate::rollup::RollupStrategy`] and [`crate::topk_reuse::TopKLimitReuseStrategy`] -/// already draw; [`crate::replacement::search_workload_with`] constructs +/// [`crate::pass1::rollup::RollupStrategy`] and [`crate::pass2::topk_reuse::TopKLimitReuseStrategy`] +/// already draw; [`crate::pass1::replacement::search_workload_with`] constructs /// this strategy from the same post-CSE `Aggregate` sibling set it already /// builds for `RollupStrategy`. pub struct AccuracyReconciliationStrategy { @@ -302,7 +302,7 @@ impl AccuracyReconciliationStrategy { /// Also requires the candidate's own *output* schema to carry a provable /// unique key ([`Schema::has_unique_key`]) — the exact legality gate /// `ir::cse::share_common_sub_dags` already applies to its own - /// sharing decisions, and [`crate::rollup::RollupStrategy`] already + /// sharing decisions, and [`crate::pass1::rollup::RollupStrategy`] already /// reuses verbatim for the identical reason (see that module's /// `is_legal_rollup_source` doc, point 4): a producer's output is only /// safely reusable across a second, independent consumer when its row @@ -388,10 +388,8 @@ impl ReplacementStrategy for AccuracyReconciliationStrategy { #[cfg(test)] mod tests { use super::*; - use crate::cost_model::{CostModel, DefaultCostModel}; use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::operator::operator_properties::{GroupKeys, Source}; - use asap_types::ir::schema::SketchAlgorithm; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; /// `[ts(0), value(1), job(2)]`. @@ -532,7 +530,7 @@ mod tests { let epsilon_only = AccuracyTarget::Epsilon(0.01); let equivalent_epsilon_delta = AccuracyTarget::EpsilonDelta { epsilon: 0.01, - delta: crate::replacement::DEFAULT_DELTA, + delta: crate::pass1::replacement::DEFAULT_DELTA, }; assert!(dominates(&epsilon_only, &equivalent_epsilon_delta)); assert!(dominates(&equivalent_epsilon_delta, &epsilon_only)); @@ -576,7 +574,8 @@ mod tests { let tight = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); let loose = quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); - let space = crate::replacement::search_workload(vec![("tight", tight), ("loose", loose)]); + let space = + crate::pass1::replacement::search_workload(vec![("tight", tight), ("loose", loose)]); let loose_root = &space.roots[1].1; let loose_group = space @@ -754,229 +753,4 @@ mod tests { assert!(!strategy.matches(&TargetSubDAG::new(&loose))); assert!(strategy.replacements(&TargetSubDAG::new(&loose)).is_empty()); } - - // ── cost: reading the sibling must not be priced like recomputing - // `target` independently per consumer ──────────────────────────────── - - #[test] - fn estimate_cost_does_not_scale_with_the_readers_own_consumer_count() { - // Regression guard for the review-reported sign inversion: pricing - // this candidate like `CseRecompute` ("rebuild `target`, once per - // consumer") made it artificially *more* expensive exactly as more - // of `target`'s own consumers stood to benefit from reading the - // already-necessary tighter sibling instead — the literal opposite - // of the intended incentive. The real cost is "one more read against - // `rc`'s own build," which must not scale with `target`'s own - // `consumer_count`. - let scan = metric_scan(); - let tight = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); - let loose = quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); - let strategy = AccuracyReconciliationStrategy::new(&[Rc::clone(&tight), Rc::clone(&loose)]); - let candidate = strategy - .replacements(&TargetSubDAG::new(&loose)) - .into_iter() - .next() - .expect("loose has a reconciliation candidate reading the tight sibling"); - - let cost_model = DefaultCostModel; - let single_consumer = TargetSubDAG::with_consumer_count(&loose, 1); - let many_consumers = TargetSubDAG::with_consumer_count(&loose, 5); - - let cost_single = cost_model.estimate_cost(&candidate, &single_consumer); - let cost_many = cost_model.estimate_cost(&candidate, &many_consumers); - - assert!( - cost_single.is_finite(), - "expected a real cost, not the NaN placeholder: {cost_single}" - ); - assert_eq!( - cost_single, cost_many, - "AccuracyReconciliation's estimate_cost must price 'read the sibling', not scale \ - with the reader's own consumer_count the way CseRecompute's 'rebuild independently \ - per consumer' formula does (single-consumer: {cost_single}, 5 consumers: \ - {cost_many})" - ); - } - - // ── cost_sorted / global_selection: single-consumer and shared-consumer ─ - - #[test] - fn cost_sorted_and_global_selection_handle_a_single_consumer_looser_target() { - let scan = metric_scan(); - let tight = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); - let loose = quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); - let space = crate::replacement::search_workload(vec![("tight", tight), ("loose", loose)]); - let loose_root = &space.roots[1].1; - - let cost_model = DefaultCostModel; - let ranked = space.cost_sorted(&cost_model); - let loose_ranked = ranked - .iter() - .find(|group| Rc::ptr_eq(group.target, loose_root)) - .expect("loose has its own ranked group"); - assert!( - loose_ranked.costs.iter().all(|cost| cost.is_finite()), - "no candidate should cost NaN under DefaultCostModel: {:?}", - loose_ranked.costs - ); - assert!( - loose_ranked - .candidates - .iter() - .any(|c| c.strategy == "AccuracyReconciliationStrategy"), - "the reconciliation candidate must still be present, ranked, not filtered" - ); - - let selected = space.global_selection(&cost_model); - let chosen = selected - .for_target(loose_root) - .and_then(|group| group.chosen); - assert!( - chosen.is_some(), - "global_selection must commit to some candidate for a single-consumer looser target" - ); - // With no recompute term at all (it never rebuilds `target`), this - // candidate strictly undercuts every ASAPStrategies - // candidate (which each pay a recompute term on top of their own - // maintenance term) under DefaultCostModel's numbers — the sane - // direction: reading an already-necessary sibling should be able to - // win on its own merit, not just fail to lose as badly as before. - assert_eq!( - chosen.map(|c| c.provenance), - Some(crate::replacement::ReplacementProvenance::AccuracyReconciliation) - ); - } - - #[test] - fn cost_sorted_and_global_selection_handle_a_shared_looser_target() { - // The loose accuracy target itself has 2 direct consumers (two - // independently-built but structurally identical loose queries - // merge onto one Rc via ordinary CSE), *and* a separate, - // single-consumer tight sibling exists over the same input — the - // scenario the issue itself targets: `SharedSubDAGStrategy`'s own - // CseShare/CseRecompute pair is on the table for the loose target's - // own 2 consumers at the same time as this strategy's "read the - // tight sibling instead" candidate. - let scan = metric_scan(); - let loose_a = (*quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan)).clone(); - let loose_b = (*quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan)).clone(); - let tight = (*quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan)).clone(); - - let space = crate::replacement::search_workload(vec![ - ("loose_a", Rc::new(loose_a)), - ("loose_b", Rc::new(loose_b)), - ("tight", Rc::new(tight)), - ]); - - // Fixture sanity: the two loose roots really did merge onto one Rc. - assert!(Rc::ptr_eq(&space.roots[0].1, &space.roots[1].1)); - let loose_group = space - .candidates_for_target(&space.roots[0].1) - .expect("the merged loose target has a group"); - assert_eq!(loose_group.consumer_count, 2); - assert!( - loose_group - .candidates - .iter() - .any(|c| c.strategy == "AccuracyReconciliationStrategy"), - "the reconciliation candidate must still be proposed alongside the CSE share/recompute \ - pair, not crowded out: {:?}", - loose_group - .candidates - .iter() - .map(|c| (c.strategy, c.provenance)) - .collect::>() - ); - - let cost_model = DefaultCostModel; - let ranked = space.cost_sorted(&cost_model); - let loose_ranked = ranked - .iter() - .find(|group| Rc::ptr_eq(group.target, &space.roots[0].1)) - .expect("loose has its own ranked group"); - assert!( - loose_ranked.costs.iter().all(|cost| cost.is_finite()), - "no candidate should cost NaN under DefaultCostModel, shared or not: {:?}", - loose_ranked.costs - ); - - let selected = space.global_selection(&cost_model); - let chosen = selected - .for_target(&space.roots[0].1) - .and_then(|group| group.chosen); - assert!( - chosen.is_some(), - "global_selection must commit to some candidate for the shared looser target" - ); - // Under `DefaultCostModel`'s numbers, `CseShare` (flat maintenance, - // no recompute term) and this strategy's own candidate (also a - // flat, non-scaling read cost after the fix) land tied, and - // `global_selection` breaks ties in `CseShare`'s favor (it only ever - // overrides the CSE choice on a *strict* `<`, not `<=`) — a sane, - // deliberate tie-break, not the "reconciliation always loses to - // CseShare regardless of its own real merit" bug this test guards - // against (see `estimate_cost_does_not_scale_with_the_readers_own_consumer_count` - // for the direct regression check that the old `* consumer_count` - // scaling — which made this an unfair, ever-widening loss instead - // of a tie — is gone). - assert_eq!( - chosen.map(|c| c.provenance), - Some(crate::replacement::ReplacementProvenance::CseShare) - ); - } - - #[test] - fn global_selection_propagates_reconciled_consumers_to_the_tighter_group() { - struct PreferReconciliation; - - impl CostModel for PreferReconciliation { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - - fn estimate_cost( - &self, - candidate: &ReplacementSubDAG, - _target: &TargetSubDAG<'_>, - ) -> f64 { - if candidate.provenance == ReplacementProvenance::AccuracyReconciliation { - 0.0 - } else { - 100.0 - } - } - } - - let scan = metric_scan(); - let tight = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); - let loose = quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); - let space = crate::replacement::search_workload(vec![ - ("tight", Rc::clone(&tight)), - ("loose", Rc::clone(&loose)), - ]); - - let selected = space.global_selection(&PreferReconciliation); - let tight_root = &space.roots[0].1; - let loose_root = &space.roots[1].1; - assert_eq!( - selected - .for_target(loose_root) - .and_then(|group| group.chosen) - .map(|candidate| candidate.provenance), - Some(ReplacementProvenance::AccuracyReconciliation), - "fixture must select the cross-sibling rewrite" - ); - assert_eq!( - selected - .for_target(tight_root) - .expect("the tighter sibling is a discovered memo group") - .effective_consumer_count, - 2, - "the tighter build serves its original root and the reconciled looser root" - ); - } } diff --git a/crates/asap-aware-mapping/src/topk_reuse.rs b/crates/logical-optimizer/src/pass2/topk_reuse.rs similarity index 99% rename from crates/asap-aware-mapping/src/topk_reuse.rs rename to crates/logical-optimizer/src/pass2/topk_reuse.rs index 1a698af2..f58c35d7 100644 --- a/crates/asap-aware-mapping/src/topk_reuse.rs +++ b/crates/logical-optimizer/src/pass2/topk_reuse.rs @@ -9,7 +9,7 @@ use std::rc::Rc; use asap_types::ir::{NonASAPOp, OperatorNode}; -use crate::replacement::{ +use crate::pass1::replacement::{ Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; diff --git a/crates/logical-optimizer/src/test_support.rs b/crates/logical-optimizer/src/test_support.rs new file mode 100644 index 00000000..368b5234 --- /dev/null +++ b/crates/logical-optimizer/src/test_support.rs @@ -0,0 +1,213 @@ +// Shared fixture helpers; not every test module uses every helper. +#![allow(dead_code)] + +use std::rc::Rc; + +use asap_types::ir::OperatorNode; +use asap_types::types::AccuracyTarget; +use asap_types::workload::{ + AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, PlanningWorkload, + Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, TimeSelection, +}; + +pub(crate) fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc { + let workload = PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some(vec![BatchEntry { + query: Query(query.into()), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit(accuracy), + ..Default::default() + }, + predictability: Predictability::Unknown, + invocations: 1, + execute_at: None, + time_selection: TimeSelection::default(), + }]), + repeating_queries: None, + }, + data_workload: Some(DataWorkload { + data_ingestion_interval: Evidence { + value: Some(DurationMs(1_000)), + ..Default::default() + }, + ..Default::default() + }), + }; + asap_frontend_promql::lower_promql_workload(&workload, 0) + .unwrap() + .pop() + .unwrap() +} + +// ── Shared pre-ASAP fixture builders ───────────────────────────────────── +// +// Every builder returns an `Rc` whose schema is derived by +// `OperatorNode::new_shared`, so a fixture is exactly what a front end +// would hand the planner. Added by the test migration; only add here, never +// rename or remove (several test modules share these). + +use std::time::Duration; + +use asap_types::ir::operator::agg_intent::AggIntent; +use asap_types::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; +use asap_types::ir::properties::timing::{ + apply_materialization_timings, MaterializationAssignment, TimingMemo, +}; +use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; +use asap_types::ir::{NonASAPOp, Predicate, ScalarExpr, TimeRangeKind}; + +/// A `TimeSeries("m")` scan over `[ts(0), value(1), labels...]`, time index 0, +/// no unique key. +pub(crate) fn metric_scan(labels: &[&str]) -> Rc { + metric_scan_with_keys(labels, vec![]) +} + +/// [`metric_scan`] with explicit `unique_keys` (a `[[0]]` key makes CSE +/// willing to hoist the scan). +pub(crate) fn metric_scan_with_keys( + labels: &[&str], + unique_keys: Vec>, +) -> Rc { + let mut columns = vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ]; + columns.extend( + labels + .iter() + .map(|n| Field::plain(*n, DataType::Utf8, true)), + ); + scan("m", Schema::with_time_index(columns, 0, unique_keys)) +} + +/// A predicate-free `TimeSeries(metric)` scan with the given schema. +pub(crate) fn scan(metric: &str, schema: Schema) -> Rc { + scan_from( + Source::TimeSeries { + metric: metric.into(), + }, + schema, + ) +} + +pub(crate) fn scan_from(source: Source, schema: Schema) -> Rc { + OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { + source, + predicates: vec![], + schema, + })) + .unwrap() +} + +/// A general aggregate node. +pub(crate) fn aggregate( + reduction: Reduction, + measures: Vec, + output_names: Vec, + having: Option, + child: Rc, +) -> Rc { + OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { + reduction, + measures, + output_names, + filters: vec![], + having, + child, + })) + .unwrap() +} + +/// `intent by (by)` — a single-measure, `HAVING`-free grouped aggregate. +pub(crate) fn agg( + by: Vec, + intent: AggIntent, + child: Rc, +) -> Rc { + aggregate(Reduction::by(by), vec![intent], vec![], None, child) +} + +/// `intent without (excluded)`. +pub(crate) fn without_agg( + excluded: Vec, + intent: AggIntent, + child: Rc, +) -> Rc { + aggregate( + Reduction::Reduce(GroupKeys::without(excluded)), + vec![intent], + vec![], + None, + child, + ) +} + +/// A per-entity (per-series) single-measure aggregate. +pub(crate) fn agg_per_entity(intent: AggIntent, child: Rc) -> Rc { + aggregate(Reduction::PerEntity, vec![intent], vec![], None, child) +} + +pub(crate) fn filter(pred: ScalarExpr, child: Rc) -> Rc { + OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(pred), + child, + })) + .unwrap() +} + +pub(crate) fn dedup(cols: Vec, child: Rc) -> Rc { + OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Dedup { + cols, + child, + })) + .unwrap() +} + +/// An explicit range selector `child[range]`. +pub(crate) fn time_range(range: Duration, child: Rc) -> Rc { + OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::TimeRange { + range, + kind: TimeRangeKind::Range, + child, + })) + .unwrap() +} + +/// `root` timed under the default (every summary at query time) +/// assignment — the shape export and the post-ASAP validators consume. +pub(crate) fn timed(root: &Rc) -> Rc { + apply_materialization_timings( + root, + &MaterializationAssignment::all_query_time(), + &mut TimingMemo::new(), + ) + .expect("default materialization timings apply") +} + +/// `root` timed with every summary maintained at ingestion time. +pub(crate) fn maintained(root: &Rc) -> Rc { + apply_materialization_timings( + root, + &MaterializationAssignment::all_ingestion_time(), + &mut TimingMemo::new(), + ) + .expect("maintained materialization timings apply") +} + +/// Time `root` under the default materialization assignment (which runs every +/// data-state / population-contract check) and export it as a physical ASAP DAG. +pub(crate) fn time_and_export( + root: &Rc, +) -> Result< + asap_types::ir::export::PhysicalASAPDAG, + asap_types::ir::properties::execution::ExecutionDataStateError, +> { + let timed = apply_materialization_timings( + root, + &MaterializationAssignment::all_query_time(), + &mut TimingMemo::new(), + )?; + asap_types::ir::export::compile_physical_asap_dag(&timed) +} diff --git a/crates/asap-aware-mapping/tests/logical_candidates.rs b/crates/logical-optimizer/tests/logical_candidates.rs similarity index 96% rename from crates/asap-aware-mapping/tests/logical_candidates.rs rename to crates/logical-optimizer/tests/logical_candidates.rs index af93507a..db2f0bc9 100644 --- a/crates/asap-aware-mapping/tests/logical_candidates.rs +++ b/crates/logical-optimizer/tests/logical_candidates.rs @@ -1,9 +1,8 @@ //! Frontend-to-Pass-1 acceptance: candidate discovery precedes empirical selection. -use asap_aware_mapping::{ - logical_candidates::{ - enumerate_local_logical_candidates, local_realizations_for_intent, LogicalCandidateError, - }, - Realization, +use asap_logical_optimizer::{ + pass1::logical_candidates::enumerate_local_logical_candidates, + pass1::logical_candidates::local_realizations_for_intent, + pass1::logical_candidates::LogicalCandidateError, Realization, }; use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::operator::AggIntent; @@ -244,7 +243,7 @@ fn topk_keeps_both_specialized_heap_choices() { /// coverage on the summary, and keeps unchosen plans identical. #[test] fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { - use asap_aware_mapping::logical_candidates::compose_logical_candidate; + use asap_logical_optimizer::pass1::logical_candidates::compose_logical_candidate; use asap_types::ir::ASAPOp; let producer = aggregate(AggIntent::Cardinality { cols: vec![0], diff --git a/crates/logical-optimizer/tests/stage1_cost_independence.rs b/crates/logical-optimizer/tests/stage1_cost_independence.rs new file mode 100644 index 00000000..a7d9bf96 --- /dev/null +++ b/crates/logical-optimizer/tests/stage1_cost_independence.rs @@ -0,0 +1,107 @@ +//! Stage 1 (logical candidate generation) stays independent of the cost +//! model: only Stage 3 prices plans (#572, decision Q36(a)). + +use std::path::{Path, PathBuf}; + +/// Module paths Stage 1 production code must not name. +const FORBIDDEN_MODULES: &[&str] = &["cost_model", "recurrence"]; + +/// Crates Stage 1 must not depend on: later stages, the facade and the executor. +const FORBIDDEN_CRATES: &[&str] = &["asap-aware-mapping", "asap-physical-operators"]; + +/// Every `.rs` file under `dir`. +fn rust_files(dir: &Path) -> Vec { + let mut files = Vec::new(); + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + files.extend(rust_files(&path)); + } else if path.extension().is_some_and(|ext| ext == "rs") { + files.push(path); + } + } + files.sort(); + files +} + +/// The source lines outside `#[cfg(test)]` items and comments, numbered. +fn production_lines(source: &str) -> Vec<(usize, &str)> { + let mut lines = Vec::new(); + let mut skip_next_item = false; + let mut depth = 0i64; + for (index, line) in source.lines().enumerate() { + let trimmed = line.trim(); + if depth > 0 { + depth += brace_balance(line); + continue; + } + if trimmed == "#[cfg(test)]" { + skip_next_item = true; + continue; + } + if skip_next_item { + if trimmed.starts_with("#[") || trimmed.is_empty() { + continue; + } + skip_next_item = false; + depth = brace_balance(line); + continue; + } + if !trimmed.starts_with("//") { + lines.push((index + 1, line)); + } + } + lines +} + +fn brace_balance(line: &str) -> i64 { + line.chars() + .map(|c| match c { + '{' => 1, + '}' => -1, + _ => 0, + }) + .sum() +} + +/// Stage 1 production code names neither `cost_model` nor `recurrence`. +#[test] +fn stage1_does_not_import_cost_model_or_recurrence() { + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let mut offenders = Vec::new(); + for file in rust_files(&src) { + let source = std::fs::read_to_string(&file).unwrap(); + for (number, line) in production_lines(&source) { + if FORBIDDEN_MODULES + .iter() + .any(|module| line.contains(&format!("{module}::"))) + { + let file = file.strip_prefix(&src).unwrap().display(); + offenders.push(format!("{file}:{number}: {}", line.trim())); + } + } + } + assert!( + offenders.is_empty(), + "Stage 1 must not depend on the cost model:\n{}", + offenders.join("\n") + ); +} + +/// The manifest names no later stage, facade or executor crate, so Cargo +/// rejects any import of them. +#[test] +fn stage1_manifest_has_no_path_back_to_later_stages() { + let manifest = + std::fs::read_to_string(Path::new(env!("CARGO_MANIFEST_DIR")).join("Cargo.toml")).unwrap(); + let offenders: Vec<&str> = manifest + .lines() + .filter(|line| !line.trim_start().starts_with('#')) + .filter(|line| FORBIDDEN_CRATES.iter().any(|name| line.contains(name))) + .collect(); + assert!( + offenders.is_empty(), + "asap-logical-optimizer must not depend on a later stage:\n{}", + offenders.join("\n") + ); +} diff --git a/crates/planner/Cargo.toml b/crates/planner/Cargo.toml index 59825a3f..0a73eecc 100644 --- a/crates/planner/Cargo.toml +++ b/crates/planner/Cargo.toml @@ -21,3 +21,4 @@ thiserror = "2" # The SQL frontend plans through DataFusion, which is async; `e2e_plan` is # therefore async and its tests need a runtime. tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } +asap-logical-optimizer = { path = "../logical-optimizer" } diff --git a/crates/planner/tests/e2e_plan.rs b/crates/planner/tests/e2e_plan.rs index 0c84d381..57c1eb4b 100644 --- a/crates/planner/tests/e2e_plan.rs +++ b/crates/planner/tests/e2e_plan.rs @@ -3,12 +3,12 @@ use std::rc::Rc; -use asap_aware_mapping::logical_candidates::enumerate_local_logical_candidates; use asap_aware_mapping::pass::{ OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, PlanningModels, }; use asap_aware_mapping::plan_selection::{select_exhaustive, MAX_ENUMERATED_CANDIDATES}; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; +use asap_logical_optimizer::pass1::logical_candidates::enumerate_local_logical_candidates; use asap_planner::{e2e_plan, FrontendInput, PlanError, UserInput, UserInputError}; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::QueryRoot; diff --git a/crates/planner/tests/stage_pipeline_selection.rs b/crates/planner/tests/stage_pipeline_selection.rs index ac1d2d70..fa76496b 100644 --- a/crates/planner/tests/stage_pipeline_selection.rs +++ b/crates/planner/tests/stage_pipeline_selection.rs @@ -3,14 +3,14 @@ //! the combination it picks is the one that building and pricing every //! combination picks. -use asap_aware_mapping::logical_candidates::{ - enumerate_local_logical_candidates, LocalLogicalCandidates, -}; use asap_aware_mapping::pass::PlanningModels; use asap_aware_mapping::plan_selection::{ select_exhaustive, select_plan, SelectionMethod, MAX_ENUMERATED_CANDIDATES, }; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; +use asap_logical_optimizer::pass1::logical_candidates::{ + enumerate_local_logical_candidates, LocalLogicalCandidates, +}; use asap_planner::{e2e_plan, FrontendInput, UserInput}; use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::schema::{DataType, Field, Schema}; diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index fd3657d4..b53c9178 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -5,17 +5,19 @@ use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::{ASAPOp, OperatorNode}; use std::rc::Rc; -use asap_aware_mapping::accuracy::{ - AccuracyModel, DefaultAccuracyModel, EqualSplitAllocator, PropagationStats, -}; use asap_aware_mapping::pass::{PlanOutput, PlanningModels, QueryPlan}; -use asap_aware_mapping::replacement::{default_size_params, DEFAULT_DELTA}; -use asap_aware_mapping::{ - search_workload_with_targets, ASAPStrategies, CostModel, DefaultCostModel, Replacement, - ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, -}; +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; +use asap_aware_mapping::{CostModel, DefaultCostModel}; use asap_frontend_promql::lower_promql_workload; use asap_frontend_sql::SqlCatalog; +use asap_logical_optimizer::accuracy::{ + AccuracyModel, DefaultAccuracyModel, EqualSplitAllocator, PropagationStats, +}; +use asap_logical_optimizer::pass1::replacement::{default_size_params, DEFAULT_DELTA}; +use asap_logical_optimizer::{ + search_workload_with_targets, ASAPStrategies, Replacement, ReplacementStrategy, + ReplacementSubDAG, TargetSubDAG, +}; use asap_planner::{e2e_plan, FrontendInput, UserInput}; use asap_types::ir::operator::agg_intent::default_quantile; use asap_types::ir::operator::AggIntent; @@ -477,7 +479,7 @@ fn certified_frequency_evaluations_share_one_univmon_state() { ASAPStrategies::new_with_planning_inputs(&UnivMonEvidence, &EqualSplitAllocator), )]; let space = search_workload_with_targets(roots, &strategies, &UnivMonEvidence); - let selection = space.global_selection(&PREFER_UNIVMON); + let selection = global_selection(&space, &PREFER_UNIVMON); let assembled = space .roots .iter() diff --git a/crates/types/src/cost.rs b/crates/types/src/cost.rs index 660d3bb7..1bab8ea1 100644 --- a/crates/types/src/cost.rs +++ b/crates/types/src/cost.rs @@ -73,7 +73,7 @@ pub enum BaselineRef { /// — "do nothing" (never apply ASAP-aware replacement at all). PreAsapRecomputation, /// The best-ranked *non-selected* legal candidate for the same target - /// (`rank` into that target's own `CandidateLogicalASAPDAGs::cost_sorted` ordering, + /// (`rank` into that target's own `candidate_selection::cost_sorted` ordering, /// `0` = best; a baseline referencing this variant is always `rank >= /// 1`, since `rank 0` is what got selected). HighestRankedNonSelectedCandidate { rank: usize }, diff --git a/crates/types/src/dag_export.rs b/crates/types/src/dag_export.rs index de9883a0..4094d239 100644 --- a/crates/types/src/dag_export.rs +++ b/crates/types/src/dag_export.rs @@ -141,11 +141,11 @@ pub struct DAGNode { #[derive(Debug, Clone, Serialize)] pub struct DAGNote { /// A short tag for the kind of annotation this is (e.g. a - /// `Debug`-formatted `asap_aware_mapping::ExplanationKind`) — opaque to + /// `Debug`-formatted `asap_logical_optimizer::ExplanationKind`) — opaque to /// `asap_types`, meant for a renderer to group or color by. pub kind: String, /// Human-readable explanation text (e.g. an - /// `asap_aware_mapping::ReplacementExplanation::reason`). + /// `asap_logical_optimizer::ReplacementExplanation::reason`). pub reason: String, } @@ -294,7 +294,7 @@ pub struct WorkloadDAG { // the `dag_export` devtools binary's `--post-asap` flag) populates after // running its own search — the exact same layering rule [`DAGNode::notes`]'s // doc above already states: this module never runs -// `asap_aware_mapping::replacement::search_workload_with` itself, never +// `asap_logical_optimizer::pass1::replacement::search_workload_with` itself, never // picks a "winning" candidate, and has no opinion on what a // `ReplacementProvenance` or a cost model even is. It only defines shapes // concrete and serializable enough for a higher layer to fill in, and for @@ -335,7 +335,7 @@ pub struct SummaryDAGNode { } /// One accuracy-illegal candidate a higher layer's search refused for a -/// target (issue #172) — `asap_aware_mapping::replacement::RejectedCandidate` +/// target (issue #172) — `asap_logical_optimizer::pass1::replacement::RejectedCandidate` /// re-shaped into this crate's own crate-agnostic vocabulary, the same /// layering rule as [`TargetReplacement`]. Carried on /// [`NamedDAG::rejections`] so a renderer can explain *why* a target kept @@ -362,8 +362,8 @@ pub struct SummaryDAG { } /// One replacement site a higher layer (the `dag_export` binary) found by -/// running `asap_aware_mapping::replacement::search_workload_with` + -/// `CandidateLogicalASAPDAGs::cost_sorted` and picking the best-ranked candidate for one +/// running `asap_logical_optimizer::pass1::replacement::search_workload_with` + +/// `candidate_selection::cost_sorted` and picking the best-ranked candidate for one /// `TargetSubDAGCandidates` — `asap_types` never runs that search itself (same layering /// rule as [`DAGNote`]: this crate defines the shape, a higher crate /// populates it). @@ -390,7 +390,7 @@ pub struct TargetReplacement { /// not re-derived here). pub rationale: String, /// This candidate's rank among its `TargetSubDAGCandidates`'s alternatives after - /// `CandidateLogicalASAPDAGs::cost_sorted` (`0` = best). Exposed so a renderer can show + /// `candidate_selection::cost_sorted` (`0` = best). Exposed so a renderer can show /// "this was the best of N candidates" without re-deriving the ranking. pub rank: usize, /// This candidate's own estimated cost, straight off @@ -419,7 +419,7 @@ pub struct TargetReplacement { /// What a [`TargetReplacement`] became — either a genuine post-ASAP binding /// or a still-relational structural rewrite, mirroring -/// `asap_aware_mapping::replacement::Replacement`'s own two variants. Both +/// `asap_logical_optimizer::pass1::replacement::Replacement`'s own two variants. Both /// carry an ordinary [`ExportDAG`]: the unified IR renders a summary sub-DAG /// and a rewritten relational sub-DAG through the same [`export`]. /// @@ -442,8 +442,8 @@ pub enum TargetReplacementAfter { /// What a higher layer found for one specific node when building a merged /// post-ASAP dag via [`export_post_asap`] — see that function's own doc /// for the full design. `asap_types` has no opinion on *how* this is -/// decided (that's `asap_aware_mapping::replacement::search_workload_with` + -/// `CandidateLogicalASAPDAGs::cost_sorted`'s job, a higher layer, exactly the layering rule +/// decided (that's `asap_logical_optimizer::pass1::replacement::search_workload_with` + +/// `candidate_selection::cost_sorted`'s job, a higher layer, exactly the layering rule /// [`DAGNode::notes`] already states); it only defines the shape a decision /// comes back in. Both variants render identically (one IR, one builder); /// they are kept apart so the caller's `Replacement` maps one-to-one. @@ -511,8 +511,8 @@ pub fn export_summary(node: &Rc) -> SummaryDAG { /// `::after` — small, independent, per-site before/after pairs — do). /// /// `find_winner` is the whole layering seam: `asap_types` never runs -/// `asap_aware_mapping::replacement::search_workload_with` or -/// `CandidateLogicalASAPDAGs::cost_sorted` itself, and has no idea what a `TargetSubDAGCandidates` or a +/// `asap_logical_optimizer::pass1::replacement::search_workload_with` or +/// `candidate_selection::cost_sorted` itself, and has no idea what a `TargetSubDAGCandidates` or a /// `ReplacementProvenance` is — it only asks, for one node at a time, "did a /// higher layer already decide something for you?" A caller (e.g. the /// `dag_export` devtools binary) builds this closure once per workload diff --git a/crates/types/src/ir/properties/guarantee.rs b/crates/types/src/ir/properties/guarantee.rs index 68cd7cea..6bc4810d 100644 --- a/crates/types/src/ir/properties/guarantee.rs +++ b/crates/types/src/ir/properties/guarantee.rs @@ -9,7 +9,7 @@ //! failure-probability expressions, the provenance trail, and the typed //! rejection reasons. The *algebra* that composes these (the `AccuracyModel` //! trait, its default conservative rules, and budget allocation) lives one -//! layer up in `asap_aware_mapping::accuracy`, the same layering +//! layer up in `asap_logical_optimizer::accuracy`, the same layering //! [`crate::dag_export`] keeps for cost decisions: this crate defines the //! shapes, the planning crate decides. //! diff --git a/crates/types/src/ir/schema/state_type.rs b/crates/types/src/ir/schema/state_type.rs index e05802e0..8146b652 100644 --- a/crates/types/src/ir/schema/state_type.rs +++ b/crates/types/src/ir/schema/state_type.rs @@ -13,7 +13,7 @@ //! [`GroupingStrategy`] is a second, orthogonal axis: how many physical //! instances of a summary exist across a grouped aggregate's `by` //! subpopulations (per-subpopulation vs. one shared Hydra instance — see -//! `asap_aware_mapping::grouping`). It rides on `ASAPOp::SummaryAgg` and on +//! `asap_logical_optimizer::pass1::grouping`). It rides on `ASAPOp::SummaryAgg` and on //! sketch-valued edge types. use serde::{Deserialize, Serialize}; @@ -161,7 +161,7 @@ pub enum SketchCategory { /// [`SketchKind::new`] is the one place `(SketchAlgorithm, SketchParams)` /// pairs get classified into a category; construct through it rather than /// naming a variant directly, so a new algorithm can't drift out of sync -/// with its category. See `asap_aware_mapping::summary_candidates` for the +/// with its category. See `asap_logical_optimizer::summary_candidates` for the /// `AggIntent -> [SketchAlgorithm]` candidate list this ultimately groups. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SketchKind { @@ -322,7 +322,7 @@ pub enum StatModelParams { // family/kind answers an intent — any family could in principle grow its own // per-subpopulation vs. shared-multi-subpopulation variant, so it is a // second, independent axis, not a member of any one family's own kind -// vocabulary. See `asap_aware_mapping::grouping`'s module docs for where this +// vocabulary. See `asap_logical_optimizer::pass1::grouping`'s module docs for where this // axis actually plugs into the post-ASAP IR and the legality rules gating // when `SharedMultiSubpopulation` is offered as a candidate at all. @@ -411,7 +411,7 @@ pub enum HydraParams { /// subpopulation. Correctly sizing this against an estimated /// subpopulation cardinality is a cost-model concern — out of scope /// for the legality axis this type lives on (see - /// `asap_aware_mapping::grouping`'s module docs) — so this is + /// `asap_logical_optimizer::pass1::grouping`'s module docs) — so this is /// deliberately not derived from any cardinality estimate here. shared_buckets: u32, }, @@ -468,7 +468,7 @@ pub fn hydra_kind_for(algorithm: &SketchAlgorithm) -> Option { /// belong to the [`SketchAlgorithm`] `kind` wraps: a caller bug, since /// [`hydra_kind_for`] and the algorithm a `SketchParams` came from must /// agree; callers that got both from the same already-ranked -/// `Realization` (as `asap_aware_mapping::grouping` does) cannot hit +/// `Realization` (as `asap_logical_optimizer::pass1::grouping` does) cannot hit /// this. /// /// This function is generic over which inner sketch type `kind` wraps diff --git a/docs/design_docs/architecture/README.md b/docs/design_docs/architecture/README.md index d0cc6102..c3a74f72 100644 --- a/docs/design_docs/architecture/README.md +++ b/docs/design_docs/architecture/README.md @@ -79,7 +79,7 @@ serving, and operational feedback. Their physical planning can reorder candidates because it has evidence that the reusable Planner does not, but it must not silently change Planner-owned semantics. -`CandidateLogicalASAPDAGs::global_selection` optionally coordinates structural choices across +`candidate_selection::global_selection` optionally coordinates structural choices across targets; `GlobalSelection::assemble_selected_dag` constructs a selected semantic DAG. Those APIs do not establish physical feasibility or a materialization decision. See the [library guide](../../develop_docs/library-api.md#optional-whole-plan-selection-and-dag-assembly) diff --git a/docs/design_docs/architecture/evidence-dependent-candidates.md b/docs/design_docs/architecture/evidence-dependent-candidates.md index 437cfe4b..39889ef4 100644 --- a/docs/design_docs/architecture/evidence-dependent-candidates.md +++ b/docs/design_docs/architecture/evidence-dependent-candidates.md @@ -22,7 +22,7 @@ The exact path (the pre-ASAP sub-DAG kept by `retain_exact`) has an exact guaran | Known guarantee | `ResultGuarantee` with evaluable bound and failure probability | Planner checks whether the guarantee satisfies the query's accuracy target. If this candidate is selected, the backend checks whether its implementation can realize the selected summary; it does not re-decide the accuracy target. | | Missing accuracy/domain evidence | Symbolic `BoundExpr::Unknown` or `ProbabilityExpr::Unknown`, or `guarantee: None` on a constructible summary | Inspect `ReplacementSubDAG::has_missing_accuracy_evidence()`, obtain applicable evidence or apply explicit policy; do not claim certification. | | Missing cost | `CostModel::candidate_cost()` returns `None` for a `ReplacementSubDAG` (including a non-finite or negative legacy estimate) | Keep that logical summary/rewrite candidate in `CandidateLogicalASAPDAGs` for inspection; provide a comparable cost before selecting it by cost. This does not make it a deployable physical plan. | -| Unknown runtime support | `ReplacementSubDAG::runtime_support_evidence(model)` returns `None` | Candidate remains visible; bind a concrete implementation and confirm support before deployment. | +| Unknown runtime support | `candidate_selection::runtime_support_evidence(candidate, model)` returns `None` | Candidate remains visible; bind a concrete implementation and confirm support before deployment. | | Known invalid evidence or impossible semantics | No candidate; where supported, a `RejectedCandidate` records the error | Do not deploy. | `ResultGuarantee::has_unknown()` detects symbolic gaps. The candidate-level diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index 18221035..133914f2 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -369,7 +369,7 @@ Then choose the operation matching the caller's responsibility: ### Ranked view -`CandidateLogicalASAPDAGs::cost_sorted` returns one `RankedTargetSubDAGCandidates` for each +`candidate_selection::cost_sorted` returns one `RankedTargetSubDAGCandidates` for each `TargetSubDAGCandidates` entry. Conceptually, it is the same target's alternatives in cost-model preference order where the model defines one (otherwise discovery order), with one displayed cost per alternative. It is @@ -395,7 +395,7 @@ physical deployability. ### Selection and DAG assembly The input is `CandidateLogicalASAPDAGs` and a cost model. Call -`CandidateLogicalASAPDAGs::global_selection(&cost_model)` once for the workload, then +`candidate_selection::global_selection(&space, &cost_model)` once for the workload, then `GlobalSelection::assemble_selected_dag(root)` for each wanted query root. These are two public APIs, not one combined call: N roots require one selection and N assembly calls. Each successful assembly returns one DAG root; the caller diff --git a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md index c1186c4a..56a5df00 100644 --- a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md +++ b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md @@ -195,7 +195,7 @@ let strategies = default_strategies_with_evidence(&cost_model, &evidence); let space = search_workload_with_targets(roots, &strategies, &accuracy_model); // 3. Select once for the whole workload. -let selection = space.global_selection(&cost_model); +let selection = global_selection(&space, &cost_model); // 4. Assemble once per root, then share common sub-DAGs across roots. let mut assembled = Vec::new(); diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index 1ee7502b..8db6e9a8 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -319,6 +319,6 @@ not require a new selection rule for every sketch. Deployment extensions to `AccuracyModel` remain possible, but carry the same obligation to justify metrics, assumptions and propagation. -For implementation details, see the [accuracy module](../../../crates/asap-aware-mapping/src/accuracy/mod.rs), +For implementation details, see the [accuracy module](../../../crates/logical-optimizer/src/accuracy/mod.rs), [guarantee representation](../../../crates/types/src/ir/properties/guarantee.rs), and [accuracy propagation companion](../../develop_docs/end-to-end-accuracy-guarantees.md). diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index 2401c491..67a5ed40 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -69,7 +69,7 @@ Exact work is represented by the ordinary operators, unchanged: - A sub-DAG the planner does not rewrite keeps its `NonASAPOp` nodes. Plan assembly marks such a sub-DAG with an exact `ResultGuarantee` - (`asap_aware_mapping::replacement::retain_exact`); a sub-DAG with no ASAP + (`asap_logical_optimizer::pass1::replacement::retain_exact`); a sub-DAG with no ASAP operator and no guarantee is a logical rewrite candidate that has not been assessed yet (`is_logical_rewrite`). - `BinaryOp` combines independently planned operands. Summary planning may set diff --git a/docs/design_docs/decisions/concat-unique-keys.md b/docs/design_docs/decisions/concat-unique-keys.md index e84e2529..8837bb86 100644 --- a/docs/design_docs/decisions/concat-unique-keys.md +++ b/docs/design_docs/decisions/concat-unique-keys.md @@ -69,7 +69,7 @@ Both current `Concat`-constructing call sites, and every consumer of - **Every consumer of `Schema::unique_keys`** in the DAG, to check for a cost beyond "a literal `Dedup` node": `ir::cse::share_common_sub_dags` (gates CSE producer-sharing on `Schema::has_unique_key()`) and - `asap_aware_mapping::rollup::is_legal_rollup_source` (gates rollup-source + `asap_logical_optimizer::pass1::rollup::is_legal_rollup_source` (gates rollup-source legality the same way, on an *`Aggregate`'s* own output schema). Neither case is exercised by a `histogram_quantiles` or `ROLLUP`/`CUBE`/ `GROUPING SETS` `Concat` in any current test, workload, or call site: no diff --git a/docs/design_docs/decisions/cse-cost-model.md b/docs/design_docs/decisions/cse-cost-model.md index 1c20382d..d4d0496a 100644 --- a/docs/design_docs/decisions/cse-cost-model.md +++ b/docs/design_docs/decisions/cse-cost-model.md @@ -58,7 +58,7 @@ This decision does not need search infrastructure of its own. Issue #252's MEMO-based search engine (`CandidateLogicalASAPDAGs`/`TargetSubDAGCandidates` in `replacement.rs`) already enumerates and ranks the larger, workload-wide candidate space. The choice between sharing and recomputing one already-detected CSE candidate is binary, -so `CandidateLogicalASAPDAGs::cost_sorted` reuses one direct +so `candidate_selection::cost_sorted` reuses one direct `CostModel::cse_share_decision` comparison per group. This preserves the policy described here—compare costs rather than applying a fixed rule—inside the larger search engine. `search_workload_with`'s @@ -78,7 +78,7 @@ gate) and the cost-aware decision is applied downstream, in ## Where it hooks in -[`CandidateLogicalASAPDAGs::cost_sorted`](../../../crates/asap-aware-mapping/src/replacement.rs) +[`candidate_selection::cost_sorted`](../../../crates/asap-aware-mapping/src/plan_selection/candidate_selection.rs) is where this hooks in today. `search_workload_with` computes each shared sub-DAG's true `consumer_count` across the whole workload up front (the same role `implement_workload_with`'s pre-pass used to play, before that function @@ -86,7 +86,7 @@ was retired along with `bind.rs` — this crate no longer commits to one physically-materialized answer at all; picking and building one final `SummaryNode` per shared sub-DAG is a downstream deployment's job, not this crate's). For a `TargetSubDAGCandidates` whose candidates are a -[`SharedSubDAGStrategy`](../../../crates/asap-aware-mapping/src/replacement.rs) +[`SharedSubDAGStrategy`](../../../crates/logical-optimizer/src/pass1/replacement.rs) share-vs-recompute pair, `cost_sorted`'s ranking step (`rank_group`/ `cse_preference`) asks `CostModel::cse_share_decision` once per group — using one representative bound `SummaryNode` built just for that comparison, not @@ -116,7 +116,7 @@ either or both, same as `size_params` already lets a deployment override ## Scope -This decision, and `cse_share_decision`'s wiring into `CandidateLogicalASAPDAGs::cost_sorted` +This decision, and `cse_share_decision`'s wiring into `candidate_selection::cost_sorted` (originally into `implement_workload_with`, before `bind.rs` was retired — see above), close out #223's stage 4 and #212's original "add CSE" tracking issue. Stage 3 (`dag_export::structural_hash` unification) landed separately diff --git a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md index 7d0cd9d4..0b865154 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md @@ -912,7 +912,7 @@ The intended end-to-end selection pipeline is: The query lowerer and physical estimator cover the supported raw-query shapes listed above. `PhysicalPlanCostModel` executes this pipeline for every -candidate supplied to `CandidateLogicalASAPDAGs::global_selection`. Logical rewrites are +candidate supplied to `candidate_selection::global_selection`. Logical rewrites are lowered recursively. Summary candidates participate only after the deployment has bound their complete `SummaryExpr` DAG; there is no optimistic generic summary fallback. Choosing between a maintained summary and raw recomputation diff --git a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md index c2b05588..91382160 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md +++ b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md @@ -1,7 +1,7 @@ # Shared maintained population rule > Status: planner rule implemented; deployment support is conditional. See -> [MaintainedPopulationStrategy](../../../../crates/asap-aware-mapping/src/maintained_population.rs). +> [MaintainedPopulationStrategy](../../../../crates/logical-optimizer/src/pass1/maintained_population.rs). > A deployment must provide the membership, freshness, state and operation > capabilities described below. Planner representation alone does not implement > population maintenance in a runtime. @@ -51,7 +51,7 @@ column and grouping determine whether consumers refer to the same population. ## Rule: share one population across compatible readouts **Realization:** `MaintainedPopulationStrategy`, an opt-in `ReplacementStrategy` -in [maintained_population.rs](../../../../crates/asap-aware-mapping/src/maintained_population.rs). +in [maintained_population.rs](../../../../crates/logical-optimizer/src/pass1/maintained_population.rs). **Target sub-DAGs:** diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index a71a0166..a606bc29 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -16,7 +16,8 @@ defined in [mapping contracts](asap-aware-mapping-contracts.md). Names such as `MyStrategy`, `MyCostModel`, and `PreferDDSketch` are illustrative; they do not ship with this crate. Samples that use real public -types and functions follow the APIs exported by `asap-aware-mapping`. +types and functions follow the APIs exported by `asap-logical-optimizer` +(Stage 1 candidate search) and `asap-aware-mapping` (cost models and selection). If you only need to find the right extension point, start with the [extension map](extend-asap-aware-mapping.md#7-current-extension-map). If you are implementing a strategy, read this mental model, the [mapping contracts](asap-aware-mapping-contracts.md), and the [extension guide](extend-asap-aware-mapping.md). @@ -109,7 +110,7 @@ flowchart TB subgraph RANKING[Optional ranked view] CM(["CostModel
selection-time preferences and costs"]):::choose - SORT["CandidateLogicalASAPDAGs::cost_sorted
use the CostModel to order each candidate set
and cost every candidate"]:::choose + SORT["candidate_selection::cost_sorted
use the CostModel to order each candidate set
and cost every candidate"]:::choose CM -.-> SORT RANKED["RankedTargetSubDAGCandidates
the same candidates in preferred order,
with costs aligned by index"]:::choose SPACE --> SORT -->|"reorder only; preserve every candidate"| RANKED @@ -220,7 +221,7 @@ The default context-free registry contains five `ReplacementStrategy` implementa `AvgToSumOverCountStrategy` alias) in its rewrite slot. The evidence-aware registry supplies the accuracy evidence provider to summary and Hydra construction. Search derives `RollupStrategy` after CSE from the actual sibling set. See the -[registry definitions](../../crates/asap-aware-mapping/src/replacement.rs). +[registry definitions](../../crates/logical-optimizer/src/pass1/replacement.rs). The important rule is: @@ -238,7 +239,7 @@ rejection reasons. This compact representation preserves independent choices without enumerating a flat list of `2^N` complete plans for `N` replaceable targets. -`CandidateLogicalASAPDAGs::cost_sorted` ranks each target's existing candidates with the +`candidate_selection::cost_sorted` ranks each target's existing candidates with the supplied `CostModel`. It returns the same candidates in preferred order, with costs aligned by index; ranking does not select or remove a candidate. @@ -257,7 +258,7 @@ order. Constructing all candidates before taking the first costs more than constructing only one, but it keeps the strategy contract consistent and preserves the full choice set for other callers. -`CandidateLogicalASAPDAGs::global_selection` optionally coordinates cross-target sharing and +`candidate_selection::global_selection` optionally coordinates cross-target sharing and composition choices. `GlobalSelection::assemble_selected_dag` constructs the selected semantic DAG. These APIs do not decide materialization or establish physical deployment feasibility. Recurrence-aware variants require the corresponding diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 7e820e3f..1d5734fd 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -76,7 +76,7 @@ counts as a bound decision, not a rewrite. Use `Replacement::ExactComposition` when an exact operation refers to a child target whose realization must remain undecided. Selection coordinates the parent/child pair; DAG assembly constructs and validates the composed DAG. -See [exact_composition.rs](../../crates/asap-aware-mapping/src/exact_composition.rs). +See [exact_composition.rs](../../crates/logical-optimizer/src/pass1/exact_composition.rs). Examples: @@ -205,7 +205,7 @@ see [code architecture §3](asap-aware-mapping-architecture.md#3-how-the-current `CostModel` covers deployment-specific preference and cost decisions. It is consulted only at selection time (`cost_sorted`, `global_selection` and their `_with_recurrence` variants), never during candidate generation: sketch parameters come from the analytical estimators (`accuracy::estimators::size_params`), and extension intents stay pass-through. -The crate cannot hardcode real deployment costs: `asap-aware-mapping` uses `asap-types` and pinned `asap_sketchlib` mapping +The crate cannot hardcode real deployment costs: `asap-aware-mapping` uses `asap-types` and the `asap_sketchlib` mapping bounds, but does not execute workloads or own deployment measurements. Most hooks therefore provide the crate's built-in static behavior as a default. Override only the decisions your deployment needs to change. | Hook | Use it to | Default? | @@ -242,7 +242,7 @@ bounds, but does not execute workloads or own deployment measurements. Most hook fn cse_share_decision(&self, candidate: &CseCandidate) -> ShareDecision; ``` -- **`estimate_cost`** — attach a comparable numeric cost to an already-constructed replacement. `CandidateLogicalASAPDAGs::cost_sorted` calls it for every candidate and keeps the returned values aligned with the ranked candidates. The trait default returns `f64::NAN` deliberately; override it when a custom model's callers need displayable or otherwise consumable numeric costs. `DefaultCostModel` provides real values derived from its CSE cost hooks. +- **`estimate_cost`** — attach a comparable numeric cost to an already-constructed replacement. `candidate_selection::cost_sorted` calls it for every candidate and keeps the returned values aligned with the ranked candidates. The trait default returns `f64::NAN` deliberately; override it when a custom model's callers need displayable or otherwise consumable numeric costs. `DefaultCostModel` provides real values derived from its CSE cost hooks. ```rust fn estimate_cost( @@ -283,7 +283,7 @@ pub struct RankedTargetSubDAGCandidates<'a> { `search_workload(roots)` runs the shared-sub-DAG pass once, discovers every target across every root's whole DAG (not just root-level sharing — a `SharedSubDAGStrategy` candidate three levels under an unshared `Filter` is exactly as real a site as a shared whole root), and asks every registered strategy to a fixpoint. Two logically different candidates at two different targets are never copied into two separate plans — they're two entries in two different `TargetSubDAGCandidates`s, sharing every other node in the workload by construction. -`CandidateLogicalASAPDAGs::cost_sorted(cost_model)` is the one ranking step: for each candidate set, it dispatches by candidate shape — the `SharedSubDAGStrategy` share/recompute pair (recognized by `ReplacementProvenance::CseShare`/`CseRecompute`) goes through `CostModel::cse_share_decision`; a set with a Hydra shared-grid alternative goes through `CostModel::grouping_state_cost`; a set whose candidates all realize sketches (a `ASAPStrategies` choice) goes through `CostModel::rank_candidates`; and any other mixed set is ordered by `CostModel::candidate_cost`. Every candidate gets a numeric cost aligned index-for-index in `costs`. Count in, count out—nothing is dropped to produce a ranking. Legality checks +`candidate_selection::cost_sorted(cost_model)` is the one ranking step: for each candidate set, it dispatches by candidate shape — the `SharedSubDAGStrategy` share/recompute pair (recognized by `ReplacementProvenance::CseShare`/`CseRecompute`) goes through `CostModel::cse_share_decision`; a set with a Hydra shared-grid alternative goes through `CostModel::grouping_state_cost`; a set whose candidates all realize sketches (a `ASAPStrategies` choice) goes through `CostModel::rank_candidates`; and any other mixed set is ordered by `CostModel::candidate_cost`. Every candidate gets a numeric cost aligned index-for-index in `costs`. Count in, count out—nothing is dropped to produce a ranking. Legality checks may already have removed proposals before this boundary. In particular, `search_workload_with_targets` checks explicit per-root targets, while retaining direct DDSketch ratios with missing domain evidence and no root guarantee for diff --git a/docs/develop_docs/end-to-end-accuracy-guarantees.md b/docs/develop_docs/end-to-end-accuracy-guarantees.md index 213e6e48..1baf0a60 100644 --- a/docs/develop_docs/end-to-end-accuracy-guarantees.md +++ b/docs/develop_docs/end-to-end-accuracy-guarantees.md @@ -41,9 +41,9 @@ The main implementation locations are: | Concern | Location | | --- | --- | | Guarantee and error vocabulary | `asap_types::ir::properties::guarantee` | -| Accuracy model and built-in propagation | `asap_aware_mapping::accuracy` | -| Candidate construction and legality filtering | `asap_aware_mapping::replacement` | -| Parameter sizing | `asap_aware_mapping::accuracy::estimators` | +| Accuracy model and built-in propagation | `asap_logical_optimizer::accuracy` | +| Candidate construction and legality filtering | `asap_logical_optimizer::pass1::replacement` | +| Parameter sizing | `asap_logical_optimizer::accuracy::estimators` | | Guarantee and rejection export | `asap_types::dag_export` and the `dag_export` devtool | Read the sections below when changing one of those contracts. diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index 4674e44e..6432ad76 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -319,7 +319,7 @@ Replacement::SubDAG( This strategy does **not** decide whether sharing is cheaper. That preference belongs to the cost model. -`CandidateLogicalASAPDAGs::cost_sorted` calls `CostModel::cse_share_decision` when it ranks a +`candidate_selection::cost_sorted` calls `CostModel::cse_share_decision` when it ranks a share-versus-recompute candidate pair. The strategy still returns both alternatives because enumeration and ranking are separate steps: @@ -536,8 +536,8 @@ Then pass it to the selection-time APIs that accept a `&dyn CostModel`: let model = PreferDDSketch; let space = search_workload_with(roots, &default_strategies()); -let ranked = space.cost_sorted(&model); -let selection = space.global_selection(&model); +let ranked = cost_sorted(&space, &model); +let selection = global_selection(&space, &model); ``` Important: `rank_candidates` changes only the selection-time ordering; `ASAPStrategies` still enumerates every valid sketch candidate, in `summary_candidates` order, sized analytically. @@ -634,7 +634,7 @@ fn estimate_cost( ) -> f64; ``` -The default returns `f64::NAN`, making the absence of a numeric model explicit. Override this hook when passing the model to `CandidateLogicalASAPDAGs::cost_sorted` if downstream code displays or otherwise consumes the `costs` values. Prefer to derive the result from the same inputs used by `rank_candidates` and the CSE cost hooks so numeric costs do not disagree with relative ordering. +The default returns `f64::NAN`, making the absence of a numeric model explicit. Override this hook when passing the model to `candidate_selection::cost_sorted` if downstream code displays or otherwise consumes the `costs` values. Prefer to derive the result from the same inputs used by `rank_candidates` and the CSE cost hooks so numeric costs do not disagree with relative ordering. --- @@ -664,7 +664,7 @@ For example: ```rust let ranked = - space.cost_sorted(&model); + cost_sorted(&space, &model); ``` The important assertion is usually not that other valid candidates disappeared. They should not. @@ -898,9 +898,9 @@ Use this table to find the right place for a change. | Produce the first-listed post-ASAP summary for one target (unranked, `summary_candidates` order) | `ASAPStrategies::replacements(...).into_iter().next()` | | Search a whole workload for supported legal candidates | `search_workload`/`search_workload_with` | | Enforce per-root result accuracy requirements | `search_workload_with_targets` | -| Coordinate compatible choices across groups | `CandidateLogicalASAPDAGs::global_selection` | +| Coordinate compatible choices across groups | `candidate_selection::global_selection` | | Assemble the selected logical DAG | `GlobalSelection::assemble_selected_dag` | -| Get every candidate ranked best-first, across a whole workload | `CandidateLogicalASAPDAGs::cost_sorted` | +| Get every candidate ranked best-first, across a whole workload | `candidate_selection::cost_sorted` | | Get a real numeric cost per candidate, not just a relative rank | `CostModel::estimate_cost` | | Enumerate valid sketch algorithms | `summary_candidates` | | Build a target with no workload context | `TargetSubDAG::new` | @@ -915,7 +915,7 @@ Use this table to find the right place for a change. ### Using it ```rust -use asap_aware_mapping::{explain_replacements, ExplanationKind}; +use asap_logical_optimizer::{explain_replacements, ExplanationKind}; let explanations = explain_replacements(vec![("dashboard_p99", query)]); for explanation in &explanations { diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index f7abce1a..d1d2f937 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -22,14 +22,17 @@ artifact, while preserving the checks required by its intended consumer. ## Dependencies -Inside this workspace, depend on the frontend you need, `asap-aware-mapping`, -and `asap-types`. External users can use Git dependencies pinned to a compatible -revision; use the same revision across these crates. For the example below: +Inside this workspace, depend on the frontend you need, +`asap-logical-optimizer` (Stage 1 candidate search), `asap-aware-mapping` +(cost models and selection) and `asap-types`. External users can use Git +dependencies pinned to a compatible revision; use the same revision across +these crates. For the example below: ```toml [dependencies] asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e7fdb2492c42c9f5b34760706a5162aa586d3025" } asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e7fdb2492c42c9f5b34760706a5162aa586d3025" } +asap-logical-optimizer = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e7fdb2492c42c9f5b34760706a5162aa586d3025" } asap-types = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e7fdb2492c42c9f5b34760706a5162aa586d3025" } ``` @@ -168,8 +171,8 @@ search_workload_with_targets<'s, Id>( accuracy_model: &dyn AccuracyModel, ) -> CandidateLogicalASAPDAGs -CandidateLogicalASAPDAGs::cost_sorted(&self, cost_model: &dyn CostModel) - -> Vec> +candidate_selection::cost_sorted<'a, Id>(space: &'a CandidateLogicalASAPDAGs, cost_model: &dyn CostModel) + -> Vec> ``` | Argument | Choices / meaning | Required? | @@ -203,9 +206,10 @@ use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, Query, PlanningWorkload, QueryLanguage, QueryRequirements, QueryWorkload, }; -use asap_aware_mapping::{ - default_strategies, search_workload_with_targets, - DefaultAccuracyModel, DefaultCostModel, +use asap_aware_mapping::plan_selection::candidate_selection::cost_sorted; +use asap_aware_mapping::DefaultCostModel; +use asap_logical_optimizer::{ + default_strategies, search_workload_with_targets, DefaultAccuracyModel, }; use asap_types::types::AccuracyTarget; @@ -243,7 +247,7 @@ fn main() -> Result<(), Box> { &strategies, &DefaultAccuracyModel, ); - for group in space.cost_sorted(&cost_model) { + for group in cost_sorted(&space, &cost_model) { for (candidate, cost) in group.candidates.iter().zip(&group.costs) { println!("candidate={candidate:?}, reported_cost={cost:?}"); } @@ -252,13 +256,13 @@ fn main() -> Result<(), Box> { } ``` -| API (`asap_aware_mapping`, unless qualified) | Inputs | Output and limits | +| API (`asap_logical_optimizer`; `candidate_selection` is `asap_aware_mapping::plan_selection::candidate_selection`) | Inputs | Output and limits | | --- | --- | --- | | `search_workload` | `(query_id, Rc)` roots | `CandidateLogicalASAPDAGs` with built-in strategies/model; no explicit per-root target argument | | `search_workload_with` | Roots, strategy slice | `CandidateLogicalASAPDAGs`; callers choose context-free replacement strategies | | `search_workload_with_targets` | Roots with optional end-to-end targets, strategies, accuracy model | Candidate space with supplied root-target checks; `None` does not supply a root-level requirement; uncertified direct DDSketch ratios remain available for backend selection | -| `CandidateLogicalASAPDAGs::cost_sorted` | Cost model | `Vec`; retains alternatives and pairs `candidates[i]` with `costs[i]` | -| `CandidateLogicalASAPDAGs::cost_sorted_with_recurrence` | Cost model, recurrence profiles, optional horizon | Ranked per-target candidate sets or `RecurrenceError`; uses recurrence for applicable share/recompute comparisons | +| `candidate_selection::cost_sorted` | Cost model | `Vec`; retains alternatives and pairs `candidates[i]` with `costs[i]` | +| `candidate_selection::cost_sorted_with_recurrence` | Cost model, recurrence profiles, optional horizon | Ranked per-target candidate sets or `RecurrenceError`; uses recurrence for applicable share/recompute comparisons | | `ASAPStrategies::replacements` through `ReplacementStrategy` | One `TargetSubDAG` | Alternatives at that target; not whole-workload search | `cost_sorted` is a ranking view, not a request to discard all but the first @@ -351,9 +355,11 @@ use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, Query, PlanningWorkload, QueryLanguage, QueryRequirements, QueryWorkload, }; -use asap_aware_mapping::{ - search_workload_with_targets, DefaultAccuracyModel, DefaultCostModel, - ReplacementStrategy, ASAPStrategies, SharedSubDAGStrategy, +use asap_aware_mapping::plan_selection::candidate_selection::cost_sorted; +use asap_aware_mapping::DefaultCostModel; +use asap_logical_optimizer::{ + search_workload_with_targets, DefaultAccuracyModel, ReplacementStrategy, + ASAPStrategies, SharedSubDAGStrategy, }; use asap_types::types::AccuracyTarget; @@ -392,7 +398,7 @@ fn main() -> Result<(), Box> { let space = search_workload_with_targets( vec![("q1", root, Some(accuracy))], &strategies, &DefaultAccuracyModel, ); - println!("{:#?}", space.cost_sorted(&model)); + println!("{:#?}", cost_sorted(&space, &model)); Ok(()) } ``` @@ -404,7 +410,8 @@ not waive semantic or accuracy requirements. ### Model and evidence options Traits permit custom implementations; the following are concrete built-in options. -Module-qualified paths below are relative to `asap_aware_mapping`. +Cost models are in `asap_aware_mapping` (module-qualified paths below are +relative to it); accuracy models and evidence are in `asap_logical_optimizer`. | Parameter | Available value / constructor | Meaning | | --- | --- | --- | @@ -440,7 +447,7 @@ accuracy guarantees. ### Example: configure all sketch-strategy providers ```rust -use asap_aware_mapping::{ +use asap_logical_optimizer::{ DefaultAccuracyModel, EqualSplitAllocator, NoAccuracyEvidence, ReplacementStrategy, ASAPStrategies, }; @@ -485,7 +492,7 @@ Accuracy models, allocators and evidence are consumed during generation; the cos model is consumed only at selection (`cost_sorted`, `global_selection` and their `_with_recurrence` variants). Sketch parameters come from the analytical estimators, not the cost model. For evidence-aware defaults, use -`asap_aware_mapping::replacement::default_strategies_with_evidence`. +`asap_logical_optimizer::pass1::replacement::default_strategies_with_evidence`. For custom accuracy/allocation/evidence on sketches, `ASAPStrategies::new_with_planning_inputs_and_evidence` exposes these providers. Keep each provider's evidence scope and freshness valid for the query population. @@ -550,10 +557,10 @@ Plain `global_selection()` does not decide materialization or establish physical deployment feasibility. Stage 2 materialization (#509) will own materialization; downstream still owns physical commitment. -| Method on `CandidateLogicalASAPDAGs` / `GlobalSelection` | Behavior | +| Function or method | Behavior | | --- | --- | -| `CandidateLogicalASAPDAGs::global_selection(&model)` | Compatible structural selection across targets; no recurrence or materialization planning implied | -| `CandidateLogicalASAPDAGs::global_selection_with_recurrence(...)` | Compatible selection using supplied recurrence profiles/horizon; no materialization commitments implied | +| `candidate_selection::global_selection(&space, &model)` | Compatible structural selection across targets; no recurrence or materialization planning implied | +| `candidate_selection::global_selection_with_recurrence(...)` | Compatible selection using supplied recurrence profiles/horizon; no materialization commitments implied | | `GlobalSelection::assemble_selected_dag(&target)` | `Result>, RealizationError>`; constructs untimed semantic IR, not stored summary data | Use a target associated with the searched space; DAG assembly can return `None` @@ -565,7 +572,8 @@ for checking complete physical alternatives and deployment constraints. ### API definition and example ```text -CandidateLogicalASAPDAGs::global_selection(&self, cost_model: &dyn CostModel) -> GlobalSelection<'_> +candidate_selection::global_selection<'a, Id>(space: &'a CandidateLogicalASAPDAGs, cost_model: &dyn CostModel) + -> CostedGlobalSelection<'a> // derefs to GlobalSelection GlobalSelection::assemble_selected_dag(&self, target: &Rc) -> Result>, RealizationError> ``` @@ -580,7 +588,9 @@ use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, Query, PlanningWorkload, QueryLanguage, QueryRequirements, QueryWorkload, }; -use asap_aware_mapping::{search_workload, DefaultCostModel}; +use asap_aware_mapping::plan_selection::candidate_selection::global_selection; +use asap_aware_mapping::DefaultCostModel; +use asap_logical_optimizer::search_workload; use asap_types::types::AccuracyTarget; fn main() -> Result<(), Box> { @@ -610,7 +620,7 @@ fn main() -> Result<(), Box> { }; let root = lower_promql_workload(&workload, 0)?.remove(0); let space = search_workload(vec![("q1", root)]); - let selection = space.global_selection(&DefaultCostModel); + let selection = global_selection(&space, &DefaultCostModel); // Search may canonicalize roots; use the root returned by CandidateLogicalASAPDAGs. if let Some(summary) = selection.assemble_selected_dag(&space.roots[0].1)? { let dag = asap_types::dag_export::export_summary(&summary); @@ -643,7 +653,7 @@ cargo doc -p asap-aware-mapping -p asap-types --no-deps ## Source references - [Frontend PromQL](../../crates/frontend-promql/src/lib.rs), [SQL](../../crates/frontend-sql/src/lib.rs), [MetricsQL](../../crates/frontend-metricsql/src/lib.rs) -- [Search, ranking and selection](../../crates/asap-aware-mapping/src/replacement.rs) +- [Search, ranking and selection](../../crates/logical-optimizer/src/pass1/replacement.rs) - [Cost models](../../crates/asap-aware-mapping/src/cost_model.rs) - [Workload types](../../crates/types/src/workload/mod.rs) - [Planner-runtime contract](../design_docs/architecture/planner-runtime-contract.md) diff --git a/docs/develop_docs/local-logical-candidates.md b/docs/develop_docs/local-logical-candidates.md index b27e9b2d..4673647b 100644 --- a/docs/develop_docs/local-logical-candidates.md +++ b/docs/develop_docs/local-logical-candidates.md @@ -1,6 +1,6 @@ # Local logical alternatives (Pass 1) -`asap_aware_mapping::logical_candidates` enumerates local realization choices over +`asap_logical_optimizer::pass1::logical_candidates` enumerates local realization choices over unified `OperatorNode` and `QueryRoot` inputs. It is the first part of logical ASAP optimization in [planner layering](../design_docs/proposals/planner-layering.md). diff --git a/docs/develop_docs/replacement-explanations.md b/docs/develop_docs/replacement-explanations.md index 4cd97300..caefd00f 100644 --- a/docs/develop_docs/replacement-explanations.md +++ b/docs/develop_docs/replacement-explanations.md @@ -24,4 +24,4 @@ Additional opportunities: - reuse finer-grained aggregation through roll-up ``` -Implemented as `asap-aware-mapping`'s `explanation` module (`explain_replacements`/`explain_replacements_with`, issue #257) +Implemented as `asap-logical-optimizer`'s `pass1::explanation` module (`explain_replacements`/`explain_replacements_with`, issue #257)