From e52e047e37d9915f1fbc6cc272c3eec866ddfa94 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:53:18 +0000 Subject: [PATCH 1/2] feat(dag-viewer): add three-lane Stages view for asap-stage-pipeline/v1 Load an asap-stage-pipeline/v1 document (file picker or ?doc=) and show Logical, Logical ASAP and Physical ASAP lanes side by side, with candidate switchers, per-node timing and cost, a cost-ranked list of physical candidates, and the stage-3 selection and rejection reasons. Selecting a physical candidate shows its from_logical candidate in lane 2. Pre/Post-ASAP stays the default unless a stage document is loaded. stages.js holds the DOM-free validation, lane construction and ranking, tested headless against a hand-written #509 Example 1 Q2 fixture. Co-Authored-By: Claude Opus 5.5 --- tools/dag-viewer/README.md | 68 +- tools/dag-viewer/RUNNING.md | 9 + .../examples/stage-pipeline.sample.json | 3552 +++++++++++++++++ tools/dag-viewer/index.html | 16 +- tools/dag-viewer/render.py | 2 +- tools/dag-viewer/stages.js | 249 ++ tools/dag-viewer/test_render.py | 146 +- tools/dag-viewer/viewer.js | 285 +- 8 files changed, 4310 insertions(+), 17 deletions(-) create mode 100644 tools/dag-viewer/examples/stage-pipeline.sample.json create mode 100644 tools/dag-viewer/stages.js diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index 182c0f94..673486b1 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -1,6 +1,9 @@ # ASAP Pre/Post-ASAP DAG viewer -The viewer has one visualization mode: **Pre/Post-ASAP**. +The viewer has two visualization modes, chosen with the header switch: +**Pre/Post-ASAP** (below) and **Stages** (see "Stages view"). Stages is +enabled, and opens by default, only when an `asap-stage-pipeline/v1` +document is loaded. - Select one query to see that query's complete pre-ASAP and post-ASAP DAGs. - Select multiple queries to see two workload-union DAGs: one pre-ASAP union @@ -26,6 +29,65 @@ The viewer has one visualization mode: **Pre/Post-ASAP**. There are no separate Single, Compare, or Union modes. +## Stages view + +The Stages view shows one planner run through the stages of +[planner layering](../../docs/design_docs/proposals/planner-layering.md), +as three lanes side by side: + +1. **Logical**: the stage-0 logical DAG. +2. **Logical ASAP**: one stage-1 candidate. Pick it with the + **Logical ASAP candidates** list above the canvas, which shows each + candidate's label. +3. **Physical ASAP**: one stage-2 candidate. Each node shows its timing + (`⏱ ingestion time` or `⏱ query time`; ingestion-time nodes also have a + double border) and its cost. The lane header shows the candidate's total + cost and unit, its rank, and whether stage 3 selected or rejected it. + +The **Physical ASAP candidates by total cost** list ranks every physical +candidate, cheapest first. It marks the selected candidate, and shows each +rejected candidate with its reason. A candidate that stage 3 neither +selects nor rejects is shown as "not selected". Clicking a row shows that +candidate in lane 3, and shows the logical candidate it implements in +lane 2. That logical candidate is outlined in the list and in the lane. + +Click a node to see its payload, output schema, guarantee and coverage. +Physical nodes also show their output timing and per-node cost. Click an +edge to see its intermediate schema, and for physical edges, its data state. + +To open the sample document, start the server (see "Interactive query +editor") and go to +. The `doc` +parameter takes any JSON file path served next to `index.html`. You can also +load a stage document with the file picker. + +The sample, `examples/stage-pipeline.sample.json`, is hand-written. It +follows #509's Example 1 query Q2, +`topk by (job) (10, sum_over_time(http_requests_total[1m]))`. It has three +logical candidates (exact, Count-Min with heap, Hydra) and four physical +candidates. The costs are illustrative, not planner output. + +### Document format + +```json +{ + "format": "asap-stage-pipeline/v1", + "workload": { "queries": [{ "id": "q1", "language": "sql", "text": "..." }] }, + "stage0_logical": { "dag": "" }, + "stage1_logical_asap": { "candidates": [{ "id": "L1", "label": "...", "dag": "" }] }, + "stage2_physical_asap": { "candidates": [{ "id": "P1", "from_logical": "L1", "label": "...", + "dag": "", + "cost": { "total": 12.5, "unit": "cpu_ms_per_s", "per_node": { "": { "cost": 3.2, "detail": "..." } } } }] }, + "stage3_selection": { "selected": "P1", "rejected": [{ "id": "P2", "reason": "..." }] } +} +``` + +DAGs use the serde JSON of `LogicalASAPDAG` and `PhysicalASAPDAG`. The viewer +checks the document before it renders it: node and edge references, +`from_logical`, `per_node` keys, physical `output_state.timing` and +`data_state`, and the stage-3 ids. If any check fails, it lists the +problems and does not load the document. + ## Interactive query editor From the repository root: @@ -199,3 +261,7 @@ All of this is additive and optional: an export with none of these fields python3 -m unittest discover -s tools/dag-viewer -p test_render.py cargo test -p asap-devtools --bin dag_export ``` + +The JavaScript tests, including the Stages validation, lane and ranking +tests, need `py_mini_racer` (`pip install py-mini-racer`). Without it they +are skipped. diff --git a/tools/dag-viewer/RUNNING.md b/tools/dag-viewer/RUNNING.md index 2a9118d2..df48b080 100644 --- a/tools/dag-viewer/RUNNING.md +++ b/tools/dag-viewer/RUNNING.md @@ -11,4 +11,13 @@ Open: http://localhost:8000/ ``` +To open the Stages view with the sample stage-pipeline document: + +```text +http://localhost:8000/?doc=examples/stage-pipeline.sample.json +``` + +Use `--port 8765` to serve on another port, and `--skip-build` to reuse an +already-built `target/debug/dag_export`. + Press `Ctrl+C` in the server terminal to stop it and free port 8000. diff --git a/tools/dag-viewer/examples/stage-pipeline.sample.json b/tools/dag-viewer/examples/stage-pipeline.sample.json new file mode 100644 index 00000000..d3ad1b56 --- /dev/null +++ b/tools/dag-viewer/examples/stage-pipeline.sample.json @@ -0,0 +1,3552 @@ +{ + "format": "asap-stage-pipeline/v1", + "workload": { + "queries": [ + { + "id": "q2", + "language": "promql", + "text": "topk by (job) (10, sum_over_time(http_requests_total[1m]))" + } + ] + }, + "stage0_logical": { + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "kind": "relational", + "operator": { + "kind": "scan", + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + } + } + }, + "result_kind": "instant_vector", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 1, + "payload": { + "kind": "relational", + "operator": { + "kind": "time_range", + "range": { + "secs": 60, + "nanos": 0 + }, + "range_kind": "range" + } + }, + "result_kind": "range_vector", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 2, + "payload": { + "kind": "relational", + "operator": { + "kind": "aggregate", + "reduction": "PerEntity", + "measures": [ + { + "kind": "sum", + "col": null + } + ], + "output_names": [ + "" + ], + "filters": [], + "having": null + } + }, + "result_kind": "instant_vector", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 3, + "payload": { + "kind": "relational", + "operator": { + "kind": "aggregate", + "reduction": { + "Reduce": [ + 2 + ] + }, + "measures": [ + { + "kind": "top_k", + "k": 10, + "accuracy": "Exact" + } + ], + "output_names": [], + "filters": [], + "having": null + } + }, + "result_kind": "instant_vector", + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "topk_10", + "dtype": { + "Plain": "utf8" + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": null, + "coverage": null + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "grouping": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "grouping": "NotApplicable" + }, + { + "producer": 2, + "consumer": 3, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "grouping": "NotApplicable" + } + ], + "root": { + "Operator": 3 + } + } + }, + "stage1_logical_asap": { + "candidates": [ + { + "id": "L1", + "label": "Exact: per-series sum, sort + limit 10 per job", + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "kind": "relational", + "operator": { + "kind": "scan", + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + } + } + }, + "result_kind": "instant_vector", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 1, + "payload": { + "kind": "relational", + "operator": { + "kind": "time_range", + "range": { + "secs": 60, + "nanos": 0 + }, + "range_kind": "range" + } + }, + "result_kind": "range_vector", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 2, + "payload": { + "kind": "relational", + "operator": { + "kind": "aggregate", + "reduction": "PerEntity", + "measures": [ + { + "kind": "sum", + "col": null + } + ], + "output_names": [ + "" + ], + "filters": [], + "having": null + } + }, + "result_kind": "instant_vector", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 3, + "payload": { + "kind": "relational", + "operator": { + "kind": "aggregate", + "reduction": { + "Reduce": [ + 2 + ] + }, + "measures": [ + { + "kind": "top_k", + "k": 10, + "accuracy": "Exact" + } + ], + "output_names": [], + "filters": [], + "having": null + } + }, + "result_kind": "instant_vector", + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "topk_10", + "dtype": { + "Plain": "utf8" + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": { + "metric": "absolute_value", + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "provenance": [ + { + "kind": "exact", + "reason": "exact per-series sum and sort within job" + } + ] + }, + "coverage": null + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "grouping": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "grouping": "NotApplicable" + }, + { + "producer": 2, + "consumer": 3, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "grouping": "NotApplicable" + } + ], + "root": { + "Operator": 3 + } + } + }, + { + "id": "L2", + "label": "Count-Min + top-10 heap per job", + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "kind": "relational", + "operator": { + "kind": "scan", + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + } + } + }, + "result_kind": "instant_vector", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 1, + "payload": { + "kind": "relational", + "operator": { + "kind": "time_range", + "range": { + "secs": 60, + "nanos": 0 + }, + "range_kind": "range" + } + }, + "result_kind": "range_vector", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 2, + "payload": { + "kind": "summary_agg", + "family": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "input": { + "item": { + "EntityIdentity": { + "PromqlLabelSet": { + "excluding": [] + } + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + }, + "grouping": "PerSubpopulationInstance", + "filter": null + }, + "result_kind": "state", + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "state", + "dtype": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": null, + "coverage": { + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "regions": [ + { + "time_ms": null, + "population": {} + } + ] + } + }, + { + "id": 3, + "payload": { + "kind": "summary_estimate", + "query": { + "TopK": { + "k": 10 + } + } + }, + "result_kind": "instant_vector", + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "topk_10", + "dtype": { + "Plain": "utf8" + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": { + "metric": "relative_value", + "bound": { + "op": "constant", + "value": 0.01 + }, + "failure_probability": { + "op": "constant", + "value": 0.001 + }, + "provenance": [ + { + "kind": "sketch_readout", + "algorithm": "CmsWithHeap", + "contract": "eps-delta frequency bound on heap members", + "params": { + "width": 272, + "depth": 7, + "heap_size": 10 + }, + "query": "TopK { k: 10 }" + } + ] + }, + "coverage": null + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "grouping": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "grouping": "NotApplicable" + }, + { + "producer": 2, + "consumer": 3, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "state", + "dtype": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "grouping": "Identical" + } + ], + "root": { + "Operator": 3 + } + } + }, + { + "id": "L3", + "label": "Hydra over (job, series) + heap per job", + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "kind": "relational", + "operator": { + "kind": "scan", + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + } + } + }, + "result_kind": "instant_vector", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 1, + "payload": { + "kind": "relational", + "operator": { + "kind": "time_range", + "range": { + "secs": 60, + "nanos": 0 + }, + "range_kind": "range" + } + }, + "result_kind": "range_vector", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 2, + "payload": { + "kind": "summary_agg", + "family": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + { + "SharedMultiSubpopulation": { + "kind": "HydraCms", + "params": { + "HydraCms": { + "width": 272, + "depth": 7, + "shared_rows": 5, + "shared_columns": 65536 + } + } + } + } + ] + }, + "input": { + "item": { + "EntityIdentity": { + "PromqlLabelSet": { + "excluding": [] + } + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + }, + "grouping": { + "SharedMultiSubpopulation": { + "kind": "HydraCms", + "params": { + "HydraCms": { + "width": 272, + "depth": 7, + "shared_rows": 5, + "shared_columns": 65536 + } + } + } + }, + "filter": null + }, + "result_kind": "state", + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "state", + "dtype": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + { + "SharedMultiSubpopulation": { + "kind": "HydraCms", + "params": { + "HydraCms": { + "width": 272, + "depth": 7, + "shared_rows": 5, + "shared_columns": 65536 + } + } + } + } + ] + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": null, + "coverage": { + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "regions": [ + { + "time_ms": null, + "population": {} + } + ] + } + }, + { + "id": 3, + "payload": { + "kind": "summary_estimate", + "query": { + "TopK": { + "k": 10 + } + } + }, + "result_kind": "instant_vector", + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "topk_10", + "dtype": { + "Plain": "utf8" + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": { + "metric": "relative_value", + "bound": { + "op": "constant", + "value": 0.01 + }, + "failure_probability": { + "op": "constant", + "value": 0.001 + }, + "provenance": [ + { + "kind": "sketch_readout", + "algorithm": "HydraCms", + "contract": "eps-delta frequency bound on heap members", + "params": { + "width": 272, + "depth": 7, + "shared_rows": 5, + "shared_columns": 65536 + }, + "query": "TopK { k: 10 }" + } + ] + }, + "coverage": null + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "grouping": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "grouping": "NotApplicable" + }, + { + "producer": 2, + "consumer": 3, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "state", + "dtype": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + { + "SharedMultiSubpopulation": { + "kind": "HydraCms", + "params": { + "HydraCms": { + "width": 272, + "depth": 7, + "shared_rows": 5, + "shared_columns": 65536 + } + } + } + } + ] + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "grouping": "Identical" + } + ], + "root": { + "Operator": 3 + } + } + } + ] + }, + "stage2_physical_asap": { + "candidates": [ + { + "id": "P1", + "from_logical": "L1", + "label": "Exact, rebuilt from raw samples at query time", + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "kind": "relational", + "operator": { + "kind": "scan", + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 1, + "payload": { + "kind": "relational", + "operator": { + "kind": "time_range", + "range": { + "secs": 60, + "nanos": 0 + }, + "range_kind": "range" + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 2, + "payload": { + "kind": "relational", + "operator": { + "kind": "aggregate", + "reduction": "PerEntity", + "measures": [ + { + "kind": "sum", + "col": null + } + ], + "output_names": [ + "" + ], + "filters": [], + "having": null + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 3, + "payload": { + "kind": "relational", + "operator": { + "kind": "aggregate", + "reduction": { + "Reduce": [ + 2 + ] + }, + "measures": [ + { + "kind": "top_k", + "k": 10, + "accuracy": "Exact" + } + ], + "output_names": [], + "filters": [], + "having": null + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "topk_10", + "dtype": { + "Plain": "utf8" + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": { + "metric": "absolute_value", + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "provenance": [ + { + "kind": "exact", + "reason": "exact per-series sum and sort within job" + } + ] + }, + "coverage": null + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 2, + "consumer": 3, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + } + ], + "root": 3 + }, + "cost": { + "total": 412.0, + "unit": "cpu_ms_per_s", + "per_node": { + "0": { + "cost": 96.0, + "detail": "read 1 min of raw samples: 4.0M samples per refresh" + }, + "1": { + "cost": 8.0, + "detail": "slice 1 min window" + }, + "2": { + "cost": 180.0, + "detail": "sum per series: 1,000,000 hash-table entries" + }, + "3": { + "cost": 128.0, + "detail": "sort 1,000,000 series within job, keep 10" + } + } + } + }, + { + "id": "P2", + "from_logical": "L2", + "label": "Count-Min + heap, sketch at ingestion time", + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "kind": "relational", + "operator": { + "kind": "scan", + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + } + } + }, + "output_state": { + "timing": "ingestion_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 1, + "payload": { + "kind": "relational", + "operator": { + "kind": "time_range", + "range": { + "secs": 60, + "nanos": 0 + }, + "range_kind": "range" + } + }, + "output_state": { + "timing": "ingestion_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 2, + "payload": { + "kind": "summary_agg", + "family": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "input": { + "item": { + "EntityIdentity": { + "PromqlLabelSet": { + "excluding": [] + } + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + }, + "grouping": "PerSubpopulationInstance", + "filter": null + }, + "output_state": { + "timing": "ingestion_time", + "primitive": "SummaryState" + }, + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "state", + "dtype": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": null, + "coverage": { + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "regions": [ + { + "time_ms": null, + "population": {} + } + ] + } + }, + { + "id": 3, + "payload": { + "kind": "summary_estimate", + "query": { + "TopK": { + "k": 10 + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "topk_10", + "dtype": { + "Plain": "utf8" + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": { + "metric": "relative_value", + "bound": { + "op": "constant", + "value": 0.01 + }, + "failure_probability": { + "op": "constant", + "value": 0.001 + }, + "provenance": [ + { + "kind": "sketch_readout", + "algorithm": "CmsWithHeap", + "contract": "eps-delta frequency bound on heap members", + "params": { + "width": 272, + "depth": 7, + "heap_size": 10 + }, + "query": "TopK { k: 10 }" + } + ] + }, + "coverage": null + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "data_state": { + "timing": "ingestion_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "data_state": { + "timing": "ingestion_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 2, + "consumer": 3, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "state", + "dtype": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "data_state": { + "timing": "ingestion_time", + "primitive": "SummaryState" + }, + "grouping": "Identical", + "window": "NotApplicable" + } + ], + "root": 3 + }, + "cost": { + "total": 18.4, + "unit": "cpu_ms_per_s", + "per_node": { + "0": { + "cost": 0.0, + "detail": "no raw read at query time" + }, + "1": { + "cost": 1.2, + "detail": "assign each sample to its 1 min window" + }, + "2": { + "cost": 15.6, + "detail": "66,667 samples/s x 7 CMS rows, plus heap updates" + }, + "3": { + "cost": 1.6, + "detail": "read 10 heap entries per job" + } + } + } + }, + { + "id": "P3", + "from_logical": "L2", + "label": "Count-Min + heap, rebuilt at query time", + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "kind": "relational", + "operator": { + "kind": "scan", + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 1, + "payload": { + "kind": "relational", + "operator": { + "kind": "time_range", + "range": { + "secs": 60, + "nanos": 0 + }, + "range_kind": "range" + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 2, + "payload": { + "kind": "summary_agg", + "family": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "input": { + "item": { + "EntityIdentity": { + "PromqlLabelSet": { + "excluding": [] + } + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + }, + "grouping": "PerSubpopulationInstance", + "filter": null + }, + "output_state": { + "timing": "query_time", + "primitive": "SummaryState" + }, + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "state", + "dtype": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": null, + "coverage": { + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "regions": [ + { + "time_ms": null, + "population": {} + } + ] + } + }, + { + "id": 3, + "payload": { + "kind": "summary_estimate", + "query": { + "TopK": { + "k": 10 + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "topk_10", + "dtype": { + "Plain": "utf8" + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": { + "metric": "relative_value", + "bound": { + "op": "constant", + "value": 0.01 + }, + "failure_probability": { + "op": "constant", + "value": 0.001 + }, + "provenance": [ + { + "kind": "sketch_readout", + "algorithm": "CmsWithHeap", + "contract": "eps-delta frequency bound on heap members", + "params": { + "width": 272, + "depth": 7, + "heap_size": 10 + }, + "query": "TopK { k: 10 }" + } + ] + }, + "coverage": null + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 2, + "consumer": 3, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "state", + "dtype": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "data_state": { + "timing": "query_time", + "primitive": "SummaryState" + }, + "grouping": "Identical", + "window": "NotApplicable" + } + ], + "root": 3 + }, + "cost": { + "total": 96.4, + "unit": "cpu_ms_per_s", + "per_node": { + "0": { + "cost": 48.0, + "detail": "read 1 min of raw samples per refresh" + }, + "1": { + "cost": 4.0, + "detail": "slice 1 min window" + }, + "2": { + "cost": 42.8, + "detail": "insert 4.0M samples x 7 rows per refresh" + }, + "3": { + "cost": 1.6, + "detail": "read 10 heap entries per job" + } + } + } + }, + { + "id": "P4", + "from_logical": "L3", + "label": "Hydra, sketch at ingestion time", + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "kind": "relational", + "operator": { + "kind": "scan", + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + } + } + }, + "output_state": { + "timing": "ingestion_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 1, + "payload": { + "kind": "relational", + "operator": { + "kind": "time_range", + "range": { + "secs": 60, + "nanos": 0 + }, + "range_kind": "range" + } + }, + "output_state": { + "timing": "ingestion_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "guarantee": null, + "coverage": null + }, + { + "id": 2, + "payload": { + "kind": "summary_agg", + "family": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + { + "SharedMultiSubpopulation": { + "kind": "HydraCms", + "params": { + "HydraCms": { + "width": 272, + "depth": 7, + "shared_rows": 5, + "shared_columns": 65536 + } + } + } + } + ] + }, + "input": { + "item": { + "EntityIdentity": { + "PromqlLabelSet": { + "excluding": [] + } + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + }, + "grouping": { + "SharedMultiSubpopulation": { + "kind": "HydraCms", + "params": { + "HydraCms": { + "width": 272, + "depth": 7, + "shared_rows": 5, + "shared_columns": 65536 + } + } + } + }, + "filter": null + }, + "output_state": { + "timing": "ingestion_time", + "primitive": "SummaryState" + }, + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "state", + "dtype": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + { + "SharedMultiSubpopulation": { + "kind": "HydraCms", + "params": { + "HydraCms": { + "width": 272, + "depth": 7, + "shared_rows": 5, + "shared_columns": 65536 + } + } + } + } + ] + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": null, + "coverage": { + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + }, + "regions": [ + { + "time_ms": null, + "population": {} + } + ] + } + }, + { + "id": 3, + "payload": { + "kind": "summary_estimate", + "query": { + "TopK": { + "k": 10 + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "output_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "topk_10", + "dtype": { + "Plain": "utf8" + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "guarantee": { + "metric": "relative_value", + "bound": { + "op": "constant", + "value": 0.01 + }, + "failure_probability": { + "op": "constant", + "value": 0.001 + }, + "provenance": [ + { + "kind": "sketch_readout", + "algorithm": "HydraCms", + "contract": "eps-delta frequency bound on heap members", + "params": { + "width": 272, + "depth": 7, + "shared_rows": 5, + "shared_columns": 65536 + }, + "query": "TopK { k: 10 }" + } + ] + }, + "coverage": null + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "data_state": { + "timing": "ingestion_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false, + "table": null + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false, + "table": null + }, + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + } + ], + "time_index": 0, + "unique_keys": [], + "closed": false + }, + "data_state": { + "timing": "ingestion_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 2, + "consumer": 3, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "job", + "dtype": { + "Plain": "utf8" + }, + "nullable": true, + "table": null + }, + { + "name": "state", + "dtype": { + "Sketch": [ + { + "category": "TopK", + "algorithm": "CmsWithHeap", + "params": { + "CmsWithHeap": { + "width": 272, + "depth": 7, + "heap_size": 10 + } + } + }, + { + "SharedMultiSubpopulation": { + "kind": "HydraCms", + "params": { + "HydraCms": { + "width": 272, + "depth": 7, + "shared_rows": 5, + "shared_columns": 65536 + } + } + } + } + ] + }, + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ], + "closed": true + }, + "data_state": { + "timing": "ingestion_time", + "primitive": "SummaryState" + }, + "grouping": "Identical", + "window": "NotApplicable" + } + ], + "root": 3 + }, + "cost": { + "total": 11.2, + "unit": "cpu_ms_per_s", + "per_node": { + "0": { + "cost": 0.0, + "detail": "no raw read at query time" + }, + "1": { + "cost": 1.2, + "detail": "assign each sample to its 1 min window" + }, + "2": { + "cost": 8.1, + "detail": "66,667 samples/s x 5 shared rows, one structure for all jobs" + }, + "3": { + "cost": 1.9, + "detail": "query shared grid and heap per job" + } + } + } + } + ] + }, + "stage3_selection": { + "selected": "P4", + "rejected": [ + { + "id": "P1", + "reason": "Estimated 340 ms latency misses the 100 ms bound: exact top 10 is rebuilt from 1,000,000 series at every refresh." + }, + { + "id": "P3", + "reason": "Valid but more expensive: 96.4 vs 11.2 cpu_ms_per_s for the selected plan." + } + ] + } +} diff --git a/tools/dag-viewer/index.html b/tools/dag-viewer/index.html index 17754d28..d394a82d 100644 --- a/tools/dag-viewer/index.html +++ b/tools/dag-viewer/index.html @@ -8,6 +8,7 @@ +