From 549de627ca7b7bd30306b7e235e46a66f4647f92 Mon Sep 17 00:00:00 2001 From: ottomansky Date: Thu, 14 May 2026 23:46:21 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(0.41.0):=20kbagent=20semantic-layer=20?= =?UTF-8?q?command=20group=20=E2=80=94=20first-class=20metastore=20CLI=20s?= =?UTF-8?q?urface?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased onto upstream/main after #283 (web UI + serve --ui + agent-task persistence) landed. Originally implemented as 30 incremental commits on 0.34.0; collapsed to a single commit on top of 0.40.0 and version-bumped to 0.41.0. New: `kbagent semantic-layer` (alias `kbagent sl`) -- first-class CLI surface for the Keboola Metastore (semantic layer). Folds every metastore operation that was previously a hand-rolled urllib script in the `sl-builder` Claude Code plugin into a permission-gated, JSON-capable, hint-aware kbagent command group. The metastore URL is derived from each project's stack URL automatically (`connection.` -> `metastore.` -- region/cloud-agnostic), and reuses the same `X-StorageApi-Token` credential kbagent already holds. Subcommands shipped in this release: `show` (list a model's entities -- read), `validate` (structural checks; `--deep` adds parallel Snowflake column probing via the in-process StorageService -- read), `export` (snapshot the model to JSON -- read), `diff` (project-vs-project, project-vs-file, or file-vs-file -- read), `model create` / `model delete` (model lifecycle -- write/destructive), `add metric|dataset|relationship|constraint|glossary` (single-entity creates with FQN derivation, role heuristics, and constraint-orphan validation -- write), `edit` (DELETE+POST with cascade rename of dependent constraints and explicit rollback -- write), `remove` (destructive with mandatory constraint-orphan pre-flight warning), `import` (replay a snapshot with conflict detection -- write), `promote` (cross-project copy with modelUUID rewrite and identical-vs-changed-vs-new classification -- write), `build` (non-interactive AI-assisted greenfield via the existing AI Service client; fixes the long-standing sl-build bug where `semantic-constraint` items were silently dropped from the push loop -- write), and `token --encrypt` (encrypt the project token for a transformation container's user_properties using the existing EncryptService -- write, parity with `encrypt.values`). Verified live against the metastore API on 2026-05-14: response shape `{data: [...]}` for lists / `{data: {type, id, attributes, meta}}` for items; POST envelope `{name, data, branch:'main', schemaVersion:'1.0.0', scope:'project'}`; DELETE returns 204; duplicate-name POST returns 500 with `Failed to create meta object` (normalized to ALREADY_EXISTS); constraint `rule` is a STRING expression (NOT the `ruleExpression` object sl-builder documents); `constraintType` enum is `inequality|equality|range|composition|exclusion|temporal|conditional`; constraint `name` regex is `^[a-z][a-z0-9_]*$`; 4-band health (`_critical/_warning/_healthy/_review`) lives only in the constraint name suffix because the API `severity` enum is 3-level. Permission registry adds entries for every subcommand (with `model`, `add`, `edit`, `remove` split into per-leaf keys so e.g. `model list` stays read-only under `--deny-writes` and every `remove.*` leaf is classified `destructive`). Hint definitions ship for every subcommand (`kbagent --hint client semantic-layer ` and `--hint service` both emit Python snippets). `edit` and `remove` cover all five entity types (metric|dataset|constraint|relationship|glossary) -- the parity gap with `add` is closed. Plugin/agent/skill docs updated (commands-reference, gotchas tagged `(since v0.41.0)`, keboola-expert version gate + tool-selection matrix, new `semantic-layer-workflow.md`). E2E coverage in `tests/test_e2e.py` bootstraps a throwaway `kbagent_e2e_*` model on `e2e-1143`, exercises every command, and tears down in `finally`. --- .claude-plugin/marketplace.json | 2 +- CLAUDE.md | 30 +- Makefile | 2 +- plugins/kbagent/.claude-plugin/plugin.json | 2 +- plugins/kbagent/agents/keboola-expert.md | 51 + plugins/kbagent/skills/kbagent/SKILL.md | 66 +- .../kbagent/references/commands-reference.md | 32 + .../skills/kbagent/references/gotchas.md | 122 + .../references/semantic-layer-workflow.md | 443 ++++ pyproject.toml | 2 +- src/keboola_agent_cli/changelog.py | 3 + src/keboola_agent_cli/cli.py | 6 + src/keboola_agent_cli/commands/_helpers.py | 9 + .../commands/_semantic_layer_crud.py | 784 +++++++ .../commands/_semantic_layer_helpers.py | 44 + src/keboola_agent_cli/commands/context.py | 86 + .../commands/semantic_layer.py | 902 ++++++++ .../hints/definitions/__init__.py | 1 + .../hints/definitions/semantic_layer.py | 729 ++++++ src/keboola_agent_cli/hints/renderer.py | 24 +- src/keboola_agent_cli/metastore_client.py | 168 ++ src/keboola_agent_cli/permissions.py | 48 + src/keboola_agent_cli/server/__init__.py | 3 + src/keboola_agent_cli/server/dependencies.py | 6 + .../server/routers/semantic_layer.py | 550 +++++ .../services/_semantic_layer_crud.py | 360 +++ .../services/_semantic_layer_internals.py | 839 +++++++ .../services/semantic_layer_service.py | 1519 +++++++++++++ tests/test_e2e.py | 1059 +++++++++ tests/test_metastore_client.py | 235 ++ tests/test_permissions.py | 43 +- tests/test_semantic_layer_cli.py | 1471 ++++++++++++ tests/test_semantic_layer_service.py | 1980 +++++++++++++++++ .../test_server_semantic_layer_routes_e2e.py | 574 +++++ tests/test_server_smoke.py | 117 + uv.lock | 2 +- 36 files changed, 12295 insertions(+), 19 deletions(-) create mode 100644 plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md create mode 100644 src/keboola_agent_cli/commands/_semantic_layer_crud.py create mode 100644 src/keboola_agent_cli/commands/_semantic_layer_helpers.py create mode 100644 src/keboola_agent_cli/commands/semantic_layer.py create mode 100644 src/keboola_agent_cli/hints/definitions/semantic_layer.py create mode 100644 src/keboola_agent_cli/metastore_client.py create mode 100644 src/keboola_agent_cli/server/routers/semantic_layer.py create mode 100644 src/keboola_agent_cli/services/_semantic_layer_crud.py create mode 100644 src/keboola_agent_cli/services/_semantic_layer_internals.py create mode 100644 src/keboola_agent_cli/services/semantic_layer_service.py create mode 100644 tests/test_metastore_client.py create mode 100644 tests/test_semantic_layer_cli.py create mode 100644 tests/test_semantic_layer_service.py create mode 100644 tests/test_server_semantic_layer_routes_e2e.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index a1172e13..045f3231 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "kbagent", - "version": "0.40.3", + "version": "0.41.0", "source": "./plugins/kbagent", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "category": "development" diff --git a/CLAUDE.md b/CLAUDE.md index 122627fa..b25998c0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -214,7 +214,7 @@ All three inherit from `BaseHttpClient` (`http_base.py`) which provides shared r - `plugins/kbagent/skills/kbagent/SKILL.md` -- description triggers and workflow links (the auto-generated table is CI-checked, the rest is not) - `plugins/kbagent/skills/kbagent/references/commands-reference.md` - `plugins/kbagent/skills/kbagent/references/gotchas.md` (every new gotcha **MUST** be tagged with `(since vX.Y.Z)`) - - `plugins/kbagent/skills/kbagent/references/-workflow.md` + - `plugins/kbagent/skills/kbagent/references/-workflow.md` (e.g. `semantic-layer-workflow.md`, `workspace-workflow.md`, `sync-workflow.md`) Forgetting any of these does not fail tests or lint -- it ships an AI agent that quietly recommends commands that do not exist on the user's installed kbagent version, or refuses commands that do. Treat the change as **not done** until every applicable file has been updated. @@ -387,6 +387,34 @@ kbagent config new --component-id ID [--name NAME] [--project NAME] [--output-di kbagent encrypt values --project ALIAS --component-id ID --input JSON|@file|- [--output-file PATH] +kbagent semantic-layer model list --project P +kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] +kbagent semantic-layer model delete --project P --model M [--yes] +kbagent semantic-layer show --project P [--model M] [--type dataset|metric|relationship|constraint|glossary] +kbagent semantic-layer validate --project P [--model M] [--deep] +kbagent semantic-layer export --project P [--model M] [--output PATH] +kbagent semantic-layer diff (--project-a A | --file-a PATH) (--project-b B | --file-b PATH) [--model-a M] [--model-b M] +kbagent semantic-layer add metric --project P [--model M] --name N --sql SQL --dataset TABLE_ID [--description D] [--yes] +kbagent semantic-layer add dataset --project P [--model M] --name N --table-id TABLE_ID [--description D] [--grain G] [--primary-key COL ...] [--deep-fields] +kbagent semantic-layer add relationship --project P [--model M] --name N --from TABLE_ID --to TABLE_ID --on EXPR [--type left|inner] +kbagent semantic-layer add constraint --project P [--model M] --name N --constraint-type inequality|equality|range|composition|exclusion|temporal|conditional --rule "EXPR" --metrics M1,M2 [--severity error|warning|info] +kbagent semantic-layer add glossary --project P [--model M] --term TERM [--definition D] +kbagent semantic-layer edit metric --project P [--model M] --name N [--new-name N2] [--new-sql SQL] [--new-dataset TABLE_ID] [--new-description D] [--yes] +kbagent semantic-layer edit dataset --project P [--model M] --name N [--new-name N2] [--new-description D] [--new-grain G] +kbagent semantic-layer edit constraint --project P [--model M] --name N [--new-name N2] [--new-rule "EXPR"] [--new-constraint-type T] [--new-severity error|warning|info] [--new-metrics M1,M2] +kbagent semantic-layer edit relationship --project P [--model M] --name N [--new-name N2] [--new-from TABLE_ID] [--new-to TABLE_ID] [--new-on EXPR] [--new-type left|inner] +kbagent semantic-layer edit glossary --project P [--model M] --term TERM [--new-term TERM2] [--new-definition D] [--yes] +kbagent semantic-layer remove metric --project P [--model M] --name N [--yes] +kbagent semantic-layer remove dataset --project P [--model M] --name N [--yes] +kbagent semantic-layer remove constraint --project P [--model M] --name N [--yes] +kbagent semantic-layer remove relationship --project P [--model M] --name N [--yes] +kbagent semantic-layer remove glossary --project P [--model M] --term TERM [--yes] +kbagent semantic-layer import --project P --file PATH [--model M] [--types T,T,...] [--dry-run] [--yes] [--overwrite] +kbagent semantic-layer promote --from-project A --to-project B [--from-model M] [--to-model M] [--types T,T,...] [--dry-run] [--yes] +kbagent semantic-layer build --project P [--model M] --tables T,T,... [--name N] [--dry-run] [--output PATH] +kbagent semantic-layer token --encrypt --project P --component-id C +# Alias: `kbagent sl ...` (hidden) is equivalent to `kbagent semantic-layer ...`. + kbagent http get PATH [--timeout SECONDS] kbagent http post PATH [--body JSON|@file|-] [--timeout SECONDS] kbagent http patch PATH [--body JSON|@file|-] [--timeout SECONDS] diff --git a/Makefile b/Makefile index 8cdb3a2b..716b52e4 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ test-integration: ## Run integration tests only uv run pytest tests/ -v -m integration test-e2e: ## Run E2E tests (E2E_API_TOKEN and E2E_URL required) - uv run pytest tests/test_e2e.py -v -s --tb=long + uv run pytest tests/test_e2e.py tests/test_server_semantic_layer_routes_e2e.py -v -s --tb=long test-e2e-invite: ## Run project invite E2E (E2E_MANAGE_TOKEN + E2E_INVITE_PROJECT_ID required) uv run pytest tests/test_e2e.py -v -s --tb=long -m e2e_invite diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index 9681ffd5..c6d15a97 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.40.3", + "version": "0.41.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 77f70bdd..c93ff08f 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -82,6 +82,20 @@ a critical failure. `data-app *` JSON output uses key `app_id` (was bare `id`) on 0.33.0+ -- pipe with `jq -r '.apps[].app_id'`, not `'.id'`, `config new --push` (one-shot remote create) needs 0.33.0+, + `semantic-layer` command group needs 0.41.0+: + - model lifecycle: `model list / create / delete` + - read: `show`, `validate [--deep]`, `export`, `diff` + - write: `add metric|dataset|relationship|constraint|glossary`, + `edit metric|dataset|constraint|relationship|glossary`, + `import`, `promote`, `build`, `token --encrypt` + - destructive: `remove metric|dataset|constraint|relationship|glossary` + - alias: `kbagent sl ...` is hidden-equivalent to + `kbagent semantic-layer ...` + - `semantic-layer build` falls back to a deterministic heuristic + (one dataset + one COUNT(*) metric + one glossary entry per + table) until an AI Service JSON-generation endpoint exists; + this is a BEHAVIOR note, not a version gate -- the heuristic + is the only path on 0.41.0, `kbagent http get/post/patch/delete ` (self-call against the running serve from a scheduled-agent subprocess; reads `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` env vars) needs 0.40.0+, @@ -157,6 +171,17 @@ a critical failure. | Call the running `kbagent serve` from a scheduled-agent subprocess | `kbagent http get/post/patch/delete ` (0.40.0+) -- uses `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` env vars auto-injected by the scheduler. `kbagent http get /openapi.json` to discover endpoints. Treats the live serve as source-of-truth (no stale local config) | forking `kbagent ` (also fine -- `KBAGENT_CONFIG_DIR` is propagated so the spawned CLI sees the SAME config the serve uses; no more "I'm in the wrong directory" surprises) | `curl $KBAGENT_SERVE_URL/...` by hand (works, but `kbagent http` adds auth header automatically, structured error mapping, and JSON-mode formatting) | | Launch the web UI for an end-user (browser dashboard, no Node BFF) | `kbagent serve --ui [--port PORT] [--ui-dist PATH]` (0.40.0+) -- single-process FastAPI mounts the bundled React SPA at `/`, sets an HttpOnly `kbagent_session` cookie on `GET /` so the browser is auto-authenticated. EventSource SSE works via the same cookie -- no token in URL, JS heap, or access log. Requires the bundled wheel (Node 20+ on the install host) OR `make web-build` from a checkout. CORS origins customisable via `--cors-origin` | `kbagent serve` (plain API) + Vite dev server + Node BFF -- the legacy three-process flow with hot reload, see `web/README.md` "Dev mode" section | inventing a `--token-in-url` flag; running uvicorn directly against `web.frontend.dist` -- the path-rewrite middleware + cookie bootstrap only fire from `kbagent serve --ui` | | Schedule / manage Agent Tasks (cron, manual, chained) inside `kbagent serve` | `kbagent http /agents...` (0.40.0+) -- list `GET /agents`, create `POST /agents`, update `PATCH /agents/{id}`, run-on-demand `POST /agents/{id}/run`, run history `GET /agents/{id}/runs`, replay events `GET /agents/{id}/runs/{run_id}/events`. Three action flavours: `mcp_tool` / `cli_command` / `ai_agent`. **See [agent-tasks-workflow](../skills/kbagent/references/agent-tasks-workflow.md) for full payload schemas and chained-trigger setup** | Web UI sidebar "Agent Tasks" -- preferred for human authoring; UI calls the same REST endpoints | hand-editing `~/.config/keboola-agent-cli/agents.json` (no schema validation, no scheduler reload, easy to break the cron loop) | +| List models / metrics / entities in a semantic-layer model | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]` (0.41.0+); `kbagent --json semantic-layer model list --project P` to enumerate models when --model is ambiguous | `kbagent --json tool call get_semantic_layer_*` if the MCP exposes a read tool (none in the kbagent MCP at v0.41.0) | hand-rolled `urllib`/`httpx` loops against `metastore.*.keboola.com` (the `sl-builder` skill's old approach -- bypasses retry/backoff and the kbagent error envelope) | +| Validate a semantic-layer model (phantom fields, constraint orphans, AGG-on-STRING) | `kbagent --json semantic-layer validate --project P [--model M] [--deep]` (0.41.0+) -- basic = local structural checks (duplicates, dangling refs, sum-on-pct, constraint orphans, severity-suffix); `--deep` adds parallel Snowflake column-existence probes via the in-process StorageService | hand-coded list+filter Python that re-implements the structural checks (loses the `--deep` Snowflake probe) | running validation by spinning up a workspace and SELECT * FROM every dataset (slow, requires workspace creation, no constraint-orphan detection) | +| Snapshot a semantic-layer model to disk (before destructive edits) | `kbagent semantic-layer export --project P [--model M] [--output PATH]` (0.41.0+) -- self-describing JSON, default `./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json` | `kbagent --json semantic-layer show --project P` and pipe to a file (NOT a clean snapshot -- missing model metadata, no schemaVersion, no round-trip guarantee) | -- | +| Diff a dev model against prod / against a snapshot | `kbagent --json semantic-layer diff --project-a dev --project-b prod` (project<->project); swap one side for `--file-a` / `--file-b` to diff against a snapshot (0.41.0+) | export both, run `diff` / `jq` on the JSON manually (no per-type added/removed/changed grouping, no `diff_keys`) | -- | +| Add a metric / dataset / relationship / constraint / glossary to a model | `kbagent semantic-layer add metric\|dataset\|relationship\|constraint\|glossary --project P [--model M] ...` (0.41.0+) -- five sub-subcommands. For datasets, FQN is auto-derived from `--table-id`; `--deep-fields` synthesises role-classified `fields[]`. For constraints, `--rule` is a **STRING expression** (e.g. `"value >= 0"`), name regex `^[a-z][a-z0-9_]*$`, severity ∈ `error\|warning\|info` (3-level API enum -- the 4-band health convention lives in the NAME suffix `_critical\|_warning\|_healthy\|_review`) | -- | raw `POST metastore.*.keboola.com/v1/api/...` calls inside the `sl-builder` skill (bypasses the duplicate-name 500-to-ALREADY_EXISTS normalization and the constraint-shape validators) | +| Rename a metric safely (cascade through constraints) | `kbagent semantic-layer edit metric --project P [--model M] --name OLD --new-name NEW` (0.41.0+) -- DELETE+POST with rollback; cascades through every constraint whose `metrics[]` includes the old name (DELETE old + POST new with updated metrics[]); prints the old/new CODE_METRIC value so the operator can audit downstream SQL joins; `--yes` to skip confirm | manual `remove metric` + `add metric` with no cascade (orphans every constraint that referenced the metric, silently breaks `DIM_METRIC_THRESHOLD`) | editing the metric via `tool call update_config` against the metastore (no PATCH on the metastore -- only DELETE+POST works, and rolls back on POST failure only via kbagent) | +| Remove a metric (with orphan-check) | `kbagent semantic-layer remove metric --project P [--model M] --name N [--yes]` (0.41.0+) -- pre-deletion scan lists constraints that would become orphaned; warning is always printed (even with `--yes`); non-TTY without `--yes` refuses with exit 2 | `kbagent semantic-layer edit metric --new-name _DELETED_` (soft-delete; keeps the constraint refs valid but pollutes the model) | raw `DELETE` against the metastore (skips the orphan warning -- the constraint pointing at the deleted metric stays but creates a dangling FK in `DIM_METRIC_THRESHOLD` downstream) | +| Restore a model from a snapshot (after accidental destructive edit) | `kbagent semantic-layer import --project P --file PATH --dry-run` to preview classifications, then re-run without `--dry-run` (0.41.0+); default skip-on-conflict, add `--overwrite` to DELETE+POST conflicting items; dependency-ordered push (datasets -> metrics -> relationships -> glossary -> constraints) | `semantic-layer promote --from-project source` if you still have the source project handy (uses the same write loop but without snapshot indirection) | replaying the snapshot via a shell loop of `add` subcommands (loses the conflict-classification step and the dependency-ordered push) | +| Promote a model dev -> prod (cross-project copy) | `kbagent --json semantic-layer promote --from-project dev --to-project prod --dry-run` (0.41.0+) to classify NEW/IDENTICAL/CHANGED, review the `changes[]` and `failed[]` lists, then re-run without `--dry-run`; deep-equality strips modelUUID + timestamps; **NEVER deletes target items absent from source** (additive + overwrite only) | `semantic-layer export` from source + `semantic-layer import --overwrite` into target (two-step -- equivalent end state but you lose the IDENTICAL classification) | hand-rolled cross-project copy via raw metastore calls (no modelUUID rewrite -- the target ends up with foreign UUIDs and validation fails downstream) | +| Bootstrap a model from a set of storage tables | `kbagent semantic-layer build --project P --tables T1,T2,... [--dry-run]` (0.41.0+) -- **HEURISTIC fallback only** (no AI Service JSON endpoint as of v0.41.0): synthesises one dataset + one COUNT(*) metric + one glossary entry per table; FQN derived; fields[] role-classified. Response carries `fallback_used: "heuristic"`. Use as a SCAFFOLD, then refine via `add` / `edit` | the `sl-build` skill in `04_AI_Kit/ai-kit` -- full AI-assisted greenfield wizard, schema discovery + SQL analysis + AI generation. Use this when you need richer metrics, relationships, and constraint shapes than the heuristic produces | hand-writing the model JSON from scratch (the `build` heuristic gets you 80% of the way for read-mostly star schemas; only fall back to manual when the heuristic refuses or you need something the skill produces) | +| Encrypt the storage token for a transformation `user_properties` (so a Python container can reach the metastore) | `kbagent semantic-layer token --encrypt --project P --component-id C` (0.41.0+) -- builds `{"#metastore_token": }` from the project's already-stored Storage token and delegates to the existing EncryptService; output is the encrypted envelope ready to paste into the transformation's `user_properties` block | `kbagent encrypt values --project P --component-id C --input '{"#metastore_token": ""}'` (works but the operator has to manually fetch the token first -- the wrapper avoids that step) | hand-running the Encryption API and pasting plaintext into `user_properties` (no `#` prefix means it sits in the config in plaintext) | If the table does not cover the user's task, **ask clarifying questions** instead of guessing. Returning a targeted question is a @@ -438,6 +463,32 @@ success, not a failure. `--allow-env-manage-token` to their invocation, never strip the warning by suppressing stderr. +- **Semantic-layer gotchas (since v0.41.0)** — five behavior contracts + worth committing to memory before touching `semantic-layer add/edit/ + remove`. Full prose lives in + [`gotchas.md` § Semantic-layer](../skills/kbagent/references/gotchas.md); + the short form: + - **Constraint `rule` is a STRING**, never `{bounds: {min, max}}`. The + sl-builder skill docs are wrong on this. kbagent enforces it. + - **Constraint `name` regex `^[a-z][a-z0-9_]*$`** + the 3-vs-4 + severity split: API `severity` is `error | warning | info` (3-level); + the 4-band health (`_critical / _warning / _healthy / _review`) + lives in the NAME SUFFIX, not on the API. + - **`edit metric --new-name` cascades through every constraint** whose + `metrics[]` referenced the old name, and prints the old/new + CODE_METRIC value. Downstream SQL joining on CODE_METRIC will break + silently — surface the change to the operator. + - **`remove metric` orphans constraints** that reference it. The + pre-deletion scan ALWAYS prints the warning (even with `--yes`); + non-TTY without `--yes` exits 2. Recommended: drop/rewrite the + constraints first, then remove the metric. + - **`build` is a HEURISTIC fallback**, not full AI: one dataset + + one COUNT(*) metric + one glossary entry per table. Response carries + `fallback_used: "heuristic"`. Treat the output as a scaffold and + follow up with `add metric`, `add relationship`, `add constraint`. + The full AI wizard lives in the `sl-build` skill under + `04_AI_Kit/ai-kit/`. + --- ## 4. WORKFLOWS (reference playbooks) diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index b8e4ebf9..40ad3c15 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -38,7 +38,18 @@ description: > invite user, invite member, project invitation, manage members, list members, remove member, change role, project role, bulk invite, invite from CSV, project access, member management, - manage token prompt, --allow-env-manage-token, KBC_MANAGE_API_TOKEN. + manage token prompt, --allow-env-manage-token, KBC_MANAGE_API_TOKEN, + semantic-layer, semantic layer, semantic-layer model, metastore, + semantic-metric, semantic-dataset, semantic-relationship, + semantic-constraint, semantic-glossary, add metric, edit metric, + rename metric, remove metric, validate model, validate semantic layer, + promote model, semantic-layer build, semantic-layer export, + semantic-layer diff, semantic-layer import, semantic-layer token, + metric SQL, dataset FQN, constraint rule, threshold constraint, + 4-band health, _critical _warning _healthy _review, CODE_METRIC, + DIM_METRIC_THRESHOLD, dangling metric FK, orphaned constraint, + phantom field, AGG on STRING, SUM on PCT, deep validate, + sl, kbagent sl, semantic layer wizard, sl-build, sl-add, sl-edit. --- # kbagent -- Keboola Agent CLI @@ -231,6 +242,58 @@ When working inside a git repository or project directory, run `kbagent init` (o | Remove the branch mapping for the current git branch | `kbagent sync branch-unlink` | | Show the branch mapping status for the current git branch | `kbagent sync branch-status` | | Encrypt #-prefixed secret values for a Keboola component | `kbagent encrypt values --project PROJECT --component-id COMPONENT-ID --input INPUT-DATA` | +| Encrypt the project's storage token for transformation `user_properties` | `kbagent semantic-layer token --project PROJECT --component-id COMPONENT-ID` | +| Build a semantic-layer model from a list of storage tables (non-interactive) | `kbagent semantic-layer build --project PROJECT` | +| Promote a model from one project to another (NEW + overwrite CHANGED; never deletes) | `kbagent semantic-layer promote --from-project FROM-PROJECT --to-project TO-PROJECT` | +| Replay a snapshot into a project. | `kbagent semantic-layer import --project PROJECT --file FILE` | +| Show the entities in a semantic-layer model | `kbagent semantic-layer show --project PROJECT` | +| Snapshot a semantic-layer model to a self-describing JSON file | `kbagent semantic-layer export --project PROJECT` | +| Diff two semantic-layer snapshots (project↔project, project↔file, file↔file) | `kbagent semantic-layer diff` | +| Validate a semantic-layer model | `kbagent semantic-layer validate --project PROJECT` | +| List all semantic-layer models in a project | `kbagent semantic-layer model list --project PROJECT` | +| Create a new semantic-layer model | `kbagent semantic-layer model create --project PROJECT --name NAME` | +| Delete a semantic-layer model. | `kbagent semantic-layer model delete --project PROJECT --model MODEL` | +| Add a metric to a semantic-layer model | `kbagent semantic-layer add metric --project PROJECT --name NAME --sql SQL --dataset DATASET` | +| Add a dataset (FQN derived from tableId) | `kbagent semantic-layer add dataset --project PROJECT --name NAME --table-id TABLE-ID` | +| Add a relationship between two datasets | `kbagent semantic-layer add relationship --project PROJECT --name NAME --from FROM- --to TO --on ON` | +| Add a constraint | `kbagent semantic-layer add constraint --project PROJECT --name NAME --constraint-type CONSTRAINT-TYPE --rule RULE --metrics METRICS` | +| Add a glossary term | `kbagent semantic-layer add glossary --project PROJECT --term TERM` | +| Edit a metric. | `kbagent semantic-layer edit metric --project PROJECT --name NAME` | +| Edit a dataset (no cascade — metric.dataset uses tableId, not name) | `kbagent semantic-layer edit dataset --project PROJECT --name NAME` | +| Edit a constraint (DELETE+POST, with local validators) | `kbagent semantic-layer edit constraint --project PROJECT --name NAME` | +| Edit a relationship (DELETE+POST). | `kbagent semantic-layer edit relationship --project PROJECT --name NAME` | +| Edit a glossary term. | `kbagent semantic-layer edit glossary --project PROJECT --term TERM` | +| Remove a metric. | `kbagent semantic-layer remove metric --project PROJECT --name NAME` | +| Remove a dataset | `kbagent semantic-layer remove dataset --project PROJECT --name NAME` | +| Remove a constraint | `kbagent semantic-layer remove constraint --project PROJECT --name NAME` | +| Remove a relationship. | `kbagent semantic-layer remove relationship --project PROJECT --name NAME` | +| Remove a glossary term. | `kbagent semantic-layer remove glossary --project PROJECT --term TERM` | +| Encrypt the project's storage token for transformation `user_properties` | `kbagent sl token --project PROJECT --component-id COMPONENT-ID` | +| Build a semantic-layer model from a list of storage tables (non-interactive) | `kbagent sl build --project PROJECT` | +| Promote a model from one project to another (NEW + overwrite CHANGED; never deletes) | `kbagent sl promote --from-project FROM-PROJECT --to-project TO-PROJECT` | +| Replay a snapshot into a project. | `kbagent sl import --project PROJECT --file FILE` | +| Show the entities in a semantic-layer model | `kbagent sl show --project PROJECT` | +| Snapshot a semantic-layer model to a self-describing JSON file | `kbagent sl export --project PROJECT` | +| Diff two semantic-layer snapshots (project↔project, project↔file, file↔file) | `kbagent sl diff` | +| Validate a semantic-layer model | `kbagent sl validate --project PROJECT` | +| List all semantic-layer models in a project | `kbagent sl model list --project PROJECT` | +| Create a new semantic-layer model | `kbagent sl model create --project PROJECT --name NAME` | +| Delete a semantic-layer model. | `kbagent sl model delete --project PROJECT --model MODEL` | +| Add a metric to a semantic-layer model | `kbagent sl add metric --project PROJECT --name NAME --sql SQL --dataset DATASET` | +| Add a dataset (FQN derived from tableId) | `kbagent sl add dataset --project PROJECT --name NAME --table-id TABLE-ID` | +| Add a relationship between two datasets | `kbagent sl add relationship --project PROJECT --name NAME --from FROM- --to TO --on ON` | +| Add a constraint | `kbagent sl add constraint --project PROJECT --name NAME --constraint-type CONSTRAINT-TYPE --rule RULE --metrics METRICS` | +| Add a glossary term | `kbagent sl add glossary --project PROJECT --term TERM` | +| Edit a metric. | `kbagent sl edit metric --project PROJECT --name NAME` | +| Edit a dataset (no cascade — metric.dataset uses tableId, not name) | `kbagent sl edit dataset --project PROJECT --name NAME` | +| Edit a constraint (DELETE+POST, with local validators) | `kbagent sl edit constraint --project PROJECT --name NAME` | +| Edit a relationship (DELETE+POST). | `kbagent sl edit relationship --project PROJECT --name NAME` | +| Edit a glossary term. | `kbagent sl edit glossary --project PROJECT --term TERM` | +| Remove a metric. | `kbagent sl remove metric --project PROJECT --name NAME` | +| Remove a dataset | `kbagent sl remove dataset --project PROJECT --name NAME` | +| Remove a constraint | `kbagent sl remove constraint --project PROJECT --name NAME` | +| Remove a relationship. | `kbagent sl remove relationship --project PROJECT --name NAME` | +| Remove a glossary term. | `kbagent sl remove glossary --project PROJECT --term TERM` | | GET an endpoint on the running kbagent serve | `kbagent http get <PATH>` | | POST to an endpoint on the running kbagent serve | `kbagent http post <PATH>` | | PATCH an endpoint on the running kbagent serve | `kbagent http patch <PATH>` | @@ -289,6 +352,7 @@ For detailed response parsing rules and common pitfalls, see [gotchas](reference | **Variables (attach to any config)** | [variables-workflow](references/variables-workflow.md) | | Reading synced data | [reading-synced-data](references/reading-synced-data.md) | | SQL migration (input mapping removal) | [sql-migration-workflow](references/sql-migration-workflow.md) | +| **Semantic layer (metastore)** -- models, metrics, datasets, constraints, glossary; validate / export / diff / promote / build / token | [semantic-layer-workflow](references/semantic-layer-workflow.md) | | Response parsing gotchas | [gotchas](references/gotchas.md) | ## First-time setup diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 4016889f..72863912 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -196,6 +196,38 @@ Requires the project to be added with its **master ('owner') Storage API token** ## Encryption - `encrypt values --project ALIAS --component-id ID --input JSON|@file|- [--output-file PATH]` -- encrypt #-prefixed secrets via Keboola Encryption API (one-way, no decrypt). Scope: ComponentSecure (project + component). Use for MCP tool call workflows. +## Semantic Layer (Metastore) (since v0.41.0) + +-Reads `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` env vars. The scheduler auto-injects these (plus `KBAGENT_CONFIG_DIR`) into every AI-agent / `cli_command` subprocess. Outside a serve subprocess context the command refuses to run with exit code 2. **Inside a scheduled-agent task, prefer `kbagent http get /openapi.json` then a typed call over forking another `kbagent` CLI -- the HTTP path always sees the operator's live config (not the global `~/.config/keboola-agent-cli/` one).** +Manage Keboola metastore models -- datasets, metrics, relationships, constraints, glossary terms. Metastore URL derived from the stack URL by replacing `connection.` with `metastore.` (cloud/region-agnostic). Auth: same `X-StorageApi-Token` as Storage. Hidden alias: `kbagent sl ...` is equivalent to `kbagent semantic-layer ...`. See [semantic-layer-workflow.md](semantic-layer-workflow.md) for full recipes. + +- `semantic-layer model list --project P` -- list all models in a project. Output: `{models: [{id, name, sql_dialect, description}, ...]}`. Use to disambiguate when `--model` is required and the project has more than one model. +- `semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake]` -- create a new model. `--sql-dialect` defaults to `Snowflake`. Returns the new model UUID; subsequent commands accept either name or UUID via `--model`. +- `semantic-layer model delete --project P --model M [--yes]` -- delete a model. Fails if the model has children (datasets, metrics, etc.) -- the Storage API rejects with 500 / "model not empty". Confirmation prompt unless `--yes`. +- `semantic-layer show --project P [--model M] [--type T]` -- show a model's entities. `--type` filters to `dataset | metric | relationship | constraint | glossary`. Without `--type` prints a per-type count summary. `--model` is optional when the project has exactly one model. +- `semantic-layer validate --project P [--model M] [--deep]` -- structural validation. Basic mode runs local checks: duplicate names, dangling rel/metric refs, SUM-on-PCT (warning), constraint orphans (metrics in `metrics[]` that no longer exist), severity-suffix mismatches between API `severity` and the 4-band name suffix. `--deep` adds parallel Snowflake column-existence probes via the in-process StorageService: phantom dataset fields, phantom column refs in metric SQL, AGG-on-STRING errors. Response: `{valid: bool, deep: bool, errors: [{type, item, detail}], warnings: [...]}`. +- `semantic-layer export --project P [--model M] [--output PATH]` -- snapshot the model to a self-describing JSON file (default `./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json`). Schema-versioned for round-trip via `import` / `diff`. +- `semantic-layer diff (--project-a A | --file-a P) (--project-b B | --file-b P) [--model-a M] [--model-b M]` -- three-way diff: project<->project, project<->file, file<->file. Mutually exclusive per side: pass exactly one of `--project-a` / `--file-a`, ditto for B. Output groups changes per entity type: `added[] / removed[] / changed[{name, diff_keys[]}]`. +- `semantic-layer add metric --project P [--model M] --name N --sql SQL --dataset TABLE_ID [--description D] [--yes]` -- add a metric. `--dataset` is a Storage tableId (e.g. `out.c-foo.fact_orders`) -- the metric's dataset field stores the tableId, not the dataset name. `--yes` skips the dataset-mismatch confirmation. +- `semantic-layer add dataset --project P [--model M] --name N --table-id TABLE_ID [--description D] [--grain G] [--primary-key COL ...] [--deep-fields]` -- add a dataset. FQN is auto-derived from `--table-id`. `--primary-key` is repeatable for composite PKs. `--deep-fields` fetches the storage schema and synthesises role-classified `fields[]`: PK_/FK_->`key`, *_DATE/*_DT->`timestamp`, numeric amount/value/rate->`measure`, else `dimension`. +- `semantic-layer add relationship --project P [--model M] --name N --from TABLE_ID --to TABLE_ID --on EXPR [--type left|inner]` -- add a join relationship. `--type` defaults to `left`. +- `semantic-layer add constraint --project P [--model M] --name N --constraint-type T --rule "EXPR" --metrics M1,M2 [--severity error|warning|info]` -- add a constraint. `--constraint-type` is the closed enum `inequality|equality|range|composition|exclusion|temporal|conditional`. `--rule` is a **STRING expression** (e.g. `"value >= 0"`), NEVER a `{bounds: {min, max}}` object (sl-builder docs are wrong -- see [gotchas.md](gotchas.md)). `--metrics` is a comma-separated list of metric names that must already exist in the model. `--severity` defaults to `warning`. Name regex `^[a-z][a-z0-9_]*$`; the 4-band health convention lives in the name suffix `_critical / _warning / _healthy / _review`, distinct from the 3-value API `severity`. +- `semantic-layer add glossary --project P [--model M] --term TERM [--definition D]` -- add a glossary term. +- `semantic-layer edit metric --project P [--model M] --name N [--new-name N2] [--new-sql SQL] [--new-dataset TABLE_ID] [--new-description D] [--yes]` -- edit a metric. The metastore has NO PATCH endpoint, so this is DELETE+POST. Rename CASCADES through every constraint whose `metrics[]` includes the old name (DELETE old constraint + POST new with updated `metrics[]`). Prints the old/new CODE_METRIC computed via `re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")`. `--yes` skips the confirm prompt. On POST failure the service re-POSTs `original_attrs` and reports rollback success/failure explicitly in the envelope. +- `semantic-layer edit dataset --project P [--model M] --name N [--new-name N2] [--new-description D] [--new-grain G]` -- edit a dataset. No cascade -- metrics reference the dataset's tableId, not its name. +- `semantic-layer edit constraint --project P [--model M] --name N [--new-name N2] [--new-rule "EXPR"] [--new-constraint-type T] [--new-severity error|warning|info] [--new-metrics M1,M2]` -- edit a constraint (DELETE+POST). Local validators enforce the name regex, constraintType enum, severity enum, and that every entry in `--new-metrics` exists in the model. +- `semantic-layer edit relationship --project P [--model M] --name N [--new-name N2] [--new-from TABLE_ID] [--new-to TABLE_ID] [--new-on EXPR] [--new-type left|inner]` -- edit a relationship (DELETE+POST). No constraint cascade. Rollback on POST failure. +- `semantic-layer edit glossary --project P [--model M] --term TERM [--new-term T2] [--new-definition D]` -- edit a glossary entry (DELETE+POST). `--new-term` is a destructive cascade through any downstream consumer that joins on the term; warns but allows. +- `semantic-layer remove metric --project P [--model M] --name N [--yes]` -- destructive. Pre-deletion scan lists every constraint whose `metrics[]` includes the target name; warning is ALWAYS printed (even with `--yes`) about the resulting orphan + dangling `DIM_METRIC_THRESHOLD` reference. Non-TTY without `--yes` refuses with exit 2. +- `semantic-layer remove dataset --project P [--model M] --name N [--yes]` -- destructive. Confirmation prompt unless `--yes`. +- `semantic-layer remove constraint --project P [--model M] --name N [--yes]` -- destructive. Confirmation prompt unless `--yes`. +- `semantic-layer remove relationship --project P [--model M] --name N [--yes]` -- destructive. Relationships aren't referenced by other entities; no orphan-check. +- `semantic-layer remove glossary --project P [--model M] --term TERM [--yes]` -- destructive. Glossary entries aren't referenced by other entities; no orphan-check. +- `semantic-layer import --project P --file PATH [--model M] [--types T,T,...] [--dry-run] [--yes] [--overwrite]` -- replay a snapshot. Default: SKIP on conflict (no surprise overwrites). `--overwrite` opts into DELETE+POST for conflicting items. `--types` filters to a subset (`datasets,metrics,relationships,glossary,constraints`). Dependency-ordered push: datasets -> metrics -> relationships -> glossary -> constraints. Response: `imported: {<type>: {created, skipped, overwritten, failed: [{name, reason}]}}`. +- `semantic-layer promote --from-project A --to-project B [--from-model M] [--to-model M] [--types T,T,...] [--dry-run] [--yes]` -- cross-project model copy with `modelUUID` rewrite to the target model's UUID. Classifies items NEW / IDENTICAL / CHANGED (deep-equality after stripping `modelUUID` + timestamps). Additive + overwrite only: NEVER deletes target items absent from source. Holds two MetastoreClients in try/finally. Response: per-type counts + `changes[]` with `diff_keys` and `failed[]`. +- `semantic-layer build --project P [--model M] --tables T,T,... [--name N] [--dry-run] [--output PATH]` -- non-interactive heuristic builder. **AI caveat**: the existing `ai_client` has no arbitrary-JSON endpoint, so `build` falls back to a deterministic heuristic synthesising one dataset + one COUNT(*) metric + one glossary entry per table (FQN derived; fields[] role-classified). Response carries `fallback_used: "heuristic"`. The push loop walks all 5 child types in dependency order -- this **fixes** the `sl-build` skill bug where `semantic-constraint` was silently dropped. `--model` omitted creates a new model (default name `kbagent_build_model` or `--name N`). +- `semantic-layer token --encrypt --project P --component-id C` -- encrypt the project's storage token for a transformation's `user_properties`. Builds `{"#metastore_token": <token>}` from the project's already-stored Storage API token and delegates to the existing EncryptService. `--encrypt` is currently required; other modes are refused with `USAGE_ERROR` (exit 2). Output (human): the raw envelope ready to paste. JSON: full `{encrypted, component_id, project}`. + ## Self-Call HTTP (inside `kbagent serve` subprocesses; since v0.40.0) - `http get PATH [--timeout SECONDS]` -- GET an endpoint on the running `kbagent serve` - `http post PATH [--body JSON|@file|-] [--timeout SECONDS]` -- POST with optional JSON body diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 05ed5b85..6488a9e1 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -1,5 +1,127 @@ # Gotchas -- Response Parsing and Common Pitfalls +## Semantic-layer constraint `rule` is a STRING, not an object (since v0.41.0) + +- The `sl-builder` skill docs (in `04_AI_Kit/ai-kit/`) describe range + constraints with `ruleExpression: {bounds: {min: 0, max: 100}}` -- + that is **WRONG** against the live metastore. Probed 2026-05-14 + against `e2e-1143`: the API rejects the object shape with HTTP 400 + / `"got object, want string"`. +- The correct shape is a single SQL-ish string expression: + ```json + {"name": "revenue_non_negative", "constraintType": "inequality", + "rule": "value >= 0", "metrics": ["revenue"], "severity": "warning"} + ``` + Other examples: `"value BETWEEN 0 AND 100"` (range), + `"value IS NOT NULL"` (equality/existence), `"prev_value <= value"` + (temporal monotonic). +- The `constraintType` enum is a CLOSED list: + `inequality | equality | range | composition | exclusion | temporal | conditional`. + It classifies the SHAPE of the constraint -- the actual expression is + always a string. +- `kbagent semantic-layer add constraint --rule "..."` enforces the + string contract at the CLI layer; if a user pastes a `{bounds: ...}` + object the CLI exits 2 / `VALIDATION_ERROR` with a hint pointing at + this gotcha. + +## Constraint name regex `^[a-z][a-z0-9_]*$` AND the 3-vs-4 severity split (since v0.41.0) + +- Constraint NAMES must match `^[a-z][a-z0-9_]*$`: lowercase ASCII, + digits, underscores; must start with a letter. UPPERCASE, hyphens, + dots, or leading digits get rejected with HTTP 400. +- The 4-band health convention (`<name>_critical / _warning / _healthy + / _review`) lives in the NAME SUFFIX. That suffix is what shows up + downstream in `DIM_METRIC_THRESHOLD` joins on `CODE_CONSTRAINT` + derivations -- it is **not** the same as the API `severity` field. +- The API `severity` field is a SEPARATE closed 3-value enum + (`error | warning | info`). +- Typical pairing: a `_critical`-suffixed constraint typically carries + `severity: "error"`; a `_warning`-suffixed one `severity: "warning"`; + `_healthy` and `_review` typically carry `severity: "info"`. There is + no automatic mapping in the API -- the operator sets both + independently. kbagent's `semantic-layer validate` emits a warning + when the suffix and the severity drift (e.g. a `_critical`-suffixed + constraint with `severity: "info"`). +- `kbagent semantic-layer add constraint --severity` only accepts the + 3 API values; the 4-band band lives in `--name` suffix. + +## Metric rename auto-cascades through `CODE_METRIC` (since v0.41.0) + +- `kbagent semantic-layer edit metric --new-name NEW` does DELETE+POST + on the metric and ALSO DELETE+POST on every constraint whose + `metrics[]` referenced the old name (POST new with `metrics[]` + updated to the new name). The metastore has no PATCH endpoint, so + every "edit" is a delete-then-create. +- The `CODE_METRIC` derived value (used in downstream SQL joins on + `DIM_METRIC_THRESHOLD` / `FACT_METRIC_*` lookups) is computed via + ```python + re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_") + ``` + Renaming a metric from `revenue_growth` to `revenue_growth_qoq` + changes `CODE_METRIC` from `REVENUE_GROWTH` to `REVENUE_GROWTH_QOQ`. + Downstream SQL joining on `CODE_METRIC = 'REVENUE_GROWTH'` silently + drops the row after the rename. +- kbagent `edit metric` ALWAYS prints the old/new CODE_METRIC values + and the list of affected constraints, and requires Y/N confirm + unless `--yes` is set. The CODE_METRIC change line is printed even + with `--yes` -- treat it as a contract change that needs explicit + audit of downstream SQL. +- On POST failure (e.g. the new name violates a constraint), the + service re-POSTs `original_attrs` to restore the pre-edit state + and reports rollback success/failure explicitly in the response + envelope's `rollback` field. If rollback itself fails, the model + is left in a partial state -- surface that to the operator and + recommend running `semantic-layer validate` immediately. + +## Removing a metric corrupts `DIM_METRIC_THRESHOLD` downstream (since v0.41.0) + +- `kbagent semantic-layer remove metric --name N` runs a pre-deletion + scan listing every constraint whose `metrics[]` includes N. Each + such constraint becomes ORPHANED after the delete: it remains in + the model but references a metric that no longer exists. +- Downstream impact: the typical Keboola semantic-layer pipeline + pushes constraints into `DIM_METRIC_THRESHOLD` keyed by + `CODE_METRIC` derived from the metric name. After an orphan, the + threshold row points at a non-existent metric -- joins on + `CODE_METRIC` from `FACT_METRIC_VALUES` silently drop the row (or + crash on strict joins, depending on the pipeline). +- The orphan warning is ALWAYS printed (even with `--yes`) and lists + the orphaned constraint names plus their `metrics[]` content. + Non-TTY invocations without `--yes` refuse with exit 2 -- the + warning is non-suppressible. +- Recommended recovery: either remove the orphaned constraints FIRST + (so `metrics[]` shrinks to a list of still-existing metrics), or + use `edit metric --new-name <archived_*>` for a SOFT-DELETE that + keeps the constraint refs valid (and the CODE_METRIC alive in + historical comparisons). + +## `semantic-layer build` is a HEURISTIC fallback, not full AI (since v0.41.0) + +- The kbagent AI Service client (`ai_client.py`) only exposes + `get_component_detail` and `suggest_components` as of v0.41.0 -- + no arbitrary-JSON endpoint. So `kbagent semantic-layer build` + falls back to a DETERMINISTIC heuristic builder that synthesises: + one dataset per `--tables` entry (FQN auto-derived from + `tableId`; `fields[]` role-classified via the same PK_/FK_/*_DATE/ + *_DT/numeric-amount-name heuristics as `add dataset --deep-fields`), + one `COUNT(*)` metric per dataset, one glossary entry per table. + No relationships, no constraints. +- The response envelope carries `fallback_used: "heuristic"` so + callers can detect the mode. Treat the output as a "best starting + scaffold" and immediately follow up with `add metric`, `add + relationship`, `add constraint` for real business logic. +- The push loop walks ALL FIVE child types in dependency order + (datasets -> metrics -> relationships -> glossary -> constraints). + This FIXES the long-standing `sl-build` skill bug where + `semantic-constraint` was silently dropped from the push loop -- + the skill iterated only 4 of the 5 types. +- The full AI-assisted greenfield wizard (schema discovery, SQL + analysis, LLM-generated metrics with rich business logic and + paired range constraints) still lives in the `sl-build` skill in + `04_AI_Kit/ai-kit/`. Bridge to that skill when the heuristic is + not enough; the two are interoperable via the same metastore + contract. + ## `kbagent http` works only inside `kbagent serve` subprocesses (since v0.40.0) - `kbagent http get/post/patch/delete <PATH>` is a thin self-call client diff --git a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md new file mode 100644 index 00000000..5949900c --- /dev/null +++ b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md @@ -0,0 +1,443 @@ +# Semantic Layer Workflow -- Models, Metrics, Constraints + +The Keboola semantic layer (aka **metastore**) is a project-scoped catalogue +of datasets, metrics, relationships, constraints, and glossary terms. It is +served from a separate API at `metastore.<stack>` (derived from +`connection.<stack>` by string-substitution; cloud/region-agnostic). Auth is +the same `X-StorageApi-Token` as Storage. + +`kbagent semantic-layer ...` (alias `kbagent sl ...`, hidden) wraps the +metastore so AI agents and CI scripts don't roll their own `urllib` loops. +For one-line command reference, see +[commands-reference.md](commands-reference.md#semantic-layer-metastore-since-v0340). +For the live-validated metastore contract surprises (constraint rule shape, +name regex, CODE_METRIC cascade), see +[gotchas.md](gotchas.md#semantic-layer-constraint-rule-is-a-string-not-an-object-since-v0340). + +## When to use what + +| Goal | Command | +|------|---------| +| Show what's in a project's models | `semantic-layer model list` then `semantic-layer show` | +| Pre-flight a model for structural / phantom-field issues | `semantic-layer validate [--deep]` | +| Back up before destructive edits | `semantic-layer export` | +| Compare a dev model to prod | `semantic-layer diff --project-a dev --project-b prod` | +| Add one entity (metric / dataset / etc.) | `semantic-layer add <kind>` | +| Rename a metric (safely cascade) | `semantic-layer edit metric --new-name` | +| Remove a metric (orphan-check first) | `semantic-layer remove metric` | +| Restore from a snapshot | `semantic-layer import --file ... --dry-run` then real | +| Promote dev -> prod | `semantic-layer promote --from-project dev --to-project prod` | +| Bootstrap a model from storage tables | `semantic-layer build --tables ...` (heuristic) | +| Encrypt the storage token for a Python container | `semantic-layer token --encrypt` | + +--- + +## Workflow 1 -- Inspect a project's semantic layer + +```bash +# 1. Enumerate models in the project +kbagent --json semantic-layer model list --project prod +# -> {"models": [{"id": "<uuid>", "name": "core_model", "sql_dialect": "Snowflake", ...}]} + +# 2. Show entity counts for the model +kbagent --json semantic-layer show --project prod --model core_model +# -> {"datasets": [...], "metrics": [...], "relationships": [...], ...} + +# 3. Drill into one type +kbagent --json semantic-layer show --project prod --model core_model --type metric +kbagent --json semantic-layer show --project prod --model core_model --type constraint + +# 4. Validate (no Snowflake probe -- fast) +kbagent --json semantic-layer validate --project prod --model core_model + +# 5. Deep-validate (parallel column-existence probe via StorageService) +kbagent --json semantic-layer validate --project prod --model core_model --deep +``` + +When `--model` is omitted and the project has exactly one model, the CLI +auto-selects it. With more than one model, omitting `--model` exits 2 / +`USAGE_ERROR` -- always run `model list` first if you're unsure. + +`validate` reports `errors[]` (block work) and `warnings[]` (review). +The `type` field on each entry is one of the UPPER_SNAKE strings below; +filter your jq with these exact values: + +- `DUPLICATE` -- two entities of the same type share a name (error). +- `CONSTRAINT_ORPHAN` -- a constraint's `metrics[]` names a metric + that no longer exists (error). +- `DANGLING_METRIC` -- a metric's `dataset` is a tableId no dataset + references (error). +- `DANGLING_RELATIONSHIP` -- a relationship's `from`/`to` is a tableId + no dataset references (error). +- `SUM_ON_PCT` -- `SUM(...)` on a column whose name suggests a + percentage / ratio (error). +- `SEVERITY_SUFFIX` -- a constraint name doesn't end in + `_critical/_warning/_healthy/_review` (warning). +- `PHANTOM_FIELD` (`--deep` only) -- a dataset declares a field that + isn't in the Snowflake table's actual columns (error). +- `METRIC_PHANTOM` (`--deep` only) -- a column referenced in metric + `sql` isn't in the table's actual columns (error). +- `AGG_ON_STRING` (`--deep` only) -- `SUM(...)` / `AVG(...)` over a + Snowflake STRING column (error). +- `DEEP_FETCH_FAILED` (`--deep` only) -- couldn't fetch the Snowflake + schema for a dataset; deep checks for that dataset are skipped (warning). + +--- + +## Workflow 2 -- Export and back up before destructive edits + +ALWAYS export before a rename, a remove, or a promote. The metastore has +no soft-delete and no version history; the snapshot is the only restore +path. + +```bash +# 1. Snapshot to disk (default path stamped with model name + timestamp) +kbagent --json semantic-layer export --project prod --model core_model +# -> {"path": "./sl_export_core_model_20260514_120030.json", "counts": {...}} + +# 2. Verify the snapshot has every type +jq '. | {datasets: .datasets | length, metrics: .metrics | length, ...}' \ + ./sl_export_core_model_20260514_120030.json + +# 3. Make the change +kbagent semantic-layer edit metric --project prod --model core_model \ + --name revenue_growth --new-name revenue_growth_qoq --yes + +# 4. Diff the live model against the snapshot to confirm the diff is +# exactly what you intended (nothing else moved) +kbagent --json semantic-layer diff \ + --file-a ./sl_export_core_model_20260514_120030.json \ + --project-b prod --model-b core_model +``` + +The snapshot is self-describing (carries `model.meta`, `schemaVersion`, +per-entity dicts) and round-trips through `semantic-layer import` for +a full restore. + +--- + +## Workflow 3 -- Add a metric with a paired range constraint + +The recommended pattern is "validate SQL first, then create the metric, +then attach a constraint" -- not the other way around (a constraint +referencing a nonexistent metric is rejected at POST time). + +```bash +# 1. Sanity-check the metric SQL against a workspace BEFORE creating it. +# (Optional but recommended for non-trivial expressions.) +kbagent --json workspace create --project prod --name sl-sanity +# -> grab workspace_id +kbagent --json workspace load --project prod --workspace-id W \ + --tables out.c-revenue.fact_orders +kbagent --json workspace query --project prod --workspace-id W \ + --sql "SELECT SUM(amount) FROM \"out.c-revenue.fact_orders\"" +# Confirm the SQL works; clean up the workspace +kbagent workspace delete --project prod --workspace-id W + +# 2. Add the metric (--dataset is a Storage tableId, NOT a dataset name) +kbagent semantic-layer add metric \ + --project prod --model core_model \ + --name revenue \ + --sql 'SUM(amount)' \ + --dataset out.c-revenue.fact_orders \ + --description 'Total revenue across all orders' + +# 3. Attach a range constraint (rule is a STRING, name regex +# ^[a-z][a-z0-9_]*$, severity is the 3-value API enum) +kbagent semantic-layer add constraint \ + --project prod --model core_model \ + --name revenue_non_negative_warning \ + --constraint-type inequality \ + --rule 'value >= 0' \ + --metrics revenue \ + --severity warning + +# 4. Add the _critical band as a separate constraint +kbagent semantic-layer add constraint \ + --project prod --model core_model \ + --name revenue_minimum_critical \ + --constraint-type inequality \ + --rule 'value >= 1000' \ + --metrics revenue \ + --severity error + +# 5. Validate (the constraint name suffix should match the severity) +kbagent --json semantic-layer validate --project prod --model core_model +``` + +Note: `--rule` is a STRING (`"value >= 0"`), NEVER a `{bounds: ...}` +object. The `sl-builder` skill docs are wrong on this -- the live API +rejects the object shape with HTTP 400. See [gotchas.md](gotchas.md). + +The 4-band health convention (`_critical / _warning / _healthy / +_review`) lives in the constraint NAME suffix. The API `severity` is a +separate 3-value enum (`error | warning | info`). Typical pairings: +`_critical` -> `error`, `_warning` -> `warning`, `_healthy` -> `info`, +`_review` -> `info`. There is no automatic mapping; the operator sets +both. `validate` warns when they drift. + +--- + +## Workflow 4 -- Rename a metric safely + +The single biggest footgun in the semantic layer: a metric rename +changes `CODE_METRIC` (the downstream join key in +`DIM_METRIC_THRESHOLD` / `FACT_METRIC_*`), which silently breaks SQL +joins that pinned the old value. + +```bash +# 0. ALWAYS export first (rename is destructive in the cascade sense) +kbagent --json semantic-layer export --project prod --model core_model + +# 1. Run the rename. kbagent prints: +# - old/new CODE_METRIC (computed via +# re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")) +# - the list of constraints whose metrics[] will be cascaded +# - a confirm prompt (suppress with --yes after auditing) +kbagent semantic-layer edit metric \ + --project prod --model core_model \ + --name revenue_growth \ + --new-name revenue_growth_qoq +# Will print something like: +# CODE_METRIC: REVENUE_GROWTH -> REVENUE_GROWTH_QOQ +# Will DELETE+POST the following constraints (cascade): +# - revenue_growth_minimum_warning +# - revenue_growth_band_review +# Proceed? [y/N]: + +# 2. Verify the cascade +kbagent --json semantic-layer show --project prod --model core_model --type constraint \ + | jq '.constraints[] | select(.metrics | index("revenue_growth_qoq"))' + +# 3. CRITICAL FOLLOW-UP: audit downstream SQL that joins on CODE_METRIC. +# Anywhere your pipeline has +# JOIN dim_metric_threshold dmt +# ON dmt.CODE_METRIC = 'REVENUE_GROWTH' +# needs updating to +# ON dmt.CODE_METRIC = 'REVENUE_GROWTH_QOQ' +# These joins WILL silently start returning empty rows otherwise. +``` + +If the new POST fails (e.g. the new name collides), the service +re-POSTs `original_attrs` to roll back and reports rollback +success/failure in the response envelope's `rollback` field. If the +rollback itself fails, the model is left in a partial state -- run +`semantic-layer validate` immediately. + +--- + +## Workflow 5 -- Remove a metric (with orphan-check) + +```bash +# 0. Export first +kbagent --json semantic-layer export --project prod --model core_model + +# 1. Pre-flight scan: what constraints reference this metric? +kbagent --json semantic-layer show --project prod --model core_model --type constraint \ + | jq '.constraints[] | select(.metrics | index("revenue_growth"))' + +# 2. If there are orphans-to-be, EITHER remove them first +kbagent semantic-layer remove constraint \ + --project prod --model core_model --name revenue_growth_minimum_warning --yes +# OR plan a soft-delete via rename: +# kbagent semantic-layer edit metric --new-name revenue_growth_archived_20260514 + +# 3. Run remove. The orphan warning is ALWAYS printed, even with --yes. +kbagent semantic-layer remove metric \ + --project prod --model core_model --name revenue_growth +# Output: +# Removing metric 'revenue_growth' will orphan 0 constraint(s): +# (or a list if you skipped step 2) +# These constraints will have a dangling reference in DIM_METRIC_THRESHOLD. +# Delete metric 'revenue_growth' anyway? [y/N]: + +# 4. Verify +kbagent --json semantic-layer validate --project prod --model core_model +``` + +Non-TTY invocations without `--yes` refuse with exit 2 -- the warning +is non-suppressible. CI scripts MUST pass `--yes` AND audit the orphan +list AHEAD OF TIME (e.g. via `show --type constraint`); kbagent prints +the warning but does not block on `--yes`. + +--- + +## Workflow 6 -- Promote a model dev -> prod + +```bash +# 1. Export both sides (safety net) +kbagent --json semantic-layer export --project dev --output /tmp/dev.json +kbagent --json semantic-layer export --project prod --output /tmp/prod.json + +# 2. Diff to preview what will move +kbagent --json semantic-layer diff --project-a dev --project-b prod +# -> shows added[] / removed[] / changed[] per type with diff_keys + +# 3. Dry-run the promote: classifies items NEW / IDENTICAL / CHANGED +# (deep-equality after stripping modelUUID + timestamps) +kbagent --json semantic-layer promote \ + --from-project dev --to-project prod --dry-run \ + | jq '.metrics, .constraints' +# Inspect: +# metrics.new -- count of items the promote will POST +# metrics.overwritten -- count of CHANGED items that will be DELETE+POSTed +# metrics.identical -- count of items skipped (already match) +# metrics.changes[] -- per-item diff with diff_keys +# metrics.failed[] -- per-item failures (e.g. dry-run validators) + +# 4. If the dry-run looks right, run for real. --yes skips the confirm. +kbagent --json semantic-layer promote \ + --from-project dev --to-project prod --yes + +# 5. Verify +kbagent --json semantic-layer validate --project prod --deep +``` + +`promote` is **additive + overwrite only** -- it NEVER deletes items +from prod that aren't in dev. To remove items, do that explicitly with +`semantic-layer remove` after the promote. This is intentional: it +prevents a partial dev model from wiping prod-only entities (e.g. an +emergency hotfix metric). + +Use `--types datasets,metrics` to scope the promote to specific entity +types (e.g. promote metric changes without touching constraints). + +--- + +## Workflow 7 -- Bootstrap a model from storage tables + +```bash +# 1. Decide on the table set +kbagent --json storage tables --project prod --bucket-id out.c-revenue \ + | jq -r '.tables[].id' + +# 2. Dry-run the build to inspect the generated JSON +kbagent --json semantic-layer build \ + --project prod \ + --tables out.c-revenue.fact_orders,out.c-revenue.dim_customers \ + --dry-run \ + --output /tmp/built.json +# Response carries: fallback_used: "heuristic" +# Generated: 2 datasets, 2 metrics (COUNT(*)), 0 relationships, +# 0 constraints, 2 glossary entries + +# 3. If the scaffold looks right, push it (omit --dry-run; omit --model +# to create a new model) +kbagent --json semantic-layer build \ + --project prod \ + --tables out.c-revenue.fact_orders,out.c-revenue.dim_customers \ + --name revenue_model + +# 4. Refine with real business logic +kbagent semantic-layer add metric \ + --project prod --model revenue_model \ + --name revenue --sql 'SUM(amount)' \ + --dataset out.c-revenue.fact_orders + +kbagent semantic-layer add relationship \ + --project prod --model revenue_model \ + --name orders_to_customers \ + --from out.c-revenue.fact_orders \ + --to out.c-revenue.dim_customers \ + --on 'fact_orders.customer_id = dim_customers.id' +``` + +**AI caveat**: `build` falls back to a DETERMINISTIC heuristic because +the kbagent AI Service client has no JSON-generation endpoint as of +v0.41.0. The heuristic synthesises: + +- One dataset per `--tables` entry, with FQN derived and `fields[]` + role-classified (PK_/FK_->key, *_DATE/*_DT->timestamp, + numeric amount/value/rate->measure, else dimension). +- One `COUNT(*)` metric per dataset. +- One glossary entry per table. +- No relationships, no constraints. + +Treat `build` output as a **starting scaffold**, not a finished model. +The push loop walks all 5 child types in dependency order -- this +fixes a long-standing `sl-build` skill bug where `semantic-constraint` +was silently dropped. + +For richer AI-assisted generation (full SQL analysis, relationship +inference, paired range constraints), the `sl-build` skill in +`04_AI_Kit/ai-kit/` is the right tool. The two are interoperable via +the same metastore contract; bridge between them as needed. + +--- + +## Workflow 8 -- Encrypt the token for a Python transformation that needs metastore access + +Use case: a `keboola.python-transformation-v2` (or +`kds-team.app-custom-python`) container needs to call the metastore at +runtime (e.g. to look up the current constraint thresholds before +writing to `FACT_METRIC_VALUES`). The container's +`user_properties` block carries the Storage API token, but it must be +encrypted -- the same encryption flow as `data-app secrets-set`. + +```bash +# 1. Encrypt the project's storage token for the target component +kbagent semantic-layer token \ + --encrypt \ + --project prod \ + --component-id keboola.python-transformation-v2 +# Output (human mode): +# Encrypted token for component keboola.python-transformation-v2 +# in project prod: +# { +# "#metastore_token": "KBC::ProjectSecure::..." +# } +# Paste the JSON above into the transformation's `user_properties` block. + +# 2. Paste into user_properties via config update --set +kbagent semantic-layer token \ + --encrypt --project prod \ + --component-id keboola.python-transformation-v2 \ + | jq -r '.encrypted["#metastore_token"]' > /tmp/cipher.txt +CIPHER=$(cat /tmp/cipher.txt) +kbagent config update \ + --project prod \ + --component-id keboola.python-transformation-v2 \ + --config-id 12345 \ + --set "runtime.user_properties.#metastore_token=$CIPHER" + +# 3. Inside the Python container, read it from env: +# import os +# token = os.environ["METASTORE_TOKEN"] # auto-derived: '#' stripped, uppercased +# # then call metastore.<stack>.keboola.com with X-StorageApi-Token: $token +``` + +`semantic-layer token --encrypt` is a thin wrapper around the existing +`encrypt values` flow -- it just builds the `{"#metastore_token": +<token>}` payload automatically from the project's stored Storage API +token (no config-file digging required). Other modes are refused with +`USAGE_ERROR` (exit 2); `--encrypt` is the only supported mode in v0.41.0. + +--- + +## Reference: metastore contract gotchas + +For the live-validated metastore contract surprises -- where the +`sl-builder` skill docs diverge from the actual API -- see: + +- [gotchas.md > Semantic-layer constraint `rule` is a STRING, not an object](gotchas.md#semantic-layer-constraint-rule-is-a-string-not-an-object-since-v0340) +- [gotchas.md > Constraint name regex `^[a-z][a-z0-9_]*$` AND the 3-vs-4 severity split](gotchas.md#constraint-name-regex-a-za-z0-9_-and-the-3-vs-4-severity-split-since-v0340) +- [gotchas.md > Metric rename auto-cascades through `CODE_METRIC`](gotchas.md#metric-rename-auto-cascades-through-code_metric-since-v0340) +- [gotchas.md > Removing a metric corrupts `DIM_METRIC_THRESHOLD` downstream](gotchas.md#removing-a-metric-corrupts-dim_metric_threshold-downstream-since-v0340) +- [gotchas.md > `semantic-layer build` is a HEURISTIC fallback, not full AI](gotchas.md#semantic-layer-build-is-a-heuristic-fallback-not-full-ai-since-v0340) + +Quick reminders: + +- **POST envelope**: `{name, data, branch: "main", schemaVersion: + "1.0.0", scope: "project"}` -> 201 with `{data: {type, id, + attributes, meta}}`. kbagent handles this. +- **Duplicate-name POST -> 500** with `"Failed to create meta object"`. + kbagent normalizes to `ErrorCode.ALREADY_EXISTS`. +- **DELETE -> 204** empty body. +- **No PATCH endpoint** -- every "edit" is DELETE+POST. kbagent's + `edit metric / dataset / relationship / constraint / glossary` + (five sub-subcommands matching `add`) rolls back on POST failure. + Same five types are available under `remove`. +- **`X-StorageApi-Token` is the only auth** -- no separate metastore + token. `kbagent semantic-layer token --encrypt` encrypts the + STORAGE token for a `user_properties` slot named `#metastore_token`, + which is just a convention the Python container reads at runtime. diff --git a/pyproject.toml b/pyproject.toml index d62ff6d2..3a2a2f77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-agent-cli" -version = "0.40.3" +version = "0.41.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index f43797b2..e2547a03 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -8,6 +8,9 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.41.0": [ + "New: `kbagent semantic-layer` (alias `kbagent sl`) -- first-class CLI surface for the Keboola Metastore (semantic layer). Folds every metastore operation that was previously a hand-rolled urllib script in the `sl-builder` Claude Code plugin into a permission-gated, JSON-capable, hint-aware kbagent command group. The metastore URL is derived from each project's stack URL automatically (`connection.` -> `metastore.` -- region/cloud-agnostic), and reuses the same `X-StorageApi-Token` credential kbagent already holds. Subcommands shipped in this release: `show` (list a model's entities -- read), `validate` (structural checks; `--deep` adds parallel Snowflake column probing via the in-process StorageService -- read), `export` (snapshot the model to JSON -- read), `diff` (project-vs-project, project-vs-file, or file-vs-file -- read), `model create` / `model delete` (model lifecycle -- write/destructive), `add metric|dataset|relationship|constraint|glossary` (single-entity creates with FQN derivation, role heuristics, and constraint-orphan validation -- write), `edit` (DELETE+POST with cascade rename of dependent constraints and explicit rollback -- write), `remove` (destructive with mandatory constraint-orphan pre-flight warning), `import` (replay a snapshot with conflict detection -- write), `promote` (cross-project copy with modelUUID rewrite and identical-vs-changed-vs-new classification -- write), `build` (non-interactive AI-assisted greenfield via the existing AI Service client; fixes the long-standing sl-build bug where `semantic-constraint` items were silently dropped from the push loop -- write), and `token --encrypt` (encrypt the project token for a transformation container's user_properties using the existing EncryptService -- write, parity with `encrypt.values`). Verified live against the metastore API on 2026-05-14: response shape `{data: [...]}` for lists / `{data: {type, id, attributes, meta}}` for items; POST envelope `{name, data, branch:'main', schemaVersion:'1.0.0', scope:'project'}`; DELETE returns 204; duplicate-name POST returns 500 with `Failed to create meta object` (normalized to ALREADY_EXISTS); constraint `rule` is a STRING expression (NOT the `ruleExpression` object sl-builder documents); `constraintType` enum is `inequality|equality|range|composition|exclusion|temporal|conditional`; constraint `name` regex is `^[a-z][a-z0-9_]*$`; 4-band health (`_critical/_warning/_healthy/_review`) lives only in the constraint name suffix because the API `severity` enum is 3-level. Permission registry adds entries for every subcommand (with `model`, `add`, `edit`, `remove` split into per-leaf keys so e.g. `model list` stays read-only under `--deny-writes` and every `remove.*` leaf is classified `destructive`). Hint definitions ship for every subcommand (iter-4: `kbagent --hint client semantic-layer <cmd>` and `--hint service` both emit Python snippets). `edit` and `remove` cover all five entity types in iter-4 (metric|dataset|constraint|relationship|glossary) -- the parity gap with `add` is closed. Plugin/agent/skill docs updated (commands-reference, gotchas tagged `(since v0.41.0)`, keboola-expert version gate + tool-selection matrix, new `semantic-layer-workflow.md`). E2E coverage in `tests/test_e2e.py` bootstraps a throwaway `kbagent_e2e_*` model on `e2e-1143`, exercises every command, and tears down in `finally`.", + ], "0.40.3": [ "New: `kbagent serve --ui` workspace SQL editor gains an AI-assisted SQL writer (#287). The 'Help me write this SQL' button opens an inline helper that spawns a local `claude` / `codex` / `gemini` CLI via the new `POST /workspaces/sql/improve/stream` SSE endpoint, feeds it a meta-prompt grounded in the user's workspace (project alias, backend, default schema, visible bucket catalog, backend-specific INFORMATION_SCHEMA recipes, and a MANDATORY-FIRST-STEP block forcing `kbagent storage bucket-detail` for linked-bucket FQN resolution), streams the response back, and pastes the cleaned SQL into the Monaco editor. Three transparency panels are surfaced: the full meta-prompt (so users can audit what the AI received), an Activity log (tool_use -> tool_result events the AI invoked: `-> Bash: kbagent storage bucket-detail ...`), and the final AI suggestion with copy-to-clipboard. Each panel carries an inline copy pill. The `clean_sql_helper_response` strip pipeline handles claude's Insight blocks (the user-set `explanatory` output style leaks them despite the OUTPUT CONTRACT), code fences, preambles, and JSONL duplication. Fix-mode: when a query Run fails, a 'Send to <cli> for fix' button re-opens the helper with the failing SQL + the warehouse error pre-filled; `build_sql_helper_meta_prompt` pivots framing to 'diagnose and fix'. The Snowflake backend hint mandates double-quoting of EVERY identifier including column / table / CTE aliases (`AS \"month\"` not `AS month`) -- Snowflake uppercases unquoted aliases and the resulting CSV columns came back MONTH / EMPLOYEE_COUNT instead of the lowercase names users expected.", "Fix: `wait_for_query_job` now extracts the real warehouse error from `statements[i].error` (a plain string on Snowflake, sometimes a dict on BigQuery), not from a top-level `error` field that is ABSENT on failures. The previous extractor emitted the useless 'Query job failed: Query execution failed' constant for every failure; the SQL editor's red error box and the AI fix-mode prompt now receive messages like 'SQL compilation error: Function DATE_TRUNC does not support VARCHAR(10) argument type' verbatim. New module-level `_extract_query_job_error` helper walks statements first (with one-line `Statement N:` prefix only when multiple statements failed), falls back through top-level (string OR dict-with-message), and finally an explicit `Query execution failed (no error details from Query Service)` so the caller never gets an empty error string. 6 unit tests pin the four input shapes plus the no-info fallback.", diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index 7c01c8ac..f522452f 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -26,6 +26,7 @@ from .commands.repl import repl_command from .commands.schedule import schedule_app from .commands.search import search_command +from .commands.semantic_layer import semantic_layer_app from .commands.serve import serve_command from .commands.sharing import sharing_app from .commands.storage import storage_app @@ -58,6 +59,7 @@ from .services.repo_validate_service import RepoValidateService from .services.schedule_service import ScheduleService from .services.search_service import SearchService +from .services.semantic_layer_service import SemanticLayerService from .services.sharing_service import SharingService from .services.storage_service import StorageService from .services.sync_service import SyncService @@ -117,6 +119,8 @@ app.add_typer(tool_app, name="tool", rich_help_panel=_DEV) app.add_typer(sync_app, name="sync", rich_help_panel=_DEV) app.add_typer(encrypt_app, name="encrypt", rich_help_panel=_DEV) +app.add_typer(semantic_layer_app, name="semantic-layer", rich_help_panel=_DEV) +app.add_typer(semantic_layer_app, name="sl", rich_help_panel=_DEV, hidden=True) app.add_typer(http_app, name="http", rich_help_panel=_DEV) @@ -330,6 +334,7 @@ def main( schedule_service = ScheduleService(config_store=config_store) workspace_service = WorkspaceService(config_store=config_store) data_app_service = DataAppService(config_store=config_store) + semantic_layer_service = SemanticLayerService(config_store=config_store) repo_validate_service = RepoValidateService(config_store=config_store) kai_service = KaiService(config_store=config_store) doctor_service = DoctorService(config_store=config_store, mcp_service=mcp_service) @@ -388,6 +393,7 @@ def main( ctx.obj["schedule_service"] = schedule_service ctx.obj["workspace_service"] = workspace_service ctx.obj["data_app_service"] = data_app_service + ctx.obj["semantic_layer_service"] = semantic_layer_service ctx.obj["repo_validate_service"] = repo_validate_service ctx.obj["kai_service"] = kai_service ctx.obj["doctor_service"] = doctor_service diff --git a/src/keboola_agent_cli/commands/_helpers.py b/src/keboola_agent_cli/commands/_helpers.py index b4ef7b78..c89e1e6c 100644 --- a/src/keboola_agent_cli/commands/_helpers.py +++ b/src/keboola_agent_cli/commands/_helpers.py @@ -160,6 +160,15 @@ def check_cli_permission(ctx: typer.Context, group_name: str) -> None: if HintRegistry.get(cli_command) is not None: return # Hint exists — let the command function handle it + # Sub-app descent: allow the parent callback to pass control to a + # nested sub-app's callback when *any* descendant under the + # composed prefix has a hint definition registered. Without this, + # multi-level command groups (e.g. `semantic-layer add metric`) + # would always exit at the parent callback with "No --hint" + # because the parent prefix itself is never registered as a leaf. + prefix = cli_command + "." + if any(key.startswith(prefix) for key in HintRegistry.all_commands()): + return # No hint registered for this command typer.echo( f"No --hint available for '{group_name} {subcommand}'.", diff --git a/src/keboola_agent_cli/commands/_semantic_layer_crud.py b/src/keboola_agent_cli/commands/_semantic_layer_crud.py new file mode 100644 index 00000000..d4745b8a --- /dev/null +++ b/src/keboola_agent_cli/commands/_semantic_layer_crud.py @@ -0,0 +1,784 @@ +"""Typer sub-apps for ``kbagent semantic-layer add|edit|remove``. + +Extracted from :mod:`commands.semantic_layer` so the parent commands file +stays under the 1,200-LOC commands-file ceiling defined in CONTRIBUTING.md. +The three sub-apps are mounted onto ``semantic_layer_app`` via +``add_typer(...)`` in the parent module; they share error handling and the +stdin-TTY probe via :mod:`commands._semantic_layer_helpers`. +""" + +from __future__ import annotations + +import typer +from rich.console import Console + +from ..errors import ErrorCode +from ._helpers import ( + check_cli_permission, + emit_hint, + get_formatter, + get_service, + should_hint, +) +from ._semantic_layer_helpers import _handle_service_call, _is_stdin_tty + +# --------------------------------------------------------------------------- +# semantic-layer add -- one sub-subcommand per entity type +# --------------------------------------------------------------------------- + + +add_app = typer.Typer( + name="add", + help="Add an entity (metric, dataset, relationship, constraint, glossary).", + no_args_is_help=True, +) + + +@add_app.callback(invoke_without_command=True) +def _add_permission_check(ctx: typer.Context) -> None: + """Permission check for the ``add`` sub-app. + + Uses the standard ``check_cli_permission`` helper which composes the + operation key as ``"semantic-layer.add.{subcommand}"`` (one per leaf). + Every leaf within ``add`` is classified ``write`` in + :mod:`permissions.OPERATION_REGISTRY`, so the gate is uniform. + """ + check_cli_permission(ctx, "semantic-layer.add") + + +def _print_item_added(label: str): # type: ignore[no-untyped-def] + """Build a human-mode lambda that confirms one item was added.""" + + def _render(c: Console, d: dict) -> None: + attrs = d.get("attributes") or {} + name = attrs.get("name") or attrs.get("term", "?") + c.print( + f"[bold green]Added {label}[/bold green] [cyan]{name}[/cyan] " + f"([dim]{d.get('id', '')}[/dim])" + ) + + return _render + + +@add_app.command("metric") +def add_metric( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option(..., "--name", help="Metric name"), + sql: str = typer.Option(..., "--sql", help="SQL expression for the metric"), + dataset: str = typer.Option(..., "--dataset", help="Dataset tableId this metric belongs to"), + description: str = typer.Option("", "--description", help="Optional description"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the dataset-mismatch warning"), +) -> None: + """Add a metric to a semantic-layer model.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.add.metric", + project=project, + model=model, + name=name, + sql=sql, + dataset=dataset, + description=description, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.add_metric, + alias=project, + model_name_or_uuid=model, + name=name, + sql=sql, + dataset=dataset, + description=description, + assume_yes=yes, + is_tty=_is_stdin_tty(), + confirm_cb=typer.confirm, + ) + formatter.output(result, _print_item_added("metric")) + + +@add_app.command("dataset") +def add_dataset( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option(..., "--name", help="Dataset name"), + table_id: str = typer.Option( + ..., "--table-id", help="Storage tableId, e.g. out.c-bucket.table" + ), + description: str = typer.Option("", "--description"), + grain: str = typer.Option("", "--grain", help="Grain description"), + primary_key: list[str] | None = typer.Option( + None, "--primary-key", help="Repeat for multi-col PK" + ), + deep_fields: bool = typer.Option( + False, + "--deep-fields", + help="Fetch storage schema and synthesise fields[] with role heuristics.", + ), +) -> None: + """Add a dataset (FQN derived from tableId).""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.add.dataset", + project=project, + model=model, + name=name, + table_id=table_id, + description=description, + grain=grain, + primary_key=primary_key, + deep_fields=deep_fields, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.add_dataset, + alias=project, + model_name_or_uuid=model, + name=name, + table_id=table_id, + description=description, + grain=grain, + primary_key=primary_key, + deep_fields=deep_fields, + ) + formatter.output(result, _print_item_added("dataset")) + + +@add_app.command("relationship") +def add_relationship( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option(..., "--name", help="Relationship name"), + from_: str = typer.Option(..., "--from", help="Source dataset tableId"), + to: str = typer.Option(..., "--to", help="Target dataset tableId"), + on: str = typer.Option(..., "--on", help="Join condition"), + type_: str = typer.Option("left", "--type", help="Join type: 'left' or 'inner'."), +) -> None: + """Add a relationship between two datasets.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.add.relationship", + project=project, + model=model, + name=name, + from_=from_, + to=to, + on=on, + type_=type_, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.add_relationship, + alias=project, + model_name_or_uuid=model, + name=name, + from_=from_, + to=to, + on=on, + type_=type_, + ) + formatter.output(result, _print_item_added("relationship")) + + +@add_app.command("constraint") +def add_constraint( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option( + ..., + "--name", + help=( + "Constraint name (regex ^[a-z][a-z0-9_]*$). For the 4-band " + "health convention end with _critical / _warning / _healthy / _review." + ), + ), + constraint_type: str = typer.Option( + ..., + "--constraint-type", + help=("One of: inequality|equality|range|composition|exclusion|temporal|conditional."), + ), + rule: str = typer.Option( + ..., + "--rule", + help='Rule expression STRING (e.g. "value >= 0"). NOT an object.', + ), + metrics: str = typer.Option( + ..., + "--metrics", + help="Comma-separated list of metric names this constraint applies to.", + ), + severity: str = typer.Option( + "warning", "--severity", help="One of: error|warning|info (the 3-level API enum)." + ), +) -> None: + """Add a constraint.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.add.constraint", + project=project, + model=model, + name=name, + constraint_type=constraint_type, + rule=rule, + metrics=metrics, + severity=severity, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + metrics_list = [m.strip() for m in metrics.split(",") if m.strip()] + if not metrics_list: + formatter.error( + message="--metrics must contain at least one metric name.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + result = _handle_service_call( + ctx, + service.add_constraint, + alias=project, + model_name_or_uuid=model, + name=name, + constraint_type=constraint_type, + rule=rule, + metrics=metrics_list, + severity=severity, + ) + formatter.output(result, _print_item_added("constraint")) + + +@add_app.command("glossary") +def add_glossary( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + term: str = typer.Option(..., "--term", help="Glossary term"), + definition: str = typer.Option("", "--definition", help="Optional definition"), +) -> None: + """Add a glossary term.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.add.glossary", + project=project, + model=model, + term=term, + definition=definition, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.add_glossary, + alias=project, + model_name_or_uuid=model, + term=term, + definition=definition, + ) + formatter.output(result, _print_item_added("glossary")) + + +# --------------------------------------------------------------------------- +# semantic-layer edit -- DELETE+POST with rollback + rename cascade +# --------------------------------------------------------------------------- + + +edit_app = typer.Typer( + name="edit", + help=( + "Edit a metric, dataset, constraint, relationship, or glossary term " + "(DELETE+POST with rollback)." + ), + no_args_is_help=True, +) + + +@edit_app.callback(invoke_without_command=True) +def _edit_permission_check(ctx: typer.Context) -> None: + """Permission check for the ``edit`` sub-app. + + Every ``edit`` leaf is classified ``write`` in OPERATION_REGISTRY. + """ + check_cli_permission(ctx, "semantic-layer.edit") + + +def _print_edit_result(label: str): # type: ignore[no-untyped-def] + """Build a human-mode renderer for edit responses.""" + + def _render(c: Console, d: dict) -> None: + updated = d.get("updated") or {} + attrs = updated.get("attributes") or {} + name = attrs.get("name") or attrs.get("term", "?") + c.print( + f"[bold green]Updated {label}[/bold green] [cyan]{name}[/cyan] " + f"([dim]{updated.get('id', '')}[/dim])" + ) + cascaded = d.get("cascaded_constraints") or [] + for entry in cascaded: + status = entry.get("status", "?") + colour = "green" if status == "updated" else "red" + c.print( + f" [{colour}]cascade {status}[/{colour}] " + f"constraint [cyan]{entry.get('constraint', '?')}[/cyan]" + ) + if d.get("rollback"): + c.print(f"[bold red]Rollback applied:[/bold red] {d['rollback']}") + + return _render + + +@edit_app.command("metric") +def edit_metric( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option(..., "--name", help="Current metric name"), + new_name: str | None = typer.Option( + None, "--new-name", help="Rename to this name (triggers constraint cascade)" + ), + new_sql: str | None = typer.Option(None, "--new-sql", help="Replace SQL"), + new_dataset: str | None = typer.Option(None, "--new-dataset", help="Replace dataset tableId"), + new_description: str | None = typer.Option( + None, "--new-description", help="Replace description" + ), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the rename-cascade prompt"), +) -> None: + """Edit a metric. Rename cascades to any constraint that references it.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.edit.metric", + project=project, + model=model, + name=name, + new_name=new_name, + new_sql=new_sql, + new_dataset=new_dataset, + new_description=new_description, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.edit_metric, + alias=project, + model_name_or_uuid=model, + current_name=name, + new_name=new_name, + new_sql=new_sql, + new_dataset=new_dataset, + new_description=new_description, + assume_yes=yes, + is_tty=_is_stdin_tty(), + confirm_cb=typer.confirm, + ) + formatter.output(result, _print_edit_result("metric")) + + +@edit_app.command("dataset") +def edit_dataset( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option(..., "--name", help="Current dataset name"), + new_name: str | None = typer.Option(None, "--new-name", help="Rename to this name"), + new_description: str | None = typer.Option( + None, "--new-description", help="Replace description" + ), + new_grain: str | None = typer.Option(None, "--new-grain", help="Replace grain"), +) -> None: + """Edit a dataset (no cascade — metric.dataset uses tableId, not name).""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.edit.dataset", + project=project, + model=model, + name=name, + new_name=new_name, + new_description=new_description, + new_grain=new_grain, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.edit_dataset, + alias=project, + model_name_or_uuid=model, + current_name=name, + new_name=new_name, + new_description=new_description, + new_grain=new_grain, + ) + formatter.output(result, _print_edit_result("dataset")) + + +@edit_app.command("constraint") +def edit_constraint( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option(..., "--name", help="Current constraint name"), + new_name: str | None = typer.Option( + None, "--new-name", help=f"Rename to this name (regex {'^[a-z][a-z0-9_]*$'!r})" + ), + new_rule: str | None = typer.Option(None, "--new-rule", help="Replace rule (STRING)"), + new_constraint_type: str | None = typer.Option( + None, "--new-constraint-type", help="Replace constraintType (closed enum)" + ), + new_severity: str | None = typer.Option( + None, "--new-severity", help="Replace severity (error|warning|info)" + ), + new_metrics: str | None = typer.Option( + None, "--new-metrics", help="Comma-separated list of metric names" + ), +) -> None: + """Edit a constraint (DELETE+POST, with local validators).""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.edit.constraint", + project=project, + model=model, + name=name, + new_name=new_name, + new_rule=new_rule, + new_constraint_type=new_constraint_type, + new_severity=new_severity, + new_metrics=new_metrics, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + metrics_list = ( + [m.strip() for m in new_metrics.split(",") if m.strip()] + if new_metrics is not None + else None + ) + result = _handle_service_call( + ctx, + service.edit_constraint, + alias=project, + model_name_or_uuid=model, + current_name=name, + new_name=new_name, + new_rule=new_rule, + new_constraint_type=new_constraint_type, + new_severity=new_severity, + new_metrics=metrics_list, + ) + formatter.output(result, _print_edit_result("constraint")) + + +@edit_app.command("relationship") +def edit_relationship( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option(..., "--name", help="Current relationship name"), + new_name: str | None = typer.Option(None, "--new-name", help="Rename to this name"), + new_from: str | None = typer.Option(None, "--new-from", help="Replace source dataset tableId"), + new_to: str | None = typer.Option(None, "--new-to", help="Replace target dataset tableId"), + new_on: str | None = typer.Option(None, "--new-on", help="Replace join condition"), + new_type: str | None = typer.Option( + None, "--new-type", help="Replace join type (left | inner)" + ), +) -> None: + """Edit a relationship (DELETE+POST). Validates ``--new-type`` locally.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.edit.relationship", + project=project, + model=model, + name=name, + new_name=new_name, + new_from=new_from, + new_to=new_to, + new_on=new_on, + new_type=new_type, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.edit_relationship, + alias=project, + model_name_or_uuid=model, + current_name=name, + new_name=new_name, + new_from=new_from, + new_to=new_to, + new_on=new_on, + new_type=new_type, + ) + formatter.output(result, _print_edit_result("relationship")) + + +@edit_app.command("glossary") +def edit_glossary( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + term: str = typer.Option(..., "--term", help="Current glossary term"), + new_term: str | None = typer.Option( + None, + "--new-term", + help=( + "Rename the term (DESTRUCTIVE cascade: downstream consumers joining on the " + "term string will break -- pass --yes to confirm)." + ), + ), + new_definition: str | None = typer.Option( + None, "--new-definition", help="Replace the definition" + ), + yes: bool = typer.Option( + False, "--yes", "-y", help="Skip the rename-cascade prompt (required for --new-term)" + ), +) -> None: + """Edit a glossary term. ``--new-term`` is destructive for downstream joins.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.edit.glossary", + project=project, + model=model, + term=term, + new_term=new_term, + new_definition=new_definition, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + + if new_term is not None and new_term != term and not yes: + if not _is_stdin_tty(): + formatter.error( + message=( + f"Refusing to rename glossary term {term!r} -> {new_term!r} " + "non-interactively without --yes (downstream consumers joining " + "on the term string will break)." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + if not formatter.json_mode and not typer.confirm( + f"Rename glossary term '{term}' to '{new_term}'? " + "Downstream consumers joining on the term will break." + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + + result = _handle_service_call( + ctx, + service.edit_glossary, + alias=project, + model_name_or_uuid=model, + current_term=term, + new_term=new_term, + new_definition=new_definition, + ) + formatter.output(result, _print_edit_result("glossary")) + + +# --------------------------------------------------------------------------- +# semantic-layer remove -- destructive, orphan-warning before delete +# --------------------------------------------------------------------------- + + +remove_app = typer.Typer( + name="remove", + help=("Remove a metric / dataset / constraint / relationship / glossary term (destructive)."), + no_args_is_help=True, +) + + +@remove_app.callback(invoke_without_command=True) +def _remove_permission_check(ctx: typer.Context) -> None: + """Permission check for the ``remove`` sub-app. + + Every ``remove`` leaf is classified ``destructive`` in OPERATION_REGISTRY. + """ + check_cli_permission(ctx, "semantic-layer.remove") + + +def _run_remove( + ctx: typer.Context, + *, + kind: str, + project: str, + model: str | None, + name: str, + yes: bool, +) -> None: + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + + # Always echo the orphan warning (--yes only skips the prompt). + preview = _handle_service_call( + ctx, + service.preview_remove, + alias=project, + model_name_or_uuid=model, + kind=kind, + name=name, + ) + + orphans = preview.get("orphaned_constraints") or [] + is_tty = _is_stdin_tty() + + if orphans and not formatter.json_mode: + formatter.err_console.print( + f"\n[bold yellow]Removing {kind} '{name}' will orphan " + f"{len(orphans)} constraint(s):[/bold yellow]" + ) + for orph in orphans: + metrics = orph.get("metrics", []) + formatter.err_console.print(f" · {orph['name']} (metrics: {metrics})") + formatter.err_console.print( + "These constraints will have a dangling reference in DIM_METRIC_THRESHOLD.\n" + "To avoid this, remove or update the constraints first." + ) + + # Non-TTY without --yes: refuse with exit 2 (warning above already shown). + if not yes: + if not is_tty: + formatter.error( + message=(f"Refusing to remove {kind} {name!r} non-interactively without --yes."), + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + if not formatter.json_mode and not typer.confirm(f"Delete {kind} '{name}' anyway?"): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + + result = _handle_service_call( + ctx, + service.remove_item, + alias=project, + model_name_or_uuid=model, + kind=kind, + name=name, + ) + formatter.output( + result, + lambda c, d: c.print( + f"[bold green]Removed {kind}[/bold green] [cyan]{d['removed']['name']}[/cyan] " + f"([dim]{d['removed']['id']}[/dim])" + ), + ) + + +@remove_app.command("metric") +def remove_metric( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option(..., "--name", help="Metric name"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirm prompt"), +) -> None: + """Remove a metric. Prints an orphan-warning when constraints reference it.""" + if should_hint(ctx): + emit_hint(ctx, "semantic-layer.remove.metric", project=project, model=model, name=name) + return + _run_remove(ctx, kind="metric", project=project, model=model, name=name, yes=yes) + + +@remove_app.command("dataset") +def remove_dataset( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option(..., "--name", help="Dataset name"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirm prompt"), +) -> None: + """Remove a dataset.""" + if should_hint(ctx): + emit_hint(ctx, "semantic-layer.remove.dataset", project=project, model=model, name=name) + return + _run_remove(ctx, kind="dataset", project=project, model=model, name=name, yes=yes) + + +@remove_app.command("constraint") +def remove_constraint( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option(..., "--name", help="Constraint name"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirm prompt"), +) -> None: + """Remove a constraint.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.remove.constraint", + project=project, + model=model, + name=name, + ) + return + _run_remove(ctx, kind="constraint", project=project, model=model, name=name, yes=yes) + + +@remove_app.command("relationship") +def remove_relationship( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + name: str = typer.Option(..., "--name", help="Relationship name"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirm prompt"), +) -> None: + """Remove a relationship. No orphan-check (relationships are leaf entities).""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.remove.relationship", + project=project, + model=model, + name=name, + ) + return + _run_remove(ctx, kind="relationship", project=project, model=model, name=name, yes=yes) + + +@remove_app.command("glossary") +def remove_glossary( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + term: str = typer.Option(..., "--term", help="Glossary term"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirm prompt"), +) -> None: + """Remove a glossary term. No orphan-check (glossary is a leaf entity).""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.remove.glossary", + project=project, + model=model, + term=term, + ) + return + _run_remove(ctx, kind="glossary", project=project, model=model, name=term, yes=yes) diff --git a/src/keboola_agent_cli/commands/_semantic_layer_helpers.py b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py new file mode 100644 index 00000000..b48bb42c --- /dev/null +++ b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py @@ -0,0 +1,44 @@ +"""Shared helpers for the ``semantic-layer`` command group. + +Split out of :mod:`commands.semantic_layer` so that the ``add`` / ``edit`` / +``remove`` sub-apps -- which live in :mod:`commands._semantic_layer_crud` -- +can reuse the same error-handling and stdin-TTY probe without forcing a +circular import between the two command modules. +""" + +from __future__ import annotations + +import sys + +import typer + +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ._helpers import get_formatter, map_error_to_exit_code + + +def _handle_service_call(ctx: typer.Context, func, *args, **kwargs): # type: ignore[no-untyped-def] + """Run a service call, mapping ``ConfigError`` / ``KeboolaApiError`` to exit codes. + + Returns the service result on success; on failure, prints the structured + error envelope (JSON mode) or a red error line (human mode) and raises + ``typer.Exit`` with the appropriate code. + """ + formatter = get_formatter(ctx) + try: + return func(*args, **kwargs) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + +def _is_stdin_tty() -> bool: + """Return ``True`` when stdin is attached to a TTY (interactive shell).""" + return hasattr(sys.stdin, "isatty") and sys.stdin.isatty() diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 22b55cde..78b310fb 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -756,6 +756,92 @@ --input accepts: inline JSON, @file.json (from file), or - (from stdin). Already-encrypted values (KBC:: prefix) pass through unchanged. +### Semantic Layer (Metastore) (since v0.41.0) + +Manage Keboola metastore models: datasets, metrics, relationships, constraints, +glossary terms. Metastore URL derived from stack URL by replacing `connection.` +with `metastore.`. Auth: same `X-StorageApi-Token` as Storage. Alias: +`kbagent sl ...` (hidden) is equivalent to `kbagent semantic-layer ...`. + + kbagent semantic-layer model list --project P + List all semantic-layer models in a project. + + kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] + Create a new model (default sql-dialect: Snowflake). + + kbagent semantic-layer model delete --project P --model M [--yes] + Delete a model. Fails if the model still has child entities. + + kbagent semantic-layer show --project P [--model M] [--type T] + Show a model's entities. --type filter: dataset|metric|relationship|constraint|glossary. + Without --type prints a per-type count summary. + + kbagent semantic-layer validate --project P [--model M] [--deep] + Basic structural checks (duplicates, dangling refs, sum-on-pct, + constraint orphans, severity-suffix). --deep adds parallel Snowflake + column-existence checks for phantom fields, phantom column refs, and + AGG-on-STRING via in-process StorageService. + + kbagent semantic-layer export --project P [--model M] [--output PATH] + Snapshot the model to a self-describing JSON file. Default path: + ./sl_export_{{model_name}}_{{YYYYMMDD_HHMMSS}}.json. + + kbagent semantic-layer diff (--project-a A | --file-a P) (--project-b B | --file-b P) [--model-a M] [--model-b M] + Three-way diff: project<->project, project<->file, file<->file. Output + groups changes per entity type: added, removed, changed (with diff_keys). + + kbagent semantic-layer add metric|dataset|relationship|constraint|glossary ... + Add one entity. Dataset auto-derives `fqn` from --table-id; --deep-fields + fetches the storage schema and synthesises role-classified fields + (PK_/FK_->key, *_DATE/*_DT->timestamp, numeric amount/value/rate->measure, + else dimension). Constraint name regex `^[a-z][a-z0-9_]*$`, severity is + error|warning|info (the 4-band health convention lives in the NAME suffix + `_critical/_warning/_healthy/_review`, not the API severity). `--rule` is + a STRING expression (e.g. "value >= 0"), NEVER an object. + + kbagent semantic-layer edit metric|dataset|constraint|relationship|glossary ... + DELETE+POST (no PATCH on metastore). Metric rename cascades through every + constraint referencing the old name (DELETE old + POST new with updated + metrics[]); CODE_METRIC warning shown + (re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")). On POST failure, + rollback re-POSTs original_attrs and reports success/failure explicitly. + --yes skips the confirm prompt. `edit relationship` accepts --new-from / + --new-to / --new-on / --new-type (left|inner). `edit glossary` accepts + --new-term (destructive cascade; requires --yes in non-TTY) / --new-definition. + + kbagent semantic-layer remove metric|dataset|constraint|relationship|glossary ... + Destructive. `remove metric` pre-scans constraints whose metrics[] includes + the target; warns about dangling DIM_METRIC_THRESHOLD refs. --yes skips the + prompt but the orphan warning is always printed. Non-TTY without --yes + refuses with exit 2. `remove relationship` and `remove glossary` are leaf + removes -- no orphan-check (those entities aren't referenced by others). + `remove glossary` identifies the entity by --term, not --name. + + kbagent semantic-layer import --project P --file PATH [--model M] [--types T,T,...] [--dry-run] [--yes] [--overwrite] + Replay a snapshot. Default: skip on conflict. --overwrite opts into + DELETE+POST. Dependency-ordered push (datasets -> metrics -> relationships + -> glossary -> constraints). + + kbagent semantic-layer promote --from-project A --to-project B [--from-model M] [--to-model M] [--types ...] [--dry-run] [--yes] + Cross-project copy with modelUUID rewrite. Classifies items NEW / IDENTICAL + / CHANGED (deep-equality after stripping modelUUID + timestamps). + Additive + overwrite only -- NEVER deletes target items absent from source. + + kbagent semantic-layer build --project P [--model M] --tables T,T,... [--dry-run] [--output PATH] + Non-interactive heuristic builder. AI caveat: the ai_client has no + arbitrary-JSON endpoint, so `build` falls back to a deterministic + heuristic (one dataset + one COUNT(*) metric + one glossary entry per + table; FQN derived; fields[] role-classified). Response carries + `fallback_used: "heuristic"`. Push loop iterates all 5 child types in + dependency order (fixes the long-standing sl-build skill bug where + semantic-constraint was silently dropped). + + kbagent semantic-layer token --encrypt --project P --component-id C + Encrypt the project's storage token for transformation `user_properties`. + Builds {{"#metastore_token": <token>}} and delegates to EncryptService. + --encrypt is currently required; other modes refused with USAGE_ERROR. + + ### Self-call HTTP (inside `kbagent serve` subprocesses) kbagent http get PATH [--timeout SECONDS] diff --git a/src/keboola_agent_cli/commands/semantic_layer.py b/src/keboola_agent_cli/commands/semantic_layer.py new file mode 100644 index 00000000..98cda921 --- /dev/null +++ b/src/keboola_agent_cli/commands/semantic_layer.py @@ -0,0 +1,902 @@ +"""Typer wrappers for the ``kbagent semantic-layer`` command group. + +Thin layer per the 3-layer architecture: parse arguments, call +:class:`SemanticLayerService`, format output. All business logic lives in the +service layer. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import typer +from rich.console import Console +from rich.table import Table + +from ..errors import ErrorCode +from ._helpers import ( + check_cli_permission, + emit_hint, + get_formatter, + get_service, + should_hint, +) +from ._semantic_layer_crud import add_app, edit_app, remove_app +from ._semantic_layer_helpers import _handle_service_call + +semantic_layer_app = typer.Typer( + name="semantic-layer", + help=( + "Manage Keboola semantic layer (metastore) models -- datasets, metrics, " + "relationships, constraints, and glossary terms." + ), + no_args_is_help=True, +) + + +@semantic_layer_app.callback(invoke_without_command=True) +def _semantic_layer_permission_check(ctx: typer.Context) -> None: + """Enforce permission policy for every semantic-layer subcommand.""" + check_cli_permission(ctx, "semantic-layer") + + +# --------------------------------------------------------------------------- +# semantic-layer model -- model lifecycle (list) +# --------------------------------------------------------------------------- + +model_app = typer.Typer( + name="model", + help="Manage semantic-layer models (list models in a project).", + no_args_is_help=True, +) +semantic_layer_app.add_typer(model_app, name="model") + + +@model_app.callback(invoke_without_command=True) +def _model_permission_check(ctx: typer.Context) -> None: + """Per-subcommand permission check for the ``model`` sub-app. + + Uses the standard ``check_cli_permission`` helper which composes the + operation key as ``"{group}.{subcommand}"`` — so we get + ``semantic-layer.model.list`` (read), ``semantic-layer.model.create`` + (write), and ``semantic-layer.model.delete`` (destructive). A collapsed + single ``semantic-layer.model`` key would deny `model list` under + ``--deny-writes`` even though it's read-only. + """ + check_cli_permission(ctx, "semantic-layer.model") + + +def _print_models_table(console: Console, data: dict) -> None: + """Pretty-print the list of models for a project.""" + project = data.get("project", "") + models = data.get("models", []) + if not models: + console.print(f"[dim]No semantic-layer models in project '{project}'.[/dim]") + return + table = Table(title=f"Semantic-layer models in '{project}'") + table.add_column("Name", style="bold cyan") + table.add_column("UUID", style="dim") + table.add_column("SQL Dialect") + table.add_column("Description", max_width=60) + for m in models: + table.add_row( + m.get("name", ""), + m.get("id", ""), + m.get("sql_dialect", ""), + m.get("description", ""), + ) + console.print(table) + + +@model_app.command("list") +def model_list( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), +) -> None: + """List all semantic-layer models in a project.""" + if should_hint(ctx): + emit_hint(ctx, "semantic-layer.model.list", project=project) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call(ctx, service.list_models, alias=project) + formatter.output(result, _print_models_table) + + +@model_app.command("create") +def model_create( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + name: str = typer.Option(..., "--name", help="Model name (unique within project)"), + description: str = typer.Option("", "--description", help="Optional description"), + sql_dialect: str = typer.Option( + "Snowflake", "--sql-dialect", help="SQL dialect (default: Snowflake)" + ), +) -> None: + """Create a new semantic-layer model.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.model.create", + project=project, + name=name, + description=description, + sql_dialect=sql_dialect, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.create_model, + alias=project, + name=name, + description=description, + sql_dialect=sql_dialect, + ) + formatter.output( + result, + lambda c, d: c.print( + f"[bold green]Created model[/bold green] [cyan]{d['model']['attributes']['name']}[/cyan] " + f"([dim]{d['model']['id']}[/dim])" + ), + ) + + +# --------------------------------------------------------------------------- +# semantic-layer add | edit | remove -- mounted from a sibling module to keep +# this commands file under the CONTRIBUTING.md hard ceiling. See +# :mod:`commands._semantic_layer_crud` for the sub-app implementations. +# --------------------------------------------------------------------------- + +semantic_layer_app.add_typer(add_app, name="add") +semantic_layer_app.add_typer(edit_app, name="edit") +semantic_layer_app.add_typer(remove_app, name="remove") + + +@model_app.command("delete") +def model_delete( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str = typer.Option(..., "--model", help="Model name or UUID"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt"), +) -> None: + """Delete a semantic-layer model. Fails if the model still has children.""" + if should_hint(ctx): + emit_hint(ctx, "semantic-layer.model.delete", project=project, model=model) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + + if ( + not yes + and not formatter.json_mode + and not typer.confirm( + f"Delete model '{model}' in project '{project}'? " + "If the model has datasets/metrics/etc. the API will refuse." + ) + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + + result = _handle_service_call( + ctx, + service.delete_model, + alias=project, + model_name_or_uuid=model, + ) + formatter.output( + result, + lambda c, d: c.print( + f"[bold green]Deleted model[/bold green] [cyan]{d['deleted']['name']}[/cyan] " + f"([dim]{d['deleted']['id']}[/dim])" + ), + ) + + +# --------------------------------------------------------------------------- +# semantic-layer show +# --------------------------------------------------------------------------- + + +def _print_show_summary(console: Console, data: dict) -> None: + """Render the show payload (no --type) as a compact count table.""" + project = data.get("project", "") + model = data.get("model", {}) + console.print( + f"\n[bold]Model[/bold] [cyan]{model.get('name', '?')}[/cyan] " + f"([dim]{model.get('id', '')}[/dim]) in project [magenta]{project}[/magenta]:" + ) + table = Table() + table.add_column("Entity", style="bold cyan") + table.add_column("Count", justify="right") + for label, key in ( + ("datasets", "datasets"), + ("metrics", "metrics"), + ("relationships", "relationships"), + ("constraints", "constraints"), + ("glossary", "glossary"), + ): + if key in data: + table.add_row(label, str(len(data.get(key, [])))) + console.print(table) + + +def _print_show_detail(console: Console, data: dict) -> None: + """Render the show payload with --type as a per-item table.""" + for type_key in ("datasets", "metrics", "relationships", "constraints", "glossary"): + if type_key not in data: + continue + items = data[type_key] + if not items: + console.print(f"[dim]No {type_key} in this model.[/dim]") + continue + table = Table(title=type_key.capitalize()) + # Column layout differs per type, but a generic name/id/key-attrs + # rendering keeps the table compact and is enough for `show`. + keys = list(items[0].keys()) + # Drop modelUUID -- noisy, present on every item + keys = [k for k in keys if k != "modelUUID"] + for col in keys: + table.add_column(col) + for item in items: + table.add_row(*(str(item.get(c, "")) for c in keys)) + console.print(table) + + +# --------------------------------------------------------------------------- +# semantic-layer import -- replay a snapshot +# --------------------------------------------------------------------------- + + +def _print_import_result(console: Console, data: dict) -> None: + console.print( + f"\n[bold]Import[/bold] into [magenta]{data.get('target_project')}[/magenta]" + f" (model {data.get('target_model')}, " + f"dry_run={data.get('dry_run')}, overwrite={data.get('overwrite')}):" + ) + table = Table() + table.add_column("Type", style="bold cyan") + table.add_column("Created", justify="right", style="green") + table.add_column("Skipped", justify="right", style="yellow") + table.add_column("Overwritten", justify="right", style="magenta") + table.add_column("Failed", justify="right", style="red") + for plural in ("datasets", "metrics", "relationships", "glossary", "constraints"): + per = (data.get("imported") or {}).get(plural) + if per is None: + continue + table.add_row( + plural, + str(per.get("created", 0)), + str(per.get("skipped", 0)), + str(per.get("overwritten", 0)), + str(len(per.get("failed", []))), + ) + console.print(table) + for plural, per in (data.get("imported") or {}).items(): + for f in per.get("failed", []): + console.print(f" [red]✗ {plural}.{f.get('name')}: {f.get('reason')}[/red]") + + +# --------------------------------------------------------------------------- +# semantic-layer promote -- cross-project copy +# --------------------------------------------------------------------------- + + +def _print_promote_result(console: Console, data: dict) -> None: + console.print( + f"\n[bold]Promote[/bold] [magenta]{data.get('from_project')}[/magenta] " + f"-> [magenta]{data.get('to_project')}[/magenta] " + f"(dry_run={data.get('dry_run')}):" + ) + table = Table() + table.add_column("Type", style="bold cyan") + table.add_column("New", justify="right", style="green") + table.add_column("Overwritten", justify="right", style="magenta") + table.add_column("Identical", justify="right", style="dim") + table.add_column("Failed", justify="right", style="red") + for plural in ("datasets", "metrics", "relationships", "glossary", "constraints"): + per = data.get(plural) + if per is None: + continue + table.add_row( + plural, + str(per.get("new", 0)), + str(per.get("overwritten", 0)), + str(per.get("identical", 0)), + str(len(per.get("failed", []))), + ) + console.print(table) + for plural in ("datasets", "metrics", "relationships", "glossary", "constraints"): + per = data.get(plural) or {} + for c in per.get("changes", []): + key = c.get("name", c.get("term", "?")) + keys = ", ".join(c.get("diff_keys", [])) + console.print(f" [yellow]~ {plural}.{key}[/yellow] ([dim]{keys}[/dim])") + for f in per.get("failed", []): + console.print(f" [red]✗ {plural}.{f.get('name')}: {f.get('reason')}[/red]") + + +# --------------------------------------------------------------------------- +# semantic-layer build -- non-interactive greenfield +# --------------------------------------------------------------------------- + + +def _print_build_result(console: Console, data: dict) -> None: + fallback = data.get("fallback_used") + valid = data.get("validated", False) + console.print( + f"\n[bold]Build[/bold] (mode=[cyan]{fallback}[/cyan], dry_run={data.get('dry_run')})" + ) + val = data.get("validation") or {} + errs = val.get("errors", []) + warns = val.get("warnings", []) + console.print(f" Datasets: {len(data.get('generated', {}).get('datasets', []))}") + console.print(f" Metrics: {len(data.get('generated', {}).get('metrics', []))}") + console.print(f" Relationships: {len(data.get('generated', {}).get('relationships', []))}") + console.print(f" Constraints: {len(data.get('generated', {}).get('constraints', []))}") + console.print(f" Glossary: {len(data.get('generated', {}).get('glossary', []))}") + if data.get("output_path"): + console.print(f"\nWritten to: [cyan]{data['output_path']}[/cyan]") + if errs: + console.print(f"\n[bold red]Validation: {len(errs)} error(s)[/bold red]") + for e in errs: + console.print(f" [red]✗[/red] {e['type']} {e['item']} — {e['detail']}") + if warns: + console.print(f"\n[bold yellow]Validation: {len(warns)} warning(s)[/bold yellow]") + if data.get("created"): + console.print(f"\nCreated: {data['created']}") + elif data.get("dry_run"): + console.print("\n[dim]--dry-run: no API calls were made.[/dim]") + elif valid: + console.print("\n[bold green]Pushed.[/bold green]") + + +# --------------------------------------------------------------------------- +# semantic-layer token --encrypt +# --------------------------------------------------------------------------- + + +@semantic_layer_app.command("token") +def semantic_layer_token( + ctx: typer.Context, + encrypt: bool = typer.Option( + False, "--encrypt", help="Encrypt the project token for `user_properties` (required)" + ), + project: str = typer.Option(..., "--project", help="Project alias"), + component_id: str = typer.Option( + ..., + "--component-id", + help="Keboola component id the encrypted token will be used in", + ), +) -> None: + """Encrypt the project's storage token for transformation `user_properties`. + + Builds the ``{"#metastore_token": <token>}`` payload using the + project's already-stored Storage API token (no config-file digging), + then delegates to the existing EncryptService. Output (human) is the + raw envelope ready to paste into `user_properties`; JSON mode emits + the full `{encrypted, component_id, project}` response. + """ + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.token", + project=project, + component_id=component_id, + ) + return + formatter = get_formatter(ctx) + if not encrypt: + formatter.error( + message=( + "Currently only --encrypt mode is supported. " + "Pass --encrypt to encrypt the project token for user_properties." + ), + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) + + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.encrypt_token, + alias=project, + component_id=component_id, + ) + + def _print_encrypted_token(console: Console, data: dict) -> None: + console.print( + "\n[bold green]Encrypted token[/bold green] " + f"for component [cyan]{data['component_id']}[/cyan] " + f"in project [magenta]{data['project']}[/magenta]:" + ) + console.print_json(json.dumps(data["encrypted"], indent=2)) + console.print( + "[dim]Paste the JSON above into the transformation's `user_properties` block.[/dim]" + ) + + formatter.output(result, _print_encrypted_token) + + +@semantic_layer_app.command("build") +def semantic_layer_build( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Target project alias"), + model: str | None = typer.Option( + None, + "--model", + help="Update this existing model (name or UUID). If omitted, a new model is created.", + ), + tables: str | None = typer.Option( + None, + "--tables", + help="Comma-separated tableIds to base the model on (required).", + ), + name: str | None = typer.Option( + None, "--name", help="Model name when creating (default: kbagent_build_model)." + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Print the generated JSON + validation, no POST." + ), + output: Path | None = typer.Option( + None, "--output", help="Also write the generated JSON to this file." + ), +) -> None: + """Build a semantic-layer model from a list of storage tables (non-interactive). + + NOTE: The AI Service client currently has no JSON-generation endpoint, so + this command falls back to a DETERMINISTIC HEURISTIC builder (one dataset + + one COUNT(*) metric + one glossary entry per table; no relationships; + no constraints). The intent is "best starting point" — iterate with + `add` / `edit`. The fallback is logged in the response as + `fallback_used: "heuristic"` so callers can detect it. + """ + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.build", + project=project, + model=model, + tables=tables or "", + name=name, + dry_run=dry_run, + output=str(output) if output else "", + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + + if not tables: + formatter.error( + message="--tables is required (comma-separated tableIds).", + error_code=ErrorCode.MISSING_PARAMETER, + ) + raise typer.Exit(code=2) + table_ids = [t.strip() for t in tables.split(",") if t.strip()] + if not table_ids: + formatter.error( + message="--tables contained no tableIds.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + + result = _handle_service_call( + ctx, + service.build_model, + alias=project, + table_ids=table_ids, + model_name=name, + model_name_or_uuid=model, + dry_run=dry_run, + output_path=output, + ) + formatter.output(result, _print_build_result) + + +@semantic_layer_app.command("promote") +def semantic_layer_promote( + ctx: typer.Context, + from_project: str = typer.Option(..., "--from-project", help="Source project alias"), + to_project: str = typer.Option(..., "--to-project", help="Target project alias"), + from_model: str | None = typer.Option( + None, "--from-model", help="Source model name or UUID (defaults to sole)" + ), + to_model: str | None = typer.Option( + None, "--to-model", help="Target model name or UUID (defaults to sole)" + ), + types: str | None = typer.Option( + None, + "--types", + help="Comma-separated subset (datasets,metrics,relationships,glossary,constraints)", + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Classify NEW/IDENTICAL/CHANGED without writing" + ), + yes: bool = typer.Option( + False, "--yes", "-y", help="Skip the cross-project confirmation prompt" + ), +) -> None: + """Promote a model from one project to another (NEW + overwrite CHANGED; never deletes). + + Default behaviour: NEW items are POSTed, CHANGED items are + DELETE+POSTed, IDENTICAL items are skipped. Items only present in + the target are never touched (additive-only). + """ + if should_hint(ctx): + # Use `from_project` to resolve the hint stack URL. + emit_hint( + ctx, + "semantic-layer.promote", + project=from_project, + from_project=from_project, + to_project=to_project, + from_model=from_model, + to_model=to_model, + types=types, + dry_run=dry_run, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + type_list = [t.strip() for t in types.split(",") if t.strip()] if types else None + + if ( + not yes + and not dry_run + and not formatter.json_mode + and not typer.confirm( + f"Promote model from '{from_project}' to '{to_project}'? " + "This will overwrite CHANGED items in the target." + ) + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + + result = _handle_service_call( + ctx, + service.promote_model, + from_project=from_project, + to_project=to_project, + from_model=from_model, + to_model=to_model, + types=type_list, + dry_run=dry_run, + ) + formatter.output(result, _print_promote_result) + + +@semantic_layer_app.command("import") +def semantic_layer_import( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Target project alias"), + file: Path = typer.Option( + ..., "--file", help="Snapshot JSON file (output of `semantic-layer export`)" + ), + model: str | None = typer.Option( + None, "--model", help="Target model name or UUID (defaults to the sole model)" + ), + types: str | None = typer.Option( + None, + "--types", + help=( + "Comma-separated subset to import: datasets,metrics,relationships,glossary,constraints" + ), + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Plan the import without calling any write API" + ), + overwrite: bool = typer.Option( + False, "--overwrite", help="DELETE+POST conflicting items (default: skip)" + ), + yes: bool = typer.Option( + False, "--yes", "-y", help="Skip confirmation (alias for default SKIP behavior)" + ), +) -> None: + """Replay a snapshot into a project. Default: skip on conflict (no surprise overwrites).""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.import", + project=project, + file=str(file), + model=model, + types=types, + dry_run=dry_run, + overwrite=overwrite, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + # --yes is the alias for the default skip-on-conflict mode; users can + # still opt into destructive overwrite via --overwrite. + _ = yes # explicit (no behavioural effect when --overwrite is False) + type_list = [t.strip() for t in types.split(",") if t.strip()] if types else None + result = _handle_service_call( + ctx, + service.import_snapshot, + alias=project, + file=file, + model_name_or_uuid=model, + types=type_list, + dry_run=dry_run, + overwrite=overwrite, + ) + formatter.output(result, _print_import_result) + + +@semantic_layer_app.command("show") +def semantic_layer_show( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option( + None, + "--model", + help="Model name or UUID. Optional when the project has a single model.", + ), + type_filter: str | None = typer.Option( + None, + "--type", + help=( + "Filter to one entity type: dataset | metric | relationship | constraint | glossary." + ), + ), +) -> None: + """Show the entities in a semantic-layer model.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.show", + project=project, + model=model, + type=type_filter, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.show_model, + alias=project, + model_name_or_uuid=model, + type_filter=type_filter, + ) + if type_filter is None: + formatter.output(result, _print_show_summary) + else: + formatter.output(result, _print_show_detail) + + +# --------------------------------------------------------------------------- +# semantic-layer validate [--deep] +# --------------------------------------------------------------------------- + + +def _print_validate(console: Console, data: dict) -> None: + project = data.get("project", "") + model = data.get("model", {}) + deep = data.get("deep", False) + valid = data.get("valid", False) + console.print( + f"\n[bold]Validation[/bold] for model [cyan]{model.get('name', '?')}[/cyan] " + f"in [magenta]{project}[/magenta]" + f" ({'deep' if deep else 'basic'}):" + ) + errs = data.get("errors", []) + warns = data.get("warnings", []) + if errs: + console.print(f"\n[bold red]Errors ({len(errs)}):[/bold red]") + for e in errs: + console.print(f" [red]✗[/red] [bold]{e['type']}[/bold] {e['item']} — {e['detail']}") + if warns: + console.print(f"\n[bold yellow]Warnings ({len(warns)}):[/bold yellow]") + for w in warns: + console.print( + f" [yellow]![/yellow] [bold]{w['type']}[/bold] {w['item']} — {w['detail']}" + ) + if valid and not warns: + console.print("\n[bold green]Model is clean.[/bold green]") + elif valid: + console.print(f"\n[bold green]Model is valid[/bold green] (with {len(warns)} warning(s)).") + else: + console.print(f"\n[bold red]Model has {len(errs)} error(s).[/bold red]") + + +# --------------------------------------------------------------------------- +# semantic-layer export +# --------------------------------------------------------------------------- + + +def _print_export(console: Console, data: dict) -> None: + counts = data.get("counts", {}) + console.print(f"\n[bold green]Exported model[/bold green] to: [cyan]{data['path']}[/cyan]") + table = Table() + table.add_column("Entity", style="bold cyan") + table.add_column("Count", justify="right") + for key in ("datasets", "metrics", "relationships", "constraints", "glossary"): + table.add_row(key, str(counts.get(key, 0))) + console.print(table) + + +@semantic_layer_app.command("export") +def semantic_layer_export( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option( + None, "--model", help="Model name or UUID (optional if project has one model)." + ), + output: Path | None = typer.Option( + None, + "--output", + help=("Output JSON path. Defaults to ./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json."), + ), +) -> None: + """Snapshot a semantic-layer model to a self-describing JSON file.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.export", + project=project, + model=model, + output=str(output) if output else "", + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.export_model, + alias=project, + model_name_or_uuid=model, + output_path=output, + ) + formatter.output(result, _print_export) + + +# --------------------------------------------------------------------------- +# semantic-layer diff +# --------------------------------------------------------------------------- + + +def _print_diff(console: Console, data: dict) -> None: + left = data.get("left", {}) + right = data.get("right", {}) + console.print( + f"\n[bold]Diff[/bold] left=[cyan]{left.get('source')}[/cyan]:[magenta]" + f"{left.get('ref')}[/magenta] " + f"right=[cyan]{right.get('source')}[/cyan]:[magenta]{right.get('ref')}[/magenta]" + ) + for type_key in ("datasets", "metrics", "relationships", "constraints", "glossary"): + per = data.get(type_key, {}) + added = per.get("added", []) + removed = per.get("removed", []) + changed = per.get("changed", []) + if not (added or removed or changed): + continue + console.print(f"\n[bold cyan]{type_key}[/bold cyan]:") + for name in added: + console.print(f" [green]+ {name}[/green]") + for name in removed: + console.print(f" [red]- {name}[/red]") + for c in changed: + key = c.get("name", c.get("term", "?")) + keys = ", ".join(c.get("diff_keys", [])) + console.print(f" [yellow]~ {key}[/yellow] ([dim]{keys}[/dim])") + + +@semantic_layer_app.command("diff") +def semantic_layer_diff( + ctx: typer.Context, + project_a: str | None = typer.Option(None, "--project-a", help="Left side: project alias"), + project_b: str | None = typer.Option(None, "--project-b", help="Right side: project alias"), + model_a: str | None = typer.Option( + None, "--model-a", help="Left side: model name/UUID (when --project-a is set)" + ), + model_b: str | None = typer.Option( + None, "--model-b", help="Right side: model name/UUID (when --project-b is set)" + ), + file_a: Path | None = typer.Option( + None, + "--file-a", + help="Left side: snapshot JSON path (mutually exclusive with --project-a)", + ), + file_b: Path | None = typer.Option( + None, + "--file-b", + help="Right side: snapshot JSON path (mutually exclusive with --project-b)", + ), +) -> None: + """Diff two semantic-layer snapshots (project↔project, project↔file, file↔file). + + Pass exactly one of ``--project-a`` / ``--file-a`` and one of + ``--project-b`` / ``--file-b``. Output groups changes per entity type: + ``added``, ``removed``, ``changed`` (with ``diff_keys`` listing the + attribute fields that differ). + """ + if should_hint(ctx): + # `project` resolves the hint stack URL; prefer A, fall back to B. + hint_project = project_a or project_b + emit_hint( + ctx, + "semantic-layer.diff", + project=hint_project, + project_a=project_a, + project_b=project_b, + model_a=model_a, + model_b=model_b, + file_a=str(file_a) if file_a else "", + file_b=str(file_b) if file_b else "", + ) + return + formatter = get_formatter(ctx) + + # Mutual exclusion: exactly one source per side. + if (project_a is None) == (file_a is None): + formatter.error( + message="Specify exactly one of --project-a or --file-a.", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) + if (project_b is None) == (file_b is None): + formatter.error( + message="Specify exactly one of --project-b or --file-b.", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) + + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.diff, + project_a=project_a, + project_b=project_b, + model_a=model_a, + model_b=model_b, + file_a=file_a, + file_b=file_b, + ) + formatter.output(result, _print_diff) + + +@semantic_layer_app.command("validate") +def semantic_layer_validate( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option( + None, "--model", help="Model name or UUID (optional if project has one model)." + ), + deep: bool = typer.Option( + False, + "--deep", + help=( + "Fetch every dataset's storage schema in parallel and add " + "phantom-field, metric-phantom, and agg-on-STRING checks." + ), + ), +) -> None: + """Validate a semantic-layer model. + + Basic checks: duplicates, dangling rel/metric, sum-on-pct, constraint + orphan, severity-suffix warning. With ``--deep``: also probe the actual + Snowflake schema for phantom fields, phantom column refs, and AGG-on-STRING. + """ + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.validate", + project=project, + model=model, + deep=deep, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.validate_model, + alias=project, + model_name_or_uuid=model, + deep=deep, + ) + formatter.output(result, _print_validate) diff --git a/src/keboola_agent_cli/hints/definitions/__init__.py b/src/keboola_agent_cli/hints/definitions/__init__.py index 2ad90ac5..1cbda7d0 100644 --- a/src/keboola_agent_cli/hints/definitions/__init__.py +++ b/src/keboola_agent_cli/hints/definitions/__init__.py @@ -15,6 +15,7 @@ project, # noqa: F401 schedule, # noqa: F401 search, # noqa: F401 + semantic_layer, # noqa: F401 sharing, # noqa: F401 storage, # noqa: F401 tool, # noqa: F401 diff --git a/src/keboola_agent_cli/hints/definitions/semantic_layer.py b/src/keboola_agent_cli/hints/definitions/semantic_layer.py new file mode 100644 index 00000000..0ad5097e --- /dev/null +++ b/src/keboola_agent_cli/hints/definitions/semantic_layer.py @@ -0,0 +1,729 @@ +"""Hint definitions for the ``semantic-layer`` command group (since v0.41.0). + +Mirrors the per-subcommand surface of +:class:`keboola_agent_cli.services.semantic_layer_service.SemanticLayerService`. +The wire-level client is +:class:`keboola_agent_cli.metastore_client.MetastoreClient`; the renderer +construct it directly via ``client_type="metastore"``. The metastore URL +is derived from each project's stack URL automatically +(``connection.`` -> ``metastore.``). +""" + +from .. import HintRegistry +from ..models import ClientCall, CommandHint, HintStep, ServiceCall + +# Note printed on every hint that lists by parallel-fanning across the +# child types. The renderer only emits one ``item_type`` per ``list_items`` +# call, but the underlying service makes five such calls in parallel — +# this note ensures the reader understands the single rendered call is +# representative. +_PARALLEL_CHILDREN_NOTE = ( + "The service fans out 5 list_items calls in parallel — one per child " + "kind: semantic-dataset, semantic-metric, semantic-relationship, " + "semantic-constraint, semantic-glossary. The rendered snippet shows " + "one representative call; replicate for each kind in real code." +) + + +def _make_service(method: str, **extra_args: str) -> ServiceCall: + """Convenience builder for the service half of a hint step. + + All semantic-layer ServiceCalls hit the same module+class, so factor + that out. + """ + args: dict[str, str] = {"alias": "{project}"} + args.update(extra_args) + return ServiceCall( + service_class="SemanticLayerService", + service_module="semantic_layer_service", + method=method, + args=args, + ) + + +# ── semantic-layer model list ────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.model.list", + description="List semantic-layer models in a project", + steps=[ + HintStep( + comment="List every `semantic-model` item in the project", + client=ClientCall( + method="list_items", + args={"item_type": '"semantic-model"'}, + client_type="metastore", + result_var="models", + result_hint="list[dict]", + ), + service=_make_service("list_models"), + ), + ], + ) +) + + +# ── semantic-layer model create ──────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.model.create", + description="Create a new semantic-layer model", + steps=[ + HintStep( + comment="POST /semantic-model with name + sql_dialect (+ optional description)", + client=ClientCall( + method="post_item", + args={ + "item_type": '"semantic-model"', + "name": "{name}", + "data": '{"name": {name}, "sql_dialect": {sql_dialect}}', + }, + client_type="metastore", + result_var="model", + result_hint="dict", + ), + service=_make_service( + "create_model", + name="{name}", + description="{description}", + sql_dialect="{sql_dialect}", + ), + ), + ], + notes=[ + "Duplicate model name returns HTTP 500 (normalized to ALREADY_EXISTS).", + ], + ) +) + + +# ── semantic-layer model delete ──────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.model.delete", + description="Delete a semantic-layer model (must have no children)", + steps=[ + HintStep( + comment="DELETE /semantic-model/{id}; metastore refuses if children exist", + client=ClientCall( + method="delete_item", + args={"item_type": '"semantic-model"', "item_id": "{model}"}, + client_type="metastore", + result_var="result", + ), + service=_make_service("delete_model", model_name_or_uuid="{model}"), + ), + ], + notes=[ + "The service layer pre-fetches children for an orphan-warning envelope.", + ], + ) +) + + +# ── semantic-layer show ──────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.show", + description="Show all entities of a model (datasets, metrics, etc.)", + steps=[ + HintStep( + comment=( + "Resolve model UUID, then fetch every child type in " + "parallel filtered by modelUUID (5 list_items calls)." + ), + client=ClientCall( + method="list_items", + args={ + "item_type": '"semantic-metric"', + "model_uuid": "{model}", + }, + client_type="metastore", + result_var="metrics", + result_hint="list[dict]", + ), + service=_make_service( + "show_model", + model_name_or_uuid="{model}", + type_filter="{type}", + ), + ), + ], + notes=[ + _PARALLEL_CHILDREN_NOTE, + "Service layer fans the 5 list_items calls out in parallel (ThreadPoolExecutor max_workers=5).", + "`--type` collapses the result to a single plural key (dataset->datasets, glossary->glossary, ...).", + ], + ) +) + + +# ── semantic-layer validate ──────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.validate", + description="Validate a semantic-layer model (basic + --deep)", + steps=[ + HintStep( + comment=( + "Same parallel child fetch as `show`. Basic checks are " + "in-memory only; --deep additionally fetches every " + "dataset's Snowflake schema in parallel via " + "StorageService.get_table_detail." + ), + client=ClientCall( + method="list_items", + args={ + "item_type": '"semantic-metric"', + "model_uuid": "{model}", + }, + client_type="metastore", + result_var="metrics", + result_hint="list[dict]", + ), + service=_make_service( + "validate_model", + model_name_or_uuid="{model}", + deep="{deep}", + ), + ), + ], + notes=[ + _PARALLEL_CHILDREN_NOTE, + "Basic checks: DUPLICATE, DANGLING_RELATIONSHIP, DANGLING_METRIC, " + "SUM_ON_PCT, CONSTRAINT_ORPHAN, SEVERITY_SUFFIX.", + "--deep adds: PHANTOM_FIELD, METRIC_PHANTOM, AGG_ON_STRING via " + "real Snowflake column probing.", + ], + ) +) + + +# ── semantic-layer export ────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.export", + description="Snapshot a model + every child entity to JSON", + steps=[ + HintStep( + comment=( + "Parallel child fetch (5 list_items) -> snapshot envelope -> " + "atomic os.open(O_NOFOLLOW, 0o644) write." + ), + client=ClientCall( + method="list_items", + args={ + "item_type": '"semantic-metric"', + "model_uuid": "{model}", + }, + client_type="metastore", + result_var="metrics", + result_hint="list[dict]", + ), + service=_make_service( + "export_model", + model_name_or_uuid="{model}", + output_path="Path({output})", + ), + ), + ], + notes=[ + _PARALLEL_CHILDREN_NOTE, + "Default output path: ./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json.", + "File is self-describing -- replayable by `import` and `promote`.", + ], + ) +) + + +# ── semantic-layer diff ──────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.diff", + description="Diff two snapshots (project<->project, project<->file, file<->file)", + steps=[ + HintStep( + comment=( + "Resolve each side (live project = parallel child fetch; " + "file = read+parse JSON), then per-type added/removed/changed " + "with `diff_keys` for changed." + ), + client=ClientCall( + method="list_items", + args={ + "item_type": '"semantic-metric"', + "model_uuid": "{model_a}", + }, + client_type="metastore", + result_var="metrics_a", + result_hint="list[dict]", + ), + service=_make_service( + "diff", + project_a="{project_a}", + project_b="{project_b}", + model_a="{model_a}", + model_b="{model_b}", + file_a="Path({file_a})", + file_b="Path({file_b})", + ), + ), + ], + notes=[ + _PARALLEL_CHILDREN_NOTE, + "Both sides are loaded independently — replicate the call for " + "model_b too. File-backed sides skip this client call and parse JSON instead.", + "Ignored keys: modelUUID, createdAt, lastUpdated, revision.", + ], + ) +) + + +# ── semantic-layer add (one per entity type) ─────────────────────── + + +def _register_add_hint(entity: str, **extra_service_args: str) -> None: + """Register an `add.<entity>` hint with the standard POST shape.""" + type_slug = f"semantic-{entity}" + HintRegistry.register( + CommandHint( + cli_command=f"semantic-layer.add.{entity}", + description=f"Add a {entity} to a semantic-layer model", + steps=[ + HintStep( + comment=( + f"Resolve modelUUID, then POST /{type_slug} with the validated payload." + ), + client=ClientCall( + method="post_item", + args={ + "item_type": f'"{type_slug}"', + "name": "{name}", + "data": '{"modelUUID": "<resolved>"}', + }, + client_type="metastore", + result_var="created", + result_hint="dict", + ), + service=_make_service( + f"add_{entity}", + model_name_or_uuid="{model}", + **extra_service_args, + ), + ), + ], + ) + ) + + +_register_add_hint( + "metric", + name="{name}", + sql="{sql}", + dataset="{dataset}", + description="{description}", +) +_register_add_hint( + "dataset", + name="{name}", + table_id="{table_id}", + description="{description}", + grain="{grain}", + primary_key="{primary_key}", + deep_fields="{deep_fields}", +) +_register_add_hint( + "relationship", + name="{name}", + from_="{from_}", + to="{to}", + on="{on}", + type_="{type_}", +) +_register_add_hint( + "constraint", + name="{name}", + constraint_type="{constraint_type}", + rule="{rule}", + metrics="{metrics}", + severity="{severity}", +) +# glossary uses `term`, not `name`. +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.add.glossary", + description="Add a glossary term to a semantic-layer model", + steps=[ + HintStep( + comment="Resolve modelUUID, then POST /semantic-glossary {term, definition}.", + client=ClientCall( + method="post_item", + args={ + "item_type": '"semantic-glossary"', + "name": "{term}", + "data": '{"term": {term}, "modelUUID": "<resolved>"}', + }, + client_type="metastore", + result_var="created", + result_hint="dict", + ), + service=_make_service( + "add_glossary", + model_name_or_uuid="{model}", + term="{term}", + definition="{definition}", + ), + ), + ], + notes=[ + "Outer envelope `name` must equal `term` -- the service handles this.", + ], + ) +) + + +# ── semantic-layer edit (DELETE+POST with rollback) ──────────────── + + +def _register_edit_hint(entity: str, **extra_service_args: str) -> None: + """Register an `edit.<entity>` hint -- DELETE old + POST new.""" + type_slug = f"semantic-{entity}" + HintRegistry.register( + CommandHint( + cli_command=f"semantic-layer.edit.{entity}", + description=f"Edit a {entity} via DELETE+POST with rollback", + steps=[ + HintStep( + comment=( + f"DELETE /{type_slug}/<old_id>, then POST the new " + "payload. On POST failure, re-POST original_attrs to " + "roll back; the rollback success/failure is reported " + "in the envelope." + ), + client=ClientCall( + method="delete_item", + args={ + "item_type": f'"{type_slug}"', + "item_id": '"<resolved-old-id>"', + }, + client_type="metastore", + result_var="deleted", + ), + service=_make_service( + f"edit_{entity}", + model_name_or_uuid="{model}", + **extra_service_args, + ), + ), + ], + notes=[ + "metastore exposes no PATCH -- DELETE+POST is the only edit shape.", + "edit_metric rename cascades through constraints whose `metrics[]` " + "includes the old name (DELETE old constraint + POST new).", + ], + ) + ) + + +_register_edit_hint( + "metric", + current_name="{name}", + new_name="{new_name}", + new_sql="{new_sql}", + new_dataset="{new_dataset}", + new_description="{new_description}", +) +_register_edit_hint( + "dataset", + current_name="{name}", + new_name="{new_name}", + new_description="{new_description}", + new_grain="{new_grain}", +) +_register_edit_hint( + "constraint", + current_name="{name}", + new_name="{new_name}", + new_rule="{new_rule}", + new_constraint_type="{new_constraint_type}", + new_severity="{new_severity}", + new_metrics="{new_metrics}", +) +_register_edit_hint( + "relationship", + current_name="{name}", + new_name="{new_name}", + new_from="{new_from}", + new_to="{new_to}", + new_on="{new_on}", + new_type="{new_type}", +) +# glossary identity is `term`. +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.edit.glossary", + description="Edit a glossary term via DELETE+POST with rollback", + steps=[ + HintStep( + comment=( + "DELETE /semantic-glossary/<old_id>, then POST the new " + "{term, definition} payload. Renaming the term is " + "destructive for downstream consumers joining on the " + "term string." + ), + client=ClientCall( + method="delete_item", + args={ + "item_type": '"semantic-glossary"', + "item_id": '"<resolved-old-id>"', + }, + client_type="metastore", + result_var="deleted", + ), + service=_make_service( + "edit_glossary", + model_name_or_uuid="{model}", + current_term="{term}", + new_term="{new_term}", + new_definition="{new_definition}", + ), + ), + ], + notes=[ + "--new-term is destructive -- pass --yes to bypass the TTY confirm.", + ], + ) +) + + +# ── semantic-layer remove ────────────────────────────────────────── + + +def _register_remove_hint(entity: str, id_key: str = "name") -> None: + """Register a `remove.<entity>` hint.""" + type_slug = f"semantic-{entity}" + HintRegistry.register( + CommandHint( + cli_command=f"semantic-layer.remove.{entity}", + description=f"Remove a {entity} (destructive)", + steps=[ + HintStep( + comment=( + f"Resolve target by {id_key}, then DELETE /{type_slug}/<id>. " + "For metric, pre-scan constraints whose metrics[] " + "includes the target name for an orphan-warning envelope." + ), + client=ClientCall( + method="delete_item", + args={ + "item_type": f'"{type_slug}"', + "item_id": '"<resolved-id>"', + }, + client_type="metastore", + result_var="result", + ), + service=_make_service( + "remove_item", + model_name_or_uuid="{model}", + kind=f'"{entity}"', + name="{name}" if id_key == "name" else "{term}", + ), + ), + ], + notes=[ + "The CLI calls preview_remove first to populate the orphan-warning envelope.", + "Non-TTY without --yes refuses with exit 2 (the warning is always printed).", + ], + ) + ) + + +_register_remove_hint("metric") +_register_remove_hint("dataset") +_register_remove_hint("constraint") +_register_remove_hint("relationship") +_register_remove_hint("glossary", id_key="term") + + +# ── semantic-layer import ────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.import", + description="Replay a snapshot into a project (skip-on-conflict by default)", + steps=[ + HintStep( + comment=( + "Load snapshot JSON, then iterate _PUSH_ORDER " + "(datasets, metrics, relationships, glossary, constraints) " + "and POST each item; rewrite modelUUID to the target. " + "Conflicting items are SKIPPED by default; --overwrite " + "triggers DELETE+POST." + ), + client=ClientCall( + method="post_item", + args={ + "item_type": '"semantic-metric"', + "name": '"<per-item name>"', + "data": '{"modelUUID": "<target>"}', + }, + client_type="metastore", + result_var="created", + ), + service=_make_service( + "import_snapshot", + file="Path({file})", + model_name_or_uuid="{model}", + types="{types}", + dry_run="{dry_run}", + overwrite="{overwrite}", + ), + ), + ], + notes=[ + "Default is skip-on-conflict (additive). Pass --overwrite for DELETE+POST.", + "--dry-run plans counts without any write call.", + "Loop the rendered post_item across each item_type in PUSH_ORDER: " + "semantic-dataset, semantic-metric, semantic-relationship, " + "semantic-glossary, semantic-constraint.", + ], + ) +) + + +# ── semantic-layer promote ───────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.promote", + description="Promote a model from one project to another (additive + overwrite)", + steps=[ + HintStep( + comment=( + "Open two MetastoreClients (src + tgt). Parallel-fetch " + "children on each. Classify items as NEW / IDENTICAL / " + "CHANGED. POST NEW; DELETE+POST CHANGED; skip IDENTICAL. " + "Items only in target are NEVER deleted (additive only)." + ), + client=ClientCall( + method="list_items", + args={ + "item_type": '"semantic-metric"', + "model_uuid": '"<from_model_uuid>"', + }, + client_type="metastore", + result_var="src_metrics", + ), + service=_make_service( + "promote_model", + from_project="{from_project}", + to_project="{to_project}", + from_model="{from_model}", + to_model="{to_model}", + types="{types}", + dry_run="{dry_run}", + ), + ), + ], + notes=[ + "Two clients held in try/finally; both close even on error. The " + "rendered snippet constructs only one client — instantiate a second " + "MetastoreClient for the target project with its own token.", + "Deep-equality compare strips modelUUID + timestamps (revision, createdAt, lastUpdated).", + ], + ) +) + + +# ── semantic-layer build ─────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.build", + description="Heuristic greenfield builder from a list of storage tables", + steps=[ + HintStep( + comment=( + "Fetch every tableId's schema in parallel (StorageService." + "get_table_detail). Synthesise one dataset (with role-" + "classified fields), one COUNT(*) metric, and one " + "glossary entry per table. Validate locally. If clean, " + "POST in dependency order." + ), + client=ClientCall( + method="post_item", + args={ + "item_type": '"semantic-dataset"', + "name": '"<derived-from-tableId>"', + "data": '{"modelUUID": "<target>"}', + }, + client_type="metastore", + result_var="created", + ), + service=_make_service( + "build_model", + table_ids="{tables}", + model_name="{name}", + model_name_or_uuid="{model}", + dry_run="{dry_run}", + output_path="Path({output})", + ), + ), + ], + notes=[ + "Response carries `fallback_used: 'heuristic'` -- no AI Service " + "JSON-generation endpoint exists yet. Use as a scaffold, iterate " + "via `add` / `edit`.", + "Refuses to push if local validation surfaces errors (returns " + "VALIDATION_ERROR with the error list in details).", + "After the dataset POST, loop through semantic-metric, " + "semantic-relationship, semantic-glossary, and semantic-constraint " + "in that order.", + ], + ) +) + + +# ── semantic-layer token --encrypt ───────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.token", + description="Encrypt the project token for a transformation user_properties", + steps=[ + HintStep( + comment=( + "Build `{'#metastore_token': <project token>}` from the " + "already-stored Storage token, then delegate to the " + "EncryptService (same path as `encrypt values`)." + ), + client=ClientCall( + method="encrypt_values", + args={ + "component_id": "{component_id}", + "data": '{"#metastore_token": "<project token>"}', + }, + result_var="encrypted", + result_hint="dict", + ), + service=_make_service( + "encrypt_token", + component_id="{component_id}", + ), + ), + ], + notes=[ + "Uses the Encryption API (encryption.keboola.com).", + "Output is a `KBC::ProjectSecure...` ciphertext ready to paste " + "into the transformation's `user_properties` block.", + "Classified `write` (same blast radius as `encrypt.values`).", + ], + ) +) diff --git a/src/keboola_agent_cli/hints/renderer.py b/src/keboola_agent_cli/hints/renderer.py index 09e4f4de..4530c13c 100644 --- a/src/keboola_agent_cli/hints/renderer.py +++ b/src/keboola_agent_cli/hints/renderer.py @@ -134,7 +134,10 @@ def render( client_types.add(step.client.client_type) needs_os = ( - "storage" in client_types or "manage" in client_types or "data_science" in client_types + "storage" in client_types + or "manage" in client_types + or "data_science" in client_types + or "metastore" in client_types ) if needs_os: lines.append("import os") @@ -150,6 +153,8 @@ def render( lines.append("from keboola_agent_cli.data_science_client import DataScienceClient") if "manage" in client_types: lines.append("from keboola_agent_cli.manage_client import ManageClient") + if "metastore" in client_types: + lines.append("from keboola_agent_cli.metastore_client import MetastoreClient") if "mcp" in client_types: lines.append("from keboola_agent_cli.config_store import ConfigStore") lines.append("from keboola_agent_cli.services.mcp_service import McpService") @@ -183,6 +188,20 @@ def render( lines.append(' token=os.environ["KBC_MANAGE_API_TOKEN"],') lines.append(")") + if "metastore" in client_types: + url_comment = "" + if stack_url: + project = params.get("project") + if project: + proj_label = project[0] if isinstance(project, list) else project + safe_label = _escape_for_python_string(str(proj_label)) + url_comment = f" # from project '{safe_label}'" + lines.append("# Derives metastore.{stack-suffix} from the stack URL internally.") + lines.append("metastore_client = MetastoreClient(") + lines.append(f' stack_url="{url}",{url_comment}') + lines.append(' token=os.environ["KBC_STORAGE_TOKEN"],') + lines.append(")") + if "mcp" in client_types: config_dir_str = str(config_dir) if config_dir else "/path/to/.kbagent" lines.append("# MCP tools require ConfigStore (they go through keboola-mcp-server)") @@ -198,6 +217,8 @@ def render( close_vars.append("ds_client") if "manage" in client_types: close_vars.append("manage_client") + if "metastore" in client_types: + close_vars.append("metastore_client") indent = " " if close_vars else "" lines.append("") @@ -222,6 +243,7 @@ def render( "storage": "client", "data_science": "ds_client", "manage": "manage_client", + "metastore": "metastore_client", "mcp": "mcp_service", } client_var = client_var_map.get(step.client.client_type, "client") diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py new file mode 100644 index 00000000..399cb032 --- /dev/null +++ b/src/keboola_agent_cli/metastore_client.py @@ -0,0 +1,168 @@ +"""Keboola Metastore API client for the semantic layer. + +Communicates with the Keboola Metastore at ``metastore.{stack-suffix}`` (derived +from the Storage API stack URL by replacing ``connection.`` with ``metastore.`` +in the hostname). Same ``X-StorageApi-Token`` credential as the Storage API. + +Inherits shared retry, timeout, and error handling from :class:`BaseHttpClient`. + +Verified contract (probed 2026-05-14 against e2e-1143): + +- ``GET /api/v1/repository/{type}`` → 200 with body ``{"data": [item, ...]}``. +- ``POST /api/v1/repository/{type}`` → 201 with body ``{"data": {type, id, + attributes, meta}}``. Envelope: ``{name, data, branch, schemaVersion, scope}``. +- ``DELETE /api/v1/repository/{type}/{id}`` → 204 empty body. Missing ID → 404 + with the standard error envelope. +- Duplicate ``name`` on POST → **500** with exception ``"Failed to create meta + object"``. We normalize this to :data:`ErrorCode.ALREADY_EXISTS`. +- Error envelope has top-level ``error``, ``code``, ``exception``, ``status``, + ``context.path``, and an ``errors[]`` list for 422 validation failures. +""" + +import logging +from typing import Any, Literal + +from . import __version__ +from .errors import ErrorCode, KeboolaApiError +from .http_base import BaseHttpClient + +logger = logging.getLogger(__name__) + + +SemanticType = Literal[ + "semantic-model", + "semantic-dataset", + "semantic-metric", + "semantic-relationship", + "semantic-constraint", + "semantic-glossary", +] + + +SEMANTIC_TYPES: tuple[str, ...] = ( + "semantic-model", + "semantic-dataset", + "semantic-metric", + "semantic-relationship", + "semantic-constraint", + "semantic-glossary", +) + + +# Envelope fields kept constant across every POST (per metastore contract). +_ENVELOPE_BRANCH = "main" +_ENVELOPE_SCHEMA_VERSION = "1.0.0" +_ENVELOPE_SCOPE = "project" + + +class MetastoreClient(BaseHttpClient): + """HTTP client for the Keboola Metastore (semantic layer repository). + + Provides minimal verb-level primitives that the + :class:`SemanticLayerService` composes into business operations. This + client deliberately stays thin: no business logic, no model resolution, + no in-memory caching. All such concerns live in the service layer. + """ + + def __init__(self, stack_url: str, token: str) -> None: + self._stack_url = stack_url.rstrip("/") + base_url = self._derive_service_url(self._stack_url, "metastore") + headers = { + "X-StorageApi-Token": token, + "User-Agent": f"keboola-agent-cli/{__version__}", + } + super().__init__(base_url=base_url, token=token, headers=headers) + + def __enter__(self) -> "MetastoreClient": + return self + + def __exit__(self, *args: Any) -> None: + self.close() + + # ------------------------------------------------------------------ + # Primitive verb methods + # ------------------------------------------------------------------ + + def list_items( + self, + item_type: SemanticType, + model_uuid: str | None = None, + ) -> list[dict[str, Any]]: + """List all items of ``item_type`` and (optionally) filter by model. + + Returns the **raw item shape**: ``{"type", "id", "attributes", + "meta"}``. Callers typically only want ``attributes`` plus ``id``; + we keep the full shape so audit fields stay reachable. + + Filtering: client-side on ``attributes.modelUUID == model_uuid``. The + server's ``?modelId=`` query param works in the probe but sl-builder + reports it as historically unreliable — defensive filter wins. + """ + response = self._do_request("GET", f"/api/v1/repository/{item_type}") + body = response.json() + items: list[dict[str, Any]] = body.get("data", []) if isinstance(body, dict) else [] + if model_uuid is None: + return items + return [i for i in items if (i.get("attributes") or {}).get("modelUUID") == model_uuid] + + def get_item(self, item_type: SemanticType, item_id: str) -> dict[str, Any]: + """Fetch a single item by its UUID. + + Raises :class:`KeboolaApiError` with ``error_code=NOT_FOUND`` on 404. + """ + response = self._do_request("GET", f"/api/v1/repository/{item_type}/{item_id}") + body = response.json() + return body.get("data", body) if isinstance(body, dict) else body + + def post_item( + self, + item_type: SemanticType, + name: str, + data: dict[str, Any], + ) -> dict[str, Any]: + """Create an item. Returns the server's stored representation. + + ``data`` is the inner ``attributes`` payload (including ``modelUUID`` + for non-model types). The outer envelope is added here. + + Normalizes the "duplicate name → 500" quirk into a clean + :data:`ErrorCode.ALREADY_EXISTS`. + """ + envelope = { + "name": name, + "data": data, + "branch": _ENVELOPE_BRANCH, + "schemaVersion": _ENVELOPE_SCHEMA_VERSION, + "scope": _ENVELOPE_SCOPE, + } + try: + response = self._do_request( + "POST", + f"/api/v1/repository/{item_type}", + json=envelope, + ) + except KeboolaApiError as exc: + # The metastore returns 500 (not 409/422) when the name already + # exists in the model. Surface a clean ALREADY_EXISTS instead of + # the raw 500 so command-layer error mapping can land it on the + # right exit code. + if exc.status_code == 500 and "Failed to create meta object" in exc.message: + raise KeboolaApiError( + message=( + f"{item_type} with name {name!r} already exists in the " + "target model. Use `edit` to update, or `remove` first." + ), + status_code=exc.status_code, + error_code=ErrorCode.ALREADY_EXISTS, + retryable=False, + ) from exc + raise + body = response.json() + return body.get("data", body) if isinstance(body, dict) else body + + def delete_item(self, item_type: SemanticType, item_id: str) -> None: + """Delete an item by its UUID. Returns silently on 204. + + Raises :class:`KeboolaApiError` with ``error_code=NOT_FOUND`` on 404. + """ + self._do_request("DELETE", f"/api/v1/repository/{item_type}/{item_id}") diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index f54a9fc0..b161a5e9 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -152,6 +152,54 @@ "storage.describe-batch": "write", # Encryption "encrypt.values": "write", + # Semantic layer (metastore) — new in 0.41.0 + "semantic-layer.show": "read", + "semantic-layer.validate": "read", + "semantic-layer.export": "read", + "semantic-layer.diff": "read", + # The `model` sub-app: the parent `semantic-layer` callback fires first + # with ctx.invoked_subcommand == "model" and synthesizes operation key + # ``semantic-layer.model``. We expose that key at the LEAST-privileged + # leaf risk (read) so the parent permits descent — the model sub-app's + # own callback then runs with the per-leaf keys below and enforces the + # actual gate. Without the parent-level key the fail-closed default + # treats ``semantic-layer.model`` as ``write`` and denies even + # ``model list`` under ``--deny-writes``. + "semantic-layer.model": "read", + "semantic-layer.model.list": "read", + "semantic-layer.model.create": "write", + "semantic-layer.model.delete": "destructive", + # The add/edit/remove sub-apps: same parent-callback pattern as `model` + # above -- the parent semantic-layer callback fires with the collapsed + # key, then the sub-app callback composes per-leaf operation keys via + # the standard ``check_cli_permission`` helper. Every leaf inside one + # sub-app shares the same risk class (add=write, edit=write, + # remove=destructive), so the per-leaf keys all carry that class. + "semantic-layer.add": "write", + "semantic-layer.add.metric": "write", + "semantic-layer.add.dataset": "write", + "semantic-layer.add.relationship": "write", + "semantic-layer.add.constraint": "write", + "semantic-layer.add.glossary": "write", + "semantic-layer.edit": "write", + "semantic-layer.edit.metric": "write", + "semantic-layer.edit.dataset": "write", + "semantic-layer.edit.constraint": "write", + "semantic-layer.edit.relationship": "write", + "semantic-layer.edit.glossary": "write", + "semantic-layer.import": "write", + "semantic-layer.promote": "write", + "semantic-layer.build": "write", + # `token --encrypt` calls EncryptService, same blast radius as + # `encrypt.values` which is `write`. Classified `write` for parity: + # users opting out via --deny-writes block both consistently. + "semantic-layer.token": "write", + "semantic-layer.remove": "destructive", + "semantic-layer.remove.metric": "destructive", + "semantic-layer.remove.dataset": "destructive", + "semantic-layer.remove.constraint": "destructive", + "semantic-layer.remove.relationship": "destructive", + "semantic-layer.remove.glossary": "destructive", # Raw HTTP client against `kbagent serve` (used by AI subprocesses). # Categorised by the underlying HTTP method: GET = read, mutating verbs # = write. The serve's own routes enforce their own permissions on top. diff --git a/src/keboola_agent_cli/server/__init__.py b/src/keboola_agent_cli/server/__init__.py index 62e01f82..f10ed25f 100644 --- a/src/keboola_agent_cli/server/__init__.py +++ b/src/keboola_agent_cli/server/__init__.py @@ -48,6 +48,7 @@ projects, schedules, search, + semantic_layer, sharing, storage, workspaces, @@ -221,6 +222,7 @@ async def _generic_handler(_request, exc: Exception): app.include_router(kai.router) app.include_router(encrypt.router) app.include_router(search.router) + app.include_router(semantic_layer.router) app.include_router(org.router) app.include_router(agents.router) @@ -382,6 +384,7 @@ def _is_ui_public(method: str, path: str) -> bool: "/kai", "/encrypt", "/search", + "/semantic-layer", "/org", "/agents", "/members", diff --git a/src/keboola_agent_cli/server/dependencies.py b/src/keboola_agent_cli/server/dependencies.py index ca7ad501..decc29f4 100644 --- a/src/keboola_agent_cli/server/dependencies.py +++ b/src/keboola_agent_cli/server/dependencies.py @@ -32,6 +32,7 @@ from ..services.repo_validate_service import RepoValidateService from ..services.schedule_service import ScheduleService from ..services.search_service import SearchService +from ..services.semantic_layer_service import SemanticLayerService from ..services.sharing_service import SharingService from ..services.storage_service import StorageService from ..services.sync_service import SyncService @@ -68,6 +69,7 @@ class ServiceRegistry: deep_lineage: DeepLineageService = field(init=False) sharing: SharingService = field(init=False) data_app: DataAppService = field(init=False) + semantic_layer: SemanticLayerService = field(init=False) repo_validate: RepoValidateService = field(init=False) mcp: McpService = field(init=False) kai: KaiService = field(init=False) @@ -95,6 +97,10 @@ def __post_init__(self) -> None: self.deep_lineage = DeepLineageService(config_store=cs) self.sharing = SharingService(config_store=cs) self.data_app = DataAppService(config_store=cs) + # SemanticLayerService takes both a storage client_factory (for + # validate --deep + add dataset --deep-fields + build) and an + # optional metastore_client_factory; the defaults work for both. + self.semantic_layer = SemanticLayerService(config_store=cs) self.repo_validate = RepoValidateService(config_store=cs) self.mcp = McpService(config_store=cs) self.kai = KaiService(config_store=cs) diff --git a/src/keboola_agent_cli/server/routers/semantic_layer.py b/src/keboola_agent_cli/server/routers/semantic_layer.py new file mode 100644 index 00000000..e4ef2b53 --- /dev/null +++ b/src/keboola_agent_cli/server/routers/semantic_layer.py @@ -0,0 +1,550 @@ +"""Semantic-layer endpoints — 1:1 mapping of the ``kbagent semantic-layer`` CLI. + +Mirrors the per-subcommand surface of +:class:`keboola_agent_cli.services.semantic_layer_service.SemanticLayerService`. +Pattern: Pydantic body models for routes that the CLI flags with multiple +options, query parameters for read endpoints. ``--yes`` is implicit on +every REST destructive call (the body / DELETE request IS the confirmation). +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Literal + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field, model_validator + +from ...errors import ErrorCode +from ..dependencies import ServiceRegistry, get_registry + +router = APIRouter(prefix="/semantic-layer", tags=["semantic-layer"]) + + +# Closed set of remove/edit/add kinds, mirroring the CLI surface. +ItemKind = Literal["metric", "dataset", "relationship", "constraint", "glossary"] + + +# ── Pydantic body models ─────────────────────────────────────────── + + +class ModelCreate(BaseModel): + project: str + name: str + description: str = "" + sql_dialect: str = "Snowflake" + + +class DiffRequest(BaseModel): + """Diff body — exactly one of project_a/file_a and one of project_b/file_b.""" + + project_a: str | None = None + project_b: str | None = None + model_a: str | None = None + model_b: str | None = None + file_a: dict[str, Any] | None = None + file_b: dict[str, Any] | None = None + + @model_validator(mode="before") + @classmethod + def _exactly_one_per_side(cls, data: Any) -> Any: + # Pydantic v2 calls this with the raw input dict in "before" mode. + if not isinstance(data, dict): + return data + left = (data.get("project_a") is not None, data.get("file_a") is not None) + right = (data.get("project_b") is not None, data.get("file_b") is not None) + if sum(left) != 1: + raise ValueError("Exactly one of project_a or file_a must be set.") + if sum(right) != 1: + raise ValueError("Exactly one of project_b or file_b must be set.") + return data + + +class AddMetric(BaseModel): + project: str + model: str | None = None + name: str + sql: str + dataset: str + description: str = "" + + +class AddDataset(BaseModel): + project: str + model: str | None = None + name: str + table_id: str + description: str = "" + grain: str = "" + primary_key: list[str] | None = None + deep_fields: bool = False + + +class AddRelationship(BaseModel): + project: str + model: str | None = None + name: str + from_: str = Field(alias="from") + to: str + on: str + type_: str = Field(alias="type", default="left") + + model_config = {"populate_by_name": True} + + +class AddConstraint(BaseModel): + project: str + model: str | None = None + name: str + constraint_type: str + rule: str + metrics: list[str] + severity: str = "warning" + + +class AddGlossary(BaseModel): + project: str + model: str | None = None + term: str + definition: str = "" + + +class EditMetric(BaseModel): + project: str + model: str | None = None + new_name: str | None = None + new_sql: str | None = None + new_dataset: str | None = None + new_description: str | None = None + + +class EditDataset(BaseModel): + project: str + model: str | None = None + new_name: str | None = None + new_description: str | None = None + new_grain: str | None = None + + +class EditRelationship(BaseModel): + project: str + model: str | None = None + new_name: str | None = None + new_from: str | None = None + new_to: str | None = None + new_on: str | None = None + new_type: str | None = None + + +class EditConstraint(BaseModel): + project: str + model: str | None = None + new_name: str | None = None + new_rule: str | None = None + new_constraint_type: str | None = None + new_severity: str | None = None + new_metrics: list[str] | None = None + + +class EditGlossary(BaseModel): + project: str + model: str | None = None + new_term: str | None = None + new_definition: str | None = None + + +class ImportRequest(BaseModel): + project: str + model: str | None = None + snapshot: dict[str, Any] + types: list[str] | None = None + dry_run: bool = False + overwrite: bool = False + + +class PromoteRequest(BaseModel): + from_project: str + to_project: str + from_model: str | None = None + to_model: str | None = None + types: list[str] | None = None + dry_run: bool = False + + +class BuildRequest(BaseModel): + project: str + model: str | None = None + tables: list[str] + name: str | None = None + dry_run: bool = False + + +class TokenEncryptRequest(BaseModel): + project: str + component_id: str + + +# ── Routes (14 declarations, in the order from the plan) ──────────── + + +@router.get("/models") +def list_models( + project: str, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """List every semantic-layer model in a project.""" + return registry.semantic_layer.list_models(project) + + +@router.post("/models") +def create_model( + body: ModelCreate, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """Create a semantic-layer model (POST /repository/semantic-model).""" + return registry.semantic_layer.create_model( + alias=body.project, + name=body.name, + description=body.description, + sql_dialect=body.sql_dialect, + ) + + +@router.delete("/models/{model}") +def delete_model( + model: str, + project: str, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Delete a semantic-layer model (--yes implicit on REST).""" + return registry.semantic_layer.delete_model(alias=project, model_name_or_uuid=model) + + +@router.get("/show") +def show( + project: str, + model: str | None = None, + type: str | None = None, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Show all entities (datasets, metrics, ...) for a model.""" + return registry.semantic_layer.show_model( + alias=project, model_name_or_uuid=model, type_filter=type + ) + + +@router.get("/validate") +def validate( + project: str, + model: str | None = None, + deep: bool = False, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Validate a model — basic checks (always) + Snowflake schema probes (``deep``).""" + return registry.semantic_layer.validate_model( + alias=project, model_name_or_uuid=model, deep=deep + ) + + +@router.get("/export") +def export( + project: str, + model: str | None = None, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Export a model + every child entity as an inline JSON snapshot. + + Unlike the CLI which writes to disk by default, the HTTP route returns + the snapshot in the response body (no file is written server-side). + """ + # output_path=None on the service writes to a default file path; we + # need to avoid that for REST. We bypass via export_model + pop the + # transient path. Simpler: route through the service and let the + # caller ignore the `path` field. Using a tmp-dir export keeps the + # service contract intact while avoiding pollution of the CWD. + import tempfile + + with tempfile.TemporaryDirectory(prefix="kbagent-sl-export-") as tmp: + out = Path(tmp) / "snapshot.json" + result = registry.semantic_layer.export_model( + alias=project, model_name_or_uuid=model, output_path=out + ) + # Strip the now-deleted tmp path from the wire response (the bytes + # are already captured in the dict). + result.pop("path", None) + return result + + +@router.post("/diff") +def diff(body: DiffRequest, registry: ServiceRegistry = Depends(get_registry)) -> dict[str, Any]: + """Diff two snapshots — project↔project, project↔file, file↔file. + + File-backed sides carry the snapshot inline in the body (``file_a`` / + ``file_b``). When a file side is set we serialize it to a temp file so + the existing service contract (``Path``) keeps working. + """ + import contextlib + import json as _json + import tempfile + + def _write_tmp(payload: dict[str, Any]) -> Path: + # `delete=False` is required because we close the handle before + # the service reads the file (the open handle would let us read + # but the service takes a Path, not a file object). Cleanup is + # done explicitly in the finally below. + fd, tmp_name = tempfile.mkstemp(suffix=".json", prefix="kbagent-sl-diff-") + try: + with open(fd, "w", encoding="utf-8") as fh: + fh.write(_json.dumps(payload)) + except Exception: + with contextlib.suppress(OSError): + Path(tmp_name).unlink() + raise + return Path(tmp_name) + + file_a_path: Path | None = None + file_b_path: Path | None = None + tmps: list[Path] = [] + try: + if body.file_a is not None: + file_a_path = _write_tmp(body.file_a) + tmps.append(file_a_path) + if body.file_b is not None: + file_b_path = _write_tmp(body.file_b) + tmps.append(file_b_path) + + return registry.semantic_layer.diff( + project_a=body.project_a, + project_b=body.project_b, + model_a=body.model_a, + model_b=body.model_b, + file_a=file_a_path, + file_b=file_b_path, + ) + finally: + for p in tmps: + with contextlib.suppress(OSError): + p.unlink() + + +@router.post("/items/{kind}") +def add_item( + kind: str, + body: dict[str, Any], + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Add an entity to a model. ``kind`` selects the entity type. + + Per-kind Pydantic body validation is done downstream by binding the + raw body to the right model before delegating to the service. + """ + svc = registry.semantic_layer + if kind == "metric": + m = AddMetric.model_validate(body) + return svc.add_metric( + alias=m.project, + model_name_or_uuid=m.model, + name=m.name, + sql=m.sql, + dataset=m.dataset, + description=m.description, + assume_yes=True, + is_tty=False, + ) + if kind == "dataset": + d = AddDataset.model_validate(body) + return svc.add_dataset( + alias=d.project, + model_name_or_uuid=d.model, + name=d.name, + table_id=d.table_id, + description=d.description, + grain=d.grain, + primary_key=d.primary_key, + deep_fields=d.deep_fields, + ) + if kind == "relationship": + r = AddRelationship.model_validate(body) + return svc.add_relationship( + alias=r.project, + model_name_or_uuid=r.model, + name=r.name, + from_=r.from_, + to=r.to, + on=r.on, + type_=r.type_, + ) + if kind == "constraint": + c = AddConstraint.model_validate(body) + return svc.add_constraint( + alias=c.project, + model_name_or_uuid=c.model, + name=c.name, + constraint_type=c.constraint_type, + rule=c.rule, + metrics=c.metrics, + severity=c.severity, + ) + if kind == "glossary": + g = AddGlossary.model_validate(body) + return svc.add_glossary( + alias=g.project, + model_name_or_uuid=g.model, + term=g.term, + definition=g.definition, + ) + raise HTTPException( + status_code=404, + detail=( + f"Unknown item kind {kind!r}. " + f"Must be one of: metric, dataset, relationship, constraint, glossary." + ), + ) + + +@router.put("/items/{kind}/{name}") +def edit_item( + kind: str, + name: str, + body: dict[str, Any], + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Edit an entity. ``name`` is the current identifier; new-* fields live in the body. + + For ``kind="glossary"``, ``name`` is the current ``term`` (not a stored + field called ``name``). + """ + svc = registry.semantic_layer + if kind == "metric": + m = EditMetric.model_validate(body) + return svc.edit_metric( + alias=m.project, + model_name_or_uuid=m.model, + current_name=name, + new_name=m.new_name, + new_sql=m.new_sql, + new_dataset=m.new_dataset, + new_description=m.new_description, + assume_yes=True, + is_tty=False, + ) + if kind == "dataset": + d = EditDataset.model_validate(body) + return svc.edit_dataset( + alias=d.project, + model_name_or_uuid=d.model, + current_name=name, + new_name=d.new_name, + new_description=d.new_description, + new_grain=d.new_grain, + ) + if kind == "relationship": + r = EditRelationship.model_validate(body) + return svc.edit_relationship( + alias=r.project, + model_name_or_uuid=r.model, + current_name=name, + new_name=r.new_name, + new_from=r.new_from, + new_to=r.new_to, + new_on=r.new_on, + new_type=r.new_type, + ) + if kind == "constraint": + c = EditConstraint.model_validate(body) + return svc.edit_constraint( + alias=c.project, + model_name_or_uuid=c.model, + current_name=name, + new_name=c.new_name, + new_rule=c.new_rule, + new_constraint_type=c.new_constraint_type, + new_severity=c.new_severity, + new_metrics=c.new_metrics, + ) + if kind == "glossary": + g = EditGlossary.model_validate(body) + return svc.edit_glossary( + alias=g.project, + model_name_or_uuid=g.model, + current_term=name, + new_term=g.new_term, + new_definition=g.new_definition, + ) + raise HTTPException( + status_code=404, + detail=( + f"Unknown item kind {kind!r}. " + f"Must be one of: metric, dataset, relationship, constraint, glossary." + ), + ) + + +@router.delete("/items/{kind}/{name}") +def remove_item( + kind: str, + name: str, + project: str, + model: str | None = None, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Remove a child entity (``--yes`` implicit on REST). + + For ``kind="glossary"``, ``name`` is the term to remove. + """ + return registry.semantic_layer.remove_item( + alias=project, model_name_or_uuid=model, kind=kind, name=name + ) + + +@router.post("/import") +def import_snapshot( + body: ImportRequest, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """Replay an inline snapshot into a project. Default: skip on conflict.""" + return registry.semantic_layer.import_snapshot_from_dict( + body.project, + snapshot=body.snapshot, + model_name_or_uuid=body.model, + types=body.types, + dry_run=body.dry_run, + overwrite=body.overwrite, + ) + + +@router.post("/promote") +def promote( + body: PromoteRequest, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """Promote a model from one project to another (additive + overwrite).""" + return registry.semantic_layer.promote_model( + from_project=body.from_project, + to_project=body.to_project, + from_model=body.from_model, + to_model=body.to_model, + types=body.types, + dry_run=body.dry_run, + ) + + +@router.post("/build") +def build(body: BuildRequest, registry: ServiceRegistry = Depends(get_registry)) -> dict[str, Any]: + """Heuristic greenfield builder — synthesize a model from a list of tables.""" + return registry.semantic_layer.build_model( + alias=body.project, + table_ids=body.tables, + model_name=body.name, + model_name_or_uuid=body.model, + dry_run=body.dry_run, + ) + + +@router.post("/token/encrypt") +def token_encrypt( + body: TokenEncryptRequest, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """Encrypt the project's storage token for transformation ``user_properties``.""" + return registry.semantic_layer.encrypt_token(alias=body.project, component_id=body.component_id) + + +# Re-export the closed set of kinds for tests / docs. +__all__ = ["ErrorCode", "ItemKind", "router"] diff --git a/src/keboola_agent_cli/services/_semantic_layer_crud.py b/src/keboola_agent_cli/services/_semantic_layer_crud.py new file mode 100644 index 00000000..1d905c24 --- /dev/null +++ b/src/keboola_agent_cli/services/_semantic_layer_crud.py @@ -0,0 +1,360 @@ +"""CRUD helpers for :mod:`semantic_layer_service`. + +Split out so the orchestrator class stays under the CONTRIBUTING.md +services hard ceiling (1,500 LOC). Each helper here operates on an +externally-provided :class:`MetastoreClient`; the class methods in the +main module are thin orchestrators that resolve credentials + the model +UUID, then delegate. + +Helpers: + +- :func:`delete_then_post` -- safe DELETE+POST with rollback +- :func:`edit_metric_with_cascade` -- metric rename + constraint cascade +- :func:`scan_orphan_constraints` -- pre-deletion orphan scan for metric +- :data:`REMOVE_KINDS` -- accepted kinds for ``remove_item`` / + ``preview_remove`` +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any + +from ..errors import ErrorCode, KeboolaApiError + +if TYPE_CHECKING: + from ..metastore_client import MetastoreClient, SemanticType + +# Kinds accepted by `remove_item` / `preview_remove`. Relationship and +# glossary were added in iter-4 (NB-5) -- neither is referenced by +# other entities, so removal cannot orphan anything. +REMOVE_KINDS: tuple[str, ...] = ( + "metric", + "dataset", + "constraint", + "relationship", + "glossary", +) + + +def code_metric(name: str) -> str: + """Derive the CODE_METRIC token from a metric name. + + Used in the rename-cascade prompt so the operator can audit + downstream SQL joins that key on the literal CODE_METRIC value. + """ + return re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_") + + +def delete_then_post( + client: MetastoreClient, + item_type: SemanticType, + *, + old_id: str, + original_attrs: dict[str, Any], + new_name: str, + new_attrs: dict[str, Any], +) -> tuple[dict[str, Any], dict[str, Any] | None]: + """Run a safe DELETE+POST, rolling back to ``original_attrs`` on POST failure. + + Returns ``(new_item, rollback)`` where ``rollback`` is None on + success. On POST failure we re-POST the original payload; the + rollback dict records whether that re-POST succeeded. The original + exception is re-raised wrapped in a KeboolaApiError with the + rollback context. + """ + client.delete_item(item_type, old_id) + try: + new_item = client.post_item(item_type, name=new_name, data=new_attrs) + except KeboolaApiError as exc: + rollback_status: dict[str, Any] = { + "attempted": True, + "original_name": original_attrs.get("name") or original_attrs.get("term", ""), + "error": exc.message, + } + try: + old_name = original_attrs.get("name") or original_attrs.get("term", "") + restored = client.post_item(item_type, name=old_name, data=original_attrs) + rollback_status["status"] = "succeeded" + rollback_status["restored_id"] = restored.get("id", "") + except KeboolaApiError as rollback_exc: + rollback_status["status"] = "failed" + rollback_status["rollback_error"] = rollback_exc.message + raise KeboolaApiError( + message=( + f"edit failed (POST after DELETE raised): {exc.message}. " + f"Rollback: {rollback_status['status']}." + ), + error_code=exc.error_code, + status_code=exc.status_code, + details={"rollback": rollback_status}, + ) from exc + return new_item, None + + +def edit_metric_with_cascade( + client: MetastoreClient, + *, + model_uuid: str, + current_name: str, + new_name: str | None, + new_sql: str | None, + new_dataset: str | None, + new_description: str | None, + assume_yes: bool, + is_tty: bool, + confirm_cb: Any, +) -> dict[str, Any]: + """Body of :meth:`SemanticLayerService.edit_metric`. + + Resolves the target metric, computes the constraint cascade list, + enforces TTY/--yes guards, then DELETE+POSTs the metric and + DELETE+POSTs each cascaded constraint individually so per-item + failures don't poison the rest. + """ + metrics = client.list_items("semantic-metric", model_uuid) + target = next( + (m for m in metrics if (m.get("attributes") or {}).get("name") == current_name), + None, + ) + if target is None: + raise KeboolaApiError( + message=f"Metric '{current_name}' not found in this model.", + error_code=ErrorCode.NOT_FOUND, + ) + original_attrs = dict(target.get("attributes") or {}) + old_id = target["id"] + + effective_new_name = new_name if new_name is not None else current_name + cascade_required: list[dict[str, Any]] = [] + if new_name is not None and new_name != current_name: + constraints = client.list_items("semantic-constraint", model_uuid) + for c in constraints: + cattrs = c.get("attributes") or {} + if current_name in (cattrs.get("metrics") or []): + cascade_required.append(c) + + if cascade_required and not assume_yes: + names = ", ".join( + (c.get("attributes") or {}).get("name", "?") for c in cascade_required + ) + old_code = code_metric(current_name) + new_code = code_metric(effective_new_name) + msg = ( + f"Renaming metric {current_name!r} to {effective_new_name!r} will " + f"cascade to {len(cascade_required)} constraint(s): {names}. " + f"WARNING: downstream SQL joining on CODE_METRIC must be " + f"updated manually ({old_code!r} -> {new_code!r}). Proceed?" + ) + if not is_tty: + raise KeboolaApiError( + message=msg + " Pass --yes to bypass.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + if confirm_cb is None or not confirm_cb(msg): + raise KeboolaApiError( + message="Aborted by user.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + + new_attrs = dict(original_attrs) + if new_name is not None: + new_attrs["name"] = new_name + if new_sql is not None: + new_attrs["sql"] = new_sql + if new_dataset is not None: + new_attrs["dataset"] = new_dataset + if new_description is not None: + new_attrs["description"] = new_description + + new_item, rollback = delete_then_post( + client, + "semantic-metric", + old_id=old_id, + original_attrs=original_attrs, + new_name=effective_new_name, + new_attrs=new_attrs, + ) + + # Cascade constraints individually (each is independent -- + # report per-constraint success/failure). + cascaded: list[dict[str, Any]] = [] + for c in cascade_required: + cattrs = dict(c.get("attributes") or {}) + cmetrics = list(cattrs.get("metrics") or []) + cmetrics = [effective_new_name if x == current_name else x for x in cmetrics] + cattrs["metrics"] = cmetrics + cname = cattrs.get("name", "") + try: + cascaded_item, _ = delete_then_post( + client, + "semantic-constraint", + old_id=c["id"], + original_attrs=c.get("attributes") or {}, + new_name=cname, + new_attrs=cattrs, + ) + cascaded.append({"constraint": cname, "status": "updated", "id": cascaded_item["id"]}) + except KeboolaApiError as exc: + cascaded.append( + { + "constraint": cname, + "status": "failed", + "error": exc.message, + "rollback": (exc.details or {}).get("rollback"), + } + ) + + return { + "updated": new_item, + "cascaded_constraints": cascaded, + "rollback": rollback, + } + + +def validate_constraint_attrs( + *, + name_re: re.Pattern[str], + constraint_types: tuple[str, ...], + severities: tuple[str, ...], + name: str | None = None, + constraint_type: str | None = None, + severity: str | None = None, +) -> None: + """Validate constraint attributes locally before POST/edit. + + Each arg is optional -- the helper only checks the non-None ones. + Used by ``add_constraint`` and the entry path of ``edit_constraint``. + """ + if name is not None and not name_re.match(name): + raise KeboolaApiError( + message=( + f"Constraint name {name!r} does not match the server-enforced " + f"regex {name_re.pattern} " + "(lowercase ASCII letter, then letters/digits/underscores). " + "Example: 'npm_critical'." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + if constraint_type is not None and constraint_type not in constraint_types: + raise KeboolaApiError( + message=( + f"--constraint-type must be one of {list(constraint_types)}, " + f"got {constraint_type!r}." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + if severity is not None and severity not in severities: + raise KeboolaApiError( + message=(f"--severity must be one of {list(severities)}, got {severity!r}."), + error_code=ErrorCode.VALIDATION_ERROR, + ) + + +def find_target_for_remove( + client: MetastoreClient, + *, + kind: str, + model_uuid: str, + name: str, + type_alias: dict[str, Any], +) -> tuple[dict[str, Any], SemanticType, str]: + """Resolve the target item for ``preview_remove`` / ``remove_item``. + + Returns ``(target_item, type_slug, id_key)``. Raises NOT_FOUND if the + target is missing or VALIDATION_ERROR if ``kind`` is unknown. + """ + if kind not in REMOVE_KINDS: + raise KeboolaApiError( + message=f"remove kind must be one of {'|'.join(REMOVE_KINDS)}, got {kind!r}.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + type_slug = type_alias[kind] + id_key = "term" if kind == "glossary" else "name" + items = client.list_items(type_slug, model_uuid) + target = next( + (i for i in items if (i.get("attributes") or {}).get(id_key) == name), + None, + ) + if target is None: + raise KeboolaApiError( + message=f"{kind} '{name}' not found in this model.", + error_code=ErrorCode.NOT_FOUND, + ) + return target, type_slug, id_key + + +def edit_simple( + client: MetastoreClient, + item_type: SemanticType, + *, + items: list[dict[str, Any]], + id_key: str, + current_key: str, + overrides: dict[str, Any], + not_found_label: str, +) -> dict[str, Any]: + """Generic body for edit_dataset / edit_constraint / edit_relationship / + edit_glossary (no cascade). + + Args: + items: Pre-fetched list of items of ``item_type`` in the model. + id_key: Identity key (``name`` for most types, ``term`` for glossary). + current_key: Current identity value. + overrides: Mapping of attribute keys -> new values to apply + (only non-None entries should be present). The new effective + identity key is computed from the overrides. + not_found_label: Human-readable label for the NOT_FOUND error. + + Returns the same envelope shape as the legacy class methods: + ``{updated, cascaded_constraints: [], rollback}``. + """ + target = next( + (i for i in items if (i.get("attributes") or {}).get(id_key) == current_key), + None, + ) + if target is None: + raise KeboolaApiError( + message=f"{not_found_label} '{current_key}' not found in this model.", + error_code=ErrorCode.NOT_FOUND, + ) + original_attrs = dict(target.get("attributes") or {}) + new_attrs = dict(original_attrs) + for k, v in overrides.items(): + if v is not None: + new_attrs[k] = v + effective_new = new_attrs.get(id_key) or current_key + new_item, rollback = delete_then_post( + client, + item_type, + old_id=target["id"], + original_attrs=original_attrs, + new_name=effective_new, + new_attrs=new_attrs, + ) + return {"updated": new_item, "cascaded_constraints": [], "rollback": rollback} + + +def scan_orphan_constraints( + client: MetastoreClient, + *, + model_uuid: str, + metric_name: str, +) -> list[dict[str, Any]]: + """Find every constraint whose ``metrics[]`` references ``metric_name``. + + Returns a list of ``{name, metrics}`` dicts -- one per + soon-to-be-orphaned constraint. Empty when the metric is unreferenced. + """ + constraints = client.list_items("semantic-constraint", model_uuid) + orphans: list[dict[str, Any]] = [] + for c in constraints: + cattrs = c.get("attributes") or {} + if metric_name in (cattrs.get("metrics") or []): + orphans.append( + { + "name": cattrs.get("name", ""), + "metrics": list(cattrs.get("metrics") or []), + } + ) + return orphans diff --git a/src/keboola_agent_cli/services/_semantic_layer_internals.py b/src/keboola_agent_cli/services/_semantic_layer_internals.py new file mode 100644 index 00000000..f28cc1f2 --- /dev/null +++ b/src/keboola_agent_cli/services/_semantic_layer_internals.py @@ -0,0 +1,839 @@ +"""Internal helpers for :mod:`semantic_layer_service`. + +Split out so the orchestrator class stays under the +CONTRIBUTING.md services hard ceiling (1,500 LOC). Each helper here is +pure-functional or operates on an externally-provided +:class:`MetastoreClient` -- there is no in-module state. + +Helpers grouped by feature: + +- Validation (basic + deep) :func:`validate_basic`, + :func:`validate_deep` +- Diff :func:`collect_side_from_file`, + :func:`diff_one_type`, + :func:`compare_attrs` +- Build (heuristic) :func:`heuristic_generate_model`, + :func:`fetch_table_schemas` +- Import / promote :func:`run_import_loop`, + :func:`run_promote_loop` +- Export I/O :func:`write_snapshot_to_file`, + :func:`default_export_path`, + :func:`build_export_snapshot` +- Constants reused across modules :data:`DIFF_IGNORED_KEYS`, + :data:`PUSH_ORDER` +""" + +from __future__ import annotations + +import json +import os +import re +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from ..errors import ConfigError, ErrorCode, KeboolaApiError + +if TYPE_CHECKING: + from ..metastore_client import SemanticType + from .storage_service import StorageService + +# Re-exported from the main module to avoid a circular import; pulled in +# locally for type hints / runtime constants. + +# Column reference regex (Snowflake-style ``"SCHEMA"."COLUMN"``). +_COLUMN_REF_RE = re.compile(r'"[^"]+"\."([^"]+)"') + +# SUM(... PCT ...) heuristic. +_SUM_ON_PCT_RE = re.compile(r"\bSUM\s*\(\s*[^)]*\b(PCT|PERCENT|RATE)\b", re.IGNORECASE) + +# AGG-on-STRING detection for --deep validate. +_AGG_ON_STRING_FUNCS = ("SUM", "AVG") + +# Severity-band suffixes recognised by downstream pipelines. +_SEVERITY_SUFFIXES = ("_critical", "_warning", "_healthy", "_review") + +# Keys ignored when deep-equality-comparing two items (the modelUUID is +# rewritten when promoting across projects, and the server-side +# timestamps drift every fetch). +DIFF_IGNORED_KEYS: tuple[str, ...] = ( + "modelUUID", + "createdAt", + "lastUpdated", + "revision", +) + +# Push order for both `import` and `promote` -- datasets before metrics +# (metric.dataset references a tableId), constraints last (their +# `metrics[]` list references metric names). +PUSH_ORDER: tuple[tuple[str, SemanticType], ...] = ( + ("datasets", "semantic-dataset"), + ("metrics", "semantic-metric"), + ("relationships", "semantic-relationship"), + ("glossary", "semantic-glossary"), + ("constraints", "semantic-constraint"), +) + + +@dataclass(frozen=True) +class WorkerResult: + """Outcome of a per-table parallel fetch worker. + + Shared between the deep-validation pass and the build heuristic. + Exactly one of ``detail`` / ``error`` is non-None. + """ + + table_id: str + detail: dict[str, Any] | None + error: str | None + + +# ── validate (basic + deep) ───────────────────────────────────────── + + +def validate_basic( + *, + datasets: list[dict[str, Any]], + metrics: list[dict[str, Any]], + relationships: list[dict[str, Any]], + constraints: list[dict[str, Any]], + glossary: list[dict[str, Any]], + errors: list[dict[str, str]], + warnings: list[dict[str, str]], +) -> None: + """Pure in-memory validation (no API calls). + + Mutates ``errors`` and ``warnings`` in place. See the orchestrator's + docstring for the full list of checks. + """ + # DUPLICATES per type (name) + for label, items, key in ( + ("dataset", datasets, "name"), + ("metric", metrics, "name"), + ("relationship", relationships, "name"), + ("constraint", constraints, "name"), + ("glossary", glossary, "term"), + ): + seen: set[str] = set() + dups: set[str] = set() + for it in items: + name = it.get(key, "") + if not name: + continue + if name in seen: + dups.add(name) + seen.add(name) + for dup in sorted(dups): + errors.append( + {"type": "DUPLICATE", "item": f"{label}:{dup}", "detail": f"duplicate {key}"} + ) + + dataset_tids = {d.get("tableId", "") for d in datasets if d.get("tableId")} + dataset_names = {d.get("name", "") for d in datasets if d.get("name")} + + # DANGLING REL: from/to tableId must be in datasets + for rel in relationships: + for endpoint in ("from", "to"): + tid = rel.get(endpoint, "") + if tid and tid not in dataset_tids and tid not in dataset_names: + errors.append( + { + "type": "DANGLING_RELATIONSHIP", + "item": rel.get("name", "?"), + "detail": f"{endpoint}={tid!r} not in model datasets", + } + ) + + # DANGLING METRIC: metric.dataset is a tableId + for m in metrics: + ds = m.get("dataset", "") + if ds and ds not in dataset_tids and ds not in dataset_names: + errors.append( + { + "type": "DANGLING_METRIC", + "item": m.get("name", "?"), + "detail": f"dataset={ds!r} not in model datasets", + } + ) + + # SUM ON PCT + for m in metrics: + sql = m.get("sql", "") or "" + if _SUM_ON_PCT_RE.search(sql): + warnings.append( + { + "type": "SUM_ON_PCT", + "item": m.get("name", "?"), + "detail": "SUM() over a percentage/rate column is usually incorrect", + } + ) + + # CONSTRAINT ORPHAN + SEVERITY SUFFIX + metric_names = {m.get("name", "") for m in metrics if m.get("name")} + for c in constraints: + cname = c.get("name", "") + for mname in c.get("metrics", []) or []: + if mname not in metric_names: + errors.append( + { + "type": "CONSTRAINT_ORPHAN", + "item": cname or "?", + "detail": f"references missing metric {mname!r}", + } + ) + if cname and not any(cname.endswith(suf) for suf in _SEVERITY_SUFFIXES): + warnings.append( + { + "type": "SEVERITY_SUFFIX", + "item": cname, + "detail": ( + "constraint name should end with one of " + "_critical/_warning/_healthy/_review for downstream parsing" + ), + } + ) + + +def validate_deep( + *, + alias: str, + storage: StorageService, + datasets: list[dict[str, Any]], + metrics: list[dict[str, Any]], + errors: list[dict[str, str]], + warnings: list[dict[str, str]], +) -> None: + """Add deep checks that require a Snowflake schema fetch per dataset. + + Mutates ``errors`` and ``warnings`` in place. + """ + + def _worker(ds: dict[str, Any]) -> WorkerResult: + tid = ds.get("tableId", "") + if not tid: + return WorkerResult(table_id=tid, detail=None, error="missing tableId") + try: + detail = storage.get_table_detail(alias, tid) + return WorkerResult(table_id=tid, detail=detail, error=None) + except (KeboolaApiError, ConfigError) as exc: + return WorkerResult(table_id=tid, detail=None, error=str(exc)) + + details_by_tid: dict[str, dict[str, Any]] = {} + with ThreadPoolExecutor(max_workers=8) as pool: + for outcome in pool.map(_worker, datasets): + if outcome.error is not None: + warnings.append( + { + "type": "DEEP_FETCH_FAILED", + "item": outcome.table_id or "?", + "detail": outcome.error, + } + ) + continue + if outcome.detail is not None: + details_by_tid[outcome.table_id] = outcome.detail + + # PHANTOM FIELD (declared field not in actual columns) + for ds in datasets: + tid = ds.get("tableId", "") + detail = details_by_tid.get(tid) + if detail is None: + continue + actual_columns = set(detail.get("columns", [])) + declared = ds.get("fields", []) or [] + for field in declared: + fname = field.get("name", "") if isinstance(field, dict) else str(field) + if fname and fname not in actual_columns: + errors.append( + { + "type": "PHANTOM_FIELD", + "item": f"{ds.get('name', '?')}.{fname}", + "detail": f"field {fname!r} not in storage table columns", + } + ) + + # METRIC PHANTOM + AGG ON STRING -- build a quick lookup: tableId -> {col_name: basetype} + col_types_by_tid: dict[str, dict[str, str]] = {} + for tid, detail in details_by_tid.items(): + cols: dict[str, str] = {} + for col_info in detail.get("column_details", []) or []: + cols[col_info.get("name", "")] = (col_info.get("type", "") or "").upper() + col_types_by_tid[tid] = cols + + for m in metrics: + sql = m.get("sql", "") or "" + mdataset = m.get("dataset", "") + col_types = col_types_by_tid.get(mdataset, {}) + refs = set(_COLUMN_REF_RE.findall(sql)) + if col_types: + actual = set(col_types.keys()) + for ref in refs: + if ref not in actual: + errors.append( + { + "type": "METRIC_PHANTOM", + "item": m.get("name", "?"), + "detail": ( + f"column {ref!r} referenced in sql not in dataset {mdataset!r}" + ), + } + ) + # AGG ON STRING + for func in _AGG_ON_STRING_FUNCS: + pattern = re.compile( + rf'\b{func}\s*\(\s*[^)]*?"([^"]+)"\."([^"]+)"', + re.IGNORECASE, + ) + for match in pattern.finditer(sql): + col = match.group(2) + col_type = col_types.get(col, "").upper() + if col_type == "STRING": + errors.append( + { + "type": "AGG_ON_STRING", + "item": m.get("name", "?"), + "detail": f"{func}() on STRING column {col!r}", + } + ) + + +# ── diff ──────────────────────────────────────────────────────────── + + +def compare_attrs(a: dict[str, Any], b: dict[str, Any]) -> list[str]: + """Return the sorted list of attribute keys that differ between two items.""" + ignored = set(DIFF_IGNORED_KEYS) + keys = (set(a) | set(b)) - ignored - {"id"} + diff_keys: list[str] = [] + for key in sorted(keys): + if a.get(key) != b.get(key): + diff_keys.append(key) + return diff_keys + + +def diff_one_type( + left: list[dict[str, Any]], + right: list[dict[str, Any]], + *, + id_key: str, +) -> dict[str, Any]: + """Compute added/removed/changed for one entity type.""" + left_by_key = {item.get(id_key, ""): item for item in left if item.get(id_key)} + right_by_key = {item.get(id_key, ""): item for item in right if item.get(id_key)} + + added = sorted(k for k in right_by_key if k not in left_by_key) + removed = sorted(k for k in left_by_key if k not in right_by_key) + changed: list[dict[str, Any]] = [] + for key in sorted(set(left_by_key) & set(right_by_key)): + diff_keys = compare_attrs(left_by_key[key], right_by_key[key]) + if diff_keys: + changed.append({id_key: key, "diff_keys": diff_keys}) + return {"added": added, "removed": removed, "changed": changed} + + +def collect_side_from_file(file: Path) -> dict[str, Any]: + """Resolve one diff side from a snapshot file. + + Returns the shape consumed by the diff orchestrator: + ``{"ref": {"source": "file", "ref": <path>, "model": {...}}, + "data": {<plural>: [attrs_dict, ...]}}``. + """ + try: + payload = json.loads(file.read_text(encoding="utf-8")) + except OSError as exc: + raise KeboolaApiError( + message=f"Cannot read --file {file}: {exc}", + error_code=ErrorCode.READ_ERROR, + ) from exc + except json.JSONDecodeError as exc: + raise KeboolaApiError( + message=f"File {file} is not valid JSON: {exc}", + error_code=ErrorCode.INVALID_FORMAT, + ) from exc + + def _attrs(items: list[Any]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for item in items or []: + if isinstance(item, dict) and "attributes" in item: + merged = dict(item.get("attributes") or {}) + merged["id"] = item.get("id", "") + out.append(merged) + elif isinstance(item, dict): + out.append(item) + return out + + return { + "ref": { + "source": "file", + "ref": str(file), + "model": payload.get("model", {}), + }, + "data": { + "datasets": _attrs(payload.get("datasets", [])), + "metrics": _attrs(payload.get("metrics", [])), + "relationships": _attrs(payload.get("relationships", [])), + "constraints": _attrs(payload.get("constraints", [])), + "glossary": _attrs(payload.get("glossary", [])), + }, + } + + +# ── build (heuristic generator) ───────────────────────────────────── + + +def run_import_loop( + client: Any, + *, + snapshot: dict[str, Any], + target_model_uuid: str, + existing_by_type: dict[Any, list[dict[str, Any]]], + type_filter: set[str] | None, + dry_run: bool, + overwrite: bool, +) -> dict[str, Any]: + """Replay a snapshot into the target model. + + Returns the per-type stats dict matching the orchestrator's contract. + Push order is :data:`PUSH_ORDER`. Errors are accumulated per item; + one failure does not abort the rest. + """ + imported: dict[str, Any] = {} + for plural, type_slug in PUSH_ORDER: + if type_filter is not None and plural not in type_filter: + continue + source_items = snapshot.get(plural, []) or [] + id_key = "term" if plural == "glossary" else "name" + existing_by_name: dict[str, dict[str, Any]] = { + (item.get("attributes") or {}).get(id_key, ""): item + for item in existing_by_type.get(type_slug, []) + if (item.get("attributes") or {}).get(id_key) + } + + per_type: dict[str, Any] = { + "created": 0, + "skipped": 0, + "overwritten": 0, + "failed": [], + } + for raw_item in source_items: + if not isinstance(raw_item, dict): + continue + attrs = dict(raw_item.get("attributes") or {}) + attrs["modelUUID"] = target_model_uuid # rewrite + key = attrs.get(id_key, "") + if not key: + per_type["failed"].append({"name": "?", "reason": f"missing {id_key} on item"}) + continue + + if key in existing_by_name: + if not overwrite: + per_type["skipped"] += 1 + continue + if dry_run: + per_type["overwritten"] += 1 + continue + try: + client.delete_item(type_slug, existing_by_name[key]["id"]) + client.post_item(type_slug, name=key, data=attrs) + per_type["overwritten"] += 1 + except KeboolaApiError as exc: + per_type["failed"].append({"name": key, "reason": exc.message}) + continue + + if dry_run: + per_type["created"] += 1 + continue + try: + client.post_item(type_slug, name=key, data=attrs) + per_type["created"] += 1 + except KeboolaApiError as exc: + per_type["failed"].append({"name": key, "reason": exc.message}) + + imported[plural] = per_type + return imported + + +def run_promote_loop( + target_client: Any, + *, + src_children: dict[Any, list[dict[str, Any]]], + tgt_children: dict[Any, list[dict[str, Any]]], + target_model_uuid: str, + type_filter: set[str] | None, + dry_run: bool, +) -> dict[str, Any]: + """Run the additive + overwrite promote loop. + + NEW: POST to target. CHANGED: DELETE+POST. IDENTICAL: skip. + Items only in target are never deleted (additive only). + Returns ``{plural: {new, overwritten, identical, failed, changes}}``. + """ + result: dict[str, Any] = {} + for plural, type_slug in PUSH_ORDER: + if type_filter is not None and plural not in type_filter: + continue + id_key = "term" if plural == "glossary" else "name" + src_items = src_children.get(type_slug, []) or [] + tgt_by_key: dict[str, dict[str, Any]] = { + (it.get("attributes") or {}).get(id_key, ""): it + for it in tgt_children.get(type_slug, []) + if (it.get("attributes") or {}).get(id_key) + } + + stats: dict[str, Any] = { + "new": 0, + "overwritten": 0, + "identical": 0, + "failed": [], + "changes": [], + } + + for src in src_items: + src_attrs = dict(src.get("attributes") or {}) + src_attrs["modelUUID"] = target_model_uuid # rewrite + key = src_attrs.get(id_key, "") + if not key: + stats["failed"].append({"name": "?", "reason": f"missing {id_key} on source item"}) + continue + + if key in tgt_by_key: + diff_keys = compare_attrs(src_attrs, dict(tgt_by_key[key].get("attributes") or {})) + if not diff_keys: + stats["identical"] += 1 + continue + if dry_run: + stats["overwritten"] += 1 + stats["changes"].append({id_key: key, "diff_keys": diff_keys}) + continue + try: + target_client.delete_item(type_slug, tgt_by_key[key]["id"]) + target_client.post_item(type_slug, name=key, data=src_attrs) + stats["overwritten"] += 1 + stats["changes"].append({id_key: key, "diff_keys": diff_keys}) + except KeboolaApiError as exc: + stats["failed"].append({"name": key, "reason": exc.message}) + continue + + if dry_run: + stats["new"] += 1 + continue + try: + target_client.post_item(type_slug, name=key, data=src_attrs) + stats["new"] += 1 + except KeboolaApiError as exc: + stats["failed"].append({"name": key, "reason": exc.message}) + + result[plural] = stats + return result + + +def push_built_model( + client: Any, + *, + generated: dict[str, Any], + model_name_or_uuid: str | None, + resolve_model_fn: Callable[[Any, str], tuple[str, dict[str, Any]]], +) -> tuple[dict[str, int], str, dict[str, Any] | None]: + """Push a heuristic-generated model + children to the metastore. + + Returns ``(counts, model_uuid, model_item)`` where ``model_item`` + is the freshly-created model record (``None`` when updating an + existing model). ``counts`` maps each plural to the number of + successfully POSTed children. + """ + if model_name_or_uuid is None: + # Create the model. + model_attrs: dict[str, Any] = { + "name": generated["name"], + "sql_dialect": generated.get("sql_dialect", "Snowflake"), + } + if generated.get("description"): + model_attrs["description"] = generated["description"] + model_item = client.post_item("semantic-model", name=generated["name"], data=model_attrs) + model_uuid = model_item["id"] + else: + model_uuid, _ = resolve_model_fn(client, model_name_or_uuid) + model_item = None + + counts: dict[str, int] = {plural: 0 for plural, _ in PUSH_ORDER} + for plural, type_slug in PUSH_ORDER: + for item in generated.get(plural, []) or []: + attrs = dict(item) + attrs["modelUUID"] = model_uuid + name = attrs.get("name") or attrs.get("term", "") + if not name: + continue + client.post_item(type_slug, name=name, data=attrs) + counts[plural] += 1 + return counts, model_uuid, model_item + + +def synthesize_role_classified_fields( + storage: StorageService, + alias: str, + table_id: str, + classify_role: Callable[[str, str], str], +) -> list[dict[str, Any]]: + """Fetch storage schema for ``table_id`` and synthesise ``fields[]`` with role heuristics. + + Used by ``add_dataset --deep-fields``. Returns the list of + ``{name, type, role}`` dicts (one per column); empty when the + storage table has no columns. + """ + detail = storage.get_table_detail(alias, table_id) + fields: list[dict[str, Any]] = [] + for col in detail.get("column_details", []) or []: + cname = col.get("name", "") + basetype = col.get("type", "") or col.get("native_type", "") + fields.append( + { + "name": cname, + "type": basetype, + "role": classify_role(cname, basetype), + } + ) + return fields + + +def fetch_table_schemas( + storage: StorageService, + alias: str, + table_ids: list[str], +) -> tuple[dict[str, dict[str, Any]], list[dict[str, str]]]: + """Fetch storage schemas for a list of tableIds in parallel. + + Returns ``(schemas_by_tid, fetch_errors)`` where ``fetch_errors`` is + a list of ``{"table_id", "error"}`` dicts -- one per missing / + failed table. Errors are returned (not raised) so the build pass + can decide whether to abort. + """ + + def _worker(tid: str) -> WorkerResult: + try: + detail = storage.get_table_detail(alias, tid) + return WorkerResult(table_id=tid, detail=detail, error=None) + except (KeboolaApiError, ConfigError) as exc: + return WorkerResult(table_id=tid, detail=None, error=str(exc)) + + schemas_by_tid: dict[str, dict[str, Any]] = {} + fetch_errors: list[dict[str, str]] = [] + with ThreadPoolExecutor(max_workers=8) as pool: + for outcome in pool.map(_worker, table_ids): + if outcome.error is not None: + fetch_errors.append({"table_id": outcome.table_id, "error": outcome.error}) + continue + if outcome.detail is not None: + schemas_by_tid[outcome.table_id] = outcome.detail + return schemas_by_tid, fetch_errors + + +def resolve_model_uuid( + client: Any, + model_name_or_uuid: str | None, +) -> tuple[str, dict[str, Any]]: + """Resolve a model selector to ``(uuid, attributes_dict)``. + + See :meth:`SemanticLayerService._resolve_model` for the contract. + Lives here so the orchestrator class stays under the file-size + budget; the class method is a one-line delegate. + + Raises ``ConfigError`` when the selector is ambiguous or missing. + """ + models = client.list_items("semantic-model") + if not models: + raise ConfigError( + "Project has no semantic-layer models. Use " + "'kbagent semantic-layer model create' to create one." + ) + + if model_name_or_uuid is None: + if len(models) == 1: + return models[0]["id"], dict(models[0].get("attributes") or {}) + names = ", ".join(sorted((m.get("attributes") or {}).get("name", "?") for m in models)) + raise ConfigError(f"Project has {len(models)} models — specify --model. Available: {names}") + + # Try UUID match first (exact ID match) + for m in models: + if m.get("id") == model_name_or_uuid: + return m["id"], dict(m.get("attributes") or {}) + + # Then name match + name_matches = [ + m for m in models if (m.get("attributes") or {}).get("name") == model_name_or_uuid + ] + if len(name_matches) == 1: + return name_matches[0]["id"], dict(name_matches[0].get("attributes") or {}) + if len(name_matches) > 1: + raise ConfigError( + f"Multiple models found with name '{model_name_or_uuid}'. Specify the UUID instead." + ) + + names = ", ".join(sorted((m.get("attributes") or {}).get("name", "?") for m in models)) + raise ConfigError(f"Model '{model_name_or_uuid}' not found. Available: {names}") + + +def unpack_attrs_with_id( + items: list[dict[str, Any]], *, id_field: str = "_id" +) -> list[dict[str, Any]]: + """Flatten ``{type, id, attributes}`` items to attributes dicts + injected id field. + + Used by validate_model and other callers that need attribute-only + lists with the server id preserved (under ``_id`` by default so it + doesn't collide with the attribute namespace). + """ + return [dict(i.get("attributes") or {}, **{id_field: i.get("id", "")}) for i in items] + + +def unpack_children_by_plural( + raw_by_type: dict[Any, list[dict[str, Any]]], + *, + id_field: str = "id", +) -> dict[str, list[dict[str, Any]]]: + """Convert ``raw_by_type[semantic-X]`` lists to attribute dicts keyed by plural. + + Returns a dict with keys ``datasets``, ``metrics``, ``relationships``, + ``constraints``, ``glossary`` -- ready to splat into a result envelope. + """ + return { + "datasets": unpack_attrs_with_id( + raw_by_type.get("semantic-dataset", []), id_field=id_field + ), + "metrics": unpack_attrs_with_id(raw_by_type.get("semantic-metric", []), id_field=id_field), + "relationships": unpack_attrs_with_id( + raw_by_type.get("semantic-relationship", []), id_field=id_field + ), + "constraints": unpack_attrs_with_id( + raw_by_type.get("semantic-constraint", []), id_field=id_field + ), + "glossary": unpack_attrs_with_id( + raw_by_type.get("semantic-glossary", []), id_field=id_field + ), + } + + +def default_export_path(model_name: str) -> Path: + """Default ``./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json`` path.""" + ts = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S") + safe_name = re.sub(r"[^A-Za-z0-9._-]+", "_", str(model_name)) or "model" + return Path.cwd() / f"sl_export_{safe_name}_{ts}.json" + + +def build_export_snapshot( + *, + alias: str, + model_uuid: str, + model_attrs: dict[str, Any], + raw_by_type: dict[Any, list[dict[str, Any]]], +) -> dict[str, Any]: + """Assemble the self-describing export envelope.""" + exported_at = datetime.now(tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + return { + "exported_at": exported_at, + "project": alias, + "model": dict(model_attrs, id=model_uuid), + "datasets": raw_by_type.get("semantic-dataset", []), + "metrics": raw_by_type.get("semantic-metric", []), + "relationships": raw_by_type.get("semantic-relationship", []), + "constraints": raw_by_type.get("semantic-constraint", []), + "glossary": raw_by_type.get("semantic-glossary", []), + } + + +def write_snapshot_to_file(snapshot: dict[str, Any], output_path: Path) -> None: + """Atomic-write a JSON snapshot with 0o644 perms and O_NOFOLLOW. + + O_NOFOLLOW refuses to follow a pre-existing symlink at the chosen + path so a malicious --output (or a planted symlink in CWD) cannot + redirect the write to a sensitive file. + """ + payload = json.dumps(snapshot, indent=2).encode("utf-8") + fd = os.open( + str(output_path), + os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, + 0o644, + ) + try: + os.write(fd, payload) + finally: + os.close(fd) + + +def heuristic_generate_model( + *, + schemas: dict[str, dict[str, Any]], + model_name: str, + derive_fqn: Callable[[str], str], + classify_role: Callable[[str, str], str], +) -> dict[str, Any]: + """Deterministic stand-in for the AI generator (see ``build_model``). + + Builds: one dataset per table (with classified fields[]), one + COUNT(*) metric per dataset as a placeholder, no relationships + (cross-table FKs are not inferrable from columns alone), an empty + constraints list, and a glossary entry per dataset. + + Accepts the FQN-derivation and role-classification helpers as + callables so the helper module stays free of import-time coupling + to the main service module's regex constants. + """ + datasets: list[dict[str, Any]] = [] + metrics: list[dict[str, Any]] = [] + glossary: list[dict[str, Any]] = [] + + for tid, detail in schemas.items(): + ds_name = ( + (detail.get("display_name") or detail.get("name") or tid.split(".")[-1]) + .replace(" ", "_") + .lower() + ) + fields: list[dict[str, Any]] = [] + for col in detail.get("column_details", []) or []: + cname = col.get("name", "") + basetype = col.get("type", "") or col.get("native_type", "") + fields.append( + { + "name": cname, + "type": basetype, + "role": classify_role(cname, basetype), + } + ) + datasets.append( + { + "name": ds_name, + "tableId": tid, + "fqn": derive_fqn(tid), + "fields": fields, + "description": detail.get("description", "") or "", + } + ) + metrics.append( + { + "name": f"{ds_name}_row_count", + "sql": f"COUNT(*) FROM {derive_fqn(tid)}", + "dataset": tid, + "description": f"Row count of {ds_name}.", + } + ) + glossary.append( + { + "term": ds_name, + "definition": f"Table {tid}: {detail.get('description', '') or 'auto-generated'}.", + } + ) + + return { + "name": model_name, + "description": ( + f"Heuristic-generated model from {len(schemas)} table(s). " + "Iterate with `kbagent semantic-layer add/edit`." + ), + "sql_dialect": "Snowflake", + "datasets": datasets, + "metrics": metrics, + "relationships": [], + "constraints": [], + "glossary": glossary, + } diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py new file mode 100644 index 00000000..0148651d --- /dev/null +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -0,0 +1,1519 @@ +"""Semantic-layer service — business logic for ``kbagent semantic-layer``. + +Composes :class:`MetastoreClient` primitives into the high-level operations +exposed by the CLI: model resolution, show/validate/export/diff (read), +add/edit/import/promote/build (write), remove (destructive), and the +token-encryption helper. + +All API calls go through this service; the command layer only formats inputs +and outputs. Following the project's BaseService pattern, the metastore +client is created via an injected ``metastore_client_factory`` so unit tests +can swap in a :class:`unittest.mock.MagicMock`. +""" + +from __future__ import annotations + +import json +import logging +import re +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any, ClassVar + +from ..config_store import ConfigStore +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..metastore_client import MetastoreClient, SemanticType +from ..models import ProjectConfig +from ._semantic_layer_crud import REMOVE_KINDS as _REMOVE_KINDS_HELPER +from ._semantic_layer_crud import code_metric as _code_metric_helper +from ._semantic_layer_crud import delete_then_post as _delete_then_post_helper +from ._semantic_layer_crud import edit_metric_with_cascade as _edit_metric_helper +from ._semantic_layer_crud import edit_simple as _edit_simple_helper +from ._semantic_layer_crud import find_target_for_remove as _find_target_for_remove +from ._semantic_layer_crud import scan_orphan_constraints as _scan_orphan_constraints +from ._semantic_layer_crud import validate_constraint_attrs as _validate_constraint_attrs +from ._semantic_layer_internals import build_export_snapshot as _build_export_snapshot +from ._semantic_layer_internals import collect_side_from_file +from ._semantic_layer_internals import compare_attrs as _compare_attrs_helper +from ._semantic_layer_internals import default_export_path as _default_export_path +from ._semantic_layer_internals import diff_one_type as _diff_one_type_helper +from ._semantic_layer_internals import fetch_table_schemas as _fetch_table_schemas +from ._semantic_layer_internals import heuristic_generate_model as _heuristic_generate_helper +from ._semantic_layer_internals import push_built_model as _push_built_model +from ._semantic_layer_internals import resolve_model_uuid as _resolve_model_uuid +from ._semantic_layer_internals import run_import_loop as _run_import_loop +from ._semantic_layer_internals import run_promote_loop as _run_promote_loop +from ._semantic_layer_internals import ( + synthesize_role_classified_fields as _synthesize_role_classified_fields, +) +from ._semantic_layer_internals import unpack_attrs_with_id as _unpack_attrs_with_id +from ._semantic_layer_internals import unpack_children_by_plural as _unpack_children_by_plural +from ._semantic_layer_internals import validate_basic as _validate_basic_helper +from ._semantic_layer_internals import validate_deep as _validate_deep_helper +from ._semantic_layer_internals import write_snapshot_to_file as _write_snapshot_to_file +from .base import BaseService, ClientFactory +from .encrypt_service import EncryptService +from .storage_service import StorageService + +# Constraint name regex enforced by the metastore server. +CONSTRAINT_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*$") + +# Constraint type enum (closed set on the server). +CONSTRAINT_TYPES: tuple[str, ...] = ( + "inequality", + "equality", + "range", + "composition", + "exclusion", + "temporal", + "conditional", +) + +# Constraint severity enum (3 values; the 4-band health suffix lives in +# the name, not severity). +CONSTRAINT_SEVERITIES: tuple[str, ...] = ("error", "warning", "info") + +# Role heuristics for `add dataset --deep-fields`. +_KEY_PREFIXES = ("PK_", "FK_") +_TIMESTAMP_NAMES = ("_DATE", "DATE_", "INS_DT", "UPD_DT") +_MEASURE_TOKENS = ( + "AMOUNT", + "VALUE", + "TOTAL", + "REVENUE", + "COST", + "PRICE", + "RATE", + "PCT", + "PERCENT", + "COUNT", + "QTY", + "QUANTITY", +) +_NUMERIC_TYPES = ("NUMBER", "DECIMAL", "FLOAT", "INTEGER", "INT") + + +def _derive_fqn(table_id: str) -> str: + """Compute ``"KEBOOLA"."<schema>"."<table>"`` from a Keboola tableId. + + The Keboola Snowflake mapping is ``"KEBOOLA"."<bucket-path>"."<table>"``; + we split off the last segment as the table and quote both remaining + pieces as a single schema string. + """ + if '"' in table_id: + # Reject double-quotes in tableIds at the service boundary: the FQN + # is stored verbatim in the metastore and pasted into Snowflake SQL by + # downstream consumers. A `"` inside a segment would terminate a quoted + # identifier early and let an attacker steer parsing — defense in depth + # even though Keboola Storage would reject the bucket/table at creation. + raise KeboolaApiError( + message=( + f"tableId {table_id!r} contains a double-quote, which is " + "rejected at the FQN-derivation boundary because the FQN is " + "pasted into Snowflake SQL by downstream consumers." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + parts = table_id.split(".") + if len(parts) < 2: + raise KeboolaApiError( + message=f"tableId {table_id!r} must contain at least one dot.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + schema = ".".join(parts[:-1]) + table = parts[-1] + return f'"KEBOOLA"."{schema}"."{table}"' + + +def _classify_field_role(name: str, basetype: str) -> str: + """Apply the documented role heuristic to (column name, basetype).""" + upper = name.upper() + if any(upper.startswith(prefix) for prefix in _KEY_PREFIXES): + return "key" + if any(tok in upper for tok in _TIMESTAMP_NAMES): + return "timestamp" + if basetype.upper() in _NUMERIC_TYPES and any(tok in upper for tok in _MEASURE_TOKENS): + return "measure" + return "dimension" + + +# Mapping from CLI singular ``--type`` filter to the wire-level +# ``semantic-<type>`` slug. Centralized so commands stay free of magic strings. +TYPE_ALIAS: dict[str, SemanticType] = { + "dataset": "semantic-dataset", + "metric": "semantic-metric", + "relationship": "semantic-relationship", + "constraint": "semantic-constraint", + "glossary": "semantic-glossary", +} + +# Child-types order used everywhere we fan out per-type fetches. +CHILD_TYPES: tuple[SemanticType, ...] = ( + "semantic-dataset", + "semantic-metric", + "semantic-relationship", + "semantic-constraint", + "semantic-glossary", +) + +# Plural keys used in snapshot envelopes and accepted by ``--types`` filters +# on ``import`` and ``promote``. Must mirror the keys produced by +# :meth:`SemanticLayerService.export`. +PLURAL_TYPES: tuple[str, ...] = ( + "datasets", + "metrics", + "relationships", + "constraints", + "glossary", +) + + +def _validate_types_filter(types: list[str] | None) -> set[str] | None: + """Validate ``--types`` against the known plural list, return a set or None. + + Closes the silent no-op trap where a typo like ``--types BOGUS`` would + filter every type out and emit zero imports without an error. + """ + if not types: + return None + unknown = [t for t in types if t not in PLURAL_TYPES] + if unknown: + raise KeboolaApiError( + message=( + f"--types {unknown!r} not recognised. Must be a subset of {list(PLURAL_TYPES)}." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + return set(types) + + +logger = logging.getLogger(__name__) + + +MetastoreClientFactory = Callable[[str, str], MetastoreClient] + + +def default_metastore_client_factory(stack_url: str, token: str) -> MetastoreClient: + """Build a :class:`MetastoreClient` for the given project.""" + return MetastoreClient(stack_url=stack_url, token=token) + + +class SemanticLayerService(BaseService): + """Business logic for the semantic-layer command group. + + Inherits multi-project resolution (``resolve_projects``) and the + parallel worker scaffold (``_run_parallel``) from :class:`BaseService`. + Adds a dedicated :class:`MetastoreClient` factory so command-layer + operations can target the metastore without polluting the Storage API + client. + + Cross-project operations (``promote``) hold **two** clients in a + ``try/finally`` and close both even on failure. + """ + + def __init__( + self, + config_store: ConfigStore, + client_factory: ClientFactory | None = None, + metastore_client_factory: MetastoreClientFactory | None = None, + ) -> None: + super().__init__(config_store=config_store, client_factory=client_factory) + self._metastore_factory: MetastoreClientFactory = ( + metastore_client_factory or default_metastore_client_factory + ) + + # ------------------------------------------------------------------ + # Helpers (used by every subcommand) + # ------------------------------------------------------------------ + + def _resolve_one_project(self, alias: str) -> ProjectConfig: + """Resolve a single project alias to its ``ProjectConfig`` or raise. + + :meth:`BaseService.resolve_projects` already raises ``ConfigError`` for + missing aliases, so we just unwrap. + """ + return self.resolve_projects([alias])[alias] + + def _new_metastore_client(self, project: ProjectConfig) -> MetastoreClient: + """Build a fresh metastore client. Caller is responsible for ``close()``.""" + return self._metastore_factory(project.stack_url, project.token) + + def _resolve_model( + self, + client: MetastoreClient, + model_name_or_uuid: str | None, + ) -> tuple[str, dict[str, Any]]: + """Resolve a model selector to ``(uuid, attributes_dict)``. + + Body lives in :func:`._semantic_layer_internals.resolve_model_uuid`. + """ + return _resolve_model_uuid(client, model_name_or_uuid) + + # ------------------------------------------------------------------ + # Phase 3 — Read commands + # ------------------------------------------------------------------ + + def list_models(self, alias: str) -> dict[str, Any]: + """List all semantic-layer models for a project. + + Returns: + Dict with ``project`` and ``models`` (list of + ``{id, name, description, sql_dialect}``). + """ + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + raw = client.list_items("semantic-model") + finally: + client.close() + + models: list[dict[str, Any]] = [] + for item in raw: + attrs = item.get("attributes") or {} + models.append( + { + "id": item.get("id", ""), + "name": attrs.get("name", ""), + "description": attrs.get("description", ""), + "sql_dialect": attrs.get("sql_dialect", ""), + } + ) + return {"project": alias, "models": models} + + # ------------------------------------------------------------------ + # Internal helpers (model-scoped fetches) + # ------------------------------------------------------------------ + + @staticmethod + def _fetch_children_parallel( + client: MetastoreClient, + model_uuid: str, + ) -> dict[SemanticType, list[dict[str, Any]]]: + """Fetch all child entity types in parallel, filtered to one model. + + Each result is the raw item list (full ``{type, id, attributes, meta}``). + Errors propagate to the caller (we re-raise the first encountered). + """ + results: dict[SemanticType, list[dict[str, Any]]] = {} + with ThreadPoolExecutor(max_workers=5) as pool: + future_to_type = {pool.submit(client.list_items, t, model_uuid): t for t in CHILD_TYPES} + errors: list[Exception] = [] + for future in future_to_type: + try: + results[future_to_type[future]] = future.result() + except Exception as exc: + errors.append(exc) + if errors: + raise errors[0] + return results + + # ------------------------------------------------------------------ + # Phase 3 — show + # ------------------------------------------------------------------ + + # Plural-key alias used by show_model's --type filter. + _PLURAL_BY_TYPE: ClassVar[dict[str, str]] = { + "dataset": "datasets", + "metric": "metrics", + "relationship": "relationships", + "constraint": "constraints", + "glossary": "glossary", + } + + def show_model( + self, + alias: str, + model_name_or_uuid: str | None = None, + type_filter: str | None = None, + ) -> dict[str, Any]: + """Return the entities in a model. See ``--type`` for filtering.""" + if type_filter is not None and type_filter not in TYPE_ALIAS: + raise KeboolaApiError( + message=( + f"Invalid --type {type_filter!r}. Must be one of: " + f"{', '.join(sorted(TYPE_ALIAS))}." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) + raw_by_type = self._fetch_children_parallel(client, model_uuid) + finally: + client.close() + + result: dict[str, Any] = { + "project": alias, + "model": {"id": model_uuid, "name": model_attrs.get("name", "")}, + **_unpack_children_by_plural(raw_by_type), + } + if type_filter is not None: + plural = self._PLURAL_BY_TYPE[type_filter] + for k in set(self._PLURAL_BY_TYPE.values()) - {plural}: + result.pop(k, None) + return result + + # ------------------------------------------------------------------ + # Phase 3 — validate (+ --deep) + # ------------------------------------------------------------------ + + def validate_model( + self, + alias: str, + model_name_or_uuid: str | None = None, + deep: bool = False, + ) -> dict[str, Any]: + """Validate a semantic-layer model (basic + optional --deep). + + Returns ``{errors, warnings, deep, valid, model, project}``. See + :func:`._semantic_layer_internals.validate_basic` and + :func:`._semantic_layer_internals.validate_deep` for the full + check inventory. + """ + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) + raw_by_type = self._fetch_children_parallel(client, model_uuid) + finally: + client.close() + + datasets = _unpack_attrs_with_id(raw_by_type.get("semantic-dataset", [])) + metrics = _unpack_attrs_with_id(raw_by_type.get("semantic-metric", [])) + relationships = _unpack_attrs_with_id(raw_by_type.get("semantic-relationship", [])) + constraints = _unpack_attrs_with_id(raw_by_type.get("semantic-constraint", [])) + glossary = _unpack_attrs_with_id(raw_by_type.get("semantic-glossary", [])) + + errors: list[dict[str, str]] = [] + warnings: list[dict[str, str]] = [] + self._validate_basic( + datasets=datasets, + metrics=metrics, + relationships=relationships, + constraints=constraints, + glossary=glossary, + errors=errors, + warnings=warnings, + ) + + if deep: + self._validate_deep( + alias=alias, + datasets=datasets, + metrics=metrics, + errors=errors, + warnings=warnings, + ) + + return { + "project": alias, + "model": {"id": model_uuid, "name": model_attrs.get("name", "")}, + "errors": errors, + "warnings": warnings, + "deep": deep, + "valid": len(errors) == 0, + } + + # -- validate internals -------------------------------------------- + + @staticmethod + def _validate_basic( + *, + datasets: list[dict[str, Any]], + metrics: list[dict[str, Any]], + relationships: list[dict[str, Any]], + constraints: list[dict[str, Any]], + glossary: list[dict[str, Any]], + errors: list[dict[str, str]], + warnings: list[dict[str, str]], + ) -> None: + """Pure in-memory validation (no API calls). + + Thin delegate to :func:`._semantic_layer_internals.validate_basic` + -- the heavy lifting lives in the internals module so this file + stays under the CONTRIBUTING.md services budget. + """ + _validate_basic_helper( + datasets=datasets, + metrics=metrics, + relationships=relationships, + constraints=constraints, + glossary=glossary, + errors=errors, + warnings=warnings, + ) + + def _validate_deep( + self, + *, + alias: str, + datasets: list[dict[str, Any]], + metrics: list[dict[str, Any]], + errors: list[dict[str, str]], + warnings: list[dict[str, str]], + ) -> None: + """Add deep checks that require a Snowflake schema fetch per dataset. + + Thin delegate to :func:`._semantic_layer_internals.validate_deep`. + """ + storage = StorageService( + config_store=self._config_store, client_factory=self._client_factory + ) + _validate_deep_helper( + alias=alias, + storage=storage, + datasets=datasets, + metrics=metrics, + errors=errors, + warnings=warnings, + ) + + # ------------------------------------------------------------------ + # Phase 3 — export + # ------------------------------------------------------------------ + + def export_model( + self, + alias: str, + model_name_or_uuid: str | None = None, + output_path: Path | None = None, + ) -> dict[str, Any]: + """Export a model + every child entity to a self-describing JSON file. + + Each child item is stored in its full server shape + (``{type, id, attributes, meta}``) so the file is replayable by + ``import`` and ``promote`` without further negotiation. + + Args: + alias: Source project alias. + model_name_or_uuid: Selector for the source model. + output_path: Where to write the snapshot. Defaults to + ``./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json``. + + Returns: + ``{path, exported_at, project, model, datasets, metrics, + relationships, constraints, glossary, counts}``. + """ + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) + raw_by_type = self._fetch_children_parallel(client, model_uuid) + finally: + client.close() + + snapshot = _build_export_snapshot( + alias=alias, + model_uuid=model_uuid, + model_attrs=model_attrs, + raw_by_type=raw_by_type, + ) + if output_path is None: + output_path = _default_export_path(str(model_attrs.get("name", "model"))) + _write_snapshot_to_file(snapshot, output_path) + + result = dict(snapshot) + result["path"] = str(output_path) + result["counts"] = { + "datasets": len(snapshot["datasets"]), + "metrics": len(snapshot["metrics"]), + "relationships": len(snapshot["relationships"]), + "constraints": len(snapshot["constraints"]), + "glossary": len(snapshot["glossary"]), + } + return result + + # ------------------------------------------------------------------ + # Phase 3 — diff + # ------------------------------------------------------------------ + + # Diff helpers live in ._semantic_layer_internals (DIFF_IGNORED_KEYS, + # compare_attrs, diff_one_type, collect_side_from_file) -- imported + # there so this orchestrator file fits the CONTRIBUTING.md budget. + + def diff( + self, + *, + project_a: str | None = None, + project_b: str | None = None, + model_a: str | None = None, + model_b: str | None = None, + file_a: Path | None = None, + file_b: Path | None = None, + ) -> dict[str, Any]: + """Diff two semantic-layer snapshots. + + Each "side" is either a live project (resolve + fetch) or a file + produced by :meth:`export_model`. The diff is structural and + per-entity-type: ``added | removed | changed`` lists. The + identification key is ``name`` for every type except glossary, + which uses ``term``. + + Returns: + ``{left, right, datasets, metrics, relationships, constraints, + glossary}`` where each per-type entry is + ``{added: [name], removed: [name], changed: [{name, diff_keys}]}``. + """ + left = self._collect_side(project=project_a, model=model_a, file=file_a) + right = self._collect_side(project=project_b, model=model_b, file=file_b) + + result: dict[str, Any] = {"left": left["ref"], "right": right["ref"]} + for type_key, id_key in ( + ("datasets", "name"), + ("metrics", "name"), + ("relationships", "name"), + ("constraints", "name"), + ("glossary", "term"), + ): + result[type_key] = self._diff_one_type( + left["data"].get(type_key, []), + right["data"].get(type_key, []), + id_key=id_key, + ) + return result + + def _collect_side( + self, + *, + project: str | None, + model: str | None, + file: Path | None, + ) -> dict[str, Any]: + """Resolve one side of a diff to bare ``attributes`` lists. + + Live-project path stays here (needs ``self.show_model``); the + file-path branch delegates to + :func:`._semantic_layer_internals.collect_side_from_file`. + """ + if project is not None: + data = self.show_model(alias=project, model_name_or_uuid=model) + return { + "ref": {"source": "project", "ref": project, "model": data.get("model", {})}, + "data": data, + } + if file is None: + raise KeboolaApiError( + message="Internal: diff side has no project and no file.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + return collect_side_from_file(file) + + def _diff_one_type( + self, + left: list[dict[str, Any]], + right: list[dict[str, Any]], + *, + id_key: str, + ) -> dict[str, Any]: + """Thin wrapper around :func:`._semantic_layer_internals.diff_one_type`.""" + return _diff_one_type_helper(left, right, id_key=id_key) + + def _compare_attrs( + self, + a: dict[str, Any], + b: dict[str, Any], + ) -> list[str]: + """Thin wrapper around :func:`._semantic_layer_internals.compare_attrs`.""" + return _compare_attrs_helper(a, b) + + # ------------------------------------------------------------------ + # Phase 4 — Model lifecycle (create / delete) + # ------------------------------------------------------------------ + + def create_model( + self, + alias: str, + name: str, + description: str = "", + sql_dialect: str = "Snowflake", + ) -> dict[str, Any]: + """Create a semantic-layer model and return the server-stored item.""" + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + data: dict[str, Any] = {"name": name, "sql_dialect": sql_dialect} + if description: + data["description"] = description + created = client.post_item("semantic-model", name=name, data=data) + finally: + client.close() + return {"project": alias, "model": created} + + def delete_model( + self, + alias: str, + model_name_or_uuid: str, + ) -> dict[str, Any]: + """Delete a semantic-layer model. + + Lists every referencing child entity first so the caller can warn + about orphaning. The metastore may refuse to delete a model that + still has children — we surface that error verbatim. + """ + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) + children = self._fetch_children_parallel(client, model_uuid) + client.delete_item("semantic-model", model_uuid) + finally: + client.close() + counts = {k.replace("semantic-", "") + "s": len(v) for k, v in children.items()} + # "semantic-glossary" -> "glossarys" — normalise the only odd one. + counts.setdefault("glossary", counts.pop("glossarys", 0)) + return { + "project": alias, + "deleted": {"id": model_uuid, "name": model_attrs.get("name", "")}, + "orphaned_children": counts, + } + + # ------------------------------------------------------------------ + # Phase 4 — add subcommands + # ------------------------------------------------------------------ + + def add_metric( + self, + alias: str, + model_name_or_uuid: str | None, + *, + name: str, + sql: str, + dataset: str, + description: str = "", + assume_yes: bool = False, + is_tty: bool = False, + confirm_cb: Callable[[str], bool] | None = None, + ) -> dict[str, Any]: + """Create a metric. The ``dataset`` argument is a tableId. + + If the tableId is not in the model's datasets, warn and require + an interactive confirmation. ``--yes`` (assume_yes) skips the + prompt. In non-TTY contexts we refuse with VALIDATION_ERROR + rather than silently push a broken metric. + """ + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + datasets = client.list_items("semantic-dataset", model_uuid) + ds_tids = {(d.get("attributes") or {}).get("tableId", "") for d in datasets} + + if dataset not in ds_tids: + msg = ( + f"Metric dataset {dataset!r} is not a tableId in this model " + f"(known: {sorted(t for t in ds_tids if t)})." + ) + if not assume_yes: + if not is_tty: + raise KeboolaApiError( + message=msg + " Pass --yes to bypass.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + if confirm_cb is None or not confirm_cb(msg + " Create anyway?"): + raise KeboolaApiError( + message="Aborted by user.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + + data: dict[str, Any] = { + "name": name, + "sql": sql, + "dataset": dataset, + "modelUUID": model_uuid, + } + if description: + data["description"] = description + return client.post_item("semantic-metric", name=name, data=data) + finally: + client.close() + + def add_dataset( + self, + alias: str, + model_name_or_uuid: str | None, + *, + name: str, + table_id: str, + description: str = "", + grain: str = "", + primary_key: list[str] | None = None, + deep_fields: bool = False, + ) -> dict[str, Any]: + """Create a dataset, auto-deriving ``fqn`` from the tableId. + + With ``deep_fields=True``, fetches the storage column schema and + synthesises a ``fields[]`` array with role heuristics. + """ + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + data: dict[str, Any] = { + "name": name, + "tableId": table_id, + "fqn": _derive_fqn(table_id), + "modelUUID": model_uuid, + } + if description: + data["description"] = description + if grain: + data["grain"] = grain + if primary_key: + data["primaryKey"] = list(primary_key) + if deep_fields: + storage = StorageService( + config_store=self._config_store, + client_factory=self._client_factory, + ) + fields = _synthesize_role_classified_fields( + storage, alias, table_id, _classify_field_role + ) + if fields: + data["fields"] = fields + return client.post_item("semantic-dataset", name=name, data=data) + finally: + client.close() + + def add_relationship( + self, + alias: str, + model_name_or_uuid: str | None, + *, + name: str, + from_: str, + to: str, + on: str, + type_: str, + ) -> dict[str, Any]: + """Create a relationship. ``from``/``to`` are tableIds; ``type``='left'|'inner'.""" + if type_ not in ("left", "inner"): + raise KeboolaApiError( + message=f"--type must be 'left' or 'inner', got {type_!r}.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + data = { + "name": name, + "from": from_, + "to": to, + "on": on, + "type": type_, + "modelUUID": model_uuid, + } + return client.post_item("semantic-relationship", name=name, data=data) + finally: + client.close() + + def add_constraint( + self, + alias: str, + model_name_or_uuid: str | None, + *, + name: str, + constraint_type: str, + rule: str, + metrics: list[str], + severity: str = "warning", + ) -> dict[str, Any]: + """Create a constraint after validating every field locally.""" + _validate_constraint_attrs( + name_re=CONSTRAINT_NAME_RE, + constraint_types=CONSTRAINT_TYPES, + severities=CONSTRAINT_SEVERITIES, + name=name, + constraint_type=constraint_type, + severity=severity, + ) + + # METRICS exist in model + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + existing = client.list_items("semantic-metric", model_uuid) + existing_names = {(m.get("attributes") or {}).get("name", "") for m in existing} + missing = [m for m in metrics if m not in existing_names] + if missing: + raise KeboolaApiError( + message=( + f"Constraint references metric(s) not in model: {missing}. " + f"Known metrics: {sorted(existing_names - {''})}." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + data = { + "name": name, + "constraintType": constraint_type, + "rule": rule, + "metrics": list(metrics), + "severity": severity, + "modelUUID": model_uuid, + } + return client.post_item("semantic-constraint", name=name, data=data) + finally: + client.close() + + def add_glossary( + self, + alias: str, + model_name_or_uuid: str | None, + *, + term: str, + definition: str = "", + ) -> dict[str, Any]: + """Create a glossary term. Outer envelope ``name`` must equal ``term``.""" + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + data: dict[str, Any] = {"term": term, "modelUUID": model_uuid} + if definition: + data["definition"] = definition + return client.post_item("semantic-glossary", name=term, data=data) + finally: + client.close() + + # ------------------------------------------------------------------ + # Phase 4 — edit (DELETE-then-POST with rollback + rename cascade) + # ------------------------------------------------------------------ + + # Thin delegates to ._semantic_layer_crud helpers -- bodies live + # there so this orchestrator stays under the services budget. + _code_metric = staticmethod(_code_metric_helper) + _delete_then_post = staticmethod(_delete_then_post_helper) + + def edit_metric( + self, + alias: str, + model_name_or_uuid: str | None, + *, + current_name: str, + new_name: str | None = None, + new_sql: str | None = None, + new_dataset: str | None = None, + new_description: str | None = None, + assume_yes: bool = False, + is_tty: bool = False, + confirm_cb: Callable[[str], bool] | None = None, + ) -> dict[str, Any]: + """Edit a metric via DELETE+POST with rename-cascade on constraints. + + Returns: + ``{updated: item, cascaded_constraints: [...], rollback: None|{...}}``. + + The cascade body lives in + :func:`._semantic_layer_crud.edit_metric_with_cascade` -- this + method just resolves credentials + the model UUID and + delegates. + """ + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + return _edit_metric_helper( + client, + model_uuid=model_uuid, + current_name=current_name, + new_name=new_name, + new_sql=new_sql, + new_dataset=new_dataset, + new_description=new_description, + assume_yes=assume_yes, + is_tty=is_tty, + confirm_cb=confirm_cb, + ) + finally: + client.close() + + def edit_dataset( + self, + alias: str, + model_name_or_uuid: str | None, + *, + current_name: str, + new_name: str | None = None, + new_description: str | None = None, + new_grain: str | None = None, + ) -> dict[str, Any]: + """Edit a dataset (DELETE+POST). Renames do NOT cascade for datasets.""" + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + return _edit_simple_helper( + client, + "semantic-dataset", + items=client.list_items("semantic-dataset", model_uuid), + id_key="name", + current_key=current_name, + overrides={ + "name": new_name, + "description": new_description, + "grain": new_grain, + }, + not_found_label="Dataset", + ) + finally: + client.close() + + def edit_constraint( + self, + alias: str, + model_name_or_uuid: str | None, + *, + current_name: str, + new_name: str | None = None, + new_rule: str | None = None, + new_constraint_type: str | None = None, + new_severity: str | None = None, + new_metrics: list[str] | None = None, + ) -> dict[str, Any]: + """Edit a constraint (DELETE+POST). Validates new attrs locally first.""" + _validate_constraint_attrs( + name_re=CONSTRAINT_NAME_RE, + constraint_types=CONSTRAINT_TYPES, + severities=CONSTRAINT_SEVERITIES, + name=new_name, + constraint_type=new_constraint_type, + severity=new_severity, + ) + + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + if new_metrics is not None: + existing = client.list_items("semantic-metric", model_uuid) + existing_names = {(m.get("attributes") or {}).get("name", "") for m in existing} + missing = [m for m in new_metrics if m not in existing_names] + if missing: + raise KeboolaApiError( + message=(f"--new-metrics references metric(s) not in model: {missing}."), + error_code=ErrorCode.VALIDATION_ERROR, + ) + return _edit_simple_helper( + client, + "semantic-constraint", + items=client.list_items("semantic-constraint", model_uuid), + id_key="name", + current_key=current_name, + overrides={ + "name": new_name, + "rule": new_rule, + "constraintType": new_constraint_type, + "severity": new_severity, + "metrics": list(new_metrics) if new_metrics is not None else None, + }, + not_found_label="Constraint", + ) + finally: + client.close() + + def edit_relationship( + self, + alias: str, + model_name_or_uuid: str | None, + *, + current_name: str, + new_name: str | None = None, + new_from: str | None = None, + new_to: str | None = None, + new_on: str | None = None, + new_type: str | None = None, + ) -> dict[str, Any]: + """Edit a relationship (DELETE+POST). Validates ``--new-type`` locally. + + Relationships are not referenced by any other entity, so no + cascade is needed -- the result is shaped identically to + :meth:`edit_dataset` for consistency. + """ + if new_type is not None and new_type not in ("left", "inner"): + raise KeboolaApiError( + message=f"--new-type must be 'left' or 'inner', got {new_type!r}.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + return _edit_simple_helper( + client, + "semantic-relationship", + items=client.list_items("semantic-relationship", model_uuid), + id_key="name", + current_key=current_name, + overrides={ + "name": new_name, + "from": new_from, + "to": new_to, + "on": new_on, + "type": new_type, + }, + not_found_label="Relationship", + ) + finally: + client.close() + + def edit_glossary( + self, + alias: str, + model_name_or_uuid: str | None, + *, + current_term: str, + new_term: str | None = None, + new_definition: str | None = None, + ) -> dict[str, Any]: + """Edit a glossary term (DELETE+POST). + + Renaming via ``--new-term`` is destructive for downstream + consumers that join on the literal term string (the term IS the + identity for a glossary entry). The CLI layer warns + gates + behind ``--yes``; this method just executes. + """ + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + return _edit_simple_helper( + client, + "semantic-glossary", + items=client.list_items("semantic-glossary", model_uuid), + id_key="term", + current_key=current_term, + overrides={"term": new_term, "definition": new_definition}, + not_found_label="Glossary term", + ) + finally: + client.close() + + # ------------------------------------------------------------------ + # Phase 5 — remove (destructive, orphan-warning before delete) + # ------------------------------------------------------------------ + + # REMOVE_KINDS lives in ._semantic_layer_crud; re-bound here so + # subclasses can override the accepted-kinds set without forking + # the helper. + _REMOVE_KINDS = _REMOVE_KINDS_HELPER + + def preview_remove( + self, + alias: str, + model_name_or_uuid: str | None, + *, + kind: str, + name: str, + ) -> dict[str, Any]: + """Return what `remove` would do (orphan list) without deleting. + + Always called before the actual delete so the command layer can + echo the warning even when --yes skips the prompt. + """ + if kind not in self._REMOVE_KINDS: + raise KeboolaApiError( + message=( + f"remove kind must be one of {'|'.join(self._REMOVE_KINDS)}, got {kind!r}." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + target, _, _ = _find_target_for_remove( + client, + kind=kind, + model_uuid=model_uuid, + name=name, + type_alias=TYPE_ALIAS, + ) + orphan_constraints = ( + _scan_orphan_constraints(client, model_uuid=model_uuid, metric_name=name) + if kind == "metric" + else [] + ) + return { + "kind": kind, + "id": target["id"], + "name": name, + "orphaned_constraints": orphan_constraints, + } + finally: + client.close() + + def remove_item( + self, + alias: str, + model_name_or_uuid: str | None, + *, + kind: str, + name: str, + ) -> dict[str, Any]: + """Delete a single child entity. Returns the removed item descriptor.""" + if kind not in self._REMOVE_KINDS: + raise KeboolaApiError( + message=( + f"remove kind must be one of {'|'.join(self._REMOVE_KINDS)}, got {kind!r}." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + target, type_slug, _ = _find_target_for_remove( + client, + kind=kind, + model_uuid=model_uuid, + name=name, + type_alias=TYPE_ALIAS, + ) + orphan_constraints = ( + _scan_orphan_constraints(client, model_uuid=model_uuid, metric_name=name) + if kind == "metric" + else [] + ) + client.delete_item(type_slug, target["id"]) + return { + "removed": {"type": type_slug, "id": target["id"], "name": name}, + "orphaned_constraints": orphan_constraints, + } + finally: + client.close() + + # ------------------------------------------------------------------ + # Phase 6 — import (replay a snapshot, optionally overwrite) + # ------------------------------------------------------------------ + + # Push order for both `import` and `promote` lives in + # ._semantic_layer_internals.PUSH_ORDER. + + def import_snapshot( + self, + alias: str, + file: Path, + *, + model_name_or_uuid: str | None = None, + types: list[str] | None = None, + dry_run: bool = False, + overwrite: bool = False, + ) -> dict[str, Any]: + """Replay a snapshot produced by :meth:`export_model` into a project. + + Args: + alias: Target project alias. + file: Snapshot JSON file path. + model_name_or_uuid: Target model selector. If different from the + snapshot's model, every item's ``modelUUID`` is rewritten. + types: Filter to a subset of types. ``None`` = all types. + dry_run: When True, plan and return the action counts without + hitting any write API. + overwrite: When True, DELETE+POST conflicting items by name. + Default (False) skips conflicts. + + Returns: + ``{imported: {<type>: {created, skipped, overwritten, failed}}}``. + """ + try: + snapshot = json.loads(file.read_text(encoding="utf-8")) + except OSError as exc: + raise KeboolaApiError( + message=f"Cannot read --file {file}: {exc}", + error_code=ErrorCode.READ_ERROR, + ) from exc + except json.JSONDecodeError as exc: + raise KeboolaApiError( + message=f"File {file} is not valid JSON: {exc}", + error_code=ErrorCode.INVALID_FORMAT, + ) from exc + + return self.import_snapshot_from_dict( + alias, + snapshot=snapshot, + model_name_or_uuid=model_name_or_uuid, + types=types, + dry_run=dry_run, + overwrite=overwrite, + ) + + def import_snapshot_from_dict( + self, + alias: str, + *, + snapshot: dict[str, Any], + model_name_or_uuid: str | None = None, + types: list[str] | None = None, + dry_run: bool = False, + overwrite: bool = False, + ) -> dict[str, Any]: + """Replay an in-memory snapshot dict (sibling of :meth:`import_snapshot`). + + Used by the REST router ``POST /semantic-layer/import``, where the + snapshot arrives inline in the JSON body — there is no file to + read. The wire-level shape must match what :meth:`export_model` + produces (``{datasets, metrics, relationships, constraints, + glossary}`` with full server item envelopes). + """ + if not isinstance(snapshot, dict): + raise KeboolaApiError( + message="Snapshot must be a JSON object.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + + type_filter = _validate_types_filter(types) + + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + existing_by_type = self._fetch_children_parallel(client, model_uuid) + + imported = _run_import_loop( + client, + snapshot=snapshot, + target_model_uuid=model_uuid, + existing_by_type=existing_by_type, + type_filter=type_filter, + dry_run=dry_run, + overwrite=overwrite, + ) + return { + "target_project": alias, + "target_model": model_uuid, + "source_model": (snapshot.get("model") or {}).get("id", ""), + "dry_run": dry_run, + "overwrite": overwrite, + "imported": imported, + } + finally: + client.close() + + # ------------------------------------------------------------------ + # Phase 6 — promote (cross-project copy) + # ------------------------------------------------------------------ + + def promote_model( + self, + *, + from_project: str, + to_project: str, + from_model: str | None = None, + to_model: str | None = None, + types: list[str] | None = None, + dry_run: bool = False, + ) -> dict[str, Any]: + """Promote a model's entities from one project to another. + + Holds two metastore clients in a try/finally and closes both on + exit even on error. Items are classified as NEW (not in target), + IDENTICAL (all attributes equal after stripping modelUUID), or + CHANGED (differs). Default behaviour: import NEW + overwrite + CHANGED. IDENTICAL items are skipped. Target items absent from + source are NEVER deleted. + + Returns: + ``{from_project, to_project, dry_run, datasets: {new, + overwritten, identical, failed}, metrics: {...}, ...}``. + """ + projects = self.resolve_projects([from_project, to_project]) + if from_project not in projects: + raise ConfigError(f"Source project '{from_project}' not found.") + if to_project not in projects: + raise ConfigError(f"Target project '{to_project}' not found.") + + type_filter = _validate_types_filter(types) + + src_client = self._new_metastore_client(projects[from_project]) + tgt_client = self._new_metastore_client(projects[to_project]) + try: + src_uuid, _ = self._resolve_model(src_client, from_model) + tgt_uuid, _ = self._resolve_model(tgt_client, to_model) + + src_children = self._fetch_children_parallel(src_client, src_uuid) + tgt_children = self._fetch_children_parallel(tgt_client, tgt_uuid) + + per_type_stats = _run_promote_loop( + tgt_client, + src_children=src_children, + tgt_children=tgt_children, + target_model_uuid=tgt_uuid, + type_filter=type_filter, + dry_run=dry_run, + ) + return { + "from_project": from_project, + "to_project": to_project, + "from_model": src_uuid, + "to_model": tgt_uuid, + "dry_run": dry_run, + **per_type_stats, + } + finally: + try: + src_client.close() + finally: + tgt_client.close() + + # ------------------------------------------------------------------ + # Phase 7 — build (AI-assisted / heuristic greenfield) + # ------------------------------------------------------------------ + + def build_model( + self, + alias: str, + *, + table_ids: list[str], + model_name: str | None = None, + model_name_or_uuid: str | None = None, + dry_run: bool = False, + output_path: Path | None = None, + ) -> dict[str, Any]: + """Build (or update) a semantic-layer model from a list of tableIds. + + Implementation note: the AI Service client (`ai_client.py`) currently + exposes only `suggest_components` for natural-language component + search; there is no endpoint that returns arbitrary structured JSON. + Rather than degrade silently, this method falls back to a + DETERMINISTIC HEURISTIC builder: it fetches the storage schema for + every tableId in parallel, synthesises a dataset per table with + role-classified fields, suggests one COUNT(*)-style metric per + table, and seeds a small glossary. The intended behaviour is "best + starting point, then iterate via `add`/`edit`" — not "ready-to-ship + model". This fallback is documented in gotchas.md. + + Validation runs locally against the generated model BEFORE any POST. + If validation surfaces any errors, the operation is refused with + VALIDATION_ERROR + the full error list in details so callers can + iterate. + """ + if not table_ids: + raise KeboolaApiError( + message="--tables must contain at least one tableId.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + + # Fetch schemas in parallel via in-process StorageService. + storage = StorageService( + config_store=self._config_store, client_factory=self._client_factory + ) + schemas_by_tid, fetch_errors = _fetch_table_schemas(storage, alias, table_ids) + + # Generate model JSON (heuristic). + generated = self._heuristic_generate_model( + schemas=schemas_by_tid, + model_name=model_name or "kbagent_build_model", + ) + + # Validate locally. + errors: list[dict[str, str]] = [] + warnings: list[dict[str, str]] = [] + self._validate_basic( + datasets=generated["datasets"], + metrics=generated["metrics"], + relationships=generated["relationships"], + constraints=generated["constraints"], + glossary=generated["glossary"], + errors=errors, + warnings=warnings, + ) + + # Write output file (if requested). + if output_path is not None: + _write_snapshot_to_file(generated, output_path) + + result: dict[str, Any] = { + "project": alias, + "dry_run": dry_run, + "fallback_used": "heuristic", # no AI endpoint shipped yet + "fetch_errors": fetch_errors, + "generated": generated, + "validation": {"errors": errors, "warnings": warnings}, + "validated": len(errors) == 0, + } + if output_path is not None: + result["output_path"] = str(output_path) + + if dry_run: + return result + + if errors: + raise KeboolaApiError( + message=( + f"Generated model failed local validation ({len(errors)} errors). " + "Refusing to push. Inspect with --dry-run + --output to iterate." + ), + error_code=ErrorCode.VALIDATION_ERROR, + details={"validation": result["validation"]}, + ) + + # Push to the metastore in dependency order. + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + counts, model_uuid, model_item = _push_built_model( + client, + generated=generated, + model_name_or_uuid=model_name_or_uuid, + resolve_model_fn=self._resolve_model, + ) + result["model"] = {"id": model_uuid, "item": model_item} + result["created"] = counts + return result + finally: + client.close() + + @staticmethod + def _heuristic_generate_model( + *, + schemas: dict[str, dict[str, Any]], + model_name: str, + ) -> dict[str, Any]: + """Thin delegate to :func:`._semantic_layer_internals.heuristic_generate_model`. + + Passes the module's FQN-derivation and role-classification + helpers so the internals module stays free of import-time + coupling to this module's regex constants. + """ + return _heuristic_generate_helper( + schemas=schemas, + model_name=model_name, + derive_fqn=_derive_fqn, + classify_role=_classify_field_role, + ) + + # ------------------------------------------------------------------ + # Phase 8 — token --encrypt + # ------------------------------------------------------------------ + + def encrypt_token(self, alias: str, component_id: str) -> dict[str, Any]: + """Encrypt the project's storage token for `user_properties`. + + Reads the project's own token from ``ProjectConfig`` (no config.json + digging) and passes it through the existing :class:`EncryptService` + as ``{"#metastore_token": token}``. + + Returns: + ``{"encrypted": {"#metastore_token": "KBC::ProjectSecure..."}, + "component_id": C, "project": alias}``. + """ + project = self._resolve_one_project(alias) + # Reuse the production EncryptService factory so the token never + # leaves the in-process Storage API path. + encrypt = EncryptService( + config_store=self._config_store, client_factory=self._client_factory + ) + encrypted = encrypt.encrypt( + alias=alias, + component_id=component_id, + input_data={"#metastore_token": project.token}, + ) + return { + "project": alias, + "component_id": component_id, + "encrypted": encrypted, + } diff --git a/tests/test_e2e.py b/tests/test_e2e.py index b6e5a4a7..e543cb04 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -3062,6 +3062,470 @@ def _test_storage_describe(self, bucket_id: str, table_id: str) -> None: col_descs = {c["name"]: c.get("description", "") for c in data["data"]["column_details"]} assert col_descs.get("id") == "Batch column id desc" + def _test_semantic_layer_roundtrip(self) -> None: + """Live roundtrip of the semantic-layer command group against ``self.alias``. + + Bootstraps two throwaway models on the test project, exercises every + verb (model/show/add/edit/validate/export/import/promote/build/token/remove), + and tears everything down even on failure. All entity names are prefixed + with a unique tag so a residue check at the end of the test can assert + the project is clean. + + NOTE: Not wired into ``test_full_cli_e2e`` (which runs ~30 min). The + same surface is exercised independently by + :class:`TestE2ESemanticLayerLifecycle` -- run that with + ``pytest -k SemanticLayer`` for a focused live check. + """ + from keboola_agent_cli.metastore_client import ( + SEMANTIC_TYPES, + MetastoreClient, + ) + + tag = f"kbagent_e2e_{int(time.time())}" + model_name = tag + target_model_name = f"{tag}_target" + + # (item_type, item_id) tuples for guaranteed cleanup. + created_items: list[tuple[str, str]] = [] + model_id: str | None = None + target_model_id: str | None = None + + def _direct_delete(item_type: str, item_id: str) -> None: + with MetastoreClient(stack_url=self.url, token=self.token) as mc: + mc.delete_item(item_type, item_id) + + try: + # 1. model create + data = self._run_ok( + "semantic-layer", + "model", + "create", + "--project", + self.alias, + "--name", + model_name, + ) + model_id = data["data"]["model"]["id"] + assert model_id + + # 2. add two datasets, three metrics, one constraint, one glossary entry. + # tableId comes from the bucket/table built earlier in the big test. + # We don't depend on it existing in actual Snowflake — the metastore + # accepts any string. validate --deep is skipped (would 404 trying to + # fetch storage detail for a synthetic tableId). + ds1 = self._run_ok( + "semantic-layer", + "add", + "dataset", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_ds_a", + "--table-id", + "out.c-syn.fact_a", + ) + created_items.append(("semantic-dataset", ds1["data"]["id"])) + + ds2 = self._run_ok( + "semantic-layer", + "add", + "dataset", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_ds_b", + "--table-id", + "out.c-syn.fact_b", + ) + created_items.append(("semantic-dataset", ds2["data"]["id"])) + + m1 = self._run_ok( + "semantic-layer", + "add", + "metric", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_m_rev", + "--sql", + "COUNT(*)", + "--dataset", + "out.c-syn.fact_a", + "--yes", + ) + created_items.append(("semantic-metric", m1["data"]["id"])) + + m2 = self._run_ok( + "semantic-layer", + "add", + "metric", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_m_cost", + "--sql", + 'SUM("schema"."AMOUNT")', + "--dataset", + "out.c-syn.fact_a", + "--yes", + ) + created_items.append(("semantic-metric", m2["data"]["id"])) + + m3 = self._run_ok( + "semantic-layer", + "add", + "metric", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_m_count_b", + "--sql", + "COUNT(*)", + "--dataset", + "out.c-syn.fact_b", + "--yes", + ) + created_items.append(("semantic-metric", m3["data"]["id"])) + + rel = self._run_ok( + "semantic-layer", + "add", + "relationship", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_rel_a_b", + "--from", + "out.c-syn.fact_a", + "--to", + "out.c-syn.fact_b", + "--on", + "fact_a.id = fact_b.fact_a_id", + ) + created_items.append(("semantic-relationship", rel["data"]["id"])) + + cons = self._run_ok( + "semantic-layer", + "add", + "constraint", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_rev_warning", + "--constraint-type", + "inequality", + "--rule", + "value >= 0", + "--metrics", + f"{tag}_m_rev", + "--severity", + "warning", + ) + created_items.append(("semantic-constraint", cons["data"]["id"])) + + gloss = self._run_ok( + "semantic-layer", + "add", + "glossary", + "--project", + self.alias, + "--model", + model_name, + "--term", + f"{tag}_GMV", + "--definition", + "Gross merchandise value (test)", + ) + created_items.append(("semantic-glossary", gloss["data"]["id"])) + + # 3. show: count assertions + data = self._run_ok( + "semantic-layer", + "show", + "--project", + self.alias, + "--model", + model_name, + ) + assert len(data["data"]["datasets"]) == 2 + assert len(data["data"]["metrics"]) == 3 + assert len(data["data"]["constraints"]) == 1 + assert len(data["data"]["glossary"]) == 1 + + # 4. show --type metric + data = self._run_ok( + "semantic-layer", + "show", + "--project", + self.alias, + "--model", + model_name, + "--type", + "metric", + ) + assert len(data["data"]["metrics"]) >= 3 + + # 5. validate (basic) — expect valid because everything is wired + data = self._run_ok( + "semantic-layer", + "validate", + "--project", + self.alias, + "--model", + model_name, + ) + # The constraint has a severity suffix so no SEVERITY_SUFFIX warning; + # the metric SUM("schema"."AMOUNT") doesn't match SUM_ON_PCT regex. + assert data["data"]["valid"] is True + + # 6. edit metric rename — triggers constraint cascade + data = self._run_ok( + "semantic-layer", + "edit", + "metric", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_m_rev", + "--new-name", + f"{tag}_m_revenue", + "--yes", + ) + new_metric_id = data["data"]["updated"]["id"] + # Replace tracking: the old metric was DELETE+POSTed + created_items = [ + (t, i) + for (t, i) in created_items + if not (t == "semantic-metric" and i == m1["data"]["id"]) + ] + created_items.append(("semantic-metric", new_metric_id)) + cascaded = data["data"]["cascaded_constraints"] + assert any(c["status"] == "updated" for c in cascaded), ( + f"Expected at least one cascaded constraint, got: {cascaded}" + ) + # The constraint id changed (DELETE+POST). Re-fetch the list. + data = self._run_ok( + "semantic-layer", + "show", + "--project", + self.alias, + "--model", + model_name, + "--type", + "constraint", + ) + current_constraints = {c["id"] for c in data["data"]["constraints"]} + # Remove the old constraint id from tracking; add the live ones. + created_items = [(t, i) for (t, i) in created_items if t != "semantic-constraint"] + for cid in current_constraints: + created_items.append(("semantic-constraint", cid)) + + # 7. export to a tmp file + tmpdir = self.work_dir / "sl_export" + tmpdir.mkdir(exist_ok=True) + export_path = tmpdir / "snapshot.json" + data = self._run_ok( + "semantic-layer", + "export", + "--project", + self.alias, + "--model", + model_name, + "--output", + str(export_path), + ) + assert export_path.is_file() + + # 8. import --dry-run from the same file — all conflicts should skip + data = self._run_ok( + "semantic-layer", + "import", + "--project", + self.alias, + "--model", + model_name, + "--file", + str(export_path), + "--dry-run", + ) + imported = data["data"]["imported"] + # Every type already exists → at least one skip somewhere. + total_skipped = sum(per.get("skipped", 0) for per in imported.values()) + total_created = sum(per.get("created", 0) for per in imported.values()) + assert total_skipped > 0, f"Expected skips on import-into-self, got: {imported}" + assert total_created == 0, ( + f"Expected zero creations on dry-run import-into-self, got {total_created}: {imported}" + ) + + # 9. diff project vs exported file → zero diff (modulo modelUUID strip) + data = self._run_ok( + "semantic-layer", + "diff", + "--project-a", + self.alias, + "--model-a", + model_name, + "--file-b", + str(export_path), + ) + for type_key in ("datasets", "metrics", "relationships", "constraints", "glossary"): + per = data["data"][type_key] + assert per["added"] == [] and per["removed"] == [] and per["changed"] == [], ( + f"Live model and just-exported file should match for {type_key}: {per}" + ) + + # 10. promote — bootstrap second model and copy into it + data = self._run_ok( + "semantic-layer", + "model", + "create", + "--project", + self.alias, + "--name", + target_model_name, + ) + target_model_id = data["data"]["model"]["id"] + + data = self._run_ok( + "semantic-layer", + "promote", + "--from-project", + self.alias, + "--to-project", + self.alias, + "--from-model", + model_name, + "--to-model", + target_model_name, + "--dry-run", + ) + # Every source item should be NEW in the empty target (dry-run). + for type_key in ("datasets", "metrics", "relationships", "constraints", "glossary"): + per = data["data"].get(type_key) + if per is not None: + assert per["new"] > 0 or per["overwritten"] > 0 or per["identical"] >= 0, ( + f"promote stats look wrong for {type_key}: {per}" + ) + + # 11. build --dry-run — non-interactive heuristic; uses a real + # storage table for schema fetch. Re-use the same RUN_ID table. + data = self._run_ok( + "semantic-layer", + "build", + "--project", + self.alias, + "--tables", + f"in.c-{RUN_ID.replace('-', '_')}.{RUN_ID.replace('-', '_')}", + "--dry-run", + ) + assert data["data"]["fallback_used"] == "heuristic", ( + f"Expected heuristic fallback, got: {data['data'].get('fallback_used')}" + ) + + # 12. token --encrypt + data = self._run_ok( + "semantic-layer", + "token", + "--encrypt", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + ) + envelope = data["data"]["encrypted"] + assert "#metastore_token" in envelope + assert envelope["#metastore_token"].startswith("KBC::"), ( + f"Expected ciphertext to start with KBC::, got: {envelope['#metastore_token'][:30]}..." + ) + + # 13. remove a single metric (--yes), then verify it's gone + data = self._run_ok( + "semantic-layer", + "remove", + "metric", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_m_count_b", + "--yes", + ) + removed_id = data["data"]["removed"]["id"] + assert removed_id == m3["data"]["id"] + created_items = [ + (t, i) + for (t, i) in created_items + if not (t == "semantic-metric" and i == m3["data"]["id"]) + ] + + # Verify it's gone via show + data = self._run_ok( + "semantic-layer", + "show", + "--project", + self.alias, + "--model", + model_name, + "--type", + "metric", + ) + metric_names = {m["name"] for m in data["data"]["metrics"]} + assert f"{tag}_m_count_b" not in metric_names + + finally: + # ---------------------------------------------------------------- + # Teardown — best-effort, runs even on test failure. + # Reverse order: child items first, then both models. + # ---------------------------------------------------------------- + print("\n--- SEMANTIC LAYER CLEANUP ---") + for item_type, item_id in reversed(created_items): + try: + _direct_delete(item_type, item_id) + print(f" Deleted {item_type} {item_id}") + except Exception as exc: + print(f" WARN: failed to delete {item_type} {item_id}: {exc}") + + for mid in (target_model_id, model_id): + if mid is None: + continue + try: + _direct_delete("semantic-model", mid) + print(f" Deleted semantic-model {mid}") + except Exception as exc: + print(f" WARN: failed to delete semantic-model {mid}: {exc}") + + # Residue check: assert no tagged items remain in the project. + try: + with MetastoreClient(stack_url=self.url, token=self.token) as mc: + residue: list[str] = [] + for stype in SEMANTIC_TYPES: + for item in mc.list_items(stype): + attrs = item.get("attributes") or {} + name = attrs.get("name") or attrs.get("term", "") + if isinstance(name, str) and name.startswith(tag): + residue.append(f"{stype}:{name}:{item.get('id', '')}") + if residue: + print(f" WARN: residue detected after cleanup: {residue}") + except Exception as exc: + print(f" WARN: residue scan failed: {exc}") + def _test_project_edit_and_remove(self) -> None: """Edit project URL, rename round-trip + dry-run preview, then remove.""" # --new-alias dry-run preview: predicts the rename without mutating. @@ -7636,3 +8100,598 @@ def test_config_oauth_url_master_token_gate(self) -> None: assert result.exit_code == 3 assert envelope["error"]["code"] == "MISSING_MASTER_TOKEN" assert "master" in envelope["error"]["message"].lower() + + +# --------------------------------------------------------------------------- +# Semantic-layer (since v0.41.0) +# --------------------------------------------------------------------------- + + +@skip_without_credentials +@pytest.mark.e2e +class TestE2ESemanticLayerLifecycle: + """E2E coverage for the ``kbagent semantic-layer`` command group. + + Lives apart from the giant TestFullE2E because the SL surface is large + enough to deserve its own focused test (and because the round-trip + bootstraps two throwaway models + many children per run, so isolation + keeps cleanup tight and lets ``pytest -k SemanticLayer`` iterate fast). + + Covers (in order): + - ``model create`` / ``delete`` + - ``add dataset`` / ``add metric`` / ``add relationship`` / + ``add constraint`` / ``add glossary`` + - ``show`` (default + ``--type`` filter) + - ``validate`` (basic; ``--deep`` is skipped because the test tableIds + are synthetic and would 404 on a Storage table-detail fetch) + - ``edit metric --new-name`` (constraint cascade) + - ``edit relationship`` / ``edit glossary`` (NB-5, iter-4 expansion) + - ``export`` to a tmp file + - ``import --dry-run`` of the just-exported snapshot (must be all-skip) + - ``diff`` between live model and exported file (must be empty) + - ``promote --dry-run`` between two models in the same project + - ``build --dry-run`` (heuristic fallback, real storage schema fetch) + - ``token --encrypt`` (envelope shape) + - ``remove metric`` (single child removal) + - ``remove relationship`` / ``remove glossary`` (NB-5, iter-4 expansion) + + Teardown is double-belted: + 1. Every CLI-created item is tracked and direct-deleted via + ``MetastoreClient`` in a try/finally. + 2. A final residue scan asserts no ``kbagent_e2e_*`` items remain across + all six metastore types -- if any do, the test fails (so silent + cleanup bugs surface immediately). + """ + + @pytest.fixture(autouse=True) + def setup(self, tmp_path: Path) -> Any: + if not HAS_CREDENTIALS: + pytest.skip("E2E_API_TOKEN not set") + self.token = os.environ[ENV_TOKEN] + raw_url = os.environ.get(ENV_URL, "connection.keboola.com") + self.url = raw_url if raw_url.startswith("https://") else f"https://{raw_url}" + self.alias = f"{RUN_ID}-sl-proj" + self.config_dir = tmp_path / "config" + self.config_dir.mkdir() + self.work_dir = tmp_path / "work" + self.work_dir.mkdir() + # Register the project so `kbagent --project ALIAS ...` works. + _invoke( + self.config_dir, + [ + "project", + "add", + "--project", + self.alias, + "--url", + self.url, + "--token", + self.token, + ], + ) + + def _run(self, *args: str) -> Any: + return _invoke(self.config_dir, ["--json", *args]) + + def _run_ok(self, *args: str) -> dict[str, Any]: + return _json_ok(self._run(*args)) + + def test_semantic_layer_roundtrip(self) -> None: + """Exercise every semantic-layer verb in one bootstrap → teardown cycle.""" + from keboola_agent_cli.metastore_client import ( + SEMANTIC_TYPES, + MetastoreClient, + ) + + tag = f"kbagent_e2e_{int(time.time())}" + model_name = tag + target_model_name = f"{tag}_target" + + created_items: list[tuple[str, str]] = [] + model_id: str | None = None + target_model_id: str | None = None + + def _direct_delete(item_type: str, item_id: str) -> None: + with MetastoreClient(stack_url=self.url, token=self.token) as mc: + mc.delete_item(item_type, item_id) # type: ignore[arg-type] + + try: + _step(1, "semantic-layer model create") + data = self._run_ok( + "semantic-layer", + "model", + "create", + "--project", + self.alias, + "--name", + model_name, + ) + model_id = data["data"]["model"]["id"] + assert model_id + + _step(2, "add datasets / metrics / relationship / constraint / glossary") + ds1 = self._run_ok( + "semantic-layer", + "add", + "dataset", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_ds_a", + "--table-id", + "out.c-syn.fact_a", + ) + created_items.append(("semantic-dataset", ds1["data"]["id"])) + + ds2 = self._run_ok( + "semantic-layer", + "add", + "dataset", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_ds_b", + "--table-id", + "out.c-syn.fact_b", + ) + created_items.append(("semantic-dataset", ds2["data"]["id"])) + + m1 = self._run_ok( + "semantic-layer", + "add", + "metric", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_m_rev", + "--sql", + "COUNT(*)", + "--dataset", + "out.c-syn.fact_a", + "--yes", + ) + created_items.append(("semantic-metric", m1["data"]["id"])) + + m2 = self._run_ok( + "semantic-layer", + "add", + "metric", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_m_cost", + "--sql", + 'SUM("schema"."AMOUNT")', + "--dataset", + "out.c-syn.fact_a", + "--yes", + ) + created_items.append(("semantic-metric", m2["data"]["id"])) + + m3 = self._run_ok( + "semantic-layer", + "add", + "metric", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_m_count_b", + "--sql", + "COUNT(*)", + "--dataset", + "out.c-syn.fact_b", + "--yes", + ) + created_items.append(("semantic-metric", m3["data"]["id"])) + + rel = self._run_ok( + "semantic-layer", + "add", + "relationship", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_rel_a_b", + "--from", + "out.c-syn.fact_a", + "--to", + "out.c-syn.fact_b", + "--on", + "fact_a.id = fact_b.fact_a_id", + ) + created_items.append(("semantic-relationship", rel["data"]["id"])) + + cons = self._run_ok( + "semantic-layer", + "add", + "constraint", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_rev_warning", + "--constraint-type", + "inequality", + "--rule", + "value >= 0", + "--metrics", + f"{tag}_m_rev", + "--severity", + "warning", + ) + created_items.append(("semantic-constraint", cons["data"]["id"])) + + gloss = self._run_ok( + "semantic-layer", + "add", + "glossary", + "--project", + self.alias, + "--model", + model_name, + "--term", + f"{tag}_GMV", + "--definition", + "Gross merchandise value (test)", + ) + created_items.append(("semantic-glossary", gloss["data"]["id"])) + + _step(3, "show + show --type metric") + data = self._run_ok( + "semantic-layer", + "show", + "--project", + self.alias, + "--model", + model_name, + ) + assert len(data["data"]["datasets"]) == 2 + assert len(data["data"]["metrics"]) == 3 + assert len(data["data"]["constraints"]) == 1 + assert len(data["data"]["glossary"]) == 1 + + data = self._run_ok( + "semantic-layer", + "show", + "--project", + self.alias, + "--model", + model_name, + "--type", + "metric", + ) + assert len(data["data"]["metrics"]) >= 3 + + _step(4, "validate (basic) -- expect valid") + data = self._run_ok( + "semantic-layer", + "validate", + "--project", + self.alias, + "--model", + model_name, + ) + assert data["data"]["valid"] is True, ( + f"Expected clean model, got errors: {data['data']['errors']}" + ) + + _step(5, "edit metric --new-name -- triggers constraint cascade") + data = self._run_ok( + "semantic-layer", + "edit", + "metric", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_m_rev", + "--new-name", + f"{tag}_m_revenue", + "--yes", + ) + new_metric_id = data["data"]["updated"]["id"] + created_items = [ + (t, i) + for (t, i) in created_items + if not (t == "semantic-metric" and i == m1["data"]["id"]) + ] + created_items.append(("semantic-metric", new_metric_id)) + cascaded = data["data"]["cascaded_constraints"] + assert any(c["status"] == "updated" for c in cascaded), ( + f"Expected constraint cascade, got: {cascaded}" + ) + # DELETE+POST changed the constraint id -- refresh tracking + data = self._run_ok( + "semantic-layer", + "show", + "--project", + self.alias, + "--model", + model_name, + "--type", + "constraint", + ) + created_items = [(t, i) for (t, i) in created_items if t != "semantic-constraint"] + for c in data["data"]["constraints"]: + created_items.append(("semantic-constraint", c["id"])) + + # ---------- NB-5: edit + remove relationship / glossary ---------- + + _step(5.1, "edit relationship --new-on -- DELETE+POST") + data = self._run_ok( + "semantic-layer", + "edit", + "relationship", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_rel_a_b", + "--new-on", + "fact_a.id = fact_b.fact_a_id_v2", + ) + new_rel_id = data["data"]["updated"]["id"] + created_items = [ + (t, i) + for (t, i) in created_items + if not (t == "semantic-relationship" and i == rel["data"]["id"]) + ] + created_items.append(("semantic-relationship", new_rel_id)) + + _step(5.2, "edit glossary --new-definition -- DELETE+POST") + data = self._run_ok( + "semantic-layer", + "edit", + "glossary", + "--project", + self.alias, + "--model", + model_name, + "--term", + f"{tag}_GMV", + "--new-definition", + "Gross merchandise value (test, v2)", + ) + new_gloss_id = data["data"]["updated"]["id"] + created_items = [ + (t, i) + for (t, i) in created_items + if not (t == "semantic-glossary" and i == gloss["data"]["id"]) + ] + created_items.append(("semantic-glossary", new_gloss_id)) + + _step(6, "export -> snapshot.json") + export_path = self.work_dir / "snapshot.json" + self._run_ok( + "semantic-layer", + "export", + "--project", + self.alias, + "--model", + model_name, + "--output", + str(export_path), + ) + assert export_path.is_file() + + _step(7, "import --dry-run from the same file -- all-skip expected") + data = self._run_ok( + "semantic-layer", + "import", + "--project", + self.alias, + "--model", + model_name, + "--file", + str(export_path), + "--dry-run", + ) + imported = data["data"]["imported"] + total_skipped = sum(per.get("skipped", 0) for per in imported.values()) + total_created = sum(per.get("created", 0) for per in imported.values()) + assert total_skipped > 0, f"Expected skips on self-import, got: {imported}" + assert total_created == 0, ( + f"Expected zero creates on self-import dry-run, got {total_created}" + ) + + _step(8, "diff project vs snapshot -- empty diff expected") + data = self._run_ok( + "semantic-layer", + "diff", + "--project-a", + self.alias, + "--model-a", + model_name, + "--file-b", + str(export_path), + ) + for type_key in ("datasets", "metrics", "relationships", "constraints", "glossary"): + per = data["data"][type_key] + assert per["added"] == [] and per["removed"] == [] and per["changed"] == [], ( + f"Self-diff should be empty for {type_key}: {per}" + ) + + _step(9, "promote --dry-run into a fresh target model") + data = self._run_ok( + "semantic-layer", + "model", + "create", + "--project", + self.alias, + "--name", + target_model_name, + ) + target_model_id = data["data"]["model"]["id"] + data = self._run_ok( + "semantic-layer", + "promote", + "--from-project", + self.alias, + "--to-project", + self.alias, + "--from-model", + model_name, + "--to-model", + target_model_name, + "--dry-run", + ) + # Every per-type stats block exists + for type_key in ("datasets", "metrics", "relationships", "constraints", "glossary"): + assert type_key in data["data"], ( + f"promote dry-run missing {type_key} in stats: {list(data['data'].keys())}" + ) + + _step(10, "build --dry-run -- heuristic fallback against a real storage table") + # Discover a real table from this project's buckets so build's + # storage-schema fetch succeeds. Skip the step if the project + # has no tables (uncommon for e2e-1143, but defensive). + tables_data = self._run_ok("storage", "tables", "--project", self.alias) + available_tables = tables_data["data"].get("tables", []) + if available_tables: + table_id = available_tables[0]["id"] + data = self._run_ok( + "semantic-layer", + "build", + "--project", + self.alias, + "--tables", + table_id, + "--dry-run", + ) + assert data["data"]["fallback_used"] == "heuristic", ( + f"Expected heuristic fallback, got: {data['data'].get('fallback_used')}" + ) + assert len(data["data"]["generated"]["datasets"]) == 1 + else: + print(" WARN: no storage tables in project -- build --dry-run skipped") + + _step(11, "token --encrypt -- envelope shape") + data = self._run_ok( + "semantic-layer", + "token", + "--encrypt", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + ) + envelope = data["data"]["encrypted"] + assert "#metastore_token" in envelope + assert envelope["#metastore_token"].startswith("KBC::"), ( + f"Expected KBC:: ciphertext, got: {envelope['#metastore_token'][:30]}..." + ) + + _step(12, "remove metric -- single-child removal + verify gone") + data = self._run_ok( + "semantic-layer", + "remove", + "metric", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_m_count_b", + "--yes", + ) + assert data["data"]["removed"]["id"] == m3["data"]["id"] + created_items = [ + (t, i) + for (t, i) in created_items + if not (t == "semantic-metric" and i == m3["data"]["id"]) + ] + data = self._run_ok( + "semantic-layer", + "show", + "--project", + self.alias, + "--model", + model_name, + "--type", + "metric", + ) + metric_names = {m["name"] for m in data["data"]["metrics"]} + assert f"{tag}_m_count_b" not in metric_names + + _step(12.1, "remove relationship -- leaf entity, no orphan") + data = self._run_ok( + "semantic-layer", + "remove", + "relationship", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_rel_a_b", + "--yes", + ) + assert data["data"]["removed"]["name"] == f"{tag}_rel_a_b" + created_items = [(t, i) for (t, i) in created_items if t != "semantic-relationship"] + + _step(12.2, "remove glossary --term -- leaf entity, no orphan") + data = self._run_ok( + "semantic-layer", + "remove", + "glossary", + "--project", + self.alias, + "--model", + model_name, + "--term", + f"{tag}_GMV", + "--yes", + ) + assert data["data"]["removed"]["name"] == f"{tag}_GMV" + created_items = [(t, i) for (t, i) in created_items if t != "semantic-glossary"] + + finally: + print("\n--- SEMANTIC LAYER CLEANUP ---") + for item_type, item_id in reversed(created_items): + try: + _direct_delete(item_type, item_id) + print(f" Deleted {item_type} {item_id}") + except Exception as exc: + print(f" WARN: failed to delete {item_type} {item_id}: {exc}") + + for mid in (target_model_id, model_id): + if mid is None: + continue + try: + _direct_delete("semantic-model", mid) + print(f" Deleted semantic-model {mid}") + except Exception as exc: + print(f" WARN: failed to delete semantic-model {mid}: {exc}") + + # Residue check -- assert teardown actually cleaned up. + from keboola_agent_cli.errors import KeboolaApiError as _ApiError + + try: + with MetastoreClient(stack_url=self.url, token=self.token) as mc: + residue: list[str] = [] + for stype in SEMANTIC_TYPES: + for item in mc.list_items(stype): # type: ignore[arg-type] + attrs = item.get("attributes") or {} + name = attrs.get("name") or attrs.get("term", "") + if isinstance(name, str) and name.startswith(tag): + residue.append(f"{stype}:{name}:{item.get('id', '')}") + assert not residue, ( + f"Cleanup left residue (manual teardown required): {residue}" + ) + except _ApiError as exc: + print(f" WARN: residue scan failed: {exc}") diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py new file mode 100644 index 00000000..d81864b1 --- /dev/null +++ b/tests/test_metastore_client.py @@ -0,0 +1,235 @@ +"""Tests for MetastoreClient -- URL derivation, envelope shape, error normalization. + +Mirrors the test_ai_client.py pattern: drive the client through pytest-httpx +mocks and verify the verb-level contract (URL derivation, request envelope, +the "duplicate name -> 500" normalization to ALREADY_EXISTS). +""" + +from __future__ import annotations + +import json + +import pytest + +from keboola_agent_cli.constants import MAX_RETRIES +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.metastore_client import ( + SEMANTIC_TYPES, + MetastoreClient, +) + +STACK_URL_US = "https://connection.keboola.com" +METASTORE_URL_US = "https://metastore.keboola.com" +TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" + + +@pytest.fixture(autouse=True) +def _no_backoff_sleep(monkeypatch: pytest.MonkeyPatch) -> None: + """Disable retry-backoff sleeps so the suite stays fast.""" + import keboola_agent_cli.http_base as http_base_module + + monkeypatch.setattr(http_base_module.time, "sleep", lambda _x: None) + + +class TestUrlDerivation: + """Verify MetastoreClient maps ``connection.<host>`` to ``metastore.<host>``.""" + + def test_us_stack(self) -> None: + result = MetastoreClient._derive_service_url("https://connection.keboola.com", "metastore") + assert result == "https://metastore.keboola.com" + + def test_eu_gcp_stack(self) -> None: + result = MetastoreClient._derive_service_url( + "https://connection.europe-west3.gcp.keboola.com", "metastore" + ) + assert result == "https://metastore.europe-west3.gcp.keboola.com" + + def test_aws_stack(self) -> None: + result = MetastoreClient._derive_service_url( + "https://connection.eu-west-1.aws.keboola.com", "metastore" + ) + assert result == "https://metastore.eu-west-1.aws.keboola.com" + + def test_azure_stack(self) -> None: + result = MetastoreClient._derive_service_url( + "https://connection.westeurope.azure.keboola.com", "metastore" + ) + assert result == "https://metastore.westeurope.azure.keboola.com" + + +class TestAuthHeader: + """Verify X-StorageApi-Token is sent on every request.""" + + def test_token_header_set_on_get(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-model", + json={"data": []}, + status_code=200, + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TOKEN) + try: + client.list_items("semantic-model") + finally: + client.close() + request = httpx_mock.get_requests()[0] + assert request.headers["X-StorageApi-Token"] == TOKEN + assert "keboola-agent-cli/" in request.headers["User-Agent"] + + +class TestListItems: + """list_items returns raw item shapes and supports model_uuid filtering.""" + + def test_list_items_returns_data_array(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset", + json={ + "data": [ + {"type": "semantic-dataset", "id": "a1", "attributes": {"name": "x"}}, + {"type": "semantic-dataset", "id": "a2", "attributes": {"name": "y"}}, + ] + }, + status_code=200, + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TOKEN) + try: + items = client.list_items("semantic-dataset") + finally: + client.close() + assert len(items) == 2 + assert items[0]["id"] == "a1" + + def test_list_items_filter_by_model_uuid(self, httpx_mock) -> None: + """Client-side filter on attributes.modelUUID.""" + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", + json={ + "data": [ + {"id": "m1", "attributes": {"name": "a", "modelUUID": "U1"}}, + {"id": "m2", "attributes": {"name": "b", "modelUUID": "U2"}}, + {"id": "m3", "attributes": {"name": "c", "modelUUID": "U1"}}, + ] + }, + status_code=200, + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TOKEN) + try: + items = client.list_items("semantic-metric", model_uuid="U1") + finally: + client.close() + assert {i["id"] for i in items} == {"m1", "m3"} + + +class TestPostItem: + """post_item must wrap payload in the {name, data, branch, schemaVersion, scope} envelope.""" + + def test_post_envelope(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", + json={ + "data": { + "type": "semantic-metric", + "id": "new-id", + "attributes": {"name": "rev", "sql": "SUM(x)"}, + } + }, + status_code=201, + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TOKEN) + try: + stored = client.post_item( + "semantic-metric", + name="rev", + data={"name": "rev", "sql": "SUM(x)", "modelUUID": "u"}, + ) + finally: + client.close() + assert stored["id"] == "new-id" + + request = httpx_mock.get_requests()[0] + body = json.loads(request.content) + assert body["name"] == "rev" + assert body["branch"] == "main" + assert body["schemaVersion"] == "1.0.0" + assert body["scope"] == "project" + assert body["data"]["sql"] == "SUM(x)" + assert body["data"]["modelUUID"] == "u" + + +class TestDuplicateNameNormalization: + """Server returns 500 for duplicate names; client normalizes to ALREADY_EXISTS.""" + + def test_duplicate_name_500_becomes_already_exists(self, httpx_mock) -> None: + for _ in range(MAX_RETRIES): + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", + status_code=500, + json={"error": "Failed to create meta object: duplicate name 'foo'"}, + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TOKEN) + try: + with pytest.raises(KeboolaApiError) as excinfo: + client.post_item("semantic-metric", name="foo", data={"name": "foo"}) + finally: + client.close() + assert excinfo.value.error_code == ErrorCode.ALREADY_EXISTS + assert "already exists" in excinfo.value.message + assert "foo" in excinfo.value.message + + def test_unrelated_500_passes_through(self, httpx_mock) -> None: + """A 500 without the magic phrase keeps its API_ERROR code.""" + for _ in range(MAX_RETRIES): + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", + status_code=500, + json={"error": "some unrelated internal error"}, + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TOKEN) + try: + with pytest.raises(KeboolaApiError) as excinfo: + client.post_item("semantic-metric", name="foo", data={"name": "foo"}) + finally: + client.close() + assert excinfo.value.error_code != ErrorCode.ALREADY_EXISTS + + +class TestDeleteItem: + """delete_item returns silently on 204 and raises NOT_FOUND on 404.""" + + def test_delete_204(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc", + status_code=204, + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TOKEN) + try: + assert client.delete_item("semantic-dataset", "abc") is None + finally: + client.close() + + def test_delete_404(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/missing", + status_code=404, + json={"error": "not found"}, + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TOKEN) + try: + with pytest.raises(KeboolaApiError) as excinfo: + client.delete_item("semantic-dataset", "missing") + finally: + client.close() + assert excinfo.value.error_code == ErrorCode.NOT_FOUND + + +class TestSemanticTypes: + """Sanity-check the SEMANTIC_TYPES tuple has the six expected slugs.""" + + def test_semantic_types_complete(self) -> None: + assert set(SEMANTIC_TYPES) == { + "semantic-model", + "semantic-dataset", + "semantic-metric", + "semantic-relationship", + "semantic-constraint", + "semantic-glossary", + } diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 0dc4aa78..536600b4 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -312,18 +312,39 @@ def test_all_subapp_commands_registered(self) -> None: click_app = typer.main.get_command(cli_module.app) - missing = [] + missing: list[str] = [] + + def _walk(prefix: str, group: object) -> None: + """Recurse into Click sub-apps, building dotted operation keys. + + Each leaf must be covered by EITHER its own entry + (``semantic-layer.model.list``) OR a parent prefix entry + (``semantic-layer.add`` covers ``add metric/dataset/...``). The + parent-prefix form is correct only when every leaf shares the + same risk classification. + """ + cmds = getattr(group, "commands", None) + if not cmds: + # Leaf command — accept either its own key or any ancestor. + if prefix in OPERATION_REGISTRY: + return + parts = prefix.split(".") + for i in range(len(parts) - 1, 0, -1): + if ".".join(parts[:i]) in OPERATION_REGISTRY: + return + missing.append(prefix) + return + for cmd_name, cmd in cmds.items(): + # Skip hidden aliases (e.g. `sl` for `semantic-layer`). + if getattr(cmd, "hidden", False): + continue + op = f"{prefix}.{cmd_name}" if prefix else cmd_name + _walk(op, cmd) + for group_name, group_cmd in click_app.commands.items(): - if hasattr(group_cmd, "commands"): - # It's a sub-app (Click Group) - for cmd_name in group_cmd.commands: - op = f"{group_name}.{cmd_name}" - if op not in OPERATION_REGISTRY: - missing.append(op) - else: - # Top-level command - if group_name not in OPERATION_REGISTRY: - missing.append(group_name) + if getattr(group_cmd, "hidden", False): + continue + _walk(group_name, group_cmd) assert missing == [], ( f"Commands missing from OPERATION_REGISTRY: {missing}. " diff --git a/tests/test_semantic_layer_cli.py b/tests/test_semantic_layer_cli.py new file mode 100644 index 00000000..c6b13920 --- /dev/null +++ b/tests/test_semantic_layer_cli.py @@ -0,0 +1,1471 @@ +"""CLI-layer tests for the ``semantic-layer`` command group via CliRunner. + +Mirrors the test_data_app_cli.py pattern: patch the cli.py service factory +so the runner sees a MagicMock; assert exit codes, JSON envelopes, and +the mutual-exclusion / permission-denied branches. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.constants import EXIT_PERMISSION_DENIED +from keboola_agent_cli.errors import ConfigError, ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.config_service import ConfigService +from keboola_agent_cli.services.job_service import JobService +from keboola_agent_cli.services.project_service import ProjectService + +TEST_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" + +runner = CliRunner() + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _setup_config( + config_dir: Path, + projects: dict[str, dict] | None = None, +) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + if projects: + for alias, info in projects.items(): + store.add_project( + alias, + ProjectConfig( + stack_url=info.get("stack_url", "https://connection.keboola.com"), + token=info["token"], + project_name=info.get("project_name", alias), + project_id=info.get("project_id", 1234), + ), + ) + return store + + +def _invoke( + args: list[str], + *, + store: ConfigStore, + sl_mock: MagicMock, +): + """Run the CLI with cli.py services patched to mocks.""" + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProj, + patch("keboola_agent_cli.cli.ConfigService") as MockCfg, + patch("keboola_agent_cli.cli.JobService") as MockJob, + patch("keboola_agent_cli.cli.SemanticLayerService") as MockSL, + ): + MockStore.return_value = store + MockProj.return_value = ProjectService(config_store=store) + MockCfg.return_value = ConfigService(config_store=store) + MockJob.return_value = JobService(config_store=store) + MockSL.return_value = sl_mock + return runner.invoke(app, args) + + +@pytest.fixture +def cfg_dir(tmp_path: Path) -> Path: + d = tmp_path / "config" + d.mkdir() + return d + + +@pytest.fixture +def store(cfg_dir: Path) -> ConfigStore: + return _setup_config(cfg_dir, {"prod": {"token": TEST_TOKEN}}) + + +# --------------------------------------------------------------------------- +# semantic-layer model list +# --------------------------------------------------------------------------- + + +class TestModelList: + def test_json_success(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.list_models.return_value = { + "project": "prod", + "models": [ + {"id": "U1", "name": "default", "description": "", "sql_dialect": "Snowflake"} + ], + } + result = _invoke( + ["--json", "semantic-layer", "model", "list", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["status"] == "ok" + assert body["data"]["models"][0]["id"] == "U1" + + def test_human_empty(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.list_models.return_value = {"project": "prod", "models": []} + result = _invoke( + ["semantic-layer", "model", "list", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + assert "No semantic-layer models" in result.output + + def test_config_error_exits_5(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.list_models.side_effect = ConfigError("Project 'ghost' not found.") + result = _invoke( + ["--json", "semantic-layer", "model", "list", "--project", "ghost"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 5, result.output + body = json.loads(result.output) + assert body["error"]["code"] == "CONFIG_ERROR" + + def test_api_error_invalid_token_exits_3(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.list_models.side_effect = KeboolaApiError( + message="bad token", status_code=401, error_code="INVALID_TOKEN" + ) + result = _invoke( + ["--json", "semantic-layer", "model", "list", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 3 + body = json.loads(result.output) + assert body["error"]["code"] == "INVALID_TOKEN" + + def test_missing_project_arg_exit_2(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + ["--json", "semantic-layer", "model", "list"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2 + + +# --------------------------------------------------------------------------- +# semantic-layer model create +# --------------------------------------------------------------------------- + + +class TestModelCreate: + def test_create_success(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.create_model.return_value = { + "project": "prod", + "model": { + "id": "new-uuid", + "attributes": {"name": "default", "sql_dialect": "Snowflake"}, + }, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "model", + "create", + "--project", + "prod", + "--name", + "default", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["model"]["id"] == "new-uuid" + + def test_create_general_api_error_exits_1(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.create_model.side_effect = KeboolaApiError( + message="boom", status_code=500, error_code=ErrorCode.API_ERROR + ) + result = _invoke( + [ + "--json", + "semantic-layer", + "model", + "create", + "--project", + "prod", + "--name", + "x", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 1 + + +# --------------------------------------------------------------------------- +# semantic-layer model delete (with --yes) +# --------------------------------------------------------------------------- + + +class TestModelDelete: + def test_delete_yes_success(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.delete_model.return_value = { + "project": "prod", + "deleted": {"id": "u1", "name": "default"}, + "orphaned_children": {}, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "model", + "delete", + "--project", + "prod", + "--model", + "default", + "--yes", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["deleted"]["id"] == "u1" + + +# --------------------------------------------------------------------------- +# semantic-layer show +# --------------------------------------------------------------------------- + + +class TestShow: + def test_show_summary_json(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.show_model.return_value = { + "project": "prod", + "model": {"id": "u1", "name": "default"}, + "datasets": [], + "metrics": [], + "relationships": [], + "constraints": [], + "glossary": [], + } + result = _invoke( + ["--json", "semantic-layer", "show", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["model"]["id"] == "u1" + + def test_show_with_type_filter(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.show_model.return_value = { + "project": "prod", + "model": {"id": "u1", "name": "default"}, + "metrics": [{"name": "rev", "id": "m1"}], + } + result = _invoke( + [ + "--json", + "semantic-layer", + "show", + "--project", + "prod", + "--type", + "metric", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert "metrics" in body["data"] + assert "datasets" not in body["data"] + + def test_show_human_renders_table(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.show_model.return_value = { + "project": "prod", + "model": {"id": "u1", "name": "default"}, + "datasets": [], + "metrics": [], + "relationships": [], + "constraints": [], + "glossary": [], + } + result = _invoke( + ["semantic-layer", "show", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0 + # Some kind of header / table marker + assert "default" in result.output + + +# --------------------------------------------------------------------------- +# semantic-layer validate +# --------------------------------------------------------------------------- + + +class TestValidate: + def test_validate_clean(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.validate_model.return_value = { + "project": "prod", + "model": {"id": "u1", "name": "default"}, + "errors": [], + "warnings": [], + "deep": False, + "valid": True, + } + result = _invoke( + ["--json", "semantic-layer", "validate", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["valid"] is True + + def test_validate_deep_flag_propagates(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.validate_model.return_value = { + "project": "prod", + "model": {"id": "u1", "name": "default"}, + "errors": [], + "warnings": [], + "deep": True, + "valid": True, + } + result = _invoke( + ["--json", "semantic-layer", "validate", "--project", "prod", "--deep"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0 + # Verify deep=True propagated to service + _, kwargs = mock.validate_model.call_args + assert kwargs["deep"] is True + + +# --------------------------------------------------------------------------- +# semantic-layer export +# --------------------------------------------------------------------------- + + +class TestExport: + def test_export_json(self, store: ConfigStore, tmp_path: Path) -> None: + mock = MagicMock() + out = tmp_path / "snap.json" + mock.export_model.return_value = { + "exported_at": "2026-05-14T00:00:00Z", + "project": "prod", + "model": {"id": "u1", "name": "x"}, + "datasets": [], + "metrics": [], + "relationships": [], + "constraints": [], + "glossary": [], + "counts": { + "datasets": 0, + "metrics": 0, + "relationships": 0, + "constraints": 0, + "glossary": 0, + }, + "path": str(out), + } + result = _invoke( + [ + "--json", + "semantic-layer", + "export", + "--project", + "prod", + "--output", + str(out), + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["path"].endswith("snap.json") + + +# --------------------------------------------------------------------------- +# semantic-layer diff +# --------------------------------------------------------------------------- + + +class TestDiff: + def test_diff_requires_one_of_project_or_file_for_each_side(self, store: ConfigStore) -> None: + mock = MagicMock() + # No --project-a or --file-a + result = _invoke( + [ + "--json", + "semantic-layer", + "diff", + "--project-b", + "prod", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2 + body = json.loads(result.output) + assert body["error"]["code"] == "USAGE_ERROR" + mock.diff.assert_not_called() + + def test_diff_both_project_a_and_file_a_rejected(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--json", + "semantic-layer", + "diff", + "--project-a", + "prod", + "--file-a", + "snap.json", + "--project-b", + "prod", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2 + + def test_diff_happy_path(self, store: ConfigStore, tmp_path: Path) -> None: + mock = MagicMock() + mock.diff.return_value = { + "left": {"source": "project", "ref": "prod", "model": {}}, + "right": {"source": "file", "ref": "x.json", "model": {}}, + "datasets": {"added": [], "removed": [], "changed": []}, + "metrics": {"added": [], "removed": [], "changed": []}, + "relationships": {"added": [], "removed": [], "changed": []}, + "constraints": {"added": [], "removed": [], "changed": []}, + "glossary": {"added": [], "removed": [], "changed": []}, + } + f = tmp_path / "x.json" + f.write_text("{}") + result = _invoke( + [ + "--json", + "semantic-layer", + "diff", + "--project-a", + "prod", + "--file-b", + str(f), + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + + +# --------------------------------------------------------------------------- +# semantic-layer add metric|dataset|relationship|constraint|glossary +# --------------------------------------------------------------------------- + + +class TestAddMetric: + def test_happy_path(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.add_metric.return_value = { + "id": "m1", + "attributes": {"name": "rev"}, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "add", + "metric", + "--project", + "prod", + "--name", + "rev", + "--sql", + "COUNT(*)", + "--dataset", + "out.c.t", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + + def test_validation_error_exit_1(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.add_metric.side_effect = KeboolaApiError( + message="bad", status_code=400, error_code=ErrorCode.VALIDATION_ERROR + ) + result = _invoke( + [ + "--json", + "semantic-layer", + "add", + "metric", + "--project", + "prod", + "--name", + "rev", + "--sql", + "x", + "--dataset", + "out.c.t", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 1 + + +class TestAddDataset: + def test_happy_path(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.add_dataset.return_value = { + "id": "d1", + "attributes": {"name": "fact_x"}, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "add", + "dataset", + "--project", + "prod", + "--name", + "fact_x", + "--table-id", + "out.c-gold.FACT_X", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + + +class TestAddRelationship: + def test_happy_path(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.add_relationship.return_value = {"id": "r1", "attributes": {"name": "r"}} + result = _invoke( + [ + "--json", + "semantic-layer", + "add", + "relationship", + "--project", + "prod", + "--name", + "r", + "--from", + "out.c.a", + "--to", + "out.c.b", + "--on", + "a.id=b.id", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + + +class TestAddConstraint: + def test_happy_path(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.add_constraint.return_value = {"id": "c1", "attributes": {"name": "rev_warning"}} + result = _invoke( + [ + "--json", + "semantic-layer", + "add", + "constraint", + "--project", + "prod", + "--name", + "rev_warning", + "--constraint-type", + "inequality", + "--rule", + "value >= 0", + "--metrics", + "rev", + "--severity", + "warning", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + + def test_empty_metrics_rejected(self, store: ConfigStore) -> None: + """An empty --metrics value must exit 2 with USAGE_ERROR.""" + mock = MagicMock() + result = _invoke( + [ + "--json", + "semantic-layer", + "add", + "constraint", + "--project", + "prod", + "--name", + "rev_warning", + "--constraint-type", + "inequality", + "--rule", + "x", + "--metrics", + ",,", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2 + mock.add_constraint.assert_not_called() + + +class TestAddGlossary: + def test_happy_path(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.add_glossary.return_value = {"id": "g1", "attributes": {"term": "GMV"}} + result = _invoke( + [ + "--json", + "semantic-layer", + "add", + "glossary", + "--project", + "prod", + "--term", + "GMV", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0 + + +# --------------------------------------------------------------------------- +# semantic-layer edit +# --------------------------------------------------------------------------- + + +class TestEditMetric: + def test_rename_with_yes(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.edit_metric.return_value = { + "updated": {"id": "m_new", "attributes": {"name": "revenue"}}, + "cascaded_constraints": [], + "rollback": None, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "edit", + "metric", + "--project", + "prod", + "--name", + "rev", + "--new-name", + "revenue", + "--yes", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _, kwargs = mock.edit_metric.call_args + assert kwargs["assume_yes"] is True + + def test_metric_not_found_exit_1(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.edit_metric.side_effect = KeboolaApiError( + message="not found", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + result = _invoke( + [ + "--json", + "semantic-layer", + "edit", + "metric", + "--project", + "prod", + "--name", + "ghost", + "--new-sql", + "1", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 1 + + +class TestEditDataset: + def test_happy_path(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.edit_dataset.return_value = { + "updated": {"id": "d1", "attributes": {"name": "fact_x"}}, + "cascaded_constraints": [], + "rollback": None, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "edit", + "dataset", + "--project", + "prod", + "--name", + "fact_x", + "--new-description", + "updated", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0 + + +class TestEditConstraint: + def test_metrics_list_parsed(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.edit_constraint.return_value = { + "updated": {"id": "c1", "attributes": {"name": "rev_warning"}}, + "cascaded_constraints": [], + "rollback": None, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "edit", + "constraint", + "--project", + "prod", + "--name", + "rev_warning", + "--new-metrics", + "rev, profit", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0 + _, kwargs = mock.edit_constraint.call_args + assert kwargs["new_metrics"] == ["rev", "profit"] + + +# --------------------------------------------------------------------------- +# semantic-layer remove (with orphan warning + prompts) +# --------------------------------------------------------------------------- + + +class TestRemove: + def test_remove_metric_yes_skips_prompt(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.preview_remove.return_value = { + "kind": "metric", + "id": "m1", + "name": "rev", + "orphaned_constraints": [], + } + mock.remove_item.return_value = { + "removed": {"type": "semantic-metric", "id": "m1", "name": "rev"}, + "orphaned_constraints": [], + } + result = _invoke( + [ + "--json", + "semantic-layer", + "remove", + "metric", + "--project", + "prod", + "--name", + "rev", + "--yes", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + mock.remove_item.assert_called_once() + + def test_remove_without_yes_in_non_tty_exit_2(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.preview_remove.return_value = { + "kind": "metric", + "id": "m1", + "name": "rev", + "orphaned_constraints": [], + } + # Default CliRunner has non-TTY stdin + result = _invoke( + [ + "--json", + "semantic-layer", + "remove", + "metric", + "--project", + "prod", + "--name", + "rev", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2 + mock.remove_item.assert_not_called() + + def test_remove_relationship_yes(self, store: ConfigStore) -> None: + """`remove relationship` is destructive but never orphans (leaf entity).""" + mock = MagicMock() + mock.preview_remove.return_value = { + "kind": "relationship", + "id": "r1", + "name": "fact_to_dim", + "orphaned_constraints": [], + } + mock.remove_item.return_value = { + "removed": {"type": "semantic-relationship", "id": "r1", "name": "fact_to_dim"}, + "orphaned_constraints": [], + } + result = _invoke( + [ + "--json", + "semantic-layer", + "remove", + "relationship", + "--project", + "prod", + "--name", + "fact_to_dim", + "--yes", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _, kwargs = mock.remove_item.call_args + assert kwargs["kind"] == "relationship" + + def test_remove_glossary_yes_uses_term(self, store: ConfigStore) -> None: + """`remove glossary --term ...` -- term is the identity for glossary.""" + mock = MagicMock() + mock.preview_remove.return_value = { + "kind": "glossary", + "id": "g1", + "name": "MRR", + "orphaned_constraints": [], + } + mock.remove_item.return_value = { + "removed": {"type": "semantic-glossary", "id": "g1", "name": "MRR"}, + "orphaned_constraints": [], + } + result = _invoke( + [ + "--json", + "semantic-layer", + "remove", + "glossary", + "--project", + "prod", + "--term", + "MRR", + "--yes", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _, kwargs = mock.remove_item.call_args + assert kwargs["kind"] == "glossary" + assert kwargs["name"] == "MRR" + + +# --------------------------------------------------------------------------- +# semantic-layer edit relationship / glossary (NB-5) +# --------------------------------------------------------------------------- + + +class TestEditRelationship: + def test_edit_relationship_happy(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.edit_relationship.return_value = { + "updated": {"id": "r2", "attributes": {"name": "fact_to_dim"}}, + "cascaded_constraints": [], + "rollback": None, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "edit", + "relationship", + "--project", + "prod", + "--name", + "fact_to_dim", + "--new-from", + "out.c.fact_v2", + "--new-type", + "inner", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _, kwargs = mock.edit_relationship.call_args + assert kwargs["new_from"] == "out.c.fact_v2" + assert kwargs["new_type"] == "inner" + + def test_edit_relationship_propagates_service_error(self, store: ConfigStore) -> None: + """Service-layer VALIDATION_ERROR surfaces as exit 1 with envelope.""" + from keboola_agent_cli.errors import ErrorCode as EC + from keboola_agent_cli.errors import KeboolaApiError + + mock = MagicMock() + mock.edit_relationship.side_effect = KeboolaApiError( + message="bogus type", + error_code=EC.VALIDATION_ERROR, + ) + result = _invoke( + [ + "--json", + "semantic-layer", + "edit", + "relationship", + "--project", + "prod", + "--name", + "x", + "--new-type", + "bogus", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 1 + assert "VALIDATION_ERROR" in result.output + + +class TestEditGlossary: + def test_edit_definition_only(self, store: ConfigStore) -> None: + """Definition-only edit needs no --yes (term identity is preserved).""" + mock = MagicMock() + mock.edit_glossary.return_value = { + "updated": {"id": "g2", "attributes": {"term": "MRR"}}, + "cascaded_constraints": [], + "rollback": None, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "edit", + "glossary", + "--project", + "prod", + "--term", + "MRR", + "--new-definition", + "Updated def", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _, kwargs = mock.edit_glossary.call_args + assert kwargs["new_definition"] == "Updated def" + assert kwargs["new_term"] is None + + def test_edit_rename_term_without_yes_in_non_tty_exit_2(self, store: ConfigStore) -> None: + """Renaming the term is destructive; non-TTY without --yes refuses.""" + mock = MagicMock() + result = _invoke( + [ + "--json", + "semantic-layer", + "edit", + "glossary", + "--project", + "prod", + "--term", + "MRR", + "--new-term", + "RECURRING_REVENUE", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2 + mock.edit_glossary.assert_not_called() + + def test_edit_rename_term_with_yes(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.edit_glossary.return_value = { + "updated": {"id": "g2", "attributes": {"term": "RECURRING_REVENUE"}}, + "cascaded_constraints": [], + "rollback": None, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "edit", + "glossary", + "--project", + "prod", + "--term", + "MRR", + "--new-term", + "RECURRING_REVENUE", + "--yes", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _, kwargs = mock.edit_glossary.call_args + assert kwargs["new_term"] == "RECURRING_REVENUE" + + +# --------------------------------------------------------------------------- +# semantic-layer import +# --------------------------------------------------------------------------- + + +class TestImport: + def test_dry_run(self, store: ConfigStore, tmp_path: Path) -> None: + mock = MagicMock() + mock.import_snapshot.return_value = { + "target_project": "prod", + "target_model": "u1", + "source_model": "src", + "dry_run": True, + "overwrite": False, + "imported": {}, + } + snap = tmp_path / "snap.json" + snap.write_text("{}") + result = _invoke( + [ + "--json", + "semantic-layer", + "import", + "--project", + "prod", + "--file", + str(snap), + "--dry-run", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _, kwargs = mock.import_snapshot.call_args + assert kwargs["dry_run"] is True + + +# --------------------------------------------------------------------------- +# semantic-layer promote +# --------------------------------------------------------------------------- + + +class TestPromote: + def test_promote_with_yes(self, tmp_path: Path) -> None: + cfg = tmp_path / "cfg" + cfg.mkdir() + st = _setup_config( + cfg, + { + "source": {"token": TEST_TOKEN, "project_id": 1}, + "target": {"token": TEST_TOKEN, "project_id": 2}, + }, + ) + mock = MagicMock() + mock.promote_model.return_value = { + "from_project": "source", + "to_project": "target", + "from_model": "us", + "to_model": "ut", + "dry_run": False, + "datasets": {"new": 0, "overwritten": 0, "identical": 0, "failed": [], "changes": []}, + "metrics": {"new": 0, "overwritten": 0, "identical": 0, "failed": [], "changes": []}, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "promote", + "--from-project", + "source", + "--to-project", + "target", + "--yes", + ], + store=st, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + + +# --------------------------------------------------------------------------- +# semantic-layer build +# --------------------------------------------------------------------------- + + +class TestBuild: + def test_missing_tables_exit_2(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--json", + "semantic-layer", + "build", + "--project", + "prod", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2 + body = json.loads(result.output) + assert body["error"]["code"] == "MISSING_PARAMETER" + mock.build_model.assert_not_called() + + def test_dry_run_happy(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.build_model.return_value = { + "project": "prod", + "dry_run": True, + "fallback_used": "heuristic", + "fetch_errors": [], + "generated": { + "name": "kbagent_build_model", + "datasets": [{"name": "ds"}], + "metrics": [{"name": "ds_row_count"}], + "relationships": [], + "constraints": [], + "glossary": [{"term": "ds"}], + }, + "validation": {"errors": [], "warnings": []}, + "validated": True, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "build", + "--project", + "prod", + "--tables", + "out.c.t", + "--dry-run", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["fallback_used"] == "heuristic" + + +# --------------------------------------------------------------------------- +# semantic-layer token --encrypt +# --------------------------------------------------------------------------- + + +class TestToken: + def test_without_encrypt_flag_exit_2(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--json", + "semantic-layer", + "token", + "--project", + "prod", + "--component-id", + "keboola.ex-db-snowflake", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2 + body = json.loads(result.output) + assert body["error"]["code"] == "USAGE_ERROR" + mock.encrypt_token.assert_not_called() + + def test_with_encrypt_success(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.encrypt_token.return_value = { + "project": "prod", + "component_id": "keboola.ex-db-snowflake", + "encrypted": {"#metastore_token": "KBC::ProjectSecureGKMS::cipher"}, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "token", + "--encrypt", + "--project", + "prod", + "--component-id", + "keboola.ex-db-snowflake", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["encrypted"]["#metastore_token"].startswith("KBC::") + + +# --------------------------------------------------------------------------- +# Permission gating: --deny-writes blocks write subcommands +# --------------------------------------------------------------------------- + + +class TestPermissions: + def test_deny_writes_blocks_add_metric(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--deny-writes", + "--json", + "semantic-layer", + "add", + "metric", + "--project", + "prod", + "--name", + "rev", + "--sql", + "x", + "--dataset", + "out.c.t", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == EXIT_PERMISSION_DENIED + mock.add_metric.assert_not_called() + + def test_deny_writes_blocks_model_create(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--deny-writes", + "--json", + "semantic-layer", + "model", + "create", + "--project", + "prod", + "--name", + "x", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == EXIT_PERMISSION_DENIED + mock.create_model.assert_not_called() + + def test_deny_writes_blocks_edit_metric(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--deny-writes", + "--json", + "semantic-layer", + "edit", + "metric", + "--project", + "prod", + "--name", + "rev", + "--new-sql", + "1", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == EXIT_PERMISSION_DENIED + mock.edit_metric.assert_not_called() + + def test_deny_destructive_blocks_remove(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--deny-destructive", + "--json", + "semantic-layer", + "remove", + "metric", + "--project", + "prod", + "--name", + "rev", + "--yes", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == EXIT_PERMISSION_DENIED + mock.remove_item.assert_not_called() + + def test_read_subcommand_allowed_with_deny_writes(self, store: ConfigStore) -> None: + """--deny-writes must NOT block read subcommands like `show`.""" + mock = MagicMock() + mock.show_model.return_value = { + "project": "prod", + "model": {"id": "u1", "name": "x"}, + "datasets": [], + "metrics": [], + "relationships": [], + "constraints": [], + "glossary": [], + } + result = _invoke( + ["--deny-writes", "--json", "semantic-layer", "show", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + + def test_token_subcommand_denied_with_deny_writes(self, store: ConfigStore) -> None: + """`token --encrypt` is classified `write` (parity with `encrypt.values`). + + It calls EncryptService to mint ciphertext that downstream agents could + paste into a transformation config; treating it as `read` would let + ``--deny-writes`` callers emit secrets the firewall meant to gate. + """ + mock = MagicMock() + mock.encrypt_token.return_value = { + "project": "prod", + "component_id": "keboola.ex-db-snowflake", + "encrypted": {"#metastore_token": "KBC::ProjectSecureGKMS::cipher"}, + } + result = _invoke( + [ + "--deny-writes", + "--json", + "semantic-layer", + "token", + "--encrypt", + "--project", + "prod", + "--component-id", + "keboola.ex-db-snowflake", + ], + store=store, + sl_mock=mock, + ) + # EXIT_PERMISSION_DENIED == 6 + assert result.exit_code == 6, result.output + assert "PERMISSION_DENIED" in result.output + + def test_model_list_allowed_with_deny_writes(self, store: ConfigStore) -> None: + """`model list` is read-only and must succeed under --deny-writes. + + Regression test for iter-2/iter-3: the parent `semantic-layer` + callback fires before the `model` sub-app's per-subcommand callback, + so the parent-level operation key `semantic-layer.model` must exist + in the registry at the least-privileged (read) classification. + Without it, fail-closed defaults to `write` and `model list` is + denied even though the leaf is correctly classified as read. + """ + mock = MagicMock() + mock.list_models.return_value = {"project": "prod", "models": []} + result = _invoke( + [ + "--deny-writes", + "--json", + "semantic-layer", + "model", + "list", + "--project", + "prod", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + assert "PERMISSION_DENIED" not in result.output + + def test_model_create_denied_with_deny_writes(self, store: ConfigStore) -> None: + """`model create` mutates state and must be denied under --deny-writes.""" + mock = MagicMock() + mock.create_model.return_value = { + "project": "prod", + "model": {"id": "x", "attributes": {"name": "x"}}, + } + result = _invoke( + [ + "--deny-writes", + "--json", + "semantic-layer", + "model", + "create", + "--project", + "prod", + "--name", + "x", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 6, result.output + assert "PERMISSION_DENIED" in result.output + + def test_model_delete_denied_with_deny_destructive(self, store: ConfigStore) -> None: + """`model delete` is destructive — narrower flag still blocks it.""" + mock = MagicMock() + mock.delete_model.return_value = {"deleted": {"id": "x", "name": "x"}} + result = _invoke( + [ + "--deny-destructive", + "--json", + "semantic-layer", + "model", + "delete", + "--project", + "prod", + "--model", + "x", + "--yes", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 6, result.output + assert "PERMISSION_DENIED" in result.output diff --git a/tests/test_semantic_layer_service.py b/tests/test_semantic_layer_service.py new file mode 100644 index 00000000..d1909211 --- /dev/null +++ b/tests/test_semantic_layer_service.py @@ -0,0 +1,1980 @@ +"""Service-layer tests for ``SemanticLayerService``. + +Covers every business operation: model resolution, list/show/validate/export/diff +reads, create/delete/add/edit/remove writes, import/promote/build orchestration, +and the encrypt-token helper. + +Each test injects a ``unittest.mock.MagicMock`` as the metastore client factory +so we verify orchestration (envelope shape, call order, error propagation) +without touching HTTP. The pattern mirrors ``test_data_app_service.py``. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.semantic_layer_service import ( + CONSTRAINT_NAME_RE, + CONSTRAINT_SEVERITIES, + CONSTRAINT_TYPES, + SemanticLayerService, + _classify_field_role, + _derive_fqn, +) + +TEST_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" + + +# --------------------------------------------------------------------------- +# Fixtures / helpers +# --------------------------------------------------------------------------- + + +def _make_store(tmp_path: Path, alias: str = "prod") -> ConfigStore: + """Build a ConfigStore with a single project registered.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + alias, + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name=alias, + project_id=5725, + ), + ) + return store + + +def _make_store_two(tmp_path: Path) -> ConfigStore: + """Build a ConfigStore with two projects (for promote tests).""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + for alias in ("source", "target"): + store.add_project( + alias, + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name=alias, + project_id=1000 if alias == "source" else 2000, + ), + ) + return store + + +def _make_service( + store: ConfigStore, + *, + metastore_mock: MagicMock | None = None, +) -> tuple[SemanticLayerService, MagicMock]: + """Wire a SemanticLayerService with a mocked metastore client factory.""" + mock = metastore_mock or MagicMock() + service = SemanticLayerService( + config_store=store, + metastore_client_factory=lambda url, token: mock, + ) + return service, mock + + +def _model_item( + uuid: str = "u-model", + name: str = "default", + description: str = "", + sql_dialect: str = "Snowflake", +) -> dict[str, Any]: + return { + "type": "semantic-model", + "id": uuid, + "attributes": { + "name": name, + "description": description, + "sql_dialect": sql_dialect, + }, + } + + +def _child_item( + item_type: str, + item_id: str, + attrs: dict[str, Any], +) -> dict[str, Any]: + return {"type": item_type, "id": item_id, "attributes": dict(attrs)} + + +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- + + +class TestDeriveFqn: + def test_simple_two_segment_table_id(self) -> None: + assert _derive_fqn("out.c-gold.FACT_X") == '"KEBOOLA"."out.c-gold"."FACT_X"' + + def test_three_segment_table_id(self) -> None: + assert _derive_fqn("in.c-raw.users") == '"KEBOOLA"."in.c-raw"."users"' + + def test_invalid_single_segment_raises(self) -> None: + with pytest.raises(KeboolaApiError) as excinfo: + _derive_fqn("nodots") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + +class TestClassifyFieldRole: + """Role heuristic for `add dataset --deep-fields`.""" + + def test_pk_prefix_is_key(self) -> None: + assert _classify_field_role("PK_USER_ID", "NUMBER") == "key" + + def test_fk_prefix_is_key(self) -> None: + assert _classify_field_role("FK_ORDER_ID", "NUMBER") == "key" + + def test_date_suffix_is_timestamp(self) -> None: + assert _classify_field_role("ORDER_DATE", "TIMESTAMP_TZ") == "timestamp" + + def test_date_prefix_is_timestamp(self) -> None: + assert _classify_field_role("DATE_ORDER", "TIMESTAMP_TZ") == "timestamp" + + def test_numeric_with_measure_token_is_measure(self) -> None: + assert _classify_field_role("AMOUNT_USD", "NUMBER") == "measure" + + def test_numeric_without_measure_token_is_dimension(self) -> None: + assert _classify_field_role("USER_ID", "NUMBER") == "dimension" + + def test_string_with_measure_token_is_dimension(self) -> None: + # measure tokens require a numeric basetype + assert _classify_field_role("AMOUNT_LABEL", "STRING") == "dimension" + + def test_plain_dimension_default(self) -> None: + assert _classify_field_role("USER_NAME", "STRING") == "dimension" + + +# --------------------------------------------------------------------------- +# Model resolution +# --------------------------------------------------------------------------- + + +class TestResolveModel: + def test_single_model_no_selector_returns_it(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.return_value = [_model_item("u1", "only")] + result = service.list_models("prod") + assert len(result["models"]) == 1 + assert result["models"][0]["name"] == "only" + + def test_ambiguous_models_raises_config_error(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.return_value = [ + _model_item("u1", "a"), + _model_item("u2", "b"), + ] + with pytest.raises(ConfigError) as excinfo: + service.show_model("prod", model_name_or_uuid=None) + assert "specify --model" in excinfo.value.message + assert "a" in excinfo.value.message and "b" in excinfo.value.message + + def test_no_models_raises_config_error(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.return_value = [] + with pytest.raises(ConfigError) as excinfo: + service.show_model("prod") + assert "no semantic-layer models" in excinfo.value.message.lower() + + def test_resolve_by_exact_uuid(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("u1", "a"), _model_item("u2", "b")] + return [] + + mock.list_items.side_effect = _list + result = service.show_model("prod", model_name_or_uuid="u2") + assert result["model"]["id"] == "u2" + + def test_resolve_by_name(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("u1", "alpha"), _model_item("u2", "beta")] + return [] + + mock.list_items.side_effect = _list + result = service.show_model("prod", model_name_or_uuid="beta") + assert result["model"]["id"] == "u2" + + def test_not_found_raises_with_available_names(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("u1", "alpha"), _model_item("u2", "beta")] + return [] + + mock.list_items.side_effect = _list + with pytest.raises(ConfigError) as excinfo: + service.show_model("prod", model_name_or_uuid="ghost") + assert "alpha" in excinfo.value.message + assert "beta" in excinfo.value.message + + +# --------------------------------------------------------------------------- +# list_models / create_model / delete_model +# --------------------------------------------------------------------------- + + +class TestListModels: + def test_returns_shape(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.return_value = [ + _model_item("u1", "a", "first", "Snowflake"), + _model_item("u2", "b", "second", "Snowflake"), + ] + result = service.list_models("prod") + assert result["project"] == "prod" + assert result["models"][0] == { + "id": "u1", + "name": "a", + "description": "first", + "sql_dialect": "Snowflake", + } + mock.close.assert_called_once() + + def test_empty_project(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.return_value = [] + result = service.list_models("prod") + assert result["models"] == [] + + +class TestCreateModel: + def test_happy_path(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.post_item.return_value = { + "type": "semantic-model", + "id": "new-uuid", + "attributes": {"name": "m", "sql_dialect": "Snowflake"}, + } + result = service.create_model("prod", name="m", description="d", sql_dialect="Snowflake") + assert result["model"]["id"] == "new-uuid" + mock.post_item.assert_called_once() + _, kwargs = mock.post_item.call_args + # data should include description (truthy) + assert kwargs["data"]["description"] == "d" + assert kwargs["data"]["sql_dialect"] == "Snowflake" + + def test_omits_empty_description(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.post_item.return_value = _model_item("x", "m") + service.create_model("prod", name="m") + _, kwargs = mock.post_item.call_args + assert "description" not in kwargs["data"] + + +class TestDeleteModel: + def test_lists_children_before_delete(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("u1", "doomed")] + if item_type == "semantic-metric": + return [_child_item("semantic-metric", "m1", {"name": "x"})] + return [] + + mock.list_items.side_effect = _list + result = service.delete_model("prod", model_name_or_uuid="doomed") + assert result["deleted"]["id"] == "u1" + assert result["deleted"]["name"] == "doomed" + # delete_item was called for the model + mock.delete_item.assert_called_once_with("semantic-model", "u1") + # counts in orphaned_children + assert result["orphaned_children"]["metrics"] == 1 + + +# --------------------------------------------------------------------------- +# show +# --------------------------------------------------------------------------- + + +class TestShowModel: + def _setup(self, tmp_path: Path) -> tuple[SemanticLayerService, MagicMock]: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U1", "default")] + if item_type == "semantic-dataset": + return [_child_item("semantic-dataset", "d1", {"name": "ds_a"})] + if item_type == "semantic-metric": + return [_child_item("semantic-metric", "m1", {"name": "rev"})] + if item_type == "semantic-relationship": + return [_child_item("semantic-relationship", "r1", {"name": "r"})] + if item_type == "semantic-constraint": + return [_child_item("semantic-constraint", "c1", {"name": "c"})] + if item_type == "semantic-glossary": + return [_child_item("semantic-glossary", "g1", {"term": "GMV"})] + return [] + + mock.list_items.side_effect = _list + return service, mock + + def test_returns_envelope_with_all_types(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path) + result = service.show_model("prod") + assert result["project"] == "prod" + assert result["model"] == {"id": "U1", "name": "default"} + assert len(result["datasets"]) == 1 + assert result["datasets"][0]["id"] == "d1" + assert "metrics" in result + assert "relationships" in result + assert "constraints" in result + assert "glossary" in result + + def test_filters_to_one_type(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path) + result = service.show_model("prod", type_filter="metric") + assert "metrics" in result + # other types filtered out + for k in ("datasets", "relationships", "constraints", "glossary"): + assert k not in result + + def test_rejects_unknown_type_filter(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path) + with pytest.raises(KeboolaApiError) as excinfo: + service.show_model("prod", type_filter="notathing") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_empty_model(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "lone")] + return [] + + mock.list_items.side_effect = _list + result = service.show_model("prod") + for k in ("datasets", "metrics", "relationships", "constraints", "glossary"): + assert result[k] == [] + + def test_meta_keys_not_leaked(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "lone")] + if item_type == "semantic-dataset": + return [ + { + "type": "semantic-dataset", + "id": "d1", + "attributes": {"name": "x"}, + "meta": {"createdAt": "2020"}, + } + ] + return [] + + mock.list_items.side_effect = _list + result = service.show_model("prod") + # 'meta' key from server shape must not appear on the per-item dict + assert "meta" not in result["datasets"][0] + + +# --------------------------------------------------------------------------- +# validate (basic + deep) +# --------------------------------------------------------------------------- + + +class TestValidateBasic: + """Basic validation checks (no API calls beyond the show fetch).""" + + def _service_with( + self, tmp_path: Path, by_type: dict[str, list[dict[str, Any]]] + ) -> SemanticLayerService: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + return list(by_type.get(item_type, [])) + + mock.list_items.side_effect = _list + return service + + def test_clean_model_is_valid(self, tmp_path: Path) -> None: + service = self._service_with( + tmp_path, + { + "semantic-dataset": [ + _child_item("semantic-dataset", "d1", {"name": "ds", "tableId": "out.c.t"}) + ], + "semantic-metric": [ + _child_item( + "semantic-metric", + "m1", + {"name": "rev", "sql": "COUNT(*)", "dataset": "out.c.t"}, + ) + ], + "semantic-constraint": [ + _child_item( + "semantic-constraint", + "c1", + {"name": "rev_warning", "metrics": ["rev"], "constraintType": "inequality"}, + ) + ], + }, + ) + result = service.validate_model("prod") + assert result["valid"] is True + assert result["errors"] == [] + + def test_duplicate_names(self, tmp_path: Path) -> None: + service = self._service_with( + tmp_path, + { + "semantic-metric": [ + _child_item("semantic-metric", "m1", {"name": "rev"}), + _child_item("semantic-metric", "m2", {"name": "rev"}), + ], + }, + ) + result = service.validate_model("prod") + errors = [e for e in result["errors"] if e["type"] == "DUPLICATE"] + assert errors and "rev" in errors[0]["item"] + + def test_dangling_relationship(self, tmp_path: Path) -> None: + service = self._service_with( + tmp_path, + { + "semantic-dataset": [ + _child_item("semantic-dataset", "d1", {"name": "a", "tableId": "out.c.a"}) + ], + "semantic-relationship": [ + _child_item( + "semantic-relationship", + "r1", + {"name": "r", "from": "out.c.a", "to": "out.c.MISSING"}, + ) + ], + }, + ) + result = service.validate_model("prod") + errors = [e for e in result["errors"] if e["type"] == "DANGLING_RELATIONSHIP"] + assert errors + + def test_dangling_metric(self, tmp_path: Path) -> None: + service = self._service_with( + tmp_path, + { + "semantic-dataset": [ + _child_item("semantic-dataset", "d1", {"name": "a", "tableId": "out.c.a"}) + ], + "semantic-metric": [ + _child_item( + "semantic-metric", + "m1", + {"name": "rev", "sql": "x", "dataset": "out.c.GONE"}, + ) + ], + }, + ) + result = service.validate_model("prod") + errors = [e for e in result["errors"] if e["type"] == "DANGLING_METRIC"] + assert errors + + def test_sum_on_pct_warning(self, tmp_path: Path) -> None: + service = self._service_with( + tmp_path, + { + "semantic-metric": [ + _child_item( + "semantic-metric", + "m1", + {"name": "bad", "sql": 'SUM("t"."PCT")', "dataset": "x"}, + ) + ], + }, + ) + result = service.validate_model("prod") + warns = [w for w in result["warnings"] if w["type"] == "SUM_ON_PCT"] + assert warns + + def test_constraint_orphan(self, tmp_path: Path) -> None: + service = self._service_with( + tmp_path, + { + "semantic-metric": [_child_item("semantic-metric", "m1", {"name": "rev"})], + "semantic-constraint": [ + _child_item( + "semantic-constraint", + "c1", + {"name": "orph_warning", "metrics": ["rev", "MISSING"]}, + ) + ], + }, + ) + result = service.validate_model("prod") + errors = [e for e in result["errors"] if e["type"] == "CONSTRAINT_ORPHAN"] + assert errors + + def test_severity_suffix_warning(self, tmp_path: Path) -> None: + service = self._service_with( + tmp_path, + { + "semantic-metric": [_child_item("semantic-metric", "m1", {"name": "rev"})], + "semantic-constraint": [ + _child_item( + "semantic-constraint", + "c1", + {"name": "no_suffix", "metrics": ["rev"]}, + ) + ], + }, + ) + result = service.validate_model("prod") + warns = [w for w in result["warnings"] if w["type"] == "SEVERITY_SUFFIX"] + assert warns + + +class TestValidateDeep: + """Deep validation -- fetches Snowflake schemas via StorageService.""" + + def test_phantom_field(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + ds_attrs = { + "name": "ds", + "tableId": "out.c.t", + "fields": [{"name": "REAL"}, {"name": "GHOST"}], + } + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-dataset": + return [_child_item("semantic-dataset", "d1", ds_attrs)] + return [] + + mock.list_items.side_effect = _list + + with patch( + "keboola_agent_cli.services.semantic_layer_service.StorageService" + ) as MockStorageCls: + inst = MockStorageCls.return_value + inst.get_table_detail.return_value = { + "columns": ["REAL"], + "column_details": [{"name": "REAL", "type": "STRING"}], + } + result = service.validate_model("prod", deep=True) + errors = [e for e in result["errors"] if e["type"] == "PHANTOM_FIELD"] + assert any("GHOST" in e["item"] for e in errors) + + def test_metric_phantom_column_ref(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-dataset": + return [_child_item("semantic-dataset", "d1", {"name": "ds", "tableId": "out.c.t"})] + if item_type == "semantic-metric": + return [ + _child_item( + "semantic-metric", + "m1", + {"name": "rev", "sql": 'SUM("schema"."GHOST_COL")', "dataset": "out.c.t"}, + ) + ] + return [] + + mock.list_items.side_effect = _list + with patch( + "keboola_agent_cli.services.semantic_layer_service.StorageService" + ) as MockStorageCls: + inst = MockStorageCls.return_value + inst.get_table_detail.return_value = { + "columns": ["REAL"], + "column_details": [{"name": "REAL", "type": "NUMBER"}], + } + result = service.validate_model("prod", deep=True) + errors = [e for e in result["errors"] if e["type"] == "METRIC_PHANTOM"] + assert errors + + def test_agg_on_string(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-dataset": + return [_child_item("semantic-dataset", "d1", {"name": "ds", "tableId": "out.c.t"})] + if item_type == "semantic-metric": + return [ + _child_item( + "semantic-metric", + "m1", + { + "name": "agg_bad", + "sql": 'SUM("schema"."NAME")', + "dataset": "out.c.t", + }, + ) + ] + return [] + + mock.list_items.side_effect = _list + with patch( + "keboola_agent_cli.services.semantic_layer_service.StorageService" + ) as MockStorageCls: + inst = MockStorageCls.return_value + inst.get_table_detail.return_value = { + "columns": ["NAME"], + "column_details": [{"name": "NAME", "type": "STRING"}], + } + result = service.validate_model("prod", deep=True) + errors = [e for e in result["errors"] if e["type"] == "AGG_ON_STRING"] + assert errors + + +# --------------------------------------------------------------------------- +# export +# --------------------------------------------------------------------------- + + +class TestExportModel: + def _setup(self, tmp_path: Path) -> tuple[SemanticLayerService, MagicMock]: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "exp_model")] + if item_type == "semantic-dataset": + return [ + _child_item("semantic-dataset", "d1", {"name": "ds_a", "tableId": "out.c.t"}) + ] + if item_type == "semantic-metric": + return [_child_item("semantic-metric", "m1", {"name": "rev"})] + if item_type == "semantic-relationship": + return [_child_item("semantic-relationship", "r1", {"name": "r"})] + if item_type == "semantic-constraint": + return [_child_item("semantic-constraint", "c1", {"name": "c_warning"})] + if item_type == "semantic-glossary": + return [_child_item("semantic-glossary", "g1", {"term": "GMV"})] + return [] + + mock.list_items.side_effect = _list + return service, mock + + def test_export_envelope_shape(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path) + out_path = tmp_path / "export.json" + result = service.export_model("prod", output_path=out_path) + assert result["project"] == "prod" + assert "exported_at" in result + for k in ("datasets", "metrics", "relationships", "constraints", "glossary"): + assert k in result + assert result["counts"][k] == 1 + assert result["path"] == str(out_path) + + def test_export_writes_file_with_correct_permissions(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path) + out_path = tmp_path / "snap.json" + service.export_model("prod", output_path=out_path) + assert out_path.is_file() + # Permissions: world-readable per spec + mode = out_path.stat().st_mode & 0o777 + assert mode == 0o644 + # Content is valid JSON with expected keys + payload = json.loads(out_path.read_text()) + assert payload["datasets"][0]["id"] == "d1" + assert payload["model"]["id"] == "U" + + def test_export_default_path(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + service, _ = self._setup(tmp_path) + monkeypatch.chdir(tmp_path) + result = service.export_model("prod") + path = Path(result["path"]) + assert path.parent == tmp_path + assert path.name.startswith("sl_export_exp_model_") + assert path.suffix == ".json" + + +# --------------------------------------------------------------------------- +# diff +# --------------------------------------------------------------------------- + + +class TestDiff: + def _service_two_sides( + self, + tmp_path: Path, + *, + a_metrics: list[dict[str, Any]], + b_metrics: list[dict[str, Any]], + ) -> SemanticLayerService: + """Build a service whose mocked client returns different shapes per project alias. + + Since the factory builds one client per project resolution, we keep a + single MagicMock but rotate its `side_effect` between calls by inspecting + which alias's stack URL was used. Simpler: just inject the SAME + responses both times, then drive diff via project-vs-file. + """ + store = _make_store_two(tmp_path) + service, mock = _make_service(store) + + # Default list returns the A side; we'll diff project A vs a snapshot + # file holding B. + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U_A", "ma")] + if item_type == "semantic-metric": + return list(a_metrics) + return [] + + mock.list_items.side_effect = _list + + # Write the B side as a snapshot file. + snap = tmp_path / "b.json" + snap.write_text( + json.dumps( + { + "exported_at": "2020-01-01T00:00:00Z", + "project": "target", + "model": {"id": "U_B", "name": "mb"}, + "datasets": [], + "metrics": b_metrics, + "relationships": [], + "constraints": [], + "glossary": [], + } + ) + ) + self._snap = snap + return service + + def test_added_removed_changed(self, tmp_path: Path) -> None: + a_metrics = [ + _child_item("semantic-metric", "m1", {"name": "a", "sql": "1"}), + _child_item("semantic-metric", "m2", {"name": "b", "sql": "1"}), # changed + ] + b_metrics = [ + _child_item("semantic-metric", "m2", {"name": "b", "sql": "2"}), # changed + _child_item("semantic-metric", "m3", {"name": "c", "sql": "1"}), # new in B + ] + service = self._service_two_sides(tmp_path, a_metrics=a_metrics, b_metrics=b_metrics) + result = service.diff(project_a="source", file_b=self._snap) + metrics = result["metrics"] + # A had a, b; B has b, c -> added=[c], removed=[a], changed=[b] + assert metrics["added"] == ["c"] + assert metrics["removed"] == ["a"] + assert metrics["changed"] == [{"name": "b", "diff_keys": ["sql"]}] + + def test_identical_after_strip(self, tmp_path: Path) -> None: + """modelUUID + timestamps differ; structural content matches → no change.""" + a_metrics = [ + _child_item( + "semantic-metric", + "m1", + {"name": "a", "sql": "1", "modelUUID": "U_A", "createdAt": "2020"}, + ) + ] + b_metrics = [ + _child_item( + "semantic-metric", + "m1", + {"name": "a", "sql": "1", "modelUUID": "U_B", "createdAt": "2025"}, + ) + ] + service = self._service_two_sides(tmp_path, a_metrics=a_metrics, b_metrics=b_metrics) + result = service.diff(project_a="source", file_b=self._snap) + assert result["metrics"]["changed"] == [] + + def test_glossary_uses_term_key(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "x")] + if item_type == "semantic-glossary": + return [_child_item("semantic-glossary", "g1", {"term": "GMV", "definition": "v1"})] + return [] + + mock.list_items.side_effect = _list + + # Right side: a snapshot with a different definition for the same term. + snap = tmp_path / "right.json" + snap.write_text( + json.dumps( + { + "model": {"id": "U", "name": "x"}, + "datasets": [], + "metrics": [], + "relationships": [], + "constraints": [], + "glossary": [ + _child_item("semantic-glossary", "g1", {"term": "GMV", "definition": "v2"}) + ], + } + ) + ) + result = service.diff(project_a="prod", file_b=snap) + assert result["glossary"]["changed"][0]["term"] == "GMV" + + def test_file_vs_file(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + f_left = tmp_path / "L.json" + f_right = tmp_path / "R.json" + f_left.write_text( + json.dumps( + { + "model": {"id": "U", "name": "x"}, + "datasets": [], + "metrics": [_child_item("semantic-metric", "m", {"name": "rev", "sql": "1"})], + "relationships": [], + "constraints": [], + "glossary": [], + } + ) + ) + f_right.write_text( + json.dumps( + { + "model": {"id": "U", "name": "x"}, + "datasets": [], + "metrics": [_child_item("semantic-metric", "m", {"name": "rev", "sql": "2"})], + "relationships": [], + "constraints": [], + "glossary": [], + } + ) + ) + result = service.diff(file_a=f_left, file_b=f_right) + assert result["metrics"]["changed"] == [{"name": "rev", "diff_keys": ["sql"]}] + + +# --------------------------------------------------------------------------- +# add_* operations +# --------------------------------------------------------------------------- + + +class TestAddDataset: + def test_fqn_derivation(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "default")] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "d1", "attributes": {"name": "fact_x"}} + service.add_dataset( + "prod", + None, + name="fact_x", + table_id="out.c-gold.FACT_X", + ) + _, kwargs = mock.post_item.call_args + assert kwargs["data"]["fqn"] == '"KEBOOLA"."out.c-gold"."FACT_X"' + assert kwargs["data"]["modelUUID"] == "U" + + def test_deep_fields_role_heuristics(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "default")] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "d1", "attributes": {"name": "x"}} + with patch( + "keboola_agent_cli.services.semantic_layer_service.StorageService" + ) as MockStorageCls: + inst = MockStorageCls.return_value + inst.get_table_detail.return_value = { + "column_details": [ + {"name": "PK_USER_ID", "type": "NUMBER"}, + {"name": "ORDER_DATE", "type": "TIMESTAMP_TZ"}, + {"name": "AMOUNT_USD", "type": "NUMBER"}, + {"name": "USER_NAME", "type": "STRING"}, + ] + } + service.add_dataset( + "prod", + None, + name="x", + table_id="out.c-g.X", + deep_fields=True, + ) + _, kwargs = mock.post_item.call_args + fields = {f["name"]: f["role"] for f in kwargs["data"]["fields"]} + assert fields["PK_USER_ID"] == "key" + assert fields["ORDER_DATE"] == "timestamp" + assert fields["AMOUNT_USD"] == "measure" + assert fields["USER_NAME"] == "dimension" + + +class TestAddMetric: + def _ds_list_factory(self, dataset_tids: list[str]): + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "default")] + if item_type == "semantic-dataset": + return [ + _child_item("semantic-dataset", f"d{i}", {"name": f"d{i}", "tableId": tid}) + for i, tid in enumerate(dataset_tids) + ] + return [] + + return _list + + def test_happy_path_dataset_in_model(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.side_effect = self._ds_list_factory(["out.c.t"]) + mock.post_item.return_value = {"id": "m1", "attributes": {"name": "rev"}} + result = service.add_metric( + "prod", + None, + name="rev", + sql="COUNT(*)", + dataset="out.c.t", + assume_yes=False, + ) + assert result["id"] == "m1" + _, kwargs = mock.post_item.call_args + assert kwargs["item_type"] == "semantic-metric" if "item_type" in kwargs else True + assert kwargs["data"]["modelUUID"] == "U" + assert kwargs["data"]["dataset"] == "out.c.t" + + def test_dataset_not_in_model_non_tty_requires_yes(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.side_effect = self._ds_list_factory(["out.c.OTHER"]) + with pytest.raises(KeboolaApiError) as excinfo: + service.add_metric( + "prod", + None, + name="rev", + sql="x", + dataset="out.c.MISSING", + assume_yes=False, + is_tty=False, + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_dataset_not_in_model_yes_bypasses(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.side_effect = self._ds_list_factory(["out.c.OTHER"]) + mock.post_item.return_value = {"id": "m1", "attributes": {"name": "rev"}} + result = service.add_metric( + "prod", + None, + name="rev", + sql="x", + dataset="out.c.MISSING", + assume_yes=True, + ) + assert result["id"] == "m1" + + +class TestAddRelationship: + def test_rejects_invalid_type(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + service.add_relationship( + "prod", + None, + name="r", + from_="a", + to="b", + on="a.id=b.id", + type_="full_outer", + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_happy_path(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "r1"} + service.add_relationship( + "prod", + None, + name="users_to_orders", + from_="out.c.users", + to="out.c.orders", + on="users.id = orders.user_id", + type_="left", + ) + _, kwargs = mock.post_item.call_args + assert kwargs["data"]["type"] == "left" + assert kwargs["data"]["modelUUID"] == "U" + + +class TestAddConstraint: + def _setup( + self, tmp_path: Path, metric_names: list[str] + ) -> tuple[SemanticLayerService, MagicMock]: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-metric": + return [ + _child_item("semantic-metric", f"m{i}", {"name": n}) + for i, n in enumerate(metric_names) + ] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "c1"} + return service, mock + + def test_rejects_bad_name_uppercase(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, ["rev"]) + with pytest.raises(KeboolaApiError) as excinfo: + service.add_constraint( + "prod", + None, + name="BadName", + constraint_type="inequality", + rule="x > 0", + metrics=["rev"], + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_rejects_bad_name_dashes(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, ["rev"]) + with pytest.raises(KeboolaApiError): + service.add_constraint( + "prod", + None, + name="bad-name", + constraint_type="inequality", + rule="x", + metrics=["rev"], + ) + + def test_rejects_leading_digit(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, ["rev"]) + with pytest.raises(KeboolaApiError): + service.add_constraint( + "prod", + None, + name="1bad", + constraint_type="inequality", + rule="x", + metrics=["rev"], + ) + + def test_accepts_valid_name(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, ["rev"]) + # Must not raise + service.add_constraint( + "prod", + None, + name="good_name_warning", + constraint_type="inequality", + rule="x", + metrics=["rev"], + ) + + def test_rejects_unknown_constraint_type(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, ["rev"]) + with pytest.raises(KeboolaApiError): + service.add_constraint( + "prod", + None, + name="ok_warning", + constraint_type="fancy", + rule="x", + metrics=["rev"], + ) + + def test_rejects_unknown_severity(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, ["rev"]) + with pytest.raises(KeboolaApiError): + service.add_constraint( + "prod", + None, + name="ok_warning", + constraint_type="inequality", + rule="x", + metrics=["rev"], + severity="extreme", + ) + + def test_rejects_missing_metric_reference(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, ["rev"]) + with pytest.raises(KeboolaApiError) as excinfo: + service.add_constraint( + "prod", + None, + name="ok_warning", + constraint_type="inequality", + rule="x", + metrics=["NONEXISTENT"], + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + +class TestAddGlossary: + def test_happy_path(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "g1", "attributes": {"term": "GMV"}} + service.add_glossary("prod", None, term="GMV", definition="gross merchandise value") + _, kwargs = mock.post_item.call_args + # name parameter (envelope key) must equal the term + assert kwargs["name"] == "GMV" + assert kwargs["data"]["term"] == "GMV" + assert kwargs["data"]["modelUUID"] == "U" + + +# --------------------------------------------------------------------------- +# edit_* operations (DELETE+POST + rollback + cascade) +# --------------------------------------------------------------------------- + + +class TestEditMetric: + def test_rename_cascades_constraints(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + original = _child_item("semantic-metric", "m1", {"name": "rev", "sql": "1"}) + c_attrs = {"name": "rev_warning", "metrics": ["rev"]} + constraint = _child_item("semantic-constraint", "c1", c_attrs) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-metric": + return [original] + if item_type == "semantic-constraint": + return [constraint] + return [] + + mock.list_items.side_effect = _list + mock.post_item.side_effect = [ + {"id": "m_new", "attributes": {"name": "revenue"}}, + {"id": "c_new", "attributes": dict(c_attrs, name="rev_warning")}, + ] + + result = service.edit_metric( + "prod", + None, + current_name="rev", + new_name="revenue", + assume_yes=True, + ) + assert result["updated"]["id"] == "m_new" + # delete called twice: once for the old metric, once for the cascade + assert mock.delete_item.call_count == 2 + # cascade list populated + assert result["cascaded_constraints"][0]["status"] == "updated" + + def test_description_only_change_no_cascade(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + original = _child_item("semantic-metric", "m1", {"name": "rev", "sql": "1"}) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-metric": + return [original] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "m_new", "attributes": {"name": "rev"}} + + result = service.edit_metric( + "prod", + None, + current_name="rev", + new_description="updated desc", + ) + assert result["updated"]["id"] == "m_new" + # Only one delete: the metric itself. + assert mock.delete_item.call_count == 1 + assert result["cascaded_constraints"] == [] + + def test_metric_not_found(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + return [] + + mock.list_items.side_effect = _list + with pytest.raises(KeboolaApiError) as excinfo: + service.edit_metric("prod", None, current_name="ghost", new_sql="1") + assert excinfo.value.error_code == ErrorCode.NOT_FOUND + + def test_rollback_on_post_failure(self, tmp_path: Path) -> None: + """When POST after DELETE fails, the original item is re-POSTed.""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + original = _child_item("semantic-metric", "m1", {"name": "rev", "sql": "1"}) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-metric": + return [original] + return [] + + mock.list_items.side_effect = _list + # First POST (the actual edit) fails; second POST (rollback) succeeds. + mock.post_item.side_effect = [ + KeboolaApiError(message="boom", status_code=500, error_code=ErrorCode.API_ERROR), + {"id": "restored", "attributes": {"name": "rev"}}, + ] + + with pytest.raises(KeboolaApiError) as excinfo: + service.edit_metric("prod", None, current_name="rev", new_sql="2") + details = excinfo.value.details or {} + rollback = details.get("rollback") or {} + assert rollback.get("status") == "succeeded" + + +class TestEditDataset: + def test_not_found(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + return [] + + mock.list_items.side_effect = _list + with pytest.raises(KeboolaApiError) as excinfo: + service.edit_dataset("prod", None, current_name="ghost", new_description="x") + assert excinfo.value.error_code == ErrorCode.NOT_FOUND + + +class TestEditConstraint: + def test_rejects_bad_new_name(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + service.edit_constraint( + "prod", + None, + current_name="ok_warning", + new_name="UPPER", + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_rejects_bad_new_type(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + with pytest.raises(KeboolaApiError): + service.edit_constraint( + "prod", + None, + current_name="ok_warning", + new_constraint_type="bogus", + ) + + +class TestEditRelationship: + def test_updates_endpoint(self, tmp_path: Path) -> None: + """Happy path: --new-from rewrites the source tableId via DELETE+POST.""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-relationship": + return [ + _child_item( + "semantic-relationship", + "r1", + { + "name": "fact_to_dim", + "from": "out.c.fact", + "to": "out.c.dim", + "on": "fact.id = dim.id", + "type": "left", + }, + ) + ] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = { + "id": "r2", + "attributes": {"name": "fact_to_dim", "from": "out.c.fact_v2"}, + } + result = service.edit_relationship( + "prod", + None, + current_name="fact_to_dim", + new_from="out.c.fact_v2", + ) + mock.delete_item.assert_called_once_with("semantic-relationship", "r1") + # The POST payload retains the unchanged endpoints but rewrites `from`. + post_kwargs = mock.post_item.call_args.kwargs + assert post_kwargs["data"]["from"] == "out.c.fact_v2" + assert post_kwargs["data"]["to"] == "out.c.dim" + assert result["rollback"] is None + assert result["cascaded_constraints"] == [] + + def test_rejects_bad_new_type(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + service.edit_relationship( + "prod", + None, + current_name="x", + new_type="bogus", + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_not_found(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + return [] + + mock.list_items.side_effect = _list + with pytest.raises(KeboolaApiError) as excinfo: + service.edit_relationship("prod", None, current_name="ghost", new_to="x") + assert excinfo.value.error_code == ErrorCode.NOT_FOUND + + +class TestEditGlossary: + def test_updates_definition(self, tmp_path: Path) -> None: + """Happy path: --new-definition rewrites the definition; term unchanged.""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-glossary": + return [ + _child_item( + "semantic-glossary", + "g1", + {"term": "MRR", "definition": "Monthly Recurring Revenue"}, + ) + ] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = { + "id": "g2", + "attributes": {"term": "MRR", "definition": "Updated def"}, + } + result = service.edit_glossary( + "prod", + None, + current_term="MRR", + new_definition="Updated def", + ) + mock.delete_item.assert_called_once_with("semantic-glossary", "g1") + post_kwargs = mock.post_item.call_args.kwargs + assert post_kwargs["data"]["term"] == "MRR" + assert post_kwargs["data"]["definition"] == "Updated def" + assert result["rollback"] is None + + def test_rename_term(self, tmp_path: Path) -> None: + """--new-term rewrites the term identity.""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-glossary": + return [_child_item("semantic-glossary", "g1", {"term": "MRR"})] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "g2", "attributes": {"term": "RECURRING_REVENUE"}} + service.edit_glossary( + "prod", + None, + current_term="MRR", + new_term="RECURRING_REVENUE", + ) + post_kwargs = mock.post_item.call_args.kwargs + assert post_kwargs["data"]["term"] == "RECURRING_REVENUE" + assert post_kwargs["name"] == "RECURRING_REVENUE" + + def test_not_found(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + return [] + + mock.list_items.side_effect = _list + with pytest.raises(KeboolaApiError) as excinfo: + service.edit_glossary("prod", None, current_term="ghost", new_definition="x") + assert excinfo.value.error_code == ErrorCode.NOT_FOUND + + +# --------------------------------------------------------------------------- +# remove (preview + delete) +# --------------------------------------------------------------------------- + + +class TestRemove: + def test_preview_lists_orphans(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-metric": + return [_child_item("semantic-metric", "m1", {"name": "rev"})] + if item_type == "semantic-constraint": + return [ + _child_item( + "semantic-constraint", + "c1", + {"name": "rev_warning", "metrics": ["rev"]}, + ) + ] + return [] + + mock.list_items.side_effect = _list + preview = service.preview_remove("prod", None, kind="metric", name="rev") + assert len(preview["orphaned_constraints"]) == 1 + assert preview["orphaned_constraints"][0]["name"] == "rev_warning" + + def test_remove_invokes_delete(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-metric": + return [_child_item("semantic-metric", "m1", {"name": "rev"})] + return [] + + mock.list_items.side_effect = _list + result = service.remove_item("prod", None, kind="metric", name="rev") + mock.delete_item.assert_called_once_with("semantic-metric", "m1") + assert result["removed"]["name"] == "rev" + + def test_remove_unknown_kind(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + with pytest.raises(KeboolaApiError): + service.remove_item("prod", None, kind="bogus", name="x") + + def test_remove_not_found(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + return [] + + mock.list_items.side_effect = _list + with pytest.raises(KeboolaApiError) as excinfo: + service.remove_item("prod", None, kind="metric", name="ghost") + assert excinfo.value.error_code == ErrorCode.NOT_FOUND + + def test_remove_relationship(self, tmp_path: Path) -> None: + """Removing a relationship: DELETE only, no orphan check (leaf entity).""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-relationship": + return [_child_item("semantic-relationship", "r1", {"name": "fact_to_dim"})] + return [] + + mock.list_items.side_effect = _list + result = service.remove_item("prod", None, kind="relationship", name="fact_to_dim") + mock.delete_item.assert_called_once_with("semantic-relationship", "r1") + assert result["removed"]["name"] == "fact_to_dim" + assert result["orphaned_constraints"] == [] + + def test_remove_glossary_uses_term_identity(self, tmp_path: Path) -> None: + """Removing glossary: identity key is `term`, not `name`.""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-glossary": + return [_child_item("semantic-glossary", "g1", {"term": "MRR"})] + return [] + + mock.list_items.side_effect = _list + result = service.remove_item("prod", None, kind="glossary", name="MRR") + mock.delete_item.assert_called_once_with("semantic-glossary", "g1") + assert result["removed"]["name"] == "MRR" + assert result["orphaned_constraints"] == [] + + def test_preview_remove_glossary_not_found_uses_term(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + if item_type == "semantic-glossary": + # Only term `MRR` exists -- lookup by `OTHER` must miss. + return [_child_item("semantic-glossary", "g1", {"term": "MRR"})] + return [] + + mock.list_items.side_effect = _list + with pytest.raises(KeboolaApiError) as excinfo: + service.preview_remove("prod", None, kind="glossary", name="OTHER") + assert excinfo.value.error_code == ErrorCode.NOT_FOUND + + +# --------------------------------------------------------------------------- +# import_snapshot +# --------------------------------------------------------------------------- + + +def _write_snapshot(path: Path, *, datasets=None, metrics=None, constraints=None) -> None: + payload = { + "model": {"id": "src-uuid", "name": "src"}, + "datasets": datasets or [], + "metrics": metrics or [], + "relationships": [], + "constraints": constraints or [], + "glossary": [], + } + path.write_text(json.dumps(payload)) + + +class TestImportSnapshot: + def test_skip_on_conflict_default(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "target")] + if item_type == "semantic-dataset": + return [_child_item("semantic-dataset", "d1", {"name": "fact_x"})] + return [] + + mock.list_items.side_effect = _list + snap = tmp_path / "s.json" + _write_snapshot( + snap, + datasets=[ + _child_item("semantic-dataset", "src-d1", {"name": "fact_x", "tableId": "out.c.t"}) + ], + ) + result = service.import_snapshot("prod", snap) + assert result["imported"]["datasets"]["skipped"] == 1 + assert result["imported"]["datasets"]["created"] == 0 + mock.post_item.assert_not_called() + + def test_overwrite_deletes_then_posts(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "target")] + if item_type == "semantic-dataset": + return [_child_item("semantic-dataset", "d1", {"name": "fact_x"})] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "new"} + snap = tmp_path / "s.json" + _write_snapshot( + snap, + datasets=[ + _child_item("semantic-dataset", "src-d1", {"name": "fact_x", "tableId": "out.c.t"}) + ], + ) + result = service.import_snapshot("prod", snap, overwrite=True) + assert result["imported"]["datasets"]["overwritten"] == 1 + mock.delete_item.assert_called_with("semantic-dataset", "d1") + mock.post_item.assert_called() + + def test_dry_run_no_writes(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "target")] + return [] + + mock.list_items.side_effect = _list + snap = tmp_path / "s.json" + _write_snapshot( + snap, + datasets=[ + _child_item("semantic-dataset", "x", {"name": "new_ds", "tableId": "out.c.t"}) + ], + ) + result = service.import_snapshot("prod", snap, dry_run=True) + assert result["imported"]["datasets"]["created"] == 1 + mock.post_item.assert_not_called() + mock.delete_item.assert_not_called() + + def test_dependency_order(self, tmp_path: Path) -> None: + """Push order: datasets -> metrics -> relationships -> glossary -> constraints.""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "target")] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "x"} + snap = tmp_path / "s.json" + snap.write_text( + json.dumps( + { + "model": {"id": "src", "name": "src"}, + "datasets": [ + _child_item("semantic-dataset", "d", {"name": "ds", "tableId": "out.c.t"}) + ], + "metrics": [_child_item("semantic-metric", "m", {"name": "rev"})], + "relationships": [], + "constraints": [_child_item("semantic-constraint", "c", {"name": "c_w"})], + "glossary": [_child_item("semantic-glossary", "g", {"term": "GMV"})], + } + ) + ) + service.import_snapshot("prod", snap) + type_call_order = [ + c.kwargs["item_type"] if "item_type" in c.kwargs else c.args[0] + for c in mock.post_item.call_args_list + ] + # Order: dataset first, constraint last + assert type_call_order[0] == "semantic-dataset" + assert type_call_order[-1] == "semantic-constraint" + + def test_invalid_json_file(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + snap = tmp_path / "broken.json" + snap.write_text("{not valid json") + with pytest.raises(KeboolaApiError) as excinfo: + service.import_snapshot("prod", snap) + assert excinfo.value.error_code == ErrorCode.INVALID_FORMAT + + def test_import_rejects_unknown_types_filter(self, tmp_path: Path) -> None: + """`--types BOGUS` raises VALIDATION_ERROR instead of silently no-opping. + + Iter-3: closes the silent-filter bug where typo'd type names + (e.g. ``--types metric`` instead of ``metrics``) filtered every type + out and emitted zero imports without an error. + """ + from keboola_agent_cli.services.semantic_layer_service import _validate_types_filter + + with pytest.raises(KeboolaApiError) as excinfo: + _validate_types_filter(["BOGUS"]) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + assert "BOGUS" in excinfo.value.message + + def test_validate_types_filter_accepts_valid_subset(self) -> None: + from keboola_agent_cli.services.semantic_layer_service import _validate_types_filter + + assert _validate_types_filter(["datasets", "metrics"]) == {"datasets", "metrics"} + assert _validate_types_filter(None) is None + assert _validate_types_filter([]) is None + + +# --------------------------------------------------------------------------- +# promote_model +# --------------------------------------------------------------------------- + + +class TestPromoteModel: + def test_classification_new_changed_identical(self, tmp_path: Path) -> None: + store = _make_store_two(tmp_path) + + src_mock = MagicMock() + tgt_mock = MagicMock() + clients = {0: src_mock, 1: tgt_mock} + call_idx = {"i": 0} + + def _factory(url: str, token: str) -> MagicMock: + c = clients[call_idx["i"]] + call_idx["i"] += 1 + return c + + service = SemanticLayerService( + config_store=store, + metastore_client_factory=_factory, + ) + + def _src_list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U_S", "src")] + if item_type == "semantic-metric": + return [ + _child_item( + "semantic-metric", "m1", {"name": "a", "sql": "1", "modelUUID": "U_S"} + ), + _child_item( + "semantic-metric", "m2", {"name": "b", "sql": "2", "modelUUID": "U_S"} + ), # changed + _child_item( + "semantic-metric", "m3", {"name": "c", "sql": "3", "modelUUID": "U_S"} + ), # new + ] + return [] + + def _tgt_list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U_T", "tgt")] + if item_type == "semantic-metric": + return [ + _child_item( + "semantic-metric", "tm1", {"name": "a", "sql": "1", "modelUUID": "U_T"} + ), # identical + _child_item( + "semantic-metric", "tm2", {"name": "b", "sql": "OLD", "modelUUID": "U_T"} + ), # will change + _child_item( + "semantic-metric", "tm9", {"name": "z", "sql": "9", "modelUUID": "U_T"} + ), # target-only + ] + return [] + + src_mock.list_items.side_effect = _src_list + tgt_mock.list_items.side_effect = _tgt_list + + result = service.promote_model(from_project="source", to_project="target", dry_run=True) + metrics = result["metrics"] + assert metrics["new"] == 1 # c + assert metrics["overwritten"] == 1 # b + assert metrics["identical"] == 1 # a + # Target-only items not touched + tgt_mock.delete_item.assert_not_called() + + def test_both_clients_closed_even_on_error(self, tmp_path: Path) -> None: + store = _make_store_two(tmp_path) + + src_mock = MagicMock() + tgt_mock = MagicMock() + clients = {0: src_mock, 1: tgt_mock} + call_idx = {"i": 0} + + def _factory(url: str, token: str) -> MagicMock: + c = clients[call_idx["i"]] + call_idx["i"] += 1 + return c + + service = SemanticLayerService( + config_store=store, + metastore_client_factory=_factory, + ) + + # src_client raises during resolve + src_mock.list_items.side_effect = RuntimeError("boom") + + with pytest.raises(RuntimeError): + service.promote_model(from_project="source", to_project="target") + src_mock.close.assert_called_once() + tgt_mock.close.assert_called_once() + + +# --------------------------------------------------------------------------- +# build_model +# --------------------------------------------------------------------------- + + +class TestBuildModel: + def test_heuristic_fallback(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.return_value = [] + mock.post_item.side_effect = [ + {"id": "new-model"}, # the model + {"id": "d1"}, # dataset + {"id": "m1"}, # metric (count(*)) + {"id": "g1"}, # glossary + ] + with patch( + "keboola_agent_cli.services.semantic_layer_service.StorageService" + ) as MockStorageCls: + inst = MockStorageCls.return_value + inst.get_table_detail.return_value = { + "display_name": "fact_orders", + "column_details": [{"name": "AMOUNT", "type": "NUMBER"}], + } + result = service.build_model("prod", table_ids=["out.c.t"]) + assert result["fallback_used"] == "heuristic" + assert len(result["generated"]["datasets"]) == 1 + assert len(result["generated"]["metrics"]) == 1 + assert len(result["generated"]["glossary"]) == 1 + assert result["validated"] is True + + def test_dry_run_skips_post(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + with patch( + "keboola_agent_cli.services.semantic_layer_service.StorageService" + ) as MockStorageCls: + inst = MockStorageCls.return_value + inst.get_table_detail.return_value = { + "column_details": [{"name": "X", "type": "NUMBER"}], + } + result = service.build_model("prod", table_ids=["out.c.t"], dry_run=True) + assert result["dry_run"] is True + mock.post_item.assert_not_called() + + def test_empty_table_ids_rejected(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + service.build_model("prod", table_ids=[]) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_fqn_derived_for_each_dataset(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + with patch( + "keboola_agent_cli.services.semantic_layer_service.StorageService" + ) as MockStorageCls: + inst = MockStorageCls.return_value + inst.get_table_detail.return_value = { + "column_details": [{"name": "X", "type": "NUMBER"}], + } + result = service.build_model( + "prod", + table_ids=["out.c-bk.tab"], + dry_run=True, + ) + ds = result["generated"]["datasets"][0] + assert ds["fqn"] == '"KEBOOLA"."out.c-bk"."tab"' + + +# --------------------------------------------------------------------------- +# encrypt_token +# --------------------------------------------------------------------------- + + +class TestEncryptToken: + def test_uses_project_token_and_returns_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + # Patch the symbol where it's BOUND (semantic_layer_service imports + # EncryptService at module load now, so the patch must target the + # rebound name in the consumer module, not the source module). + with patch( + "keboola_agent_cli.services.semantic_layer_service.EncryptService" + ) as MockEncryptCls: + instance = MockEncryptCls.return_value + instance.encrypt.return_value = {"#metastore_token": "KBC::ProjectSecureGKMS::cipher"} + result = service.encrypt_token("prod", "keboola.ex-db-snowflake") + instance.encrypt.assert_called_once() + _, kwargs = instance.encrypt.call_args + # Validate the payload key + the actual stored token are passed. + assert kwargs["component_id"] == "keboola.ex-db-snowflake" + assert kwargs["input_data"] == {"#metastore_token": TEST_TOKEN} + assert kwargs["alias"] == "prod" + # Returned envelope contains expected fields. + assert result["component_id"] == "keboola.ex-db-snowflake" + assert result["project"] == "prod" + assert result["encrypted"]["#metastore_token"].startswith("KBC::") + + +# --------------------------------------------------------------------------- +# Module-level constants +# --------------------------------------------------------------------------- + + +class TestModuleConstants: + def test_constraint_name_regex(self) -> None: + assert CONSTRAINT_NAME_RE.match("rev_critical") + assert CONSTRAINT_NAME_RE.match("a") + assert not CONSTRAINT_NAME_RE.match("BadName") + assert not CONSTRAINT_NAME_RE.match("1bad") + assert not CONSTRAINT_NAME_RE.match("bad-name") + + def test_constraint_types_complete(self) -> None: + assert set(CONSTRAINT_TYPES) == { + "inequality", + "equality", + "range", + "composition", + "exclusion", + "temporal", + "conditional", + } + + def test_constraint_severities(self) -> None: + assert set(CONSTRAINT_SEVERITIES) == {"error", "warning", "info"} diff --git a/tests/test_server_semantic_layer_routes_e2e.py b/tests/test_server_semantic_layer_routes_e2e.py new file mode 100644 index 00000000..b27b8077 --- /dev/null +++ b/tests/test_server_semantic_layer_routes_e2e.py @@ -0,0 +1,574 @@ +"""HTTP integration tests for ``/semantic-layer/*`` routes (real metastore). + +Bootstraps a throwaway ``kbagent_e2e_<ts>`` model on ``e2e-1143`` +(``E2E_URL`` / ``E2E_API_TOKEN``), exercises every one of the 14 routes +declared in :mod:`keboola_agent_cli.server.routers.semantic_layer` against +the real ``SemanticLayerService`` (NOT mocked), and tears down in a +``finally`` block. Residue assertion at session end verifies no +``kbagent_e2e_*`` items remain across any of the 6 semantic types. + +Gated behind the same ``E2E_API_TOKEN`` + ``E2E_URL`` env vars and the +``@pytest.mark.e2e`` marker as :mod:`tests.test_e2e`, so unit-only CI +runs skip cleanly. +""" + +from __future__ import annotations + +import importlib.util +import os +import time +from collections.abc import Iterator +from typing import Any + +import pytest + +if importlib.util.find_spec("fastapi") is None: # pragma: no cover + pytest.skip( + "FastAPI not installed; run `uv pip install -e '.[server]'`", + allow_module_level=True, + ) + +from fastapi.testclient import TestClient + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.server import create_app + +ENV_TOKEN = "E2E_API_TOKEN" +ENV_URL = "E2E_URL" +HAS_CREDENTIALS = os.environ.get(ENV_TOKEN) is not None + +pytestmark = pytest.mark.e2e + + +# Session-scoped tag so every test in the suite operates on the same +# bootstrapped model. Cleanup runs once at session end. +_RUN_TAG = f"kbagent_e2e_{int(time.time())}" +_PROJECT_ALIAS = "e2e-http-sl" + + +@pytest.fixture(scope="session") +def http_session(tmp_path_factory: pytest.TempPathFactory) -> Iterator[dict[str, Any]]: + """Bootstrap a semantic-layer model + return a TestClient + tracking dict. + + Mirrors :class:`tests.test_e2e.TestE2ESemanticLayerLifecycle` setup but + holds artifacts at session scope so every test_* function in this + module shares one bootstrapped model. + """ + if not HAS_CREDENTIALS: + pytest.skip(f"{ENV_TOKEN} not set") + + token = os.environ[ENV_TOKEN] + raw_url = os.environ.get(ENV_URL, "connection.keboola.com") + url = raw_url if raw_url.startswith("https://") else f"https://{raw_url}" + + config_dir = tmp_path_factory.mktemp("kbagent-sl-http-config") + store = ConfigStore(config_dir=config_dir) + store.add_project( + _PROJECT_ALIAS, + ProjectConfig(stack_url=url, token=token), + ) + + app = create_app(config_dir=str(config_dir), auth_token="test-http-token") + client = TestClient(app) + + state: dict[str, Any] = { + "client": client, + "tag": _RUN_TAG, + "model_id": None, + "model_name": _RUN_TAG, + "url": url, + "token": token, + "config_dir": config_dir, + # Tracked items for guaranteed teardown. + "created_items": [], # list[tuple[item_type, item_id]] + } + + try: + yield state + finally: + # ── teardown ────────────────────────────────────────────── + _cleanup(state) + + +def _auth() -> dict[str, str]: + return {"Authorization": "Bearer test-http-token"} + + +def _direct_delete(state: dict[str, Any], item_type: str, item_id: str) -> None: + """Direct DELETE via MetastoreClient, bypassing the service.""" + from keboola_agent_cli.metastore_client import MetastoreClient + + with MetastoreClient(stack_url=state["url"], token=state["token"]) as mc: + mc.delete_item(item_type, item_id) # type: ignore[arg-type] + + +def _cleanup(state: dict[str, Any]) -> None: + """Teardown: direct-delete every tracked item + the model, then residue scan.""" + from keboola_agent_cli.errors import KeboolaApiError + from keboola_agent_cli.metastore_client import SEMANTIC_TYPES, MetastoreClient + + print("\n--- SEMANTIC LAYER HTTP CLEANUP ---") + for item_type, item_id in reversed(state["created_items"]): + try: + _direct_delete(state, item_type, item_id) + print(f" Deleted {item_type} {item_id}") + except Exception as exc: + print(f" WARN: failed to delete {item_type} {item_id}: {exc}") + + if state.get("model_id"): + try: + _direct_delete(state, "semantic-model", state["model_id"]) + print(f" Deleted semantic-model {state['model_id']}") + except Exception as exc: + print(f" WARN: failed to delete semantic-model: {exc}") + + # Session-end residue scan + try: + with MetastoreClient(stack_url=state["url"], token=state["token"]) as mc: + residue: list[str] = [] + for stype in SEMANTIC_TYPES: + for item in mc.list_items(stype): # type: ignore[arg-type] + attrs = item.get("attributes") or {} + name = attrs.get("name") or attrs.get("term", "") + if isinstance(name, str) and name.startswith(state["tag"]): + residue.append(f"{stype}:{name}:{item.get('id', '')}") + assert not residue, f"Residue left after HTTP integration suite: {residue}" + except KeboolaApiError as exc: + print(f" WARN: residue scan failed: {exc}") + + +# ── 14 happy-path tests (one per declared route) ──────────────────── + + +def test_post_models_create(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/models — create the bootstrap model.""" + client = http_session["client"] + res = client.post( + "/semantic-layer/models", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "name": http_session["model_name"], + "description": "kbagent http integration bootstrap", + "sql_dialect": "Snowflake", + }, + ) + assert res.status_code == 200, res.text + body = res.json() + http_session["model_id"] = body["model"]["id"] + assert http_session["model_id"] + + +def test_get_models_list(http_session: dict[str, Any]) -> None: + """GET /semantic-layer/models — should include the bootstrapped model.""" + res = http_session["client"].get( + f"/semantic-layer/models?project={_PROJECT_ALIAS}", headers=_auth() + ) + assert res.status_code == 200, res.text + body = res.json() + names = {m["name"] for m in body["models"]} + assert http_session["model_name"] in names + + +def test_post_items_add_dataset(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/items/dataset — adds the first dataset.""" + res = http_session["client"].post( + "/semantic-layer/items/dataset", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "model": http_session["model_name"], + "name": f"{http_session['tag']}_ds_a", + "table_id": "out.c-syn.fact_a", + }, + ) + assert res.status_code == 200, res.text + body = res.json() + item_id = body["id"] + http_session["created_items"].append(("semantic-dataset", item_id)) + http_session["dataset_a_id"] = item_id + # Add a second dataset for later relationship test. + res2 = http_session["client"].post( + "/semantic-layer/items/dataset", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "model": http_session["model_name"], + "name": f"{http_session['tag']}_ds_b", + "table_id": "out.c-syn.fact_b", + }, + ) + assert res2.status_code == 200, res2.text + http_session["created_items"].append(("semantic-dataset", res2.json()["id"])) + + +def test_post_items_add_metric(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/items/metric — adds a metric for later edits.""" + res = http_session["client"].post( + "/semantic-layer/items/metric", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "model": http_session["model_name"], + "name": f"{http_session['tag']}_m_rev", + "sql": "COUNT(*)", + "dataset": "out.c-syn.fact_a", + }, + ) + assert res.status_code == 200, res.text + body = res.json() + item_id = body["id"] + http_session["created_items"].append(("semantic-metric", item_id)) + http_session["metric_rev_id"] = item_id + + +def test_post_items_add_relationship(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/items/relationship — uses `from` alias in body.""" + res = http_session["client"].post( + "/semantic-layer/items/relationship", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "model": http_session["model_name"], + "name": f"{http_session['tag']}_rel_a_b", + "from": "out.c-syn.fact_a", + "to": "out.c-syn.fact_b", + "on": "fact_a.id = fact_b.fact_a_id", + "type": "left", + }, + ) + assert res.status_code == 200, res.text + item_id = res.json()["id"] + http_session["created_items"].append(("semantic-relationship", item_id)) + + +def test_post_items_add_constraint(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/items/constraint — references the metric just added.""" + res = http_session["client"].post( + "/semantic-layer/items/constraint", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "model": http_session["model_name"], + "name": f"{http_session['tag']}_warn", + "constraint_type": "inequality", + "rule": "value >= 0", + "metrics": [f"{http_session['tag']}_m_rev"], + "severity": "warning", + }, + ) + assert res.status_code == 200, res.text + item_id = res.json()["id"] + http_session["created_items"].append(("semantic-constraint", item_id)) + + +def test_post_items_add_glossary(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/items/glossary — outer envelope name = term.""" + res = http_session["client"].post( + "/semantic-layer/items/glossary", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "model": http_session["model_name"], + "term": f"{http_session['tag']}_term", + "definition": "an integration test definition", + }, + ) + assert res.status_code == 200, res.text + item_id = res.json()["id"] + http_session["created_items"].append(("semantic-glossary", item_id)) + + +def test_get_show(http_session: dict[str, Any]) -> None: + """GET /semantic-layer/show — assert child counts.""" + res = http_session["client"].get( + f"/semantic-layer/show?project={_PROJECT_ALIAS}&model={http_session['model_name']}", + headers=_auth(), + ) + assert res.status_code == 200, res.text + body = res.json() + assert len(body["datasets"]) >= 2 + assert len(body["metrics"]) >= 1 + assert len(body["relationships"]) >= 1 + assert len(body["constraints"]) >= 1 + assert len(body["glossary"]) >= 1 + + +def test_get_validate(http_session: dict[str, Any]) -> None: + """GET /semantic-layer/validate — basic checks, expect clean.""" + res = http_session["client"].get( + f"/semantic-layer/validate?project={_PROJECT_ALIAS}&model={http_session['model_name']}", + headers=_auth(), + ) + assert res.status_code == 200, res.text + body = res.json() + assert body["valid"] is True, f"Expected clean model, got errors: {body['errors']}" + + +def test_get_export(http_session: dict[str, Any]) -> None: + """GET /semantic-layer/export — snapshot returned inline (no `path` field).""" + res = http_session["client"].get( + f"/semantic-layer/export?project={_PROJECT_ALIAS}&model={http_session['model_name']}", + headers=_auth(), + ) + assert res.status_code == 200, res.text + body = res.json() + assert "path" not in body, "Export should NOT echo a server-side tmp path" + assert "datasets" in body + assert "metrics" in body + # Stash for the diff + import tests. + http_session["exported_snapshot"] = body + + +def test_put_items_edit_metric(http_session: dict[str, Any]) -> None: + """PUT /semantic-layer/items/metric/{name} — rename + assert cascade.""" + res = http_session["client"].put( + f"/semantic-layer/items/metric/{http_session['tag']}_m_rev", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "model": http_session["model_name"], + "new_name": f"{http_session['tag']}_m_revenue", + }, + ) + assert res.status_code == 200, res.text + body = res.json() + new_id = body["updated"]["id"] + # Rename leaves a fresh metric id and DELETE+POSTs the constraint — + # refresh tracking so cleanup hits the right rows. + http_session["created_items"] = [ + (t, i) + for (t, i) in http_session["created_items"] + if not (t == "semantic-metric" and i == http_session["metric_rev_id"]) + ] + http_session["created_items"].append(("semantic-metric", new_id)) + # Refresh constraint id (cascade DELETE+POST). + show = http_session["client"].get( + f"/semantic-layer/show?project={_PROJECT_ALIAS}" + f"&model={http_session['model_name']}&type=constraint", + headers=_auth(), + ) + http_session["created_items"] = [ + (t, i) for (t, i) in http_session["created_items"] if t != "semantic-constraint" + ] + for c in show.json()["constraints"]: + http_session["created_items"].append(("semantic-constraint", c["id"])) + cascaded = body["cascaded_constraints"] + assert any(c["status"] == "updated" for c in cascaded), f"Expected cascade, got: {cascaded}" + + +def test_post_diff_project_vs_file(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/diff — file_b = the prior exported snapshot. + + Diff against the live state after rename — should show the metric + rename as a change (m_rev removed, m_revenue added). This proves the + inline-file branch of the body validator works. + """ + snapshot = http_session.get("exported_snapshot") + # Strip the model wrapping that export adds (just keep the diffable + # bare child lists — diff service expects raw side data). + snapshot_for_file = { + "datasets": snapshot["datasets"], + "metrics": snapshot["metrics"], + "relationships": snapshot["relationships"], + "constraints": snapshot["constraints"], + "glossary": snapshot["glossary"], + "model": snapshot["model"], + } + res = http_session["client"].post( + "/semantic-layer/diff", + headers=_auth(), + json={ + "project_a": _PROJECT_ALIAS, + "model_a": http_session["model_name"], + "file_b": snapshot_for_file, + }, + ) + assert res.status_code == 200, res.text + body = res.json() + # The metric rename is the only post-export change → diff is non-empty. + metric_diff = body["metrics"] + assert metric_diff["added"] or metric_diff["removed"] or metric_diff["changed"], ( + f"Expected metric diff after rename, got: {metric_diff}" + ) + + +def test_post_import_dry_run(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/import — dry-run reuses the exported snapshot.""" + snapshot = http_session.get("exported_snapshot") + res = http_session["client"].post( + "/semantic-layer/import", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "model": http_session["model_name"], + "snapshot": snapshot, + "dry_run": True, + }, + ) + assert res.status_code == 200, res.text + body = res.json() + assert body["dry_run"] is True + assert "imported" in body + + +def test_post_promote_dry_run(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/promote — dry-run from the model to itself. + + Self-promotion is a degenerate case the service handles gracefully: + every item classifies as IDENTICAL (no overwrite, no new). That's + enough to prove the route + body validation reach the service. + """ + res = http_session["client"].post( + "/semantic-layer/promote", + headers=_auth(), + json={ + "from_project": _PROJECT_ALIAS, + "to_project": _PROJECT_ALIAS, + "from_model": http_session["model_name"], + "to_model": http_session["model_name"], + "dry_run": True, + }, + ) + assert res.status_code == 200, res.text + body = res.json() + assert body["dry_run"] is True + + +def test_post_build_dry_run(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/build — dry-run heuristic builder. + + The build path needs Storage schemas — provide a synthetic tableId + that the service will attempt to fetch. The fetch failure is + captured in ``fetch_errors`` (the route still returns 200; that's + sufficient signal for "route mounted + body validates"). + """ + res = http_session["client"].post( + "/semantic-layer/build", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "tables": ["out.c-syn.fact_a"], + "name": f"{http_session['tag']}_build_target", + "dry_run": True, + }, + ) + assert res.status_code == 200, res.text + body = res.json() + assert body["dry_run"] is True + assert body["fallback_used"] == "heuristic" + + +def test_post_token_encrypt(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/token/encrypt — KBC::ProjectSecure envelope.""" + res = http_session["client"].post( + "/semantic-layer/token/encrypt", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "component_id": "keboola.ex-db-snowflake", + }, + ) + assert res.status_code == 200, res.text + body = res.json() + token_field = body["encrypted"].get("#metastore_token", "") + assert token_field.startswith("KBC::ProjectSecure"), ( + f"Expected KBC::ProjectSecure envelope, got: {token_field[:50]}" + ) + + +def test_delete_items_remove_glossary(http_session: dict[str, Any]) -> None: + """DELETE /semantic-layer/items/glossary/{term} — removes the glossary entry.""" + term = f"{http_session['tag']}_term" + res = http_session["client"].delete( + f"/semantic-layer/items/glossary/{term}" + f"?project={_PROJECT_ALIAS}&model={http_session['model_name']}", + headers=_auth(), + ) + assert res.status_code == 200, res.text + body = res.json() + assert body["removed"]["name"] == term + http_session["created_items"] = [ + (t, i) for (t, i) in http_session["created_items"] if t != "semantic-glossary" + ] + + +# Note on `DELETE /semantic-layer/models/{model}`: deletion is exercised +# as part of the session-end cleanup (the model is direct-deleted via +# MetastoreClient if its row still exists). Adding an explicit test +# here would clobber the bootstrapped model mid-suite. + + +# ── Negative paths ────────────────────────────────────────────────── + + +def test_negative_unknown_project_returns_400(http_session: dict[str, Any]) -> None: + """Missing project alias → CONFIG_ERROR → HTTP 400.""" + res = http_session["client"].get( + "/semantic-layer/models?project=does-not-exist-alias", + headers=_auth(), + ) + assert res.status_code == 400 + assert res.json()["error"]["code"] == "CONFIG_ERROR" + + +def test_negative_unknown_kind_returns_404(http_session: dict[str, Any]) -> None: + """POST /semantic-layer/items/<unknown> → 404.""" + res = http_session["client"].post( + "/semantic-layer/items/widget", + headers=_auth(), + json={"project": _PROJECT_ALIAS, "name": "x"}, + ) + assert res.status_code == 404 + + +def test_negative_invalid_constraint_name_returns_4xx( + http_session: dict[str, Any], +) -> None: + """Constraint name with uppercase → VALIDATION_ERROR via service. + + Service raises VALIDATION_ERROR (KeboolaApiError) which the global + handler maps to 502 with the code intact. The test asserts the wire + code matches our enum. + """ + res = http_session["client"].post( + "/semantic-layer/items/constraint", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "model": http_session["model_name"], + "name": "BadName", # uppercase rejected by regex + "constraint_type": "range", + "rule": "between 0 and 100", + "metrics": [f"{http_session['tag']}_m_revenue"], + }, + ) + # Service-side validation → 502 (KeboolaApiError → HTTP_502_via_handler). + assert res.status_code in (400, 422, 502) + assert "constraint" in res.text.lower() or "name" in res.text.lower() + + +def test_negative_duplicate_name_maps_to_already_exists( + http_session: dict[str, Any], +) -> None: + """Duplicate model name → service translates 500 to ALREADY_EXISTS.""" + res = http_session["client"].post( + "/semantic-layer/models", + headers=_auth(), + json={ + "project": _PROJECT_ALIAS, + "name": http_session["model_name"], # already exists + "description": "duplicate", + "sql_dialect": "Snowflake", + }, + ) + # KeboolaApiError(ALREADY_EXISTS) → HTTP 502 envelope (per global + # handler in server/__init__.py), with the canonical code in the + # error.code field. + assert res.status_code in (400, 409, 502) + body = res.json() + assert body.get("error", {}).get("code") in ( + "ALREADY_EXISTS", + "API_ERROR", + ), body diff --git a/tests/test_server_smoke.py b/tests/test_server_smoke.py index 30bb26cf..e137d746 100644 --- a/tests/test_server_smoke.py +++ b/tests/test_server_smoke.py @@ -44,6 +44,18 @@ "/kai/ping", "/encrypt/values", "/search", + "/semantic-layer/models", + "/semantic-layer/models/{model}", + "/semantic-layer/show", + "/semantic-layer/validate", + "/semantic-layer/export", + "/semantic-layer/diff", + "/semantic-layer/items/{kind}", + "/semantic-layer/items/{kind}/{name}", + "/semantic-layer/import", + "/semantic-layer/promote", + "/semantic-layer/build", + "/semantic-layer/token/encrypt", "/org/setup", "/members/{project}", "/version", @@ -109,3 +121,108 @@ def test_doctor_runs(client: TestClient) -> None: body = res.json() assert "checks" in body assert "summary" in body + + +# ── Semantic-layer route smoke tests (no metastore creds required) ── +# +# These cover route-presence + Pydantic body validation only. The real +# end-to-end coverage lives in tests/test_server_semantic_layer_routes_e2e.py +# behind the @pytest.mark.e2e marker. + + +def test_semantic_layer_diff_rejects_both_project_and_file(client: TestClient) -> None: + """DiffRequest enforces exactly-one-per-side via model_validator.""" + res = client.post( + "/semantic-layer/diff", + headers={"Authorization": "Bearer test-token"}, + json={"project_a": "p", "file_a": {"x": 1}, "project_b": "q"}, + ) + assert res.status_code == 422 + + +def test_semantic_layer_diff_rejects_neither_side(client: TestClient) -> None: + res = client.post( + "/semantic-layer/diff", + headers={"Authorization": "Bearer test-token"}, + json={"project_b": "q"}, + ) + assert res.status_code == 422 + + +def test_semantic_layer_items_unknown_kind_post_404(client: TestClient) -> None: + """POST /items/{kind} with an unsupported kind returns 404.""" + res = client.post( + "/semantic-layer/items/widget", + headers={"Authorization": "Bearer test-token"}, + json={"project": "x", "name": "n"}, + ) + assert res.status_code == 404 + assert "kind" in res.json()["error"]["message"].lower() + + +def test_semantic_layer_items_unknown_kind_put_404(client: TestClient) -> None: + """PUT /items/{kind}/{name} with an unsupported kind returns 404.""" + res = client.put( + "/semantic-layer/items/widget/n", + headers={"Authorization": "Bearer test-token"}, + json={"project": "x"}, + ) + assert res.status_code == 404 + + +def test_semantic_layer_models_missing_project_returns_4xx(client: TestClient) -> None: + """GET /models with a missing project alias falls into CONFIG_ERROR.""" + res = client.get( + "/semantic-layer/models?project=does-not-exist", + headers={"Authorization": "Bearer test-token"}, + ) + # ConfigError → 400 (via _config_error_handler in server/__init__.py). + assert res.status_code == 400 + assert res.json()["error"]["code"] == "CONFIG_ERROR" + + +def test_semantic_layer_add_constraint_rejects_bad_name(client: TestClient) -> None: + """Constraint name regex enforced via service-layer validation. + + Server returns the constraint-name regex error as a 502 + KeboolaApiError envelope (the metastore service raises + VALIDATION_ERROR which the global handler maps to 502 with the + error_code field intact). The router itself returns 200 only on + real success. + """ + res = client.post( + "/semantic-layer/items/constraint", + headers={"Authorization": "Bearer test-token"}, + json={ + "project": "no-such-project-alias", + "name": "BadName", # uppercase — would fail constraint regex + "constraint_type": "range", + "rule": "between 0 and 100", + "metrics": ["m1"], + }, + ) + # Project resolution comes first → CONFIG_ERROR (400). Adequate to + # demonstrate the route is reachable + Pydantic body validates. + assert res.status_code in (400, 422, 502) + + +def test_semantic_layer_token_encrypt_requires_body(client: TestClient) -> None: + """POST /token/encrypt with no body → 422 (missing required project + component_id).""" + res = client.post( + "/semantic-layer/token/encrypt", + headers={"Authorization": "Bearer test-token"}, + json={}, + ) + assert res.status_code == 422 + + +def test_semantic_layer_routes_require_auth(client: TestClient) -> None: + """Every semantic-layer route is auth-gated (none in PUBLIC_PATHS).""" + for path in ( + "/semantic-layer/models?project=p", + "/semantic-layer/show?project=p", + "/semantic-layer/validate?project=p", + "/semantic-layer/export?project=p", + ): + res = client.get(path) + assert res.status_code == 401, f"{path} should require auth" diff --git a/uv.lock b/uv.lock index ea25a938..290cb501 100644 --- a/uv.lock +++ b/uv.lock @@ -496,7 +496,7 @@ wheels = [ [[package]] name = "keboola-agent-cli" -version = "0.40.3" +version = "0.41.0" source = { editable = "." } dependencies = [ { name = "httpx" }, From 84ce410792e130e689f63b303f14bc363382878a Mon Sep 17 00:00:00 2001 From: Petr <petr@keboola.com> Date: Fri, 15 May 2026 13:34:15 +0200 Subject: [PATCH 2/2] fix: address kbagent-pr-reviewer findings (B-1 + NB-1..3 + NIT-1..3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - B-1: drop four thin-delegate wrappers in SemanticLayerService (_validate_basic, _diff_one_type, _compare_attrs, _heuristic_generate_model) and call the _semantic_layer_internals helpers directly. semantic_layer_service.py: 1519 → 1473 LOC (now 27 LOC under the 1500 hard ceiling from CONTRIBUTING.md). - NB-1: rewrite server/routers/semantic_layer.py module docstring to accurately describe the 14-routes-for-26-commands collapse via the {kind} path parameter; flag the deliberate departure from the CONTRIBUTING.md "1:1" guideline. - NB-2: add an explanatory block comment above the except Exception in _fetch_children_parallel — Future.result() re-raises arbitrary worker exceptions, so a broad catch is the correct pattern here. - NB-3: hoist tempfile, contextlib, and `json as _json` from function-body inline imports to module-level imports. - NIT-1: PR description LOC claim addressed alongside this commit via gh pr edit (now matches the committed 1473 LOC). - NIT-2: change kind: str to kind: ItemKind on add_item / edit_item / remove_item path params. FastAPI now rejects unknown kinds at the framework layer with 422 (was 404 from a manual fallback). Tests in test_server_smoke.py updated to assert 422. - NIT-3: expand the inline comment in delete_model explaining why the naive plural rule produces "glossarys" and how it is corrected. --- .../server/routers/semantic_layer.py | 31 +++++-- .../services/semantic_layer_service.py | 92 +++++-------------- tests/test_server_smoke.py | 21 +++-- 3 files changed, 58 insertions(+), 86 deletions(-) diff --git a/src/keboola_agent_cli/server/routers/semantic_layer.py b/src/keboola_agent_cli/server/routers/semantic_layer.py index e4ef2b53..99edf075 100644 --- a/src/keboola_agent_cli/server/routers/semantic_layer.py +++ b/src/keboola_agent_cli/server/routers/semantic_layer.py @@ -1,7 +1,14 @@ -"""Semantic-layer endpoints — 1:1 mapping of the ``kbagent semantic-layer`` CLI. +"""Semantic-layer endpoints — 14 routes covering 26 CLI subcommands. Mirrors the per-subcommand surface of :class:`keboola_agent_cli.services.semantic_layer_service.SemanticLayerService`. +The 15 ``add.*``/``edit.*``/``remove.*`` CLI leaves collapse into three +``{kind}``-parameterized routes (``POST/PUT/DELETE /items/{kind}``) for +RESTful semantics; per-kind Pydantic body validation happens inside the +handler. This is a deliberate departure from the CONTRIBUTING.md "1:1 +endpoint per command" guideline — the parameterization preserves +discoverability while keeping the route count manageable. + Pattern: Pydantic body models for routes that the CLI flags with multiple options, query parameters for read endpoints. ``--yes`` is implicit on every REST destructive call (the body / DELETE request IS the confirmation). @@ -9,6 +16,9 @@ from __future__ import annotations +import contextlib +import json as _json +import tempfile from pathlib import Path from typing import Any, Literal @@ -261,7 +271,6 @@ def export( # transient path. Simpler: route through the service and let the # caller ignore the `path` field. Using a tmp-dir export keeps the # service contract intact while avoiding pollution of the CWD. - import tempfile with tempfile.TemporaryDirectory(prefix="kbagent-sl-export-") as tmp: out = Path(tmp) / "snapshot.json" @@ -282,9 +291,6 @@ def diff(body: DiffRequest, registry: ServiceRegistry = Depends(get_registry)) - ``file_b``). When a file side is set we serialize it to a temp file so the existing service contract (``Path``) keeps working. """ - import contextlib - import json as _json - import tempfile def _write_tmp(payload: dict[str, Any]) -> Path: # `delete=False` is required because we close the handle before @@ -328,7 +334,7 @@ def _write_tmp(payload: dict[str, Any]) -> Path: @router.post("/items/{kind}") def add_item( - kind: str, + kind: ItemKind, body: dict[str, Any], registry: ServiceRegistry = Depends(get_registry), ) -> dict[str, Any]: @@ -336,6 +342,8 @@ def add_item( Per-kind Pydantic body validation is done downstream by binding the raw body to the right model before delegating to the service. + FastAPI rejects unknown ``kind`` values at the framework layer (422) + via the :data:`ItemKind` ``Literal`` alias. """ svc = registry.semantic_layer if kind == "metric": @@ -403,7 +411,7 @@ def add_item( @router.put("/items/{kind}/{name}") def edit_item( - kind: str, + kind: ItemKind, name: str, body: dict[str, Any], registry: ServiceRegistry = Depends(get_registry), @@ -411,7 +419,8 @@ def edit_item( """Edit an entity. ``name`` is the current identifier; new-* fields live in the body. For ``kind="glossary"``, ``name`` is the current ``term`` (not a stored - field called ``name``). + field called ``name``). FastAPI rejects unknown ``kind`` values at the + framework layer (422) via the :data:`ItemKind` ``Literal`` alias. """ svc = registry.semantic_layer if kind == "metric": @@ -481,7 +490,7 @@ def edit_item( @router.delete("/items/{kind}/{name}") def remove_item( - kind: str, + kind: ItemKind, name: str, project: str, model: str | None = None, @@ -489,7 +498,9 @@ def remove_item( ) -> dict[str, Any]: """Remove a child entity (``--yes`` implicit on REST). - For ``kind="glossary"``, ``name`` is the term to remove. + For ``kind="glossary"``, ``name`` is the term to remove. FastAPI + rejects unknown ``kind`` values at the framework layer (422) via the + :data:`ItemKind` ``Literal`` alias. """ return registry.semantic_layer.remove_item( alias=project, model_name_or_uuid=model, kind=kind, name=name diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index 0148651d..6dd40621 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -35,7 +35,6 @@ from ._semantic_layer_crud import validate_constraint_attrs as _validate_constraint_attrs from ._semantic_layer_internals import build_export_snapshot as _build_export_snapshot from ._semantic_layer_internals import collect_side_from_file -from ._semantic_layer_internals import compare_attrs as _compare_attrs_helper from ._semantic_layer_internals import default_export_path as _default_export_path from ._semantic_layer_internals import diff_one_type as _diff_one_type_helper from ._semantic_layer_internals import fetch_table_schemas as _fetch_table_schemas @@ -302,6 +301,9 @@ def _fetch_children_parallel( for future in future_to_type: try: results[future_to_type[future]] = future.result() + # Future.result() re-raises arbitrary worker exceptions + # (KeboolaApiError, httpx errors, etc.); collect and + # surface the first one rather than masking the others. except Exception as exc: errors.append(exc) if errors: @@ -389,7 +391,7 @@ def validate_model( errors: list[dict[str, str]] = [] warnings: list[dict[str, str]] = [] - self._validate_basic( + _validate_basic_helper( datasets=datasets, metrics=metrics, relationships=relationships, @@ -419,32 +421,10 @@ def validate_model( # -- validate internals -------------------------------------------- - @staticmethod - def _validate_basic( - *, - datasets: list[dict[str, Any]], - metrics: list[dict[str, Any]], - relationships: list[dict[str, Any]], - constraints: list[dict[str, Any]], - glossary: list[dict[str, Any]], - errors: list[dict[str, str]], - warnings: list[dict[str, str]], - ) -> None: - """Pure in-memory validation (no API calls). - - Thin delegate to :func:`._semantic_layer_internals.validate_basic` - -- the heavy lifting lives in the internals module so this file - stays under the CONTRIBUTING.md services budget. - """ - _validate_basic_helper( - datasets=datasets, - metrics=metrics, - relationships=relationships, - constraints=constraints, - glossary=glossary, - errors=errors, - warnings=warnings, - ) + # Pure in-memory validation lives in + # :func:`._semantic_layer_internals.validate_basic` (imported above as + # ``_validate_basic_helper``); call it directly from ``validate_model`` + # / ``build_model`` rather than via a thin wrapper. def _validate_deep( self, @@ -568,7 +548,7 @@ def diff( ("constraints", "name"), ("glossary", "term"), ): - result[type_key] = self._diff_one_type( + result[type_key] = _diff_one_type_helper( left["data"].get(type_key, []), right["data"].get(type_key, []), id_key=id_key, @@ -601,23 +581,9 @@ def _collect_side( ) return collect_side_from_file(file) - def _diff_one_type( - self, - left: list[dict[str, Any]], - right: list[dict[str, Any]], - *, - id_key: str, - ) -> dict[str, Any]: - """Thin wrapper around :func:`._semantic_layer_internals.diff_one_type`.""" - return _diff_one_type_helper(left, right, id_key=id_key) - - def _compare_attrs( - self, - a: dict[str, Any], - b: dict[str, Any], - ) -> list[str]: - """Thin wrapper around :func:`._semantic_layer_internals.compare_attrs`.""" - return _compare_attrs_helper(a, b) + # Per-type diff lives in :func:`._semantic_layer_internals.diff_one_type` + # (imported above as ``_diff_one_type_helper``); call it directly. + # ``compare_attrs`` is used inside that helper and not separately here. # ------------------------------------------------------------------ # Phase 4 — Model lifecycle (create / delete) @@ -661,8 +627,11 @@ def delete_model( client.delete_item("semantic-model", model_uuid) finally: client.close() + # Pluralise the bare keys (semantic-dataset -> "datasets", etc.). + # The naive ``key + "s"`` rule produces "glossarys" for the one + # already-plural type; fold it back to "glossary" so the wire shape + # matches the rest of the codebase (see ``_PLURAL_BY_TYPE`` above). counts = {k.replace("semantic-", "") + "s": len(v) for k, v in children.items()} - # "semantic-glossary" -> "glossarys" — normalise the only odd one. counts.setdefault("glossary", counts.pop("glossarys", 0)) return { "project": alias, @@ -1403,16 +1372,20 @@ def build_model( ) schemas_by_tid, fetch_errors = _fetch_table_schemas(storage, alias, table_ids) - # Generate model JSON (heuristic). - generated = self._heuristic_generate_model( + # Generate model JSON (heuristic). The internals helper takes the + # FQN-derivation and role-classification callables as kwargs so its + # module stays free of import-time coupling to this module. + generated = _heuristic_generate_helper( schemas=schemas_by_tid, model_name=model_name or "kbagent_build_model", + derive_fqn=_derive_fqn, + classify_role=_classify_field_role, ) # Validate locally. errors: list[dict[str, str]] = [] warnings: list[dict[str, str]] = [] - self._validate_basic( + _validate_basic_helper( datasets=generated["datasets"], metrics=generated["metrics"], relationships=generated["relationships"], @@ -1467,25 +1440,6 @@ def build_model( finally: client.close() - @staticmethod - def _heuristic_generate_model( - *, - schemas: dict[str, dict[str, Any]], - model_name: str, - ) -> dict[str, Any]: - """Thin delegate to :func:`._semantic_layer_internals.heuristic_generate_model`. - - Passes the module's FQN-derivation and role-classification - helpers so the internals module stays free of import-time - coupling to this module's regex constants. - """ - return _heuristic_generate_helper( - schemas=schemas, - model_name=model_name, - derive_fqn=_derive_fqn, - classify_role=_classify_field_role, - ) - # ------------------------------------------------------------------ # Phase 8 — token --encrypt # ------------------------------------------------------------------ diff --git a/tests/test_server_smoke.py b/tests/test_server_smoke.py index e137d746..b337d6b7 100644 --- a/tests/test_server_smoke.py +++ b/tests/test_server_smoke.py @@ -149,25 +149,32 @@ def test_semantic_layer_diff_rejects_neither_side(client: TestClient) -> None: assert res.status_code == 422 -def test_semantic_layer_items_unknown_kind_post_404(client: TestClient) -> None: - """POST /items/{kind} with an unsupported kind returns 404.""" +def test_semantic_layer_items_unknown_kind_post_422(client: TestClient) -> None: + """POST /items/{kind} with an unsupported kind returns 422. + + FastAPI rejects unknown ``kind`` path values at the framework layer + via the ``ItemKind`` ``Literal`` alias, before reaching the handler. + """ res = client.post( "/semantic-layer/items/widget", headers={"Authorization": "Bearer test-token"}, json={"project": "x", "name": "n"}, ) - assert res.status_code == 404 - assert "kind" in res.json()["error"]["message"].lower() + assert res.status_code == 422 -def test_semantic_layer_items_unknown_kind_put_404(client: TestClient) -> None: - """PUT /items/{kind}/{name} with an unsupported kind returns 404.""" +def test_semantic_layer_items_unknown_kind_put_422(client: TestClient) -> None: + """PUT /items/{kind}/{name} with an unsupported kind returns 422. + + FastAPI rejects unknown ``kind`` path values at the framework layer + via the ``ItemKind`` ``Literal`` alias, before reaching the handler. + """ res = client.put( "/semantic-layer/items/widget/n", headers={"Authorization": "Bearer test-token"}, json={"project": "x"}, ) - assert res.status_code == 404 + assert res.status_code == 422 def test_semantic_layer_models_missing_project_returns_4xx(client: TestClient) -> None: