diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 47dc852d..a5e090e8 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "kbagent", - "version": "0.46.1", + "version": "0.47.0", "source": "./plugins/kbagent", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "category": "development" diff --git a/CLAUDE.md b/CLAUDE.md index 8ab8ec2e..0fc7dc34 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -346,7 +346,7 @@ kbagent storage bucket-detail --project NAME --bucket-id ID [--branch ID] kbagent storage tables [--project NAME ...] [--bucket-id ID] [--branch ID] kbagent storage table-detail --project NAME --table-id ID [--branch ID] kbagent storage create-bucket --project NAME --stage STAGE --name NAME [--description D] [--backend B] [--branch ID] -kbagent storage create-table --project NAME --bucket-id ID --name NAME --column COL:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID] +kbagent storage create-table --project NAME --bucket-id ID --name NAME --column COL:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID] [--if-not-exists] kbagent storage upload-table --project NAME --table-id ID --file PATH [--incremental] [--branch ID] kbagent storage download-table --project NAME --table-id ID [--output FILE] [--columns COL ...] [--limit N] [--branch ID] kbagent storage delete-table --project NAME --table-id ID [--table-id ...] [--force] [--dry-run] [--yes] [--branch ID] @@ -432,6 +432,8 @@ kbagent semantic-layer model list --project P kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] kbagent semantic-layer model delete --project P --model M [--yes] kbagent semantic-layer show --project P [--model M] [--type dataset|metric|relationship|constraint|glossary] +kbagent semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N] +kbagent semantic-layer get-context --project P --context-id ID kbagent semantic-layer validate --project P [--model M] [--deep] kbagent semantic-layer export --project P [--model M] [--output PATH] kbagent semantic-layer diff (--project-a A | --file-a PATH) (--project-b B | --file-b PATH) [--model-a M] [--model-b M] diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index 159ba29d..9e924d60 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.46.1", + "version": "0.47.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 4f9c0e64..099f6677 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -91,13 +91,8 @@ a critical failure. `edit metric|dataset|constraint|relationship|glossary`, `import`, `promote`, `build`, `token --encrypt` - destructive: `remove metric|dataset|constraint|relationship|glossary` - - alias: `kbagent sl ...` is hidden-equivalent to - `kbagent semantic-layer ...` - - `semantic-layer build` falls back to a deterministic heuristic - (one dataset + one COUNT(*) metric + one glossary entry per - table) until an AI Service JSON-generation endpoint exists; - this is a BEHAVIOR note, not a version gate -- the heuristic - is the only path on 0.41.0, + - alias: `kbagent sl ...` = `kbagent semantic-layer ...` + - `semantic-layer build` is heuristic-only on 0.41.0+ (one dataset + one COUNT(*) metric + one glossary entry per table; not a version gate), `kbagent http get/post/patch/delete ` (self-call against the running serve from a scheduled-agent subprocess; reads `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` env vars) needs 0.40.0+, @@ -115,6 +110,7 @@ a critical failure. `kbagent update --beta` = 0.43.3+, `data-app logs` = 0.43.8+, `kbagent agent ` (CLI parity /agents REST) = 0.44.0+, + `semantic-layer search-context|get-context`, `storage create-table --if-not-exists`, `sync push|pull|diff --branch`, `sync push --no-name-drift-warnings`, fresh-CREATE writeback + KBC.* = 0.47.0+, `storage retype` is a future composite), you MUST refuse the task and return a handoff message to the parent: `"Cannot proceed safely on kbagent . Missing: . @@ -183,7 +179,7 @@ a critical failure. | Call the running `kbagent serve` from a scheduled-agent subprocess | `kbagent http get/post/patch/delete ` (0.40.0+) -- uses `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` env vars auto-injected by the scheduler. `kbagent http get /openapi.json` to discover endpoints. Treats the live serve as source-of-truth (no stale local config) | forking `kbagent ` (also fine -- `KBAGENT_CONFIG_DIR` is propagated so the spawned CLI sees the SAME config the serve uses; no more "I'm in the wrong directory" surprises) | `curl $KBAGENT_SERVE_URL/...` by hand (works, but `kbagent http` adds auth header automatically, structured error mapping, and JSON-mode formatting) | | Launch the web UI for an end-user (browser dashboard, no Node BFF) | `kbagent serve --ui [--port PORT] [--ui-dist PATH]` (0.40.0+) -- single-process FastAPI mounts the bundled React SPA at `/`, sets an HttpOnly `kbagent_session` cookie on `GET /` so the browser is auto-authenticated. EventSource SSE works via the same cookie -- no token in URL, JS heap, or access log. Requires the bundled wheel (Node 20+ on the install host) OR `make web-build` from a checkout. CORS origins customisable via `--cors-origin` | `kbagent serve` (plain API) + Vite dev server + Node BFF -- the legacy three-process flow with hot reload, see `web/README.md` "Dev mode" section | inventing a `--token-in-url` flag; running uvicorn directly against `web.frontend.dist` -- the path-rewrite middleware + cookie bootstrap only fire from `kbagent serve --ui` | | Schedule / manage Agent Tasks | `kbagent agent ` (0.44.0+) -- CRUD `list/show/create/update/delete`, exec `run [--stream]`, history `runs/run-detail/run-events`, util `test/cron-preview/prompt-improve`. Local-only; cron needs `kbagent serve`. See [agent-tasks-cli-workflow](../skills/kbagent/references/agent-tasks-cli-workflow.md) | `kbagent http /agents...` (0.40.0+) in scheduled subprocesses; Web UI for human authoring | hand-editing `agents.json` | -| List models / metrics / entities in a semantic-layer model | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]` (0.41.0+); `kbagent --json semantic-layer model list --project P` to enumerate models when --model is ambiguous | `kbagent --json tool call get_semantic_layer_*` if the MCP exposes a read tool (none in the kbagent MCP at v0.41.0) | hand-rolled `urllib`/`httpx` loops against `metastore.*.keboola.com` (the `sl-builder` skill's old approach -- bypasses retry/backoff and the kbagent error envelope) | +| List models / metrics / entities in a semantic-layer model | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]` (0.41.0+); `kbagent --json semantic-layer model list --project P`; `search-context` / `get-context` (0.47.0+) for glob/id lookup | `kbagent --json tool call get_semantic_layer_*` if the MCP exposes a read tool (none at v0.41.0) | hand-rolled `urllib`/`httpx` loops against `metastore.*.keboola.com` (the `sl-builder` skill's old approach -- bypasses retry/backoff and the kbagent error envelope) | | Validate a semantic-layer model (phantom fields, constraint orphans, AGG-on-STRING) | `kbagent --json semantic-layer validate --project P [--model M] [--deep]` (0.41.0+) -- basic = local structural checks (duplicates, dangling refs, sum-on-pct, constraint orphans, severity-suffix); `--deep` adds parallel Snowflake column-existence probes via the in-process StorageService | hand-coded list+filter Python that re-implements the structural checks (loses the `--deep` Snowflake probe) | running validation by spinning up a workspace and SELECT * FROM every dataset (slow, requires workspace creation, no constraint-orphan detection) | | Snapshot a semantic-layer model to disk (before destructive edits) | `kbagent semantic-layer export --project P [--model M] [--output PATH]` (0.41.0+) -- self-describing JSON, default `./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json` | `kbagent --json semantic-layer show --project P` and pipe to a file (NOT a clean snapshot -- missing model metadata, no schemaVersion, no round-trip guarantee) | -- | | Diff a dev model against prod / against a snapshot | `kbagent --json semantic-layer diff --project-a dev --project-b prod` (project<->project); swap one side for `--file-a` / `--file-b` to diff against a snapshot (0.41.0+) | export both, run `diff` / `jq` on the JSON manually (no per-type added/removed/changed grouping, no `diff_keys`) | -- | diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 8be14f20..134d3a0f 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -253,6 +253,8 @@ When working inside a git repository or project directory, run `kbagent init` (o | Snapshot a semantic-layer model to a self-describing JSON file | `kbagent semantic-layer export --project PROJECT` | | Diff two semantic-layer snapshots (project↔project, project↔file, file↔file) | `kbagent semantic-layer diff` | | Validate a semantic-layer model | `kbagent semantic-layer validate --project PROJECT` | +| Search semantic-layer entities across a project by name pattern | `kbagent semantic-layer search-context --project PROJECT` | +| Fetch a single semantic-layer entity by id, irrespective of its type | `kbagent semantic-layer get-context --project PROJECT --context-id CONTEXT-ID` | | List all semantic-layer models in a project | `kbagent semantic-layer model list --project PROJECT` | | Create a new semantic-layer model | `kbagent semantic-layer model create --project PROJECT --name NAME` | | Delete a semantic-layer model and cascade-delete its children | `kbagent semantic-layer model delete --project PROJECT --model MODEL` | @@ -279,6 +281,8 @@ When working inside a git repository or project directory, run `kbagent init` (o | Snapshot a semantic-layer model to a self-describing JSON file | `kbagent sl export --project PROJECT` | | Diff two semantic-layer snapshots (project↔project, project↔file, file↔file) | `kbagent sl diff` | | Validate a semantic-layer model | `kbagent sl validate --project PROJECT` | +| Search semantic-layer entities across a project by name pattern | `kbagent sl search-context --project PROJECT` | +| Fetch a single semantic-layer entity by id, irrespective of its type | `kbagent sl get-context --project PROJECT --context-id CONTEXT-ID` | | List all semantic-layer models in a project | `kbagent sl model list --project PROJECT` | | Create a new semantic-layer model | `kbagent sl model create --project PROJECT --name NAME` | | Delete a semantic-layer model and cascade-delete its children | `kbagent sl model delete --project PROJECT --model MODEL` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 8d0b7005..c575fced 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -84,7 +84,7 @@ All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the - `storage tables [--project NAME ...] [--bucket-id ID] [--branch ID]` -- list tables across all connected projects in parallel (multi-project by default, same as `storage buckets`); repeat `--project` to target a subset; `--bucket-id` is applied independently per project (missing buckets become per-project errors); `--branch` requires exactly one `--project` - `storage table-detail --project NAME --table-id ID [--branch ID]` -- table detail with columns, types, primary key, row count (branch-aware) - `storage create-bucket --project NAME --stage STAGE --name NAME [--description D] [--backend B] [--branch ID]` -- create bucket (branch-aware). With `--branch ID` on a project lacking the `storage-branches` feature (legacy fake-branch), response carries `legacy_branch_storage: true` and human mode prints a warning -- the runner will create a parallel `out.c--*` bucket at job time. See `storage-types-workflow.md` -- `storage create-table --project NAME --bucket-id ID --name NAME --column col:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID]` -- create typed table. Base types `STRING/INTEGER/NUMERIC/FLOAT/BOOLEAN/DATE/TIMESTAMP` plus native backend types with length (`VARCHAR(40)`, `NUMBER(18,2)`, `TIMESTAMP_TZ`, `VARIANT`, etc.) -- type/length validation delegated to the Storage API. `--not-null` marks a column `nullable=false`; `--default NAME=VALUE` sets a DEFAULT expression (booleans must be lowercase `true`/`false`). In a dev branch, the target bucket is auto-materialized if it has not yet been written to there -- response surfaces this via `auto_created_bucket: bool`. On legacy fake-branch projects (no `storage-branches` feature), `legacy_branch_storage: true` flags that the runner will use a separate `out.c--*` bucket at job time. See `storage-types-workflow.md` +- `storage create-table --project NAME --bucket-id ID --name NAME --column col:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID] [--if-not-exists]` -- create typed table. Base types `STRING/INTEGER/NUMERIC/FLOAT/BOOLEAN/DATE/TIMESTAMP` plus native backend types with length (`VARCHAR(40)`, `NUMBER(18,2)`, `TIMESTAMP_TZ`, `VARIANT`, etc.) -- type/length validation delegated to the Storage API. `--not-null` marks a column `nullable=false`; `--default NAME=VALUE` sets a DEFAULT expression (booleans must be lowercase `true`/`false`). In a dev branch, the target bucket is auto-materialized if it has not yet been written to there -- response surfaces this via `auto_created_bucket: bool`. On legacy fake-branch projects (no `storage-branches` feature), `legacy_branch_storage: true` flags that the runner will use a separate `out.c--*` bucket at job time. `--if-not-exists` (0.47.0+) turns a duplicate-display-name failure into `action: skipped` when the table really exists at the expected id (safe for parallel workers). See `storage-types-workflow.md` - `storage upload-table --project NAME --table-id ID --file PATH [--incremental] [--branch ID]` -- upload CSV (branch-aware) - `storage download-table --project NAME --table-id ID [--output FILE] [--columns COL ...] [--limit N] [--branch ID]` -- export table to CSV (branch-aware) - `storage delete-table --project NAME --table-id ID [--table-id ...] [--force] [--dry-run] [--yes] [--branch ID]` -- delete tables, --force cascade-deletes aliased tables (branch-aware) @@ -186,9 +186,9 @@ Requires the project to be added with its **master ('owner') Storage API token** ## Sync (GitOps) - `sync init --project ALIAS [--directory DIR] [--git-branching] [--adopt-existing]` -- initialize sync working directory; `--adopt-existing` (since v0.22.0) adopts a `.keboola/manifest.json` already written by the kbc Go CLI without overwriting (idempotent; validates `project_id` against the alias token) -- `sync pull --project ALIAS [--all-projects] [--force] [--dry-run] [--with-samples] [--no-storage] [--no-jobs] [--job-limit N]` -- download configs to local files. For large projects (>100 configs), automatically fetches jobs per-config when the grouped API limit is insufficient -- `sync push --project ALIAS [--all-projects] [--dry-run] [--force] [--allow-plaintext-on-encrypt-failure]` -- push local changes (auto-encrypts secrets, fails if encryption fails) -- `sync diff --project ALIAS [--all-projects]` -- 3-way diff (local vs base vs remote), detects conflicts +- `sync pull --project ALIAS [--all-projects] [--force] [--dry-run] [--with-samples] [--no-storage] [--no-jobs] [--job-limit N] [--branch ID]` -- download configs to local files. For large projects (>100 configs), automatically fetches jobs per-config when the grouped API limit is insufficient. `--branch` (0.47.0+) per-invocation dev-branch override, beats every other branch source. +- `sync push --project ALIAS [--all-projects] [--dry-run] [--force] [--allow-plaintext-on-encrypt-failure] [--branch ID] [--no-name-drift-warnings]` -- push local changes (auto-encrypts secrets, fails if encryption fails). Fresh-CREATE writeback updates placeholder manifest entries in place (since 0.47.0) and propagates any `KBC.configuration.*` metadata via `set_config_metadata`. `--branch` (0.47.0+) per-invocation override; `--no-name-drift-warnings` (0.47.0+) drops the cosmetic warnings array. +- `sync diff --project ALIAS [--all-projects] [--branch ID]` -- 3-way diff (local vs base vs remote), detects conflicts. `--branch` (0.47.0+) per-invocation dev-branch override. - `sync status [--directory DIR]` -- show locally modified/added/deleted configs - `sync branch-link --project ALIAS [--branch-id ID] [--branch-name NAME]` -- link git branch to Keboola dev branch - `sync branch-unlink [--directory DIR]` -- remove git-to-Keboola branch mapping @@ -206,6 +206,8 @@ Manage Keboola metastore models -- datasets, metrics, relationships, constraints - `semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake]` -- create a new model. `--sql-dialect` defaults to `Snowflake`. Returns the new model UUID; subsequent commands accept either name or UUID via `--model`. - `semantic-layer model delete --project P --model M [--yes]` -- delete a model **and cascade-delete every child entity** (datasets, metrics, relationships, constraints, glossary terms) in `reversed(PUSH_ORDER)` (constraints first, datasets last) before the parent. Confirmation prompt unless `--yes`. **Cascade is unconditional in 0.43.4+** -- before that release the call only DELETEd the parent, silently leaking children pointing at the dead `modelUUID` and breaking subsequent `build` / `import` retries with HTTP 422 name collisions (closes #306). On any child-DELETE failure the parent is **preserved** and the response carries `details.cascade = {attempted, deleted, failures: [{type, id, name, error}], parent_deleted: False, model_uuid}` so the user can re-run after fixing the underlying error. Happy-path envelope adds `cascade.deleted` per-type counts. Legacy `orphaned_children` top-level key kept for back-compat (same shape, meaning flipped from "leaked" to "cascaded") but **deprecated -- removal scheduled for a future minor release**; new callers should read `cascade.deleted` instead. See [gotchas.md](gotchas.md) for the meaning-flip + deprecation note. - `semantic-layer show --project P [--model M] [--type T]` -- show a model's entities. `--type` filters to `dataset | metric | relationship | constraint | glossary`. Without `--type` prints a per-type count summary. `--model` is optional when the project has exactly one model. +- `semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N]` (since 0.47.0) -- project-wide glob search across semantic-layer entity names. Mirrors the upstream `keboola-mcp-server search_semantic_context` MCP tool so a downstream caller can drop the MCP dependency for the pre-flight "is the model populated?" check. Patterns are case-sensitive `fnmatch`, repeatable (union); default `*`. Default `--type all` searches every CHILD type (`model` searches semantic models). `--limit N` short-circuits both per-type and outer loops. Envelope: `{project, contexts: [{id, type, name, description, attributes}], total_count}`; the `type` field is the CLI-friendly singular (no `semantic-` prefix). +- `semantic-layer get-context --project P --context-id ID` (since 0.47.0) -- single-entry fetch by id, irrespective of type. Probes `semantic-model` first then every CHILD type (dataset / metric / relationship / constraint / glossary) until a 200 lands. 404 on any one type is non-terminal; only a full miss raises `NOT_FOUND` (exit 1). Non-404 errors (500, etc.) propagate immediately rather than being swallowed by the next probe. - `semantic-layer validate --project P [--model M] [--deep]` -- structural validation. Basic mode runs local checks: duplicate names, dangling rel/metric refs, SUM-on-PCT (warning), constraint orphans (metrics in `metrics[]` that no longer exist), severity-suffix mismatches between API `severity` and the 4-band name suffix. `--deep` adds parallel Snowflake column-existence probes via the in-process StorageService: phantom dataset fields, phantom column refs in metric SQL, AGG-on-STRING errors. Response: `{valid: bool, deep: bool, errors: [{type, item, detail}], warnings: [...]}`. - `semantic-layer export --project P [--model M] [--output PATH]` -- snapshot the model to a self-describing JSON file (default `./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json`). Schema-versioned for round-trip via `import` / `diff`. - `semantic-layer diff (--project-a A | --file-a P) (--project-b B | --file-b P) [--model-a M] [--model-b M]` -- three-way diff: project<->project, project<->file, file<->file. Mutually exclusive per side: pass exactly one of `--project-a` / `--file-a`, ditto for B. Output groups changes per entity type: `added[] / removed[] / changed[{name, diff_keys[]}]`. diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 0f8255e1..eb045ac3 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -11,6 +11,103 @@ Versioning convention: behavior; the inline `(updated vX.Y.Z)` records when the refinement landed. --> +## `sync push` fresh-CREATE writeback now updates placeholders in place (since v0.47.0) + +Before v0.47.0, `kbagent sync push` always **appended** new `ManifestConfiguration` +(and `ManifestConfigRow`) entries to `.keboola/manifest.json` on every CREATE. +The FIIA / scaffold emit pattern — pre-populating manifest entries with +placeholder ids before the first push — therefore produced manifests with +N placeholders + N real entries (= 2N) after one push, and every placeholder +still looked `added` on re-push (spurious duplicates on remote). + +Starting in v0.47.0, the create path looks up an existing entry by +`(component_id, path)` and **updates it in place** (id, branch_id, pull_hash, +pull_config_hash refreshed; user-declared `KBC.configuration.*` metadata +preserved). When no placeholder is found, the legacy append path still fires +(so commands like `sync init` followed by direct push of newly-pulled remote +configs are unaffected). + +Two follow-on contract notes: + +- **Re-push idempotency comes for free.** After the first push the manifest + holds the real ULID, so the diff engine matches against remote_configs and + reports `status: no_changes, created: 0`. No more "every fresh-create + emit doubles the manifest" workaround needed. +- **`KBC.configuration.*` metadata propagates on CREATE.** If a placeholder + entry's `metadata` dict contains keys starting with `KBC.` (e.g. + `KBC.configuration.folderName`), they are POSTed to the metadata API + via `client.set_config_metadata` immediately after the create call. + Bookkeeping keys (`pull_hash`, `pull_config_hash`, ...) are filtered + out by the `KBC.` prefix check. `_push_update` does **not** propagate + metadata — use `kbagent config set-metadata` (or `config set-folder`) + for that. + +If a downstream consumer has been working around the duplication by +post-processing the manifest, drop that workaround. The single-entry +manifest is the new contract. + +## `sync push` / `sync pull` / `sync diff` accept `--branch ` for per-invocation dev-branch targeting (since v0.47.0) + +The `--branch` override wins over every other branch source: `manifest.branches[0]`, +`active_branch_id` (set by `kbagent branch use`), and the git-branching +`branch-mapping.json`. Required exactly one `--project` (branch id is per-project). +Useful for targeting a freshly-created dev branch without running `branch use` or +`sync branch-link` first. The override is per-invocation only — it does not persist +to the manifest or to the config store, so subsequent commands without `--branch` +fall back to the normal priority chain. + +## `storage create-table --if-not-exists` returns `action: skipped` instead of raising on duplicate display name (since v0.47.0) + +Opt-in flag (default `False`, so existing callers are unaffected). When set, +catches the specific `STORAGE_JOB_FAILED` + "already has the same display name" +error from the Storage API, probes `get_table_detail(target_id)`, and returns +`{action: "skipped", skip_reason: "table already exists", table_id: ...}` when +the table really exists at the expected id. A different table that happens to +share the display name still raises (real conflict to resolve). The response +envelope now always carries `action: "created" | "skipped"` so programmatic +callers can branch on outcome. Safe for parallel workers (e.g. FIIA's +8-worker scaffold pattern that previously surfaced ~12 spurious errors per run). + +**Caveat — skipped envelope returns REQUESTED schema, not ACTUAL schema** (tracked +in keboola/cli#349; planned fix in a follow-up). When `action == "skipped"`, the +`columns` and `primary_key` fields in the envelope reflect what the CALLER asked +for, not what the existing table actually has. If you want the real shape, call +`kbagent storage table-detail --table-id --branch ` after a skip. This +matters when a caller hits a pre-existing table with a different shape — until +keboola/cli#349 lands, the response is a re-echo of the request, not a discovery +mechanism. + +## `sync push --no-name-drift-warnings` suppresses the cosmetic warnings array (since v0.47.0) + +When local directory names diverge from the canonical kbagent naming (e.g. +FIIA's `var-07-fi-daily-date-refresh` pattern), `sync push` normally returns +a `name_drift_warnings: [...]` array on the result envelope. The +`--no-name-drift-warnings` flag drops that field. The underlying detection +still runs, so a future operator who wants to audit can flip the flag off +without losing data. + +## `semantic-layer search-context` + `get-context` cover the MCP `search_semantic_context` / `get_semantic_context` parity (since v0.47.0) + +`kbagent semantic-layer search-context --project P [--pattern G ...] [--type T] [--limit N]` +is project-wide (not model-scoped). Patterns are **case-sensitive `fnmatch`** against +`attributes.name`, repeatable (union). Default `--type all` searches every CHILD +type (datasets, metrics, relationships, constraints, glossary) and does **not** +include semantic models — pass `--type model` to search those. The response +envelope is `{project, contexts: [{id, type, name, description, attributes}], total_count}`. +The `type` field is the CLI-friendly singular (no `semantic-` prefix on the wire form). + +`kbagent semantic-layer get-context --project P --context-id ID` probes +`semantic-model` first then every child type until a 200 lands. A 404 on any +single probe is non-terminal (keeps trying); only a full miss across all 6 +types raises `NOT_FOUND` (exit 1). **Non-404 errors propagate immediately** +without continuing the probe — a 500 on the dataset type does not get +swallowed by the subsequent metric probe. + +These two subcommands cover the pre-flight pattern FIIA uses to verify a +project's semantic model is populated before kicking off a downstream +pipeline; the previous workaround (a `keboola-mcp-server` MCP server entry +in `.mcp.json` solely for these two tools) can be dropped. + ## `workspace list` / `workspace detail` now expose loginType + RO + qs_compatible (since v0.42.0, closes #304) Before v0.42.0 the Storage workspace endpoint already returned diff --git a/plugins/kbagent/skills/kbagent/references/sync-workflow.md b/plugins/kbagent/skills/kbagent/references/sync-workflow.md index 0b44247a..6eaafcab 100644 --- a/plugins/kbagent/skills/kbagent/references/sync-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/sync-workflow.md @@ -67,6 +67,59 @@ kbagent sync push --all-projects # apply Each project gets its own subdirectory (named by alias). Projects are processed in parallel. +## Per-invocation dev-branch override (since v0.47.0) + +`sync push`, `sync pull`, and `sync diff` accept `--branch ` to target a +dev branch for a single invocation. The override beats every other branch +source: `manifest.branches[0]`, the project's `active_branch_id` (`branch use`), +and the git-branching `branch-mapping.json`. + +```bash +# Push the current working tree to dev branch 388072 without `branch use` +# or `sync branch-link` first. Required exactly one --project. +kbagent sync push --project prod --branch 388072 + +# Same dev branch on pull (`sync diff` accepts it too). +kbagent sync pull --project prod --branch 388072 +kbagent sync diff --project prod --branch 388072 +``` + +Use cases: +- Spin up a throwaway dev branch via `kbagent branch create`, push a + candidate change to it for testing, then `kbagent branch delete` to clean + up — all without touching the persisted active-branch state. +- Scripted / scheduled flows where the branch id comes from an upstream + job (e.g. a CI pipeline computes the branch id and passes it via env). + +Mutually exclusive with `--all-projects` at the CLI layer (branch id is +per-project; the validator returns exit 2 + `USAGE_ERROR` if combined). +The override is per-invocation only — it does not write into the manifest +or the config store, so a subsequent command without `--branch` falls back +to the normal priority chain. + +## Fresh-CREATE writeback (since v0.47.0) + +If you (or a tool like FIIA) seed `.keboola/manifest.json` with placeholder +entries before the first `sync push`, the writeback updates each placeholder +**in place** rather than appending a new entry. Pre-v0.47.0 this produced +manifests of length 2N after one push (placeholders + new entries both +retained); from v0.47.0 the manifest stays at length N and the placeholder +entry's id is updated to the API-assigned ULID. Re-pushes against the +now-real id are naturally idempotent. + +If a placeholder entry's `metadata` dict contains `KBC.configuration.*` +keys (e.g. `KBC.configuration.folderName`), they are propagated to the +metadata API immediately after the create call. This was the previous +"set folderName via `config set-folder` after push" workaround for +fresh-create flows; from v0.47.0 a single push handles it. + +`sync push --no-name-drift-warnings` (since v0.47.0) suppresses the +cosmetic `name_drift_warnings` array on the result envelope. The +detection still runs; only the report is dropped. Useful for downstream +tools that already audit drift their own way (e.g. FIIA's +`var-07-fi-daily-date-refresh` pattern legitimately differs from the +canonical kbagent naming and the warnings are noise). + ## Adopting an existing kbc Go CLI checkout (since v0.22.0) If you already have a `.keboola/manifest.json` produced by the official diff --git a/pyproject.toml b/pyproject.toml index f4575c55..413dfb78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-agent-cli" -version = "0.46.1" +version = "0.47.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 00b1c52d..6d9d6be3 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -8,6 +8,14 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.47.0": [ + "Fix (sync push, fresh-CREATE): pre-existing placeholder manifest entries -- the FIIA / scaffold emit pattern, where a downstream tool seeds `.keboola/manifest.json` with placeholder ids and (optionally) `KBC.configuration.*` metadata before the first push -- are now updated **in place** by the create path instead of unconditionally appended. Pre-0.47.0 every create did `manifest.configurations.append(ManifestConfiguration(...))` (and `parent.rows.append(...)` for rows), so N placeholders -> 2N manifest entries after one push, every placeholder still looked `added` on re-push (spurious duplicates on remote), and any `metadata.KBC.configuration.folderName` declared in the placeholder was silently dropped on the floor. Two new private helpers do the work: `SyncService._writeback_create_config_in_manifest(...)` finds the placeholder by `(branch_id, component_id, path)` -- branch is part of the key so a multi-branch manifest with the same logical path under two branches updates the right entry -- and refreshes its id + pull_hash / pull_config_hash while preserving every non-bookkeeping metadata key; `SyncService._writeback_create_row_in_manifest(...)` does the same for rows under their parent. Idempotency on re-push falls out for free: the now-real config id flows through the existing diff engine and the second push reports `status: no_changes, created: 0`. Tests: `tests/test_sync_service.py::TestFreshCreateWriteback` (7 cases incl. an end-to-end placeholder + KBC-metadata round-trip). Manifest contract change for downstream parsers: a single CREATE now produces a single manifest entry (not placeholder + new). Downstream tooling that has been working around the duplication by post-processing must drop that workaround. Live-validated against project 1143 / dev branch 388071: placeholder with `KBC.configuration.folderName: 'Area B E2E Folder'` -> `created=1, errors=0`, manifest length 1, folderName visible via `config metadata-list`, re-push -> `no_changes`.", + "New: `kbagent semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N]` and `kbagent semantic-layer get-context --project P --context-id ID`. Two project-wide read subcommands that mirror the upstream `keboola-mcp-server` semantic-context tools (`search_semantic_context`, `get_semantic_context`) so downstream callers (FIIA, scheduled agents, pre-flight scripts) can drop the MCP dependency for the common 'is the model populated?' + 'what's at this id?' lookups. `search-context` is project-wide (not model-scoped); patterns are repeatable, taking the union; case-sensitive `fnmatch` against `attributes.name`; `--limit` short-circuits both inner and outer loops. `get-context` probes `semantic-model` first (model hits short-circuit on the first probe) then every `CHILD_TYPES` entry until a 200 lands; raises `NOT_FOUND` after all 6 misses; non-404 errors (500, etc.) propagate immediately rather than being swallowed. Response envelope: `{project, contexts: [{id, type, name, description, attributes}], total_count}` for search; `{project, id, type, name, description, attributes}` for get. The wire-level `\"semantic-\"` prefix is stripped from the response `type` field for CLI ergonomics (`dataset` not `semantic-dataset`). Both registered as `read` operations in the permission engine. Sync surfaces touched: `commands/semantic_layer.py`, `services/semantic_layer_service.py`, `server/routers/semantic_layer.py` (1:1 CLI->HTTP), `hints/definitions/semantic_layer.py`, `permissions.py`. Tests: `tests/test_semantic_layer_service.py::TestSearchContext` (12) + `::TestGetContext` (6); `tests/test_semantic_layer_cli.py::TestSearchContext` (4) + `::TestGetContext` (3). Live-validated against project 1143: returns 8 contexts spanning 4 types; pattern `rev_*` + `--type metric` narrows to 1 hit; round-trip search -> get-context on the returned id resolves correctly; UUID `00000000-0000-0000-0000-000000000000` returns NOT_FOUND after probing all 6 types.", + "New: `kbagent sync push --branch `, `sync pull --branch `, `sync diff --branch `. Per-invocation dev-branch override that wins over `manifest.branches[0]`, `active_branch_id` (`kbagent branch use`), and the git-branching `branch-mapping.json` -- new priority 0 in `SyncService._resolve_branch_id`. Lets an operator or downstream tool target a freshly-created dev branch without first running `branch use` or `sync branch-link`. Validated mutually exclusive with `--all-projects` at the CLI layer (branch id is per-project). Symmetric across push / pull / diff for predictable UX. Threaded through `branch_override=` kwargs on `SyncService.push / pull / diff`. Live-validated against project 1143: `sync diff --branch 388072` reports `remote_only: 31` (configs visible on the dev branch); without `--branch` the same call reports no remote diff.", + 'New: `kbagent storage create-table --if-not-exists`. Opt-in idempotency flag for parallel-worker patterns (e.g. FIIA\'s 8-worker `scaffold_storage.py`). When set, catches the specific `STORAGE_JOB_FAILED` + \'already has the same display name\' error from the Storage API, probes `get_table_detail(target_id)` to confirm the table really exists at the expected id, and returns `{action: "skipped", skip_reason: "table already exists", table_id: ...}` instead of raising. A different table with the same display name still surfaces the original error (a real conflict to resolve). Defaults to `False` so existing callers are byte-for-byte unaffected. Response envelope gains `action: "created" | "skipped"` so programmatic callers can branch on outcome. Error-code gate uses `ErrorCode.STORAGE_JOB_FAILED` (no raw string literal -- `make check-error-codes` enforces enum usage). Live-validated against project 1143 / dev branch 388072: first create -> `action: created`; second create (same name, with flag) -> `action: skipped`; third create (same name, no flag) -> original `STORAGE_JOB_FAILED` envelope.', + "New: `kbagent sync push --no-name-drift-warnings`. Opt-out flag that drops the cosmetic `name_drift_warnings` array from the result envelope when local directory names differ from the canonical kbagent naming (e.g. FIIA's `var-07-fi-daily-date-refresh` pattern). The underlying detection still runs, so flipping the flag does not lose any audit data; only the report is suppressed. Defaults to `False` so existing callers see the warnings exactly as before.", + 'Note (sync `serve` exposure): the four sync subcommands (`init`, `pull`, `push`, `diff`) remain filesystem-local and intentionally have no HTTP endpoints in `src/keboola_agent_cli/server/routers/`. The plugin-sync map permits this exemption for terminal-only / filesystem-bound commands (CONTRIBUTING.md "Plugin synchronization map"). The new `--branch` and `--no-name-drift-warnings` flags consequently also have no REST counterpart.', + ], "0.46.1": [ "Fix (plugin): the `kbagent` Claude Code skill and the `keboola-expert` subagent now surface `kbagent data-app logs` (shipped in 0.43.8). The SKILL.md `description:` trigger list gained `data-app logs, container logs, app logs, tail logs, build logs, app stdout, app stderr, troubleshoot data app, debug data app`, and the keboola-expert Tool Selection Matrix gained a row for `kbagent data-app logs --project P --app-id N [--lines N | --since ISO8601]` (0.43.8+). Before this, asking the agent for a data app's container logs fell back to the UI Terminal Log tab or the 20-line-capped `get_data_apps` MCP tool. No CLI behavior change. (#335 / #336)", "Chore (frontend dev tooling): bumped the `web/frontend` dev dependencies -- vite 5 -> 8, vitest 2 -> 4, and `@vitejs/plugin-react` 4 -> 5.2 to keep the peer range consistent with vite 8. The earlier Dependabot PRs (#337, #338) bumped only vite + vitest, which left plugin-react pinned below vite 8 and broke `npm ci` (ERESOLVE) in the Windows wheel-build job, silently shipping a UI-less wheel. No runtime change to the CLI or the bundled SPA. (#341)", diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index b2431af6..e60478c6 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -342,8 +342,12 @@ time. Response includes `legacy_branch_storage: true` and human mode prints a warning when this applies. See storage-types-workflow.md. - kbagent storage create-table --project NAME --bucket-id BUCKET_ID --name TABLE_NAME --column col:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID] + kbagent storage create-table --project NAME --bucket-id BUCKET_ID --name TABLE_NAME --column col:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID] [--if-not-exists] Create a typed table. --column repeatable. + - --if-not-exists (since 0.47.0): opt-in idempotency. On a duplicate-display-name failure, + probe get-table-detail at the expected id and, if the table really exists, return + `action: "skipped", skip_reason: "table already exists"` instead of raising. A different + table with the same display name still surfaces the original error. Safe for parallel workers. - Base types: STRING, INTEGER, NUMERIC, FLOAT, BOOLEAN, DATE, TIMESTAMP. Type defaults to STRING if omitted. - Native backend types with length pass through to the Storage API: VARCHAR(40), NUMBER(18,2), CHAR(10), TIMESTAMP_TZ, TIMESTAMP_NTZ, VARIANT, OBJECT, ARRAY, etc. The API validates type/length per backend; e.g. INTEGER(10) is rejected with "'10' is not valid length for INTEGER". @@ -755,21 +759,32 @@ kbagent sync init --project ALIAS [--directory DIR] [--git-branching] [--adopt-existing] Initialize sync working directory. --git-branching enables git-to-Keboola branch mapping. - kbagent sync pull --project ALIAS [--all-projects] [--force] [--dry-run] [--with-samples] [--no-storage] [--no-jobs] [--job-limit N] + kbagent sync pull --project ALIAS [--all-projects] [--force] [--dry-run] [--with-samples] [--no-storage] [--no-jobs] [--job-limit N] [--branch ID] Download configs as local files. Idempotent, protects local modifications. --job-limit controls max recent jobs per config (default 5). For large projects, automatically falls back to per-config job fetching to ensure all configs get job history. Auto-detects renamed configs and renames local directories to match (uses git mv in git repos). + --branch (since 0.47.0): per-invocation dev-branch override. Same semantics as sync push/diff. kbagent sync status [--directory DIR] Show local changes since last pull (SHA256-based). - kbagent sync diff --project ALIAS [--all-projects] [--directory DIR] + kbagent sync diff --project ALIAS [--all-projects] [--directory DIR] [--branch ID] 3-way diff: local vs pull-time snapshot vs remote. Detects conflicts. + --branch (since 0.47.0): per-invocation dev-branch override. Wins over + manifest.branches[0] / 'branch use' active branch / git-branching mapping. + Requires exactly one --project. - kbagent sync push --project ALIAS [--all-projects] [--dry-run] [--force] [--allow-plaintext-on-encrypt-failure] + kbagent sync push --project ALIAS [--all-projects] [--dry-run] [--force] [--allow-plaintext-on-encrypt-failure] [--branch ID] [--no-name-drift-warnings] Push local changes. Auto-encrypts secrets. Skips conflicts (pull first). Fails if encryption fails (plaintext secrets never pushed). Use escape hatch flag only if you know what you are doing. + Fresh-CREATE behavior (since 0.47.0): if the manifest contains a placeholder entry at + (component_id, path), the create path updates it in place (no manifest duplication) + and propagates any KBC.configuration.* metadata via set_config_metadata. Re-pushes + against the now-real config id are naturally idempotent. + --branch (since 0.47.0): per-invocation dev-branch override. Same semantics as sync diff. + --no-name-drift-warnings (since 0.47.0): suppress the cosmetic name_drift_warnings + array from the result envelope. kbagent sync branch-link --project ALIAS [--branch-id ID] [--branch-name NAME] Link git branch to Keboola dev branch. Auto-creates if needed. @@ -809,6 +824,19 @@ Show a model's entities. --type filter: dataset|metric|relationship|constraint|glossary. Without --type prints a per-type count summary. + kbagent semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N] + (since 0.47.0) Project-wide glob search across semantic-layer entity names. + Mirrors the upstream keboola-mcp-server search_semantic_context tool so a + downstream caller can verify the model is populated without an MCP dependency. + Patterns are case-sensitive fnmatch, repeatable (union). Default pattern is "*". + Default --type is "all" (every CHILD type; "model" searches semantic models). + Returns {{project, contexts: [{{id, type, name, description, attributes}}], total_count}}. + + kbagent semantic-layer get-context --project P --context-id ID + (since 0.47.0) Single-entry fetch by id, irrespective of type. Probes model first, + then datasets/metrics/relationships/constraints/glossary in order; raises NOT_FOUND + if no type matches (exit 1). + kbagent semantic-layer validate --project P [--model M] [--deep] Basic structural checks (duplicates, dangling refs, sum-on-pct, constraint orphans, severity-suffix). --deep adds parallel Snowflake diff --git a/src/keboola_agent_cli/commands/semantic_layer.py b/src/keboola_agent_cli/commands/semantic_layer.py index 07b1ac4c..eb3bcf35 100644 --- a/src/keboola_agent_cli/commands/semantic_layer.py +++ b/src/keboola_agent_cli/commands/semantic_layer.py @@ -917,3 +917,146 @@ def semantic_layer_validate( deep=deep, ) formatter.output(result, _print_validate) + + +# --------------------------------------------------------------------------- +# semantic-layer search-context / get-context +# +# Project-wide read surface that mirrors the upstream +# ``keboola-mcp-server`` semantic-context tools. Lets downstream callers +# (FIIA, scheduled agents) drop the MCP dependency for the common +# "is the model populated?" + "what's at this id?" lookups. +# --------------------------------------------------------------------------- + + +def _print_search_context(console: Console, data: dict) -> None: + project = data.get("project", "") + total = data.get("total_count", 0) + console.print( + f"\n[bold]Semantic contexts[/bold] in [magenta]{project}[/magenta]: " + f"{total} match{'es' if total != 1 else ''}" + ) + contexts = data.get("contexts", []) or [] + if not contexts: + console.print("[dim](no matches)[/dim]") + return + table = Table() + table.add_column("Type", style="bold cyan") + table.add_column("Name", style="bold") + table.add_column("ID") + table.add_column("Description") + for c in contexts: + table.add_row( + str(c.get("type", "")), + str(c.get("name", "")), + str(c.get("id", "")), + str(c.get("description", ""))[:60], + ) + console.print(table) + + +def _print_get_context(console: Console, data: dict) -> None: + console.print( + f"\n[bold]{data.get('type', '?')}[/bold] " + f"[cyan]{data.get('name', '')}[/cyan] " + f"([dim]{data.get('id', '')}[/dim]) in " + f"[magenta]{data.get('project', '')}[/magenta]" + ) + desc = data.get("description", "") + if desc: + console.print(f"\n{desc}\n") + attrs = data.get("attributes") or {} + if attrs: + console.print("[bold]Attributes:[/bold]") + console.print(json.dumps(attrs, indent=2, sort_keys=True, default=str)) + + +@semantic_layer_app.command("search-context") +def semantic_layer_search_context( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + pattern: list[str] = typer.Option( + ["*"], + "--pattern", + help=( + "Glob pattern matched against entity name (case-sensitive " + "fnmatch). Repeatable; matches the union. Default: '*'." + ), + ), + type_filter: str = typer.Option( + "all", + "--type", + help=( + "Restrict to one type: model | dataset | metric | relationship | " + "constraint | glossary | all. Default: all (every child type)." + ), + ), + limit: int | None = typer.Option( + None, + "--limit", + help="Maximum number of results to return. Default: no cap.", + ), +) -> None: + """Search semantic-layer entities across a project by name pattern. + + Project-wide (not model-scoped). Equivalent to the upstream + ``keboola-mcp-server`` ``search_semantic_context`` tool. Use this as a + pre-flight check ("is the semantic model populated?") before kicking + off a downstream pipeline that depends on it. + """ + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.search-context", + project=project, + pattern=pattern, + type_filter=type_filter, + limit=limit, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.search_context, + alias=project, + patterns=pattern, + type_filter=type_filter, + limit=limit, + ) + formatter.output(result, _print_search_context) + + +@semantic_layer_app.command("get-context") +def semantic_layer_get_context( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + context_id: str = typer.Option( + ..., + "--context-id", + help="UUID of the entity to fetch (model, dataset, metric, ...).", + ), +) -> None: + """Fetch a single semantic-layer entity by id, irrespective of its type. + + Probes every type (model + datasets / metrics / relationships / + constraints / glossary) until it finds the entity, then returns the + full attribute dict. Exits 1 if no type matches. + """ + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.get-context", + project=project, + context_id=context_id, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.get_context, + alias=project, + context_id=context_id, + ) + formatter.output(result, _print_get_context) diff --git a/src/keboola_agent_cli/commands/storage.py b/src/keboola_agent_cli/commands/storage.py index 1766af51..34d411b5 100644 --- a/src/keboola_agent_cli/commands/storage.py +++ b/src/keboola_agent_cli/commands/storage.py @@ -586,6 +586,16 @@ def storage_create_table( "--branch", help="Dev branch ID (defaults to active branch if set via 'branch use')", ), + if_not_exists: bool = typer.Option( + False, + "--if-not-exists", + help=( + "Treat a duplicate-display-name failure as a successful no-op " + "when the table already exists at the expected id. Safe for " + "parallel workers (FIIA scaffold pattern). A different table " + "with the same display name still surfaces the original error." + ), + ), ) -> None: """Create a new storage table with typed columns. @@ -622,6 +632,7 @@ def storage_create_table( not_null=not_null, default=default, branch=branch, + if_not_exists=if_not_exists, ) formatter = get_formatter(ctx) @@ -639,6 +650,7 @@ def storage_create_table( branch_id=effective_branch, not_null_columns=not_null, defaults=default, + if_not_exists=if_not_exists, ) except ValueError as exc: formatter.error(message=str(exc), error_code=ErrorCode.INVALID_ARGUMENT) @@ -653,17 +665,25 @@ def storage_create_table( if formatter.json_mode: formatter.output(result) else: - formatter.console.print(f"[bold green]Created table:[/bold green] {result['table_id']}") - if result.get("auto_created_bucket"): + if result.get("action") == "skipped": formatter.console.print( - f" [yellow]Note:[/yellow] bucket {result['bucket_id']} was " - f"auto-materialized in this branch." + f"[bold yellow]Skipped[/bold yellow] (already exists): {result['table_id']}" ) - if result["primary_key"]: - formatter.console.print(f" Primary key: {', '.join(result['primary_key'])}") - formatter.console.print(f" Columns: {', '.join(result['columns'])}") - if result.get("legacy_branch_storage"): - formatter.console.print(_LEGACY_BRANCH_STORAGE_WARNING) + reason = result.get("skip_reason") + if reason: + formatter.console.print(f" [dim]{reason}[/dim]") + else: + formatter.console.print(f"[bold green]Created table:[/bold green] {result['table_id']}") + if result.get("auto_created_bucket"): + formatter.console.print( + f" [yellow]Note:[/yellow] bucket {result['bucket_id']} was " + f"auto-materialized in this branch." + ) + if result["primary_key"]: + formatter.console.print(f" Primary key: {', '.join(result['primary_key'])}") + formatter.console.print(f" Columns: {', '.join(result['columns'])}") + if result.get("legacy_branch_storage"): + formatter.console.print(_LEGACY_BRANCH_STORAGE_WARNING) @storage_app.command("upload-table", rich_help_panel=_TABLES) diff --git a/src/keboola_agent_cli/commands/sync.py b/src/keboola_agent_cli/commands/sync.py index fb92a64c..7c0be9b9 100644 --- a/src/keboola_agent_cli/commands/sync.py +++ b/src/keboola_agent_cli/commands/sync.py @@ -446,6 +446,14 @@ def sync_pull( "--max-samples", help="Max number of tables to sample (default 50)", ), + branch: int | None = typer.Option( + None, + "--branch", + help=( + "Dev branch ID. Overrides the manifest / 'branch use' active " + "branch for this single invocation. Requires exactly one --project." + ), + ), ) -> None: """Download configurations from a Keboola project to local files. @@ -467,6 +475,12 @@ def sync_pull( error_code=ErrorCode.USAGE_ERROR, ) raise typer.Exit(code=2) + if branch is not None and all_projects: + formatter.error( + message="--branch requires --project (branch id is per-project)", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) if all_projects: base_dir = _safe_resolve_dir(directory) @@ -515,6 +529,7 @@ def sync_pull( with_samples=with_samples, sample_limit=sample_limit, max_samples=max_samples, + branch_override=branch, ) except FileNotFoundError as exc: formatter.error(message=str(exc), error_code=ErrorCode.NOT_INITIALIZED) @@ -615,6 +630,14 @@ def sync_diff( "-d", help="Project root directory (must contain .keboola/)", ), + branch: int | None = typer.Option( + None, + "--branch", + help=( + "Dev branch ID. Overrides the manifest / 'branch use' active " + "branch for this single invocation. Requires exactly one --project." + ), + ), ) -> None: """Show detailed diff between local and remote configurations. @@ -636,6 +659,12 @@ def sync_diff( error_code=ErrorCode.USAGE_ERROR, ) raise typer.Exit(code=2) + if branch is not None and all_projects: + formatter.error( + message="--branch requires --project (branch id is per-project)", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) if all_projects: base_dir = _safe_resolve_dir(directory) @@ -654,7 +683,7 @@ def sync_diff( project_root = _resolve_project_root(directory, project) try: - result = service.diff(alias=project, project_root=project_root) + result = service.diff(alias=project, project_root=project_root, branch_override=branch) except FileNotFoundError as exc: formatter.error(message=str(exc), error_code=ErrorCode.NOT_INITIALIZED) raise typer.Exit(code=1) from None @@ -791,6 +820,22 @@ def sync_push( "--allow-plaintext-on-encrypt-failure", help="Allow push even if secret encryption fails (DANGEROUS: secrets stored as plaintext)", ), + branch: int | None = typer.Option( + None, + "--branch", + help=( + "Dev branch ID. Overrides the manifest / 'branch use' active " + "branch for this single invocation. Requires exactly one --project." + ), + ), + no_name_drift_warnings: bool = typer.Option( + False, + "--no-name-drift-warnings", + help=( + "Suppress the cosmetic name_drift_warnings array in the result " + "envelope (the underlying detection still runs)." + ), + ), ) -> None: """Push local configuration changes to a Keboola project. @@ -812,6 +857,12 @@ def sync_push( error_code=ErrorCode.USAGE_ERROR, ) raise typer.Exit(code=2) + if branch is not None and all_projects: + formatter.error( + message="--branch requires --project (branch id is per-project)", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) if all_projects: base_dir = _safe_resolve_dir(directory) @@ -841,6 +892,8 @@ def sync_push( dry_run=dry_run, force=force, allow_plaintext_fallback=allow_plaintext, + branch_override=branch, + no_name_drift_warnings=no_name_drift_warnings, ) except FileNotFoundError as exc: formatter.error(message=str(exc), error_code=ErrorCode.NOT_INITIALIZED) diff --git a/src/keboola_agent_cli/hints/definitions/semantic_layer.py b/src/keboola_agent_cli/hints/definitions/semantic_layer.py index c14b21e7..6d752667 100644 --- a/src/keboola_agent_cli/hints/definitions/semantic_layer.py +++ b/src/keboola_agent_cli/hints/definitions/semantic_layer.py @@ -162,6 +162,84 @@ def _make_service(method: str, **extra_args: str) -> ServiceCall: ) +# ── semantic-layer search-context (since v0.47.0) ───────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.search-context", + description=( + "Search semantic-layer entities project-wide by glob pattern " + "(mirrors the upstream keboola-mcp-server search_semantic_context)" + ), + steps=[ + HintStep( + comment=( + "List every entity of the requested type (or every " + "child type if --type=all) and filter by name pattern." + ), + client=ClientCall( + method="list_items", + args={"item_type": '"semantic-dataset"'}, + client_type="metastore", + result_var="datasets", + result_hint="list[dict]", + ), + service=_make_service( + "search_context", + patterns="{pattern}", + type_filter="{type_filter}", + limit="{limit}", + ), + ), + ], + notes=[ + _PARALLEL_CHILDREN_NOTE, + "Pattern matching is case-sensitive fnmatch against attributes.name.", + "`--limit` short-circuits both inner and outer loops.", + ], + ) +) + + +# ── semantic-layer get-context (since v0.47.0) ──────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.get-context", + description=( + "Fetch a single semantic-layer entity by id, irrespective of type " + "(mirrors the upstream keboola-mcp-server get_semantic_context)" + ), + steps=[ + HintStep( + comment=("Try each type until one returns 200. Raise NOT_FOUND if none match."), + client=ClientCall( + method="get_item", + args={ + "item_type": '"semantic-dataset"', + "item_id": "{context_id}", + }, + client_type="metastore", + result_var="entity", + result_hint="dict", + ), + service=_make_service( + "get_context", + context_id="{context_id}", + ), + ), + ], + notes=[ + ( + "Iteration order is: semantic-model, then semantic-dataset / " + "metric / relationship / constraint / glossary." + ), + "404 on any one type is non-terminal; only a full miss raises NOT_FOUND.", + ], + ) +) + + # ── semantic-layer validate ──────────────────────────────────────── HintRegistry.register( diff --git a/src/keboola_agent_cli/hints/definitions/storage.py b/src/keboola_agent_cli/hints/definitions/storage.py index 90c7ce6a..ccc737ab 100644 --- a/src/keboola_agent_cli/hints/definitions/storage.py +++ b/src/keboola_agent_cli/hints/definitions/storage.py @@ -232,6 +232,7 @@ "branch_id": "{branch}", "not_null_columns": "{not_null}", "defaults": "{default}", + "if_not_exists": "{if_not_exists}", }, ), ), @@ -242,6 +243,7 @@ "Service mode: --not-null and --default flags add nullable/default to column definitions.", "Client mode: build column dicts directly as [{'name': 'pk', 'definition': {'type': 'VARCHAR', 'length': '40', 'nullable': False}}].", "In a dev branch, service layer auto-materializes the bucket on 404 (mirrors Keboola Go CLI's EnsureBucketExists). Client mode does not -- call get_bucket_detail + create_bucket first.", + "if_not_exists=True (0.47.0+) returns {action: 'skipped'} on a duplicate-display-name failure when the table really exists at the expected id. Safe for parallel workers.", ], ) ) diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 384d2b71..51fe1a42 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -174,6 +174,8 @@ "semantic-layer.validate": "read", "semantic-layer.export": "read", "semantic-layer.diff": "read", + "semantic-layer.search-context": "read", + "semantic-layer.get-context": "read", # The `model` sub-app: the parent `semantic-layer` callback fires first # with ctx.invoked_subcommand == "model" and synthesizes operation key # ``semantic-layer.model``. We expose that key at the LEAST-privileged diff --git a/src/keboola_agent_cli/server/routers/semantic_layer.py b/src/keboola_agent_cli/server/routers/semantic_layer.py index 7f28b0a5..43182560 100644 --- a/src/keboola_agent_cli/server/routers/semantic_layer.py +++ b/src/keboola_agent_cli/server/routers/semantic_layer.py @@ -22,7 +22,7 @@ from pathlib import Path from typing import Any, Literal -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel, Field, model_validator from ...errors import ErrorCode @@ -256,6 +256,34 @@ def validate( ) +@router.get("/search-context", summary="Search semantic contexts by name pattern") +def search_context( + project: str, + pattern: list[str] = Query(default=["*"]), + type: str = "all", + limit: int | None = None, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Project-wide glob search across semantic-layer entities. + + Mirrors ``kbagent semantic-layer search-context``. See the service-layer + docstring for matching semantics and the returned envelope shape. + """ + return registry.semantic_layer.search_context( + alias=project, patterns=pattern, type_filter=type, limit=limit + ) + + +@router.get("/get-context", summary="Fetch one semantic context by id") +def get_context( + project: str, + context_id: str, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Single-entry fetch by id; probes every type until found.""" + return registry.semantic_layer.get_context(alias=project, context_id=context_id) + + @router.get("/export", summary="Export model snapshot") def export( project: str, diff --git a/src/keboola_agent_cli/server/routers/storage.py b/src/keboola_agent_cli/server/routers/storage.py index 2ee5f2a4..256349b3 100644 --- a/src/keboola_agent_cli/server/routers/storage.py +++ b/src/keboola_agent_cli/server/routers/storage.py @@ -31,6 +31,7 @@ class CreateTable(BaseModel): not_null_columns: list[str] | None = None defaults: list[str] | None = None branch_id: int | None = None + if_not_exists: bool = False class DescribeBucket(BaseModel): @@ -237,6 +238,7 @@ def create_table( branch_id=body.branch_id, not_null_columns=body.not_null_columns, defaults=body.defaults, + if_not_exists=body.if_not_exists, ) diff --git a/src/keboola_agent_cli/services/_semantic_layer_lookup.py b/src/keboola_agent_cli/services/_semantic_layer_lookup.py new file mode 100644 index 00000000..cce29827 --- /dev/null +++ b/src/keboola_agent_cli/services/_semantic_layer_lookup.py @@ -0,0 +1,181 @@ +"""Project-wide context search / lookup helpers for :mod:`semantic_layer_service`. + +Split out so :class:`SemanticLayerService` stays under the CONTRIBUTING.md +services hard ceiling (1,500 LOC). Each helper opens + closes its own +metastore client via the factory the service injects; the service methods +are 1-line delegators. + +Helpers: + +- :func:`run_search_context` -- project-wide glob search across semantic-layer + entity names (mirrors MCP ``search_semantic_context``). +- :func:`run_get_context` -- single fetch by id, irrespective of type (mirrors + MCP ``get_semantic_context``). +""" + +from __future__ import annotations + +import fnmatch +from typing import TYPE_CHECKING, Any + +from ..errors import ErrorCode, KeboolaApiError + +if TYPE_CHECKING: + from collections.abc import Callable + + from ..metastore_client import MetastoreClient, SemanticType + + +# Probed first by :func:`run_get_context`; child types follow the canonical +# iteration order so the sweep is deterministic. +_MODEL_TYPE: SemanticType = "semantic-model" + + +def _strip_semantic_prefix(wire_type: str) -> str: + """``"semantic-dataset"`` -> ``"dataset"`` for the CLI surface.""" + return wire_type[len("semantic-") :] if wire_type.startswith("semantic-") else wire_type + + +def _matches_any_pattern(name: str, patterns: list[str]) -> bool: + """Case-sensitive ``fnmatch`` against any of the supplied patterns.""" + return any(fnmatch.fnmatchcase(name, pat) for pat in patterns) + + +def _resolve_search_types( + type_filter: str | None, + child_types: tuple[SemanticType, ...], + type_alias: dict[str, SemanticType], +) -> tuple[SemanticType, ...]: + """Map the CLI ``--type`` flag to the list of wire types to scan.""" + if type_filter is None or type_filter == "all": + return child_types + if type_filter == "model": + return (_MODEL_TYPE,) + if type_filter in type_alias: + return (type_alias[type_filter],) + allowed = ["all", "model", *sorted(type_alias)] + raise KeboolaApiError( + message=f"Invalid --type {type_filter!r}. Must be one of: {', '.join(allowed)}.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + + +def run_search_context( + *, + open_client: Callable[[], MetastoreClient], + alias: str, + child_types: tuple[SemanticType, ...], + type_alias: dict[str, SemanticType], + patterns: list[str] | None, + type_filter: str | None, + limit: int | None, +) -> dict[str, Any]: + """Project-wide glob search across semantic-layer entity names. + + Mirrors the upstream ``keboola-mcp-server`` ``search_semantic_context`` + MCP tool. ``patterns`` default to ``["*"]`` and are matched case- + sensitively against ``attributes.name``; multiple patterns take the + union. ``type_filter`` ``None`` / ``"all"`` -> every entry in + ``child_types``; ``"model"`` -> semantic models; any other CLI + singular narrows to that single wire type via ``type_alias``. + + Returns ``{"project", "contexts", "total_count"}``; each context is + ``{"id", "type", "name", "description", "attributes"}`` with the + wire ``"semantic-"`` prefix stripped from ``type`` for CLI ergonomics. + Raises :data:`ErrorCode.VALIDATION_ERROR` for empty patterns, non- + positive ``limit``, or an unknown ``type_filter``. + + Opens + closes its own metastore client via ``open_client()`` so the + service method is a one-line delegator. + """ + eff_patterns: list[str] = patterns or ["*"] + if any(not p for p in eff_patterns): + raise KeboolaApiError( + message="--pattern values must be non-empty strings", + error_code=ErrorCode.VALIDATION_ERROR, + ) + if limit is not None and limit <= 0: + raise KeboolaApiError( + message="--limit must be a positive integer", + error_code=ErrorCode.VALIDATION_ERROR, + ) + types_to_search = _resolve_search_types(type_filter, child_types, type_alias) + + contexts: list[dict[str, Any]] = [] + with open_client() as client: + for wire_type in types_to_search: + for item in client.list_items(wire_type): + attrs = item.get("attributes") or {} + name = str(attrs.get("name", "")) + if not _matches_any_pattern(name, eff_patterns): + continue + contexts.append( + { + "id": item.get("id", ""), + "type": _strip_semantic_prefix(wire_type), + "name": name, + "description": attrs.get("description", ""), + "attributes": attrs, + } + ) + if limit is not None and len(contexts) >= limit: + break + if limit is not None and len(contexts) >= limit: + break + + return {"project": alias, "contexts": contexts, "total_count": len(contexts)} + + +def run_get_context( + *, + open_client: Callable[[], MetastoreClient], + alias: str, + child_types: tuple[SemanticType, ...], + context_id: str, +) -> dict[str, Any]: + """Single-id fetch across every semantic type. + + Probes ``semantic-model`` first then every entry in ``child_types``, + stopping on the first 200. A 404 on any one type is non-terminal; + only a full miss raises ``NOT_FOUND``. Non-404 errors (e.g. 500) + propagate immediately rather than being swallowed by the next probe. + + Returns ``{"project", "id", "type", "name", "description", "attributes"}`` + on hit (type stripped of the ``"semantic-"`` wire prefix). Raises + :data:`ErrorCode.VALIDATION_ERROR` on empty id, or + :data:`ErrorCode.NOT_FOUND` after the full sweep. + + Opens + closes its own metastore client via ``open_client()``. + """ + if not context_id: + raise KeboolaApiError( + message="--context-id is required", + error_code=ErrorCode.VALIDATION_ERROR, + ) + + lookup_order: tuple[SemanticType, ...] = (_MODEL_TYPE, *child_types) + with open_client() as client: + for wire_type in lookup_order: + try: + item = client.get_item(wire_type, context_id) + except KeboolaApiError as exc: + if exc.error_code == ErrorCode.NOT_FOUND: + continue + raise + attrs = item.get("attributes") or {} + return { + "project": alias, + "id": item.get("id", ""), + "type": _strip_semantic_prefix(wire_type), + "name": attrs.get("name", ""), + "description": attrs.get("description", ""), + "attributes": attrs, + } + + raise KeboolaApiError( + message=( + f"Semantic context with id {context_id!r} not found in project " + f"{alias!r}. Tried: semantic-model + {', '.join(child_types)}." + ), + error_code=ErrorCode.NOT_FOUND, + ) diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index 2a0a1c2b..58ff1791 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -52,6 +52,8 @@ from ._semantic_layer_internals import validate_basic as _validate_basic_helper from ._semantic_layer_internals import validate_deep as _validate_deep_helper from ._semantic_layer_internals import write_snapshot_to_file as _write_snapshot_to_file +from ._semantic_layer_lookup import run_get_context as _run_get_context_helper +from ._semantic_layer_lookup import run_search_context as _run_search_context_helper from .base import BaseService, ClientFactory from .encrypt_service import EncryptService from .storage_service import StorageService @@ -223,10 +225,7 @@ def __init__( metastore_client_factory or default_metastore_client_factory ) - # ------------------------------------------------------------------ - # Helpers (used by every subcommand) - # ------------------------------------------------------------------ - + # Helpers (used by every subcommand). def _resolve_one_project(self, alias: str) -> ProjectConfig: """Resolve a single project alias to its ``ProjectConfig`` or raise. @@ -240,20 +239,12 @@ def _new_metastore_client(self, project: ProjectConfig) -> MetastoreClient: return self._metastore_factory(project.stack_url, project.token) def _resolve_model( - self, - client: MetastoreClient, - model_name_or_uuid: str | None, + self, client: MetastoreClient, model_name_or_uuid: str | None ) -> tuple[str, dict[str, Any]]: - """Resolve a model selector to ``(uuid, attributes_dict)``. - - Body lives in :func:`._semantic_layer_internals.resolve_model_uuid`. - """ + """Resolve a model selector via :func:`._semantic_layer_internals.resolve_model_uuid`.""" return _resolve_model_uuid(client, model_name_or_uuid) - # ------------------------------------------------------------------ - # Phase 3 — Read commands - # ------------------------------------------------------------------ - + # Phase 3 — Read commands. def list_models(self, alias: str) -> dict[str, Any]: """List all semantic-layer models for a project. @@ -262,11 +253,8 @@ def list_models(self, alias: str) -> dict[str, Any]: ``{id, name, description, sql_dialect}``). """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: raw = client.list_items("semantic-model") - finally: - client.close() models: list[dict[str, Any]] = [] for item in raw: @@ -281,9 +269,34 @@ def list_models(self, alias: str) -> dict[str, Any]: ) return {"project": alias, "models": models} - # ------------------------------------------------------------------ - # Internal helpers (model-scoped fetches) - # ------------------------------------------------------------------ + def search_context( + self, + alias: str, + patterns: list[str] | None = None, + type_filter: str | None = None, + limit: int | None = None, + ) -> dict[str, Any]: + """Project-wide glob search; see :func:`_semantic_layer_lookup.run_search_context`.""" + return _run_search_context_helper( + open_client=lambda: self._new_metastore_client(self._resolve_one_project(alias)), + alias=alias, + child_types=CHILD_TYPES, + type_alias=TYPE_ALIAS, + patterns=patterns, + type_filter=type_filter, + limit=limit, + ) + + def get_context(self, alias: str, context_id: str) -> dict[str, Any]: + """Single-id lookup; see :func:`_semantic_layer_lookup.run_get_context`.""" + return _run_get_context_helper( + open_client=lambda: self._new_metastore_client(self._resolve_one_project(alias)), + alias=alias, + child_types=CHILD_TYPES, + context_id=context_id, + ) + + # Internal helpers (model-scoped fetches). @staticmethod def _fetch_children_parallel( @@ -341,12 +354,9 @@ def show_model( ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) raw_by_type = self._fetch_children_parallel(client, model_uuid) - finally: - client.close() result: dict[str, Any] = { "project": alias, @@ -377,12 +387,9 @@ def validate_model( check inventory. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) raw_by_type = self._fetch_children_parallel(client, model_uuid) - finally: - client.close() datasets = _unpack_attrs_with_id(raw_by_type.get("semantic-dataset", [])) metrics = _unpack_attrs_with_id(raw_by_type.get("semantic-metric", [])) @@ -479,12 +486,9 @@ def export_model( relationships, constraints, glossary, counts}``. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) raw_by_type = self._fetch_children_parallel(client, model_uuid) - finally: - client.close() snapshot = _build_export_snapshot( alias=alias, @@ -599,14 +603,11 @@ def create_model( ) -> dict[str, Any]: """Create a semantic-layer model and return the server-stored item.""" project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: data: dict[str, Any] = {"name": name, "sql_dialect": sql_dialect} if description: data["description"] = description created = client.post_item("semantic-model", name=name, data=data) - finally: - client.close() return {"project": alias, "model": created} def delete_model( @@ -623,8 +624,7 @@ def delete_model( in the helper to keep this file under the 1500 LOC ceiling. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) children = self._fetch_children_parallel(client, model_uuid) return _cascade_delete_model_impl( @@ -634,8 +634,6 @@ def delete_model( model_attrs=model_attrs, children=children, ) - finally: - client.close() # ------------------------------------------------------------------ # Phase 4 — add subcommands @@ -662,8 +660,7 @@ def add_metric( rather than silently push a broken metric. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) datasets = client.list_items("semantic-dataset", model_uuid) ds_tids = {(d.get("attributes") or {}).get("tableId", "") for d in datasets} @@ -694,8 +691,6 @@ def add_metric( if description: data["description"] = description return client.post_item("semantic-metric", name=name, data=data) - finally: - client.close() def add_dataset( self, @@ -715,8 +710,7 @@ def add_dataset( synthesises a ``fields[]`` array with role heuristics. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) data: dict[str, Any] = { "name": name, @@ -741,8 +735,6 @@ def add_dataset( if fields: data["fields"] = fields return client.post_item("semantic-dataset", name=name, data=data) - finally: - client.close() def add_relationship( self, @@ -762,8 +754,7 @@ def add_relationship( error_code=ErrorCode.VALIDATION_ERROR, ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) data = { "name": name, @@ -774,8 +765,6 @@ def add_relationship( "modelUUID": model_uuid, } return client.post_item("semantic-relationship", name=name, data=data) - finally: - client.close() def add_constraint( self, @@ -800,8 +789,7 @@ def add_constraint( # METRICS exist in model project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) existing = client.list_items("semantic-metric", model_uuid) existing_names = {(m.get("attributes") or {}).get("name", "") for m in existing} @@ -823,8 +811,6 @@ def add_constraint( "modelUUID": model_uuid, } return client.post_item("semantic-constraint", name=name, data=data) - finally: - client.close() def add_glossary( self, @@ -836,15 +822,12 @@ def add_glossary( ) -> dict[str, Any]: """Create a glossary term. Outer envelope ``name`` must equal ``term``.""" project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) data: dict[str, Any] = {"term": term, "modelUUID": model_uuid} if definition: data["definition"] = definition return client.post_item("semantic-glossary", name=term, data=data) - finally: - client.close() # ------------------------------------------------------------------ # Phase 4 — edit (DELETE-then-POST with rollback + rename cascade) @@ -880,8 +863,7 @@ def edit_metric( delegates. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) return _edit_metric_helper( client, @@ -895,8 +877,6 @@ def edit_metric( is_tty=is_tty, confirm_cb=confirm_cb, ) - finally: - client.close() def edit_dataset( self, @@ -910,8 +890,7 @@ def edit_dataset( ) -> dict[str, Any]: """Edit a dataset (DELETE+POST). Renames do NOT cascade for datasets.""" project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) return _edit_simple_helper( client, @@ -926,8 +905,6 @@ def edit_dataset( }, not_found_label="Dataset", ) - finally: - client.close() def edit_constraint( self, @@ -952,8 +929,7 @@ def edit_constraint( ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) if new_metrics is not None: existing = client.list_items("semantic-metric", model_uuid) @@ -979,8 +955,6 @@ def edit_constraint( }, not_found_label="Constraint", ) - finally: - client.close() def edit_relationship( self, @@ -1006,8 +980,7 @@ def edit_relationship( error_code=ErrorCode.VALIDATION_ERROR, ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) return _edit_simple_helper( client, @@ -1024,8 +997,6 @@ def edit_relationship( }, not_found_label="Relationship", ) - finally: - client.close() def edit_glossary( self, @@ -1044,8 +1015,7 @@ def edit_glossary( behind ``--yes``; this method just executes. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) return _edit_simple_helper( client, @@ -1056,8 +1026,6 @@ def edit_glossary( overrides={"term": new_term, "definition": new_definition}, not_found_label="Glossary term", ) - finally: - client.close() # ------------------------------------------------------------------ # Phase 5 — remove (destructive, orphan-warning before delete) @@ -1089,8 +1057,7 @@ def preview_remove( error_code=ErrorCode.VALIDATION_ERROR, ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) target, _, _ = _find_target_for_remove( client, @@ -1110,8 +1077,6 @@ def preview_remove( "name": name, "orphaned_constraints": orphan_constraints, } - finally: - client.close() def remove_item( self, @@ -1130,8 +1095,7 @@ def remove_item( error_code=ErrorCode.VALIDATION_ERROR, ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) target, type_slug, _ = _find_target_for_remove( client, @@ -1150,8 +1114,6 @@ def remove_item( "removed": {"type": type_slug, "id": target["id"], "name": name}, "orphaned_constraints": orphan_constraints, } - finally: - client.close() # ------------------------------------------------------------------ # Phase 6 — import (replay a snapshot, optionally overwrite) @@ -1235,8 +1197,7 @@ def import_snapshot_from_dict( type_filter = _validate_types_filter(types) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) existing_by_type = self._fetch_children_parallel(client, model_uuid) @@ -1257,8 +1218,6 @@ def import_snapshot_from_dict( "overwrite": overwrite, "imported": imported, } - finally: - client.close() # ------------------------------------------------------------------ # Phase 6 — promote (cross-project copy) @@ -1295,9 +1254,10 @@ def promote_model( type_filter = _validate_types_filter(types) - src_client = self._new_metastore_client(projects[from_project]) - tgt_client = self._new_metastore_client(projects[to_project]) - try: + with ( + self._new_metastore_client(projects[from_project]) as src_client, + self._new_metastore_client(projects[to_project]) as tgt_client, + ): src_uuid, _ = self._resolve_model(src_client, from_model) tgt_uuid, _ = self._resolve_model(tgt_client, to_model) @@ -1320,11 +1280,6 @@ def promote_model( "dry_run": dry_run, **per_type_stats, } - finally: - try: - src_client.close() - finally: - tgt_client.close() # ------------------------------------------------------------------ # Phase 7 — build (AI-assisted / heuristic greenfield) @@ -1432,8 +1387,7 @@ def build_model( # Push to the metastore in dependency order. project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: counts, model_uuid, model_item = _push_built_model( client, generated=generated, @@ -1444,8 +1398,6 @@ def build_model( result["model"] = {"id": model_uuid, "item": model_item} result["created"] = counts return result - finally: - client.close() # ------------------------------------------------------------------ # Phase 8 — token --encrypt diff --git a/src/keboola_agent_cli/services/storage_service.py b/src/keboola_agent_cli/services/storage_service.py index 19423cf3..bf90a39c 100644 --- a/src/keboola_agent_cli/services/storage_service.py +++ b/src/keboola_agent_cli/services/storage_service.py @@ -711,6 +711,7 @@ def create_table( branch_id: int | None = None, not_null_columns: list[str] | None = None, defaults: list[str] | None = None, + if_not_exists: bool = False, ) -> dict[str, Any]: """Create a new table with typed columns. @@ -777,28 +778,63 @@ def create_table( project = projects[alias] client = self._client_factory(project.stack_url, project.token) + target_table_id = f"{bucket_id}.{name}" try: auto_created_bucket = _ensure_bucket_exists_in_branch(client, bucket_id, branch_id) - results = client.create_table( - bucket_id=bucket_id, - name=name, - columns=parsed_columns, - primary_key=primary_key, - branch_id=branch_id, - ) + try: + results = client.create_table( + bucket_id=bucket_id, + name=name, + columns=parsed_columns, + primary_key=primary_key, + branch_id=branch_id, + ) + except KeboolaApiError as exc: + # IF-NOT-EXISTS: if the create failed because the table + # already has the same display name AND the table at the + # expected id resolves, treat as a successful skip. A + # different table with the same display name still + # surfaces the original error (the user has a real + # conflict to resolve). + if ( + if_not_exists + and exc.error_code == ErrorCode.STORAGE_JOB_FAILED + and "already has the same display name" in (exc.message or "") + ): + try: + existing = client.get_table_detail(target_table_id, branch_id=branch_id) + except KeboolaApiError: + existing = None + if existing is not None: + return { + "project_alias": alias, + "table_id": target_table_id, + "name": name, + "bucket_id": bucket_id, + "primary_key": primary_key or [], + "columns": [c["name"] for c in parsed_columns], + "auto_created_bucket": auto_created_bucket, + "legacy_branch_storage": _detect_legacy_branch_storage( + client, branch_id + ), + "action": "skipped", + "skip_reason": "table already exists", + } + raise legacy_branch_storage = _detect_legacy_branch_storage(client, branch_id) finally: client.close() return { "project_alias": alias, - "table_id": results.get("id", f"{bucket_id}.{name}"), + "table_id": results.get("id", target_table_id), "name": name, "bucket_id": bucket_id, "primary_key": primary_key or [], "columns": [c["name"] for c in parsed_columns], "auto_created_bucket": auto_created_bucket, "legacy_branch_storage": legacy_branch_storage, + "action": "created", } def upload_table( diff --git a/src/keboola_agent_cli/services/sync_service.py b/src/keboola_agent_cli/services/sync_service.py index 54369cc5..5681f86a 100644 --- a/src/keboola_agent_cli/services/sync_service.py +++ b/src/keboola_agent_cli/services/sync_service.py @@ -281,6 +281,7 @@ def pull( with_samples: bool = False, sample_limit: int = DEFAULT_SAMPLE_LIMIT, max_samples: int = DEFAULT_MAX_SAMPLES, + branch_override: int | None = None, ) -> dict[str, Any]: """Download all configurations from Keboola to local filesystem. @@ -295,6 +296,8 @@ def pull( with_samples: Download table data samples (opt-in). sample_limit: Max rows per sample (default 100). max_samples: Max number of tables to sample (default 50). + branch_override: If set, pull from this branch ID rather than + the resolved active/manifest branch (CLI ``--branch``). Returns: Dict with pull statistics (configs, rows, files written). @@ -306,7 +309,9 @@ def pull( manifest = load_manifest(project_root) # Determine branch to pull from (git-branching aware) - branch_id = self._resolve_branch_id(project, manifest, project_root) + branch_id = self._resolve_branch_id( + project, manifest, project_root, branch_override=branch_override + ) # Fetch all components with configs from API (+ storage metadata + jobs) client = self._client_factory(project.stack_url, project.token) @@ -840,12 +845,19 @@ def diff( self, alias: str, project_root: Path, + branch_override: int | None = None, ) -> dict[str, Any]: """Compare local configs against the remote API state. Fetches current state from API, reads local _config.yml files, and runs the diff engine to produce a detailed changeset. + Args: + alias: Project alias. + project_root: Sync working-tree root. + branch_override: If set, diff against this branch ID rather + than the resolved active/manifest branch. + Returns: Dict with 'changes' list and summary counts. """ @@ -853,7 +865,9 @@ def diff( project = projects[alias] manifest = load_manifest(project_root) - branch_id = self._resolve_branch_id(project, manifest, project_root) + branch_id = self._resolve_branch_id( + project, manifest, project_root, branch_override=branch_override + ) # Fetch remote state client = self._client_factory(project.stack_url, project.token) @@ -1086,6 +1100,8 @@ def push( dry_run: bool = False, force: bool = False, allow_plaintext_fallback: bool = False, + branch_override: int | None = None, + no_name_drift_warnings: bool = False, ) -> dict[str, Any]: """Push local changes to Keboola. @@ -1097,11 +1113,20 @@ def push( project_root: Root directory of the sync working tree. dry_run: If True, compute changes but don't execute them. force: If True, allow deletions without extra confirmation. + allow_plaintext_fallback: If True, allow push when secret + encryption fails (DANGEROUS). + branch_override: If set, target this dev-branch ID for the push. + Wins over ``active_branch_id`` / ``manifest.branches[0]`` / + git-branching mapping. Used by the CLI ``--branch`` flag. + no_name_drift_warnings: If True, omit the ``name_drift_warnings`` + array from the result envelope (the underlying detection + still runs; only the report is suppressed). Used by the CLI + ``--no-name-drift-warnings`` flag. Returns: Dict with push results (created, updated, deleted, errors). """ - diff_result = self.diff(alias, project_root) + diff_result = self.diff(alias, project_root, branch_override=branch_override) all_changes = diff_result["changes"] # Only push local-side changes (added, modified, deleted). @@ -1136,7 +1161,9 @@ def push( project = projects[alias] manifest = load_manifest(project_root) - branch_id = self._resolve_branch_id(project, manifest, project_root) + branch_id = self._resolve_branch_id( + project, manifest, project_root, branch_override=branch_override + ) # Detect name drift: local dir name doesn't match config name name_drift_warnings = self._detect_name_drift(manifest, project_root) @@ -1197,7 +1224,6 @@ def push( ) if result: new_id = str(result.get("id", "")) - # Add to manifest with the API-assigned ID config_dir = project_root / branch_path / config_path_str config_file = config_dir / CONFIG_FILENAME file_hash = self._file_hash(config_file) if config_file.exists() else "" @@ -1207,18 +1233,30 @@ def push( cfg_hash = config_hash(local_data) else: cfg_hash = "" - manifest.configurations.append( - ManifestConfiguration( - branchId=branch_id or 0, - componentId=component_id, - id=new_id, - path=config_path_str, - metadata={ - "pull_hash": file_hash, - "pull_config_hash": cfg_hash, - }, - ) + entry = self._writeback_create_config_in_manifest( + manifest=manifest, + component_id=component_id, + branch_id=branch_id, + config_path_str=config_path_str, + new_id=new_id, + file_hash=file_hash, + cfg_hash=cfg_hash, ) + metadata_error = self._propagate_kbc_metadata(client, entry, branch_id) + if metadata_error is not None: + # The config IS on the remote; only the + # follow-up metadata POST failed. Accumulate + # like any other per-change error so the rest + # of the push continues, and surface the + # original cause in the envelope. + errors.append( + { + "change_type": "metadata_propagation", + "component_id": component_id, + "config_id": new_id, + "message": metadata_error, + } + ) manifest_dirty = True created += 1 pushed_details.append(change) @@ -1319,7 +1357,7 @@ def push( "errors": errors, "pushed_details": pushed_details, } - if name_drift_warnings: + if name_drift_warnings and not no_name_drift_warnings: result_data["name_drift_warnings"] = name_drift_warnings return result_data @@ -1463,12 +1501,12 @@ def _push_create_row( row_file = row_dir / CONFIG_FILENAME new_file_hash = self._file_hash(row_file) if row_file.exists() else "" cfg_hash_value = config_hash(pristine_data) - parent.rows.append( - ManifestConfigRow( - id=new_row_id, - path=row_path_str, - metadata={"pull_hash": new_file_hash, "pull_config_hash": cfg_hash_value}, - ) + self._writeback_create_row_in_manifest( + parent=parent, + row_path_str=row_path_str, + new_row_id=new_row_id, + file_hash=new_file_hash, + cfg_hash=cfg_hash_value, ) def _push_update_row( @@ -1656,6 +1694,123 @@ def _push_update( # Use pristine_data so blocks/code stay only in their code files. self._writeback_after_push(pristine_data, config_dir, config_id, configuration) + def _writeback_create_config_in_manifest( + self, + *, + manifest: Manifest, + component_id: str, + branch_id: int | None, + config_path_str: str, + new_id: str, + file_hash: str, + cfg_hash: str, + ) -> ManifestConfiguration: + """Record a freshly-created config in the manifest. + + If a placeholder entry already exists at + ``(branch_id, component_id, path)`` -- the FIIA / scaffold emit + pattern -- update it in place, preserving any user-declared metadata + (e.g. ``KBC.configuration.folderName``) and refreshing only the + bookkeeping hashes. Otherwise append a new entry. + + Matching includes ``branch_id`` because a single manifest can hold + entries from multiple branches in git-branching mode; matching on + ``(component_id, path)`` alone would risk updating the wrong branch's + entry when the same logical path exists under two branches. + """ + target_branch = branch_id or 0 + for entry in manifest.configurations: + if ( + entry.branch_id == target_branch + and entry.component_id == component_id + and entry.path == config_path_str + ): + entry.id = new_id + entry.metadata["pull_hash"] = file_hash + entry.metadata["pull_config_hash"] = cfg_hash + return entry + new_entry = ManifestConfiguration( + branchId=target_branch, + componentId=component_id, + id=new_id, + path=config_path_str, + metadata={"pull_hash": file_hash, "pull_config_hash": cfg_hash}, + ) + manifest.configurations.append(new_entry) + return new_entry + + def _writeback_create_row_in_manifest( + self, + *, + parent: ManifestConfiguration, + row_path_str: str, + new_row_id: str, + file_hash: str, + cfg_hash: str, + ) -> ManifestConfigRow: + """Record a freshly-created row under its parent in the manifest. + + Mirrors :meth:`_writeback_create_config_in_manifest` for rows: update + any placeholder row entry in place, otherwise append. + """ + for row in parent.rows: + if row.path == row_path_str: + row.id = new_row_id + row.metadata["pull_hash"] = file_hash + row.metadata["pull_config_hash"] = cfg_hash + return row + new_row = ManifestConfigRow( + id=new_row_id, + path=row_path_str, + metadata={"pull_hash": file_hash, "pull_config_hash": cfg_hash}, + ) + parent.rows.append(new_row) + return new_row + + def _propagate_kbc_metadata( + self, + client: Any, + entry: ManifestConfiguration, + branch_id: int | None, + ) -> str | None: + """POST any ``KBC.*`` keys from the manifest entry to the metadata API. + + Bookkeeping keys (``pull_hash``, ``pull_config_hash``, ...) live in the + same metadata dict but are filtered by the ``KBC.`` prefix. Called only + on CREATE; updates use ``kbagent config set-metadata`` explicitly. The + metadata API stores configuration-level annotations only -- this is + **not** a secret store; do not place tokens or passwords under + ``KBC.*`` keys. + + Returns ``None`` on success (or when there are no KBC.* keys to + propagate). Returns the API error message on a non-fatal write + failure: the config is already created on the remote and the + manifest writeback is complete, so a single failed metadata POST + is reported back to the push loop as an accumulated error rather + than aborting the rest of the push. + """ + entries = [ + (key, str(value)) for key, value in entry.metadata.items() if key.startswith("KBC.") + ] + if not entries: + return None + try: + client.set_config_metadata( + component_id=entry.component_id, + config_id=entry.id, + entries=entries, + branch_id=branch_id, + ) + except KeboolaApiError as exc: + logger.warning( + "Failed to propagate KBC.* metadata for %s/%s: %s", + entry.component_id, + entry.id, + exc, + ) + return exc.message + return None + def _writeback_after_push( self, local_data: dict[str, Any], @@ -2120,10 +2275,12 @@ def _resolve_branch_id( project: Any, manifest: "Manifest", project_root: Path, + branch_override: int | None = None, ) -> int | None: """Resolve the Keboola branch ID for sync operations. Priority: + 0. ``branch_override`` (CLI ``--branch ``) -- wins over everything 1. Git-branching mode: read branch-mapping.json for current git branch 2. ``active_branch_id`` from project config (``kbagent branch use``) 3. First branch in manifest (production fallback) @@ -2139,6 +2296,12 @@ def _resolve_branch_id( from ..sync.branch_mapping import load_branch_mapping from ..sync.git_utils import get_current_branch + # CLI override beats every persisted source so a user can target a + # dev branch from a clean git workspace without first running + # `branch use` or `branch-link`. + if branch_override is not None: + return branch_override + if manifest.git_branching.enabled: git_branch = get_current_branch(project_root) if git_branch: diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 9cc96ef1..9a80c249 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -58,6 +58,7 @@ from unittest.mock import patch import pytest +import yaml from typer.testing import CliRunner from helpers import metastore_scope_available @@ -9596,3 +9597,524 @@ def _direct_delete(item_type: str, item_id: str) -> None: ) except _ApiError as exc: print(f" WARN: residue scan failed: {exc}") + + +# --------------------------------------------------------------------------- +# v0.47.0 -- fresh-CREATE writeback + new ergonomic flags (E2E coverage per +# CLAUDE.md convention #16: "Every new CLI command MUST have a corresponding +# E2E test in tests/test_e2e.py"). +# --------------------------------------------------------------------------- + + +@skip_without_credentials +@pytest.mark.e2e +class TestE2E_0_47_0_NewSurfaces: + """E2E coverage for v0.47.0 additions. + + - ``storage create-table --if-not-exists`` -- idempotent re-create + - ``semantic-layer search-context`` + ``get-context`` -- project-wide read + - ``sync diff --branch `` -- per-invocation dev-branch override + + All three touch a real Keboola project via the configured E2E token. The + test creates a throwaway dev branch where needed and deletes it in the + teardown so residue does not accumulate across re-runs. + """ + + @pytest.fixture(autouse=True) + def setup(self, tmp_path: Path): + self.token = os.environ[ENV_TOKEN] + raw_url = os.environ.get(ENV_URL, "connection.keboola.com") + self.url = raw_url if raw_url.startswith("https://") else f"https://{raw_url}" + self.alias = f"{RUN_ID}-v0470" + self.config_dir = tmp_path / "config" + self.config_dir.mkdir() + self.tmp_path = tmp_path + + result = _invoke( + self.config_dir, + [ + "--json", + "project", + "add", + "--project", + self.alias, + "--url", + self.url, + "--token", + self.token, + ], + ) + assert result.exit_code == 0, f"project add failed: {result.output}" + self._dev_branch_id: int | None = None + self._created_bucket_id: str | None = None + try: + yield + finally: + if self._dev_branch_id is not None: + try: + self._run( + "branch", + "delete", + "--project", + self.alias, + "--branch", + str(self._dev_branch_id), + ) + except Exception as exc: + print(f" WARN: branch delete failed: {exc}") + if self._created_bucket_id is not None: + try: + self._run( + "storage", + "delete-bucket", + "--project", + self.alias, + "--bucket-id", + self._created_bucket_id, + "--force", + "--yes", + ) + except Exception as exc: + print(f" WARN: bucket delete failed: {exc}") + + def _run(self, *args: str) -> Any: + return _invoke(self.config_dir, ["--json", *args]) + + def _run_ok(self, *args: str) -> dict[str, Any]: + return _json_ok(self._run(*args)) + + # ------------------------------------------------------------------ + # storage create-table --if-not-exists + # ------------------------------------------------------------------ + + def test_storage_create_table_if_not_exists_round_trip(self) -> None: + """First call: action=created. Second call with --if-not-exists: + action=skipped. Third call without the flag: STORAGE_JOB_FAILED.""" + _step("v0470-1", "storage create-table --if-not-exists") + bucket_name = f"v0470_{RUN_ID.replace('-', '_')[:20]}" + bucket_data = self._run_ok( + "storage", + "create-bucket", + "--project", + self.alias, + "--stage", + "in", + "--name", + bucket_name, + ) + bucket_id = bucket_data["data"]["id"] + assert bucket_id.startswith("in.c-") + self._created_bucket_id = bucket_id + + table_name = f"v0470_tbl_{RUN_ID.replace('-', '_')[:16]}" + + first = self._run_ok( + "storage", + "create-table", + "--project", + self.alias, + "--bucket-id", + bucket_id, + "--name", + table_name, + "--column", + "id:INTEGER", + "--column", + "label:STRING", + "--primary-key", + "id", + "--if-not-exists", + ) + assert first["data"]["action"] == "created" + assert first["data"]["table_id"] == f"{bucket_id}.{table_name}" + + second = self._run_ok( + "storage", + "create-table", + "--project", + self.alias, + "--bucket-id", + bucket_id, + "--name", + table_name, + "--column", + "id:INTEGER", + "--column", + "label:STRING", + "--primary-key", + "id", + "--if-not-exists", + ) + assert second["data"]["action"] == "skipped" + assert second["data"]["skip_reason"] == "table already exists" + assert second["data"]["table_id"] == f"{bucket_id}.{table_name}" + + third = self._run( + "storage", + "create-table", + "--project", + self.alias, + "--bucket-id", + bucket_id, + "--name", + table_name, + "--column", + "id:INTEGER", + "--primary-key", + "id", + ) + assert third.exit_code != 0, ( + "default behavior must still error on duplicate (no silent skip)" + ) + body = json.loads(third.output) + assert body.get("status") == "error" + assert body.get("error", {}).get("code") == "STORAGE_JOB_FAILED" + + # ------------------------------------------------------------------ + # semantic-layer search-context / get-context + # ------------------------------------------------------------------ + + def test_semantic_layer_search_and_get_context(self) -> None: + """search-context with default pattern returns a valid envelope. + get-context with an all-zero UUID returns NOT_FOUND.""" + _step("v0470-2", "semantic-layer search-context + get-context") + + search = self._run_ok( + "semantic-layer", + "search-context", + "--project", + self.alias, + "--pattern", + "*", + ) + data = search["data"] + assert "contexts" in data + assert "total_count" in data + assert isinstance(data["contexts"], list) + assert isinstance(data["total_count"], int) + assert data["total_count"] == len(data["contexts"]) + for ctx in data["contexts"]: + assert ctx["type"] in { + "model", + "dataset", + "metric", + "relationship", + "constraint", + "glossary", + }, f"unexpected type slug: {ctx['type']!r}" + + only_datasets = self._run_ok( + "semantic-layer", + "search-context", + "--project", + self.alias, + "--type", + "dataset", + ) + for ctx in only_datasets["data"]["contexts"]: + assert ctx["type"] == "dataset" + + missing = self._run( + "semantic-layer", + "get-context", + "--project", + self.alias, + "--context-id", + "00000000-0000-0000-0000-000000000000", + ) + assert missing.exit_code != 0 + body = json.loads(missing.output) + assert body.get("status") == "error" + assert body.get("error", {}).get("code") == "NOT_FOUND" + + if data["contexts"]: + first_id = data["contexts"][0]["id"] + roundtrip = self._run_ok( + "semantic-layer", + "get-context", + "--project", + self.alias, + "--context-id", + first_id, + ) + assert roundtrip["data"]["id"] == first_id + assert roundtrip["data"]["type"] == data["contexts"][0]["type"] + + # ------------------------------------------------------------------ + # sync diff --branch + # ------------------------------------------------------------------ + + def test_sync_diff_branch_override(self) -> None: + """A dev branch created on the fly is targetable via `sync diff --branch` + without first running `branch use` or `sync branch-link`.""" + _step("v0470-3", "sync diff --branch ") + + branch_name = f"v0470-e2e-{RUN_ID[:20]}" + branch_data = self._run_ok( + "branch", + "create", + "--project", + self.alias, + "--name", + branch_name, + ) + dev_branch_id = int(branch_data["data"]["branch_id"]) + self._dev_branch_id = dev_branch_id + + project_dir = self.tmp_path / "v0470-sync" + project_dir.mkdir() + _git(project_dir, "init") + _git(project_dir, "config", "user.email", "e2e@test.local") + _git(project_dir, "config", "user.name", "E2E Test") + _git(project_dir, "commit", "--allow-empty", "-m", "init") + + init_result = _invoke( + self.config_dir, + [ + "--json", + "sync", + "init", + "--project", + self.alias, + "--directory", + str(project_dir), + ], + ) + assert init_result.exit_code == 0, init_result.output + + with_override = _invoke( + self.config_dir, + [ + "--json", + "sync", + "diff", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + ], + ) + body = json.loads(with_override.output) + assert body.get("status") == "ok", body + assert body["data"].get("changes") is not None + summary = body["data"].get("summary", {}) + assert summary.get("remote_only", 0) >= 0 + + # ------------------------------------------------------------------ + # sync push -- fresh-CREATE writeback + KBC.configuration.* propagation + # (Area B headline fix; against real Storage API + metadata API) + # ------------------------------------------------------------------ + + def test_sync_push_fresh_create_writeback_and_kbc_metadata(self) -> None: + """Round-trip the FIIA / scaffold emit pattern against a real + Keboola project: hand-author a placeholder ManifestConfiguration + with ``KBC.configuration.folderName`` declared, run ``sync push``, + and assert: + 1. The push reports ``created=1, errors=0``. + 2. The manifest entry was updated in place (length stays at 1, + not 2; placeholder id is now the assigned ULID; folderName + metadata is preserved on the entry). + 3. The remote configuration's metadata-list returns the + KBC.configuration.folderName key with the declared value. + 4. A second ``sync push`` against the same workspace is a no-op + (``status=no_changes, created=0, errors=0``). + + Cleanup: delete the freshly-created remote config + the dev branch + in the teardown so re-runs do not accumulate residue. + """ + from keboola_agent_cli.constants import CONFIG_FILENAME, CONFIG_YML_VERSION + from keboola_agent_cli.sync.manifest import ( + ManifestConfiguration, + load_manifest, + save_manifest, + ) + + _step("v0470-4", "sync push fresh-CREATE writeback + KBC.* propagation") + + # Throwaway dev branch so we never pollute main. + branch_name = f"v0470-fcw-{RUN_ID[:20]}" + branch_data = self._run_ok( + "branch", + "create", + "--project", + self.alias, + "--name", + branch_name, + ) + dev_branch_id = int(branch_data["data"]["branch_id"]) + self._dev_branch_id = dev_branch_id # teardown will delete + + # Fresh sync workspace. + project_dir = self.tmp_path / "v0470-fcw" + project_dir.mkdir() + _git(project_dir, "init") + _git(project_dir, "config", "user.email", "e2e@test.local") + _git(project_dir, "config", "user.name", "E2E Test") + _git(project_dir, "commit", "--allow-empty", "-m", "init") + + init_result = _invoke( + self.config_dir, + [ + "--json", + "sync", + "init", + "--project", + self.alias, + "--directory", + str(project_dir), + ], + ) + assert init_result.exit_code == 0, init_result.output + + # Pull the dev branch so its branch directory + entry land in the + # manifest (otherwise the placeholder's target branch is not + # tracked and sync push can't resolve a path for it). + pull_result = _invoke( + self.config_dir, + [ + "--json", + "sync", + "pull", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + "--no-storage", + "--no-jobs", + ], + ) + assert pull_result.exit_code == 0, pull_result.output + + manifest = load_manifest(project_dir) + dev_branch_entry = next((b for b in manifest.branches if b.id == dev_branch_id), None) + assert dev_branch_entry is not None, ( + "sync pull --branch must register the dev branch in the manifest" + ) + dev_branch_path = dev_branch_entry.path + + # Hand-author a placeholder ManifestConfiguration with the + # KBC.configuration.folderName key (FIIA / scaffold emit pattern). + component_id = "keboola.snowflake-transformation" + config_dir_name = f"v0470-fcw-{RUN_ID[:18]}" + cfg_rel_path = f"transformation/{component_id}/{config_dir_name}" + folder_name = "v0.47.0 E2E Fresh-CREATE" + manifest.configurations.append( + ManifestConfiguration( + branchId=dev_branch_id, + componentId=component_id, + id="PLACEHOLDER-FCW", + path=cfg_rel_path, + metadata={"KBC.configuration.folderName": folder_name}, + ) + ) + save_manifest(project_dir, manifest) + pre_push_n = len(manifest.configurations) + + # Local _config.yml for the placeholder. + local_dir = project_dir / dev_branch_path / cfg_rel_path + local_dir.mkdir(parents=True) + (local_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": "v0.47.0 e2e fresh-create", + "description": "E2E test: fresh-CREATE writeback in place", + "parameters": {}, + "_keboola": {"component_id": component_id, "config_id": ""}, + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + + # First push: should CREATE the config + propagate the folder. + push_result = _invoke( + self.config_dir, + [ + "--json", + "sync", + "push", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + ], + ) + assert push_result.exit_code == 0, push_result.output + body = json.loads(push_result.output) + assert body.get("status") == "ok", body + data = body["data"] + assert data["created"] == 1, data + assert data["errors"] == [], data["errors"] + + # Manifest contract: updated in place (length unchanged). + post = load_manifest(project_dir) + matching = [ + c + for c in post.configurations + if c.component_id == component_id + and c.path == cfg_rel_path + and c.branch_id == dev_branch_id + ] + assert len(matching) == 1, ( + "writeback must update placeholder in place, not duplicate; " + f"found {len(matching)} matching entries" + ) + assigned_id = matching[0].id + assert assigned_id != "PLACEHOLDER-FCW", ( + "placeholder id must be replaced with the API-assigned ULID" + ) + assert matching[0].metadata.get("KBC.configuration.folderName") == folder_name + assert len(post.configurations) == pre_push_n, "manifest must not grow on a single CREATE" + + # Remote metadata: folderName landed via the metadata API. + meta_result = _invoke( + self.config_dir, + [ + "--json", + "config", + "metadata-list", + "--project", + self.alias, + "--component-id", + component_id, + "--config-id", + assigned_id, + "--branch", + str(dev_branch_id), + ], + ) + meta = _json_ok(meta_result) + meta_keys = {m.get("key"): m.get("value") for m in meta["data"]["metadata"]} + assert meta_keys.get("KBC.configuration.folderName") == folder_name, ( + f"folderName missing or wrong on remote metadata-list: {meta_keys}" + ) + + # Second push: idempotent (no_changes; create_config NOT called again). + repush = _invoke( + self.config_dir, + [ + "--json", + "sync", + "push", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + ], + ) + assert repush.exit_code == 0, repush.output + repush_body = json.loads(repush.output) + repush_status = repush_body.get("data", {}).get("status") or repush_body.get("status") + assert repush_status in ("no_changes", "pushed"), repush_body + assert repush_body["data"].get("created", 0) == 0, ( + "re-push must be idempotent: created=0 after writeback in place" + ) diff --git a/tests/test_semantic_layer_cli.py b/tests/test_semantic_layer_cli.py index 9b7ddb74..046278c6 100644 --- a/tests/test_semantic_layer_cli.py +++ b/tests/test_semantic_layer_cli.py @@ -1610,3 +1610,199 @@ def test_model_delete_denied_with_deny_destructive(self, store: ConfigStore) -> ) assert result.exit_code == 6, result.output assert "PERMISSION_DENIED" in result.output + + +# --------------------------------------------------------------------------- +# semantic-layer search-context / get-context (v0.47.0) +# --------------------------------------------------------------------------- + + +class TestSearchContext: + """CLI surface for the project-wide name-pattern search.""" + + def test_default_pattern_json(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.search_context.return_value = { + "project": "prod", + "contexts": [ + { + "id": "d1", + "type": "dataset", + "name": "users", + "description": "", + "attributes": {"name": "users"}, + } + ], + "total_count": 1, + } + result = _invoke( + ["--json", "semantic-layer", "search-context", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["total_count"] == 1 + assert body["data"]["contexts"][0]["type"] == "dataset" + # Default pattern propagates to the service. + call_kwargs = mock.search_context.call_args.kwargs + assert call_kwargs["patterns"] == ["*"] + + def test_pattern_and_type_filter_propagate(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.search_context.return_value = { + "project": "prod", + "contexts": [], + "total_count": 0, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "search-context", + "--project", + "prod", + "--pattern", + "DIM_*", + "--pattern", + "FACT_*", + "--type", + "dataset", + "--limit", + "5", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + call_kwargs = mock.search_context.call_args.kwargs + assert call_kwargs["alias"] == "prod" + assert call_kwargs["patterns"] == ["DIM_*", "FACT_*"] + assert call_kwargs["type_filter"] == "dataset" + assert call_kwargs["limit"] == 5 + + def test_human_renders_table(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.search_context.return_value = { + "project": "prod", + "contexts": [ + { + "id": "m1", + "type": "metric", + "name": "revenue", + "description": "GMV", + "attributes": {}, + } + ], + "total_count": 1, + } + result = _invoke( + ["semantic-layer", "search-context", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0 + assert "revenue" in result.output + + def test_service_error_maps_to_exit_1(self, store: ConfigStore) -> None: + from keboola_agent_cli.errors import ErrorCode, KeboolaApiError + + mock = MagicMock() + mock.search_context.side_effect = KeboolaApiError( + message="Invalid type", error_code=ErrorCode.VALIDATION_ERROR + ) + result = _invoke( + [ + "--json", + "semantic-layer", + "search-context", + "--project", + "prod", + "--type", + "bogus", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 1, result.output + + +class TestGetContext: + """CLI surface for the single-id semantic context lookup.""" + + def test_get_context_json(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.get_context.return_value = { + "project": "prod", + "id": "u-model", + "type": "model", + "name": "default", + "description": "", + "attributes": {"name": "default", "sql_dialect": "Snowflake"}, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "get-context", + "--project", + "prod", + "--context-id", + "u-model", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["id"] == "u-model" + assert body["data"]["type"] == "model" + + def test_get_context_not_found_exits_nonzero(self, store: ConfigStore) -> None: + from keboola_agent_cli.errors import ErrorCode, KeboolaApiError + + mock = MagicMock() + mock.get_context.side_effect = KeboolaApiError( + message="not found", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + result = _invoke( + [ + "--json", + "semantic-layer", + "get-context", + "--project", + "prod", + "--context-id", + "ghost", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code != 0 + assert "not found" in result.output.lower() or "NOT_FOUND" in result.output + + def test_get_context_human_renders_attributes(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.get_context.return_value = { + "project": "prod", + "id": "d1", + "type": "dataset", + "name": "users", + "description": "User dimension table", + "attributes": {"name": "users", "primary_key": ["id"]}, + } + result = _invoke( + [ + "semantic-layer", + "get-context", + "--project", + "prod", + "--context-id", + "d1", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0 + assert "users" in result.output + assert "dataset" in result.output diff --git a/tests/test_semantic_layer_service.py b/tests/test_semantic_layer_service.py index 1b62e0dc..1f213aef 100644 --- a/tests/test_semantic_layer_service.py +++ b/tests/test_semantic_layer_service.py @@ -78,8 +78,15 @@ def _make_service( *, metastore_mock: MagicMock | None = None, ) -> tuple[SemanticLayerService, MagicMock]: - """Wire a SemanticLayerService with a mocked metastore client factory.""" + """Wire a SemanticLayerService with a mocked metastore client factory. + + The mock supports the context-manager protocol (``__enter__`` returns + self, ``__exit__`` is a no-op) so the service-layer `with` blocks see + the same MagicMock body that tests configure side-effects on. + """ mock = metastore_mock or MagicMock() + mock.__enter__ = MagicMock(return_value=mock) + mock.__exit__ = MagicMock(return_value=False) service = SemanticLayerService( config_store=store, metastore_client_factory=lambda url, token: mock, @@ -256,7 +263,7 @@ def test_returns_shape(self, tmp_path: Path) -> None: "description": "first", "sql_dialect": "Snowflake", } - mock.close.assert_called_once() + mock.__exit__.assert_called_once() def test_empty_project(self, tmp_path: Path) -> None: store = _make_store(tmp_path) @@ -1928,7 +1935,11 @@ def test_classification_new_changed_identical(self, tmp_path: Path) -> None: store = _make_store_two(tmp_path) src_mock = MagicMock() + src_mock.__enter__ = MagicMock(return_value=src_mock) + src_mock.__exit__ = MagicMock(return_value=False) tgt_mock = MagicMock() + tgt_mock.__enter__ = MagicMock(return_value=tgt_mock) + tgt_mock.__exit__ = MagicMock(return_value=False) clients = {0: src_mock, 1: tgt_mock} call_idx = {"i": 0} @@ -1991,7 +2002,11 @@ def test_both_clients_closed_even_on_error(self, tmp_path: Path) -> None: store = _make_store_two(tmp_path) src_mock = MagicMock() + src_mock.__enter__ = MagicMock(return_value=src_mock) + src_mock.__exit__ = MagicMock(return_value=False) tgt_mock = MagicMock() + tgt_mock.__enter__ = MagicMock(return_value=tgt_mock) + tgt_mock.__exit__ = MagicMock(return_value=False) clients = {0: src_mock, 1: tgt_mock} call_idx = {"i": 0} @@ -2010,8 +2025,8 @@ def _factory(url: str, token: str) -> MagicMock: with pytest.raises(RuntimeError): service.promote_model(from_project="source", to_project="target") - src_mock.close.assert_called_once() - tgt_mock.close.assert_called_once() + src_mock.__exit__.assert_called_once() + tgt_mock.__exit__.assert_called_once() # --------------------------------------------------------------------------- @@ -2397,3 +2412,271 @@ def test_constraint_types_complete(self) -> None: def test_constraint_severities(self) -> None: assert set(CONSTRAINT_SEVERITIES) == {"error", "warning", "info"} + + +# --------------------------------------------------------------------------- +# search-context / get-context (v0.47.0) +# --------------------------------------------------------------------------- + + +class TestSearchContext: + """Project-wide name-pattern search across semantic-layer entities.""" + + def _setup( + self, tmp_path: Path, items_by_type: dict[str, list[dict[str, Any]]] + ) -> tuple[SemanticLayerService, MagicMock]: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + return items_by_type.get(item_type, []) + + mock.list_items.side_effect = _list + return service, mock + + def test_default_pattern_matches_everything(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [_child_item("semantic-dataset", "d1", {"name": "users"})], + "semantic-metric": [_child_item("semantic-metric", "m1", {"name": "revenue"})], + } + service, _ = self._setup(tmp_path, items) + + result = service.search_context("prod") + + assert result["project"] == "prod" + assert result["total_count"] == 2 + names = sorted(c["name"] for c in result["contexts"]) + assert names == ["revenue", "users"] + # Types are CLI-friendly singular (no "semantic-" prefix). + types = {c["type"] for c in result["contexts"]} + assert types == {"dataset", "metric"} + + def test_pattern_glob_narrows_results(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [ + _child_item("semantic-dataset", "d1", {"name": "DIM_users"}), + _child_item("semantic-dataset", "d2", {"name": "FACT_orders"}), + _child_item("semantic-dataset", "d3", {"name": "DIM_products"}), + ], + } + service, _ = self._setup(tmp_path, items) + + result = service.search_context("prod", patterns=["DIM_*"]) + + assert result["total_count"] == 2 + names = sorted(c["name"] for c in result["contexts"]) + assert names == ["DIM_products", "DIM_users"] + + def test_pattern_is_case_sensitive(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [ + _child_item("semantic-dataset", "d1", {"name": "DIM_x"}), + _child_item("semantic-dataset", "d2", {"name": "dim_y"}), + ], + } + service, _ = self._setup(tmp_path, items) + + upper = service.search_context("prod", patterns=["DIM_*"]) + lower = service.search_context("prod", patterns=["dim_*"]) + + assert [c["name"] for c in upper["contexts"]] == ["DIM_x"] + assert [c["name"] for c in lower["contexts"]] == ["dim_y"] + + def test_multiple_patterns_take_union(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [ + _child_item("semantic-dataset", "d1", {"name": "DIM_a"}), + _child_item("semantic-dataset", "d2", {"name": "FACT_b"}), + _child_item("semantic-dataset", "d3", {"name": "AGG_c"}), + ], + } + service, _ = self._setup(tmp_path, items) + + result = service.search_context("prod", patterns=["DIM_*", "FACT_*"]) + + assert result["total_count"] == 2 + assert {c["name"] for c in result["contexts"]} == {"DIM_a", "FACT_b"} + + def test_type_filter_narrows_to_one_kind(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [_child_item("semantic-dataset", "d1", {"name": "users"})], + "semantic-metric": [_child_item("semantic-metric", "m1", {"name": "revenue"})], + "semantic-relationship": [_child_item("semantic-relationship", "r1", {"name": "r"})], + "semantic-constraint": [_child_item("semantic-constraint", "c1", {"name": "c"})], + "semantic-glossary": [_child_item("semantic-glossary", "g1", {"name": "term"})], + } + service, mock = self._setup(tmp_path, items) + + result = service.search_context("prod", type_filter="metric") + + assert result["total_count"] == 1 + assert result["contexts"][0]["type"] == "metric" + called_types = {call.args[0] for call in mock.list_items.call_args_list} + assert called_types == {"semantic-metric"}, ( + "type_filter must short-circuit the per-type loop" + ) + + def test_type_filter_model(self, tmp_path: Path) -> None: + items = {"semantic-model": [_model_item("u1", "default")]} + service, mock = self._setup(tmp_path, items) + + result = service.search_context("prod", type_filter="model") + + assert result["total_count"] == 1 + assert result["contexts"][0]["type"] == "model" + assert mock.list_items.call_args_list[0].args[0] == "semantic-model" + + def test_type_filter_all_iterates_every_child(self, tmp_path: Path) -> None: + items = { + t: [_child_item(t, "x", {"name": "n"})] + for t in ( + "semantic-dataset", + "semantic-metric", + "semantic-relationship", + "semantic-constraint", + "semantic-glossary", + ) + } + service, mock = self._setup(tmp_path, items) + + result = service.search_context("prod", type_filter="all") + + assert result["total_count"] == 5 + called_types = {call.args[0] for call in mock.list_items.call_args_list} + assert called_types == set(items) + assert "semantic-model" not in called_types, ( + "type=all means every CHILD type, not the model itself" + ) + + def test_limit_caps_results_and_short_circuits(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [ + _child_item("semantic-dataset", f"d{i}", {"name": f"n{i}"}) for i in range(10) + ], + "semantic-metric": [ + _child_item("semantic-metric", "m1", {"name": "rev"}), + ], + } + service, mock = self._setup(tmp_path, items) + + result = service.search_context("prod", limit=3) + + assert result["total_count"] == 3 + # short-circuit: never reached semantic-metric + called_types = [call.args[0] for call in mock.list_items.call_args_list] + assert "semantic-metric" not in called_types + + def test_invalid_type_filter_rejected(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, {}) + + with pytest.raises(KeboolaApiError) as excinfo: + service.search_context("prod", type_filter="bogus") + + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_empty_pattern_rejected(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, {}) + + with pytest.raises(KeboolaApiError) as excinfo: + service.search_context("prod", patterns=["valid", ""]) + + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_zero_limit_rejected(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, {}) + + with pytest.raises(KeboolaApiError) as excinfo: + service.search_context("prod", limit=0) + + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_client_closed_even_on_api_error(self, tmp_path: Path) -> None: + """try/finally must close the client even when list_items raises.""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.side_effect = KeboolaApiError( + message="boom", status_code=500, error_code=ErrorCode.API_ERROR + ) + + with pytest.raises(KeboolaApiError): + service.search_context("prod") + + mock.__exit__.assert_called_once() + + +class TestGetContext: + """Single-id lookup across every semantic type.""" + + def _setup(self, tmp_path: Path) -> tuple[SemanticLayerService, MagicMock]: + return _make_service(_make_store(tmp_path)) + + def test_finds_dataset(self, tmp_path: Path) -> None: + service, mock = self._setup(tmp_path) + + def _get(item_type: str, item_id: str) -> dict[str, Any]: + if item_type == "semantic-dataset" and item_id == "d1": + return _child_item("semantic-dataset", "d1", {"name": "users"}) + raise KeboolaApiError(message="404", status_code=404, error_code=ErrorCode.NOT_FOUND) + + mock.get_item.side_effect = _get + result = service.get_context("prod", "d1") + + assert result["id"] == "d1" + assert result["type"] == "dataset" + assert result["name"] == "users" + + def test_finds_model(self, tmp_path: Path) -> None: + """Lookup probes model first; a model hit short-circuits the scan.""" + service, mock = self._setup(tmp_path) + mock.get_item.return_value = _model_item("u-model", "default") + + result = service.get_context("prod", "u-model") + + assert result["type"] == "model" + # Only one client call needed when model is the first probe. + assert mock.get_item.call_count == 1 + assert mock.get_item.call_args.args[0] == "semantic-model" + + def test_missing_id_raises_not_found(self, tmp_path: Path) -> None: + service, mock = self._setup(tmp_path) + mock.get_item.side_effect = KeboolaApiError( + message="404", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + + with pytest.raises(KeboolaApiError) as excinfo: + service.get_context("prod", "missing") + + assert excinfo.value.error_code == ErrorCode.NOT_FOUND + # Probed every type before giving up: model + 5 child types = 6 calls. + assert mock.get_item.call_count == 6 + + def test_non_404_error_propagates_immediately(self, tmp_path: Path) -> None: + """A 500 on one type must surface as-is, not be swallowed.""" + service, mock = self._setup(tmp_path) + mock.get_item.side_effect = KeboolaApiError( + message="boom", status_code=500, error_code=ErrorCode.API_ERROR + ) + + with pytest.raises(KeboolaApiError) as excinfo: + service.get_context("prod", "x") + + assert excinfo.value.error_code == ErrorCode.API_ERROR + + def test_empty_id_rejected(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path) + + with pytest.raises(KeboolaApiError) as excinfo: + service.get_context("prod", "") + + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_client_closed_even_on_api_error(self, tmp_path: Path) -> None: + service, mock = self._setup(tmp_path) + mock.get_item.side_effect = KeboolaApiError( + message="500", status_code=500, error_code=ErrorCode.API_ERROR + ) + + with pytest.raises(KeboolaApiError): + service.get_context("prod", "x") + + mock.__exit__.assert_called_once() diff --git a/tests/test_storage_write.py b/tests/test_storage_write.py index fc7a2d39..ce95454a 100644 --- a/tests/test_storage_write.py +++ b/tests/test_storage_write.py @@ -9,7 +9,7 @@ from keboola_agent_cli.cli import app from keboola_agent_cli.config_store import ConfigStore -from keboola_agent_cli.errors import KeboolaApiError +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError from keboola_agent_cli.models import AppConfig, ProjectConfig from keboola_agent_cli.services.storage_service import StorageService @@ -1041,6 +1041,128 @@ def test_create_bucket_api_error(self, tmp_path: Path) -> None: assert result.exit_code != 0 +# --------------------------------------------------------------------------- +# Service tests: create_table --if-not-exists (v0.47.0) +# --------------------------------------------------------------------------- + + +class TestCreateTableIfNotExists: + """`if_not_exists=True` turns duplicate-display-name into a skip.""" + + @staticmethod + def _duplicate_display_name_error() -> KeboolaApiError: + return KeboolaApiError( + message=( + "Bucket in.c-b.users already has the same display name in " + "bucket in.c-b. Please rename one of them." + ), + status_code=500, + error_code=ErrorCode.STORAGE_JOB_FAILED, + ) + + def test_skip_on_existing_when_flag_set(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.side_effect = self._duplicate_display_name_error() + mock_client.get_table_detail.return_value = {"id": "in.c-b.users", "name": "users"} + service = _make_service(store, mock_client) + + result = service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER", "name:STRING"], + if_not_exists=True, + ) + + assert result["action"] == "skipped" + assert result["skip_reason"] == "table already exists" + assert result["table_id"] == "in.c-b.users" + mock_client.get_table_detail.assert_called_once_with("in.c-b.users", branch_id=None) + mock_client.close.assert_called_once() + + def test_reraises_when_flag_unset(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.side_effect = self._duplicate_display_name_error() + service = _make_service(store, mock_client) + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER"], + ) + assert excinfo.value.error_code == ErrorCode.STORAGE_JOB_FAILED + # No probe when flag is off. + mock_client.get_table_detail.assert_not_called() + mock_client.close.assert_called_once() + + def test_reraises_when_target_table_missing(self, tmp_path: Path) -> None: + """Duplicate-name error but the table at the expected id doesn't + resolve → a different table is conflicting; surface the real error.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.side_effect = self._duplicate_display_name_error() + mock_client.get_table_detail.side_effect = KeboolaApiError( + message="404", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + service = _make_service(store, mock_client) + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER"], + if_not_exists=True, + ) + # The ORIGINAL error must propagate, not the lookup error. + assert excinfo.value.error_code == ErrorCode.STORAGE_JOB_FAILED + + def test_non_duplicate_error_reraises_even_with_flag(self, tmp_path: Path) -> None: + """A non-duplicate STORAGE_JOB_FAILED still surfaces — the IF-NOT- + EXISTS path is gated on the specific message substring.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.side_effect = KeboolaApiError( + message="quota exceeded", + status_code=500, + error_code=ErrorCode.STORAGE_JOB_FAILED, + ) + service = _make_service(store, mock_client) + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER"], + if_not_exists=True, + ) + assert "quota" in str(excinfo.value.message).lower() + mock_client.get_table_detail.assert_not_called() + + def test_success_path_unchanged_with_flag(self, tmp_path: Path) -> None: + """When the create succeeds, the flag has no effect on the envelope.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.return_value = {"id": "in.c-b.users"} + service = _make_service(store, mock_client) + + result = service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER"], + if_not_exists=True, + ) + + assert result["action"] == "created" + assert result["table_id"] == "in.c-b.users" + + # --------------------------------------------------------------------------- # CLI tests: create-table # --------------------------------------------------------------------------- @@ -1049,6 +1171,50 @@ def test_create_bucket_api_error(self, tmp_path: Path) -> None: class TestCreateTableCLI: """CLI tests for `kbagent storage create-table`.""" + def test_human_renders_skip_when_action_is_skipped(self, tmp_path: Path) -> None: + """When --if-not-exists triggers a skip, human mode prints + 'Skipped (already exists)' instead of the misleading 'Created table'.""" + store = _make_store(tmp_path) + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.StorageService") as MockSvc, + ): + MockStore.return_value = store + svc = MockSvc.return_value + svc.create_table.return_value = { + "project_alias": "test", + "table_id": "in.c-b.users", + "name": "users", + "bucket_id": "in.c-b", + "primary_key": ["id"], + "columns": ["id", "name"], + "action": "skipped", + "skip_reason": "table already exists", + } + result = runner.invoke( + app, + [ + "storage", + "create-table", + "--project", + "test", + "--bucket-id", + "in.c-b", + "--name", + "users", + "--column", + "id:INTEGER", + "--if-not-exists", + ], + ) + assert result.exit_code == 0, result.output + assert "Skipped" in result.output + assert "in.c-b.users" in result.output + assert "table already exists" in result.output + assert "Created table" not in result.output, ( + "must NOT print the misleading success line on a skipped row" + ) + def test_create_table_json(self, tmp_path: Path) -> None: store = _make_store(tmp_path) with ( @@ -1097,6 +1263,7 @@ def test_create_table_json(self, tmp_path: Path) -> None: branch_id=None, not_null_columns=None, defaults=None, + if_not_exists=False, ) def test_create_table_native_types_and_attributes(self, tmp_path: Path) -> None: diff --git a/tests/test_sync_service.py b/tests/test_sync_service.py index 856feba3..73e325fa 100644 --- a/tests/test_sync_service.py +++ b/tests/test_sync_service.py @@ -2799,3 +2799,598 @@ def test_passes_for_absolute_path_inside_branch(self, tmp_path: Path) -> None: # Resolve and then re-pass: should still pass _ensure_within_branch(branch_dir, config_dir.resolve(), "comp", "id") + + +# --------------------------------------------------------------------------- +# Fresh-CREATE writeback + KBC.* metadata propagation (v0.47.0 / FIIA migration) +# --------------------------------------------------------------------------- + + +class TestFreshCreateWriteback: + """Cover the fresh-CREATE manifest writeback + KBC.* metadata propagation. + + Closes the gap where a downstream caller (FIIA / scaffold-style emitter) + pre-populates manifest entries with placeholder ids and folder metadata + before the first ``sync push``. Pre-v0.47.0 every create unconditionally + appended a new manifest entry (manifest doubled in size, re-pushes flagged + every placeholder as ``added`` again, ``KBC.configuration.folderName`` + silently dropped on the floor). + """ + + @staticmethod + def _make_svc(tmp_config_dir: Path) -> SyncService: + return SyncService(config_store=setup_single_project(tmp_config_dir)) + + def test_writeback_config_in_place_updates_placeholder(self, tmp_config_dir: Path) -> None: + """A placeholder entry at the same (branch_id, component_id, path) is + updated in place; the manifest does not grow.""" + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + manifest = Manifest.model_construct( + project={"id": 1, "apiHost": "connection.keboola.com"}, # type: ignore[arg-type] + naming={"config": "{component_type}/{component_id}/{config_name}"}, # type: ignore[arg-type] + configurations=[ + ManifestConfiguration( + branchId=12345, + componentId="keboola.snowflake-transformation", + id="PLACEHOLDER-TX1", + path="transformation/keboola.snowflake-transformation/01_stage", + metadata={"KBC.configuration.folderName": "FI Pipeline"}, + ) + ], + ) + + entry = svc._writeback_create_config_in_manifest( + manifest=manifest, + component_id="keboola.snowflake-transformation", + branch_id=12345, + config_path_str="transformation/keboola.snowflake-transformation/01_stage", + new_id="123456789", + file_hash="abc123", + cfg_hash="def456", + ) + + assert len(manifest.configurations) == 1, "must not append a duplicate" + assert entry.id == "123456789" + assert entry.branch_id == 12345 + assert entry.metadata["pull_hash"] == "abc123" + assert entry.metadata["pull_config_hash"] == "def456" + assert entry.metadata["KBC.configuration.folderName"] == "FI Pipeline", ( + "user-declared KBC.* metadata must survive the writeback" + ) + + def test_writeback_config_does_not_match_across_branches(self, tmp_config_dir: Path) -> None: + """A placeholder at the same (component_id, path) but a different + branch must NOT be matched. The new entry is appended; the other + branch's entry is left untouched.""" + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + # Placeholder for branch 12345 (e.g. main); we push to dev branch 99999. + manifest = Manifest.model_construct( + project={"id": 1, "apiHost": "connection.keboola.com"}, # type: ignore[arg-type] + naming={"config": "{component_type}/{component_id}/{config_name}"}, # type: ignore[arg-type] + configurations=[ + ManifestConfiguration( + branchId=12345, + componentId="keboola.snowflake-transformation", + id="main-id-001", + path="transformation/keboola.snowflake-transformation/01_stage", + metadata={"KBC.configuration.folderName": "Main FI"}, + ) + ], + ) + + entry = svc._writeback_create_config_in_manifest( + manifest=manifest, + component_id="keboola.snowflake-transformation", + branch_id=99999, + config_path_str="transformation/keboola.snowflake-transformation/01_stage", + new_id="dev-id-002", + file_hash="h1", + cfg_hash="h2", + ) + + # Two entries: the main-branch one untouched, plus the new dev-branch + # one we just appended. + assert len(manifest.configurations) == 2 + main_entry = next(c for c in manifest.configurations if c.branch_id == 12345) + assert main_entry.id == "main-id-001" + assert main_entry.metadata == {"KBC.configuration.folderName": "Main FI"} + # The returned entry is the newly-appended dev-branch one. + assert entry.branch_id == 99999 + assert entry.id == "dev-id-002" + + def test_writeback_config_appends_when_no_placeholder(self, tmp_config_dir: Path) -> None: + """If no placeholder exists at the path, append (legacy fallback).""" + svc = self._make_svc(tmp_config_dir) + manifest = Manifest.model_construct( + project={"id": 1, "apiHost": "connection.keboola.com"}, # type: ignore[arg-type] + naming={"config": "{component_type}/{component_id}/{config_name}"}, # type: ignore[arg-type] + configurations=[], + ) + + entry = svc._writeback_create_config_in_manifest( + manifest=manifest, + component_id="keboola.ex-http", + branch_id=0, + config_path_str="extractor/keboola.ex-http/my-new-config", + new_id="999", + file_hash="h1", + cfg_hash="h2", + ) + + assert len(manifest.configurations) == 1 + assert manifest.configurations[0] is entry + assert entry.id == "999" + assert entry.metadata == {"pull_hash": "h1", "pull_config_hash": "h2"} + + def test_propagate_kbc_metadata_calls_set_config_metadata(self, tmp_config_dir: Path) -> None: + """KBC.* keys are POSTed via client.set_config_metadata; bookkeeping + keys (``pull_hash``, ...) are filtered out.""" + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + entry = ManifestConfiguration( + branchId=0, + componentId="keboola.snowflake-transformation", + id="cfg-123", + path="x", + metadata={ + "pull_hash": "h1", + "pull_config_hash": "h2", + "KBC.configuration.folderName": "FI Pipeline", + "KBC.configuration.category": "transformation", + }, + ) + client = MagicMock() + + svc._propagate_kbc_metadata(client, entry, branch_id=99) + + client.set_config_metadata.assert_called_once() + call = client.set_config_metadata.call_args + assert call.kwargs["component_id"] == "keboola.snowflake-transformation" + assert call.kwargs["config_id"] == "cfg-123" + assert call.kwargs["branch_id"] == 99 + entries = dict(call.kwargs["entries"]) + assert entries == { + "KBC.configuration.folderName": "FI Pipeline", + "KBC.configuration.category": "transformation", + }, "pull_* bookkeeping keys must not be sent to the metadata API" + + def test_propagate_kbc_metadata_noop_when_no_kbc_keys(self, tmp_config_dir: Path) -> None: + """No KBC.* keys → no API call (don't waste a round-trip).""" + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + entry = ManifestConfiguration( + branchId=0, + componentId="x", + id="y", + path="z", + metadata={"pull_hash": "h", "pull_config_hash": "h2"}, + ) + client = MagicMock() + + result = svc._propagate_kbc_metadata(client, entry, branch_id=None) + + client.set_config_metadata.assert_not_called() + assert result is None + + def test_propagate_kbc_metadata_returns_error_message_on_api_failure( + self, tmp_config_dir: Path + ) -> None: + """A failed metadata POST returns the error message (caller accumulates + into the push errors list) instead of aborting the push mid-loop.""" + from keboola_agent_cli.errors import ErrorCode, KeboolaApiError + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + entry = ManifestConfiguration( + branchId=0, + componentId="keboola.snowflake-transformation", + id="cfg-123", + path="x", + metadata={"KBC.configuration.folderName": "FI Pipeline"}, + ) + client = MagicMock() + client.set_config_metadata.side_effect = KeboolaApiError( + message="metastore 500", + status_code=500, + error_code=ErrorCode.API_ERROR, + ) + + result = svc._propagate_kbc_metadata(client, entry, branch_id=None) + + assert result == "metastore 500", ( + "non-fatal metadata failure must return the message for the caller " + "to accumulate into the push error list" + ) + client.set_config_metadata.assert_called_once() + + def test_writeback_row_in_place_updates_placeholder(self, tmp_config_dir: Path) -> None: + """A placeholder row at the same ``path`` is updated in place; parent's + rows list does not grow.""" + from keboola_agent_cli.sync.manifest import ManifestConfigRow, ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + parent = ManifestConfiguration( + branchId=0, + componentId="keboola.variables", + id="vars-001", + path="other/keboola.variables/shared", + rows=[ + ManifestConfigRow( + id="PLACEHOLDER-ROW", + path="rows/default", + metadata={}, + ) + ], + ) + + row = svc._writeback_create_row_in_manifest( + parent=parent, + row_path_str="rows/default", + new_row_id="vals-real-id", + file_hash="rh1", + cfg_hash="rh2", + ) + + assert len(parent.rows) == 1, "must not append a duplicate row" + assert row.id == "vals-real-id" + assert row.metadata == {"pull_hash": "rh1", "pull_config_hash": "rh2"} + + def test_writeback_row_appends_when_no_placeholder(self, tmp_config_dir: Path) -> None: + """No placeholder row → append (legacy fallback for untracked rows).""" + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + parent = ManifestConfiguration( + branchId=0, + componentId="keboola.ex-http", + id="cfg-1", + path="extractor/keboola.ex-http/my-ext", + rows=[], + ) + + row = svc._writeback_create_row_in_manifest( + parent=parent, + row_path_str="rows/new", + new_row_id="row-001", + file_hash="rh1", + cfg_hash="rh2", + ) + + assert len(parent.rows) == 1 + assert parent.rows[0] is row + assert row.id == "row-001" + + def test_push_create_with_placeholder_is_idempotent_and_propagates_folder( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """End-to-end push: placeholder + KBC.configuration.folderName. + + Round-trip: + 1. Init a project (empty manifest). + 2. Hand-populate a placeholder ManifestConfiguration with a + ``KBC.configuration.folderName`` metadata key, save it, and write + a matching ``_config.yml`` file. + 3. Run sync push — assert: created=1, manifest length stays at 1 + (placeholder updated in place to real ULID), client.create_config + was called, client.set_config_metadata was called with the folder + metadata. + 4. Run sync push a second time — assert: created=0, errors=0 + (idempotency naturally follows from writeback-in-place). + """ + from keboola_agent_cli.constants import CONFIG_YML_VERSION + from keboola_agent_cli.sync.manifest import ManifestConfiguration, save_manifest + + project_root = tmp_path / "project" + project_root.mkdir() + + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + # Hand-author a placeholder manifest entry + local _config.yml at the + # corresponding path. This is the FIIA / scaffold emit pattern. + manifest = load_manifest(project_root) + placeholder_path = "transformation/keboola.snowflake-transformation/01_stage" + manifest.configurations.append( + ManifestConfiguration( + branchId=12345, + componentId="keboola.snowflake-transformation", + id="PLACEHOLDER-TX1", + path=placeholder_path, + metadata={"KBC.configuration.folderName": "FI Pipeline"}, + ) + ) + save_manifest(project_root, manifest) + + branch_path = manifest.branches[0].path + config_dir = project_root / branch_path / placeholder_path + config_dir.mkdir(parents=True) + (config_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": "01 Stage", + "description": "Staging transformation", + "parameters": {}, + "_keboola": { + "component_id": "keboola.snowflake-transformation", + "config_id": "", + }, + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + + push_client = _make_sync_mock_client(components_response=[]) + push_client.create_config.return_value = {"id": "999000111"} + + push_svc = SyncService( + config_store=store, + client_factory=lambda url, token: push_client, + ) + + # First push: placeholder → real ULID, KBC.* propagated. + result = push_svc.push(alias="prod", project_root=project_root) + assert result["status"] == "pushed" + assert result["created"] == 1 + assert result["errors"] == [] + push_client.create_config.assert_called_once() + push_client.set_config_metadata.assert_called_once() + meta_call = push_client.set_config_metadata.call_args + assert meta_call.kwargs["component_id"] == "keboola.snowflake-transformation" + assert meta_call.kwargs["config_id"] == "999000111" + assert dict(meta_call.kwargs["entries"]) == { + "KBC.configuration.folderName": "FI Pipeline", + } + + # Manifest must have updated the placeholder in place — NOT appended. + post = load_manifest(project_root) + matching = [ + c + for c in post.configurations + if c.component_id == "keboola.snowflake-transformation" and c.path == placeholder_path + ] + assert len(matching) == 1, "writeback must update placeholder in place" + assert matching[0].id == "999000111" + assert matching[0].metadata.get("KBC.configuration.folderName") == "FI Pipeline" + + # Second push against the now-real manifest must be a no-op. + # The remote side reports the created config so the diff sees it as + # present and unchanged. + push_client2 = _make_sync_mock_client( + components_response=[ + { + "id": "keboola.snowflake-transformation", + "type": "transformation", + "configurations": [ + { + "id": "999000111", + "name": "01 Stage", + "description": "Staging transformation", + "configuration": {"parameters": {}}, + "rows": [], + } + ], + } + ], + ) + push_svc2 = SyncService( + config_store=store, + client_factory=lambda url, token: push_client2, + ) + result2 = push_svc2.push(alias="prod", project_root=project_root) + # Idempotent re-push: diff sees no changes, so service short-circuits + # to status="no_changes" without ever entering the create path. + assert result2["status"] in ("no_changes", "pushed") + assert result2.get("created", 0) == 0, "re-push must be idempotent" + push_client2.create_config.assert_not_called() + + +# --------------------------------------------------------------------------- +# Ergonomics: --branch override + --no-name-drift-warnings (v0.47.0) +# --------------------------------------------------------------------------- + + +class TestBranchOverrideAndNameDriftFlag: + """Cover the `--branch` override (push / pull / diff) and the + `--no-name-drift-warnings` opt-out at the service boundary.""" + + def test_resolve_branch_id_override_wins(self, tmp_path: Path) -> None: + from keboola_agent_cli.sync.manifest import ( + ManifestBranch, + ManifestNaming, + ManifestProject, + ) + + project = MagicMock() + project.active_branch_id = 12345 + manifest = Manifest.model_construct( + project=ManifestProject(id=1, apiHost="connection.keboola.com"), + naming=ManifestNaming(), + branches=[ManifestBranch(id=999, path="main", metadata={})], + ) + + # Without override -> falls back to active_branch_id (priority 2). + assert ( + SyncService._resolve_branch_id(project, manifest, tmp_path, branch_override=None) + == 12345 + ) + # Override wins (priority 0). + assert ( + SyncService._resolve_branch_id(project, manifest, tmp_path, branch_override=388071) + == 388071 + ) + + def test_push_branch_override_reaches_client( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """push(branch_override=X) must thread X into list_components_with_configs.""" + project_root = tmp_path / "project" + project_root.mkdir() + + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + push_client = _make_sync_mock_client(components_response=[]) + push_svc = SyncService( + config_store=store, + client_factory=lambda url, token: push_client, + ) + + push_svc.push(alias="prod", project_root=project_root, branch_override=99999) + + push_client.list_components_with_configs.assert_called() + call = push_client.list_components_with_configs.call_args + assert call.kwargs.get("branch_id") == 99999 + + def test_diff_branch_override_reaches_client( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + project_root = tmp_path / "project" + project_root.mkdir() + + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + diff_client = _make_sync_mock_client(components_response=[]) + diff_svc = SyncService( + config_store=store, + client_factory=lambda url, token: diff_client, + ) + + diff_svc.diff(alias="prod", project_root=project_root, branch_override=77777) + + diff_client.list_components_with_configs.assert_called() + call = diff_client.list_components_with_configs.call_args + assert call.kwargs.get("branch_id") == 77777 + + def test_no_name_drift_warnings_flag_suppresses_field( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """When name drift is detected, the suppression flag drops the + ``name_drift_warnings`` array from the result envelope.""" + from keboola_agent_cli.constants import CONFIG_YML_VERSION + from keboola_agent_cli.sync.manifest import ( + ManifestConfiguration, + save_manifest, + ) + + project_root = tmp_path / "project" + project_root.mkdir() + + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + # Author a tracked manifest entry whose dirname does NOT match the + # config name -> name-drift detector will surface a warning. + manifest = load_manifest(project_root) + cfg_path = "transformation/keboola.snowflake-transformation/dir-name-NEQ-config-name" + manifest.configurations.append( + ManifestConfiguration( + branchId=12345, + componentId="keboola.snowflake-transformation", + id="01abc", + path=cfg_path, + # No pull_hash -> diff falls into 2-way mode and any + # difference is classified "modified" (a pushable change), + # so the name-drift detector actually runs end-to-end. + metadata={}, + ) + ) + save_manifest(project_root, manifest) + branch_path = manifest.branches[0].path + config_dir = project_root / branch_path / cfg_path + config_dir.mkdir(parents=True) + (config_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": "Some Pretty Config Name", + "description": "", + "parameters": {"x": "y_new"}, + "_keboola": { + "component_id": "keboola.snowflake-transformation", + "config_id": "01abc", + }, + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + + # Remote returns a stale param value so the diff sees a "modified" + # change and the push actually enters the warning-emitting path. + push_client = _make_sync_mock_client( + components_response=[ + { + "id": "keboola.snowflake-transformation", + "type": "transformation", + "configurations": [ + { + "id": "01abc", + "name": "Some Pretty Config Name", + "description": "", + "configuration": {"parameters": {"x": "y_old"}}, + "rows": [], + } + ], + } + ], + ) + push_client.update_config.return_value = {"id": "01abc"} + push_svc = SyncService( + config_store=store, + client_factory=lambda url, token: push_client, + ) + + default_result = push_svc.push(alias="prod", project_root=project_root) + assert "name_drift_warnings" in default_result, ( + "control: the warning must surface without the suppression flag" + ) + + suppressed_result = push_svc.push( + alias="prod", + project_root=project_root, + no_name_drift_warnings=True, + ) + assert "name_drift_warnings" not in suppressed_result, ( + "--no-name-drift-warnings must drop the field from the envelope" + ) diff --git a/uv.lock b/uv.lock index c7189963..a9ec1375 100644 --- a/uv.lock +++ b/uv.lock @@ -496,7 +496,7 @@ wheels = [ [[package]] name = "keboola-agent-cli" -version = "0.46.1" +version = "0.47.0" source = { editable = "." } dependencies = [ { name = "croniter" },