diff --git a/CLAUDE.md b/CLAUDE.md index d252de59..d8cc8ab6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -399,6 +399,20 @@ kbagent config state-set --project NAME --component-id ID --config-id ID [--row- # currentVersion, changeDescription, created, creatorToken, isDeleted, isDisabled) -- use # state-set / --name / --description / row-update --is-disabled instead, or --configuration # for a genuine configuration. key. +kbagent config clone --project P --component-id ID --config-id ID --name NAME [--target-project P2] [--description D] [--set PATH=VALUE ...] [--secret PATH=VALUE ...] [--branch ID] [--target-branch ID] [--dry-run] [--allow-plaintext-on-encrypt-failure] +# clone (0.84.2+, #587): duplicates a configuration WHOLE. Hand-rebuilding a body from `config +# detail` drops siblings of `parameters` (`runtime`, `storage`, `authorization`) silently -- a +# lost `runtime.parallelism` makes Keboola fall back to parallelism 1 (a 65-row writer then ran +# sequentially, 140 min instead of ~60-90, with nothing reported). +# SAME project (default): server-side copy (`POST .../configs/{id}/versions/{v}/create`) -- +# rows AND `KBC::` values come along (verified live). `--set` edits are applied as a follow-up +# update on the copy, so an override can never be why a key went missing. +# CROSS project (`--target-project`): reassembled client-side, rows recreated one by one. +# Encrypted values CANNOT travel (ciphertext is project-scoped), so the clone is REFUSED, +# listing every path, until each is re-supplied via `--secret PATH=VALUE`; those are encrypted +# in the TARGET project on write. `--dry-run` reports the paths instead of refusing -- that is +# how you discover what to gather. Storage bucket/table IDs are copied VERBATIM, never remapped +# (`sync clone` is the command that remaps). kbagent search QUERY [--project NAME] [--type table|bucket|config|flow|data-app|transformation] [--search-type textual|config-based] [--regex] [--limit N] # --regex (0.67.0+): opt-in regex mode (mode=regex). Case-insensitive whole-term match on ENTITY NAMES diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 3cb236a3..9c3f1222 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -50,7 +50,7 @@ a critical failure. 4. **PREFER CLI OVER MCP**. If a `kbagent ` native subcommand exists, use it. Only fall back to `kbagent tool call ...` (MCP) when the native command does not cover the operation -- the `tool` group is - REMOVED in v0.85.0 (end of August 2026), so never build a new workflow on it. When an MCP + REMOVED in v0.85.0, so never build a new workflow on it. When an MCP `tool call` returns `isError: true`, DO NOT retry with reformatted inputs. Fall back to the `kbagent serve` REST API for the equivalent operation. @@ -121,7 +121,7 @@ a critical failure. | Manage feature flags (stack catalogue / project / user) | `kbagent feature list\|project-show\|project-add\|project-remove\|user-show\|user-add\|user-remove --project P [--email E] [--feature NAME] [--dry-run] [--yes]` (0.48.0+) -- Manage API; needs a SUPER-ADMIN manage token (interactive prompt; `--allow-env-manage-token`+`KBC_MANAGE_API_TOKEN` for CI); `--project` resolves the stack URL (+project_id for `project-*`); add=admin, remove=destructive; add body is `{"feature":NAME}` | `kbagent project info` for a project's *enabled* features (read-only, no super-admin) | raw `/manage/...` calls; manage token via a CLI flag | | Create a new config (one-shot remote, no scaffold to disk) | `kbagent config new --project P --component-id C --name N --push --no-files [--configuration @body.json] [--branch ID]` (0.33.0+) -- single CLI call POSTs to `/v2/storage/components/{cid}/configs`; default body is `{}` (FIIA empty-shell pattern, validation auto-skips); explicit `--configuration` body is schema-validated by default (`--no-validate` opts out); works for ALL component types incl. `keboola.snowflake-transformation` | `kbagent config new --output-dir D` then edit + `kbagent sync push` (scaffold-then-push GitOps flow) | `tool call create_config` (refuses keboola.snowflake-transformation; raw MCP envelope, no validation) | | Create / update / delete a config row | `kbagent config row-create\|row-update\|row-delete --project P --component-id C --config-id K [--row-id R] [--name N] [--configuration JSON] [--yes]` (0.30.0+) -- `row-delete` is destructive (gated behind `--allow-destructive`); all three are branch-aware | `tool call create_config_row` / `update_config_row` / `delete_config_row` | raw REST against `/v2/storage/components/C/configs/K/rows` | -| Get OAuth authorization URL | `kbagent config oauth-url --project P --component-id C --config-id K` (0.30.0+) -- returns URL to open in browser to complete OAuth flow | -- | raw `GET /v2/storage/components/C/configs/K/oauth/authorize` | +| Get OAuth authorization URL | `kbagent config oauth-url --project P --component-id C --config-id K` (0.30.0+) -- URL to open in a browser for OAuth | -- | raw `GET /v2/storage/components/C/configs/K/oauth/authorize` | | Inventory data apps | `kbagent data-app list --project P` (0.27.0+; 0.43.9+ skips sandboxes) | `tool call get_configs --component_id keboola.data-apps` (Storage view only -- no state/URL/configVersion) | per-project `tool call` joined to Data Science | | Bring a new data app online from a git repo | `kbagent data-app create --project P --name N --slug S --git-repo URL [--git-pat-env VAR \| --git-public]` (0.27.0+) -- OR `--use-managed-git-repo` (0.65.0+) for an empty Keboola-hosted repo instead of `--git-repo` (mutually exclusive; forces --no-deploy; deploy flow = create -> git-credentials-create + push -> deploy; platform injects clone creds; see gotchas) | `tool call create_config keboola.data-apps` + manual `kbagent encrypt values` + raw `POST /apps` -- only for custom shapes | raw `POST data-science/apps` then `PATCH desiredState=running` without `configVersion + restartIfRunning` (the §9 footgun -- pins to v2 empty shell, errors `dataApp.git.repository is required`) | | Roll out a new code or config version on a data app | `kbagent data-app deploy --project P --app-id N --wait` (0.27.0+) -- always sends the §9 trio | -- | `tool call update_config` then `tool call run_component` (data apps are not jobs -- the queue runner does not deploy them) | @@ -149,7 +149,7 @@ a critical failure. | Remove a metric (with orphan-check) | `kbagent semantic-layer remove metric --project P [--model M] --name N [--yes]` (0.41.0+) -- pre-deletion scan lists constraints that would become orphaned; warning is always printed (even with `--yes`); non-TTY without `--yes` refuses with exit 2 | `kbagent semantic-layer edit metric --new-name _DELETED_` (soft-delete; keeps the constraint refs valid but pollutes the model) | raw `DELETE` against the metastore (skips the orphan warning -- the constraint pointing at the deleted metric stays but creates a dangling FK in `DIM_METRIC_THRESHOLD` downstream) | | Restore a model from a snapshot (after accidental destructive edit) | `kbagent semantic-layer import --project P --file PATH --dry-run` to preview classifications, then re-run without `--dry-run` (0.41.0+); default skip-on-conflict, add `--overwrite` to DELETE+POST conflicting items; dependency-ordered push (datasets -> metrics -> relationships -> glossary -> constraints) | `semantic-layer promote --from-project source` if you still have the source project handy (uses the same write loop but without snapshot indirection) | replaying the snapshot via a shell loop of `add` subcommands (loses the conflict-classification step and the dependency-ordered push) | | Promote a model dev -> prod (cross-project copy) | `kbagent --json semantic-layer promote --from-project dev --to-project prod --dry-run` (0.41.0+) to classify NEW/IDENTICAL/CHANGED, review the `changes[]` and `failed[]` lists, then re-run without `--dry-run`; deep-equality strips modelUUID + timestamps; **NEVER deletes target items absent from source** (additive + overwrite only) | `semantic-layer export` from source + `semantic-layer import --overwrite` into target (two-step -- equivalent end state but you lose the IDENTICAL classification) | hand-rolled cross-project copy via raw metastore calls (no modelUUID rewrite -- the target ends up with foreign UUIDs and validation fails downstream) | -| Bootstrap a model from a set of storage tables | `kbagent semantic-layer build --project P --tables T1,T2,... [--dry-run] [--keep-on-failure]` (0.41.0+) -- **HEURISTIC fallback only** (no AI Service JSON endpoint): synthesises one dataset + one COUNT(*) metric + one glossary entry per table; FQN derived; fields[] role-classified. Response carries `fallback_used: "heuristic"`. Use as a SCAFFOLD, then refine via `add` / `edit`. Rollback on push failure (0.41.10+): every successfully-POSTed child is DELETEd in reverse + model deleted if we created it; pass `--keep-on-failure` to preserve partial state | the `sl-build` skill in `04_AI_Kit/ai-kit` -- full AI-assisted greenfield wizard, schema discovery + SQL analysis + AI generation. Use this when you need richer metrics, relationships, and constraint shapes than the heuristic produces | hand-writing the model JSON from scratch (the `build` heuristic gets you 80% of the way for read-mostly star schemas; only fall back to manual when the heuristic refuses or you need something the skill produces) | +| Bootstrap a model from a set of storage tables | `kbagent semantic-layer build --project P --tables T1,T2,... [--dry-run] [--keep-on-failure]` (0.41.0+) -- **HEURISTIC fallback only** (no AI Service JSON endpoint): synthesises one dataset + one COUNT(*) metric + one glossary entry per table; FQN derived, fields[] role-classified. Response carries `fallback_used: "heuristic"`. Use as a SCAFFOLD, then refine via `add` / `edit`. Rollback on push failure (0.41.10+): every successfully-POSTed child is DELETEd in reverse + model deleted if we created it; pass `--keep-on-failure` to preserve partial state | the `sl-build` skill in `04_AI_Kit/ai-kit` -- full AI-assisted greenfield wizard, schema discovery + SQL analysis + AI generation. Use this when you need richer metrics, relationships, and constraint shapes than the heuristic produces | hand-writing the model JSON from scratch (the `build` heuristic gets you 80% of the way for read-mostly star schemas; only fall back to manual when the heuristic refuses or you need something the skill produces) | | Encrypt the storage token for a transformation `user_properties` (so a Python container can reach the metastore) | `kbagent semantic-layer token --encrypt --project P --component-id C` (0.41.0+) -- builds `{"#metastore_token": }` from the project's already-stored Storage token and delegates to the existing EncryptService; output is the encrypted envelope ready to paste into the transformation's `user_properties` block | `kbagent encrypt values --project P --component-id C --input '{"#metastore_token": ""}'` (works but the operator has to manually fetch the token first -- the wrapper avoids that step) | hand-running the Encryption API and pasting plaintext into `user_properties` (no `#` prefix means it sits in the config in plaintext) | | User asks to "log in" / "authenticate via browser" / set up programmatic auth, or to register a session's projects as aliases | **DO NOT RUN `kbagent auth login` YOURSELF** -- needs a human at the keyboard, no headless path. Tell the user to run `kbagent auth login [--register-projects]` themselves, then continue with `kbagent auth status`. To register projects from an EXISTING session (no re-login), `kbagent auth register-projects --all` or `--project-id ID` (0.80.0+) is non-interactive and agent-safe | -- | attempting `auth login`/the flagless `register-projects` picker from an unattended task; reading the token out of `auth.json`; using the numeric project id as an alias (aliases come from the project NAME) | | CI task has account creds | `kbagent auth login-password --email E (--password-stdin\|--password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | static token | `auth login` unattended | @@ -198,26 +198,28 @@ read it when a trigger fires. Each `(X.Y.Z+)` tag is the version floor. with an admin token. **VERSION GATE**: schedules created by < 0.66.1 stay dormant until `flow schedule` is re-run on 0.66.1+. - **Snowflake transformation scaffolding**: MCP `create_config` REFUSES - `keboola.snowflake-transformation`. Use `config new --push --no-files` - (0.33.0+) or `config new --output-dir` then `config update`, or MCP - `create_sql_transformation`. `config new --push` hits the Storage API - directly, so it does NOT inherit the refusal. + `keboola.snowflake-transformation`; `config new --push` hits Storage + directly and does not inherit it. +- **Never rebuild a body to duplicate a config** -- `config clone` (0.84.2+, + #587): copying `parameters` alone drops `runtime`/`storage`/`authorization` + (silent parallelism 1). Cross-project needs `--secret` per `KBC::` value, + listed by `--dry-run`. - **`script[]` normalization**: `config update` auto-fixes string-vs-array - (0.28.0+, #245) and re-splits multi-statement list elements (0.31.0+, #274); - inspect the result envelope's `normalizations: [...]`. The trap STILL fires - via MCP `update_sql_transformation` / raw `PUT` -- prefer `config update`. + (#245) and re-splits multi-statement elements (#274); see envelope's + `normalizations: [...]`. Still fires via MCP `update_sql_transformation` / + raw `PUT` -- prefer `config update`. - **`config create/update/row-*` auto-encrypt `#`-secrets** (0.54.0+, #378): pre-encrypt via the Encryption API; fail-closed (`--allow-plaintext-on-encrypt-failure` overrides); `--dry-run` is not encrypted; covers CLI + `serve` + MCP passthrough. **VERSION GATE**: < 0.54.0 - wrote `#`-secrets to Storage in PLAINTEXT -- warn + recommend `kbagent update`. - To find pre-0.54.0 leaks in a synced tree use `sync status` / `doctor` + wrote `#`-secrets to Storage in PLAINTEXT -- warn/recommend `kbagent update`. + For pre-0.54.0 leaks in a synced tree use `sync status` / `doctor` (0.55.0+) -- they flag in-sync configs whose `#`-secrets are still plaintext; fix = re-push to encrypt AND rotate (version history keeps the plaintext). - **`source` vs `destination`** in output mappings: `source` = the SQL alias - your query creates; `destination` = the full `in.c-bucket.table` path. + your query creates, `destination` = the full `in.c-bucket.table` path. Swapping them breaks the config SILENTLY (no save-time error). -- **Primary keys on new output tables**: columns are nullable on first insert, +- **Primary keys on new output tables**: columns are nullable on first insert so a PK crashes the first run. Strip PKs, run, restore. Warn the user BEFORE the crash. diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 288f4dee..a259e152 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -125,9 +125,10 @@ When working inside a git repository or project directory, run `kbagent init` (o | Create a new configuration row | `kbagent config row-create --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --name NAME` | | Update an existing configuration row | `kbagent config row-update --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --row-id ROW-ID` | | Delete a configuration row | `kbagent config row-delete --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --row-id ROW-ID` | -| Requires master token. | `kbagent config oauth-url --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | Read the runtime ``state`` dict of a configuration or one of its rows | `kbagent config state-get --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | Overwrite the runtime ``state`` dict of a configuration or one of its rows | `kbagent config state-set --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --state STATE` | +| Duplicate a configuration, whole -- including runtime, storage and authorization | `kbagent config clone --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --name NAME` | +| Requires master token. | `kbagent config oauth-url --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | List data apps across one or more registered projects | `kbagent data-app list` | | Show merged Data Science + Storage detail for one data app | `kbagent data-app detail --project PROJECT --app-id APP-ID` | | Create a Keboola data app end-to-end (POST + encrypt + PUT + deploy) | `kbagent data-app create --project PROJECT --name NAME --slug SLUG` | @@ -411,7 +412,6 @@ For detailed response parsing rules and common pitfalls, see [gotchas](reference | **Semantic layer (metastore)** -- models, metrics, datasets, constraints, glossary; validate / export / diff / promote / build / token | [semantic-layer-workflow](references/semantic-layer-workflow.md) | | **Developer Portal** (identity CRUD, list/get apps, create/patch/upload-icon/publish/deprecate; TTY-confirm on writes) | [dev-portal-workflow](references/dev-portal-workflow.md) | | **Config metadata** (list/get/set/delete arbitrary key-value metadata on a configuration) | [config-metadata-workflow](references/config-metadata-workflow.md) | -| **Config runtime state** (`state-get`/`state-set`; root vs row; seeding a dev branch before testing `changed_since: adaptive`) | [config-state-workflow](references/config-state-workflow.md) | | **Storage descriptions** (describe bucket / table / column, batch from YAML) | [storage-describe-workflow](references/storage-describe-workflow.md) | | **Deep column-level lineage** (`lineage build --ai`, column graph, ER + HTML output) | [lineage-deep-workflow](references/lineage-deep-workflow.md) | | **Session permissions firewall** (`--deny-writes` / `--deny-destructive`, persisted policies, `permissions check`) | [permissions-workflow](references/permissions-workflow.md) | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index bff8368c..3015c390 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -130,6 +130,7 @@ Requires a **super-admin** Manage API token (same kind as `org setup`). Same def - `config rename --project NAME --component-id ID --config-id ID --name "New Name" [--branch ID] [--directory DIR]` -- rename a configuration (API update + local sync directory rename with git mv support) - `config delete --project NAME --component-id ID --config-id ID [--branch ID]` -- delete a configuration - `config new --component-id ID [--project NAME] [--name NAME] [--output-dir DIR] [--push --no-files --description D --configuration JSON|@file|- --configuration-file PATH --no-validate --branch ID --dry-run --allow-plaintext-on-encrypt-failure]` -- **two modes**. **Default (no `--push`)**: scaffold new config from component schema; writes files to `--output-dir` or prints to stdout. **Zero API calls.** **With `--push`** (0.33.0+, requires `--project` + non-empty `--name`): also POSTs to `/v2/storage/components/{cid}/configs` for a one-shot remote create. `#`-prefixed secrets in the pushed body auto-encrypt via the Encryption API first (fail-closed; since 0.54.0, #378; `--allow-plaintext-on-encrypt-failure` overrides). `--no-files` skips the filesystem step entirely (FIIA-style empty-shell pattern). `--configuration` / `--configuration-file` override the POSTed body (default is `{}`, with validation auto-skipped for the default empty shell). `--dry-run` previews the planned POST + validation result without creating. Schema validation runs by default when an explicit body is given (fail-closed: `ConfigError` exit 5 on mismatch) but skips silently if the AI Service has no schema for the component or returns an error; `--no-validate` opts out. Works for ALL component types including `keboola.snowflake-transformation` (unlike `tool call create_config`, which refuses that component). +- `config clone --project P --component-id ID --config-id ID --name NAME [--target-project P2] [--description D] [--set PATH=VALUE ...] [--secret PATH=VALUE ...] [--branch ID] [--target-branch ID] [--dry-run] [--allow-plaintext-on-encrypt-failure]` (0.84.2+, #587) -- duplicate a configuration **whole**. Reach for this instead of reading `config detail` and rebuilding a body: copying only `configuration["parameters"]` silently drops its siblings (`runtime`, `storage`, `authorization`), and a lost `runtime.parallelism` makes Keboola fall back to `parallelism: 1` -- the reporter's 65-row writer went sequential, 140 min instead of ~60-90, with nothing in any output pointing at it. **Same project** (default): server-side copy via `POST .../configs/{id}/versions/{v}/create`; rows and `KBC::` encrypted values travel with it (verified live). `--set PATH=VALUE` is applied as a follow-up update on the copy, so an override can never be the reason a key went missing. **Cross project** (`--target-project`): reassembled client-side and rows recreated one by one, because encrypted values **cannot** travel -- a Keboola ciphertext is scoped to the project it was encrypted in. Any `KBC::` value makes the clone **fail with exit 5**, listing every path, until re-supplied via `--secret PATH=VALUE` (encrypted in the TARGET project on write). `--dry-run` reports those paths instead of refusing -- run it first to learn what to gather. Storage bucket/table IDs are copied **verbatim, never remapped**; `sync clone` is the command that remaps. - `config variables-set --project NAME --component-id ID --config-id ID --var KEY=VALUE [--var ...] [--replace] [--variables-id ID] [--values-id ID] [--branch ID] [--dry-run] [--allow-plaintext-on-encrypt-failure] [--yes]` -- attach variable values to a config. Auto-creates a sibling `keboola.variables` config + default row on first use and links it via the parent's `runtime.variables_id` / `variables_values_id`. Defaults to merge; `--replace` drops keys not in `--var`. `#`-prefixed values encrypt via the Encryption API (fail-closed; exit non-zero on `ENCRYPTION_FAILED`). See `variables-workflow.md` - `config variables-get --project NAME --component-id ID --config-id ID [--branch ID]` -- resolve `variables_id` + `values_id` from the parent config and fetch the current KEY=VALUE map. Returns `{linked: bool, variables_id, values_id, values}`; `linked=false` means the parent has no variables attached - `config variables-clear --project NAME --component-id ID --config-id ID [--branch ID] [--yes]` -- unlink variables from the parent config (strips `variables_id` + `variables_values_id`). **Does NOT delete** the backing `keboola.variables` config -- use `config delete` explicitly if you've verified nothing else references it diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index abbbbd09..84f75c4c 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -1178,20 +1178,53 @@ events and emits a final `done` SSE frame mirroring the same record. configuration still validates `ok`, because it is indistinguishable from a flow-style config. Always POST the full object. -## Cloning a config by hand: copy the WHOLE object, not just `parameters` (documented since v0.84.1, issue #587) - -- A configuration's root has siblings of `parameters` that carry real - behavior: `runtime` (e.g. `runtime.parallelism`), `storage` (input/output - mapping), and `authorization` (OAuth). `config examples` shows only a - `parameters` shape, so reconstructing a config from `config detail` output - by copying `configuration["parameters"]` alone **silently drops them**. -- The failure is silent and slow, not loud: a dropped `runtime.parallelism` - makes Keboola fall back to `parallelism: 1`, so a 65-row writer runs - strictly sequentially instead of 20-at-a-time. Nothing errors -- you only - see it in per-row job start/end timestamps (issue #587). -- There is **no `config clone` command**. To duplicate a config, take the - whole `configuration` object from `config detail` and pass it verbatim to - `config new --push --configuration-file`, changing only what must differ. +## `config clone` duplicates a config whole; cross-project cannot carry secrets (since v0.84.2) + +- **Use it instead of rebuilding a body.** Reading `config detail` and + POSTing back `configuration["parameters"]` drops the siblings + (`runtime`, `storage`, `authorization`) silently. That is issue #587: a + dropped `runtime.parallelism` made Keboola fall back to `parallelism: 1` + and a 65-row writer ran sequentially, 140 min instead of ~60-90. +- **Same project (default): server-side copy.** Goes through + `POST .../configs/{id}/versions/{version}/create` at the source's current + version. Verified live: **rows come along** (a 2-row source produced a + 2-row clone) and so do `KBC::` encrypted values, which stay decryptable + because the project is unchanged. +- **`--set PATH=VALUE` is applied AFTER the copy**, as a normal update on the + new config. So an override can never be the reason a key went missing -- + the copy is complete first, then edited. +- **Cross-project (`--target-project`) REFUSES on encrypted values.** A + Keboola ciphertext is scoped to the project it was encrypted in; no other + project can decrypt it. The clone exits 5 and lists every `KBC::` path + (parent config AND rows, reported as `rows[N].path`) until each is + re-supplied via `--secret PATH=VALUE`. Those are encrypted in the **target** + project on write. Run `--dry-run` first: it reports the paths in + `missing_secrets` instead of refusing, which is how you learn what to gather. +- **Cross-project copies storage mappings VERBATIM.** Bucket and table IDs are + not remapped, so an `in.c-main.orders` input still says `in.c-main.orders` + in the target. Check those buckets exist there; `sync clone` is the command + that actually remaps. +- `--secret` and `--set` split on the FIRST `=` only, so values may contain + `=` (base64 padding, connection strings). +- **Row secrets are supplied under their `rows[N].` path** and are routed back + into that row, not the parent body. Paths for secrets inside a list use a + plain index segment (`parameters.values.0.value`, not `[0]`) so they can be + passed straight back to `--secret`. +- **Ciphertext under a plain (non-`#`) key is refused outright.** The + Encryption API round-trip in this CLI keys off `#` names, so a replacement + supplied there would be written to the target project in PLAINTEXT. + Encrypt it yourself for the target with `kbagent encrypt values` and pass + the ciphertext via `--set`. +- **`--set` is encrypted on both paths.** A `--set 'parameters.db.#password=...'` + goes through the Encryption API before the write, same as `config update`. +- **`--target-branch` is rejected on a same-project clone** when it differs + from `--branch`: the server-side copy writes into the source's own branch, + so honouring it is impossible and writing to the wrong branch silently is + not acceptable. +- **A cross-project clone is not transactional.** If a row fails mid-copy the + error names the created configuration id and how many rows landed, so you + can delete it and re-run. + ## `data-app` JSON output: key for the app's own id is `app_id` (since v0.33.0) - Every `kbagent --json data-app <subcommand>` envelope emits the diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 76b53584..427f1e79 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -25,6 +25,60 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { "0.84.2": [ + "New: `kbagent config clone` duplicates a configuration WHOLE (closes #587). " + "`--project P --component-id C --config-id ID --name N [--target-project P2] " + "[--set PATH=VALUE ...] [--secret PATH=VALUE ...] [--dry-run]`. Until now there was " + "no way to copy a configuration, so people rebuilt the body from `config detail` " + "output -- typically copying " + '`configuration["parameters"]` and nothing else. A configuration\'s root also carries ' + "`storage`, `runtime` and `authorization`, and dropping one is silent: the reporter " + "lost `runtime.parallelism`, Keboola fell back to `parallelism: 1`, and a 65-row " + "Snowflake writer ran strictly sequentially -- 140 minutes instead of the expected " + "60-90, caught only by hand-comparing per-row job timestamps afterwards. WITHIN a " + "project the Storage API copies server-side (`POST .../configs/{id}/versions/{v}/" + "create`) so nothing is rebuilt; verified live that rows travel with the copy (a " + "2-row source produced a 2-row clone) along with `runtime` and `KBC::` values, which " + "stay decryptable because the project is unchanged. `--set` edits are applied " + "afterwards as a normal update on the new config, so an override can never be why a " + "key went missing. ACROSS projects the configuration is reassembled client-side and " + "rows are recreated one by one, because a Keboola ciphertext is scoped to the project " + "it was encrypted in: copying it verbatim would yield a configuration that looks " + "complete and fails at runtime, in a project nobody is watching. So any `KBC::` value " + "-- in the parent body or in a row, reported as `rows[N].path` -- makes the clone fail " + "fast (exit 5) listing every affected path, until each is re-supplied via `--secret " + "PATH=VALUE`; those are then encrypted in the TARGET project. `--dry-run` reports the " + "same paths in `missing_secrets` instead of refusing, which is how a caller discovers " + "what to gather. Storage input/output mappings are copied VERBATIM -- bucket and table " + "IDs are never remapped (`sync clone` is the command that does that), and cross-project " + "output says so. New `KeboolaClient.create_config_copy`; the flow itself lives in the " + "new `services/_config_clone.py` because `config_service.py` is already over its " + "`make loc-check` budget, and `config clone` / `config oauth-url` moved into private " + "command modules so `commands/config.py` SHRANK (1953 code lines, from 2007) rather " + "than needing its grandfathered ceiling raised. A matching " + "`POST /configs/{project}/{component}/{config}/clone` route keeps the 1:1 CLI/HTTP " + "rule.", + "Security: `--set` is encrypted on BOTH clone paths. The same-project path applied " + "overrides through a direct `update_config`, bypassing the Encryption API -- so a " + "`--set 'parameters.db.#password=...'` (repointing a copy at another database is the " + "documented use case) would have landed in Storage, and in version history, in " + "plaintext. Ciphertext stored under a plain non-`#` key is now REFUSED outright " + "instead of accepting a `--secret` for it: `collect_secrets` only picks up `#` keys, " + "so the replacement would never have been re-encrypted. Encrypt such a value for the " + "target project yourself (`kbagent encrypt values`) and pass the ciphertext via " + "`--set`.", + "Fix: a re-supplied row secret now lands in that ROW. Row ciphertext is reported " + "under a `rows[N].` prefix and the refusal check accepted a `--secret` at that exact " + "path, but the substitution applied every override to the PARENT body -- so the " + "copied row kept the source project's undecryptable ciphertext while the command " + "reported success, which is precisely the outcome this command exists to prevent. " + "Paths for secrets inside a list also changed from `parameters.values.[0].#token` to " + "`parameters.values.0.value`: the bracketed form raised `ValueError` in " + "`set_nested_value`, so the CLI was instructing operators to run a command that then " + "crashed. Cross-project clones also inherit the source description instead of " + "blanking it, `--target-branch` is rejected on a same-project clone when it differs " + "from `--branch` (the server-side copy cannot honour it) rather than silently writing " + "to the wrong branch, and a row failing mid-copy now reports the created " + "configuration id and how many rows landed so the partial clone can be cleaned up.", "New (#594): `kbagent billing credits [--project ALIAS ...]` reads the Pay-As-You-Go " "credit balance, fanned out across every registered project in parallel. Wraps `GET " "/credits` on the `billing.{stack}` host, which accepts a plain per-project Storage " diff --git a/src/keboola_agent_cli/client/configs.py b/src/keboola_agent_cli/client/configs.py index 002297ef..6e8f4889 100644 --- a/src/keboola_agent_cli/client/configs.py +++ b/src/keboola_agent_cli/client/configs.py @@ -439,6 +439,52 @@ def create_config( ) return resp.json() + def create_config_copy( + self, + component_id: str, + config_id: str, + version: int, + name: str, + description: str = "", + branch_id: int | None = None, + ) -> dict[str, Any]: + """Copy an existing configuration into a NEW independent configuration. + + POST /v2/storage/[branch/{id}/]components/{comp_id}/configs/{config_id} + /versions/{version}/create + + This is the server-side duplicate. It copies the configuration exactly + as stored -- every top-level key (``parameters``, ``storage``, + ``runtime``, ``authorization``) travels with it, which is precisely + what hand-rebuilding a body from ``config detail`` fails to do (issue + #587: a dropped ``runtime.parallelism`` silently serialized a 65-row + writer). Encrypted (``KBC::``) values stay valid because the copy + lands in the same project. + + Args: + component_id: Component identifier. + config_id: Source configuration ID. + version: Source configuration version to copy from. + name: Name for the new configuration. + description: Optional description. When empty the field is omitted + and the copy inherits the source's description. + branch_id: If set, target a specific dev branch. + + Returns: + Dict carrying the new configuration's ``id``. + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + data: dict[str, Any] = {"name": name} + if description: + data["description"] = description + resp = self._request( + "POST", + f"{prefix}/components/{quote(component_id, safe='')}/configs/" + f"{quote(config_id, safe='')}/versions/{version}/create", + data=data, + ) + return resp.json() + def update_config( self, component_id: str, diff --git a/src/keboola_agent_cli/commands/_config_clone_cmd.py b/src/keboola_agent_cli/commands/_config_clone_cmd.py new file mode 100644 index 00000000..97cb58a1 --- /dev/null +++ b/src/keboola_agent_cli/commands/_config_clone_cmd.py @@ -0,0 +1,196 @@ +"""``kbagent config clone`` -- whole-configuration duplicate (issue #587). + +Thin CLI layer over :meth:`services.config_service.ConfigService.clone_config`. + +Lives in a private module because ``commands/config.py`` is already past its +grandfathered size ceiling (``make loc-check``: "shrink it, do not extend +it"). Mounted onto ``config_app`` via :func:`register`, so the permission key +stays ``config.clone`` and the command shows up in ``kbagent config --help`` +alongside the other lifecycle commands. +""" + +from __future__ import annotations + +from typing import Any + +import typer +from rich.console import Console +from rich.markup import escape + +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ._helpers import get_formatter, get_service, map_error_to_exit_code + + +def _parse_pair_options(items: list[str] | None, flag: str, formatter: Any) -> dict[str, str]: + """Parse repeatable ``--flag PATH=VALUE`` options into a dict. + + Splits on the FIRST ``=`` only, so a value may itself contain ``=`` + (base64 padding and connection strings routinely do). + """ + parsed: dict[str, str] = {} + for item in items or []: + if "=" not in item: + formatter.error( + message=f"Invalid {flag} format: '{item}'. Expected PATH=VALUE.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) from None + path, _, value = item.partition("=") + parsed[path.strip()] = value + return parsed + + +def _format_clone_result(console: Console, data: dict) -> None: + """Render the clone result for humans.""" + mode = data.get("mode", "same-project") + if data.get("dry_run"): + console.print( + f"[bold yellow]DRY RUN[/bold yellow] -- {escape(mode)} clone of " + f"[cyan]{escape(str(data.get('source_config_id')))}[/cyan] " + f"(version {data.get('source_version')})" + ) + console.print(f" New name : {escape(str(data.get('name')))}") + console.print(f" Target : {escape(str(data.get('target_project')))}") + console.print(f" Rows : {data.get('row_count', 0)}") + missing = data.get("missing_secrets") or [] + if missing: + console.print( + f"\n[bold red]{len(missing)} encrypted value(s) must be re-supplied[/bold red] " + "-- no other project can decrypt them:" + ) + for path in missing: + console.print(f" [red]-[/red] {escape(path)}") + console.print("\n[dim]Pass each one as --secret 'PATH=VALUE'.[/dim]") + return + + console.print( + f"[bold green]Cloned[/bold green] -> config id [cyan]{escape(str(data.get('id')))}[/cyan] " + f"({escape(mode)}, project {escape(str(data.get('target_project')))})" + ) + rows = data.get("copied_rows") or [] + if rows: + console.print(f" Copied {len(rows)} row(s)") + if mode == "cross-project": + console.print( + "\n[dim]Note: storage input/output mappings were copied verbatim. " + "Bucket and table IDs are NOT remapped -- check they exist in the " + "target project (`kbagent sync clone` handles remapping).[/dim]" + ) + + +def register(app: typer.Typer) -> None: + """Mount the clone command onto ``app`` (the ``config`` Typer group).""" + + @app.command("clone", rich_help_panel="Lifecycle") + def config_clone( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Source project alias"), + component_id: str = typer.Option( + ..., "--component-id", help="Component ID (e.g. keboola.wr-db-snowflake)" + ), + config_id: str = typer.Option(..., "--config-id", help="Configuration ID to clone"), + name: str = typer.Option(..., "--name", help="Name for the new configuration"), + target_project: str | None = typer.Option( + None, + "--target-project", + help="Clone into a different project (default: same project)", + ), + description: str = typer.Option( + "", "--description", help="Description for the clone (default: inherit the source's)" + ), + set_values: list[str] | None = typer.Option( + None, + "--set", + help="Override a value in the clone: PATH=VALUE (repeatable)", + ), + secret_values: list[str] | None = typer.Option( + None, + "--secret", + help=( + "Re-supply an encrypted value for a cross-project clone: PATH=VALUE " + "(repeatable). Encrypted in the TARGET project on write." + ), + ), + branch: int | None = typer.Option( + None, "--branch", help="Source dev branch (defaults to the active branch)" + ), + target_branch: int | None = typer.Option( + None, + "--target-branch", + help="Target dev branch (defaults to the target's active branch)", + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Show the plan (and any missing secrets) without writing" + ), + allow_plaintext: bool = typer.Option( + False, + "--allow-plaintext-on-encrypt-failure", + help="Allow the clone even if secret encryption fails (DANGEROUS: plaintext secrets)", + ), + ) -> None: + """Duplicate a configuration, whole -- including runtime, storage and authorization. + + \b + Rebuilding a configuration body by hand drops sibling keys of `parameters` + silently. A lost `runtime.parallelism` makes Keboola fall back to + parallelism 1, which is invisible until you compare job timestamps. + Cloning copies everything instead. + + \b + Within one project the Storage API copies server-side, so rows and + encrypted values come along untouched. Across projects the configuration + is reassembled here, and any encrypted (`KBC::`) value must be re-supplied + with --secret: ciphertext belongs to the project it was encrypted in and + no other project can decrypt it. Run with --dry-run first to list exactly + which paths need one. + + \b + Examples: + # Duplicate inside a project and point the copy at new tables + kbagent config clone --project prod --component-id keboola.wr-db-snowflake \\ + --config-id 123 --name "Writer (staging tables)" \\ + --set 'parameters.db.schema=STAGING' + + # See what a cross-project clone would need, without writing + kbagent config clone --project prod --component-id keboola.ex-db-mysql \\ + --config-id 123 --name "Copy" --target-project dev --dry-run + + # Cross-project clone, re-supplying the credential + kbagent config clone --project prod --component-id keboola.ex-db-mysql \\ + --config-id 123 --name "Copy" --target-project dev \\ + --secret 'parameters.db.#password=hunter2' + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "config_service") + + set_overrides = _parse_pair_options(set_values, "--set", formatter) + secret_overrides = _parse_pair_options(secret_values, "--secret", formatter) + + try: + result = service.clone_config( + alias=project, + component_id=component_id, + config_id=config_id, + name=name, + description=description, + target_alias=target_project, + set_overrides=set_overrides, + secret_overrides=secret_overrides, + branch_id=branch, + target_branch_id=target_branch, + dry_run=dry_run, + allow_plaintext_fallback=allow_plaintext, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error( + message=exc.message, + error_code=exc.error_code, + project=project, + retryable=exc.retryable, + ) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + formatter.output(result, _format_clone_result) diff --git a/src/keboola_agent_cli/commands/_config_oauth.py b/src/keboola_agent_cli/commands/_config_oauth.py new file mode 100644 index 00000000..8f6016cd --- /dev/null +++ b/src/keboola_agent_cli/commands/_config_oauth.py @@ -0,0 +1,98 @@ +"""``kbagent config oauth-url`` -- OAuth authorization URL (issue #587 fallout). + +Split out of ``commands/config.py`` purely for size: that module sits at its +grandfathered ``make loc-check`` ceiling, which may only shrink, and adding the +two-line clone registration hook pushed it over. Moving this self-contained +command out buys the file back its headroom instead of raising the recorded +limit -- CONTRIBUTING.md is explicit that the baseline is never regenerated to +silence a file you just grew. + +Mounted onto ``config_app`` via :func:`register`, so the permission key stays +``config.oauth-url`` and it still shows up in ``kbagent config --help``. +""" + +from __future__ import annotations + +import typer +from rich.markup import escape + +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ._helpers import get_formatter, get_service, map_error_to_exit_code + + +def register(app: typer.Typer) -> None: + """Mount the oauth-url command onto ``app`` (the ``config`` Typer group).""" + + @app.command( + "oauth-url", + rich_help_panel="OAuth", + help=( + "Requires master token. Generate an OAuth authorization URL for a component configuration." + ), + ) + def config_oauth_url( + ctx: typer.Context, + project: str = typer.Option( + ..., + "--project", + help="Project alias", + ), + component_id: str = typer.Option( + ..., + "--component-id", + help="Component ID (e.g. keboola.ex-google-drive)", + ), + config_id: str = typer.Option( + ..., + "--config-id", + help="Configuration ID to authorize", + ), + redirect_url: str | None = typer.Option( + None, + "--redirect-url", + help="Optional URL to return to after the OAuth flow completes (sets returnUrl query param)", + ), + ) -> None: + """Generate an OAuth authorization URL for a component configuration. + + Opens a short-lived, component-scoped authorization link. + The user must open this URL in a browser and grant access. + + \b + Examples: + kbagent config oauth-url --project P --component-id keboola.ex-google-drive --config-id ID + + # Redirect back to a custom URL after the OAuth flow completes + kbagent config oauth-url --project P --component-id keboola.ex-google-drive --config-id ID \\ + --redirect-url https://example.com/oauth-done + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "config_service") + + try: + result = service.get_oauth_url( + alias=project, + component_id=component_id, + config_id=config_id, + redirect_url=redirect_url, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + ) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + else: + formatter.console.print( + f"[bold]OAuth URL for[/bold] [cyan]{escape(component_id)}[/cyan]/" + f"[cyan]{escape(config_id)}[/cyan]:\n" + ) + formatter.console.print(f" [link]{result['url']}[/link]") + formatter.console.print("\n[dim]Open this URL in a browser and grant access.[/dim]") diff --git a/src/keboola_agent_cli/commands/config.py b/src/keboola_agent_cli/commands/config.py index 0965ad55..6f923fca 100644 --- a/src/keboola_agent_cli/commands/config.py +++ b/src/keboola_agent_cli/commands/config.py @@ -2460,75 +2460,16 @@ def config_row_delete( # ── config oauth-url ─────────────────────────────────────────────────────────── - -@config_app.command( - "oauth-url", - rich_help_panel="OAuth", - help=( - "Requires master token. Generate an OAuth authorization URL for a component configuration." - ), -) -def config_oauth_url( - ctx: typer.Context, - project: str = typer.Option( - ..., - "--project", - help="Project alias", - ), - component_id: str = typer.Option( - ..., - "--component-id", - help="Component ID (e.g. keboola.ex-google-drive)", - ), - config_id: str = typer.Option( - ..., - "--config-id", - help="Configuration ID to authorize", - ), - redirect_url: str | None = typer.Option( - None, - "--redirect-url", - help="Optional URL to return to after the OAuth flow completes (sets returnUrl query param)", - ), -) -> None: - """Generate an OAuth authorization URL for a component configuration. - - Opens a short-lived, component-scoped authorization link. - The user must open this URL in a browser and grant access. - - \b - Examples: - kbagent config oauth-url --project P --component-id keboola.ex-google-drive --config-id ID - - # Redirect back to a custom URL after the OAuth flow completes - kbagent config oauth-url --project P --component-id keboola.ex-google-drive --config-id ID \\ - --redirect-url https://example.com/oauth-done - """ - formatter = get_formatter(ctx) - service = get_service(ctx, "config_service") - - try: - result = service.get_oauth_url( - alias=project, - component_id=component_id, - config_id=config_id, - redirect_url=redirect_url, - ) - except (ConfigError, KeboolaApiError) as exc: - _handle_config_service_error(formatter, exc) - - if formatter.json_mode: - formatter.output(result) - else: - formatter.console.print( - f"[bold]OAuth URL for[/bold] [cyan]{escape(component_id)}[/cyan]/" - f"[cyan]{escape(config_id)}[/cyan]:\n" - ) - formatter.console.print(f" [link]{result['url']}[/link]") - formatter.console.print("\n[dim]Open this URL in a browser and grant access.[/dim]") - - # `config state-get` / `config state-set` live in config_state.py (file-size-budget # split, see that module's docstring) but register on THIS module's `config_app`. # Import for the side effect only -- nothing here references the module by name. from . import config_state as _config_state # noqa: E402,F401 + +# The clone command (issue #587) lives in a private module -- config.py is +# past its grandfathered commands-file size ceiling. Mounted here so it shares +# the `config.*` permission namespace and appears in `kbagent config --help`. +from ._config_clone_cmd import register as _register_clone_command # noqa: E402 +from ._config_oauth import register as _register_oauth_command # noqa: E402 + +_register_clone_command(config_app) +_register_oauth_command(config_app) diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index ee5e2646..a229b3fd 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -419,6 +419,21 @@ (empty shell, validation auto-skipped). Works for ALL component types including keboola.snowflake-transformation (unlike tool call create_config which refuses it). + kbagent config clone --project P --component-id ID --config-id ID --name NAME + [--target-project P2] [--description D] [--set PATH=VALUE ...] + [--secret PATH=VALUE ...] [--branch ID] [--target-branch ID] + [--dry-run] [--allow-plaintext-on-encrypt-failure] + (since 0.84.2) Duplicate a configuration WHOLE. Use this instead of reading config detail + and rebuilding a body by hand -- that drops siblings of parameters (runtime, storage, + authorization) silently, and a lost runtime.parallelism means Keboola falls back to + parallelism 1 (issue #587). + Same project (default): server-side copy; rows and KBC:: encrypted values come along. + Cross project (--target-project): reassembled client-side. Encrypted values CANNOT travel + (ciphertext is project-scoped), so the clone is REFUSED until each path is re-supplied via + --secret PATH=VALUE; they are encrypted in the TARGET project on write. Run --dry-run first + to list exactly which paths need one. Storage bucket/table IDs are copied verbatim, NOT + remapped -- use sync clone for that. + kbagent config search --query PATTERN [--project NAME] [--component-type TYPE] [-i] [-r] [--branch ID] Search config bodies for string/regex. Reports match location in JSON tree. Branch-aware. diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 57881635..e98261b8 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -74,6 +74,8 @@ "config.rename": "write", "config.delete": "destructive", "config.new": "write", + # Creates a new configuration; never modifies or deletes the source. + "config.clone": "write", "config.variables-set": "write", "config.variables-get": "read", "config.variables-clear": "destructive", diff --git a/src/keboola_agent_cli/server/routers/configs.py b/src/keboola_agent_cli/server/routers/configs.py index ab458f13..442698ea 100644 --- a/src/keboola_agent_cli/server/routers/configs.py +++ b/src/keboola_agent_cli/server/routers/configs.py @@ -48,6 +48,19 @@ class ConfigCreate(BaseModel): branch_id: int | None = None +class ConfigClone(BaseModel): + name: str + target_project: str | None = None + description: str | None = None + # Dotted paths, exactly as the CLI's --set / --secret take them. + set_overrides: dict[str, str] | None = None + secret_overrides: dict[str, str] | None = None + branch_id: int | None = None + target_branch_id: int | None = None + dry_run: bool = False + allow_plaintext_fallback: bool = False + + class SetDefaultBucket(BaseModel): bucket: str | None = None clear: bool = False @@ -221,6 +234,36 @@ def config_create( ) +@router.post("/{project}/{component_id}/{config_id}/clone", summary="Clone a configuration") +def config_clone( + project: str, + component_id: str, + config_id: str, + body: ConfigClone, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Duplicate a configuration whole. Mirrors `kbagent config clone`. + + A cross-project clone raises ConfigError (HTTP 4xx via the shared handler) + listing every encrypted path that must be re-supplied in + `secret_overrides` -- call with `dry_run: true` first to collect them. + """ + return registry.config.clone_config( + alias=project, + component_id=component_id, + config_id=config_id, + name=body.name, + description=body.description or "", + target_alias=body.target_project, + set_overrides=body.set_overrides, + secret_overrides=body.secret_overrides, + branch_id=body.branch_id, + target_branch_id=body.target_branch_id, + dry_run=body.dry_run, + allow_plaintext_fallback=body.allow_plaintext_fallback, + ) + + @router.post( "/{project}/{component_id}/{config_id}/set-default-bucket", summary="Set or clear default bucket", diff --git a/src/keboola_agent_cli/services/_config_clone.py b/src/keboola_agent_cli/services/_config_clone.py new file mode 100644 index 00000000..0e8c54eb --- /dev/null +++ b/src/keboola_agent_cli/services/_config_clone.py @@ -0,0 +1,545 @@ +"""Configuration cloning for ``kbagent config clone`` (issue #587). + +Lives outside ``config_service.py`` because that module is already over its +per-layer size budget (``make loc-check``), and the clone flow is +self-contained: it takes clients and resolved projects, and returns a result +envelope. ``ConfigService.clone_config`` is a thin delegation. + +Why a command exists at all +--------------------------- +There was no way to duplicate a configuration, so people rebuilt the body from +``config detail`` output -- typically by copying ``configuration["parameters"]`` +and nothing else. A configuration's root also carries ``storage``, ``runtime`` +and ``authorization``, and dropping one is silent: a lost +``runtime.parallelism`` makes Keboola fall back to ``parallelism: 1``, which +turned a 65-row writer from 20-at-a-time into strictly sequential -- 140 +minutes instead of the expected 60-90, with nothing reported anywhere. + +Two paths, for one reason: encryption +------------------------------------- +Within a project the Storage API can copy server-side +(``POST .../versions/{v}/create``), which duplicates the stored configuration +exactly -- every sibling key, every row -- and ``KBC::`` ciphertexts remain +decryptable because the project is unchanged. + +Across projects that is not available, and more importantly not sufficient: a +Keboola ciphertext is bound to the project it was encrypted for. Copying it +verbatim yields a configuration that looks complete and fails at runtime. So +the cross-project path assembles the configuration itself, and refuses to +write until every encrypted value has been re-supplied in plaintext, which it +then encrypts in the TARGET project. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Protocol + +from ..errors import ConfigError, KeboolaApiError +from ..json_utils import set_nested_value +from ..models import ProjectConfig +from ._encryption import find_encrypted_secret_paths, find_unencryptable_secret_paths + +logger = logging.getLogger(__name__) + + +class _EncryptFn(Protocol): + """The encrypt-before-write callable supplied by ``ConfigService``.""" + + def __call__( + self, + client: Any, + project: ProjectConfig, + component_id: str, + configuration: dict[str, Any] | None, + *, + allow_plaintext_fallback: bool, + ) -> dict[str, Any] | None: ... + + +def is_same_project(source: ProjectConfig, target: ProjectConfig) -> bool: + """True when both aliases point at one and the same Keboola project. + + Identity short-circuits the no-``--target-project`` case. Otherwise both + the stack and the project id must match, and an unknown (``None``) id + never counts as a match: two aliases with no id recorded could be + different projects, and guessing "same" would skip the encrypted-value + check that protects the cross-project path. + """ + if source is target: + return True + return ( + source.stack_url == target.stack_url + and source.project_id is not None + and source.project_id == target.project_id + ) + + +def _apply_overrides(configuration: dict[str, Any], overrides: dict[str, str]) -> dict[str, Any]: + """Apply ``--set path=value`` edits, returning a new configuration. + + ``set_nested_value`` deep-copies, so the source body is never mutated -- + which matters because the same dict is also reported in the dry-run + envelope. + """ + result = configuration + for path, value in overrides.items(): + result = set_nested_value(result, path, value) + return result + + +_ROW_PATH = re.compile(r"^rows\[(\d+)\]\.(.+)$") + + +def split_row_secret_overrides( + secrets: dict[str, str], +) -> tuple[dict[str, str], dict[int, dict[str, str]]]: + """Split ``--secret`` paths into parent-body ones and per-row ones. + + Row ciphertext is *detected* and reported as ``rows[N].<path>`` so the + operator knows which row to fix, which means a re-supplied value arrives + under that same prefix. It has to be routed back to that row: applying it + to the parent body would leave the row carrying the source project's + undecryptable ciphertext while the command reported success. + + Returns ``(parent_overrides, {row_index: {path: value}})``. + """ + parent: dict[str, str] = {} + per_row: dict[int, dict[str, str]] = {} + for path, value in secrets.items(): + match = _ROW_PATH.match(path) + if match: + per_row.setdefault(int(match.group(1)), {})[match.group(2)] = value + else: + parent[path] = value + return parent, per_row + + +def _apply_secret_overrides( + configuration: dict[str, Any], secrets: dict[str, str] +) -> dict[str, Any]: + """Substitute re-supplied plaintext at the paths that held ciphertext. + + Written back as plaintext on purpose: the caller encrypts the assembled + body in the target project immediately afterwards, so this value never + reaches the API unencrypted. + """ + result = configuration + for path, value in secrets.items(): + result = set_nested_value(result, path, value) + return result + + +def _clone_same_project( + *, + client: Any, + component_id: str, + config_id: str, + source: dict[str, Any], + name: str, + description: str, + set_overrides: dict[str, str], + branch_id: int | None, + project: ProjectConfig, + encrypt_fn: _EncryptFn, + allow_plaintext_fallback: bool, +) -> dict[str, Any]: + """Duplicate within one project using the server-side copy endpoint. + + Nothing is rebuilt here, which is the entire value: whatever the source + holds -- sibling keys, rows, ciphertexts -- lands in the copy untouched. + ``--set`` edits are applied afterwards as a normal update on the new + configuration, so an override can never be the reason a key goes missing. + + The patched body still goes through the encrypt-before-write step: a + ``--set 'parameters.db.#password=...'`` is expected traffic here (this is + how you repoint a copy at another database), and every other config write + path in this CLI pre-encrypts ``#``-prefixed values (issue #378). Skipping + it would put the credential in Storage -- and in version history -- in the + clear. + """ + created = client.create_config_copy( + component_id=component_id, + config_id=config_id, + version=source["version"], + name=name, + description=description, + branch_id=branch_id, + ) + new_id = str(created["id"]) + + if set_overrides: + # Re-read rather than patching the source body we already hold: the + # copy is what we are editing, and only the API can tell us what it + # actually contains. + clone_detail = client.get_config_detail(component_id, new_id, branch_id=branch_id) + patched = _apply_overrides(clone_detail.get("configuration") or {}, set_overrides) + encrypted = encrypt_fn( + client, + project, + component_id, + patched, + allow_plaintext_fallback=allow_plaintext_fallback, + ) + client.update_config( + component_id=component_id, + config_id=new_id, + configuration=encrypted if encrypted is not None else patched, + change_description=f"config clone: applied {len(set_overrides)} override(s)", + branch_id=branch_id, + ) + + return created + + +def _clone_cross_project( + *, + source_rows: list[dict[str, Any]], + target_client: Any, + target_project: ProjectConfig, + component_id: str, + configuration: dict[str, Any], + name: str, + description: str, + target_branch_id: int | None, + encrypt_fn: _EncryptFn, + allow_plaintext_fallback: bool, + row_secret_overrides: dict[int, dict[str, str]] | None = None, +) -> dict[str, Any]: + """Assemble the configuration in the target project, then copy its rows. + + Rows are created one by one because there is no bulk endpoint. Each row + body gets its own re-supplied secrets substituted and is then encrypted in + the target project -- a row can carry its own ``#``-secrets, and skipping + either step would leave the row unusable or write plaintext. + """ + row_secret_overrides = row_secret_overrides or {} + encrypted_body = encrypt_fn( + target_client, + target_project, + component_id, + configuration, + allow_plaintext_fallback=allow_plaintext_fallback, + ) + created = target_client.create_config( + component_id=component_id, + name=name, + configuration=encrypted_body if encrypted_body is not None else {}, + description=description, + branch_id=target_branch_id, + ) + new_id = str(created["id"]) + + copied_rows: list[dict[str, str]] = [] + for index, row in enumerate(source_rows): + source_row_body: dict[str, Any] = row.get("configuration") or {} + if index in row_secret_overrides: + source_row_body = _apply_secret_overrides(source_row_body, row_secret_overrides[index]) + row_body = encrypt_fn( + target_client, + target_project, + component_id, + source_row_body, + allow_plaintext_fallback=allow_plaintext_fallback, + ) + try: + created_row = target_client.create_config_row( + component_id=component_id, + config_id=new_id, + name=row.get("name") or "", + configuration=row_body if row_body is not None else {}, + description=row.get("description") or "", + is_disabled=bool(row.get("isDisabled")), + branch_id=target_branch_id, + ) + except KeboolaApiError as exc: + # There is no bulk row endpoint and no rollback, so a mid-way + # failure leaves a half-populated configuration in the target + # project. Name it and say how far we got -- the caller cannot + # retry or clean up something it cannot identify. + raise KeboolaApiError( + message=( + f"{exc.message}\n" + f"PARTIAL CLONE: configuration '{new_id}' was created in the target " + f"project with {len(copied_rows)} of {len(source_rows)} row(s) copied " + f"before row '{row.get('name') or row.get('id')}' failed. Delete it " + f"with `kbagent config delete --component-id {component_id} " + f"--config-id {new_id}` and re-run, or add the missing rows by hand." + ), + status_code=exc.status_code, + error_code=exc.error_code, + retryable=exc.retryable, + ) from exc + copied_rows.append({"source_row_id": str(row.get("id")), "id": str(created_row["id"])}) + + created["copied_rows"] = copied_rows + return created + + +def clone_config( + *, + source_client: Any, + source_project: ProjectConfig, + source_alias: str, + target_client: Any, + target_project: ProjectConfig, + target_alias: str, + component_id: str, + config_id: str, + name: str, + description: str = "", + set_overrides: dict[str, str] | None = None, + secret_overrides: dict[str, str] | None = None, + branch_id: int | None = None, + target_branch_id: int | None = None, + dry_run: bool = False, + allow_plaintext_fallback: bool = False, + encrypt_fn: _EncryptFn, +) -> dict[str, Any]: + """Clone a configuration, within one project or into another. + + Raises: + ConfigError: On a cross-project clone whose source carries encrypted + values that were not re-supplied via ``secret_overrides``. This is + deliberately fatal rather than a warning: the resulting + configuration would look complete and fail at runtime, in a + different project from the one the operator is watching. + """ + set_overrides = set_overrides or {} + secret_overrides = secret_overrides or {} + + source = source_client.get_config_detail(component_id, config_id, branch_id=branch_id) + source_body: dict[str, Any] = source.get("configuration") or {} + source_rows: list[dict[str, Any]] = source.get("rows") or [] + cross_project = not is_same_project(source_project, target_project) + + # The server-side copy writes into the branch it reads from, so it cannot + # honour a different target branch. Refuse rather than write to the wrong + # branch silently -- the caller asked for something this path cannot do. + if not cross_project and target_branch_id is not None and target_branch_id != branch_id: + raise ConfigError( + f"--target-branch {target_branch_id} cannot be honoured for a clone within one " + f"project: the Storage API copies into the source's own branch " + f"({branch_id if branch_id is not None else 'production'}). Clone without " + f"--target-branch, or clone into a different project." + ) + + # Same project omits an empty description so the copy endpoint inherits the + # source's; the cross-project path assembles the body itself, so it has to + # carry that inheritance over explicitly or the copy comes out blank. + effective_description = description or (source.get("description") or "") + + # Ciphertext is project-scoped, so it only blocks the cross-project path. + encrypted_paths = find_encrypted_secret_paths(source_body) if cross_project else [] + row_encrypted_paths = ( + [ + f"rows[{index}].{path}" + for index, row in enumerate(source_rows) + for path in find_encrypted_secret_paths(row.get("configuration") or {}) + ] + if cross_project + else [] + ) + all_encrypted = encrypted_paths + row_encrypted_paths + + # Ciphertext under a plain (non-``#``) key has no supported round-trip: + # the encrypt step keys off ``#`` names, so a replacement supplied here + # would be written to the target project in the clear. Refuse instead -- + # leaking a credential is a worse failure than not cloning. + unencryptable = ( + find_unencryptable_secret_paths(source_body) + + [ + f"rows[{index}].{path}" + for index, row in enumerate(source_rows) + for path in find_unencryptable_secret_paths(row.get("configuration") or {}) + ] + if cross_project + else [] + ) + if unencryptable and not dry_run: + listed = "\n - ".join(unencryptable) + raise ConfigError( + f"Cannot clone into project '{target_alias}': the source holds " + f"{len(unencryptable)} encrypted value(s) under a plain (non-'#') key, which " + f"this CLI cannot re-encrypt -- supplying one would write it to " + f"'{target_alias}' in PLAINTEXT.\n" + f" - {listed}\n" + f"Encrypt the replacement yourself for the target project with " + f"`kbagent encrypt values --project {target_alias} --component-id {component_id}` " + f"and pass the ciphertext via --set, or clone within the source project instead." + ) + + missing = [path for path in all_encrypted if path not in secret_overrides] + + if missing and not dry_run: + listed = "\n - ".join(missing) + raise ConfigError( + f"Cannot clone into project '{target_alias}': the source configuration " + f"holds {len(missing)} encrypted value(s) that no other project can decrypt.\n" + f" - {listed}\n" + f"Re-supply each one with --secret PATH=VALUE (they are encrypted in the target " + f"project on write), or clone within the source project instead." + ) + + # Row secrets are reported (and therefore re-supplied) under a `rows[N].` + # prefix, but they belong to the row body, not the parent -- keep them + # apart so each half is applied to the document it actually describes. + parent_secrets, row_secrets = split_row_secret_overrides(secret_overrides) + + planned_body = _apply_overrides(source_body, set_overrides) + if cross_project: + planned_body = _apply_secret_overrides(planned_body, parent_secrets) + + if dry_run: + return { + "dry_run": True, + "mode": "cross-project" if cross_project else "same-project", + "source_project": source_alias, + "target_project": target_alias, + "component_id": component_id, + "source_config_id": config_id, + "source_version": source.get("version"), + "name": name, + "description": description, + "configuration": planned_body, + "row_count": len(source_rows), + "encrypted_paths": all_encrypted, + "missing_secrets": missing, + "branch_id": target_branch_id if cross_project else branch_id, + } + + if cross_project: + created = _clone_cross_project( + source_rows=source_rows, + target_client=target_client, + target_project=target_project, + component_id=component_id, + configuration=planned_body, + name=name, + description=effective_description, + target_branch_id=target_branch_id, + encrypt_fn=encrypt_fn, + allow_plaintext_fallback=allow_plaintext_fallback, + row_secret_overrides=row_secrets, + ) + else: + created = _clone_same_project( + client=source_client, + component_id=component_id, + config_id=config_id, + source=source, + name=name, + description=description, + set_overrides=set_overrides, + branch_id=branch_id, + project=source_project, + encrypt_fn=encrypt_fn, + allow_plaintext_fallback=allow_plaintext_fallback, + ) + + created["mode"] = "cross-project" if cross_project else "same-project" + created["source_project"] = source_alias + created["target_project"] = target_alias + created["component_id"] = component_id + created["source_config_id"] = config_id + created["source_version"] = source.get("version") + created["encrypted_paths"] = all_encrypted + created.setdefault("copied_rows", []) + return created + + +# --- ConfigService binding ------------------------------------------------- +# +# Bound onto ConfigService as ``clone_config``. It lives here rather than on +# the service because ``config_service.py`` sits at its HARD file-size +# ceiling, and CONTRIBUTING.md is explicit that such a file is split before +# more functionality is merged into it. ``self`` is a ConfigService. +def clone_config_method( + self, + alias: str, + component_id: str, + config_id: str, + name: str, + description: str = "", + target_alias: str | None = None, + set_overrides: dict[str, str] | None = None, + secret_overrides: dict[str, str] | None = None, + branch_id: int | None = None, + target_branch_id: int | None = None, + dry_run: bool = False, + allow_plaintext_fallback: bool = False, +) -> dict[str, Any]: + """Duplicate a configuration (``kbagent config clone``, issue #587). + + Thin delegation to :mod:`._config_clone`, which holds the flow itself. + + Args: + alias: Source project alias. + component_id: Component ID of the configuration being cloned. + config_id: Source configuration ID. + name: Name for the new configuration. + description: Optional description; empty inherits the source's. + target_alias: Target project alias. Defaults to the source, which + selects the server-side copy path. + set_overrides: ``{dotted.path: value}`` edits applied to the clone. + secret_overrides: ``{dotted.path: plaintext}`` replacements for + encrypted values, required for a cross-project clone. + branch_id: Source dev branch; falls back to the project's active one. + target_branch_id: Target dev branch; falls back to the target + project's active one. + dry_run: Report the plan (including any missing secrets) without + writing anything. + allow_plaintext_fallback: Permit a plaintext write if the + Encryption API fails. DANGEROUS. + + Returns: + The created configuration annotated with ``mode``, + ``source_project`` / ``target_project``, ``source_version``, + ``encrypted_paths`` and ``copied_rows``; or the planned envelope + when ``dry_run``. + + Raises: + ConfigError: If either alias is unknown, or a cross-project clone + is missing plaintext for an encrypted value. + KeboolaApiError: If any API call fails. + """ + aliases = [alias] if target_alias is None else [alias, target_alias] + projects = self.resolve_projects(aliases) + source_project = projects[alias] + target_project = projects[target_alias] if target_alias else source_project + + source_client = self._client_factory(source_project.stack_url, source_project.token) + # Must use the SAME same/cross decision as the clone flow itself: + # reusing the source client for what the flow treats as a + # cross-project write would create the configuration in the source + # project while reporting the target. + target_client = ( + source_client + if is_same_project(source_project, target_project) + else self._client_factory(target_project.stack_url, target_project.token) + ) + try: + return clone_config( + source_client=source_client, + source_project=source_project, + source_alias=alias, + target_client=target_client, + target_project=target_project, + target_alias=target_alias or alias, + component_id=component_id, + config_id=config_id, + name=name, + description=description, + set_overrides=set_overrides, + secret_overrides=secret_overrides, + branch_id=branch_id or source_project.active_branch_id, + target_branch_id=target_branch_id or target_project.active_branch_id, + dry_run=dry_run, + allow_plaintext_fallback=allow_plaintext_fallback, + encrypt_fn=self._encrypt_secrets_before_write, + ) + finally: + source_client.close() + if target_client is not source_client: + target_client.close() diff --git a/src/keboola_agent_cli/services/_encryption.py b/src/keboola_agent_cli/services/_encryption.py index 8a02883b..5b43231f 100644 --- a/src/keboola_agent_cli/services/_encryption.py +++ b/src/keboola_agent_cli/services/_encryption.py @@ -220,6 +220,85 @@ def encrypt_secrets_in_config( return configuration +def collect_encrypted_paths( + obj: Any, + path_prefix: str, + result: list[str], + unencryptable: list[str] | None = None, +) -> None: + """Recursively collect the dotted paths of ``KBC::``-encrypted values. + + The mirror of :func:`collect_secrets`: that one finds values still in + plaintext (so they can be encrypted), this one finds values already + encrypted (so a caller can discover what cannot be moved). + + Paths are plain dotted config paths (``parameters.db.#password``), in the + exact form :func:`~keboola_agent_cli.json_utils.set_nested_value` accepts -- + including a bare integer segment for a list index + (``parameters.values.0.value``). They are shown to a human and fed back + through ``--secret PATH=VALUE``, so a path that cannot be applied is worse + than useless: it instructs the operator to run a command that then fails. + Values are never returned -- only where they live. + + When ``unencryptable`` is supplied, ciphertext found under a key that is + NOT ``#``-prefixed is collected there instead. :func:`collect_secrets` + only picks up ``#`` keys, so a plaintext substituted at such a path would + never be re-encrypted and would reach Storage in the clear. + """ + if isinstance(obj, dict): + for key, value in obj.items(): + path = f"{path_prefix}{key}" + if is_already_encrypted(value): + if unencryptable is not None and not is_secret_key(key): + unencryptable.append(path) + else: + result.append(path) + elif isinstance(value, (dict, list)): + collect_encrypted_paths(value, f"{path}.", result, unencryptable) + elif isinstance(obj, list): + for i, item in enumerate(obj): + if _is_secret_name_value_pair(item) and is_already_encrypted(item["value"]): + # Point at the ``value`` field itself: that is what gets + # replaced, and `name`/`value` pairs keep the secret marker in + # ``name``, so the path must not end in the ``#name``. + result.append(f"{path_prefix}{i}.value") + else: + collect_encrypted_paths(item, f"{path_prefix}{i}.", result, unencryptable) + + +def find_encrypted_secret_paths(configuration: Any) -> list[str]: + """Return the paths of every ``KBC::``-encrypted value that CAN be re-supplied. + + A Keboola ciphertext is bound to the project (and often the component) it + was encrypted for -- no other project can decrypt it. Copying one across + projects therefore produces a configuration that looks complete and fails + at runtime. ``config clone`` uses this to refuse a cross-project clone + until each path is re-supplied in plaintext, which it then encrypts in the + TARGET project. + + Ciphertext under a non-``#`` key is excluded here and reported by + :func:`find_unencryptable_secret_paths` instead. + """ + paths: list[str] = [] + collect_encrypted_paths(configuration, "", paths, []) + return sorted(paths) + + +def find_unencryptable_secret_paths(configuration: Any) -> list[str]: + """Return paths holding ciphertext that this CLI cannot re-encrypt. + + The Encryption API contract in this codebase keys off ``#``-prefixed + names (:func:`collect_secrets`), so a ciphertext stored under a plain key + has no supported round-trip: accepting a ``--secret`` for it would write + the replacement to Storage in plaintext. Callers should refuse instead, + and point at ``kbagent encrypt values`` + ``--set`` for the deliberate + manual path. + """ + unencryptable: list[str] = [] + collect_encrypted_paths(configuration, "", [], unencryptable) + return sorted(unencryptable) + + def find_plaintext_secret_keys(configuration: dict[str, Any]) -> list[str]: """Return the flattened paths of *unencrypted* ``#``-prefixed secrets. diff --git a/src/keboola_agent_cli/services/config_service.py b/src/keboola_agent_cli/services/config_service.py index 33d64e9d..2c4f557d 100644 --- a/src/keboola_agent_cli/services/config_service.py +++ b/src/keboola_agent_cli/services/config_service.py @@ -25,6 +25,7 @@ from ..sync.code_extraction import normalize_blocks_codes_script from ..sync.manifest import Manifest, load_manifest, save_manifest from ..sync.naming import sanitize_name +from ._config_clone import clone_config_method from ._config_set_guard import validate_set_paths from ._encryption import collect_secrets, encrypt_secrets_in_config, find_plaintext_secret_keys from .base import BaseService, ClientFactory, sanitize_unexpected_error @@ -1982,6 +1983,12 @@ def create_config( ) return result + # Bound from ._config_clone: the flow, and the client/project wiring it + # needs, live there because this module is at its HARD size ceiling. + # Assigned rather than delegated so the 12-parameter signature is not + # spelled out twice. + clone_config = clone_config_method + def _validate_config_body( self, project: ProjectConfig, diff --git a/tests/test_client.py b/tests/test_client.py index f6b6f05c..84472270 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -2167,6 +2167,113 @@ def test_delete_config_row_not_found_raises(self, httpx_mock) -> None: assert excinfo.value.status_code == 404 +class TestCreateConfigCopy: + """Server-side configuration copy (issue #587). + + ``POST .../configs/{id}/versions/{version}/create`` duplicates a + configuration into a NEW independent configuration and returns its id. It + is the only way to duplicate a config without rebuilding the body by hand + -- which is what drops sibling keys like ``runtime.parallelism``. + """ + + TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + + @staticmethod + def _parse_form_body(request: httpx.Request) -> dict[str, str]: + """Parse form-encoded request body to a flat str->str dict.""" + parsed = parse_qs(request.content.decode("utf-8"), keep_blank_values=True) + return {k: v[0] for k, v in parsed.items()} + + def test_create_config_copy_no_branch(self, httpx_mock) -> None: + """POST to the versions/{v}/create endpoint; name/description are form fields.""" + httpx_mock.add_response( + url=( + "https://connection.keboola.com/v2/storage/components/" + "keboola.wr-db-snowflake/configs/src-1/versions/7/create" + ), + method="POST", + json={"id": "364494012"}, + status_code=201, + ) + + with KeboolaClient(stack_url="https://connection.keboola.com", token=self.TOKEN) as client: + result = client.create_config_copy( + component_id="keboola.wr-db-snowflake", + config_id="src-1", + version=7, + name="Clone of writer", + description="copied for the new tables", + ) + + assert result["id"] == "364494012" + body = self._parse_form_body(httpx_mock.get_requests()[0]) + assert body["name"] == "Clone of writer" + assert body["description"] == "copied for the new tables" + + def test_create_config_copy_with_branch(self, httpx_mock) -> None: + """branch_id routes the POST through the branch-scoped prefix.""" + httpx_mock.add_response( + url=( + "https://connection.keboola.com/v2/storage/branch/200/components/" + "keboola.wr-db-snowflake/configs/src-1/versions/7/create" + ), + method="POST", + json={"id": "new-1"}, + status_code=201, + ) + + with KeboolaClient(stack_url="https://connection.keboola.com", token=self.TOKEN) as client: + client.create_config_copy( + component_id="keboola.wr-db-snowflake", + config_id="src-1", + version=7, + name="Clone", + branch_id=200, + ) + + def test_create_config_copy_omits_empty_description(self, httpx_mock) -> None: + """An empty description is not sent, so the copy inherits the source's.""" + httpx_mock.add_response( + url=( + "https://connection.keboola.com/v2/storage/components/" + "keboola.ex-http/configs/src-1/versions/2/create" + ), + method="POST", + json={"id": "new-2"}, + status_code=201, + ) + + with KeboolaClient(stack_url="https://connection.keboola.com", token=self.TOKEN) as client: + client.create_config_copy( + component_id="keboola.ex-http", + config_id="src-1", + version=2, + name="Clone", + ) + + assert "description" not in self._parse_form_body(httpx_mock.get_requests()[0]) + + def test_create_config_copy_component_id_is_url_quoted(self, httpx_mock) -> None: + """A component id containing a slash must not break out of the path.""" + httpx_mock.add_response( + url=( + "https://connection.keboola.com/v2/storage/components/" + "vendor%2Fcomp/configs/src-1/versions/1/create" + ), + method="POST", + json={"id": "new-3"}, + status_code=201, + ) + + with KeboolaClient(stack_url="https://connection.keboola.com", token=self.TOKEN) as client: + client.create_config_copy( + component_id="vendor/comp", + config_id="src-1", + version=1, + name="Clone", + ) + + class TestConfigIsDisabledField: """isDisabled form-field contract on create_config / update_config (issue #467). diff --git a/tests/test_config_clone_cli.py b/tests/test_config_clone_cli.py new file mode 100644 index 00000000..91f3abda --- /dev/null +++ b/tests/test_config_clone_cli.py @@ -0,0 +1,225 @@ +"""CLI tests for ``kbagent config clone`` (issue #587). + +Scope: +- --set / --secret PATH=VALUE parsing, including values containing '='. +- Both output modes actually render. The human renderer is the reason this + file exists: the service-layer suite passes with a broken renderer, because + it never goes through OutputFormatter. A first live run crashed on exactly + that (a human_formatter taking one argument instead of two), which no + service test could have caught. +- Error propagation: a refused cross-project clone surfaces as exit 5. +""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner, Result + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.project_service import ProjectService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + +runner = CliRunner() + + +def _setup_config(config_dir: Path) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="Production", + project_id=1234, + ), + ) + return store + + +def _clone_result(**overrides: object) -> dict: + result = { + "id": "clone-1", + "mode": "same-project", + "source_project": "prod", + "target_project": "prod", + "component_id": "keboola.wr-db-snowflake", + "source_config_id": "src-1", + "source_version": 7, + "encrypted_paths": [], + "copied_rows": [], + } + result.update(overrides) + return result + + +def _invoke( + args: list[str], + *, + config_dir: Path, + service_mock: MagicMock | None = None, +) -> Result: + store = _setup_config(config_dir) + svc = service_mock or MagicMock() + if service_mock is None: + svc.clone_config.return_value = _clone_result() + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockConfigService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockConfigService.return_value = svc + return runner.invoke(app, args) + + +BASE_ARGS = [ + "config", + "clone", + "--project", + "prod", + "--component-id", + "keboola.wr-db-snowflake", + "--config-id", + "src-1", + "--name", + "Clone", +] + + +class TestCloneOptionParsing: + def test_set_and_secret_pairs_reach_the_service(self, tmp_path: Path) -> None: + svc = MagicMock() + svc.clone_config.return_value = _clone_result() + + result = _invoke( + [*BASE_ARGS, "--set", "parameters.db.host=h2", "--secret", "parameters.db.#password=p"], + config_dir=tmp_path, + service_mock=svc, + ) + + assert result.exit_code == 0, result.output + kwargs = svc.clone_config.call_args.kwargs + assert kwargs["set_overrides"] == {"parameters.db.host": "h2"} + assert kwargs["secret_overrides"] == {"parameters.db.#password": "p"} + + def test_value_containing_equals_is_preserved(self, tmp_path: Path) -> None: + """Split on the FIRST '=' only -- base64 padding and connection + strings routinely contain more. + """ + svc = MagicMock() + svc.clone_config.return_value = _clone_result() + + result = _invoke( + [*BASE_ARGS, "--secret", "parameters.#token=abc==def="], + config_dir=tmp_path, + service_mock=svc, + ) + + assert result.exit_code == 0, result.output + assert svc.clone_config.call_args.kwargs["secret_overrides"] == { + "parameters.#token": "abc==def=" + } + + def test_malformed_pair_exits_2(self, tmp_path: Path) -> None: + result = _invoke([*BASE_ARGS, "--set", "no-equals-sign"], config_dir=tmp_path) + + assert result.exit_code == 2, result.output + assert "Expected PATH=VALUE" in result.output + + def test_target_project_is_forwarded(self, tmp_path: Path) -> None: + svc = MagicMock() + svc.clone_config.return_value = _clone_result(mode="cross-project", target_project="dev") + + result = _invoke( + [*BASE_ARGS, "--target-project", "dev"], config_dir=tmp_path, service_mock=svc + ) + + assert result.exit_code == 0, result.output + assert svc.clone_config.call_args.kwargs["target_alias"] == "dev" + + +class TestCloneOutput: + """Both renderers must survive a real OutputFormatter round-trip.""" + + def test_human_output_reports_the_new_id(self, tmp_path: Path) -> None: + result = _invoke(BASE_ARGS, config_dir=tmp_path) + + assert result.exit_code == 0, result.output + assert "clone-1" in result.output + + def test_human_dry_run_lists_missing_secrets(self, tmp_path: Path) -> None: + """--dry-run is how a caller learns which --secret values to gather, + so the paths must actually appear in human output. + """ + svc = MagicMock() + svc.clone_config.return_value = { + "dry_run": True, + "mode": "cross-project", + "source_project": "prod", + "target_project": "dev", + "component_id": "keboola.wr-db-snowflake", + "source_config_id": "src-1", + "source_version": 7, + "name": "Clone", + "row_count": 2, + "encrypted_paths": ["parameters.db.#password"], + "missing_secrets": ["parameters.db.#password"], + } + + result = _invoke( + [*BASE_ARGS, "--target-project", "dev", "--dry-run"], + config_dir=tmp_path, + service_mock=svc, + ) + + assert result.exit_code == 0, result.output + assert "parameters.db.#password" in result.output + assert "--secret" in result.output + + def test_cross_project_human_output_warns_about_bucket_mapping(self, tmp_path: Path) -> None: + """Storage mappings are copied verbatim; the operator has to be told.""" + svc = MagicMock() + svc.clone_config.return_value = _clone_result( + mode="cross-project", + target_project="dev", + copied_rows=[{"source_row_id": "r1", "id": "n1"}], + ) + + result = _invoke( + [*BASE_ARGS, "--target-project", "dev"], config_dir=tmp_path, service_mock=svc + ) + + assert result.exit_code == 0, result.output + assert "NOT remapped" in result.output + + def test_json_output_is_the_service_envelope(self, tmp_path: Path) -> None: + import json + + result = _invoke(["--json", *BASE_ARGS], config_dir=tmp_path) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["status"] == "ok" + assert payload["data"]["id"] == "clone-1" + assert payload["data"]["mode"] == "same-project" + + +class TestCloneErrors: + def test_refused_clone_exits_5(self, tmp_path: Path) -> None: + svc = MagicMock() + svc.clone_config.side_effect = ConfigError( + "Cannot clone into project 'dev': 1 encrypted value(s)" + ) + + result = _invoke( + [*BASE_ARGS, "--target-project", "dev"], config_dir=tmp_path, service_mock=svc + ) + + assert result.exit_code == 5, result.output + assert "encrypted value" in result.output diff --git a/tests/test_config_clone_service.py b/tests/test_config_clone_service.py new file mode 100644 index 00000000..97528028 --- /dev/null +++ b/tests/test_config_clone_service.py @@ -0,0 +1,630 @@ +"""Tests for ConfigService.clone_config (the `config clone` command, issue #587). + +Cloning a configuration by hand -- reading `config detail` and rebuilding the +body -- silently drops sibling keys of `parameters` (`runtime`, `storage`, +`authorization`). The reporter of #587 lost `runtime.parallelism` that way and +a 65-row writer ran sequentially for 140 minutes instead of ~60-90. + +Two paths, deliberately different: + +- **Same project**: the Storage API copies server-side + (`POST .../versions/{v}/create`), so nothing is rebuilt and encrypted + `KBC::` values stay valid. +- **Cross project**: we assemble it ourselves, because encrypted values are + scoped to their project and cannot travel. Those are detected and must be + re-supplied, or the clone is refused. +""" + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from helpers import setup_single_project +from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.services.config_service import ConfigService + +SOURCE_DETAIL = { + "id": "src-1", + "name": "Snowflake writer", + "description": "writes to prod", + "version": 7, + "configuration": { + "parameters": {"db": {"host": "h", "#password": "KBC::ProjectSecure::abc"}}, + "runtime": {"parallelism": "20"}, + "storage": {"input": {"tables": [{"source": "in.c-main.orders"}]}}, + }, + "rows": [ + {"id": "r1", "name": "orders", "configuration": {"parameters": {"table": "orders"}}}, + {"id": "r2", "name": "users", "configuration": {"parameters": {"table": "users"}}}, + ], +} + + +def _make_service( + tmp_config_dir: Path, + *, + detail: dict | None = None, +) -> tuple[ConfigService, MagicMock]: + """Build a ConfigService wired to a single mock Storage client.""" + store = setup_single_project(tmp_config_dir) + + client = MagicMock() + client.get_config_detail.return_value = dict(detail or SOURCE_DETAIL) + client.create_config_copy.return_value = {"id": "clone-1"} + client.create_config.return_value = {"id": "clone-1", "version": 1} + client.create_config_row.return_value = {"id": "row-new"} + client.update_config.return_value = {"id": "clone-1", "version": 2} + + service = ConfigService( + config_store=store, + client_factory=lambda url, token: client, + ) + return service, client + + +class TestCloneSameProject: + """Same-project clone delegates to the server-side copy endpoint.""" + + def test_uses_server_side_copy_at_the_source_version(self, tmp_config_dir: Path) -> None: + """The copy is taken from the source's CURRENT version, and nothing is + rebuilt client-side -- that is the whole point of the same-project path. + """ + service, client = _make_service(tmp_config_dir) + + result = service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="Snowflake writer (copy)", + ) + + client.create_config_copy.assert_called_once() + kwargs = client.create_config_copy.call_args.kwargs + assert kwargs["config_id"] == "src-1" + assert kwargs["version"] == 7 + assert kwargs["name"] == "Snowflake writer (copy)" + # Nothing is assembled by hand on this path. + client.create_config.assert_not_called() + assert result["id"] == "clone-1" + assert result["mode"] == "same-project" + + def test_encrypted_values_are_not_an_obstacle_within_a_project( + self, tmp_config_dir: Path + ) -> None: + """`KBC::` values stay valid in the same project, so a clone carrying + them must not be refused (the cross-project path is where they block). + """ + service, client = _make_service(tmp_config_dir) + + result = service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + ) + + assert result["encrypted_paths"] == [] + client.create_config_copy.assert_called_once() + + def test_dry_run_makes_no_write_call(self, tmp_config_dir: Path) -> None: + """Dry-run reports the plan; no copy, no create, no update.""" + service, client = _make_service(tmp_config_dir) + + result = service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + dry_run=True, + ) + + client.create_config_copy.assert_not_called() + client.create_config.assert_not_called() + client.update_config.assert_not_called() + assert result["dry_run"] is True + assert result["mode"] == "same-project" + assert result["source_version"] == 7 + + def test_set_overrides_are_applied_after_the_copy(self, tmp_config_dir: Path) -> None: + """--set edits land via a follow-up update on the NEW config, leaving + every key the copy brought along intact. + """ + service, client = _make_service(tmp_config_dir) + # The clone is re-read before patching, so return its (copied) body. + client.get_config_detail.side_effect = [ + dict(SOURCE_DETAIL), + {"id": "clone-1", "version": 1, "configuration": SOURCE_DETAIL["configuration"]}, + ] + + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + set_overrides={"parameters.db.host": "new-host"}, + ) + + client.update_config.assert_called_once() + patched = client.update_config.call_args.kwargs["configuration"] + assert patched["parameters"]["db"]["host"] == "new-host" + # The sibling that #587 is about survives the override step. + assert patched["runtime"] == {"parallelism": "20"} + + def test_no_overrides_means_no_update_call(self, tmp_config_dir: Path) -> None: + """Without --set the copy is already final; no pointless second write.""" + service, client = _make_service(tmp_config_dir) + + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + ) + + client.update_config.assert_not_called() + + +def _make_two_project_service( + tmp_config_dir: Path, + *, + detail: dict | None = None, +) -> tuple[ConfigService, MagicMock, MagicMock]: + """Build a service over two DISTINCT projects, returning (service, source, target).""" + from keboola_agent_cli.models import ProjectConfig + + store = setup_single_project(tmp_config_dir) + config = store.load() + config.projects["prod"].project_id = 100 + config.projects["dev"] = ProjectConfig( + stack_url="https://connection.keboola.com", + token="901-55555-fakeTestTokenDoNotUseXXXXXXXX", + project_name="Dev", + project_id=200, + ) + store.save(config) + + source_client = MagicMock() + source_client.get_config_detail.return_value = dict(detail or SOURCE_DETAIL) + target_client = MagicMock() + target_client.create_config.return_value = {"id": "clone-1"} + target_client.create_config_row.side_effect = [{"id": "new-r1"}, {"id": "new-r2"}] + clients = [source_client, target_client] + + def factory(url: str, token: str) -> MagicMock: + return clients.pop(0) if clients else target_client + + service = ConfigService(config_store=store, client_factory=factory) + return service, source_client, target_client + + +class TestCloneCrossProjectEncryptedValues: + """Ciphertext is project-scoped, so a cross-project clone must not carry it. + + Copying a `KBC::` value into another project produces a configuration that + looks complete and fails at runtime -- in a project the operator is not + watching. The clone is refused until each value is re-supplied. + """ + + def test_refuses_when_an_encrypted_value_was_not_resupplied(self, tmp_config_dir: Path) -> None: + service, _, target = _make_two_project_service(tmp_config_dir) + + with pytest.raises(ConfigError, match="encrypted value"): + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="dev", + ) + target.create_config.assert_not_called() + + def test_refusal_names_every_path_that_needs_a_value(self, tmp_config_dir: Path) -> None: + """The error must be actionable -- a path the caller can pass to --secret.""" + service, _, _ = _make_two_project_service(tmp_config_dir) + + with pytest.raises(ConfigError) as exc_info: + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="dev", + ) + assert "parameters.db.#password" in str(exc_info.value) + assert "--secret" in str(exc_info.value) + + def test_supplied_secret_is_written_and_encrypted_in_the_target( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The re-supplied plaintext replaces the ciphertext, and encryption is + scoped to the TARGET project -- encrypting against the source would + produce a value the target still cannot read. + """ + service, _, target = _make_two_project_service(tmp_config_dir) + seen: list[tuple[int | None, dict]] = [] + + def fake_encrypt(client, project, component_id, configuration, *, allow_plaintext_fallback): + seen.append((project.project_id, configuration)) + return configuration + + monkeypatch.setattr(service, "_encrypt_secrets_before_write", fake_encrypt) + + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="dev", + secret_overrides={"parameters.db.#password": "fresh-secret"}, + ) + + target.create_config.assert_called_once() + body = target.create_config.call_args.kwargs["configuration"] + assert body["parameters"]["db"]["#password"] == "fresh-secret" + # Every encryption call was scoped to the target project (id 200). + assert {project_id for project_id, _ in seen} == {200} + + def test_row_secret_is_written_into_that_row_not_the_parent(self, tmp_config_dir: Path) -> None: + """A re-supplied `rows[N].…` secret must land in that row's body. + + The detection reports row ciphertext with a `rows[N].` prefix and the + refusal check accepts a `--secret` at that exact path -- so if the + substitution then applies it to the PARENT body, the command reports + success while the copied row still carries the source project's + undecryptable ciphertext. That is precisely the outcome `config clone` + exists to prevent, and it would be discovered only at runtime, in the + other project. + """ + detail = dict(SOURCE_DETAIL) + detail["configuration"] = {"parameters": {"db": {"host": "h"}}} + detail["rows"] = [ + { + "id": "r1", + "name": "orders", + "configuration": {"parameters": {"#token": "KBC::ProjectSecure::xyz"}}, + } + ] + service, _, target = _make_two_project_service(tmp_config_dir, detail=detail) + + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="dev", + secret_overrides={"rows[0].parameters.#token": "fresh-row-token"}, + ) + + row_body = target.create_config_row.call_args.kwargs["configuration"] + assert row_body["parameters"]["#token"] == "fresh-row-token", row_body + + # And the parent must not have grown a bogus "rows[0]" key. + parent_body = target.create_config.call_args.kwargs["configuration"] + assert "rows[0]" not in parent_body, parent_body + assert parent_body == {"parameters": {"db": {"host": "h"}}}, parent_body + + def test_ciphertext_under_a_plain_key_is_refused_not_silently_leaked( + self, tmp_config_dir: Path + ) -> None: + """A ``KBC::`` value under a non-``#`` key has no encryption round-trip. + + Accepting a ``--secret`` for it would write the replacement to the + target project in plaintext, because the encrypt step only picks up + ``#``-prefixed keys. Refusing is the only outcome that does not either + break the clone or leak the credential. + """ + detail = dict(SOURCE_DETAIL) + detail["configuration"] = {"parameters": {"token": "KBC::ProjectSecure::plain"}} + detail["rows"] = [] + service, _, target = _make_two_project_service(tmp_config_dir, detail=detail) + + with pytest.raises(ConfigError, match="cannot re-encrypt"): + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="dev", + secret_overrides={"parameters.token": "fresh"}, + ) + target.create_config.assert_not_called() + + def test_rows_are_copied_into_the_new_configuration(self, tmp_config_dir: Path) -> None: + """Cross-project has no server-side copy, so rows are recreated by hand.""" + service, _, target = _make_two_project_service(tmp_config_dir) + + result = service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="dev", + secret_overrides={"parameters.db.#password": "fresh"}, + ) + + assert target.create_config_row.call_count == 2 + names = [c.kwargs["name"] for c in target.create_config_row.call_args_list] + assert names == ["orders", "users"] + assert result["copied_rows"] == [ + {"source_row_id": "r1", "id": "new-r1"}, + {"source_row_id": "r2", "id": "new-r2"}, + ] + + def test_sibling_keys_survive_a_cross_project_clone(self, tmp_config_dir: Path) -> None: + """The whole reason the command exists: runtime/storage must travel.""" + service, _, target = _make_two_project_service(tmp_config_dir) + + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="dev", + secret_overrides={"parameters.db.#password": "fresh"}, + ) + + body = target.create_config.call_args.kwargs["configuration"] + assert body["runtime"] == {"parallelism": "20"} + assert body["storage"] == {"input": {"tables": [{"source": "in.c-main.orders"}]}} + + def test_dry_run_reports_missing_secrets_instead_of_raising(self, tmp_config_dir: Path) -> None: + """--dry-run is how a caller discovers what --secret values to gather, + so it must report rather than refuse. + """ + service, _, target = _make_two_project_service(tmp_config_dir) + + result = service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="dev", + dry_run=True, + ) + + target.create_config.assert_not_called() + assert result["mode"] == "cross-project" + assert result["missing_secrets"] == ["parameters.db.#password"] + assert result["row_count"] == 2 + + def test_row_level_encrypted_values_are_detected_too(self, tmp_config_dir: Path) -> None: + """A row can hold its own secret; missing it breaks the clone just as + thoroughly, so rows are scanned as well as the parent body. + """ + detail = dict(SOURCE_DETAIL) + detail["rows"] = [ + { + "id": "r1", + "name": "orders", + "configuration": {"parameters": {"#token": "KBC::ProjectSecure::xyz"}}, + } + ] + service, _, _ = _make_two_project_service(tmp_config_dir, detail=detail) + + result = service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="dev", + dry_run=True, + ) + + assert "rows[0].parameters.#token" in result["missing_secrets"] + + +class TestCloneWriteSafety: + """Behaviour the two paths must share, because a flag that works on one and + silently does nothing on the other is worse than an unsupported flag. + """ + + def test_same_project_set_encrypts_hash_prefixed_values( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """`--set 'parameters.db.#password=…'` must not reach Storage in the clear. + + Repointing the copy at another database is the documented use case, so + a `#`-prefixed --set is expected traffic. Every other config write path + pre-encrypts through the Encryption API (issue #378, fail-closed); this + one bypassed it entirely, and version history would keep the plaintext. + """ + service, client = _make_service(tmp_config_dir) + client.get_config_detail.side_effect = [ + dict(SOURCE_DETAIL), + {"id": "clone-1", "version": 1, "configuration": {"parameters": {"db": {}}}}, + ] + encrypted: list[dict] = [] + + def fake_encrypt(cl, project, component_id, configuration, *, allow_plaintext_fallback): + encrypted.append(configuration) + return {"encrypted": True} + + monkeypatch.setattr(service, "_encrypt_secrets_before_write", fake_encrypt) + + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + set_overrides={"parameters.db.#password": "plaintext"}, + ) + + assert encrypted, "the patched body was never sent through encryption" + assert client.update_config.call_args.kwargs["configuration"] == {"encrypted": True} + + def test_cross_project_inherits_the_source_description(self, tmp_config_dir: Path) -> None: + """Same-project omits the field so the API copies it; cross-project has + to do that itself or the copy comes out blank. + """ + service, _, target = _make_two_project_service(tmp_config_dir) + + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="dev", + secret_overrides={"parameters.db.#password": "fresh"}, + ) + + assert target.create_config.call_args.kwargs["description"] == "writes to prod" + + def test_explicit_description_still_wins(self, tmp_config_dir: Path) -> None: + service, _, target = _make_two_project_service(tmp_config_dir) + + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + description="my own", + target_alias="dev", + secret_overrides={"parameters.db.#password": "fresh"}, + ) + + assert target.create_config.call_args.kwargs["description"] == "my own" + + def test_same_project_rejects_a_differing_target_branch(self, tmp_config_dir: Path) -> None: + """The server-side copy writes into the SOURCE branch, so a different + --target-branch cannot be honoured. Silently writing to the wrong + branch is the one outcome that must not happen. + """ + service, client = _make_service(tmp_config_dir) + + with pytest.raises(ConfigError, match="--target-branch"): + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + branch_id=100, + target_branch_id=200, + ) + client.create_config_copy.assert_not_called() + + def test_same_project_accepts_a_matching_target_branch(self, tmp_config_dir: Path) -> None: + """Passing the same branch on both sides is redundant, not wrong.""" + service, client = _make_service(tmp_config_dir) + + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + branch_id=100, + target_branch_id=100, + ) + + assert client.create_config_copy.call_args.kwargs["branch_id"] == 100 + + def test_partial_cross_project_clone_reports_what_landed(self, tmp_config_dir: Path) -> None: + """A row failing mid-copy leaves a half-populated configuration behind. + + The caller cannot clean up what it cannot name, so the error has to + carry the created configuration id and how many rows made it. + """ + service, _, target = _make_two_project_service(tmp_config_dir) + target.create_config_row.side_effect = [ + {"id": "new-r1"}, + KeboolaApiError(message="boom", error_code="API_ERROR", status_code=500), + ] + + with pytest.raises(KeboolaApiError) as exc_info: + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="dev", + secret_overrides={"parameters.db.#password": "fresh"}, + ) + + message = str(exc_info.value) + assert "clone-1" in message, message + assert "1 of 2" in message, message + + +class TestCloneTargetsTheRightClient: + """The client used for writes must agree with the same/cross decision. + + If the service decided "same project" (reuse the source client) while the + clone logic decided "cross project", the cross-project write would go to + the SOURCE project -- creating a configuration in the wrong place while + reporting success for the target. + """ + + def test_two_aliases_without_project_ids_do_not_share_a_client( + self, tmp_config_dir: Path + ) -> None: + """An unrecorded project_id must not be treated as "same project".""" + from keboola_agent_cli.models import ProjectConfig + + store = setup_single_project(tmp_config_dir) + config = store.load() + config.projects["prod"].project_id = None + config.projects["other"] = ProjectConfig( + stack_url="https://connection.keboola.com", + token="901-55555-fakeTestTokenDoNotUseXXXXXXXX", + project_name="Other", + project_id=None, + ) + store.save(config) + + source_client = MagicMock() + source_client.get_config_detail.return_value = dict(SOURCE_DETAIL) + target_client = MagicMock() + target_client.create_config.return_value = {"id": "clone-1"} + target_client.create_config_row.return_value = {"id": "row-new"} + handed_out: list[MagicMock] = [] + + def factory(url: str, token: str) -> MagicMock: + client = source_client if not handed_out else target_client + handed_out.append(client) + return client + + service = ConfigService(config_store=store, client_factory=factory) + + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="other", + # The source body holds one KBC:: value; supply it so the clone + # is not refused and we can observe WHERE it gets written. + secret_overrides={"parameters.db.#password": "fresh"}, + ) + + # Two distinct clients were requested, and the write went to the target. + assert len(handed_out) == 2 + target_client.create_config.assert_called_once() + source_client.create_config.assert_not_called() + source_client.create_config_copy.assert_not_called() + + +class TestCloneRejectsUnknownProjects: + def test_unknown_source_alias_raises(self, tmp_config_dir: Path) -> None: + service, client = _make_service(tmp_config_dir) + + with pytest.raises(ConfigError): + service.clone_config( + alias="nope", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + ) + client.create_config_copy.assert_not_called() + + def test_unknown_target_alias_raises(self, tmp_config_dir: Path) -> None: + service, client = _make_service(tmp_config_dir) + + with pytest.raises(ConfigError): + service.clone_config( + alias="prod", + component_id="keboola.wr-db-snowflake", + config_id="src-1", + name="copy", + target_alias="nope", + ) + client.create_config_copy.assert_not_called() diff --git a/tests/test_config_row_cli.py b/tests/test_config_row_cli.py index 6f6082de..8af879fa 100644 --- a/tests/test_config_row_cli.py +++ b/tests/test_config_row_cli.py @@ -515,7 +515,7 @@ def test_returns_url_json(self, tmp_config_dir: Path) -> None: with pytest.MonkeyPatch.context() as mp: mp.setattr( - "keboola_agent_cli.commands.config.get_service", + "keboola_agent_cli.commands._config_oauth.get_service", lambda ctx, name: service, ) result = _invoke( @@ -561,7 +561,7 @@ def test_api_error_exits_nonzero(self, tmp_config_dir: Path) -> None: with pytest.MonkeyPatch.context() as mp: mp.setattr( - "keboola_agent_cli.commands.config.get_service", + "keboola_agent_cli.commands._config_oauth.get_service", lambda ctx, name: service, ) result = _invoke( @@ -620,7 +620,7 @@ def test_redirect_url_propagates(self, tmp_config_dir: Path) -> None: with pytest.MonkeyPatch.context() as mp: mp.setattr( - "keboola_agent_cli.commands.config.get_service", + "keboola_agent_cli.commands._config_oauth.get_service", lambda ctx, name: service, ) result = _invoke( @@ -758,7 +758,7 @@ def test_non_master_token_exits_3(self, tmp_config_dir: Path) -> None: with pytest.MonkeyPatch.context() as mp: mp.setattr( - "keboola_agent_cli.commands.config.get_service", + "keboola_agent_cli.commands._config_oauth.get_service", lambda ctx, name: service, ) result = _invoke( diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 42d5b0e2..ee413f64 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -543,6 +543,9 @@ def test_full_cli_e2e(self) -> None: _step("19c", "config new --push validation", "real schema vs real body (#587)") self._test_config_new_push_schema_validation() + _step("19d", "config clone", "whole-config duplicate incl. rows (#587)") + self._test_config_clone() + # ============================================================== # PHASE 5: Component commands # ============================================================== @@ -2279,6 +2282,130 @@ def _test_config_new_push_schema_validation(self) -> None: # which pointed the reader at the wrong level of their own config. assert all(e.startswith("parameters") for e in invalid["validation_errors"]), invalid + def _test_config_clone(self) -> None: + """Test ``config clone`` -- whole-configuration duplicate (0.84.2+, #587). + + The point of the command is that NOTHING is left behind, so the + assertions are about completeness, not about the happy path: the + source is given a `runtime` sibling and two rows, and the clone must + come back with both. `runtime.parallelism` is exactly what went + missing in #587, and rows are what makes hand-rebuilding hopeless at + 65 of them. + + Everything is created and deleted inside this test. + """ + clone_name = f"{RUN_ID} clone-source" + source = self._run_ok( + "config", + "new", + "--component-id", + "keboola.ex-http", + "--project", + self.alias, + "--name", + clone_name, + "--push", + "--no-files", + )["data"] + source_id = str(source["id"]) + self._created_config_ids.append(("keboola.ex-http", source_id)) + + clone_id: str | None = None + try: + # Give the source a sibling key and rows -- the things that get lost. + self._run_ok( + "config", + "update", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + source_id, + "--set", + "runtime.parallelism=20", + ) + for row_name in ("alpha", "beta"): + self._run_ok( + "config", + "row-create", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + source_id, + "--name", + f"row-{row_name}", + "--configuration", + json.dumps({"parameters": {"table": row_name}}), + ) + + # Dry-run first: reports the plan, writes nothing. + planned = self._run_ok( + "config", + "clone", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + source_id, + "--name", + f"{RUN_ID} clone-dry", + "--dry-run", + )["data"] + assert planned["dry_run"] is True, planned + assert planned["mode"] == "same-project", planned + assert planned["row_count"] == 2, planned + + cloned = self._run_ok( + "config", + "clone", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + source_id, + "--name", + f"{RUN_ID} clone-target", + )["data"] + clone_id = str(cloned["id"]) + self._created_config_ids.append(("keboola.ex-http", clone_id)) + assert clone_id != source_id, cloned + assert cloned["mode"] == "same-project", cloned + + detail = self._run_ok( + "config", + "detail", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + clone_id, + )["data"] + configuration = detail.get("configuration") or {} + # The sibling key that #587 is about survived the copy. + assert configuration.get("runtime") == {"parallelism": 20}, configuration + # And so did the rows -- no client-side row copying involved. + rows = detail.get("rows") or [] + assert len(rows) == 2, rows + assert sorted(r["name"] for r in rows) == ["row-alpha", "row-beta"], rows + finally: + for config_id in filter(None, (clone_id, source_id)): + self._run_ok( + "config", + "delete", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + config_id, + ) + def _test_component_commands(self) -> None: """List components and get detail for one. diff --git a/tests/test_encryption.py b/tests/test_encryption.py index 046c68b1..7a8b345f 100644 --- a/tests/test_encryption.py +++ b/tests/test_encryption.py @@ -12,11 +12,14 @@ import pytest from keboola_agent_cli.errors import KeboolaApiError +from keboola_agent_cli.json_utils import set_nested_value from keboola_agent_cli.services._encryption import ( apply_encrypted, apply_encrypted_to_local, collect_secrets, encrypt_secrets_in_config, + find_encrypted_secret_paths, + find_unencryptable_secret_paths, ) @@ -251,3 +254,47 @@ def test_noop_when_all_already_encrypted(self) -> None: ) client.encrypt_values.assert_not_called() + + +class TestFindEncryptedSecretPaths: + """Paths are shown to a human and fed back through ``--secret PATH=VALUE``, + so they must be in a form the substitution step actually accepts. + """ + + def test_dict_paths_are_dotted(self) -> None: + config = {"parameters": {"db": {"#password": "KBC::ProjectSecure::abc"}}} + + assert find_encrypted_secret_paths(config) == ["parameters.db.#password"] + + def test_list_element_paths_use_a_plain_index_segment(self) -> None: + """`keboola.variables` hoists secrets into a list of {name, value}. + + The reported path has to round-trip through ``set_nested_value``: a + bracketed ``[0]`` segment raises ValueError there, so the operator is + told to supply a path that then crashes the command. + """ + config = {"parameters": {"values": [{"name": "#token", "value": "KBC::ProjectSecure::x"}]}} + + paths = find_encrypted_secret_paths(config) + + assert paths == ["parameters.values.0.value"] + # The contract that matters: the path can actually be applied. + patched = set_nested_value(config, paths[0], "fresh") + assert patched["parameters"]["values"][0]["value"] == "fresh" + + def test_plain_key_ciphertext_is_reported_separately(self) -> None: + """Ciphertext under a non-``#`` key cannot be re-encrypted. + + ``collect_secrets`` only picks up ``#``-prefixed keys, so a plaintext + substituted at such a path would be written to Storage as-is. These + paths are returned apart so the caller can refuse rather than leak. + """ + config = { + "parameters": { + "#password": "KBC::ProjectSecure::a", + "token": "KBC::ProjectSecure::b", + } + } + + assert find_encrypted_secret_paths(config) == ["parameters.#password"] + assert find_unencryptable_secret_paths(config) == ["parameters.token"]