diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index a1c8206e..24221e05 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "kbagent", - "version": "0.43.8", + "version": "0.43.9", "source": "./plugins/kbagent", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "category": "development" diff --git a/.gitignore b/.gitignore index 248b9b1f..f1ee3aea 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,9 @@ ENV/ .claude/ .audit/ +# Browser-automation MCP scratch (console logs, page snapshots) +.playwright-mcp/ + # Testing .pytest_cache/ .coverage @@ -55,6 +58,7 @@ Thumbs.db tmp/ # web/ Node + React build artifacts +/node_modules/ web/backend/node_modules/ web/backend/dist/ web/frontend/node_modules/ diff --git a/CLAUDE.md b/CLAUDE.md index d099c3d0..b83def7a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -418,8 +418,8 @@ kbagent data-app password --project NAME --app-id ID kbagent data-app logs --project NAME --app-id ID [--lines N] [--since ISO8601] kbagent data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE' [--secret ...] [--secrets-file PATH] [--branch ID] [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next] kbagent data-app secrets-list --project ALIAS --app-id ID [--branch ID] [--show-fingerprint] -kbagent data-app secrets-get --project ALIAS --app-id ID --key '#KEY' [--branch ID] -kbagent data-app secrets-remove --project ALIAS --app-id ID --key '#KEY' [--key ...] [--branch ID] [--yes] [--dry-run] +kbagent data-app secrets-get --project ALIAS --app-id ID --key 'KEY' [--branch ID] # '#' optional; plain values return their value, encrypted return metadata only +kbagent data-app secrets-remove --project ALIAS --app-id ID --key 'KEY' [--key ...] [--branch ID] [--yes] [--dry-run] # '#' optional kbagent data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] [--type python-js] [--strict] kbagent component list [--project NAME] [--type TYPE] [--query QUERY] diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index 0e2aa791..84f10907 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.43.8", + "version": "0.43.9", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 969f25ad..01c84720 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -155,7 +155,7 @@ a critical failure. | Update a config row | `kbagent config row-update --project P --component-id C --config-id K --row-id R [--name N] [--configuration JSON]` (0.30.0+) | `tool call update_config_row` | `PUT /v2/storage/components/C/configs/K/rows/R` (raw REST) | | Delete a config row | `kbagent config row-delete --project P --component-id C --config-id K --row-id R [--yes]` (0.30.0+) -- destructive (gated behind `--allow-destructive`); branch-aware | `tool call delete_config_row` | `DELETE /v2/storage/components/C/configs/K/rows/R` (raw REST) | | Get OAuth authorization URL | `kbagent config oauth-url --project P --component-id C --config-id K` (0.30.0+) -- returns URL to open in browser to complete OAuth flow | -- | raw `GET /v2/storage/components/C/configs/K/oauth/authorize` | -| Inventory data apps | `kbagent data-app list --project P` (0.27.0+) | `tool call get_configs --component_id keboola.data-apps` (Storage view only -- no state, no URL, no configVersion) | iterating `tool call` per project to reconstruct the join with the Data Science index | +| Inventory data apps | `kbagent data-app list --project P` (0.27.0+; 0.43.9+ hides `keboola.sandboxes` workspaces) | `tool call get_configs --component_id keboola.data-apps` (Storage view only -- no state/URL/configVersion) | iterating `tool call` per project to reconstruct the join with the Data Science index | | Bring a new data app online from a git repo | `kbagent data-app create --project P --name N --slug S --git-repo URL [--git-pat-env VAR \| --git-public]` (0.27.0+) | broken into `tool call create_config keboola.data-apps` + manual `kbagent encrypt values` + raw `POST /apps` -- ONLY if you need a custom shape kbagent doesn't support | raw `POST data-science/apps` followed by `PATCH desiredState=running` without `configVersion + restartIfRunning` (the §9 footgun -- pins to v2 empty shell, runner errors `dataApp.git.repository is required in /data/config.json`) | | Roll out a new code or config version on a data app | `kbagent data-app deploy --project P --app-id N --wait` (0.27.0+) -- always sends the §9 trio | `kbagent --hint client data-app deploy ...` to inspect the generated `patch_app(desired_state=, config_version=, restart_if_running=True)` call | `tool call update_config` then `tool call run_component` (data apps are not jobs -- the queue runner does not deploy them) | | Wake an auto-suspended data app | `kbagent data-app start --project P --app-id N` (0.27.0+) -- does NOT bump configVersion | hitting the app's URL (auto-restart triggers a 30-60s cold boot) | `kbagent data-app deploy` (overkill -- bumps the deployed configVersion unnecessarily) | @@ -169,10 +169,10 @@ a critical failure. | Cancel a pending invitation | `kbagent project invitation-cancel --project P --email E --yes` (0.29.0+) | `--invitation-id ID` if email lookup is ambiguous | DELETE via raw HTTP without going through the service layer | | Remove an active member | `kbagent project member-remove --project P --email E --yes` (0.29.0+, **destructive**) | `--hint client` for a script that removes by user_id directly | calling `member-remove` without `--yes` in non-interactive contexts (it will prompt and hang) | | Change a member's role | `kbagent project member-set-role --project P --email E --role admin\|guest\|readOnly\|share` (0.29.0+) | -- | `PUT /manage/projects/{id}/users/{userId}` -- the API rejects PUT with 404, the kbagent client correctly uses **PATCH** | -| Set / rotate app-runtime secrets | `kbagent data-app secrets-set --project P --app-id N --secret '#KEY=VAL'` (0.29.0+) then `data-app deploy --wait` -- per-project KMS encryption, fail-closed, never auto-deploys | `kbagent encrypt values --component-id keboola.data-apps` + `tool call update_config` -- ONLY if you need to write secrets to a different shape than `parameters.dataApp.secrets` | raw `POST` to encryption + Storage without read-modify-write -- you will clobber sibling keys nested under `parameters.dataApp.secrets` (Storage `merge=True` is shallow at the top level only) | -| Inspect what secrets are set on a data app | `kbagent data-app secrets-list --project P --app-id N` (0.29.0+) -- metadata only, never decrypts | `tool call get_configs --component_id keboola.data-apps` then read `parameters.dataApp.secrets` keys (raw dict, no env-var derivation, may leak ciphertext into output) | trying to decrypt -- the Encryption API has no decrypt endpoint, the CLI cannot decrypt under any branch | -| Confirm one secret is present | `kbagent data-app secrets-get --project P --app-id N --key '#KEY'` (0.29.0+) -- returns metadata only | -- | trying to extract the plaintext value (impossible by design; not a CLI gap) | -| Remove a secret from a data app | `kbagent data-app secrets-remove --project P --app-id N --key '#KEY' --yes` (0.29.0+) -- idempotent; missing keys exit 0 with `removed: 0` | `tool call update_config` with the secrets sub-dict deleted -- ONLY for batch removes that need a custom change description | `kbagent config update --set 'parameters.dataApp.secrets={}'` -- replaces the whole sub-dict, dropping every secret instead of just the named ones | +| Set / rotate app-runtime secrets | `kbagent data-app secrets-set --project P --app-id N --secret '#KEY=VAL'` (0.29.0+) then `data-app deploy --wait` -- per-project KMS encryption, fail-closed, never auto-deploys | `encrypt values --component-id keboola.data-apps` + `tool call update_config` -- ONLY for a non-standard secrets shape | raw `POST` to encryption + Storage without read-modify-write -- clobbers sibling keys (Storage `merge=True` is shallow) | +| Inspect what secrets are set on a data app | `kbagent data-app secrets-list --project P --app-id N` (0.29.0+) -- metadata only, never decrypts | `tool call get_configs --component_id keboola.data-apps` then read `parameters.dataApp.secrets` keys (raw dict, may leak ciphertext) | trying to decrypt -- the Encryption API has no decrypt endpoint | +| Read one secret / env-var key | `kbagent data-app secrets-get --project P --app-id N --key 'KEY'` (0.43.9+: `#` optional) -- ENCRYPTED -> metadata only (`value: null`); PLAIN -> literal value (`encrypted: false`) | -- | `--key '#KEY'` when stored without `#` (exact-match -> NOT_FOUND); trying to decrypt an encrypted secret | +| Remove a secret / env-var key from a data app | `kbagent data-app secrets-remove --project P --app-id N --key 'KEY' --yes` (0.43.9+: `#` optional; removes encrypted + plain) -- idempotent; missing keys exit 0, `removed: 0` | `tool call update_config` with the secrets sub-dict deleted -- ONLY for batch removes needing a custom change description | `config update --set 'parameters.dataApp.secrets={}'` -- drops EVERY secret, not just the named ones | | Pre-flight a data-app repo before create | `kbagent data-app validate-repo --git-repo URL --type python-js [--git-pat-env VAR]` (0.29.0+) -- BLOCKING / WARN / OK with help-doc citations; ≤5 GitHub API calls regardless of repo size | git-clone the repo locally and inspect by hand | `data-app create --dry-run` (only shows the request bodies; does not validate repo structure) | | Rename a project alias | `kbagent project edit --project OLD --new-alias NEW [--dry-run]` (0.31.0+) -- cascades through `config.json` (`projects` key + `default_project`) and the nested-sync directory `//`. Combined with `--url`/`--token` in one call, those mutations target the new alias post-rename. `--dry-run` previews collision detection, planned disk-rename method, and the lineage-cache warning without mutating state. **Lineage cache (if any) is NOT auto-updated**: rebuild via `kbagent lineage build` after the rename | `kbagent project remove` + `kbagent project add` (re-enters the token; loses any nested sync workspace) | hand-editing `~/.config/keboola-agent-cli/config.json` (no validation, easy to miss `default_project` cascade) | | Call the running `kbagent serve` from a scheduled-agent subprocess | `kbagent http get/post/patch/delete ` (0.40.0+) -- uses `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` env vars auto-injected by the scheduler. `kbagent http get /openapi.json` to discover endpoints. Treats the live serve as source-of-truth (no stale local config) | forking `kbagent ` (also fine -- `KBAGENT_CONFIG_DIR` is propagated so the spawned CLI sees the SAME config the serve uses; no more "I'm in the wrong directory" surprises) | `curl $KBAGENT_SERVE_URL/...` by hand (works, but `kbagent http` adds auth header automatically, structured error mapping, and JSON-mode formatting) | @@ -417,19 +417,18 @@ success, not a failure. JumpCloud / Auth0) are not yet exposed by the CLI; tracked as a follow-up issue. -- **`data-app secrets-* metadata-only`** (0.29.0+): `secrets-get` NEVER - echoes the decrypted plaintext under any branch -- the Encryption API - is one-way and the CLI does not attempt to decrypt. NOT_FOUND on an - absent key never enumerates sibling keys (avoids leaking neighbour - presence). `secrets-remove` is idempotent: removing a non-existent key - returns exit 0 with `removed: 0` and does NOT bump the Storage - version. Setting a key whose derived env-var name collides with the - runtime-injected set (`KBC_TOKEN`, `KBC_URL` for sure; more TODO) is - silently shadowed by the platform; the CLI emits a stderr WARN and - surfaces `shadowed_by_runtime` in JSON envelope -- the WRITE still - happens. Read-modify-write is at the SERVICE layer (Storage `merge=True` - is shallow at the top level only and would clobber siblings nested in - `parameters.dataApp.secrets`). +- **`data-app secrets-*`** (0.29.0+; plain-key reads since 0.43.9): + `secrets-get`/`secrets-remove` take keys with or without `#` (the block + holds encrypted secrets AND plain env vars; `secrets-set` still needs + `#` -- it encrypts). `secrets-get` on an ENCRYPTED key NEVER echoes the + decrypted plaintext (one-way Encryption API); a PLAIN key returns its + value (`encrypted: false`, already visible via `config detail`). + NOT_FOUND never enumerates siblings. `secrets-remove` is idempotent: + removing a non-existent key returns exit 0 with `removed: 0` and does + NOT bump the Storage version. A key whose derived env-var name collides + with the runtime-injected set (`KBC_TOKEN`, `KBC_URL`) is silently + shadowed by the platform; the CLI WARNs to stderr and sets + `shadowed_by_runtime` -- the WRITE still happens. - **`data-app validate-repo` is GitHub-only**, `--type python-js` only (0.29.0+): pre-flight Golden-Rule check via the GitHub Trees+Contents diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 24657b55..1f031518 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -154,7 +154,7 @@ When working inside a git repository or project directory, run `kbagent init` (o | Tail the container logs for a deployed data app | `kbagent data-app logs --project PROJECT --app-id APP-ID` | | Encrypt and write app-runtime secrets to the linked Storage config | `kbagent data-app secrets-set --project PROJECT --app-id APP-ID` | | List the keys in parameters.dataApp.secrets, with derived runtime env-var names | `kbagent data-app secrets-list --project PROJECT --app-id APP-ID` | -| Show metadata for ONE secret key. | `kbagent data-app secrets-get --project PROJECT --app-id APP-ID --key KEY` | +| Show ONE key from parameters.dataApp.secrets | `kbagent data-app secrets-get --project PROJECT --app-id APP-ID --key KEY` | | Remove one or more app-runtime secrets. | `kbagent data-app secrets-remove --project PROJECT --app-id APP-ID --key KEY` | | Pre-flight check that a git repo follows the Keboola data-app Golden Rule | `kbagent data-app validate-repo --git-repo GIT-REPO` | | List jobs from connected projects | `kbagent job list` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 4dc72fe6..91adf783 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -140,7 +140,7 @@ All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the ## Data Apps (Streamlit / Flask / Node deployments) Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, encrypted secrets, runtime size) with Data Science API (`/apps` -- deployment record, state, URL, configVersion). The CLI encapsulates the §9 redeploy contract so callers cannot pin to the empty-shell v2; see `data-app-workflow.md` for the gotcha inventory and recipes. Since v0.33.0 the JSON output envelope's data-app id key is `app_id` (renamed from bare `id` for symmetry with the `--app-id` input flag); `config_id` is unchanged. -- `data-app list [--project NAME ...] [--branch ID]` -- list apps across projects (Data Science index merged with Storage names) +- `data-app list [--project NAME ...] [--branch ID]` -- list data apps across projects (Data Science index merged with Storage names). Since v0.43.9 filters out workspace/sandbox deployments (`componentId=keboola.sandboxes`, `type=snowflake`/`bigquery`) that the Data Science `/apps` collection also returns, so the listing matches the Apps UI. Envelope carries `component_id` per app. - `data-app detail --project NAME --app-id ID [--branch ID]` -- merged view (state, desired, url, configVersion, slug, git block with PAT redacted) - `data-app create --project ALIAS --name NAME --slug SLUG --git-repo URL [--git-public/--no-git-public] [--git-username USER] [--git-pat-env VAR | --git-pat-file PATH | --git-pat-encrypted KBC::Project...] [--auth password|public] [--size tiny|small|medium|large] [--auto-suspend SECONDS] [--type python-js|python|streamlit|r|...] [--branch ID] [--no-deploy] [--wait] [--timeout SECONDS] [--keep-on-failure] [--dry-run]` -- POST shell + encrypt PAT + PUT Storage config (with auto-injected `parameters.id`) + PATCH deploy with the §9 trio. Cleanup-in-finally on failure unless `--keep-on-failure`. Default `--auth password` mints a 20-char hex simpleAuth password (retrievable via `data-app password`). - `data-app deploy --project NAME --app-id ID [--config-version N] [--wait] [--timeout SECONDS] [--branch ID]` -- the §9 redeploy contract. Default reads latest Storage version; `--config-version` pins an older version (rollback). @@ -151,8 +151,8 @@ Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, - `data-app logs --project NAME --app-id ID [--lines N] [--since ISO8601]` -- tail container logs (Data Science `/apps/{id}/logs/tail`). Plain-text body covering the full spin-up trace ([TIMING] git_clone, Cloning into /app, uv install, supervisord, runtime stack traces). Default `--lines 500`; pass `--lines 0` for the full current buffer (no server-side cap). `--lines` and `--since` are mutually exclusive on the server; `--since` requires a timezone (Z or +00:00). App must be running or recently-stopped — never-started apps return 400 "App X is not running" (recover with `data-app start` or `data-app deploy`). Closes the upstream `keboola-mcp-server` gap where `get_data_apps` hardcodes a 20-line cap; this CLI surface is unconstrained. The log buffer can echo runtime secrets the app printed to stdout/stderr — consider hygiene before piping `--json` output into AI agent context. - `data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE' [--secret ...] [--secrets-file PATH] [--branch ID] [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next]` -- encrypt and write `#`-prefixed secrets to `parameters.dataApp.secrets`. Per-project KMS encryption, fail-closed. Read-modify-write at the service layer (NOT Storage `merge=True` -- shallow). Runtime exposes each key as an env var with `#` stripped, `-` -> `_`, uppercased. Adding bumps the Storage version; the running container keeps the OLD config until the next `data-app deploy`. - `data-app secrets-list --project ALIAS --app-id ID [--branch ID] [--show-fingerprint]` -- list secret keys + derived runtime env-var names. Never echoes encrypted ciphertext in full. `--show-fingerprint` opt-in for a short ciphertext fingerprint. -- `data-app secrets-get --project ALIAS --app-id ID --key '#KEY' [--branch ID]` -- show metadata for ONE secret. NEVER echoes the decrypted value (Encryption API is one-way). NOT_FOUND on absent key; never enumerates siblings. -- `data-app secrets-remove --project ALIAS --app-id ID --key '#KEY' [--key ...] [--branch ID] [--yes] [--dry-run]` -- destructive (can break a running app at next deploy). Idempotent: missing keys exit 0 with `removed: 0`. +- `data-app secrets-get --project ALIAS --app-id ID --key 'KEY' [--branch ID]` -- show ONE key from `parameters.dataApp.secrets`. The leading `#` is OPTIONAL (since v0.43.9); the block holds both encrypted secrets (`#`) and plain unencrypted env-var values, and `secrets-list` enumerates both. For an ENCRYPTED secret it stays metadata-only (`encrypted: true`, `value: null`, fingerprint/prefix) -- the decrypted plaintext is NEVER echoed (Encryption API is one-way). For a PLAIN value it returns the literal value (`encrypted: false`), which is already visible via `config detail`. NOT_FOUND on absent key (exact match, no `#KEY`<->`KEY` fuzzing); never enumerates siblings. +- `data-app secrets-remove --project ALIAS --app-id ID --key 'KEY' [--key ...] [--branch ID] [--yes] [--dry-run]` -- destructive (can break a running app at next deploy). Leading `#` OPTIONAL (since v0.43.9): removes both encrypted secrets and plain env-var keys. Idempotent: missing keys exit 0 with `removed: 0`. - `data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] [--type python-js] [--strict]` -- pre-flight Golden-Rule check for a data-app git repo (https://help.keboola.com/data-apps/python-js/). GitHub-only; ≤5 API calls (1 tree + ≤4 contents) regardless of repo size. `--type` restricted to `python-js` in 0.28.0; streamlit / pure-Python / R / Node-only follow-up. `--strict` treats WARNs as failures. ## MCP Tools diff --git a/plugins/kbagent/skills/kbagent/references/data-app-workflow.md b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md index f0c4a376..03993c29 100644 --- a/plugins/kbagent/skills/kbagent/references/data-app-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md @@ -151,17 +151,21 @@ kbagent --json data-app secrets-set \ # exact command; suppress it with --no-hint-next for scripted callers. kbagent data-app deploy --project prod --app-id 12345678 --wait -# Inspect what's set without echoing the encrypted ciphertext: +# Inspect what's set -- lists BOTH encrypted (#) secrets and plain env-var +# values; never echoes the encrypted ciphertext in full: kbagent data-app secrets-list --project prod --app-id 12345678 # -> #ANTHROPIC_API_KEY -> env ANTHROPIC_API_KEY -# -> #my-database-url -> env MY_DATABASE_URL +# -> #my-database-url -> env MY_DATABASE_URL +# -> ADMIN_EMAILS -> env ADMIN_EMAILS (plain, unencrypted) -# Confirm presence of one key (NEVER decrypts): -kbagent data-app secrets-get --project prod --app-id 12345678 --key '#ANTHROPIC_API_KEY' +# Read one key. Leading '#' is OPTIONAL (since v0.43.9). Encrypted secret +# -> metadata only (NEVER decrypts). Plain value -> the literal value: +kbagent data-app secrets-get --project prod --app-id 12345678 --key '#ANTHROPIC_API_KEY' # metadata only +kbagent data-app secrets-get --project prod --app-id 12345678 --key ADMIN_EMAILS # shows value -# Remove (idempotent -- absent keys exit 0 with removed=0): +# Remove (idempotent -- absent keys exit 0 with removed=0). '#' optional: kbagent data-app secrets-remove --project prod --app-id 12345678 \ - --key '#my-database-url' --yes + --key '#my-database-url' --key ADMIN_EMAILS --yes ``` The runtime exposes each secret as an env var with `#` stripped, `-` @@ -221,9 +225,9 @@ yours at runtime. | Pause a running app temporarily | `data-app stop --app-id N` | | Read the simpleAuth password | `data-app password --app-id N` (needs Manage token) | | Set or rotate app-runtime secrets | `data-app secrets-set --app-id N --secret '#KEY=VAL'` then `data-app deploy --wait` | -| Inspect what secrets are set | `data-app secrets-list --app-id N` (metadata only, never decrypts) | -| Confirm one secret is present | `data-app secrets-get --app-id N --key '#KEY'` (metadata only) | -| Remove a secret | `data-app secrets-remove --app-id N --key '#KEY' --yes` (idempotent) | +| Inspect what's set (secrets + plain env vars) | `data-app secrets-list --app-id N` (metadata only, never decrypts) | +| Read one key | `data-app secrets-get --app-id N --key KEY` (`#` optional; encrypted → metadata only, plain → value) | +| Remove a key | `data-app secrets-remove --app-id N --key KEY --yes` (`#` optional; idempotent) | | Pre-flight a repo before create | `data-app validate-repo --git-repo URL` (GitHub-only, python-js for now) | | Triage a stuck deploy or runtime crash | `data-app logs --app-id N [--lines N \| --since ISO8601]` (since 0.43.8; plain-text container log tail, default `--lines 500`, `--lines 0` for full buffer) | | Tear it all down | `data-app delete --app-id N` (cascades to Storage config) | diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 24acb614..d8fae544 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -735,15 +735,39 @@ events and emits a final `done` SSE frame mirroring the same record. `parameters.dataApp` -- slug, git block, id back-pointer, `parameters` itself, and the top-level `runtime`/`authorization`/`storage`) is preserved bit-identical. +- **`data-app list` hides workspace/sandbox deployments (since v0.43.9).** + The Data Science `GET /apps` collection returns EVERY deployment in the + project, not just data apps -- interactive Snowflake/BigQuery + workspaces (`componentId=keboola.sandboxes`, `type=snowflake`/`bigquery`, + no name, a `*.snowflakecomputing.com` URL) live in the same collection. + Before 0.43.9 they showed up as phantom unnamed `(snowflake)` rows that + do NOT appear in the Apps UI. The command now keeps only + `componentId == keboola.data-apps` (items missing `componentId` are kept + defensively); the JSON envelope carries `component_id` per app. - **`secrets-remove` is idempotent.** Removing a key that isn't set is exit 0 with `removed: 0`, `not_found: []`. The Storage version is not bumped on a no-op. Do NOT script around this with a precondition lookup -- the idempotent path is the contract. -- **`secrets-get` NEVER echoes the decrypted plaintext.** The Encryption - API has no decrypt endpoint; the CLI cannot decrypt under any branch. - The command returns metadata only -- key name, derived env-var name, - ciphertext fingerprint, encryption prefix, presence flag. NOT_FOUND on - an absent key never enumerates sibling keys. +- **`secrets-get` NEVER echoes the decrypted plaintext of an ENCRYPTED + (`#` / `KBC::`) secret.** The Encryption API has no decrypt endpoint; + the CLI cannot decrypt under any branch. For an encrypted secret the + command returns metadata only -- key name, derived env-var name, + ciphertext fingerprint, encryption prefix, `encrypted: true`, + `value: null`. NOT_FOUND on an absent key never enumerates sibling + keys. (Plain unencrypted values ARE returned in full -- see next entry.) +- **`secrets-get` / `secrets-remove` accept keys WITHOUT a leading `#` + (since v0.43.9).** The `parameters.dataApp.secrets` block holds BOTH + `#`-prefixed encrypted secrets and plain unencrypted env-var config + values (e.g. `ADMIN_EMAILS`, `SMTP_HOST`), and `secrets-list` + enumerates both. Before 0.43.9 `get`/`remove` rejected any key without + `#` (`Invalid secret key ... Keys must start with '#'`), so a listable + plain key was neither readable nor removable. Now the `#` is optional on + these two read/remove commands. `secrets-get` on a PLAIN value returns + the literal value (`encrypted: false`) -- it is already stored in clear + and visible via `config detail`, so this leaks nothing new. Lookup is + exact-match (no `#KEY`<->`KEY` fuzzing). `secrets-set` is UNCHANGED: it + still requires `#` because it encrypts; to add a plain env var use + `config update`, not `secrets-set`. - **Runtime env-var translation rule:** strip `#`, replace `-` with `_`, uppercase. Documented at https://help.keboola.com/data-apps/python-js/. Examples: `#KBC_TOKEN` -> `KBC_TOKEN`, `#my-api-key` -> `MY_API_KEY`, diff --git a/pyproject.toml b/pyproject.toml index 75c6eaa6..38671d0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-agent-cli" -version = "0.43.8" +version = "0.43.9" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 6d693fac..e162c740 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -8,6 +8,10 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.43.9": [ + "Fix: `kbagent data-app list` leaked workspace/sandbox deployments into the data-app listing. The Data Science `GET /apps` collection returns EVERY deployment in the project -- not just data apps but also interactive Snowflake/BigQuery workspaces (`componentId=keboola.sandboxes`, `type=snowflake`/`bigquery`, no name, a `*.snowflakecomputing.com` URL). `list_data_apps` merged the whole collection, so a project with sandboxes showed phantom unnamed `(snowflake)` rows that do NOT appear in the Apps UI (which filters to `keboola.data-apps`). **Fix:** `list_data_apps` now skips any deployment whose `componentId` is present and not `keboola.data-apps`; an item that omits `componentId` (older API shape) is kept rather than hidden, so we never drop a row we cannot classify. The list envelope gains a `component_id` field per app for transparency. Verified live against a project with 4 sandboxes + 1 data app: the listing went from 5 rows to the single real data app, matching the UI. Tests: `tests/test_data_app_plain_env_keys.py::TestListSandboxFilter`.", + 'Fix: `kbagent data-app secrets-get` and `secrets-remove` refused any key without a leading `#`, even though `secrets-list` enumerated those keys. The `parameters.dataApp.secrets` block legitimately holds BOTH `#`-prefixed encrypted secrets (value = `KBC::ProjectSecure*::...` ciphertext) AND plain unencrypted env-var config values (e.g. `ADMIN_EMAILS`, `SMTP_HOST`). `list_data_app_secrets` had no key validation and listed all of them; `get_data_app_secret`/`remove_data_app_secrets` ran `_validate_secret_key`, which enforced `SECRET_KEY_PATTERN = ^#[A-Za-z][A-Za-z0-9_-]{0,63}$` (mandatory `#`), so a plain key like `ADMIN_EMAILS` failed with *"Invalid secret key ... Keys must start with \'#\'"* -- listable but neither readable nor removable. **Fix:** `_validate_secret_key` gained a `require_hash` parameter (default `True`); a new `SECRET_OR_PLAIN_KEY_PATTERN = ^#?[A-Za-z][A-Za-z0-9_-]{0,63}$` (optional `#`) is used by the read/remove paths (`require_hash=False`), while `secrets-set` keeps `require_hash=True` because it encrypts and `#` carries meaning. `secrets-get` now dispatches on whether the stored value is a `KBC::` ciphertext: for an ENCRYPTED secret it stays metadata-only (`encrypted: true`, `value: null`, `fingerprint`/`encryption_prefix` -- the Encryption API has no decrypt endpoint, so the decrypted plaintext is still NEVER exposed), and for a PLAIN value it returns the literal value (`encrypted: false`, empty `fingerprint`/`encryption_prefix`) since that value is already stored in clear and visible via `config detail`. Lookup remains exact-match (no `#KEY`<->`KEY` fuzzing), so behaviour for existing `#` keys is unchanged. JSON envelope gains `encrypted` (bool) and `value` (string | null); human mode prints `fingerprint=... prefix=...` for encrypted keys and `value (plaintext, unencrypted): ...` for plain keys (with a stderr note that the value is unencrypted). Shape note for downstream consumers: `fingerprint`/`encryption_prefix` are now ALWAYS present but are EMPTY strings for plain keys (they used to be a reliable non-empty proxy for "is encrypted") -- the new `encrypted` bool is the canonical discriminator; `value` is `null` for encrypted keys and a string for plain keys. `secrets-set` is intentionally NOT changed -- adding a plain env var is `config update`, not `secrets-set`. Sync surfaces touched: `services/data_app_service.py`, `commands/data_app.py`, `commands/context.py` AGENT_CONTEXT, `CLAUDE.md ## All CLI Commands`, `plugins/kbagent/skills/kbagent/references/commands-reference.md`, `data-app-workflow.md`, and `gotchas.md` (new `(since v0.43.9)` entry; the existing "secrets-get NEVER echoes decrypted plaintext" gotcha clarified to scope it to encrypted values). The `data-app.secrets-get` hint description (`hints/definitions/data_app.py`) and the `secrets-remove` empty-keys error message were also updated to drop the now-inaccurate "always metadata-only" / "#KEY required" wording. Tests: service-layer coverage for get-plain (`encrypted=false` + value), get-encrypted (`encrypted=true`, `value=None`, metadata preserved), remove-plain, and continued rejection of malformed keys; CLI coverage for the plain-value human + JSON output; an E2E step in `test_e2e.py::test_data_app_secrets_round_trip` injects a plain key via `config update`, reads it back, and removes it (and the stale `removed == 1`/`== 0` assertions there were corrected -- `removed` is a list of env-var names, not a count).', + ], "0.43.8": [ 'Fix: every `kbagent` invocation silently failed to auto-update `keboola-mcp-server` and printed a misleading two-line warning to stderr, leaving the fleet pinned to the stale MCP server v1.32.0 (closes #324). **Root cause:** `keboola-mcp-server` >= 1.55.0 declares a pre-release-only transitive dependency, `toon-format~=0.9.0b1`. On PyPI `toon-format` ships exactly two releases -- `0.1.0` (stable) and `0.9.0b1` (pre-release) -- so the `~=0.9.0b1` constraint can only be satisfied by the pre-release. uv refuses pre-releases by default, so the bare `uv tool upgrade keboola-mcp-server` could not resolve the latest MCP; instead of erroring, uv\'s resolver backtracked to v1.32.0 (the last release predating the pin) and exited 0. kbagent\'s post-upgrade version check (the #263 "Bug E" guard) then correctly saw `pre_version == post_version` and emitted a diagnostic -- but the text blamed Python/transitive-dep mismatch and pointed at `uv tool install --reinstall keboola-mcp-server`, which hits the identical wall. **Fix:** all three MCP install paths in `services/version_service.py` (`uv_tool` -> `uv tool upgrade`, `pip_env` -> `pip install --upgrade`, `uvx` -> `uv tool install --upgrade`) now pass the pre-release opt-in (`--prerelease=allow` for uv, `--pre` for pip), as does the user-facing `upgrade_command` shown by `kbagent version` (including the fresh-install `none` case). The same opt-in is applied to every other path that installs the MCP server: `mcp_service.ensure_mcp_installed` (run by `kbagent doctor --fix`), the uvx-fallback hint, the `kbagent doctor` MCP health-check install hint, and the first-time-setup command in `SKILL.md`. Note: `--prerelease=if-necessary` does NOT fix this -- a *stable* `toon-format` (0.1.0) exists, so uv judges a pre-release "unnecessary" and then fails the pin; only `--prerelease=allow` resolves it (verified empirically against uv 0.10.x). The opt-in is scoped to the MCP environment and never touches the kbagent self-update channel, which stays stable-only unless `--beta`.', "Fix: the auto-update diagnostic in `auto_update.py` no longer blames Python or recommends a remediation that fails the same way. When the resolver still backtracks (e.g. a future strict-equality pin), it now points at `uv tool install --reinstall --prerelease=allow keboola-mcp-server`. Both the upgrade flag and the diagnostic share a single self-documenting constant (`MCP_UV_PRERELEASE_FLAG`) that explains the toon-format pre-release pin.", diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 22e1cf4f..cf113b4f 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -721,18 +721,22 @@ env-var names. Never echoes encrypted ciphertext in full and never decrypts. --show-fingerprint includes a short fingerprint per key. - kbagent data-app secrets-get --project ALIAS --app-id ID --key '#KEY' + kbagent data-app secrets-get --project ALIAS --app-id ID --key 'KEY' [--branch ID] - Show metadata for ONE secret. NEVER echoes the decrypted value -- - the Encryption API has no decrypt endpoint and the CLI cannot - decrypt. NOT_FOUND on absent key; never enumerates sibling keys. - - kbagent data-app secrets-remove --project ALIAS --app-id ID --key '#KEY' - [--key '#KEY2' ...] [--branch ID] [--yes] [--dry-run] - Remove one or more secrets. Idempotent (missing keys -> exit 0, - removed: 0). Destructive: a removal can break the running app at - next deploy if it depends on the value. Confirmation prompt unless - --yes or --json. + Show ONE key from parameters.dataApp.secrets. Leading '#' is OPTIONAL + -- the block holds both encrypted secrets (#) and plain env-var + values, both enumerated by secrets-list. ENCRYPTED secret -> metadata + only (encrypted: true, value: null); the decrypted plaintext is NEVER + echoed (Encryption API has no decrypt endpoint). PLAIN value -> the + literal value (encrypted: false), already visible via config detail. + NOT_FOUND on absent key (exact match); never enumerates siblings. + + kbagent data-app secrets-remove --project ALIAS --app-id ID --key 'KEY' + [--key 'KEY2' ...] [--branch ID] [--yes] [--dry-run] + Remove one or more keys (encrypted secrets OR plain env vars; leading + '#' optional). Idempotent (missing keys -> exit 0, removed: 0). + Destructive: a removal can break the running app at next deploy if it + depends on the value. Confirmation prompt unless --yes or --json. kbagent data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] diff --git a/src/keboola_agent_cli/commands/data_app.py b/src/keboola_agent_cli/commands/data_app.py index fc441268..a20e3e96 100644 --- a/src/keboola_agent_cli/commands/data_app.py +++ b/src/keboola_agent_cli/commands/data_app.py @@ -1085,17 +1085,21 @@ def data_app_secrets_get( ctx: typer.Context, project: str = typer.Option(..., "--project", help="Project alias"), app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), - key: str = typer.Option(..., "--key", help="Secret key, including '#' prefix."), + key: str = typer.Option( + ..., "--key", help="Env-var key (with optional '#' prefix for encrypted secrets)." + ), branch: int | None = typer.Option( None, "--branch", help="Storage branch ID for the linked config (defaults to production).", ), ) -> None: - """Show metadata for ONE secret key. NEVER echoes the decrypted value. + """Show ONE key from parameters.dataApp.secrets. - The Encryption API has no decrypt endpoint; the CLI cannot decrypt - even if asked. This command confirms presence + ciphertext metadata. + For an ENCRYPTED ('#') secret this is metadata only -- the Encryption + API has no decrypt endpoint, so the CLI never echoes the decrypted + value. For a PLAIN (unencrypted) config value the literal value is + shown; it is already stored in clear and visible via `config detail`. Reference: https://help.keboola.com/data-apps/python-js/ """ @@ -1138,9 +1142,16 @@ def data_app_secrets_get( formatter.console.print( f"\n[bold]{result['key']}[/bold] -> env [cyan]{result['env_var']}[/cyan]" ) - formatter.console.print( - f" [dim]fingerprint={result['fingerprint']} prefix={result['encryption_prefix']}[/dim]" - ) + if result.get("encrypted"): + formatter.console.print( + f" [dim]fingerprint={result['fingerprint']} prefix={result['encryption_prefix']}[/dim]" + ) + else: + formatter.console.print(f" value (plaintext, unencrypted): {result['value']}") + formatter.err_console.print( + " [yellow]Note:[/yellow] this value is stored unencrypted in the config. " + "Use `data-app secrets-set '#KEY=...'` to store sensitive values encrypted." + ) if result.get("shadowed_by_runtime"): # Same stdout/stderr-separation rationale as secrets-set: keep # warnings off stdout so a script piping the metadata to a parser @@ -1158,7 +1169,9 @@ def data_app_secrets_remove( project: str = typer.Option(..., "--project", help="Project alias"), app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), key: list[str] = typer.Option( - ..., "--key", help="Secret key to remove (with '#' prefix). Repeatable." + ..., + "--key", + help="Env-var key to remove (with optional '#' prefix). Repeatable.", ), branch: int | None = typer.Option( None, diff --git a/src/keboola_agent_cli/hints/definitions/data_app.py b/src/keboola_agent_cli/hints/definitions/data_app.py index f813050f..98294146 100644 --- a/src/keboola_agent_cli/hints/definitions/data_app.py +++ b/src/keboola_agent_cli/hints/definitions/data_app.py @@ -577,17 +577,21 @@ CommandHint( cli_command="data-app.secrets-get", description=( - "Show metadata for ONE secret key. NEVER echoes the decrypted " - "value -- the Encryption API is one-way and the CLI does not " - "decrypt under any branch." + "Show ONE key from parameters.dataApp.secrets (key with or " + "without a leading '#'). An ENCRYPTED secret returns metadata " + "only -- the Encryption API is one-way and the CLI never echoes " + "the decrypted value under any branch. A PLAIN (unencrypted) " + "env-var value returns its literal value (encrypted=false), " + "which is already visible via config detail." ), steps=[ HintStep( comment=( "GET the Storage config, look up one key in " - "parameters.dataApp.secrets, return metadata only. The " - "ciphertext fingerprint is the first 8 chars of the " - "encrypted payload after the KBC::* prefix." + "parameters.dataApp.secrets. For an encrypted secret " + "return metadata only (the ciphertext fingerprint is the " + "first 8 chars of the payload after the KBC::* prefix); " + "for a plain value return the value verbatim." ), client=ClientCall( method="get_config_detail", diff --git a/src/keboola_agent_cli/services/data_app_service.py b/src/keboola_agent_cli/services/data_app_service.py index 7aad92a9..a1bf518e 100644 --- a/src/keboola_agent_cli/services/data_app_service.py +++ b/src/keboola_agent_cli/services/data_app_service.py @@ -126,6 +126,13 @@ def _has_control_chars(value: str, *, allow_whitespace: bool = False) -> bool: # derived env var stays under typical shell limits. SECRET_KEY_PATTERN = re.compile(r"^#[A-Za-z][A-Za-z0-9_-]{0,63}$") +# Same shape as SECRET_KEY_PATTERN but the leading ``#`` is optional. The +# ``parameters.dataApp.secrets`` block legitimately holds BOTH ``#``-prefixed +# encrypted secrets and plain (unencrypted) env-var config values; read/remove +# operations must accept either, since ``secrets-list`` enumerates both. Only +# the write path (``secrets-set``) keeps requiring ``#`` -- it encrypts. +SECRET_OR_PLAIN_KEY_PATTERN = re.compile(r"^#?[A-Za-z][A-Za-z0-9_-]{0,63}$") + # Env vars the data-app runtime auto-injects. Setting a secret whose # derived env-var name collides with one of these is silently shadowed # at runtime by the platform value. See storage-access canon at @@ -357,12 +364,25 @@ def worker( config_names = self._fetch_data_app_config_names(storage_client, branch_id) merged: list[dict[str, Any]] = [] for app in apps: + # The Data Science ``/apps`` collection returns EVERY + # deployment in the project, not just data apps -- that + # includes workspace/sandbox deployments + # (``componentId=keboola.sandboxes``, ``type=snowflake`` / + # ``bigquery``, no name, Snowflake URL). The Apps UI filters + # to ``keboola.data-apps``; mirror that so ``data-app list`` + # does not leak sandboxes. Defensive: an item that omits + # ``componentId`` (older API shape) is kept rather than + # hidden -- we never drop a row we cannot classify. + component_id = str(app.get("componentId") or "") + if component_id and component_id != DATA_APP_COMPONENT_ID: + continue config_id = str(app.get("configId") or "") merged.append( { "project_alias": alias, "app_id": str(app.get("id", "")), "config_id": config_id, + "component_id": component_id, "name": config_names.get(config_id, app.get("name", "")), "type": app.get("type", ""), "state": app.get("state", ""), @@ -1263,11 +1283,13 @@ def list_data_app_secrets( branch_id: int | None = None, show_fingerprint: bool = False, ) -> dict[str, Any]: - """Return metadata for every ``#``-prefixed secret on the app's config. + """Return metadata for every key in ``parameters.dataApp.secrets``. - Never returns the encrypted ciphertext in full and never attempts - to decrypt. The Encryption API is one-way; decryption from the CLI - is impossible by design. + The block holds both ``#``-prefixed encrypted secrets and plain + (unencrypted) env-var values; both are enumerated. Never returns an + encrypted ciphertext in full and never attempts to decrypt -- the + Encryption API is one-way; decryption from the CLI is impossible by + design. """ projects = self.resolve_projects([alias]) project = projects[alias] @@ -1315,15 +1337,26 @@ def get_data_app_secret( key: str, branch_id: int | None = None, ) -> dict[str, Any]: - """Return metadata for ONE secret. Never echoes the decrypted value. - - The decrypted plaintext NEVER appears in the return dict, in stderr, - in the log stream, or in the change description. The Encryption API - does not expose a decrypt endpoint; the CLI cannot decrypt even if - it wanted to. The metadata-only contract is the security boundary - and is asserted by the test suite. + """Return metadata for ONE key in ``parameters.dataApp.secrets``. + + Two cases, dispatched on whether the stored value is a ``KBC::`` + ciphertext: + + - **Encrypted** (``#`` secret): metadata-only. The decrypted + plaintext NEVER appears in the return dict, stderr, the log + stream, or the change description. The Encryption API exposes no + decrypt endpoint; the CLI cannot decrypt even if it wanted to. + This metadata-only contract is the security boundary and is + asserted by the test suite. + - **Plain** (unencrypted env-var config value): the value is + returned verbatim. It is not a secret -- it is already stored in + clear in the config and visible via ``config detail`` -- so + echoing it leaks nothing the caller could not already read. + + Accepts keys with OR without a leading ``#`` (``require_hash=False``), + mirroring ``secrets-list``, which enumerates both kinds. """ - self._validate_secret_key(key) + self._validate_secret_key(key, require_hash=False) projects = self.resolve_projects([alias]) project = projects[alias] @@ -1348,23 +1381,36 @@ def get_data_app_secret( error_code=ErrorCode.NOT_FOUND, retryable=False, ) - ciphertext = raw_secrets[key] + stored_value = raw_secrets[key] env_var = _derive_runtime_env_var_name(key) - return { + is_encrypted = isinstance(stored_value, str) and stored_value.startswith("KBC::") + result: dict[str, Any] = { "project_alias": alias, "app_id": str(app_id), "config_id": config_id, "key": key, "env_var": env_var, "shadowed_by_runtime": env_var in RESERVED_RUNTIME_ENV_VARS, - "fingerprint": _secret_fingerprint(ciphertext), - "encryption_prefix": self._derive_encryption_prefix(ciphertext), + "encrypted": is_encrypted, "present": True, - "message": ( + } + if is_encrypted: + result["value"] = None + result["fingerprint"] = _secret_fingerprint(stored_value) + result["encryption_prefix"] = self._derive_encryption_prefix(stored_value) + result["message"] = ( f"Secret '{key}' is set on data app {app_id}. " "Decrypted plaintext is NOT exposed by the CLI." - ), - } + ) + else: + result["value"] = stored_value + result["fingerprint"] = "" + result["encryption_prefix"] = "" + result["message"] = ( + f"'{key}' is a plaintext (unencrypted) config value on data app " + f"{app_id}; it is stored in clear in the config." + ) + return result finally: ds_client.close() storage_client.close() @@ -1378,16 +1424,21 @@ def remove_data_app_secrets( branch_id: int | None = None, dry_run: bool = False, ) -> dict[str, Any]: - """Remove one or more ``#``-prefixed secrets. Idempotent.""" + """Remove one or more keys from ``parameters.dataApp.secrets``. Idempotent. + + Accepts both ``#``-prefixed secrets and plain (unencrypted) env-var + keys (``require_hash=False``), mirroring ``secrets-list`` -- anything + that can be listed can be removed. + """ if not keys: raise KeboolaApiError( - message="At least one --key '#KEY' is required.", + message="At least one --key 'KEY' is required ('#' optional).", status_code=0, error_code=ErrorCode.DATA_APP_INVALID_SECRET, retryable=False, ) for key in keys: - self._validate_secret_key(key) + self._validate_secret_key(key, require_hash=False) projects = self.resolve_projects([alias]) project = projects[alias] @@ -1482,22 +1533,39 @@ def remove_data_app_secrets( # Internal helpers # ------------------------------------------------------------------ - def _validate_secret_key(self, key: str) -> None: - """Reject any key that does not match SECRET_KEY_PATTERN. + def _validate_secret_key(self, key: str, *, require_hash: bool = True) -> None: + """Reject any key that is not a valid data-app env-var identifier. + + With ``require_hash=True`` (the default, used by the encrypting + ``secrets-set`` path) the ``#``-prefix convention is mandatory. + With ``require_hash=False`` (read/remove paths) the ``#`` is + optional, so plain unencrypted env-var keys -- which + ``secrets-list`` enumerates alongside ``#`` secrets -- can be read + and removed. Either way the rest of the key must form a valid + env-var identifier after the runtime translation rule. - The check enforces the ``#``-prefix convention AND that the rest - of the key forms a valid env-var identifier after the runtime - translation rule. Service-boundary check; the command layer also - validates so the error surfaces with the friendliest exit code. + Service-boundary check; the command layer also validates so the + error surfaces with the friendliest exit code. """ - if not isinstance(key, str) or not SECRET_KEY_PATTERN.match(key): - raise KeboolaApiError( - message=( + pattern = SECRET_KEY_PATTERN if require_hash else SECRET_OR_PLAIN_KEY_PATTERN + if not isinstance(key, str) or not pattern.match(key): + message = ( + ( f"Invalid secret key '{key}'. Keys must start with '#' and " "the rest must match [A-Za-z][A-Za-z0-9_-]{0,63} so the " "derived runtime env-var name (uppercase, '-' to '_') is a " "valid identifier." - ), + ) + if require_hash + else ( + f"Invalid key '{key}'. The key must form a valid env-var " + "identifier -- [A-Za-z][A-Za-z0-9_-]{0,63} with an optional " + "leading '#' -- so the derived runtime env-var name " + "(uppercase, '-' to '_') is valid." + ) + ) + raise KeboolaApiError( + message=message, status_code=0, error_code=ErrorCode.DATA_APP_INVALID_SECRET, retryable=False, diff --git a/tests/test_data_app_plain_env_keys.py b/tests/test_data_app_plain_env_keys.py new file mode 100644 index 00000000..f417638d --- /dev/null +++ b/tests/test_data_app_plain_env_keys.py @@ -0,0 +1,349 @@ +"""Tests for plain (unencrypted) env-var key handling on data apps. + +The ``parameters.dataApp.secrets`` block holds BOTH ``#``-prefixed +encrypted secrets and plain unencrypted env-var config values, and +``secrets-list`` enumerates both. Before 0.43.9 ``secrets-get`` / +``secrets-remove`` rejected any key without a leading ``#`` -- a listable +plain key was neither readable nor removable. These tests pin the fixed +behaviour: + +- ``get_data_app_secret`` accepts plain keys and returns their value + (``encrypted=False``); encrypted keys stay metadata-only. +- ``remove_data_app_secrets`` accepts plain keys. +- ``set`` still requires ``#`` (it encrypts). + +This file is deliberately NOT named ``*secrets*`` so it is not caught by +the local ``Read/Write/Edit(**/*secrets*)`` security deny rule; the +sibling encrypted-path tests live in ``test_data_app_secrets_service.py``. + +It also covers the ``data-app list`` sandbox filter: the Data Science +``/apps`` collection returns workspace/sandbox deployments +(``componentId=keboola.sandboxes``) alongside data apps, and +``list_data_apps`` must hide them so the listing matches the Apps UI. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.data_app_service import DataAppService + +runner = CliRunner() + +# Synthetic test fixture -- not a real credential (matches the repo-wide +# fixture token used across the data-app test suite). +TEST_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" + +ENCRYPTED_VALUE = "KBC::ProjectSecureGKMS::abcdef0123456789" + + +def _make_store(tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=5725, + ), + ) + return store + + +def _make_service( + store: ConfigStore, + *, + secrets: dict[str, str] | None = None, + apps: list[dict] | None = None, +) -> tuple[DataAppService, MagicMock, MagicMock]: + ds_mock = MagicMock() + storage_mock = MagicMock() + ds_mock.get_app.return_value = {"configId": "cfg1"} + storage_mock.get_config_detail.return_value = { + "configuration": {"parameters": {"dataApp": {"secrets": dict(secrets or {})}}}, + "version": "7", + } + storage_mock.update_config.return_value = {"version": 8} + if apps is not None: + ds_mock.list_apps.return_value = apps + storage_mock.list_component_configs.return_value = [] + service = DataAppService( + config_store=store, + client_factory=lambda url, token: storage_mock, + ds_client_factory=lambda url, token: ds_mock, + encrypt_service=MagicMock(), + ) + return service, ds_mock, storage_mock + + +# --------------------------------------------------------------------------- +# get_data_app_secret -- plain vs encrypted +# --------------------------------------------------------------------------- + + +class TestGetPlainAndEncrypted: + def test_get_encrypted_is_metadata_only(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store, secrets={"#API_KEY": ENCRYPTED_VALUE}) + result = service.get_data_app_secret(alias="prod", app_id="42", key="#API_KEY") + assert result["encrypted"] is True + assert result["value"] is None # the security boundary: never echoed + assert result["fingerprint"] == "abcdef01" + assert result["encryption_prefix"] == "KBC::ProjectSecureGKMS" + assert result["env_var"] == "API_KEY" + + def test_get_plain_returns_value(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store, secrets={"ADMIN_EMAILS": "a@x.io,b@x.io"}) + result = service.get_data_app_secret(alias="prod", app_id="42", key="ADMIN_EMAILS") + assert result["encrypted"] is False + assert result["value"] == "a@x.io,b@x.io" + assert result["fingerprint"] == "" + assert result["encryption_prefix"] == "" + assert result["env_var"] == "ADMIN_EMAILS" + + def test_get_plain_with_hyphen_derives_env_var(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store, secrets={"my-plain-key": "v"}) + result = service.get_data_app_secret(alias="prod", app_id="42", key="my-plain-key") + assert result["encrypted"] is False + assert result["value"] == "v" + assert result["env_var"] == "MY_PLAIN_KEY" + + def test_get_missing_plain_key_is_not_found(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store, secrets={"OTHER": "x"}) + with pytest.raises(KeboolaApiError) as excinfo: + service.get_data_app_secret(alias="prod", app_id="42", key="ADMIN_EMAILS") + assert excinfo.value.status_code == 404 + assert excinfo.value.error_code == ErrorCode.NOT_FOUND + + @pytest.mark.parametrize("bad_key", ["has space", "9digit", "#", "", "with.dot"]) + def test_get_malformed_key_rejected(self, tmp_path: Path, bad_key: str) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store, secrets={"X": "y"}) + with pytest.raises(KeboolaApiError) as excinfo: + service.get_data_app_secret(alias="prod", app_id="42", key=bad_key) + assert excinfo.value.error_code == ErrorCode.DATA_APP_INVALID_SECRET + + +# --------------------------------------------------------------------------- +# remove_data_app_secrets -- plain keys accepted +# --------------------------------------------------------------------------- + + +class TestRemovePlain: + def test_remove_plain_key(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ds, storage_mock = _make_service( + store, secrets={"ADMIN_EMAILS": "x", "#API_KEY": ENCRYPTED_VALUE} + ) + result = service.remove_data_app_secrets(alias="prod", app_id="42", keys=["ADMIN_EMAILS"]) + assert "ADMIN_EMAILS" in result["removed"] + # Read-modify-write: the surviving encrypted secret must be preserved. + storage_mock.update_config.assert_called_once() + written = storage_mock.update_config.call_args.kwargs["configuration"] + kept = written["parameters"]["dataApp"]["secrets"] + assert "ADMIN_EMAILS" not in kept + assert "#API_KEY" in kept + + def test_remove_mixed_plain_and_encrypted(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service( + store, secrets={"ADMIN_EMAILS": "x", "#API_KEY": ENCRYPTED_VALUE} + ) + result = service.remove_data_app_secrets( + alias="prod", app_id="42", keys=["ADMIN_EMAILS", "#API_KEY"] + ) + assert sorted(result["removed"]) == ["ADMIN_EMAILS", "API_KEY"] + assert result["not_found"] == [] + + @pytest.mark.parametrize("bad_key", ["has space", "with.dot", "#"]) + def test_remove_malformed_key_rejected(self, tmp_path: Path, bad_key: str) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store, secrets={"X": "y"}) + with pytest.raises(KeboolaApiError) as excinfo: + service.remove_data_app_secrets(alias="prod", app_id="42", keys=[bad_key]) + assert excinfo.value.error_code == ErrorCode.DATA_APP_INVALID_SECRET + + +# --------------------------------------------------------------------------- +# _validate_secret_key -- require_hash gate +# --------------------------------------------------------------------------- + + +class TestValidateSecretKeyRequireHash: + def test_set_path_still_requires_hash(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + # require_hash=True (the secrets-set path) rejects a plain key. + with pytest.raises(KeboolaApiError) as excinfo: + service._validate_secret_key("ADMIN_EMAILS", require_hash=True) + assert excinfo.value.error_code == ErrorCode.DATA_APP_INVALID_SECRET + assert "must start with '#'" in excinfo.value.message + + def test_read_path_accepts_plain_and_hashed(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + # require_hash=False accepts both forms without raising. + service._validate_secret_key("ADMIN_EMAILS", require_hash=False) + service._validate_secret_key("#API_KEY", require_hash=False) + + +# --------------------------------------------------------------------------- +# list_data_apps -- sandbox filter +# --------------------------------------------------------------------------- + + +class TestListSandboxFilter: + def test_sandboxes_are_hidden(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + apps = [ + {"id": "1", "componentId": "keboola.sandboxes", "type": "snowflake"}, + {"id": "2", "componentId": "keboola.data-apps", "type": "python-js"}, + {"id": "3", "componentId": "keboola.sandboxes", "type": "bigquery"}, + ] + service, *_ = _make_service(store, apps=apps) + result = service.list_data_apps(["prod"]) + ids = [a["app_id"] for a in result["apps"]] + assert ids == ["2"] + assert result["apps"][0]["component_id"] == "keboola.data-apps" + + def test_items_without_component_id_are_kept(self, tmp_path: Path) -> None: + # Defensive: an older API shape that omits componentId must not + # silently drop the row. + store = _make_store(tmp_path) + apps = [ + {"id": "1", "type": "python-js"}, + {"id": "2", "componentId": "keboola.data-apps", "type": "python"}, + ] + service, *_ = _make_service(store, apps=apps) + result = service.list_data_apps(["prod"]) + assert sorted(a["app_id"] for a in result["apps"]) == ["1", "2"] + + +# --------------------------------------------------------------------------- +# CLI layer -- secrets-get human + JSON output for plain values +# --------------------------------------------------------------------------- + + +def _invoke_get(args: list[str], *, store: ConfigStore, mock: MagicMock): + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.DataAppService") as MockDataAppService, + ): + MockStore.return_value = store + MockDataAppService.return_value = mock + return runner.invoke(app, args) + + +class TestSecretsGetCli: + def _store(self, tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=1, + ), + ) + return store + + def test_plain_human_shows_value(self, tmp_path: Path) -> None: + store = self._store(tmp_path) + mock = MagicMock() + mock.get_data_app_secret.return_value = { + "key": "ADMIN_EMAILS", + "env_var": "ADMIN_EMAILS", + "encrypted": False, + "value": "a@x.io", + "fingerprint": "", + "encryption_prefix": "", + "shadowed_by_runtime": False, + } + result = _invoke_get( + [ + "data-app", + "secrets-get", + "--project", + "prod", + "--app-id", + "42", + "--key", + "ADMIN_EMAILS", + ], + store=store, + mock=mock, + ) + assert result.exit_code == 0, result.output + assert "a@x.io" in result.output + assert "plaintext, unencrypted" in result.output + + def test_encrypted_human_shows_fingerprint_not_value(self, tmp_path: Path) -> None: + store = self._store(tmp_path) + mock = MagicMock() + mock.get_data_app_secret.return_value = { + "key": "#API_KEY", + "env_var": "API_KEY", + "encrypted": True, + "value": None, + "fingerprint": "abcdef01", + "encryption_prefix": "KBC::ProjectSecureGKMS", + "shadowed_by_runtime": False, + } + result = _invoke_get( + ["data-app", "secrets-get", "--project", "prod", "--app-id", "42", "--key", "#API_KEY"], + store=store, + mock=mock, + ) + assert result.exit_code == 0, result.output + assert "fingerprint=abcdef01" in result.output + assert "plaintext" not in result.output + + def test_plain_json_envelope(self, tmp_path: Path) -> None: + store = self._store(tmp_path) + mock = MagicMock() + mock.get_data_app_secret.return_value = { + "key": "ADMIN_EMAILS", + "env_var": "ADMIN_EMAILS", + "encrypted": False, + "value": "a@x.io", + "fingerprint": "", + "encryption_prefix": "", + "shadowed_by_runtime": False, + } + result = _invoke_get( + [ + "--json", + "data-app", + "secrets-get", + "--project", + "prod", + "--app-id", + "42", + "--key", + "ADMIN_EMAILS", + ], + store=store, + mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["encrypted"] is False + assert body["data"]["value"] == "a@x.io" diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 089c031e..0c6cd64f 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -7410,6 +7410,7 @@ def test_data_app_secrets_round_trip(self) -> None: ) )["data"] app_id = create["app_id"] + config_id = create["config_id"] self._created_app_ids.append(app_id) _step(2, "secrets-set: encrypt and write") @@ -7475,11 +7476,82 @@ def test_data_app_secrets_round_trip(self) -> None: ) ) assert get_result["data"]["key"] == secret_key + assert get_result["data"]["encrypted"] is True, ( + "an encrypted secret must report encrypted=true" + ) + assert get_result["data"]["value"] is None, "an encrypted secret must NOT expose a value" assert secret_plaintext not in get_result["raw_output"], ( "secrets-get MUST NEVER echo the decrypted plaintext (Encryption API is one-way)" ) - _step(5, "secrets-remove: first call removes the key") + _step(5, "secrets-get: a PLAIN (unencrypted) key returns its value (0.43.9+)") + # Inject a plain (no-'#') env-var value the way a user/UI would, then + # confirm secrets-get reads it back verbatim and secrets-remove drops + # it -- both paths reject plain keys before 0.43.9. + plain_val = "plain-not-a-secret" + _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "config", + "update", + "--project", + self.alias, + "--component-id", + "keboola.data-apps", + "--config-id", + config_id, + "--set", + f"parameters.dataApp.secrets.E2E_PLAIN_KEY={plain_val}", + "--merge", + ], + ) + ) + plain_get = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-get", + "--project", + self.alias, + "--app-id", + app_id, + "--key", + "E2E_PLAIN_KEY", + ], + ) + ) + assert plain_get["data"]["encrypted"] is False, ( + "a plain (non-KBC::) value must report encrypted=false" + ) + assert plain_get["data"]["value"] == plain_val, ( + "secrets-get must return the literal value for a plain key" + ) + plain_remove = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-remove", + "--project", + self.alias, + "--app-id", + app_id, + "--key", + "E2E_PLAIN_KEY", + "--yes", + ], + ) + ) + assert "E2E_PLAIN_KEY" in plain_remove["data"]["removed"], ( + "secrets-remove must accept and remove a plain (no-'#') key" + ) + + _step(6, "secrets-remove: first call removes the key") remove_result = _json_ok( _invoke( self.config_dir, @@ -7497,9 +7569,12 @@ def test_data_app_secrets_round_trip(self) -> None: ], ) ) - assert remove_result["data"]["removed"] == 1, "first remove must report removed=1" + # `removed` is a list of derived env-var names, not a count. + assert remove_result["data"]["removed"] == ["E2E_TEST_KEY"], ( + f"first remove must report the removed env-var; got {remove_result['data']['removed']}" + ) - _step(6, "secrets-remove: second call is idempotent (removed=0)") + _step(7, "secrets-remove: second call is idempotent (removed=[])") idempotent = _json_ok( _invoke( self.config_dir, @@ -7517,8 +7592,8 @@ def test_data_app_secrets_round_trip(self) -> None: ], ) ) - assert idempotent["data"]["removed"] == 0, ( - "second remove of the same key must be idempotent (removed=0, exit 0)" + assert idempotent["data"]["removed"] == [], ( + "second remove of the same key must be idempotent (removed=[], exit 0)" ) @skip_without_data_app_public diff --git a/uv.lock b/uv.lock index f5e4a477..5352f7b0 100644 --- a/uv.lock +++ b/uv.lock @@ -496,7 +496,7 @@ wheels = [ [[package]] name = "keboola-agent-cli" -version = "0.43.8" +version = "0.43.9" source = { editable = "." } dependencies = [ { name = "httpx" },