diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 36d079a5..f1a50902 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "kbagent", - "version": "0.28.0", + "version": "0.29.0", "source": "./plugins/kbagent", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "category": "development" diff --git a/docs/TUTORIAL.md b/docs/TUTORIAL.md index f17d8272..3328dd78 100644 --- a/docs/TUTORIAL.md +++ b/docs/TUTORIAL.md @@ -102,7 +102,7 @@ use `org setup --project-ids`. kbagent creates a Storage API token in each listed project and registers them all locally, in parallel. ```bash -# Interactive (default since v0.28.0): kbagent prompts for the Manage API +# Interactive (default since v0.29.0): kbagent prompts for the Manage API # token on stdin. No env var, no shell history. kbagent org setup \ --project-ids 901,9621,10539 \ @@ -139,7 +139,7 @@ Flags worth knowing: The command is **idempotent**: running it again skips projects that are already registered. Safe to re-run after adding new project IDs. -**Security note (since v0.28.0)**: `KBC_MANAGE_API_TOKEN` is **ignored +**Security note (since v0.29.0)**: `KBC_MANAGE_API_TOKEN` is **ignored by default** -- the env var is read only when the top-level `--allow-env-manage-token` flag is passed. Without the flag, kbagent prompts on stdin (hidden input). kbagent never accepts the token as a @@ -156,7 +156,7 @@ If you are an org admin with a Manage API token, register **every** project in an organization in one shot: ```bash -# Interactive (default since v0.28.0): +# Interactive (default since v0.29.0): kbagent org setup \ --org-id 123 \ --url https://connection.keboola.com \ @@ -876,7 +876,7 @@ kbagent --json data-app create \ deploy. To retrieve it: ```bash -# Manage API token: interactive prompt by default (since v0.28.0). For CI, +# Manage API token: interactive prompt by default (since v0.29.0). For CI, # add `--allow-env-manage-token` and set KBC_MANAGE_API_TOKEN in env. kbagent --json data-app password \ --project prod --app-id 12345678 \ @@ -967,7 +967,7 @@ shapes, and the `--hint client|service` code-generation contract, see | `kbagent: command not found` after `uv tool install` | Ensure `~/.local/bin` (or uv's tool dir) is on your PATH. `uv tool update-shell` can help. | | `kbagent doctor` reports `warn` for plugin | Run the two `/plugin` commands shown in the warning, from inside Claude Code. | | Plugin version != CLI version | In Claude Code: `/plugin update kbagent`. | -| `org setup` exits 2 with `Warning: KBC_MANAGE_API_TOKEN found in environment but ignored` | Default-deny since v0.28.0 -- pass `--allow-env-manage-token` (top-level flag) to opt in to env resolution, or run interactively to use the prompt. | +| `org setup` exits 2 with `Warning: KBC_MANAGE_API_TOKEN found in environment but ignored` | Default-deny since v0.29.0 -- pass `--allow-env-manage-token` (top-level flag) to opt in to env resolution, or run interactively to use the prompt. | | `org setup` fails with `401 Unauthorized` | Your manage token is wrong for this stack or role. Manage tokens are stack-specific and require the right scope. | | `org setup --org-id` fails with `403` | You are not an org admin. Use `--project-ids` with a Personal Access Token instead (works for any project member). | | Changes from `kbagent config update` do not show in UI | You are on a dev branch. Run `kbagent branch list` and `kbagent project current` to verify the active branch; changes in a dev branch merge to production only via the UI merge step (`kbagent branch merge` returns the merge URL). | diff --git a/docs/e2e-scenarios.md b/docs/e2e-scenarios.md index e78cd87d..b5fd98cf 100644 --- a/docs/e2e-scenarios.md +++ b/docs/e2e-scenarios.md @@ -211,7 +211,7 @@ Skipped if `keboola-mcp-server` is not installed. | Command | Reason | |---------|--------| -| `project refresh` | Requires Manage API token (interactive prompt by default since v0.28.0; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for non-interactive runners) | +| `project refresh` | Requires Manage API token (interactive prompt by default since v0.29.0; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for non-interactive runners) | | `org setup` | Requires Manage API token (same prompt-or-flag as above) + destructive (registers projects in org) | | `sharing share/unshare` | Requires org-level permissions or second project | | `sharing link/unlink` | Requires shared bucket from another project | diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index 2d06ad7c..fde66363 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.28.0", + "version": "0.29.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 828a46cc..cd41b595 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -67,11 +67,11 @@ a critical failure. need 0.27.0+, `config update` script[] auto-normalize against #245 trap needs 0.28.0+, `storage swap-tables` needs 0.28.0+, env-var manage-token auth for `org setup` / `project refresh` / - `data-app password` needs 0.28.0+ with `--allow-env-manage-token` - (the env var is default-deny on 0.28.0+), + `data-app password` needs 0.29.0+ with `--allow-env-manage-token` + (the env var is default-deny on 0.29.0+), `project invite` / `project member-*` / `project invitation-*` - need 0.26.1+, - `data-app secrets-* / validate-repo` need 0.28.0+, + need 0.29.0+, + `data-app secrets-* / validate-repo` need 0.29.0+, `storage retype` is a future composite), you MUST refuse the task and return a handoff message to the parent: `"Cannot proceed safely on kbagent . Missing: . @@ -112,20 +112,20 @@ a critical failure. | Roll out a new code or config version on a data app | `kbagent data-app deploy --project P --app-id N --wait` (0.27.0+) -- always sends the §9 trio | `kbagent --hint client data-app deploy ...` to inspect the generated `patch_app(desired_state=, config_version=, restart_if_running=True)` call | `tool call update_config` then `tool call run_component` (data apps are not jobs -- the queue runner does not deploy them) | | Wake an auto-suspended data app | `kbagent data-app start --project P --app-id N` (0.27.0+) -- does NOT bump configVersion | hitting the app's URL (auto-restart triggers a 30-60s cold boot) | `kbagent data-app deploy` (overkill -- bumps the deployed configVersion unnecessarily) | | Pause a running data app | `kbagent data-app stop --project P --app-id N` (0.27.0+) | -- | `kbagent data-app delete` (irreversible; cascades to Storage config) | -| Read the simpleAuth password for a password-gated app | `kbagent data-app password --project P --app-id N` (0.27.0+) -- needs Manage API token (interactive prompt by default; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+) | -- | trying to "rotate" the password (not supported by the API; delete + recreate to mint a new one) | +| Read the simpleAuth password for a password-gated app | `kbagent data-app password --project P --app-id N` (0.27.0+) -- needs Manage API token (interactive prompt by default; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+) | -- | trying to "rotate" the password (not supported by the API; delete + recreate to mint a new one) | | Tear down a data app | `kbagent data-app delete --project P --app-id N` (0.27.0+) -- cascades to Storage config; URL retired | -- | manually `tool call delete_config keboola.data-apps` while leaving the deployment record orphaned | -| Invite a user to a project (single) | `kbagent project invite --project P --email E --role admin\|guest\|readOnly\|share` (0.26.1+) | raw `requests.post(/manage/projects/{id}/invitations)` only if version-gated out | `kbagent project invite` without `KBC_MANAGE_API_TOKEN` set; passing manage token via CLI flag | -| Invite many users (bulk) | `kbagent project invite --from-csv FILE [--default-role guest] [--workers N] [--dry-run]` (0.26.1+) | `--hint client` to generate a parallel script using `ManageClient` | per-row shell loop calling the CLI -- defeats the parallelism + idempotency the service already does | -| List active project members | `kbagent project member-list --project P [--include-pending]` (0.26.1+) | `tool call run_sync_action` against the Manage API | reading `.kbagent/config.json` to infer membership (it only stores the local user's token) | -| List pending invitations | `kbagent project invitation-list --project P` (0.26.1+) | -- | -- | -| Cancel a pending invitation | `kbagent project invitation-cancel --project P --email E --yes` (0.26.1+) | `--invitation-id ID` if email lookup is ambiguous | DELETE via raw HTTP without going through the service layer | -| Remove an active member | `kbagent project member-remove --project P --email E --yes` (0.26.1+, **destructive**) | `--hint client` for a script that removes by user_id directly | calling `member-remove` without `--yes` in non-interactive contexts (it will prompt and hang) | -| Change a member's role | `kbagent project member-set-role --project P --email E --role admin\|guest\|readOnly\|share` (0.26.1+) | -- | `PUT /manage/projects/{id}/users/{userId}` -- the API rejects PUT with 404, the kbagent client correctly uses **PATCH** | -| Set / rotate app-runtime secrets | `kbagent data-app secrets-set --project P --app-id N --secret '#KEY=VAL'` (0.28.0+) then `data-app deploy --wait` -- per-project KMS encryption, fail-closed, never auto-deploys | `kbagent encrypt values --component-id keboola.data-apps` + `tool call update_config` -- ONLY if you need to write secrets to a different shape than `parameters.dataApp.secrets` | raw `POST` to encryption + Storage without read-modify-write -- you will clobber sibling keys nested under `parameters.dataApp.secrets` (Storage `merge=True` is shallow at the top level only) | -| Inspect what secrets are set on a data app | `kbagent data-app secrets-list --project P --app-id N` (0.28.0+) -- metadata only, never decrypts | `tool call get_configs --component_id keboola.data-apps` then read `parameters.dataApp.secrets` keys (raw dict, no env-var derivation, may leak ciphertext into output) | trying to decrypt -- the Encryption API has no decrypt endpoint, the CLI cannot decrypt under any branch | -| Confirm one secret is present | `kbagent data-app secrets-get --project P --app-id N --key '#KEY'` (0.28.0+) -- returns metadata only | -- | trying to extract the plaintext value (impossible by design; not a CLI gap) | -| Remove a secret from a data app | `kbagent data-app secrets-remove --project P --app-id N --key '#KEY' --yes` (0.28.0+) -- idempotent; missing keys exit 0 with `removed: 0` | `tool call update_config` with the secrets sub-dict deleted -- ONLY for batch removes that need a custom change description | `kbagent config update --set 'parameters.dataApp.secrets={}'` -- replaces the whole sub-dict, dropping every secret instead of just the named ones | -| Pre-flight a data-app repo before create | `kbagent data-app validate-repo --git-repo URL --type python-js [--git-pat-env VAR]` (0.28.0+) -- BLOCKING / WARN / OK with help-doc citations; ≤5 GitHub API calls regardless of repo size | git-clone the repo locally and inspect by hand | `data-app create --dry-run` (only shows the request bodies; does not validate repo structure) | +| Invite a user to a project (single) | `kbagent project invite --project P --email E --role admin\|guest\|readOnly\|share` (0.29.0+) | raw `requests.post(/manage/projects/{id}/invitations)` only if version-gated out | `kbagent project invite` without `KBC_MANAGE_API_TOKEN` set; passing manage token via CLI flag | +| Invite many users (bulk) | `kbagent project invite --from-csv FILE [--default-role guest] [--workers N] [--dry-run]` (0.29.0+) | `--hint client` to generate a parallel script using `ManageClient` | per-row shell loop calling the CLI -- defeats the parallelism + idempotency the service already does | +| List active project members | `kbagent project member-list --project P [--include-pending]` (0.29.0+) | `tool call run_sync_action` against the Manage API | reading `.kbagent/config.json` to infer membership (it only stores the local user's token) | +| List pending invitations | `kbagent project invitation-list --project P` (0.29.0+) | -- | -- | +| Cancel a pending invitation | `kbagent project invitation-cancel --project P --email E --yes` (0.29.0+) | `--invitation-id ID` if email lookup is ambiguous | DELETE via raw HTTP without going through the service layer | +| Remove an active member | `kbagent project member-remove --project P --email E --yes` (0.29.0+, **destructive**) | `--hint client` for a script that removes by user_id directly | calling `member-remove` without `--yes` in non-interactive contexts (it will prompt and hang) | +| Change a member's role | `kbagent project member-set-role --project P --email E --role admin\|guest\|readOnly\|share` (0.29.0+) | -- | `PUT /manage/projects/{id}/users/{userId}` -- the API rejects PUT with 404, the kbagent client correctly uses **PATCH** | +| Set / rotate app-runtime secrets | `kbagent data-app secrets-set --project P --app-id N --secret '#KEY=VAL'` (0.29.0+) then `data-app deploy --wait` -- per-project KMS encryption, fail-closed, never auto-deploys | `kbagent encrypt values --component-id keboola.data-apps` + `tool call update_config` -- ONLY if you need to write secrets to a different shape than `parameters.dataApp.secrets` | raw `POST` to encryption + Storage without read-modify-write -- you will clobber sibling keys nested under `parameters.dataApp.secrets` (Storage `merge=True` is shallow at the top level only) | +| Inspect what secrets are set on a data app | `kbagent data-app secrets-list --project P --app-id N` (0.29.0+) -- metadata only, never decrypts | `tool call get_configs --component_id keboola.data-apps` then read `parameters.dataApp.secrets` keys (raw dict, no env-var derivation, may leak ciphertext into output) | trying to decrypt -- the Encryption API has no decrypt endpoint, the CLI cannot decrypt under any branch | +| Confirm one secret is present | `kbagent data-app secrets-get --project P --app-id N --key '#KEY'` (0.29.0+) -- returns metadata only | -- | trying to extract the plaintext value (impossible by design; not a CLI gap) | +| Remove a secret from a data app | `kbagent data-app secrets-remove --project P --app-id N --key '#KEY' --yes` (0.29.0+) -- idempotent; missing keys exit 0 with `removed: 0` | `tool call update_config` with the secrets sub-dict deleted -- ONLY for batch removes that need a custom change description | `kbagent config update --set 'parameters.dataApp.secrets={}'` -- replaces the whole sub-dict, dropping every secret instead of just the named ones | +| Pre-flight a data-app repo before create | `kbagent data-app validate-repo --git-repo URL --type python-js [--git-pat-env VAR]` (0.29.0+) -- BLOCKING / WARN / OK with help-doc citations; ≤5 GitHub API calls regardless of repo size | git-clone the repo locally and inspect by hand | `data-app create --dry-run` (only shows the request bodies; does not validate repo structure) | If the table does not cover the user's task, **ask clarifying questions** instead of guessing. Returning a targeted question is a @@ -201,7 +201,7 @@ success, not a failure. verification payload but do not treat it as a failure signal. Production writes never materialize anything. -- **`project invite` "already invited / already member" is a no-op, not a failure** (0.26.1+): +- **`project invite` "already invited / already member" is a no-op, not a failure** (0.29.0+): Re-inviting a user the project already knows returns HTTP 400 from the Manage API. kbagent normalises both "...already been invited..." and "...already a member..." to `status="noop"` with a `note` field, exit 0. @@ -210,7 +210,7 @@ success, not a failure. toward `noop`, not `failed`, in the summary; surface that distinction to the user when reporting bulk results. -- **`project invite --from-csv` ordering is non-deterministic** (0.26.1+): +- **`project invite --from-csv` ordering is non-deterministic** (0.29.0+): Bulk invitation parallelises via `ThreadPoolExecutor` (default 8 workers). The `rows[]` array in the JSON result is in completion order, not CSV order. When reporting per-row outcomes to the user, **match by `email`, @@ -218,7 +218,7 @@ success, not a failure. the JSON -- treat that as a soft failure that needs review, not a catastrophe. -- **`project member-set-role` uses PATCH, not PUT** (0.26.1+): The Manage +- **`project member-set-role` uses PATCH, not PUT** (0.29.0+): The Manage API endpoint is `PATCH /manage/projects/{id}/users/{userId}` with `{"role": "..."}`. PUT returns 404 even on real members. kbagent's `ManageClient.update_project_member_role` emits PATCH; if you write a @@ -298,13 +298,13 @@ success, not a failure. the round-trip does not return a `KBC::Project*` ciphertext. - **`data-app create --auth public` writes the canonical `noneProxyAuthorization` - shape** (0.28.0+, fixes a v0.27.0 silent-503 bug): v0.27.0 wrote NO + shape** (0.29.0+, fixes a v0.27.0 silent-503 bug): v0.27.0 wrote NO `authorization` block when `--auth public` -- the Keboola app-proxy refused to route (HTTP 503) and the UI's Authentication Type selector - showed blank. v0.28.0 writes + showed blank. v0.29.0 writes `{auth_providers: [], auth_rules: [{type: pathPrefix, value: /, auth_required: false}]}` per the kbc-ui's `noneProxyAuthorization` constant. If a user reports a - v0.27.0 public app returning 503, the fix is to recreate on 0.28.0+ + v0.27.0 public app returning 503, the fix is to recreate on 0.29.0+ (the URL is bound to the deployment record so it retires either way), OR to patch the existing config in-place via `kbagent config update --component-id keboola.data-apps --config-id ID @@ -313,7 +313,7 @@ success, not a failure. JumpCloud / Auth0) are not yet exposed by the CLI; tracked as a follow-up issue. -- **`data-app secrets-* metadata-only`** (0.28.0+): `secrets-get` NEVER +- **`data-app secrets-* metadata-only`** (0.29.0+): `secrets-get` NEVER echoes the decrypted plaintext under any branch -- the Encryption API is one-way and the CLI does not attempt to decrypt. NOT_FOUND on an absent key never enumerates sibling keys (avoids leaking neighbour @@ -328,7 +328,7 @@ success, not a failure. `parameters.dataApp.secrets`). - **`data-app validate-repo` is GitHub-only**, `--type python-js` only - (0.28.0+): pre-flight Golden-Rule check via the GitHub Trees+Contents + (0.29.0+): pre-flight Golden-Rule check via the GitHub Trees+Contents API. Total <=5 calls regardless of repo size. Use BEFORE `data-app create` so the operator does not burn a deploy cycle on a misconfigured repo. WARNs are advisory unless `--strict` is set; diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index d83cbfa8..649656aa 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -290,7 +290,7 @@ kbagent --json project add --project prod --url https://connection.keboola.com - # Or bulk-onboard from organization (org admin) # Manage token: interactive prompt by default; for CI add --allow-env-manage-token -# alongside KBC_MANAGE_API_TOKEN (required since v0.28.0). +# alongside KBC_MANAGE_API_TOKEN (required since v0.29.0). KBC_MANAGE_API_TOKEN=xxx kbagent --allow-env-manage-token --json org setup --org-id 123 --url https://connection.keboola.com --yes # Or onboard specific projects (any project member, uses Personal Access Token) diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 30756e03..91ad739a 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -21,7 +21,7 @@ All commands support `--json` for structured output. Multi-project flags (`--pro - `project use ALIAS` -- pin `ALIAS` as the persistent default project. Stored as `default_project` in config.json. Overridden at runtime by `KBAGENT_PROJECT=ALIAS` (env, beats pin) and by `--project ALIAS` (CLI flag, beats both) - `project current` -- print the effective default project and its source (`env` / `pin` / `none`). Reports both the env override AND the persisted pin so misconfigurations are visible. Returns `{"alias": null, "source": "none"}` when neither is set -## Project Members & Invitations (since v0.26.1) +## Project Members & Invitations (since v0.29.0) All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the project's Storage token. Allowed roles are exactly `admin`, `guest`, `readOnly`, `share` -- the API self-reports this list in its 400 validation error and `constants.PROJECT_ROLES` mirrors it. @@ -31,17 +31,17 @@ All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the - `project invitation-list --project ALIAS` -- list pending (unaccepted) invitations only. - `project invitation-cancel --project ALIAS --email EMAIL [--invitation-id ID] [--yes]` -- cancel a pending invitation. Without `--invitation-id`, the service resolves it by listing pending invitations and matching `--email` (case-insensitive). 204 No Content on success; `KeboolaApiError(NOT_FOUND)` if the email has no pending invitation. - `project member-remove --project ALIAS --email EMAIL [--yes]` -- destructive: remove an active member. The service resolves `--email` to the numeric `user_id` (case-insensitive) and DELETEs `/manage/projects/{id}/users/{userId}`. Re-add the user via `project invite`. -- `project member-set-role --project ALIAS --email EMAIL --role admin|guest|readOnly|share` -- change an existing member's role. Uses **PATCH** `/manage/projects/{id}/users/{userId}` with `{"role": "..."}`. PUT does *not* work on this endpoint -- pre-v0.26.1 implementations that tried PUT got a misleading 404. +- `project member-set-role --project ALIAS --email EMAIL --role admin|guest|readOnly|share` -- change an existing member's role. Uses **PATCH** `/manage/projects/{id}/users/{userId}` with `{"role": "..."}`. PUT does *not* work on this endpoint -- pre-v0.29.0 implementations that tried PUT got a misleading 404. ## Permission flags (top-level, session-only) - `--deny-writes` -- block all write/destructive/admin operations for this single invocation. Merges with any persisted permission policy; never written to config.json. Exit code 6 (PERMISSION_DENIED) on blocked operations - `--deny-destructive` -- block only destructive operations (delete-table, delete-bucket, terminate-job, etc.) for this invocation. Pure-write ops like create-table stay allowed. Use this when you want to keep build-up capabilities but lock out tear-downs -- `--allow-env-manage-token` -- opt in to reading `KBC_MANAGE_API_TOKEN` from env (default-deny since v0.28.0). Without it the env var is ignored and an interactive hidden prompt is required for `org setup` / `project refresh` / `data-app password`. Closes the AI-exfiltration risk where any subprocess inherits the manage token via env. Session-only; not persisted; no env-var equivalent (intentional, would re-create the hole). REPL forwards this flag to nested invocations the same way it forwards the deny-* flags +- `--allow-env-manage-token` -- opt in to reading `KBC_MANAGE_API_TOKEN` from env (default-deny since v0.29.0). Without it the env var is ignored and an interactive hidden prompt is required for `org setup` / `project refresh` / `data-app password`. Closes the AI-exfiltration risk where any subprocess inherits the manage token via env. Session-only; not persisted; no env-var equivalent (intentional, would re-create the hole). REPL forwards this flag to nested invocations the same way it forwards the deny-* flags - All three flags compose: `kbagent --deny-writes --deny-destructive --allow-env-manage-token ...` is the safest CI-friendly invocation ## Organization -- `org setup --org-id ID --url URL [--dry-run] [--yes]` -- bulk-onboard all projects from an org (org admin; manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+) -- `org setup --project-ids 1,2,3 --url URL [--dry-run] [--yes]` -- onboard specific projects by ID (any project member; manage token / Personal Access Token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+) +- `org setup --org-id ID --url URL [--dry-run] [--yes]` -- bulk-onboard all projects from an org (org admin; manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+) +- `org setup --project-ids 1,2,3 --url URL [--dry-run] [--yes]` -- onboard specific projects by ID (any project member; manage token / Personal Access Token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+) ## Component Discovery - `component list [--project NAME] [--type TYPE] [--query "text"]` -- list/search components (AI-powered with `--query`) @@ -139,7 +139,7 @@ Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, - `data-app start --project NAME --app-id ID [--wait] [--timeout SECONDS]` -- wake an auto-suspended app at the currently-pinned version. Distinct from deploy: does NOT bump configVersion. - `data-app stop --project NAME --app-id ID [--wait] [--timeout SECONDS]` -- stop a running app (URL and Storage config preserved). - `data-app delete --project NAME --app-id ID [--yes]` -- destructive, cascades to Storage config; URL retired permanently. -- `data-app password --project NAME --app-id ID` -- read the simpleAuth password. Manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+. Auto-generated, not rotatable -- delete + recreate to mint a new one. +- `data-app password --project NAME --app-id ID` -- read the simpleAuth password. Manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+. Auto-generated, not rotatable -- delete + recreate to mint a new one. - `data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE' [--secret ...] [--secrets-file PATH] [--branch ID] [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next]` -- encrypt and write `#`-prefixed secrets to `parameters.dataApp.secrets`. Per-project KMS encryption, fail-closed. Read-modify-write at the service layer (NOT Storage `merge=True` -- shallow). Runtime exposes each key as an env var with `#` stripped, `-` -> `_`, uppercased. Adding bumps the Storage version; the running container keeps the OLD config until the next `data-app deploy`. - `data-app secrets-list --project ALIAS --app-id ID [--branch ID] [--show-fingerprint]` -- list secret keys + derived runtime env-var names. Never echoes encrypted ciphertext in full. `--show-fingerprint` opt-in for a short ciphertext fingerprint. - `data-app secrets-get --project ALIAS --app-id ID --key '#KEY' [--branch ID]` -- show metadata for ONE secret. NEVER echoes the decrypted value (Encryption API is one-way). NOT_FOUND on absent key; never enumerates siblings. diff --git a/plugins/kbagent/skills/kbagent/references/data-app-workflow.md b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md index ffbd40a6..3a8956b6 100644 --- a/plugins/kbagent/skills/kbagent/references/data-app-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md @@ -58,7 +58,7 @@ with: ```bash kbagent data-app password --project prod --app-id -# Manage token: interactive prompt by default (since v0.28.0); for CI add +# Manage token: interactive prompt by default (since v0.29.0); for CI add # --allow-env-manage-token alongside KBC_MANAGE_API_TOKEN. Storage token # is read from .kbagent/config.json as usual. ``` @@ -116,7 +116,7 @@ kbagent data-app deploy --project prod --app-id 12345678 \ (rollback). Subsequent deploys without the flag will jump back to the latest. -### Pre-flight repo validation (since v0.28.0) +### Pre-flight repo validation (since v0.29.0) ```bash kbagent data-app validate-repo \ @@ -136,7 +136,7 @@ repo. Public repos: drop `--git-pat-env` and use `--git-public`. Total GitHub call budget per run is ≤5 (1 tree + ≤4 contents) regardless of repo size, so the 60/hour unauth limit rarely fires; pass a PAT for CI loops. -### Manage app-runtime secrets (since v0.28.0) +### Manage app-runtime secrets (since v0.29.0) ```bash # Set two secrets at once. Plaintext values; the CLI encrypts under diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index a68b51b0..529fce82 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -17,16 +17,16 @@ swaps it back into the original name. After merging the branch the original table now carries the typed schema with no downstream config rewrite required. -## `data-app create --auth public` writes the canonical noneProxyAuthorization shape (since v0.28.0; fixes v0.27.0 silent HTTP 503) +## `data-app create --auth public` writes the canonical noneProxyAuthorization shape (since v0.29.0; fixes v0.27.0 silent HTTP 503) - **What changed.** v0.27.0's `--auth public` wrote NO `authorization` key into the Storage config at all. The Keboola app-proxy refused to route to the resulting URL (HTTP 503 / "Service Unavailable") and the UI's "Authentication Type" selector showed blank. Operators got a - silently broken app. v0.28.0 fixes this: `--auth public` now writes + silently broken app. v0.29.0 fixes this: `--auth public` now writes the canonical `noneProxyAuthorization` shape that the kbc-ui exports for the "None" UI option. -- **Exact shape written by 0.28.0:** +- **Exact shape written by 0.29.0:** ```json { "app_proxy": { @@ -62,7 +62,7 @@ flag. Use the Keboola UI to configure them after `data-app create`. Tracked as a follow-up issue. -## `data-app secrets-*` -- per-project KMS, idempotent remove, never decryptable (since v0.28.0) +## `data-app secrets-*` -- per-project KMS, idempotent remove, never decryptable (since v0.29.0) - **Encryption is per-project KMS.** `kbagent data-app secrets-set` calls the project's Encryption API to wrap each plaintext value before @@ -110,7 +110,7 @@ field with the exact redeploy command to run; suppress it with `--no-hint-next` for scripted callers. -## `data-app validate-repo` -- pre-flight against the Golden Rule, GitHub-only (since v0.28.0) +## `data-app validate-repo` -- pre-flight against the Golden Rule, GitHub-only (since v0.29.0) - `kbagent data-app validate-repo --git-repo URL` walks the repo via the GitHub Contents + Trees API and verifies the documented "Golden Rule" @@ -120,7 +120,7 @@ GitHub API calls regardless of repo size (one trees-recursive + up to four contents fetches), so the 60/hour unauthenticated GitHub rate limit is no longer the common-case failure mode. -- **`--type` is restricted to `python-js` in 0.28.0.** Streamlit / +- **`--type` is restricted to `python-js` in 0.29.0.** Streamlit / pure-Python / R / Node-only repos have different layouts (Streamlit does not require the `keboola-config/` tree, for instance) and need per-type canon citations. Tracked as a follow-up. @@ -137,12 +137,12 @@ [issue #240](https://github.com/padak/keboola_agent_cli/issues/240) (needs platform-side API exposure first). -## Manage token: env var is ignored without `--allow-env-manage-token` (since v0.28.0) +## Manage token: env var is ignored without `--allow-env-manage-token` (since v0.29.0) - `KBC_MANAGE_API_TOKEN` is no longer auto-resolved on the three surfaces that consume it (`kbagent org setup`, `kbagent project refresh`, `kbagent data-app password`). Default - behaviour on 0.28.0+ is **default-deny**: the env var is ignored, a + behaviour on 0.29.0+ is **default-deny**: the env var is ignored, a TTY hidden-input prompt is shown instead. With no TTY (CI / cron / systemd / `< /dev/null`) the resolver exits **2** with the message `Error: No manage token available. Run interactively, or pass @@ -223,7 +223,7 @@ container after `autoSuspendAfterSeconds` of inactivity. Hit the URL to wake it (auto-restart triggers a 30-60s cold boot) or run `kbagent data-app start --app-id N`. -## `project invite` "already invited / already member" returns HTTP 400, not 422 (since v0.26.1) +## `project invite` "already invited / already member" returns HTTP 400, not 422 (since v0.29.0) - Re-inviting a user the project already knows about returns HTTP **400** with one of two error strings: @@ -233,11 +233,11 @@ `note="already_invited"` / `"already_member"` -- they are *not* exit-1 failures. Bulk runs (`--from-csv`) count them as `noop` in the summary, not `failed`. -- The 422 heuristic in pre-v0.26.1 orchestrator scripts (`invite_participants.py:25`) +- The 422 heuristic in pre-v0.29.0 orchestrator scripts (`invite_participants.py:25`) is **wrong** for this API. If you write a parallel implementation, key off status_code 400 + the substring marker, not 422. -## `project member-set-role` is PATCH, not PUT (since v0.26.1) +## `project member-set-role` is PATCH, not PUT (since v0.29.0) - The Manage API role-change endpoint is `PATCH /manage/projects/{id}/users/{userId}` with body `{"role": "..."}`. **PUT returns 404** ("resource not found") even @@ -245,7 +245,7 @@ - The kbagent `ManageClient.update_project_member_role` method emits PATCH; any custom code re-implementing the call must do the same. -## `project invite --from-csv` order is not deterministic (since v0.26.1) +## `project invite --from-csv` order is not deterministic (since v0.29.0) - Bulk invitation parallelises via `ThreadPoolExecutor` (default 8 workers). The `rows[]` array in the result is in completion order, not CSV order. @@ -570,7 +570,7 @@ type inventory and examples. - Tokens are always masked in output (e.g. `901-...pt0k`) -- this is normal - Token can be passed via `--token`, `KBC_TOKEN` env var, or interactive prompt -- Manage API token (since v0.28.0): default-deny on env -- via interactive hidden prompt; pass top-level `--allow-env-manage-token` to opt in to `KBC_MANAGE_API_TOKEN`. Never as CLI argument. See the `(since v0.28.0)` entry at the top of this file. +- Manage API token (since v0.29.0): default-deny on env -- via interactive hidden prompt; pass top-level `--allow-env-manage-token` to opt in to `KBC_MANAGE_API_TOKEN`. Never as CLI argument. See the `(since v0.29.0)` entry at the top of this file. - Master token for sharing: `KBC_MASTER_TOKEN_{ALIAS}` (e.g. `KBC_MASTER_TOKEN_PROD`) or `KBC_MASTER_TOKEN` as global fallback. Alias is uppercased, hyphens become underscores. Required for `sharing share` and `sharing unshare`; `sharing list/link/unlink` use regular project tokens. ## MCP tool call gotchas @@ -980,7 +980,7 @@ See [docs/hint-mode.md](../../../../../docs/hint-mode.md) for full documentation - **Forgetting `--json`**: without it, output is human-formatted Rich text, not parseable - **Assuming `data.projects`**: `project list` returns data as a flat list -- **Passing manage token as argument**: use the interactive prompt (default since v0.28.0), or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` env var for CI +- **Passing manage token as argument**: use the interactive prompt (default since v0.29.0), or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` env var for CI - **Polling after branch create**: kbagent already waits for async completion - **Not saving workspace password**: only returned once on creation - **Putting SQL in _config.yml**: SQL transformations must use `transform.sql` with block markers (see above) diff --git a/plugins/kbagent/skills/kbagent/references/member-workflow.md b/plugins/kbagent/skills/kbagent/references/member-workflow.md index 218172bf..061c67cc 100644 --- a/plugins/kbagent/skills/kbagent/references/member-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/member-workflow.md @@ -1,4 +1,4 @@ -# Project Member & Invitation Workflow (since v0.26.1) +# Project Member & Invitation Workflow (since v0.29.0) Closes the long-standing Manage API gap that forced every Keboola-internal automation (most recently the Cuesta-training orchestrator) to bypass kbagent @@ -161,11 +161,11 @@ Permission category: `destructive` (re-adding requires sending a fresh invite). | HTTP 403 manage token lacks org-admin | `KeboolaApiError(ACCESS_DENIED)` | 1 | | HTTP 404 project / invitation not found | `KeboolaApiError(NOT_FOUND)` | 1 | -## When to use the Manage API direct-add (not in v0.26.1) +## When to use the Manage API direct-add (not in v0.29.0) The Manage API also exposes `POST /manage/projects/{id}/users` with body `{"email": "...", "role": "..."}`. This **directly creates a member without sending an email** -- useful for org-internal automation, dangerous for -public-facing flows. v0.26.1 deliberately does NOT expose this path because +public-facing flows. v0.29.0 deliberately does NOT expose this path because its semantics differ from `invite`. If you need it, talk to the maintainers about a future `member-add-direct` command. diff --git a/pyproject.toml b/pyproject.toml index a4ff6b89..4952d430 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-agent-cli" -version = "0.28.0" +version = "0.29.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index f0df35d1..b1dc6d09 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -8,6 +8,27 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.29.0": [ + "BREAKING: `KBC_MANAGE_API_TOKEN` is now ignored by default. The three commands that consume it (`org setup`, `project refresh`, `data-app password`) prompt for the token on a TTY by default. Pass the new top-level flag `--allow-env-manage-token` to restore the legacy env-var behaviour (e.g. for CI/CD). Without the flag and without a TTY, the resolver exits 2 with an actionable message naming the flag. The change closes the AI-exfiltration risk where any subprocess running as the same user (including the AI agent itself) inherits the manage token via env. Migration: prepend `--allow-env-manage-token` to existing CI invocations. Storage tokens (`KBC_TOKEN`) are unaffected. Closes the manage-token UX flagged on #236; supersedes the per-stack design discussed in #238.", + "Security: `resolve_manage_token` (`src/keboola_agent_cli/commands/_helpers.py`) refactored to default-deny env, TTY-first. When the env var is set but the flag is not passed, a one-shot stderr warning fires (`Warning: KBC_MANAGE_API_TOKEN found in environment but ignored. Pass --allow-env-manage-token to opt in.`) and the resolver falls through to the TTY prompt. No cache, no keyring, no temp file -- next invocation prompts again. The bulk-prompt-once contract (`project refresh --all`) is preserved by construction: the resolver lives at command entry, before any per-project loop.", + "New: top-level CLI flag `--allow-env-manage-token` (session-only, mirrors `--deny-writes` / `--deny-destructive`). Plumbed via `ctx.obj['allow_env_manage_token']` and forwarded by the three call sites into `resolve_manage_token(allow_env=...)`. Not persisted, no env-var equivalent (intentional; an env-var equivalent would re-create the AI-exfiltration hole this default-deny is closing).", + "Tests: 12 new (`tests/test_helpers.py::TestResolveManageToken` x7 covering allow_env-True/False x env-set/unset x TTY/non-TTY combinations + token-leak regression pin; `tests/test_manage_token_cli.py::TestAllowEnvManageTokenFlag` x4 covering project-refresh / org-setup / data-app-password through CliRunner with services mocked; `tests/test_manage_token_bulk.py::TestBulkPromptOnce` pinning the contract that `project refresh --all` resolves the token exactly once at command entry, not per-project).", + "Docs: `commands/context.py` AGENT_CONTEXT updated (org-setup example + env-var help block); `CLAUDE.md` convention #12 + global-flag list; `keboola-expert.md` Rule 6 VERSION GATE adds the 0.29.0+ env-flag requirement, tool-selection-matrix updated, new inline-gotcha block; `gotchas.md` new `(since v0.29.0)` entry naming the warning text and the one-line CI fix; `commands-reference.md` updated for `org setup`, `data-app password`, env-var table.", + "New: project member & invitation lifecycle. Closes the long-standing Manage API gap that forced every Keboola-internal automation (most recently `17_CuestaDemo/scripts/replicate_master.py` and `invite_participants.py`) to bypass kbagent and POST raw HTTP at `/manage/projects/{id}/invitations`. Seven new commands under `kbagent project`: `invite` (single-shot or `--from-csv` bulk with `ThreadPoolExecutor` parallelism, default 8 workers), `member-list` (active members, `--include-pending` adds pending invitations), `invitation-list`, `invitation-cancel` (resolves invitation_id by email lookup so callers don't have to), `member-remove` (destructive; resolves user_id by email), `member-set-role` (PATCH `/manage/projects/{id}/users/{userId}` with `{role}`). All seven require `KBC_MANAGE_API_TOKEN`; the manage token is never logged, never persisted, never accepted on the CLI line. Permission registry: `member-remove` is `destructive`, `member-list` / `invitation-list` are `read`, the rest are `admin`.", + "New: role whitelist `PROJECT_ROLES = ('admin', 'guest', 'readOnly', 'share')` in `constants.py`, lifted verbatim from the Manage API's own validation error message (verified empirically on 2026-05-01 against `connection.us-east4.gcp.keboola.com`). Typer enforces the whitelist via `click.Choice` at the command layer; `MemberService` double-checks for defence-in-depth. Invalid role values now fail-fast with `Role 'X' is not valid. Allowed roles are: admin, guest, readOnly, share` instead of letting the API return an opaque 400.", + "New: `MemberService` (`src/keboola_agent_cli/services/member_service.py`) wrapping six new `ManageClient` methods (`create_project_invitation`, `list_project_invitations`, `cancel_project_invitation`, `list_project_members`, `remove_project_member`, `update_project_member_role`). Resolves project alias -> (stack_url, project_id) via `ConfigStore`; resolves email -> numeric user_id / invitation_id by listing + matching case-insensitively. Treats the Manage API's HTTP 400 'already been invited' / 'already a member' responses as `status=noop` rather than errors (the heuristic the orchestrator scripts had to do via substring matching, now typed to `status_code == 400` AND message-substring marker constants). `--from-csv` enforces a single-stack-URL invariant per file (rows referencing multiple stacks raise `ConfigError` upfront).", + "New: hint definitions (`hints/definitions/member.py`) for all seven commands. Both `--hint client` (direct `ManageClient` calls) and `--hint service` (`MemberService` calls) generate runnable Python.", + "New: e2e marker `e2e_invite` (registered in `pyproject.toml`). `make test-e2e-invite` runs `tests/test_e2e.py::test_project_invite_e2e` against a real Manage API; gated on `E2E_MANAGE_TOKEN` + `E2E_INVITE_PROJECT_ID` (skips cleanly when missing). The test invites `ottomansky.max@gmail.com` (override via `E2E_INVITE_EMAIL`) as `guest`, asserts the invitation appears in `invitation-list`, then cancels it -- the same run that proves the system can send confirms it can clean up.", + "Docs (members): new `references/member-workflow.md` (golden paths for single invite, bulk invite, audit, role change, remove). `gotchas.md` gains three `(since v0.29.0)` entries -- 'already invited / already member' returns HTTP 400 not 422; role-change is PATCH not PUT (PUT returns 404 even on real members); bulk-invite ordering is not deterministic (parallel workers). `keboola-expert.md` adds seven matrix rows under 'Project administration' plus a Rule 6 VERSION GATE entry. `commands-reference.md` adds a 'Project members & invitations' section.", + "New: `kbagent data-app secrets-set / secrets-list / secrets-get / secrets-remove` — manage `#`-prefixed app-runtime secrets in `parameters.dataApp.secrets`. Encryption is per-project KMS via the existing `EncryptService` (same fail-closed semantics as `--git-pat-encrypted`: refuses to write plaintext if the Encryption API does not return a project-scoped ciphertext). Read-modify-write at the service layer (NOT Storage `merge=True` — that flag is shallow at the top level only and would clobber sibling keys nested inside `parameters.dataApp.secrets`). The runtime exposes each key as an env var with `#` stripped, `-` replaced with `_`, and uppercased (`#my-api-key` → `MY_API_KEY` per help.keboola.com/data-apps/python-js/). `secrets-get` is metadata-only — never echoes decrypted plaintext to stdout / stderr / logs / change descriptions; the Encryption API is one-way and the CLI does not attempt to decrypt under any branch. `secrets-remove` is idempotent (missing keys exit 0 with `removed: 0`). `secrets-set` warns when a derived env-var name collides with `RESERVED_RUNTIME_ENV_VARS` (KBC_TOKEN, KBC_URL — verified canon floor; full runtime list TODO follow-up). Adding/removing a secret bumps the Storage version but the running container keeps the OLD config until the next `data-app deploy`.", + "New: `kbagent data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] [--type python-js] [--strict]` — pre-flight check that a git repo follows the documented Golden Rule (https://help.keboola.com/data-apps/python-js/) BEFORE `data-app create` so operators don't burn a deploy cycle on a misconfigured repo. Each check emits BLOCKING / WARN / OK with a help-doc citation: `keboola-config/nginx/sites/default.conf` exists, `keboola-config/supervisord/services/app.conf` exists, `pyproject.toml` at root, `keboola-config/setup.sh` content has no `pip install` (BLOCKING per the help canon's pip prohibition) and contains `uv sync` if `pyproject.toml` declares deps, `requires-python` consistent with the runtime image (when the pin is available), nginx `proxy_pass` port matches `app.conf` declared port. Uses `GET /repos/{owner}/{repo}/git/trees/{ref}?recursive=1` (one call) + up to 4 `GET .../contents/{path}` for files whose contents the rules need to inspect — total ≤5 GitHub API calls (1 tree + 0-4 contents) regardless of repo size, sidesteps the 60/hour unauth rate limit for typical use. `--git-pat-env` / `--git-pat-file` raises the limit to 5,000/hour. Read-only; never touches a Keboola project. `--type` is restricted to `python-js` in 0.29.0; streamlit / pure-Python / R / Node-only follow-up.", + "New: `RepoValidateService` (`src/keboola_agent_cli/services/repo_validate_service.py`) — pure validation function `validate_keboola_repo(snapshot, type_, runtime_python_pin)` plus a tiny `GitHubContentsClient` (HTTPS GET to `api.github.com`, optional bearer PAT, no token persistence). Service module is the only place GitHub HTTP lives; the rest of kbagent stays Keboola-API-only. (Future refactor: extract to `src/keboola_agent_cli/github_client.py` to follow the existing 3-layer architecture; `github_client_factory` injection preserves test coverage today.)", + "New: `ErrorCode` entries `DATA_APP_INVALID_SECRET`, `DATA_APP_INVALID_REPO`, `DATA_APP_REPO_VALIDATION_BLOCKING`. Permission registry entries `data-app.secrets-set` (write), `data-app.secrets-list` / `data-app.secrets-get` (read), `data-app.secrets-remove` (destructive — removing a secret can break a running app), `data-app.validate-repo` (read).", + "New: `--hint client/service` for all five new data-app commands. `secrets-get` hint snippet asserts the metadata-only contract; `validate-repo` snippet uses `RepoValidateService.validate_repo(...)` and the hint comment notes that GitHub-side detail is not shown.", + "Fix: `kbagent data-app create --auth public` now writes the canonical `noneProxyAuthorization` shape (kbc-ui exact constant: `auth_providers: []` + `auth_rules: [{type: pathPrefix, value: /, auth_required: false}]`). v0.27.0 wrote NO `authorization` key when `--auth public`, leaving the Keboola app-proxy unable to route (HTTP 503) and the UI Authentication Type selector blank — silently broken. Authoritative source: the public backend validator at `keboola/job-queue-job-configuration` `AppProxyDefinition.php` (when `auth_required=false`, `auth` MUST NOT be set). The private `keboola/ui` repo `apps/kbc-ui/src/scripts/modules/data-apps/constants.ts` corroborates: its `noneProxyAuthorization` constant exports this exact shape for the None UI option (Keboola org members can verify; external readers rely on the validator). Live-validated end-to-end on a real project: HTTP 200 on the resulting URL, written block bit-identical to canon, UI auth selector now shows None pre-selected. Existing `--auth password` behaviour unchanged.", + "Tests (data-app secrets / validate-repo): 27 secrets service tests + 20 validate-repo service tests + 22 CLI tests (13 secrets/validate-repo CLI methods + 9 hint-compile AST-parse cases) + 4 new auth-block tests (`TestDataAppCreateAuthBlock` asserts both `--auth public` and `--auth password` write the canonical shape on POST `/apps` AND PUT Storage). E2E coverage in `tests/test_e2e.py::TestE2EDataAppLifecycle::test_data_app_secrets_round_trip` and `::test_data_app_validate_repo_against_public_repo` exercises the full path. Sibling-preservation regression test for `secrets-set` asserts every untouched key under `parameters.dataApp.secrets`, `parameters.dataApp` (slug, git block), `parameters` (id), and the top-level config (`runtime`, `authorization`, `storage`) is preserved bit-identical after the read-modify-write.", + 'Plugin: `keboola-expert.md` matrix gains five new data-app rows (one per `secrets-set / -list / -get / -remove + validate-repo`); §1 Rule 6 VERSION GATE example updated for `secrets / validate-repo need 0.29.0+`. New `(since v0.29.0)` `gotchas.md` entries: (a) secrets are per-project KMS encrypted, `secrets-remove` on missing key is exit 0, `secrets-get` never echoes decrypted plaintext, `#KBC_TOKEN` is silently shadowed by the runtime; (b) `validate-repo` GitHub-only Golden-Rule check; (c) `--auth public` writes the canonical `noneProxyAuthorization` shape (fixes v0.27.0 silent 503). New "Managing app-runtime secrets" + "Pre-flight repo validation" recipe sections in `data-app-workflow.md`. Logs / auto-log-dump deferred to issue #240 (the Data Science API does not expose Terminal Logs as JSON per help canon).', + ], "0.28.0": [ 'Fix: `kbagent config update` now auto-normalizes `parameters.blocks[].codes[].script` from string to array before pushing to the Storage API. Closes #245. The Storage API silently accepts a string for `script` while the runtime schema validator requires an array (`Invalid type for path "root.parameters.blocks.0.codes.X.script". Expected "array", but got "string"`); the broken push lands silently and crashes only at job-run time, often hours later, with no attribution back to the offending write. The CLI now closes the gap on the write side: SQL transformations (`keboola.snowflake-transformation`, `keboola.synapse-transformation`, `keboola.oracle-transformation`, `keboola.redshift-sql-transformation`, `keboola.google-bigquery-transformation`, `keboola.duckdb-transformation`, plus fragment-fallback for self-hosted variants like `*-exasol-transformation` / `*-teradata-transformation`) get statement-level split via the existing `split_statements()` state-machine (respects `\'...\'` / `"..."` / `$$...$$` / `--` / `#` / `//` / `/* ... */`); Python / R / `kds-team.app-custom-python` and any other component sharing the schema get a single-element array wrap. Already-array `script` values pass through unchanged.', 'Observability: every normalization is surfaced -- the JSON envelope gains a `normalizations: [{path, action: "sql_split"|"wrap_array", before_type, after_type, after_length}]` field per write (and on `--dry-run` the `new_configuration` reflects the post-normalize shape). Human mode prints a yellow `Auto-normalized N script field(s) to array (string -> list). See --json for details.` warning followed by a per-element trace, so the silent fix is observable to operators and AI agents alike. Default behaviour is silent normalize -- the issue\'s preferred design -- because the Keboola UI splitter and `keboola-as-code` produce the same array shape kbagent now writes; the audit fields exist precisely so callers who want to detect "my agent produced a string" can.', @@ -17,19 +38,6 @@ "New: `kbagent storage swap-tables --project P --table-id A --target-table-id B [--branch ID] [--dry-run] [--yes]` -- thin wrapper around the Storage API `POST /v2/storage/branch/{branch}/tables/{id}/swap` endpoint. Both tables exchange physical positions; aliases are NOT transferred (they keep pointing at the same physical position and therefore expose the OTHER table's data after the swap). The Storage API queues this as an async storage job (`operationName: tableSwap`); the client polls to completion before returning, so callers can rely on the schemas already being exchanged on return (~10s observed on Snowflake). The API restricts this to dev branches; the service refuses with exit 5 (`ConfigError`) before any HTTP call when neither `--branch` nor an active branch (via `kbagent branch use`) is set. Same-source-and-target IDs also rejected pre-flight. The use case is: AI agent profiles a typeless table, builds a typed rebuild via CTAS in a workspace, then swaps the typed copy into the original name without touching downstream config references that point at the original table ID. Permission classification: `destructive` (gated behind `--allow-destructive`). The PHP reference client docstring claims a synchronous response, but live calls against the platform consistently return a queued job -- this client polls the job to completion to make the `delete_table` / `create_table` semantics consistent. Companion entry in `storage-types-workflow.md` explains the typify-via-CTAS pattern; gotchas + commands-reference + agent prompt all updated.", "Tests (swap-tables): `tests/test_storage_swap.py` (14 tests) covers all three layers -- HTTP shape (POST + body + URL encoding + immediate-success path + async-poll path + 4xx propagation via `pytest_httpx`), service business logic (success, dry-run, branch enforcement, same-id guard, API error propagation, unknown project), and CLI integration (JSON happy path, dry-run, explicit `--branch` overrides active, missing-branch error path with exit 5). E2E coverage in `tests/test_e2e.py::TestE2EStorageSwapTables` runs three scenarios against a live API: live swap of two tables with different VARCHAR lengths verifies definitions exchange in both directions; dry-run skips API call and `lastChangeDate` is unchanged; and the production-rejection path (no branch + no active branch) returns exit 5.", "Plugin docs: new `plugins/kbagent/skills/kbagent/references/typify-table-workflow.md` -- end-to-end procedure for converting a typeless Storage table (every column `STRING(16M)`) into one with proper Snowflake / BigQuery native types. 8 phases: (0) decide-or-skip rubric; (1) isolate in dev branch; (2) profile the typeless table in a workspace with length / cardinality / parse-failure / scale-precision queries + decision matrix mapping profile signals to Snowflake types; (3) build typed sibling via `storage create-table` + copy data via in-workspace INSERT or SQL transformation, with row-count / NULL-count verification; (4) validate downstream consumers in the dev branch (search configs that reference the table, run a representative transformation against the typeless source as baseline); (5) `swap-tables` (dry-run + actual + verify); (6) re-run downstream as smoke test; (7) cleanup the sibling after merge; (8) handoff protocol -- structured summary the AI agent hands to the user with phase-by-phase receipts, the merge URL, and rollback / cleanup commands. Cross-references `storage-types-workflow.md`, `branch-workflow.md`, `workspace-workflow.md`, `gotchas.md`. SKILL.md workflow-references table gains the new entry.", - "BREAKING: `KBC_MANAGE_API_TOKEN` is now ignored by default. The three commands that consume it (`org setup`, `project refresh`, `data-app password`) prompt for the token on a TTY by default. Pass the new top-level flag `--allow-env-manage-token` to restore the legacy env-var behaviour (e.g. for CI/CD). Without the flag and without a TTY, the resolver exits 2 with an actionable message naming the flag. The change closes the AI-exfiltration risk where any subprocess running as the same user (including the AI agent itself) inherits the manage token via env. Migration: prepend `--allow-env-manage-token` to existing CI invocations. Storage tokens (`KBC_TOKEN`) are unaffected. Closes the manage-token UX flagged on #236; supersedes the per-stack design discussed in #238.", - "Security: `resolve_manage_token` (`src/keboola_agent_cli/commands/_helpers.py`) refactored to default-deny env, TTY-first. When the env var is set but the flag is not passed, a one-shot stderr warning fires (`Warning: KBC_MANAGE_API_TOKEN found in environment but ignored. Pass --allow-env-manage-token to opt in.`) and the resolver falls through to the TTY prompt. No cache, no keyring, no temp file -- next invocation prompts again. The bulk-prompt-once contract (`project refresh --all`) is preserved by construction: the resolver lives at command entry, before any per-project loop.", - "New: top-level CLI flag `--allow-env-manage-token` (session-only, mirrors `--deny-writes` / `--deny-destructive`). Plumbed via `ctx.obj['allow_env_manage_token']` and forwarded by the three call sites into `resolve_manage_token(allow_env=...)`. Not persisted, no env-var equivalent (intentional; an env-var equivalent would re-create the AI-exfiltration hole this default-deny is closing).", - "Tests: 12 new (`tests/test_helpers.py::TestResolveManageToken` x7 covering allow_env-True/False x env-set/unset x TTY/non-TTY combinations + token-leak regression pin; `tests/test_manage_token_cli.py::TestAllowEnvManageTokenFlag` x4 covering project-refresh / org-setup / data-app-password through CliRunner with services mocked; `tests/test_manage_token_bulk.py::TestBulkPromptOnce` pinning the contract that `project refresh --all` resolves the token exactly once at command entry, not per-project).", - "Docs: `commands/context.py` AGENT_CONTEXT updated (org-setup example + env-var help block); `CLAUDE.md` convention #12 + global-flag list; `keboola-expert.md` Rule 6 VERSION GATE adds the 0.28.0+ env-flag requirement, tool-selection-matrix updated, new inline-gotcha block; `gotchas.md` new `(since v0.28.0)` entry naming the warning text and the one-line CI fix; `commands-reference.md` updated for `org setup`, `data-app password`, env-var table.", - "New: `kbagent data-app secrets-set / secrets-list / secrets-get / secrets-remove` — manage `#`-prefixed app-runtime secrets in `parameters.dataApp.secrets`. Encryption is per-project KMS via the existing `EncryptService` (same fail-closed semantics as `--git-pat-encrypted`: refuses to write plaintext if the Encryption API does not return a project-scoped ciphertext). Read-modify-write at the service layer (NOT Storage `merge=True` — that flag is shallow at the top level only and would clobber sibling keys nested inside `parameters.dataApp.secrets`). The runtime exposes each key as an env var with `#` stripped, `-` replaced with `_`, and uppercased (`#my-api-key` → `MY_API_KEY` per help.keboola.com/data-apps/python-js/). `secrets-get` is metadata-only — never echoes decrypted plaintext to stdout / stderr / logs / change descriptions; the Encryption API is one-way and the CLI does not attempt to decrypt under any branch. `secrets-remove` is idempotent (missing keys exit 0 with `removed: 0`). `secrets-set` warns when a derived env-var name collides with `RESERVED_RUNTIME_ENV_VARS` (KBC_TOKEN, KBC_URL — verified canon floor; full runtime list TODO follow-up). Adding/removing a secret bumps the Storage version but the running container keeps the OLD config until the next `data-app deploy`.", - "New: `kbagent data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] [--type python-js] [--strict]` — pre-flight check that a git repo follows the documented Golden Rule (https://help.keboola.com/data-apps/python-js/) BEFORE `data-app create` so operators don't burn a deploy cycle on a misconfigured repo. Each check emits BLOCKING / WARN / OK with a help-doc citation: `keboola-config/nginx/sites/default.conf` exists, `keboola-config/supervisord/services/app.conf` exists, `pyproject.toml` at root, `keboola-config/setup.sh` content has no `pip install` (BLOCKING per the help canon's pip prohibition) and contains `uv sync` if `pyproject.toml` declares deps, `requires-python` consistent with the runtime image (when the pin is available), nginx `proxy_pass` port matches `app.conf` declared port. Uses `GET /repos/{owner}/{repo}/git/trees/{ref}?recursive=1` (one call) + up to 4 `GET .../contents/{path}` for files whose contents the rules need to inspect — total ≤5 GitHub API calls (1 tree + 0-4 contents) regardless of repo size, sidesteps the 60/hour unauth rate limit for typical use. `--git-pat-env` / `--git-pat-file` raises the limit to 5,000/hour. Read-only; never touches a Keboola project. `--type` is restricted to `python-js` in 0.28.0; streamlit / pure-Python / R / Node-only follow-up.", - "New: `RepoValidateService` (`src/keboola_agent_cli/services/repo_validate_service.py`) — pure validation function `validate_keboola_repo(snapshot, type_, runtime_python_pin)` plus a tiny `GitHubContentsClient` (HTTPS GET to `api.github.com`, optional bearer PAT, no token persistence). Service module is the only place GitHub HTTP lives; the rest of kbagent stays Keboola-API-only.", - "New: `ErrorCode` entries `DATA_APP_INVALID_SECRET`, `DATA_APP_INVALID_REPO`, `DATA_APP_REPO_VALIDATION_BLOCKING`. Permission registry entries `data-app.secrets-set` (write), `data-app.secrets-list` / `data-app.secrets-get` (read), `data-app.secrets-remove` (destructive — removing a secret can break a running app), `data-app.validate-repo` (read).", - "New: `--hint client/service` for all five new commands. `secrets-get` hint snippet asserts the metadata-only contract; `validate-repo` snippet uses `RepoValidateService.validate_repo(...)` and the hint comment notes that GitHub-side detail is not shown.", - "Fix: `kbagent data-app create --auth public` now writes the canonical `noneProxyAuthorization` shape (kbc-ui exact constant: `auth_providers: []` + `auth_rules: [{type: pathPrefix, value: /, auth_required: false}]`). v0.27.0 wrote NO `authorization` key when `--auth public`, leaving the Keboola app-proxy unable to route (HTTP 503) and the UI Authentication Type selector blank — silently broken. Authoritative source: the public backend validator at `keboola/job-queue-job-configuration` `AppProxyDefinition.php` (when `auth_required=false`, `auth` MUST NOT be set). The private `keboola/ui` repo `apps/kbc-ui/src/scripts/modules/data-apps/constants.ts` corroborates: its `noneProxyAuthorization` constant exports this exact shape for the None UI option (Keboola org members can verify; external readers rely on the validator). Live-validated end-to-end on a real project: HTTP 200 on the resulting URL, written block bit-identical to canon, UI auth selector now shows None pre-selected. Existing `--auth password` behaviour unchanged.", - "Tests: 27 secrets service tests + 20 validate-repo service tests + 22 CLI tests (13 secrets/validate-repo CLI methods + 9 hint-compile AST-parse cases) + 4 new auth-block tests (`TestDataAppCreateAuthBlock` asserts both `--auth public` and `--auth password` write the canonical shape on POST `/apps` AND PUT Storage). 2505 total tests green. Sibling-preservation regression test for `secrets-set` asserts every untouched key under `parameters.dataApp.secrets`, `parameters.dataApp` (slug, git block), `parameters` (id), and the top-level config (`runtime`, `authorization`, `storage`) is preserved bit-identical after the read-modify-write.", - 'Plugin: `keboola-expert.md` matrix gains five new rows (one per `secrets-set / -list / -get / -remove + validate-repo`); §1 Rule 6 VERSION GATE example updated for `secrets / validate-repo need 0.28.0+`. New `(since v0.28.0)` `gotchas.md` entries: (a) secrets are per-project KMS encrypted, `secrets-remove` on missing key is exit 0, `secrets-get` never echoes decrypted plaintext, `#KBC_TOKEN` is silently shadowed by the runtime; (b) `validate-repo` GitHub-only Golden-Rule check; (c) `--auth public` writes the canonical `noneProxyAuthorization` shape (fixes v0.27.0 silent 503). New "Managing app-runtime secrets" + "Pre-flight repo validation" recipe sections in `data-app-workflow.md`. Logs / auto-log-dump deferred to issue #240 (the Data Science API does not expose Terminal Logs as JSON per help canon).', ], "0.27.0": [ "New: `kbagent data-app` command group — first-class lifecycle for Keboola data apps (`keboola.data-apps` Storage component + Data Science API `/apps`). Eight subcommands: `list`, `detail`, `create`, `deploy`, `start`, `stop`, `delete`, `password`. The CLI encapsulates the **§9 redeploy contract** (always sends the `{desiredState=running, configVersion, restartIfRunning=true}` trio together; without it, `PATCH /apps {desiredState:running}` silently pins to the empty-shell v2 and the runner errors `dataApp.git.repository is required in /data/config.json`), per-project KMS encryption of git PATs (refuses to write plaintext if the Encryption API does not return a project-scoped ciphertext), cleanup-in-finally on initial-deploy failure (orphan shell deleted by default; `--keep-on-failure` opts out), and a poll loop that respects pitfall #1 — `state == stopped` is NOT terminal while `desiredState == running` (the platform transitions `created → stopped → starting → running` during initial deploy). `data-app create` accepts `--git-pat-env VAR` (recommended; no argv leak), `--git-pat-file PATH`, or `--git-pat-encrypted KBC::Project...` (must be encrypted under THIS project's KMS — ciphertext does not cross projects).", @@ -39,14 +47,6 @@ "Tests: 30 service-level tests in `tests/test_data_app_service.py` (validation, dry-run, happy-path orchestration, cleanup-in-finally, encryption-failure-aborts-loud, poll-loop semantics including the transient-stopped invariant), 10 CLI tests in `tests/test_data_app_cli.py` (mutual-exclusion validation, dual JSON+human output, `--yes` for delete, manage-token forwarding for password without leaking the token to stdout/stderr).", "Plugin: new `data-app-workflow.md` reference + two `(since v0.27.0)` gotcha entries (the §9 redeploy contract; cross-project KMS ciphertext mismatch). `keboola-expert.md` matrix gains five rows (`create`, `deploy`, `start`, `stop`, `delete`).", ], - "0.26.1": [ - "New: project member & invitation lifecycle. Closes the long-standing Manage API gap that forced every Keboola-internal automation (most recently `17_CuestaDemo/scripts/replicate_master.py` and `invite_participants.py`) to bypass kbagent and POST raw HTTP at `/manage/projects/{id}/invitations`. Seven new commands under `kbagent project`: `invite` (single-shot or `--from-csv` bulk with `ThreadPoolExecutor` parallelism, default 8 workers), `member-list` (active members, `--include-pending` adds pending invitations), `invitation-list`, `invitation-cancel` (resolves invitation_id by email lookup so callers don't have to), `member-remove` (destructive; resolves user_id by email), `member-set-role` (PATCH `/manage/projects/{id}/users/{userId}` with `{role}`). All seven require `KBC_MANAGE_API_TOKEN`; the manage token is never logged, never persisted, never accepted on the CLI line. Permission registry: `member-remove` is `destructive`, `member-list` / `invitation-list` are `read`, the rest are `admin`.", - "New: role whitelist `PROJECT_ROLES = ('admin', 'guest', 'readOnly', 'share')` in `constants.py`, lifted verbatim from the Manage API's own validation error message (verified empirically on 2026-05-01 against `connection.us-east4.gcp.keboola.com`). Typer enforces the whitelist via `click.Choice` at the command layer; `MemberService` double-checks for defence-in-depth. Invalid role values now fail-fast with `Role 'X' is not valid. Allowed roles are: admin, guest, readOnly, share` instead of letting the API return an opaque 400.", - "New: `MemberService` (`src/keboola_agent_cli/services/member_service.py`) wrapping six new `ManageClient` methods (`create_project_invitation`, `list_project_invitations`, `cancel_project_invitation`, `list_project_members`, `remove_project_member`, `update_project_member_role`). Resolves project alias -> (stack_url, project_id) via `ConfigStore`; resolves email -> numeric user_id / invitation_id by listing + matching case-insensitively. Treats the Manage API's HTTP 400 'already been invited' / 'already a member' responses as `status=noop` rather than errors (the heuristic the orchestrator scripts had to do via substring matching, now typed to `status_code == 400` AND message-substring marker constants). `--from-csv` enforces a single-stack-URL invariant per file (rows referencing multiple stacks raise `ConfigError` upfront).", - "New: hint definitions (`hints/definitions/member.py`) for all seven commands. Both `--hint client` (direct `ManageClient` calls) and `--hint service` (`MemberService` calls) generate runnable Python.", - "New: e2e marker `e2e_invite` (registered in `pyproject.toml`). `make test-e2e-invite` runs `tests/test_e2e.py::test_project_invite_e2e` against a real Manage API; gated on `E2E_MANAGE_TOKEN` + `E2E_INVITE_PROJECT_ID` (skips cleanly when missing). The test invites `ottomansky.max@gmail.com` (override via `E2E_INVITE_EMAIL`) as `guest`, asserts the invitation appears in `invitation-list`, then cancels it -- the same run that proves the system can send confirms it can clean up.", - "Docs: new `references/member-workflow.md` (golden paths for single invite, bulk invite, audit, role change, remove). `gotchas.md` gains three `(since v0.26.1)` entries -- 'already invited / already member' returns HTTP 400 not 422; role-change is PATCH not PUT (PUT returns 404 even on real members); bulk-invite ordering is not deterministic (parallel workers). `keboola-expert.md` adds seven matrix rows under 'Project administration' plus a Rule 6 VERSION GATE entry. `commands-reference.md` adds a 'Project members & invitations' section.", - ], "0.26.0": [ "New: `kbagent config set-default-bucket --bucket BUCKET_ID | --clear [--dry-run] [--branch ID]` -- discoverable wrapper around the raw-mode `storage.output.default_bucket` workaround documented at https://keboola.atlassian.net/wiki/spaces/SUP/pages/3770155030/ (epic KBCP-108). Read-modify-write that preserves all sibling keys under `storage.output` and the rest of the configuration. Same-value writes short-circuit with `{\"changed\": false}` (no API call, no version bump). `--clear` removes only the `default_bucket` key, leaving an empty `storage.output: {}` if no other siblings live there (intentional -- mirrors `set_nested_value`'s parent-creation semantics; Storage API treats `output: {}` and missing `output` identically as 'use the auto-derived bucket'). Live-validated end-to-end on three component types -- row-based GCS extractor, root-only `keboola.ex-cnb-exchange-rates`, and `ex-generic-v2` with multiple jobs -- output tables routed to the configured bucket at job runtime in every case. The per-table `destination` override (the second method shown in the support article) keeps using the existing `kbagent config update --set 'storage.output.tables=[...]'` -- no new wrapper there because per-table mappings have many fields that don't fit a single-purpose flag.", "Fix: `kbagent sync pull --with-samples` no longer crashes with `TypeError: '>' not supported between instances of 'NoneType' and 'int'` when one or more tables in the project return `rowsCount: null` from the Storage API (typical for newly-created or empty tables on some backends, reproduced live against `kosik-sales`). `dict.get(\"rowsCount\", 0)` returns the default `0` only when the key is **missing** -- if the key is present with a `null` value, `.get()` returns `None`, and the `> 0` comparison crashed Python 3 before any sample was fetched. The filter and sort key in `SyncService._fetch_samples()` now coerce `None` to `0` via a small `_rows()` helper used in both places (`t.get(\"rowsCount\") or 0`), so empty/null-rowcount tables are gracefully skipped exactly like `rowsCount: 0` ones. Closes #233.", diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index bad48673..4c967e5a 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -94,7 +94,7 @@ Print the effective default project and its source (env / pin / none). Resolution order for single-project operations: --project > KBAGENT_PROJECT > pin. -### Project Members & Invitations (since v0.26.1) +### Project Members & Invitations (since v0.29.0) Requires KBC_MANAGE_API_TOKEN (Manage API auth). Allowed roles: admin, guest, readOnly, share. @@ -452,7 +452,7 @@ Use --org-id OR --project-ids (at least one required). Token via interactive hidden prompt by default; pass top-level --allow-env-manage-token to read KBC_MANAGE_API_TOKEN from env (CI/CD). - Default-deny since 0.28.0 -- closes the AI-exfiltration risk where + Default-deny since 0.29.0 -- closes the AI-exfiltration risk where subprocesses inherit the manage token via env. ### Flows (Orchestrator + Conditional) @@ -630,7 +630,7 @@ Retrieve the simpleAuth password. Requires the Manage API token in addition to the project's Storage token. Token is read from interactive hidden prompt by default; pass top-level --allow-env-manage-token to - use KBC_MANAGE_API_TOKEN from env (default-deny since 0.28.0). Never + use KBC_MANAGE_API_TOKEN from env (default-deny since 0.29.0). Never persisted, never logged. Password is auto-generated at create time and CANNOT be rotated -- delete and recreate the app to mint a new one. @@ -811,7 +811,7 @@ KBC_TOKEN Storage API token (fallback for --token) KBC_STORAGE_API_URL Default stack URL (fallback for --url) KBC_MANAGE_API_TOKEN Manage API token (org setup, project refresh, data-app password). - Default-DENY since 0.28.0: pass --allow-env-manage-token + Default-DENY since 0.29.0: pass --allow-env-manage-token to opt in, otherwise this var is ignored and a TTY prompt is required. Closes AI-exfiltration via subprocess env. KBC_MASTER_TOKEN Master token for sharing ops (global fallback) diff --git a/src/keboola_agent_cli/commands/project.py b/src/keboola_agent_cli/commands/project.py index f202d2a8..38c3e55e 100644 --- a/src/keboola_agent_cli/commands/project.py +++ b/src/keboola_agent_cli/commands/project.py @@ -859,7 +859,7 @@ def project_invite( ) return - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: @@ -922,7 +922,7 @@ def project_member_list( ) return - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: result = service.list_members( @@ -960,7 +960,7 @@ def project_invitation_list( ) return - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: result = service.list_invitations(manage_token=manage_token, alias=project) @@ -1014,7 +1014,7 @@ def project_invitation_cancel( formatter.console.print("Aborted.") raise typer.Exit(code=0) - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: result = service.cancel_invitation( @@ -1069,7 +1069,7 @@ def project_member_remove( formatter.console.print("Aborted.") raise typer.Exit(code=0) - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: result = service.remove_member( @@ -1122,7 +1122,7 @@ def project_member_set_role( ) return - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: result = service.set_member_role( diff --git a/src/keboola_agent_cli/services/repo_validate_service.py b/src/keboola_agent_cli/services/repo_validate_service.py index 2e71ac76..afa96a0c 100644 --- a/src/keboola_agent_cli/services/repo_validate_service.py +++ b/src/keboola_agent_cli/services/repo_validate_service.py @@ -23,6 +23,14 @@ Scope of this PR: ``--type python-js`` only. Streamlit / pure-Python / R repo layouts differ and need their own per-type canon citations -- a follow-up PR adds them. + +TODO: extract ``GitHubContentsClient`` (defined below) into a top-level +``src/keboola_agent_cli/github_client.py`` module that inherits from +``BaseHttpClient`` so this stays consistent with the 3-layer architecture +(LAYER 3 = clients, LAYER 2 = services). The ``github_client_factory`` +dependency-injection pattern in ``RepoValidateService`` already isolates +the client for testing; the extraction is a pure refactor with no +behaviour change. """ from __future__ import annotations diff --git a/tests/test_e2e.py b/tests/test_e2e.py index b18fc679..52985bb2 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -45,6 +45,7 @@ from keboola_agent_cli.cli import app from keboola_agent_cli.client import KeboolaClient from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.models import ProjectConfig # --------------------------------------------------------------------------- # Environment & skip logic @@ -6139,8 +6140,12 @@ def test_swap_without_branch_is_rejected(self) -> None: reason=( f"requires {ENV_TOKEN} + {ENV_DATA_APP_GIT_REPO_PRIVATE} + " f"{ENV_DATA_APP_GIT_USER} + {ENV_DATA_APP_GIT_PAT}" + ), +) + + # ────────────────────────────────────────────────────────────────────── -# Project invite E2E (since v0.26.1) +# Project invite E2E (since v0.29.0) # # Opt-in via `make test-e2e-invite`. Default-skipped in `make test-e2e` because # (a) it sends a real invitation email and (b) it depends on a separate manage @@ -6148,7 +6153,6 @@ def test_swap_without_branch_is_rejected(self) -> None: # ────────────────────────────────────────────────────────────────────── -ENV_MANAGE_TOKEN = "E2E_MANAGE_TOKEN" ENV_INVITE_PROJECT_ID = "E2E_INVITE_PROJECT_ID" ENV_INVITE_EMAIL = "E2E_INVITE_EMAIL" DEFAULT_INVITE_EMAIL = "ottomansky.max@gmail.com" @@ -6354,6 +6358,210 @@ def test_data_app_lifecycle_private_and_redeploy(self) -> None: )["data"] assert deploy["config_version"], "deploy must pin a configVersion" + @skip_without_data_app_public + def test_data_app_secrets_round_trip(self) -> None: + """secrets-set -> secrets-list -> secrets-get -> secrets-remove on a real app. + + Uses --no-deploy + --auth public to mint a cheap shell app, then + verifies the four-step lifecycle: + 1. set: encrypts via per-project KMS, writes to parameters.dataApp.secrets. + 2. list: enumerates keys + derived runtime env-var names; never decrypts. + 3. get: returns metadata only (NEVER plaintext). + 4. remove: idempotent (second remove returns removed: 0, exit 0). + + The decrypted plaintext value must NEVER appear in any CLI output. + """ + repo = os.environ[ENV_DATA_APP_GIT_REPO_PUBLIC] + slug = f"e2e-secrets-{RUN_ID}"[:60] + secret_key = "#E2E_TEST_KEY" + secret_plaintext = "supersecret-do-not-leak" + + _step(1, "Create shell app for secrets round-trip") + create = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "create", + "--project", + self.alias, + "--name", + f"E2E Secrets {RUN_ID}", + "--slug", + slug, + "--git-repo", + repo, + "--git-public", + "--auth", + "public", + "--no-deploy", + ], + ) + )["data"] + app_id = create["id"] + self._created_app_ids.append(app_id) + + _step(2, "secrets-set: encrypt and write") + set_result = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-set", + "--project", + self.alias, + "--app-id", + app_id, + "--secret", + f"{secret_key}={secret_plaintext}", + "--no-hint-next", + ], + ) + ) + assert secret_plaintext not in set_result["raw_output"], ( + "Plaintext value MUST NEVER appear in secrets-set output" + ) + + _step(3, "secrets-list: enumerate keys (never decrypts)") + list_result = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-list", + "--project", + self.alias, + "--app-id", + app_id, + ], + ) + ) + keys_in_list = [s["key"] for s in list_result["data"]["secrets"]] + assert secret_key in keys_in_list, ( + f"secrets-list must surface the just-written key; got {keys_in_list}" + ) + assert secret_plaintext not in list_result["raw_output"], ( + "Plaintext value MUST NEVER appear in secrets-list output" + ) + + _step(4, "secrets-get: metadata only (never plaintext)") + get_result = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-get", + "--project", + self.alias, + "--app-id", + app_id, + "--key", + secret_key, + ], + ) + ) + assert get_result["data"]["key"] == secret_key + assert secret_plaintext not in get_result["raw_output"], ( + "secrets-get MUST NEVER echo the decrypted plaintext (Encryption API is one-way)" + ) + + _step(5, "secrets-remove: first call removes the key") + remove_result = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-remove", + "--project", + self.alias, + "--app-id", + app_id, + "--key", + secret_key, + "--yes", + ], + ) + ) + assert remove_result["data"]["removed"] == 1, "first remove must report removed=1" + + _step(6, "secrets-remove: second call is idempotent (removed=0)") + idempotent = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-remove", + "--project", + self.alias, + "--app-id", + app_id, + "--key", + secret_key, + "--yes", + ], + ) + ) + assert idempotent["data"]["removed"] == 0, ( + "second remove of the same key must be idempotent (removed=0, exit 0)" + ) + + +# --------------------------------------------------------------------------- +# Data-app validate-repo (since v0.29.0) -- GitHub-only, no Keboola creds needed +# --------------------------------------------------------------------------- + + +@pytest.mark.e2e +def test_data_app_validate_repo_against_public_repo(tmp_path: Path) -> None: + """validate-repo against a real public GitHub repo. + + Does NOT require Keboola credentials -- the command only hits GitHub. + Uses a known-public Keboola example repo via E2E_DATA_APP_GIT_REPO_PUBLIC + when set; otherwise skipped (no hard-coded URL to keep the test + independent of upstream-template renames). + + Asserts the command exits cleanly and emits the expected envelope + shape (status + checks list with BLOCKING / WARN / OK severities). + """ + repo = os.environ.get(ENV_DATA_APP_GIT_REPO_PUBLIC) + if not repo: + pytest.skip(f"requires {ENV_DATA_APP_GIT_REPO_PUBLIC} (any public Keboola data-app repo)") + + config_dir = tmp_path / "kbagent-config" + config_dir.mkdir() + + result = _invoke( + config_dir, + [ + "--json", + "data-app", + "validate-repo", + "--git-repo", + repo, + "--git-public", + "--type", + "python-js", + ], + ) + # Exit 0 when no BLOCKING; exit 1 when at least one BLOCKING. Either is + # a successful invocation -- the assertion is on shape, not verdict. + assert result.exit_code in (0, 1), result.output + + body = json.loads(result.output) + assert body["status"] in ("ok", "error"), f"unexpected status: {body['status']}" + if body["status"] == "ok": + assert "checks" in body["data"], "envelope must list per-rule checks" + # Every check must carry severity + a citation back to the help-doc. + for check in body["data"]["checks"]: + assert check["severity"] in ("BLOCKING", "WARN", "OK") + assert "citation" in check, "each check must cite the help-doc canon" + # --------------------------------------------------------------------------- # Issue #245: parameters.blocks[].codes[].script auto-normalize on config update @@ -6686,6 +6894,8 @@ def test_config_update_auto_normalizes_script_array(self, tmp_path: Path) -> Non assert "Expected" not in rendered or "script" not in rendered, ( f"job envelope still mentions the script type-mismatch failure: {rendered}" ) + + @pytest.mark.e2e_invite @skip_without_invite_credentials def test_project_invite_e2e(tmp_path: Path) -> None: @@ -6695,9 +6905,6 @@ def test_project_invite_e2e(tmp_path: Path) -> None: invalidates the invitation link before the inbox sees it, so this is a "the system can send + clean up" check, not a "join my project" check. """ - from keboola_agent_cli.config_store import ConfigStore as _Store - from keboola_agent_cli.models import ProjectConfig as _Project - invite_email = os.environ.get(ENV_INVITE_EMAIL, DEFAULT_INVITE_EMAIL) project_id = int(os.environ[ENV_INVITE_PROJECT_ID]) stack_url = ( @@ -6712,10 +6919,10 @@ def test_project_invite_e2e(tmp_path: Path) -> None: # field is unused. Write a minimal config.json with a placeholder token. config_dir = tmp_path / "kbagent-config" config_dir.mkdir() - store = _Store(config_dir=config_dir) + store = ConfigStore(config_dir=config_dir) store.add_project( alias, - _Project( + ProjectConfig( stack_url=stack_url, token="901-e2e-placeholder-not-used-by-member-commands-xxxxxxxxxx", project_id=project_id, diff --git a/tests/test_member_cli.py b/tests/test_member_cli.py index 7a2eaea8..379555dd 100644 --- a/tests/test_member_cli.py +++ b/tests/test_member_cli.py @@ -60,6 +60,7 @@ def test_json_happy_path(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -89,6 +90,7 @@ def test_missing_required_args_exits_2(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -110,6 +112,7 @@ def test_invalid_role_blocked_by_choice(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -145,6 +148,7 @@ def test_dry_run_short_circuits(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -181,6 +185,7 @@ def test_invalid_token_maps_to_exit_3(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -239,6 +244,7 @@ def test_json_bulk_summary(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -266,6 +272,7 @@ def test_from_csv_mutually_exclusive_with_project(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -308,6 +315,7 @@ def test_json_output(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -343,6 +351,7 @@ def test_include_pending_flag_propagates(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -380,6 +389,7 @@ def test_yes_skips_confirmation(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -417,6 +427,7 @@ def test_destructive_yes(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -455,6 +466,7 @@ def test_propagates_role(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", diff --git a/uv.lock b/uv.lock index d39aa76e..73ad4950 100644 --- a/uv.lock +++ b/uv.lock @@ -439,7 +439,7 @@ wheels = [ [[package]] name = "keboola-agent-cli" -version = "0.28.0" +version = "0.29.0" source = { editable = "." } dependencies = [ { name = "httpx" },