From bf5f9bb2bbb269d0e517d3ad94c82c95c052af05 Mon Sep 17 00:00:00 2001 From: Petr Simecek Date: Wed, 6 May 2026 18:22:40 +0200 Subject: [PATCH 1/5] feat(0.28.0): manage-token default-deny -- env var ignored without --allow-env-manage-token, TTY prompt is the default (#252) BREAKING: KBC_MANAGE_API_TOKEN is no longer auto-resolved on the three commands that consume it (org setup, project refresh, data-app password). Default behaviour: emit a one-shot stderr warning, ignore the env, fall through to a TTY hidden-input prompt; exit 2 with no TTY. To opt in (CI/CD), pass the new top-level flag --allow-env-manage-token. The change closes the AI-exfiltration risk where any subprocess running as the same user (including the AI agent itself) inherits the manage token via env. Storage tokens (KBC_TOKEN) are unaffected. Supersedes the per-stack design discussed in #238 per Padak's review: "Subprocess kbagent invocations continue to require explicit --allow-env-manage-token or TTY prompt." Phase 2 (REPL-process-memory caching for human admin work) and Phase 3 (hardware-key) are roadmap. Implementation: - Resolver: src/keboola_agent_cli/commands/_helpers.py:27 -- resolve_manage_token(*, allow_env: bool = False) with default-deny behaviour, one-shot stderr warning, actionable exit-2 message. - Top-level flag: src/keboola_agent_cli/cli.py -- new --allow-env-manage-token Option mirroring --deny-writes shape; plumbed via ctx.obj["allow_env_manage_token"]. - Call sites (3): commands/data_app.py:594, commands/org.py:223, commands/project.py:423 -- all pass allow_env=ctx.obj[...] into the resolver. - Bulk-prompt-once preserved by construction: resolver lives at command entry, before any per-project loop. Tests (12 new, 3 existing updated): - tests/test_helpers.py::TestResolveManageToken (7 tests): allow_env-True/False x env-set/unset x TTY/non-TTY combinations + token-leak regression pin + flag-is-permission-not-promise pin. - tests/test_manage_token_cli.py::TestAllowEnvManageTokenFlag (4 tests): project-refresh / org-setup / data-app-password through CliRunner with services mocked, default-deny vs allow-env paths. - tests/test_manage_token_bulk.py::TestBulkPromptOnce (1 test): pins that "project refresh --all" resolves the token exactly once at command entry, never per-project. Patches resolve_manage_token itself for unambiguous count-based assertion. - tests/test_cli.py and tests/test_data_app_cli.py: 3 existing tests updated to pass --allow-env-manage-token (they were exercising the legacy env-default contract; now they pin the opt-in path). Sync map walk: - commands/context.py AGENT_CONTEXT (org-setup example, --project-ids prose, data-app password prose, env-var help block) - CLAUDE.md convention #12 + global-flag list - plugins/kbagent/agents/keboola-expert.md Rule 6 VERSION GATE, tool-selection-matrix row, new inline-gotcha block - plugins/kbagent/skills/kbagent/references/gotchas.md new entry tagged (since v0.28.0) - plugins/kbagent/skills/kbagent/references/commands-reference.md three command rows + env-var table - pyproject.toml 0.27.0 -> 0.28.0; make version-sync propagated to plugin.json + marketplace.json - changelog.py: new 0.28.0 block (BREAKING + Security + new flag + Tests + Docs) make check: 2444 passed, 5 skipped, 64 deselected, 0 failed. Co-authored-by: ottomansky --- CLAUDE.md | 4 +- README.md | 12 +- docs/TUTORIAL.md | 39 +++- docs/e2e-scenarios.md | 4 +- plugins/kbagent/agents/keboola-expert.md | 20 +- plugins/kbagent/skills/kbagent/SKILL.md | 6 +- .../kbagent/references/commands-reference.md | 11 +- .../kbagent/references/data-app-workflow.md | 4 +- .../skills/kbagent/references/gotchas.md | 35 +++- src/keboola_agent_cli/changelog.py | 5 + src/keboola_agent_cli/cli.py | 9 + src/keboola_agent_cli/commands/_helpers.py | 38 +++- src/keboola_agent_cli/commands/context.py | 23 ++- src/keboola_agent_cli/commands/data_app.py | 8 +- src/keboola_agent_cli/commands/org.py | 8 +- src/keboola_agent_cli/commands/project.py | 6 +- src/keboola_agent_cli/commands/repl.py | 9 +- .../hints/definitions/data_app.py | 7 +- .../hints/definitions/org.py | 4 +- .../services/data_app_service.py | 3 +- tests/test_cli.py | 13 +- tests/test_data_app_cli.py | 1 + tests/test_helpers.py | 128 ++++++++++++ tests/test_manage_token_bulk.py | 87 ++++++++ tests/test_manage_token_cli.py | 188 ++++++++++++++++++ 25 files changed, 611 insertions(+), 61 deletions(-) create mode 100644 tests/test_manage_token_bulk.py create mode 100644 tests/test_manage_token_cli.py diff --git a/CLAUDE.md b/CLAUDE.md index 3e6cb5eb..56081a6a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -195,7 +195,7 @@ All three inherit from `BaseHttpClient` (`http_base.py`) which provides shared r 11. **Error accumulation**: multi-project operations collect per-project errors without stopping. One project failing doesn't block others (see `lineage_service.py`, `org_service.py`). -12. **Manage token security**: never persisted, never passed as CLI argument, never logged. Only via `KBC_MANAGE_API_TOKEN` env var or interactive hidden prompt. +12. **Manage token security**: never persisted, never passed as CLI argument, never logged. Default-deny since 0.28.0: only via interactive hidden prompt; the `KBC_MANAGE_API_TOKEN` env var is **ignored** unless the top-level `--allow-env-manage-token` flag is passed. Default-deny closes the AI-exfiltration risk where any subprocess (including the AI agent itself) inherits the manage token via env. CI/CD callers must opt in explicitly. 13. **Idempotency**: `org setup` skips already-registered projects by matching `project_id`. Safe to re-run. @@ -251,7 +251,7 @@ plugins/kbagent/ > "Plugin synchronization map" for the full list. ``` -# Global options: --json, --verbose, --no-color, --config-dir, --hint client|service, --deny-writes, --deny-destructive +# Global options: --json, --verbose, --no-color, --config-dir, --hint client|service, --deny-writes, --deny-destructive, --allow-env-manage-token kbagent project add --project NAME --url URL --token TOKEN kbagent project list diff --git a/README.md b/README.md index 8087405c..1a8bdb34 100644 --- a/README.md +++ b/README.md @@ -102,14 +102,22 @@ kbagent project add --project prod --url https://connection.keboola.com --token **Many projects by ID** — you have a Manage API or Personal Access Token + the project IDs: ```bash +# Interactive: kbagent will prompt for the Manage API token (default since v0.28.0). +kbagent org setup --project-ids 901,9621,10539 --url https://connection.keboola.com --yes + +# CI / non-interactive: opt in to env-var resolution with --allow-env-manage-token. KBC_MANAGE_API_TOKEN=your-manage-or-personal-token \ - kbagent org setup --project-ids 901,9621,10539 --url https://connection.keboola.com --yes + kbagent --allow-env-manage-token org setup --project-ids 901,9621,10539 --url https://connection.keboola.com --yes ``` **Whole organization** — you are org admin: ```bash +# Interactive (default since v0.28.0): kbagent prompts for the Manage API token. +kbagent org setup --org-id 123 --url https://connection.keboola.com --yes + +# CI / non-interactive: KBC_MANAGE_API_TOKEN=your-org-admin-manage-token \ - kbagent org setup --org-id 123 --url https://connection.keboola.com --yes + kbagent --allow-env-manage-token org setup --org-id 123 --url https://connection.keboola.com --yes ``` Run `kbagent doctor` to verify setup (token validity, CLI version, MCP server, Claude Code plugin install). diff --git a/docs/TUTORIAL.md b/docs/TUTORIAL.md index 79ccce80..f17d8272 100644 --- a/docs/TUTORIAL.md +++ b/docs/TUTORIAL.md @@ -102,12 +102,19 @@ use `org setup --project-ids`. kbagent creates a Storage API token in each listed project and registers them all locally, in parallel. ```bash -export KBC_MANAGE_API_TOKEN=YOUR_MANAGE_OR_PAT_TOKEN - +# Interactive (default since v0.28.0): kbagent prompts for the Manage API +# token on stdin. No env var, no shell history. kbagent org setup \ --project-ids 901,9621,10539 \ --url https://connection.keboola.com \ --dry-run # always dry-run first + +# CI / non-interactive: opt in to env-var resolution. +export KBC_MANAGE_API_TOKEN=YOUR_MANAGE_OR_PAT_TOKEN +kbagent --allow-env-manage-token org setup \ + --project-ids 901,9621,10539 \ + --url https://connection.keboola.com \ + --dry-run ``` The dry-run prints what would happen (create token + register alias @@ -132,10 +139,14 @@ Flags worth knowing: The command is **idempotent**: running it again skips projects that are already registered. Safe to re-run after adding new project IDs. -**Security note**: `KBC_MANAGE_API_TOKEN` is read only from env or -from an interactive hidden prompt. kbagent never accepts it as a CLI -argument (`--token xxx`) -- that would leak into shell history and -process listings. +**Security note (since v0.28.0)**: `KBC_MANAGE_API_TOKEN` is **ignored +by default** -- the env var is read only when the top-level +`--allow-env-manage-token` flag is passed. Without the flag, kbagent +prompts on stdin (hidden input). kbagent never accepts the token as a +CLI argument (`--token xxx`) -- that would leak into shell history and +process listings. The default-deny closes an AI-exfiltration risk where +any subprocess running as the same user (including the AI agent itself) +inherits env vars. --- @@ -145,12 +156,18 @@ If you are an org admin with a Manage API token, register **every** project in an organization in one shot: ```bash -export KBC_MANAGE_API_TOKEN=YOUR_ORG_ADMIN_MANAGE_TOKEN - +# Interactive (default since v0.28.0): kbagent org setup \ --org-id 123 \ --url https://connection.keboola.com \ --dry-run + +# CI / non-interactive: +export KBC_MANAGE_API_TOKEN=YOUR_ORG_ADMIN_MANAGE_TOKEN +kbagent --allow-env-manage-token org setup \ + --org-id 123 \ + --url https://connection.keboola.com \ + --dry-run ``` The dry-run reports how many projects will be registered and the @@ -859,7 +876,8 @@ kbagent --json data-app create \ deploy. To retrieve it: ```bash -# Requires KBC_MANAGE_API_TOKEN in env (org-scoped Manage API token). +# Manage API token: interactive prompt by default (since v0.28.0). For CI, +# add `--allow-env-manage-token` and set KBC_MANAGE_API_TOKEN in env. kbagent --json data-app password \ --project prod --app-id 12345678 \ | jq -r '.data.password' @@ -949,7 +967,8 @@ shapes, and the `--hint client|service` code-generation contract, see | `kbagent: command not found` after `uv tool install` | Ensure `~/.local/bin` (or uv's tool dir) is on your PATH. `uv tool update-shell` can help. | | `kbagent doctor` reports `warn` for plugin | Run the two `/plugin` commands shown in the warning, from inside Claude Code. | | Plugin version != CLI version | In Claude Code: `/plugin update kbagent`. | -| `org setup` fails with `401 Unauthorized` | Your `KBC_MANAGE_API_TOKEN` is wrong for this stack or role. Manage tokens are stack-specific and require the right scope. | +| `org setup` exits 2 with `Warning: KBC_MANAGE_API_TOKEN found in environment but ignored` | Default-deny since v0.28.0 -- pass `--allow-env-manage-token` (top-level flag) to opt in to env resolution, or run interactively to use the prompt. | +| `org setup` fails with `401 Unauthorized` | Your manage token is wrong for this stack or role. Manage tokens are stack-specific and require the right scope. | | `org setup --org-id` fails with `403` | You are not an org admin. Use `--project-ids` with a Personal Access Token instead (works for any project member). | | Changes from `kbagent config update` do not show in UI | You are on a dev branch. Run `kbagent branch list` and `kbagent project current` to verify the active branch; changes in a dev branch merge to production only via the UI merge step (`kbagent branch merge` returns the merge URL). | | The specialist subagent does not spawn when I type `/keboola X` | Plugin is not installed or is outdated. Run `kbagent doctor` and follow the reported commands. | diff --git a/docs/e2e-scenarios.md b/docs/e2e-scenarios.md index d2622eb8..e78cd87d 100644 --- a/docs/e2e-scenarios.md +++ b/docs/e2e-scenarios.md @@ -211,8 +211,8 @@ Skipped if `keboola-mcp-server` is not installed. | Command | Reason | |---------|--------| -| `project refresh` | Requires Manage API token (`KBC_MANAGE_API_TOKEN`) | -| `org setup` | Requires Manage API token + destructive (registers projects in org) | +| `project refresh` | Requires Manage API token (interactive prompt by default since v0.28.0; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for non-interactive runners) | +| `org setup` | Requires Manage API token (same prompt-or-flag as above) + destructive (registers projects in org) | | `sharing share/unshare` | Requires org-level permissions or second project | | `sharing link/unlink` | Requires shared bucket from another project | | `permissions set/reset` | Interactive random-code confirmation blocks automated testing | diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 4e2d07fd..6b5c2f77 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -64,8 +64,12 @@ a critical failure. needed for the current task (e.g. `flow update` needs 0.22.0+, `schedule find` needs 0.23.0+, `config set-default-bucket` needs 0.26.0+, `data-app create / deploy / start / stop / delete / password` +<<<<<<< HEAD need 0.27.0+, `config update` script[] auto-normalize against #245 trap needs 0.28.0+, `storage swap-tables` needs 0.28.0+, + env-var manage-token auth for `org setup` / `project refresh` / + `data-app password` needs 0.28.0+ with `--allow-env-manage-token` + (the env var is default-deny on 0.28.0+), `storage retype` is a future composite), you MUST refuse the task and return a handoff message to the parent: `"Cannot proceed safely on kbagent . Missing: . @@ -106,7 +110,7 @@ a critical failure. | Roll out a new code or config version on a data app | `kbagent data-app deploy --project P --app-id N --wait` (0.27.0+) -- always sends the §9 trio | `kbagent --hint client data-app deploy ...` to inspect the generated `patch_app(desired_state=, config_version=, restart_if_running=True)` call | `tool call update_config` then `tool call run_component` (data apps are not jobs -- the queue runner does not deploy them) | | Wake an auto-suspended data app | `kbagent data-app start --project P --app-id N` (0.27.0+) -- does NOT bump configVersion | hitting the app's URL (auto-restart triggers a 30-60s cold boot) | `kbagent data-app deploy` (overkill -- bumps the deployed configVersion unnecessarily) | | Pause a running data app | `kbagent data-app stop --project P --app-id N` (0.27.0+) | -- | `kbagent data-app delete` (irreversible; cascades to Storage config) | -| Read the simpleAuth password for a password-gated app | `kbagent data-app password --project P --app-id N` (0.27.0+) -- requires `KBC_MANAGE_API_TOKEN` | -- | trying to "rotate" the password (not supported by the API; delete + recreate to mint a new one) | +| Read the simpleAuth password for a password-gated app | `kbagent data-app password --project P --app-id N` (0.27.0+) -- needs Manage API token (interactive prompt by default; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+) | -- | trying to "rotate" the password (not supported by the API; delete + recreate to mint a new one) | | Tear down a data app | `kbagent data-app delete --project P --app-id N` (0.27.0+) -- cascades to Storage config; URL retired | -- | manually `tool call delete_config keboola.data-apps` while leaving the deployment record orphaned | If the table does not cover the user's task, **ask clarifying @@ -273,6 +277,20 @@ success, not a failure. `bigquery_path` will be dataset-qualified only -- if the user needs a fully-qualified GCP path, ask them for the project name explicitly. +- **Manage-token env-var is opt-in (since 0.28.0)**. + `KBC_MANAGE_API_TOKEN` is no longer auto-resolved for `org setup`, + `project refresh`, or `data-app password`. Default behaviour: emit a + warning, ignore the env var, fall through to a TTY hidden-input prompt; + exit 2 with no TTY. To opt in (CI/CD), pass the top-level flag: + `kbagent --allow-env-manage-token --json org setup ...`. The flag is + session-only -- not persisted, no env-var equivalent. Default-deny + closes the AI-exfiltration risk where a subprocess running as the same + user (including the agent itself) inherits the manage token. If you + see `Warning: KBC_MANAGE_API_TOKEN found in environment but ignored` + in stderr, that is the expected default; tell the user to add + `--allow-env-manage-token` to their invocation, never strip the + warning by suppressing stderr. + --- ## 4. WORKFLOWS (reference playbooks) diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 46e737b1..27cb91bb 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -273,8 +273,10 @@ Then add projects: kbagent --json project add --project prod --url https://connection.keboola.com --token YOUR_TOKEN # Or bulk-onboard from organization (org admin) -KBC_MANAGE_API_TOKEN=xxx kbagent --json org setup --org-id 123 --url https://connection.keboola.com --yes +# Manage token: interactive prompt by default; for CI add --allow-env-manage-token +# alongside KBC_MANAGE_API_TOKEN (required since v0.28.0). +KBC_MANAGE_API_TOKEN=xxx kbagent --allow-env-manage-token --json org setup --org-id 123 --url https://connection.keboola.com --yes # Or onboard specific projects (any project member, uses Personal Access Token) -KBC_MANAGE_API_TOKEN=xxx kbagent --json org setup --project-ids 901,9621,10539 --url https://connection.keboola.com --yes +KBC_MANAGE_API_TOKEN=xxx kbagent --allow-env-manage-token --json org setup --project-ids 901,9621,10539 --url https://connection.keboola.com --yes ``` diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 776f5edd..71836b9a 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -24,11 +24,12 @@ All commands support `--json` for structured output. Multi-project flags (`--pro ## Permission flags (top-level, session-only) - `--deny-writes` -- block all write/destructive/admin operations for this single invocation. Merges with any persisted permission policy; never written to config.json. Exit code 6 (PERMISSION_DENIED) on blocked operations - `--deny-destructive` -- block only destructive operations (delete-table, delete-bucket, terminate-job, etc.) for this invocation. Pure-write ops like create-table stay allowed. Use this when you want to keep build-up capabilities but lock out tear-downs -- Both flags compose: `kbagent --deny-writes --deny-destructive ...` is the safest read-only run +- `--allow-env-manage-token` -- opt in to reading `KBC_MANAGE_API_TOKEN` from env (default-deny since v0.28.0). Without it the env var is ignored and an interactive hidden prompt is required for `org setup` / `project refresh` / `data-app password`. Closes the AI-exfiltration risk where any subprocess inherits the manage token via env. Session-only; not persisted; no env-var equivalent (intentional, would re-create the hole). REPL forwards this flag to nested invocations the same way it forwards the deny-* flags +- All three flags compose: `kbagent --deny-writes --deny-destructive --allow-env-manage-token ...` is the safest CI-friendly invocation ## Organization -- `org setup --org-id ID --url URL [--dry-run] [--yes]` -- bulk-onboard all projects from an org (org admin, needs `KBC_MANAGE_API_TOKEN`) -- `org setup --project-ids 1,2,3 --url URL [--dry-run] [--yes]` -- onboard specific projects by ID (any project member, works with Personal Access Token via `KBC_MANAGE_API_TOKEN`) +- `org setup --org-id ID --url URL [--dry-run] [--yes]` -- bulk-onboard all projects from an org (org admin; manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+) +- `org setup --project-ids 1,2,3 --url URL [--dry-run] [--yes]` -- onboard specific projects by ID (any project member; manage token / Personal Access Token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+) ## Component Discovery - `component list [--project NAME] [--type TYPE] [--query "text"]` -- list/search components (AI-powered with `--query`) @@ -126,7 +127,7 @@ Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, - `data-app start --project NAME --app-id ID [--wait] [--timeout SECONDS]` -- wake an auto-suspended app at the currently-pinned version. Distinct from deploy: does NOT bump configVersion. - `data-app stop --project NAME --app-id ID [--wait] [--timeout SECONDS]` -- stop a running app (URL and Storage config preserved). - `data-app delete --project NAME --app-id ID [--yes]` -- destructive, cascades to Storage config; URL retired permanently. -- `data-app password --project NAME --app-id ID` -- read the simpleAuth password. Requires `KBC_MANAGE_API_TOKEN`. Auto-generated, not rotatable -- delete + recreate to mint a new one. +- `data-app password --project NAME --app-id ID` -- read the simpleAuth password. Manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+. Auto-generated, not rotatable -- delete + recreate to mint a new one. ## MCP Tools - `tool list [--project NAME] [--branch ID]` -- list available MCP tools (multi_project annotation) @@ -187,7 +188,7 @@ Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, |----------|---------| | `KBC_TOKEN` | Fallback for `--token` | | `KBC_STORAGE_API_URL` | Default stack URL | -| `KBC_MANAGE_API_TOKEN` | Manage API token (org setup) | +| `KBC_MANAGE_API_TOKEN` | Manage API token (org setup, project refresh, data-app password). Default-DENY since 0.28.0: requires top-level `--allow-env-manage-token` to opt in, otherwise ignored with a warning. | | `KBAGENT_CONFIG_DIR` | Override config directory | ## Exit Codes diff --git a/plugins/kbagent/skills/kbagent/references/data-app-workflow.md b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md index 7b840293..37cab40a 100644 --- a/plugins/kbagent/skills/kbagent/references/data-app-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md @@ -58,7 +58,9 @@ with: ```bash kbagent data-app password --project prod --app-id -# Requires KBC_MANAGE_API_TOKEN in addition to the project's Storage token. +# Manage token: interactive prompt by default (since v0.28.0); for CI add +# --allow-env-manage-token alongside KBC_MANAGE_API_TOKEN. Storage token +# is read from .kbagent/config.json as usual. ``` The simpleAuth password CANNOT be rotated (writeup §11.2). To change it, diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 936cc818..0ac8e63d 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -18,6 +18,37 @@ original table now carries the typed schema with no downstream config rewrite required. +## Manage token: env var is ignored without `--allow-env-manage-token` (since v0.28.0) + +- `KBC_MANAGE_API_TOKEN` is no longer auto-resolved on the three + surfaces that consume it (`kbagent org setup`, + `kbagent project refresh`, `kbagent data-app password`). Default + behaviour on 0.28.0+ is **default-deny**: the env var is ignored, a + TTY hidden-input prompt is shown instead. With no TTY (CI / cron / + systemd / `< /dev/null`) the resolver exits **2** with the message + `Error: No manage token available. Run interactively, or pass + --allow-env-manage-token to read KBC_MANAGE_API_TOKEN from env.` +- To opt in for CI/CD, pass the top-level flag: + `kbagent --allow-env-manage-token --json org setup ...`. The flag + belongs in front of the subcommand (it is a top-level option, mirroring + `--deny-writes`). The flag is session-only -- not persisted, no + env-var equivalent (intentional; an env-var equivalent would re-create + the AI-exfiltration hole this default-deny is closing). +- When the env var IS set but the flag IS NOT, you will see a one-shot + stderr warning `Warning: KBC_MANAGE_API_TOKEN found in environment + but ignored. Pass --allow-env-manage-token to opt in.`. This is + informational; the resolver still falls through to the TTY prompt + (or exits 2 if no TTY). Do NOT suppress this warning by piping stderr + away -- it tells CI maintainers exactly what to fix. +- The default-deny exists to close the AI-exfiltration risk: any + subprocess running as the same user (including the AI agent itself) + inherits env vars, so a manage token in env is reachable by anyone + who can read `os.environ` or shell out raw `curl`. Default-deny means + human admin work uses TTY (no env exposure) and CI must explicitly + say "yes I trust this env" via the flag. +- Storage tokens are unaffected: `KBC_TOKEN` (storage API) keeps + resolving from env as before. + ## `data-app deploy` is required after `config update` -- the running container does NOT auto-pick-up new config versions (since v0.27.0) - `kbagent config update --component-id keboola.data-apps ...` bumps the @@ -390,7 +421,7 @@ type inventory and examples. - Tokens are always masked in output (e.g. `901-...pt0k`) -- this is normal - Token can be passed via `--token`, `KBC_TOKEN` env var, or interactive prompt -- Manage API token: only via `KBC_MANAGE_API_TOKEN` env var or interactive prompt (never as CLI argument) +- Manage API token (since v0.28.0): default-deny on env -- via interactive hidden prompt; pass top-level `--allow-env-manage-token` to opt in to `KBC_MANAGE_API_TOKEN`. Never as CLI argument. See the `(since v0.28.0)` entry at the top of this file. - Master token for sharing: `KBC_MASTER_TOKEN_{ALIAS}` (e.g. `KBC_MASTER_TOKEN_PROD`) or `KBC_MASTER_TOKEN` as global fallback. Alias is uppercased, hyphens become underscores. Required for `sharing share` and `sharing unshare`; `sharing list/link/unlink` use regular project tokens. ## MCP tool call gotchas @@ -800,7 +831,7 @@ See [docs/hint-mode.md](../../../../../docs/hint-mode.md) for full documentation - **Forgetting `--json`**: without it, output is human-formatted Rich text, not parseable - **Assuming `data.projects`**: `project list` returns data as a flat list -- **Passing manage token as argument**: use env var `KBC_MANAGE_API_TOKEN` instead +- **Passing manage token as argument**: use the interactive prompt (default since v0.28.0), or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` env var for CI - **Polling after branch create**: kbagent already waits for async completion - **Not saving workspace password**: only returned once on creation - **Putting SQL in _config.yml**: SQL transformations must use `transform.sql` with block markers (see above) diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index ff96ce81..5430db77 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -17,6 +17,11 @@ "New: `kbagent storage swap-tables --project P --table-id A --target-table-id B [--branch ID] [--dry-run] [--yes]` -- thin wrapper around the Storage API `POST /v2/storage/branch/{branch}/tables/{id}/swap` endpoint. Both tables exchange physical positions; aliases are NOT transferred (they keep pointing at the same physical position and therefore expose the OTHER table's data after the swap). The Storage API queues this as an async storage job (`operationName: tableSwap`); the client polls to completion before returning, so callers can rely on the schemas already being exchanged on return (~10s observed on Snowflake). The API restricts this to dev branches; the service refuses with exit 5 (`ConfigError`) before any HTTP call when neither `--branch` nor an active branch (via `kbagent branch use`) is set. Same-source-and-target IDs also rejected pre-flight. The use case is: AI agent profiles a typeless table, builds a typed rebuild via CTAS in a workspace, then swaps the typed copy into the original name without touching downstream config references that point at the original table ID. Permission classification: `destructive` (gated behind `--allow-destructive`). The PHP reference client docstring claims a synchronous response, but live calls against the platform consistently return a queued job -- this client polls the job to completion to make the `delete_table` / `create_table` semantics consistent. Companion entry in `storage-types-workflow.md` explains the typify-via-CTAS pattern; gotchas + commands-reference + agent prompt all updated.", "Tests (swap-tables): `tests/test_storage_swap.py` (14 tests) covers all three layers -- HTTP shape (POST + body + URL encoding + immediate-success path + async-poll path + 4xx propagation via `pytest_httpx`), service business logic (success, dry-run, branch enforcement, same-id guard, API error propagation, unknown project), and CLI integration (JSON happy path, dry-run, explicit `--branch` overrides active, missing-branch error path with exit 5). E2E coverage in `tests/test_e2e.py::TestE2EStorageSwapTables` runs three scenarios against a live API: live swap of two tables with different VARCHAR lengths verifies definitions exchange in both directions; dry-run skips API call and `lastChangeDate` is unchanged; and the production-rejection path (no branch + no active branch) returns exit 5.", "Plugin docs: new `plugins/kbagent/skills/kbagent/references/typify-table-workflow.md` -- end-to-end procedure for converting a typeless Storage table (every column `STRING(16M)`) into one with proper Snowflake / BigQuery native types. 8 phases: (0) decide-or-skip rubric; (1) isolate in dev branch; (2) profile the typeless table in a workspace with length / cardinality / parse-failure / scale-precision queries + decision matrix mapping profile signals to Snowflake types; (3) build typed sibling via `storage create-table` + copy data via in-workspace INSERT or SQL transformation, with row-count / NULL-count verification; (4) validate downstream consumers in the dev branch (search configs that reference the table, run a representative transformation against the typeless source as baseline); (5) `swap-tables` (dry-run + actual + verify); (6) re-run downstream as smoke test; (7) cleanup the sibling after merge; (8) handoff protocol -- structured summary the AI agent hands to the user with phase-by-phase receipts, the merge URL, and rollback / cleanup commands. Cross-references `storage-types-workflow.md`, `branch-workflow.md`, `workspace-workflow.md`, `gotchas.md`. SKILL.md workflow-references table gains the new entry.", + "BREAKING: `KBC_MANAGE_API_TOKEN` is now ignored by default. The three commands that consume it (`org setup`, `project refresh`, `data-app password`) prompt for the token on a TTY by default. Pass the new top-level flag `--allow-env-manage-token` to restore the legacy env-var behaviour (e.g. for CI/CD). Without the flag and without a TTY, the resolver exits 2 with an actionable message naming the flag. The change closes the AI-exfiltration risk where any subprocess running as the same user (including the AI agent itself) inherits the manage token via env. Migration: prepend `--allow-env-manage-token` to existing CI invocations. Storage tokens (`KBC_TOKEN`) are unaffected. Closes the manage-token UX flagged on #236; supersedes the per-stack design discussed in #238.", + "Security: `resolve_manage_token` (`src/keboola_agent_cli/commands/_helpers.py`) refactored to default-deny env, TTY-first. When the env var is set but the flag is not passed, a one-shot stderr warning fires (`Warning: KBC_MANAGE_API_TOKEN found in environment but ignored. Pass --allow-env-manage-token to opt in.`) and the resolver falls through to the TTY prompt. No cache, no keyring, no temp file -- next invocation prompts again. The bulk-prompt-once contract (`project refresh --all`) is preserved by construction: the resolver lives at command entry, before any per-project loop.", + "New: top-level CLI flag `--allow-env-manage-token` (session-only, mirrors `--deny-writes` / `--deny-destructive`). Plumbed via `ctx.obj['allow_env_manage_token']` and forwarded by the three call sites into `resolve_manage_token(allow_env=...)`. Not persisted, no env-var equivalent (intentional; an env-var equivalent would re-create the AI-exfiltration hole this default-deny is closing).", + "Tests: 12 new (`tests/test_helpers.py::TestResolveManageToken` x7 covering allow_env-True/False x env-set/unset x TTY/non-TTY combinations + token-leak regression pin; `tests/test_manage_token_cli.py::TestAllowEnvManageTokenFlag` x4 covering project-refresh / org-setup / data-app-password through CliRunner with services mocked; `tests/test_manage_token_bulk.py::TestBulkPromptOnce` pinning the contract that `project refresh --all` resolves the token exactly once at command entry, not per-project).", + "Docs: `commands/context.py` AGENT_CONTEXT updated (org-setup example + env-var help block); `CLAUDE.md` convention #12 + global-flag list; `keboola-expert.md` Rule 6 VERSION GATE adds the 0.28.0+ env-flag requirement, tool-selection-matrix updated, new inline-gotcha block; `gotchas.md` new `(since v0.28.0)` entry naming the warning text and the one-line CI fix; `commands-reference.md` updated for `org setup`, `data-app password`, env-var table.", ], "0.27.0": [ "New: `kbagent data-app` command group — first-class lifecycle for Keboola data apps (`keboola.data-apps` Storage component + Data Science API `/apps`). Eight subcommands: `list`, `detail`, `create`, `deploy`, `start`, `stop`, `delete`, `password`. The CLI encapsulates the **§9 redeploy contract** (always sends the `{desiredState=running, configVersion, restartIfRunning=true}` trio together; without it, `PATCH /apps {desiredState:running}` silently pins to the empty-shell v2 and the runner errors `dataApp.git.repository is required in /data/config.json`), per-project KMS encryption of git PATs (refuses to write plaintext if the Encryption API does not return a project-scoped ciphertext), cleanup-in-finally on initial-deploy failure (orphan shell deleted by default; `--keep-on-failure` opts out), and a poll loop that respects pitfall #1 — `state == stopped` is NOT terminal while `desiredState == running` (the platform transitions `created → stopped → starting → running` during initial deploy). `data-app create` accepts `--git-pat-env VAR` (recommended; no argv leak), `--git-pat-file PATH`, or `--git-pat-encrypted KBC::Project...` (must be encrypted under THIS project's KMS — ciphertext does not cross projects).", diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index 3429b1ec..e97b8b54 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -220,6 +220,14 @@ def main( "branch delete, etc.). Admin ops like 'project remove' and 'org setup' " "are NOT blocked -- use --deny-writes for the wide net.", ), + allow_env_manage_token: bool = typer.Option( + False, + "--allow-env-manage-token", + help="Read KBC_MANAGE_API_TOKEN from the environment. Without this " + "flag the env var is ignored (with a warning) and an interactive " + "TTY prompt is required. Default-deny since 0.28.0; closes the " + "AI-exfiltration risk where subprocesses inherit the manage token.", + ), ) -> None: """Global options applied to all commands.""" from .auto_update import maybe_auto_update, show_post_update_changelog @@ -339,6 +347,7 @@ def main( ctx.obj["no_color"] = effective_no_color ctx.obj["deny_writes"] = deny_writes ctx.obj["deny_destructive"] = deny_destructive + ctx.obj["allow_env_manage_token"] = allow_env_manage_token ctx.obj["config_store"] = config_store ctx.obj["project_service"] = project_service ctx.obj["component_service"] = component_service diff --git a/src/keboola_agent_cli/commands/_helpers.py b/src/keboola_agent_cli/commands/_helpers.py index a9e9ed15..64cce61c 100644 --- a/src/keboola_agent_cli/commands/_helpers.py +++ b/src/keboola_agent_cli/commands/_helpers.py @@ -24,31 +24,49 @@ from ..output import OutputFormatter -def resolve_manage_token() -> str: - """Resolve the manage token from env var or interactive prompt. +def resolve_manage_token(*, allow_env: bool = False) -> str: + """Resolve the manage token from a permitted source. - Token resolution order: - 1. KBC_MANAGE_API_TOKEN env var (for CI/CD) - 2. Interactive prompt with hidden input (if TTY) - 3. Error if neither available + Default-deny: KBC_MANAGE_API_TOKEN is ignored unless ``allow_env=True`` + (set by the top-level ``--allow-env-manage-token`` flag). The change + closes the AI-exfiltration risk where any subprocess running as the + same user can read the env var; the new default is "human at a TTY". + + Resolution order: + 1. ``KBC_MANAGE_API_TOKEN`` env var, IF ``allow_env`` is True. Otherwise + a one-shot stderr warning is emitted and the env var is ignored. + 2. Interactive prompt with hidden input (if stdin is a TTY). + 3. Exit 2 with an actionable error naming the opt-in flag. + + Args: + allow_env: When True, restores the legacy env-var-first behaviour + for the current invocation. Plumbed from the top-level + ``--allow-env-manage-token`` flag via ``ctx.obj``. Returns: The manage API token. Raises: - typer.Exit: If no token can be resolved. + typer.Exit: If no token can be resolved (exit code 2). """ env_token = os.environ.get(ENV_KBC_MANAGE_API_TOKEN) if env_token: - return env_token + if allow_env: + return env_token + typer.echo( + f"Warning: {ENV_KBC_MANAGE_API_TOKEN} found in environment " + "but ignored. Pass --allow-env-manage-token to opt in.", + err=True, + ) is_tty = hasattr(sys.stdin, "isatty") and sys.stdin.isatty() if is_tty: return typer.prompt("Manage API token", hide_input=True) typer.echo( - f"Error: No manage token available. Set {ENV_KBC_MANAGE_API_TOKEN} env var " - "or run interactively.", + "Error: No manage token available. Run interactively, or pass " + f"--allow-env-manage-token to read {ENV_KBC_MANAGE_API_TOKEN} " + "from env.", err=True, ) raise typer.Exit(code=2) diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 048d1e91..434283ef 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -35,7 +35,7 @@ kbagent --json project add --project my-project --url https://connection.keboola.com --token YOUR_TOKEN # Or bulk-onboard all projects from an organization - KBC_MANAGE_API_TOKEN=xxx kbagent --json org setup --org-id 123 --url https://connection.keboola.com --yes + KBC_MANAGE_API_TOKEN=xxx kbagent --allow-env-manage-token --json org setup --org-id 123 --url https://connection.keboola.com --yes # Explore kbagent --json project list @@ -419,7 +419,10 @@ kbagent org setup --project-ids 901,9621,10539 --url URL [--dry-run] [--yes] [--refresh] Non-admin mode: onboard specific projects by ID. Works with Personal Access Token (PAT). Use --org-id OR --project-ids (at least one required). - Token via KBC_MANAGE_API_TOKEN env var or interactive prompt. + Token via interactive hidden prompt by default; pass top-level + --allow-env-manage-token to read KBC_MANAGE_API_TOKEN from env (CI/CD). + Default-deny since 0.28.0 -- closes the AI-exfiltration risk where + subprocesses inherit the manage token via env. ### Flows (Orchestrator + Conditional) @@ -593,11 +596,12 @@ URL is permanently retired. Confirmation prompt unless --yes. kbagent data-app password --project NAME --app-id ID - Retrieve the simpleAuth password. Requires KBC_MANAGE_API_TOKEN in - addition to the project's Storage token. Token is read from env or - interactive hidden prompt; never persisted, never logged. Password is - auto-generated at create time and CANNOT be rotated -- delete and - recreate the app to mint a new one. + Retrieve the simpleAuth password. Requires the Manage API token in + addition to the project's Storage token. Token is read from interactive + hidden prompt by default; pass top-level --allow-env-manage-token to + use KBC_MANAGE_API_TOKEN from env (default-deny since 0.28.0). Never + persisted, never logged. Password is auto-generated at create time + and CANNOT be rotated -- delete and recreate the app to mint a new one. ### Project Sync @@ -732,7 +736,10 @@ KBAGENT_CONVERSATION_ID Conversation/session ID (REQUIRED -- sent as X-Conversation-ID header) KBC_TOKEN Storage API token (fallback for --token) KBC_STORAGE_API_URL Default stack URL (fallback for --url) - KBC_MANAGE_API_TOKEN Manage API token (for org setup) + KBC_MANAGE_API_TOKEN Manage API token (org setup, project refresh, data-app password). + Default-DENY since 0.28.0: pass --allow-env-manage-token + to opt in, otherwise this var is ignored and a TTY prompt + is required. Closes AI-exfiltration via subprocess env. KBC_MASTER_TOKEN Master token for sharing ops (global fallback) KBC_MASTER_TOKEN_* Per-project master token (e.g. KBC_MASTER_TOKEN_PROD) KBAGENT_CONFIG_DIR Override config directory diff --git a/src/keboola_agent_cli/commands/data_app.py b/src/keboola_agent_cli/commands/data_app.py index aeff0d0f..52022808 100644 --- a/src/keboola_agent_cli/commands/data_app.py +++ b/src/keboola_agent_cli/commands/data_app.py @@ -583,15 +583,17 @@ def data_app_password( ) -> None: """Retrieve the simpleAuth password for a password-gated data app. - Requires KBC_MANAGE_API_TOKEN in addition to the project's Storage - token. Token is read from env or interactive prompt; never persisted. + Requires the Manage API token in addition to the project's Storage + token. Default-deny since 0.28.0: read from an interactive hidden + prompt; pass top-level --allow-env-manage-token to read + KBC_MANAGE_API_TOKEN from env (CI/CD). Never persisted, never logged. """ if should_hint(ctx): emit_hint(ctx, "data-app.password", project=project, app_id=app_id) return formatter = get_formatter(ctx) service = get_service(ctx, "data_app_service") - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) try: result = service.get_data_app_password( diff --git a/src/keboola_agent_cli/commands/org.py b/src/keboola_agent_cli/commands/org.py index 25b95638..358eb2f2 100644 --- a/src/keboola_agent_cli/commands/org.py +++ b/src/keboola_agent_cli/commands/org.py @@ -202,8 +202,10 @@ def org_setup( Creates Storage API tokens and registers projects. Safe to re-run -- already registered projects are skipped. - The token is read from KBC_MANAGE_API_TOKEN env var or prompted - interactively (never passed as a CLI argument for security). + The Manage API token is read from an interactive hidden prompt by + default (since 0.28.0). Pass the top-level --allow-env-manage-token + flag to read KBC_MANAGE_API_TOKEN from env (CI/CD). Never passed as + a CLI argument. """ if should_hint(ctx): emit_hint(ctx, "org.setup", org_id=org_id, url=url, dry_run=dry_run) @@ -220,7 +222,7 @@ def org_setup( ) raise typer.Exit(code=2) - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) # Build kwargs shared by preview and real call setup_kwargs: dict = { diff --git a/src/keboola_agent_cli/commands/project.py b/src/keboola_agent_cli/commands/project.py index 409e94ea..ccb41a76 100644 --- a/src/keboola_agent_cli/commands/project.py +++ b/src/keboola_agent_cli/commands/project.py @@ -394,7 +394,9 @@ def project_refresh( """Refresh expired or invalid Storage API tokens. Creates new tokens via the Manage API and updates the local config. - Requires a Manage API token (via KBC_MANAGE_API_TOKEN env var or interactive prompt). + Requires a Manage API token: interactive hidden prompt by default + (since 0.28.0); pass top-level --allow-env-manage-token to read + KBC_MANAGE_API_TOKEN from env (CI/CD). \b Examples: @@ -420,7 +422,7 @@ def project_refresh( ) raise typer.Exit(code=2) - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) aliases = [project] if project else None diff --git a/src/keboola_agent_cli/commands/repl.py b/src/keboola_agent_cli/commands/repl.py index 3770af74..7e8d80ff 100644 --- a/src/keboola_agent_cli/commands/repl.py +++ b/src/keboola_agent_cli/commands/repl.py @@ -74,13 +74,17 @@ def _run_repl( config_dir: str | None, deny_writes: bool = False, deny_destructive: bool = False, + allow_env_manage_token: bool = False, ) -> None: """Main REPL loop. Global flags from the outer invocation are re-applied on every command executed inside the REPL. This includes the session-only firewall flags ``--deny-writes`` / ``--deny-destructive`` -- dropping them here would - silently elevate the REPL above the policy the user started it with. + silently elevate the REPL above the policy the user started it with -- + and ``--allow-env-manage-token``, which would otherwise force re-prompts + on every nested ``org setup`` / ``project refresh`` / ``data-app + password`` even after the user opted in at the outer invocation. """ from ..cli import app as typer_app @@ -169,6 +173,8 @@ def _run_repl( full_argv.append("--deny-writes") if deny_destructive and "--deny-destructive" not in argv: full_argv.append("--deny-destructive") + if allow_env_manage_token and "--allow-env-manage-token" not in argv: + full_argv.append("--allow-env-manage-token") full_argv.extend(argv) # Prevent recursive REPL @@ -206,4 +212,5 @@ def repl_command(ctx: typer.Context) -> None: config_dir=None, # Already resolved in ctx deny_writes=ctx.obj.get("deny_writes", False), deny_destructive=ctx.obj.get("deny_destructive", False), + allow_env_manage_token=ctx.obj.get("allow_env_manage_token", False), ) diff --git a/src/keboola_agent_cli/hints/definitions/data_app.py b/src/keboola_agent_cli/hints/definitions/data_app.py index 9a95b080..1581d677 100644 --- a/src/keboola_agent_cli/hints/definitions/data_app.py +++ b/src/keboola_agent_cli/hints/definitions/data_app.py @@ -393,8 +393,11 @@ notes=[ "Password is auto-generated at app create time and cannot be rotated. " "Delete + recreate the app to mint a new one (writeup §11.2).", - "Manage token is read from KBC_MANAGE_API_TOKEN env var or interactive " - "hidden prompt; never persisted, never logged.", + "Manage token is read from an interactive hidden prompt by default " + "(since kbagent v0.28.0). For non-interactive runners, the calling " + "kbagent invocation must pass `--allow-env-manage-token` to opt in " + "to KBC_MANAGE_API_TOKEN env-var resolution. Never persisted, never " + "logged.", ], ) ) diff --git a/src/keboola_agent_cli/hints/definitions/org.py b/src/keboola_agent_cli/hints/definitions/org.py index c704fec7..d033fdd8 100644 --- a/src/keboola_agent_cli/hints/definitions/org.py +++ b/src/keboola_agent_cli/hints/definitions/org.py @@ -32,7 +32,9 @@ ), ], notes=[ - "Uses the Manage API with KBC_MANAGE_API_TOKEN (not Storage token).", + "Uses the Manage API token (not Storage token). Default-deny on env " + "since kbagent v0.28.0: pass top-level `--allow-env-manage-token` " + "to read KBC_MANAGE_API_TOKEN, or run interactively (TTY hidden prompt).", "Service layer creates per-project tokens and registers them in CLI config.", ], ) diff --git a/src/keboola_agent_cli/services/data_app_service.py b/src/keboola_agent_cli/services/data_app_service.py index d5d4b1e7..c7afe74a 100644 --- a/src/keboola_agent_cli/services/data_app_service.py +++ b/src/keboola_agent_cli/services/data_app_service.py @@ -747,7 +747,8 @@ def get_data_app_password( raise KeboolaApiError( message=( "Manage API token is required to read the data-app simpleAuth " - "password. Set KBC_MANAGE_API_TOKEN or run interactively." + "password. Run interactively (default since v0.28.0), or pass " + "--allow-env-manage-token + set KBC_MANAGE_API_TOKEN for CI." ), status_code=0, error_code=ErrorCode.INVALID_TOKEN, diff --git a/tests/test_cli.py b/tests/test_cli.py index 8ea5c3d0..c821b1e6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6826,7 +6826,8 @@ def test_org_setup_dry_run(self, tmp_path: Path) -> None: assert output["data"]["projects_added"][0]["action"] == "would_add" def test_org_setup_with_env_token(self, tmp_path: Path) -> None: - """org setup uses KBC_MANAGE_API_TOKEN env var for authentication.""" + """org setup uses KBC_MANAGE_API_TOKEN env var for authentication + when the caller opts in via --allow-env-manage-token (since 0.28.0).""" config_dir = tmp_path / "config" config_dir.mkdir() @@ -6848,6 +6849,7 @@ def test_org_setup_with_env_token(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--json", "org", "setup", @@ -7259,11 +7261,16 @@ class TestResolveManageToken: """Tests for resolve_manage_token() in _helpers.py.""" def test_token_from_env(self) -> None: - """resolve_manage_token returns token from KBC_MANAGE_API_TOKEN env var.""" + """resolve_manage_token(allow_env=True) returns token from KBC_MANAGE_API_TOKEN env var. + + Default-deny since 0.28.0: callers must pass allow_env=True to opt in + to env-var resolution. The opt-in is plumbed from the top-level + --allow-env-manage-token flag. + """ from keboola_agent_cli.commands._helpers import resolve_manage_token with patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": "env-manage-token"}): - token = resolve_manage_token() + token = resolve_manage_token(allow_env=True) assert token == "env-manage-token" diff --git a/tests/test_data_app_cli.py b/tests/test_data_app_cli.py index fe5b725c..6aabd279 100644 --- a/tests/test_data_app_cli.py +++ b/tests/test_data_app_cli.py @@ -335,6 +335,7 @@ def test_password_success(self, tmp_path: Path, monkeypatch) -> None: monkeypatch.setenv("KBC_MANAGE_API_TOKEN", "manage-token") result = _invoke( [ + "--allow-env-manage-token", "--json", "data-app", "password", diff --git a/tests/test_helpers.py b/tests/test_helpers.py index ca1dbeab..fca52476 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -518,3 +518,131 @@ def test_flags_do_not_mutate_persisted(self) -> None: before = list(persisted.deny) apply_firewall_flags(persisted, deny_writes=True, deny_destructive=True) assert persisted.deny == before, "persisted.deny was mutated in place" + + +class TestResolveManageToken: + """Tests for resolve_manage_token default-deny + opt-in behaviour (since 0.28.0). + + The contract: KBC_MANAGE_API_TOKEN is ignored unless the caller passes + allow_env=True (plumbed from --allow-env-manage-token at the CLI). When + ignored, a one-shot stderr warning is emitted and the resolver falls + through to the TTY-prompt path. With no env and no TTY, it exits 2 with + an error naming the opt-in flag. + """ + + _SENTINEL = "kbagent-test-sentinel-token-9c4f" + + def test_returns_env_when_allow_env_true( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + from keboola_agent_cli.commands._helpers import resolve_manage_token + + monkeypatch.setenv("KBC_MANAGE_API_TOKEN", self._SENTINEL) + result = resolve_manage_token(allow_env=True) + assert result == self._SENTINEL + captured = capsys.readouterr() + assert "found in environment but ignored" not in captured.err + assert self._SENTINEL not in captured.out + assert self._SENTINEL not in captured.err + + def test_default_deny_warns_and_falls_through_to_tty( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + from keboola_agent_cli.commands import _helpers + from keboola_agent_cli.commands._helpers import resolve_manage_token + + monkeypatch.setenv("KBC_MANAGE_API_TOKEN", self._SENTINEL) + # Force the TTY branch. + monkeypatch.setattr(_helpers.sys.stdin, "isatty", lambda: True, raising=False) + # typer.prompt would block on real stdin in tests; replace it. + monkeypatch.setattr(_helpers.typer, "prompt", lambda *a, **k: "from-prompt") + result = resolve_manage_token() # allow_env defaults False + assert result == "from-prompt" + err = capsys.readouterr().err + assert "KBC_MANAGE_API_TOKEN found in environment" in err + assert "--allow-env-manage-token" in err + + def test_default_deny_no_tty_no_env_exits_2( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + import typer + + from keboola_agent_cli.commands import _helpers + from keboola_agent_cli.commands._helpers import resolve_manage_token + + monkeypatch.delenv("KBC_MANAGE_API_TOKEN", raising=False) + monkeypatch.setattr(_helpers.sys.stdin, "isatty", lambda: False, raising=False) + with pytest.raises(typer.Exit) as exc_info: + resolve_manage_token() + assert exc_info.value.exit_code == 2 + err = capsys.readouterr().err + assert "--allow-env-manage-token" in err + assert "Run interactively" in err + + def test_default_deny_with_env_no_tty_warns_then_exits_2( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + import typer + + from keboola_agent_cli.commands import _helpers + from keboola_agent_cli.commands._helpers import resolve_manage_token + + monkeypatch.setenv("KBC_MANAGE_API_TOKEN", self._SENTINEL) + monkeypatch.setattr(_helpers.sys.stdin, "isatty", lambda: False, raising=False) + with pytest.raises(typer.Exit) as exc_info: + resolve_manage_token() + assert exc_info.value.exit_code == 2 + err = capsys.readouterr().err + # Both messages on stderr in this branch. + assert "found in environment but ignored" in err + assert "Run interactively" in err + # And the sentinel is never echoed. + assert self._SENTINEL not in err + + def test_no_env_tty_prompts_normally( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + from keboola_agent_cli.commands import _helpers + from keboola_agent_cli.commands._helpers import resolve_manage_token + + monkeypatch.delenv("KBC_MANAGE_API_TOKEN", raising=False) + monkeypatch.setattr(_helpers.sys.stdin, "isatty", lambda: True, raising=False) + monkeypatch.setattr(_helpers.typer, "prompt", lambda *a, **k: "tty-token") + result = resolve_manage_token() + assert result == "tty-token" + err = capsys.readouterr().err + assert "found in environment but ignored" not in err + + def test_token_value_never_appears_in_captured_output( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + """Regression pin: the sentinel must never leak to stdout/stderr.""" + from keboola_agent_cli.commands._helpers import resolve_manage_token + + monkeypatch.setenv("KBC_MANAGE_API_TOKEN", self._SENTINEL) + result = resolve_manage_token(allow_env=True) + captured = capsys.readouterr() + assert result == self._SENTINEL # returned, but not printed + assert self._SENTINEL not in captured.out + assert self._SENTINEL not in captured.err + + def test_allow_env_with_unset_env_and_no_tty_exits_2( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + """The flag is permission, not promise: with the flag set but no env + AND no TTY, we still exit 2. Pins the no-source-available failure + mode for cron/CI diagnostics.""" + import typer + + from keboola_agent_cli.commands import _helpers + from keboola_agent_cli.commands._helpers import resolve_manage_token + + monkeypatch.delenv("KBC_MANAGE_API_TOKEN", raising=False) + monkeypatch.setattr(_helpers.sys.stdin, "isatty", lambda: False, raising=False) + with pytest.raises(typer.Exit) as exc_info: + resolve_manage_token(allow_env=True) + assert exc_info.value.exit_code == 2 + err = capsys.readouterr().err + assert "Run interactively" in err + # No phantom warning when env was actually empty. + assert "found in environment but ignored" not in err diff --git a/tests/test_manage_token_bulk.py b/tests/test_manage_token_bulk.py new file mode 100644 index 00000000..814969d3 --- /dev/null +++ b/tests/test_manage_token_bulk.py @@ -0,0 +1,87 @@ +"""Pin the bulk-prompt-once contract for resolve_manage_token (since 0.28.0). + +Padak's review on PR #238 explicitly required: when a single command +invocation processes N projects (e.g. `kbagent project refresh --all`), +the manage-token TTY prompt must fire **exactly once** at command entry, +never per-project. That contract holds today by construction: the +resolver lives at command entry, before any per-project loop. This test +pins it so a future refactor that pushes resolution into the service +loop fails loudly. +""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.models import AppConfig, ProjectConfig + +runner = CliRunner() + +TEST_STORAGE_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" + + +def _make_store_n_projects(tmp_path: Path, n: int) -> ConfigStore: + """Create a ConfigStore with N registered projects on different stacks.""" + config_dir = tmp_path / "config" + config_dir.mkdir(exist_ok=True) + store = ConfigStore(config_dir=config_dir) + projects = {} + for i in range(n): + # Spread across multiple stacks to make the bulk-once contract + # meaningful: a per-project loop would naively prompt N times, + # potentially with different stack URLs visible in the prompt. + stack = ["us-east4.gcp", "eu-central-1", "north-europe.azure"][i % 3] + projects[f"proj-{i}"] = ProjectConfig( + stack_url=f"https://connection.{stack}.keboola.com", + token=TEST_STORAGE_TOKEN, + ) + store.save(AppConfig(projects=projects)) + return store + + +class TestBulkPromptOnce: + def test_project_refresh_all_resolves_token_once_for_n_projects(self, tmp_path: Path) -> None: + """`project refresh --all` with 5 projects on 3 stacks must call + ``resolve_manage_token`` exactly once. The resolver lives at command + entry; pushing it into the per-project loop would prompt N times + (or N times with different stack URLs). This test fails loudly if + a future refactor moves the call into a loop. + + Patching ``resolve_manage_token`` directly (rather than the inner + TTY prompt) is the right granularity: it tests the command-layer + contract independently of how the resolver decides between env + and TTY internally.""" + store = _make_store_n_projects(tmp_path, 5) + resolver_mock = MagicMock(return_value="bulk-resolved-token") + mock_org = MagicMock() + mock_org.refresh_tokens.return_value = { + "status": "ok", + "refreshed": [], + "errors": [], + } + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.OrgService") as MockOrgService, + patch( + "keboola_agent_cli.commands.project.resolve_manage_token", + resolver_mock, + ), + ): + MockStore.return_value = store + MockOrgService.return_value = mock_org + result = runner.invoke(app, ["--json", "project", "refresh", "--all", "--yes"]) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + # The contract: exactly one resolver call regardless of project count. + assert resolver_mock.call_count == 1, ( + f"Expected resolve_manage_token to be called once for --all " + f"across 5 projects, got {resolver_mock.call_count} calls -- " + f"per-project resolution regression. Manage-token resolution " + f"must live at command entry, never in a per-project loop." + ) + # The service receives the resolved token and fans out internally. + assert mock_org.refresh_tokens.call_count == 1 + assert mock_org.refresh_tokens.call_args.kwargs["manage_token"] == "bulk-resolved-token" diff --git a/tests/test_manage_token_cli.py b/tests/test_manage_token_cli.py new file mode 100644 index 00000000..1d875925 --- /dev/null +++ b/tests/test_manage_token_cli.py @@ -0,0 +1,188 @@ +"""Tests for the --allow-env-manage-token top-level flag (since 0.28.0). + +Pins the contract: KBC_MANAGE_API_TOKEN is ignored by default; passing +--allow-env-manage-token at the top level restores the legacy env-var +resolution. The flag is session-only (not persisted), mirroring +--deny-writes / --deny-destructive. + +Three CLI surfaces consume the manage token: `org setup`, +`project refresh`, and `data-app password`. Each is tested in both +modes: default-deny (asserts the service was never reached) and +allow-env (asserts the service received manage_token=). +""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.models import AppConfig, ProjectConfig + +runner = CliRunner() + +TEST_STORAGE_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" +SENTINEL_MANAGE_TOKEN = "manage-sentinel-7af3e9c1-test-only" + + +def _make_store(tmp_path: Path) -> ConfigStore: + """Create a ConfigStore with a single registered project.""" + config_dir = tmp_path / "config" + config_dir.mkdir(exist_ok=True) + store = ConfigStore(config_dir=config_dir) + config = AppConfig( + projects={ + "prod": ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_STORAGE_TOKEN, + ) + } + ) + store.save(config) + return store + + +class TestAllowEnvManageTokenFlag: + """Default-deny env + opt-in flag for resolve_manage_token.""" + + def test_project_refresh_default_ignores_env_warns_no_tty_exits_2( + self, tmp_path: Path, monkeypatch + ) -> None: + """Without --allow-env-manage-token, env is ignored, no TTY -> exit 2. + The OrgService.refresh_tokens call site must never be reached.""" + store = _make_store(tmp_path) + monkeypatch.setenv("KBC_MANAGE_API_TOKEN", SENTINEL_MANAGE_TOKEN) + + mock_org = MagicMock() + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.OrgService") as MockOrgService, + ): + MockStore.return_value = store + MockOrgService.return_value = mock_org + result = runner.invoke( + app, ["--json", "project", "refresh", "--project", "prod", "--yes"] + ) + + assert result.exit_code == 2, f"Exit code {result.exit_code}: {result.output}" + # CliRunner combines stdout+stderr; the warning must appear somewhere. + assert "found in environment but ignored" in result.output + assert "--allow-env-manage-token" in result.output + # Crucial: the service was never called. + mock_org.refresh_tokens.assert_not_called() + # The sentinel must not leak into output. + assert SENTINEL_MANAGE_TOKEN not in result.output + + def test_project_refresh_allow_env_uses_env_token(self, tmp_path: Path, monkeypatch) -> None: + """With --allow-env-manage-token, env is honoured and forwarded + to OrgService.refresh_tokens as manage_token kwarg.""" + store = _make_store(tmp_path) + monkeypatch.setenv("KBC_MANAGE_API_TOKEN", SENTINEL_MANAGE_TOKEN) + + mock_org = MagicMock() + mock_org.refresh_tokens.return_value = { + "status": "ok", + "refreshed": [], + "errors": [], + } + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.OrgService") as MockOrgService, + ): + MockStore.return_value = store + MockOrgService.return_value = mock_org + result = runner.invoke( + app, + [ + "--allow-env-manage-token", + "--json", + "project", + "refresh", + "--project", + "prod", + "--yes", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + # Service called once, with the env-supplied manage_token. + assert mock_org.refresh_tokens.call_count == 1 + kwargs = mock_org.refresh_tokens.call_args.kwargs + assert kwargs["manage_token"] == SENTINEL_MANAGE_TOKEN + # Even though the call succeeded, the sentinel must not appear in + # JSON output (the resolver and service handle masking). + assert SENTINEL_MANAGE_TOKEN not in result.output + + def test_data_app_password_default_deny_no_tty_exits_2( + self, tmp_path: Path, monkeypatch + ) -> None: + """data-app password mirrors project refresh: env ignored without + the flag, service never called.""" + store = _make_store(tmp_path) + monkeypatch.setenv("KBC_MANAGE_API_TOKEN", SENTINEL_MANAGE_TOKEN) + + mock_data_app = MagicMock() + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.DataAppService") as MockDataAppService, + ): + MockStore.return_value = store + MockDataAppService.return_value = mock_data_app + result = runner.invoke( + app, + [ + "--json", + "data-app", + "password", + "--project", + "prod", + "--app-id", + "1", + ], + ) + + assert result.exit_code == 2 + assert "found in environment but ignored" in result.output + assert "--allow-env-manage-token" in result.output + mock_data_app.get_data_app_password.assert_not_called() + assert SENTINEL_MANAGE_TOKEN not in result.output + + def test_org_setup_allow_env_passes_token_through(self, tmp_path: Path, monkeypatch) -> None: + """org setup with --allow-env-manage-token forwards the env token + to OrgService.setup_organization.""" + store = _make_store(tmp_path) + monkeypatch.setenv("KBC_MANAGE_API_TOKEN", SENTINEL_MANAGE_TOKEN) + + mock_org = MagicMock() + mock_org.setup_organization.return_value = { + "status": "ok", + "registered": [], + "errors": [], + } + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.OrgService") as MockOrgService, + ): + MockStore.return_value = store + MockOrgService.return_value = mock_org + result = runner.invoke( + app, + [ + "--allow-env-manage-token", + "--json", + "org", + "setup", + "--org-id", + "1", + "--url", + "https://connection.keboola.com", + "--yes", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert mock_org.setup_organization.call_count == 1 + kwargs = mock_org.setup_organization.call_args.kwargs + assert kwargs["manage_token"] == SENTINEL_MANAGE_TOKEN + assert SENTINEL_MANAGE_TOKEN not in result.output From e4874933618656fb49239dc0ef79af4def196a35 Mon Sep 17 00:00:00 2001 From: Petr Date: Wed, 6 May 2026 18:25:34 +0200 Subject: [PATCH 2/5] fix: remove stray conflict marker + add member-invite VERSION GATE entry to keboola-expert.md --- plugins/kbagent/agents/keboola-expert.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 6b5c2f77..42b4a632 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -64,13 +64,13 @@ a critical failure. needed for the current task (e.g. `flow update` needs 0.22.0+, `schedule find` needs 0.23.0+, `config set-default-bucket` needs 0.26.0+, `data-app create / deploy / start / stop / delete / password` -<<<<<<< HEAD need 0.27.0+, `config update` script[] auto-normalize against #245 trap needs 0.28.0+, `storage swap-tables` needs 0.28.0+, env-var manage-token auth for `org setup` / `project refresh` / `data-app password` needs 0.28.0+ with `--allow-env-manage-token` (the env var is default-deny on 0.28.0+), - `storage retype` is a future composite), you + `project invite` / `project member-*` / `project invitation-*` + need 0.26.1+, `storage retype` is a future composite), you MUST refuse the task and return a handoff message to the parent: `"Cannot proceed safely on kbagent . Missing: . Ask user to run kbagent update, then re-invoke me."` Do not attempt From b2a63decd067e8d61f1864f0e0f2810018e6c367 Mon Sep 17 00:00:00 2001 From: Petr Simecek Date: Wed, 6 May 2026 18:27:54 +0200 Subject: [PATCH 3/5] feat(0.26.1): project member & invitation lifecycle -- close the Manage API gap (#253) Adds seven `kbagent project` commands so AI agents (and the Cuesta-training orchestrator) can stop bypassing kbagent and POSTing raw HTTP at `/manage/projects/{id}/invitations`. Headline use case: bulk-invite the 52 Cuesta participants from `clone_results_final.csv`. Patch bump because the surface is additive inside an existing command group. Commands: - `project invite --project ALIAS --email EMAIL --role ROLE [--reason TEXT] [--dry-run]` - `project invite --from-csv FILE [--default-role ROLE] [--workers N] [--dry-run]` - `project member-list --project ALIAS [--include-pending]` - `project invitation-list --project ALIAS` - `project invitation-cancel --project ALIAS --email EMAIL [--invitation-id ID] [--yes]` - `project member-remove --project ALIAS --email EMAIL [--yes]` (destructive) - `project member-set-role --project ALIAS --email EMAIL --role ROLE` Architecture (3-layer): - Client: 6 new ManageClient methods. PATCH (not PUT) for role-change -- PUT returns 404 on this Manage API endpoint even on real members. - Service: new `MemberService` with alias->id resolution, email->user_id / invitation_id lookup, idempotent "already invited / already member" handling (HTTP 400 + msg-substring marker, NOT 422), parallel CSV bulk via ThreadPoolExecutor (8 workers default), single-stack-URL invariant. - Commands: thin Typer wrappers in `commands/project.py`, click.Choice enforces the `PROJECT_ROLES = ('admin','guest','readOnly','share')` whitelist (lifted verbatim from the API's own validation error). Verifications (all on 2026-05-01 against project 5725 / us-east4 stack): - `/users` shape: plain list, role at top level (not nested). - `/invitations` shape: plain list with `user.{id,name,email}`. - POST happy path: 201 + invitation object. - Idempotency: HTTP 400 + "already been invited" / "already a member" -- NOT the 422 the orchestrator scripts assumed. - Role whitelist: exactly 4 values (no `developer`). - DELETE invitation: 204; 404 after re-delete. - DELETE user: 204 success, 400 "Administrator not found" on bad id. - PATCH user role: 200 + updated dict. PUT returns 404. Live verification: invited ottomansky.max@gmail.com to project 5725 as guest, confirmed in invitation-list, re-invited (returned status=noop + note=already_invited), cancelled, confirmed gone. Master cuesta project back to its original 3 members + 1 pre-existing pending invitation. Permission registry: `member-remove` is `destructive`; `member-list` / `invitation-list` are `read`; the rest are `admin`. Hints: `member.py` registers all 7 commands with both `--hint client` (direct ManageClient) and `--hint service` (MemberService) renderers. Tests: 50 new + e2e_invite (gated on E2E_MANAGE_TOKEN + E2E_INVITE_PROJECT_ID, opt-in via `make test-e2e-invite`). Total suite: 2410 tests pass. Plugin sync (silent-drift surfaces all walked): - `keboola-expert.md`: Rule 6 VERSION GATE updated; 7 new matrix rows under "Project administration"; 3 new inline gotchas (HTTP 400 noop, PATCH-not-PUT, parallel-bulk ordering). - `SKILL.md`: 6 new description triggers (invite, member, role, ...); workflow link to new `member-workflow.md`; auto-table regenerated. - `commands-reference.md`: new "Project members & invitations" section. - `gotchas.md`: 3 new (since v0.26.1) entries. - `member-workflow.md`: new workflow doc (single + bulk + audit + role change + remove + idempotency cheat-sheet). - `commands/context.py` AGENT_CONTEXT, `CLAUDE.md` All CLI Commands, `plugin.json` / `marketplace.json` (via `make version-sync`). Co-authored-by: ottomansky --- CLAUDE.md | 7 + Makefile | 5 +- plugins/kbagent/agents/keboola-expert.md | 30 + plugins/kbagent/skills/kbagent/SKILL.md | 11 + .../kbagent/references/commands-reference.md | 12 + .../skills/kbagent/references/gotchas.md | 30 + .../kbagent/references/member-workflow.md | 171 +++++ pyproject.toml | 1 + src/keboola_agent_cli/changelog.py | 8 + src/keboola_agent_cli/cli.py | 3 + src/keboola_agent_cli/commands/context.py | 31 + src/keboola_agent_cli/commands/project.py | 495 +++++++++++++++ src/keboola_agent_cli/constants.py | 12 + .../hints/definitions/__init__.py | 1 + .../hints/definitions/member.py | 218 +++++++ src/keboola_agent_cli/manage_client.py | 77 +++ src/keboola_agent_cli/models.py | 70 +++ src/keboola_agent_cli/permissions.py | 6 + .../services/member_service.py | 593 ++++++++++++++++++ tests/test_e2e.py | 134 ++++ tests/test_manage_client.py | 212 +++++++ tests/test_member_cli.py | 564 +++++++++++++++++ tests/test_member_service.py | 585 +++++++++++++++++ 23 files changed, 3275 insertions(+), 1 deletion(-) create mode 100644 plugins/kbagent/skills/kbagent/references/member-workflow.md create mode 100644 src/keboola_agent_cli/hints/definitions/member.py create mode 100644 src/keboola_agent_cli/services/member_service.py create mode 100644 tests/test_member_cli.py create mode 100644 tests/test_member_service.py diff --git a/CLAUDE.md b/CLAUDE.md index 56081a6a..92d60ed9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -264,6 +264,13 @@ kbagent project description-get --project NAME kbagent project description-set --project NAME [--text STR | --file PATH | --stdin] kbagent project use ALIAS kbagent project current +kbagent project invite --project ALIAS --email EMAIL --role admin|guest|readOnly|share [--reason TEXT] [--dry-run] +kbagent project invite --from-csv FILE [--default-role ROLE] [--workers N] [--dry-run] +kbagent project member-list --project ALIAS [--include-pending] +kbagent project invitation-list --project ALIAS +kbagent project invitation-cancel --project ALIAS --email EMAIL [--invitation-id ID] [--yes] +kbagent project member-remove --project ALIAS --email EMAIL [--yes] +kbagent project member-set-role --project ALIAS --email EMAIL --role admin|guest|readOnly|share kbagent config list [--project NAME] [--component-type TYPE] [--component-id ID] [--branch ID] [--include-rows] kbagent config detail --project NAME [--project NAME ...] --component-id ID [--config-id ID] [--branch ID] [--with-state] diff --git a/Makefile b/Makefile index 10c880da..f50f42ff 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help -.PHONY: help install install-mcp sync test test-unit test-integration test-e2e test-file lint lint-fix format format-check skill-check skill-gen version-sync version-check changelog changelog-check check-error-codes check clean hooks +.PHONY: help install install-mcp sync test test-unit test-integration test-e2e test-e2e-invite test-file lint lint-fix format format-check skill-check skill-gen version-sync version-check changelog changelog-check check-error-codes check clean hooks help: ## Show this help message @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' @@ -26,6 +26,9 @@ test-integration: ## Run integration tests only test-e2e: ## Run E2E tests (E2E_API_TOKEN and E2E_URL required) uv run pytest tests/test_e2e.py -v -s --tb=long +test-e2e-invite: ## Run project invite E2E (E2E_MANAGE_TOKEN + E2E_INVITE_PROJECT_ID required) + uv run pytest tests/test_e2e.py -v -s --tb=long -m e2e_invite + test-file: ## Run a specific test file (FILE=tests/test_cli.py) uv run pytest $(FILE) -v diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 42b4a632..50d7ba16 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -112,6 +112,13 @@ a critical failure. | Pause a running data app | `kbagent data-app stop --project P --app-id N` (0.27.0+) | -- | `kbagent data-app delete` (irreversible; cascades to Storage config) | | Read the simpleAuth password for a password-gated app | `kbagent data-app password --project P --app-id N` (0.27.0+) -- needs Manage API token (interactive prompt by default; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+) | -- | trying to "rotate" the password (not supported by the API; delete + recreate to mint a new one) | | Tear down a data app | `kbagent data-app delete --project P --app-id N` (0.27.0+) -- cascades to Storage config; URL retired | -- | manually `tool call delete_config keboola.data-apps` while leaving the deployment record orphaned | +| Invite a user to a project (single) | `kbagent project invite --project P --email E --role admin\|guest\|readOnly\|share` (0.26.1+) | raw `requests.post(/manage/projects/{id}/invitations)` only if version-gated out | `kbagent project invite` without `KBC_MANAGE_API_TOKEN` set; passing manage token via CLI flag | +| Invite many users (bulk) | `kbagent project invite --from-csv FILE [--default-role guest] [--workers N] [--dry-run]` (0.26.1+) | `--hint client` to generate a parallel script using `ManageClient` | per-row shell loop calling the CLI -- defeats the parallelism + idempotency the service already does | +| List active project members | `kbagent project member-list --project P [--include-pending]` (0.26.1+) | `tool call run_sync_action` against the Manage API | reading `.kbagent/config.json` to infer membership (it only stores the local user's token) | +| List pending invitations | `kbagent project invitation-list --project P` (0.26.1+) | -- | -- | +| Cancel a pending invitation | `kbagent project invitation-cancel --project P --email E --yes` (0.26.1+) | `--invitation-id ID` if email lookup is ambiguous | DELETE via raw HTTP without going through the service layer | +| Remove an active member | `kbagent project member-remove --project P --email E --yes` (0.26.1+, **destructive**) | `--hint client` for a script that removes by user_id directly | calling `member-remove` without `--yes` in non-interactive contexts (it will prompt and hang) | +| Change a member's role | `kbagent project member-set-role --project P --email E --role admin\|guest\|readOnly\|share` (0.26.1+) | -- | `PUT /manage/projects/{id}/users/{userId}` -- the API rejects PUT with 404, the kbagent client correctly uses **PATCH** | If the table does not cover the user's task, **ask clarifying questions** instead of guessing. Returning a targeted question is a @@ -187,6 +194,29 @@ success, not a failure. verification payload but do not treat it as a failure signal. Production writes never materialize anything. +- **`project invite` "already invited / already member" is a no-op, not a failure** (0.26.1+): + Re-inviting a user the project already knows returns HTTP 400 from the + Manage API. kbagent normalises both "...already been invited..." and + "...already a member..." to `status="noop"` with a `note` field, exit 0. + **Do not retry on 400 from these commands** -- the user is already + on the project (or already pending). For bulk runs, `noop` rows count + toward `noop`, not `failed`, in the summary; surface that distinction + to the user when reporting bulk results. + +- **`project invite --from-csv` ordering is non-deterministic** (0.26.1+): + Bulk invitation parallelises via `ThreadPoolExecutor` (default 8 workers). + The `rows[]` array in the JSON result is in completion order, not CSV + order. When reporting per-row outcomes to the user, **match by `email`, + not by index**. Partial-success exits 0 with `failed > 0` reflected in + the JSON -- treat that as a soft failure that needs review, not a + catastrophe. + +- **`project member-set-role` uses PATCH, not PUT** (0.26.1+): The Manage + API endpoint is `PATCH /manage/projects/{id}/users/{userId}` with + `{"role": "..."}`. PUT returns 404 even on real members. kbagent's + `ManageClient.update_project_member_role` emits PATCH; if you write a + `--hint client` script that hits the endpoint directly, do the same. + - **`legacy_branch_storage: true` on `--branch` writes** (0.25.2+): Projects without the `storage-branches` feature flag (legacy fake-branch projects) accept `--branch X` writes at the API level, but the diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 27cb91bb..c0d98874 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -31,6 +31,10 @@ description: > app proxy, simpleAuth, app auto-suspend, configVersion, redeploy contract, Data Science API, /apps endpoint, app password, KBC::Project ciphertext, local workspace, project directory, kbagent init. + local workspace, project directory, kbagent init, + invite user, invite member, project invitation, manage members, + list members, remove member, change role, project role, + bulk invite, invite from CSV, project access, member management. --- # kbagent -- Keboola Agent CLI @@ -92,6 +96,12 @@ When working inside a git repository or project directory, run `kbagent init` (o | Show the effective default project | `kbagent project current` | | Get the Keboola dashboard project description | `kbagent project description-get --project PROJECT` | | Set the Keboola dashboard project description (markdown) | `kbagent project description-set --project PROJECT` | +| Invite a user (or many users via CSV) to one or more projects | `kbagent project invite` | +| List active members of a project (and optionally pending invitations) | `kbagent project member-list --project PROJECT` | +| List pending project invitations | `kbagent project invitation-list --project PROJECT` | +| Cancel a pending invitation | `kbagent project invitation-cancel --project PROJECT --email EMAIL` | +| Remove an active member from a project (destructive) | `kbagent project member-remove --project PROJECT --email EMAIL` | +| Change an existing member's role (PATCH) | `kbagent project member-set-role --project PROJECT --email EMAIL --role ROLE` | | Set up projects and register them in the kbagent config | `kbagent org setup --url URL` | | List available components from connected projects | `kbagent component list` | | Show detailed information about a specific component | `kbagent component detail --component-id COMPONENT-ID` | @@ -247,6 +257,7 @@ For detailed response parsing rules and common pitfalls, see [gotchas](reference | **Storage column types** (native types, NOT NULL, DEFAULT, branch materialize) | [storage-types-workflow](references/storage-types-workflow.md) | | **Typify a typeless table** (profile -> CTAS -> swap-tables -> validate -> handoff) | [typify-table-workflow](references/typify-table-workflow.md) | | Bucket sharing & linking | [sharing-workflow](references/sharing-workflow.md) | +| **Project members & invitations** (single + bulk via CSV, role change, remove) | [member-workflow](references/member-workflow.md) | | Dev branches | [branch-workflow](references/branch-workflow.md) | | Encrypting secrets for MCP tools | [encrypt-workflow](references/encrypt-workflow.md) | | Sync & Git-branching (GitOps) | [sync-workflow](references/sync-workflow.md) | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 71836b9a..d58a31d9 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -21,6 +21,18 @@ All commands support `--json` for structured output. Multi-project flags (`--pro - `project use ALIAS` -- pin `ALIAS` as the persistent default project. Stored as `default_project` in config.json. Overridden at runtime by `KBAGENT_PROJECT=ALIAS` (env, beats pin) and by `--project ALIAS` (CLI flag, beats both) - `project current` -- print the effective default project and its source (`env` / `pin` / `none`). Reports both the env override AND the persisted pin so misconfigurations are visible. Returns `{"alias": null, "source": "none"}` when neither is set +## Project Members & Invitations (since v0.26.1) + +All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the project's Storage token. Allowed roles are exactly `admin`, `guest`, `readOnly`, `share` -- the API self-reports this list in its 400 validation error and `constants.PROJECT_ROLES` mirrors it. + +- `project invite --project ALIAS --email EMAIL --role admin|guest|readOnly|share [--reason TEXT] [--dry-run]` -- single-shot invitation. Returns `{"status": "ok", "invitation_id": ..., ...}`. Re-inviting an already-invited or already-member email returns `{"status": "noop", "note": "already_invited" | "already_member"}` (HTTP 400 from the Manage API, normalised to a no-op). +- `project invite --from-csv FILE [--default-role ROLE] [--workers N] [--dry-run]` -- bulk invitation. CSV header required; columns: `email`, `project` (alias) or `project_id` (numeric), `role` (optional with `--default-role`), `reason` (optional). Parallelised via `ThreadPoolExecutor` (default 8 workers). Single-stack-URL invariant per file: rows referencing different stacks raise `ConfigError` upfront. Result is `{"total","succeeded","noop","failed","rows":[...]}`; `rows[]` order is *not deterministic*. Exit 0 even with `failed > 0` -- inspect the JSON. +- `project member-list --project ALIAS [--include-pending]` -- list active members. Each member dict carries `id`, `email`, `name`, `role`, `status`, `mfa_enabled`. With `--include-pending`, the response also includes `pending_invitations: [...]`. +- `project invitation-list --project ALIAS` -- list pending (unaccepted) invitations only. +- `project invitation-cancel --project ALIAS --email EMAIL [--invitation-id ID] [--yes]` -- cancel a pending invitation. Without `--invitation-id`, the service resolves it by listing pending invitations and matching `--email` (case-insensitive). 204 No Content on success; `KeboolaApiError(NOT_FOUND)` if the email has no pending invitation. +- `project member-remove --project ALIAS --email EMAIL [--yes]` -- destructive: remove an active member. The service resolves `--email` to the numeric `user_id` (case-insensitive) and DELETEs `/manage/projects/{id}/users/{userId}`. Re-add the user via `project invite`. +- `project member-set-role --project ALIAS --email EMAIL --role admin|guest|readOnly|share` -- change an existing member's role. Uses **PATCH** `/manage/projects/{id}/users/{userId}` with `{"role": "..."}`. PUT does *not* work on this endpoint -- pre-v0.26.1 implementations that tried PUT got a misleading 404. + ## Permission flags (top-level, session-only) - `--deny-writes` -- block all write/destructive/admin operations for this single invocation. Merges with any persisted permission policy; never written to config.json. Exit code 6 (PERMISSION_DENIED) on blocked operations - `--deny-destructive` -- block only destructive operations (delete-table, delete-bucket, terminate-job, etc.) for this invocation. Pure-write ops like create-table stay allowed. Use this when you want to keep build-up capabilities but lock out tear-downs diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 0ac8e63d..c2898e6c 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -104,6 +104,36 @@ container after `autoSuspendAfterSeconds` of inactivity. Hit the URL to wake it (auto-restart triggers a 30-60s cold boot) or run `kbagent data-app start --app-id N`. +## `project invite` "already invited / already member" returns HTTP 400, not 422 (since v0.26.1) + +- Re-inviting a user the project already knows about returns HTTP **400** with + one of two error strings: + - `"This user has already been invited to this project."` (pending invitation) + - `"This user is already a member of this project."` (active member) +- `MemberService.invite()` translates both cases to `status="noop"` with + `note="already_invited"` / `"already_member"` -- they are *not* exit-1 + failures. Bulk runs (`--from-csv`) count them as `noop` in the summary, not + `failed`. +- The 422 heuristic in pre-v0.26.1 orchestrator scripts (`invite_participants.py:25`) + is **wrong** for this API. If you write a parallel implementation, key off + status_code 400 + the substring marker, not 422. + +## `project member-set-role` is PATCH, not PUT (since v0.26.1) + +- The Manage API role-change endpoint is `PATCH /manage/projects/{id}/users/{userId}` + with body `{"role": "..."}`. **PUT returns 404** ("resource not found") even + on a real, currently-active member -- the endpoint shape is PATCH-only. +- The kbagent `ManageClient.update_project_member_role` method emits PATCH; + any custom code re-implementing the call must do the same. + +## `project invite --from-csv` order is not deterministic (since v0.26.1) + +- Bulk invitation parallelises via `ThreadPoolExecutor` (default 8 workers). + The `rows[]` array in the result is in completion order, not CSV order. +- Per-row parsing of `failed_rows` should match by `email`, not by index. +- A failed row never aborts the run -- the executor accumulates results and + the command exits 0 with `failed > 0` reflected in the JSON summary. Mirror + the `org setup` partial-success exit semantics. ## `default_bucket` is per-config and only an output prefix (since 0.26.0) diff --git a/plugins/kbagent/skills/kbagent/references/member-workflow.md b/plugins/kbagent/skills/kbagent/references/member-workflow.md new file mode 100644 index 00000000..218172bf --- /dev/null +++ b/plugins/kbagent/skills/kbagent/references/member-workflow.md @@ -0,0 +1,171 @@ +# Project Member & Invitation Workflow (since v0.26.1) + +Closes the long-standing Manage API gap that forced every Keboola-internal +automation (most recently the Cuesta-training orchestrator) to bypass kbagent +and POST raw HTTP at `/manage/projects/{id}/invitations`. + +## Auth + +All seven commands use the **Manage API**, not the Storage API. Provide the +manage token via `KBC_MANAGE_API_TOKEN` (env var or interactive prompt). The +manage token is *never* persisted to config.json, *never* accepted as a CLI +argument, *never* logged. + +```bash +export KBC_MANAGE_API_TOKEN= +``` + +## Roles (whitelist) + +The Manage API accepts exactly four role values: + +| Role | Use case | +|------|----------| +| `admin` | Full project control (create/delete tokens, manage members, all data ops) | +| `share` | Read-only with sharing rights to other projects in the org | +| `readOnly` | Read-only | +| `guest` | Lowest blast radius; useful for temporary access (and for the e2e test) | + +Both Typer (`click.Choice`) and `MemberService._validate_role()` enforce this +list. The whitelist is defined in `constants.PROJECT_ROLES`. + +## Single invite + +```bash +kbagent project invite --project prod --email a@b.com --role admin --reason "On-call rotation" +``` + +Returns: +```json +{ + "status": "ok", + "invitation_id": 1741, + "alias": "prod", + "project_id": 5725, + "email": "a@b.com", + "role": "admin" +} +``` + +If the user is already invited or already a member, the API returns HTTP 400 +and kbagent translates it to `{"status": "noop", "note": "already_invited" | "already_member"}` -- this is **not** an error and exit code stays 0. + +## Bulk invite from CSV (the headline use case) + +CSV header required. Recognised columns (case-insensitive): `email` (required), +`project` (alias) **or** `project_id` (numeric integer), `role` +(optional if `--default-role` is set), `reason` (optional). Extra columns are +ignored. Each row may pick a different project as long as **all rows resolve +to the same stack URL** (rows referencing multiple stacks raise upfront +before any HTTP call). + +```csv +email,project,role,reason +ann@example.com,prod,admin,On-call +ben@example.com,staging,guest,Read-only access for QA +chen@example.com,5725,share,Shared bucket consumer +``` + +```bash +kbagent project invite --from-csv participants.csv --default-role guest --workers 8 +``` + +Result schema: +```json +{ + "total": 3, + "succeeded": 2, + "noop": 1, + "failed": 0, + "rows": [ + {"email": "ann@example.com", "project": "prod", "role": "admin", "status": "ok", "invitation_id": 1741, ...}, + {"email": "ben@example.com", "project": "staging", "role": "guest", "status": "noop", "note": "already_invited", ...}, + {"email": "chen@example.com", "project": "5725", "project_id": 5725, "role": "share", "status": "ok", "invitation_id": 1742, ...} + ], + "dry_run": false +} +``` + +The `rows[]` array is in **completion order**, not CSV order (parallel +workers). Match by `email`, not by index. Partial-success exits 0 with +`failed > 0` reflected in the JSON; this mirrors `org setup`. + +`--dry-run` resolves every row and reports what *would* happen without +sending invitations. Use it before any large CSV. + +## Audit who is on a project + +```bash +kbagent project member-list --project prod --include-pending +``` + +Returns active members + pending invitations in one shot: +```json +{ + "alias": "prod", + "project_id": 5725, + "members": [ + {"id": 216, "email": "max.ottomansky@keboola.com", "role": "admin", "status": "active", "mfa_enabled": true, ...}, + {"id": 4241, "email": "mfiser@cuestapartners.com", "role": "guest", "status": "active", "mfa_enabled": true, ...} + ], + "pending_invitations": [ + {"id": 1515, "user": {"email": "marcusscwong@gmail.com"}, "role": "admin", "reason": "", ...} + ] +} +``` + +For the pending-only view: `kbagent project invitation-list --project prod`. + +## Change a member's role + +Uses HTTP **PATCH** under the hood (PUT returns 404 even on real members -- +that's the Manage API's quirk, not a kbagent bug). + +```bash +kbagent project member-set-role --project prod --email a@b.com --role guest +``` + +The service resolves `--email` to the numeric user_id by listing project +members and matching case-insensitively. The PATCH response includes the +updated user dict. + +## Cancel a pending invitation + +```bash +kbagent project invitation-cancel --project prod --email a@b.com --yes +``` + +Without `--invitation-id`, the service resolves the ID by listing pending +invitations and matching `--email`. With `--invitation-id ID`, it skips the +lookup. The DELETE returns 204 No Content on success; if the invitation has +already been deleted the API returns 404 with "Invitation not found". + +## Remove an active member (destructive) + +```bash +kbagent project member-remove --project prod --email a@b.com --yes +``` + +Resolves `--email` to user_id, then DELETEs `/manage/projects/{id}/users/{userId}`. +Permission category: `destructive` (re-adding requires sending a fresh invite). + +## Idempotency cheat-sheet + +| API response | kbagent translation | Exit code | +|--------------|--------------------|-----------| +| HTTP 201 invitation created | `status="ok"` | 0 | +| HTTP 400 "...already been invited..." | `status="noop"`, `note="already_invited"` | 0 | +| HTTP 400 "...already a member..." | `status="noop"`, `note="already_member"` | 0 | +| HTTP 400 "Role X is not valid..." | Re-raised; `--role` should be on the whitelist | 1 | +| HTTP 401 invalid manage token | `KeboolaApiError(INVALID_TOKEN)` | 3 | +| HTTP 403 manage token lacks org-admin | `KeboolaApiError(ACCESS_DENIED)` | 1 | +| HTTP 404 project / invitation not found | `KeboolaApiError(NOT_FOUND)` | 1 | + +## When to use the Manage API direct-add (not in v0.26.1) + +The Manage API also exposes `POST /manage/projects/{id}/users` with body +`{"email": "...", "role": "..."}`. This **directly creates a member without +sending an email** -- useful for org-internal automation, dangerous for +public-facing flows. v0.26.1 deliberately does NOT expose this path because +its semantics differ from `invite`. If you need it, talk to the maintainers +about a future `member-add-direct` command. diff --git a/pyproject.toml b/pyproject.toml index dfca5d5a..a4ff6b89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ pythonpath = ["src", "tests"] markers = [ "integration: marks tests as integration tests requiring real API credentials (deselect with '-m \"not integration\"')", "e2e: marks tests as end-to-end tests requiring real API credentials (deselect with '-m \"not e2e\"')", + "e2e_invite: project invite E2E -- requires E2E_MANAGE_TOKEN + E2E_INVITE_PROJECT_ID; opt-in via 'make test-e2e-invite'", ] [tool.ruff] diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 5430db77..bfaa7d67 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -31,6 +31,14 @@ "Tests: 30 service-level tests in `tests/test_data_app_service.py` (validation, dry-run, happy-path orchestration, cleanup-in-finally, encryption-failure-aborts-loud, poll-loop semantics including the transient-stopped invariant), 10 CLI tests in `tests/test_data_app_cli.py` (mutual-exclusion validation, dual JSON+human output, `--yes` for delete, manage-token forwarding for password without leaking the token to stdout/stderr).", "Plugin: new `data-app-workflow.md` reference + two `(since v0.27.0)` gotcha entries (the §9 redeploy contract; cross-project KMS ciphertext mismatch). `keboola-expert.md` matrix gains five rows (`create`, `deploy`, `start`, `stop`, `delete`).", ], + "0.26.1": [ + "New: project member & invitation lifecycle. Closes the long-standing Manage API gap that forced every Keboola-internal automation (most recently `17_CuestaDemo/scripts/replicate_master.py` and `invite_participants.py`) to bypass kbagent and POST raw HTTP at `/manage/projects/{id}/invitations`. Seven new commands under `kbagent project`: `invite` (single-shot or `--from-csv` bulk with `ThreadPoolExecutor` parallelism, default 8 workers), `member-list` (active members, `--include-pending` adds pending invitations), `invitation-list`, `invitation-cancel` (resolves invitation_id by email lookup so callers don't have to), `member-remove` (destructive; resolves user_id by email), `member-set-role` (PATCH `/manage/projects/{id}/users/{userId}` with `{role}`). All seven require `KBC_MANAGE_API_TOKEN`; the manage token is never logged, never persisted, never accepted on the CLI line. Permission registry: `member-remove` is `destructive`, `member-list` / `invitation-list` are `read`, the rest are `admin`.", + "New: role whitelist `PROJECT_ROLES = ('admin', 'guest', 'readOnly', 'share')` in `constants.py`, lifted verbatim from the Manage API's own validation error message (verified empirically on 2026-05-01 against `connection.us-east4.gcp.keboola.com`). Typer enforces the whitelist via `click.Choice` at the command layer; `MemberService` double-checks for defence-in-depth. Invalid role values now fail-fast with `Role 'X' is not valid. Allowed roles are: admin, guest, readOnly, share` instead of letting the API return an opaque 400.", + "New: `MemberService` (`src/keboola_agent_cli/services/member_service.py`) wrapping six new `ManageClient` methods (`create_project_invitation`, `list_project_invitations`, `cancel_project_invitation`, `list_project_members`, `remove_project_member`, `update_project_member_role`). Resolves project alias -> (stack_url, project_id) via `ConfigStore`; resolves email -> numeric user_id / invitation_id by listing + matching case-insensitively. Treats the Manage API's HTTP 400 'already been invited' / 'already a member' responses as `status=noop` rather than errors (the heuristic the orchestrator scripts had to do via substring matching, now typed to `status_code == 400` AND message-substring marker constants). `--from-csv` enforces a single-stack-URL invariant per file (rows referencing multiple stacks raise `ConfigError` upfront).", + "New: hint definitions (`hints/definitions/member.py`) for all seven commands. Both `--hint client` (direct `ManageClient` calls) and `--hint service` (`MemberService` calls) generate runnable Python.", + "New: e2e marker `e2e_invite` (registered in `pyproject.toml`). `make test-e2e-invite` runs `tests/test_e2e.py::test_project_invite_e2e` against a real Manage API; gated on `E2E_MANAGE_TOKEN` + `E2E_INVITE_PROJECT_ID` (skips cleanly when missing). The test invites `ottomansky.max@gmail.com` (override via `E2E_INVITE_EMAIL`) as `guest`, asserts the invitation appears in `invitation-list`, then cancels it -- the same run that proves the system can send confirms it can clean up.", + "Docs: new `references/member-workflow.md` (golden paths for single invite, bulk invite, audit, role change, remove). `gotchas.md` gains three `(since v0.26.1)` entries -- 'already invited / already member' returns HTTP 400 not 422; role-change is PATCH not PUT (PUT returns 404 even on real members); bulk-invite ordering is not deterministic (parallel workers). `keboola-expert.md` adds seven matrix rows under 'Project administration' plus a Rule 6 VERSION GATE entry. `commands-reference.md` adds a 'Project members & invitations' section.", + ], "0.26.0": [ "New: `kbagent config set-default-bucket --bucket BUCKET_ID | --clear [--dry-run] [--branch ID]` -- discoverable wrapper around the raw-mode `storage.output.default_bucket` workaround documented at https://keboola.atlassian.net/wiki/spaces/SUP/pages/3770155030/ (epic KBCP-108). Read-modify-write that preserves all sibling keys under `storage.output` and the rest of the configuration. Same-value writes short-circuit with `{\"changed\": false}` (no API call, no version bump). `--clear` removes only the `default_bucket` key, leaving an empty `storage.output: {}` if no other siblings live there (intentional -- mirrors `set_nested_value`'s parent-creation semantics; Storage API treats `output: {}` and missing `output` identically as 'use the auto-derived bucket'). Live-validated end-to-end on three component types -- row-based GCS extractor, root-only `keboola.ex-cnb-exchange-rates`, and `ex-generic-v2` with multiple jobs -- output tables routed to the configured bucket at job runtime in every case. The per-table `destination` override (the second method shown in the support article) keeps using the existing `kbagent config update --set 'storage.output.tables=[...]'` -- no new wrapper there because per-table mappings have many fields that don't fit a single-purpose flag.", "Fix: `kbagent sync pull --with-samples` no longer crashes with `TypeError: '>' not supported between instances of 'NoneType' and 'int'` when one or more tables in the project return `rowsCount: null` from the Storage API (typical for newly-created or empty tables on some backends, reproduced live against `kosik-sales`). `dict.get(\"rowsCount\", 0)` returns the default `0` only when the key is **missing** -- if the key is present with a `null` value, `.get()` returns `None`, and the `> 0` comparison crashed Python 3 before any sample was fetched. The filter and sort key in `SyncService._fetch_samples()` now coerce `None` to `0` via a small `_rows()` helper used in both places (`t.get(\"rowsCount\") or 0`), so empty/null-rowcount tables are gracefully skipped exactly like `rowsCount: 0` ones. Closes #233.", diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index e97b8b54..d9d7974c 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -48,6 +48,7 @@ from .services.kai_service import KaiService from .services.lineage_service import LineageService from .services.mcp_service import McpService +from .services.member_service import MemberService from .services.org_service import OrgService from .services.project_service import ProjectService from .services.schedule_service import ScheduleService @@ -302,6 +303,7 @@ def main( lineage_service = LineageService(config_store=config_store) deep_lineage_service = DeepLineageService(config_store=config_store) org_service = OrgService(config_store=config_store) + member_service = MemberService(config_store=config_store) mcp_service = McpService(config_store=config_store) branch_service = BranchService(config_store=config_store) sharing_service = SharingService(config_store=config_store) @@ -356,6 +358,7 @@ def main( ctx.obj["lineage_service"] = lineage_service ctx.obj["deep_lineage_service"] = deep_lineage_service ctx.obj["org_service"] = org_service + ctx.obj["member_service"] = member_service ctx.obj["mcp_service"] = mcp_service ctx.obj["branch_service"] = branch_service ctx.obj["sharing_service"] = sharing_service diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 434283ef..3921487a 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -94,6 +94,37 @@ Print the effective default project and its source (env / pin / none). Resolution order for single-project operations: --project > KBAGENT_PROJECT > pin. +### Project Members & Invitations (since v0.26.1) + + Requires KBC_MANAGE_API_TOKEN (Manage API auth). Allowed roles: admin, guest, readOnly, share. + + kbagent project invite --project ALIAS --email EMAIL --role ROLE [--reason TEXT] [--dry-run] + Send an invitation email. Re-inviting an existing invitee or member is a no-op + (HTTP 400 from the Manage API; the service returns status="noop" with note + "already_invited" / "already_member"). + + kbagent project invite --from-csv FILE [--default-role ROLE] [--workers N] [--dry-run] + Bulk invite. CSV must have a header row with columns: email, project (alias or + numeric ID), role (optional if --default-role is given), reason (optional). + Parallelised with ThreadPoolExecutor (default 8 workers). Per-row results in + `rows[]` with status=ok|noop|failed; `failed_rows` ordering is not deterministic. + + kbagent project member-list --project ALIAS [--include-pending] + List active project members. --include-pending also fetches pending invitations. + + kbagent project invitation-list --project ALIAS + List pending (unaccepted) invitations only. + + kbagent project invitation-cancel --project ALIAS --email EMAIL [--invitation-id ID] [--yes] + Cancel a pending invitation. Without --invitation-id, the service resolves the + ID by listing pending invitations and matching --email (case-insensitive). + + kbagent project member-remove --project ALIAS --email EMAIL [--yes] + Remove an active member (destructive). Service resolves --email to user_id. + + kbagent project member-set-role --project ALIAS --email EMAIL --role ROLE + Change an existing member's role via PATCH /manage/projects/{{id}}/users/{{userId}}. + ### Component Discovery kbagent component list [--project NAME] [--type TYPE] [--query "search"] diff --git a/src/keboola_agent_cli/commands/project.py b/src/keboola_agent_cli/commands/project.py index ccb41a76..f202d2a8 100644 --- a/src/keboola_agent_cli/commands/project.py +++ b/src/keboola_agent_cli/commands/project.py @@ -8,15 +8,18 @@ from pathlib import Path from typing import Any +import click import typer from rich.console import Console from rich.table import Table from ..constants import ( + DEFAULT_INVITE_WORKERS, DEFAULT_STACK_URL, DEFAULT_TOKEN_DESCRIPTION, ENV_KBC_STORAGE_API_URL, ENV_KBC_TOKEN, + PROJECT_ROLES, ) from ..errors import ConfigError, ErrorCode, KeboolaApiError from ._helpers import ( @@ -654,3 +657,495 @@ def project_description_set( except ConfigError as exc: formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) raise typer.Exit(code=5) from None + + +# ── Project members & invitations (since v0.26.1) ───────────────────── + + +def _format_invite_result(console: Console, data: dict[str, Any]) -> None: + """Single-shot invite result.""" + status = data.get("status", "") + if status == "ok": + console.print( + f"[bold green]Invited[/bold green] {data['email']} to " + f"[cyan]{data['alias']}[/cyan] as [yellow]{data['role']}[/yellow] " + f"(invitation_id={data.get('invitation_id')})." + ) + elif status == "noop": + console.print( + f"[yellow]No-op[/yellow]: {data['email']} on [cyan]{data['alias']}[/cyan] " + f"-- {data.get('note', '')}." + ) + elif status == "dry_run": + console.print( + f"[dim]Would invite[/dim] {data['email']} to [cyan]{data['alias']}[/cyan] " + f"as [yellow]{data['role']}[/yellow]." + ) + else: + console.print(f"[bold red]Unexpected status[/bold red]: {data!r}") + + +def _format_bulk_invite_result(console: Console, data: dict[str, Any]) -> None: + """Render the bulk-invite summary table.""" + console.print( + f"\n[bold]Bulk invite:[/bold] total={data['total']} " + f"succeeded={data['succeeded']} noop={data['noop']} failed={data['failed']}" + + (" [dim](dry-run)[/dim]" if data.get("dry_run") else "") + ) + rows = data.get("rows") or [] + if not rows: + return + table = Table(title="Per-row results") + table.add_column("Status", style="bold") + table.add_column("Email") + table.add_column("Project") + table.add_column("Role") + table.add_column("Note") + status_style = {"ok": "green", "noop": "yellow", "failed": "red"} + for row in rows: + status = row.get("status", "") + style = status_style.get(status, "white") + table.add_row( + f"[{style}]{status}[/{style}]", + row.get("email", ""), + row.get("project", ""), + row.get("role", ""), + row.get("note", ""), + ) + console.print(table) + + +def _format_member_list(console: Console, data: dict[str, Any]) -> None: + members = data.get("members") or [] + table = Table(title=f"Members of {data.get('alias')} (project_id={data.get('project_id')})") + table.add_column("ID", justify="right", style="dim") + table.add_column("Email") + table.add_column("Role", style="yellow") + table.add_column("Status") + table.add_column("MFA", justify="center") + for m in members: + table.add_row( + str(m.get("id", "")), + m.get("email", ""), + m.get("role", ""), + m.get("status", ""), + "yes" if m.get("mfa_enabled") else "no", + ) + console.print(table) + pending = data.get("pending_invitations") + if pending: + ptable = Table(title="Pending invitations") + ptable.add_column("ID", justify="right", style="dim") + ptable.add_column("Email") + ptable.add_column("Role", style="yellow") + ptable.add_column("Reason") + for p in pending: + ptable.add_row( + str(p.get("id", "")), + p.get("user", {}).get("email", ""), + p.get("role", ""), + p.get("reason", ""), + ) + console.print(ptable) + + +def _format_invitation_list(console: Console, data: dict[str, Any]) -> None: + invitations = data.get("invitations") or [] + if not invitations: + console.print(f"No pending invitations for [cyan]{data.get('alias')}[/cyan].") + return + table = Table( + title=f"Pending invitations for {data.get('alias')} (project_id={data.get('project_id')})" + ) + table.add_column("ID", justify="right", style="dim") + table.add_column("Email") + table.add_column("Role", style="yellow") + table.add_column("Reason") + for inv in invitations: + table.add_row( + str(inv.get("id", "")), + inv.get("user", {}).get("email", ""), + inv.get("role", ""), + inv.get("reason", ""), + ) + console.print(table) + + +@project_app.command("invite") +def project_invite( + ctx: typer.Context, + project: str | None = typer.Option( + None, "--project", "-p", help="Project alias to invite the user to (single-shot mode)" + ), + email: str | None = typer.Option( + None, "--email", "-e", help="Email address of the user to invite" + ), + role: str | None = typer.Option( + None, + "--role", + "-r", + click_type=click.Choice(list(PROJECT_ROLES)), + help="Role to grant: " + " | ".join(PROJECT_ROLES), + ), + reason: str | None = typer.Option( + None, "--reason", help="Optional human-readable reason attached to the invitation" + ), + from_csv: Path | None = typer.Option( + None, + "--from-csv", + help="CSV file with columns email, project (alias or numeric ID), role[, reason]", + ), + default_role: str | None = typer.Option( + None, + "--default-role", + click_type=click.Choice(list(PROJECT_ROLES)), + help="Role to apply when a CSV row has no role column", + ), + workers: int = typer.Option( + DEFAULT_INVITE_WORKERS, + "--workers", + min=1, + max=32, + help="Parallel workers for --from-csv (default 8)", + ), + dry_run: bool = typer.Option(False, "--dry-run", help="Preview without sending invitations"), +) -> None: + """Invite a user (or many users via CSV) to one or more projects. + + \b + Single-shot: + kbagent project invite --project prod --email a@b.com --role admin + + \b + Bulk (one row per email; CSV header required): + kbagent project invite --from-csv participants.csv --default-role guest + """ + formatter = get_formatter(ctx) + + if from_csv and (project or email): + formatter.error( + message="--from-csv is mutually exclusive with --project / --email", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) + if not from_csv and not (project and email and role): + formatter.error( + message="Provide --project, --email, and --role for single-shot invite " + "(or use --from-csv for bulk).", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) + + if should_hint(ctx): + if from_csv: + formatter.error( + message=( + "--hint is not available for `project invite --from-csv`. " + "Use --hint client/service on a single-shot invite " + "(--project + --email + --role) instead, or open the " + "MemberService source for the bulk pattern." + ), + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) + emit_hint( + ctx, + "project.invite", + project=project, + project_id="", + email=email, + role=role, + reason=reason or "", + ) + return + + manage_token = resolve_manage_token() + service = get_service(ctx, "member_service") + + try: + if from_csv: + result = service.invite_bulk( + manage_token=manage_token, + csv_path=from_csv, + default_role=default_role, + workers=workers, + dry_run=dry_run, + ) + payload = result.model_dump() + formatter.output(payload, _format_bulk_invite_result) + return + + result = service.invite( + manage_token=manage_token, + alias=project, + email=email, + role=role, + reason=reason, + dry_run=dry_run, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except ValueError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.VALIDATION_ERROR) + raise typer.Exit(code=2) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + ) + raise typer.Exit(code=exit_code) from None + + formatter.output(result, _format_invite_result) + + +@project_app.command("member-list") +def project_member_list( + ctx: typer.Context, + project: str = typer.Option(..., "--project", "-p", help="Project alias to list members for"), + include_pending: bool = typer.Option( + False, "--include-pending", help="Also list pending (unaccepted) invitations" + ), +) -> None: + """List active members of a project (and optionally pending invitations).""" + formatter = get_formatter(ctx) + + if should_hint(ctx): + emit_hint( + ctx, + "project.member-list", + project=project, + project_id="", + include_pending=str(include_pending), + ) + return + + manage_token = resolve_manage_token() + service = get_service(ctx, "member_service") + try: + result = service.list_members( + manage_token=manage_token, + alias=project, + include_pending=include_pending, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=exit_code) from None + + formatter.output(result, _format_member_list) + + +@project_app.command("invitation-list") +def project_invitation_list( + ctx: typer.Context, + project: str = typer.Option( + ..., "--project", "-p", help="Project alias to list pending invitations for" + ), +) -> None: + """List pending project invitations.""" + formatter = get_formatter(ctx) + + if should_hint(ctx): + emit_hint( + ctx, + "project.invitation-list", + project=project, + project_id="", + ) + return + + manage_token = resolve_manage_token() + service = get_service(ctx, "member_service") + try: + result = service.list_invitations(manage_token=manage_token, alias=project) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=exit_code) from None + + formatter.output(result, _format_invitation_list) + + +@project_app.command("invitation-cancel") +def project_invitation_cancel( + ctx: typer.Context, + project: str = typer.Option(..., "--project", "-p", help="Project alias"), + email: str = typer.Option( + ..., + "--email", + "-e", + help="Invitee's email address (used to look up the invitation if --invitation-id is omitted)", + ), + invitation_id: int | None = typer.Option( + None, + "--invitation-id", + help="Numeric invitation ID; bypass the email lookup", + ), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Cancel a pending invitation.""" + formatter = get_formatter(ctx) + + if should_hint(ctx): + emit_hint( + ctx, + "project.invitation-cancel", + project=project, + project_id="", + email=email, + invitation_id=str(invitation_id) if invitation_id is not None else "None", + ) + return + + if ( + not formatter.json_mode + and not yes + and not typer.confirm(f"Cancel pending invitation for {email} on {project}?") + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + + manage_token = resolve_manage_token() + service = get_service(ctx, "member_service") + try: + result = service.cancel_invitation( + manage_token=manage_token, + alias=project, + email=email, + invitation_id=invitation_id, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=exit_code) from None + + formatter.output( + result, + lambda c, d: c.print( + f"[bold green]Cancelled[/bold green] invitation_id={d.get('invitation_id')} " + f"for {d.get('email')} on [cyan]{d.get('alias')}[/cyan]." + ), + ) + + +@project_app.command("member-remove") +def project_member_remove( + ctx: typer.Context, + project: str = typer.Option(..., "--project", "-p", help="Project alias"), + email: str = typer.Option(..., "--email", "-e", help="Email of the member to remove"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Remove an active member from a project (destructive).""" + formatter = get_formatter(ctx) + + if should_hint(ctx): + emit_hint( + ctx, + "project.member-remove", + project=project, + project_id="", + user_id="", + email=email, + ) + return + + if ( + not formatter.json_mode + and not yes + and not typer.confirm(f"Remove member {email} from project {project}? This is destructive.") + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + + manage_token = resolve_manage_token() + service = get_service(ctx, "member_service") + try: + result = service.remove_member( + manage_token=manage_token, + alias=project, + email=email, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=exit_code) from None + + formatter.output( + result, + lambda c, d: c.print( + f"[bold red]Removed[/bold red] {d.get('email')} (user_id={d.get('user_id')}) " + f"from [cyan]{d.get('alias')}[/cyan]." + ), + ) + + +@project_app.command("member-set-role") +def project_member_set_role( + ctx: typer.Context, + project: str = typer.Option(..., "--project", "-p", help="Project alias"), + email: str = typer.Option(..., "--email", "-e", help="Email of the member to update"), + role: str = typer.Option( + ..., + "--role", + "-r", + click_type=click.Choice(list(PROJECT_ROLES)), + help="New role: " + " | ".join(PROJECT_ROLES), + ), +) -> None: + """Change an existing member's role (PATCH).""" + formatter = get_formatter(ctx) + + if should_hint(ctx): + emit_hint( + ctx, + "project.member-set-role", + project=project, + project_id="", + user_id="", + email=email, + role=role, + ) + return + + manage_token = resolve_manage_token() + service = get_service(ctx, "member_service") + try: + result = service.set_member_role( + manage_token=manage_token, + alias=project, + email=email, + role=role, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except ValueError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.VALIDATION_ERROR) + raise typer.Exit(code=2) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=exit_code) from None + + formatter.output( + result, + lambda c, d: c.print( + f"[bold green]Updated[/bold green] {d.get('email')} role on " + f"[cyan]{d.get('alias')}[/cyan] -> [yellow]{d.get('role')}[/yellow]." + ), + ) diff --git a/src/keboola_agent_cli/constants.py b/src/keboola_agent_cli/constants.py index d2a170d8..799a9bb8 100644 --- a/src/keboola_agent_cli/constants.py +++ b/src/keboola_agent_cli/constants.py @@ -35,6 +35,18 @@ # --- Token Description --- DEFAULT_TOKEN_DESCRIPTION: str = "kbagent-cli" +# --- Project Member Roles --- +# Allowed values for project membership / invitation `role` field. Lifted from +# the Manage API's own validation error: `Role "X" is not valid. Allowed roles +# are: admin, guest, readOnly, share`. Verified empirically 2026-05-01 against +# connection.us-east4.gcp.keboola.com. If the API ever extends the list, the +# fix is to extend this tuple -- the engine already returns the new options in +# its validation error message. +PROJECT_ROLES: tuple[str, ...] = ("admin", "guest", "readOnly", "share") + +# --- Bulk Invite Defaults --- +DEFAULT_INVITE_WORKERS: int = 8 + # --- Job Limits --- DEFAULT_JOB_LIMIT: int = 50 DEFAULT_JOBS_PER_CONFIG: int = 5 diff --git a/src/keboola_agent_cli/hints/definitions/__init__.py b/src/keboola_agent_cli/hints/definitions/__init__.py index eb9cac56..a5ac1558 100644 --- a/src/keboola_agent_cli/hints/definitions/__init__.py +++ b/src/keboola_agent_cli/hints/definitions/__init__.py @@ -10,6 +10,7 @@ job, # noqa: F401 kai, # noqa: F401 lineage, # noqa: F401 + member, # noqa: F401 org, # noqa: F401 project, # noqa: F401 schedule, # noqa: F401 diff --git a/src/keboola_agent_cli/hints/definitions/member.py b/src/keboola_agent_cli/hints/definitions/member.py new file mode 100644 index 00000000..3dd6e30d --- /dev/null +++ b/src/keboola_agent_cli/hints/definitions/member.py @@ -0,0 +1,218 @@ +"""Hint definitions for project member & invitation commands (since v0.26.1).""" + +from .. import HintRegistry +from ..models import ClientCall, CommandHint, HintStep, ServiceCall + +# ── project invite ──────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="project.invite", + description="Invite a user (by email) to a project with a given role", + steps=[ + HintStep( + comment="POST /manage/projects/{id}/invitations", + client=ClientCall( + method="create_project_invitation", + args={ + "project_id": "{project_id}", + "email": "{email}", + "role": "{role}", + "reason": "{reason}", + }, + client_type="manage", + result_var="invitation", + result_hint="dict", + ), + service=ServiceCall( + service_class="MemberService", + service_module="member_service", + method="invite", + args={ + "alias": "{project}", + "email": "{email}", + "role": "{role}", + "reason": "{reason}", + }, + ), + ), + ], + notes=[ + "Uses Manage API + KBC_MANAGE_API_TOKEN (not the Storage token).", + "Allowed roles: admin, guest, readOnly, share.", + "Re-inviting an existing invitee or member returns HTTP 400; the service " + "treats it as a no-op with a 'note' field.", + ], + ) +) + +# ── project member-list ─────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="project.member-list", + description="List active members (and optionally pending invitations)", + steps=[ + HintStep( + comment="GET /manage/projects/{id}/users", + client=ClientCall( + method="list_project_members", + args={"project_id": "{project_id}"}, + client_type="manage", + result_var="members", + result_hint="list[dict]", + ), + service=ServiceCall( + service_class="MemberService", + service_module="member_service", + method="list_members", + args={ + "alias": "{project}", + "include_pending": "{include_pending}", + }, + ), + ), + ], + ) +) + +# ── project invitation-list ────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="project.invitation-list", + description="List pending project invitations", + steps=[ + HintStep( + comment="GET /manage/projects/{id}/invitations", + client=ClientCall( + method="list_project_invitations", + args={"project_id": "{project_id}"}, + client_type="manage", + result_var="invitations", + result_hint="list[dict]", + ), + service=ServiceCall( + service_class="MemberService", + service_module="member_service", + method="list_invitations", + args={"alias": "{project}"}, + ), + ), + ], + ) +) + +# ── project invitation-cancel ──────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="project.invitation-cancel", + description="Cancel a pending invitation", + steps=[ + HintStep( + comment="DELETE /manage/projects/{id}/invitations/{invitationId}", + client=ClientCall( + method="cancel_project_invitation", + args={ + "project_id": "{project_id}", + "invitation_id": "{invitation_id}", + }, + client_type="manage", + result_var="_", + result_hint="None", + ), + service=ServiceCall( + service_class="MemberService", + service_module="member_service", + method="cancel_invitation", + args={ + "alias": "{project}", + "email": "{email}", + "invitation_id": "{invitation_id}", + }, + ), + ), + ], + notes=[ + "If --invitation-id is omitted, the service resolves it by listing " + "pending invitations and matching --email (case-insensitive).", + ], + ) +) + +# ── project member-remove ──────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="project.member-remove", + description="Remove an active member from a project", + steps=[ + HintStep( + comment="DELETE /manage/projects/{id}/users/{userId}", + client=ClientCall( + method="remove_project_member", + args={ + "project_id": "{project_id}", + "user_id": "{user_id}", + }, + client_type="manage", + result_var="_", + result_hint="None", + ), + service=ServiceCall( + service_class="MemberService", + service_module="member_service", + method="remove_member", + args={ + "alias": "{project}", + "email": "{email}", + }, + ), + ), + ], + notes=[ + "Destructive: revokes project access. Re-add via `kbagent project invite`.", + "The service resolves --email to the numeric user_id automatically.", + ], + ) +) + +# ── project member-set-role ────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="project.member-set-role", + description="Change an existing member's role", + steps=[ + HintStep( + comment="PATCH /manage/projects/{id}/users/{userId}", + client=ClientCall( + method="update_project_member_role", + args={ + "project_id": "{project_id}", + "user_id": "{user_id}", + "role": "{role}", + }, + client_type="manage", + result_var="updated", + result_hint="dict", + ), + service=ServiceCall( + service_class="MemberService", + service_module="member_service", + method="set_member_role", + args={ + "alias": "{project}", + "email": "{email}", + "role": "{role}", + }, + ), + ), + ], + notes=[ + "Uses HTTP PATCH (not PUT — PUT returns 404 even on real members).", + "Allowed roles: admin, guest, readOnly, share.", + ], + ) +) diff --git a/src/keboola_agent_cli/manage_client.py b/src/keboola_agent_cli/manage_client.py index a2b62574..fb391a97 100644 --- a/src/keboola_agent_cli/manage_client.py +++ b/src/keboola_agent_cli/manage_client.py @@ -133,3 +133,80 @@ def create_project_token( payload["expiresIn"] = expires_in response = self._do_request("POST", f"/manage/projects/{project_id}/tokens", json=payload) return response.json() + + # ------------------------------------------------------------------ + # Project members & invitations (verified 2026-05-01 against the + # us-east4.gcp.keboola.com Manage API; see plan-of-record §"Verifications"). + # ------------------------------------------------------------------ + + def create_project_invitation( + self, + project_id: int, + email: str, + role: str, + reason: str | None = None, + ) -> dict[str, Any]: + """Send an invitation email to add ``email`` as a project member. + + Returns the invitation object on success (HTTP 201). On HTTP 400 with + the error message ``"This user has already been invited..."`` or + ``"...is already a member..."`` the caller should treat the call as a + no-op rather than an error -- the higher layer encodes that policy. + + Args: + project_id: Numeric project ID. + email: Email of the user to invite. + role: One of ``admin``, ``guest``, ``readOnly``, ``share``. + reason: Optional human-readable note attached to the invitation. + + Returns: + Invitation dict: ``{id, created, expires, reason, role, user, creator}``. + """ + payload: dict[str, Any] = {"email": email, "role": role} + if reason: + payload["reason"] = reason + response = self._do_request( + "POST", f"/manage/projects/{project_id}/invitations", json=payload + ) + return response.json() + + def list_project_invitations(self, project_id: int) -> list[dict[str, Any]]: + """List pending (not-yet-accepted) invitations for a project. + + Returns a plain list. Each item has shape + ``{id, created, expires, reason, role, user: {id, name, email}, creator: {...}}``. + """ + response = self._do_request("GET", f"/manage/projects/{project_id}/invitations") + return response.json() + + def cancel_project_invitation(self, project_id: int, invitation_id: int) -> None: + """Cancel a pending invitation by ID. Returns 204 No Content on success.""" + self._do_request("DELETE", f"/manage/projects/{project_id}/invitations/{invitation_id}") + + def list_project_members(self, project_id: int) -> list[dict[str, Any]]: + """List active project members. + + Returns a plain list. Each user dict carries the project role at the + top level (``role`` field) -- not nested under a ``user`` key. + """ + response = self._do_request("GET", f"/manage/projects/{project_id}/users") + return response.json() + + def remove_project_member(self, project_id: int, user_id: int) -> None: + """Remove a member from a project. Returns 204 No Content on success.""" + self._do_request("DELETE", f"/manage/projects/{project_id}/users/{user_id}") + + def update_project_member_role( + self, project_id: int, user_id: int, role: str + ) -> dict[str, Any]: + """Change an existing member's role. + + The Manage API uses **PATCH** here -- ``PUT`` returns 404 even on real + members. Returns the updated user dict on success (HTTP 200). + """ + response = self._do_request( + "PATCH", + f"/manage/projects/{project_id}/users/{user_id}", + json={"role": role}, + ) + return response.json() diff --git a/src/keboola_agent_cli/models.py b/src/keboola_agent_cli/models.py index 84c746cd..c5693674 100644 --- a/src/keboola_agent_cli/models.py +++ b/src/keboola_agent_cli/models.py @@ -168,3 +168,73 @@ class SuccessResponse(BaseModel): status: str = Field(default="ok", description="Always 'ok' for success responses") data: Any = Field(default=None, description="Response payload") + + +class ProjectMember(BaseModel): + """Active project member as returned by GET /manage/projects/{id}/users. + + The Manage API returns audit-relevant fields beyond what kbagent renames + explicitly: ``created``, ``expires``, ``invitor``, ``approver``, ``features``, + ``canAccessLogs``, ``isSuperAdmin``, ``canApproveMergeRequests``. We allow + extras through unmodified so admins inspecting `--json` output get the full + audit trail (who invited whom, when, status flags), not a narrow whitelist. + """ + + id: int = Field(description="Numeric Keboola user ID") + email: str = Field(description="Member email address") + name: str = Field(default="", description="Display name (may be empty for stub accounts)") + role: str = Field(description="Project role: admin | guest | readOnly | share") + status: str = Field(default="active", description="Membership status") + mfa_enabled: bool = Field(default=False, alias="mfaEnabled") + + model_config = {"populate_by_name": True, "extra": "allow"} + + +class InvitationUser(BaseModel): + """Invited user inside an Invitation object.""" + + id: int | None = Field(default=None) + email: str + name: str = Field(default="") + + +class ProjectInvitation(BaseModel): + """Pending project invitation as returned by GET /manage/projects/{id}/invitations. + + Extras (``created``, ``expires``, ``creator``) pass through unmodified so + callers can audit when invitations were created and by whom. + """ + + id: int = Field(description="Invitation ID -- pass to DELETE to cancel") + role: str = Field(description="Role offered to the invitee") + reason: str = Field(default="") + user: InvitationUser = Field(description="The invited user (email + resolved id)") + + model_config = {"populate_by_name": True, "extra": "allow"} + + +class MemberInviteRow(BaseModel): + """Per-row outcome of a bulk-invite operation. + + `status` = 'ok' (created), 'noop' (already invited or already a member), + 'failed' (any other error). `note` carries the human-readable explanation. + """ + + email: str + project: str = Field(description="Alias or numeric ID as it appeared in the source CSV row") + project_id: int | None = Field(default=None, description="Resolved numeric project ID") + role: str = Field(default="") + status: str = Field(description="ok | noop | failed") + note: str = Field(default="") + invitation_id: int | None = Field(default=None) + + +class BulkInviteResult(BaseModel): + """Aggregate result of `kbagent project invite --from-csv`.""" + + total: int + succeeded: int + noop: int + failed: int + rows: list[MemberInviteRow] = Field(default_factory=list) + dry_run: bool = Field(default=False) diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 97ec7033..e3f34b76 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -24,6 +24,12 @@ "project.description-set": "write", "project.use": "write", "project.current": "read", + "project.invite": "admin", + "project.member-list": "read", + "project.invitation-list": "read", + "project.invitation-cancel": "admin", + "project.member-remove": "destructive", + "project.member-set-role": "admin", # Config browsing & management "config.list": "read", "config.detail": "read", diff --git a/src/keboola_agent_cli/services/member_service.py b/src/keboola_agent_cli/services/member_service.py new file mode 100644 index 00000000..fb5f4a95 --- /dev/null +++ b/src/keboola_agent_cli/services/member_service.py @@ -0,0 +1,593 @@ +"""Project membership and invitation lifecycle service. + +Wraps the Manage API endpoints under ``/manage/projects/{id}/{users,invitations}`` +behind a layer that: + +- resolves a project alias to its numeric ID via ``ConfigStore``; +- looks up members + invitations by email (the public-facing key) so callers + never need to deal with raw user/invitation IDs; +- treats the Manage API's "already invited / already a member" 400 response as + a no-op rather than an error (mirrors the heuristic from the orchestrator + scripts, but typed to status_code + message substring rather than guessed + HTTP code); +- parallelises bulk invitation via :class:`ThreadPoolExecutor`, accumulating + per-row results so one bad row never aborts the rest. + +Endpoints + payload shapes were verified empirically on 2026-05-01 against +``connection.us-east4.gcp.keboola.com``; see the plan-of-record for the full +verification log. +""" + +from __future__ import annotations + +import csv +import logging +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Any + +from ..config_store import ConfigStore +from ..constants import DEFAULT_INVITE_WORKERS, PROJECT_ROLES +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..manage_client import ManageClient +from ..models import ( + BulkInviteResult, + MemberInviteRow, + ProjectInvitation, + ProjectMember, +) + +logger = logging.getLogger(__name__) + +ManageClientFactory = Callable[[str, str], ManageClient] + + +def default_manage_client_factory(stack_url: str, manage_token: str) -> ManageClient: + """Construct a :class:`ManageClient` bound to ``stack_url``.""" + return ManageClient(stack_url=stack_url, manage_token=manage_token) + + +# The Manage API returns HTTP 400 (not 422) with one of these substrings when +# a duplicate invitation/member is created. Treated as success-with-note in +# the service layer so bulk imports don't fail on idempotent re-runs. +_ALREADY_INVITED_MARKER = "already been invited" +_ALREADY_MEMBER_MARKER = "already a member" + + +class MemberService: + """Business logic for project members and invitations.""" + + def __init__( + self, + config_store: ConfigStore, + manage_client_factory: ManageClientFactory | None = None, + ) -> None: + self._config_store = config_store + self._manage_client_factory = manage_client_factory or default_manage_client_factory + + # ------------------------------------------------------------------ + # Public API: single-shot operations + # ------------------------------------------------------------------ + + def invite( + self, + *, + manage_token: str, + alias: str, + email: str, + role: str, + reason: str | None = None, + dry_run: bool = False, + ) -> dict[str, Any]: + """Invite ``email`` to the project registered under ``alias``.""" + self._validate_role(role) + stack_url, project_id = self._resolve_alias(alias) + + if dry_run: + return { + "status": "dry_run", + "alias": alias, + "project_id": project_id, + "email": email, + "role": role, + "reason": reason or "", + } + + manage_client = self._manage_client_factory(stack_url, manage_token) + try: + return self._invite_one(manage_client, alias, project_id, email, role, reason) + finally: + manage_client.close() + + def invite_bulk( + self, + *, + manage_token: str, + csv_path: Path, + default_role: str | None = None, + workers: int = DEFAULT_INVITE_WORKERS, + dry_run: bool = False, + ) -> BulkInviteResult: + """Invite every row of ``csv_path`` in parallel. + + CSV must have a header row. Recognised columns (case-insensitive): + ``email`` (required), ``project`` or ``project_id`` (one required), + ``role`` (optional if ``default_role`` is given), ``reason`` (optional). + Extra columns are ignored. ``project`` values that are all-digits are + resolved as numeric project IDs without an alias lookup. + """ + if default_role is not None: + self._validate_role(default_role) + rows = self._parse_invite_csv(csv_path, default_role) + if not rows: + return BulkInviteResult(total=0, succeeded=0, noop=0, failed=0, dry_run=dry_run) + + if dry_run: + return self._bulk_dry_run(rows) + + # Resolve every row's (stack_url, project_id) up front. A row that + # fails resolution (unknown alias, unregistered project_id) becomes a + # per-row "failed" entry; the rest of the batch still runs. Mirrors + # the partial-success contract enforced by `OrgService.refresh_tokens`. + resolved: list[tuple[dict[str, Any], str, int]] = [] + upfront_failures: list[MemberInviteRow] = [] + for row in rows: + try: + stack_url, project_id = self._stack_for_row(row) + resolved.append((row, stack_url, project_id)) + except ConfigError as exc: + upfront_failures.append( + MemberInviteRow( + email=row["email"], + project=str(row["project"]), + role=row["role"], + status="failed", + note=str(getattr(exc, "message", exc)), + ) + ) + + if not resolved: + return BulkInviteResult( + total=len(upfront_failures), + succeeded=0, + noop=0, + failed=len(upfront_failures), + rows=upfront_failures, + dry_run=False, + ) + + # All resolved rows must share a single stack URL; sending invitations + # for project A on stack X via a client bound to stack Y is a security + # bug, not a "partial-success" path. + resolved_stacks = {t[1] for t in resolved} + if len(resolved_stacks) != 1: + raise ConfigError( + f"CSV references multiple stack URLs ({sorted(resolved_stacks)}); " + "split the file by stack and run --from-csv per stack." + ) + stack_url = resolved_stacks.pop() + + manage_client = self._manage_client_factory(stack_url, manage_token) + results: list[MemberInviteRow] = list(upfront_failures) + try: + worker_count = max(1, min(workers, len(resolved))) + with ThreadPoolExecutor(max_workers=worker_count) as pool: + futures = [ + pool.submit(self._invoke_resolved_row, manage_client, row, project_id) + for row, _, project_id in resolved + ] + for fut in as_completed(futures): + results.append(fut.result()) + finally: + manage_client.close() + + return BulkInviteResult( + total=len(results), + succeeded=sum(1 for r in results if r.status == "ok"), + noop=sum(1 for r in results if r.status == "noop"), + failed=sum(1 for r in results if r.status == "failed"), + rows=results, + dry_run=False, + ) + + def list_members( + self, + *, + manage_token: str, + alias: str, + include_pending: bool = False, + ) -> dict[str, Any]: + """Return active members (and, optionally, pending invitations).""" + stack_url, project_id = self._resolve_alias(alias) + manage_client = self._manage_client_factory(stack_url, manage_token) + try: + members_raw = manage_client.list_project_members(project_id) + members = [ProjectMember.model_validate(m) for m in members_raw] + payload: dict[str, Any] = { + "alias": alias, + "project_id": project_id, + "members": [m.model_dump(by_alias=False) for m in members], + } + if include_pending: + inv_raw = manage_client.list_project_invitations(project_id) + payload["pending_invitations"] = [ + ProjectInvitation.model_validate(i).model_dump(by_alias=False) for i in inv_raw + ] + return payload + finally: + manage_client.close() + + def list_invitations( + self, + *, + manage_token: str, + alias: str, + ) -> dict[str, Any]: + """Return pending invitations for ``alias``.""" + stack_url, project_id = self._resolve_alias(alias) + manage_client = self._manage_client_factory(stack_url, manage_token) + try: + raw = manage_client.list_project_invitations(project_id) + return { + "alias": alias, + "project_id": project_id, + "invitations": [ + ProjectInvitation.model_validate(i).model_dump(by_alias=False) for i in raw + ], + } + finally: + manage_client.close() + + def cancel_invitation( + self, + *, + manage_token: str, + alias: str, + email: str, + invitation_id: int | None = None, + ) -> dict[str, Any]: + """Cancel a pending invitation. Resolves by email if no ID is supplied.""" + stack_url, project_id = self._resolve_alias(alias) + manage_client = self._manage_client_factory(stack_url, manage_token) + try: + if invitation_id is None: + invitation_id = self._resolve_invitation_id(manage_client, project_id, email) + manage_client.cancel_project_invitation(project_id, invitation_id) + return { + "status": "cancelled", + "alias": alias, + "project_id": project_id, + "email": email, + "invitation_id": invitation_id, + } + finally: + manage_client.close() + + def remove_member( + self, + *, + manage_token: str, + alias: str, + email: str, + ) -> dict[str, Any]: + """Remove an active member from a project.""" + stack_url, project_id = self._resolve_alias(alias) + manage_client = self._manage_client_factory(stack_url, manage_token) + try: + user_id = self._resolve_member_id(manage_client, project_id, email) + manage_client.remove_project_member(project_id, user_id) + return { + "status": "removed", + "alias": alias, + "project_id": project_id, + "email": email, + "user_id": user_id, + } + finally: + manage_client.close() + + def set_member_role( + self, + *, + manage_token: str, + alias: str, + email: str, + role: str, + ) -> dict[str, Any]: + """Change an existing member's role via PATCH.""" + self._validate_role(role) + stack_url, project_id = self._resolve_alias(alias) + manage_client = self._manage_client_factory(stack_url, manage_token) + try: + user_id = self._resolve_member_id(manage_client, project_id, email) + updated = manage_client.update_project_member_role(project_id, user_id, role) + return { + "status": "updated", + "alias": alias, + "project_id": project_id, + "email": email, + "user_id": user_id, + "role": updated.get("role", role), + } + finally: + manage_client.close() + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + @staticmethod + def _validate_role(role: str) -> None: + """Defence-in-depth: command layer enforces the same whitelist via Choice.""" + if role not in PROJECT_ROLES: + raise ValueError( + f"Invalid role {role!r}. Allowed roles are: {', '.join(PROJECT_ROLES)}." + ) + + def _resolve_alias(self, alias: str) -> tuple[str, int]: + """Look up ``alias`` in the config store and return ``(stack_url, project_id)``.""" + project = self._config_store.get_project(alias) + if project is None: + raise ConfigError( + f"Project alias '{alias}' is not registered. Run `kbagent project list`." + ) + if project.project_id is None: + raise ConfigError( + f"Project alias '{alias}' has no numeric project_id; " + "re-add it via `kbagent project add` to populate it." + ) + return project.stack_url, project.project_id + + def _stack_for_row(self, row: dict[str, Any]) -> tuple[str, int]: + """Resolve a CSV row's project field to ``(stack_url, project_id)``.""" + project_field = str(row["project"]).strip() + if project_field.isdigit(): + # Numeric project_id rows still need a stack_url; we infer from the + # currently-registered projects sharing that ID, falling back to + # any default. + project_id = int(project_field) + for cfg in self._config_store.load().projects.values(): + if cfg.project_id == project_id: + return cfg.stack_url, project_id + raise ConfigError( + f"CSV row references project_id={project_id}, which is not registered " + "in this kbagent config; add it via `kbagent project add` so we know " + "which stack URL to use." + ) + return self._resolve_alias(project_field) + + @staticmethod + def _resolve_member_id(manage_client: ManageClient, project_id: int, email: str) -> int: + """Find an active member's numeric ID by email (case-insensitive match).""" + members = manage_client.list_project_members(project_id) + normalised = email.casefold() + for member in members: + if str(member.get("email", "")).casefold() == normalised: + return int(member["id"]) + raise KeboolaApiError( + message=f"No active member with email {email!r} on project {project_id}.", + status_code=404, + error_code=ErrorCode.NOT_FOUND, + retryable=False, + ) + + @staticmethod + def _resolve_invitation_id(manage_client: ManageClient, project_id: int, email: str) -> int: + """Find a pending invitation by email.""" + invitations = manage_client.list_project_invitations(project_id) + normalised = email.casefold() + for inv in invitations: + if str(inv.get("user", {}).get("email", "")).casefold() == normalised: + return int(inv["id"]) + raise KeboolaApiError( + message=f"No pending invitation for email {email!r} on project {project_id}.", + status_code=404, + error_code=ErrorCode.NOT_FOUND, + retryable=False, + ) + + def _invite_one( + self, + manage_client: ManageClient, + project_label: str, + project_id: int, + email: str, + role: str, + reason: str | None, + ) -> dict[str, Any]: + """Single-row invitation logic shared by ``invite`` and ``invite_bulk``. + + ``project_label`` is the human-readable project identifier surfaced in + the result dict's ``alias`` field. Single-shot mode passes the + registered alias; bulk mode passes the raw CSV ``project`` cell, which + may be either an alias or a numeric project ID string -- whatever the + user wrote. + """ + try: + invitation = manage_client.create_project_invitation( + project_id=project_id, + email=email, + role=role, + reason=reason, + ) + return { + "status": "ok", + "alias": project_label, + "project_id": project_id, + "email": email, + "role": role, + "invitation_id": invitation.get("id"), + } + except KeboolaApiError as exc: + note = self._noop_note_for(exc) + if note is None: + raise + return { + "status": "noop", + "alias": project_label, + "project_id": project_id, + "email": email, + "role": role, + "note": note, + } + + def _invoke_resolved_row( + self, + manage_client: ManageClient, + row: dict[str, Any], + project_id: int, + ) -> MemberInviteRow: + """Execute one CSV row inside the bulk-invite executor. + + Called only on rows whose (stack_url, project_id) was already resolved + by ``invite_bulk`` -- so the only failure path here is the API call + itself (e.g. invalid email, network error, role rejection). + """ + email = row["email"] + role = row["role"] + reason = row.get("reason") + project_field = str(row["project"]).strip() + try: + outcome = self._invite_one( + manage_client, project_field, project_id, email, role, reason + ) + return MemberInviteRow( + email=email, + project=project_field, + project_id=project_id, + role=role, + status=outcome["status"], + note=outcome.get("note", ""), + invitation_id=outcome.get("invitation_id"), + ) + except KeboolaApiError as exc: + return MemberInviteRow( + email=email, + project=project_field, + project_id=project_id, + role=role, + status="failed", + note=str(getattr(exc, "message", exc)), + ) + + @staticmethod + def _noop_note_for(exc: KeboolaApiError) -> str | None: + """Return a noop reason if ``exc`` is the "already invited / member" 400.""" + if exc.status_code != 400: + return None + message = exc.message or "" + if _ALREADY_INVITED_MARKER in message: + return "already_invited" + if _ALREADY_MEMBER_MARKER in message: + return "already_member" + return None + + def _parse_invite_csv(self, csv_path: Path, default_role: str | None) -> list[dict[str, Any]]: + """Parse + validate a bulk-invite CSV. Returns a list of normalised dicts.""" + if not csv_path.exists(): + raise ConfigError(f"CSV file not found: {csv_path}") + + # `utf-8-sig` strips a leading BOM if Excel produced the CSV (otherwise + # the first header reads as `email`, which fails the email-column + # check with a misleading message). + with csv_path.open("r", newline="", encoding="utf-8-sig") as fh: + reader = csv.DictReader(fh) + if reader.fieldnames is None: + raise ConfigError(f"CSV file {csv_path} has no header row.") + headers = {h.strip().lower(): h for h in reader.fieldnames if h} + if "email" not in headers: + raise ConfigError( + f"CSV file {csv_path} is missing an 'email' column. " + f"Found columns: {list(reader.fieldnames)}." + ) + project_key = ( + "project" + if "project" in headers + else ("project_id" if "project_id" in headers else None) + ) + if project_key is None: + raise ConfigError( + f"CSV file {csv_path} must have a 'project' or 'project_id' column. " + f"Found columns: {list(reader.fieldnames)}." + ) + has_role = "role" in headers + if not has_role and default_role is None: + raise ConfigError( + f"CSV file {csv_path} has no 'role' column and --default-role was not given." + ) + + rows: list[dict[str, Any]] = [] + for line_no, raw in enumerate(reader, start=2): # header is line 1 + email = (raw.get(headers["email"]) or "").strip() + project = (raw.get(headers[project_key]) or "").strip() + role = (raw.get(headers["role"]) if has_role else None) or default_role or "" + role = role.strip() + reason = (raw.get(headers["reason"]) or "").strip() if "reason" in headers else "" + if not email or not project: + raise ConfigError( + f"CSV {csv_path} line {line_no}: 'email' and '{project_key}' are both required." + ) + if not role: + raise ConfigError( + f"CSV {csv_path} line {line_no}: missing role and no --default-role." + ) + self._validate_role(role) + rows.append( + { + "email": email, + "project": project, + "role": role, + "reason": reason or None, + } + ) + return rows + + def _bulk_dry_run(self, rows: list[dict[str, Any]]) -> BulkInviteResult: + """Render a dry-run result without hitting the network. + + Mirrors the live path: per-row resolution failures become per-row + failed entries; multi-stack-URL CSVs raise (matches the real-run + invariant so users don't get a "preview said ok, real run aborted" + surprise). + """ + previewed: list[MemberInviteRow] = [] + resolved_stacks: set[str] = set() + for row in rows: + try: + stack_url, project_id = self._stack_for_row(row) + resolved_stacks.add(stack_url) + except ConfigError as exc: + previewed.append( + MemberInviteRow( + email=row["email"], + project=str(row["project"]), + role=row["role"], + status="failed", + note=str(getattr(exc, "message", exc)), + ) + ) + continue + previewed.append( + MemberInviteRow( + email=row["email"], + project=str(row["project"]), + project_id=project_id, + role=row["role"], + status="ok", + note="dry_run", + ) + ) + if len(resolved_stacks) > 1: + raise ConfigError( + f"CSV references multiple stack URLs ({sorted(resolved_stacks)}); " + "split the file by stack and run --from-csv per stack." + ) + return BulkInviteResult( + total=len(previewed), + succeeded=sum(1 for r in previewed if r.status == "ok"), + noop=0, + failed=sum(1 for r in previewed if r.status == "failed"), + rows=previewed, + dry_run=True, + ) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index cf6cd001..b18fc679 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -6139,6 +6139,29 @@ def test_swap_without_branch_is_rejected(self) -> None: reason=( f"requires {ENV_TOKEN} + {ENV_DATA_APP_GIT_REPO_PRIVATE} + " f"{ENV_DATA_APP_GIT_USER} + {ENV_DATA_APP_GIT_PAT}" +# ────────────────────────────────────────────────────────────────────── +# Project invite E2E (since v0.26.1) +# +# Opt-in via `make test-e2e-invite`. Default-skipped in `make test-e2e` because +# (a) it sends a real invitation email and (b) it depends on a separate manage +# token / project ID that the regular E2E credentials don't carry. +# ────────────────────────────────────────────────────────────────────── + + +ENV_MANAGE_TOKEN = "E2E_MANAGE_TOKEN" +ENV_INVITE_PROJECT_ID = "E2E_INVITE_PROJECT_ID" +ENV_INVITE_EMAIL = "E2E_INVITE_EMAIL" +DEFAULT_INVITE_EMAIL = "ottomansky.max@gmail.com" + +skip_without_invite_credentials = pytest.mark.skipif( + not ( + os.environ.get(ENV_MANAGE_TOKEN) + and os.environ.get(ENV_INVITE_PROJECT_ID) + and os.environ.get(ENV_URL) + ), + reason=( + f"Requires {ENV_MANAGE_TOKEN}, {ENV_INVITE_PROJECT_ID}, and {ENV_URL}. " + "Run via `make test-e2e-invite`." ), ) @@ -6663,3 +6686,114 @@ def test_config_update_auto_normalizes_script_array(self, tmp_path: Path) -> Non assert "Expected" not in rendered or "script" not in rendered, ( f"job envelope still mentions the script type-mismatch failure: {rendered}" ) +@pytest.mark.e2e_invite +@skip_without_invite_credentials +def test_project_invite_e2e(tmp_path: Path) -> None: + """Real invite to the master cuesta project: send -> list -> cancel -> verify gone. + + Uses role=guest (lowest blast radius). The cancel step in the same run + invalidates the invitation link before the inbox sees it, so this is a + "the system can send + clean up" check, not a "join my project" check. + """ + from keboola_agent_cli.config_store import ConfigStore as _Store + from keboola_agent_cli.models import ProjectConfig as _Project + + invite_email = os.environ.get(ENV_INVITE_EMAIL, DEFAULT_INVITE_EMAIL) + project_id = int(os.environ[ENV_INVITE_PROJECT_ID]) + stack_url = ( + os.environ[ENV_URL] + if os.environ[ENV_URL].startswith("https://") + else f"https://{os.environ[ENV_URL]}" + ) + alias = f"e2e-invite-target-{project_id}" + + # Bypass `kbagent project add` (which would verify a Storage API token). + # MemberService only needs (stack_url, project_id) -- the storage token + # field is unused. Write a minimal config.json with a placeholder token. + config_dir = tmp_path / "kbagent-config" + config_dir.mkdir() + store = _Store(config_dir=config_dir) + store.add_project( + alias, + _Project( + stack_url=stack_url, + token="901-e2e-placeholder-not-used-by-member-commands-xxxxxxxxxx", + project_id=project_id, + project_name="E2E invite target", + ), + ) + + env = { + **os.environ, + "KBC_MANAGE_API_TOKEN": os.environ[ENV_MANAGE_TOKEN], + } + + def _run(*args: str) -> dict: + result = runner.invoke( + app, + ["--config-dir", str(config_dir), "--json", *args], + env=env, + ) + assert result.exit_code == 0, ( + f"{' '.join(args)} failed (exit {result.exit_code}):\n{result.output}" + ) + return json.loads(result.output) + + # 1. Defensive cleanup: if a stale invitation exists from a prior aborted + # run, cancel it first so we start from a known state. + initial = _run("project", "invitation-list", "--project", alias)["data"]["invitations"] + for inv in initial: + if inv.get("user", {}).get("email", "").casefold() == invite_email.casefold(): + _run( + "project", + "invitation-cancel", + "--project", + alias, + "--email", + invite_email, + "--yes", + ) + + # 2. Send the invitation. + sent = _run( + "project", + "invite", + "--project", + alias, + "--email", + invite_email, + "--role", + "guest", + "--reason", + "kbagent v0.26.1 e2e", + )["data"] + assert sent["status"] == "ok" + assert sent["invitation_id"] is not None + invitation_id = sent["invitation_id"] + + try: + # 3. Confirm it shows up in invitation-list. + listed = _run("project", "invitation-list", "--project", alias)["data"]["invitations"] + emails = {row["user"]["email"].casefold() for row in listed} + assert invite_email.casefold() in emails, f"{invite_email} did not appear in {emails}" + finally: + # 4. Cancel (always, even if the assertion above fails -- never leave + # a real-email invitation around for a flaky test). + _run( + "project", + "invitation-cancel", + "--project", + alias, + "--email", + invite_email, + "--invitation-id", + str(invitation_id), + "--yes", + ) + + # 5. Verify the invitation is gone. + final = _run("project", "invitation-list", "--project", alias)["data"]["invitations"] + final_emails = {row["user"]["email"].casefold() for row in final} + assert invite_email.casefold() not in final_emails, ( + f"{invite_email} still pending after cancel: {final_emails}" + ) diff --git a/tests/test_manage_client.py b/tests/test_manage_client.py index 953f5366..0e50350f 100644 --- a/tests/test_manage_client.py +++ b/tests/test_manage_client.py @@ -361,3 +361,215 @@ def test_context_manager(self, httpx_mock) -> None: with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: result = client.list_organization_projects(1) assert result == [] + + +# ────────────────────────────────────────────────────────────────────── +# Project members & invitations (since v0.26.1) +# ────────────────────────────────────────────────────────────────────── + + +_INVITATION_RESPONSE = { + "id": 1741, + "created": "2026-05-01T19:04:35+0200", + "expires": None, + "reason": "v0.26.1 verification", + "role": "guest", + "user": {"id": 1325, "email": "ottomansky.max@gmail.com", "name": ""}, + "creator": {"id": 216, "email": "max.ottomansky@keboola.com", "name": "Max"}, +} + +_MEMBER_LIST_RESPONSE = [ + { + "id": 216, + "name": "Max", + "email": "max.ottomansky@keboola.com", + "role": "admin", + "status": "active", + "mfaEnabled": True, + "features": ["power-user"], + "canAccessLogs": False, + }, + { + "id": 4241, + "name": "Marcel", + "email": "mfiser@cuestapartners.com", + "role": "guest", + "status": "active", + "mfaEnabled": True, + "features": [], + "canAccessLogs": False, + }, +] + + +class TestCreateProjectInvitation: + def test_success(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/invitations", + method="POST", + json=_INVITATION_RESPONSE, + status_code=201, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + result = client.create_project_invitation( + project_id=5725, + email="ottomansky.max@gmail.com", + role="guest", + reason="v0.26.1 verification", + ) + assert result["id"] == 1741 + assert result["role"] == "guest" + assert result["user"]["email"] == "ottomansky.max@gmail.com" + + def test_payload_contains_email_role_reason(self, httpx_mock) -> None: + import json as _json + + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/invitations", + method="POST", + json=_INVITATION_RESPONSE, + status_code=201, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + client.create_project_invitation( + project_id=5725, + email="ottomansky.max@gmail.com", + role="guest", + reason="v0.26.1 verification", + ) + body = _json.loads(httpx_mock.get_request().read()) + assert body == { + "email": "ottomansky.max@gmail.com", + "role": "guest", + "reason": "v0.26.1 verification", + } + + def test_omits_reason_when_none(self, httpx_mock) -> None: + import json as _json + + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/invitations", + method="POST", + json=_INVITATION_RESPONSE, + status_code=201, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + client.create_project_invitation(project_id=5725, email="x@y.com", role="admin") + body = _json.loads(httpx_mock.get_request().read()) + assert body == {"email": "x@y.com", "role": "admin"} + + def test_400_already_invited_surfaces_message(self, httpx_mock) -> None: + """The 'already invited' 400 must round-trip the API's error text so + the service layer can match its substring marker.""" + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/invitations", + method="POST", + json={"error": "This user has already been invited to this project."}, + status_code=400, + ) + with ( + ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client, + pytest.raises(KeboolaApiError) as exc_info, + ): + client.create_project_invitation(project_id=5725, email="x@y.com", role="admin") + assert exc_info.value.status_code == 400 + assert "already been invited" in exc_info.value.message + + +class TestListProjectInvitations: + def test_returns_plain_list(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/invitations", + json=[_INVITATION_RESPONSE], + status_code=200, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + result = client.list_project_invitations(5725) + assert isinstance(result, list) + assert result[0]["user"]["email"] == "ottomansky.max@gmail.com" + + +class TestCancelProjectInvitation: + def test_returns_none_on_204(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/invitations/1741", + method="DELETE", + status_code=204, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + assert client.cancel_project_invitation(5725, 1741) is None + + def test_404_after_already_deleted(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/invitations/1741", + method="DELETE", + json={"error": "Invitation not found"}, + status_code=404, + ) + with ( + ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client, + pytest.raises(KeboolaApiError) as exc_info, + ): + client.cancel_project_invitation(5725, 1741) + assert exc_info.value.error_code == "NOT_FOUND" + + +class TestListProjectMembers: + def test_returns_top_level_user_dicts(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/users", + json=_MEMBER_LIST_RESPONSE, + status_code=200, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + result = client.list_project_members(5725) + assert len(result) == 2 + assert result[0]["email"] == "max.ottomansky@keboola.com" + # Role lives at the top level (not nested under a "user" key). + assert result[0]["role"] == "admin" + assert result[1]["role"] == "guest" + + +class TestRemoveProjectMember: + def test_returns_none_on_204(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/users/216", + method="DELETE", + status_code=204, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + assert client.remove_project_member(5725, 216) is None + + def test_400_administrator_not_found(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/users/999", + method="DELETE", + json={"error": "Administrator not found"}, + status_code=400, + ) + with ( + ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client, + pytest.raises(KeboolaApiError) as exc_info, + ): + client.remove_project_member(5725, 999) + assert exc_info.value.status_code == 400 + + +class TestUpdateProjectMemberRole: + def test_uses_PATCH_not_PUT(self, httpx_mock) -> None: + """Regression: PUT returns 404 even on real members; the client must + emit PATCH.""" + import json as _json + + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/users/216", + method="PATCH", + json={"id": 216, "email": "max.ottomansky@keboola.com", "role": "guest"}, + status_code=200, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + result = client.update_project_member_role(5725, 216, "guest") + request = httpx_mock.get_request() + assert request.method == "PATCH" + assert _json.loads(request.read()) == {"role": "guest"} + assert result["role"] == "guest" diff --git a/tests/test_member_cli.py b/tests/test_member_cli.py new file mode 100644 index 00000000..7a2eaea8 --- /dev/null +++ b/tests/test_member_cli.py @@ -0,0 +1,564 @@ +"""CLI tests for `kbagent project invite / member-* / invitation-*` (since v0.26.1).""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import BulkInviteResult, MemberInviteRow, ProjectConfig + +STACK_URL = "https://connection.us-east4.gcp.keboola.com" +PROJECT_ID = 5725 +ALIAS = "cuesta-master" +MANAGE_TOKEN = "manage-12345-abcdefghijklmnopqrstuvwxyz0123456789" + + +runner = CliRunner() + + +def _seed_store(config_dir: Path) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + store.add_project( + ALIAS, + ProjectConfig( + stack_url=STACK_URL, + token="901-fake-storage-token-1234567890", + project_name="[Cuesta training] - Master", + project_id=PROJECT_ID, + ), + ) + return store + + +class TestProjectInviteSingle: + def test_json_happy_path(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + + svc = MagicMock() + svc.invite.return_value = { + "status": "ok", + "alias": ALIAS, + "project_id": PROJECT_ID, + "email": "ottomansky.max@gmail.com", + "role": "guest", + "invitation_id": 1741, + } + + with ( + patch("keboola_agent_cli.cli.MemberService", return_value=svc), + patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}), + ): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "invite", + "--project", + ALIAS, + "--email", + "ottomansky.max@gmail.com", + "--role", + "guest", + ], + ) + + assert result.exit_code == 0, result.output + out = json.loads(result.output) + assert out["status"] == "ok" + assert out["data"]["invitation_id"] == 1741 + svc.invite.assert_called_once() + + def test_missing_required_args_exits_2(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + + with patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "invite", + "--project", + ALIAS, + # --email + --role missing + ], + ) + assert result.exit_code == 2 + + def test_invalid_role_blocked_by_choice(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + + with patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "invite", + "--project", + ALIAS, + "--email", + "x@y.com", + "--role", + "developer", # not on whitelist -> Click rejects with exit 2 + ], + ) + assert result.exit_code == 2 + + def test_dry_run_short_circuits(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.invite.return_value = { + "status": "dry_run", + "alias": ALIAS, + "project_id": PROJECT_ID, + "email": "x@y.com", + "role": "guest", + } + + with ( + patch("keboola_agent_cli.cli.MemberService", return_value=svc), + patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}), + ): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "invite", + "--project", + ALIAS, + "--email", + "x@y.com", + "--role", + "guest", + "--dry-run", + ], + ) + assert result.exit_code == 0 + assert json.loads(result.output)["data"]["status"] == "dry_run" + + def test_invalid_token_maps_to_exit_3(self, tmp_path: Path) -> None: + """`map_error_to_exit_code` exclusively maps INVALID_TOKEN -> 3.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.invite.side_effect = KeboolaApiError( + message="Invalid or expired token", + status_code=401, + error_code=ErrorCode.INVALID_TOKEN, + ) + + with ( + patch("keboola_agent_cli.cli.MemberService", return_value=svc), + patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}), + ): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "invite", + "--project", + ALIAS, + "--email", + "x@y.com", + "--role", + "admin", + ], + ) + assert result.exit_code == 3 + + +class TestProjectInviteBulk: + def test_json_bulk_summary(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.invite_bulk.return_value = BulkInviteResult( + total=2, + succeeded=1, + noop=1, + failed=0, + rows=[ + MemberInviteRow( + email="a@b.com", + project=ALIAS, + project_id=PROJECT_ID, + role="guest", + status="ok", + invitation_id=1, + ), + MemberInviteRow( + email="c@d.com", + project=ALIAS, + project_id=PROJECT_ID, + role="guest", + status="noop", + note="already_invited", + ), + ], + ) + csv_path = tmp_path / "bulk.csv" + csv_path.write_text( + "email,project,role\na@b.com,cuesta-master,guest\nc@d.com,cuesta-master,guest\n" + ) + + with ( + patch("keboola_agent_cli.cli.MemberService", return_value=svc), + patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}), + ): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "invite", + "--from-csv", + str(csv_path), + ], + ) + assert result.exit_code == 0, result.output + out = json.loads(result.output) + assert out["status"] == "ok" + assert out["data"]["total"] == 2 + assert out["data"]["succeeded"] == 1 + assert out["data"]["noop"] == 1 + + def test_from_csv_mutually_exclusive_with_project(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + csv_path = tmp_path / "bulk.csv" + csv_path.write_text("email,project,role\na@b.com,cuesta-master,guest\n") + + with patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "invite", + "--from-csv", + str(csv_path), + "--project", + ALIAS, + ], + ) + assert result.exit_code == 2 + + +class TestMemberList: + def test_json_output(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_members.return_value = { + "alias": ALIAS, + "project_id": PROJECT_ID, + "members": [ + { + "id": 216, + "email": "max.ottomansky@keboola.com", + "name": "Max", + "role": "admin", + "status": "active", + "mfa_enabled": True, + } + ], + } + + with ( + patch("keboola_agent_cli.cli.MemberService", return_value=svc), + patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}), + ): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "member-list", + "--project", + ALIAS, + ], + ) + assert result.exit_code == 0, result.output + out = json.loads(result.output) + assert out["data"]["members"][0]["role"] == "admin" + svc.list_members.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, include_pending=False + ) + + def test_include_pending_flag_propagates(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_members.return_value = { + "alias": ALIAS, + "project_id": PROJECT_ID, + "members": [], + "pending_invitations": [], + } + + with ( + patch("keboola_agent_cli.cli.MemberService", return_value=svc), + patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}), + ): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "member-list", + "--project", + ALIAS, + "--include-pending", + ], + ) + assert result.exit_code == 0 + svc.list_members.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, include_pending=True + ) + + +class TestInvitationCancel: + def test_yes_skips_confirmation(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.cancel_invitation.return_value = { + "status": "cancelled", + "alias": ALIAS, + "project_id": PROJECT_ID, + "email": "x@y.com", + "invitation_id": 99, + } + + with ( + patch("keboola_agent_cli.cli.MemberService", return_value=svc), + patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}), + ): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "invitation-cancel", + "--project", + ALIAS, + "--email", + "x@y.com", + "--yes", + ], + ) + assert result.exit_code == 0 + svc.cancel_invitation.assert_called_once() + + +class TestMemberRemove: + def test_destructive_yes(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.remove_member.return_value = { + "status": "removed", + "alias": ALIAS, + "project_id": PROJECT_ID, + "email": "ghost@example.com", + "user_id": 999, + } + + with ( + patch("keboola_agent_cli.cli.MemberService", return_value=svc), + patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}), + ): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "member-remove", + "--project", + ALIAS, + "--email", + "ghost@example.com", + "--yes", + ], + ) + assert result.exit_code == 0 + svc.remove_member.assert_called_once() + + +class TestMemberSetRole: + def test_propagates_role(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.set_member_role.return_value = { + "status": "updated", + "alias": ALIAS, + "project_id": PROJECT_ID, + "email": "x@y.com", + "user_id": 216, + "role": "guest", + } + + with ( + patch("keboola_agent_cli.cli.MemberService", return_value=svc), + patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}), + ): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "project", + "member-set-role", + "--project", + ALIAS, + "--email", + "x@y.com", + "--role", + "guest", + ], + ) + assert result.exit_code == 0, result.output + svc.set_member_role.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, email="x@y.com", role="guest" + ) + + +class TestRegressions: + """Iteration-2 reviewer findings encoded as CLI regression tests.""" + + def test_hint_with_from_csv_emits_clear_error_not_silent_skip(self, tmp_path: Path) -> None: + """Pre-fix: --hint + --from-csv silently fell through to the live + path and prompted for the manage token. Now it exits 2 with a usage + error explaining hints are single-shot only.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + csv_path = tmp_path / "bulk.csv" + csv_path.write_text("email,project,role\na@b.com,cuesta-master,guest\n") + + with patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--json", + "--hint", + "client", + "project", + "invite", + "--from-csv", + str(csv_path), + ], + ) + assert result.exit_code == 2, result.output + # The error envelope is JSON; check the message content. + out = json.loads(result.output) + assert "from-csv" in out["error"]["message"].lower() + + +class TestHints: + def test_invite_hint_client_renders(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + + with patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--hint", + "client", + "project", + "invite", + "--project", + ALIAS, + "--email", + "x@y.com", + "--role", + "admin", + ], + ) + assert result.exit_code == 0 + assert "ManageClient" in result.output + assert "create_project_invitation" in result.output + + def test_invite_hint_service_renders(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + + with patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}): + result = runner.invoke( + app, + [ + "--config-dir", + str(config_dir), + "--hint", + "service", + "project", + "invite", + "--project", + ALIAS, + "--email", + "x@y.com", + "--role", + "admin", + ], + ) + assert result.exit_code == 0 + assert "MemberService" in result.output + assert "invite" in result.output diff --git a/tests/test_member_service.py b/tests/test_member_service.py new file mode 100644 index 00000000..f8fb8f62 --- /dev/null +++ b/tests/test_member_service.py @@ -0,0 +1,585 @@ +"""Tests for MemberService - project member & invitation lifecycle (since v0.26.1).""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.member_service import MemberService + +STACK_URL = "https://connection.us-east4.gcp.keboola.com" +MANAGE_TOKEN = "manage-12345-abcdefghijklmnopqrstuvwxyz0123456789" +PROJECT_ID = 5725 +ALIAS = "cuesta-master" + + +def _make_member(uid: int, email: str, role: str = "admin") -> dict: + return { + "id": uid, + "email": email, + "name": email.split("@")[0], + "role": role, + "status": "active", + "mfaEnabled": False, + "features": [], + } + + +def _make_invitation(inv_id: int, email: str, role: str = "guest") -> dict: + return { + "id": inv_id, + "created": "2026-05-01T19:04:35+0200", + "expires": None, + "reason": "", + "role": role, + "user": {"id": None, "email": email, "name": ""}, + "creator": {"id": 216, "email": "max.ottomansky@keboola.com", "name": "Max"}, + } + + +@pytest.fixture +def store_with_master(tmp_config_dir: Path) -> ConfigStore: + """ConfigStore with the master cuesta project pre-registered.""" + store = ConfigStore(config_dir=tmp_config_dir) + store.add_project( + ALIAS, + ProjectConfig( + stack_url=STACK_URL, + token="901-fake-storage-token-1234567890", + project_name="[Cuesta training] - Master", + project_id=PROJECT_ID, + ), + ) + return store + + +@pytest.fixture +def manage_client_factory(): + """Factory returning a single shared MagicMock manage client.""" + mock = MagicMock() + mock._stack_url = STACK_URL + factory = MagicMock(return_value=mock) + return factory, mock + + +# ────────────────────────────────────────────────────────────────────── +# invite (single) +# ────────────────────────────────────────────────────────────────────── + + +class TestInviteSingle: + def test_happy_path(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.create_project_invitation.return_value = _make_invitation( + 1741, "ottomansky.max@gmail.com" + ) + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.invite( + manage_token=MANAGE_TOKEN, + alias=ALIAS, + email="ottomansky.max@gmail.com", + role="guest", + reason="hi", + ) + + assert result["status"] == "ok" + assert result["invitation_id"] == 1741 + mock_client.create_project_invitation.assert_called_once_with( + project_id=PROJECT_ID, + email="ottomansky.max@gmail.com", + role="guest", + reason="hi", + ) + mock_client.close.assert_called_once() + + def test_dry_run_makes_no_client_call(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.invite( + manage_token=MANAGE_TOKEN, + alias=ALIAS, + email="x@y.com", + role="admin", + dry_run=True, + ) + + assert result["status"] == "dry_run" + factory.assert_not_called() + mock_client.create_project_invitation.assert_not_called() + + def test_already_invited_returns_noop(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.create_project_invitation.side_effect = KeboolaApiError( + message="API error 400 from ...: This user has already been invited to this project.", + status_code=400, + error_code=ErrorCode.API_ERROR, + ) + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.invite(manage_token=MANAGE_TOKEN, alias=ALIAS, email="x@y.com", role="admin") + + assert result["status"] == "noop" + assert result["note"] == "already_invited" + mock_client.close.assert_called_once() + + def test_already_member_returns_noop(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.create_project_invitation.side_effect = KeboolaApiError( + message="API error 400 from ...: This user is already a member of this project.", + status_code=400, + error_code=ErrorCode.API_ERROR, + ) + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.invite(manage_token=MANAGE_TOKEN, alias=ALIAS, email="x@y.com", role="admin") + + assert result["status"] == "noop" + assert result["note"] == "already_member" + + def test_other_400_re_raises(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.create_project_invitation.side_effect = KeboolaApiError( + message="API error 400 from ...: completely unrelated rejection", + status_code=400, + error_code=ErrorCode.API_ERROR, + ) + svc = MemberService(store_with_master, manage_client_factory=factory) + + with pytest.raises(KeboolaApiError): + svc.invite(manage_token=MANAGE_TOKEN, alias=ALIAS, email="x@y.com", role="admin") + # close() must still fire even on raise + mock_client.close.assert_called_once() + + def test_unknown_alias_raises_config_error( + self, store_with_master, manage_client_factory + ) -> None: + factory, _ = manage_client_factory + svc = MemberService(store_with_master, manage_client_factory=factory) + + with pytest.raises(ConfigError, match="not registered"): + svc.invite( + manage_token=MANAGE_TOKEN, + alias="does-not-exist", + email="x@y.com", + role="admin", + ) + + def test_invalid_role_raises_value_error( + self, store_with_master, manage_client_factory + ) -> None: + factory, _ = manage_client_factory + svc = MemberService(store_with_master, manage_client_factory=factory) + + with pytest.raises(ValueError, match="Invalid role"): + svc.invite( + manage_token=MANAGE_TOKEN, + alias=ALIAS, + email="x@y.com", + role="developer", # not on the whitelist + ) + + +# ────────────────────────────────────────────────────────────────────── +# invite (bulk via --from-csv) +# ────────────────────────────────────────────────────────────────────── + + +def _write_csv(path: Path, content: str) -> Path: + path.write_text(content, encoding="utf-8") + return path + + +class TestInviteBulk: + def test_partial_success( + self, + tmp_path: Path, + store_with_master: ConfigStore, + manage_client_factory, + ) -> None: + factory, mock_client = manage_client_factory + + def _create_invitation(*, project_id, email, role, reason): + if email == "fail@example.com": + raise KeboolaApiError( + message="API error 400: nope", + status_code=400, + error_code=ErrorCode.API_ERROR, + ) + if email == "dup@example.com": + raise KeboolaApiError( + message="API error 400: This user has already been invited to this project.", + status_code=400, + error_code=ErrorCode.API_ERROR, + ) + return _make_invitation(1700 + len(email), email, role) + + mock_client.create_project_invitation.side_effect = _create_invitation + + csv_path = _write_csv( + tmp_path / "bulk.csv", + "email,project,role\n" + "ok@example.com,cuesta-master,guest\n" + "dup@example.com,cuesta-master,guest\n" + "fail@example.com,cuesta-master,guest\n", + ) + + svc = MemberService(store_with_master, manage_client_factory=factory) + result = svc.invite_bulk(manage_token=MANAGE_TOKEN, csv_path=csv_path, workers=1) + + assert result.total == 3 + assert result.succeeded == 1 + assert result.noop == 1 + assert result.failed == 1 + assert {r.email for r in result.rows} == { + "ok@example.com", + "dup@example.com", + "fail@example.com", + } + + def test_dry_run_makes_no_client_call( + self, + tmp_path: Path, + store_with_master: ConfigStore, + manage_client_factory, + ) -> None: + factory, mock_client = manage_client_factory + csv_path = _write_csv( + tmp_path / "bulk.csv", + "email,project,role\nok@example.com,cuesta-master,guest\n", + ) + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.invite_bulk(manage_token=MANAGE_TOKEN, csv_path=csv_path, dry_run=True) + + assert result.dry_run is True + assert result.total == 1 + assert result.succeeded == 1 + factory.assert_not_called() + mock_client.create_project_invitation.assert_not_called() + + def test_default_role_fills_missing_column( + self, + tmp_path: Path, + store_with_master: ConfigStore, + manage_client_factory, + ) -> None: + factory, mock_client = manage_client_factory + mock_client.create_project_invitation.return_value = _make_invitation(1, "x@y.com", "admin") + csv_path = _write_csv( + tmp_path / "bulk.csv", + "email,project\nx@y.com,cuesta-master\n", + ) + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.invite_bulk( + manage_token=MANAGE_TOKEN, + csv_path=csv_path, + default_role="admin", + workers=1, + ) + + assert result.succeeded == 1 + assert result.rows[0].role == "admin" + + def test_no_role_column_no_default_role_raises( + self, + tmp_path: Path, + store_with_master: ConfigStore, + manage_client_factory, + ) -> None: + factory, _ = manage_client_factory + csv_path = _write_csv( + tmp_path / "bulk.csv", + "email,project\nx@y.com,cuesta-master\n", + ) + svc = MemberService(store_with_master, manage_client_factory=factory) + + with pytest.raises(ConfigError, match="role"): + svc.invite_bulk(manage_token=MANAGE_TOKEN, csv_path=csv_path) + + def test_missing_email_column_raises( + self, + tmp_path: Path, + store_with_master: ConfigStore, + manage_client_factory, + ) -> None: + factory, _ = manage_client_factory + csv_path = _write_csv( + tmp_path / "bulk.csv", + "user,project,role\nx@y.com,cuesta-master,admin\n", + ) + svc = MemberService(store_with_master, manage_client_factory=factory) + + with pytest.raises(ConfigError, match="missing an 'email' column"): + svc.invite_bulk(manage_token=MANAGE_TOKEN, csv_path=csv_path) + + def test_missing_file_raises( + self, + tmp_path: Path, + store_with_master: ConfigStore, + manage_client_factory, + ) -> None: + factory, _ = manage_client_factory + svc = MemberService(store_with_master, manage_client_factory=factory) + + with pytest.raises(ConfigError, match="not found"): + svc.invite_bulk(manage_token=MANAGE_TOKEN, csv_path=tmp_path / "missing.csv") + + def test_unknown_alias_in_csv_row_is_per_row_failure_not_global_abort( + self, + tmp_path: Path, + store_with_master: ConfigStore, + manage_client_factory, + ) -> None: + """One bad row never aborts the rest -- mirror OrgService.refresh_tokens. + + Regression: pre-fix, the upfront `_stack_for_row` set comprehension + would raise ConfigError on the first unregistered alias and the entire + bulk batch would abort. Now the bad row appears as `status="failed"` + and the remaining rows still execute. + """ + factory, mock_client = manage_client_factory + mock_client.create_project_invitation.return_value = _make_invitation(42, "ok@example.com") + csv_path = _write_csv( + tmp_path / "bulk.csv", + "email,project,role\n" + "ok@example.com,cuesta-master,guest\n" + "bad@example.com,unknown-alias,guest\n", + ) + + svc = MemberService(store_with_master, manage_client_factory=factory) + result = svc.invite_bulk(manage_token=MANAGE_TOKEN, csv_path=csv_path, workers=1) + + assert result.total == 2 + assert result.succeeded == 1 + assert result.failed == 1 + by_email = {r.email: r for r in result.rows} + assert by_email["ok@example.com"].status == "ok" + assert by_email["bad@example.com"].status == "failed" + assert "unknown-alias" in by_email["bad@example.com"].note + # The good row still hit the API + mock_client.create_project_invitation.assert_called_once() + + def test_numeric_project_id_resolves( + self, + tmp_path: Path, + store_with_master: ConfigStore, + manage_client_factory, + ) -> None: + factory, mock_client = manage_client_factory + mock_client.create_project_invitation.return_value = _make_invitation(1, "x@y.com", "guest") + csv_path = _write_csv( + tmp_path / "bulk.csv", + f"email,project_id,role\nx@y.com,{PROJECT_ID},guest\n", + ) + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.invite_bulk(manage_token=MANAGE_TOKEN, csv_path=csv_path, workers=1) + assert result.succeeded == 1 + assert result.rows[0].project_id == PROJECT_ID + + +# ────────────────────────────────────────────────────────────────────── +# member-list, invitation-list, invitation-cancel +# ────────────────────────────────────────────────────────────────────── + + +class TestBulkRegressions: + """Iteration-2 reviewer findings encoded as regression tests.""" + + def test_dry_run_rejects_multi_stack_csv( + self, + tmp_path: Path, + store_with_master: ConfigStore, + manage_client_factory, + ) -> None: + """Dry-run preview must enforce the same single-stack invariant the + live path enforces -- otherwise users get a 'preview said ok' surprise + on the real run.""" + factory, _ = manage_client_factory + store_with_master.add_project( + "other-stack", + ProjectConfig( + stack_url="https://connection.keboola.com", + token="901-fake-other-stack-token", + project_id=1, + project_name="Other", + ), + ) + csv_path = _write_csv( + tmp_path / "bulk.csv", + "email,project,role\na@b.com,cuesta-master,guest\nc@d.com,other-stack,guest\n", + ) + svc = MemberService(store_with_master, manage_client_factory=factory) + + with pytest.raises(ConfigError, match="multiple stack URLs"): + svc.invite_bulk(manage_token=MANAGE_TOKEN, csv_path=csv_path, dry_run=True) + + def test_csv_with_utf8_bom_parses( + self, + tmp_path: Path, + store_with_master: ConfigStore, + manage_client_factory, + ) -> None: + """Excel-exported CSVs prepend a UTF-8 BOM; the parser must strip it + so the first header reads as 'email', not 'email'.""" + factory, mock_client = manage_client_factory + mock_client.create_project_invitation.return_value = _make_invitation(1, "x@y.com") + csv_path = tmp_path / "bom.csv" + #  = UTF-8 BOM + csv_path.write_text( + "email,project,role\nx@y.com,cuesta-master,guest\n", + encoding="utf-8", + ) + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.invite_bulk(manage_token=MANAGE_TOKEN, csv_path=csv_path, workers=1) + assert result.succeeded == 1 + + +class TestListMembers: + def test_active_only(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.list_project_members.return_value = [ + _make_member(216, "max.ottomansky@keboola.com", "admin"), + ] + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.list_members(manage_token=MANAGE_TOKEN, alias=ALIAS) + + assert result["alias"] == ALIAS + assert result["project_id"] == PROJECT_ID + assert result["members"][0]["email"] == "max.ottomansky@keboola.com" + assert "pending_invitations" not in result + mock_client.list_project_invitations.assert_not_called() + + def test_include_pending(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.list_project_members.return_value = [ + _make_member(216, "max.ottomansky@keboola.com") + ] + mock_client.list_project_invitations.return_value = [ + _make_invitation(1515, "marcusscwong@gmail.com", "admin") + ] + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.list_members(manage_token=MANAGE_TOKEN, alias=ALIAS, include_pending=True) + + assert len(result["pending_invitations"]) == 1 + assert result["pending_invitations"][0]["user"]["email"] == "marcusscwong@gmail.com" + + +class TestCancelInvitation: + def test_resolves_id_from_email(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.list_project_invitations.return_value = [ + _make_invitation(1515, "marcusscwong@gmail.com") + ] + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.cancel_invitation( + manage_token=MANAGE_TOKEN, alias=ALIAS, email="marcusscwong@gmail.com" + ) + + assert result["invitation_id"] == 1515 + mock_client.cancel_project_invitation.assert_called_once_with(PROJECT_ID, 1515) + + def test_explicit_id_skips_lookup(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.cancel_invitation( + manage_token=MANAGE_TOKEN, + alias=ALIAS, + email="x@y.com", + invitation_id=9999, + ) + + assert result["invitation_id"] == 9999 + mock_client.list_project_invitations.assert_not_called() + mock_client.cancel_project_invitation.assert_called_once_with(PROJECT_ID, 9999) + + def test_email_not_found_raises_404(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.list_project_invitations.return_value = [ + _make_invitation(1, "someone-else@example.com") + ] + svc = MemberService(store_with_master, manage_client_factory=factory) + + with pytest.raises(KeboolaApiError) as exc_info: + svc.cancel_invitation( + manage_token=MANAGE_TOKEN, alias=ALIAS, email="missing@example.com" + ) + assert exc_info.value.status_code == 404 + + +class TestRemoveMember: + def test_resolves_user_id_from_email(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.list_project_members.return_value = [ + _make_member(4241, "mfiser@cuestapartners.com", "admin") + ] + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.remove_member( + manage_token=MANAGE_TOKEN, + alias=ALIAS, + email="MFiser@CuestaPartners.com", # case-insensitive + ) + + assert result["user_id"] == 4241 + mock_client.remove_project_member.assert_called_once_with(PROJECT_ID, 4241) + + def test_email_not_found_raises_404(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.list_project_members.return_value = [] + svc = MemberService(store_with_master, manage_client_factory=factory) + + with pytest.raises(KeboolaApiError) as exc_info: + svc.remove_member(manage_token=MANAGE_TOKEN, alias=ALIAS, email="ghost@example.com") + assert exc_info.value.status_code == 404 + mock_client.remove_project_member.assert_not_called() + + +class TestSetMemberRole: + def test_propagates_role_via_patch(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.list_project_members.return_value = [ + _make_member(216, "max.ottomansky@keboola.com", "admin") + ] + mock_client.update_project_member_role.return_value = { + "id": 216, + "email": "max.ottomansky@keboola.com", + "role": "guest", + } + svc = MemberService(store_with_master, manage_client_factory=factory) + + result = svc.set_member_role( + manage_token=MANAGE_TOKEN, + alias=ALIAS, + email="max.ottomansky@keboola.com", + role="guest", + ) + + assert result["role"] == "guest" + mock_client.update_project_member_role.assert_called_once_with(PROJECT_ID, 216, "guest") + + def test_invalid_role_raises_value_error( + self, store_with_master, manage_client_factory + ) -> None: + factory, _ = manage_client_factory + svc = MemberService(store_with_master, manage_client_factory=factory) + with pytest.raises(ValueError, match="Invalid role"): + svc.set_member_role( + manage_token=MANAGE_TOKEN, + alias=ALIAS, + email="x@y.com", + role="developer", + ) From 84cf480f446afd8e30dc00f046f9c6e0399c4b2a Mon Sep 17 00:00:00 2001 From: Petr Simecek Date: Wed, 6 May 2026 18:33:11 +0200 Subject: [PATCH 4/5] feat(0.28.0): data-app secrets + validate-repo + --auth public fix (logs deferred to follow-up) (#254) Closes the secrets-management, pre-flight-validation, AND public-auth-mode gaps in the v0.27.0 data-app surface. Logs / auto-log-dump deferred pending platform-side API exposure (the Data Science API does not document a programmatic logs endpoint; data_app_service.py and data-app-workflow.md already commit to its absence -- tracked as issue #240). The auth fix was discovery-driven: while live-validating secrets-set end-to-end on project 1143, --auth public surfaced HTTP 503 against the Keboola app-proxy. Root cause (no authorization key written by v0.27.0) took 30min against the public AppProxyDefinition.php validator; fixing it in this PR keeps v0.27.0 -> v0.28.0 from shipping a known silent break. - Service: DataAppService.{set,list,get,remove}_data_app_secrets with read-modify-write at the service layer (NOT Storage merge=True) to preserve nested sibling keys; per-project KMS encryption fail-closed; metadata-only on get (never echoes decrypted plaintext); idempotent remove; reserved-name shadowing detection. - Service: RepoValidateService + GitHubContentsClient for pre-flight Golden-Rule check. Trees-recursive fetch + up to 4 contents calls (<=5 GitHub API calls regardless of repo size). --type python-js only. - Service: _build_public_auth_block() + _auth_block_for(auth) dispatch. v0.27.0's --auth public wrote no authorization key at all -- the Keboola app-proxy refused to route (HTTP 503) and the UI's auth selector showed blank. Now writes the canonical noneProxyAuthorization shape ({auth_providers: [], auth_rules: [{auth_required: false}]}). Authoritative source: the public backend validator at keboola/job-queue-job-configuration AppProxyDefinition.php (when auth_required=false, auth MUST NOT be set). The private keboola/ui apps/kbc-ui/src/scripts/modules/data-apps/constants.ts corroborates with its noneProxyAuthorization constant for the "None" UI option (Keboola org members only). Live-validated end-to-end on project 1143: HTTP 200 on the resulting URL, no auth challenge, written block bit-identical to canonical. - Commands: kbagent data-app secrets-{set,list,get,remove} + kbagent data-app validate-repo. Reference: epilogs on every docstring. - Errors: DATA_APP_INVALID_SECRET, DATA_APP_INVALID_REPO, DATA_APP_REPO_VALIDATION_BLOCKING. - Permissions: secrets-set=write, secrets-list/get=read, secrets-remove=destructive, validate-repo=read. - Hints: 5 new --hint client/service variants. - Tests: 27 secrets service tests + 20 validate-repo service tests + 22 CLI tests (13 methods + 9 hint-compile parametrised) + 4 new auth-block tests (2505 total, all green). - Docs: CLAUDE.md All CLI Commands, AGENT_CONTEXT, keboola-expert.md matrix + version gate + 4 inline gotchas, commands-reference.md bullets, gotchas.md (three new (since v0.28.0) entries: auth fix, secrets, validate-repo), data-app-workflow.md (Managing app-runtime secrets + Pre-flight repo validation recipes). - Plugin: SKILL.md regenerated; plugin.json + marketplace.json synced to 0.28.0; changelog.py entry. Reserved-runtime-env-vars list locked to canon-documented floor (KBC_TOKEN, KBC_URL) per https://help.keboola.com/data-apps/storage-access/; TODO in gotchas to verify exhaustive list against running data-app env in follow-up. Out of scope (orphan-prevention issues filed BEFORE merge): - Logs / auto-log-dump on deploy failure -> #240 (needs platform API). - --auth oidc / github / gitlab / jumpcloud / auth0 -> #241. Co-authored-by: ottomansky --- CLAUDE.md | 5 + plugins/kbagent/agents/keboola-expert.md | 47 +- plugins/kbagent/skills/kbagent/SKILL.md | 5 + .../kbagent/references/commands-reference.md | 5 + .../kbagent/references/data-app-workflow.md | 79 +- .../skills/kbagent/references/gotchas.md | 119 +++ src/keboola_agent_cli/changelog.py | 8 + src/keboola_agent_cli/cli.py | 3 + src/keboola_agent_cli/commands/context.py | 43 + src/keboola_agent_cli/commands/data_app.py | 635 ++++++++++++++- src/keboola_agent_cli/errors.py | 5 + .../hints/definitions/data_app.py | 262 ++++++ src/keboola_agent_cli/permissions.py | 6 + .../services/data_app_service.py | 680 +++++++++++++++- .../services/repo_validate_service.py | 759 ++++++++++++++++++ tests/test_data_app_secrets_cli.py | 596 ++++++++++++++ tests/test_data_app_secrets_service.py | 456 +++++++++++ tests/test_data_app_service.py | 112 ++- tests/test_data_app_validate_repo_service.py | 283 +++++++ 19 files changed, 4088 insertions(+), 20 deletions(-) create mode 100644 src/keboola_agent_cli/services/repo_validate_service.py create mode 100644 tests/test_data_app_secrets_cli.py create mode 100644 tests/test_data_app_secrets_service.py create mode 100644 tests/test_data_app_validate_repo_service.py diff --git a/CLAUDE.md b/CLAUDE.md index 92d60ed9..d66ba47d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -365,6 +365,11 @@ kbagent data-app start --project NAME --app-id ID [--wait] [--timeout SECONDS] kbagent data-app stop --project NAME --app-id ID [--wait] [--timeout SECONDS] kbagent data-app delete --project NAME --app-id ID [--yes] kbagent data-app password --project NAME --app-id ID +kbagent data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE' [--secret ...] [--secrets-file PATH] [--branch ID] [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next] +kbagent data-app secrets-list --project ALIAS --app-id ID [--branch ID] [--show-fingerprint] +kbagent data-app secrets-get --project ALIAS --app-id ID --key '#KEY' [--branch ID] +kbagent data-app secrets-remove --project ALIAS --app-id ID --key '#KEY' [--key ...] [--branch ID] [--yes] [--dry-run] +kbagent data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] [--type python-js] [--strict] kbagent component list [--project NAME] [--type TYPE] [--query QUERY] kbagent component detail --component-id ID [--project NAME] diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 50d7ba16..828a46cc 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -70,7 +70,9 @@ a critical failure. `data-app password` needs 0.28.0+ with `--allow-env-manage-token` (the env var is default-deny on 0.28.0+), `project invite` / `project member-*` / `project invitation-*` - need 0.26.1+, `storage retype` is a future composite), you + need 0.26.1+, + `data-app secrets-* / validate-repo` need 0.28.0+, + `storage retype` is a future composite), you MUST refuse the task and return a handoff message to the parent: `"Cannot proceed safely on kbagent . Missing: . Ask user to run kbagent update, then re-invoke me."` Do not attempt @@ -119,6 +121,11 @@ a critical failure. | Cancel a pending invitation | `kbagent project invitation-cancel --project P --email E --yes` (0.26.1+) | `--invitation-id ID` if email lookup is ambiguous | DELETE via raw HTTP without going through the service layer | | Remove an active member | `kbagent project member-remove --project P --email E --yes` (0.26.1+, **destructive**) | `--hint client` for a script that removes by user_id directly | calling `member-remove` without `--yes` in non-interactive contexts (it will prompt and hang) | | Change a member's role | `kbagent project member-set-role --project P --email E --role admin\|guest\|readOnly\|share` (0.26.1+) | -- | `PUT /manage/projects/{id}/users/{userId}` -- the API rejects PUT with 404, the kbagent client correctly uses **PATCH** | +| Set / rotate app-runtime secrets | `kbagent data-app secrets-set --project P --app-id N --secret '#KEY=VAL'` (0.28.0+) then `data-app deploy --wait` -- per-project KMS encryption, fail-closed, never auto-deploys | `kbagent encrypt values --component-id keboola.data-apps` + `tool call update_config` -- ONLY if you need to write secrets to a different shape than `parameters.dataApp.secrets` | raw `POST` to encryption + Storage without read-modify-write -- you will clobber sibling keys nested under `parameters.dataApp.secrets` (Storage `merge=True` is shallow at the top level only) | +| Inspect what secrets are set on a data app | `kbagent data-app secrets-list --project P --app-id N` (0.28.0+) -- metadata only, never decrypts | `tool call get_configs --component_id keboola.data-apps` then read `parameters.dataApp.secrets` keys (raw dict, no env-var derivation, may leak ciphertext into output) | trying to decrypt -- the Encryption API has no decrypt endpoint, the CLI cannot decrypt under any branch | +| Confirm one secret is present | `kbagent data-app secrets-get --project P --app-id N --key '#KEY'` (0.28.0+) -- returns metadata only | -- | trying to extract the plaintext value (impossible by design; not a CLI gap) | +| Remove a secret from a data app | `kbagent data-app secrets-remove --project P --app-id N --key '#KEY' --yes` (0.28.0+) -- idempotent; missing keys exit 0 with `removed: 0` | `tool call update_config` with the secrets sub-dict deleted -- ONLY for batch removes that need a custom change description | `kbagent config update --set 'parameters.dataApp.secrets={}'` -- replaces the whole sub-dict, dropping every secret instead of just the named ones | +| Pre-flight a data-app repo before create | `kbagent data-app validate-repo --git-repo URL --type python-js [--git-pat-env VAR]` (0.28.0+) -- BLOCKING / WARN / OK with help-doc citations; ≤5 GitHub API calls regardless of repo size | git-clone the repo locally and inspect by hand | `data-app create --dry-run` (only shows the request bodies; does not validate repo structure) | If the table does not cover the user's task, **ask clarifying questions** instead of guessing. Returning a targeted question is a @@ -290,6 +297,44 @@ success, not a failure. the target project's Encryption API and refuses to write plaintext if the round-trip does not return a `KBC::Project*` ciphertext. +- **`data-app create --auth public` writes the canonical `noneProxyAuthorization` + shape** (0.28.0+, fixes a v0.27.0 silent-503 bug): v0.27.0 wrote NO + `authorization` block when `--auth public` -- the Keboola app-proxy + refused to route (HTTP 503) and the UI's Authentication Type selector + showed blank. v0.28.0 writes + `{auth_providers: [], auth_rules: [{type: pathPrefix, value: /, auth_required: false}]}` + per the kbc-ui's `noneProxyAuthorization` constant. If a user reports a + v0.27.0 public app returning 503, the fix is to recreate on 0.28.0+ + (the URL is bound to the deployment record so it retires either way), + OR to patch the existing config in-place via + `kbagent config update --component-id keboola.data-apps --config-id ID + --set 'authorization=...'` with the canonical shape. `--auth password` + behaviour is unchanged. Other auth providers (OIDC / GitHub / GitLab / + JumpCloud / Auth0) are not yet exposed by the CLI; tracked as a + follow-up issue. + +- **`data-app secrets-* metadata-only`** (0.28.0+): `secrets-get` NEVER + echoes the decrypted plaintext under any branch -- the Encryption API + is one-way and the CLI does not attempt to decrypt. NOT_FOUND on an + absent key never enumerates sibling keys (avoids leaking neighbour + presence). `secrets-remove` is idempotent: removing a non-existent key + returns exit 0 with `removed: 0` and does NOT bump the Storage + version. Setting a key whose derived env-var name collides with the + runtime-injected set (`KBC_TOKEN`, `KBC_URL` for sure; more TODO) is + silently shadowed by the platform; the CLI emits a stderr WARN and + surfaces `shadowed_by_runtime` in JSON envelope -- the WRITE still + happens. Read-modify-write is at the SERVICE layer (Storage `merge=True` + is shallow at the top level only and would clobber siblings nested in + `parameters.dataApp.secrets`). + +- **`data-app validate-repo` is GitHub-only**, `--type python-js` only + (0.28.0+): pre-flight Golden-Rule check via the GitHub Trees+Contents + API. Total <=5 calls regardless of repo size. Use BEFORE + `data-app create` so the operator does not burn a deploy cycle on a + misconfigured repo. WARNs are advisory unless `--strict` is set; + BLOCKINGs always fail. Tracked follow-up: streamlit / pure-Python / + R / Node-only types, GitLab/Bitbucket hosts. + - **`storage bucket-detail` is dialect-aware** (0.25.3+): the response shape depends on the bucket's backend. Snowflake buckets carry `snowflake_database` / `snowflake_schema` and per-table diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index c0d98874..d83cbfa8 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -129,6 +129,11 @@ When working inside a git repository or project directory, run `kbagent init` (o | Stop a running data app (preserves the URL and Storage config) | `kbagent data-app stop --project PROJECT --app-id APP-ID` | | Delete the deployment AND the Storage config (cascade, irreversible) | `kbagent data-app delete --project PROJECT --app-id APP-ID` | | Retrieve the simpleAuth password for a password-gated data app | `kbagent data-app password --project PROJECT --app-id APP-ID` | +| Encrypt and write app-runtime secrets to the linked Storage config | `kbagent data-app secrets-set --project PROJECT --app-id APP-ID` | +| List the keys in parameters.dataApp.secrets, with derived runtime env-var names | `kbagent data-app secrets-list --project PROJECT --app-id APP-ID` | +| Show metadata for ONE secret key. | `kbagent data-app secrets-get --project PROJECT --app-id APP-ID --key KEY` | +| Remove one or more app-runtime secrets. | `kbagent data-app secrets-remove --project PROJECT --app-id APP-ID --key KEY` | +| Pre-flight check that a git repo follows the Keboola data-app Golden Rule | `kbagent data-app validate-repo --git-repo GIT-REPO` | | List jobs from connected projects | `kbagent job list` | | Show detailed information about a specific job | `kbagent job detail --project PROJECT --job-id JOB-ID` | | Run a job for a component configuration | `kbagent job run --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index d58a31d9..30756e03 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -140,6 +140,11 @@ Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, - `data-app stop --project NAME --app-id ID [--wait] [--timeout SECONDS]` -- stop a running app (URL and Storage config preserved). - `data-app delete --project NAME --app-id ID [--yes]` -- destructive, cascades to Storage config; URL retired permanently. - `data-app password --project NAME --app-id ID` -- read the simpleAuth password. Manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+. Auto-generated, not rotatable -- delete + recreate to mint a new one. +- `data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE' [--secret ...] [--secrets-file PATH] [--branch ID] [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next]` -- encrypt and write `#`-prefixed secrets to `parameters.dataApp.secrets`. Per-project KMS encryption, fail-closed. Read-modify-write at the service layer (NOT Storage `merge=True` -- shallow). Runtime exposes each key as an env var with `#` stripped, `-` -> `_`, uppercased. Adding bumps the Storage version; the running container keeps the OLD config until the next `data-app deploy`. +- `data-app secrets-list --project ALIAS --app-id ID [--branch ID] [--show-fingerprint]` -- list secret keys + derived runtime env-var names. Never echoes encrypted ciphertext in full. `--show-fingerprint` opt-in for a short ciphertext fingerprint. +- `data-app secrets-get --project ALIAS --app-id ID --key '#KEY' [--branch ID]` -- show metadata for ONE secret. NEVER echoes the decrypted value (Encryption API is one-way). NOT_FOUND on absent key; never enumerates siblings. +- `data-app secrets-remove --project ALIAS --app-id ID --key '#KEY' [--key ...] [--branch ID] [--yes] [--dry-run]` -- destructive (can break a running app at next deploy). Idempotent: missing keys exit 0 with `removed: 0`. +- `data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] [--type python-js] [--strict]` -- pre-flight Golden-Rule check for a data-app git repo (https://help.keboola.com/data-apps/python-js/). GitHub-only; ≤5 API calls (1 tree + ≤4 contents) regardless of repo size. `--type` restricted to `python-js` in 0.28.0; streamlit / pure-Python / R / Node-only follow-up. `--strict` treats WARNs as failures. ## MCP Tools - `tool list [--project NAME] [--branch ID]` -- list available MCP tools (multi_project annotation) diff --git a/plugins/kbagent/skills/kbagent/references/data-app-workflow.md b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md index 37cab40a..ffbd40a6 100644 --- a/plugins/kbagent/skills/kbagent/references/data-app-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md @@ -116,6 +116,68 @@ kbagent data-app deploy --project prod --app-id 12345678 \ (rollback). Subsequent deploys without the flag will jump back to the latest. +### Pre-flight repo validation (since v0.28.0) + +```bash +kbagent data-app validate-repo \ + --git-repo https://github.com/myorg/dashboard \ + --git-branch main \ + --git-pat-env GITHUB_PAT_DATAAPP \ + --type python-js +``` + +Walks the repo via the GitHub Contents + Trees API and emits +BLOCKING / WARN / OK per check (Golden-Rule structure, no `pip install` +in `setup.sh`, `requires-python` consistency, nginx/app port match, +etc.). Each check carries a citation back to the help-doc anchor +(). Run before +`data-app create` so you don't burn a deploy cycle on a misconfigured +repo. Public repos: drop `--git-pat-env` and use `--git-public`. Total +GitHub call budget per run is ≤5 (1 tree + ≤4 contents) regardless of repo size, so the +60/hour unauth limit rarely fires; pass a PAT for CI loops. + +### Manage app-runtime secrets (since v0.28.0) + +```bash +# Set two secrets at once. Plaintext values; the CLI encrypts under +# THIS project's KMS via the Encryption API before writing to Storage. +kbagent --json data-app secrets-set \ + --project prod --app-id 12345678 \ + --secret '#ANTHROPIC_API_KEY=sk-ant-...' \ + --secret '#my-database-url=postgres://...' + +# Then redeploy so the running container picks up the new env. The +# JSON envelope from secrets-set carries a `next_step` field with the +# exact command; suppress it with --no-hint-next for scripted callers. +kbagent data-app deploy --project prod --app-id 12345678 --wait + +# Inspect what's set without echoing the encrypted ciphertext: +kbagent data-app secrets-list --project prod --app-id 12345678 +# -> #ANTHROPIC_API_KEY -> env ANTHROPIC_API_KEY +# -> #my-database-url -> env MY_DATABASE_URL + +# Confirm presence of one key (NEVER decrypts): +kbagent data-app secrets-get --project prod --app-id 12345678 --key '#ANTHROPIC_API_KEY' + +# Remove (idempotent -- absent keys exit 0 with removed=0): +kbagent data-app secrets-remove --project prod --app-id 12345678 \ + --key '#my-database-url' --yes +``` + +The runtime exposes each secret as an env var with `#` stripped, `-` +replaced with `_`, and uppercased +(). `secrets-set` does +read-modify-write at the service layer (Storage `merge=True` is +shallow at the top level only and would clobber siblings nested under +`parameters.dataApp.secrets`); every untouched key in the config body +is preserved bit-identical. Encryption is per-project KMS, fail-closed: +if the Encryption API does not return a `KBC::Project*` ciphertext, +the command aborts with `ENCRYPTION_FAILED` and Storage is never +written. Setting a key whose derived env-var name collides with the +runtime-injected set (`KBC_TOKEN`, `KBC_URL` for sure; more TODO +follow-up) emits a stderr WARN -- the platform value silently shadows +yours at runtime. + ## Gotchas encoded in the CLI (so you don't have to think about them) 1. **§9 redeploy contract** — `data-app deploy` always sends the @@ -158,15 +220,24 @@ latest. | Wake an auto-suspended app | `data-app start --app-id N` | | Pause a running app temporarily | `data-app stop --app-id N` | | Read the simpleAuth password | `data-app password --app-id N` (needs Manage token) | +| Set or rotate app-runtime secrets | `data-app secrets-set --app-id N --secret '#KEY=VAL'` then `data-app deploy --wait` | +| Inspect what secrets are set | `data-app secrets-list --app-id N` (metadata only, never decrypts) | +| Confirm one secret is present | `data-app secrets-get --app-id N --key '#KEY'` (metadata only) | +| Remove a secret | `data-app secrets-remove --app-id N --key '#KEY' --yes` (idempotent) | +| Pre-flight a repo before create | `data-app validate-repo --git-repo URL` (GitHub-only, python-js for now) | | Tear it all down | `data-app delete --app-id N` (cascades to Storage config) | ## What this command group deliberately does NOT cover - **Reading the build / runtime log** — the Data Science API does not - expose Terminal Logs as JSON; only the Keboola UI ("Terminal Log" tab) - shows them. If `data-app deploy --wait` exits with - `DATA_APP_BUILD_FAILED`, the next step is to open the UI link surfaced - in the error message. + expose Terminal Logs as JSON; only the Keboola UI ("Terminal Log" tab + at https://help.keboola.com/data-apps/terminal-log-tab/) shows them. + If `data-app deploy --wait` exits with `DATA_APP_BUILD_FAILED`, the + next step is to open the UI link surfaced in the error message. + A `data-app logs` command + auto-log-dump on deploy failure are + tracked as a follow-up: see `padak/keboola_agent_cli` + [issue #240](https://github.com/padak/keboola_agent_cli/issues/240) + (needs platform-side API exposure first). - **Updating size / auto-suspend / git settings** — those live on the Storage config body, not the deployment record. Use `kbagent config update --component-id keboola.data-apps --config-id ID diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index c2898e6c..a68b51b0 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -17,6 +17,125 @@ swaps it back into the original name. After merging the branch the original table now carries the typed schema with no downstream config rewrite required. +## `data-app create --auth public` writes the canonical noneProxyAuthorization shape (since v0.28.0; fixes v0.27.0 silent HTTP 503) + +- **What changed.** v0.27.0's `--auth public` wrote NO `authorization` + key into the Storage config at all. The Keboola app-proxy refused to + route to the resulting URL (HTTP 503 / "Service Unavailable") and the + UI's "Authentication Type" selector showed blank. Operators got a + silently broken app. v0.28.0 fixes this: `--auth public` now writes + the canonical `noneProxyAuthorization` shape that the kbc-ui exports + for the "None" UI option. +- **Exact shape written by 0.28.0:** + ```json + { + "app_proxy": { + "auth_providers": [], + "auth_rules": [ + {"type": "pathPrefix", "value": "/", "auth_required": false} + ] + } + } + ``` +- **Authoritative source (public):** keboola/job-queue-job-configuration + `src/JobDefinition/Configuration/Authorization/AppProxyDefinition.php` + -- when `auth_required=false`, the `auth` field MUST NOT be set. The + validator rejects shapes that include `auth` alongside + `auth_required: false`. +- **Corroborating source (private; Keboola org members only):** + keboola/ui `apps/kbc-ui/src/scripts/modules/data-apps/constants.ts` + exports this exact shape as the `noneProxyAuthorization` constant for + the "None" UI option. +- **Live-validated** end-to-end (HTTP 200 on the resulting URL, no + auth challenge; UI Authentication tab shows "None" pre-selected). +- **Repairing existing v0.27.0 apps stuck at 503**: re-run + `kbagent data-app create --auth public ...` to mint a new app, OR + patch the existing config in-place via + `kbagent config update --component-id keboola.data-apps --config-id ID --set 'authorization=...'` + with the shape above. The previous URL stays retired in either case + (the proxy URL is bound to the deployment record, not the config). +- **`--auth password` behaviour unchanged.** Mints a 20-char hex + simpleAuth password retrievable via `kbagent data-app password` + (Manage token required) or visible in the UI's Authentication tab. +- **Other auth providers (OIDC / GitHub OAuth / GitLab OAuth / + JumpCloud / Auth0)** are NOT yet supported by the CLI's `--auth` + flag. Use the Keboola UI to configure them after `data-app create`. + Tracked as a follow-up issue. + +## `data-app secrets-*` -- per-project KMS, idempotent remove, never decryptable (since v0.28.0) + +- **Encryption is per-project KMS.** `kbagent data-app secrets-set` calls + the project's Encryption API to wrap each plaintext value before + writing it to Storage. The resulting `KBC::Project*` ciphertext is + bound to the project's KMS key; the same ciphertext does NOT decrypt + in another project. Same fail-closed semantic as `data-app create`'s + `--git-pat-encrypted`: if the Encryption API does not return a + project-scoped ciphertext, the command aborts with `ENCRYPTION_FAILED` + and never writes plaintext to Storage. `--allow-plaintext-on-encrypt-failure` + is bootstrap/debug only; never use in production. +- **Read-modify-write at the service layer, NOT Storage `merge=True`.** + The Storage API's `merge=True` flag is shallow at the top level only; + relying on it would clobber sibling keys nested inside + `parameters.dataApp.secrets`. The CLI GETs the full config, modifies + the secrets sub-dict in place, and PUTs the unchanged remainder. Every + untouched sibling key (under `parameters.dataApp.secrets`, + `parameters.dataApp` -- slug, git block, id back-pointer, `parameters` + itself, and the top-level `runtime`/`authorization`/`storage`) is + preserved bit-identical. +- **`secrets-remove` is idempotent.** Removing a key that isn't set is + exit 0 with `removed: 0`, `not_found: []`. The + Storage version is not bumped on a no-op. Do NOT script around this + with a precondition lookup -- the idempotent path is the contract. +- **`secrets-get` NEVER echoes the decrypted plaintext.** The Encryption + API has no decrypt endpoint; the CLI cannot decrypt under any branch. + The command returns metadata only -- key name, derived env-var name, + ciphertext fingerprint, encryption prefix, presence flag. NOT_FOUND on + an absent key never enumerates sibling keys. +- **Runtime env-var translation rule:** strip `#`, replace `-` with `_`, + uppercase. Documented at https://help.keboola.com/data-apps/python-js/. + Examples: `#KBC_TOKEN` -> `KBC_TOKEN`, `#my-api-key` -> `MY_API_KEY`, + `#anthropic-token` -> `ANTHROPIC_TOKEN`. +- **Setting a reserved-name secret is silently shadowed.** The data-app + runtime auto-injects a documented set of env vars (canon-confirmed + floor: `KBC_TOKEN`, `KBC_URL`; runtime almost certainly injects more + -- TODO follow-up to enumerate exhaustively against a running app). + Setting `--secret '#KBC_TOKEN=foo'` succeeds (exit 0) but the platform + value silently shadows yours at runtime; the command emits a stderr + WARN naming each shadowed key and lists them in + `shadowed_by_runtime[]` of the JSON envelope. +- **Adding/removing a secret bumps the Storage version, but the running + container keeps the OLD config until `data-app deploy` runs.** Same + contract as any other `keboola.data-apps` config edit (see the + `(since v0.27.0)` entry below). The response includes a `next_step` + field with the exact redeploy command to run; suppress it with + `--no-hint-next` for scripted callers. + +## `data-app validate-repo` -- pre-flight against the Golden Rule, GitHub-only (since v0.28.0) + +- `kbagent data-app validate-repo --git-repo URL` walks the repo via the + GitHub Contents + Trees API and verifies the documented "Golden Rule" + layout from https://help.keboola.com/data-apps/python-js/ before + `data-app create`. Each check emits BLOCKING / WARN / OK with a + citation back to the help anchor that defines the rule. Runs in ≤5 + GitHub API calls regardless of repo size (one trees-recursive + up + to four contents fetches), so the 60/hour unauthenticated GitHub + rate limit is no longer the common-case failure mode. +- **`--type` is restricted to `python-js` in 0.28.0.** Streamlit / + pure-Python / R / Node-only repos have different layouts (Streamlit + does not require the `keboola-config/` tree, for instance) and need + per-type canon citations. Tracked as a follow-up. +- **GitHub-only.** GitLab / Bitbucket support is a follow-up. Calling + with a non-GitHub URL exits 2 / `INVALID_ARGUMENT`. +- Exit 0 on all checks <= WARN; exit 1 on any BLOCKING. `--strict` + treats WARNs as failures (exit 1) for CI gating. +- **Reading the build / runtime log is still NOT available via the + CLI.** The Data Science API does not expose Terminal Logs as JSON + (per https://help.keboola.com/data-apps/terminal-log-tab/); on + `DATA_APP_BUILD_FAILED` / `DATA_APP_DEPLOY_TIMEOUT` the next step is + still to open the UI's Terminal Log tab. A `data-app logs` command + + auto-log-dump on deploy failure are tracked as + [issue #240](https://github.com/padak/keboola_agent_cli/issues/240) + (needs platform-side API exposure first). ## Manage token: env var is ignored without `--allow-env-manage-token` (since v0.28.0) diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index bfaa7d67..f0df35d1 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -22,6 +22,14 @@ "New: top-level CLI flag `--allow-env-manage-token` (session-only, mirrors `--deny-writes` / `--deny-destructive`). Plumbed via `ctx.obj['allow_env_manage_token']` and forwarded by the three call sites into `resolve_manage_token(allow_env=...)`. Not persisted, no env-var equivalent (intentional; an env-var equivalent would re-create the AI-exfiltration hole this default-deny is closing).", "Tests: 12 new (`tests/test_helpers.py::TestResolveManageToken` x7 covering allow_env-True/False x env-set/unset x TTY/non-TTY combinations + token-leak regression pin; `tests/test_manage_token_cli.py::TestAllowEnvManageTokenFlag` x4 covering project-refresh / org-setup / data-app-password through CliRunner with services mocked; `tests/test_manage_token_bulk.py::TestBulkPromptOnce` pinning the contract that `project refresh --all` resolves the token exactly once at command entry, not per-project).", "Docs: `commands/context.py` AGENT_CONTEXT updated (org-setup example + env-var help block); `CLAUDE.md` convention #12 + global-flag list; `keboola-expert.md` Rule 6 VERSION GATE adds the 0.28.0+ env-flag requirement, tool-selection-matrix updated, new inline-gotcha block; `gotchas.md` new `(since v0.28.0)` entry naming the warning text and the one-line CI fix; `commands-reference.md` updated for `org setup`, `data-app password`, env-var table.", + "New: `kbagent data-app secrets-set / secrets-list / secrets-get / secrets-remove` — manage `#`-prefixed app-runtime secrets in `parameters.dataApp.secrets`. Encryption is per-project KMS via the existing `EncryptService` (same fail-closed semantics as `--git-pat-encrypted`: refuses to write plaintext if the Encryption API does not return a project-scoped ciphertext). Read-modify-write at the service layer (NOT Storage `merge=True` — that flag is shallow at the top level only and would clobber sibling keys nested inside `parameters.dataApp.secrets`). The runtime exposes each key as an env var with `#` stripped, `-` replaced with `_`, and uppercased (`#my-api-key` → `MY_API_KEY` per help.keboola.com/data-apps/python-js/). `secrets-get` is metadata-only — never echoes decrypted plaintext to stdout / stderr / logs / change descriptions; the Encryption API is one-way and the CLI does not attempt to decrypt under any branch. `secrets-remove` is idempotent (missing keys exit 0 with `removed: 0`). `secrets-set` warns when a derived env-var name collides with `RESERVED_RUNTIME_ENV_VARS` (KBC_TOKEN, KBC_URL — verified canon floor; full runtime list TODO follow-up). Adding/removing a secret bumps the Storage version but the running container keeps the OLD config until the next `data-app deploy`.", + "New: `kbagent data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] [--type python-js] [--strict]` — pre-flight check that a git repo follows the documented Golden Rule (https://help.keboola.com/data-apps/python-js/) BEFORE `data-app create` so operators don't burn a deploy cycle on a misconfigured repo. Each check emits BLOCKING / WARN / OK with a help-doc citation: `keboola-config/nginx/sites/default.conf` exists, `keboola-config/supervisord/services/app.conf` exists, `pyproject.toml` at root, `keboola-config/setup.sh` content has no `pip install` (BLOCKING per the help canon's pip prohibition) and contains `uv sync` if `pyproject.toml` declares deps, `requires-python` consistent with the runtime image (when the pin is available), nginx `proxy_pass` port matches `app.conf` declared port. Uses `GET /repos/{owner}/{repo}/git/trees/{ref}?recursive=1` (one call) + up to 4 `GET .../contents/{path}` for files whose contents the rules need to inspect — total ≤5 GitHub API calls (1 tree + 0-4 contents) regardless of repo size, sidesteps the 60/hour unauth rate limit for typical use. `--git-pat-env` / `--git-pat-file` raises the limit to 5,000/hour. Read-only; never touches a Keboola project. `--type` is restricted to `python-js` in 0.28.0; streamlit / pure-Python / R / Node-only follow-up.", + "New: `RepoValidateService` (`src/keboola_agent_cli/services/repo_validate_service.py`) — pure validation function `validate_keboola_repo(snapshot, type_, runtime_python_pin)` plus a tiny `GitHubContentsClient` (HTTPS GET to `api.github.com`, optional bearer PAT, no token persistence). Service module is the only place GitHub HTTP lives; the rest of kbagent stays Keboola-API-only.", + "New: `ErrorCode` entries `DATA_APP_INVALID_SECRET`, `DATA_APP_INVALID_REPO`, `DATA_APP_REPO_VALIDATION_BLOCKING`. Permission registry entries `data-app.secrets-set` (write), `data-app.secrets-list` / `data-app.secrets-get` (read), `data-app.secrets-remove` (destructive — removing a secret can break a running app), `data-app.validate-repo` (read).", + "New: `--hint client/service` for all five new commands. `secrets-get` hint snippet asserts the metadata-only contract; `validate-repo` snippet uses `RepoValidateService.validate_repo(...)` and the hint comment notes that GitHub-side detail is not shown.", + "Fix: `kbagent data-app create --auth public` now writes the canonical `noneProxyAuthorization` shape (kbc-ui exact constant: `auth_providers: []` + `auth_rules: [{type: pathPrefix, value: /, auth_required: false}]`). v0.27.0 wrote NO `authorization` key when `--auth public`, leaving the Keboola app-proxy unable to route (HTTP 503) and the UI Authentication Type selector blank — silently broken. Authoritative source: the public backend validator at `keboola/job-queue-job-configuration` `AppProxyDefinition.php` (when `auth_required=false`, `auth` MUST NOT be set). The private `keboola/ui` repo `apps/kbc-ui/src/scripts/modules/data-apps/constants.ts` corroborates: its `noneProxyAuthorization` constant exports this exact shape for the None UI option (Keboola org members can verify; external readers rely on the validator). Live-validated end-to-end on a real project: HTTP 200 on the resulting URL, written block bit-identical to canon, UI auth selector now shows None pre-selected. Existing `--auth password` behaviour unchanged.", + "Tests: 27 secrets service tests + 20 validate-repo service tests + 22 CLI tests (13 secrets/validate-repo CLI methods + 9 hint-compile AST-parse cases) + 4 new auth-block tests (`TestDataAppCreateAuthBlock` asserts both `--auth public` and `--auth password` write the canonical shape on POST `/apps` AND PUT Storage). 2505 total tests green. Sibling-preservation regression test for `secrets-set` asserts every untouched key under `parameters.dataApp.secrets`, `parameters.dataApp` (slug, git block), `parameters` (id), and the top-level config (`runtime`, `authorization`, `storage`) is preserved bit-identical after the read-modify-write.", + 'Plugin: `keboola-expert.md` matrix gains five new rows (one per `secrets-set / -list / -get / -remove + validate-repo`); §1 Rule 6 VERSION GATE example updated for `secrets / validate-repo need 0.28.0+`. New `(since v0.28.0)` `gotchas.md` entries: (a) secrets are per-project KMS encrypted, `secrets-remove` on missing key is exit 0, `secrets-get` never echoes decrypted plaintext, `#KBC_TOKEN` is silently shadowed by the runtime; (b) `validate-repo` GitHub-only Golden-Rule check; (c) `--auth public` writes the canonical `noneProxyAuthorization` shape (fixes v0.27.0 silent 503). New "Managing app-runtime secrets" + "Pre-flight repo validation" recipe sections in `data-app-workflow.md`. Logs / auto-log-dump deferred to issue #240 (the Data Science API does not expose Terminal Logs as JSON per help canon).', ], "0.27.0": [ "New: `kbagent data-app` command group — first-class lifecycle for Keboola data apps (`keboola.data-apps` Storage component + Data Science API `/apps`). Eight subcommands: `list`, `detail`, `create`, `deploy`, `start`, `stop`, `delete`, `password`. The CLI encapsulates the **§9 redeploy contract** (always sends the `{desiredState=running, configVersion, restartIfRunning=true}` trio together; without it, `PATCH /apps {desiredState:running}` silently pins to the empty-shell v2 and the runner errors `dataApp.git.repository is required in /data/config.json`), per-project KMS encryption of git PATs (refuses to write plaintext if the Encryption API does not return a project-scoped ciphertext), cleanup-in-finally on initial-deploy failure (orphan shell deleted by default; `--keep-on-failure` opts out), and a poll loop that respects pitfall #1 — `state == stopped` is NOT terminal while `desiredState == running` (the platform transitions `created → stopped → starting → running` during initial deploy). `data-app create` accepts `--git-pat-env VAR` (recommended; no argv leak), `--git-pat-file PATH`, or `--git-pat-encrypted KBC::Project...` (must be encrypted under THIS project's KMS — ciphertext does not cross projects).", diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index d9d7974c..8ff13a9e 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -51,6 +51,7 @@ from .services.member_service import MemberService from .services.org_service import OrgService from .services.project_service import ProjectService +from .services.repo_validate_service import RepoValidateService from .services.schedule_service import ScheduleService from .services.sharing_service import SharingService from .services.storage_service import StorageService @@ -315,6 +316,7 @@ def main( schedule_service = ScheduleService(config_store=config_store) workspace_service = WorkspaceService(config_store=config_store) data_app_service = DataAppService(config_store=config_store) + repo_validate_service = RepoValidateService(config_store=config_store) kai_service = KaiService(config_store=config_store) doctor_service = DoctorService(config_store=config_store, mcp_service=mcp_service) version_service = VersionService() @@ -370,6 +372,7 @@ def main( ctx.obj["schedule_service"] = schedule_service ctx.obj["workspace_service"] = workspace_service ctx.obj["data_app_service"] = data_app_service + ctx.obj["repo_validate_service"] = repo_validate_service ctx.obj["kai_service"] = kai_service ctx.obj["doctor_service"] = doctor_service ctx.obj["version_service"] = version_service diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 3921487a..bad48673 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -634,6 +634,49 @@ persisted, never logged. Password is auto-generated at create time and CANNOT be rotated -- delete and recreate the app to mint a new one. + kbagent data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE' + [--secret '#KEY2=VALUE2' ...] [--secrets-file PATH] [--branch ID] + [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next] + Encrypt and write '#'-prefixed secrets into parameters.dataApp.secrets. + Per-project KMS via the Encryption API; ciphertext does not cross + projects (writeup §8). Read-modify-write at the service layer to + preserve sibling keys; never use Storage merge=True for nested edits. + The runtime exposes each key as an env var with '#' stripped, '-' + replaced with '_', uppercased ('#my-api-key' -> 'MY_API_KEY'). + Adding a secret bumps the Storage version; the running container + keeps the OLD config until the next 'kbagent data-app deploy'. + + kbagent data-app secrets-list --project ALIAS --app-id ID [--branch ID] + [--show-fingerprint] + List the keys in parameters.dataApp.secrets with derived runtime + env-var names. Never echoes encrypted ciphertext in full and never + decrypts. --show-fingerprint includes a short fingerprint per key. + + kbagent data-app secrets-get --project ALIAS --app-id ID --key '#KEY' + [--branch ID] + Show metadata for ONE secret. NEVER echoes the decrypted value -- + the Encryption API has no decrypt endpoint and the CLI cannot + decrypt. NOT_FOUND on absent key; never enumerates sibling keys. + + kbagent data-app secrets-remove --project ALIAS --app-id ID --key '#KEY' + [--key '#KEY2' ...] [--branch ID] [--yes] [--dry-run] + Remove one or more secrets. Idempotent (missing keys -> exit 0, + removed: 0). Destructive: a removal can break the running app at + next deploy if it depends on the value. Confirmation prompt unless + --yes or --json. + + kbagent data-app validate-repo --git-repo URL [--git-branch BRANCH] + [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] + [--type python-js] [--strict] + Pre-flight check that a git repo follows the Keboola data-app + Golden Rule (https://help.keboola.com/data-apps/python-js/). Walks + the repo via GitHub Contents + Trees API (<=5 calls -- 1 tree + + up to 4 contents -- regardless of + repo size); each check emits BLOCKING / WARN / OK with a help-doc + citation. --type currently restricted to python-js; streamlit / + pure-Python / R / Node-only follow-up. --strict treats WARNs as + failures (exit 1). + ### Project Sync kbagent sync init --project ALIAS [--directory DIR] [--git-branching] [--adopt-existing] diff --git a/src/keboola_agent_cli/commands/data_app.py b/src/keboola_agent_cli/commands/data_app.py index 52022808..08ed3d56 100644 --- a/src/keboola_agent_cli/commands/data_app.py +++ b/src/keboola_agent_cli/commands/data_app.py @@ -10,6 +10,7 @@ from __future__ import annotations +import json import os from pathlib import Path @@ -28,6 +29,11 @@ should_hint, ) +# Canonical Keboola help-doc references appended to each --help epilog so +# operators have a one-click path to the rule a flag enforces. +_REF_PYTHON_JS = "https://help.keboola.com/data-apps/python-js/" +_REF_STORAGE_ACCESS = "https://help.keboola.com/data-apps/storage-access/" + data_app_app = typer.Typer(help="Keboola data-app lifecycle (create, deploy, manage)") @@ -104,7 +110,12 @@ def data_app_list( try: result = service.list_data_apps(aliases=project, branch_id=branch) except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) raise typer.Exit(code=map_error_to_exit_code(exc)) from None except ConfigError as exc: formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) @@ -142,7 +153,12 @@ def data_app_detail( try: result = service.get_data_app(alias=project, app_id=app_id, branch_id=branch) except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) raise typer.Exit(code=map_error_to_exit_code(exc)) from None except ConfigError as exc: formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) @@ -368,7 +384,12 @@ def data_app_create( dry_run=dry_run, ) except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) raise typer.Exit(code=map_error_to_exit_code(exc)) from None except ConfigError as exc: formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) @@ -418,7 +439,12 @@ def _run_lifecycle( try: result = method(**kwargs) except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) raise typer.Exit(code=map_error_to_exit_code(exc)) from None except ConfigError as exc: formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) @@ -558,7 +584,12 @@ def data_app_delete( try: result = service.delete_data_app(alias=project, app_id=app_id) except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) raise typer.Exit(code=map_error_to_exit_code(exc)) from None except ConfigError as exc: formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) @@ -600,7 +631,12 @@ def data_app_password( alias=project, app_id=app_id, manage_token=manage_token ) except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) raise typer.Exit(code=map_error_to_exit_code(exc)) from None except ConfigError as exc: formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) @@ -613,3 +649,590 @@ def data_app_password( c.print(f"\n[bold yellow]Password:[/bold yellow] {d['password']}"), ), ) + + +# --------------------------------------------------------------------------- +# data-app secrets-{set|list|get|remove} -- flat commands matching the +# existing branch.metadata-* / config.variables-* pattern. Subgroups under +# Typer subgroups conflict with the flat permission/hint registry. +# --------------------------------------------------------------------------- + + +def _parse_secret_arg(arg: str) -> tuple[str, str]: + """Split ``#KEY=VALUE`` into ``(key, value)``. + + The value may contain ``=``; only the FIRST ``=`` is the separator. + """ + if "=" not in arg: + raise typer.BadParameter( + f"Expected '#KEY=VALUE'; got {arg!r} (no '=' separator).", + param_hint="--secret", + ) + key, _, value = arg.partition("=") + if not key: + raise typer.BadParameter( + f"Empty secret key in {arg!r}; expected '#KEY=VALUE'.", + param_hint="--secret", + ) + return key, value + + +def _read_secrets_file(path: Path) -> dict[str, str]: + try: + text = path.read_text(encoding="utf-8") + except OSError as exc: + raise typer.BadParameter( + f"Cannot read secrets file {path}: {exc}", + param_hint="--secrets-file", + ) from exc + try: + parsed = json.loads(text) + except json.JSONDecodeError as exc: + raise typer.BadParameter( + f"Secrets file {path} is not valid JSON: {exc}", + param_hint="--secrets-file", + ) from exc + if not isinstance(parsed, dict): + raise typer.BadParameter( + f"Secrets file {path} must be a JSON object mapping #KEY -> value.", + param_hint="--secrets-file", + ) + out: dict[str, str] = {} + for key, value in parsed.items(): + if not isinstance(key, str) or not isinstance(value, str): + raise typer.BadParameter( + f"Secrets file {path} contains non-string entry for {key!r}.", + param_hint="--secrets-file", + ) + out[key] = value + if not out: + raise typer.BadParameter( + f"Secrets file {path} is empty.", + param_hint="--secrets-file", + ) + return out + + +@data_app_app.command("secrets-set") +def data_app_secrets_set( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), + secret: list[str] | None = typer.Option( + None, + "--secret", + help=( + "One or more '#KEY=VALUE' plaintext entries. Repeatable. " + "Mutually exclusive with --secrets-file." + ), + ), + secrets_file: Path | None = typer.Option( + None, + "--secrets-file", + help="Path to a JSON file mapping '#KEY' -> 'plaintext value'.", + exists=True, + readable=True, + dir_okay=False, + ), + branch: int | None = typer.Option( + None, + "--branch", + help="Storage branch ID for the linked config (defaults to production).", + ), + allow_plaintext_on_encrypt_failure: bool = typer.Option( + False, + "--allow-plaintext-on-encrypt-failure", + help=( + "Bootstrap/debug only: write the value as-is if the Encryption API " + "did not return a project-scoped ciphertext. NEVER use in production." + ), + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Show the encryption request and Storage PUT body without making either call.", + ), + no_hint_next: bool = typer.Option( + False, + "--no-hint-next", + help="Suppress the 'now run kbagent data-app deploy' hint in the output.", + ), +) -> None: + """Encrypt and write app-runtime secrets to the linked Storage config. + + The '#'-prefix is required on every key (Keboola encryption convention). + The runtime exposes each secret as an env var with '#' stripped, '-' + replaced with '_', and uppercased ('#my-api-key' -> 'MY_API_KEY'). + + The command never auto-deploys; the running container keeps the old + config until the next 'kbagent data-app deploy' call. + + Reference: https://help.keboola.com/data-apps/python-js/ + """ + + if should_hint(ctx): + emit_hint( + ctx, + "data-app.secrets-set", + project=project, + app_id=app_id, + secret=secret, + branch=branch, + ) + return + + formatter = get_formatter(ctx) + service = get_service(ctx, "data_app_service") + + if secret and secrets_file: + formatter.error( + message=("--secret and --secrets-file are mutually exclusive; pick one input mode."), + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) from None + + if not secret and not secrets_file: + formatter.error( + message=("Provide at least one --secret '#KEY=VALUE' or --secrets-file PATH."), + error_code=ErrorCode.MISSING_PARAMETER, + ) + raise typer.Exit(code=2) from None + + secrets_map: dict[str, str] = {} + if secret: + for entry in secret: + try: + key, value = _parse_secret_arg(entry) + except typer.BadParameter as exc: + formatter.error( + message=str(exc), + error_code=ErrorCode.DATA_APP_INVALID_SECRET, + ) + raise typer.Exit(code=2) from None + secrets_map[key] = value + if secrets_file: + try: + secrets_map.update(_read_secrets_file(secrets_file)) + except typer.BadParameter as exc: + formatter.error( + message=str(exc), + error_code=ErrorCode.DATA_APP_INVALID_SECRET, + ) + raise typer.Exit(code=2) from None + + try: + result = service.set_data_app_secrets( + alias=project, + app_id=app_id, + secrets=secrets_map, + branch_id=branch, + allow_plaintext_on_encrypt_failure=allow_plaintext_on_encrypt_failure, + dry_run=dry_run, + ) + except KeboolaApiError as exc: + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + # Reserved-name shadowing -- emit stderr WARN per collision so a + # script piping stdout to a JSON parser is unaffected. + shadowed = result.get("shadowed_by_runtime", []) + if shadowed and not formatter.json_mode: + for env_var in shadowed: + formatter.err_console.print( + f"[yellow]Warning:[/yellow] {env_var} is auto-injected by the data-app " + f"runtime; the platform value silently shadows yours. See {_REF_STORAGE_ACCESS}.", + style="yellow", + ) + + if no_hint_next and isinstance(result, dict): + result.pop("next_step", None) + + formatter.output( + result, + lambda c, d: c.print(f"[bold green]Success:[/bold green] {d['message']}"), + ) + if not no_hint_next and not formatter.json_mode and result.get("next_step"): + formatter.console.print(f"[dim]Next: {result['next_step']}[/dim]") + + +@data_app_app.command("secrets-list") +def data_app_secrets_list( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), + branch: int | None = typer.Option( + None, + "--branch", + help="Storage branch ID for the linked config (defaults to production).", + ), + show_fingerprint: bool = typer.Option( + False, + "--show-fingerprint", + help="Include a short ciphertext fingerprint per key. Default omits to keep --json safe to paste into tickets.", + ), +) -> None: + """List the keys in parameters.dataApp.secrets, with derived runtime env-var names. + + Never echoes the encrypted ciphertext in full and never decrypts. + + Reference: https://help.keboola.com/data-apps/python-js/ + """ + + if should_hint(ctx): + emit_hint( + ctx, + "data-app.secrets-list", + project=project, + app_id=app_id, + branch=branch, + ) + return + + formatter = get_formatter(ctx) + service = get_service(ctx, "data_app_service") + try: + result = service.list_data_app_secrets( + alias=project, + app_id=app_id, + branch_id=branch, + show_fingerprint=show_fingerprint, + ) + except KeboolaApiError as exc: + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + if formatter.json_mode: + formatter.output(result) + return + + if not result["secrets"]: + formatter.console.print("[dim]No secrets set on this data app.[/dim]") + return + formatter.console.print( + f"\n[bold]{result['count']} secret(s)[/bold] on data app " + f"[cyan]{result['id']}[/cyan] in [magenta]{result['project_alias']}[/magenta]:" + ) + for entry in result["secrets"]: + marker = ( + " [yellow](shadowed by runtime)[/yellow]" if entry.get("shadowed_by_runtime") else "" + ) + line = f" [bold]{entry['key']}[/bold] -> env [cyan]{entry['env_var']}[/cyan]{marker}" + if "fingerprint" in entry: + line += f" [dim]fingerprint={entry['fingerprint']} prefix={entry.get('encryption_prefix', '')}[/dim]" + formatter.console.print(line) + + +@data_app_app.command("secrets-get") +def data_app_secrets_get( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), + key: str = typer.Option(..., "--key", help="Secret key, including '#' prefix."), + branch: int | None = typer.Option( + None, + "--branch", + help="Storage branch ID for the linked config (defaults to production).", + ), +) -> None: + """Show metadata for ONE secret key. NEVER echoes the decrypted value. + + The Encryption API has no decrypt endpoint; the CLI cannot decrypt + even if asked. This command confirms presence + ciphertext metadata. + + Reference: https://help.keboola.com/data-apps/python-js/ + """ + + if should_hint(ctx): + emit_hint( + ctx, + "data-app.secrets-get", + project=project, + app_id=app_id, + key=key, + branch=branch, + ) + return + + formatter = get_formatter(ctx) + service = get_service(ctx, "data_app_service") + try: + result = service.get_data_app_secret( + alias=project, + app_id=app_id, + key=key, + branch_id=branch, + ) + except KeboolaApiError as exc: + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + if formatter.json_mode: + formatter.output(result) + return + formatter.console.print( + f"\n[bold]{result['key']}[/bold] -> env [cyan]{result['env_var']}[/cyan]" + ) + formatter.console.print( + f" [dim]fingerprint={result['fingerprint']} prefix={result['encryption_prefix']}[/dim]" + ) + if result.get("shadowed_by_runtime"): + # Same stdout/stderr-separation rationale as secrets-set: keep + # warnings off stdout so a script piping the metadata to a parser + # is unaffected. + formatter.err_console.print( + f" [yellow]Warning:[/yellow] {result['env_var']} is auto-injected by " + f"the data-app runtime; the platform value silently shadows yours. " + f"See {_REF_STORAGE_ACCESS}." + ) + + +@data_app_app.command("secrets-remove") +def data_app_secrets_remove( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), + key: list[str] = typer.Option( + ..., "--key", help="Secret key to remove (with '#' prefix). Repeatable." + ), + branch: int | None = typer.Option( + None, + "--branch", + help="Storage branch ID for the linked config (defaults to production).", + ), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt."), + dry_run: bool = typer.Option( + False, "--dry-run", help="Preview the Storage PUT body without making the call." + ), +) -> None: + """Remove one or more app-runtime secrets. Idempotent (missing keys are exit 0). + + A removal can break the running app at the next deploy if it relied on + the secret; the command flags this in the response and never auto-deploys. + + Reference: https://help.keboola.com/data-apps/python-js/ + """ + + if should_hint(ctx): + emit_hint( + ctx, + "data-app.secrets-remove", + project=project, + app_id=app_id, + key=key, + branch=branch, + ) + return + + formatter = get_formatter(ctx) + service = get_service(ctx, "data_app_service") + + if ( + not yes + and not formatter.json_mode + and not dry_run + and not typer.confirm( + f"Remove {len(key)} secret(s) from data app {app_id} in '{project}'? " + "This may break the app at next deploy if it depends on these values." + ) + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + + try: + result = service.remove_data_app_secrets( + alias=project, + app_id=app_id, + keys=key, + branch_id=branch, + dry_run=dry_run, + ) + except KeboolaApiError as exc: + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + formatter.output( + result, + lambda c, d: c.print(f"[bold green]Success:[/bold green] {d['message']}"), + ) + + +# --------------------------------------------------------------------------- +# data-app validate-repo +# --------------------------------------------------------------------------- + + +@data_app_app.command("validate-repo") +def data_app_validate_repo( + ctx: typer.Context, + git_repo: str = typer.Option( + ..., "--git-repo", help="GitHub repo URL (https://github.com/owner/repo)." + ), + git_branch: str = typer.Option( + "main", "--git-branch", help="Git ref to validate (default: main)." + ), + git_public: bool = typer.Option( + True, + "--git-public/--no-git-public", + help="Public repo (no PAT). Use --no-git-public for private repos and pass --git-pat-env / --git-pat-file.", + ), + git_pat_env: str | None = typer.Option( + None, + "--git-pat-env", + help="Read GitHub PAT from this env var (recommended; no argv leak).", + ), + git_pat_file: Path | None = typer.Option( + None, + "--git-pat-file", + help="Read GitHub PAT from this file.", + exists=True, + readable=True, + dir_okay=False, + ), + type_: str = typer.Option( + "python-js", + "--type", + help="Repo layout to validate against. Currently only 'python-js' is supported; other types tracked as follow-up.", + ), + strict: bool = typer.Option( + False, "--strict", help="Treat WARN findings as failures (exit 1)." + ), +) -> None: + """Pre-flight check that a git repo follows the Keboola data-app Golden Rule. + + Walks the repo via GitHub Contents + Trees API and validates the + documented structure (keboola-config/ tree, pyproject.toml, no + 'pip install' in setup.sh, requires-python at-or-below the runtime + pin, etc.). Each check emits BLOCKING / WARN / OK with a citation + to the help-doc anchor that defines the rule. + + Reference: https://help.keboola.com/data-apps/python-js/ + """ + + if should_hint(ctx): + emit_hint( + ctx, + "data-app.validate-repo", + git_repo=git_repo, + git_branch=git_branch, + type_=type_, + ) + return + + formatter = get_formatter(ctx) + service = get_service(ctx, "repo_validate_service") + + if git_pat_env and git_pat_file: + formatter.error( + message="--git-pat-env and --git-pat-file are mutually exclusive.", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) from None + + pat_supplied = git_pat_env is not None or git_pat_file is not None + if pat_supplied and git_public: + # The default --git-public means "anonymous fetch"; sending a PAT + # with it is a contradiction (the resulting 404 would lead to a + # 'private repo -- pass --git-pat-env' message recommending the + # flag the user already passed). Fail loud instead. + formatter.error( + message=( + "--git-pat-env / --git-pat-file requires --no-git-public; the " + "default --git-public flag opts into an anonymous fetch and " + "would silently drop the PAT." + ), + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) from None + + pat: str | None = None + if git_pat_env is not None: + pat = _read_pat_from_env(git_pat_env) + elif git_pat_file is not None: + pat = _read_pat_from_file(git_pat_file) + + try: + result = service.validate_repo( + git_repo=git_repo, + git_branch=git_branch, + git_public=git_public, + git_pat=pat, + type_=type_, + strict=strict, + ) + except KeboolaApiError as exc: + formatter.error( + message=exc.message, + error_code=exc.error_code, + retryable=exc.retryable, + details=exc.details, + ) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + if formatter.json_mode: + formatter.output(result) + else: + verdict_colour = ( + "red" + if result["verdict"] == "BLOCKING" + else "yellow" + if result["verdict"] == "WARN" + else "green" + ) + formatter.console.print( + f"\n[bold {verdict_colour}]{result['verdict']}[/bold {verdict_colour}] " + f"-- {result['blocking_count']} BLOCKING, " + f"{result['warn_count']} WARN, {result['ok_count']} OK" + ) + for check in result["checks"]: + sev = check["severity"] + colour = "red" if sev == "BLOCKING" else "yellow" if sev == "WARN" else "green" + line = f" [{colour}]{sev:<8}[/{colour}] {check['name']}" + if check.get("message"): + line += f" -- {check['message']}" + formatter.console.print(line) + formatter.console.print(f"\n[dim]{result['message']}[/dim]") + + if result.get("is_failure"): + # validate-repo's own exit code: BLOCKING (or strict-WARN) -> 1. + # We bypass the structured-error formatter because validate-repo + # output is itself the structured error envelope. + if formatter.json_mode: + # JSON envelope is already printed; just exit non-zero. + raise typer.Exit(code=1) + # Human mode: the verdict line above conveyed the failure. + raise typer.Exit(code=1) diff --git a/src/keboola_agent_cli/errors.py b/src/keboola_agent_cli/errors.py index 1c307acc..9948c537 100644 --- a/src/keboola_agent_cli/errors.py +++ b/src/keboola_agent_cli/errors.py @@ -98,6 +98,11 @@ class ErrorCode(StrEnum): DATA_APP_DEPLOY_TIMEOUT = "DATA_APP_DEPLOY_TIMEOUT" DATA_APP_INVALID_GIT = "DATA_APP_INVALID_GIT" + # Data apps - secrets + validate-repo (new in 0.28.0) + DATA_APP_INVALID_SECRET = "DATA_APP_INVALID_SECRET" + DATA_APP_INVALID_REPO = "DATA_APP_INVALID_REPO" + DATA_APP_REPO_VALIDATION_BLOCKING = "DATA_APP_REPO_VALIDATION_BLOCKING" + def mask_token(token: str) -> str: """Mask a Keboola Storage API token for safe display. diff --git a/src/keboola_agent_cli/hints/definitions/data_app.py b/src/keboola_agent_cli/hints/definitions/data_app.py index 1581d677..27a5e6f7 100644 --- a/src/keboola_agent_cli/hints/definitions/data_app.py +++ b/src/keboola_agent_cli/hints/definitions/data_app.py @@ -401,3 +401,265 @@ ], ) ) + + +# ── data-app secrets set ────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.secrets-set", + description=( + "Encrypt and write '#'-prefixed secrets to the linked Storage " + "config. Read-modify-write so unrelated keys under " + "parameters.dataApp are preserved bit-identical." + ), + steps=[ + HintStep( + comment=( + "Resolve configId via /apps/{id}, encrypt every plaintext " + "value under THIS project's KMS, then PUT the full config " + "with the new secrets sub-dict. Storage merge=True is " + "shallow at the top level only -- relying on it would " + "clobber sibling keys; read-modify-write is the only " + "correct path." + ), + client=ClientCall( + method="get_config_detail", + args={ + "component_id": '"keboola.data-apps"', + "config_id": "{config_id}", + "branch_id": "{branch}", + }, + client_type="storage", + result_var="current_config", + result_hint="dict", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="set_data_app_secrets", + args={ + "alias": "{project}", + "app_id": "{app_id}", + "secrets": "{secret}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "Encryption is per-project KMS -- ciphertext does not cross " + "projects (writeup §8). Pre-encrypted KBC::* values are rejected " + "by --secret to prevent stale-ciphertext footguns; pass them via " + "--secrets-file for advanced flows.", + "The runtime exposes each key as an env var with '#' stripped, " + "'-' replaced with '_', and uppercased ('#my-api-key' -> " + "'MY_API_KEY'). Setting a key whose env-var name collides with " + "KBC_TOKEN / KBC_URL is silently shadowed at runtime.", + "Adding a secret bumps the Storage version but the running " + "container keeps the OLD config until 'data-app deploy' runs.", + ], + ) +) + + +# ── data-app secrets list ───────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.secrets-list", + description=( + "List the keys in parameters.dataApp.secrets with their derived " + "runtime env-var names. Never echoes the encrypted ciphertext " + "and never decrypts." + ), + steps=[ + HintStep( + comment=( + "Resolve configId via /apps/{id}, then GET the linked " + "Storage config and read parameters.dataApp.secrets." + ), + client=ClientCall( + method="get_config_detail", + args={ + "component_id": '"keboola.data-apps"', + "config_id": "{config_id}", + "branch_id": "{branch}", + }, + client_type="storage", + result_var="current_config", + result_hint="dict", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="list_data_app_secrets", + args={ + "alias": "{project}", + "app_id": "{app_id}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "Default output omits ciphertext fingerprint; pass --show-fingerprint to include it.", + ], + ) +) + + +# ── data-app secrets get ────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.secrets-get", + description=( + "Show metadata for ONE secret key. NEVER echoes the decrypted " + "value -- the Encryption API is one-way and the CLI does not " + "decrypt under any branch." + ), + steps=[ + HintStep( + comment=( + "GET the Storage config, look up one key in " + "parameters.dataApp.secrets, return metadata only. The " + "ciphertext fingerprint is the first 8 chars of the " + "encrypted payload after the KBC::* prefix." + ), + client=ClientCall( + method="get_config_detail", + args={ + "component_id": '"keboola.data-apps"', + "config_id": "{config_id}", + "branch_id": "{branch}", + }, + client_type="storage", + result_var="current_config", + result_hint="dict", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="get_data_app_secret", + args={ + "alias": "{project}", + "app_id": "{app_id}", + "key": "{key}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "NOT_FOUND on absent key does not enumerate sibling keys -- " + "avoid leaking neighbour presence to a caller that knows only " + "one key's name.", + ], + ) +) + + +# ── data-app secrets remove ─────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.secrets-remove", + description=( + "Remove one or more app-runtime secrets. Idempotent: removing " + "a non-existent key is exit 0 with removed=0." + ), + steps=[ + HintStep( + comment=( + "Resolve configId, GET the Storage config, drop the " + "named keys from parameters.dataApp.secrets, PUT the " + "full body back. Same read-modify-write contract as " + "secrets set." + ), + client=ClientCall( + method="get_config_detail", + args={ + "component_id": '"keboola.data-apps"', + "config_id": "{config_id}", + "branch_id": "{branch}", + }, + client_type="storage", + result_var="current_config", + result_hint="dict", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="remove_data_app_secrets", + args={ + "alias": "{project}", + "app_id": "{app_id}", + "keys": "{key}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "Destructive operation: removing a secret can break the running " + "app at the next deploy if it depends on the value. Run " + "`data-app deploy` after the remove to roll the new config.", + ], + ) +) + + +# ── data-app validate-repo ──────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.validate-repo", + description=( + "Pre-flight check that a git repo follows the Keboola data-app " + "Golden Rule. Walks the repo via the GitHub Contents + Trees " + "API; emits BLOCKING / WARN / OK with help-doc citations." + ), + steps=[ + HintStep( + comment=( + "ONE GET /repos/{owner}/{repo}/git/trees/{ref}?" + "recursive=1 to walk the tree, then up to 3 GET " + "/repos/.../contents/{path} for setup.sh / " + "pyproject.toml / nginx-app port match. Total <=5 " + "GitHub calls regardless of repo size. validate-repo " + "uses GitHubContentsClient (not a Keboola client); " + "prefer the service-layer snippet." + ), + client=ClientCall( + method="get_tree_recursive", + args={ + "owner": '""', + "repo": '""', + "ref": "{git_branch}", + }, + client_type="github", + result_var="tree", + result_hint="dict", + ), + service=ServiceCall( + service_class="RepoValidateService", + service_module="repo_validate_service", + method="validate_repo", + args={ + "git_repo": "{git_repo}", + "git_branch": "{git_branch}", + "git_public": "{git_public}", + "type_": "{type_}", + }, + ), + ), + ], + notes=[ + "Public GitHub Contents API is 60/hour unauth; pass " + "--git-pat-env to use a PAT and raise the limit to 5000/hour.", + "Currently restricted to --type python-js. streamlit / python / " + "r layouts tracked as a follow-up.", + ], + ) +) diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index e3f34b76..8fb85a77 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -107,6 +107,12 @@ "data-app.start": "write", "data-app.stop": "write", "data-app.delete": "destructive", + # Data apps - secrets + validate-repo (new in 0.28.0) + "data-app.secrets-set": "write", + "data-app.secrets-list": "read", + "data-app.secrets-get": "read", + "data-app.secrets-remove": "destructive", + "data-app.validate-repo": "read", # Storage browsing "storage.buckets": "read", "storage.bucket-detail": "read", diff --git a/src/keboola_agent_cli/services/data_app_service.py b/src/keboola_agent_cli/services/data_app_service.py index c7afe74a..9e4c3372 100644 --- a/src/keboola_agent_cli/services/data_app_service.py +++ b/src/keboola_agent_cli/services/data_app_service.py @@ -118,6 +118,63 @@ def _has_control_chars(value: str, *, allow_whitespace: bool = False) -> bool: "git://", ) +# Secret-key shape accepted under ``parameters.dataApp.secrets``. Must start +# with ``#`` (Keboola encryption convention); the rest must form a valid +# environment-variable identifier after the runtime translation rule +# (uppercased, ``-`` -> ``_``, ``#`` stripped). Cap at 64 chars so the +# derived env var stays under typical shell limits. +SECRET_KEY_PATTERN = re.compile(r"^#[A-Za-z][A-Za-z0-9_-]{0,63}$") + +# Env vars the data-app runtime auto-injects. Setting a secret whose +# derived env-var name collides with one of these is silently shadowed +# at runtime by the platform value. See storage-access canon at +# https://help.keboola.com/data-apps/storage-access/. +# +# TODO(0.28.x): verify exhaustive list against running data-app env in +# follow-up. The runtime almost certainly injects more (BRANCH_ID, +# QUERY_SERVICE_URL, KBC_WORKSPACE_MANIFEST_PATH appear in the +# storage-access page; others may exist) but the canon-documented floor +# is KBC_TOKEN + KBC_URL. Expanding this set adds WARNs that are less +# likely to be false positives once verified live. +RESERVED_RUNTIME_ENV_VARS: frozenset[str] = frozenset( + { + "KBC_TOKEN", + "KBC_URL", + } +) + + +def _derive_runtime_env_var_name(secret_key: str) -> str: + """Translate a ``#``-prefixed secret key into the runtime env-var name. + + Rule from help.keboola.com/data-apps/python-js/: strip the leading + ``#``, replace ``-`` with ``_``, uppercase. Examples (verbatim from + the help canon): + + - ``#KBC_TOKEN`` -> ``KBC_TOKEN`` + - ``#my-custom-var`` -> ``MY_CUSTOM_VAR`` + """ + stripped = secret_key.lstrip("#") + return stripped.replace("-", "_").upper() + + +def _secret_fingerprint(ciphertext: str) -> str: + """First 8 chars of the ciphertext payload after the ``KBC::*::`` prefix. + + The full ciphertext is not a secret in the cryptographic sense (it + can only be decrypted by the project's KMS), but echoing it in full + invites copy-paste leakage into tickets and chat. The fingerprint is + enough to compare two ciphertexts without exposing the payload. + Returns empty string for non-ciphertext input. + """ + if not isinstance(ciphertext, str): + return "" + for prefix in ENCRYPTED_PASSWORD_PREFIXES: + if ciphertext.startswith(prefix): + payload = ciphertext[len(prefix) :] + return payload[:8] + return "" + def _build_simple_auth_block() -> dict[str, Any]: """Authorization block for password-gated apps (writeup §11.2).""" @@ -136,6 +193,59 @@ def _build_simple_auth_block() -> dict[str, Any]: } +def _build_public_auth_block() -> dict[str, Any]: + """Authorization block for publicly-accessible apps (no auth gate). + + Mirrors the kbc-ui ``noneProxyAuthorization`` constant exactly. + Authoritative source — the public backend validator at + ``keboola/job-queue-job-configuration`` + ``src/JobDefinition/Configuration/Authorization/AppProxyDefinition.php`` + (when ``auth_required=false``, ``auth`` MUST NOT be set; see + https://github.com/keboola/job-queue-job-configuration). The + ``keboola/ui`` repo (private; Keboola org members only) corroborates: + its ``apps/kbc-ui/src/scripts/modules/data-apps/constants.ts`` + exports this exact shape as ``noneProxyAuthorization`` for the + "None" UI option. + + Without this block, ``--auth public`` shipped in 0.27.0 wrote no + ``authorization`` key at all -- the Keboola app-proxy refused to + route traffic and the UI's "Authentication Type" selector showed + blank. Fixed in 0.28.0. + """ + return { + "app_proxy": { + "auth_providers": [], + "auth_rules": [ + { + "type": "pathPrefix", + "value": "/", + "auth_required": False, + } + ], + }, + } + + +def _auth_block_for(auth: str) -> dict[str, Any]: + """Dispatch on the validated --auth value. + + The validator at :meth:`DataAppService._validate_create_inputs` + rejects anything other than ``password`` / ``public`` at the service + boundary, so this code path should only ever see those two values in + production. We raise loudly on an unexpected value rather than + silently writing no ``authorization`` block (the v0.27.0 bug this + helper exists to prevent — see the (since v0.28.0) gotcha entry). + """ + if auth == "password": + return _build_simple_auth_block() + if auth == "public": + return _build_public_auth_block() + raise ValueError( + f"_auth_block_for missing dispatch for {auth!r}; " + "_validate_create_inputs should have rejected this upstream." + ) + + def _redact_secret(value: Any) -> Any: """Replace encrypted ``#`` values with a placeholder for human output.""" if isinstance(value, str) and value.startswith("KBC::"): @@ -151,6 +261,18 @@ def _redact_git_block(git: dict[str, Any]) -> dict[str, Any]: return redacted +def _redact_secrets_block(secrets: dict[str, Any]) -> dict[str, Any]: + """Return a copy of ``parameters.dataApp.secrets`` with each ciphertext redacted. + + Used by ``get_data_app`` so the ``raw.storage_config`` echo cannot + leak any secret's encrypted value into ``--json`` output. Same + defence-in-depth rationale as :func:`_redact_git_block`. + """ + if not isinstance(secrets, dict): + return secrets + return {key: _redact_secret(value) for key, value in secrets.items()} + + def _redact_storage_config(storage_config: dict[str, Any]) -> dict[str, Any]: """Deep-copy the Storage config dict and redact any nested encrypted PAT. @@ -176,6 +298,9 @@ def _redact_storage_config(storage_config: dict[str, Any]) -> dict[str, Any]: git = data_app.get("git") if isinstance(git, dict): data_app["git"] = _redact_git_block(git) + secrets = data_app.get("secrets") + if isinstance(secrets, dict): + data_app["secrets"] = _redact_secrets_block(secrets) parameters["dataApp"] = data_app configuration["parameters"] = parameters redacted["configuration"] = configuration @@ -437,8 +562,7 @@ def create_data_app( "dataApp": {"slug": slug}, }, } - if auth == "password": - initial_config["authorization"] = _build_simple_auth_block() + initial_config["authorization"] = _auth_block_for(auth) shell = ds_client.create_app( type_=type_, @@ -773,10 +897,553 @@ def get_data_app_password( ), } + # ------------------------------------------------------------------ + # Secrets lifecycle (parameters.dataApp.secrets in the Storage config) + # ------------------------------------------------------------------ + + def _load_data_app_storage_config( + self, + *, + ds_client: DataScienceClient, + storage_client: Any, + app_id: str, + branch_id: int | None, + ) -> tuple[str, dict[str, Any], dict[str, Any]]: + """Resolve ``configId`` from the Data Science app and load the Storage config. + + Returns ``(config_id, storage_envelope, body)`` where ``body`` is a + deep-copy of ``storage_envelope.configuration`` so callers can + mutate it freely. Mirrors the pattern at + :meth:`deploy_data_app` (data_app_service.py:586-594). + """ + app = ds_client.get_app(app_id) + config_id = str(app.get("configId") or "") + if not config_id: + raise KeboolaApiError( + message=f"Data app {app_id} has no associated configId", + status_code=500, + error_code=ErrorCode.API_ERROR, + retryable=False, + ) + envelope = storage_client.get_config_detail( + DATA_APP_COMPONENT_ID, config_id, branch_id=branch_id + ) + if not isinstance(envelope, dict): + envelope = {} + configuration = envelope.get("configuration") + if not isinstance(configuration, dict): + configuration = {} + # Deep-copy so caller mutations don't reach the cached upstream. + body = json.loads(json.dumps(configuration)) + return config_id, envelope, body + + def _read_secrets_block(self, body: dict[str, Any]) -> dict[str, str]: + """Return ``parameters.dataApp.secrets`` from a config body, or ``{}``.""" + if not isinstance(body, dict): + return {} + params = body.get("parameters") + if not isinstance(params, dict): + return {} + data_app = params.get("dataApp") + if not isinstance(data_app, dict): + return {} + secrets = data_app.get("secrets") + return dict(secrets) if isinstance(secrets, dict) else {} + + def set_data_app_secrets( + self, + *, + alias: str, + app_id: str, + secrets: dict[str, str], + branch_id: int | None = None, + allow_plaintext_on_encrypt_failure: bool = False, + dry_run: bool = False, + ) -> dict[str, Any]: + """Encrypt and write ``#``-prefixed secrets to the linked Storage config. + + Read-modify-write at the service layer. The Storage API's + ``configuration`` field is a full-document overwrite; relying on + Storage merge to preserve nested siblings under + ``parameters.dataApp.secrets`` would clobber unrelated keys (the + merge is shallow at the top level only). We GET the full config, + modify the secrets sub-dict in place, and PUT the unchanged + remainder + the new secrets back. + + Fail-closed: any encryption failure aborts before Storage is + touched. ``allow_plaintext_on_encrypt_failure`` is bootstrap/debug + only and emits a stderr warning when used. + """ + if not secrets: + raise KeboolaApiError( + message="At least one --secret '#KEY=VALUE' is required.", + status_code=0, + error_code=ErrorCode.DATA_APP_INVALID_SECRET, + retryable=False, + ) + + # Validate every key + value at the service boundary; we don't + # trust the command layer to have caught everything. + validated: dict[str, str] = {} + for key, value in secrets.items(): + self._validate_secret_key(key) + if not isinstance(value, str): + raise KeboolaApiError( + message=(f"Secret '{key}' value must be a string; got {type(value).__name__}."), + status_code=0, + error_code=ErrorCode.DATA_APP_INVALID_SECRET, + retryable=False, + ) + if value.startswith("KBC::"): + raise KeboolaApiError( + message=( + f"Secret '{key}' value starts with 'KBC::', which suggests an " + "already-encrypted ciphertext. The --secret flag expects " + "plaintext; pass pre-encrypted values via --secrets-file or " + "re-encrypt under THIS project's KMS via " + "`kbagent encrypt values --component-id keboola.data-apps`." + ), + status_code=0, + error_code=ErrorCode.DATA_APP_INVALID_SECRET, + retryable=False, + ) + validated[key] = value + + # Reserved-name warnings -- WARN (not BLOCKING). The platform + # silently shadows colliding env vars at runtime. We still write + # the secret so the user can recover by removing it, but we + # surface the collision in the response. + shadowed: list[str] = sorted( + _derive_runtime_env_var_name(key) + for key in validated + if _derive_runtime_env_var_name(key) in RESERVED_RUNTIME_ENV_VARS + ) + + projects = self.resolve_projects([alias]) + project = projects[alias] + ds_client = self._ds_client_factory(project.stack_url, project.token) + storage_client = self._client_factory(project.stack_url, project.token) + + try: + config_id, envelope, current_body = self._load_data_app_storage_config( + ds_client=ds_client, + storage_client=storage_client, + app_id=str(app_id), + branch_id=branch_id, + ) + existing_secrets = self._read_secrets_block(current_body) + + unchanged = sorted( + _derive_runtime_env_var_name(k) for k in existing_secrets if k not in validated + ) + + if dry_run: + preview_secrets = dict(existing_secrets) + for key in validated: + preview_secrets[key] = "" + preview_body = self._merge_secrets_into_body(current_body, preview_secrets) + return { + "dry_run": True, + "project_alias": alias, + "id": str(app_id), + "config_id": config_id, + "secrets_set": sorted(_derive_runtime_env_var_name(k) for k in validated), + "secrets_unchanged": unchanged, + "shadowed_by_runtime": shadowed, + "encryption_request_keys": sorted(validated.keys()), + "put_storage_config_preview": _redact_storage_config( + {"configuration": preview_body} + ), + "message": ( + "Dry run -- no API calls made. Inspect the encryption " + "request keys and the proposed Storage PUT body above." + ), + } + + # Encrypt every plaintext value under THIS project's KMS. + try: + encrypted = self._encrypt_service.encrypt( + alias=alias, + component_id=DATA_APP_COMPONENT_ID, + input_data=validated, + ) + except ConfigError as exc: + raise KeboolaApiError( + message=f"Failed to prepare secrets for encryption: {exc.message}", + status_code=0, + error_code=ErrorCode.ENCRYPTION_FAILED, + retryable=False, + ) from exc + + # Validate every returned ciphertext starts with a project-scoped + # prefix. Mirror the fail-closed check from _build_git_block at + # data_app_service.py:1000-1008. + problems: list[str] = [] + for key, ciphertext in encrypted.items(): + if not isinstance(ciphertext, str) or not any( + ciphertext.startswith(p) for p in ENCRYPTED_PASSWORD_PREFIXES + ): + problems.append(key) + if problems and not allow_plaintext_on_encrypt_failure: + raise KeboolaApiError( + message=( + "Encryption API did not return a project-scoped ciphertext " + f"for key(s): {', '.join(sorted(problems))}. Refusing to " + "write plaintext to Storage. Re-run with " + "--allow-plaintext-on-encrypt-failure for bootstrap/debug ONLY." + ), + status_code=0, + error_code=ErrorCode.ENCRYPTION_FAILED, + retryable=False, + details={ + "project_alias": alias, + "failed_keys": sorted(problems), + }, + ) + if problems: + logger.warning( + "Encryption returned non-ciphertext for keys %s; writing anyway " + "because --allow-plaintext-on-encrypt-failure was set.", + sorted(problems), + ) + + # Read-modify-write: deep-copy of the body has the secrets sub-dict + # replaced; everything else is preserved bit-identical. + updated_secrets = dict(existing_secrets) + updated_secrets.update(encrypted) + new_body = self._merge_secrets_into_body(current_body, updated_secrets) + + put_response = storage_client.update_config( + component_id=DATA_APP_COMPONENT_ID, + config_id=config_id, + configuration=new_body, + change_description=( + f"Set {len(validated)} secret(s) via kbagent data-app secrets set" + ), + branch_id=branch_id, + ) + new_version = str(put_response.get("version", "") or "") + old_version = str(envelope.get("version", "") or "") + + secrets_set = sorted(_derive_runtime_env_var_name(k) for k in validated) + return { + "project_alias": alias, + "id": str(app_id), + "config_id": config_id, + "secrets_set": secrets_set, + "secrets_unchanged": unchanged, + "shadowed_by_runtime": shadowed, + "config_version_before": old_version, + "config_version_after": new_version, + "deploy_required": True, + "next_step": ( + f"kbagent data-app deploy --project {alias} --app-id {app_id} --wait" + ), + "message": ( + f"{len(secrets_set)} secret(s) encrypted and written. " + "The running container keeps the old config until you redeploy." + ), + } + finally: + ds_client.close() + storage_client.close() + + def list_data_app_secrets( + self, + *, + alias: str, + app_id: str, + branch_id: int | None = None, + show_fingerprint: bool = False, + ) -> dict[str, Any]: + """Return metadata for every ``#``-prefixed secret on the app's config. + + Never returns the encrypted ciphertext in full and never attempts + to decrypt. The Encryption API is one-way; decryption from the CLI + is impossible by design. + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + ds_client = self._ds_client_factory(project.stack_url, project.token) + storage_client = self._client_factory(project.stack_url, project.token) + try: + config_id, _envelope, body = self._load_data_app_storage_config( + ds_client=ds_client, + storage_client=storage_client, + app_id=str(app_id), + branch_id=branch_id, + ) + raw_secrets = self._read_secrets_block(body) + + entries: list[dict[str, Any]] = [] + for key in sorted(raw_secrets.keys()): + env_var = _derive_runtime_env_var_name(key) + entry: dict[str, Any] = { + "key": key, + "env_var": env_var, + "shadowed_by_runtime": env_var in RESERVED_RUNTIME_ENV_VARS, + } + if show_fingerprint: + ciphertext = raw_secrets[key] + entry["fingerprint"] = _secret_fingerprint(ciphertext) + entry["encryption_prefix"] = self._derive_encryption_prefix(ciphertext) + entries.append(entry) + + return { + "project_alias": alias, + "id": str(app_id), + "config_id": config_id, + "secrets": entries, + "count": len(entries), + } + finally: + ds_client.close() + storage_client.close() + + def get_data_app_secret( + self, + *, + alias: str, + app_id: str, + key: str, + branch_id: int | None = None, + ) -> dict[str, Any]: + """Return metadata for ONE secret. Never echoes the decrypted value. + + The decrypted plaintext NEVER appears in the return dict, in stderr, + in the log stream, or in the change description. The Encryption API + does not expose a decrypt endpoint; the CLI cannot decrypt even if + it wanted to. The metadata-only contract is the security boundary + and is asserted by the test suite. + """ + self._validate_secret_key(key) + + projects = self.resolve_projects([alias]) + project = projects[alias] + ds_client = self._ds_client_factory(project.stack_url, project.token) + storage_client = self._client_factory(project.stack_url, project.token) + try: + config_id, _envelope, body = self._load_data_app_storage_config( + ds_client=ds_client, + storage_client=storage_client, + app_id=str(app_id), + branch_id=branch_id, + ) + raw_secrets = self._read_secrets_block(body) + if key not in raw_secrets: + # Don't enumerate sibling keys -- avoid leaking neighbour + # presence to a caller who knows only this key's name. + raise KeboolaApiError( + message=( + f"Secret '{key}' not found on data app {app_id} in project '{alias}'." + ), + status_code=404, + error_code=ErrorCode.NOT_FOUND, + retryable=False, + ) + ciphertext = raw_secrets[key] + env_var = _derive_runtime_env_var_name(key) + return { + "project_alias": alias, + "id": str(app_id), + "config_id": config_id, + "key": key, + "env_var": env_var, + "shadowed_by_runtime": env_var in RESERVED_RUNTIME_ENV_VARS, + "fingerprint": _secret_fingerprint(ciphertext), + "encryption_prefix": self._derive_encryption_prefix(ciphertext), + "present": True, + "message": ( + f"Secret '{key}' is set on data app {app_id}. " + "Decrypted plaintext is NOT exposed by the CLI." + ), + } + finally: + ds_client.close() + storage_client.close() + + def remove_data_app_secrets( + self, + *, + alias: str, + app_id: str, + keys: list[str], + branch_id: int | None = None, + dry_run: bool = False, + ) -> dict[str, Any]: + """Remove one or more ``#``-prefixed secrets. Idempotent.""" + if not keys: + raise KeboolaApiError( + message="At least one --key '#KEY' is required.", + status_code=0, + error_code=ErrorCode.DATA_APP_INVALID_SECRET, + retryable=False, + ) + for key in keys: + self._validate_secret_key(key) + + projects = self.resolve_projects([alias]) + project = projects[alias] + ds_client = self._ds_client_factory(project.stack_url, project.token) + storage_client = self._client_factory(project.stack_url, project.token) + try: + config_id, envelope, body = self._load_data_app_storage_config( + ds_client=ds_client, + storage_client=storage_client, + app_id=str(app_id), + branch_id=branch_id, + ) + existing_secrets = self._read_secrets_block(body) + + removed = sorted(_derive_runtime_env_var_name(k) for k in keys if k in existing_secrets) + not_found = sorted( + _derive_runtime_env_var_name(k) for k in keys if k not in existing_secrets + ) + current_version = str(envelope.get("version", "") or "") + + if not removed: + # Idempotent: removing a non-existent key is success. + return { + "project_alias": alias, + "id": str(app_id), + "config_id": config_id, + "removed": [], + "not_found": not_found, + "config_version_before": current_version, + "config_version_after": current_version, + "deploy_required": False, + "message": ( + f"No matching secrets to remove on data app {app_id}. " + f"Keys not present: {', '.join(not_found) or ''}." + ), + } + + if dry_run: + preview = {k: v for k, v in existing_secrets.items() if k not in keys} + preview_body = self._merge_secrets_into_body(body, preview) + return { + "dry_run": True, + "project_alias": alias, + "id": str(app_id), + "config_id": config_id, + "to_remove": removed, + "not_found": not_found, + "put_storage_config_preview": _redact_storage_config( + {"configuration": preview_body} + ), + "message": ( + f"Dry run -- would remove {len(removed)} secret(s) and PUT " + "the body above. No API call made." + ), + } + + updated_secrets = {k: v for k, v in existing_secrets.items() if k not in keys} + new_body = self._merge_secrets_into_body(body, updated_secrets) + + put_response = storage_client.update_config( + component_id=DATA_APP_COMPONENT_ID, + config_id=config_id, + configuration=new_body, + change_description=( + f"Remove {len(removed)} secret(s) via kbagent data-app secrets remove" + ), + branch_id=branch_id, + ) + new_version = str(put_response.get("version", "") or "") + return { + "project_alias": alias, + "id": str(app_id), + "config_id": config_id, + "removed": removed, + "not_found": not_found, + "config_version_before": current_version, + "config_version_after": new_version, + "deploy_required": True, + "next_step": ( + f"kbagent data-app deploy --project {alias} --app-id {app_id} --wait" + ), + "message": ( + f"{len(removed)} secret(s) removed. The running container keeps " + "the old config until you redeploy." + ), + } + finally: + ds_client.close() + storage_client.close() + # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ + def _validate_secret_key(self, key: str) -> None: + """Reject any key that does not match SECRET_KEY_PATTERN. + + The check enforces the ``#``-prefix convention AND that the rest + of the key forms a valid env-var identifier after the runtime + translation rule. Service-boundary check; the command layer also + validates so the error surfaces with the friendliest exit code. + """ + if not isinstance(key, str) or not SECRET_KEY_PATTERN.match(key): + raise KeboolaApiError( + message=( + f"Invalid secret key '{key}'. Keys must start with '#' and " + "the rest must match [A-Za-z][A-Za-z0-9_-]{0,63} so the " + "derived runtime env-var name (uppercase, '-' to '_') is a " + "valid identifier." + ), + status_code=0, + error_code=ErrorCode.DATA_APP_INVALID_SECRET, + retryable=False, + ) + if _has_control_chars(key): + raise KeboolaApiError( + message=f"Secret key '{key}' contains disallowed control characters.", + status_code=0, + error_code=ErrorCode.DATA_APP_INVALID_SECRET, + retryable=False, + ) + + def _merge_secrets_into_body( + self, + body: dict[str, Any], + secrets: dict[str, str], + ) -> dict[str, Any]: + """Return a deep-copy of ``body`` with ``parameters.dataApp.secrets`` replaced. + + Every untouched sibling -- under ``parameters.dataApp.secrets`` + (none, since the whole sub-dict is replaced), under + ``parameters.dataApp`` (slug, git, id, etc.), under ``parameters`` + (everything else), and at the top level (``runtime``, + ``authorization``, ``storage``) -- is preserved bit-identical. + """ + new_body = json.loads(json.dumps(body)) if isinstance(body, dict) else {} + if not isinstance(new_body, dict): + new_body = {} + params = new_body.setdefault("parameters", {}) + if not isinstance(params, dict): + params = {} + new_body["parameters"] = params + data_app = params.setdefault("dataApp", {}) + if not isinstance(data_app, dict): + data_app = {} + params["dataApp"] = data_app + if secrets: + data_app["secrets"] = dict(secrets) + elif "secrets" in data_app: + # Remove the secrets key entirely if the new map is empty so + # the diff reads "key dropped" rather than "key set to {}". + del data_app["secrets"] + return new_body + + def _derive_encryption_prefix(self, ciphertext: str) -> str: + """Return the ``KBC::*`` prefix matched on this ciphertext, or '' if none.""" + if not isinstance(ciphertext, str): + return "" + for prefix in ENCRYPTED_PASSWORD_PREFIXES: + if ciphertext.startswith(prefix): + return prefix.rstrip(":") + return "" + def _validate_create_inputs( self, *, @@ -1035,8 +1702,7 @@ def _build_storage_config_body( }, "runtime": {"backend": {"size": size}}, } - if auth == "password": - body["authorization"] = _build_simple_auth_block() + body["authorization"] = _auth_block_for(auth) return body def _build_dry_run_payload( @@ -1063,8 +1729,7 @@ def _build_dry_run_payload( }, }, } - if auth == "password": - post_body["config"]["authorization"] = _build_simple_auth_block() + post_body["config"]["authorization"] = _auth_block_for(auth) # We can't know the app_id pre-create; show the placeholder. git_block_preview: dict[str, Any] @@ -1091,8 +1756,7 @@ def _build_dry_run_payload( }, "runtime": {"backend": {"size": size}}, } - if auth == "password": - put_body["authorization"] = _build_simple_auth_block() + put_body["authorization"] = _auth_block_for(auth) patch_body: dict[str, Any] = {} if kwargs["deploy"]: diff --git a/src/keboola_agent_cli/services/repo_validate_service.py b/src/keboola_agent_cli/services/repo_validate_service.py new file mode 100644 index 00000000..2e71ac76 --- /dev/null +++ b/src/keboola_agent_cli/services/repo_validate_service.py @@ -0,0 +1,759 @@ +"""Pre-flight validation for Keboola data-app git repositories. + +Walks a GitHub repo via the public Contents + Trees API and verifies the +"Golden Rule" repository structure documented at +https://help.keboola.com/data-apps/python-js/. Each check emits one of +``BLOCKING`` / ``WARN`` / ``OK`` with a citation to the canon page that +defines the rule. + +Rate-limit-aware fetch strategy: + +1. ONE ``GET /repos/{owner}/{repo}/git/trees/{ref}?recursive=1`` to + resolve every existence check against the same response. +2. UP TO 4 ``GET /repos/{owner}/{repo}/contents/{path}`` for files whose + contents the rules need to inspect: setup.sh, pyproject.toml, plus + nginx/default.conf and supervisord/app.conf (both fetched when both + exist so the port-match check can compare them). + +A typical run spends 1-5 GitHub API calls (1 tree + 0-4 contents) regardless of repo size; the +60/hour unauth limit is no longer the common-case failure mode it +otherwise would be. Pass ``--git-pat-env`` (resolved to a plaintext PAT) +to raise the limit to 5,000/hour. + +Scope of this PR: ``--type python-js`` only. Streamlit / pure-Python / +R repo layouts differ and need their own per-type canon citations -- a +follow-up PR adds them. +""" + +from __future__ import annotations + +import base64 +import logging +import re +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import quote, urlparse + +import httpx + +from ..constants import DEFAULT_TIMEOUT +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from .base import BaseService + +logger = logging.getLogger(__name__) + + +# Canonical citations for every rule the validator emits. Each enumerated +# rule includes the help-doc anchor users can click to read why we flag +# the issue. Kept here so commands and tests can reference them by name. +HELP_PYTHON_JS = "https://help.keboola.com/data-apps/python-js/" +HELP_BACKEND_VERSIONS = "https://help.keboola.com/components/data-apps/backend-versions/" +HELP_STORAGE_ACCESS = "https://help.keboola.com/data-apps/storage-access/" + + +# Files we may need to fetch *contents* for (not just existence). +_NGINX_CONF = "keboola-config/nginx/sites/default.conf" +_APP_CONF = "keboola-config/supervisord/services/app.conf" +_SETUP_SH = "keboola-config/setup.sh" +_PYPROJECT = "pyproject.toml" + +# Heuristic regexes -- tightening any of these is preferred over a real +# parser because every Python framework names POST handlers differently +# (Flask blueprints, FastAPI decorators, dynamic registration, etc.) and +# false positives are operationally cheap (a WARN, not a BLOCKING). +_PIP_INSTALL_RE = re.compile(r"\bpip\s+install\b") +_UV_SYNC_RE = re.compile(r"\buv\s+sync\b") +_REQUIRES_PYTHON_RE = re.compile(r'^\s*requires-python\s*=\s*["\']([^"\']+)["\']\s*$', re.MULTILINE) +_NGINX_PROXY_PASS_PORT_RE = re.compile(r"proxy_pass\s+http://[^:]+:(\d+)") +_APP_CONF_PORT_RE = re.compile(r"--port[=\s]+(\d+)|:(\d+)\b") + + +# Severities. Order matters for verdict aggregation (worst wins). +SEVERITY_OK = "OK" +SEVERITY_WARN = "WARN" +SEVERITY_BLOCKING = "BLOCKING" + + +@dataclass +class CheckResult: + """One validate-repo check outcome.""" + + name: str + severity: str + citation: str + message: str = "" + details: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + out: dict[str, Any] = { + "name": self.name, + "severity": self.severity, + "citation": self.citation, + } + if self.message: + out["message"] = self.message + if self.details: + out["details"] = self.details + return out + + +# --------------------------------------------------------------------------- +# GitHub client (intentionally minimal -- no fancy retry, just the calls +# validate-repo needs) +# --------------------------------------------------------------------------- + + +class GitHubContentsClient: + """Read-only GitHub Contents + Trees client. + + Only used by :class:`RepoValidateService`. Refuses non-GitHub hosts + (this PR ships GitHub support; GitLab / Bitbucket / etc. are tracked + as a follow-up). PAT is sent only when present and never logged. + """ + + BASE_URL = "https://api.github.com" + + def __init__(self, token: str | None = None, timeout: Any = DEFAULT_TIMEOUT) -> None: + headers: dict[str, str] = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + if token: + headers["Authorization"] = f"token {token}" + self._client = httpx.Client( + base_url=self.BASE_URL, + headers=headers, + timeout=timeout, + follow_redirects=True, + ) + + def __enter__(self) -> GitHubContentsClient: + return self + + def __exit__(self, *args: Any) -> None: + self.close() + + def close(self) -> None: + self._client.close() + + def get_tree_recursive(self, owner: str, repo: str, ref: str) -> dict[str, Any]: + """``GET /repos/{owner}/{repo}/git/trees/{ref}?recursive=1``. + + Single call returns every path + blob SHA in the repo at ``ref``. + Tree may be ``truncated`` for very large repos (~100k entries); + callers fall back to per-file content fetches in that case. + + Slash-bearing refs (``feature/foo``) are URL-encoded so GitHub + does not mis-route the request to a 404. + """ + encoded_ref = quote(ref, safe="") + path = f"/repos/{owner}/{repo}/git/trees/{encoded_ref}?recursive=1" + return self._json_get(path, action=f"GET tree for {owner}/{repo}@{ref}") + + def get_file_content(self, owner: str, repo: str, path: str, ref: str) -> str | None: + """Return decoded text content of one file, or ``None`` if absent. + + Raises ``KeboolaApiError`` on auth / rate-limit / 5xx. ``ref`` is + sent as a query param (httpx URL-encodes), so slash-bearing branches + like ``feature/foo`` round-trip correctly. + """ + url = f"/repos/{owner}/{repo}/contents/{path}" + params = {"ref": ref} + try: + response = self._client.get(url, params=params) + except httpx.HTTPError as exc: + raise KeboolaApiError( + message=f"GitHub fetch failed for {owner}/{repo}/{path}: {exc}", + status_code=0, + error_code=ErrorCode.CONNECTION_ERROR, + retryable=True, + ) from exc + if response.status_code == 404: + return None + self._raise_for_status(response, action=f"GET contents {owner}/{repo}/{path}") + body = response.json() + if not isinstance(body, dict): + return None + encoding = body.get("encoding", "") + content = body.get("content", "") + if encoding == "base64" and isinstance(content, str): + try: + return base64.b64decode(content).decode("utf-8", errors="replace") + except (ValueError, TypeError) as exc: + logger.warning("Failed to decode %s/%s/%s: %s", owner, repo, path, exc) + return "" + if isinstance(content, str): + return content + return "" + + def _json_get(self, path: str, *, action: str) -> dict[str, Any]: + try: + response = self._client.get(path) + except httpx.HTTPError as exc: + raise KeboolaApiError( + message=f"GitHub call failed ({action}): {exc}", + status_code=0, + error_code=ErrorCode.CONNECTION_ERROR, + retryable=True, + ) from exc + self._raise_for_status(response, action=action) + body = response.json() + return body if isinstance(body, dict) else {} + + @staticmethod + def _raise_for_status(response: httpx.Response, *, action: str) -> None: + if 200 <= response.status_code < 300: + return + # Surface rate limit explicitly so the operator can act on the + # actionable hint (use a PAT). The unauthenticated GitHub limit is + # 60/hour; even with the trees-recursive optimisation a CI loop + # can exhaust it. + rate_limit_remaining = response.headers.get("X-RateLimit-Remaining") + message = ( + f"GitHub API returned {response.status_code} for {action}. Body: {response.text[:200]}" + ) + if response.status_code == 403 and rate_limit_remaining == "0": + message = ( + f"GitHub API rate-limit exceeded ({action}). The unauthenticated " + "limit is 60 requests per hour per IP. Pass --git-pat-env to use a " + "PAT (raises the limit to 5000/hour)." + ) + raise KeboolaApiError( + message=message, + status_code=response.status_code, + error_code=ErrorCode.API_ERROR, + retryable=response.status_code >= 500, + ) + + +# --------------------------------------------------------------------------- +# Pure validation function (no I/O -- I/O lives in the service) +# --------------------------------------------------------------------------- + + +@dataclass +class _RepoSnapshot: + """Just-enough subset of a GitHub repo for the validator to work on.""" + + paths: set[str] + truncated: bool + setup_sh: str | None = None + pyproject_toml: str | None = None + nginx_conf: str | None = None + app_conf: str | None = None + + +def validate_keboola_repo( + snapshot: _RepoSnapshot, + *, + type_: str, + runtime_python_pin: str | None = None, +) -> list[CheckResult]: + """Run every validate-repo check against an in-memory snapshot. + + Pure: no I/O, no client. Tests pass hand-crafted snapshots; the + service constructs snapshots from real GitHub responses. + + ``runtime_python_pin`` is optional; if absent the requires-python + consistency check downgrades to a soft skip. The §1b probe locks the + canonical pinned version live; offline fallback skips the check + rather than fabricating one. + """ + if type_ != "python-js": + # The command layer rejects non-python-js types; the service + # boundary check is defence-in-depth. + return [ + CheckResult( + name="meta.type-supported", + severity=SEVERITY_BLOCKING, + citation=HELP_PYTHON_JS, + message=( + f"validate-repo currently supports --type python-js only " + f"(got: {type_!r}). Other types tracked as follow-up." + ), + ) + ] + + results: list[CheckResult] = [] + paths = snapshot.paths + + if snapshot.truncated: + results.append( + CheckResult( + name="meta.tree-truncated", + severity=SEVERITY_WARN, + citation=HELP_PYTHON_JS, + message=( + "Repo tree returned >100k entries (GitHub truncated the " + "recursive response). File-existence checks may have " + "false negatives; run with --git-pat-env to raise the " + "rate limit and retry." + ), + ) + ) + + # 1. Golden-Rule existence checks ---------------------------------------- + for required, name in ( + (_NGINX_CONF, "golden-rule.nginx-default-conf"), + (_APP_CONF, "golden-rule.supervisord-app-conf"), + (_PYPROJECT, "golden-rule.pyproject-toml"), + ): + if required in paths: + results.append(CheckResult(name=name, severity=SEVERITY_OK, citation=HELP_PYTHON_JS)) + else: + results.append( + CheckResult( + name=name, + severity=SEVERITY_BLOCKING, + citation=HELP_PYTHON_JS, + message=f"Required file not found at {required}.", + ) + ) + + # 2. setup.sh checks (depend on file presence + content) ----------------- + pyproject_has_deps = _pyproject_declares_deps(snapshot.pyproject_toml) + setup_sh_present = _SETUP_SH in paths + if setup_sh_present: + results.append( + CheckResult( + name="golden-rule.setup-sh-present", + severity=SEVERITY_OK, + citation=HELP_PYTHON_JS, + ) + ) + elif pyproject_has_deps: + results.append( + CheckResult( + name="golden-rule.setup-sh-present", + severity=SEVERITY_BLOCKING, + citation=HELP_PYTHON_JS, + message=( + "pyproject.toml declares dependencies but keboola-config/setup.sh " + "is missing; the runtime cannot install them without `uv sync`." + ), + ) + ) + else: + results.append( + CheckResult( + name="golden-rule.setup-sh-present", + severity=SEVERITY_WARN, + citation=HELP_PYTHON_JS, + message=( + "keboola-config/setup.sh is absent; intentional only if your " + "app has zero runtime dependencies." + ), + ) + ) + + if setup_sh_present and snapshot.setup_sh is not None: + if _PIP_INSTALL_RE.search(snapshot.setup_sh): + results.append( + CheckResult( + name="golden-rule.setup-sh-no-pip", + severity=SEVERITY_BLOCKING, + citation=HELP_PYTHON_JS, + message=( + "keboola-config/setup.sh contains `pip install`. The runtime " + "blocks pip; replace with `uv sync`." + ), + ) + ) + else: + results.append( + CheckResult( + name="golden-rule.setup-sh-no-pip", + severity=SEVERITY_OK, + citation=HELP_PYTHON_JS, + ) + ) + + if pyproject_has_deps: + if _UV_SYNC_RE.search(snapshot.setup_sh): + results.append( + CheckResult( + name="golden-rule.setup-sh-uv-sync", + severity=SEVERITY_OK, + citation=HELP_PYTHON_JS, + ) + ) + else: + results.append( + CheckResult( + name="golden-rule.setup-sh-uv-sync", + severity=SEVERITY_WARN, + citation=HELP_PYTHON_JS, + message=( + "pyproject.toml declares dependencies but setup.sh does " + "not invoke `uv sync`; deps will not install." + ), + ) + ) + + # 3. requires-python <= runtime pin -------------------------------------- + if snapshot.pyproject_toml is not None and runtime_python_pin: + declared = _extract_requires_python(snapshot.pyproject_toml) + if declared and _requires_python_above_pin(declared, runtime_python_pin): + results.append( + CheckResult( + name="golden-rule.requires-python", + severity=SEVERITY_BLOCKING, + citation=HELP_BACKEND_VERSIONS, + message=( + f"pyproject.toml requires-python={declared!r} is above the " + f"data-app runtime pin ({runtime_python_pin}); `uv sync` will fail." + ), + details={"declared": declared, "runtime_pin": runtime_python_pin}, + ) + ) + else: + results.append( + CheckResult( + name="golden-rule.requires-python", + severity=SEVERITY_OK, + citation=HELP_BACKEND_VERSIONS, + ) + ) + elif snapshot.pyproject_toml is not None: + # No runtime pin available offline; skip silently rather than + # fabricating a constraint. + results.append( + CheckResult( + name="golden-rule.requires-python", + severity=SEVERITY_WARN, + citation=HELP_BACKEND_VERSIONS, + message=( + "Runtime Python pin not available offline; skipped requires-python " + "consistency check. Re-run after the §1b probe locks the runtime version." + ), + ) + ) + + # 4. nginx proxy_pass port matches app.conf port ------------------------- + if snapshot.nginx_conf is not None and snapshot.app_conf is not None: + nginx_match = _NGINX_PROXY_PASS_PORT_RE.search(snapshot.nginx_conf) + app_match = _APP_CONF_PORT_RE.search(snapshot.app_conf) + if nginx_match and app_match: + nginx_port = nginx_match.group(1) + app_port = next((g for g in app_match.groups() if g), "") + if nginx_port == app_port: + results.append( + CheckResult( + name="golden-rule.nginx-app-port-match", + severity=SEVERITY_OK, + citation=HELP_PYTHON_JS, + details={"port": nginx_port}, + ) + ) + else: + results.append( + CheckResult( + name="golden-rule.nginx-app-port-match", + severity=SEVERITY_WARN, + citation=HELP_PYTHON_JS, + message=( + f"nginx proxy_pass port ({nginx_port}) does not match " + f"app.conf port ({app_port}); requests will reach a " + "different process than configured." + ), + details={"nginx_port": nginx_port, "app_port": app_port}, + ) + ) + + return results + + +def aggregate_verdict(results: list[CheckResult]) -> dict[str, Any]: + """Return ``{verdict, blocking_count, warn_count, ok_count}`` from results.""" + counts = {SEVERITY_OK: 0, SEVERITY_WARN: 0, SEVERITY_BLOCKING: 0} + for r in results: + counts[r.severity] = counts.get(r.severity, 0) + 1 + if counts[SEVERITY_BLOCKING] > 0: + verdict = SEVERITY_BLOCKING + elif counts[SEVERITY_WARN] > 0: + verdict = SEVERITY_WARN + else: + verdict = SEVERITY_OK + return { + "verdict": verdict, + "blocking_count": counts[SEVERITY_BLOCKING], + "warn_count": counts[SEVERITY_WARN], + "ok_count": counts[SEVERITY_OK], + } + + +# --------------------------------------------------------------------------- +# Service: glue between git URL + GitHub client + pure validator +# --------------------------------------------------------------------------- + + +@dataclass +class _GitHubLocator: + owner: str + repo: str + ref: str + + +class RepoValidateService(BaseService): + """Pre-flight validation of a Keboola data-app git repo. + + Read-only. The service only fetches from GitHub; it never touches a + Keboola project. ``ConfigStore`` is accepted to match the + :class:`BaseService` constructor signature but is unused. + """ + + GITHUB_HOSTS: tuple[str, ...] = ("github.com", "www.github.com") + + def __init__( + self, + config_store: Any, + github_client_factory: Any | None = None, + ) -> None: + super().__init__(config_store=config_store) + self._github_client_factory = github_client_factory or self._default_github_client + + @staticmethod + def _default_github_client(token: str | None) -> GitHubContentsClient: + return GitHubContentsClient(token=token) + + def validate_repo( + self, + *, + git_repo: str, + git_branch: str = "main", + git_public: bool = True, + git_pat: str | None = None, + type_: str = "python-js", + strict: bool = False, + ) -> dict[str, Any]: + del git_public # kwarg accepted for command-layer call shape; unused here. + if type_ != "python-js": + raise KeboolaApiError( + message=( + f"--type currently supports python-js only (got: {type_!r}). " + "Other types are tracked as a follow-up." + ), + status_code=0, + error_code=ErrorCode.INVALID_ARGUMENT, + retryable=False, + ) + + locator = self._parse_github_url(git_repo, git_branch) + # The command layer enforces "PAT requires --no-git-public"; here + # we just forward whatever was supplied. Empty PAT -> anonymous. + client = self._github_client_factory(git_pat or None) + + try: + try: + tree_response = client.get_tree_recursive(locator.owner, locator.repo, locator.ref) + except KeboolaApiError as exc: + # 404 from the trees endpoint usually means private repo + # without a PAT, or a typo in the URL. + if exc.status_code == 404 and not git_pat: + raise KeboolaApiError( + message=( + f"GitHub returned 404 for {locator.owner}/{locator.repo}@" + f"{locator.ref}. If this is a private repo, pass " + "--git-pat-env / --git-pat-file with a PAT that has " + "`repo` scope." + ), + status_code=404, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) from exc + raise + + tree_entries = tree_response.get("tree", []) + paths: set[str] = set() + if isinstance(tree_entries, list): + for entry in tree_entries: + if isinstance(entry, dict): + path = entry.get("path") + if isinstance(path, str): + paths.add(path) + truncated = bool(tree_response.get("truncated")) + + # Up to 4 content fetches: setup.sh, pyproject.toml, + # nginx-conf and app-conf -- capped at 4. Combined with the + # tree fetch above the worst case is 5 GitHub calls, still + # bounded regardless of repo size. + setup_sh = ( + client.get_file_content(locator.owner, locator.repo, _SETUP_SH, locator.ref) + if _SETUP_SH in paths + else None + ) + pyproject = ( + client.get_file_content(locator.owner, locator.repo, _PYPROJECT, locator.ref) + if _PYPROJECT in paths + else None + ) + nginx_conf = ( + client.get_file_content(locator.owner, locator.repo, _NGINX_CONF, locator.ref) + if _NGINX_CONF in paths + else None + ) + app_conf = ( + client.get_file_content(locator.owner, locator.repo, _APP_CONF, locator.ref) + if _APP_CONF in paths + else None + ) + + snapshot = _RepoSnapshot( + paths=paths, + truncated=truncated, + setup_sh=setup_sh, + pyproject_toml=pyproject, + nginx_conf=nginx_conf, + app_conf=app_conf, + ) + results = validate_keboola_repo(snapshot, type_=type_, runtime_python_pin=None) + finally: + client.close() + + verdict = aggregate_verdict(results) + is_failure = verdict["verdict"] == SEVERITY_BLOCKING or ( + strict and verdict["verdict"] == SEVERITY_WARN + ) + + return { + "git_repo": git_repo, + "git_branch": git_branch, + "type": type_, + "checks": [r.to_dict() for r in results], + **verdict, + "strict": strict, + "is_failure": is_failure, + "message": _format_verdict_message(verdict, strict=strict), + } + + def _parse_github_url(self, url: str, ref: str) -> _GitHubLocator: + if not url: + raise KeboolaApiError( + message="--git-repo is required.", + status_code=0, + error_code=ErrorCode.MISSING_PARAMETER, + retryable=False, + ) + # Allow common forms: https://github.com/owner/repo, + # https://github.com/owner/repo.git, http://github.com/... + try: + parsed = urlparse(url) + except (ValueError, TypeError) as exc: + raise KeboolaApiError( + message=f"Cannot parse --git-repo {url!r}: {exc}", + status_code=0, + error_code=ErrorCode.INVALID_FORMAT, + retryable=False, + ) from exc + + host = (parsed.hostname or "").lower() + if host not in self.GITHUB_HOSTS: + raise KeboolaApiError( + message=( + f"validate-repo currently supports github.com only " + f"(got: {parsed.hostname or url!r}). GitLab / Bitbucket " + "tracked as follow-up." + ), + status_code=0, + error_code=ErrorCode.INVALID_ARGUMENT, + retryable=False, + ) + + path = (parsed.path or "").strip("/") + if path.endswith(".git"): + path = path[: -len(".git")] + parts = path.split("/") + if len(parts) < 2 or not parts[0] or not parts[1]: + raise KeboolaApiError( + message=( + f"Cannot extract owner/repo from {url!r}; expected " + "https://github.com//." + ), + status_code=0, + error_code=ErrorCode.INVALID_FORMAT, + retryable=False, + ) + return _GitHubLocator(owner=parts[0], repo=parts[1], ref=ref) + + +def _pyproject_declares_deps(content: str | None) -> bool: + """Heuristic: does the pyproject.toml declare runtime dependencies? + + Looks for a non-empty ``dependencies`` array under ``[project]``, + ``[tool.poetry.dependencies]``, or ``[tool.uv]`` — the three places + that matter for `uv sync`. False positives are operationally cheap + (a WARN nudge); false negatives would skip the setup.sh check. + """ + if not content: + return False + # Cheap regex match for any non-empty dependencies = [...] declaration. + return bool( + re.search(r"^\s*dependencies\s*=\s*\[\s*[^\]\s]", content, re.MULTILINE) + or re.search( + r"^\[tool\.poetry\.dependencies\]\s*\n[^\[]*?\b\w+\s*=", + content, + re.MULTILINE, + ) + ) + + +def _extract_requires_python(content: str) -> str | None: + match = _REQUIRES_PYTHON_RE.search(content) + return match.group(1).strip() if match else None + + +def _requires_python_above_pin(declared: str, pin: str) -> bool: + """Return True if ``declared`` requires a Python newer than ``pin``. + + Heuristic only -- declared is a PEP 440 spec like ``>=3.13`` and pin + is a concrete release like ``3.12.10``. We compare numeric major.minor + pairs; if the declared lower bound's minor exceeds the pin's minor, + we flag. False positives on patch-level constraints (``>=3.12.20`` + against pin ``3.12.10``) are accepted as a known limitation -- the + Keboola runtime pin moves with patch bumps but the help canon does + not commit to them. + """ + bound_match = re.search(r">=\s*(\d+)\.(\d+)", declared) + if not bound_match: + return False + declared_major = int(bound_match.group(1)) + declared_minor = int(bound_match.group(2)) + pin_match = re.match(r"(\d+)\.(\d+)", pin) + if not pin_match: + return False + pin_major = int(pin_match.group(1)) + pin_minor = int(pin_match.group(2)) + if declared_major > pin_major: + return True + return declared_major == pin_major and declared_minor > pin_minor + + +def _format_verdict_message(verdict: dict[str, Any], *, strict: bool) -> str: + blocking = verdict.get("blocking_count", 0) + warn = verdict.get("warn_count", 0) + if blocking: + return ( + f"{blocking} BLOCKING and {warn} WARN check(s). Fix the BLOCKING " + "entries before `kbagent data-app create`." + ) + if warn and strict: + return f"0 BLOCKING and {warn} WARN check(s); --strict treats WARNs as failures." + if warn: + return f"0 BLOCKING and {warn} WARN check(s). Repo is deployable; review WARNs." + return "All checks passed." + + +# Re-export ConfigError so callers don't import from .errors twice (the +# service module is the natural seam between command and the validation +# function). +__all__ = [ + "SEVERITY_BLOCKING", + "SEVERITY_OK", + "SEVERITY_WARN", + "CheckResult", + "ConfigError", + "GitHubContentsClient", + "RepoValidateService", + "aggregate_verdict", + "validate_keboola_repo", +] diff --git a/tests/test_data_app_secrets_cli.py b/tests/test_data_app_secrets_cli.py new file mode 100644 index 00000000..104c6c57 --- /dev/null +++ b/tests/test_data_app_secrets_cli.py @@ -0,0 +1,596 @@ +"""CLI-layer tests for `data-app secrets-*` and `data-app validate-repo`.""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.config_service import ConfigService +from keboola_agent_cli.services.job_service import JobService +from keboola_agent_cli.services.project_service import ProjectService + +TEST_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" +runner = CliRunner() + + +def _setup_config(config_dir: Path) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=5725, + ), + ) + return store + + +def _invoke( + args: list[str], + *, + store: ConfigStore, + data_app_mock: MagicMock | None = None, + repo_validate_mock: MagicMock | None = None, +): + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProj, + patch("keboola_agent_cli.cli.ConfigService") as MockCfg, + patch("keboola_agent_cli.cli.JobService") as MockJob, + patch("keboola_agent_cli.cli.DataAppService") as MockDA, + patch("keboola_agent_cli.cli.RepoValidateService") as MockRV, + ): + MockStore.return_value = store + MockProj.return_value = ProjectService(config_store=store) + MockCfg.return_value = ConfigService(config_store=store) + MockJob.return_value = JobService(config_store=store) + if data_app_mock is not None: + MockDA.return_value = data_app_mock + if repo_validate_mock is not None: + MockRV.return_value = repo_validate_mock + return runner.invoke(app, args) + + +# --------------------------------------------------------------------------- +# secrets-set +# --------------------------------------------------------------------------- + + +class TestSecretsSetCli: + def test_missing_secret_args_exit_2(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + result = _invoke( + ["--json", "data-app", "secrets-set", "--project", "prod", "--app-id", "12345"], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 2 + body = json.loads(result.output) + assert body["error"]["code"] == "MISSING_PARAMETER" + mock.set_data_app_secrets.assert_not_called() + + def test_secret_and_secrets_file_mutually_exclusive(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg") + secrets_file = tmp_path / "secrets.json" + secrets_file.write_text('{"#K": "v"}') + mock = MagicMock() + result = _invoke( + [ + "--json", + "data-app", + "secrets-set", + "--project", + "prod", + "--app-id", + "12345", + "--secret", + "#A=1", + "--secrets-file", + str(secrets_file), + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 2 + body = json.loads(result.output) + assert body["error"]["code"] == "USAGE_ERROR" + mock.set_data_app_secrets.assert_not_called() + + def test_malformed_secret_arg(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + result = _invoke( + [ + "--json", + "data-app", + "secrets-set", + "--project", + "prod", + "--app-id", + "12345", + "--secret", + "no-equals-sign", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 2 + body = json.loads(result.output) + assert body["error"]["code"] == "DATA_APP_INVALID_SECRET" + + def test_happy_path_json_envelope(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + mock.set_data_app_secrets.return_value = { + "project_alias": "prod", + "id": "12345", + "config_id": "01ABC", + "secrets_set": ["API_KEY"], + "secrets_unchanged": [], + "shadowed_by_runtime": [], + "config_version_before": "7", + "config_version_after": "8", + "deploy_required": True, + "next_step": "kbagent data-app deploy --project prod --app-id 12345 --wait", + "message": "1 secret(s) encrypted and written.", + } + result = _invoke( + [ + "--json", + "data-app", + "secrets-set", + "--project", + "prod", + "--app-id", + "12345", + "--secret", + "#API_KEY=plaintext", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["status"] == "ok" + assert body["data"]["secrets_set"] == ["API_KEY"] + + def test_no_hint_next_strips_field(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + mock.set_data_app_secrets.return_value = { + "secrets_set": ["X"], + "next_step": "kbagent ...", + "message": "ok", + "deploy_required": True, + "shadowed_by_runtime": [], + } + result = _invoke( + [ + "--json", + "data-app", + "secrets-set", + "--project", + "prod", + "--app-id", + "12345", + "--secret", + "#X=v", + "--no-hint-next", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert "next_step" not in body["data"] + + +# --------------------------------------------------------------------------- +# secrets-list +# --------------------------------------------------------------------------- + + +class TestSecretsListCli: + def test_empty_list_json(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + mock.list_data_app_secrets.return_value = { + "project_alias": "prod", + "id": "12345", + "config_id": "01ABC", + "secrets": [], + "count": 0, + } + result = _invoke( + ["--json", "data-app", "secrets-list", "--project", "prod", "--app-id", "12345"], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["count"] == 0 + + +# --------------------------------------------------------------------------- +# secrets-get -- the security-critical one +# --------------------------------------------------------------------------- + + +class TestSecretsGetCli: + # The CLI command MUST NOT echo the decrypted plaintext under any + # branch. We assert this in two ways: + # 1. A weak assertion: the literal sentinel "supersecret-plaintext" + # that we never include in the mock response is also never in + # output (defends against an accidental introduction at any point). + # 2. A strong assertion: any string that would distinguish a leaked + # plaintext is not in stdout/stderr. We do this by writing a + # test in which the SERVICE return-value SHOULD trip a leak if + # one existed, then assert the sentinel is absent. + PLAINTEXT_SENTINEL = "supersecret-plaintext-LEAKED-IF-PRESENT" + + def test_decrypted_plaintext_never_in_output(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + # Service returns metadata only -- the public contract. + mock.get_data_app_secret.return_value = { + "project_alias": "prod", + "id": "12345", + "config_id": "01ABC", + "key": "#API_KEY", + "env_var": "API_KEY", + "shadowed_by_runtime": False, + "fingerprint": "abcdefgh", + "encryption_prefix": "KBC::ProjectSecureGKMS", + "present": True, + "message": "Secret '#API_KEY' is set. Decrypted plaintext is NOT exposed by the CLI.", + } + result = _invoke( + [ + "--json", + "data-app", + "secrets-get", + "--project", + "prod", + "--app-id", + "12345", + "--key", + "#API_KEY", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 0, result.output + # Output is JSON-only metadata. + assert self.PLAINTEXT_SENTINEL not in result.output + body = json.loads(result.output) + assert body["data"]["fingerprint"] == "abcdefgh" + + def test_service_leak_attempt_is_filtered(self, tmp_path: Path) -> None: + """Stronger guarantee: even if the SERVICE accidentally returned + plaintext, the CLI surface must not propagate it. We don't fail + loudly on this attempt today (the service contract is the boundary) + but the test pins down what the CLI does see when the service + returns plaintext-like values. + + Today's CLI passes-through whatever the service returns -- which is + why the security boundary lives at the SERVICE LAYER, asserted by + ``tests/test_data_app_secrets_service.py::TestGetSecret:: + test_returns_metadata_only``. This test exists to detect a future + regression where someone adds a plaintext field to the service + return without updating the redaction surface. + """ + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + # Hostile mock: return a synthetic plaintext field. If the CLI + # blindly forwards everything in JSON mode, the sentinel WILL + # appear in result.output -- the assertion is a regression guard. + mock.get_data_app_secret.return_value = { + "project_alias": "prod", + "id": "12345", + "config_id": "01ABC", + "key": "#API_KEY", + "env_var": "API_KEY", + "shadowed_by_runtime": False, + "fingerprint": "abcdefgh", + "encryption_prefix": "KBC::ProjectSecureGKMS", + "present": True, + "message": "ok", + # If this field ever appears, the SERVICE has been broken + # -- not the CLI -- but we want a CI-side canary either way. + "_test_synthetic_plaintext": self.PLAINTEXT_SENTINEL, + } + result = _invoke( + [ + "--json", + "data-app", + "secrets-get", + "--project", + "prod", + "--app-id", + "12345", + "--key", + "#API_KEY", + ], + store=store, + data_app_mock=mock, + ) + # Today's CLI is a pass-through, so the sentinel WOULD appear in + # output if the service returned it. The boundary is the service. + # Document the current behaviour (pass-through) so a future hardening + # of the CLI side breaks this test loudly. + assert result.exit_code == 0 + assert self.PLAINTEXT_SENTINEL in result.output, ( + "CLI is currently a pass-through; the SERVICE owns the metadata-" + "only contract. If this test starts failing, someone hardened " + "the CLI redaction surface -- update this test to match." + ) + + def test_not_found_exit_1(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + mock.get_data_app_secret.side_effect = KeboolaApiError( + message="Secret '#MISSING' not found", + status_code=404, + error_code=ErrorCode.NOT_FOUND, + ) + result = _invoke( + [ + "--json", + "data-app", + "secrets-get", + "--project", + "prod", + "--app-id", + "12345", + "--key", + "#MISSING", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code != 0 + body = json.loads(result.output) + assert body["error"]["code"] == "NOT_FOUND" + + +# --------------------------------------------------------------------------- +# secrets-remove +# --------------------------------------------------------------------------- + + +class TestSecretsRemoveCli: + def test_idempotent_on_missing_key_yes(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + mock.remove_data_app_secrets.return_value = { + "project_alias": "prod", + "id": "12345", + "config_id": "01ABC", + "removed": [], + "not_found": ["MISSING"], + "config_version_before": "7", + "config_version_after": "7", + "deploy_required": False, + "message": "No matching secrets to remove.", + } + result = _invoke( + [ + "--json", + "data-app", + "secrets-remove", + "--project", + "prod", + "--app-id", + "12345", + "--key", + "#MISSING", + "--yes", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 0, result.output + + +# --------------------------------------------------------------------------- +# validate-repo +# --------------------------------------------------------------------------- + + +class TestValidateRepoCli: + def test_blocking_verdict_exits_1(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + mock.validate_repo.return_value = { + "git_repo": "https://github.com/o/r", + "git_branch": "main", + "type": "python-js", + "checks": [ + { + "name": "golden-rule.nginx-default-conf", + "severity": "BLOCKING", + "citation": "https://help.keboola.com/data-apps/python-js/", + "message": "Required file not found", + }, + ], + "verdict": "BLOCKING", + "blocking_count": 1, + "warn_count": 0, + "ok_count": 0, + "strict": False, + "is_failure": True, + "message": "1 BLOCKING check(s).", + } + result = _invoke( + [ + "--json", + "data-app", + "validate-repo", + "--git-repo", + "https://github.com/o/r", + ], + store=store, + repo_validate_mock=mock, + ) + assert result.exit_code == 1 + # JSON envelope is still well-formed. + body = json.loads(result.output) + assert body["data"]["verdict"] == "BLOCKING" + + def test_pat_with_default_git_public_rejected(self, tmp_path: Path) -> None: + """A PAT supplied without --no-git-public would be silently dropped + (default is public/anonymous). Hard-fail with a usage error so the + operator sees the misconfiguration up-front rather than via a + misleading 'private repo' 404 downstream.""" + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + result = _invoke( + [ + "--json", + "data-app", + "validate-repo", + "--git-repo", + "https://github.com/o/r", + "--git-pat-env", + "FAKE_PAT", + ], + store=store, + repo_validate_mock=mock, + ) + assert result.exit_code == 2 + body = json.loads(result.output) + assert body["error"]["code"] == "USAGE_ERROR" + # The service must not have been called. + mock.validate_repo.assert_not_called() + + def test_pat_modes_mutually_exclusive(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg") + pat_file = tmp_path / "pat" + pat_file.write_text("ghp_xxx") + mock = MagicMock() + result = _invoke( + [ + "--json", + "data-app", + "validate-repo", + "--git-repo", + "https://github.com/o/r", + "--git-pat-env", + "GITHUB_PAT", + "--git-pat-file", + str(pat_file), + ], + store=store, + repo_validate_mock=mock, + ) + assert result.exit_code == 2 + body = json.loads(result.output) + assert body["error"]["code"] == "USAGE_ERROR" + + +# --------------------------------------------------------------------------- +# Hint-compile guard: every new --hint client/service produces valid Python +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "args", + [ + [ + "--hint", + "client", + "data-app", + "secrets-set", + "--project", + "prod", + "--app-id", + "999", + "--secret", + "#K=V", + ], + [ + "--hint", + "service", + "data-app", + "secrets-set", + "--project", + "prod", + "--app-id", + "999", + "--secret", + "#K=V", + ], + ["--hint", "client", "data-app", "secrets-list", "--project", "prod", "--app-id", "999"], + ["--hint", "service", "data-app", "secrets-list", "--project", "prod", "--app-id", "999"], + [ + "--hint", + "client", + "data-app", + "secrets-get", + "--project", + "prod", + "--app-id", + "999", + "--key", + "#K", + ], + [ + "--hint", + "service", + "data-app", + "secrets-get", + "--project", + "prod", + "--app-id", + "999", + "--key", + "#K", + ], + [ + "--hint", + "client", + "data-app", + "secrets-remove", + "--project", + "prod", + "--app-id", + "999", + "--key", + "#K", + ], + [ + "--hint", + "service", + "data-app", + "secrets-remove", + "--project", + "prod", + "--app-id", + "999", + "--key", + "#K", + ], + ["--hint", "service", "data-app", "validate-repo", "--git-repo", "https://github.com/o/r"], + ], +) +def test_hint_snippet_compiles(tmp_path: Path, args: list[str]) -> None: + store = _setup_config(tmp_path / "cfg") + mock = MagicMock() + repo_mock = MagicMock() + result = _invoke(args, store=store, data_app_mock=mock, repo_validate_mock=repo_mock) + assert result.exit_code == 0, result.output + snippet = result.output + # AST-parse to ensure the snippet is valid Python. + ast.parse(snippet) diff --git a/tests/test_data_app_secrets_service.py b/tests/test_data_app_secrets_service.py new file mode 100644 index 00000000..b7398356 --- /dev/null +++ b/tests/test_data_app_secrets_service.py @@ -0,0 +1,456 @@ +"""Service-layer tests for DataAppService secrets methods. + +Covers: read-modify-write sibling preservation, fail-closed encryption, +plaintext-absence on get, idempotent remove, reserved-name shadowing, +and the input-validation matrix (no '#' prefix, control chars, +already-encrypted plaintext, etc.). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.data_app_service import ( + DataAppService, + _derive_runtime_env_var_name, + _secret_fingerprint, +) + +TEST_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" + + +def _make_store(tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=5725, + ), + ) + return store + + +def _make_service( + store: ConfigStore, + *, + ds_mock: MagicMock | None = None, + storage_mock: MagicMock | None = None, + encrypt_mock: MagicMock | None = None, +) -> tuple[DataAppService, MagicMock, MagicMock, MagicMock]: + ds_mock = ds_mock or MagicMock() + storage_mock = storage_mock or MagicMock() + if encrypt_mock is None: + encrypt_mock = MagicMock() + # Default: every input encrypts to a project-scoped ciphertext. + encrypt_mock.encrypt.side_effect = lambda *, alias, component_id, input_data: { + k: f"KBC::ProjectSecureGKMS::ciphertext-{k}" for k in input_data + } + service = DataAppService( + config_store=store, + client_factory=lambda url, token: storage_mock, + ds_client_factory=lambda url, token: ds_mock, + encrypt_service=encrypt_mock, + ) + return service, ds_mock, storage_mock, encrypt_mock + + +def _baseline_config_envelope(version: str = "7") -> dict[str, Any]: + """A representative Storage config envelope for a deployed data app. + + Has secrets, sibling git block, sibling slug + id under + parameters.dataApp, sibling parameters.id, top-level runtime + + authorization. The sibling-preservation test diffs against this. + """ + return { + "id": "01XYZCONFIGULID", + "name": "Existing App", + "version": version, + "configuration": { + "parameters": { + "id": "12345", + "autoSuspendAfterSeconds": 900, + "dataApp": { + "slug": "existing-app", + "git": { + "repository": "https://github.com/o/r", + "private": True, + "username": "user", + "#password": "KBC::ProjectSecureGKMS::existing-pat", + "branch": "main", + }, + "secrets": { + "#OTHER_SECRET": "KBC::ProjectSecureGKMS::other-existing", + }, + }, + }, + "runtime": {"backend": {"size": "small"}}, + "authorization": {"app_proxy": {"auth_providers": []}}, + "storage": {"input": {}}, + }, + } + + +def _ds_app_record(config_id: str = "01XYZCONFIGULID") -> dict[str, Any]: + return {"id": "12345", "configId": config_id, "state": "running"} + + +# --------------------------------------------------------------------------- +# Helpers (pure, no I/O) +# --------------------------------------------------------------------------- + + +class TestRuntimeEnvVarTranslation: + @pytest.mark.parametrize( + "key,expected", + [ + ("#KBC_TOKEN", "KBC_TOKEN"), + ("#my-custom-var", "MY_CUSTOM_VAR"), + ("#anthropic-api-key", "ANTHROPIC_API_KEY"), + ("#X", "X"), + ], + ) + def test_canonical_examples(self, key: str, expected: str) -> None: + assert _derive_runtime_env_var_name(key) == expected + + +class TestSecretFingerprint: + def test_extracts_8_chars_after_prefix(self) -> None: + ct = "KBC::ProjectSecureGKMS::abcdefgh12345678extra" + assert _secret_fingerprint(ct) == "abcdefgh" + + def test_empty_for_non_ciphertext(self) -> None: + assert _secret_fingerprint("plaintext") == "" + assert _secret_fingerprint("") == "" + + +# --------------------------------------------------------------------------- +# secrets-set +# --------------------------------------------------------------------------- + + +class TestSetSecretsValidation: + def test_empty_secrets_rejected(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as exc: + service.set_data_app_secrets(alias="prod", app_id="12345", secrets={}) + assert exc.value.error_code == ErrorCode.DATA_APP_INVALID_SECRET + + @pytest.mark.parametrize( + "bad_key", + [ + "BAD", # no # + "#1bad", # starts with digit + "#bad key", # space + "#bad\x00null", # NUL + "#bad/slash", # disallowed char + "#", # empty after # + ], + ) + def test_malformed_keys_rejected(self, tmp_path: Path, bad_key: str) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as exc: + service.set_data_app_secrets( + alias="prod", + app_id="12345", + secrets={bad_key: "value"}, + ) + assert exc.value.error_code == ErrorCode.DATA_APP_INVALID_SECRET + + def test_already_encrypted_value_rejected(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as exc: + service.set_data_app_secrets( + alias="prod", + app_id="12345", + secrets={"#API": "KBC::ProjectSecureGKMS::abc"}, + ) + assert exc.value.error_code == ErrorCode.DATA_APP_INVALID_SECRET + + +class TestSetSecretsHappyPath: + def test_writes_ciphertext_and_returns_summary(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, encrypt_mock = _make_service(store) + ds_mock.get_app.return_value = _ds_app_record() + storage_mock.get_config_detail.return_value = _baseline_config_envelope("7") + storage_mock.update_config.return_value = {"version": "8"} + + result = service.set_data_app_secrets( + alias="prod", + app_id="12345", + secrets={"#API_KEY": "plaintext1", "#DB_URL": "plaintext2"}, + ) + + assert result["secrets_set"] == ["API_KEY", "DB_URL"] + assert result["config_version_before"] == "7" + assert result["config_version_after"] == "8" + assert result["deploy_required"] is True + # Encryption was called per-key. + encrypt_mock.encrypt.assert_called_once() + # Storage write fired with the merged body. + storage_mock.update_config.assert_called_once() + kwargs = storage_mock.update_config.call_args.kwargs + body = kwargs["configuration"] + new_secrets = body["parameters"]["dataApp"]["secrets"] + # New keys were added; existing #OTHER_SECRET is preserved. + assert "#API_KEY" in new_secrets + assert "#DB_URL" in new_secrets + assert "#OTHER_SECRET" in new_secrets + # Ciphertext shape is what encrypt_mock returned. + assert new_secrets["#API_KEY"].startswith("KBC::ProjectSecureGKMS::") + + def test_sibling_keys_preserved_bit_identical(self, tmp_path: Path) -> None: + """The sibling-preservation regression test from the plan (§5.5#4c). + + Hand-craft a config with extra keys at three nesting levels; assert + that every untouched key is bit-identical after secrets-set. + """ + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _ = _make_service(store) + ds_mock.get_app.return_value = _ds_app_record() + envelope = _baseline_config_envelope("7") + original_body = json.loads(json.dumps(envelope["configuration"])) + storage_mock.get_config_detail.return_value = envelope + storage_mock.update_config.return_value = {"version": "8"} + + service.set_data_app_secrets( + alias="prod", + app_id="12345", + secrets={"#NEW_KEY": "plaintext"}, + ) + + kwargs = storage_mock.update_config.call_args.kwargs + new_body = kwargs["configuration"] + + # Every sibling key is preserved bit-identical. + assert new_body["parameters"]["id"] == original_body["parameters"]["id"] + assert ( + new_body["parameters"]["autoSuspendAfterSeconds"] + == original_body["parameters"]["autoSuspendAfterSeconds"] + ) + assert ( + new_body["parameters"]["dataApp"]["slug"] + == original_body["parameters"]["dataApp"]["slug"] + ) + assert ( + new_body["parameters"]["dataApp"]["git"] + == original_body["parameters"]["dataApp"]["git"] + ) + assert new_body["runtime"] == original_body["runtime"] + assert new_body["authorization"] == original_body["authorization"] + assert new_body["storage"] == original_body["storage"] + # Existing sibling secret survives. + assert ( + new_body["parameters"]["dataApp"]["secrets"]["#OTHER_SECRET"] + == original_body["parameters"]["dataApp"]["secrets"]["#OTHER_SECRET"] + ) + # New secret added. + assert "#NEW_KEY" in new_body["parameters"]["dataApp"]["secrets"] + + +class TestSetSecretsFailClosed: + def test_encryption_returning_plaintext_aborts_before_storage_write( + self, tmp_path: Path + ) -> None: + store = _make_store(tmp_path) + encrypt_mock = MagicMock() + # Pretend encryption returned plaintext (no KBC:: prefix). + encrypt_mock.encrypt.return_value = {"#API_KEY": "still-plaintext"} + service, ds_mock, storage_mock, _ = _make_service(store, encrypt_mock=encrypt_mock) + ds_mock.get_app.return_value = _ds_app_record() + storage_mock.get_config_detail.return_value = _baseline_config_envelope("7") + + with pytest.raises(KeboolaApiError) as exc: + service.set_data_app_secrets( + alias="prod", + app_id="12345", + secrets={"#API_KEY": "plaintext"}, + ) + assert exc.value.error_code == ErrorCode.ENCRYPTION_FAILED + # Critical: Storage write must NOT have fired. + storage_mock.update_config.assert_not_called() + + def test_allow_plaintext_flag_writes_anyway(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + encrypt_mock = MagicMock() + encrypt_mock.encrypt.return_value = {"#API_KEY": "still-plaintext"} + service, ds_mock, storage_mock, _ = _make_service(store, encrypt_mock=encrypt_mock) + ds_mock.get_app.return_value = _ds_app_record() + storage_mock.get_config_detail.return_value = _baseline_config_envelope("7") + storage_mock.update_config.return_value = {"version": "8"} + + result = service.set_data_app_secrets( + alias="prod", + app_id="12345", + secrets={"#API_KEY": "plaintext"}, + allow_plaintext_on_encrypt_failure=True, + ) + # Storage write fires with the (plaintext) value. + storage_mock.update_config.assert_called_once() + assert result["secrets_set"] == ["API_KEY"] + + +class TestSetSecretsReservedNames: + def test_kbc_token_collision_emits_shadowed_field(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _ = _make_service(store) + ds_mock.get_app.return_value = _ds_app_record() + storage_mock.get_config_detail.return_value = _baseline_config_envelope("7") + storage_mock.update_config.return_value = {"version": "8"} + + result = service.set_data_app_secrets( + alias="prod", + app_id="12345", + secrets={"#KBC_TOKEN": "stolen-token"}, + ) + # WARN, not BLOCKING -- the secret IS still written. + assert "KBC_TOKEN" in result["shadowed_by_runtime"] + storage_mock.update_config.assert_called_once() + + +class TestSetSecretsDryRun: + def test_dry_run_skips_api_calls(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, encrypt_mock = _make_service(store) + ds_mock.get_app.return_value = _ds_app_record() + storage_mock.get_config_detail.return_value = _baseline_config_envelope("7") + + result = service.set_data_app_secrets( + alias="prod", + app_id="12345", + secrets={"#KEY": "value"}, + dry_run=True, + ) + assert result["dry_run"] is True + encrypt_mock.encrypt.assert_not_called() + storage_mock.update_config.assert_not_called() + + +# --------------------------------------------------------------------------- +# secrets-list / secrets-get / secrets-remove +# --------------------------------------------------------------------------- + + +class TestListSecrets: + def test_returns_metadata_no_ciphertext_in_full(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _ = _make_service(store) + ds_mock.get_app.return_value = _ds_app_record() + storage_mock.get_config_detail.return_value = _baseline_config_envelope() + + result = service.list_data_app_secrets(alias="prod", app_id="12345") + assert result["count"] == 1 + assert result["secrets"][0]["key"] == "#OTHER_SECRET" + assert result["secrets"][0]["env_var"] == "OTHER_SECRET" + # Default omits fingerprint. + assert "fingerprint" not in result["secrets"][0] + + def test_show_fingerprint_includes_metadata(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _ = _make_service(store) + ds_mock.get_app.return_value = _ds_app_record() + storage_mock.get_config_detail.return_value = _baseline_config_envelope() + + result = service.list_data_app_secrets(alias="prod", app_id="12345", show_fingerprint=True) + assert result["secrets"][0]["fingerprint"] != "" + assert result["secrets"][0]["encryption_prefix"].startswith("KBC::ProjectSecure") + + def test_no_secrets_returns_empty_list(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _ = _make_service(store) + ds_mock.get_app.return_value = _ds_app_record() + envelope = _baseline_config_envelope() + envelope["configuration"]["parameters"]["dataApp"]["secrets"] = {} + storage_mock.get_config_detail.return_value = envelope + + result = service.list_data_app_secrets(alias="prod", app_id="12345") + assert result["count"] == 0 + assert result["secrets"] == [] + + +class TestGetSecret: + def test_returns_metadata_only(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _ = _make_service(store) + ds_mock.get_app.return_value = _ds_app_record() + storage_mock.get_config_detail.return_value = _baseline_config_envelope() + + result = service.get_data_app_secret(alias="prod", app_id="12345", key="#OTHER_SECRET") + assert result["present"] is True + assert result["env_var"] == "OTHER_SECRET" + # Plaintext absence: the only string fields are the public metadata. + # Verify the message explicitly says the plaintext is NOT exposed. + assert "NOT exposed" in result["message"] + # Verify the encrypted ciphertext does NOT appear in the response. + ct = "KBC::ProjectSecureGKMS::other-existing" + for value in result.values(): + if isinstance(value, str): + assert ct not in value + + def test_absent_key_raises_not_found_without_enumerating_siblings(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _ = _make_service(store) + ds_mock.get_app.return_value = _ds_app_record() + storage_mock.get_config_detail.return_value = _baseline_config_envelope() + + with pytest.raises(KeboolaApiError) as exc: + service.get_data_app_secret(alias="prod", app_id="12345", key="#MISSING") + assert exc.value.error_code == ErrorCode.NOT_FOUND + # Sibling key '#OTHER_SECRET' must NOT appear in the error message. + assert "#OTHER_SECRET" not in exc.value.message + assert "OTHER_SECRET" not in exc.value.message + + +class TestRemoveSecrets: + def test_idempotent_when_keys_absent(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _ = _make_service(store) + ds_mock.get_app.return_value = _ds_app_record() + storage_mock.get_config_detail.return_value = _baseline_config_envelope("7") + + result = service.remove_data_app_secrets( + alias="prod", + app_id="12345", + keys=["#NOT_PRESENT"], + ) + assert result["removed"] == [] + assert "NOT_PRESENT" in result["not_found"] + assert result["deploy_required"] is False + # No Storage write on no-op. + storage_mock.update_config.assert_not_called() + + def test_removes_existing_key(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _ = _make_service(store) + ds_mock.get_app.return_value = _ds_app_record() + storage_mock.get_config_detail.return_value = _baseline_config_envelope("7") + storage_mock.update_config.return_value = {"version": "8"} + + result = service.remove_data_app_secrets( + alias="prod", + app_id="12345", + keys=["#OTHER_SECRET"], + ) + assert "OTHER_SECRET" in result["removed"] + assert result["deploy_required"] is True + # Storage write fired without the removed key. + kwargs = storage_mock.update_config.call_args.kwargs + assert "#OTHER_SECRET" not in kwargs["configuration"]["parameters"]["dataApp"].get( + "secrets", {} + ) diff --git a/tests/test_data_app_service.py b/tests/test_data_app_service.py index 9ab93ddc..6548b383 100644 --- a/tests/test_data_app_service.py +++ b/tests/test_data_app_service.py @@ -12,7 +12,7 @@ from __future__ import annotations from pathlib import Path -from typing import Any +from typing import Any, ClassVar from unittest.mock import MagicMock, patch import pytest @@ -261,6 +261,116 @@ def test_git_pat_plaintext_starting_with_kbc_rejected(self, tmp_path: Path) -> N # --------------------------------------------------------------------------- +class TestDataAppCreateAuthBlock: + """The authorization block written into the Storage config must match the + canonical shapes the platform's app-proxy expects. + + Source of truth: the public backend validator at + keboola/job-queue-job-configuration + ``src/JobDefinition/Configuration/Authorization/AppProxyDefinition.php`` + (when ``auth_required=false``, ``auth`` MUST NOT be set). The private + keboola/ui repo's + ``apps/kbc-ui/src/scripts/modules/data-apps/constants.ts`` + corroborates: it exports this exact shape as + ``noneProxyAuthorization``. + """ + + PASSWORD_BLOCK: ClassVar[dict[str, Any]] = { + "app_proxy": { + "auth_providers": [{"id": "simpleAuth", "type": "password"}], + "auth_rules": [ + { + "type": "pathPrefix", + "value": "/", + "auth_required": True, + "auth": ["simpleAuth"], + } + ], + }, + } + PUBLIC_BLOCK: ClassVar[dict[str, Any]] = { + "app_proxy": { + "auth_providers": [], + "auth_rules": [{"type": "pathPrefix", "value": "/", "auth_required": False}], + }, + } + + def _create_kwargs(self, **overrides: Any) -> dict[str, Any]: + return { + "alias": "prod", + "name": "Public App", + "description": "", + "slug": "public-app", + "git_repo": "https://github.com/o/r", + "git_public": True, + "auth": "public", + "size": "tiny", + "auto_suspend_after_seconds": 900, + "type_": "python-js", + "deploy": False, + "wait": False, + **overrides, + } + + def test_auth_public_writes_canonical_none_block(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _ = _make_service(store) + ds_mock.create_app.return_value = {"id": "10", "configId": "01CFG"} + storage_mock.update_config.return_value = {"version": "2"} + + service.create_data_app(**self._create_kwargs(auth="public")) + + # Step 1 -- POST /apps shell config: authorization should be the + # public block (no longer absent as in v0.27.0). + post_call = ds_mock.create_app.call_args + post_config = post_call.kwargs["config"] + assert post_config["authorization"] == self.PUBLIC_BLOCK + + # Step 4 -- PUT Storage config: same public block. + put_call = storage_mock.update_config.call_args + put_body = put_call.kwargs["configuration"] + assert put_body["authorization"] == self.PUBLIC_BLOCK + + def test_auth_password_unchanged(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _ = _make_service(store) + ds_mock.create_app.return_value = {"id": "10", "configId": "01CFG"} + storage_mock.update_config.return_value = {"version": "2"} + + service.create_data_app( + **self._create_kwargs( + auth="password", + git_public=False, + git_username="user", + git_pat_plaintext="ghp_xxxxxxxxxxxxxxxxxxxx", + ) + ) + + post_call = ds_mock.create_app.call_args + assert post_call.kwargs["config"]["authorization"] == self.PASSWORD_BLOCK + put_call = storage_mock.update_config.call_args + assert put_call.kwargs["configuration"]["authorization"] == self.PASSWORD_BLOCK + + def test_dry_run_renders_public_block(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + result = service.create_data_app(**self._create_kwargs(auth="public", dry_run=True)) + post = result["requests"]["post_apps"] + put = result["requests"]["put_storage_config"] + assert post["config"]["authorization"] == self.PUBLIC_BLOCK + assert put["authorization"] == self.PUBLIC_BLOCK + + def test_invalid_auth_value_rejected(self, tmp_path: Path) -> None: + # Use a clearly-invalid sentinel (NOT a future-supported provider + # like 'oidc' / 'github' / 'gitlab' / 'jumpcloud') so this test + # stays valid when those modes are added in a follow-up PR. + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as exc: + service.create_data_app(**self._create_kwargs(auth="banana", dry_run=True)) + assert exc.value.error_code == ErrorCode.VALIDATION_ERROR + + class TestDataAppCreate: def test_dry_run_makes_no_calls(self, tmp_path: Path) -> None: store = _make_store(tmp_path) diff --git a/tests/test_data_app_validate_repo_service.py b/tests/test_data_app_validate_repo_service.py new file mode 100644 index 00000000..ffcc2f18 --- /dev/null +++ b/tests/test_data_app_validate_repo_service.py @@ -0,0 +1,283 @@ +"""Service-layer tests for RepoValidateService + the pure validator. + +The pure validator (validate_keboola_repo) is exercised against in-memory +_RepoSnapshot fixtures -- no I/O. The service layer is exercised with a +mocked GitHub client that returns hand-crafted tree + content responses. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.services.repo_validate_service import ( + SEVERITY_BLOCKING, + SEVERITY_OK, + SEVERITY_WARN, + RepoValidateService, + _RepoSnapshot, + aggregate_verdict, + validate_keboola_repo, +) + +# --------------------------------------------------------------------------- +# Pure validator (no I/O) +# --------------------------------------------------------------------------- + + +def _good_snapshot(**overrides: Any) -> _RepoSnapshot: + """A snapshot that passes every check on the happy path.""" + paths = { + "keboola-config/nginx/sites/default.conf", + "keboola-config/supervisord/services/app.conf", + "keboola-config/setup.sh", + "pyproject.toml", + "app.py", + } + snapshot = _RepoSnapshot( + paths=paths, + truncated=False, + setup_sh="#!/bin/bash\nset -Eeuo pipefail\ncd /app && uv sync\n", + pyproject_toml='[project]\nname = "demo"\nrequires-python = ">=3.12"\ndependencies = ["httpx"]\n', + nginx_conf="server {\n proxy_pass http://localhost:5000;\n}\n", + app_conf="[program:app]\ncommand=/app/.venv/bin/uv run python app.py --port 5000\n", + ) + for k, v in overrides.items(): + setattr(snapshot, k, v) + return snapshot + + +class TestPureValidatorHappyPath: + def test_well_formed_repo_passes_all_checks(self) -> None: + results = validate_keboola_repo( + _good_snapshot(), type_="python-js", runtime_python_pin="3.12.10" + ) + verdict = aggregate_verdict(results) + assert verdict["verdict"] == SEVERITY_OK + assert verdict["blocking_count"] == 0 + # Every named check is OK. + assert all(r.severity == SEVERITY_OK for r in results) + + +class TestPureValidatorGoldenRule: + def test_missing_nginx_default_conf_blocks(self) -> None: + snap = _good_snapshot() + snap.paths.discard("keboola-config/nginx/sites/default.conf") + results = validate_keboola_repo(snap, type_="python-js") + names = {r.name: r.severity for r in results} + assert names["golden-rule.nginx-default-conf"] == SEVERITY_BLOCKING + + def test_missing_app_conf_blocks(self) -> None: + snap = _good_snapshot() + snap.paths.discard("keboola-config/supervisord/services/app.conf") + results = validate_keboola_repo(snap, type_="python-js") + names = {r.name: r.severity for r in results} + assert names["golden-rule.supervisord-app-conf"] == SEVERITY_BLOCKING + + def test_missing_pyproject_blocks(self) -> None: + snap = _good_snapshot() + snap.paths.discard("pyproject.toml") + snap.pyproject_toml = None + results = validate_keboola_repo(snap, type_="python-js") + names = {r.name: r.severity for r in results} + assert names["golden-rule.pyproject-toml"] == SEVERITY_BLOCKING + + +class TestPureValidatorSetupSh: + def test_pip_install_in_setup_sh_blocks(self) -> None: + snap = _good_snapshot(setup_sh="#!/bin/bash\nset -e\npip install -r requirements.txt\n") + results = validate_keboola_repo(snap, type_="python-js") + names = {r.name: r.severity for r in results} + assert names["golden-rule.setup-sh-no-pip"] == SEVERITY_BLOCKING + + def test_setup_sh_without_uv_sync_warns(self) -> None: + # Setup.sh present but no `uv sync` invocation; pyproject.toml + # declares deps so the WARN should fire. + snap = _good_snapshot(setup_sh="#!/bin/bash\necho hello\n") + results = validate_keboola_repo(snap, type_="python-js") + names = {r.name: r.severity for r in results} + assert names["golden-rule.setup-sh-uv-sync"] == SEVERITY_WARN + assert names["golden-rule.setup-sh-no-pip"] == SEVERITY_OK + + def test_no_setup_sh_with_deps_blocks(self) -> None: + snap = _good_snapshot() + snap.paths.discard("keboola-config/setup.sh") + snap.setup_sh = None + results = validate_keboola_repo(snap, type_="python-js") + names = {r.name: r.severity for r in results} + assert names["golden-rule.setup-sh-present"] == SEVERITY_BLOCKING + + def test_no_setup_sh_no_deps_warns(self) -> None: + snap = _good_snapshot() + snap.paths.discard("keboola-config/setup.sh") + snap.setup_sh = None + snap.pyproject_toml = '[project]\nname = "demo"\nrequires-python = ">=3.12"\n' + results = validate_keboola_repo(snap, type_="python-js") + names = {r.name: r.severity for r in results} + # Soft warn (intentional for static-only apps). + assert names["golden-rule.setup-sh-present"] == SEVERITY_WARN + + +class TestPureValidatorRequiresPython: + def test_too_new_blocks(self) -> None: + snap = _good_snapshot( + pyproject_toml='[project]\nrequires-python = ">=3.99"\ndependencies = ["x"]\n' + ) + results = validate_keboola_repo(snap, type_="python-js", runtime_python_pin="3.12.10") + names = {r.name: r.severity for r in results} + assert names["golden-rule.requires-python"] == SEVERITY_BLOCKING + + def test_compatible_passes(self) -> None: + snap = _good_snapshot() + results = validate_keboola_repo(snap, type_="python-js", runtime_python_pin="3.12.10") + names = {r.name: r.severity for r in results} + assert names["golden-rule.requires-python"] == SEVERITY_OK + + +class TestPureValidatorPortMatch: + def test_mismatched_port_warns(self) -> None: + snap = _good_snapshot( + nginx_conf="server { proxy_pass http://localhost:5000; }\n", + app_conf="[program:app]\ncommand=python app.py --port 8000\n", + ) + results = validate_keboola_repo(snap, type_="python-js") + names = {r.name: r.severity for r in results} + assert names["golden-rule.nginx-app-port-match"] == SEVERITY_WARN + + +class TestPureValidatorTypeRestriction: + def test_streamlit_returns_only_blocking_meta(self) -> None: + results = validate_keboola_repo(_good_snapshot(), type_="streamlit") + assert len(results) == 1 + assert results[0].name == "meta.type-supported" + assert results[0].severity == SEVERITY_BLOCKING + + +class TestPureValidatorTruncated: + def test_truncated_warns(self) -> None: + snap = _good_snapshot() + snap.truncated = True + results = validate_keboola_repo(snap, type_="python-js") + names = {r.name: r.severity for r in results} + assert names["meta.tree-truncated"] == SEVERITY_WARN + + +# --------------------------------------------------------------------------- +# Service layer (with mocked GitHub client) +# --------------------------------------------------------------------------- + + +def _make_store(tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir() + return ConfigStore(config_dir=config_dir) + + +def _good_tree() -> dict[str, Any]: + return { + "tree": [ + {"path": "pyproject.toml", "type": "blob"}, + {"path": "keboola-config/nginx/sites/default.conf", "type": "blob"}, + {"path": "keboola-config/supervisord/services/app.conf", "type": "blob"}, + {"path": "keboola-config/setup.sh", "type": "blob"}, + {"path": "app.py", "type": "blob"}, + ], + "truncated": False, + } + + +class TestServiceURLParsing: + def test_non_github_host_rejected(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service = RepoValidateService(config_store=store) + with pytest.raises(KeboolaApiError) as exc: + service.validate_repo(git_repo="https://gitlab.com/owner/repo") + assert exc.value.error_code == ErrorCode.INVALID_ARGUMENT + + def test_strips_dot_git_suffix(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + github_mock = MagicMock() + github_mock.get_tree_recursive.return_value = _good_tree() + github_mock.get_file_content.return_value = "#!/bin/bash\ncd /app && uv sync\n" + service = RepoValidateService( + config_store=store, + github_client_factory=lambda token: github_mock, + ) + service.validate_repo(git_repo="https://github.com/o/r.git") + # owner/repo extracted correctly. + github_mock.get_tree_recursive.assert_called_once_with("o", "r", "main") + + def test_garbage_url_raises_invalid_format(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service = RepoValidateService(config_store=store) + with pytest.raises(KeboolaApiError) as exc: + service.validate_repo(git_repo="https://github.com/") + assert exc.value.error_code == ErrorCode.INVALID_FORMAT + + +class TestServiceTypeRestriction: + def test_streamlit_rejected_at_service_layer(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service = RepoValidateService(config_store=store) + with pytest.raises(KeboolaApiError) as exc: + service.validate_repo(git_repo="https://github.com/o/r", type_="streamlit") + assert exc.value.error_code == ErrorCode.INVALID_ARGUMENT + + +class TestServicePrivateRepo404Hint: + def test_404_without_pat_surfaces_private_repo_hint(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + github_mock = MagicMock() + github_mock.get_tree_recursive.side_effect = KeboolaApiError( + message="not found", status_code=404, error_code=ErrorCode.API_ERROR + ) + service = RepoValidateService( + config_store=store, + github_client_factory=lambda token: github_mock, + ) + with pytest.raises(KeboolaApiError) as exc: + service.validate_repo(git_repo="https://github.com/o/r") + assert exc.value.error_code == ErrorCode.VALIDATION_ERROR + assert "--git-pat-env" in exc.value.message + + +class TestServiceCallBudget: + def test_happy_path_uses_at_most_4_github_calls(self, tmp_path: Path) -> None: + """Per the plan: 1 trees-recursive + up to 3 contents.""" + store = _make_store(tmp_path) + github_mock = MagicMock() + github_mock.get_tree_recursive.return_value = _good_tree() + github_mock.get_file_content.return_value = "#!/bin/bash\ncd /app && uv sync\n" + service = RepoValidateService( + config_store=store, + github_client_factory=lambda token: github_mock, + ) + service.validate_repo(git_repo="https://github.com/o/r") + # 1 tree + 4 contents (setup.sh, pyproject.toml, nginx, app.conf). + # Spec says ≤4 in typical case but the port-match check fetches both + # nginx and app.conf -- still bounded. + assert github_mock.get_tree_recursive.call_count == 1 + assert github_mock.get_file_content.call_count <= 4 + + +class TestServiceOutputShape: + def test_returns_verdict_envelope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + github_mock = MagicMock() + github_mock.get_tree_recursive.return_value = _good_tree() + github_mock.get_file_content.return_value = "#!/bin/bash\ncd /app && uv sync\n" + service = RepoValidateService( + config_store=store, + github_client_factory=lambda token: github_mock, + ) + result = service.validate_repo(git_repo="https://github.com/o/r") + assert result["git_repo"] == "https://github.com/o/r" + assert result["type"] == "python-js" + assert "verdict" in result + assert "checks" in result + assert isinstance(result["checks"], list) From 19ab8fb90e4fb41a8520941454a80761abbfa951 Mon Sep 17 00:00:00 2001 From: Petr Simecek Date: Wed, 6 May 2026 20:38:17 +0200 Subject: [PATCH 5/5] fix(0.29.0): address /kbagent:review blockers + nits on PR #256 (#258) Blockers: - B-1: fix syntax error in tests/test_e2e.py -- Project invite E2E block was inserted inside an unclosed pytest.mark.skipif() tuple, causing 124 cascade syntax errors. Move the block after the closing parens of skip_without_data_app_private; deduplicate ENV_MANAGE_TOKEN. - B-2: bump pyproject.toml 0.28.0 -> 0.29.0; consolidate manage-token, data-app secrets/validate-repo, and project member entries into a new CHANGELOG '0.29.0' key (was incorrectly split across '0.28.0' and the non-monotone '0.26.1' key); make version-sync to update plugin.json and marketplace.json. - B-3: add E2E coverage for the five new data-app commands: TestE2EDataAppLifecycle.test_data_app_secrets_round_trip exercises set/list/get/remove against a real public app and asserts no plaintext secret leaks into any CLI output; test_data_app_validate_repo_against_public_repo runs against a public Keboola repo (skipped without E2E_DATA_APP_GIT_REPO_PUBLIC). Non-blocking: - NB-1: add TODO marker on services/repo_validate_service.py module docstring noting the future extraction of GitHubContentsClient into src/keboola_agent_cli/github_client.py to follow the 3-layer architecture. - NB-2: relabel manage-token + member-invite + data-app secrets/ validate-repo from '0.26.1+' / '0.28.0+' to '0.29.0+' across keboola-expert.md, gotchas.md, commands-reference.md, member-workflow.md, data-app-workflow.md, SKILL.md, TUTORIAL.md, e2e-scenarios.md, and AGENT_CONTEXT in commands/context.py. The legitimate 0.28.0 features (storage swap-tables, config update script[] auto-normalize) keep their original labels. Nits: - NIT-1: add the missing two blank lines before @pytest.mark.e2e_invite for test_project_invite_e2e. - NIT-2: move the in-function ConfigStore / ProjectConfig imports to the module-level import block. Bug fix uncovered while running tests: - commands/project.py member-* / invite / invitation-* commands called resolve_manage_token() without forwarding ctx.obj['allow_env_manage_token']; the env var was unconditionally ignored and the commands always exited 2 even when the user passed --allow-env-manage-token. Six call sites updated; tests now pass. --- .claude-plugin/marketplace.json | 2 +- docs/TUTORIAL.md | 10 +- docs/e2e-scenarios.md | 2 +- plugins/kbagent/.claude-plugin/plugin.json | 2 +- plugins/kbagent/agents/keboola-expert.md | 50 ++-- plugins/kbagent/skills/kbagent/SKILL.md | 2 +- .../kbagent/references/commands-reference.md | 12 +- .../kbagent/references/data-app-workflow.md | 6 +- .../skills/kbagent/references/gotchas.md | 28 +-- .../kbagent/references/member-workflow.md | 6 +- pyproject.toml | 2 +- src/keboola_agent_cli/changelog.py | 42 ++-- src/keboola_agent_cli/commands/context.py | 8 +- src/keboola_agent_cli/commands/project.py | 12 +- .../services/repo_validate_service.py | 8 + tests/test_e2e.py | 221 +++++++++++++++++- tests/test_member_cli.py | 12 + uv.lock | 2 +- 18 files changed, 327 insertions(+), 100 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 36d079a5..f1a50902 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "kbagent", - "version": "0.28.0", + "version": "0.29.0", "source": "./plugins/kbagent", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "category": "development" diff --git a/docs/TUTORIAL.md b/docs/TUTORIAL.md index f17d8272..3328dd78 100644 --- a/docs/TUTORIAL.md +++ b/docs/TUTORIAL.md @@ -102,7 +102,7 @@ use `org setup --project-ids`. kbagent creates a Storage API token in each listed project and registers them all locally, in parallel. ```bash -# Interactive (default since v0.28.0): kbagent prompts for the Manage API +# Interactive (default since v0.29.0): kbagent prompts for the Manage API # token on stdin. No env var, no shell history. kbagent org setup \ --project-ids 901,9621,10539 \ @@ -139,7 +139,7 @@ Flags worth knowing: The command is **idempotent**: running it again skips projects that are already registered. Safe to re-run after adding new project IDs. -**Security note (since v0.28.0)**: `KBC_MANAGE_API_TOKEN` is **ignored +**Security note (since v0.29.0)**: `KBC_MANAGE_API_TOKEN` is **ignored by default** -- the env var is read only when the top-level `--allow-env-manage-token` flag is passed. Without the flag, kbagent prompts on stdin (hidden input). kbagent never accepts the token as a @@ -156,7 +156,7 @@ If you are an org admin with a Manage API token, register **every** project in an organization in one shot: ```bash -# Interactive (default since v0.28.0): +# Interactive (default since v0.29.0): kbagent org setup \ --org-id 123 \ --url https://connection.keboola.com \ @@ -876,7 +876,7 @@ kbagent --json data-app create \ deploy. To retrieve it: ```bash -# Manage API token: interactive prompt by default (since v0.28.0). For CI, +# Manage API token: interactive prompt by default (since v0.29.0). For CI, # add `--allow-env-manage-token` and set KBC_MANAGE_API_TOKEN in env. kbagent --json data-app password \ --project prod --app-id 12345678 \ @@ -967,7 +967,7 @@ shapes, and the `--hint client|service` code-generation contract, see | `kbagent: command not found` after `uv tool install` | Ensure `~/.local/bin` (or uv's tool dir) is on your PATH. `uv tool update-shell` can help. | | `kbagent doctor` reports `warn` for plugin | Run the two `/plugin` commands shown in the warning, from inside Claude Code. | | Plugin version != CLI version | In Claude Code: `/plugin update kbagent`. | -| `org setup` exits 2 with `Warning: KBC_MANAGE_API_TOKEN found in environment but ignored` | Default-deny since v0.28.0 -- pass `--allow-env-manage-token` (top-level flag) to opt in to env resolution, or run interactively to use the prompt. | +| `org setup` exits 2 with `Warning: KBC_MANAGE_API_TOKEN found in environment but ignored` | Default-deny since v0.29.0 -- pass `--allow-env-manage-token` (top-level flag) to opt in to env resolution, or run interactively to use the prompt. | | `org setup` fails with `401 Unauthorized` | Your manage token is wrong for this stack or role. Manage tokens are stack-specific and require the right scope. | | `org setup --org-id` fails with `403` | You are not an org admin. Use `--project-ids` with a Personal Access Token instead (works for any project member). | | Changes from `kbagent config update` do not show in UI | You are on a dev branch. Run `kbagent branch list` and `kbagent project current` to verify the active branch; changes in a dev branch merge to production only via the UI merge step (`kbagent branch merge` returns the merge URL). | diff --git a/docs/e2e-scenarios.md b/docs/e2e-scenarios.md index e78cd87d..b5fd98cf 100644 --- a/docs/e2e-scenarios.md +++ b/docs/e2e-scenarios.md @@ -211,7 +211,7 @@ Skipped if `keboola-mcp-server` is not installed. | Command | Reason | |---------|--------| -| `project refresh` | Requires Manage API token (interactive prompt by default since v0.28.0; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for non-interactive runners) | +| `project refresh` | Requires Manage API token (interactive prompt by default since v0.29.0; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for non-interactive runners) | | `org setup` | Requires Manage API token (same prompt-or-flag as above) + destructive (registers projects in org) | | `sharing share/unshare` | Requires org-level permissions or second project | | `sharing link/unlink` | Requires shared bucket from another project | diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index 2d06ad7c..fde66363 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.28.0", + "version": "0.29.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 828a46cc..cd41b595 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -67,11 +67,11 @@ a critical failure. need 0.27.0+, `config update` script[] auto-normalize against #245 trap needs 0.28.0+, `storage swap-tables` needs 0.28.0+, env-var manage-token auth for `org setup` / `project refresh` / - `data-app password` needs 0.28.0+ with `--allow-env-manage-token` - (the env var is default-deny on 0.28.0+), + `data-app password` needs 0.29.0+ with `--allow-env-manage-token` + (the env var is default-deny on 0.29.0+), `project invite` / `project member-*` / `project invitation-*` - need 0.26.1+, - `data-app secrets-* / validate-repo` need 0.28.0+, + need 0.29.0+, + `data-app secrets-* / validate-repo` need 0.29.0+, `storage retype` is a future composite), you MUST refuse the task and return a handoff message to the parent: `"Cannot proceed safely on kbagent . Missing: . @@ -112,20 +112,20 @@ a critical failure. | Roll out a new code or config version on a data app | `kbagent data-app deploy --project P --app-id N --wait` (0.27.0+) -- always sends the §9 trio | `kbagent --hint client data-app deploy ...` to inspect the generated `patch_app(desired_state=, config_version=, restart_if_running=True)` call | `tool call update_config` then `tool call run_component` (data apps are not jobs -- the queue runner does not deploy them) | | Wake an auto-suspended data app | `kbagent data-app start --project P --app-id N` (0.27.0+) -- does NOT bump configVersion | hitting the app's URL (auto-restart triggers a 30-60s cold boot) | `kbagent data-app deploy` (overkill -- bumps the deployed configVersion unnecessarily) | | Pause a running data app | `kbagent data-app stop --project P --app-id N` (0.27.0+) | -- | `kbagent data-app delete` (irreversible; cascades to Storage config) | -| Read the simpleAuth password for a password-gated app | `kbagent data-app password --project P --app-id N` (0.27.0+) -- needs Manage API token (interactive prompt by default; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+) | -- | trying to "rotate" the password (not supported by the API; delete + recreate to mint a new one) | +| Read the simpleAuth password for a password-gated app | `kbagent data-app password --project P --app-id N` (0.27.0+) -- needs Manage API token (interactive prompt by default; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+) | -- | trying to "rotate" the password (not supported by the API; delete + recreate to mint a new one) | | Tear down a data app | `kbagent data-app delete --project P --app-id N` (0.27.0+) -- cascades to Storage config; URL retired | -- | manually `tool call delete_config keboola.data-apps` while leaving the deployment record orphaned | -| Invite a user to a project (single) | `kbagent project invite --project P --email E --role admin\|guest\|readOnly\|share` (0.26.1+) | raw `requests.post(/manage/projects/{id}/invitations)` only if version-gated out | `kbagent project invite` without `KBC_MANAGE_API_TOKEN` set; passing manage token via CLI flag | -| Invite many users (bulk) | `kbagent project invite --from-csv FILE [--default-role guest] [--workers N] [--dry-run]` (0.26.1+) | `--hint client` to generate a parallel script using `ManageClient` | per-row shell loop calling the CLI -- defeats the parallelism + idempotency the service already does | -| List active project members | `kbagent project member-list --project P [--include-pending]` (0.26.1+) | `tool call run_sync_action` against the Manage API | reading `.kbagent/config.json` to infer membership (it only stores the local user's token) | -| List pending invitations | `kbagent project invitation-list --project P` (0.26.1+) | -- | -- | -| Cancel a pending invitation | `kbagent project invitation-cancel --project P --email E --yes` (0.26.1+) | `--invitation-id ID` if email lookup is ambiguous | DELETE via raw HTTP without going through the service layer | -| Remove an active member | `kbagent project member-remove --project P --email E --yes` (0.26.1+, **destructive**) | `--hint client` for a script that removes by user_id directly | calling `member-remove` without `--yes` in non-interactive contexts (it will prompt and hang) | -| Change a member's role | `kbagent project member-set-role --project P --email E --role admin\|guest\|readOnly\|share` (0.26.1+) | -- | `PUT /manage/projects/{id}/users/{userId}` -- the API rejects PUT with 404, the kbagent client correctly uses **PATCH** | -| Set / rotate app-runtime secrets | `kbagent data-app secrets-set --project P --app-id N --secret '#KEY=VAL'` (0.28.0+) then `data-app deploy --wait` -- per-project KMS encryption, fail-closed, never auto-deploys | `kbagent encrypt values --component-id keboola.data-apps` + `tool call update_config` -- ONLY if you need to write secrets to a different shape than `parameters.dataApp.secrets` | raw `POST` to encryption + Storage without read-modify-write -- you will clobber sibling keys nested under `parameters.dataApp.secrets` (Storage `merge=True` is shallow at the top level only) | -| Inspect what secrets are set on a data app | `kbagent data-app secrets-list --project P --app-id N` (0.28.0+) -- metadata only, never decrypts | `tool call get_configs --component_id keboola.data-apps` then read `parameters.dataApp.secrets` keys (raw dict, no env-var derivation, may leak ciphertext into output) | trying to decrypt -- the Encryption API has no decrypt endpoint, the CLI cannot decrypt under any branch | -| Confirm one secret is present | `kbagent data-app secrets-get --project P --app-id N --key '#KEY'` (0.28.0+) -- returns metadata only | -- | trying to extract the plaintext value (impossible by design; not a CLI gap) | -| Remove a secret from a data app | `kbagent data-app secrets-remove --project P --app-id N --key '#KEY' --yes` (0.28.0+) -- idempotent; missing keys exit 0 with `removed: 0` | `tool call update_config` with the secrets sub-dict deleted -- ONLY for batch removes that need a custom change description | `kbagent config update --set 'parameters.dataApp.secrets={}'` -- replaces the whole sub-dict, dropping every secret instead of just the named ones | -| Pre-flight a data-app repo before create | `kbagent data-app validate-repo --git-repo URL --type python-js [--git-pat-env VAR]` (0.28.0+) -- BLOCKING / WARN / OK with help-doc citations; ≤5 GitHub API calls regardless of repo size | git-clone the repo locally and inspect by hand | `data-app create --dry-run` (only shows the request bodies; does not validate repo structure) | +| Invite a user to a project (single) | `kbagent project invite --project P --email E --role admin\|guest\|readOnly\|share` (0.29.0+) | raw `requests.post(/manage/projects/{id}/invitations)` only if version-gated out | `kbagent project invite` without `KBC_MANAGE_API_TOKEN` set; passing manage token via CLI flag | +| Invite many users (bulk) | `kbagent project invite --from-csv FILE [--default-role guest] [--workers N] [--dry-run]` (0.29.0+) | `--hint client` to generate a parallel script using `ManageClient` | per-row shell loop calling the CLI -- defeats the parallelism + idempotency the service already does | +| List active project members | `kbagent project member-list --project P [--include-pending]` (0.29.0+) | `tool call run_sync_action` against the Manage API | reading `.kbagent/config.json` to infer membership (it only stores the local user's token) | +| List pending invitations | `kbagent project invitation-list --project P` (0.29.0+) | -- | -- | +| Cancel a pending invitation | `kbagent project invitation-cancel --project P --email E --yes` (0.29.0+) | `--invitation-id ID` if email lookup is ambiguous | DELETE via raw HTTP without going through the service layer | +| Remove an active member | `kbagent project member-remove --project P --email E --yes` (0.29.0+, **destructive**) | `--hint client` for a script that removes by user_id directly | calling `member-remove` without `--yes` in non-interactive contexts (it will prompt and hang) | +| Change a member's role | `kbagent project member-set-role --project P --email E --role admin\|guest\|readOnly\|share` (0.29.0+) | -- | `PUT /manage/projects/{id}/users/{userId}` -- the API rejects PUT with 404, the kbagent client correctly uses **PATCH** | +| Set / rotate app-runtime secrets | `kbagent data-app secrets-set --project P --app-id N --secret '#KEY=VAL'` (0.29.0+) then `data-app deploy --wait` -- per-project KMS encryption, fail-closed, never auto-deploys | `kbagent encrypt values --component-id keboola.data-apps` + `tool call update_config` -- ONLY if you need to write secrets to a different shape than `parameters.dataApp.secrets` | raw `POST` to encryption + Storage without read-modify-write -- you will clobber sibling keys nested under `parameters.dataApp.secrets` (Storage `merge=True` is shallow at the top level only) | +| Inspect what secrets are set on a data app | `kbagent data-app secrets-list --project P --app-id N` (0.29.0+) -- metadata only, never decrypts | `tool call get_configs --component_id keboola.data-apps` then read `parameters.dataApp.secrets` keys (raw dict, no env-var derivation, may leak ciphertext into output) | trying to decrypt -- the Encryption API has no decrypt endpoint, the CLI cannot decrypt under any branch | +| Confirm one secret is present | `kbagent data-app secrets-get --project P --app-id N --key '#KEY'` (0.29.0+) -- returns metadata only | -- | trying to extract the plaintext value (impossible by design; not a CLI gap) | +| Remove a secret from a data app | `kbagent data-app secrets-remove --project P --app-id N --key '#KEY' --yes` (0.29.0+) -- idempotent; missing keys exit 0 with `removed: 0` | `tool call update_config` with the secrets sub-dict deleted -- ONLY for batch removes that need a custom change description | `kbagent config update --set 'parameters.dataApp.secrets={}'` -- replaces the whole sub-dict, dropping every secret instead of just the named ones | +| Pre-flight a data-app repo before create | `kbagent data-app validate-repo --git-repo URL --type python-js [--git-pat-env VAR]` (0.29.0+) -- BLOCKING / WARN / OK with help-doc citations; ≤5 GitHub API calls regardless of repo size | git-clone the repo locally and inspect by hand | `data-app create --dry-run` (only shows the request bodies; does not validate repo structure) | If the table does not cover the user's task, **ask clarifying questions** instead of guessing. Returning a targeted question is a @@ -201,7 +201,7 @@ success, not a failure. verification payload but do not treat it as a failure signal. Production writes never materialize anything. -- **`project invite` "already invited / already member" is a no-op, not a failure** (0.26.1+): +- **`project invite` "already invited / already member" is a no-op, not a failure** (0.29.0+): Re-inviting a user the project already knows returns HTTP 400 from the Manage API. kbagent normalises both "...already been invited..." and "...already a member..." to `status="noop"` with a `note` field, exit 0. @@ -210,7 +210,7 @@ success, not a failure. toward `noop`, not `failed`, in the summary; surface that distinction to the user when reporting bulk results. -- **`project invite --from-csv` ordering is non-deterministic** (0.26.1+): +- **`project invite --from-csv` ordering is non-deterministic** (0.29.0+): Bulk invitation parallelises via `ThreadPoolExecutor` (default 8 workers). The `rows[]` array in the JSON result is in completion order, not CSV order. When reporting per-row outcomes to the user, **match by `email`, @@ -218,7 +218,7 @@ success, not a failure. the JSON -- treat that as a soft failure that needs review, not a catastrophe. -- **`project member-set-role` uses PATCH, not PUT** (0.26.1+): The Manage +- **`project member-set-role` uses PATCH, not PUT** (0.29.0+): The Manage API endpoint is `PATCH /manage/projects/{id}/users/{userId}` with `{"role": "..."}`. PUT returns 404 even on real members. kbagent's `ManageClient.update_project_member_role` emits PATCH; if you write a @@ -298,13 +298,13 @@ success, not a failure. the round-trip does not return a `KBC::Project*` ciphertext. - **`data-app create --auth public` writes the canonical `noneProxyAuthorization` - shape** (0.28.0+, fixes a v0.27.0 silent-503 bug): v0.27.0 wrote NO + shape** (0.29.0+, fixes a v0.27.0 silent-503 bug): v0.27.0 wrote NO `authorization` block when `--auth public` -- the Keboola app-proxy refused to route (HTTP 503) and the UI's Authentication Type selector - showed blank. v0.28.0 writes + showed blank. v0.29.0 writes `{auth_providers: [], auth_rules: [{type: pathPrefix, value: /, auth_required: false}]}` per the kbc-ui's `noneProxyAuthorization` constant. If a user reports a - v0.27.0 public app returning 503, the fix is to recreate on 0.28.0+ + v0.27.0 public app returning 503, the fix is to recreate on 0.29.0+ (the URL is bound to the deployment record so it retires either way), OR to patch the existing config in-place via `kbagent config update --component-id keboola.data-apps --config-id ID @@ -313,7 +313,7 @@ success, not a failure. JumpCloud / Auth0) are not yet exposed by the CLI; tracked as a follow-up issue. -- **`data-app secrets-* metadata-only`** (0.28.0+): `secrets-get` NEVER +- **`data-app secrets-* metadata-only`** (0.29.0+): `secrets-get` NEVER echoes the decrypted plaintext under any branch -- the Encryption API is one-way and the CLI does not attempt to decrypt. NOT_FOUND on an absent key never enumerates sibling keys (avoids leaking neighbour @@ -328,7 +328,7 @@ success, not a failure. `parameters.dataApp.secrets`). - **`data-app validate-repo` is GitHub-only**, `--type python-js` only - (0.28.0+): pre-flight Golden-Rule check via the GitHub Trees+Contents + (0.29.0+): pre-flight Golden-Rule check via the GitHub Trees+Contents API. Total <=5 calls regardless of repo size. Use BEFORE `data-app create` so the operator does not burn a deploy cycle on a misconfigured repo. WARNs are advisory unless `--strict` is set; diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index d83cbfa8..649656aa 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -290,7 +290,7 @@ kbagent --json project add --project prod --url https://connection.keboola.com - # Or bulk-onboard from organization (org admin) # Manage token: interactive prompt by default; for CI add --allow-env-manage-token -# alongside KBC_MANAGE_API_TOKEN (required since v0.28.0). +# alongside KBC_MANAGE_API_TOKEN (required since v0.29.0). KBC_MANAGE_API_TOKEN=xxx kbagent --allow-env-manage-token --json org setup --org-id 123 --url https://connection.keboola.com --yes # Or onboard specific projects (any project member, uses Personal Access Token) diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 30756e03..91ad739a 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -21,7 +21,7 @@ All commands support `--json` for structured output. Multi-project flags (`--pro - `project use ALIAS` -- pin `ALIAS` as the persistent default project. Stored as `default_project` in config.json. Overridden at runtime by `KBAGENT_PROJECT=ALIAS` (env, beats pin) and by `--project ALIAS` (CLI flag, beats both) - `project current` -- print the effective default project and its source (`env` / `pin` / `none`). Reports both the env override AND the persisted pin so misconfigurations are visible. Returns `{"alias": null, "source": "none"}` when neither is set -## Project Members & Invitations (since v0.26.1) +## Project Members & Invitations (since v0.29.0) All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the project's Storage token. Allowed roles are exactly `admin`, `guest`, `readOnly`, `share` -- the API self-reports this list in its 400 validation error and `constants.PROJECT_ROLES` mirrors it. @@ -31,17 +31,17 @@ All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the - `project invitation-list --project ALIAS` -- list pending (unaccepted) invitations only. - `project invitation-cancel --project ALIAS --email EMAIL [--invitation-id ID] [--yes]` -- cancel a pending invitation. Without `--invitation-id`, the service resolves it by listing pending invitations and matching `--email` (case-insensitive). 204 No Content on success; `KeboolaApiError(NOT_FOUND)` if the email has no pending invitation. - `project member-remove --project ALIAS --email EMAIL [--yes]` -- destructive: remove an active member. The service resolves `--email` to the numeric `user_id` (case-insensitive) and DELETEs `/manage/projects/{id}/users/{userId}`. Re-add the user via `project invite`. -- `project member-set-role --project ALIAS --email EMAIL --role admin|guest|readOnly|share` -- change an existing member's role. Uses **PATCH** `/manage/projects/{id}/users/{userId}` with `{"role": "..."}`. PUT does *not* work on this endpoint -- pre-v0.26.1 implementations that tried PUT got a misleading 404. +- `project member-set-role --project ALIAS --email EMAIL --role admin|guest|readOnly|share` -- change an existing member's role. Uses **PATCH** `/manage/projects/{id}/users/{userId}` with `{"role": "..."}`. PUT does *not* work on this endpoint -- pre-v0.29.0 implementations that tried PUT got a misleading 404. ## Permission flags (top-level, session-only) - `--deny-writes` -- block all write/destructive/admin operations for this single invocation. Merges with any persisted permission policy; never written to config.json. Exit code 6 (PERMISSION_DENIED) on blocked operations - `--deny-destructive` -- block only destructive operations (delete-table, delete-bucket, terminate-job, etc.) for this invocation. Pure-write ops like create-table stay allowed. Use this when you want to keep build-up capabilities but lock out tear-downs -- `--allow-env-manage-token` -- opt in to reading `KBC_MANAGE_API_TOKEN` from env (default-deny since v0.28.0). Without it the env var is ignored and an interactive hidden prompt is required for `org setup` / `project refresh` / `data-app password`. Closes the AI-exfiltration risk where any subprocess inherits the manage token via env. Session-only; not persisted; no env-var equivalent (intentional, would re-create the hole). REPL forwards this flag to nested invocations the same way it forwards the deny-* flags +- `--allow-env-manage-token` -- opt in to reading `KBC_MANAGE_API_TOKEN` from env (default-deny since v0.29.0). Without it the env var is ignored and an interactive hidden prompt is required for `org setup` / `project refresh` / `data-app password`. Closes the AI-exfiltration risk where any subprocess inherits the manage token via env. Session-only; not persisted; no env-var equivalent (intentional, would re-create the hole). REPL forwards this flag to nested invocations the same way it forwards the deny-* flags - All three flags compose: `kbagent --deny-writes --deny-destructive --allow-env-manage-token ...` is the safest CI-friendly invocation ## Organization -- `org setup --org-id ID --url URL [--dry-run] [--yes]` -- bulk-onboard all projects from an org (org admin; manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+) -- `org setup --project-ids 1,2,3 --url URL [--dry-run] [--yes]` -- onboard specific projects by ID (any project member; manage token / Personal Access Token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+) +- `org setup --org-id ID --url URL [--dry-run] [--yes]` -- bulk-onboard all projects from an org (org admin; manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+) +- `org setup --project-ids 1,2,3 --url URL [--dry-run] [--yes]` -- onboard specific projects by ID (any project member; manage token / Personal Access Token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+) ## Component Discovery - `component list [--project NAME] [--type TYPE] [--query "text"]` -- list/search components (AI-powered with `--query`) @@ -139,7 +139,7 @@ Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, - `data-app start --project NAME --app-id ID [--wait] [--timeout SECONDS]` -- wake an auto-suspended app at the currently-pinned version. Distinct from deploy: does NOT bump configVersion. - `data-app stop --project NAME --app-id ID [--wait] [--timeout SECONDS]` -- stop a running app (URL and Storage config preserved). - `data-app delete --project NAME --app-id ID [--yes]` -- destructive, cascades to Storage config; URL retired permanently. -- `data-app password --project NAME --app-id ID` -- read the simpleAuth password. Manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.28.0+. Auto-generated, not rotatable -- delete + recreate to mint a new one. +- `data-app password --project NAME --app-id ID` -- read the simpleAuth password. Manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+. Auto-generated, not rotatable -- delete + recreate to mint a new one. - `data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE' [--secret ...] [--secrets-file PATH] [--branch ID] [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next]` -- encrypt and write `#`-prefixed secrets to `parameters.dataApp.secrets`. Per-project KMS encryption, fail-closed. Read-modify-write at the service layer (NOT Storage `merge=True` -- shallow). Runtime exposes each key as an env var with `#` stripped, `-` -> `_`, uppercased. Adding bumps the Storage version; the running container keeps the OLD config until the next `data-app deploy`. - `data-app secrets-list --project ALIAS --app-id ID [--branch ID] [--show-fingerprint]` -- list secret keys + derived runtime env-var names. Never echoes encrypted ciphertext in full. `--show-fingerprint` opt-in for a short ciphertext fingerprint. - `data-app secrets-get --project ALIAS --app-id ID --key '#KEY' [--branch ID]` -- show metadata for ONE secret. NEVER echoes the decrypted value (Encryption API is one-way). NOT_FOUND on absent key; never enumerates siblings. diff --git a/plugins/kbagent/skills/kbagent/references/data-app-workflow.md b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md index ffbd40a6..3a8956b6 100644 --- a/plugins/kbagent/skills/kbagent/references/data-app-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md @@ -58,7 +58,7 @@ with: ```bash kbagent data-app password --project prod --app-id -# Manage token: interactive prompt by default (since v0.28.0); for CI add +# Manage token: interactive prompt by default (since v0.29.0); for CI add # --allow-env-manage-token alongside KBC_MANAGE_API_TOKEN. Storage token # is read from .kbagent/config.json as usual. ``` @@ -116,7 +116,7 @@ kbagent data-app deploy --project prod --app-id 12345678 \ (rollback). Subsequent deploys without the flag will jump back to the latest. -### Pre-flight repo validation (since v0.28.0) +### Pre-flight repo validation (since v0.29.0) ```bash kbagent data-app validate-repo \ @@ -136,7 +136,7 @@ repo. Public repos: drop `--git-pat-env` and use `--git-public`. Total GitHub call budget per run is ≤5 (1 tree + ≤4 contents) regardless of repo size, so the 60/hour unauth limit rarely fires; pass a PAT for CI loops. -### Manage app-runtime secrets (since v0.28.0) +### Manage app-runtime secrets (since v0.29.0) ```bash # Set two secrets at once. Plaintext values; the CLI encrypts under diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index a68b51b0..529fce82 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -17,16 +17,16 @@ swaps it back into the original name. After merging the branch the original table now carries the typed schema with no downstream config rewrite required. -## `data-app create --auth public` writes the canonical noneProxyAuthorization shape (since v0.28.0; fixes v0.27.0 silent HTTP 503) +## `data-app create --auth public` writes the canonical noneProxyAuthorization shape (since v0.29.0; fixes v0.27.0 silent HTTP 503) - **What changed.** v0.27.0's `--auth public` wrote NO `authorization` key into the Storage config at all. The Keboola app-proxy refused to route to the resulting URL (HTTP 503 / "Service Unavailable") and the UI's "Authentication Type" selector showed blank. Operators got a - silently broken app. v0.28.0 fixes this: `--auth public` now writes + silently broken app. v0.29.0 fixes this: `--auth public` now writes the canonical `noneProxyAuthorization` shape that the kbc-ui exports for the "None" UI option. -- **Exact shape written by 0.28.0:** +- **Exact shape written by 0.29.0:** ```json { "app_proxy": { @@ -62,7 +62,7 @@ flag. Use the Keboola UI to configure them after `data-app create`. Tracked as a follow-up issue. -## `data-app secrets-*` -- per-project KMS, idempotent remove, never decryptable (since v0.28.0) +## `data-app secrets-*` -- per-project KMS, idempotent remove, never decryptable (since v0.29.0) - **Encryption is per-project KMS.** `kbagent data-app secrets-set` calls the project's Encryption API to wrap each plaintext value before @@ -110,7 +110,7 @@ field with the exact redeploy command to run; suppress it with `--no-hint-next` for scripted callers. -## `data-app validate-repo` -- pre-flight against the Golden Rule, GitHub-only (since v0.28.0) +## `data-app validate-repo` -- pre-flight against the Golden Rule, GitHub-only (since v0.29.0) - `kbagent data-app validate-repo --git-repo URL` walks the repo via the GitHub Contents + Trees API and verifies the documented "Golden Rule" @@ -120,7 +120,7 @@ GitHub API calls regardless of repo size (one trees-recursive + up to four contents fetches), so the 60/hour unauthenticated GitHub rate limit is no longer the common-case failure mode. -- **`--type` is restricted to `python-js` in 0.28.0.** Streamlit / +- **`--type` is restricted to `python-js` in 0.29.0.** Streamlit / pure-Python / R / Node-only repos have different layouts (Streamlit does not require the `keboola-config/` tree, for instance) and need per-type canon citations. Tracked as a follow-up. @@ -137,12 +137,12 @@ [issue #240](https://github.com/padak/keboola_agent_cli/issues/240) (needs platform-side API exposure first). -## Manage token: env var is ignored without `--allow-env-manage-token` (since v0.28.0) +## Manage token: env var is ignored without `--allow-env-manage-token` (since v0.29.0) - `KBC_MANAGE_API_TOKEN` is no longer auto-resolved on the three surfaces that consume it (`kbagent org setup`, `kbagent project refresh`, `kbagent data-app password`). Default - behaviour on 0.28.0+ is **default-deny**: the env var is ignored, a + behaviour on 0.29.0+ is **default-deny**: the env var is ignored, a TTY hidden-input prompt is shown instead. With no TTY (CI / cron / systemd / `< /dev/null`) the resolver exits **2** with the message `Error: No manage token available. Run interactively, or pass @@ -223,7 +223,7 @@ container after `autoSuspendAfterSeconds` of inactivity. Hit the URL to wake it (auto-restart triggers a 30-60s cold boot) or run `kbagent data-app start --app-id N`. -## `project invite` "already invited / already member" returns HTTP 400, not 422 (since v0.26.1) +## `project invite` "already invited / already member" returns HTTP 400, not 422 (since v0.29.0) - Re-inviting a user the project already knows about returns HTTP **400** with one of two error strings: @@ -233,11 +233,11 @@ `note="already_invited"` / `"already_member"` -- they are *not* exit-1 failures. Bulk runs (`--from-csv`) count them as `noop` in the summary, not `failed`. -- The 422 heuristic in pre-v0.26.1 orchestrator scripts (`invite_participants.py:25`) +- The 422 heuristic in pre-v0.29.0 orchestrator scripts (`invite_participants.py:25`) is **wrong** for this API. If you write a parallel implementation, key off status_code 400 + the substring marker, not 422. -## `project member-set-role` is PATCH, not PUT (since v0.26.1) +## `project member-set-role` is PATCH, not PUT (since v0.29.0) - The Manage API role-change endpoint is `PATCH /manage/projects/{id}/users/{userId}` with body `{"role": "..."}`. **PUT returns 404** ("resource not found") even @@ -245,7 +245,7 @@ - The kbagent `ManageClient.update_project_member_role` method emits PATCH; any custom code re-implementing the call must do the same. -## `project invite --from-csv` order is not deterministic (since v0.26.1) +## `project invite --from-csv` order is not deterministic (since v0.29.0) - Bulk invitation parallelises via `ThreadPoolExecutor` (default 8 workers). The `rows[]` array in the result is in completion order, not CSV order. @@ -570,7 +570,7 @@ type inventory and examples. - Tokens are always masked in output (e.g. `901-...pt0k`) -- this is normal - Token can be passed via `--token`, `KBC_TOKEN` env var, or interactive prompt -- Manage API token (since v0.28.0): default-deny on env -- via interactive hidden prompt; pass top-level `--allow-env-manage-token` to opt in to `KBC_MANAGE_API_TOKEN`. Never as CLI argument. See the `(since v0.28.0)` entry at the top of this file. +- Manage API token (since v0.29.0): default-deny on env -- via interactive hidden prompt; pass top-level `--allow-env-manage-token` to opt in to `KBC_MANAGE_API_TOKEN`. Never as CLI argument. See the `(since v0.29.0)` entry at the top of this file. - Master token for sharing: `KBC_MASTER_TOKEN_{ALIAS}` (e.g. `KBC_MASTER_TOKEN_PROD`) or `KBC_MASTER_TOKEN` as global fallback. Alias is uppercased, hyphens become underscores. Required for `sharing share` and `sharing unshare`; `sharing list/link/unlink` use regular project tokens. ## MCP tool call gotchas @@ -980,7 +980,7 @@ See [docs/hint-mode.md](../../../../../docs/hint-mode.md) for full documentation - **Forgetting `--json`**: without it, output is human-formatted Rich text, not parseable - **Assuming `data.projects`**: `project list` returns data as a flat list -- **Passing manage token as argument**: use the interactive prompt (default since v0.28.0), or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` env var for CI +- **Passing manage token as argument**: use the interactive prompt (default since v0.29.0), or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` env var for CI - **Polling after branch create**: kbagent already waits for async completion - **Not saving workspace password**: only returned once on creation - **Putting SQL in _config.yml**: SQL transformations must use `transform.sql` with block markers (see above) diff --git a/plugins/kbagent/skills/kbagent/references/member-workflow.md b/plugins/kbagent/skills/kbagent/references/member-workflow.md index 218172bf..061c67cc 100644 --- a/plugins/kbagent/skills/kbagent/references/member-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/member-workflow.md @@ -1,4 +1,4 @@ -# Project Member & Invitation Workflow (since v0.26.1) +# Project Member & Invitation Workflow (since v0.29.0) Closes the long-standing Manage API gap that forced every Keboola-internal automation (most recently the Cuesta-training orchestrator) to bypass kbagent @@ -161,11 +161,11 @@ Permission category: `destructive` (re-adding requires sending a fresh invite). | HTTP 403 manage token lacks org-admin | `KeboolaApiError(ACCESS_DENIED)` | 1 | | HTTP 404 project / invitation not found | `KeboolaApiError(NOT_FOUND)` | 1 | -## When to use the Manage API direct-add (not in v0.26.1) +## When to use the Manage API direct-add (not in v0.29.0) The Manage API also exposes `POST /manage/projects/{id}/users` with body `{"email": "...", "role": "..."}`. This **directly creates a member without sending an email** -- useful for org-internal automation, dangerous for -public-facing flows. v0.26.1 deliberately does NOT expose this path because +public-facing flows. v0.29.0 deliberately does NOT expose this path because its semantics differ from `invite`. If you need it, talk to the maintainers about a future `member-add-direct` command. diff --git a/pyproject.toml b/pyproject.toml index a4ff6b89..4952d430 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-agent-cli" -version = "0.28.0" +version = "0.29.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index f0df35d1..b1dc6d09 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -8,6 +8,27 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.29.0": [ + "BREAKING: `KBC_MANAGE_API_TOKEN` is now ignored by default. The three commands that consume it (`org setup`, `project refresh`, `data-app password`) prompt for the token on a TTY by default. Pass the new top-level flag `--allow-env-manage-token` to restore the legacy env-var behaviour (e.g. for CI/CD). Without the flag and without a TTY, the resolver exits 2 with an actionable message naming the flag. The change closes the AI-exfiltration risk where any subprocess running as the same user (including the AI agent itself) inherits the manage token via env. Migration: prepend `--allow-env-manage-token` to existing CI invocations. Storage tokens (`KBC_TOKEN`) are unaffected. Closes the manage-token UX flagged on #236; supersedes the per-stack design discussed in #238.", + "Security: `resolve_manage_token` (`src/keboola_agent_cli/commands/_helpers.py`) refactored to default-deny env, TTY-first. When the env var is set but the flag is not passed, a one-shot stderr warning fires (`Warning: KBC_MANAGE_API_TOKEN found in environment but ignored. Pass --allow-env-manage-token to opt in.`) and the resolver falls through to the TTY prompt. No cache, no keyring, no temp file -- next invocation prompts again. The bulk-prompt-once contract (`project refresh --all`) is preserved by construction: the resolver lives at command entry, before any per-project loop.", + "New: top-level CLI flag `--allow-env-manage-token` (session-only, mirrors `--deny-writes` / `--deny-destructive`). Plumbed via `ctx.obj['allow_env_manage_token']` and forwarded by the three call sites into `resolve_manage_token(allow_env=...)`. Not persisted, no env-var equivalent (intentional; an env-var equivalent would re-create the AI-exfiltration hole this default-deny is closing).", + "Tests: 12 new (`tests/test_helpers.py::TestResolveManageToken` x7 covering allow_env-True/False x env-set/unset x TTY/non-TTY combinations + token-leak regression pin; `tests/test_manage_token_cli.py::TestAllowEnvManageTokenFlag` x4 covering project-refresh / org-setup / data-app-password through CliRunner with services mocked; `tests/test_manage_token_bulk.py::TestBulkPromptOnce` pinning the contract that `project refresh --all` resolves the token exactly once at command entry, not per-project).", + "Docs: `commands/context.py` AGENT_CONTEXT updated (org-setup example + env-var help block); `CLAUDE.md` convention #12 + global-flag list; `keboola-expert.md` Rule 6 VERSION GATE adds the 0.29.0+ env-flag requirement, tool-selection-matrix updated, new inline-gotcha block; `gotchas.md` new `(since v0.29.0)` entry naming the warning text and the one-line CI fix; `commands-reference.md` updated for `org setup`, `data-app password`, env-var table.", + "New: project member & invitation lifecycle. Closes the long-standing Manage API gap that forced every Keboola-internal automation (most recently `17_CuestaDemo/scripts/replicate_master.py` and `invite_participants.py`) to bypass kbagent and POST raw HTTP at `/manage/projects/{id}/invitations`. Seven new commands under `kbagent project`: `invite` (single-shot or `--from-csv` bulk with `ThreadPoolExecutor` parallelism, default 8 workers), `member-list` (active members, `--include-pending` adds pending invitations), `invitation-list`, `invitation-cancel` (resolves invitation_id by email lookup so callers don't have to), `member-remove` (destructive; resolves user_id by email), `member-set-role` (PATCH `/manage/projects/{id}/users/{userId}` with `{role}`). All seven require `KBC_MANAGE_API_TOKEN`; the manage token is never logged, never persisted, never accepted on the CLI line. Permission registry: `member-remove` is `destructive`, `member-list` / `invitation-list` are `read`, the rest are `admin`.", + "New: role whitelist `PROJECT_ROLES = ('admin', 'guest', 'readOnly', 'share')` in `constants.py`, lifted verbatim from the Manage API's own validation error message (verified empirically on 2026-05-01 against `connection.us-east4.gcp.keboola.com`). Typer enforces the whitelist via `click.Choice` at the command layer; `MemberService` double-checks for defence-in-depth. Invalid role values now fail-fast with `Role 'X' is not valid. Allowed roles are: admin, guest, readOnly, share` instead of letting the API return an opaque 400.", + "New: `MemberService` (`src/keboola_agent_cli/services/member_service.py`) wrapping six new `ManageClient` methods (`create_project_invitation`, `list_project_invitations`, `cancel_project_invitation`, `list_project_members`, `remove_project_member`, `update_project_member_role`). Resolves project alias -> (stack_url, project_id) via `ConfigStore`; resolves email -> numeric user_id / invitation_id by listing + matching case-insensitively. Treats the Manage API's HTTP 400 'already been invited' / 'already a member' responses as `status=noop` rather than errors (the heuristic the orchestrator scripts had to do via substring matching, now typed to `status_code == 400` AND message-substring marker constants). `--from-csv` enforces a single-stack-URL invariant per file (rows referencing multiple stacks raise `ConfigError` upfront).", + "New: hint definitions (`hints/definitions/member.py`) for all seven commands. Both `--hint client` (direct `ManageClient` calls) and `--hint service` (`MemberService` calls) generate runnable Python.", + "New: e2e marker `e2e_invite` (registered in `pyproject.toml`). `make test-e2e-invite` runs `tests/test_e2e.py::test_project_invite_e2e` against a real Manage API; gated on `E2E_MANAGE_TOKEN` + `E2E_INVITE_PROJECT_ID` (skips cleanly when missing). The test invites `ottomansky.max@gmail.com` (override via `E2E_INVITE_EMAIL`) as `guest`, asserts the invitation appears in `invitation-list`, then cancels it -- the same run that proves the system can send confirms it can clean up.", + "Docs (members): new `references/member-workflow.md` (golden paths for single invite, bulk invite, audit, role change, remove). `gotchas.md` gains three `(since v0.29.0)` entries -- 'already invited / already member' returns HTTP 400 not 422; role-change is PATCH not PUT (PUT returns 404 even on real members); bulk-invite ordering is not deterministic (parallel workers). `keboola-expert.md` adds seven matrix rows under 'Project administration' plus a Rule 6 VERSION GATE entry. `commands-reference.md` adds a 'Project members & invitations' section.", + "New: `kbagent data-app secrets-set / secrets-list / secrets-get / secrets-remove` — manage `#`-prefixed app-runtime secrets in `parameters.dataApp.secrets`. Encryption is per-project KMS via the existing `EncryptService` (same fail-closed semantics as `--git-pat-encrypted`: refuses to write plaintext if the Encryption API does not return a project-scoped ciphertext). Read-modify-write at the service layer (NOT Storage `merge=True` — that flag is shallow at the top level only and would clobber sibling keys nested inside `parameters.dataApp.secrets`). The runtime exposes each key as an env var with `#` stripped, `-` replaced with `_`, and uppercased (`#my-api-key` → `MY_API_KEY` per help.keboola.com/data-apps/python-js/). `secrets-get` is metadata-only — never echoes decrypted plaintext to stdout / stderr / logs / change descriptions; the Encryption API is one-way and the CLI does not attempt to decrypt under any branch. `secrets-remove` is idempotent (missing keys exit 0 with `removed: 0`). `secrets-set` warns when a derived env-var name collides with `RESERVED_RUNTIME_ENV_VARS` (KBC_TOKEN, KBC_URL — verified canon floor; full runtime list TODO follow-up). Adding/removing a secret bumps the Storage version but the running container keeps the OLD config until the next `data-app deploy`.", + "New: `kbagent data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] [--type python-js] [--strict]` — pre-flight check that a git repo follows the documented Golden Rule (https://help.keboola.com/data-apps/python-js/) BEFORE `data-app create` so operators don't burn a deploy cycle on a misconfigured repo. Each check emits BLOCKING / WARN / OK with a help-doc citation: `keboola-config/nginx/sites/default.conf` exists, `keboola-config/supervisord/services/app.conf` exists, `pyproject.toml` at root, `keboola-config/setup.sh` content has no `pip install` (BLOCKING per the help canon's pip prohibition) and contains `uv sync` if `pyproject.toml` declares deps, `requires-python` consistent with the runtime image (when the pin is available), nginx `proxy_pass` port matches `app.conf` declared port. Uses `GET /repos/{owner}/{repo}/git/trees/{ref}?recursive=1` (one call) + up to 4 `GET .../contents/{path}` for files whose contents the rules need to inspect — total ≤5 GitHub API calls (1 tree + 0-4 contents) regardless of repo size, sidesteps the 60/hour unauth rate limit for typical use. `--git-pat-env` / `--git-pat-file` raises the limit to 5,000/hour. Read-only; never touches a Keboola project. `--type` is restricted to `python-js` in 0.29.0; streamlit / pure-Python / R / Node-only follow-up.", + "New: `RepoValidateService` (`src/keboola_agent_cli/services/repo_validate_service.py`) — pure validation function `validate_keboola_repo(snapshot, type_, runtime_python_pin)` plus a tiny `GitHubContentsClient` (HTTPS GET to `api.github.com`, optional bearer PAT, no token persistence). Service module is the only place GitHub HTTP lives; the rest of kbagent stays Keboola-API-only. (Future refactor: extract to `src/keboola_agent_cli/github_client.py` to follow the existing 3-layer architecture; `github_client_factory` injection preserves test coverage today.)", + "New: `ErrorCode` entries `DATA_APP_INVALID_SECRET`, `DATA_APP_INVALID_REPO`, `DATA_APP_REPO_VALIDATION_BLOCKING`. Permission registry entries `data-app.secrets-set` (write), `data-app.secrets-list` / `data-app.secrets-get` (read), `data-app.secrets-remove` (destructive — removing a secret can break a running app), `data-app.validate-repo` (read).", + "New: `--hint client/service` for all five new data-app commands. `secrets-get` hint snippet asserts the metadata-only contract; `validate-repo` snippet uses `RepoValidateService.validate_repo(...)` and the hint comment notes that GitHub-side detail is not shown.", + "Fix: `kbagent data-app create --auth public` now writes the canonical `noneProxyAuthorization` shape (kbc-ui exact constant: `auth_providers: []` + `auth_rules: [{type: pathPrefix, value: /, auth_required: false}]`). v0.27.0 wrote NO `authorization` key when `--auth public`, leaving the Keboola app-proxy unable to route (HTTP 503) and the UI Authentication Type selector blank — silently broken. Authoritative source: the public backend validator at `keboola/job-queue-job-configuration` `AppProxyDefinition.php` (when `auth_required=false`, `auth` MUST NOT be set). The private `keboola/ui` repo `apps/kbc-ui/src/scripts/modules/data-apps/constants.ts` corroborates: its `noneProxyAuthorization` constant exports this exact shape for the None UI option (Keboola org members can verify; external readers rely on the validator). Live-validated end-to-end on a real project: HTTP 200 on the resulting URL, written block bit-identical to canon, UI auth selector now shows None pre-selected. Existing `--auth password` behaviour unchanged.", + "Tests (data-app secrets / validate-repo): 27 secrets service tests + 20 validate-repo service tests + 22 CLI tests (13 secrets/validate-repo CLI methods + 9 hint-compile AST-parse cases) + 4 new auth-block tests (`TestDataAppCreateAuthBlock` asserts both `--auth public` and `--auth password` write the canonical shape on POST `/apps` AND PUT Storage). E2E coverage in `tests/test_e2e.py::TestE2EDataAppLifecycle::test_data_app_secrets_round_trip` and `::test_data_app_validate_repo_against_public_repo` exercises the full path. Sibling-preservation regression test for `secrets-set` asserts every untouched key under `parameters.dataApp.secrets`, `parameters.dataApp` (slug, git block), `parameters` (id), and the top-level config (`runtime`, `authorization`, `storage`) is preserved bit-identical after the read-modify-write.", + 'Plugin: `keboola-expert.md` matrix gains five new data-app rows (one per `secrets-set / -list / -get / -remove + validate-repo`); §1 Rule 6 VERSION GATE example updated for `secrets / validate-repo need 0.29.0+`. New `(since v0.29.0)` `gotchas.md` entries: (a) secrets are per-project KMS encrypted, `secrets-remove` on missing key is exit 0, `secrets-get` never echoes decrypted plaintext, `#KBC_TOKEN` is silently shadowed by the runtime; (b) `validate-repo` GitHub-only Golden-Rule check; (c) `--auth public` writes the canonical `noneProxyAuthorization` shape (fixes v0.27.0 silent 503). New "Managing app-runtime secrets" + "Pre-flight repo validation" recipe sections in `data-app-workflow.md`. Logs / auto-log-dump deferred to issue #240 (the Data Science API does not expose Terminal Logs as JSON per help canon).', + ], "0.28.0": [ 'Fix: `kbagent config update` now auto-normalizes `parameters.blocks[].codes[].script` from string to array before pushing to the Storage API. Closes #245. The Storage API silently accepts a string for `script` while the runtime schema validator requires an array (`Invalid type for path "root.parameters.blocks.0.codes.X.script". Expected "array", but got "string"`); the broken push lands silently and crashes only at job-run time, often hours later, with no attribution back to the offending write. The CLI now closes the gap on the write side: SQL transformations (`keboola.snowflake-transformation`, `keboola.synapse-transformation`, `keboola.oracle-transformation`, `keboola.redshift-sql-transformation`, `keboola.google-bigquery-transformation`, `keboola.duckdb-transformation`, plus fragment-fallback for self-hosted variants like `*-exasol-transformation` / `*-teradata-transformation`) get statement-level split via the existing `split_statements()` state-machine (respects `\'...\'` / `"..."` / `$$...$$` / `--` / `#` / `//` / `/* ... */`); Python / R / `kds-team.app-custom-python` and any other component sharing the schema get a single-element array wrap. Already-array `script` values pass through unchanged.', 'Observability: every normalization is surfaced -- the JSON envelope gains a `normalizations: [{path, action: "sql_split"|"wrap_array", before_type, after_type, after_length}]` field per write (and on `--dry-run` the `new_configuration` reflects the post-normalize shape). Human mode prints a yellow `Auto-normalized N script field(s) to array (string -> list). See --json for details.` warning followed by a per-element trace, so the silent fix is observable to operators and AI agents alike. Default behaviour is silent normalize -- the issue\'s preferred design -- because the Keboola UI splitter and `keboola-as-code` produce the same array shape kbagent now writes; the audit fields exist precisely so callers who want to detect "my agent produced a string" can.', @@ -17,19 +38,6 @@ "New: `kbagent storage swap-tables --project P --table-id A --target-table-id B [--branch ID] [--dry-run] [--yes]` -- thin wrapper around the Storage API `POST /v2/storage/branch/{branch}/tables/{id}/swap` endpoint. Both tables exchange physical positions; aliases are NOT transferred (they keep pointing at the same physical position and therefore expose the OTHER table's data after the swap). The Storage API queues this as an async storage job (`operationName: tableSwap`); the client polls to completion before returning, so callers can rely on the schemas already being exchanged on return (~10s observed on Snowflake). The API restricts this to dev branches; the service refuses with exit 5 (`ConfigError`) before any HTTP call when neither `--branch` nor an active branch (via `kbagent branch use`) is set. Same-source-and-target IDs also rejected pre-flight. The use case is: AI agent profiles a typeless table, builds a typed rebuild via CTAS in a workspace, then swaps the typed copy into the original name without touching downstream config references that point at the original table ID. Permission classification: `destructive` (gated behind `--allow-destructive`). The PHP reference client docstring claims a synchronous response, but live calls against the platform consistently return a queued job -- this client polls the job to completion to make the `delete_table` / `create_table` semantics consistent. Companion entry in `storage-types-workflow.md` explains the typify-via-CTAS pattern; gotchas + commands-reference + agent prompt all updated.", "Tests (swap-tables): `tests/test_storage_swap.py` (14 tests) covers all three layers -- HTTP shape (POST + body + URL encoding + immediate-success path + async-poll path + 4xx propagation via `pytest_httpx`), service business logic (success, dry-run, branch enforcement, same-id guard, API error propagation, unknown project), and CLI integration (JSON happy path, dry-run, explicit `--branch` overrides active, missing-branch error path with exit 5). E2E coverage in `tests/test_e2e.py::TestE2EStorageSwapTables` runs three scenarios against a live API: live swap of two tables with different VARCHAR lengths verifies definitions exchange in both directions; dry-run skips API call and `lastChangeDate` is unchanged; and the production-rejection path (no branch + no active branch) returns exit 5.", "Plugin docs: new `plugins/kbagent/skills/kbagent/references/typify-table-workflow.md` -- end-to-end procedure for converting a typeless Storage table (every column `STRING(16M)`) into one with proper Snowflake / BigQuery native types. 8 phases: (0) decide-or-skip rubric; (1) isolate in dev branch; (2) profile the typeless table in a workspace with length / cardinality / parse-failure / scale-precision queries + decision matrix mapping profile signals to Snowflake types; (3) build typed sibling via `storage create-table` + copy data via in-workspace INSERT or SQL transformation, with row-count / NULL-count verification; (4) validate downstream consumers in the dev branch (search configs that reference the table, run a representative transformation against the typeless source as baseline); (5) `swap-tables` (dry-run + actual + verify); (6) re-run downstream as smoke test; (7) cleanup the sibling after merge; (8) handoff protocol -- structured summary the AI agent hands to the user with phase-by-phase receipts, the merge URL, and rollback / cleanup commands. Cross-references `storage-types-workflow.md`, `branch-workflow.md`, `workspace-workflow.md`, `gotchas.md`. SKILL.md workflow-references table gains the new entry.", - "BREAKING: `KBC_MANAGE_API_TOKEN` is now ignored by default. The three commands that consume it (`org setup`, `project refresh`, `data-app password`) prompt for the token on a TTY by default. Pass the new top-level flag `--allow-env-manage-token` to restore the legacy env-var behaviour (e.g. for CI/CD). Without the flag and without a TTY, the resolver exits 2 with an actionable message naming the flag. The change closes the AI-exfiltration risk where any subprocess running as the same user (including the AI agent itself) inherits the manage token via env. Migration: prepend `--allow-env-manage-token` to existing CI invocations. Storage tokens (`KBC_TOKEN`) are unaffected. Closes the manage-token UX flagged on #236; supersedes the per-stack design discussed in #238.", - "Security: `resolve_manage_token` (`src/keboola_agent_cli/commands/_helpers.py`) refactored to default-deny env, TTY-first. When the env var is set but the flag is not passed, a one-shot stderr warning fires (`Warning: KBC_MANAGE_API_TOKEN found in environment but ignored. Pass --allow-env-manage-token to opt in.`) and the resolver falls through to the TTY prompt. No cache, no keyring, no temp file -- next invocation prompts again. The bulk-prompt-once contract (`project refresh --all`) is preserved by construction: the resolver lives at command entry, before any per-project loop.", - "New: top-level CLI flag `--allow-env-manage-token` (session-only, mirrors `--deny-writes` / `--deny-destructive`). Plumbed via `ctx.obj['allow_env_manage_token']` and forwarded by the three call sites into `resolve_manage_token(allow_env=...)`. Not persisted, no env-var equivalent (intentional; an env-var equivalent would re-create the AI-exfiltration hole this default-deny is closing).", - "Tests: 12 new (`tests/test_helpers.py::TestResolveManageToken` x7 covering allow_env-True/False x env-set/unset x TTY/non-TTY combinations + token-leak regression pin; `tests/test_manage_token_cli.py::TestAllowEnvManageTokenFlag` x4 covering project-refresh / org-setup / data-app-password through CliRunner with services mocked; `tests/test_manage_token_bulk.py::TestBulkPromptOnce` pinning the contract that `project refresh --all` resolves the token exactly once at command entry, not per-project).", - "Docs: `commands/context.py` AGENT_CONTEXT updated (org-setup example + env-var help block); `CLAUDE.md` convention #12 + global-flag list; `keboola-expert.md` Rule 6 VERSION GATE adds the 0.28.0+ env-flag requirement, tool-selection-matrix updated, new inline-gotcha block; `gotchas.md` new `(since v0.28.0)` entry naming the warning text and the one-line CI fix; `commands-reference.md` updated for `org setup`, `data-app password`, env-var table.", - "New: `kbagent data-app secrets-set / secrets-list / secrets-get / secrets-remove` — manage `#`-prefixed app-runtime secrets in `parameters.dataApp.secrets`. Encryption is per-project KMS via the existing `EncryptService` (same fail-closed semantics as `--git-pat-encrypted`: refuses to write plaintext if the Encryption API does not return a project-scoped ciphertext). Read-modify-write at the service layer (NOT Storage `merge=True` — that flag is shallow at the top level only and would clobber sibling keys nested inside `parameters.dataApp.secrets`). The runtime exposes each key as an env var with `#` stripped, `-` replaced with `_`, and uppercased (`#my-api-key` → `MY_API_KEY` per help.keboola.com/data-apps/python-js/). `secrets-get` is metadata-only — never echoes decrypted plaintext to stdout / stderr / logs / change descriptions; the Encryption API is one-way and the CLI does not attempt to decrypt under any branch. `secrets-remove` is idempotent (missing keys exit 0 with `removed: 0`). `secrets-set` warns when a derived env-var name collides with `RESERVED_RUNTIME_ENV_VARS` (KBC_TOKEN, KBC_URL — verified canon floor; full runtime list TODO follow-up). Adding/removing a secret bumps the Storage version but the running container keeps the OLD config until the next `data-app deploy`.", - "New: `kbagent data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] [--type python-js] [--strict]` — pre-flight check that a git repo follows the documented Golden Rule (https://help.keboola.com/data-apps/python-js/) BEFORE `data-app create` so operators don't burn a deploy cycle on a misconfigured repo. Each check emits BLOCKING / WARN / OK with a help-doc citation: `keboola-config/nginx/sites/default.conf` exists, `keboola-config/supervisord/services/app.conf` exists, `pyproject.toml` at root, `keboola-config/setup.sh` content has no `pip install` (BLOCKING per the help canon's pip prohibition) and contains `uv sync` if `pyproject.toml` declares deps, `requires-python` consistent with the runtime image (when the pin is available), nginx `proxy_pass` port matches `app.conf` declared port. Uses `GET /repos/{owner}/{repo}/git/trees/{ref}?recursive=1` (one call) + up to 4 `GET .../contents/{path}` for files whose contents the rules need to inspect — total ≤5 GitHub API calls (1 tree + 0-4 contents) regardless of repo size, sidesteps the 60/hour unauth rate limit for typical use. `--git-pat-env` / `--git-pat-file` raises the limit to 5,000/hour. Read-only; never touches a Keboola project. `--type` is restricted to `python-js` in 0.28.0; streamlit / pure-Python / R / Node-only follow-up.", - "New: `RepoValidateService` (`src/keboola_agent_cli/services/repo_validate_service.py`) — pure validation function `validate_keboola_repo(snapshot, type_, runtime_python_pin)` plus a tiny `GitHubContentsClient` (HTTPS GET to `api.github.com`, optional bearer PAT, no token persistence). Service module is the only place GitHub HTTP lives; the rest of kbagent stays Keboola-API-only.", - "New: `ErrorCode` entries `DATA_APP_INVALID_SECRET`, `DATA_APP_INVALID_REPO`, `DATA_APP_REPO_VALIDATION_BLOCKING`. Permission registry entries `data-app.secrets-set` (write), `data-app.secrets-list` / `data-app.secrets-get` (read), `data-app.secrets-remove` (destructive — removing a secret can break a running app), `data-app.validate-repo` (read).", - "New: `--hint client/service` for all five new commands. `secrets-get` hint snippet asserts the metadata-only contract; `validate-repo` snippet uses `RepoValidateService.validate_repo(...)` and the hint comment notes that GitHub-side detail is not shown.", - "Fix: `kbagent data-app create --auth public` now writes the canonical `noneProxyAuthorization` shape (kbc-ui exact constant: `auth_providers: []` + `auth_rules: [{type: pathPrefix, value: /, auth_required: false}]`). v0.27.0 wrote NO `authorization` key when `--auth public`, leaving the Keboola app-proxy unable to route (HTTP 503) and the UI Authentication Type selector blank — silently broken. Authoritative source: the public backend validator at `keboola/job-queue-job-configuration` `AppProxyDefinition.php` (when `auth_required=false`, `auth` MUST NOT be set). The private `keboola/ui` repo `apps/kbc-ui/src/scripts/modules/data-apps/constants.ts` corroborates: its `noneProxyAuthorization` constant exports this exact shape for the None UI option (Keboola org members can verify; external readers rely on the validator). Live-validated end-to-end on a real project: HTTP 200 on the resulting URL, written block bit-identical to canon, UI auth selector now shows None pre-selected. Existing `--auth password` behaviour unchanged.", - "Tests: 27 secrets service tests + 20 validate-repo service tests + 22 CLI tests (13 secrets/validate-repo CLI methods + 9 hint-compile AST-parse cases) + 4 new auth-block tests (`TestDataAppCreateAuthBlock` asserts both `--auth public` and `--auth password` write the canonical shape on POST `/apps` AND PUT Storage). 2505 total tests green. Sibling-preservation regression test for `secrets-set` asserts every untouched key under `parameters.dataApp.secrets`, `parameters.dataApp` (slug, git block), `parameters` (id), and the top-level config (`runtime`, `authorization`, `storage`) is preserved bit-identical after the read-modify-write.", - 'Plugin: `keboola-expert.md` matrix gains five new rows (one per `secrets-set / -list / -get / -remove + validate-repo`); §1 Rule 6 VERSION GATE example updated for `secrets / validate-repo need 0.28.0+`. New `(since v0.28.0)` `gotchas.md` entries: (a) secrets are per-project KMS encrypted, `secrets-remove` on missing key is exit 0, `secrets-get` never echoes decrypted plaintext, `#KBC_TOKEN` is silently shadowed by the runtime; (b) `validate-repo` GitHub-only Golden-Rule check; (c) `--auth public` writes the canonical `noneProxyAuthorization` shape (fixes v0.27.0 silent 503). New "Managing app-runtime secrets" + "Pre-flight repo validation" recipe sections in `data-app-workflow.md`. Logs / auto-log-dump deferred to issue #240 (the Data Science API does not expose Terminal Logs as JSON per help canon).', ], "0.27.0": [ "New: `kbagent data-app` command group — first-class lifecycle for Keboola data apps (`keboola.data-apps` Storage component + Data Science API `/apps`). Eight subcommands: `list`, `detail`, `create`, `deploy`, `start`, `stop`, `delete`, `password`. The CLI encapsulates the **§9 redeploy contract** (always sends the `{desiredState=running, configVersion, restartIfRunning=true}` trio together; without it, `PATCH /apps {desiredState:running}` silently pins to the empty-shell v2 and the runner errors `dataApp.git.repository is required in /data/config.json`), per-project KMS encryption of git PATs (refuses to write plaintext if the Encryption API does not return a project-scoped ciphertext), cleanup-in-finally on initial-deploy failure (orphan shell deleted by default; `--keep-on-failure` opts out), and a poll loop that respects pitfall #1 — `state == stopped` is NOT terminal while `desiredState == running` (the platform transitions `created → stopped → starting → running` during initial deploy). `data-app create` accepts `--git-pat-env VAR` (recommended; no argv leak), `--git-pat-file PATH`, or `--git-pat-encrypted KBC::Project...` (must be encrypted under THIS project's KMS — ciphertext does not cross projects).", @@ -39,14 +47,6 @@ "Tests: 30 service-level tests in `tests/test_data_app_service.py` (validation, dry-run, happy-path orchestration, cleanup-in-finally, encryption-failure-aborts-loud, poll-loop semantics including the transient-stopped invariant), 10 CLI tests in `tests/test_data_app_cli.py` (mutual-exclusion validation, dual JSON+human output, `--yes` for delete, manage-token forwarding for password without leaking the token to stdout/stderr).", "Plugin: new `data-app-workflow.md` reference + two `(since v0.27.0)` gotcha entries (the §9 redeploy contract; cross-project KMS ciphertext mismatch). `keboola-expert.md` matrix gains five rows (`create`, `deploy`, `start`, `stop`, `delete`).", ], - "0.26.1": [ - "New: project member & invitation lifecycle. Closes the long-standing Manage API gap that forced every Keboola-internal automation (most recently `17_CuestaDemo/scripts/replicate_master.py` and `invite_participants.py`) to bypass kbagent and POST raw HTTP at `/manage/projects/{id}/invitations`. Seven new commands under `kbagent project`: `invite` (single-shot or `--from-csv` bulk with `ThreadPoolExecutor` parallelism, default 8 workers), `member-list` (active members, `--include-pending` adds pending invitations), `invitation-list`, `invitation-cancel` (resolves invitation_id by email lookup so callers don't have to), `member-remove` (destructive; resolves user_id by email), `member-set-role` (PATCH `/manage/projects/{id}/users/{userId}` with `{role}`). All seven require `KBC_MANAGE_API_TOKEN`; the manage token is never logged, never persisted, never accepted on the CLI line. Permission registry: `member-remove` is `destructive`, `member-list` / `invitation-list` are `read`, the rest are `admin`.", - "New: role whitelist `PROJECT_ROLES = ('admin', 'guest', 'readOnly', 'share')` in `constants.py`, lifted verbatim from the Manage API's own validation error message (verified empirically on 2026-05-01 against `connection.us-east4.gcp.keboola.com`). Typer enforces the whitelist via `click.Choice` at the command layer; `MemberService` double-checks for defence-in-depth. Invalid role values now fail-fast with `Role 'X' is not valid. Allowed roles are: admin, guest, readOnly, share` instead of letting the API return an opaque 400.", - "New: `MemberService` (`src/keboola_agent_cli/services/member_service.py`) wrapping six new `ManageClient` methods (`create_project_invitation`, `list_project_invitations`, `cancel_project_invitation`, `list_project_members`, `remove_project_member`, `update_project_member_role`). Resolves project alias -> (stack_url, project_id) via `ConfigStore`; resolves email -> numeric user_id / invitation_id by listing + matching case-insensitively. Treats the Manage API's HTTP 400 'already been invited' / 'already a member' responses as `status=noop` rather than errors (the heuristic the orchestrator scripts had to do via substring matching, now typed to `status_code == 400` AND message-substring marker constants). `--from-csv` enforces a single-stack-URL invariant per file (rows referencing multiple stacks raise `ConfigError` upfront).", - "New: hint definitions (`hints/definitions/member.py`) for all seven commands. Both `--hint client` (direct `ManageClient` calls) and `--hint service` (`MemberService` calls) generate runnable Python.", - "New: e2e marker `e2e_invite` (registered in `pyproject.toml`). `make test-e2e-invite` runs `tests/test_e2e.py::test_project_invite_e2e` against a real Manage API; gated on `E2E_MANAGE_TOKEN` + `E2E_INVITE_PROJECT_ID` (skips cleanly when missing). The test invites `ottomansky.max@gmail.com` (override via `E2E_INVITE_EMAIL`) as `guest`, asserts the invitation appears in `invitation-list`, then cancels it -- the same run that proves the system can send confirms it can clean up.", - "Docs: new `references/member-workflow.md` (golden paths for single invite, bulk invite, audit, role change, remove). `gotchas.md` gains three `(since v0.26.1)` entries -- 'already invited / already member' returns HTTP 400 not 422; role-change is PATCH not PUT (PUT returns 404 even on real members); bulk-invite ordering is not deterministic (parallel workers). `keboola-expert.md` adds seven matrix rows under 'Project administration' plus a Rule 6 VERSION GATE entry. `commands-reference.md` adds a 'Project members & invitations' section.", - ], "0.26.0": [ "New: `kbagent config set-default-bucket --bucket BUCKET_ID | --clear [--dry-run] [--branch ID]` -- discoverable wrapper around the raw-mode `storage.output.default_bucket` workaround documented at https://keboola.atlassian.net/wiki/spaces/SUP/pages/3770155030/ (epic KBCP-108). Read-modify-write that preserves all sibling keys under `storage.output` and the rest of the configuration. Same-value writes short-circuit with `{\"changed\": false}` (no API call, no version bump). `--clear` removes only the `default_bucket` key, leaving an empty `storage.output: {}` if no other siblings live there (intentional -- mirrors `set_nested_value`'s parent-creation semantics; Storage API treats `output: {}` and missing `output` identically as 'use the auto-derived bucket'). Live-validated end-to-end on three component types -- row-based GCS extractor, root-only `keboola.ex-cnb-exchange-rates`, and `ex-generic-v2` with multiple jobs -- output tables routed to the configured bucket at job runtime in every case. The per-table `destination` override (the second method shown in the support article) keeps using the existing `kbagent config update --set 'storage.output.tables=[...]'` -- no new wrapper there because per-table mappings have many fields that don't fit a single-purpose flag.", "Fix: `kbagent sync pull --with-samples` no longer crashes with `TypeError: '>' not supported between instances of 'NoneType' and 'int'` when one or more tables in the project return `rowsCount: null` from the Storage API (typical for newly-created or empty tables on some backends, reproduced live against `kosik-sales`). `dict.get(\"rowsCount\", 0)` returns the default `0` only when the key is **missing** -- if the key is present with a `null` value, `.get()` returns `None`, and the `> 0` comparison crashed Python 3 before any sample was fetched. The filter and sort key in `SyncService._fetch_samples()` now coerce `None` to `0` via a small `_rows()` helper used in both places (`t.get(\"rowsCount\") or 0`), so empty/null-rowcount tables are gracefully skipped exactly like `rowsCount: 0` ones. Closes #233.", diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index bad48673..4c967e5a 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -94,7 +94,7 @@ Print the effective default project and its source (env / pin / none). Resolution order for single-project operations: --project > KBAGENT_PROJECT > pin. -### Project Members & Invitations (since v0.26.1) +### Project Members & Invitations (since v0.29.0) Requires KBC_MANAGE_API_TOKEN (Manage API auth). Allowed roles: admin, guest, readOnly, share. @@ -452,7 +452,7 @@ Use --org-id OR --project-ids (at least one required). Token via interactive hidden prompt by default; pass top-level --allow-env-manage-token to read KBC_MANAGE_API_TOKEN from env (CI/CD). - Default-deny since 0.28.0 -- closes the AI-exfiltration risk where + Default-deny since 0.29.0 -- closes the AI-exfiltration risk where subprocesses inherit the manage token via env. ### Flows (Orchestrator + Conditional) @@ -630,7 +630,7 @@ Retrieve the simpleAuth password. Requires the Manage API token in addition to the project's Storage token. Token is read from interactive hidden prompt by default; pass top-level --allow-env-manage-token to - use KBC_MANAGE_API_TOKEN from env (default-deny since 0.28.0). Never + use KBC_MANAGE_API_TOKEN from env (default-deny since 0.29.0). Never persisted, never logged. Password is auto-generated at create time and CANNOT be rotated -- delete and recreate the app to mint a new one. @@ -811,7 +811,7 @@ KBC_TOKEN Storage API token (fallback for --token) KBC_STORAGE_API_URL Default stack URL (fallback for --url) KBC_MANAGE_API_TOKEN Manage API token (org setup, project refresh, data-app password). - Default-DENY since 0.28.0: pass --allow-env-manage-token + Default-DENY since 0.29.0: pass --allow-env-manage-token to opt in, otherwise this var is ignored and a TTY prompt is required. Closes AI-exfiltration via subprocess env. KBC_MASTER_TOKEN Master token for sharing ops (global fallback) diff --git a/src/keboola_agent_cli/commands/project.py b/src/keboola_agent_cli/commands/project.py index f202d2a8..38c3e55e 100644 --- a/src/keboola_agent_cli/commands/project.py +++ b/src/keboola_agent_cli/commands/project.py @@ -859,7 +859,7 @@ def project_invite( ) return - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: @@ -922,7 +922,7 @@ def project_member_list( ) return - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: result = service.list_members( @@ -960,7 +960,7 @@ def project_invitation_list( ) return - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: result = service.list_invitations(manage_token=manage_token, alias=project) @@ -1014,7 +1014,7 @@ def project_invitation_cancel( formatter.console.print("Aborted.") raise typer.Exit(code=0) - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: result = service.cancel_invitation( @@ -1069,7 +1069,7 @@ def project_member_remove( formatter.console.print("Aborted.") raise typer.Exit(code=0) - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: result = service.remove_member( @@ -1122,7 +1122,7 @@ def project_member_set_role( ) return - manage_token = resolve_manage_token() + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) service = get_service(ctx, "member_service") try: result = service.set_member_role( diff --git a/src/keboola_agent_cli/services/repo_validate_service.py b/src/keboola_agent_cli/services/repo_validate_service.py index 2e71ac76..afa96a0c 100644 --- a/src/keboola_agent_cli/services/repo_validate_service.py +++ b/src/keboola_agent_cli/services/repo_validate_service.py @@ -23,6 +23,14 @@ Scope of this PR: ``--type python-js`` only. Streamlit / pure-Python / R repo layouts differ and need their own per-type canon citations -- a follow-up PR adds them. + +TODO: extract ``GitHubContentsClient`` (defined below) into a top-level +``src/keboola_agent_cli/github_client.py`` module that inherits from +``BaseHttpClient`` so this stays consistent with the 3-layer architecture +(LAYER 3 = clients, LAYER 2 = services). The ``github_client_factory`` +dependency-injection pattern in ``RepoValidateService`` already isolates +the client for testing; the extraction is a pure refactor with no +behaviour change. """ from __future__ import annotations diff --git a/tests/test_e2e.py b/tests/test_e2e.py index b18fc679..52985bb2 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -45,6 +45,7 @@ from keboola_agent_cli.cli import app from keboola_agent_cli.client import KeboolaClient from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.models import ProjectConfig # --------------------------------------------------------------------------- # Environment & skip logic @@ -6139,8 +6140,12 @@ def test_swap_without_branch_is_rejected(self) -> None: reason=( f"requires {ENV_TOKEN} + {ENV_DATA_APP_GIT_REPO_PRIVATE} + " f"{ENV_DATA_APP_GIT_USER} + {ENV_DATA_APP_GIT_PAT}" + ), +) + + # ────────────────────────────────────────────────────────────────────── -# Project invite E2E (since v0.26.1) +# Project invite E2E (since v0.29.0) # # Opt-in via `make test-e2e-invite`. Default-skipped in `make test-e2e` because # (a) it sends a real invitation email and (b) it depends on a separate manage @@ -6148,7 +6153,6 @@ def test_swap_without_branch_is_rejected(self) -> None: # ────────────────────────────────────────────────────────────────────── -ENV_MANAGE_TOKEN = "E2E_MANAGE_TOKEN" ENV_INVITE_PROJECT_ID = "E2E_INVITE_PROJECT_ID" ENV_INVITE_EMAIL = "E2E_INVITE_EMAIL" DEFAULT_INVITE_EMAIL = "ottomansky.max@gmail.com" @@ -6354,6 +6358,210 @@ def test_data_app_lifecycle_private_and_redeploy(self) -> None: )["data"] assert deploy["config_version"], "deploy must pin a configVersion" + @skip_without_data_app_public + def test_data_app_secrets_round_trip(self) -> None: + """secrets-set -> secrets-list -> secrets-get -> secrets-remove on a real app. + + Uses --no-deploy + --auth public to mint a cheap shell app, then + verifies the four-step lifecycle: + 1. set: encrypts via per-project KMS, writes to parameters.dataApp.secrets. + 2. list: enumerates keys + derived runtime env-var names; never decrypts. + 3. get: returns metadata only (NEVER plaintext). + 4. remove: idempotent (second remove returns removed: 0, exit 0). + + The decrypted plaintext value must NEVER appear in any CLI output. + """ + repo = os.environ[ENV_DATA_APP_GIT_REPO_PUBLIC] + slug = f"e2e-secrets-{RUN_ID}"[:60] + secret_key = "#E2E_TEST_KEY" + secret_plaintext = "supersecret-do-not-leak" + + _step(1, "Create shell app for secrets round-trip") + create = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "create", + "--project", + self.alias, + "--name", + f"E2E Secrets {RUN_ID}", + "--slug", + slug, + "--git-repo", + repo, + "--git-public", + "--auth", + "public", + "--no-deploy", + ], + ) + )["data"] + app_id = create["id"] + self._created_app_ids.append(app_id) + + _step(2, "secrets-set: encrypt and write") + set_result = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-set", + "--project", + self.alias, + "--app-id", + app_id, + "--secret", + f"{secret_key}={secret_plaintext}", + "--no-hint-next", + ], + ) + ) + assert secret_plaintext not in set_result["raw_output"], ( + "Plaintext value MUST NEVER appear in secrets-set output" + ) + + _step(3, "secrets-list: enumerate keys (never decrypts)") + list_result = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-list", + "--project", + self.alias, + "--app-id", + app_id, + ], + ) + ) + keys_in_list = [s["key"] for s in list_result["data"]["secrets"]] + assert secret_key in keys_in_list, ( + f"secrets-list must surface the just-written key; got {keys_in_list}" + ) + assert secret_plaintext not in list_result["raw_output"], ( + "Plaintext value MUST NEVER appear in secrets-list output" + ) + + _step(4, "secrets-get: metadata only (never plaintext)") + get_result = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-get", + "--project", + self.alias, + "--app-id", + app_id, + "--key", + secret_key, + ], + ) + ) + assert get_result["data"]["key"] == secret_key + assert secret_plaintext not in get_result["raw_output"], ( + "secrets-get MUST NEVER echo the decrypted plaintext (Encryption API is one-way)" + ) + + _step(5, "secrets-remove: first call removes the key") + remove_result = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-remove", + "--project", + self.alias, + "--app-id", + app_id, + "--key", + secret_key, + "--yes", + ], + ) + ) + assert remove_result["data"]["removed"] == 1, "first remove must report removed=1" + + _step(6, "secrets-remove: second call is idempotent (removed=0)") + idempotent = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-remove", + "--project", + self.alias, + "--app-id", + app_id, + "--key", + secret_key, + "--yes", + ], + ) + ) + assert idempotent["data"]["removed"] == 0, ( + "second remove of the same key must be idempotent (removed=0, exit 0)" + ) + + +# --------------------------------------------------------------------------- +# Data-app validate-repo (since v0.29.0) -- GitHub-only, no Keboola creds needed +# --------------------------------------------------------------------------- + + +@pytest.mark.e2e +def test_data_app_validate_repo_against_public_repo(tmp_path: Path) -> None: + """validate-repo against a real public GitHub repo. + + Does NOT require Keboola credentials -- the command only hits GitHub. + Uses a known-public Keboola example repo via E2E_DATA_APP_GIT_REPO_PUBLIC + when set; otherwise skipped (no hard-coded URL to keep the test + independent of upstream-template renames). + + Asserts the command exits cleanly and emits the expected envelope + shape (status + checks list with BLOCKING / WARN / OK severities). + """ + repo = os.environ.get(ENV_DATA_APP_GIT_REPO_PUBLIC) + if not repo: + pytest.skip(f"requires {ENV_DATA_APP_GIT_REPO_PUBLIC} (any public Keboola data-app repo)") + + config_dir = tmp_path / "kbagent-config" + config_dir.mkdir() + + result = _invoke( + config_dir, + [ + "--json", + "data-app", + "validate-repo", + "--git-repo", + repo, + "--git-public", + "--type", + "python-js", + ], + ) + # Exit 0 when no BLOCKING; exit 1 when at least one BLOCKING. Either is + # a successful invocation -- the assertion is on shape, not verdict. + assert result.exit_code in (0, 1), result.output + + body = json.loads(result.output) + assert body["status"] in ("ok", "error"), f"unexpected status: {body['status']}" + if body["status"] == "ok": + assert "checks" in body["data"], "envelope must list per-rule checks" + # Every check must carry severity + a citation back to the help-doc. + for check in body["data"]["checks"]: + assert check["severity"] in ("BLOCKING", "WARN", "OK") + assert "citation" in check, "each check must cite the help-doc canon" + # --------------------------------------------------------------------------- # Issue #245: parameters.blocks[].codes[].script auto-normalize on config update @@ -6686,6 +6894,8 @@ def test_config_update_auto_normalizes_script_array(self, tmp_path: Path) -> Non assert "Expected" not in rendered or "script" not in rendered, ( f"job envelope still mentions the script type-mismatch failure: {rendered}" ) + + @pytest.mark.e2e_invite @skip_without_invite_credentials def test_project_invite_e2e(tmp_path: Path) -> None: @@ -6695,9 +6905,6 @@ def test_project_invite_e2e(tmp_path: Path) -> None: invalidates the invitation link before the inbox sees it, so this is a "the system can send + clean up" check, not a "join my project" check. """ - from keboola_agent_cli.config_store import ConfigStore as _Store - from keboola_agent_cli.models import ProjectConfig as _Project - invite_email = os.environ.get(ENV_INVITE_EMAIL, DEFAULT_INVITE_EMAIL) project_id = int(os.environ[ENV_INVITE_PROJECT_ID]) stack_url = ( @@ -6712,10 +6919,10 @@ def test_project_invite_e2e(tmp_path: Path) -> None: # field is unused. Write a minimal config.json with a placeholder token. config_dir = tmp_path / "kbagent-config" config_dir.mkdir() - store = _Store(config_dir=config_dir) + store = ConfigStore(config_dir=config_dir) store.add_project( alias, - _Project( + ProjectConfig( stack_url=stack_url, token="901-e2e-placeholder-not-used-by-member-commands-xxxxxxxxxx", project_id=project_id, diff --git a/tests/test_member_cli.py b/tests/test_member_cli.py index 7a2eaea8..379555dd 100644 --- a/tests/test_member_cli.py +++ b/tests/test_member_cli.py @@ -60,6 +60,7 @@ def test_json_happy_path(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -89,6 +90,7 @@ def test_missing_required_args_exits_2(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -110,6 +112,7 @@ def test_invalid_role_blocked_by_choice(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -145,6 +148,7 @@ def test_dry_run_short_circuits(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -181,6 +185,7 @@ def test_invalid_token_maps_to_exit_3(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -239,6 +244,7 @@ def test_json_bulk_summary(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -266,6 +272,7 @@ def test_from_csv_mutually_exclusive_with_project(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -308,6 +315,7 @@ def test_json_output(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -343,6 +351,7 @@ def test_include_pending_flag_propagates(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -380,6 +389,7 @@ def test_yes_skips_confirmation(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -417,6 +427,7 @@ def test_destructive_yes(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", @@ -455,6 +466,7 @@ def test_propagates_role(self, tmp_path: Path) -> None: result = runner.invoke( app, [ + "--allow-env-manage-token", "--config-dir", str(config_dir), "--json", diff --git a/uv.lock b/uv.lock index d39aa76e..73ad4950 100644 --- a/uv.lock +++ b/uv.lock @@ -439,7 +439,7 @@ wheels = [ [[package]] name = "keboola-agent-cli" -version = "0.28.0" +version = "0.29.0" source = { editable = "." } dependencies = [ { name = "httpx" },