From bf20800ba43a85c6a534d14041d48169fac0dfe6 Mon Sep 17 00:00:00 2001 From: ottomansky Date: Fri, 1 May 2026 21:06:55 +0200 Subject: [PATCH] feat(0.27.0): data-app command group -- first-class lifecycle for Keboola data apps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `kbagent data-app` group covering create, list, detail, deploy, start, stop, delete, and password retrieval for `keboola.data-apps` + the Data Science API `/apps`. The CLI encapsulates the documented footguns so callers cannot hit them: - §9 redeploy contract: `data-app deploy` always sends the `{desiredState=running, configVersion, restartIfRunning=true}` trio together. Sending bare `desiredState=running` (the obvious shape) silently pins to the empty-shell v2 from `POST /apps`; the runner then errors `dataApp.git.repository is required in /data/config.json` with no top-level error surfaced. - Per-project KMS encryption: `data-app create` always re-encrypts the plaintext PAT via the target project's Encryption API. Refuses to write plaintext if the round-trip does not return a `KBC::Project*` ciphertext. Ciphertext does not cross projects. - Cleanup-in-finally: if the Storage PUT or initial deploy fails after `POST /apps`, the orphan shell is deleted unless `--keep-on-failure`. - Poll loop: `state == stopped` is NOT terminal while `desiredState == running` -- the platform transitions `created -> stopped -> starting -> running` during initial deploy. Layering follows the project convention (commands -> services -> clients) and mirrors workspace.py end-to-end: - Client: new `DataScienceClient` (`data_science_client.py`) inheriting `BaseHttpClient`. URL derived as `data-science.` from the connection URL. `get_app_password` accepts the Manage token per-call so it never lives on the persistent client. - Service: `DataAppService` (`services/data_app_service.py`) accepts both a Storage-client factory and a Data-Science-client factory plus the existing `EncryptService`. One method per CLI subcommand. - Commands: `commands/data_app.py` -- thin Typer subcommands; mutual exclusion validation for git auth modes; dual JSON / human output; confirmation prompt on delete. - Permissions: 8 entries added to OPERATION_REGISTRY (read for list / detail / password; write for create / deploy / start / stop; destructive for delete). `data-app.password` is `read` for parity with the existing `workspace.password`. - Errors: 3 new `ErrorCode` members (`DATA_APP_BUILD_FAILED`, `DATA_APP_DEPLOY_TIMEOUT`, `DATA_APP_INVALID_GIT`); existing codes cover the rest. - Hints: new `data_science` `client_type` in the renderer. Hint mode generates `DataScienceClient` instantiation + the §9 trio inline. Tests: 30 service-level tests (validation, dry-run, happy path, cleanup-in-finally, encryption-failure-aborts-loud, poll-loop semantics including the transient-stopped invariant), 10 CLI tests (mutual exclusion, dual output, manage-token forwarding without leaking the token to stdout), 2 E2E tests gated on `E2E_DATA_APP_GIT_REPO_PUBLIC` / `E2E_DATA_APP_GIT_REPO_PRIVATE`. Sync map: pyproject 0.27.0; changelog entry; AGENT_CONTEXT block; CLAUDE.md `## All CLI Commands`; keboola-expert.md matrix (5 rows) + inline gotcha + version-gate bump; SKILL.md description triggers + workflow link + auto-table regen; commands-reference; gotchas.md (two `(since v0.27.0)` entries -- the redeploy contract and the cross-project KMS); new data-app-workflow.md; plugin.json / marketplace.json synced via `make version-sync`. Bump 0.26.0 -> 0.27.0 (minor): new top-level command group + new underlying API surface (Data Science). --- .claude-plugin/marketplace.json | 2 +- CLAUDE.md | 9 + README.md | 2 + docs/TUTORIAL.md | 188 +++ plugins/kbagent/.claude-plugin/plugin.json | 2 +- plugins/kbagent/agents/keboola-expert.md | 31 +- plugins/kbagent/skills/kbagent/SKILL.md | 14 + .../kbagent/references/commands-reference.md | 11 + .../kbagent/references/data-app-workflow.md | 189 +++ .../skills/kbagent/references/gotchas.md | 56 + pyproject.toml | 2 +- src/keboola_agent_cli/changelog.py | 8 + src/keboola_agent_cli/cli.py | 5 + src/keboola_agent_cli/commands/context.py | 55 + src/keboola_agent_cli/commands/data_app.py | 613 ++++++++ src/keboola_agent_cli/data_science_client.py | 192 +++ src/keboola_agent_cli/errors.py | 5 + .../hints/definitions/__init__.py | 1 + .../hints/definitions/data_app.py | 400 ++++++ src/keboola_agent_cli/hints/renderer.py | 37 +- src/keboola_agent_cli/permissions.py | 9 + .../services/data_app_service.py | 1243 +++++++++++++++++ tests/test_data_app_cli.py | 560 ++++++++ tests/test_data_app_service.py | 806 +++++++++++ tests/test_e2e.py | 218 +++ uv.lock | 2 +- 26 files changed, 4653 insertions(+), 7 deletions(-) create mode 100644 plugins/kbagent/skills/kbagent/references/data-app-workflow.md create mode 100644 src/keboola_agent_cli/commands/data_app.py create mode 100644 src/keboola_agent_cli/data_science_client.py create mode 100644 src/keboola_agent_cli/hints/definitions/data_app.py create mode 100644 src/keboola_agent_cli/services/data_app_service.py create mode 100644 tests/test_data_app_cli.py create mode 100644 tests/test_data_app_service.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 4ecc89f6..703a3762 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "kbagent", - "version": "0.26.0", + "version": "0.27.0", "source": "./plugins/kbagent", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "category": "development" diff --git a/CLAUDE.md b/CLAUDE.md index 88a75ae3..a69ab61e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -349,6 +349,15 @@ kbagent workspace query --project ALIAS --workspace-id ID --file query.sql kbagent workspace gc [--project NAME ...] [--dry-run] [--yes] kbagent workspace from-transformation --project ALIAS --component-id ID --config-id ID [--row-id ID] +kbagent data-app list [--project NAME ...] [--branch ID] +kbagent data-app detail --project NAME --app-id ID [--branch ID] +kbagent data-app create --project ALIAS --name NAME --slug SLUG --git-repo URL [--description STR | --description-file PATH] [--git-branch main] [--git-public/--no-git-public] [--git-username USER] [--git-pat-env VAR | --git-pat-file PATH | --git-pat-encrypted KBC::Project...] [--auth password|public] [--size tiny|small|medium|large] [--auto-suspend SECONDS] [--type python-js|python|streamlit|r|...] [--branch ID] [--no-deploy] [--wait] [--timeout SECONDS] [--keep-on-failure] [--dry-run] +kbagent data-app deploy --project NAME --app-id ID [--config-version N] [--wait] [--timeout SECONDS] [--branch ID] +kbagent data-app start --project NAME --app-id ID [--wait] [--timeout SECONDS] +kbagent data-app stop --project NAME --app-id ID [--wait] [--timeout SECONDS] +kbagent data-app delete --project NAME --app-id ID [--yes] +kbagent data-app password --project NAME --app-id ID + kbagent component list [--project NAME] [--type TYPE] [--query QUERY] kbagent component detail --component-id ID [--project NAME] kbagent config new --component-id ID [--name NAME] [--project NAME] [--output-dir DIR] diff --git a/README.md b/README.md index 1b3f91c7..8087405c 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ kbagent workspace query --project prod --workspace-id WS_ID \ | **MCP tools** | Call `keboola-mcp-server` tools with auto-expand, multi-project fan-out, branch propagation, schema validation. | | **Workspaces** | Create Snowflake/BQ workspace, load tables, run SQL. Create from transformation config for instant debugging. Orphan detection + garbage collection. | | **Sharing** | Cross-project bucket sharing with org/project/user access control. Share, link, unlink. | +| **Data apps** | First-class lifecycle for Streamlit / Flask / Node deployments (`keboola.data-apps`). Create, deploy, start, stop, password, delete. Hides the redeploy contract and per-project KMS encryption of git PATs. | | **Lineage** | Column-level dependency analysis across projects. SQL/Python parsing, AI-enhanced detection, interactive web browser, Mermaid/HTML/ER export. | | **Kai (AI Assistant)** | Ask Keboola's built-in AI questions about your project. One-shot or chat sessions with full MCP context. | | **Encryption** | Encrypt secrets (`#password`, `#api_token`) via Keboola Encryption API. Works with sync push and MCP. | @@ -135,6 +136,7 @@ kbagent storage buckets | bucket-detail | create-bucket | delete-bucket files | file-detail | file-upload | file-download | file-tag | file-delete load-file | unload-table kbagent sharing list | share | unshare | link | unlink | edges +kbagent data-app list | detail | create | deploy | start | stop | delete | password kbagent lineage build | show | info | server kbagent branch list | create | use | reset | delete | merge metadata-list | metadata-get | metadata-set | metadata-delete diff --git a/docs/TUTORIAL.md b/docs/TUTORIAL.md index 42d339e2..79ccce80 100644 --- a/docs/TUTORIAL.md +++ b/docs/TUTORIAL.md @@ -754,6 +754,194 @@ For the full reference including the BOOLEAN/INTEGER gotchas and the --- +## 9. Data apps lifecycle + +A Keboola data app is **not one resource** -- it's a deployment record on +the Data Science API plus a configuration on the Storage API +(`keboola.data-apps`), and they have to stay in sync. Each API only owns +part of the picture, and the obvious-looking calls have four documented +footguns the platform does not surface as errors: + +1. **The redeploy contract.** `PATCH /apps/{id}` with bare + `desiredState=running` silently pins to the empty shell that + `POST /apps` minted, so the runner errors `dataApp.git.repository is + required in /data/config.json` -- only visible in the UI Terminal Log, + never in any HTTP response. +2. **Per-project KMS encryption.** Encrypted git PATs (`KBC::ProjectSecure*`) + are bound to the project that minted them; ciphertext from project A + does not decrypt in project B. +3. **Cleanup-in-finally.** A failed initial deploy leaks the empty shell + from `POST /apps` if the caller does not delete it manually. +4. **Transient `state == stopped`.** During the very first deploy the + platform transitions `created -> stopped -> starting -> running`, so a + naive poll that exits on `stopped` reports a phantom failure. + +`kbagent data-app` (since 0.27.0) encodes all four in the service layer, +so the `--json` output you see at the CLI is what would have happened if +you had done everything right at the raw HTTP level. The eight +subcommands -- `list`, `detail`, `create`, `deploy`, `start`, `stop`, +`delete`, `password` -- cover the full lifecycle end to end. + +### 9.1 Public-repo golden path + +The simplest path: a public git repo, no auth gate, three commands from +zero to a running container. + +```bash +# Register the project once (Storage API token from the UI). +kbagent project add --project prod \ + --url https://connection.keboola.com \ + --token YOUR_STORAGE_TOKEN + +# Create the deployment shell + Storage config in one shot. +kbagent --json data-app create \ + --project prod \ + --name "Hello data app" \ + --slug hello \ + --git-repo https://github.com// \ + --git-public --auth public --no-deploy \ + | jq -r '.data | "id=\(.id) config_id=\(.config_id) config_version=\(.config_version)"' +# id=12345678 config_id=01abcdefghijklmnopqrstuvwxyz config_version=3 +``` + +`type: "python-js"` (the default) covers BOTH a Node app +(`package.json` + entry point) and a Python app (`requirements.txt` + +`app.py`). The runtime auto-detects from what's in the repo. + +The `configVersion=3` is the Storage config version after kbagent's PUT. +Storage went `1 -> 2` (the empty shell that `POST /apps` minted, with an +auto-injected `parameters.id` back-pointer to the deployment record) and +`2 -> 3` (the full body with the git block + runtime block + auth block). + +```bash +# Deploy: pins the deployment record to configVersion=3 and waits. +kbagent --json data-app deploy \ + --project prod --app-id 12345678 --wait --timeout 300 \ + | jq -r '.data | "state=\(.state) url=\(.url)"' +# state=running url=https://hello-12345678.hub.keboola.com +``` + +Visit the URL. That's the entire round-trip: ~30s for a small Node +app, longer for the first cold-boot of a heavier Python app. + +### 9.2 Private-repo golden path + +For a private repo you need a GitHub PAT with `repo:read` scope. +Two non-negotiables: + +- **Pass it via env, not argv.** `--git-pat-env GITHUB_PAT_DATAAPP` + reads the PAT from the named environment variable. The plaintext + never appears in your shell history, in `ps aux`, or in any kbagent + output. +- **The PAT is encrypted under THIS project's KMS** before reaching + Storage. `kbagent data-app create` calls the Encryption API + (`encryption./encrypt`) and only writes the resulting + `KBC::ProjectSecure*` ciphertext. Ciphertext from one project's + config does NOT decrypt in another -- copying an encrypted git block + across projects via raw `kbagent config update` produces a runtime + failure, not a clear error. + +```bash +export GITHUB_PAT_DATAAPP=ghp_xxxxxxxxxxxxxxxxxxxx + +kbagent --json data-app create \ + --project prod \ + --name "Internal dashboard" \ + --slug internal-dashboard \ + --git-repo https://github.com// \ + --git-username YOUR_GITHUB_USER \ + --git-pat-env GITHUB_PAT_DATAAPP \ + --auth password --wait --timeout 300 +``` + +`--auth password` (the default) wraps the app in a simpleAuth gate. The +20-character hex password is auto-generated by the platform on first +deploy. To retrieve it: + +```bash +# Requires KBC_MANAGE_API_TOKEN in env (org-scoped Manage API token). +kbagent --json data-app password \ + --project prod --app-id 12345678 \ + | jq -r '.data.password' +# <20-character hex password, e.g. a1b2c3d4e5f6a7b8c9d0> +``` + +The password cannot be rotated; to change it, delete and recreate the +app. (See [§3](#3-add-a-whole-organization) for `KBC_MANAGE_API_TOKEN` +setup -- it is the same Manage token `org setup` uses.) + +### 9.3 Roll out a new version: `data-app deploy` after `config update` + +This is the easiest gotcha to fall into. Editing the data-app's Storage +config bumps the **Storage** version, but the deployment record's +`configVersion` is a **pinned pointer** that does NOT auto-advance: + +```bash +# Bump auto-suspend from 15 minutes to 60. +kbagent --json config update \ + --project prod --component-id keboola.data-apps --config-id 01abcdefghijklmnopqrstuvwxyz \ + --set 'parameters.autoSuspendAfterSeconds=3600' --merge \ + | jq -r '.data.version' +# 4 + +# At this point the running container is still at configVersion=3. +# Verify with detail: +kbagent --json data-app detail --project prod --app-id 12345678 \ + | jq -r '.data | "storage=\(.config_version_storage) deployed=\(.config_version_deployed)"' +# storage=4 deployed=3 <-- not in sync + +# Roll out: re-pin the deployment to the latest Storage version. +kbagent --json data-app deploy --project prod --app-id 12345678 --wait +``` + +`data-app deploy` reads the latest Storage version, then PATCHes the +deployment record with the trio +`{desiredState=running, configVersion=, restartIfRunning=true}` +together. Sending only one or two of those returns HTTP 422 from the +platform; the CLI always sends all three. This is the **redeploy +contract** -- the headline of the feature. + +`data-app start` is a different command for a different job. It wakes a +container the platform parked due to `autoSuspendAfterSeconds` of +inactivity. It does NOT bump the deployed `configVersion` -- it just +reuses whatever was pinned. Use `start` for waking a parked container; +use `deploy` for rolling out a new code or config version. + +### 9.4 `stop` is reversible, `delete` is not + +```bash +# Reversible: tears down the container, preserves the URL + Storage config. +kbagent --json data-app stop --project prod --app-id 12345678 --wait +# state=stopped, desiredState=stopped + +# Wake it back up at the same configVersion (no version bump). +kbagent --json data-app start --project prod --app-id 12345678 --wait +# state=running + +# Or: just hit the URL. The platform typically auto-wakes parked +# containers on incoming HTTP traffic (~30-60s cold-boot). Use +# `data-app start` when you want an explicit, observable wake. +``` + +```bash +# Irreversible: the Data Science API cascade-deletes the deployment +# record AND the linked Storage config server-side. The URL is +# permanently retired. +kbagent --json data-app delete --project prod --app-id 12345678 --yes +``` + +A second `data-app list` after the delete returns the project's other +apps; the deleted app does not come back, and a fresh `create` against +the same slug mints a new numeric `id` (the URL hostname embeds the +numeric id, so even with the same slug the new URL differs). + +For the full reference including the API endpoints behind each command, +the four-footgun mental model in detail, encrypted-PAT round-trip +shapes, and the `--hint client|service` code-generation contract, see +[plugins/kbagent/skills/kbagent/references/data-app-workflow.md](../plugins/kbagent/skills/kbagent/references/data-app-workflow.md). + +--- + ## Troubleshooting | Symptom | Fix | diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index f77d4265..cf6d1764 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.26.0", + "version": "0.27.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 84417f27..c6cc4fa7 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -63,8 +63,9 @@ a critical failure. `kbagent --json context` and inspect the version. If missing commands needed for the current task (e.g. `flow update` needs 0.22.0+, `schedule find` needs 0.23.0+, `config set-default-bucket` needs - 0.26.0+, `storage retype` is a future composite), you MUST refuse the - task and return a handoff message to the parent: `"Cannot proceed + 0.26.0+, `data-app create / deploy / start / stop / delete / password` + need 0.27.0+, `storage retype` is a future composite), you MUST refuse + the task and return a handoff message to the parent: `"Cannot proceed safely on kbagent . Missing: . Ask user to run kbagent update, then re-invoke me."` Do not attempt the task with workarounds that use MCP strip-bug-prone tools. @@ -96,6 +97,13 @@ a critical failure. | Debug a failed job | `kbagent job detail --project P --job-id J --json` + `kbagent job run ... --log-tail-lines 200` | `kbagent workspace from-transformation` for SQL repro | "I think the issue is..." without reading logs | | Ad-hoc SQL / row-count / type audit | `kbagent workspace create` + `kbagent workspace load` + `kbagent workspace query --sql "..."` | `kbagent workspace from-transformation` for existing transform debugging | querying Keboola Storage directly via Snowflake credentials outside the workspace abstraction | | Inspect dev branch | `kbagent branch list --project P`, `kbagent branch use --project P --branch ID` | `tool call get_branch` | acting on `main` when a dev branch exists | +| Inventory data apps | `kbagent data-app list --project P` (0.27.0+) | `tool call get_configs --component_id keboola.data-apps` (Storage view only -- no state, no URL, no configVersion) | iterating `tool call` per project to reconstruct the join with the Data Science index | +| Bring a new data app online from a git repo | `kbagent data-app create --project P --name N --slug S --git-repo URL [--git-pat-env VAR \| --git-public]` (0.27.0+) | broken into `tool call create_config keboola.data-apps` + manual `kbagent encrypt values` + raw `POST /apps` -- ONLY if you need a custom shape kbagent doesn't support | raw `POST data-science/apps` followed by `PATCH desiredState=running` without `configVersion + restartIfRunning` (the §9 footgun -- pins to v2 empty shell, runner errors `dataApp.git.repository is required in /data/config.json`) | +| Roll out a new code or config version on a data app | `kbagent data-app deploy --project P --app-id N --wait` (0.27.0+) -- always sends the §9 trio | `kbagent --hint client data-app deploy ...` to inspect the generated `patch_app(desired_state=, config_version=, restart_if_running=True)` call | `tool call update_config` then `tool call run_component` (data apps are not jobs -- the queue runner does not deploy them) | +| Wake an auto-suspended data app | `kbagent data-app start --project P --app-id N` (0.27.0+) -- does NOT bump configVersion | hitting the app's URL (auto-restart triggers a 30-60s cold boot) | `kbagent data-app deploy` (overkill -- bumps the deployed configVersion unnecessarily) | +| Pause a running data app | `kbagent data-app stop --project P --app-id N` (0.27.0+) | -- | `kbagent data-app delete` (irreversible; cascades to Storage config) | +| Read the simpleAuth password for a password-gated app | `kbagent data-app password --project P --app-id N` (0.27.0+) -- requires `KBC_MANAGE_API_TOKEN` | -- | trying to "rotate" the password (not supported by the API; delete + recreate to mint a new one) | +| Tear down a data app | `kbagent data-app delete --project P --app-id N` (0.27.0+) -- cascades to Storage config; URL retired | -- | manually `tool call delete_config keboola.data-apps` while leaving the deployment record orphaned | If the table does not cover the user's task, **ask clarifying questions** instead of guessing. Returning a targeted question is a @@ -209,6 +217,25 @@ success, not a failure. feature flag, not by this setting -- see the `legacy_branch_storage` gotcha above for what the runner actually does on `--branch` writes. +- **Data apps need `data-app deploy` after `config update`** (0.27.0+): + the deployment record's `configVersion` is a pinned pointer that does + NOT auto-advance when Storage advances. Editing the `keboola.data-apps` + config via `kbagent config update` bumps the Storage version, but the + running container keeps using the OLD version until a `kbagent data-app + deploy --project P --app-id N` PATCHes the deployment with the + §9 trio `{desiredState=running, configVersion, restartIfRunning=true}`. + Sending bare `desiredState=running` (or just `configVersion`) silently + pins to v2 (the empty shell from `POST /apps`) and the runner errors + `dataApp.git.repository is required in /data/config.json` with no + top-level error surfaced -- only visible in the UI's Terminal Logs. + `kbagent data-app start` is the cheap restart for an auto-suspended + app; it does NOT bump the configVersion. Use `data-app deploy` for new + code/config rollouts, `data-app start` for waking a parked container. + PAT encryption is per-project KMS -- ciphertext does NOT cross + projects, so `kbagent data-app create` always re-encrypts plaintext via + the target project's Encryption API and refuses to write plaintext if + the round-trip does not return a `KBC::Project*` ciphertext. + - **`storage bucket-detail` is dialect-aware** (0.25.3+): the response shape depends on the bucket's backend. Snowflake buckets carry `snowflake_database` / `snowflake_schema` and per-table diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 07dfb1f6..39bbda12 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -25,6 +25,11 @@ description: > stale local config file, config version overwrite, default bucket, output bucket, default_bucket, storage.output, raw mode bucket override, custom output bucket name, + data app, data apps, keboola data app, streamlit app, streamlit deployment, + flask app, fastapi app, node app, python-js, deploy data app, + data-app create, data-app deploy, data-app password, data-app start, + app proxy, simpleAuth, app auto-suspend, configVersion, redeploy contract, + Data Science API, /apps endpoint, app password, KBC::Project ciphertext, local workspace, project directory, kbagent init. --- @@ -106,6 +111,14 @@ When working inside a git repository or project directory, run `kbagent init` (o | Assign variables to a config (auto-creates backing keboola.variables on first call) | `kbagent config variables-set --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | Read the current variable values attached to a config | `kbagent config variables-get --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | Unlink variables from a config (does NOT delete the underlying keboola.variables) | `kbagent config variables-clear --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | +| List data apps across one or more registered projects | `kbagent data-app list` | +| Show merged Data Science + Storage detail for one data app | `kbagent data-app detail --project PROJECT --app-id APP-ID` | +| Create a Keboola data app end-to-end (POST + encrypt + PUT + deploy) | `kbagent data-app create --project PROJECT --name NAME --slug SLUG --git-repo GIT-REPO` | +| Deploy the latest Storage config (the §9 redeploy contract) | `kbagent data-app deploy --project PROJECT --app-id APP-ID` | +| Wake an auto-suspended data app at its currently-pinned configVersion | `kbagent data-app start --project PROJECT --app-id APP-ID` | +| Stop a running data app (preserves the URL and Storage config) | `kbagent data-app stop --project PROJECT --app-id APP-ID` | +| Delete the deployment AND the Storage config (cascade, irreversible) | `kbagent data-app delete --project PROJECT --app-id APP-ID` | +| Retrieve the simpleAuth password for a password-gated data app | `kbagent data-app password --project PROJECT --app-id APP-ID` | | List jobs from connected projects | `kbagent job list` | | Show detailed information about a specific job | `kbagent job detail --project PROJECT --job-id JOB-ID` | | Run a job for a component configuration | `kbagent job run --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | @@ -228,6 +241,7 @@ For detailed response parsing rules and common pitfalls, see [gotchas](reference | Creating new configurations | [scaffold-workflow](references/scaffold-workflow.md) | | MCP tools (multi-project read/write) | [mcp-workflow](references/mcp-workflow.md) | | Workspace SQL debugging | [workspace-workflow](references/workspace-workflow.md) | +| **Data apps** (create / deploy / start / stop / password / delete; the §9 redeploy contract) | [data-app-workflow](references/data-app-workflow.md) | | Storage Files (upload, download, tags, load/unload) | [storage-files-workflow](references/storage-files-workflow.md) | | **Storage column types** (native types, NOT NULL, DEFAULT, branch materialize) | [storage-types-workflow](references/storage-types-workflow.md) | | Bucket sharing & linking | [sharing-workflow](references/sharing-workflow.md) | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 0de9730d..2976b66c 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -116,6 +116,17 @@ All commands support `--json` for structured output. Multi-project flags (`--pro - `workspace gc [--project NAME ...] [--dry-run] [--yes]` -- garbage-collect orphaned workspaces (and any lingering `keboola.sandboxes` configs). `--dry-run` previews without deleting; `--project` repeatable, omit to GC across all connected projects - `workspace from-transformation --project ALIAS --component-id ID --config-id ID [--row-id ID]` -- workspace from existing transform +## Data Apps (Streamlit / Flask / Node deployments) +Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, encrypted secrets, runtime size) with Data Science API (`/apps` -- deployment record, state, URL, configVersion). The CLI encapsulates the §9 redeploy contract so callers cannot pin to the empty-shell v2; see `data-app-workflow.md` for the gotcha inventory and recipes. +- `data-app list [--project NAME ...] [--branch ID]` -- list apps across projects (Data Science index merged with Storage names) +- `data-app detail --project NAME --app-id ID [--branch ID]` -- merged view (state, desired, url, configVersion, slug, git block with PAT redacted) +- `data-app create --project ALIAS --name NAME --slug SLUG --git-repo URL [--git-public/--no-git-public] [--git-username USER] [--git-pat-env VAR | --git-pat-file PATH | --git-pat-encrypted KBC::Project...] [--auth password|public] [--size tiny|small|medium|large] [--auto-suspend SECONDS] [--type python-js|python|streamlit|r|...] [--branch ID] [--no-deploy] [--wait] [--timeout SECONDS] [--keep-on-failure] [--dry-run]` -- POST shell + encrypt PAT + PUT Storage config (with auto-injected `parameters.id`) + PATCH deploy with the §9 trio. Cleanup-in-finally on failure unless `--keep-on-failure`. Default `--auth password` mints a 20-char hex simpleAuth password (retrievable via `data-app password`). +- `data-app deploy --project NAME --app-id ID [--config-version N] [--wait] [--timeout SECONDS] [--branch ID]` -- the §9 redeploy contract. Default reads latest Storage version; `--config-version` pins an older version (rollback). +- `data-app start --project NAME --app-id ID [--wait] [--timeout SECONDS]` -- wake an auto-suspended app at the currently-pinned version. Distinct from deploy: does NOT bump configVersion. +- `data-app stop --project NAME --app-id ID [--wait] [--timeout SECONDS]` -- stop a running app (URL and Storage config preserved). +- `data-app delete --project NAME --app-id ID [--yes]` -- destructive, cascades to Storage config; URL retired permanently. +- `data-app password --project NAME --app-id ID` -- read the simpleAuth password. Requires `KBC_MANAGE_API_TOKEN`. Auto-generated, not rotatable -- delete + recreate to mint a new one. + ## MCP Tools - `tool list [--project NAME] [--branch ID]` -- list available MCP tools (multi_project annotation) - `tool call TOOL_NAME [--project NAME] [--input JSON|@file|-] [--branch ID]` -- call MCP tool (read = all projects, write = single). `--input` accepts inline JSON, `@file.json`, or `-` (stdin) diff --git a/plugins/kbagent/skills/kbagent/references/data-app-workflow.md b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md new file mode 100644 index 00000000..7b840293 --- /dev/null +++ b/plugins/kbagent/skills/kbagent/references/data-app-workflow.md @@ -0,0 +1,189 @@ +# Data App Workflow -- Streamlit / Flask / Node Lifecycle + +Data apps in Keboola are deployed from a git repo into a managed container +that auto-suspends after idle. Two API surfaces own them: + +| Layer | What it owns | +|---|---| +| **Storage API** (`keboola.data-apps` config) | git block, encrypted secrets, slug, runtime size, name, description | +| **Data Science API** (`/apps`) | deployment record: state, desiredState, url, configVersion | + +`kbagent data-app` orchestrates both, plus the project's Encryption API for +git PATs. The CLI encapsulates four documented footguns so callers cannot +hit them; see "Gotchas encoded" below. + +## Quick recipes + +### Public-repo Streamlit app from scratch (no auth gate) + +```bash +kbagent --json data-app create \ + --project prod \ + --name "Hello Streamlit" \ + --slug hello-streamlit \ + --git-repo https://github.com/streamlit/streamlit-example \ + --git-public \ + --auth public \ + --wait +``` + +Three calls under the hood: `POST /apps` (mint id + configId) → `PUT +Storage config` (full body with git block + parameters.id back-pointer) → +`PATCH /apps {desiredState=running, configVersion, restartIfRunning=true}`. +The `--wait` flag polls until `state == running` (writeup §8 pitfall #1 +encoded: a transient `state == stopped` during initial deploy is *not* +treated as terminal). + +### Private-repo simpleAuth app + +```bash +export GITHUB_PAT_DATAAPP=ghp_xxxxxxxxxxxxxxxxxxxx + +kbagent --json data-app create \ + --project prod \ + --name "Internal Dashboard" \ + --slug internal-dashboard \ + --git-repo https://github.com/myorg/dashboard \ + --git-username myuser \ + --git-pat-env GITHUB_PAT_DATAAPP \ + --auth password \ + --wait +``` + +`--git-pat-env` is the recommended PAT input mode -- the plaintext token +never appears in argv. The service encrypts it under THIS project's KMS via +the Encryption API before writing it to Storage. `--auth password` (the +default) auto-mints a 20-character hex simpleAuth password; retrieve it +with: + +```bash +kbagent data-app password --project prod --app-id +# Requires KBC_MANAGE_API_TOKEN in addition to the project's Storage token. +``` + +The simpleAuth password CANNOT be rotated (writeup §11.2). To change it, +delete and recreate the app. + +### Roll out a new code version (no Storage edit) + +```bash +git push origin main # the app's configured branch +kbagent data-app deploy --project prod --app-id 12345678 --wait +``` + +`deploy` reads the latest Storage config version and PATCHes the §9 trio. +The runner clones the configured git ref at container start, so a fresh +`git push` is picked up by the next deploy without any Storage edit. + +### Roll out a new config (e.g. change size or auto-suspend) + +```bash +kbagent --json config update \ + --project prod \ + --component-id keboola.data-apps \ + --config-id 01abcdefghijklmnopqrstuvwxyz \ + --set 'runtime.backend.size="medium"' --merge + +kbagent data-app deploy --project prod --app-id 12345678 --wait +``` + +`config update` bumps the Storage version; `data-app deploy` reads the +latest and pins the deployment to it. Without the deploy step, the running +container keeps the OLD config (the deploy-record `configVersion` does not +auto-advance when Storage advances -- writeup §9 mental model). + +### Wake an auto-suspended app + +```bash +kbagent data-app start --project prod --app-id 12345678 --wait +``` + +Distinct from `deploy`: `start` does NOT bump the deployed configVersion. +It is the cheap restart for an app the platform parked due to +`autoSuspendAfterSeconds` of inactivity (writeup §8 pitfall #2). Hitting +the app's URL also auto-wakes it (cold-boot ~30-60s). + +### Rollback to an older config version + +```bash +kbagent data-app deploy --project prod --app-id 12345678 \ + --config-version 5 --wait +``` + +`--config-version` pins the deployment to a specific Storage version +(rollback). Subsequent deploys without the flag will jump back to the +latest. + +## Gotchas encoded in the CLI (so you don't have to think about them) + +1. **§9 redeploy contract** — `data-app deploy` always sends the + `{desiredState=running, configVersion, restartIfRunning=true}` trio + together. Sending just `desiredState=running` would silently pin to the + empty-shell v2 from `POST /apps`; the runner then errors + `dataApp.git.repository is required in /data/config.json` (writeup §9). + +2. **Per-project KMS encryption** — `data-app create` re-encrypts the PAT + under the target project's KMS via the Encryption API. Pre-encrypted + PATs (`--git-pat-encrypted KBC::Project...`) MUST already be encrypted + under the same project; ciphertext does not cross projects (writeup §8 + row 1). The service refuses to write plaintext if the encryption step + does not return a project-scoped ciphertext. + +3. **Cleanup-in-finally** — if the Storage PUT or initial deploy fails + after the `POST /apps` shell was created, the orphan shell is deleted + automatically. Pass `--keep-on-failure` to preserve it for forensics. + +4. **Transient `state == stopped` during initial deploy** — the platform + transitions `created → stopped → starting → running` when the deploy + starts. The CLI's poll loop refuses to treat `stopped` as terminal + while `desiredState == running`. Naive callers that exit on `stopped` + would falsely report a failure (writeup §8 row 1). + +5. **Auto-injected `parameters.id`** — after `POST /apps`, the platform + writes the numeric app id into the Storage config's `parameters.id`. The + service preserves it on every subsequent update. Stripping it breaks + the URL minting and produces inconsistent state. + +## When to use what + +| Goal | Command | +|---|---| +| Inventory: "what data apps does this project have?" | `data-app list` | +| Inspect one: "is this app running? what's its URL?" | `data-app detail --app-id N` | +| Bring a new app online from a git repo | `data-app create` (encrypts + PUTs + deploys) | +| Roll out new code already pushed to git | `data-app deploy --app-id N` | +| Roll out a new Storage config | `config update` (any field) → `data-app deploy` | +| Wake an auto-suspended app | `data-app start --app-id N` | +| Pause a running app temporarily | `data-app stop --app-id N` | +| Read the simpleAuth password | `data-app password --app-id N` (needs Manage token) | +| Tear it all down | `data-app delete --app-id N` (cascades to Storage config) | + +## What this command group deliberately does NOT cover + +- **Reading the build / runtime log** — the Data Science API does not + expose Terminal Logs as JSON; only the Keboola UI ("Terminal Log" tab) + shows them. If `data-app deploy --wait` exits with + `DATA_APP_BUILD_FAILED`, the next step is to open the UI link surfaced + in the error message. +- **Updating size / auto-suspend / git settings** — those live on the + Storage config body, not the deployment record. Use + `kbagent config update --component-id keboola.data-apps --config-id ID + --set 'runtime.backend.size="medium"' --merge` then `data-app deploy`. + `PATCH /apps {config:{...}}` is silently dropped by the API (writeup §8 + row 3). +- **Rotating the simpleAuth password** — not supported by the API. To + change the password, delete and recreate the app (writeup §11.2). + +## Endpoints used + +| HTTP | Path | When | +|---|---|---| +| `POST` | `data-science./apps` | `data-app create` step 1 | +| `GET` | `data-science./apps` | `data-app list` | +| `GET` | `data-science./apps/{id}` | `data-app detail`, poll loop | +| `PATCH` | `data-science./apps/{id}` | `data-app deploy / start / stop` | +| `DELETE` | `data-science./apps/{id}` | `data-app delete` (cascades to Storage) | +| `GET` | `data-science./apps/{id}/password` | `data-app password` (needs Manage) | +| `POST` | `encryption./encrypt` | `data-app create` step 2 (private repo) | +| `PUT` | `connection./v2/storage/.../keboola.data-apps/configs/{id}` | `data-app create` step 3, also `config update` | +| `GET` | `connection./v2/storage/.../keboola.data-apps/configs/{id}` | `data-app detail` (latest version), `data-app deploy` (read latest) | diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 5690f9c2..82321578 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -1,5 +1,61 @@ # Gotchas -- Response Parsing and Common Pitfalls +## `data-app deploy` is required after `config update` -- the running container does NOT auto-pick-up new config versions (since v0.27.0) + +- `kbagent config update --component-id keboola.data-apps ...` bumps the + Storage config version; the deployed container keeps running at the + OLD version. The Data Science deployment record's `configVersion` + field is a *pinned pointer* that does not auto-advance when Storage + advances. +- To roll out the new config, run `kbagent data-app deploy --project P + --app-id N` (optionally with `--wait`). The CLI reads the latest + Storage version and `PATCH`es the deployment with the §9 trio + `{desiredState=running, configVersion, restartIfRunning=true}`. +- **Do NOT** call `PATCH /apps/{id} {desiredState:running}` directly -- + the API silently pins to whatever `configVersion` the deployment + already had (often the empty shell from `POST /apps`), and the runner + errors `dataApp.git.repository is required in /data/config.json` with + no top-level error surfaced. The CLI's `data-app deploy` always sends + the trio together; sending only `configVersion` returns HTTP 422. +- Same goes for `kbagent data-app start`: it WAKES an auto-suspended app + at the currently-pinned version. It does NOT roll out new code or + config -- use `data-app deploy` for that. + +## Cross-project KMS ciphertext does NOT decrypt; re-encrypt per project (since v0.27.0) + +- The Encryption API's `KBC::Project*` ciphertext is bound to the + **target project's KMS key**. A `#password` encrypted in project A + will not decrypt in project B; the Storage API accepts the value but + the runner fails the `git clone` with "Invalid cipher text for key + #password" at deploy time. +- `kbagent data-app create` always re-encrypts the plaintext PAT under + the target project's KMS via the project's Encryption API. Pass the + PAT via `--git-pat-env VAR` (recommended; no argv leak) or + `--git-pat-file PATH`. Pre-encrypted ciphertext (`--git-pat-encrypted + KBC::Project...`) is accepted only when it was encrypted under the + same project's KMS -- the service refuses to write plaintext if the + encryption round-trip does not return a project-scoped ciphertext. +- Practical implication: you cannot copy-paste a `KBC::Project*` value + from one project's `keboola.data-apps` config into another's. + +## Transient `state == stopped` during initial data-app deploy is not a failure (since v0.27.0) + +- After `data-app create` (or any `data-app deploy --wait`), polling + may observe `state == stopped` once for ~5-15s before the container + reaches `running`. This is normal: the platform transitions + `created → stopped → starting → running` while spinning up the + runtime. A naive poll that exits on `stopped` would falsely report + a failure. +- The CLI's `--wait` flag refuses to treat `stopped` as terminal while + `desiredState == running`. Only `state == running` (success) and + `state == error` (build failure) and `--timeout` exhaustion are + terminal in that mode. +- A LATER `state == stopped` (after the app has been running a while) + is a different beast: it means the platform auto-suspended the + container after `autoSuspendAfterSeconds` of inactivity. Hit the URL + to wake it (auto-restart triggers a 30-60s cold boot) or run + `kbagent data-app start --app-id N`. + ## `default_bucket` is per-config and only an output prefix (since 0.26.0) - `kbagent config set-default-bucket` writes diff --git a/pyproject.toml b/pyproject.toml index c330f326..8884aef6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-agent-cli" -version = "0.26.0" +version = "0.27.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 405d7310..c0ee1a56 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -8,6 +8,14 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.27.0": [ + "New: `kbagent data-app` command group — first-class lifecycle for Keboola data apps (`keboola.data-apps` Storage component + Data Science API `/apps`). Eight subcommands: `list`, `detail`, `create`, `deploy`, `start`, `stop`, `delete`, `password`. The CLI encapsulates the **§9 redeploy contract** (always sends the `{desiredState=running, configVersion, restartIfRunning=true}` trio together; without it, `PATCH /apps {desiredState:running}` silently pins to the empty-shell v2 and the runner errors `dataApp.git.repository is required in /data/config.json`), per-project KMS encryption of git PATs (refuses to write plaintext if the Encryption API does not return a project-scoped ciphertext), cleanup-in-finally on initial-deploy failure (orphan shell deleted by default; `--keep-on-failure` opts out), and a poll loop that respects pitfall #1 — `state == stopped` is NOT terminal while `desiredState == running` (the platform transitions `created → stopped → starting → running` during initial deploy). `data-app create` accepts `--git-pat-env VAR` (recommended; no argv leak), `--git-pat-file PATH`, or `--git-pat-encrypted KBC::Project...` (must be encrypted under THIS project's KMS — ciphertext does not cross projects).", + "New: `DataScienceClient` (`src/keboola_agent_cli/data_science_client.py`) — third HTTP client class alongside `KeboolaClient` and `AiServiceClient`. Auth via `X-StorageApi-Token`; URL derived as `data-science.{stack-suffix}` from the connection URL; inherits `BaseHttpClient` for retry/backoff/token-masking. `get_app_password()` accepts the Manage token per-call so it never lives on the persistent client.", + "New: `ErrorCode` entries `DATA_APP_BUILD_FAILED`, `DATA_APP_DEPLOY_TIMEOUT`, `DATA_APP_INVALID_GIT` for surfacing data-app-specific failure modes; `data-app deploy` and `data-app create --wait` map to these on poll-loop terminal states. Existing codes (`NOT_FOUND`, `VALIDATION_ERROR`, `ENCRYPTION_FAILED`, `INVALID_TOKEN`) cover the rest.", + "New: `--hint` mode supports `client_type=data_science`. `kbagent --hint client data-app deploy …` now generates `DataScienceClient` instantiation + `patch_app(...)` call with the §9 trio inline.", + "Tests: 30 service-level tests in `tests/test_data_app_service.py` (validation, dry-run, happy-path orchestration, cleanup-in-finally, encryption-failure-aborts-loud, poll-loop semantics including the transient-stopped invariant), 10 CLI tests in `tests/test_data_app_cli.py` (mutual-exclusion validation, dual JSON+human output, `--yes` for delete, manage-token forwarding for password without leaking the token to stdout/stderr).", + "Plugin: new `data-app-workflow.md` reference + two `(since v0.27.0)` gotcha entries (the §9 redeploy contract; cross-project KMS ciphertext mismatch). `keboola-expert.md` matrix gains five rows (`create`, `deploy`, `start`, `stop`, `delete`).", + ], "0.26.0": [ "New: `kbagent config set-default-bucket --bucket BUCKET_ID | --clear [--dry-run] [--branch ID]` -- discoverable wrapper around the raw-mode `storage.output.default_bucket` workaround documented at https://keboola.atlassian.net/wiki/spaces/SUP/pages/3770155030/ (epic KBCP-108). Read-modify-write that preserves all sibling keys under `storage.output` and the rest of the configuration. Same-value writes short-circuit with `{\"changed\": false}` (no API call, no version bump). `--clear` removes only the `default_bucket` key, leaving an empty `storage.output: {}` if no other siblings live there (intentional -- mirrors `set_nested_value`'s parent-creation semantics; Storage API treats `output: {}` and missing `output` identically as 'use the auto-derived bucket'). Live-validated end-to-end on three component types -- row-based GCS extractor, root-only `keboola.ex-cnb-exchange-rates`, and `ex-generic-v2` with multiple jobs -- output tables routed to the configured bucket at job runtime in every case. The per-table `destination` override (the second method shown in the support article) keeps using the existing `kbagent config update --set 'storage.output.tables=[...]'` -- no new wrapper there because per-table mappings have many fields that don't fit a single-purpose flag.", "Fix: `kbagent sync pull --with-samples` no longer crashes with `TypeError: '>' not supported between instances of 'NoneType' and 'int'` when one or more tables in the project return `rowsCount: null` from the Storage API (typical for newly-created or empty tables on some backends, reproduced live against `kosik-sales`). `dict.get(\"rowsCount\", 0)` returns the default `0` only when the key is **missing** -- if the key is present with a `null` value, `.get()` returns `None`, and the `> 0` comparison crashed Python 3 before any sample was fetched. The filter and sort key in `SyncService._fetch_samples()` now coerce `None` to `0` via a small `_rows()` helper used in both places (`t.get(\"rowsCount\") or 0`), so empty/null-rowcount tables are gracefully skipped exactly like `rowsCount: 0` ones. Closes #233.", diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index 7d7c0928..3429b1ec 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -11,6 +11,7 @@ from .commands.component import component_app from .commands.config import config_app from .commands.context import context_command +from .commands.data_app import data_app_app from .commands.doctor import doctor_command from .commands.encrypt import encrypt_app from .commands.flow import flow_app @@ -38,6 +39,7 @@ from .services.branch_service import BranchService from .services.component_service import ComponentService from .services.config_service import ConfigService +from .services.data_app_service import DataAppService from .services.deep_lineage_service import DeepLineageService from .services.doctor_service import DoctorService from .services.encrypt_service import EncryptService @@ -82,6 +84,7 @@ _BROWSE = "Browse & Inspect" app.add_typer(component_app, name="component", rich_help_panel=_BROWSE) app.add_typer(config_app, name="config", rich_help_panel=_BROWSE) +app.add_typer(data_app_app, name="data-app", rich_help_panel=_BROWSE) app.add_typer(job_app, name="job", rich_help_panel=_BROWSE) app.add_typer(storage_app, name="storage", rich_help_panel=_BROWSE) app.add_typer(sharing_app, name="sharing", rich_help_panel=_BROWSE) @@ -301,6 +304,7 @@ def main( flow_service = FlowService(config_store=config_store) schedule_service = ScheduleService(config_store=config_store) workspace_service = WorkspaceService(config_store=config_store) + data_app_service = DataAppService(config_store=config_store) kai_service = KaiService(config_store=config_store) doctor_service = DoctorService(config_store=config_store, mcp_service=mcp_service) version_service = VersionService() @@ -353,6 +357,7 @@ def main( ctx.obj["flow_service"] = flow_service ctx.obj["schedule_service"] = schedule_service ctx.obj["workspace_service"] = workspace_service + ctx.obj["data_app_service"] = data_app_service ctx.obj["kai_service"] = kai_service ctx.obj["doctor_service"] = doctor_service ctx.obj["version_service"] = version_service diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index c4d033c9..ea00b6d5 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -528,6 +528,61 @@ kbagent workspace gc [--project NAME] [--dry-run] [--yes] Garbage-collect orphaned workspaces (keboola.sandboxes config missing). Use --dry-run to preview. +### Data Apps (Streamlit / Flask / Node deployments) + +Lifecycle for `keboola.data-apps`. Combines the Storage API (config body -- +git block, slug, runtime size, encrypted secrets) with the Data Science API +(/apps -- deployment record, state, URL, configVersion). Encapsulates the +§9 redeploy contract so callers cannot pin to the empty-shell v2. + + kbagent data-app list [--project NAME ...] [--branch ID] + List data apps across one or many projects. Merges Data Science /apps + index with Storage config names. Multi-project parallel. + + kbagent data-app detail --project NAME --app-id ID [--branch ID] + Full merged view: state, desiredState, url, deployed configVersion, slug, + runtime size, git settings (PAT redacted as ). + + kbagent data-app create --project ALIAS --name NAME --slug SLUG --git-repo URL + [--description STR | --description-file PATH] [--git-branch main] + [--git-public/--no-git-public] [--git-username USER] + [--git-pat-env VAR | --git-pat-file PATH | --git-pat-encrypted KBC::Project...] + [--auth password|public] [--size tiny|small|medium|large] [--auto-suspend SECONDS] + [--type python-js|python|streamlit|r|...] [--branch ID] + [--no-deploy] [--wait] [--timeout SECONDS] [--keep-on-failure] [--dry-run] + Create + configure + deploy in one call. Default `--auth password` mints + a 20-char hex simpleAuth password (retrievable via `data-app password`). + PAT input (private repo): env var (recommended) > file > pre-encrypted. + Pre-encrypted PATs MUST start with KBC::Project (project-scoped KMS). + Cleanup-in-finally if PUT or initial deploy fails (orphan shell deleted + by default; --keep-on-failure preserves it for forensics). + + kbagent data-app deploy --project NAME --app-id ID [--config-version N] + [--wait] [--timeout SECONDS] [--branch ID] + The §9 redeploy contract. Default reads the latest Storage config version + and pins to it; --config-version pins an older version (rollback). + Always sends {{desiredState=running, configVersion, restartIfRunning=true}} + together -- HTTP 422 otherwise. + + kbagent data-app start --project NAME --app-id ID [--wait] [--timeout SECONDS] + Wake an auto-suspended data app at its currently-pinned configVersion. + Distinct from deploy: does NOT bump the version. + + kbagent data-app stop --project NAME --app-id ID [--wait] [--timeout SECONDS] + Stop a running data app. Preserves URL and Storage config; container is + torn down. + + kbagent data-app delete --project NAME --app-id ID [--yes] + Delete the deployment AND the Storage config (cascade, irreversible). + URL is permanently retired. Confirmation prompt unless --yes. + + kbagent data-app password --project NAME --app-id ID + Retrieve the simpleAuth password. Requires KBC_MANAGE_API_TOKEN in + addition to the project's Storage token. Token is read from env or + interactive hidden prompt; never persisted, never logged. Password is + auto-generated at create time and CANNOT be rotated -- delete and + recreate the app to mint a new one. + ### Project Sync kbagent sync init --project ALIAS [--directory DIR] [--git-branching] [--adopt-existing] diff --git a/src/keboola_agent_cli/commands/data_app.py b/src/keboola_agent_cli/commands/data_app.py new file mode 100644 index 00000000..aeff0d0f --- /dev/null +++ b/src/keboola_agent_cli/commands/data_app.py @@ -0,0 +1,613 @@ +"""Data-app commands -- create, list, detail, deploy, start, stop, delete, password. + +Thin CLI layer that delegates to :class:`DataAppService`. The underlying +Keboola Data Science API is not idempotent and has several footguns +(redeploy contract, cross-project KMS, transient stopped during initial +deploy); the service encodes them. The command layer's job is argument +parsing, mutual-exclusion validation, dual JSON / human output, and +exit-code mapping. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import typer + +from ..constants import DEFAULT_JOB_RUN_TIMEOUT +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ._helpers import ( + check_cli_permission, + emit_hint, + emit_project_warnings, + get_formatter, + get_service, + map_error_to_exit_code, + resolve_manage_token, + should_hint, +) + +data_app_app = typer.Typer(help="Keboola data-app lifecycle (create, deploy, manage)") + + +@data_app_app.callback(invoke_without_command=True) +def _data_app_permission_check(ctx: typer.Context) -> None: + check_cli_permission(ctx, "data-app") + + +def _print_data_app_table(formatter, result: dict) -> None: + """Compact human-readable list of data apps across projects.""" + apps = result.get("apps", []) + if not apps: + formatter.console.print("[dim]No data apps found.[/dim]") + return + for app in apps: + formatter.console.print( + f" [bold]{app['id']}[/bold] " + f"[cyan]{app.get('name', '')}[/cyan] " + f"({app.get('type', '?')}) " + f"state=[yellow]{app.get('state', '?')}[/yellow] " + f"desired={app.get('desired_state', '?')} " + f"v{app.get('config_version', '?')} " + f"in [magenta]{app['project_alias']}[/magenta]" + ) + if app.get("url"): + formatter.console.print(f" [dim]{app['url']}[/dim]") + + +def _read_pat_from_env(env_var: str) -> str: + value = os.environ.get(env_var, "") + if not value: + raise typer.BadParameter( + f"Environment variable {env_var} is unset or empty.", + param_hint="--git-pat-env", + ) + return value + + +def _read_pat_from_file(path: Path) -> str: + try: + return path.read_text(encoding="utf-8").strip() + except OSError as exc: + raise typer.BadParameter( + f"Cannot read PAT file {path}: {exc}", + param_hint="--git-pat-file", + ) from exc + + +# --------------------------------------------------------------------------- +# data-app list +# --------------------------------------------------------------------------- + + +@data_app_app.command("list") +def data_app_list( + ctx: typer.Context, + project: list[str] | None = typer.Option( + None, + "--project", + help="Project alias to query (repeatable). None = all projects.", + ), + branch: int | None = typer.Option( + None, + "--branch", + help="Storage branch ID for the config-name lookup (defaults to production).", + ), +) -> None: + """List data apps across one or more registered projects.""" + if should_hint(ctx): + emit_hint(ctx, "data-app.list", project=project, branch=branch) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "data_app_service") + try: + result = service.list_data_apps(aliases=project, branch_id=branch) + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + if formatter.json_mode: + formatter.output(result) + else: + _print_data_app_table(formatter, result) + emit_project_warnings(formatter, result) + + +# --------------------------------------------------------------------------- +# data-app detail +# --------------------------------------------------------------------------- + + +@data_app_app.command("detail") +def data_app_detail( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), + branch: int | None = typer.Option( + None, + "--branch", + help="Storage branch ID for the linked config (defaults to production).", + ), +) -> None: + """Show merged Data Science + Storage detail for one data app.""" + if should_hint(ctx): + emit_hint(ctx, "data-app.detail", project=project, app_id=app_id, branch=branch) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "data_app_service") + try: + result = service.get_data_app(alias=project, app_id=app_id, branch_id=branch) + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + formatter.output( + result, + lambda c, d: ( + c.print(f"\n[bold]Data app:[/bold] {d.get('name', '')} ({d['id']})"), + c.print(f" [bold]Project:[/bold] {d['project_alias']}"), + c.print(f" [bold]Slug:[/bold] {d.get('slug', '')}"), + c.print(f" [bold]Type:[/bold] {d.get('type', '')}"), + c.print( + f" [bold]State:[/bold] [yellow]{d.get('state', '?')}[/yellow] " + f"(desired={d.get('desired_state', '?')})" + ), + c.print( + f" [bold]Config version:[/bold] storage=" + f"{d.get('config_version_storage', '?')}, " + f"deployed={d.get('config_version_deployed', '?')}" + ), + c.print(f" [bold]Size:[/bold] {d.get('size', '')}"), + c.print(f" [bold]Auto-suspend:[/bold] {d.get('auto_suspend_after_seconds', '?')}s"), + c.print(f" [bold]URL:[/bold] {d.get('url', '')}"), + c.print(f" [bold]Last started:[/bold] {d.get('last_start_timestamp', '')}"), + c.print(f" [bold]Git:[/bold] {d.get('git', {})}"), + ), + ) + + +# --------------------------------------------------------------------------- +# data-app create +# --------------------------------------------------------------------------- + + +@data_app_app.command("create") +def data_app_create( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + name: str = typer.Option(..., "--name", help="Display name shown in the Keboola UI"), + description: str = typer.Option( + "", + "--description", + help="Long-form description (markdown). Mutually exclusive with --description-file.", + ), + description_file: Path | None = typer.Option( + None, + "--description-file", + help="Read description from a file. Mutually exclusive with --description.", + exists=True, + readable=True, + ), + slug: str = typer.Option( + ..., "--slug", help="URL slug (lowercase alphanumeric, hyphens; 2-64 chars)" + ), + git_repo: str = typer.Option(..., "--git-repo", help="GitHub repository URL"), + git_branch: str = typer.Option("main", "--git-branch", help="Git branch to clone"), + git_public: bool = typer.Option( + False, + "--git-public/--no-git-public", + help="Mark the repository as public (no credentials needed).", + ), + git_username: str | None = typer.Option( + None, "--git-username", help="GitHub username (required for private repos)" + ), + git_pat_env: str | None = typer.Option( + None, + "--git-pat-env", + help="Environment variable containing the plaintext PAT (recommended).", + ), + git_pat_file: Path | None = typer.Option( + None, + "--git-pat-file", + help="File containing the plaintext PAT.", + exists=True, + readable=True, + ), + git_pat_encrypted: str | None = typer.Option( + None, + "--git-pat-encrypted", + help=( + "Pre-encrypted PAT (KBC::Project... ciphertext). Must be encrypted " + "against THIS project's KMS -- ciphertext does not cross projects." + ), + ), + auth: str = typer.Option( + "password", + "--auth", + help="Authentication mode: 'password' (simpleAuth) or 'public' (no auth gate).", + ), + size: str = typer.Option("tiny", "--size", help="Runtime size: tiny, small, medium, or large."), + auto_suspend: int = typer.Option( + 900, + "--auto-suspend", + help="Auto-suspend after N seconds idle (0 disables).", + ), + type_: str = typer.Option( + "python-js", + "--type", + help="Runtime type. Default 'python-js' covers Python AND Node apps.", + ), + branch: int | None = typer.Option( + None, + "--branch", + help="Keboola dev branch ID (defaults to production).", + ), + no_deploy: bool = typer.Option( + False, + "--no-deploy", + help="Skip the deploy step; create the shell + Storage config only.", + ), + wait: bool = typer.Option( + False, + "--wait", + help="Block until state == running (or error). Respects pitfall #1: stopped is not terminal.", + ), + timeout: float = typer.Option( + DEFAULT_JOB_RUN_TIMEOUT, + "--timeout", + help="Maximum seconds to wait for state == running (default 300).", + ), + keep_on_failure: bool = typer.Option( + False, + "--keep-on-failure", + help="Keep the orphan deployment shell if PUT or initial deploy fails (forensics).", + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Print the three request bodies without making any API call.", + ), +) -> None: + """Create a Keboola data app end-to-end (POST + encrypt + PUT + deploy).""" + if should_hint(ctx): + emit_hint( + ctx, + "data-app.create", + project=project, + name=name, + description=description, + slug=slug, + git_repo=git_repo, + git_branch=git_branch, + git_public=git_public, + git_username=git_username, + git_pat_env=git_pat_env, + git_pat_file=str(git_pat_file) if git_pat_file else None, + git_pat_encrypted=git_pat_encrypted, + auth=auth, + size=size, + auto_suspend=auto_suspend, + type_=type_, + branch=branch, + no_deploy=no_deploy, + wait=wait, + timeout=timeout, + keep_on_failure=keep_on_failure, + dry_run=dry_run, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "data_app_service") + + # Mutual exclusion: --description vs --description-file + if description and description_file: + formatter.error( + message="Specify either --description or --description-file, not both.", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) + effective_description = description + if description_file is not None: + try: + effective_description = description_file.read_text(encoding="utf-8") + except OSError as exc: + formatter.error( + message=f"Cannot read --description-file {description_file}: {exc}", + error_code=ErrorCode.READ_ERROR, + ) + raise typer.Exit(code=2) from None + + # Mutual exclusion of git PAT input modes (CLI layer). + pat_inputs_set = sum(1 for v in (git_pat_env, git_pat_file, git_pat_encrypted) if v is not None) + if pat_inputs_set > 1: + formatter.error( + message=( + "Specify exactly one of --git-pat-env / --git-pat-file / " + "--git-pat-encrypted; they are mutually exclusive." + ), + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) + + # Resolve PAT plaintext if needed (env / file). Encrypted form passes + # through; service validates the prefix. + pat_plaintext: str | None = None + if git_pat_env is not None: + pat_plaintext = _read_pat_from_env(git_pat_env) + elif git_pat_file is not None: + pat_plaintext = _read_pat_from_file(git_pat_file) + + try: + result = service.create_data_app( + alias=project, + name=name, + description=effective_description, + slug=slug, + git_repo=git_repo, + git_branch=git_branch, + git_public=git_public, + git_username=git_username, + git_pat_plaintext=pat_plaintext, + git_pat_encrypted=git_pat_encrypted, + auth=auth, + size=size, + auto_suspend_after_seconds=auto_suspend, + type_=type_, + branch_id=branch, + deploy=not no_deploy, + wait=wait, + timeout_seconds=timeout, + keep_on_failure=keep_on_failure, + dry_run=dry_run, + ) + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + if formatter.json_mode: + formatter.output(result) + else: + if result.get("dry_run"): + formatter.console.print("[bold]DRY RUN -- no API calls were made.[/bold]") + formatter.console.print(result["requests"]) + else: + formatter.console.print( + f"[bold green]Success:[/bold green] {result.get('message', '')}" + ) + formatter.console.print(f" [bold]App ID:[/bold] {result['id']}") + formatter.console.print(f" [bold]Config ID:[/bold] {result['config_id']}") + if result.get("url"): + formatter.console.print(f" [bold]URL:[/bold] {result['url']}") + formatter.console.print( + f" [bold]State:[/bold] {result.get('state', '?')} " + f"(desired={result.get('desired_state', '?')})" + ) + + +# --------------------------------------------------------------------------- +# data-app deploy / start / stop +# --------------------------------------------------------------------------- + + +def _run_lifecycle( + ctx: typer.Context, + service_method: str, + *, + project: str, + app_id: str, + wait: bool, + timeout: float, + extra: dict | None = None, +) -> None: + formatter = get_formatter(ctx) + service = get_service(ctx, "data_app_service") + method = getattr(service, service_method) + kwargs = {"alias": project, "app_id": app_id, "wait": wait, "timeout_seconds": timeout} + if extra: + kwargs.update(extra) + try: + result = method(**kwargs) + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + formatter.output( + result, + lambda c, d: c.print(f"[bold green]Success:[/bold green] {d.get('message', '')}"), + ) + + +@data_app_app.command("deploy") +def data_app_deploy( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), + config_version: str | None = typer.Option( + None, + "--config-version", + help="Pin a specific Storage config version (defaults to latest).", + ), + wait: bool = typer.Option(False, "--wait", help="Block until running or error."), + timeout: float = typer.Option( + DEFAULT_JOB_RUN_TIMEOUT, "--timeout", help="Max seconds to wait." + ), + branch: int | None = typer.Option( + None, + "--branch", + help="Storage branch for reading the latest version (defaults to production).", + ), +) -> None: + """Deploy the latest Storage config (the §9 redeploy contract).""" + if should_hint(ctx): + emit_hint( + ctx, + "data-app.deploy", + project=project, + app_id=app_id, + config_version=config_version, + wait=wait, + branch=branch, + ) + return + _run_lifecycle( + ctx, + "deploy_data_app", + project=project, + app_id=app_id, + wait=wait, + timeout=timeout, + extra={"config_version": config_version, "branch_id": branch}, + ) + + +@data_app_app.command("start") +def data_app_start( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), + wait: bool = typer.Option(False, "--wait", help="Block until running or error."), + timeout: float = typer.Option( + DEFAULT_JOB_RUN_TIMEOUT, "--timeout", help="Max seconds to wait." + ), +) -> None: + """Wake an auto-suspended data app at its currently-pinned configVersion.""" + if should_hint(ctx): + emit_hint(ctx, "data-app.start", project=project, app_id=app_id, wait=wait) + return + _run_lifecycle( + ctx, + "start_data_app", + project=project, + app_id=app_id, + wait=wait, + timeout=timeout, + ) + + +@data_app_app.command("stop") +def data_app_stop( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), + wait: bool = typer.Option(False, "--wait", help="Block until stopped."), + timeout: float = typer.Option( + DEFAULT_JOB_RUN_TIMEOUT, "--timeout", help="Max seconds to wait." + ), +) -> None: + """Stop a running data app (preserves the URL and Storage config).""" + if should_hint(ctx): + emit_hint(ctx, "data-app.stop", project=project, app_id=app_id, wait=wait) + return + _run_lifecycle( + ctx, + "stop_data_app", + project=project, + app_id=app_id, + wait=wait, + timeout=timeout, + ) + + +# --------------------------------------------------------------------------- +# data-app delete +# --------------------------------------------------------------------------- + + +@data_app_app.command("delete") +def data_app_delete( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip the confirmation prompt.", + ), +) -> None: + """Delete the deployment AND the Storage config (cascade, irreversible).""" + if should_hint(ctx): + emit_hint(ctx, "data-app.delete", project=project, app_id=app_id) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "data_app_service") + + if ( + not yes + and not formatter.json_mode + and not typer.confirm( + f"Delete data app {app_id} in '{project}'? " + "This deletes the deployment AND the Storage config (irreversible)." + ) + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + + try: + result = service.delete_data_app(alias=project, app_id=app_id) + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + formatter.output( + result, + lambda c, d: c.print(f"[bold green]Success:[/bold green] {d['message']}"), + ) + + +# --------------------------------------------------------------------------- +# data-app password (requires Manage token) +# --------------------------------------------------------------------------- + + +@data_app_app.command("password") +def data_app_password( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + app_id: str = typer.Option(..., "--app-id", help="Data Science numeric app id"), +) -> None: + """Retrieve the simpleAuth password for a password-gated data app. + + Requires KBC_MANAGE_API_TOKEN in addition to the project's Storage + token. Token is read from env or interactive prompt; never persisted. + """ + if should_hint(ctx): + emit_hint(ctx, "data-app.password", project=project, app_id=app_id) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "data_app_service") + manage_token = resolve_manage_token() + + try: + result = service.get_data_app_password( + alias=project, app_id=app_id, manage_token=manage_token + ) + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + formatter.output( + result, + lambda c, d: ( + c.print(f"[bold green]Success:[/bold green] {d['message']}"), + c.print(f"\n[bold yellow]Password:[/bold yellow] {d['password']}"), + ), + ) diff --git a/src/keboola_agent_cli/data_science_client.py b/src/keboola_agent_cli/data_science_client.py new file mode 100644 index 00000000..7396957b --- /dev/null +++ b/src/keboola_agent_cli/data_science_client.py @@ -0,0 +1,192 @@ +"""Keboola Data Science API client (data-app deployment records). + +The Data Science API owns the *deployment* side of a data app — id, state, +desiredState, url, configVersion. The Storage API +(``keboola.data-apps`` configs) owns the *configuration* side — git block, +encrypted secrets, slug, runtime size. Both must stay in sync; see +``services/data_app_service.py`` for the orchestration. + +URL derivation: ``https://data-science.`` from the project's +connection URL via ``BaseHttpClient._derive_service_url``. Auth: same +``X-StorageApi-Token`` as the Storage API. The single exception is +``GET /apps/{id}/password`` which additionally requires +``X-KBC-ManageApiToken`` -- the manage token is passed per-call so the +client itself stays project-scoped. + +Verified shapes (writeup §2 / §6 / §9, replayed in this PR's live +validation): + + POST /apps -> 201, {id, configId, ...} + GET /apps -> 200, [{id, configId, state, desiredState, url}, ...] + GET /apps/{id} -> 200, full deployment record + PATCH /apps/{id} -> 200, deployment record (only + desiredState / configVersion / + restartIfRunning persist; + ``config:{...}`` is silently + dropped) + DELETE /apps/{id} -> 202, cascades to Storage config + GET /apps/{id}/password -> 200, {password: "<20 hex>"} + (requires both Storage and + Manage tokens) +""" + +from __future__ import annotations + +import json +import logging +from typing import Any +from urllib.parse import quote + +from . import __version__ +from .constants import DEFAULT_TIMEOUT +from .http_base import BaseHttpClient + +logger = logging.getLogger(__name__) + + +class DataScienceClient(BaseHttpClient): + """HTTP client for the Keboola Data Science API (``/apps``). + + Inherits retry / backoff / token-masking from ``BaseHttpClient``. + """ + + def __init__(self, stack_url: str, token: str) -> None: + self._stack_url = stack_url.rstrip("/") + ds_base_url = self._derive_service_url(self._stack_url, "data-science") + headers = { + "X-StorageApi-Token": token, + "User-Agent": f"keboola-agent-cli/{__version__}", + } + super().__init__( + base_url=ds_base_url, + token=token, + headers=headers, + timeout=DEFAULT_TIMEOUT, + ) + + def __enter__(self) -> DataScienceClient: + return self + + def __exit__(self, *args: Any) -> None: + self.close() + + def list_apps(self) -> list[dict[str, Any]]: + """Return the thin index of data apps in the project (no body filter). + + The Data Science API scopes responses by the token's project; there + is no ``branchId`` query parameter on the list endpoint. + """ + response = self._do_request("GET", "/apps") + body = response.json() + # Some stacks wrap the list in {"data": [...]}; fall back gracefully. + apps = (body.get("data") or body.get("apps") or []) if isinstance(body, dict) else body + return apps if isinstance(apps, list) else [] + + def get_app(self, app_id: str) -> dict[str, Any]: + """Fetch a single deployment record by numeric app id.""" + response = self._do_request("GET", f"/apps/{quote(str(app_id), safe='')}") + return response.json() + + def create_app( + self, + *, + type_: str, + name: str, + description: str, + config: dict[str, Any], + branch_id: int | None = None, + ) -> dict[str, Any]: + """Create the deployment shell + linked Storage config in one call. + + Server-generated identifiers: ``id`` (numeric) and ``configId`` + (ULID). The ``configId`` field in the request body is silently + ignored (writeup §5) -- callers must accept whatever ULID the + server assigns and round-trip it on subsequent updates. + + ``config`` carries the *initial* Storage configuration body. The + full config (git block, encrypted secrets, etc.) is added via + ``KeboolaClient.update_config`` after creation; sending it here is + possible but the encryption step depends on knowing + ``config_id`` first, so the canonical flow is: + ``create_app`` -> encrypt secrets -> ``update_config``. + """ + payload: dict[str, Any] = { + "branchId": branch_id, + "type": type_, + "name": name, + "description": description, + "config": config, + } + response = self._do_request( + "POST", + "/apps", + content=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + return response.json() + + def patch_app( + self, + app_id: str, + *, + desired_state: str | None = None, + config_version: str | None = None, + restart_if_running: bool | None = None, + ) -> dict[str, Any]: + """Update the deployment record (state / pinned config version). + + IMPORTANT: never sends a ``config`` block — that surface is owned + by the Storage API (writeup §2.1, §8 pitfall row 3). Updating + size / autoSuspend / git settings goes through ``update_config`` + on the Storage API. + + The §9 redeploy contract requires + ``desired_state="running"`` + ``config_version=`` + + ``restart_if_running=True`` together when bumping the deployed + config version; sending ``config_version`` alone yields HTTP 422. + """ + payload: dict[str, Any] = {} + if desired_state is not None: + payload["desiredState"] = desired_state + if config_version is not None: + payload["configVersion"] = config_version + if restart_if_running is not None: + payload["restartIfRunning"] = restart_if_running + response = self._do_request( + "PATCH", + f"/apps/{quote(str(app_id), safe='')}", + content=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + return response.json() + + def delete_app(self, app_id: str) -> None: + """Delete the deployment AND the linked Storage config (cascade). + + Returns HTTP 202 on success; the body is empty. + """ + self._do_request("DELETE", f"/apps/{quote(str(app_id), safe='')}") + + def get_app_password(self, app_id: str, manage_token: str) -> dict[str, Any]: + """Retrieve the auto-generated simpleAuth password. + + Requires both the project's Storage token (already on + ``self._client``) AND a Manage API token, supplied per-call so the + manage token never lives on the client instance. + + The 20-character hex password is auto-generated at app create time + and is NOT rotatable -- to change it you must delete and recreate + the app (writeup §11.2). + """ + path = f"/apps/{quote(str(app_id), safe='')}/password" + # Pass the Manage token via per-request `headers=`. httpx merges these + # with the client's persistent headers for this call only, so the + # manage token never lives on `self._client`. Using `_do_request` + # gives us the same retry/backoff and uniform error mapping as every + # other call in this client (no bespoke try/except needed). + response = self._do_request( + "GET", + path, + headers={"X-KBC-ManageApiToken": manage_token}, + ) + return response.json() diff --git a/src/keboola_agent_cli/errors.py b/src/keboola_agent_cli/errors.py index 6a7f8dda..1c307acc 100644 --- a/src/keboola_agent_cli/errors.py +++ b/src/keboola_agent_cli/errors.py @@ -93,6 +93,11 @@ class ErrorCode(StrEnum): INVALID_FLOW_DAG = "INVALID_FLOW_DAG" SCHEDULE_DELETE_FAILED = "SCHEDULE_DELETE_FAILED" + # Data apps (new in 0.27.0) + DATA_APP_BUILD_FAILED = "DATA_APP_BUILD_FAILED" + DATA_APP_DEPLOY_TIMEOUT = "DATA_APP_DEPLOY_TIMEOUT" + DATA_APP_INVALID_GIT = "DATA_APP_INVALID_GIT" + def mask_token(token: str) -> str: """Mask a Keboola Storage API token for safe display. diff --git a/src/keboola_agent_cli/hints/definitions/__init__.py b/src/keboola_agent_cli/hints/definitions/__init__.py index b332bd37..eb9cac56 100644 --- a/src/keboola_agent_cli/hints/definitions/__init__.py +++ b/src/keboola_agent_cli/hints/definitions/__init__.py @@ -4,6 +4,7 @@ branch, # noqa: F401 component, # noqa: F401 config, # noqa: F401 + data_app, # noqa: F401 encrypt, # noqa: F401 flow, # noqa: F401 job, # noqa: F401 diff --git a/src/keboola_agent_cli/hints/definitions/data_app.py b/src/keboola_agent_cli/hints/definitions/data_app.py new file mode 100644 index 00000000..9a95b080 --- /dev/null +++ b/src/keboola_agent_cli/hints/definitions/data_app.py @@ -0,0 +1,400 @@ +"""Hint definitions for the ``data-app`` command group.""" + +from .. import HintRegistry +from ..models import ClientCall, CommandHint, HintStep, ServiceCall + +# ── data-app list ────────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.list", + description=( + "List data apps across one or more registered projects. " + "Merges the Data Science /apps thin index with each app's " + "Storage config name (one extra GET per project)." + ), + steps=[ + HintStep( + comment=( + "Fetch the thin /apps index from the Data Science API " + "(scoped to the token's project) and join with Storage " + "config names." + ), + client=ClientCall( + method="list_apps", + args={}, + client_type="data_science", + result_var="apps", + result_hint="list[dict]", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="list_data_apps", + args={ + "aliases": "{project}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "Uses DataScienceClient(stack_url, token) -- new client class " + "added in 0.27.0; do not confuse with KeboolaClient or AiServiceClient.", + "Service envelope: {'apps': [...], 'errors': [...]} with one row per app.", + ], + ) +) + + +# ── data-app detail ──────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.detail", + description=( + "Show merged Data Science + Storage view of one data app. " + "Reads the deployment record and the linked Storage config " + "(the configId on the deployment)." + ), + steps=[ + HintStep( + comment=( + "Two GETs: the Data Science deployment record and the " + "linked Storage config. Service merges them and redacts " + "the encrypted git PAT in human mode." + ), + client=ClientCall( + method="get_app", + args={"app_id": "{app_id}"}, + client_type="data_science", + result_var="app", + result_hint="dict", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="get_data_app", + args={ + "alias": "{project}", + "app_id": "{app_id}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "config_version_storage and config_version_deployed often differ -- " + "the deployed pin is stale until the next `data-app deploy`.", + ], + ) +) + + +# ── data-app create ──────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.create", + description=( + "Create + configure + deploy a data app in one call. " + "Encapsulates the §9 redeploy contract so callers cannot pin " + "to the empty-shell v2." + ), + steps=[ + HintStep( + comment=( + "End-to-end: POST /apps shell, encrypt git PAT (private " + "repo), PUT Storage config with auto-injected " + "parameters.id back-pointer, PATCH deploy with the " + "{desiredState, configVersion, restartIfRunning} trio." + ), + client=ClientCall( + method="create_app", + args={ + "type_": "{type_}", + "name": "{name}", + "description": "{description}", + # Placeholder rendered as a string literal so the + # snippet parses as valid Python. The actual shell + # body is built by the service in production -- + # the client-side hint exists for illustration. + "config": '""', + "branch_id": "{branch}", + }, + client_type="data_science", + result_var="shell", + result_hint="dict", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="create_data_app", + args={ + "alias": "{project}", + "name": "{name}", + "description": "{description}", + "slug": "{slug}", + "git_repo": "{git_repo}", + "git_branch": "{git_branch}", + "git_public": "{git_public}", + "git_username": "{git_username}", + # Git PAT input modes (mutually exclusive). The + # service expects either a plaintext PAT (the + # service re-encrypts it under THIS project's KMS) + # or a project-scoped ciphertext. The renderer + # auto-quotes the {git_pat_env} placeholder, so the + # rendered snippet becomes ``os.environ["VAR"]``. + "git_pat_plaintext": "os.environ[{git_pat_env}]", + "git_pat_encrypted": "{git_pat_encrypted}", + "auth": "{auth}", + "size": "{size}", + "auto_suspend_after_seconds": "{auto_suspend}", + "type_": "{type_}", + "branch_id": "{branch}", + "deploy": "not {no_deploy}", + "wait": "{wait}", + "timeout_seconds": "{timeout}", + "keep_on_failure": "{keep_on_failure}", + "dry_run": "{dry_run}", + }, + ), + ), + ], + notes=[ + "Encryption is per-project KMS -- ciphertext does not cross projects " + "(writeup §8). Always pass the plaintext PAT via env var so the service " + "re-encrypts under the target project's key.", + "On failure between POST and PUT, the orphan shell is cleaned up " + "automatically unless --keep-on-failure is set.", + "--dry-run prints all three request bodies without making any API call.", + "Service-call arguments above include all CLI flags. Drop the " + "git_* keys when the repo is public (--git-public).", + ], + ) +) + + +# ── data-app deploy ──────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.deploy", + description=("Deploy the latest Storage config to a data app -- the §9 redeploy contract."), + steps=[ + HintStep( + comment=( + "GET app -> read configId. GET Storage config -> read " + "version. PATCH /apps/{id} with the trio " + "{desiredState=running, configVersion, restartIfRunning=true}. " + "Sending configVersion alone returns HTTP 422." + ), + client=ClientCall( + method="patch_app", + args={ + "app_id": "{app_id}", + "desired_state": '"running"', + "config_version": "{config_version}", + "restart_if_running": "True", + }, + client_type="data_science", + result_var="deployed", + result_hint="dict", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="deploy_data_app", + args={ + "alias": "{project}", + "app_id": "{app_id}", + "config_version": "{config_version}", + "wait": "{wait}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "Without --config-version, the service reads the latest from " + "Storage and pins to it. With --config-version, the caller can " + "deploy an older version (rollback).", + "Always sends restart_if_running=True -- HTTP 422 otherwise.", + ], + ) +) + + +# ── data-app start ───────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.start", + description=( + "Wake an auto-suspended data app at its currently-pinned " + "configVersion. Distinct from deploy: does NOT bump the version." + ), + steps=[ + HintStep( + comment=( + "PATCH /apps/{id} with {desiredState=running, " + "restartIfRunning=true} -- no configVersion -- so the " + "platform reuses the currently-pinned version." + ), + client=ClientCall( + method="patch_app", + args={ + "app_id": "{app_id}", + "desired_state": '"running"', + "restart_if_running": "True", + }, + client_type="data_science", + result_var="deployed", + result_hint="dict", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="start_data_app", + args={ + "alias": "{project}", + "app_id": "{app_id}", + "wait": "{wait}", + }, + ), + ), + ], + notes=[ + "Use this to wake an app after autoSuspendAfterSeconds expired. " + "For a code change, use `data-app deploy` instead.", + ], + ) +) + + +# ── data-app stop ────────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.stop", + description="Stop a running data app (preserves URL and Storage config).", + steps=[ + HintStep( + comment="PATCH /apps/{id} with {desiredState=stopped}.", + client=ClientCall( + method="patch_app", + args={ + "app_id": "{app_id}", + "desired_state": '"stopped"', + }, + client_type="data_science", + result_var="deployed", + result_hint="dict", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="stop_data_app", + args={ + "alias": "{project}", + "app_id": "{app_id}", + "wait": "{wait}", + }, + ), + ), + ], + ) +) + + +# ── data-app delete ──────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.delete", + description=( + "Delete the deployment AND the Storage config (cascade). URL is permanently retired." + ), + steps=[ + HintStep( + comment=( + "DELETE /apps/{id}. Returns HTTP 202; cascades to " + "Storage config delete. There is no recovery." + ), + client=ClientCall( + method="delete_app", + args={"app_id": "{app_id}"}, + client_type="data_science", + result_var="result", + result_hint="None", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="delete_data_app", + args={ + "alias": "{project}", + "app_id": "{app_id}", + }, + ), + ), + ], + notes=[ + "If you only want to stop the app temporarily, use `data-app stop` " + "instead -- it preserves the URL and config.", + ], + ) +) + + +# ── data-app password ────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="data-app.password", + description=( + "Retrieve the auto-generated simpleAuth password for a " + "password-gated data app. Requires both Storage and Manage tokens." + ), + steps=[ + HintStep( + comment=( + "GET /apps/{id}/password requires both X-StorageApi-Token " + "(already on the client) and X-KBC-ManageApiToken (passed " + "per-call so it never lives on the persistent client)." + ), + client=ClientCall( + method="get_app_password", + args={ + "app_id": "{app_id}", + "manage_token": 'os.environ["KBC_MANAGE_API_TOKEN"]', + }, + client_type="data_science", + result_var="payload", + result_hint="dict", + ), + service=ServiceCall( + service_class="DataAppService", + service_module="data_app_service", + method="get_data_app_password", + args={ + "alias": "{project}", + "app_id": "{app_id}", + # Same env-var resolution as the client-side hint + # above. The literal string ``os.environ[...]`` is + # emitted verbatim into the rendered snippet (no + # placeholder substitution) so it parses as a + # subscript expression, not a comparison op. + "manage_token": 'os.environ["KBC_MANAGE_API_TOKEN"]', + }, + ), + ), + ], + notes=[ + "Password is auto-generated at app create time and cannot be rotated. " + "Delete + recreate the app to mint a new one (writeup §11.2).", + "Manage token is read from KBC_MANAGE_API_TOKEN env var or interactive " + "hidden prompt; never persisted, never logged.", + ], + ) +) diff --git a/src/keboola_agent_cli/hints/renderer.py b/src/keboola_agent_cli/hints/renderer.py index 1e25d91f..09e4f4de 100644 --- a/src/keboola_agent_cli/hints/renderer.py +++ b/src/keboola_agent_cli/hints/renderer.py @@ -133,7 +133,9 @@ def render( for step in hint.steps: client_types.add(step.client.client_type) - needs_os = "storage" in client_types or "manage" in client_types + needs_os = ( + "storage" in client_types or "manage" in client_types or "data_science" in client_types + ) if needs_os: lines.append("import os") if needs_time: @@ -144,6 +146,8 @@ def render( if "storage" in client_types: lines.append("from keboola_agent_cli.client import KeboolaClient") + if "data_science" in client_types: + lines.append("from keboola_agent_cli.data_science_client import DataScienceClient") if "manage" in client_types: lines.append("from keboola_agent_cli.manage_client import ManageClient") if "mcp" in client_types: @@ -167,6 +171,12 @@ def render( lines.append(' token=os.environ["KBC_STORAGE_TOKEN"],') lines.append(")") + if "data_science" in client_types: + lines.append("ds_client = DataScienceClient(") + lines.append(f' stack_url="{url}",') + lines.append(' token=os.environ["KBC_STORAGE_TOKEN"],') + lines.append(")") + if "manage" in client_types: lines.append("manage_client = ManageClient(") lines.append(f' base_url="{url}",') @@ -184,6 +194,8 @@ def render( close_vars = [] if "storage" in client_types: close_vars.append("client") + if "data_science" in client_types: + close_vars.append("ds_client") if "manage" in client_types: close_vars.append("manage_client") @@ -208,6 +220,7 @@ def render( client_var_map = { "storage": "client", + "data_science": "ds_client", "manage": "manage_client", "mcp": "mcp_service", } @@ -292,7 +305,29 @@ def render( if step.service: service_imports[step.service.service_module] = step.service.service_class + # If any resolved arg references ``os.environ`` or ``os.path`` + # (e.g. the data-app create hint pipes the PAT through + # ``os.environ[VAR]``), the rendered snippet must ``import os`` -- + # otherwise running it raises NameError. The match list is + # restricted to known stdlib accessors so an unrelated arg with + # the substring ``os.`` (e.g. a hypothetical ``infos.path``) + # cannot trigger a spurious unused import. + _os_token_indicators = ("os.environ", "os.path", "os.getenv") + needs_os = False + for step in hint.steps: + if step.service is None: + continue + resolved = _substitute_params(step.service.args, params) + for value in resolved.values(): + if isinstance(value, str) and any(tok in value for tok in _os_token_indicators): + needs_os = True + break + if needs_os: + break + # Imports + if needs_os: + lines.append("import os") lines.append("from pathlib import Path") lines.append("") lines.append("from keboola_agent_cli.config_store import ConfigStore") diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 86bbed7d..e2887204 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -92,6 +92,15 @@ # Component discovery "component.list": "read", "component.detail": "read", + # Data apps (Data Science API + keboola.data-apps Storage component) + "data-app.list": "read", + "data-app.detail": "read", + "data-app.password": "read", + "data-app.create": "write", + "data-app.deploy": "write", + "data-app.start": "write", + "data-app.stop": "write", + "data-app.delete": "destructive", # Storage browsing "storage.buckets": "read", "storage.bucket-detail": "read", diff --git a/src/keboola_agent_cli/services/data_app_service.py b/src/keboola_agent_cli/services/data_app_service.py new file mode 100644 index 00000000..d5d4b1e7 --- /dev/null +++ b/src/keboola_agent_cli/services/data_app_service.py @@ -0,0 +1,1243 @@ +"""Data-app service — Keboola Data Science API + ``keboola.data-apps``. + +Owns the orchestration that the underlying APIs do *not* provide: + +- the §9 redeploy contract (read latest Storage version, then PATCH with + ``{desiredState=running, configVersion, restartIfRunning=true}`` together) +- per-project KMS encryption of git PATs via :class:`EncryptService` +- cleanup-in-finally on initial-deploy failure so a failed + ``data-app create`` does not leak an empty deployment shell +- a poll loop that refuses to treat ``state == stopped`` as terminal while + ``desiredState == running`` (writeup §8 pitfall row 1 — the transient + ``stopped`` between the initial container teardown and runtime spin-up) + +Naming convention: callers pass an integer-like ``app_id``; we coerce to +str so paths build cleanly regardless of input type. +""" + +from __future__ import annotations + +import json +import logging +import re +import time +from collections.abc import Callable +from typing import Any + +from ..constants import DEFAULT_JOB_RUN_TIMEOUT +from ..data_science_client import DataScienceClient +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..models import ProjectConfig +from .base import BaseService, ClientFactory +from .encrypt_service import EncryptService + +logger = logging.getLogger(__name__) + + +DataScienceClientFactory = Callable[[str, str], DataScienceClient] + + +def _default_ds_client_factory(stack_url: str, token: str) -> DataScienceClient: + return DataScienceClient(stack_url=stack_url, token=token) + + +# --------------------------------------------------------------------------- +# Constants encoded from the writeup +# --------------------------------------------------------------------------- + +DATA_APP_COMPONENT_ID = "keboola.data-apps" + +VALID_TYPES: tuple[str, ...] = ( + "python-js", + "python", + "streamlit", + "r", + "python-databricks", + "python-snowpark", + "python-mlflow", +) +DEFAULT_TYPE = "python-js" + +VALID_SIZES: tuple[str, ...] = ("tiny", "small", "medium", "large") +DEFAULT_SIZE = "tiny" + +DEFAULT_AUTO_SUSPEND_SECONDS = 900 + +# Slug must match the URL-safe segment used in the auto-minted hostname. +SLUG_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$") + +# Encrypted-secret prefixes produced by the Encryption API for project-scoped +# (KMS) ciphertext. The platform emits both ``KBC::ProjectSecure`` (legacy) +# and ``KBC::ProjectSecureGKMS`` (GCP); both are project-bound and decrypt +# only with the originating project's KMS key. +ENCRYPTED_PASSWORD_PREFIXES: tuple[str, ...] = ( + "KBC::ProjectSecure::", + "KBC::ProjectSecureGKMS::", + "KBC::ProjectSecureKMS::", +) + +# Defence-in-depth caps for free-form user input. The platform may accept +# longer values, but kbagent refuses anything beyond these bounds at the +# service boundary so an external caller using the service directly cannot +# exfiltrate giant payloads or smuggle control characters into audit logs. +MAX_NAME_LENGTH = 255 +MAX_DESCRIPTION_LENGTH = 65_536 # 64 KiB +MAX_GIT_REPO_LENGTH = 1024 +MAX_GIT_BRANCH_LENGTH = 255 +MAX_GIT_USERNAME_LENGTH = 255 + +POLL_INTERVAL_SECONDS = 5.0 +TERMINAL_ERROR_STATE = "error" +RUNNING_STATE = "running" +STOPPED_STATE = "stopped" + + +def _has_control_chars(value: str, *, allow_whitespace: bool = False) -> bool: + """Return True if ``value`` contains any ASCII control byte (0x00-0x1f / 0x7f). + + Set ``allow_whitespace=True`` to permit ``\\t \\n \\r`` (description markdown); + everything else under 0x20 + 0x7f is still rejected. + """ + allowed = {0x09, 0x0A, 0x0D} if allow_whitespace else set() + for ch in value: + code = ord(ch) + if code in allowed: + continue + if code < 0x20 or code == 0x7F: + return True + return False + + +# URL schemes accepted for ``--git-repo``. Anything else (file://, gopher://, +# bare ssh syntax like ``git@host:path``) is rejected at the service +# boundary -- the data-app runner only ever talks https / http / git / ssh. +ALLOWED_GIT_REPO_SCHEMES: tuple[str, ...] = ( + "https://", + "http://", + "ssh://", + "git://", +) + + +def _build_simple_auth_block() -> dict[str, Any]: + """Authorization block for password-gated apps (writeup §11.2).""" + return { + "app_proxy": { + "auth_providers": [{"id": "simpleAuth", "type": "password"}], + "auth_rules": [ + { + "type": "pathPrefix", + "value": "/", + "auth_required": True, + "auth": ["simpleAuth"], + } + ], + }, + } + + +def _redact_secret(value: Any) -> Any: + """Replace encrypted ``#`` values with a placeholder for human output.""" + if isinstance(value, str) and value.startswith("KBC::"): + return "" + return value + + +def _redact_git_block(git: dict[str, Any]) -> dict[str, Any]: + """Return a copy of the git block with the encrypted password redacted.""" + redacted = dict(git) + if "#password" in redacted: + redacted["#password"] = _redact_secret(redacted["#password"]) + return redacted + + +def _redact_storage_config(storage_config: dict[str, Any]) -> dict[str, Any]: + """Deep-copy the Storage config dict and redact any nested encrypted PAT. + + Used by ``get_data_app`` so the ``raw.storage_config`` echo cannot leak + the encrypted git PAT verbatim into ``--json`` output. The redaction is + cosmetic (the ciphertext is not a secret in the cryptographic sense -- + it can only be decrypted by Keboola's KMS), but defense-in-depth: + keeping ciphertext out of consumed JSON limits its blast radius if a + downstream consumer logs it. + """ + if not isinstance(storage_config, dict): + return storage_config + redacted = dict(storage_config) + configuration = redacted.get("configuration") + if isinstance(configuration, dict): + configuration = dict(configuration) + parameters = configuration.get("parameters") + if isinstance(parameters, dict): + parameters = dict(parameters) + data_app = parameters.get("dataApp") + if isinstance(data_app, dict): + data_app = dict(data_app) + git = data_app.get("git") + if isinstance(git, dict): + data_app["git"] = _redact_git_block(git) + parameters["dataApp"] = data_app + configuration["parameters"] = parameters + redacted["configuration"] = configuration + return redacted + + +class DataAppService(BaseService): + """Lifecycle service for Keboola data apps. + + Wires together the Data Science API (deployment record), the Storage + API (``keboola.data-apps`` config), and the Encryption API (per-project + KMS for git PATs). + """ + + def __init__( + self, + config_store: Any, + client_factory: ClientFactory | None = None, + ds_client_factory: DataScienceClientFactory | None = None, + encrypt_service: EncryptService | None = None, + ) -> None: + super().__init__(config_store=config_store, client_factory=client_factory) + self._ds_client_factory = ds_client_factory or _default_ds_client_factory + self._encrypt_service = encrypt_service or EncryptService( + config_store=config_store, client_factory=client_factory + ) + + # ------------------------------------------------------------------ + # Public lifecycle methods (one per CLI subcommand) + # ------------------------------------------------------------------ + + def list_data_apps( + self, + aliases: list[str] | None = None, + branch_id: int | None = None, + ) -> dict[str, Any]: + """Return data apps across one or more projects. + + Returns ``{"apps": [...], "errors": [...]}`` to match the envelope + used by ``ConfigService.list_configs`` / ``StorageService.list_buckets``. + Per-project failures are captured in ``errors``; they never abort + the others. + """ + projects = self.resolve_projects(aliases) + + def worker( + alias: str, project: ProjectConfig + ) -> tuple[str, list[dict[str, Any]], bool] | tuple[str, dict[str, str]]: + ds_client = self._ds_client_factory(project.stack_url, project.token) + storage_client = self._client_factory(project.stack_url, project.token) + try: + apps = ds_client.list_apps() + config_names = self._fetch_data_app_config_names(storage_client, branch_id) + merged: list[dict[str, Any]] = [] + for app in apps: + config_id = str(app.get("configId") or "") + merged.append( + { + "project_alias": alias, + "id": str(app.get("id", "")), + "config_id": config_id, + "name": config_names.get(config_id, app.get("name", "")), + "type": app.get("type", ""), + "state": app.get("state", ""), + "desired_state": app.get("desiredState", ""), + "config_version": str(app.get("configVersion", "") or ""), + "url": app.get("url", ""), + "size": app.get("size", ""), + "auto_suspend_after_seconds": app.get("autoSuspendAfterSeconds"), + "last_start_timestamp": app.get("lastStartTimestamp"), + } + ) + # Per-project sort dropped: the global sort below + # subsumes it after we concatenate every worker's output. + return (alias, merged, True) + except KeboolaApiError as exc: + return ( + alias, + { + "project_alias": alias, + "error_code": str(exc.error_code), + "message": exc.message, + }, + ) + finally: + ds_client.close() + storage_client.close() + + successes, errors = self._run_parallel(projects, worker) + all_apps: list[dict[str, Any]] = [] + for _alias, apps, _ok in successes: + all_apps.extend(apps) + all_apps.sort(key=lambda a: (a["project_alias"], a.get("id", ""))) + errors.sort(key=lambda e: e.get("project_alias", "")) + return {"apps": all_apps, "errors": errors} + + def get_data_app( + self, + alias: str, + app_id: str, + branch_id: int | None = None, + ) -> dict[str, Any]: + """Merge the Data Science deployment record with the Storage config. + + The Data Science record is the source of truth for state / URL / + configVersion; the Storage config carries slug / git settings / + runtime size / human description. Callers normally want both. + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + ds_client = self._ds_client_factory(project.stack_url, project.token) + storage_client = self._client_factory(project.stack_url, project.token) + try: + app = ds_client.get_app(app_id) + config_id = str(app.get("configId") or "") + storage_config: dict[str, Any] = {} + if config_id: + try: + storage_config = storage_client.get_config_detail( + DATA_APP_COMPONENT_ID, config_id, branch_id=branch_id + ) + except KeboolaApiError as exc: + if exc.error_code != ErrorCode.NOT_FOUND: + raise + finally: + ds_client.close() + storage_client.close() + + configuration = storage_config.get("configuration") or {} + if isinstance(configuration, str): + try: + configuration = json.loads(configuration) + except (ValueError, TypeError): + configuration = {} + parameters = configuration.get("parameters", {}) if isinstance(configuration, dict) else {} + data_app_block = parameters.get("dataApp", {}) if isinstance(parameters, dict) else {} + git_block = data_app_block.get("git", {}) if isinstance(data_app_block, dict) else {} + + return { + "project_alias": alias, + "id": str(app.get("id", "")), + "config_id": config_id, + "config_version_storage": str(storage_config.get("version", "") or ""), + "config_version_deployed": str(app.get("configVersion", "") or ""), + "name": storage_config.get("name", app.get("name", "")), + "description": storage_config.get("description", ""), + "type": app.get("type", ""), + "state": app.get("state", ""), + "desired_state": app.get("desiredState", ""), + "url": app.get("url", ""), + "size": app.get("size", "") + or ( + configuration.get("runtime", {}).get("backend", {}).get("size", "") + if isinstance(configuration, dict) + else "" + ), + "auto_suspend_after_seconds": app.get( + "autoSuspendAfterSeconds", + parameters.get("autoSuspendAfterSeconds"), + ), + "last_start_timestamp": app.get("lastStartTimestamp"), + "slug": data_app_block.get("slug", ""), + "git": _redact_git_block(git_block) if git_block else {}, + "raw": { + "deployment": app, + "storage_config": _redact_storage_config(storage_config), + }, + } + + def create_data_app( + self, + *, + alias: str, + name: str, + description: str, + slug: str, + git_repo: str, + git_branch: str = "main", + git_public: bool = False, + git_username: str | None = None, + git_pat_plaintext: str | None = None, + git_pat_encrypted: str | None = None, + auth: str = "password", + size: str = DEFAULT_SIZE, + auto_suspend_after_seconds: int = DEFAULT_AUTO_SUSPEND_SECONDS, + type_: str = DEFAULT_TYPE, + branch_id: int | None = None, + deploy: bool = True, + wait: bool = False, + timeout_seconds: float = DEFAULT_JOB_RUN_TIMEOUT, + keep_on_failure: bool = False, + dry_run: bool = False, + ) -> dict[str, Any]: + """End-to-end create flow per writeup §6/§10/§11. + + Steps in order: + + 1. Validate inputs (slug, size, type, git auth combination). + 2. POST a minimal shell to ``/apps`` to mint id + configId. + 3. If the repo is private, encrypt the PAT under THIS project's KMS + via :class:`EncryptService`. + 4. PUT the full Storage config body, including the auto-injected + ``parameters.id`` back-pointer (writeup §5 — required). + 5. If ``deploy``: read the latest Storage version, PATCH the + deployment record with the §9 trio. + 6. If ``wait``: poll until terminal (running / error / timeout). + 7. On failure between (2) and (5), DELETE the orphan shell unless + ``keep_on_failure`` is set. + """ + self._validate_create_inputs( + type_=type_, + slug=slug, + size=size, + auth=auth, + name=name, + description=description, + git_repo=git_repo, + git_branch=git_branch, + git_public=git_public, + git_username=git_username, + git_pat_plaintext=git_pat_plaintext, + git_pat_encrypted=git_pat_encrypted, + ) + + if dry_run: + return self._build_dry_run_payload( + alias=alias, + name=name, + description=description, + slug=slug, + git_repo=git_repo, + git_branch=git_branch, + git_public=git_public, + git_username=git_username, + auth=auth, + size=size, + auto_suspend_after_seconds=auto_suspend_after_seconds, + type_=type_, + branch_id=branch_id, + deploy=deploy, + ) + + projects = self.resolve_projects([alias]) + project = projects[alias] + + ds_client = self._ds_client_factory(project.stack_url, project.token) + storage_client = self._client_factory(project.stack_url, project.token) + + shell: dict[str, Any] | None = None + app_id: str | None = None + config_id: str | None = None + + try: + # Step 2: create the shell. Smallest body the API will accept. + initial_config = { + "parameters": { + "size": size, + "autoSuspendAfterSeconds": auto_suspend_after_seconds, + "dataApp": {"slug": slug}, + }, + } + if auth == "password": + initial_config["authorization"] = _build_simple_auth_block() + + shell = ds_client.create_app( + type_=type_, + name=name, + description="", # full description goes onto the Storage config below + config=initial_config, + branch_id=branch_id, + ) + app_id = str(shell.get("id", "")) + config_id = str(shell.get("configId", "")) + if not app_id or not config_id: + raise KeboolaApiError( + message="POST /apps response missing id or configId", + status_code=500, + error_code=ErrorCode.API_ERROR, + retryable=False, + ) + + # Step 3: encrypt PAT under target-project KMS if private repo. + git_block = self._build_git_block( + alias=alias, + git_repo=git_repo, + git_branch=git_branch, + git_public=git_public, + git_username=git_username, + git_pat_plaintext=git_pat_plaintext, + git_pat_encrypted=git_pat_encrypted, + ) + + # Step 4: PUT Storage config with full body + parameters.id back-pointer. + full_config = self._build_storage_config_body( + size=size, + auto_suspend_after_seconds=auto_suspend_after_seconds, + slug=slug, + git_block=git_block, + auth=auth, + app_id=app_id, + ) + storage_response = storage_client.update_config( + component_id=DATA_APP_COMPONENT_ID, + config_id=config_id, + name=name, + description=description, + configuration=full_config, + change_description=f"Initial data-app config via kbagent data-app create ({slug})", + branch_id=branch_id, + ) + storage_version = str(storage_response.get("version", "") or "") + + deployed_record: dict[str, Any] | None = None + poll_result: dict[str, Any] | None = None + + if deploy: + # Step 5: §9 redeploy contract. + if not storage_version: + raise KeboolaApiError( + message=( + "Storage API did not return a version after PUT; " + "cannot pin configVersion for deploy." + ), + status_code=500, + error_code=ErrorCode.API_ERROR, + retryable=False, + ) + deployed_record = ds_client.patch_app( + app_id, + desired_state=RUNNING_STATE, + config_version=storage_version, + restart_if_running=True, + ) + + if wait: + poll_result = self._poll_until_terminal( + ds_client, + app_id, + target_desired_state=RUNNING_STATE, + timeout_seconds=timeout_seconds, + ) + + return { + "project_alias": alias, + "id": app_id, + "config_id": config_id, + "name": name, + "slug": slug, + "type": type_, + "size": size, + "auto_suspend_after_seconds": auto_suspend_after_seconds, + "auth": auth, + "git": _redact_git_block(git_block), + "branch_id": branch_id, + "config_version": storage_version, + "deployed": bool(deploy), + "wait": bool(wait), + "url": (deployed_record or shell).get("url", ""), + "state": (poll_result or deployed_record or shell).get("state", ""), + "desired_state": (poll_result or deployed_record or shell).get("desiredState", ""), + "last_start_timestamp": (poll_result or deployed_record or {}).get( + "lastStartTimestamp" + ), + "message": self._format_create_message( + name=name, + auth=auth, + deployed=bool(deploy), + wait=bool(wait), + state=(poll_result or deployed_record or shell).get("state", ""), + ), + } + except Exception: + # Step 7: clean up the orphan shell unless caller asked us to + # preserve it for forensics. + if app_id and not keep_on_failure: + try: + ds_client.delete_app(app_id) + logger.info("Cleaned up orphan data-app shell %s after failure", app_id) + except Exception: + logger.exception("Failed to clean up orphan data-app shell %s", app_id) + raise + finally: + ds_client.close() + storage_client.close() + + def deploy_data_app( + self, + alias: str, + app_id: str, + config_version: str | None = None, + wait: bool = False, + timeout_seconds: float = DEFAULT_JOB_RUN_TIMEOUT, + branch_id: int | None = None, + ) -> dict[str, Any]: + """Encapsulate the §9 redeploy contract. + + Default reads the latest Storage version and pins to it; pass + ``config_version`` to deploy an older version. ALWAYS sends + ``restartIfRunning=true`` together with ``configVersion`` -- the + server returns HTTP 422 for any other shape. + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + + ds_client = self._ds_client_factory(project.stack_url, project.token) + storage_client: Any | None = None + try: + app = ds_client.get_app(app_id) + config_id = str(app.get("configId") or "") + if not config_id: + raise KeboolaApiError( + message=f"Data app {app_id} has no associated configId", + status_code=500, + error_code=ErrorCode.API_ERROR, + retryable=False, + ) + + effective_version = config_version + if effective_version is None: + # Only build the Storage client when we actually need to + # read the latest version. Callers that pass an explicit + # --config-version skip this path and the second client. + storage_client = self._client_factory(project.stack_url, project.token) + storage_config = storage_client.get_config_detail( + DATA_APP_COMPONENT_ID, config_id, branch_id=branch_id + ) + effective_version = str(storage_config.get("version", "") or "") + if not effective_version: + raise KeboolaApiError( + message=( + f"Cannot resolve a Storage configVersion for app {app_id}; " + "Storage config returned no version." + ), + status_code=500, + error_code=ErrorCode.API_ERROR, + retryable=False, + ) + + deployed = ds_client.patch_app( + app_id, + desired_state=RUNNING_STATE, + config_version=str(effective_version), + restart_if_running=True, + ) + poll_result: dict[str, Any] | None = None + if wait: + poll_result = self._poll_until_terminal( + ds_client, + app_id, + target_desired_state=RUNNING_STATE, + timeout_seconds=timeout_seconds, + ) + return self._format_lifecycle_result( + alias=alias, + app_id=app_id, + action="deploy", + deployed=deployed, + poll_result=poll_result, + config_version=str(effective_version), + ) + finally: + ds_client.close() + if storage_client is not None: + storage_client.close() + + def start_data_app( + self, + alias: str, + app_id: str, + wait: bool = False, + timeout_seconds: float = DEFAULT_JOB_RUN_TIMEOUT, + ) -> dict[str, Any]: + """Wake an auto-suspended app at its currently-pinned configVersion. + + Distinct from :meth:`deploy_data_app`: ``start`` does NOT bump the + deployed version. This is the cheap restart path for the + auto-suspend wake (writeup §8 pitfall row 2). + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + ds_client = self._ds_client_factory(project.stack_url, project.token) + try: + deployed = ds_client.patch_app( + app_id, + desired_state=RUNNING_STATE, + restart_if_running=True, + ) + poll_result: dict[str, Any] | None = None + if wait: + poll_result = self._poll_until_terminal( + ds_client, + app_id, + target_desired_state=RUNNING_STATE, + timeout_seconds=timeout_seconds, + ) + return self._format_lifecycle_result( + alias=alias, + app_id=app_id, + action="start", + deployed=deployed, + poll_result=poll_result, + ) + finally: + ds_client.close() + + def stop_data_app( + self, + alias: str, + app_id: str, + wait: bool = False, + timeout_seconds: float = DEFAULT_JOB_RUN_TIMEOUT, + ) -> dict[str, Any]: + projects = self.resolve_projects([alias]) + project = projects[alias] + ds_client = self._ds_client_factory(project.stack_url, project.token) + try: + deployed = ds_client.patch_app(app_id, desired_state=STOPPED_STATE) + poll_result: dict[str, Any] | None = None + if wait: + poll_result = self._poll_until_terminal( + ds_client, + app_id, + target_desired_state=STOPPED_STATE, + timeout_seconds=timeout_seconds, + ) + return self._format_lifecycle_result( + alias=alias, + app_id=app_id, + action="stop", + deployed=deployed, + poll_result=poll_result, + ) + finally: + ds_client.close() + + def delete_data_app(self, alias: str, app_id: str) -> dict[str, Any]: + """Delete the deployment AND the Storage config (cascade).""" + projects = self.resolve_projects([alias]) + project = projects[alias] + ds_client = self._ds_client_factory(project.stack_url, project.token) + try: + ds_client.delete_app(app_id) + finally: + ds_client.close() + return { + "project_alias": alias, + "id": str(app_id), + "deleted": True, + "message": ( + f"Data app {app_id} deleted from project '{alias}'. " + "Both the deployment record and the Storage config are gone; " + "the URL is permanently retired." + ), + } + + def get_data_app_password( + self, + alias: str, + app_id: str, + manage_token: str, + ) -> dict[str, Any]: + """Return the auto-generated simpleAuth password. + + Requires both project Storage token and a Manage API token. The + Manage token is passed per-call -- it is never persisted, never + attached to the long-lived client, and never logged. + """ + if not manage_token: + raise KeboolaApiError( + message=( + "Manage API token is required to read the data-app simpleAuth " + "password. Set KBC_MANAGE_API_TOKEN or run interactively." + ), + status_code=0, + error_code=ErrorCode.INVALID_TOKEN, + retryable=False, + ) + projects = self.resolve_projects([alias]) + project = projects[alias] + ds_client = self._ds_client_factory(project.stack_url, project.token) + try: + payload = ds_client.get_app_password(app_id, manage_token=manage_token) + finally: + ds_client.close() + password = payload.get("password", "") if isinstance(payload, dict) else "" + return { + "project_alias": alias, + "id": str(app_id), + "password": password, + "message": ( + f"Retrieved simpleAuth password for data app {app_id}. " + "This password is auto-generated and cannot be rotated; " + "delete and recreate the app to mint a new one." + ), + } + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _validate_create_inputs( + self, + *, + type_: str, + slug: str, + size: str, + auth: str, + name: str, + description: str, + git_repo: str, + git_branch: str, + git_public: bool, + git_username: str | None, + git_pat_plaintext: str | None, + git_pat_encrypted: str | None, + ) -> None: + # Defence-in-depth length / control-char checks at the service + # boundary. The service can be invoked directly (via --hint service + # snippets or external Python callers) so we do not rely on the + # command layer alone. + for field_name, field_value, max_len, allow_ws in ( + ("--name", name, MAX_NAME_LENGTH, False), + ("--description", description, MAX_DESCRIPTION_LENGTH, True), + ("--git-repo", git_repo, MAX_GIT_REPO_LENGTH, False), + ("--git-branch", git_branch, MAX_GIT_BRANCH_LENGTH, False), + ("--git-username", git_username or "", MAX_GIT_USERNAME_LENGTH, False), + ): + if not isinstance(field_value, str): + continue + if len(field_value) > max_len: + raise KeboolaApiError( + message=( + f"{field_name} exceeds the {max_len}-character limit " + "enforced at the service boundary." + ), + status_code=0, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) + # Description allows tab/LF/CR (markdown); other fields reject any + # control char including CR/LF (would break URL host derivation, + # JSON serialization, or audit-log change descriptions). + if _has_control_chars(field_value, allow_whitespace=allow_ws): + raise KeboolaApiError( + message=(f"{field_name} contains disallowed control characters."), + status_code=0, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) + + # Reject git_repo URLs that don't use a known clone scheme. The + # data-app runner only handles https / http / git / ssh; anything + # else (file://, gopher://, etc.) is either nonsense or an SSRF / + # local-file-read footgun and must not reach Storage. + if git_repo and not any(git_repo.startswith(scheme) for scheme in ALLOWED_GIT_REPO_SCHEMES): + raise KeboolaApiError( + message=( + f"--git-repo must use one of {', '.join(ALLOWED_GIT_REPO_SCHEMES)}." + " Bare ssh syntax (git@host:path) and other schemes are not" + " accepted." + ), + status_code=0, + error_code=ErrorCode.DATA_APP_INVALID_GIT, + retryable=False, + ) + + if type_ not in VALID_TYPES: + raise KeboolaApiError( + message=(f"Invalid --type '{type_}'. Valid values: {', '.join(VALID_TYPES)}"), + status_code=0, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) + if size not in VALID_SIZES: + raise KeboolaApiError( + message=(f"Invalid --size '{size}'. Valid values: {', '.join(VALID_SIZES)}"), + status_code=0, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) + if auth not in ("password", "public"): + raise KeboolaApiError( + message=f"Invalid --auth '{auth}'. Valid values: password, public", + status_code=0, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) + if not SLUG_PATTERN.match(slug): + raise KeboolaApiError( + message=( + f"Invalid --slug '{slug}'. Slug must be lowercase alphanumeric " + "with hyphens, 2-64 chars, and cannot start or end with a hyphen." + ), + status_code=0, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) + + if git_public: + if git_username or git_pat_plaintext or git_pat_encrypted: + raise KeboolaApiError( + message=( + "--git-public is incompatible with --git-username / " + "--git-pat-env / --git-pat-file / --git-pat-encrypted." + ), + status_code=0, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) + else: + if not git_username: + raise KeboolaApiError( + message="--git-username is required for private repositories.", + status_code=0, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) + provided = sum( + 1 for v in (git_pat_plaintext, git_pat_encrypted) if v is not None and v != "" + ) + if provided == 0: + raise KeboolaApiError( + message=( + "Private repository requires a PAT. Pass one of: " + "--git-pat-env VAR / --git-pat-file PATH / " + "--git-pat-encrypted KBC::Project..." + ), + status_code=0, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) + if provided > 1: + raise KeboolaApiError( + message=( + "Specify exactly one of --git-pat-env / --git-pat-file / " + "--git-pat-encrypted; they are mutually exclusive." + ), + status_code=0, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) + if git_pat_plaintext is not None and git_pat_plaintext.startswith("KBC::"): + # A plaintext input that already looks like a Keboola + # ciphertext is almost certainly someone pasting an + # encrypted value into --git-pat-env / --git-pat-file by + # mistake. EncryptService.encrypt() short-circuits any + # ``KBC::``-prefixed value and would pass it through + # unchanged, so a stray ciphertext from another project + # could reach Storage and silently fail at runtime + # decrypt. Reject up front. + raise KeboolaApiError( + message=( + "--git-pat-env / --git-pat-file expect plaintext PATs. " + "The value starts with 'KBC::' which suggests an " + "already-encrypted ciphertext; if so, pass it via " + "--git-pat-encrypted instead so the prefix is validated." + ), + status_code=0, + error_code=ErrorCode.DATA_APP_INVALID_GIT, + retryable=False, + ) + if git_pat_encrypted is not None and not any( + git_pat_encrypted.startswith(p) for p in ENCRYPTED_PASSWORD_PREFIXES + ): + raise KeboolaApiError( + message=( + "--git-pat-encrypted must be a project-scoped Encryption " + f"API ciphertext (one of: {', '.join(ENCRYPTED_PASSWORD_PREFIXES)}). " + "Re-encrypt with `kbagent encrypt values --component-id " + f"{DATA_APP_COMPONENT_ID}` against THIS project; ciphertext " + "from another project will not decrypt (writeup §8)." + ), + status_code=0, + error_code=ErrorCode.VALIDATION_ERROR, + retryable=False, + ) + + def _build_git_block( + self, + *, + alias: str, + git_repo: str, + git_branch: str, + git_public: bool, + git_username: str | None, + git_pat_plaintext: str | None, + git_pat_encrypted: str | None, + ) -> dict[str, Any]: + if git_public: + return { + "repository": git_repo, + "private": False, + "branch": git_branch, + } + + # Plaintext PATs are encrypted under the target project's KMS via + # EncryptService. Pre-encrypted ciphertext was prefix-validated in + # _validate_create_inputs; EncryptService short-circuits anything + # starting with ``KBC::`` (encrypt_service.py) and returns the value + # unchanged, so we do not pay the encryption round-trip on a + # caller-supplied ciphertext. We still re-validate the result below + # so a misconfigured input cannot reach Storage as plaintext. + secret_input = git_pat_plaintext or git_pat_encrypted or "" + try: + encrypted = self._encrypt_service.encrypt( + alias=alias, + component_id=DATA_APP_COMPONENT_ID, + input_data={"#password": secret_input}, + ) + except ConfigError as exc: + raise KeboolaApiError( + message=f"Failed to prepare git PAT for encryption: {exc.message}", + status_code=0, + error_code=ErrorCode.ENCRYPTION_FAILED, + retryable=False, + ) from exc + encrypted_pat = encrypted.get("#password", "") + if not encrypted_pat or not any( + encrypted_pat.startswith(p) for p in ENCRYPTED_PASSWORD_PREFIXES + ): + raise KeboolaApiError( + message=( + "Encryption API did not return a project-scoped ciphertext " + "for the git PAT; refusing to write plaintext to Storage." + ), + status_code=0, + error_code=ErrorCode.ENCRYPTION_FAILED, + retryable=False, + ) + return { + "repository": git_repo, + "private": True, + "username": git_username or "", + "#password": encrypted_pat, + "branch": git_branch, + } + + def _build_storage_config_body( + self, + *, + size: str, + auto_suspend_after_seconds: int, + slug: str, + git_block: dict[str, Any], + auth: str, + app_id: str, + ) -> dict[str, Any]: + body: dict[str, Any] = { + "parameters": { + "autoSuspendAfterSeconds": auto_suspend_after_seconds, + "dataApp": { + "slug": slug, + "git": git_block, + }, + "id": str(app_id), # writeup §5: required back-pointer + }, + "runtime": {"backend": {"size": size}}, + } + if auth == "password": + body["authorization"] = _build_simple_auth_block() + return body + + def _build_dry_run_payload( + self, + **kwargs: Any, + ) -> dict[str, Any]: + """Render the three request bodies without making any API call.""" + size = kwargs["size"] + slug = kwargs["slug"] + auth = kwargs["auth"] + auto_suspend = kwargs["auto_suspend_after_seconds"] + type_ = kwargs["type_"] + + post_body = { + "branchId": kwargs["branch_id"], + "type": type_, + "name": kwargs["name"], + "description": "", + "config": { + "parameters": { + "size": size, + "autoSuspendAfterSeconds": auto_suspend, + "dataApp": {"slug": slug}, + }, + }, + } + if auth == "password": + post_body["config"]["authorization"] = _build_simple_auth_block() + + # We can't know the app_id pre-create; show the placeholder. + git_block_preview: dict[str, Any] + if kwargs["git_public"]: + git_block_preview = { + "repository": kwargs["git_repo"], + "private": False, + "branch": kwargs["git_branch"], + } + else: + git_block_preview = { + "repository": kwargs["git_repo"], + "private": True, + "username": kwargs["git_username"] or "", + "#password": "", + "branch": kwargs["git_branch"], + } + + put_body = { + "parameters": { + "autoSuspendAfterSeconds": auto_suspend, + "dataApp": {"slug": slug, "git": git_block_preview}, + "id": "", + }, + "runtime": {"backend": {"size": size}}, + } + if auth == "password": + put_body["authorization"] = _build_simple_auth_block() + + patch_body: dict[str, Any] = {} + if kwargs["deploy"]: + patch_body = { + "desiredState": "running", + "configVersion": "", + "restartIfRunning": True, + } + + return { + "dry_run": True, + "project_alias": kwargs["alias"], + "requests": { + "post_apps": post_body, + "put_storage_config": put_body, + "patch_apps": patch_body, + }, + "message": ( + "Dry run -- no API calls made. " + "Inspect the three request bodies above before re-running without --dry-run." + ), + } + + def _fetch_data_app_config_names( + self, + storage_client: Any, + branch_id: int | None, + ) -> dict[str, str]: + """Map ``configId -> name`` for ``keboola.data-apps`` configs. + + Used by ``list_data_apps`` to enrich the thin Data Science index + with the human-readable names that live on the Storage config. + """ + try: + configs = storage_client.list_component_configs( + DATA_APP_COMPONENT_ID, branch_id=branch_id + ) + return {str(cfg.get("id", "")): cfg.get("name", "") for cfg in configs} + except Exception: + return {} + + def _poll_until_terminal( + self, + ds_client: DataScienceClient, + app_id: str, + *, + target_desired_state: str, + timeout_seconds: float, + ) -> dict[str, Any]: + """Poll ``GET /apps/{id}`` until terminal. + + Terminal definition: + + - ``state == target_desired_state`` (the deploy succeeded), OR + - ``state == "error"`` (the build / runtime failed). + + IMPORTANT: while ``desiredState == "running"``, observing + ``state == "stopped"`` is NOT terminal -- the platform transitions + ``created -> stopped -> starting -> running`` during initial + deploy, and a naive poll exits prematurely (writeup §8 pitfall 1). + """ + deadline = time.monotonic() + timeout_seconds + last_record: dict[str, Any] = {} + while True: + last_record = ds_client.get_app(app_id) + state = str(last_record.get("state", "")) + if state == TERMINAL_ERROR_STATE: + raise KeboolaApiError( + message=( + f"Data app {app_id} reached state=error during deploy. " + "See the app's Terminal Log in the Keboola UI for the build " + "error -- the Data Science API does not expose it as JSON." + ), + status_code=0, + error_code=ErrorCode.DATA_APP_BUILD_FAILED, + retryable=False, + ) + if state == target_desired_state: + return last_record + if time.monotonic() >= deadline: + raise KeboolaApiError( + message=( + f"Timed out after {timeout_seconds:.0f}s waiting for data app " + f"{app_id} to reach state={target_desired_state} " + f"(last observed: state={state}, desired={last_record.get('desiredState')})." + ), + status_code=0, + error_code=ErrorCode.DATA_APP_DEPLOY_TIMEOUT, + retryable=True, + ) + time.sleep(POLL_INTERVAL_SECONDS) + + def _format_lifecycle_result( + self, + *, + alias: str, + app_id: str, + action: str, + deployed: dict[str, Any], + poll_result: dict[str, Any] | None, + config_version: str | None = None, + ) -> dict[str, Any]: + record = poll_result or deployed + return { + "project_alias": alias, + "id": str(app_id), + "action": action, + "state": record.get("state", ""), + "desired_state": record.get("desiredState", ""), + "config_version": str(record.get("configVersion", "") or "") or (config_version or ""), + "url": record.get("url", ""), + "last_start_timestamp": record.get("lastStartTimestamp"), + "message": ( + f"Data app {app_id} {action} requested in project '{alias}'. " + f"state={record.get('state', '?')}, " + f"desiredState={record.get('desiredState', '?')}." + ), + } + + def _format_create_message( + self, + *, + name: str, + auth: str, + deployed: bool, + wait: bool, + state: str, + ) -> str: + if not deployed: + return ( + f"Data app '{name}' created and configured. " + "No deploy attempted (--no-deploy). Run `kbagent data-app deploy` " + "to start the app." + ) + if not wait: + return ( + f"Data app '{name}' created and deploy requested. " + "Use `kbagent data-app detail` to track state, or pass --wait " + "to block until running." + ) + # wait=True + if state == RUNNING_STATE: + tail = ( + " Run `kbagent data-app password` to retrieve the simpleAuth password." + if auth == "password" + else "" + ) + return f"Data app '{name}' is running.{tail}" + return f"Data app '{name}' deploy reached state={state}." diff --git a/tests/test_data_app_cli.py b/tests/test_data_app_cli.py new file mode 100644 index 00000000..fe5b725c --- /dev/null +++ b/tests/test_data_app_cli.py @@ -0,0 +1,560 @@ +"""CLI-layer tests for the ``data-app`` command group via CliRunner. + +Mirrors the test_workspace_cli.py pattern: patch the cli.py service factory +so the runner sees a MagicMock; assert exit codes, JSON envelopes, and the +mutual-exclusion validation. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.config_service import ConfigService +from keboola_agent_cli.services.job_service import JobService +from keboola_agent_cli.services.project_service import ProjectService + +TEST_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" + +runner = CliRunner() + + +def _setup_config(config_dir: Path, projects: dict[str, dict] | None = None) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + if projects: + for alias, info in projects.items(): + store.add_project( + alias, + ProjectConfig( + stack_url=info.get("stack_url", "https://connection.keboola.com"), + token=info["token"], + project_name=info.get("project_name", alias), + project_id=info.get("project_id", 1234), + ), + ) + return store + + +def _invoke( + args: list[str], + *, + store: ConfigStore, + data_app_mock: MagicMock, +): + """Run the CLI with cli.py services patched to mocks.""" + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProj, + patch("keboola_agent_cli.cli.ConfigService") as MockCfg, + patch("keboola_agent_cli.cli.JobService") as MockJob, + patch("keboola_agent_cli.cli.DataAppService") as MockDataAppService, + ): + MockStore.return_value = store + MockProj.return_value = ProjectService(config_store=store) + MockCfg.return_value = ConfigService(config_store=store) + MockJob.return_value = JobService(config_store=store) + MockDataAppService.return_value = data_app_mock + return runner.invoke(app, args) + + +# --------------------------------------------------------------------------- +# data-app list +# --------------------------------------------------------------------------- + + +class TestDataAppList: + def test_json_success(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + mock = MagicMock() + mock.list_data_apps.return_value = { + "apps": [ + { + "project_alias": "prod", + "id": "42", + "config_id": "ulid", + "name": "App", + "type": "python-js", + "state": "running", + "desired_state": "running", + "config_version": "3", + "url": "https://x.hub.example.com", + } + ], + "errors": [], + } + result = _invoke( + ["--json", "data-app", "list", "--project", "prod"], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["status"] == "ok" + assert body["data"]["apps"][0]["id"] == "42" + + +# --------------------------------------------------------------------------- +# data-app create -- mutual-exclusion validation (CLI layer) +# --------------------------------------------------------------------------- + + +class TestDataAppCreateValidation: + def test_pat_modes_mutually_exclusive(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + mock = MagicMock() + result = _invoke( + [ + "--json", + "data-app", + "create", + "--project", + "prod", + "--name", + "App", + "--slug", + "my-app", + "--git-repo", + "https://github.com/o/r", + "--git-username", + "user", + "--git-pat-env", + "X", + "--git-pat-encrypted", + "KBC::Project::abc", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 2, result.output + body = json.loads(result.output) + assert body["status"] == "error" + assert body["error"]["code"] == "USAGE_ERROR" + mock.create_data_app.assert_not_called() + + def test_missing_pat_env_var_rejected(self, tmp_path: Path, monkeypatch) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + mock = MagicMock() + monkeypatch.delenv("MISSING_PAT", raising=False) + result = _invoke( + [ + "--json", + "data-app", + "create", + "--project", + "prod", + "--name", + "App", + "--slug", + "my-app", + "--git-repo", + "https://github.com/o/r", + "--git-username", + "user", + "--git-pat-env", + "MISSING_PAT", + ], + store=store, + data_app_mock=mock, + ) + # typer.BadParameter -> exit code 2 + assert result.exit_code == 2 + + def test_dry_run_human_output(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + mock = MagicMock() + mock.create_data_app.return_value = { + "dry_run": True, + "project_alias": "prod", + "requests": { + "post_apps": {}, + "put_storage_config": {}, + "patch_apps": {}, + }, + "message": "Dry run -- no API calls made.", + } + result = _invoke( + [ + "data-app", + "create", + "--project", + "prod", + "--name", + "App", + "--slug", + "my-app", + "--git-repo", + "https://github.com/o/r", + "--git-public", + "--auth", + "public", + "--dry-run", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 0, result.output + assert "DRY RUN" in result.output + + +# --------------------------------------------------------------------------- +# data-app deploy +# --------------------------------------------------------------------------- + + +class TestDataAppDeploy: + def test_deploy_success(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + mock = MagicMock() + mock.deploy_data_app.return_value = { + "project_alias": "prod", + "id": "42", + "action": "deploy", + "state": "starting", + "desired_state": "running", + "config_version": "5", + "url": "https://x.hub.example.com", + "message": "Data app 42 deploy requested.", + } + result = _invoke( + [ + "--json", + "data-app", + "deploy", + "--project", + "prod", + "--app-id", + "42", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 0 + body = json.loads(result.output) + assert body["data"]["config_version"] == "5" + mock.deploy_data_app.assert_called_once() + + def test_api_error_exit_code(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + mock = MagicMock() + mock.deploy_data_app.side_effect = KeboolaApiError( + message="boom", + status_code=500, + error_code="API_ERROR", + retryable=False, + ) + result = _invoke( + [ + "--json", + "data-app", + "deploy", + "--project", + "prod", + "--app-id", + "42", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 1 + body = json.loads(result.output) + assert body["error"]["code"] == "API_ERROR" + + +# --------------------------------------------------------------------------- +# data-app delete (confirmation) +# --------------------------------------------------------------------------- + + +class TestDataAppDelete: + def test_delete_with_yes(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + mock = MagicMock() + mock.delete_data_app.return_value = { + "project_alias": "prod", + "id": "42", + "deleted": True, + "message": "Data app 42 deleted.", + } + result = _invoke( + [ + "--json", + "data-app", + "delete", + "--project", + "prod", + "--app-id", + "42", + "--yes", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 0 + mock.delete_data_app.assert_called_once_with(alias="prod", app_id="42") + + +# --------------------------------------------------------------------------- +# data-app password (manage token) +# --------------------------------------------------------------------------- + + +class TestDataAppPassword: + def test_password_success(self, tmp_path: Path, monkeypatch) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + mock = MagicMock() + mock.get_data_app_password.return_value = { + "project_alias": "prod", + "id": "42", + "password": "deadbeefcafe", + "message": "Retrieved.", + } + monkeypatch.setenv("KBC_MANAGE_API_TOKEN", "manage-token") + result = _invoke( + [ + "--json", + "data-app", + "password", + "--project", + "prod", + "--app-id", + "42", + ], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 0 + body = json.loads(result.output) + assert body["data"]["password"] == "deadbeefcafe" + # The Manage token should have been forwarded but never logged. + assert "manage-token" not in result.output + mock.get_data_app_password.assert_called_once_with( + alias="prod", app_id="42", manage_token="manage-token" + ) + + def test_password_missing_manage_token_no_tty(self, tmp_path: Path, monkeypatch) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + mock = MagicMock() + monkeypatch.delenv("KBC_MANAGE_API_TOKEN", raising=False) + result = _invoke( + [ + "--json", + "data-app", + "password", + "--project", + "prod", + "--app-id", + "42", + ], + store=store, + data_app_mock=mock, + ) + # CliRunner stdin is non-TTY, so resolve_manage_token returns exit 2. + assert result.exit_code == 2 + + +# --------------------------------------------------------------------------- +# Generic error mapping (ConfigError -> 5) +# --------------------------------------------------------------------------- + + +class TestDataAppHintMode: + """Compile-check every rendered ``--hint`` snippet for the data-app group. + + Iterations 4 and 5 each caught a hint that rendered to invalid Python: + iteration 4 found ``os.environ[""PAT_VAR""]`` (doubled quotes) and a + missing ``import os`` in ``create``; iteration 5 found a sibling + ``manage_token=`` placeholder reaching the + snippet verbatim in ``password``. The class below enumerates every + ``data-app`` subcommand and ``ast.parse``s both the client- and + service-mode renders so this entire bug class is caught at CI rather + than by a fresh-context reviewer. + """ + + def _setup(self, tmp_path: Path) -> tuple[ConfigStore, MagicMock]: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + return store, MagicMock() + + def _hint(self, store: ConfigStore, mock: MagicMock, args: list[str]) -> str: + result = _invoke(args, store=store, data_app_mock=mock) + assert result.exit_code == 0, result.output + return result.output + + # One sample CLI invocation per subcommand, chosen to exercise the + # most variable args (private repo for create, --config-version for + # deploy, etc.). Both client and service renders are compile-checked. + _SAMPLE_INVOCATIONS: tuple[tuple[str, list[str]], ...] = ( + ("list", ["data-app", "list", "--project", "prod"]), + ( + "detail", + ["data-app", "detail", "--project", "prod", "--app-id", "42"], + ), + ( + "create", + [ + "data-app", + "create", + "--project", + "prod", + "--name", + "X", + "--slug", + "x-x", + "--git-repo", + "https://github.com/o/r", + "--git-username", + "u", + "--git-pat-env", + "PAT_VAR", + "--auth", + "password", + ], + ), + ( + "deploy", + [ + "data-app", + "deploy", + "--project", + "prod", + "--app-id", + "42", + "--config-version", + "5", + ], + ), + ( + "start", + ["data-app", "start", "--project", "prod", "--app-id", "42"], + ), + ( + "stop", + ["data-app", "stop", "--project", "prod", "--app-id", "42"], + ), + ( + "delete", + ["data-app", "delete", "--project", "prod", "--app-id", "42"], + ), + ( + "password", + ["data-app", "password", "--project", "prod", "--app-id", "42"], + ), + ) + + @pytest.mark.parametrize( + "name,subcommand_args", + _SAMPLE_INVOCATIONS, + ids=[name for name, _ in _SAMPLE_INVOCATIONS], + ) + @pytest.mark.parametrize("mode", ["client", "service"]) + def test_hint_snippet_compiles( + self, tmp_path: Path, mode: str, name: str, subcommand_args: list[str] + ) -> None: + import ast + + store, mock = self._setup(tmp_path) + snippet = self._hint( + store, + mock, + ["--hint", mode, *subcommand_args], + ) + ast.parse(snippet) # raises SyntaxError if the render is broken + + def test_create_service_hint_imports_os_and_quotes_pat_env(self, tmp_path: Path) -> None: + """Anchor for the iteration-4 fix: ``import os`` is emitted and + the ``os.environ`` access uses single quotes (no doubling).""" + store, mock = self._setup(tmp_path) + snippet = self._hint( + store, + mock, + [ + "--hint", + "service", + "data-app", + "create", + "--project", + "prod", + "--name", + "X", + "--slug", + "x-x", + "--git-repo", + "https://github.com/o/r", + "--git-username", + "u", + "--git-pat-env", + "PAT_VAR", + "--auth", + "password", + ], + ) + assert "import os" in snippet + assert 'os.environ["PAT_VAR"]' in snippet + # The doubled-quote bug must never recur. + assert 'os.environ[""' not in snippet + + def test_password_service_hint_uses_os_environ(self, tmp_path: Path) -> None: + """Anchor for the iteration-5 fix: the ``manage_token`` kwarg + renders as a parseable ``os.environ[...]`` lookup, not a literal + ```` placeholder.""" + store, mock = self._setup(tmp_path) + snippet = self._hint( + store, + mock, + [ + "--hint", + "service", + "data-app", + "password", + "--project", + "prod", + "--app-id", + "42", + ], + ) + assert "manage_token=os.environ" in snippet + assert "" not in snippet + + +class TestDataAppErrorMapping: + def test_config_error_maps_to_exit_5(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + mock = MagicMock() + mock.list_data_apps.side_effect = ConfigError("Project 'foo' not found.") + result = _invoke( + ["--json", "data-app", "list", "--project", "foo"], + store=store, + data_app_mock=mock, + ) + assert result.exit_code == 5 + body = json.loads(result.output) + assert body["error"]["code"] == "CONFIG_ERROR" diff --git a/tests/test_data_app_service.py b/tests/test_data_app_service.py new file mode 100644 index 00000000..9ab93ddc --- /dev/null +++ b/tests/test_data_app_service.py @@ -0,0 +1,806 @@ +"""Service-layer tests for DataAppService. + +Covers: input validation, the §9 redeploy contract, cleanup-in-finally, +the §8 pitfall #1 (transient stopped during initial deploy), encryption +round-trip, and password retrieval. + +The tests speak to a fully-mocked Data Science + Storage + Encryption +stack -- they verify orchestration, not HTTP shapes (those live in +test_data_science_client.py / test_e2e.py). +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.data_app_service import ( + DataAppService, + _redact_git_block, + _redact_storage_config, +) + +TEST_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" +TEST_MANAGE_TOKEN = "manage-test-token" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_store(tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=5725, + ), + ) + return store + + +def _make_service( + store: ConfigStore, + *, + ds_mock: MagicMock | None = None, + storage_mock: MagicMock | None = None, + encrypt_mock: MagicMock | None = None, +) -> tuple[DataAppService, MagicMock, MagicMock, MagicMock]: + ds_mock = ds_mock or MagicMock() + storage_mock = storage_mock or MagicMock() + if encrypt_mock is None: + encrypt_mock = MagicMock() + encrypt_mock.encrypt.return_value = {"#password": "KBC::ProjectSecureGKMS::ciphertext-prod"} + + service = DataAppService( + config_store=store, + client_factory=lambda url, token: storage_mock, + ds_client_factory=lambda url, token: ds_mock, + encrypt_service=encrypt_mock, + ) + return service, ds_mock, storage_mock, encrypt_mock + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +class TestDataAppCreateValidation: + """The service is the single source of truth for input shape.""" + + def _create(self, service: DataAppService, **overrides: Any) -> Any: + kwargs = dict( + alias="prod", + name="My App", + description="", + slug="my-app", + git_repo="https://github.com/o/r", + git_public=False, + git_username="user", + git_pat_plaintext="ghp_xxxxxxxxxxxxxxxxxxxx", + auth="password", + size="tiny", + auto_suspend_after_seconds=900, + type_="python-js", + deploy=False, + wait=False, + dry_run=True, + ) + kwargs.update(overrides) + return service.create_data_app(**kwargs) + + def test_invalid_size(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, size="huge") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_invalid_type(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, type_="rust") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_invalid_auth(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, auth="oauth") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_invalid_slug(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, slug="UPPER") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_public_repo_rejects_credentials(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create( + service, + git_public=True, + git_username="user", + git_pat_plaintext="x", + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_private_repo_requires_username(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, git_username=None) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_private_repo_requires_pat(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, git_pat_plaintext=None) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_pat_modes_mutually_exclusive(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create( + service, + git_pat_plaintext="ghp_x", + git_pat_encrypted="KBC::ProjectSecureGKMS::abc", + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_pre_encrypted_must_be_project_scoped(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create( + service, + git_pat_plaintext=None, + git_pat_encrypted="KBC::Encrypted::not-project-scoped", + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_pre_encrypted_loose_project_prefix_rejected(self, tmp_path: Path) -> None: + """A bare 'KBC::Project' prefix is no longer enough — validator now + requires a known full prefix (Secure / SecureGKMS / SecureKMS).""" + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create( + service, + git_pat_plaintext=None, + git_pat_encrypted="KBC::ProjectAttacker::xyz", + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_oversize_name_rejected(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, name="A" * 1000) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_control_char_in_name_rejected(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, name="bad\x00name") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_control_char_in_git_username_rejected(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, git_username="user\nname") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_description_allows_markdown_newlines(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + result = self._create(service, description="line one\nline two\n\ttabbed") + assert result["dry_run"] is True + + def test_description_rejects_nul_byte(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, description="oops\x00") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_git_repo_rejects_file_scheme(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, git_repo="file:///etc/passwd") + assert excinfo.value.error_code == ErrorCode.DATA_APP_INVALID_GIT + + def test_git_repo_rejects_bare_ssh_syntax(self, tmp_path: Path) -> None: + """``git@github.com:org/repo`` style. Some clients accept this but + the data-app runner does not -- reject upfront.""" + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create(service, git_repo="git@github.com:org/repo") + assert excinfo.value.error_code == ErrorCode.DATA_APP_INVALID_GIT + + def test_git_pat_plaintext_starting_with_kbc_rejected(self, tmp_path: Path) -> None: + """A plaintext PAT that already looks like a ciphertext is almost + certainly someone pasting an encrypted value into the wrong flag. + Reject upfront so EncryptService's KBC:: short-circuit cannot + ferry a stale ciphertext into Storage.""" + store = _make_store(tmp_path) + service, *_ = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + self._create( + service, + git_pat_plaintext="KBC::ProjectSecureGKMS::accidentally-pasted", + ) + assert excinfo.value.error_code == ErrorCode.DATA_APP_INVALID_GIT + + +# --------------------------------------------------------------------------- +# Create flow +# --------------------------------------------------------------------------- + + +class TestDataAppCreate: + def test_dry_run_makes_no_calls(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, encrypt_mock = _make_service(store) + result = service.create_data_app( + alias="prod", + name="App", + description="", + slug="my-app", + git_repo="https://github.com/o/r", + git_public=True, + auth="public", + size="tiny", + auto_suspend_after_seconds=900, + type_="python-js", + deploy=True, + wait=False, + dry_run=True, + ) + assert result["dry_run"] is True + assert "post_apps" in result["requests"] + assert "put_storage_config" in result["requests"] + assert "patch_apps" in result["requests"] + ds_mock.assert_not_called() + storage_mock.assert_not_called() + encrypt_mock.encrypt.assert_not_called() + + def test_happy_path_private_repo(self, tmp_path: Path) -> None: + """Verifies POST -> encrypt -> PUT -> PATCH order and arguments.""" + store = _make_store(tmp_path) + service, ds_mock, storage_mock, encrypt_mock = _make_service(store) + + ds_mock.create_app.return_value = { + "id": "43661269", + "configId": "01kqj88t0vktxe0vfhk6ps5kzs", + } + storage_mock.update_config.return_value = {"version": "3"} + ds_mock.patch_app.return_value = { + "id": "43661269", + "state": "starting", + "desiredState": "running", + "url": "https://my-app-43661269.hub.us-east4.gcp.keboola.com", + "configVersion": "3", + } + + result = service.create_data_app( + alias="prod", + name="My App", + description="long form", + slug="my-app", + git_repo="https://github.com/o/r", + git_branch="main", + git_public=False, + git_username="user", + git_pat_plaintext="ghp_xxxxxxxxxxxxxxxxxxxx", + auth="password", + size="tiny", + auto_suspend_after_seconds=900, + type_="python-js", + deploy=True, + wait=False, + dry_run=False, + ) + + # 1. Shell created + ds_mock.create_app.assert_called_once() + create_kwargs = ds_mock.create_app.call_args.kwargs + assert create_kwargs["type_"] == "python-js" + assert create_kwargs["name"] == "My App" + assert create_kwargs["description"] == "" # description goes to Storage + + # 2. PAT encrypted under target project's KMS + encrypt_mock.encrypt.assert_called_once_with( + alias="prod", + component_id="keboola.data-apps", + input_data={"#password": "ghp_xxxxxxxxxxxxxxxxxxxx"}, + ) + + # 3. Storage config written with parameters.id back-pointer + storage_mock.update_config.assert_called_once() + put_kwargs = storage_mock.update_config.call_args.kwargs + body = put_kwargs["configuration"] + assert body["parameters"]["id"] == "43661269" + assert body["parameters"]["dataApp"]["slug"] == "my-app" + assert body["parameters"]["dataApp"]["git"]["#password"].startswith("KBC::Project") + assert body["parameters"]["dataApp"]["git"]["private"] is True + assert body["runtime"]["backend"]["size"] == "tiny" + assert "authorization" in body # simpleAuth on by default + + # 4. PATCH /apps deploys the trio + ds_mock.patch_app.assert_called_once_with( + "43661269", + desired_state="running", + config_version="3", + restart_if_running=True, + ) + + assert result["id"] == "43661269" + assert result["config_id"] == "01kqj88t0vktxe0vfhk6ps5kzs" + assert result["url"].endswith("hub.us-east4.gcp.keboola.com") + # Encrypted PAT is redacted in the returned dict for human display. + assert result["git"]["#password"] == "" + + def test_no_deploy_skips_patch(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.create_app.return_value = {"id": "1", "configId": "ulid"} + storage_mock.update_config.return_value = {"version": "2"} + + service.create_data_app( + alias="prod", + name="App", + description="", + slug="my-app", + git_repo="https://github.com/o/r", + git_public=True, + auth="public", + size="tiny", + auto_suspend_after_seconds=900, + type_="python-js", + deploy=False, + wait=False, + dry_run=False, + ) + ds_mock.patch_app.assert_not_called() + + def test_public_repo_skips_encryption(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, encrypt_mock = _make_service(store) + ds_mock.create_app.return_value = {"id": "1", "configId": "ulid"} + storage_mock.update_config.return_value = {"version": "2"} + + service.create_data_app( + alias="prod", + name="App", + description="", + slug="my-app", + git_repo="https://github.com/o/r", + git_public=True, + auth="public", + size="tiny", + auto_suspend_after_seconds=900, + type_="python-js", + deploy=False, + wait=False, + dry_run=False, + ) + encrypt_mock.encrypt.assert_not_called() + + def test_cleanup_on_storage_put_failure(self, tmp_path: Path) -> None: + """If PUT fails after POST, the orphan shell is deleted by default.""" + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.create_app.return_value = {"id": "999", "configId": "ulid"} + storage_mock.update_config.side_effect = KeboolaApiError( + message="boom", + status_code=500, + error_code=ErrorCode.API_ERROR, + retryable=False, + ) + + with pytest.raises(KeboolaApiError): + service.create_data_app( + alias="prod", + name="App", + description="", + slug="my-app", + git_repo="https://github.com/o/r", + git_public=True, + auth="public", + size="tiny", + auto_suspend_after_seconds=900, + type_="python-js", + deploy=False, + wait=False, + dry_run=False, + ) + ds_mock.delete_app.assert_called_once_with("999") + + def test_keep_on_failure_skips_cleanup(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.create_app.return_value = {"id": "999", "configId": "ulid"} + storage_mock.update_config.side_effect = KeboolaApiError( + message="boom", + status_code=500, + error_code=ErrorCode.API_ERROR, + retryable=False, + ) + + with pytest.raises(KeboolaApiError): + service.create_data_app( + alias="prod", + name="App", + description="", + slug="my-app", + git_repo="https://github.com/o/r", + git_public=True, + auth="public", + size="tiny", + auto_suspend_after_seconds=900, + type_="python-js", + deploy=False, + wait=False, + keep_on_failure=True, + dry_run=False, + ) + ds_mock.delete_app.assert_not_called() + + def test_encryption_failure_aborts_loud(self, tmp_path: Path) -> None: + """The service refuses to write plaintext if the Encryption API + does not return a project-scoped ciphertext.""" + store = _make_store(tmp_path) + service, ds_mock, storage_mock, encrypt_mock = _make_service(store) + ds_mock.create_app.return_value = {"id": "999", "configId": "ulid"} + encrypt_mock.encrypt.return_value = {"#password": "not-a-ciphertext"} + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_data_app( + alias="prod", + name="App", + description="", + slug="my-app", + git_repo="https://github.com/o/r", + git_public=False, + git_username="user", + git_pat_plaintext="ghp_xxxxxxxxxxxxxxxxxxxx", + auth="password", + size="tiny", + auto_suspend_after_seconds=900, + type_="python-js", + deploy=False, + wait=False, + dry_run=False, + ) + assert excinfo.value.error_code == ErrorCode.ENCRYPTION_FAILED + # Plaintext never reached Storage. + storage_mock.update_config.assert_not_called() + # Shell was cleaned up. + ds_mock.delete_app.assert_called_once_with("999") + + +# --------------------------------------------------------------------------- +# Deploy / start / stop +# --------------------------------------------------------------------------- + + +class TestDataAppDeploy: + def test_deploy_reads_latest_storage_version(self, tmp_path: Path) -> None: + """The §9 redeploy contract.""" + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.get_app.return_value = {"configId": "ulid"} + storage_mock.get_config_detail.return_value = {"version": 7} + ds_mock.patch_app.return_value = {"state": "starting"} + + service.deploy_data_app(alias="prod", app_id="42") + + ds_mock.patch_app.assert_called_once_with( + "42", + desired_state="running", + config_version="7", + restart_if_running=True, + ) + + def test_deploy_pins_explicit_version(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.get_app.return_value = {"configId": "ulid"} + ds_mock.patch_app.return_value = {"state": "starting"} + + service.deploy_data_app(alias="prod", app_id="42", config_version="3") + + # Service did NOT need to read Storage to derive a version + storage_mock.get_config_detail.assert_not_called() + ds_mock.patch_app.assert_called_once_with( + "42", + desired_state="running", + config_version="3", + restart_if_running=True, + ) + + def test_deploy_never_sends_config_block(self, tmp_path: Path) -> None: + """`PATCH /apps {config: ...}` is silently dropped (writeup §8 row 3). + We never construct that payload in the first place.""" + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.get_app.return_value = {"configId": "ulid"} + storage_mock.get_config_detail.return_value = {"version": "5"} + ds_mock.patch_app.return_value = {"state": "running", "desiredState": "running"} + + service.deploy_data_app(alias="prod", app_id="42") + kwargs = ds_mock.patch_app.call_args.kwargs + assert "config" not in kwargs + + +class TestDataAppStartStop: + def test_start_does_not_send_config_version(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, _storage, _enc = _make_service(store) + ds_mock.patch_app.return_value = {"state": "starting"} + + service.start_data_app(alias="prod", app_id="42") + + kwargs = ds_mock.patch_app.call_args.kwargs + assert kwargs["desired_state"] == "running" + assert kwargs["restart_if_running"] is True + assert "config_version" not in kwargs + + def test_stop_sends_only_desired_state(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, _storage, _enc = _make_service(store) + ds_mock.patch_app.return_value = {"state": "stopping"} + + service.stop_data_app(alias="prod", app_id="42") + kwargs = ds_mock.patch_app.call_args.kwargs + assert kwargs["desired_state"] == "stopped" + assert "config_version" not in kwargs + assert "restart_if_running" not in kwargs + + +# --------------------------------------------------------------------------- +# Poll loop -- writeup §8 pitfall #1 +# --------------------------------------------------------------------------- + + +class TestDataAppPoll: + def test_stopped_is_not_terminal_during_initial_deploy(self, tmp_path: Path) -> None: + """While desiredState=running, observing state=stopped MUST NOT exit + the poll. The platform transitions created -> stopped -> starting -> + running on initial deploy.""" + store = _make_store(tmp_path) + service, ds_mock, _storage, _enc = _make_service(store) + + # Sequence: stopped (transient) -> starting -> running + ds_mock.get_app.side_effect = [ + {"state": "stopped", "desiredState": "running"}, + {"state": "starting", "desiredState": "running"}, + { + "state": "running", + "desiredState": "running", + "url": "https://x.hub.example.com", + }, + ] + + with patch("keboola_agent_cli.services.data_app_service.time.sleep", lambda _: None): + result = service._poll_until_terminal( + ds_mock, + "42", + target_desired_state="running", + timeout_seconds=60.0, + ) + assert result["state"] == "running" + assert ds_mock.get_app.call_count == 3 + + def test_error_state_raises(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, _storage, _enc = _make_service(store) + ds_mock.get_app.return_value = {"state": "error"} + + with ( + patch("keboola_agent_cli.services.data_app_service.time.sleep", lambda _: None), + pytest.raises(KeboolaApiError) as excinfo, + ): + service._poll_until_terminal( + ds_mock, + "42", + target_desired_state="running", + timeout_seconds=60.0, + ) + assert excinfo.value.error_code == ErrorCode.DATA_APP_BUILD_FAILED + + def test_timeout_raises(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, _storage, _enc = _make_service(store) + ds_mock.get_app.return_value = {"state": "starting"} + + # time.monotonic returns 0 then 100 -> exceeds the 1-second deadline + # immediately on the second iteration. + with ( + patch("keboola_agent_cli.services.data_app_service.time.sleep", lambda _: None), + patch( + "keboola_agent_cli.services.data_app_service.time.monotonic", + side_effect=[0.0, 100.0, 200.0], + ), + pytest.raises(KeboolaApiError) as excinfo, + ): + service._poll_until_terminal( + ds_mock, + "42", + target_desired_state="running", + timeout_seconds=1.0, + ) + assert excinfo.value.error_code == ErrorCode.DATA_APP_DEPLOY_TIMEOUT + + +# --------------------------------------------------------------------------- +# Password retrieval +# --------------------------------------------------------------------------- + + +class TestDataAppPassword: + def test_returns_password(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, _storage, _enc = _make_service(store) + ds_mock.get_app_password.return_value = {"password": "deadbeefcafe"} + + result = service.get_data_app_password( + alias="prod", app_id="42", manage_token=TEST_MANAGE_TOKEN + ) + assert result["password"] == "deadbeefcafe" + ds_mock.get_app_password.assert_called_once_with("42", manage_token=TEST_MANAGE_TOKEN) + + def test_missing_manage_token(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ds, _storage, _enc = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + service.get_data_app_password(alias="prod", app_id="42", manage_token="") + assert excinfo.value.error_code == ErrorCode.INVALID_TOKEN + + +# --------------------------------------------------------------------------- +# Delete +# --------------------------------------------------------------------------- + + +class TestDataAppDelete: + def test_delete_calls_data_science_delete(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, _storage, _enc = _make_service(store) + result = service.delete_data_app(alias="prod", app_id="42") + ds_mock.delete_app.assert_called_once_with("42") + assert result["deleted"] is True + assert result["id"] == "42" + + +# --------------------------------------------------------------------------- +# Detail +# --------------------------------------------------------------------------- + + +class TestDataAppDetail: + def test_detail_merges_data_science_and_storage(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, ds_mock, storage_mock, _enc = _make_service(store) + ds_mock.get_app.return_value = { + "id": "42", + "configId": "ulid", + "state": "running", + "desiredState": "running", + "url": "https://x.hub.example.com", + "configVersion": "3", + "type": "python-js", + "size": "tiny", + "autoSuspendAfterSeconds": 900, + "lastStartTimestamp": "2026-05-01T00:00:00Z", + } + storage_mock.get_config_detail.return_value = { + "version": 5, + "name": "App", + "description": "long", + "configuration": { + "parameters": { + "dataApp": { + "slug": "my-app", + "git": { + "repository": "https://github.com/o/r", + "private": True, + "username": "user", + "#password": "KBC::ProjectSecure::xyz", + "branch": "main", + }, + }, + "id": "42", + }, + "runtime": {"backend": {"size": "tiny"}}, + }, + } + result = service.get_data_app(alias="prod", app_id="42") + + assert result["id"] == "42" + assert result["state"] == "running" + assert result["config_version_storage"] == "5" + assert result["config_version_deployed"] == "3" + assert result["slug"] == "my-app" + # PAT redaction + assert result["git"]["#password"] == "" + # Plaintext repository / branch preserved. + assert result["git"]["repository"] == "https://github.com/o/r" + + +# --------------------------------------------------------------------------- +# Misc helpers +# --------------------------------------------------------------------------- + + +class TestRedactGitBlock: + def test_redacts_encrypted_password(self) -> None: + block = {"#password": "KBC::Project::xyz", "username": "u"} + out = _redact_git_block(block) + assert out["#password"] == "" + assert out["username"] == "u" + + def test_no_password_key_is_noop(self) -> None: + block = {"username": "u", "repository": "https://x"} + out = _redact_git_block(block) + assert out == block + + +class TestRedactStorageConfig: + def test_redacts_nested_password(self) -> None: + cfg = { + "id": "ulid", + "version": 3, + "configuration": { + "parameters": { + "dataApp": { + "slug": "x", + "git": { + "repository": "https://github.com/o/r", + "#password": "KBC::ProjectSecureGKMS::deadbeef", + "username": "u", + }, + } + } + }, + } + out = _redact_storage_config(cfg) + assert out["configuration"]["parameters"]["dataApp"]["git"]["#password"] == "" + # Original input not mutated (function returns a deep-copy of the + # affected branch). + assert ( + cfg["configuration"]["parameters"]["dataApp"]["git"]["#password"] + == "KBC::ProjectSecureGKMS::deadbeef" + ) + + def test_no_git_block_is_noop(self) -> None: + cfg = {"id": "ulid", "configuration": {"parameters": {}}} + out = _redact_storage_config(cfg) + assert out == cfg + + def test_empty_dict_passes_through(self) -> None: + assert _redact_storage_config({}) == {} diff --git a/tests/test_e2e.py b/tests/test_e2e.py index e781471b..096b7ed8 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -5864,3 +5864,221 @@ def test_native_types_and_branch_materialize(self) -> None: assert second.get("auto_created_bucket") is False, ( "Second create against an already-materialized bucket should not re-create it." ) + + +# --------------------------------------------------------------------------- +# TestE2EDataAppLifecycle -- data-app create / detail / deploy / start / stop / delete +# --------------------------------------------------------------------------- + + +ENV_DATA_APP_GIT_REPO_PUBLIC = "E2E_DATA_APP_GIT_REPO_PUBLIC" +ENV_DATA_APP_GIT_REPO_PRIVATE = "E2E_DATA_APP_GIT_REPO_PRIVATE" +ENV_DATA_APP_GIT_USER = "E2E_DATA_APP_GIT_USER" +ENV_DATA_APP_GIT_PAT = "E2E_DATA_APP_GIT_PAT" +ENV_MANAGE_TOKEN = "E2E_MANAGE_TOKEN" + +skip_without_data_app_public = pytest.mark.skipif( + not (HAS_CREDENTIALS and os.environ.get(ENV_DATA_APP_GIT_REPO_PUBLIC)), + reason=f"requires {ENV_TOKEN} + {ENV_DATA_APP_GIT_REPO_PUBLIC}", +) +skip_without_data_app_private = pytest.mark.skipif( + not ( + HAS_CREDENTIALS + and os.environ.get(ENV_DATA_APP_GIT_REPO_PRIVATE) + and os.environ.get(ENV_DATA_APP_GIT_USER) + and os.environ.get(ENV_DATA_APP_GIT_PAT) + ), + reason=( + f"requires {ENV_TOKEN} + {ENV_DATA_APP_GIT_REPO_PRIVATE} + " + f"{ENV_DATA_APP_GIT_USER} + {ENV_DATA_APP_GIT_PAT}" + ), +) + + +@pytest.mark.e2e +class TestE2EDataAppLifecycle: + """Live validation of the data-app command group against a real stack. + + Three scenarios: + + 1. Public-repo + ``--auth public`` -- minimum recipe (no encryption). + 2. Private-repo + simpleAuth -- full recipe including KMS encryption. + 3. Lifecycle: stop / start / deploy on the just-created private app. + + Each test cleans up its own apps. Cleanup is best-effort (delete is + idempotent on the platform side -- a 404 on cleanup is not a failure). + """ + + @pytest.fixture(autouse=True) + def setup(self, tmp_path: Path) -> None: + if not HAS_CREDENTIALS: + pytest.skip("E2E_API_TOKEN not set") + self.token = os.environ[ENV_TOKEN] + raw_url = os.environ.get(ENV_URL, "connection.keboola.com") + self.url = raw_url if raw_url.startswith("https://") else f"https://{raw_url}" + self.alias = f"{RUN_ID}-da-proj" + self.config_dir = tmp_path / "config" + self.config_dir.mkdir() + # Register the project so `kbagent --project ALIAS ...` works. + _invoke( + self.config_dir, + [ + "project", + "add", + "--project", + self.alias, + "--url", + self.url, + "--token", + self.token, + ], + ) + self._created_app_ids: list[str] = [] + + @pytest.fixture(autouse=True) + def cleanup(self) -> Any: + yield + print("\n--- DATA-APP CLEANUP ---") + for app_id in self._created_app_ids: + try: + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "delete", + "--project", + self.alias, + "--app-id", + app_id, + "--yes", + ], + ) + print(f" Deleted data app {app_id}") + except Exception as exc: + print(f" WARN: failed to delete data app {app_id}: {exc}") + + @skip_without_data_app_public + def test_data_app_lifecycle_public(self) -> None: + _step(1, "Create public-repo data app", "no auth gate, no encryption") + repo = os.environ[ENV_DATA_APP_GIT_REPO_PUBLIC] + slug = f"e2e-pub-{RUN_ID}"[:60] + result = _invoke( + self.config_dir, + [ + "--json", + "data-app", + "create", + "--project", + self.alias, + "--name", + f"E2E Public {RUN_ID}", + "--slug", + slug, + "--git-repo", + repo, + "--git-public", + "--auth", + "public", + "--no-deploy", # avoid waiting on a real container build in CI + ], + ) + assert result.exit_code == 0, result.output + body = _json_ok(result) + app_id = body["data"]["id"] + assert app_id, "expected a numeric app id from POST /apps" + self._created_app_ids.append(app_id) + + _step(2, "Detail merges Data Science + Storage") + detail = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "detail", + "--project", + self.alias, + "--app-id", + app_id, + ], + ) + )["data"] + assert detail["id"] == app_id + assert detail["slug"] == slug + assert detail["config_version_storage"], ( + "Storage config version should be populated after PUT" + ) + + @skip_without_data_app_private + def test_data_app_lifecycle_private_and_redeploy(self) -> None: + _step(1, "Create private-repo simpleAuth data app", "encryption + git PAT") + repo = os.environ[ENV_DATA_APP_GIT_REPO_PRIVATE] + username = os.environ[ENV_DATA_APP_GIT_USER] + # Pass the PAT via env var so plaintext never appears in argv. + pat_var = "E2E_DATA_APP_GIT_PAT" + slug = f"e2e-priv-{RUN_ID}"[:60] + result = _invoke( + self.config_dir, + [ + "--json", + "data-app", + "create", + "--project", + self.alias, + "--name", + f"E2E Private {RUN_ID}", + "--slug", + slug, + "--git-repo", + repo, + "--git-username", + username, + "--git-pat-env", + pat_var, + "--auth", + "password", + "--no-deploy", + ], + ) + assert result.exit_code == 0, result.output + body = _json_ok(result) + app_id = body["data"]["id"] + self._created_app_ids.append(app_id) + # The encrypted PAT must NEVER appear in the JSON output. + plaintext_pat = os.environ[ENV_DATA_APP_GIT_PAT] + assert plaintext_pat not in result.output, "Plaintext PAT must never reach the CLI output" + + _step(2, "Stop (idempotent on a non-running app)") + stop = _invoke( + self.config_dir, + [ + "--json", + "data-app", + "stop", + "--project", + self.alias, + "--app-id", + app_id, + ], + ) + # `stop` on a never-deployed app may return a 4xx; we don't fail the + # test on that -- the next deploy step is the real assertion. + _ = stop + + _step(3, "Deploy via the §9 redeploy contract") + deploy = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "deploy", + "--project", + self.alias, + "--app-id", + app_id, + ], + ) + )["data"] + assert deploy["config_version"], "deploy must pin a configVersion" diff --git a/uv.lock b/uv.lock index 0c65edbf..0363ff09 100644 --- a/uv.lock +++ b/uv.lock @@ -439,7 +439,7 @@ wheels = [ [[package]] name = "keboola-agent-cli" -version = "0.26.0" +version = "0.27.0" source = { editable = "." } dependencies = [ { name = "httpx" },