diff --git a/CLAUDE.md b/CLAUDE.md index 9d3103b0..054f4081 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -253,6 +253,11 @@ kbagent config rename --project NAME --component-id ID --config-id ID --name "Ne kbagent config variables-set --project NAME --component-id ID --config-id ID --var KEY=VALUE [--var ...] [--replace] [--variables-id ID] [--values-id ID] [--branch ID] [--dry-run] kbagent config variables-get --project NAME --component-id ID --config-id ID [--branch ID] kbagent config variables-clear --project NAME --component-id ID --config-id ID [--branch ID] [--yes] +kbagent config metadata-list --project NAME --component-id ID --config-id ID [--branch ID] +kbagent config get-metadata --project NAME --component-id ID --config-id ID --key KEY [--branch ID] +kbagent config set-metadata --project NAME --component-id ID --config-id ID --key KEY --value VALUE [--branch ID] +kbagent config delete-metadata --project NAME --component-id ID --config-id ID --metadata-id ID [--branch ID] [--yes] +kbagent config set-folder --project NAME --component-id ID --config-id ID --name FOLDER [--branch ID] kbagent job list [--project NAME] [--component-id ID] [--status STATUS] [--limit N] kbagent job detail --project NAME --job-id ID @@ -309,13 +314,14 @@ kbagent branch metadata-set --project NAME --key KEY [--text STR | --file PATH | kbagent branch metadata-delete --project NAME --metadata-id ID [--branch ID|default] kbagent workspace create --project ALIAS [--name NAME] [--backend TYPE] [--ui] [--read-only/--no-read-only] -kbagent workspace list [--project NAME] +kbagent workspace list [--project NAME ...] [--orphaned] kbagent workspace detail --project ALIAS --workspace-id ID kbagent workspace delete --project ALIAS --workspace-id ID kbagent workspace password --project ALIAS --workspace-id ID kbagent workspace load --project ALIAS --workspace-id ID --tables TABLE_ID [--tables ...] [--preserve] kbagent workspace query --project ALIAS --workspace-id ID --sql "SELECT ..." [--transactional] kbagent workspace query --project ALIAS --workspace-id ID --file query.sql +kbagent workspace gc [--project NAME ...] [--dry-run] [--yes] kbagent workspace from-transformation --project ALIAS --component-id ID --config-id ID [--row-id ID] kbagent component list [--project NAME] [--type TYPE] [--query QUERY] diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index 8ec26950..d4cd469c 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.21.1", + "version": "0.22.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 3050395d..afcf7e7c 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -93,6 +93,11 @@ When working inside a git repository or project directory, run `kbagent init` (o | Rename a configuration (update name via API + rename local sync directory) | `kbagent config rename --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --name NAME` | | Delete a configuration from a project | `kbagent config delete --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | Generate boilerplate configuration files for a Keboola component | `kbagent config new --component-id COMPONENT-ID` | +| List all metadata entries on a configuration | `kbagent config metadata-list --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | +| Read a single metadata value by key | `kbagent config get-metadata --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --key KEY` | +| Set a metadata key/value on a configuration (upsert) | `kbagent config set-metadata --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --key KEY --value VALUE` | +| Delete a configuration metadata entry by its numeric ID | `kbagent config delete-metadata --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --metadata-id METADATA-ID` | +| Set the folder (KBC.configuration.folderName) on a configuration | `kbagent config set-folder --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID --name NAME` | | Assign variables to a config (auto-creates backing keboola.variables on first call) | `kbagent config variables-set --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | Read the current variable values attached to a config | `kbagent config variables-get --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | Unlink variables from a config (does NOT delete the underlying keboola.variables) | `kbagent config variables-clear --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | @@ -150,6 +155,7 @@ When working inside a git repository or project directory, run `kbagent init` (o | Reset workspace password and show the new one | `kbagent workspace password --project PROJECT --workspace-id WORKSPACE-ID` | | Load tables into a workspace | `kbagent workspace load --project PROJECT --workspace-id WORKSPACE-ID --tables TABLES` | | Execute SQL query in a workspace via Query Service | `kbagent workspace query --project PROJECT --workspace-id WORKSPACE-ID` | +| Garbage-collect orphaned workspaces | `kbagent workspace gc` | | Create a workspace from a transformation config | `kbagent workspace from-transformation --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | List available MCP tools from the keboola-mcp-server | `kbagent tool list` | | Call an MCP tool on keboola-mcp-server | `kbagent tool call ` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 06ed9679..74d8670d 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -38,6 +38,11 @@ All commands support `--json` for structured output. Multi-project flags (`--pro - `config variables-set --project NAME --component-id ID --config-id ID --var KEY=VALUE [--var ...] [--replace] [--variables-id ID] [--values-id ID] [--branch ID] [--dry-run] [--allow-plaintext-on-encrypt-failure] [--yes]` -- attach variable values to a config. Auto-creates a sibling `keboola.variables` config + default row on first use and links it via the parent's `runtime.variables_id` / `variables_values_id`. Defaults to merge; `--replace` drops keys not in `--var`. `#`-prefixed values encrypt via the Encryption API (fail-closed; exit non-zero on `ENCRYPTION_FAILED`). See `variables-workflow.md` - `config variables-get --project NAME --component-id ID --config-id ID [--branch ID]` -- resolve `variables_id` + `values_id` from the parent config and fetch the current KEY=VALUE map. Returns `{linked: bool, variables_id, values_id, values}`; `linked=false` means the parent has no variables attached - `config variables-clear --project NAME --component-id ID --config-id ID [--branch ID] [--yes]` -- unlink variables from the parent config (strips `variables_id` + `variables_values_id`). **Does NOT delete** the backing `keboola.variables` config -- use `config delete` explicitly if you've verified nothing else references it +- `config metadata-list --project NAME --component-id ID --config-id ID [--branch ID]` -- list all metadata entries on a configuration (id, key, value, provider, timestamp). Branch-aware +- `config get-metadata --project NAME --component-id ID --config-id ID --key KEY [--branch ID]` -- read a single metadata value by key. Exits with `NOT_FOUND` (exit 1) if absent +- `config set-metadata --project NAME --component-id ID --config-id ID --key KEY --value VALUE [--branch ID]` -- set (upsert) a metadata key/value on a configuration. Common keys: `KBC.configuration.folderName`, plus any custom `KBC.*` agent-facing tags +- `config delete-metadata --project NAME --component-id ID --config-id ID --metadata-id ID [--branch ID] [--yes]` -- delete a configuration metadata entry by its numeric ID (from `metadata-list`) +- `config set-folder --project NAME --component-id ID --config-id ID --name FOLDER [--branch ID]` -- set (or clear, with empty `--name`) the `KBC.configuration.folderName` metadata, which groups configs into named folders in the Keboola UI. See `config-metadata-workflow.md` ## Job History - `job list [--project NAME] [--component-id ID] [--config-id ID] [--status STATUS] [--limit N]` -- list jobs (default 50, max 500) @@ -90,12 +95,13 @@ All commands support `--json` for structured output. Multi-project flags (`--pro ## Workspaces (SQL Debugging) - `workspace create --project ALIAS [--name NAME] [--ui] [--read-only]` -- create workspace (headless ~1s, `--ui` ~15s) -- `workspace list [--project NAME]` -- list workspaces +- `workspace list [--project NAME ...] [--orphaned]` -- list workspaces. `--project` repeatable for multi-project; `--orphaned` filters to workspaces whose backing `keboola.sandboxes` config is missing - `workspace detail --project ALIAS --workspace-id ID` -- show connection details - `workspace delete --project ALIAS --workspace-id ID` -- delete workspace - `workspace password --project ALIAS --workspace-id ID` -- reset and return new password - `workspace load --project ALIAS --workspace-id ID --tables TABLE_ID [...] [--preserve]` -- load storage tables - `workspace query --project ALIAS --workspace-id ID --sql "..." [--file F] [--transactional]` -- run SQL via Query Service +- `workspace gc [--project NAME ...] [--dry-run] [--yes]` -- garbage-collect orphaned workspaces (and any lingering `keboola.sandboxes` configs). `--dry-run` previews without deleting; `--project` repeatable, omit to GC across all connected projects - `workspace from-transformation --project ALIAS --component-id ID --config-id ID [--row-id ID]` -- workspace from existing transform ## MCP Tools diff --git a/plugins/kbagent/skills/kbagent/references/config-metadata-workflow.md b/plugins/kbagent/skills/kbagent/references/config-metadata-workflow.md new file mode 100644 index 00000000..d210b78d --- /dev/null +++ b/plugins/kbagent/skills/kbagent/references/config-metadata-workflow.md @@ -0,0 +1,205 @@ +# Config Metadata Workflow -- Tags, folders, and agent breadcrumbs + +Keboola stores free-form `key / value / provider` metadata on every configuration. +The Keboola UI uses a handful of `KBC.*` keys for user-facing behavior (most +notably `KBC.configuration.folderName` to group configs into folders), but the +surface is open-ended: agents can stamp their own keys to leave breadcrumbs for +later runs (e.g. `agent.owner`, `agent.lastAudit`, `agent.domain`). + +kbagent exposes the CRUD surface as five commands on `kbagent config`: + +``` +metadata-list -- list all entries (id, key, value, provider, timestamp) +get-metadata -- read one value by key +set-metadata -- upsert a single key/value +delete-metadata -- remove an entry by its numeric id +set-folder -- convenience wrapper that writes KBC.configuration.folderName +``` + +All five are branch-aware; omit `--branch` to use the project's active branch. + +## When to use this + +- **Folder organization**: group related configs under a named folder in the + Keboola UI (`set-folder`). Works across all component types; no schema change + needed. +- **Agent breadcrumbs**: tag configs an agent has touched so later runs can + skip, re-audit, or attribute them (`set-metadata --key agent.* ...`). +- **Ownership / governance tags**: stamp `owner`, `domain`, `cost-center`, + etc. and filter via `config list --json | jq` downstream. +- **Provenance tracking**: record when a config was last generated or + refactored by an automated workflow. + +## CLI cheatsheet + +```bash +# List everything on a config (sorted by key) +kbagent --json config metadata-list --project prod \ + --component-id keboola.snowflake-transformation --config-id 15815157 + +# Read a specific key (exits 1 / NOT_FOUND if absent) +kbagent --json config get-metadata --project prod \ + --component-id keboola.snowflake-transformation --config-id 15815157 \ + --key KBC.configuration.folderName + +# Upsert (create if new, overwrite if existing) +kbagent config set-metadata --project prod \ + --component-id keboola.snowflake-transformation --config-id 15815157 \ + --key agent.owner --value analytics-team + +# Delete by numeric ID (from metadata-list) +kbagent config delete-metadata --project prod \ + --component-id keboola.snowflake-transformation --config-id 15815157 \ + --metadata-id 4281 --yes + +# Folder sugar (writes KBC.configuration.folderName) +kbagent config set-folder --project prod \ + --component-id keboola.snowflake-transformation --config-id 15815157 \ + --name "Customer 360" + +# Clear the folder assignment (empty string) +kbagent config set-folder --project prod \ + --component-id keboola.snowflake-transformation --config-id 15815157 \ + --name "" +``` + +## Folder organization pattern + +The Keboola UI reads `KBC.configuration.folderName` and groups configs sharing +the same value into a named folder under the component. This is purely a +presentation feature -- the config itself is unchanged, and there are no +parent/child resources to manage. That makes it ideal for programmatic +taxonomy: + +```bash +# Tag every config in an onboarding flow with one folder +for cfg_id in 15815157 15815158 15815159 15815160; do + kbagent config set-folder --project prod \ + --component-id keboola.snowflake-transformation \ + --config-id "$cfg_id" \ + --name "Customer 360 - Onboarding" +done +``` + +Guidelines: + +- `set-folder --name ""` removes the grouping (passes an empty string to + `set-metadata`, which the UI treats as "no folder"). +- Folder names are free-form strings; keep them short and stable -- the UI + sorts alphabetically. +- Prefer `set-folder` over raw `set-metadata --key KBC.configuration.folderName`; + the wrapper exists so the key spelling is not a moving target for agents. +- Folders are **per component**. Two configs under different components with + the same folder name render as two separate folders in the UI -- this is + intentional, not a bug. + +## Full lifecycle example + +```bash +PROJECT=prod +COMPONENT=keboola.snowflake-transformation +CONFIG=15815157 + +# 1. Inspect what's already on the config +kbagent --json config metadata-list \ + --project "$PROJECT" --component-id "$COMPONENT" --config-id "$CONFIG" + +# 2. Stamp an agent breadcrumb +kbagent config set-metadata \ + --project "$PROJECT" --component-id "$COMPONENT" --config-id "$CONFIG" \ + --key agent.lastAudit --value "2026-04-23" + +# 3. Read it back +kbagent --json config get-metadata \ + --project "$PROJECT" --component-id "$COMPONENT" --config-id "$CONFIG" \ + --key agent.lastAudit + +# 4. File it into the Customer 360 folder +kbagent config set-folder \ + --project "$PROJECT" --component-id "$COMPONENT" --config-id "$CONFIG" \ + --name "Customer 360" + +# 5. Later: clean the breadcrumb (list to find its numeric id, then delete) +METADATA_ID=$(kbagent --json config metadata-list \ + --project "$PROJECT" --component-id "$COMPONENT" --config-id "$CONFIG" \ + | jq -r '.data.metadata[] | select(.key=="agent.lastAudit") | .id') + +kbagent config delete-metadata \ + --project "$PROJECT" --component-id "$COMPONENT" --config-id "$CONFIG" \ + --metadata-id "$METADATA_ID" --yes +``` + +## Response shapes (`--json` mode) + +### `metadata-list` +```json +{ + "status": "ok", + "data": { + "project_alias": "prod", + "component_id": "keboola.snowflake-transformation", + "config_id": "15815157", + "branch_id": 12345, + "metadata": [ + {"id": "4281", "key": "KBC.configuration.folderName", "value": "Customer 360", "provider": "user", "timestamp": "2026-04-23T10:15:00Z"}, + {"id": "4282", "key": "agent.lastAudit", "value": "2026-04-23", "provider": "user", "timestamp": "2026-04-23T10:16:02Z"} + ] + } +} +``` + +The `metadata` list is key-sorted for deterministic output. + +### `get-metadata` +```json +{ + "status": "ok", + "data": { + "project_alias": "prod", + "component_id": "keboola.snowflake-transformation", + "config_id": "15815157", + "branch_id": 12345, + "key": "agent.lastAudit", + "value": "2026-04-23", + "metadata_id": "4282" + } +} +``` + +Returns `NOT_FOUND` (exit 1) when the key is absent -- there is no sentinel +"empty" value; missing means missing. + +### `set-metadata` / `set-folder` / `delete-metadata` +Each returns `project_alias`, `component_id`, `config_id`, `branch_id`, and a +human-readable `message`. `set-folder` additionally returns `folder` so +callers don't have to re-parse the message string. + +## Provider semantics + +The `provider` field on each metadata entry distinguishes `user` (anything you +wrote via CLI/API/UI) from `system` (set by the Keboola platform itself). +kbagent never filters these out -- `metadata-list` surfaces both. Do not +attempt to `set-metadata` on a system-provider key; the API will return a +validation error and `delete-metadata` on a system entry is likewise rejected. + +## Relation to `config update` + +`config update` mutates the configuration body (`parameters`, `storage`, +`processors`, etc.). `config set-metadata` mutates the sibling `metadata` +array on the same configuration resource. They touch different endpoints and +never conflict. Use `config update` for anything that affects runtime +behavior; use `set-metadata` for everything else (tags, folders, audit +breadcrumbs). + +## Branch awareness + +All five commands resolve `--branch` the same way the rest of `kbagent config` +does: + +1. Explicit `--branch ID` wins. +2. Otherwise the project's **active branch** (set by `kbagent branch use` / + `branch create`) is used. +3. If the project has no active branch, the main / default branch is used. + +Metadata in a dev branch is independent of production; it merges back via the +same merge URL as the rest of the branch. See [branch-workflow.md](branch-workflow.md). diff --git a/pyproject.toml b/pyproject.toml index 0eee9ad7..d622abf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-agent-cli" -version = "0.21.1" +version = "0.22.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 29f65c04..b19cf6d9 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -8,6 +8,12 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.22.0": [ + "New: config metadata-list / get-metadata / set-metadata / delete-metadata -- CRUD for arbitrary metadata key/value pairs on any configuration, using the branch-aware Storage API metadata endpoint (FIIA P1-3)", + "New: config set-folder -- sugar over set-metadata for KBC.configuration.folderName; organises configs into named folder groups visible in the Keboola UI (FIIA P1-3)", + "New: workspace list --orphaned -- lists workspaces backed by keboola.sandboxes whose sandbox config no longer exists (FIIA P1-4)", + "New: workspace gc [--dry-run] [--yes] -- deletes all orphaned workspaces; dry-run previews without touching anything; --yes skips interactive confirmation (FIIA P1-4)", + ], "0.21.1": [ "Fix: sync pull on a newly created dev branch now writes config rows (#193) -- idempotent skip guard for rows was missing a file-existence check, causing rows to be silently skipped when the branch directory was new (hash matched main because the branch is a clone)", ], diff --git a/src/keboola_agent_cli/client.py b/src/keboola_agent_cli/client.py index d92b07e3..ee763918 100644 --- a/src/keboola_agent_cli/client.py +++ b/src/keboola_agent_cli/client.py @@ -342,6 +342,64 @@ def list_config_folder_metadata(self, branch_id: int) -> dict[str, str]: folder_map[f"{comp_id}/{config_id}"] = meta["value"] return folder_map + def list_config_metadata( + self, + component_id: str, + config_id: str, + branch_id: int | None = None, + ) -> list[dict[str, Any]]: + """List metadata entries on a configuration. + + GET /v2/storage/[branch/{b}/]components/{c}/configs/{id}/metadata + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + response = self._request( + "GET", + f"{prefix}/components/{quote(component_id, safe='')}/configs/{quote(config_id, safe='')}/metadata", + ) + return response.json() + + def set_config_metadata( + self, + component_id: str, + config_id: str, + entries: list[tuple[str, str]], + branch_id: int | None = None, + ) -> list[dict[str, Any]]: + """Bulk-set metadata key/value pairs on a configuration. + + POST /v2/storage/[branch/{b}/]components/{c}/configs/{id}/metadata + Same PHP-style indexed form as set_branch_metadata. + """ + form: dict[str, str] = {} + for i, (key, value) in enumerate(entries): + form[f"metadata[{i}][key]"] = key + form[f"metadata[{i}][value]"] = value + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + response = self._request( + "POST", + f"{prefix}/components/{quote(component_id, safe='')}/configs/{quote(config_id, safe='')}/metadata", + data=form, + ) + return response.json() + + def delete_config_metadata( + self, + component_id: str, + config_id: str, + metadata_id: int | str, + branch_id: int | None = None, + ) -> None: + """Delete a single metadata entry on a configuration by its numeric ID. + + DELETE /v2/storage/[branch/{b}/]components/{c}/configs/{id}/metadata/{mid} + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + self._request( + "DELETE", + f"{prefix}/components/{quote(component_id, safe='')}/configs/{quote(config_id, safe='')}/metadata/{metadata_id}", + ) + def create_config( self, component_id: str, diff --git a/src/keboola_agent_cli/commands/config.py b/src/keboola_agent_cli/commands/config.py index 705345ca..37e2e4b0 100644 --- a/src/keboola_agent_cli/commands/config.py +++ b/src/keboola_agent_cli/commands/config.py @@ -836,6 +836,263 @@ def config_new( formatter.console.print() +# ── Config metadata commands ─────────────────────────────────────────── + + +@config_app.command("metadata-list") +def config_metadata_list( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + component_id: str = typer.Option(..., "--component-id", help="Component ID"), + config_id: str = typer.Option(..., "--config-id", help="Configuration ID"), + branch: int | None = typer.Option( + None, "--branch", help="Dev branch ID (defaults to active branch)" + ), +) -> None: + """List all metadata entries on a configuration.""" + if should_hint(ctx): + emit_hint( + ctx, + "config.metadata-list", + project=project, + component_id=component_id, + config_id=config_id, + branch=branch, + ) + return + formatter = get_formatter(ctx) + config_store: ConfigStore = ctx.obj["config_store"] + _, effective_branch = resolve_branch(config_store, formatter, project, branch) + service = get_service(ctx, "config_service") + try: + result = service.list_config_metadata( + alias=project, + component_id=component_id, + config_id=config_id, + branch_id=effective_branch, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + else: + entries = result.get("metadata", []) + if not entries: + formatter.console.print("[dim]No metadata entries.[/dim]") + else: + for e in entries: + formatter.console.print( + f" [dim]{escape(str(e.get('id', '')))}[/dim] [green]{escape(e.get('key', ''))}[/green] = {escape(str(e.get('value', '')))} [dim]{escape(e.get('provider', 'user'))}[/dim]" + ) + + +@config_app.command("get-metadata") +def config_get_metadata( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + component_id: str = typer.Option(..., "--component-id", help="Component ID"), + config_id: str = typer.Option(..., "--config-id", help="Configuration ID"), + key: str = typer.Option(..., "--key", help="Metadata key to read"), + branch: int | None = typer.Option( + None, "--branch", help="Dev branch ID (defaults to active branch)" + ), +) -> None: + """Read a single metadata value by key. + + Exits with code 1 (NOT_FOUND) if the key is not present. + """ + if should_hint(ctx): + emit_hint( + ctx, + "config.get-metadata", + project=project, + component_id=component_id, + config_id=config_id, + key=key, + branch=branch, + ) + return + formatter = get_formatter(ctx) + config_store: ConfigStore = ctx.obj["config_store"] + _, effective_branch = resolve_branch(config_store, formatter, project, branch) + service = get_service(ctx, "config_service") + try: + result = service.get_config_metadata_value( + alias=project, + component_id=component_id, + config_id=config_id, + key=key, + branch_id=effective_branch, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output(result, lambda c, d: c.print(d["value"])) + + +@config_app.command("set-metadata") +def config_set_metadata( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + component_id: str = typer.Option(..., "--component-id", help="Component ID"), + config_id: str = typer.Option(..., "--config-id", help="Configuration ID"), + key: str = typer.Option(..., "--key", help="Metadata key to set"), + value: str = typer.Option(..., "--value", help="Metadata value (string)"), + branch: int | None = typer.Option( + None, "--branch", help="Dev branch ID (defaults to active branch)" + ), +) -> None: + """Set a metadata key/value on a configuration (upsert).""" + if should_hint(ctx): + emit_hint( + ctx, + "config.set-metadata", + project=project, + component_id=component_id, + config_id=config_id, + key=key, + value=value, + branch=branch, + ) + return + formatter = get_formatter(ctx) + config_store: ConfigStore = ctx.obj["config_store"] + _, effective_branch = resolve_branch(config_store, formatter, project, branch) + service = get_service(ctx, "config_service") + try: + result = service.set_config_metadata( + alias=project, + component_id=component_id, + config_id=config_id, + key=key, + value=value, + branch_id=effective_branch, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output( + result, lambda c, d: c.print(f"[bold green]Success:[/bold green] {d['message']}") + ) + + +@config_app.command("delete-metadata") +def config_delete_metadata( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + component_id: str = typer.Option(..., "--component-id", help="Component ID"), + config_id: str = typer.Option(..., "--config-id", help="Configuration ID"), + metadata_id: int = typer.Option(..., "--metadata-id", help="Numeric ID from metadata-list"), + branch: int | None = typer.Option( + None, "--branch", help="Dev branch ID (defaults to active branch)" + ), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation"), +) -> None: + """Delete a configuration metadata entry by its numeric ID.""" + if should_hint(ctx): + emit_hint( + ctx, + "config.delete-metadata", + project=project, + component_id=component_id, + config_id=config_id, + metadata_id=metadata_id, + branch=branch, + ) + return + formatter = get_formatter(ctx) + config_store: ConfigStore = ctx.obj["config_store"] + _, effective_branch = resolve_branch(config_store, formatter, project, branch) + + if ( + not yes + and not formatter.json_mode + and not typer.confirm(f"Delete metadata ID {metadata_id} from {component_id}/{config_id}?") + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + + service = get_service(ctx, "config_service") + try: + result = service.delete_config_metadata( + alias=project, + component_id=component_id, + config_id=config_id, + metadata_id=metadata_id, + branch_id=effective_branch, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output( + result, lambda c, d: c.print(f"[bold green]Success:[/bold green] {d['message']}") + ) + + +@config_app.command("set-folder") +def config_set_folder( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + component_id: str = typer.Option(..., "--component-id", help="Component ID"), + config_id: str = typer.Option(..., "--config-id", help="Configuration ID"), + name: str = typer.Option(..., "--name", help="Folder name (empty string to clear)"), + branch: int | None = typer.Option( + None, "--branch", help="Dev branch ID (defaults to active branch)" + ), +) -> None: + """Set the folder (KBC.configuration.folderName) on a configuration. + + Organises configs into named groups in the Keboola UI. + Pass an empty string to remove the folder assignment. + """ + if should_hint(ctx): + emit_hint( + ctx, + "config.set-folder", + project=project, + component_id=component_id, + config_id=config_id, + name=name, + branch=branch, + ) + return + formatter = get_formatter(ctx) + config_store: ConfigStore = ctx.obj["config_store"] + _, effective_branch = resolve_branch(config_store, formatter, project, branch) + service = get_service(ctx, "config_service") + try: + result = service.set_config_folder( + alias=project, + component_id=component_id, + config_id=config_id, + folder_name=name, + branch_id=effective_branch, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output( + result, lambda c, d: c.print(f"[bold green]Success:[/bold green] {d['message']}") + ) + + def _parse_kv_var(raw: str) -> tuple[str, str]: """Split a ``KEY=VALUE`` token into ``(key, value)``; ``#``-prefix preserved.""" if "=" not in raw: diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index cd32ec30..e0e1ee95 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -131,6 +131,24 @@ Unlink variables from a config. Does NOT delete the underlying keboola.variables config (it may be shared). Delete it explicitly via `kbagent config delete` if needed. +### Config Metadata (folder organisation + arbitrary key/value) + + kbagent config metadata-list --project NAME --component-id ID --config-id ID [--branch ID] + List all metadata entries on a configuration. Each entry: id, key, value, provider, timestamp. + + kbagent config get-metadata --project NAME --component-id ID --config-id ID --key KEY [--branch ID] + Read a single metadata value by key. Exits 1 (NOT_FOUND) if absent. + + kbagent config set-metadata --project NAME --component-id ID --config-id ID --key KEY --value VALUE [--branch ID] + Set (upsert) a metadata key/value on a configuration. + + kbagent config delete-metadata --project NAME --component-id ID --config-id ID --metadata-id ID [--branch ID] [--yes] + Delete a configuration metadata entry by numeric ID (from metadata-list). + + kbagent config set-folder --project NAME --component-id ID --config-id ID --name "FolderName" [--branch ID] + Sugar: writes KBC.configuration.folderName metadata. Groups the config in the Keboola UI. + Pass --name "" to remove the folder assignment. + ### Job History kbagent job list [--project NAME] [--component-id ID] [--config-id ID] [--status STATUS] [--limit N] @@ -318,8 +336,8 @@ kbagent workspace create --project ALIAS [--name NAME] [--backend TYPE] [--ui] [--read-only/--no-read-only] Create workspace. Backend auto-detected from project (or override with --backend). Default: headless (~1s). --ui: visible in KBC UI (~15s). - kbagent workspace list [--project NAME] - List workspaces. --project repeatable. + kbagent workspace list [--project NAME] [--orphaned] + List workspaces. --orphaned shows only orphaned workspaces (sandboxes config missing). kbagent workspace detail --project ALIAS --workspace-id ID Workspace connection details (no password). @@ -339,6 +357,9 @@ kbagent workspace from-transformation --project ALIAS --component-id ID --config-id ID [--row-id ID] Create workspace from transformation config. Loads input tables automatically. + kbagent workspace gc [--project NAME] [--dry-run] [--yes] + Garbage-collect orphaned workspaces (keboola.sandboxes config missing). Use --dry-run to preview. + ### Project Sync kbagent sync init --project ALIAS [--directory DIR] [--git-branching] diff --git a/src/keboola_agent_cli/commands/workspace.py b/src/keboola_agent_cli/commands/workspace.py index 0be72a0b..048acdfd 100644 --- a/src/keboola_agent_cli/commands/workspace.py +++ b/src/keboola_agent_cli/commands/workspace.py @@ -7,6 +7,7 @@ from pathlib import Path import typer +from rich.markup import escape from ..errors import ConfigError, KeboolaApiError from ..output import format_query_results, format_workspaces_table @@ -115,6 +116,11 @@ def workspace_list( "--project", help="Project alias to query (can be repeated for multiple projects)", ), + orphaned: bool = typer.Option( + False, + "--orphaned", + help="Show only orphaned workspaces (keboola.sandboxes config missing)", + ), ) -> None: """List workspaces from connected projects.""" if should_hint(ctx): @@ -124,7 +130,11 @@ def workspace_list( service = get_service(ctx, "workspace_service") try: - result = service.list_workspaces(aliases=project) + result = service.list_workspaces(aliases=project, orphaned_only=orphaned) + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=exit_code) from None except ConfigError as exc: formatter.error(message=exc.message, error_code="CONFIG_ERROR") raise typer.Exit(code=5) from None @@ -420,6 +430,79 @@ def workspace_query( raise typer.Exit(code=5) from None +@workspace_app.command("gc") +def workspace_gc( + ctx: typer.Context, + project: list[str] | None = typer.Option( + None, + "--project", + help="Project alias to query (can be repeated). None = all projects.", + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="List orphaned workspaces without deleting them", + ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip confirmation prompt", + ), +) -> None: + """Garbage-collect orphaned workspaces. + + An orphaned workspace is one backed by keboola.sandboxes whose + sandbox config no longer exists. Running gc deletes those workspaces + (and any lingering sandbox configs). Use --dry-run to preview first. + """ + if should_hint(ctx): + emit_hint(ctx, "workspace.gc", project=project, dry_run=dry_run) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "workspace_service") + + if ( + not dry_run + and not yes + and not formatter.json_mode + and not typer.confirm("Delete all orphaned workspaces in the selected project(s)?") + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + + try: + result = service.gc_workspaces(aliases=project, dry_run=dry_run) + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=exit_code) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + + if formatter.json_mode: + formatter.output(result) + else: + formatter.console.print(result.get("message", "")) + if dry_run: + would_delete = result.get("would_delete", []) + for ws in would_delete: + formatter.console.print( + f" [dim]would delete[/dim] workspace {ws['id']} " + f"([cyan]{escape(ws.get('name', ''))}[/cyan]) in '{escape(ws['project_alias'])}'" + ) + else: + for ws in result.get("deleted", []): + formatter.console.print( + f" [green]deleted[/green] workspace {ws['id']} in '{escape(ws['project_alias'])}'" + ) + for err in result.get("errors", []): + formatter.console.print( + f" [red]error[/red] workspace {err.get('workspace_id', '?')}: {escape(err.get('error', ''))}" + ) + + @workspace_app.command("from-transformation") def workspace_from_transformation( ctx: typer.Context, diff --git a/src/keboola_agent_cli/hints/definitions/config.py b/src/keboola_agent_cli/hints/definitions/config.py index d019bb16..aaa4ce59 100644 --- a/src/keboola_agent_cli/hints/definitions/config.py +++ b/src/keboola_agent_cli/hints/definitions/config.py @@ -286,3 +286,193 @@ ], ) ) + +# ── config metadata-list ─────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="config.metadata-list", + description="List all metadata entries on a configuration", + steps=[ + HintStep( + comment="List configuration metadata", + client=ClientCall( + method="list_config_metadata", + args={ + "component_id": "{component_id}", + "config_id": "{config_id}", + "branch_id": "{branch}", + }, + result_var="entries", + result_hint="list[dict]", + ), + service=ServiceCall( + service_class="ConfigService", + service_module="config_service", + method="list_config_metadata", + args={ + "alias": "{project}", + "component_id": "{component_id}", + "config_id": "{config_id}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=["Each entry has: id, key, value, provider, timestamp."], + ) +) + +# ── config get-metadata ──────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="config.get-metadata", + description="Read a single metadata value by key from a configuration", + steps=[ + HintStep( + comment="Get single metadata value", + client=ClientCall( + method="list_config_metadata", + args={ + "component_id": "{component_id}", + "config_id": "{config_id}", + "branch_id": "{branch}", + }, + result_var="entries", + result_hint="list[dict]", + ), + service=ServiceCall( + service_class="ConfigService", + service_module="config_service", + method="get_config_metadata_value", + args={ + "alias": "{project}", + "component_id": "{component_id}", + "config_id": "{config_id}", + "key": "{key}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=["Raises NOT_FOUND (exit 1) if key is absent."], + ) +) + +# ── config set-metadata ──────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="config.set-metadata", + description="Set (upsert) a metadata key/value on a configuration", + steps=[ + HintStep( + comment="Upsert metadata entry on configuration", + client=ClientCall( + method="set_config_metadata", + args={ + "component_id": "{component_id}", + "config_id": "{config_id}", + "entries": "[({key}, {value})]", + "branch_id": "{branch}", + }, + result_var="result", + result_hint="list[dict]", + ), + service=ServiceCall( + service_class="ConfigService", + service_module="config_service", + method="set_config_metadata", + args={ + "alias": "{project}", + "component_id": "{component_id}", + "config_id": "{config_id}", + "key": "{key}", + "value": "{value}", + "branch_id": "{branch}", + }, + ), + ), + ], + ) +) + +# ── config delete-metadata ───────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="config.delete-metadata", + description="Delete a configuration metadata entry by its numeric ID", + steps=[ + HintStep( + comment="Delete metadata entry by ID", + client=ClientCall( + method="delete_config_metadata", + args={ + "component_id": "{component_id}", + "config_id": "{config_id}", + "metadata_id": "{metadata_id}", + "branch_id": "{branch}", + }, + result_var=None, + result_hint="None", + ), + service=ServiceCall( + service_class="ConfigService", + service_module="config_service", + method="delete_config_metadata", + args={ + "alias": "{project}", + "component_id": "{component_id}", + "config_id": "{config_id}", + "metadata_id": "{metadata_id}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=["Use metadata-list first to find the numeric metadata_id."], + ) +) + +# ── config set-folder ────────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="config.set-folder", + description="Set the folder (KBC.configuration.folderName) on a configuration", + steps=[ + HintStep( + comment="Write KBC.configuration.folderName metadata", + client=ClientCall( + method="set_config_metadata", + args={ + "component_id": "{component_id}", + "config_id": "{config_id}", + "entries": "[('KBC.configuration.folderName', {name})]", + "branch_id": "{branch}", + }, + result_var="result", + result_hint="list[dict]", + ), + service=ServiceCall( + service_class="ConfigService", + service_module="config_service", + method="set_config_folder", + args={ + "alias": "{project}", + "component_id": "{component_id}", + "config_id": "{config_id}", + "folder_name": "{name}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "Folder names appear in the Keboola UI to group configurations.", + "config list already shows folder names in the 'folder' column.", + ], + ) +) diff --git a/src/keboola_agent_cli/hints/definitions/workspace.py b/src/keboola_agent_cli/hints/definitions/workspace.py index 436ecb68..17694956 100644 --- a/src/keboola_agent_cli/hints/definitions/workspace.py +++ b/src/keboola_agent_cli/hints/definitions/workspace.py @@ -252,3 +252,34 @@ ], ) ) + +# ── workspace gc ─────────────────────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="workspace.gc", + description="Garbage-collect orphaned workspaces (keboola.sandboxes config missing)", + steps=[ + HintStep( + comment="List orphaned workspaces then delete each one", + client=ClientCall( + method="list_workspaces", + args={"branch_id": "{branch}"}, + result_var="workspaces", + result_hint="list[dict]", + ), + service=ServiceCall( + service_class="WorkspaceService", + service_module="workspace_service", + method="gc_workspaces", + args={"aliases": "{project}", "dry_run": "{dry_run}"}, + ), + ), + ], + notes=[ + "Orphan = workspace whose keboola.sandboxes config no longer exists.", + "Use --dry-run first to preview without deleting.", + "Reuses delete_workspace internally (also cleans up any lingering sandbox config).", + ], + ) +) diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 1955f61c..16a8bd83 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -33,6 +33,11 @@ "config.variables-set": "write", "config.variables-get": "read", "config.variables-clear": "destructive", + "config.metadata-list": "read", + "config.get-metadata": "read", + "config.set-metadata": "write", + "config.delete-metadata": "destructive", + "config.set-folder": "write", # Job history "job.list": "read", "job.detail": "read", @@ -72,6 +77,7 @@ "workspace.load": "write", "workspace.query": "write", "workspace.from-transformation": "write", + "workspace.gc": "destructive", # MCP tools "tool.list": "read", "tool.call": "write", diff --git a/src/keboola_agent_cli/services/config_service.py b/src/keboola_agent_cli/services/config_service.py index 6ab01835..b2f02d81 100644 --- a/src/keboola_agent_cli/services/config_service.py +++ b/src/keboola_agent_cli/services/config_service.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any -from ..errors import KeboolaApiError +from ..errors import ConfigError, KeboolaApiError from ..json_utils import compute_diff, deep_merge, set_nested_value from ..models import ProjectConfig from ..sync.manifest import Manifest, load_manifest, save_manifest @@ -642,6 +642,189 @@ def _find_sync_branch_dir(manifest: Manifest, project_root: Path) -> Path | None branch_dir = project_root / branch_path return branch_dir if branch_dir.exists() else None + def _resolve_metadata_branch_id( + self, project: ProjectConfig, client: Any, branch_id: int | None + ) -> int: + """Resolve the branch ID required by the config metadata API. + + Config metadata endpoints only support the branch-aware route + (/v2/storage/branch/{id}/...). This method resolves the effective + branch: explicit arg → active branch → default branch from API. + + Raises ConfigError if no default branch can be found. + """ + effective = branch_id or project.active_branch_id + if effective: + return int(effective) + try: + branches = client.list_dev_branches() + except KeboolaApiError as exc: + raise ConfigError( + f"Could not list branches to resolve metadata branch: {exc.message}. " + "Pass --branch explicitly." + ) from exc + except Exception as exc: + raise ConfigError( + f"Unexpected error listing branches for metadata route: {exc}. " + "Pass --branch explicitly." + ) from exc + default = next((b for b in branches if b.get("isDefault")), None) + if default: + return int(default["id"]) + raise ConfigError( + "Could not determine a branch for config metadata. " + "Set an active branch with 'kbagent branch use' or pass --branch." + ) + + def list_config_metadata( + self, + alias: str, + component_id: str, + config_id: str, + branch_id: int | None = None, + ) -> dict[str, Any]: + """List all metadata entries on a configuration. + + Returns: + Dict with project_alias, component_id, config_id, branch_id, + and a key-sorted metadata list. + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + client = self._client_factory(project.stack_url, project.token) + try: + effective_branch_id = self._resolve_metadata_branch_id(project, client, branch_id) + entries = client.list_config_metadata( + component_id, config_id, branch_id=effective_branch_id + ) + return { + "project_alias": alias, + "component_id": component_id, + "config_id": config_id, + "branch_id": effective_branch_id, + "metadata": sorted(entries, key=lambda e: e.get("key", "")), + } + finally: + client.close() + + def get_config_metadata_value( + self, + alias: str, + component_id: str, + config_id: str, + key: str, + branch_id: int | None = None, + ) -> dict[str, Any]: + """Get a single metadata value by key. + + Raises KeboolaApiError(NOT_FOUND) if the key is absent. + """ + result = self.list_config_metadata(alias, component_id, config_id, branch_id=branch_id) + for entry in result["metadata"]: + if entry.get("key") == key: + return { + "project_alias": alias, + "component_id": component_id, + "config_id": config_id, + "branch_id": result["branch_id"], + "key": key, + "value": entry.get("value"), + "metadata_id": entry.get("id"), + } + raise KeboolaApiError( + message=f"Metadata key '{key}' not found on config '{component_id}/{config_id}'.", + status_code=404, + error_code="NOT_FOUND", + retryable=False, + ) + + def set_config_metadata( + self, + alias: str, + component_id: str, + config_id: str, + key: str, + value: str, + branch_id: int | None = None, + ) -> dict[str, Any]: + """Set a single metadata key/value on a configuration (upsert).""" + projects = self.resolve_projects([alias]) + project = projects[alias] + client = self._client_factory(project.stack_url, project.token) + try: + effective_branch_id = self._resolve_metadata_branch_id(project, client, branch_id) + result = client.set_config_metadata( + component_id, config_id, entries=[(key, value)], branch_id=effective_branch_id + ) + return { + "project_alias": alias, + "component_id": component_id, + "config_id": config_id, + "branch_id": effective_branch_id, + "key": key, + "value": value, + "result": result, + "message": ( + f"Metadata '{key}' set on config '{component_id}/{config_id}' in project '{alias}'." + ), + } + finally: + client.close() + + def delete_config_metadata( + self, + alias: str, + component_id: str, + config_id: str, + metadata_id: int | str, + branch_id: int | None = None, + ) -> dict[str, Any]: + """Delete a metadata entry by its numeric ID.""" + projects = self.resolve_projects([alias]) + project = projects[alias] + client = self._client_factory(project.stack_url, project.token) + try: + effective_branch_id = self._resolve_metadata_branch_id(project, client, branch_id) + client.delete_config_metadata( + component_id, config_id, metadata_id, branch_id=effective_branch_id + ) + return { + "project_alias": alias, + "component_id": component_id, + "config_id": config_id, + "branch_id": effective_branch_id, + "metadata_id": metadata_id, + "message": ( + f"Metadata ID {metadata_id} deleted from config " + f"'{component_id}/{config_id}' in project '{alias}'." + ), + } + finally: + client.close() + + def set_config_folder( + self, + alias: str, + component_id: str, + config_id: str, + folder_name: str, + branch_id: int | None = None, + ) -> dict[str, Any]: + """Set the folder name on a configuration (KBC.configuration.folderName).""" + result = self.set_config_metadata( + alias, + component_id, + config_id, + key="KBC.configuration.folderName", + value=folder_name, + branch_id=branch_id, + ) + result["folder"] = folder_name + result["message"] = ( + f"Folder '{folder_name}' set on config '{component_id}/{config_id}' in project '{alias}'." + ) + return result + def search_configs( self, query: str, diff --git a/src/keboola_agent_cli/services/workspace_service.py b/src/keboola_agent_cli/services/workspace_service.py index 072e2650..8edc0ee4 100644 --- a/src/keboola_agent_cli/services/workspace_service.py +++ b/src/keboola_agent_cli/services/workspace_service.py @@ -15,6 +15,21 @@ logger = logging.getLogger(__name__) +def _is_orphaned_workspace(ws: dict[str, Any], config_names: dict[str, str]) -> bool: + """Return True if a workspace has no backing keboola.sandboxes config. + + A workspace is orphaned when it is tied to keboola.sandboxes (the normal + kbagent creation path) but the sandbox config no longer exists — either it + was deleted separately or was never created. + """ + component_id = ws.get("component_id", "") + config_id = str(ws.get("config_id", "")) + if component_id != "keboola.sandboxes": + return False + # config_names keys are sandbox config IDs; absence means orphan + return not config_id or config_id not in config_names + + class WorkspaceService(BaseService): """Business logic for managing Keboola workspaces. @@ -257,11 +272,14 @@ def _create_workspace_via_job( def list_workspaces( self, aliases: list[str] | None = None, + orphaned_only: bool = False, ) -> dict[str, Any]: """List workspaces across one or multiple projects. Args: aliases: Project aliases to query. None means all projects. + orphaned_only: If True, return only orphaned workspaces — those + whose keboola.sandboxes config no longer exists. Returns: Dict with "workspaces" and "errors" lists. @@ -283,20 +301,24 @@ def worker( for ws in raw_workspaces: connection = ws.get("connection", {}) config_id = ws.get("configurationId") or "" - workspaces.append( - { - "project_alias": alias, - "id": ws.get("id"), - "name": config_names.get(str(config_id), ws.get("name", "")), - "backend": connection.get("backend", ""), - "host": connection.get("host", ""), - "schema": connection.get("schema", ""), - "user": connection.get("user", ""), - "created": ws.get("created", ""), - "component_id": ws.get("component") or "", - "config_id": config_id, - } - ) + component_id = ws.get("component") or "" + entry = { + "project_alias": alias, + "id": ws.get("id"), + "name": config_names.get(str(config_id), ws.get("name", "")), + "backend": connection.get("backend", ""), + "host": connection.get("host", ""), + "schema": connection.get("schema", ""), + "user": connection.get("user", ""), + "created": ws.get("created", ""), + "component_id": component_id, + "config_id": config_id, + } + if orphaned_only: + if _is_orphaned_workspace(entry, config_names): + workspaces.append(entry) + else: + workspaces.append(entry) return (alias, workspaces, True) except KeboolaApiError as exc: return ( @@ -333,6 +355,75 @@ def worker( "errors": errors, } + def gc_workspaces( + self, + aliases: list[str] | None = None, + dry_run: bool = False, + ) -> dict[str, Any]: + """Delete all orphaned workspaces (workspace GC). + + An orphan is a keboola.sandboxes-backed workspace whose config no longer + exists. Reuses delete_workspace for each orphan so the sandbox config + cleanup path is also exercised. + + Args: + aliases: Project aliases to query. None means all projects. + dry_run: If True, list orphans without deleting. + + Returns: + Dict with dry_run flag, would_delete/deleted list, errors, count. + """ + orphan_result = self.list_workspaces(aliases=aliases, orphaned_only=True) + orphans = orphan_result["workspaces"] + list_errors = orphan_result["errors"] + + if dry_run: + return { + "dry_run": True, + "would_delete": orphans, + "count": len(orphans), + "errors": list_errors, + "message": ( + f"DRY RUN: {len(orphans)} orphaned workspace(s) would be deleted." + + (" No errors." if not list_errors else f" {len(list_errors)} list error(s).") + ), + } + + deleted: list[dict[str, Any]] = [] + delete_errors: list[dict[str, Any]] = [] + for ws in orphans: + try: + self.delete_workspace(alias=ws["project_alias"], workspace_id=ws["id"]) + deleted.append(ws) + except Exception as exc: + # Full traceback goes to the logger (observability for unexpected + # errors like AttributeError); user-facing flow is unchanged. + logger.exception( + "Failed to delete orphaned workspace %s in project %s", + ws["id"], + ws["project_alias"], + ) + delete_errors.append( + { + "workspace_id": ws["id"], + "project_alias": ws["project_alias"], + "error": str(exc), + } + ) + + all_errors = list_errors + delete_errors + return { + "dry_run": False, + "deleted": deleted, + "errors": all_errors, + "count_deleted": len(deleted), + "count_errors": len(all_errors), + "message": ( + f"GC complete: {len(deleted)} orphaned workspace(s) deleted" + + (f", {len(all_errors)} error(s)." if all_errors else ".") + ), + } + def get_workspace(self, alias: str, workspace_id: int) -> dict[str, Any]: """Get workspace details (password NOT included). @@ -384,6 +475,7 @@ def delete_workspace(self, alias: str, workspace_id: int) -> dict[str, Any]: try: # Get workspace details to find associated config config_id = None + component = None try: ws_data = client.get_workspace(workspace_id, branch_id=branch_id) component = ws_data.get("component") diff --git a/tests/test_config_metadata.py b/tests/test_config_metadata.py new file mode 100644 index 00000000..9cb85392 --- /dev/null +++ b/tests/test_config_metadata.py @@ -0,0 +1,662 @@ +"""Tests for config metadata CLI commands and service methods. + +Covers: metadata-list, get-metadata, set-metadata, delete-metadata, set-folder. +Uses mocked services (CLI layer) and mocked HTTP client (service layer). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.config_service import ConfigService +from keboola_agent_cli.services.job_service import JobService +from keboola_agent_cli.services.project_service import ProjectService +from keboola_agent_cli.services.workspace_service import WorkspaceService + +runner = CliRunner() + +TEST_TOKEN = "test-token-123" +TEST_URL = "https://connection.keboola.com" +COMP_ID = "keboola.ex-db-snowflake" +CFG_ID = "my-config" + +SAMPLE_ENTRIES = [ + { + "id": 1, + "key": "KBC.configuration.folderName", + "value": "extractors", + "provider": "user", + "timestamp": "2025-01-01T00:00:00Z", + }, + { + "id": 2, + "key": "my.custom.tag", + "value": "production", + "provider": "user", + "timestamp": "2025-01-01T00:00:00Z", + }, +] + + +def _setup_store(tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url=TEST_URL, + token=TEST_TOKEN, + project_name="Prod", + project_id=1, + ), + ) + return store + + +def _invoke(store: ConfigStore, mock_cfg_svc: MagicMock, *args: str) -> Any: + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjSvc, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgSvc, + patch("keboola_agent_cli.cli.JobService") as MockJobSvc, + patch("keboola_agent_cli.cli.WorkspaceService") as MockWsSvc, + ): + MockStore.return_value = store + MockProjSvc.return_value = ProjectService(config_store=store) + MockCfgSvc.return_value = mock_cfg_svc + MockJobSvc.return_value = JobService(config_store=store) + MockWsSvc.return_value = WorkspaceService(config_store=store) + return runner.invoke(app, list(args)) + + +# ── metadata-list ────────────────────────────────────────────────────── + + +class TestConfigMetadataList: + def test_list_json_success(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.list_config_metadata.return_value = { + "project_alias": "prod", + "component_id": COMP_ID, + "config_id": CFG_ID, + "branch_id": None, + "metadata": SAMPLE_ENTRIES, + } + result = _invoke( + store, + mock_svc, + "--json", + "config", + "metadata-list", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["status"] == "ok" + assert len(data["data"]["metadata"]) == 2 + mock_svc.list_config_metadata.assert_called_once_with( + alias="prod", component_id=COMP_ID, config_id=CFG_ID, branch_id=None + ) + + def test_list_empty(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.list_config_metadata.return_value = { + "project_alias": "prod", + "component_id": COMP_ID, + "config_id": CFG_ID, + "branch_id": None, + "metadata": [], + } + result = _invoke( + store, + mock_svc, + "--json", + "config", + "metadata-list", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["data"]["metadata"] == [] + + def test_list_api_error(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.list_config_metadata.side_effect = KeboolaApiError( + message="Not found", status_code=404, error_code="NOT_FOUND", retryable=False + ) + result = _invoke( + store, + mock_svc, + "--json", + "config", + "metadata-list", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + ) + assert result.exit_code == 1 + data = json.loads(result.output) + assert data["status"] == "error" + + def test_list_config_error_exits_5(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.list_config_metadata.side_effect = ConfigError("No active branch") + result = _invoke( + store, + mock_svc, + "--json", + "config", + "metadata-list", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + ) + assert result.exit_code == 5 + data = json.loads(result.output) + assert data["status"] == "error" + assert data["error"]["code"] == "CONFIG_ERROR" + + +# ── get-metadata ─────────────────────────────────────────────────────── + + +class TestConfigGetMetadata: + def test_get_json_success(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.get_config_metadata_value.return_value = { + "project_alias": "prod", + "component_id": COMP_ID, + "config_id": CFG_ID, + "branch_id": None, + "key": "my.custom.tag", + "value": "production", + "metadata_id": 2, + } + result = _invoke( + store, + mock_svc, + "--json", + "config", + "get-metadata", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + "--key", + "my.custom.tag", + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["data"]["value"] == "production" + assert data["data"]["metadata_id"] == 2 + + def test_get_not_found_exits_1(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.get_config_metadata_value.side_effect = KeboolaApiError( + message="Metadata key 'missing' not found.", + status_code=404, + error_code="NOT_FOUND", + retryable=False, + ) + result = _invoke( + store, + mock_svc, + "--json", + "config", + "get-metadata", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + "--key", + "missing", + ) + assert result.exit_code == 1 + data = json.loads(result.output) + assert data["error"]["code"] == "NOT_FOUND" + + def test_get_config_error_exits_5(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.get_config_metadata_value.side_effect = ConfigError("Branch resolution failed") + result = _invoke( + store, + mock_svc, + "--json", + "config", + "get-metadata", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + "--key", + "k", + ) + assert result.exit_code == 5 + data = json.loads(result.output) + assert data["error"]["code"] == "CONFIG_ERROR" + + +# ── set-metadata ─────────────────────────────────────────────────────── + + +class TestConfigSetMetadata: + def test_set_json_success(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.set_config_metadata.return_value = { + "project_alias": "prod", + "component_id": COMP_ID, + "config_id": CFG_ID, + "branch_id": None, + "key": "env", + "value": "production", + "result": [{"id": 5, "key": "env", "value": "production"}], + "message": "Metadata 'env' set on config ...", + } + result = _invoke( + store, + mock_svc, + "--json", + "config", + "set-metadata", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + "--key", + "env", + "--value", + "production", + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["status"] == "ok" + assert data["data"]["key"] == "env" + mock_svc.set_config_metadata.assert_called_once_with( + alias="prod", + component_id=COMP_ID, + config_id=CFG_ID, + key="env", + value="production", + branch_id=None, + ) + + def test_set_api_error(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.set_config_metadata.side_effect = KeboolaApiError( + message="Config not found", status_code=404, error_code="NOT_FOUND", retryable=False + ) + result = _invoke( + store, + mock_svc, + "--json", + "config", + "set-metadata", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + "bad-id", + "--key", + "k", + "--value", + "v", + ) + assert result.exit_code == 1 + + def test_set_config_error_exits_5(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.set_config_metadata.side_effect = ConfigError("Branch resolution failed") + result = _invoke( + store, + mock_svc, + "--json", + "config", + "set-metadata", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + "--key", + "k", + "--value", + "v", + ) + assert result.exit_code == 5 + data = json.loads(result.output) + assert data["error"]["code"] == "CONFIG_ERROR" + + +# ── delete-metadata ──────────────────────────────────────────────────── + + +class TestConfigDeleteMetadata: + def test_delete_with_yes_flag(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.delete_config_metadata.return_value = { + "project_alias": "prod", + "component_id": COMP_ID, + "config_id": CFG_ID, + "branch_id": None, + "metadata_id": 2, + "message": "Metadata ID 2 deleted.", + } + result = _invoke( + store, + mock_svc, + "--json", + "config", + "delete-metadata", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + "--metadata-id", + "2", + "--yes", + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["data"]["metadata_id"] == 2 + mock_svc.delete_config_metadata.assert_called_once_with( + alias="prod", + component_id=COMP_ID, + config_id=CFG_ID, + metadata_id=2, + branch_id=None, + ) + + def test_delete_config_error_exits_5(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.delete_config_metadata.side_effect = ConfigError("Branch resolution failed") + result = _invoke( + store, + mock_svc, + "--json", + "config", + "delete-metadata", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + "--metadata-id", + "1", + "--yes", + ) + assert result.exit_code == 5 + data = json.loads(result.output) + assert data["error"]["code"] == "CONFIG_ERROR" + + def test_delete_api_error(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.delete_config_metadata.side_effect = KeboolaApiError( + message="Not found", status_code=404, error_code="NOT_FOUND", retryable=False + ) + result = _invoke( + store, + mock_svc, + "--json", + "config", + "delete-metadata", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + "--metadata-id", + "999", + "--yes", + ) + assert result.exit_code == 1 + + +# ── set-folder ───────────────────────────────────────────────────────── + + +class TestConfigSetFolder: + def test_set_folder_success(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.set_config_folder.return_value = { + "project_alias": "prod", + "component_id": COMP_ID, + "config_id": CFG_ID, + "branch_id": None, + "key": "KBC.configuration.folderName", + "value": "My Folder", + "folder": "My Folder", + "result": [{"id": 3, "key": "KBC.configuration.folderName", "value": "My Folder"}], + "message": "Folder 'My Folder' set on config ...", + } + result = _invoke( + store, + mock_svc, + "--json", + "config", + "set-folder", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + "--name", + "My Folder", + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["data"]["folder"] == "My Folder" + mock_svc.set_config_folder.assert_called_once_with( + alias="prod", + component_id=COMP_ID, + config_id=CFG_ID, + folder_name="My Folder", + branch_id=None, + ) + + def test_set_folder_config_error_exits_5(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.set_config_folder.side_effect = ConfigError("Branch resolution failed") + result = _invoke( + store, + mock_svc, + "--json", + "config", + "set-folder", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + "--name", + "Bad Folder", + ) + assert result.exit_code == 5 + data = json.loads(result.output) + assert data["error"]["code"] == "CONFIG_ERROR" + + def test_set_folder_empty_clears(self, tmp_path: Path) -> None: + """Empty folder name is a valid call (clearing the folder).""" + store = _setup_store(tmp_path) + mock_svc = MagicMock() + mock_svc.set_config_folder.return_value = { + "project_alias": "prod", + "component_id": COMP_ID, + "config_id": CFG_ID, + "branch_id": None, + "folder": "", + "message": "Folder '' set.", + } + result = _invoke( + store, + mock_svc, + "--json", + "config", + "set-folder", + "--project", + "prod", + "--component-id", + COMP_ID, + "--config-id", + CFG_ID, + "--name", + "", + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["data"]["folder"] == "" + mock_svc.set_config_folder.assert_called_once_with( + alias="prod", + component_id=COMP_ID, + config_id=CFG_ID, + folder_name="", + branch_id=None, + ) + + +# ── ConfigService unit tests (mocked client) ────────────────────────── + + +class TestConfigServiceMetadata: + """Test ConfigService metadata methods with a mocked KeboolaClient.""" + + def _make_service(self, tmp_path: Path) -> tuple[ConfigService, MagicMock]: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + # active_branch_id set so _resolve_metadata_branch_id skips the API call + ProjectConfig( + stack_url=TEST_URL, + token=TEST_TOKEN, + project_name="Prod", + project_id=1, + active_branch_id=1, + ), + ) + mock_client = MagicMock() + mock_client.close = MagicMock() + svc = ConfigService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + return svc, mock_client + + def test_list_config_metadata_sorted(self, tmp_path: Path) -> None: + svc, mock_client = self._make_service(tmp_path) + mock_client.list_config_metadata.return_value = [ + {"id": 2, "key": "z.key", "value": "b"}, + {"id": 1, "key": "a.key", "value": "a"}, + ] + result = svc.list_config_metadata("prod", COMP_ID, CFG_ID) + assert result["metadata"][0]["key"] == "a.key" + assert result["metadata"][1]["key"] == "z.key" + mock_client.list_config_metadata.assert_called_once_with(COMP_ID, CFG_ID, branch_id=1) + mock_client.close.assert_called_once() + + def test_get_config_metadata_value_found(self, tmp_path: Path) -> None: + svc, mock_client = self._make_service(tmp_path) + mock_client.list_config_metadata.return_value = [ + {"id": 5, "key": "my.key", "value": "hello", "provider": "user"}, + ] + result = svc.get_config_metadata_value("prod", COMP_ID, CFG_ID, "my.key") + assert result["value"] == "hello" + assert result["metadata_id"] == 5 + + def test_get_config_metadata_value_not_found(self, tmp_path: Path) -> None: + svc, mock_client = self._make_service(tmp_path) + mock_client.list_config_metadata.return_value = [] + with pytest.raises(KeboolaApiError) as exc_info: + svc.get_config_metadata_value("prod", COMP_ID, CFG_ID, "missing") + assert exc_info.value.error_code == "NOT_FOUND" + + def test_set_config_metadata_wire_shape(self, tmp_path: Path) -> None: + svc, mock_client = self._make_service(tmp_path) + mock_client.set_config_metadata.return_value = [{"id": 7, "key": "env", "value": "prod"}] + result = svc.set_config_metadata("prod", COMP_ID, CFG_ID, key="env", value="prod") + assert result["key"] == "env" + assert result["value"] == "prod" + # Verify the client receives entries as list of tuples + mock_client.set_config_metadata.assert_called_once_with( + COMP_ID, CFG_ID, entries=[("env", "prod")], branch_id=1 + ) + + def test_delete_config_metadata(self, tmp_path: Path) -> None: + svc, mock_client = self._make_service(tmp_path) + mock_client.delete_config_metadata.return_value = None + result = svc.delete_config_metadata("prod", COMP_ID, CFG_ID, metadata_id=7) + assert result["metadata_id"] == 7 + mock_client.delete_config_metadata.assert_called_once_with(COMP_ID, CFG_ID, 7, branch_id=1) + mock_client.close.assert_called_once() + + def test_set_config_folder_uses_correct_key(self, tmp_path: Path) -> None: + svc, mock_client = self._make_service(tmp_path) + mock_client.set_config_metadata.return_value = [ + {"id": 9, "key": "KBC.configuration.folderName", "value": "My Group"} + ] + result = svc.set_config_folder("prod", COMP_ID, CFG_ID, folder_name="My Group") + assert result["folder"] == "My Group" + mock_client.set_config_metadata.assert_called_once_with( + COMP_ID, + CFG_ID, + entries=[("KBC.configuration.folderName", "My Group")], + branch_id=1, + ) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index b8b8e778..f1f1adb2 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -28,6 +28,7 @@ from __future__ import annotations +import contextlib import csv import json import os @@ -419,6 +420,9 @@ def test_full_cli_e2e(self) -> None: _step("18b", "config rename", "rename config via API") self._test_config_rename(config_id) + _step("18c", "config metadata CRUD + set-folder", "metadata round-trip") + self._test_config_metadata(config_id) + _step(19, "config new scaffold", "generate boilerplate for component") self._test_config_new_scaffold() @@ -455,6 +459,9 @@ def test_full_cli_e2e(self) -> None: _step(27, "workspace delete") self._test_workspace_delete(workspace_id) + _step("27b", "workspace list --orphaned + workspace gc", "orphan GC round-trip") + self._test_workspace_gc() + # ============================================================== # PHASE 7: Transformation job run (Snowflake SQL) # ============================================================== @@ -1280,6 +1287,137 @@ def _test_config_rename(self, config_id: str) -> None: "E2E Test Config", ) + def _test_config_metadata(self, config_id: str) -> None: + """Config metadata CRUD round-trip + set-folder sugar.""" + custom_key = f"E2E.{RUN_ID}.meta" + + # metadata-list on fresh config -- should be empty + data = self._run_ok( + "config", + "metadata-list", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + ) + initial_entries = data["data"]["metadata"] + assert isinstance(initial_entries, list) + + # set-metadata -- upsert a custom key + data = self._run_ok( + "config", + "set-metadata", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + "--key", + custom_key, + "--value", + "e2e-value", + ) + assert data["data"]["key"] == custom_key + assert data["data"]["value"] == "e2e-value" + + # get-metadata -- verify the value round-trips + data = self._run_ok( + "config", + "get-metadata", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + "--key", + custom_key, + ) + assert data["data"]["value"] == "e2e-value" + + # metadata-list -- custom key should now appear + data = self._run_ok( + "config", + "metadata-list", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + ) + entries = data["data"]["metadata"] + match = next((e for e in entries if e.get("key") == custom_key), None) + assert match is not None, f"{custom_key} not in metadata list" + metadata_id = str(match["id"]) + + # set-folder -- sugar over KBC.configuration.folderName + folder_name = f"E2E-folder-{RUN_ID}" + data = self._run_ok( + "config", + "set-folder", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + "--name", + folder_name, + ) + assert data["data"]["folder"] == folder_name + + # verify folder key is visible in list + data = self._run_ok( + "config", + "metadata-list", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + ) + folder_entry = next( + (e for e in data["data"]["metadata"] if e.get("key") == "KBC.configuration.folderName"), + None, + ) + assert folder_entry is not None + assert folder_entry["value"] == folder_name + + # delete-metadata -- remove the custom key by its ID + data = self._run_ok( + "config", + "delete-metadata", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + "--metadata-id", + metadata_id, + "--yes", + ) + assert metadata_id in data["data"]["message"] + + # verify it's gone + data = self._run_ok( + "config", + "metadata-list", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + ) + remaining_keys = {e.get("key") for e in data["data"]["metadata"]} + assert custom_key not in remaining_keys + def _test_config_new_scaffold(self) -> None: """Test config new -- generate scaffold for a component.""" scaffold_dir = self.data_dir / "scaffold" @@ -1433,6 +1571,78 @@ def _test_workspace_delete(self, workspace_id: int) -> None: assert data["status"] == "ok" self._created_workspace_ids.remove(workspace_id) + def _test_workspace_gc(self) -> None: + """workspace list --orphaned + workspace gc orphan GC round-trip. + + Creates a workspace, then manually deletes the backing sandbox config + via direct API call to manufacture an orphan, then verifies that: + - workspace list --orphaned reports it + - workspace gc --dry-run counts but does not delete + - workspace gc --yes deletes it + """ + # Create a workspace to orphan + result = self._run( + "workspace", + "create", + "--project", + self.alias, + ) + if result.exit_code != 0: + print( + f" {_YELLOW}SKIP: workspace create failed " + f"(exit {result.exit_code}), skipping workspace GC test{_RESET}" + ) + return + + data = _json_ok(result) + ws_id = data["data"]["workspace_id"] + assert ws_id > 0 + # Track for cleanup in case the test fails mid-way + self._created_workspace_ids.append(ws_id) + + # Retrieve the workspace detail to find its config_id + ws_data = self.api.get_workspace(ws_id) + config_id = str(ws_data.get("configurationId") or ws_data.get("config_id") or "") + + if not config_id: + # Cannot manufacture orphan without config_id; just clean up and skip + print(f" {_YELLOW}SKIP: workspace has no config_id, cannot manufacture orphan{_RESET}") + self.api.delete_workspace(ws_id) + self._created_workspace_ids.remove(ws_id) + return + + # Delete the sandbox config directly (bypassing workspace delete) to create an orphan + try: + self.api.delete_config("keboola.sandboxes", config_id) + except Exception as exc: + print(f" {_YELLOW}SKIP: could not delete sandbox config: {exc}{_RESET}") + self.api.delete_workspace(ws_id) + self._created_workspace_ids.remove(ws_id) + return + + # workspace list --orphaned -- the workspace should now appear + data = self._run_ok("workspace", "list", "--project", self.alias, "--orphaned") + orphan_ids = [w["id"] for w in data["data"]["workspaces"]] + assert ws_id in orphan_ids, f"ws {ws_id} not listed as orphan; got: {orphan_ids}" + + # workspace gc --dry-run -- count but do not delete + data = self._run_ok("workspace", "gc", "--project", self.alias, "--dry-run") + gc_data = data["data"] + assert gc_data["dry_run"] is True + would_delete_ids = [w["id"] for w in gc_data.get("would_delete", [])] + assert ws_id in would_delete_ids + + # workspace gc --yes -- actually delete orphans + data = self._run_ok("workspace", "gc", "--project", self.alias, "--yes") + gc_data = data["data"] + assert gc_data["dry_run"] is False + deleted_ids = [w["id"] for w in gc_data.get("deleted", [])] + assert ws_id in deleted_ids + + # workspace no longer tracked for cleanup (GC deleted it) + if ws_id in self._created_workspace_ids: + self._created_workspace_ids.remove(ws_id) + # ------------------------------------------------------------------ # Transformation job run # ------------------------------------------------------------------ @@ -3413,3 +3623,354 @@ def test_resolver_prefers_explicit_values_id_over_first_row(self) -> None: ) print(f" {_DIM}resolved={resolved} pinned={pinned_row_id} first={first_row_id}{_RESET}") assert resolved == pinned_row_id + + +# --------------------------------------------------------------------------- +# PR8: Config metadata + Workspace GC (standalone, no storage dependency) +# --------------------------------------------------------------------------- + + +@skip_without_credentials +@pytest.mark.e2e +class TestE2EPR8ConfigMetadata: + """End-to-end tests for config metadata CRUD commands (PR8). + + Creates a real keboola.ex-db-snowflake config, exercises the full + metadata round-trip (metadata-list / set-metadata / get-metadata / + delete-metadata / set-folder), then deletes the config. + """ + + @pytest.fixture(autouse=True) + def setup(self, tmp_path: Path) -> None: + self.token = os.environ[ENV_TOKEN] + raw_url = os.environ.get(ENV_URL, "connection.keboola.com") + self.url = raw_url if raw_url.startswith("https://") else f"https://{raw_url}" + self.alias = f"{RUN_ID}-meta" + + self.config_dir = tmp_path / "config" + self.config_dir.mkdir() + + self.api = KeboolaClient(self.url, self.token) + self._created_config_ids: list[tuple[str, str]] = [] + + # Register project + result = _invoke( + self.config_dir, + [ + "--json", + "project", + "add", + "--project", + self.alias, + "--url", + self.url, + "--token", + self.token, + ], + ) + assert result.exit_code == 0, f"project add failed: {result.output}" + + @pytest.fixture(autouse=True) + def cleanup(self) -> Any: + yield + for comp_id, cfg_id in self._created_config_ids: + with contextlib.suppress(Exception): + self.api.delete_config(comp_id, cfg_id) + + def _run(self, *args: str) -> Any: + return _invoke(self.config_dir, ["--json", *args]) + + def _run_ok(self, *args: str) -> dict[str, Any]: + return _json_ok(self._run(*args)) + + def test_config_metadata_crud_roundtrip(self) -> None: + """Full metadata CRUD: list (empty) → set → get → list (present) → delete → list (gone).""" + # Create a config to attach metadata to + cfg = self.api.create_config( + component_id=TEST_COMPONENT_ID, + name=f"{RUN_ID}-meta-test", + configuration={}, + description="E2E PR8 metadata test", + ) + config_id = str(cfg["id"]) + self._created_config_ids.append((TEST_COMPONENT_ID, config_id)) + + custom_key = f"E2E.PR8.{RUN_ID}" + + _step(1, "metadata-list on fresh config -- should be empty") + data = self._run_ok( + "config", + "metadata-list", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + ) + assert isinstance(data["data"]["metadata"], list) + initial_count = len(data["data"]["metadata"]) + + _step(2, "set-metadata -- upsert custom key") + data = self._run_ok( + "config", + "set-metadata", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + "--key", + custom_key, + "--value", + "pr8-value", + ) + assert data["data"]["key"] == custom_key + assert data["data"]["value"] == "pr8-value" + + _step(3, "get-metadata -- value round-trips") + data = self._run_ok( + "config", + "get-metadata", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + "--key", + custom_key, + ) + assert data["data"]["value"] == "pr8-value" + + _step(4, "metadata-list -- custom key appears") + data = self._run_ok( + "config", + "metadata-list", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + ) + entries = data["data"]["metadata"] + assert len(entries) == initial_count + 1 + match = next((e for e in entries if e.get("key") == custom_key), None) + assert match is not None + metadata_id = str(match["id"]) + + _step(5, "delete-metadata -- remove by ID") + data = self._run_ok( + "config", + "delete-metadata", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + "--metadata-id", + metadata_id, + "--yes", + ) + assert metadata_id in data["data"]["message"] + + _step(6, "metadata-list after delete -- key is gone") + data = self._run_ok( + "config", + "metadata-list", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + ) + remaining = {e.get("key") for e in data["data"]["metadata"]} + assert custom_key not in remaining + + def test_set_folder_sugar(self) -> None: + """set-folder writes KBC.configuration.folderName metadata.""" + cfg = self.api.create_config( + component_id=TEST_COMPONENT_ID, + name=f"{RUN_ID}-folder-test", + configuration={}, + description="E2E PR8 set-folder test", + ) + config_id = str(cfg["id"]) + self._created_config_ids.append((TEST_COMPONENT_ID, config_id)) + + folder_name = f"PR8-Folder-{RUN_ID}" + + _step(1, "set-folder -- write KBC.configuration.folderName") + data = self._run_ok( + "config", + "set-folder", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + "--name", + folder_name, + ) + assert data["data"]["folder"] == folder_name + assert data["data"]["key"] == "KBC.configuration.folderName" + + _step(2, "metadata-list -- folder key is visible") + data = self._run_ok( + "config", + "metadata-list", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + ) + folder_entry = next( + (e for e in data["data"]["metadata"] if e.get("key") == "KBC.configuration.folderName"), + None, + ) + assert folder_entry is not None + assert folder_entry["value"] == folder_name + + def test_get_metadata_missing_key_exits_1(self) -> None: + """get-metadata for a non-existent key returns exit code 1.""" + cfg = self.api.create_config( + component_id=TEST_COMPONENT_ID, + name=f"{RUN_ID}-meta-missing", + configuration={}, + description="E2E PR8 missing key test", + ) + config_id = str(cfg["id"]) + self._created_config_ids.append((TEST_COMPONENT_ID, config_id)) + + result = self._run( + "config", + "get-metadata", + "--project", + self.alias, + "--component-id", + TEST_COMPONENT_ID, + "--config-id", + config_id, + "--key", + "does.not.exist", + ) + assert result.exit_code == 1 + data = json.loads(result.output) + assert data["status"] == "error" + + +@skip_without_credentials +@pytest.mark.e2e +class TestE2EPR8WorkspaceGC: + """End-to-end tests for workspace list --orphaned and workspace gc (PR8). + + Creates a real workspace, deletes its backing sandbox config via direct API + call to manufacture an orphan, then verifies the GC commands detect and + remove it. + """ + + @pytest.fixture(autouse=True) + def setup(self, tmp_path: Path) -> None: + self.token = os.environ[ENV_TOKEN] + raw_url = os.environ.get(ENV_URL, "connection.keboola.com") + self.url = raw_url if raw_url.startswith("https://") else f"https://{raw_url}" + self.alias = f"{RUN_ID}-gc" + + self.config_dir = tmp_path / "config" + self.config_dir.mkdir() + + self.api = KeboolaClient(self.url, self.token) + self._created_workspace_ids: list[int] = [] + + # Register project + result = _invoke( + self.config_dir, + [ + "--json", + "project", + "add", + "--project", + self.alias, + "--url", + self.url, + "--token", + self.token, + ], + ) + assert result.exit_code == 0, f"project add failed: {result.output}" + + @pytest.fixture(autouse=True) + def cleanup(self) -> Any: + yield + for ws_id in self._created_workspace_ids: + with contextlib.suppress(Exception): + self.api.delete_workspace(ws_id) + + def _run(self, *args: str) -> Any: + return _invoke(self.config_dir, ["--json", *args]) + + def _run_ok(self, *args: str) -> dict[str, Any]: + return _json_ok(self._run(*args)) + + def test_workspace_gc_orphan_roundtrip(self) -> None: + """Create workspace, orphan it by deleting sandbox config, verify GC finds and removes it.""" + _step(1, "workspace create") + result = self._run("workspace", "create", "--project", self.alias) + if result.exit_code != 0: + pytest.skip(f"workspace create not supported: {result.output}") + + data = _json_ok(result) + ws_id = data["data"]["workspace_id"] + assert ws_id > 0 + self._created_workspace_ids.append(ws_id) + + _step(2, "retrieve workspace to find sandbox config_id") + ws_data = self.api.get_workspace(ws_id) + config_id = str(ws_data.get("configurationId") or ws_data.get("config_id") or "") + if not config_id: + pytest.skip("workspace has no configurationId, cannot manufacture orphan") + + _step(3, "delete sandbox config to make the workspace orphaned") + try: + self.api.delete_config("keboola.sandboxes", config_id) + except Exception as exc: + pytest.skip(f"could not delete sandbox config: {exc}") + + _step(4, "workspace list --orphaned -- workspace should appear") + data = self._run_ok("workspace", "list", "--project", self.alias, "--orphaned") + orphan_ids = [w["id"] for w in data["data"]["workspaces"]] + assert ws_id in orphan_ids, f"ws {ws_id} not listed as orphan; got: {orphan_ids}" + + _step(5, "workspace gc --dry-run -- counts but does not delete") + data = self._run_ok("workspace", "gc", "--project", self.alias, "--dry-run") + gc_data = data["data"] + assert gc_data["dry_run"] is True + would_delete_ids = [w["id"] for w in gc_data.get("would_delete", [])] + assert ws_id in would_delete_ids + + # Verify workspace still exists after dry-run + remaining = self._run_ok("workspace", "list", "--project", self.alias, "--orphaned") + assert ws_id in [w["id"] for w in remaining["data"]["workspaces"]] + + _step(6, "workspace gc --yes -- deletes the orphan") + data = self._run_ok("workspace", "gc", "--project", self.alias, "--yes") + gc_data = data["data"] + assert gc_data["dry_run"] is False + deleted_ids = [w["id"] for w in gc_data.get("deleted", [])] + assert ws_id in deleted_ids + + # Remove from cleanup tracker since GC deleted it + if ws_id in self._created_workspace_ids: + self._created_workspace_ids.remove(ws_id) + + _step(7, "workspace list --orphaned -- workspace is gone") + data = self._run_ok("workspace", "list", "--project", self.alias, "--orphaned") + remaining_ids = [w["id"] for w in data["data"]["workspaces"]] + assert ws_id not in remaining_ids diff --git a/tests/test_workspace_gc.py b/tests/test_workspace_gc.py new file mode 100644 index 00000000..8a3b0f73 --- /dev/null +++ b/tests/test_workspace_gc.py @@ -0,0 +1,382 @@ +"""Tests for workspace --orphaned flag and workspace gc command. + +Covers: + - workspace list --orphaned (service-level filtering + CLI) + - workspace gc --dry-run (preview without deletion) + - workspace gc (delete orphans, error accumulation) + - _is_orphaned_workspace helper +""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.config_service import ConfigService +from keboola_agent_cli.services.job_service import JobService +from keboola_agent_cli.services.project_service import ProjectService +from keboola_agent_cli.services.workspace_service import WorkspaceService, _is_orphaned_workspace + +runner = CliRunner() + +TEST_TOKEN = "test-token-456" +TEST_URL = "https://connection.keboola.com" + + +def _setup_store(tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url=TEST_URL, + token=TEST_TOKEN, + project_name="Prod", + project_id=1, + ), + ) + return store + + +def _invoke(store: ConfigStore, mock_ws_svc: MagicMock, *args: str, input: str | None = None): + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjSvc, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgSvc, + patch("keboola_agent_cli.cli.JobService") as MockJobSvc, + patch("keboola_agent_cli.cli.WorkspaceService") as MockWsSvc, + ): + MockStore.return_value = store + MockProjSvc.return_value = ProjectService(config_store=store) + MockCfgSvc.return_value = ConfigService(config_store=store) + MockJobSvc.return_value = JobService(config_store=store) + MockWsSvc.return_value = mock_ws_svc + return runner.invoke(app, list(args), input=input) + + +# ── _is_orphaned_workspace unit tests ───────────────────────────────── + + +class TestIsOrphanedWorkspace: + def test_non_sandboxes_component_never_orphan(self) -> None: + ws = {"component_id": "keboola.snowflake-transformation", "config_id": "cfg-1"} + assert not _is_orphaned_workspace(ws, {"cfg-1": "My Config"}) + + def test_sandboxes_with_existing_config_not_orphan(self) -> None: + ws = {"component_id": "keboola.sandboxes", "config_id": "cfg-1"} + assert not _is_orphaned_workspace(ws, {"cfg-1": "My Workspace"}) + + def test_sandboxes_with_missing_config_is_orphan(self) -> None: + ws = {"component_id": "keboola.sandboxes", "config_id": "cfg-missing"} + assert _is_orphaned_workspace(ws, {"cfg-other": "Other"}) + + def test_sandboxes_with_empty_config_id_is_orphan(self) -> None: + ws = {"component_id": "keboola.sandboxes", "config_id": ""} + assert _is_orphaned_workspace(ws, {"cfg-1": "My Workspace"}) + + def test_sandboxes_with_no_config_id_key_is_orphan(self) -> None: + ws = {"component_id": "keboola.sandboxes"} + assert _is_orphaned_workspace(ws, {"cfg-1": "My Workspace"}) + + +# ── workspace list --orphaned CLI tests ─────────────────────────────── + + +class TestWorkspaceListOrphaned: + def test_list_orphaned_json(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_ws = MagicMock() + orphan_ws = { + "project_alias": "prod", + "id": 99, + "name": "orphan-ws", + "backend": "snowflake", + "host": "host.snowflake.com", + "schema": "WORKSPACE_99", + "user": "u", + "created": "2025-01-01", + "component_id": "keboola.sandboxes", + "config_id": "", + } + mock_ws.list_workspaces.return_value = { + "workspaces": [orphan_ws], + "errors": [], + } + result = _invoke( + store, + mock_ws, + "--json", + "workspace", + "list", + "--project", + "prod", + "--orphaned", + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert len(data["data"]["workspaces"]) == 1 + # Verify service was called with orphaned_only=True + mock_ws.list_workspaces.assert_called_once_with(aliases=["prod"], orphaned_only=True) + + def test_list_without_orphaned_flag(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_ws = MagicMock() + mock_ws.list_workspaces.return_value = {"workspaces": [], "errors": []} + result = _invoke( + store, + mock_ws, + "--json", + "workspace", + "list", + "--project", + "prod", + ) + assert result.exit_code == 0, result.output + # orphaned_only=False (default) when flag absent + mock_ws.list_workspaces.assert_called_once_with(aliases=["prod"], orphaned_only=False) + + +# ── workspace gc CLI tests ───────────────────────────────────────────── + + +class TestWorkspaceGc: + def test_gc_dry_run_json(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_ws = MagicMock() + mock_ws.gc_workspaces.return_value = { + "dry_run": True, + "would_delete": [{"id": 5, "project_alias": "prod", "name": "orphan"}], + "count": 1, + "errors": [], + "message": "DRY RUN: 1 orphaned workspace(s) would be deleted.", + } + result = _invoke( + store, + mock_ws, + "--json", + "workspace", + "gc", + "--project", + "prod", + "--dry-run", + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["data"]["dry_run"] is True + assert data["data"]["count"] == 1 + mock_ws.gc_workspaces.assert_called_once_with(aliases=["prod"], dry_run=True) + + def test_gc_delete_with_yes_flag(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_ws = MagicMock() + mock_ws.gc_workspaces.return_value = { + "dry_run": False, + "deleted": [{"id": 5, "project_alias": "prod"}], + "errors": [], + "count_deleted": 1, + "count_errors": 0, + "message": "GC complete: 1 orphaned workspace(s) deleted.", + } + result = _invoke( + store, + mock_ws, + "--json", + "workspace", + "gc", + "--project", + "prod", + "--yes", + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["data"]["count_deleted"] == 1 + assert data["data"]["dry_run"] is False + + def test_gc_no_confirmation_aborts(self, tmp_path: Path) -> None: + """In non-JSON mode without --yes, answering 'n' exits 0 without calling service.""" + store = _setup_store(tmp_path) + mock_ws = MagicMock() + result = _invoke( + store, + mock_ws, + "workspace", + "gc", + "--project", + "prod", + input="n\n", + ) + assert result.exit_code == 0, result.output + assert "Aborted" in result.output + mock_ws.gc_workspaces.assert_not_called() + + def test_gc_nothing_to_delete(self, tmp_path: Path) -> None: + store = _setup_store(tmp_path) + mock_ws = MagicMock() + mock_ws.gc_workspaces.return_value = { + "dry_run": False, + "deleted": [], + "errors": [], + "count_deleted": 0, + "count_errors": 0, + "message": "GC complete: 0 orphaned workspace(s) deleted.", + } + result = _invoke( + store, + mock_ws, + "--json", + "workspace", + "gc", + "--project", + "prod", + "--yes", + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data["data"]["count_deleted"] == 0 + + +# ── WorkspaceService.gc_workspaces unit tests ────────────────────────── + + +class TestWorkspaceServiceGc: + """Test gc_workspaces service method with a mocked client.""" + + def _make_service(self, tmp_path: Path) -> tuple[WorkspaceService, MagicMock]: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig(stack_url=TEST_URL, token=TEST_TOKEN, project_name="Prod", project_id=1), + ) + mock_client = MagicMock() + mock_client.close = MagicMock() + svc = WorkspaceService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + return svc, mock_client + + def _setup_client_for_list( + self, + mock_client: MagicMock, + raw_workspaces: list[dict], + sandbox_configs: list[dict], + ) -> None: + # list_dev_branches → branch_id + mock_client.list_dev_branches.return_value = [{"id": 100, "isDefault": True}] + mock_client.list_workspaces.return_value = raw_workspaces + mock_client.list_component_configs.return_value = sandbox_configs + + def test_gc_dry_run_returns_orphans_without_deleting(self, tmp_path: Path) -> None: + svc, mock_client = self._make_service(tmp_path) + orphan_ws = { + "id": 77, + "name": "WORKSPACE_77", + "component": "keboola.sandboxes", + "configurationId": "orphan-cfg", + "connection": {"backend": "snowflake"}, + "created": "2025-01-01", + } + self._setup_client_for_list( + mock_client, + raw_workspaces=[orphan_ws], + sandbox_configs=[], # no sandbox configs → orphan + ) + result = svc.gc_workspaces(aliases=["prod"], dry_run=True) + assert result["dry_run"] is True + assert result["count"] == 1 + assert result["would_delete"][0]["id"] == 77 + # delete_workspace should NOT have been called + mock_client.delete_workspace.assert_not_called() + + def test_gc_deletes_orphan_and_sandbox_config(self, tmp_path: Path) -> None: + svc, mock_client = self._make_service(tmp_path) + orphan_ws = { + "id": 99, + "name": "WORKSPACE_99", + "component": "keboola.sandboxes", + "configurationId": "orphan-cfg", + "connection": {"backend": "snowflake"}, + "created": "2025-01-01", + } + self._setup_client_for_list( + mock_client, + raw_workspaces=[orphan_ws], + sandbox_configs=[], + ) + # delete_workspace in service also calls get_workspace then delete + mock_client.get_workspace.return_value = { + "id": 99, + "component": "keboola.sandboxes", + "configurationId": "orphan-cfg", + "connection": {}, + } + mock_client.delete_workspace.return_value = None + mock_client.delete_config.return_value = None + + result = svc.gc_workspaces(aliases=["prod"], dry_run=False) + assert result["dry_run"] is False + assert result["count_deleted"] == 1 + assert result["count_errors"] == 0 + mock_client.delete_workspace.assert_called_once_with(99, branch_id=100) + mock_client.delete_config.assert_called_once_with( + "keboola.sandboxes", "orphan-cfg", branch_id=100 + ) + + def test_gc_skips_non_sandbox_workspaces(self, tmp_path: Path) -> None: + svc, mock_client = self._make_service(tmp_path) + # Transformation workspace — should NOT be considered orphaned + tfm_ws = { + "id": 55, + "name": "WORKSPACE_55", + "component": "keboola.snowflake-transformation", + "configurationId": "tfm-cfg", + "connection": {"backend": "snowflake"}, + "created": "2025-01-01", + } + self._setup_client_for_list( + mock_client, + raw_workspaces=[tfm_ws], + sandbox_configs=[], + ) + result = svc.gc_workspaces(aliases=["prod"], dry_run=True) + assert result["count"] == 0 + assert result["would_delete"] == [] + + def test_gc_delete_error_accumulated(self, tmp_path: Path) -> None: + svc, mock_client = self._make_service(tmp_path) + orphan_ws = { + "id": 77, + "name": "WORKSPACE_77", + "component": "keboola.sandboxes", + "configurationId": "orphan-cfg", + "connection": {"backend": "snowflake"}, + "created": "2025-01-01", + } + self._setup_client_for_list( + mock_client, + raw_workspaces=[orphan_ws], + sandbox_configs=[], + ) + mock_client.get_workspace.return_value = { + "id": 77, + "component": "keboola.sandboxes", + "configurationId": "orphan-cfg", + "connection": {}, + } + mock_client.delete_workspace.side_effect = KeboolaApiError( + message="Delete failed", status_code=500, error_code="INTERNAL_ERROR", retryable=True + ) + result = svc.gc_workspaces(aliases=["prod"], dry_run=False) + # Error should be accumulated, not raised + assert result["count_deleted"] == 0 + assert result["count_errors"] == 1 diff --git a/uv.lock b/uv.lock index 400d03e1..576ed251 100644 --- a/uv.lock +++ b/uv.lock @@ -439,7 +439,7 @@ wheels = [ [[package]] name = "keboola-agent-cli" -version = "0.21.1" +version = "0.22.0" source = { editable = "." } dependencies = [ { name = "httpx" },