diff --git a/CLAUDE.md b/CLAUDE.md index 9d3103b0..732fa004 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -270,6 +270,10 @@ kbagent storage download-table --project NAME --table-id ID [--output FILE] [--c kbagent storage delete-table --project NAME --table-id ID [--table-id ...] [--force] [--dry-run] [--yes] [--branch ID] kbagent storage delete-column --project NAME --table-id ID --column COL [--column ...] [--force] [--dry-run] [--yes] [--branch ID] kbagent storage delete-bucket --project NAME --bucket-id ID [--bucket-id ...] [--force] [--dry-run] [--yes] [--branch ID] +kbagent storage describe-bucket --project NAME --bucket-id ID [--text STR | --file PATH | --stdin] [--branch ID] +kbagent storage describe-table --project NAME --table-id ID [--text STR | --file PATH | --stdin] [--branch ID] +kbagent storage describe-column --project NAME --table-id ID --column NAME=DESC [--column ...] [--branch ID] +kbagent storage describe-batch --project NAME --from-file YAML [--branch ID] kbagent storage files --project NAME [--tag TAG ...] [--limit N] [--offset N] [--query Q] [--branch ID] kbagent storage file-upload --project NAME --file PATH [--name NAME] [--tag TAG ...] [--permanent] [--branch ID] kbagent storage file-download --project NAME [--file-id ID | --tag TAG ...] [--output FILE] diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index 8ec26950..d4cd469c 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.21.1", + "version": "0.22.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 3050395d..5d393836 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -111,6 +111,10 @@ When working inside a git repository or project directory, run `kbagent init` (o | Delete one or more storage tables | `kbagent storage delete-table --project PROJECT --table-id TABLE-ID` | | Delete one or more columns from a storage table | `kbagent storage delete-column --project PROJECT --table-id TABLE-ID --column COLUMN` | | Delete one or more storage buckets | `kbagent storage delete-bucket --project PROJECT --bucket-id BUCKET-ID` | +| Set the description on a storage bucket | `kbagent storage describe-bucket --project PROJECT --bucket-id BUCKET-ID` | +| Set the description on a storage table | `kbagent storage describe-table --project PROJECT --table-id TABLE-ID` | +| Set descriptions on one or more columns of a storage table | `kbagent storage describe-column --project PROJECT --table-id TABLE-ID --column COLUMN` | +| Apply descriptions to buckets, tables, and columns from a YAML file | `kbagent storage describe-batch --project PROJECT --from-file FROM-FILE` | | List Storage Files with optional tag filtering | `kbagent storage files --project PROJECT` | | Show Storage File metadata (without downloading) | `kbagent storage file-detail --project PROJECT --file-id FILE-ID` | | Upload a local file to Storage Files | `kbagent storage file-upload --project PROJECT --file FILE` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 06ed9679..73acc939 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -57,6 +57,10 @@ All commands support `--json` for structured output. Multi-project flags (`--pro - `storage delete-table --project NAME --table-id ID [--table-id ...] [--force] [--dry-run] [--yes] [--branch ID]` -- delete tables, --force cascade-deletes aliased tables (branch-aware) - `storage delete-column --project NAME --table-id ID --column COL [--column ...] [--force] [--dry-run] [--yes] [--branch ID]` -- delete columns from a table (branch-aware) - `storage delete-bucket --project NAME --bucket-id ID [--bucket-id ...] [--force] [--dry-run] [--yes] [--branch ID]` -- delete buckets (branch-aware) +- `storage describe-bucket --project NAME --bucket-id ID [--text STR | --file PATH | --stdin] [--branch ID]` -- set a bucket description (stored as `KBC.description` in bucket metadata, upsert). Provide exactly one of `--text`, `--file`, `--stdin`. Read back via `storage bucket-detail` +- `storage describe-table --project NAME --table-id ID [--text STR | --file PATH | --stdin] [--branch ID]` -- set a table description (stored as `KBC.description` in table metadata, upsert). Provide exactly one of `--text`, `--file`, `--stdin`. Read back via `storage table-detail` +- `storage describe-column --project NAME --table-id ID --column NAME=DESCRIPTION [--column ...] [--branch ID]` -- set one or more column descriptions. Stored as `KBC.column.{name}.description` keys in the table's metadata (Keboola has no user-writable column-metadata endpoint). Read back in `storage table-detail` under `column_details[].description` +- `storage describe-batch --project NAME --from-file PATH [--branch ID]` -- apply bucket/table/column descriptions from a YAML file (top-level `buckets`, `tables`, `columns` sections, all optional). Partial-failure tolerant: per-item errors are collected and reported, the batch does not abort. Non-zero exit only when at least one item failed ## Storage Files - `storage files --project NAME [--tag TAG ...] [--limit N] [--offset N] [--query Q] [--branch ID]` -- list Storage Files, optionally filtered by tag/query diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index c0d7b08d..ccbe0a5d 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -450,6 +450,35 @@ They live at different endpoints in the Storage API (`/v2/storage/branch/{id}/metadata` vs. `/v2/storage/dev-branches/{id}`), so setting a branch's description will **not** update the dashboard. +## Storage descriptions: key convention + precedence + partial failures + +`kbagent storage describe-bucket / describe-table / describe-column / describe-batch` +write descriptive metadata onto storage objects. Three behaviors are easy to miss: + +- **Column descriptions use a metadata-key convention, not a column endpoint.** + The Keboola Storage API has no user-writable column-level metadata endpoint, + so `describe-column` stores each description as a `KBC.column.{name}.description` + entry on the **table's** metadata (upsert). `storage table-detail` reads them + back via the same key and surfaces them under `column_details[].description`. + Renaming or deleting a column does NOT automatically clean these entries up + (they remain on the table's metadata under the old name). Same convention for + table and bucket descriptions: stored as `KBC.description` (provider=user) on + the object's metadata. +- **`describe-batch` is partial-failure-tolerant.** Item-level errors are + collected into `result.errors[]` but the batch keeps processing the remaining + items. The CLI exits non-zero only if `error_count > 0`, so in scripts always + inspect `errors[]` (or at least `error_count`) rather than relying solely on + the exit code — and when consuming `--json` output, never trust a zero-exit + as "everything applied." +- **Description-field precedence: metadata wins.** When both the native Storage + API `description` field and a user-provided `KBC.description` (provider=user) + metadata entry are present, `storage bucket-detail` / `storage table-detail` + surface the **metadata value**. The native field is only settable at object + creation time via the Storage API; all user updates flow through the metadata + endpoint, so the metadata entry is the authoritative source. `KBC.description` + entries whose provider is not `user` (e.g. `system`) are ignored during + read-back and the native field is used as fallback. + ## `job terminate` quirks Queue API's kill endpoint (`POST /jobs/{id}/kill`) has a few non-obvious behaviors the diff --git a/plugins/kbagent/skills/kbagent/references/storage-describe-workflow.md b/plugins/kbagent/skills/kbagent/references/storage-describe-workflow.md new file mode 100644 index 00000000..e4044464 --- /dev/null +++ b/plugins/kbagent/skills/kbagent/references/storage-describe-workflow.md @@ -0,0 +1,258 @@ +# Storage Describe Workflow + +`kbagent storage describe-*` attaches human-readable descriptions to storage +buckets, tables, and columns so that downstream consumers (dashboards, the +MCP `get_buckets`/`get_tables` tools, AI agents) can surface meaningful +documentation rather than raw IDs. Descriptions are stored as metadata on +the storage object and round-trip via `storage bucket-detail` / `storage +table-detail`. + +## Quick reference + +| Command | Purpose | +|---------|---------| +| `storage describe-bucket` | Set a bucket description | +| `storage describe-table` | Set a table description | +| `storage describe-column` | Set descriptions on one or more columns | +| `storage describe-batch` | Apply bucket/table/column descriptions from a YAML file | +| `storage bucket-detail` | Read back the bucket description | +| `storage table-detail` | Read back the table description and `column_details[].description` | + +## When to use + +- Onboarding a new project: document every source bucket, output table, and + business-critical column so new engineers (or Kai) can self-serve. +- After a schema migration: refresh column descriptions so SQL reviews can + spot intent mismatches. +- Before sharing a bucket cross-project: the description is visible in the + receiving project's dashboard. +- From CI: write a batch YAML alongside the repo and call `describe-batch` + after every `sync push` to keep documentation in lockstep with config. + +## Storage model (what actually gets written) + +Descriptions are stored as metadata entries on the object: + +- **Bucket description** -- `KBC.description` (provider=user) on bucket metadata +- **Table description** -- `KBC.description` (provider=user) on table metadata +- **Column description** -- `KBC.column.{column_name}.description` on the + **table's** metadata. Keboola has no user-writable column-metadata endpoint, + so this key convention is the storage layer for column descriptions. Read + them back via `storage table-detail` (`column_details[].description`). + +Descriptions are `upsert`: calling `describe-*` with a new text replaces +whatever was there before. There is no append mode. + +## Single-item: bucket + +```bash +# Inline text +kbagent --json storage describe-bucket \ + --project ALIAS \ + --bucket-id in.c-sales \ + --text "Daily sales fact data, partitioned by region" + +# From a file (markdown supported) +kbagent --json storage describe-bucket \ + --project ALIAS \ + --bucket-id in.c-sales \ + --file ./docs/sales-bucket.md + +# From stdin (useful in pipelines) +echo "Generated description" | kbagent --json storage describe-bucket \ + --project ALIAS \ + --bucket-id in.c-sales \ + --stdin +``` + +Exactly one of `--text`, `--file`, `--stdin` must be provided. + +Read back: + +```bash +kbagent --json storage bucket-detail --project ALIAS --bucket-id in.c-sales \ + | jq '.data.description, .data.metadata' +``` + +## Single-item: table + +Identical shape to `describe-bucket`: + +```bash +kbagent --json storage describe-table \ + --project ALIAS \ + --table-id in.c-sales.orders \ + --text "All sales orders, one row per line item" +``` + +Read back: + +```bash +kbagent --json storage table-detail --project ALIAS --table-id in.c-sales.orders \ + | jq '.data.description, .data.column_details' +``` + +## Single-item: columns + +`describe-column` takes **one or more** `--column NAME=DESCRIPTION` flags in +a single call. All entries are applied in one API roundtrip: + +```bash +kbagent --json storage describe-column \ + --project ALIAS \ + --table-id in.c-sales.orders \ + --column "order_id=Unique order identifier" \ + --column "total=Order total in USD (gross)" \ + --column "created_at=Server-side creation timestamp (UTC)" +``` + +Column descriptions live under `KBC.column.{name}.description` on the +**table's** metadata -- they are NOT attached to the column record itself. +If you rename or delete a column, the old key lingers until you manually +clean it up (there is no `--delete-column-description` command today). + +Read back via `storage table-detail`: + +```json +{ + "data": { + "table_id": "in.c-sales.orders", + "description": "All sales orders, one row per line item", + "column_details": [ + {"name": "order_id", "type": "INTEGER", "description": "Unique order identifier"}, + {"name": "total", "type": "NUMERIC", "description": "Order total in USD (gross)"} + ] + } +} +``` + +Columns without a matching metadata entry simply omit `description`. + +## Batch: YAML schema + +For more than a handful of items, hand-maintain a YAML file and apply it +with `storage describe-batch`. The schema has three top-level sections, +all optional: + +```yaml +# descriptions.yaml +buckets: + in.c-sales: | + Sales fact and dimension tables. + Refreshed nightly from the production OLTP via Keboola ex-db-postgres. + in.c-marketing: Marketing funnel events + +tables: + in.c-sales.orders: All sales orders (one row per line item) + in.c-sales.customers: Customer master list, PII-scrubbed + in.c-marketing.events: Raw funnel events + +columns: + in.c-sales.orders: + order_id: Unique order identifier + total: Order total in USD (gross) + created_at: Server-side creation timestamp (UTC) + in.c-sales.customers: + customer_id: Primary key + email_hash: SHA-256 of the customer email (PII-scrubbed) +``` + +Apply it: + +```bash +kbagent --json storage describe-batch \ + --project ALIAS \ + --from-file ./descriptions.yaml +``` + +Response shape: + +```json +{ + "status": "ok", + "data": { + "project_alias": "ALIAS", + "applied": [ + {"type": "bucket", "id": "in.c-sales", "description": "Sales fact..."}, + {"type": "table", "id": "in.c-sales.orders", "description": "All sales orders..."}, + {"type": "columns", "id": "in.c-sales.orders", "columns": {"order_id": "...", "total": "..."}} + ], + "errors": [], + "applied_count": 3, + "error_count": 0 + } +} +``` + +In human mode, a Rich progress spinner shows per-item progress ("Describing +bucket in.c-sales", "Describing table in.c-sales.orders", ...) so large +batches do not look frozen. The spinner is suppressed under `--json` so +structured output is the only thing on stdout. + +## Partial-failure semantics + +`describe-batch` does **not** abort on the first error. Each item is +attempted independently; failures are collected into `errors[]` and the +batch continues: + +```json +{ + "data": { + "applied": [{"type": "bucket", "id": "in.c-good", ...}], + "errors": [ + {"type": "bucket", "id": "in.c-typo", "error": "Bucket in.c-typo not found"}, + {"type": "table", "id": "in.c-x.missing", "error": "Table not found"} + ], + "applied_count": 1, + "error_count": 2 + } +} +``` + +The CLI exits **1** when `error_count > 0`. In scripts, always inspect the +`errors[]` list -- a zero exit alone does not mean the whole batch went in +without issues (it means there were no partial failures). A non-zero exit +means *some* items failed; the successful items still landed. + +## End-to-end example: onboarding a new bucket + +```bash +# 1. Create the bucket and tables (or sync them from another project) +kbagent storage create-bucket --project ALIAS --stage in --name c-sales +kbagent storage create-table --project ALIAS --bucket-id in.c-sales --name orders \ + --column order_id:INTEGER --column total:NUMERIC --primary-key order_id + +# 2. Apply all descriptions from a tracked YAML file +kbagent --json storage describe-batch \ + --project ALIAS \ + --from-file ./docs/keboola/descriptions.yaml + +# 3. Verify by reading back +kbagent --json storage table-detail --project ALIAS --table-id in.c-sales.orders \ + | jq '{description: .data.description, columns: .data.column_details}' +``` + +## Precedence vs the native description field + +The Storage API has a native `description` field on buckets and tables, but +it is only settable at creation time. Anything you set with `describe-*` +lives on the metadata endpoint. When both are present, `storage bucket-detail` +/ `storage table-detail` surface the metadata value (the one you wrote with +`describe-*`). The native field is the fallback for legacy objects where +no metadata entry exists. System-provided `KBC.description` entries (e.g. +those auto-stamped by components) are filtered out on read-back -- only +entries with `provider="user"` are considered the canonical description. + +## Key behaviors + +- `describe-*` is **upsert** -- no append mode; re-running replaces the value. +- Column descriptions piggy-back on table metadata via the + `KBC.column.{name}.description` key convention. +- `describe-batch` is **partial-failure-tolerant** -- check `errors[]` even + on exit code 0. +- All commands support `--branch ID` to target a dev branch. +- Read back via `storage bucket-detail` / `storage table-detail` -- the + `metadata` field on those responses contains the raw metadata array if + you need to inspect timestamps or providers. +- Non-user (`system`) `KBC.description` entries are ignored on read-back; + they do not override the native `description` field. diff --git a/pyproject.toml b/pyproject.toml index 0eee9ad7..d622abf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-agent-cli" -version = "0.21.1" +version = "0.22.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 29f65c04..ccdec319 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -8,6 +8,14 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.22.0": [ + "New: storage describe-bucket -- set KBC.description on a bucket via metadata POST (upsert-by-key, provider=user)", + "New: storage describe-table -- set KBC.description on a table via metadata POST; description surfaces in table-detail", + "New: storage describe-column -- set per-column descriptions using KBC.column.{name}.description convention in table metadata; readable via table-detail column_details[].description", + "New: storage describe-batch --from-file -- apply bucket/table/column descriptions from a YAML file in one shot; failures collected, remaining items continue", + "Fix: storage table-detail now returns 'description' and 'metadata' fields (extracted from table metadata array)", + "Fix: storage bucket-detail now returns 'description' and 'metadata' fields (KBC.description in metadata takes precedence over native creation-time description field)", + ], "0.21.1": [ "Fix: sync pull on a newly created dev branch now writes config rows (#193) -- idempotent skip guard for rows was missing a file-existence check, causing rows to be silently skipped when the branch directory was new (hash matched main because the branch is a clone)", ], diff --git a/src/keboola_agent_cli/client.py b/src/keboola_agent_cli/client.py index d92b07e3..a8a0dc2a 100644 --- a/src/keboola_agent_cli/client.py +++ b/src/keboola_agent_cli/client.py @@ -695,6 +695,69 @@ def list_buckets_with_metadata(self) -> list[dict[str, Any]]: """ return self.list_buckets(include="metadata") + def set_bucket_metadata( + self, + bucket_id: str, + entries: list[tuple[str, str]], + branch_id: int | None = None, + ) -> list[dict[str, Any]]: + """Upsert metadata key/value pairs on a storage bucket. + + POST /v2/storage/buckets/{id}/metadata + + Uses the same PHP-style array form encoding as ``set_branch_metadata``. + Provider is always ``"user"`` for CLI-originated descriptions. + + Args: + bucket_id: Bucket ID (e.g. 'in.c-db'). + entries: Ordered list of ``(key, value)`` metadata tuples. + branch_id: If set, target a specific dev branch. + + Returns: + Full metadata list for the bucket after the upsert. + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + safe_id = quote(bucket_id, safe="") + form: dict[str, str] = {"provider": "user"} + for i, (key, value) in enumerate(entries): + form[f"metadata[{i}][key]"] = key + form[f"metadata[{i}][value]"] = value + response = self._request("POST", f"{prefix}/buckets/{safe_id}/metadata", data=form) + return response.json() + + def set_table_metadata( + self, + table_id: str, + entries: list[tuple[str, str]], + branch_id: int | None = None, + ) -> list[dict[str, Any]]: + """Upsert metadata key/value pairs on a storage table. + + POST /v2/storage/tables/{id}/metadata + + Provider is always ``"user"`` for CLI-originated descriptions. + Column-level descriptions use the namespaced key convention + ``KBC.column.{colname}.description`` stored at table-metadata level + (Keboola Storage API does not expose a user-writable column-metadata + endpoint; ``columnMetadata`` is populated exclusively by components). + + Args: + table_id: Full table ID (e.g. "in.c-bucket.table"). + entries: Ordered list of ``(key, value)`` metadata tuples. + branch_id: If set, target a specific dev branch. + + Returns: + Full metadata list for the table after the upsert. + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + safe_id = quote(table_id, safe="") + form: dict[str, str] = {"provider": "user"} + for i, (key, value) in enumerate(entries): + form[f"metadata[{i}][key]"] = key + form[f"metadata[{i}][value]"] = value + response = self._request("POST", f"{prefix}/tables/{safe_id}/metadata", data=form) + return response.json() + def get_bucket_detail( self, bucket_id: str, diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index cd32ec30..9755c96d 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -193,6 +193,22 @@ kbagent storage delete-bucket --project NAME --bucket-id ID [--bucket-id ...] [--force] [--dry-run] [--yes] [--branch ID] Delete one or more buckets. --force cascade-deletes tables. Linked/shared buckets protected. Branch-aware. +### Storage Descriptions + + kbagent storage describe-bucket --project NAME --bucket-id ID [--text STR | --file PATH | --stdin] [--branch ID] + Set the KBC.description metadata on a bucket (upsert). Visible in bucket-detail. + + kbagent storage describe-table --project NAME --table-id ID [--text STR | --file PATH | --stdin] [--branch ID] + Set the KBC.description metadata on a table (upsert). Readable via table-detail --json .data.description. + + kbagent storage describe-column --project NAME --table-id ID --column NAME=DESC [--column ...] [--branch ID] + Set per-column descriptions stored as KBC.column.{{name}}.description in table metadata (upsert). + Readable via table-detail --json .data.column_details[].description. + + kbagent storage describe-batch --project NAME --from-file YAML [--branch ID] + Apply bucket/table/column descriptions from a YAML file. Sections: buckets, tables, columns (all optional). + Failures collected; one error does not abort remaining items. + ### Storage Files kbagent storage files --project NAME [--tag TAG ...] [--limit N] [--offset N] [--query Q] [--branch ID] diff --git a/src/keboola_agent_cli/commands/storage.py b/src/keboola_agent_cli/commands/storage.py index 837c9519..ecc6ff02 100644 --- a/src/keboola_agent_cli/commands/storage.py +++ b/src/keboola_agent_cli/commands/storage.py @@ -5,6 +5,7 @@ """ from pathlib import Path +from typing import Any import typer @@ -1079,6 +1080,401 @@ def storage_delete_bucket( raise typer.Exit(code=1) +# ------------------------------------------------------------------ +# Describe (metadata write) commands +# ------------------------------------------------------------------ + +_DESCRIBE = "Descriptions" + + +@storage_app.command("describe-bucket", rich_help_panel=_DESCRIBE) +def storage_describe_bucket( + ctx: typer.Context, + project: str = typer.Option( + ..., + "--project", + help="Project alias", + ), + bucket_id: str = typer.Option( + ..., + "--bucket-id", + help="Bucket ID (e.g. 'in.c-my-bucket')", + ), + text: str | None = typer.Option( + None, + "--text", + help="Description text (inline)", + ), + file: Path | None = typer.Option( + None, + "--file", + help="Path to a file containing the description", + ), + stdin: bool = typer.Option( + False, + "--stdin", + help="Read description from standard input", + ), + branch: int | None = typer.Option( + None, + "--branch", + help="Dev branch ID (defaults to active branch if set via 'branch use')", + ), +) -> None: + """Set the description on a storage bucket. + + Stores the description as KBC.description in bucket metadata (upsert). + Provide the text via --text, --file, or --stdin (exactly one required). + """ + if should_hint(ctx): + emit_hint( + ctx, "storage.describe-bucket", project=project, bucket_id=bucket_id, branch=branch + ) + + formatter = get_formatter(ctx) + service = get_service(ctx, "storage_service") + config_store: ConfigStore = ctx.obj["config_store"] + _, effective_branch = resolve_branch(config_store, formatter, project, branch) + + from ._metadata_input import resolve_text_input + + try: + description = resolve_text_input(text=text, file=file, stdin=stdin) + except ConfigError as exc: + formatter.error(message=exc.message, error_code="INVALID_ARGUMENT") + raise typer.Exit(code=2) from None + + try: + result = service.describe_bucket( + alias=project, + bucket_id=bucket_id, + description=description, + branch_id=effective_branch, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + else: + formatter.console.print(f"[bold green]Description set:[/bold green] {bucket_id}") + formatter.console.print(f" {description[:120]}") + + +@storage_app.command("describe-table", rich_help_panel=_DESCRIBE) +def storage_describe_table( + ctx: typer.Context, + project: str = typer.Option( + ..., + "--project", + help="Project alias", + ), + table_id: str = typer.Option( + ..., + "--table-id", + help="Table ID (e.g. 'in.c-my-bucket.my-table')", + ), + text: str | None = typer.Option( + None, + "--text", + help="Description text (inline)", + ), + file: Path | None = typer.Option( + None, + "--file", + help="Path to a file containing the description", + ), + stdin: bool = typer.Option( + False, + "--stdin", + help="Read description from standard input", + ), + branch: int | None = typer.Option( + None, + "--branch", + help="Dev branch ID (defaults to active branch if set via 'branch use')", + ), +) -> None: + """Set the description on a storage table. + + Stores the description as KBC.description in table metadata (upsert). + Provide the text via --text, --file, or --stdin (exactly one required). + """ + if should_hint(ctx): + emit_hint(ctx, "storage.describe-table", project=project, table_id=table_id, branch=branch) + + formatter = get_formatter(ctx) + service = get_service(ctx, "storage_service") + config_store: ConfigStore = ctx.obj["config_store"] + _, effective_branch = resolve_branch(config_store, formatter, project, branch) + + from ._metadata_input import resolve_text_input + + try: + description = resolve_text_input(text=text, file=file, stdin=stdin) + except ConfigError as exc: + formatter.error(message=exc.message, error_code="INVALID_ARGUMENT") + raise typer.Exit(code=2) from None + + try: + result = service.describe_table( + alias=project, + table_id=table_id, + description=description, + branch_id=effective_branch, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + else: + formatter.console.print(f"[bold green]Description set:[/bold green] {table_id}") + formatter.console.print(f" {description[:120]}") + + +@storage_app.command("describe-column", rich_help_panel=_DESCRIBE) +def storage_describe_column( + ctx: typer.Context, + project: str = typer.Option( + ..., + "--project", + help="Project alias", + ), + table_id: str = typer.Option( + ..., + "--table-id", + help="Table ID (e.g. 'in.c-my-bucket.my-table')", + ), + column: list[str] = typer.Option( + ..., + "--column", + help="Column description as 'NAME=DESCRIPTION' (can be repeated)", + ), + branch: int | None = typer.Option( + None, + "--branch", + help="Dev branch ID (defaults to active branch if set via 'branch use')", + ), +) -> None: + """Set descriptions on one or more columns of a storage table. + + Descriptions are stored as KBC.column.{name}.description keys in table + metadata (upsert). Keboola Storage does not expose a user-writable + column-level metadata endpoint; this convention lets you annotate columns + and read them back via 'storage table-detail'. + + Example: + + kbagent storage describe-column \\ + --project myproj \\ + --table-id in.c-bucket.orders \\ + --column order_id="Unique order identifier" \\ + --column total="Order total in USD" + """ + if should_hint(ctx): + emit_hint(ctx, "storage.describe-column", project=project, table_id=table_id, branch=branch) + + formatter = get_formatter(ctx) + service = get_service(ctx, "storage_service") + config_store: ConfigStore = ctx.obj["config_store"] + _, effective_branch = resolve_branch(config_store, formatter, project, branch) + + parsed: dict[str, str] = {} + for entry in column: + if "=" not in entry: + formatter.error( + message=f"--column must be NAME=DESCRIPTION, got: {entry!r}", + error_code="INVALID_ARGUMENT", + ) + raise typer.Exit(code=2) from None + name, _, desc = entry.partition("=") + name = name.strip() + if not name: + formatter.error( + message=f"Column name cannot be empty in: {entry!r}", + error_code="INVALID_ARGUMENT", + ) + raise typer.Exit(code=2) from None + parsed[name] = desc + + try: + result = service.describe_columns( + alias=project, + table_id=table_id, + columns=parsed, + branch_id=effective_branch, + ) + except ValueError as exc: + formatter.error(message=str(exc), error_code="INVALID_ARGUMENT") + raise typer.Exit(code=2) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + else: + formatter.console.print( + f"[bold green]Column descriptions set:[/bold green] {table_id} " + f"({len(parsed)} column(s))" + ) + for name, desc in parsed.items(): + formatter.console.print(f" {name}: {desc[:80]}") + + +@storage_app.command("describe-batch", rich_help_panel=_DESCRIBE) +def storage_describe_batch( + ctx: typer.Context, + project: str = typer.Option( + ..., + "--project", + help="Project alias", + ), + from_file: Path = typer.Option( + ..., + "--from-file", + help="Path to a YAML file with bucket/table/column descriptions", + ), + branch: int | None = typer.Option( + None, + "--branch", + help="Dev branch ID (defaults to active branch if set via 'branch use')", + ), +) -> None: + """Apply descriptions to buckets, tables, and columns from a YAML file. + + YAML schema: + + buckets: + in.c-my-bucket: "Bucket description" + + tables: + in.c-my-bucket.my-table: "Table description" + + columns: + in.c-my-bucket.my-table: + col1: "Column 1 description" + col2: "Column 2 description" + + All sections are optional. A failure in one item does not abort the + rest -- all results are collected and reported. + """ + if should_hint(ctx): + emit_hint( + ctx, "storage.describe-batch", project=project, from_file=from_file, branch=branch + ) + + formatter = get_formatter(ctx) + service = get_service(ctx, "storage_service") + config_store: ConfigStore = ctx.obj["config_store"] + _, effective_branch = resolve_branch(config_store, formatter, project, branch) + + # In human mode, show a live progress indicator so that large batches + # (100+ items) do not look frozen. JSON mode must remain silent on stderr + # so structured output is the only thing on stdout. + progress_cm: Any = None + progress_task: Any = None + progress_callback = None + if not formatter.json_mode: + from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, + ) + + progress_cm = Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + MofNCompleteColumn(), + TextColumn("•"), + TimeElapsedColumn(), + console=formatter.console, + transient=True, + ) + + def _on_item(obj_type: str, obj_id: str, current: int, total: int) -> None: + # Guard against progress_task/progress_cm not being ready yet. + if progress_task is None or progress_cm is None: + return + # total is known up-front (passed the first time), but re-setting + # is a no-op after the first call. + progress_cm.update( + progress_task, + total=total, + completed=max(current - 1, 0), + description=f"Describing {obj_type} {obj_id}", + ) + + progress_callback = _on_item + + try: + if progress_cm is not None: + progress_cm.start() + progress_task = progress_cm.add_task("Applying descriptions...", total=None) + result = service.describe_batch( + alias=project, + from_file=from_file, + branch_id=effective_branch, + progress_callback=progress_callback, + ) + if progress_cm is not None and progress_task is not None: + # Mark the task complete so the final render shows N / N. + progress_cm.update( + progress_task, + completed=result["applied_count"] + result["error_count"], + ) + except ValueError as exc: + formatter.error(message=str(exc), error_code="INVALID_ARGUMENT") + raise typer.Exit(code=2) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + finally: + if progress_cm is not None: + # .stop() is idempotent; safe for both happy and error paths. + progress_cm.stop() + + if formatter.json_mode: + formatter.output(result) + else: + applied = result["applied_count"] + errors = result["error_count"] + formatter.console.print( + f"[bold green]Batch complete:[/bold green] {applied} applied, {errors} error(s)" + ) + for item in result["applied"]: + obj_type = item["type"] + obj_id = item["id"] + if obj_type == "columns": + n = len(item.get("columns", {})) + formatter.console.print(f" [green]✓[/green] {obj_type} {obj_id} ({n} cols)") + else: + formatter.console.print(f" [green]✓[/green] {obj_type} {obj_id}") + for item in result["errors"]: + formatter.console.print(f" [red]✗[/red] {item['type']} {item['id']}: {item['error']}") + if errors: + raise typer.Exit(code=1) from None + + # ------------------------------------------------------------------ # File operations # ------------------------------------------------------------------ diff --git a/src/keboola_agent_cli/hints/definitions/storage.py b/src/keboola_agent_cli/hints/definitions/storage.py index 0e4400cb..2b20d200 100644 --- a/src/keboola_agent_cli/hints/definitions/storage.py +++ b/src/keboola_agent_cli/hints/definitions/storage.py @@ -681,3 +681,157 @@ ], ) ) + +# ── storage describe-bucket ──────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="storage.describe-bucket", + description="Set the description on a storage bucket", + steps=[ + HintStep( + comment="Upsert KBC.description in bucket metadata (provider='user')", + client=ClientCall( + method="set_bucket_metadata", + args={ + "bucket_id": "{bucket_id}", + "entries": '[("KBC.description", "{description}")]', + "branch_id": "{branch}", + }, + result_var="result", + result_hint="list[dict]", + ), + service=ServiceCall( + service_class="StorageService", + service_module="storage_service", + method="describe_bucket", + args={ + "alias": "{project}", + "bucket_id": "{bucket_id}", + "description": "{description}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "POST /v2/storage/buckets/{id}/metadata with provider='user' is an upsert-by-key.", + "Description is readable via 'storage bucket-detail --json .data.description'.", + ], + ) +) + +# ── storage describe-table ──────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="storage.describe-table", + description="Set the description on a storage table", + steps=[ + HintStep( + comment="Upsert KBC.description in table metadata (provider='user')", + client=ClientCall( + method="set_table_metadata", + args={ + "table_id": "{table_id}", + "entries": '[("KBC.description", "{description}")]', + "branch_id": "{branch}", + }, + result_var="result", + result_hint="list[dict]", + ), + service=ServiceCall( + service_class="StorageService", + service_module="storage_service", + method="describe_table", + args={ + "alias": "{project}", + "table_id": "{table_id}", + "description": "{description}", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "Description is readable via 'storage table-detail --json | .data.description'.", + ], + ) +) + +# ── storage describe-column ─────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="storage.describe-column", + description="Set per-column descriptions on a storage table", + steps=[ + HintStep( + comment="Store column descriptions as KBC.column.{name}.description in table metadata", + client=ClientCall( + method="set_table_metadata", + args={ + "table_id": "{table_id}", + "entries": '[("KBC.column.{col}.description", "{description}")]', + "branch_id": "{branch}", + }, + result_var="result", + result_hint="list[dict]", + ), + service=ServiceCall( + service_class="StorageService", + service_module="storage_service", + method="describe_columns", + args={ + "alias": "{project}", + "table_id": "{table_id}", + "columns": '{"{col}": "{description}"}', + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "Column descriptions use key KBC.column.{name}.description in table metadata.", + "They appear under column_details[].description in 'storage table-detail --json'.", + "Keboola does not provide a user-writable column-metadata endpoint; this is the supported convention.", + ], + ) +) + +# ── storage describe-batch ──────────────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="storage.describe-batch", + description="Apply bucket/table/column descriptions from a YAML file", + steps=[ + HintStep( + comment="Load YAML and apply descriptions to all listed assets", + client=ClientCall( + method="set_bucket_metadata / set_table_metadata", + args={ + "from_file": "{from_file}", + "branch_id": "{branch}", + }, + result_var="result", + result_hint="dict", + ), + service=ServiceCall( + service_class="StorageService", + service_module="storage_service", + method="describe_batch", + args={ + "alias": "{project}", + "from_file": "Path('{from_file}')", + "branch_id": "{branch}", + }, + ), + ), + ], + notes=[ + "YAML sections: 'buckets', 'tables', 'columns' (all optional).", + "Failures are collected -- one error does not abort the rest.", + ], + ) +) diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 1955f61c..7ed7dc23 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -106,6 +106,11 @@ "storage.delete-column": "destructive", "storage.delete-bucket": "destructive", "storage.file-delete": "destructive", + # Storage descriptions + "storage.describe-bucket": "write", + "storage.describe-table": "write", + "storage.describe-column": "write", + "storage.describe-batch": "write", # Encryption "encrypt.values": "write", # Sync / git workflow diff --git a/src/keboola_agent_cli/services/storage_service.py b/src/keboola_agent_cli/services/storage_service.py index 67a506f3..d3456c43 100644 --- a/src/keboola_agent_cli/services/storage_service.py +++ b/src/keboola_agent_cli/services/storage_service.py @@ -6,6 +6,7 @@ import csv import logging +from collections.abc import Callable from pathlib import Path from typing import Any @@ -172,15 +173,27 @@ def get_bucket_detail( project_id = token_info.project_id source = bucket.get("sourceBucket") + # KBC.description in the metadata array takes precedence over the + # native description field (which can only be set at creation time + # via the Storage API; user updates go through the metadata endpoint). + raw_metadata: list[dict[str, Any]] = bucket.get("metadata", []) + metadata_description = "" + for m in raw_metadata: + if m.get("key") == "KBC.description" and m.get("provider") == "user": + metadata_description = m.get("value", "") or "" + break + description = metadata_description or bucket.get("description", "") + result: dict[str, Any] = { "project_alias": alias, "project_id": project_id, "bucket_id": bucket.get("id", ""), "display_name": bucket.get("displayName", ""), "stage": bucket.get("stage", ""), - "description": bucket.get("description", ""), + "description": description, "backend": bucket.get("backend", ""), "is_linked": source is not None, + "metadata": raw_metadata, } # Resolve Snowflake paths using backendPath from API (preserves correct case). @@ -260,6 +273,18 @@ def get_table_detail( columns = table.get("columns", []) column_metadata = table.get("columnMetadata", {}) + raw_metadata: list[dict[str, Any]] = table.get("metadata", []) + + # Extract description and per-column descriptions from metadata list + description = "" + col_descriptions: dict[str, str] = {} + for m in raw_metadata: + key = m.get("key", "") + if key == "KBC.description" and m.get("provider") == "user": + description = m.get("value", "") or "" + elif key.startswith("KBC.column.") and key.endswith(".description"): + col_name = key[len("KBC.column.") : -len(".description")] + col_descriptions[col_name] = m.get("value", "") or "" column_details = [] for col in columns: @@ -270,6 +295,8 @@ def get_table_detail( col_info["type"] = m.get("value", "") elif m.get("key") == "KBC.datatype.nullable": col_info["nullable"] = m.get("value", "") == "1" + if col in col_descriptions: + col_info["description"] = col_descriptions[col] column_details.append(col_info) return { @@ -278,6 +305,7 @@ def get_table_detail( "name": table.get("name", ""), "display_name": table.get("displayName", ""), "bucket_id": table.get("bucket", {}).get("id", ""), + "description": description, "columns": columns, "column_details": column_details, "primary_key": table.get("primaryKey", []), @@ -287,6 +315,7 @@ def get_table_detail( "last_import_date": table.get("lastImportDate", ""), "last_change_date": table.get("lastChangeDate", ""), "created": table.get("created", ""), + "metadata": raw_metadata, } def list_tables( @@ -1445,6 +1474,253 @@ def unload_table_to_file( return result + # ------------------------------------------------------------------ + # Describe (metadata write) methods + # ------------------------------------------------------------------ + + def describe_bucket( + self, + alias: str, + bucket_id: str, + description: str, + branch_id: int | None = None, + ) -> dict[str, Any]: + """Set the KBC.description metadata on a storage bucket. + + Idempotent upsert: re-running with a different value overwrites the + existing entry (Keboola metadata POST is upsert-by-key). + + Args: + alias: Project alias. + bucket_id: Bucket ID (e.g. 'in.c-my-bucket'). + description: Human-readable description text. + branch_id: If set, target a specific dev branch. + + Returns: + Dict with project_alias, bucket_id, description, result, message. + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + client = self._client_factory(project.stack_url, project.token) + try: + result = client.set_bucket_metadata( + bucket_id=bucket_id, + entries=[("KBC.description", description)], + branch_id=branch_id, + ) + finally: + client.close() + return { + "project_alias": alias, + "bucket_id": bucket_id, + "description": description, + "result": result, + "message": f"Description set on bucket '{bucket_id}' in project '{alias}'.", + } + + def describe_table( + self, + alias: str, + table_id: str, + description: str, + branch_id: int | None = None, + ) -> dict[str, Any]: + """Set the KBC.description metadata on a storage table. + + Idempotent upsert: re-running with a different value overwrites. + + Args: + alias: Project alias. + table_id: Full table ID (e.g. 'in.c-bucket.table'). + description: Human-readable description text. + branch_id: If set, target a specific dev branch. + + Returns: + Dict with project_alias, table_id, description, result, message. + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + client = self._client_factory(project.stack_url, project.token) + try: + result = client.set_table_metadata( + table_id=table_id, + entries=[("KBC.description", description)], + branch_id=branch_id, + ) + finally: + client.close() + return { + "project_alias": alias, + "table_id": table_id, + "description": description, + "result": result, + "message": f"Description set on table '{table_id}' in project '{alias}'.", + } + + def describe_columns( + self, + alias: str, + table_id: str, + columns: dict[str, str], + branch_id: int | None = None, + ) -> dict[str, Any]: + """Set per-column descriptions on a storage table. + + Column descriptions are stored as namespaced table metadata using the + key convention ``KBC.column.{colname}.description``. Keboola's + Storage API does not provide a user-writable column-level metadata + endpoint (``columnMetadata`` is populated exclusively by processing + components); this convention is the supported alternative for + annotating columns from the CLI. + + Args: + alias: Project alias. + table_id: Full table ID. + columns: Mapping of column name -> description text. + branch_id: If set, target a specific dev branch. + + Returns: + Dict with project_alias, table_id, columns dict, result, message. + """ + if not columns: + raise ValueError("At least one column description must be provided.") + projects = self.resolve_projects([alias]) + project = projects[alias] + entries = [(f"KBC.column.{name}.description", desc) for name, desc in columns.items()] + client = self._client_factory(project.stack_url, project.token) + try: + result = client.set_table_metadata( + table_id=table_id, + entries=entries, + branch_id=branch_id, + ) + finally: + client.close() + return { + "project_alias": alias, + "table_id": table_id, + "columns": columns, + "result": result, + "message": ( + f"Descriptions set for {len(columns)} column(s) on table '{table_id}' " + f"in project '{alias}'." + ), + } + + def describe_batch( + self, + alias: str, + from_file: Path, + branch_id: int | None = None, + progress_callback: Callable[[str, str, int, int], None] | None = None, + ) -> dict[str, Any]: + """Apply bucket, table, and column descriptions from a YAML file. + + YAML schema:: + + buckets: + in.c-my-bucket: "Bucket description" + tables: + in.c-my-bucket.my-table: "Table description" + columns: + in.c-my-bucket.my-table: + col1: "Column 1 description" + col2: "Column 2 description" + + All sections are optional; empty or absent sections are silently + skipped. Within each section the operations are applied in order. + A failure in one item does not skip remaining items — all results + (success and error) are collected and returned. + + Args: + alias: Project alias. + from_file: Path to a YAML file with the schema above. + branch_id: If set, target a specific dev branch. + progress_callback: Optional ``(obj_type, obj_id, current, total)`` + callable invoked **before** each item is processed. ``obj_type`` + is ``"bucket"``, ``"table"``, or ``"columns"``; ``current`` is + 1-based; ``total`` is the total number of items across all + sections. Used by the CLI to render a Rich progress indicator + in human mode; JSON mode leaves it unset. + + Returns: + Dict with project_alias, applied, errors, applied_count, error_count. + """ + import yaml + + from ..errors import KeboolaApiError + + if not from_file.is_file(): + raise ValueError(f"Batch file not found: {from_file}") + raw = yaml.safe_load(from_file.read_text(encoding="utf-8")) or {} + if not isinstance(raw, dict): + raise ValueError("Batch file must be a YAML mapping.") + + applied: list[dict[str, Any]] = [] + errors: list[dict[str, Any]] = [] + + buckets: dict[str, str] = raw.get("buckets") or {} + tables: dict[str, str] = raw.get("tables") or {} + columns: dict[str, dict[str, str]] = raw.get("columns") or {} + + total = len(buckets) + len(tables) + len(columns) + current = 0 + + for bucket_id, desc in buckets.items(): + current += 1 + if progress_callback is not None: + progress_callback("bucket", bucket_id, current, total) + try: + self.describe_bucket(alias, bucket_id, str(desc), branch_id=branch_id) + applied.append({"type": "bucket", "id": bucket_id, "description": desc}) + logger.debug("describe_batch bucket %s: ok", bucket_id) + except Exception as exc: + msg = exc.message if isinstance(exc, KeboolaApiError) else str(exc) + errors.append({"type": "bucket", "id": bucket_id, "error": msg}) + + for table_id, desc in tables.items(): + current += 1 + if progress_callback is not None: + progress_callback("table", table_id, current, total) + try: + self.describe_table(alias, table_id, str(desc), branch_id=branch_id) + applied.append({"type": "table", "id": table_id, "description": desc}) + except Exception as exc: + msg = exc.message if isinstance(exc, KeboolaApiError) else str(exc) + errors.append({"type": "table", "id": table_id, "error": msg}) + + for table_id, col_map in columns.items(): + current += 1 + if progress_callback is not None: + progress_callback("columns", table_id, current, total) + if not isinstance(col_map, dict): + errors.append( + {"type": "columns", "id": table_id, "error": "columns entry must be a mapping"} + ) + continue + try: + self.describe_columns( + alias, table_id, {k: str(v) for k, v in col_map.items()}, branch_id=branch_id + ) + applied.append( + { + "type": "columns", + "id": table_id, + "columns": {k: str(v) for k, v in col_map.items()}, + } + ) + except Exception as exc: + msg = exc.message if isinstance(exc, KeboolaApiError) else str(exc) + errors.append({"type": "columns", "id": table_id, "error": msg}) + + return { + "project_alias": alias, + "applied": applied, + "errors": errors, + "applied_count": len(applied), + "error_count": len(errors), + } + # ------------------------------------------------------------------ # Parallel workers # ------------------------------------------------------------------ diff --git a/tests/test_client.py b/tests/test_client.py index 75638e50..87446e82 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -2653,3 +2653,154 @@ def test_get_branch_metadata_value_missing(self, httpx_mock) -> None: with KeboolaClient(stack_url=_BASE, token=_TOKEN) as client: value = client.get_branch_metadata_value(key="KBC.projectDescription") assert value is METADATA_NOT_FOUND + + +# --------------------------------------------------------------------------- +# Storage object metadata (bucket + table) +# --------------------------------------------------------------------------- + +_STORAGE_META_RESPONSE = [ + { + "id": "9001", + "key": "KBC.description", + "value": "A test description", + "provider": "user", + "timestamp": "2026-04-22T10:00:00+0200", + }, +] + + +class TestSetBucketMetadata: + """Tests for set_bucket_metadata() - POST /v2/storage/buckets/{id}/metadata.""" + + def test_set_bucket_metadata_php_form_body(self, httpx_mock) -> None: + """Encodes provider + PHP-array indices in the form body.""" + from urllib.parse import quote as url_quote + + safe_id = url_quote("in.c-my-bucket", safe="") + httpx_mock.add_response( + url=f"{_BASE}/v2/storage/buckets/{safe_id}/metadata", + method="POST", + json=_STORAGE_META_RESPONSE, + status_code=201, + ) + with KeboolaClient(stack_url=_BASE, token=_TOKEN) as client: + result = client.set_bucket_metadata( + bucket_id="in.c-my-bucket", + entries=[("KBC.description", "A test description")], + ) + + assert result == _STORAGE_META_RESPONSE + req = httpx_mock.get_request() + body = req.content.decode().replace("%5B", "[").replace("%5D", "]") + assert "provider=user" in body + assert "metadata[0][key]=KBC.description" in body + assert ( + "metadata[0][value]=A+test+description" in body + or "A%20test%20description" in body + or "A test description" in body + ) + assert req.headers["content-type"].startswith("application/x-www-form-urlencoded") + + def test_set_bucket_metadata_with_branch(self, httpx_mock) -> None: + """Uses branch prefix when branch_id is provided.""" + from urllib.parse import quote as url_quote + + safe_id = url_quote("in.c-my-bucket", safe="") + httpx_mock.add_response( + url=f"{_BASE}/v2/storage/branch/42/buckets/{safe_id}/metadata", + method="POST", + json=_STORAGE_META_RESPONSE, + status_code=201, + ) + with KeboolaClient(stack_url=_BASE, token=_TOKEN) as client: + result = client.set_bucket_metadata( + bucket_id="in.c-my-bucket", + entries=[("KBC.description", "Branch desc")], + branch_id=42, + ) + assert result == _STORAGE_META_RESPONSE + + def test_set_bucket_metadata_multiple_entries(self, httpx_mock) -> None: + """Multiple entries get sequential PHP indices.""" + from urllib.parse import quote as url_quote + + safe_id = url_quote("in.c-bucket", safe="") + httpx_mock.add_response( + url=f"{_BASE}/v2/storage/buckets/{safe_id}/metadata", + method="POST", + json=_STORAGE_META_RESPONSE, + status_code=201, + ) + with KeboolaClient(stack_url=_BASE, token=_TOKEN) as client: + client.set_bucket_metadata( + bucket_id="in.c-bucket", + entries=[("k1", "v1"), ("k2", "v2")], + ) + body = httpx_mock.get_request().content.decode().replace("%5B", "[").replace("%5D", "]") + assert "metadata[0][key]=k1" in body + assert "metadata[1][key]=k2" in body + + +class TestSetTableMetadata: + """Tests for set_table_metadata() - POST /v2/storage/tables/{id}/metadata.""" + + def test_set_table_metadata_php_form_body(self, httpx_mock) -> None: + """Encodes provider + PHP-array indices for a table metadata POST.""" + from urllib.parse import quote as url_quote + + safe_id = url_quote("in.c-b.tbl", safe="") + httpx_mock.add_response( + url=f"{_BASE}/v2/storage/tables/{safe_id}/metadata", + method="POST", + json=_STORAGE_META_RESPONSE, + status_code=201, + ) + with KeboolaClient(stack_url=_BASE, token=_TOKEN) as client: + result = client.set_table_metadata( + table_id="in.c-b.tbl", + entries=[("KBC.description", "A test description")], + ) + assert result == _STORAGE_META_RESPONSE + req = httpx_mock.get_request() + body = req.content.decode().replace("%5B", "[").replace("%5D", "]") + assert "provider=user" in body + assert "metadata[0][key]=KBC.description" in body + + def test_set_table_metadata_with_branch(self, httpx_mock) -> None: + """Uses branch prefix when branch_id is provided.""" + from urllib.parse import quote as url_quote + + safe_id = url_quote("in.c-b.tbl", safe="") + httpx_mock.add_response( + url=f"{_BASE}/v2/storage/branch/7/tables/{safe_id}/metadata", + method="POST", + json=_STORAGE_META_RESPONSE, + status_code=201, + ) + with KeboolaClient(stack_url=_BASE, token=_TOKEN) as client: + result = client.set_table_metadata( + table_id="in.c-b.tbl", + entries=[("KBC.description", "Branch desc")], + branch_id=7, + ) + assert result == _STORAGE_META_RESPONSE + + def test_set_table_metadata_column_convention(self, httpx_mock) -> None: + """Column descriptions use KBC.column.{name}.description key convention.""" + from urllib.parse import quote as url_quote + + safe_id = url_quote("in.c-b.tbl", safe="") + httpx_mock.add_response( + url=f"{_BASE}/v2/storage/tables/{safe_id}/metadata", + method="POST", + json=_STORAGE_META_RESPONSE, + status_code=201, + ) + with KeboolaClient(stack_url=_BASE, token=_TOKEN) as client: + client.set_table_metadata( + table_id="in.c-b.tbl", + entries=[("KBC.column.city.description", "City name")], + ) + body = httpx_mock.get_request().content.decode().replace("%5B", "[").replace("%5D", "]") + assert "KBC.column.city.description" in body diff --git a/tests/test_e2e.py b/tests/test_e2e.py index b8b8e778..c08749e0 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -400,6 +400,9 @@ def test_full_cli_e2e(self) -> None: _step(14.1, "storage unload-table --file-type parquet", "Parquet export + sliced download") self._test_unload_table_parquet(table_id) + _step(14.2, "storage describe-bucket/table/column/batch", "description metadata round-trip") + self._test_storage_describe(bucket_id, table_id) + _step(15, "storage load-file", "upload CSV as file then load into table") self._test_load_file(table_id) @@ -2382,6 +2385,102 @@ def _test_storage_cleanup(self, bucket_id: str, table_id: str) -> None: assert bucket_id in data["data"]["deleted"] self._created_buckets.remove(bucket_id) + def _test_storage_describe(self, bucket_id: str, table_id: str) -> None: + """Round-trip describe commands: write description, read it back.""" + # describe-bucket: set KBC.description, verify via bucket-detail + data = self._run_ok( + "storage", + "describe-bucket", + "--project", + self.alias, + "--bucket-id", + bucket_id, + "--text", + "E2E bucket description", + ) + assert data["data"]["bucket_id"] == bucket_id + assert data["data"]["description"] == "E2E bucket description" + + data = self._run_ok( + "storage", "bucket-detail", "--project", self.alias, "--bucket-id", bucket_id + ) + assert data["data"]["description"] == "E2E bucket description" + + # describe-table: set KBC.description, verify via table-detail + data = self._run_ok( + "storage", + "describe-table", + "--project", + self.alias, + "--table-id", + table_id, + "--text", + "E2E table description", + ) + assert data["data"]["table_id"] == table_id + assert data["data"]["description"] == "E2E table description" + + data = self._run_ok( + "storage", "table-detail", "--project", self.alias, "--table-id", table_id + ) + assert data["data"]["description"] == "E2E table description" + + # describe-column: set per-column descriptions, verify via table-detail column_details + data = self._run_ok( + "storage", + "describe-column", + "--project", + self.alias, + "--table-id", + table_id, + "--column", + "id=Unique row identifier", + "--column", + "name=Human-readable name", + ) + assert data["data"]["table_id"] == table_id + assert data["data"]["columns"]["id"] == "Unique row identifier" + assert data["data"]["columns"]["name"] == "Human-readable name" + + data = self._run_ok( + "storage", "table-detail", "--project", self.alias, "--table-id", table_id + ) + col_descs = {c["name"]: c.get("description", "") for c in data["data"]["column_details"]} + assert col_descs.get("id") == "Unique row identifier" + assert col_descs.get("name") == "Human-readable name" + + # describe-batch: apply all three sections from a YAML file + batch_yaml = ( + f"buckets:\n" + f" {bucket_id}: Batch bucket desc\n" + f"tables:\n" + f" {table_id}: Batch table desc\n" + f"columns:\n" + f" {table_id}:\n" + f" id: Batch column id desc\n" + ) + batch_file = self.work_dir / "batch_describe.yaml" + batch_file.write_text(batch_yaml, encoding="utf-8") + data = self._run_ok( + "storage", + "describe-batch", + "--project", + self.alias, + "--from-file", + str(batch_file), + ) + assert data["data"]["project_alias"] == self.alias + assert len(data["data"]["applied"]) == 3 + assert data["data"]["errors"] == [] + + # Verify the batch updated the descriptions + data = self._run_ok( + "storage", "table-detail", "--project", self.alias, "--table-id", table_id + ) + assert data["data"]["description"] == "Batch table desc" + col_descs = {c["name"]: c.get("description", "") for c in data["data"]["column_details"]} + assert col_descs.get("id") == "Batch column id desc" + def _test_project_edit_and_remove(self) -> None: """Edit project URL, then remove it.""" # project edit -- change URL back to same (just verify command works) diff --git a/tests/test_storage_describe_cli.py b/tests/test_storage_describe_cli.py new file mode 100644 index 00000000..756e6502 --- /dev/null +++ b/tests/test_storage_describe_cli.py @@ -0,0 +1,757 @@ +"""Tests for storage describe CLI commands via CliRunner. + +Covers describe-bucket, describe-table, describe-column, and describe-batch. +Follows the CLI test pattern from test_workspace_cli.py with patched services. +""" + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.config_service import ConfigService +from keboola_agent_cli.services.job_service import JobService +from keboola_agent_cli.services.project_service import ProjectService + +TEST_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" + +runner = CliRunner() + + +def _setup_config(config_dir: Path, projects: dict[str, dict] | None = None) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + if projects: + for alias, info in projects.items(): + store.add_project( + alias, + ProjectConfig( + stack_url=info.get("stack_url", "https://connection.keboola.com"), + token=info["token"], + project_name=info.get("project_name", alias), + project_id=info.get("project_id", 1234), + ), + ) + return store + + +class TestStorageDescribeBucket: + """Tests for `kbagent storage describe-bucket`.""" + + def test_describe_bucket_json(self, tmp_path: Path) -> None: + """describe-bucket --text returns structured JSON on success.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_storage = MagicMock() + mock_storage.describe_bucket.return_value = { + "project_alias": "prod", + "bucket_id": "in.c-my-bucket", + "description": "My bucket description", + "result": [{"id": "1", "key": "KBC.description", "value": "My bucket description"}], + "message": "Description set on bucket 'in.c-my-bucket' in project 'prod'.", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-bucket", + "--project", + "prod", + "--bucket-id", + "in.c-my-bucket", + "--text", + "My bucket description", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"]["bucket_id"] == "in.c-my-bucket" + assert output["data"]["description"] == "My bucket description" + mock_storage.describe_bucket.assert_called_once_with( + alias="prod", + bucket_id="in.c-my-bucket", + description="My bucket description", + branch_id=None, + ) + + def test_describe_bucket_missing_source(self, tmp_path: Path) -> None: + """describe-bucket without --text/--file/--stdin exits with code 2.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_storage = MagicMock() + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-bucket", + "--project", + "prod", + "--bucket-id", + "in.c-my-bucket", + ], + ) + + assert result.exit_code == 2, f"Expected 2, got {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "error" + assert output["error"]["code"] == "INVALID_ARGUMENT" + mock_storage.describe_bucket.assert_not_called() + + def test_describe_bucket_api_error(self, tmp_path: Path) -> None: + """describe-bucket propagates API errors with appropriate exit code.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_storage = MagicMock() + mock_storage.describe_bucket.side_effect = KeboolaApiError( + message="Bucket not found", + status_code=404, + error_code="BUCKET_NOT_FOUND", + retryable=False, + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-bucket", + "--project", + "prod", + "--bucket-id", + "in.c-missing", + "--text", + "desc", + ], + ) + + assert result.exit_code != 0 + output = json.loads(result.output) + assert output["status"] == "error" + assert output["error"]["code"] == "BUCKET_NOT_FOUND" + + +class TestStorageDescribeTable: + """Tests for `kbagent storage describe-table`.""" + + def test_describe_table_json(self, tmp_path: Path) -> None: + """describe-table --text returns structured JSON on success.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_storage = MagicMock() + mock_storage.describe_table.return_value = { + "project_alias": "prod", + "table_id": "in.c-bucket.orders", + "description": "All orders", + "result": [{"id": "2", "key": "KBC.description", "value": "All orders"}], + "message": "Description set on table 'in.c-bucket.orders' in project 'prod'.", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-table", + "--project", + "prod", + "--table-id", + "in.c-bucket.orders", + "--text", + "All orders", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"]["table_id"] == "in.c-bucket.orders" + assert output["data"]["description"] == "All orders" + mock_storage.describe_table.assert_called_once_with( + alias="prod", + table_id="in.c-bucket.orders", + description="All orders", + branch_id=None, + ) + + def test_describe_table_missing_source(self, tmp_path: Path) -> None: + """describe-table without description source exits with code 2.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_storage = MagicMock() + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-table", + "--project", + "prod", + "--table-id", + "in.c-bucket.orders", + ], + ) + + assert result.exit_code == 2, f"Expected 2, got {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "error" + mock_storage.describe_table.assert_not_called() + + def test_describe_table_from_file(self, tmp_path: Path) -> None: + """describe-table --file reads description from a text file.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + desc_file = tmp_path / "desc.txt" + desc_file.write_text("Description from file", encoding="utf-8") + + mock_storage = MagicMock() + mock_storage.describe_table.return_value = { + "project_alias": "prod", + "table_id": "in.c-bucket.orders", + "description": "Description from file", + "result": [], + "message": "Description set.", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-table", + "--project", + "prod", + "--table-id", + "in.c-bucket.orders", + "--file", + str(desc_file), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + mock_storage.describe_table.assert_called_once_with( + alias="prod", + table_id="in.c-bucket.orders", + description="Description from file", + branch_id=None, + ) + + +class TestStorageDescribeColumn: + """Tests for `kbagent storage describe-column`.""" + + def test_describe_column_json(self, tmp_path: Path) -> None: + """describe-column with NAME=DESC pairs returns structured JSON.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_storage = MagicMock() + mock_storage.describe_columns.return_value = { + "project_alias": "prod", + "table_id": "in.c-bucket.orders", + "columns": {"order_id": "Unique order identifier", "total": "Order total in USD"}, + "result": [], + "message": "Descriptions set for 2 column(s) on table 'in.c-bucket.orders' in project 'prod'.", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-column", + "--project", + "prod", + "--table-id", + "in.c-bucket.orders", + "--column", + "order_id=Unique order identifier", + "--column", + "total=Order total in USD", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"]["table_id"] == "in.c-bucket.orders" + assert output["data"]["columns"]["order_id"] == "Unique order identifier" + assert output["data"]["columns"]["total"] == "Order total in USD" + mock_storage.describe_columns.assert_called_once_with( + alias="prod", + table_id="in.c-bucket.orders", + columns={"order_id": "Unique order identifier", "total": "Order total in USD"}, + branch_id=None, + ) + + def test_describe_column_missing_equals(self, tmp_path: Path) -> None: + """describe-column with malformed --column (no =) exits with code 2.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_storage = MagicMock() + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-column", + "--project", + "prod", + "--table-id", + "in.c-bucket.orders", + "--column", + "order_id_no_equals", + ], + ) + + assert result.exit_code == 2, f"Expected 2, got {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "error" + assert output["error"]["code"] == "INVALID_ARGUMENT" + mock_storage.describe_columns.assert_not_called() + + def test_describe_column_empty_name(self, tmp_path: Path) -> None: + """describe-column with empty column name exits with code 2.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_storage = MagicMock() + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-column", + "--project", + "prod", + "--table-id", + "in.c-bucket.orders", + "--column", + "=description with empty name", + ], + ) + + assert result.exit_code == 2, f"Expected 2, got {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "error" + mock_storage.describe_columns.assert_not_called() + + def test_describe_column_with_branch(self, tmp_path: Path) -> None: + """describe-column passes branch_id when --branch is given.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_storage = MagicMock() + mock_storage.describe_columns.return_value = { + "project_alias": "prod", + "table_id": "in.c-bucket.orders", + "columns": {"col1": "Column 1"}, + "result": [], + "message": "Descriptions set for 1 column(s).", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-column", + "--project", + "prod", + "--table-id", + "in.c-bucket.orders", + "--column", + "col1=Column 1", + "--branch", + "999", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + mock_storage.describe_columns.assert_called_once_with( + alias="prod", + table_id="in.c-bucket.orders", + columns={"col1": "Column 1"}, + branch_id=999, + ) + + +class TestStorageDescribeBatch: + """Tests for `kbagent storage describe-batch`.""" + + def test_describe_batch_json(self, tmp_path: Path) -> None: + """describe-batch with a valid YAML file returns structured JSON.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + batch_file = tmp_path / "descriptions.yaml" + batch_file.write_text( + "buckets:\n" + " in.c-sales: Sales bucket\n" + "tables:\n" + " in.c-sales.orders: All orders\n" + "columns:\n" + " in.c-sales.orders:\n" + " order_id: Unique order ID\n", + encoding="utf-8", + ) + + mock_storage = MagicMock() + mock_storage.describe_batch.return_value = { + "project_alias": "prod", + "applied": [ + {"type": "bucket", "id": "in.c-sales", "description": "Sales bucket"}, + {"type": "table", "id": "in.c-sales.orders", "description": "All orders"}, + { + "type": "columns", + "id": "in.c-sales.orders", + "columns": {"order_id": "Unique order ID"}, + }, + ], + "errors": [], + "message": "Batch complete: 3 applied, 0 errors.", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-batch", + "--project", + "prod", + "--from-file", + str(batch_file), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert len(output["data"]["applied"]) == 3 + assert output["data"]["errors"] == [] + # JSON mode must never wire a progress callback; human mode does. + mock_storage.describe_batch.assert_called_once_with( + alias="prod", + from_file=batch_file, + branch_id=None, + progress_callback=None, + ) + + def test_describe_batch_file_not_found(self, tmp_path: Path) -> None: + """describe-batch raises ValueError when YAML file does not exist.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + missing_file = tmp_path / "missing.yaml" + + mock_storage = MagicMock() + mock_storage.describe_batch.side_effect = ValueError( + f"Batch file not found: {missing_file}" + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-batch", + "--project", + "prod", + "--from-file", + str(missing_file), + ], + ) + + assert result.exit_code == 2, f"Expected 2, got {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "error" + assert output["error"]["code"] == "INVALID_ARGUMENT" + + def test_describe_batch_human_mode_wires_progress_callback(self, tmp_path: Path) -> None: + """Human mode must pass a progress_callback; JSON mode must not.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + batch_file = tmp_path / "descriptions.yaml" + batch_file.write_text("buckets:\n in.c-sales: Sales data\n", encoding="utf-8") + + mock_storage = MagicMock() + mock_storage.describe_batch.return_value = { + "project_alias": "prod", + "applied": [{"type": "bucket", "id": "in.c-sales", "description": "Sales data"}], + "errors": [], + "applied_count": 1, + "error_count": 0, + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + # No --json flag -> human mode -> progress_callback must be wired. + result = runner.invoke( + app, + [ + "storage", + "describe-batch", + "--project", + "prod", + "--from-file", + str(batch_file), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + kwargs = mock_storage.describe_batch.call_args.kwargs + assert kwargs["alias"] == "prod" + assert kwargs["from_file"] == batch_file + assert kwargs["branch_id"] is None + # Key assertion: human mode supplies a callable; JSON mode supplies None. + assert callable(kwargs["progress_callback"]) + + def test_describe_batch_partial_errors(self, tmp_path: Path) -> None: + """describe-batch with partial errors still exits 0 (errors collected, not raised).""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + batch_file = tmp_path / "partial.yaml" + batch_file.write_text("buckets:\n in.c-good: Good\n in.c-bad: Bad\n", encoding="utf-8") + + mock_storage = MagicMock() + mock_storage.describe_batch.return_value = { + "project_alias": "prod", + "applied": [{"type": "bucket", "id": "in.c-good", "description": "Good"}], + "errors": [{"type": "bucket", "id": "in.c-bad", "error": "Not found"}], + "message": "Batch complete: 1 applied, 1 error.", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.StorageService") as MockStorageService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockStorageService.return_value = mock_storage + + result = runner.invoke( + app, + [ + "--json", + "storage", + "describe-batch", + "--project", + "prod", + "--from-file", + str(batch_file), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert len(output["data"]["applied"]) == 1 + assert len(output["data"]["errors"]) == 1 diff --git a/tests/test_storage_describe_service.py b/tests/test_storage_describe_service.py new file mode 100644 index 00000000..1d643a8c --- /dev/null +++ b/tests/test_storage_describe_service.py @@ -0,0 +1,564 @@ +"""Tests for StorageService describe_* methods (bucket, table, columns, batch). + +Also covers the read-back side: extraction of ``KBC.description`` and +``KBC.column.{name}.description`` metadata keys in ``get_bucket_detail`` / +``get_table_detail``, including the precedence between the native API +``description`` field and the ``KBC.description`` metadata entry. +""" + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import KeboolaApiError +from keboola_agent_cli.models import AppConfig, ProjectConfig, TokenVerifyResponse +from keboola_agent_cli.services.storage_service import StorageService + +TEST_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" + +_META_RESPONSE = [ + { + "id": "9001", + "key": "KBC.description", + "value": "A test description", + "provider": "user", + "timestamp": "2026-04-22T10:00:00Z", + } +] + + +def _make_store(tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir(exist_ok=True) + store = ConfigStore(config_dir=config_dir) + store.save( + AppConfig( + projects={ + "prod": ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + ) + } + ) + ) + return store + + +def _make_service(store: ConfigStore, mock_client: MagicMock) -> StorageService: + return StorageService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + +class TestDescribeBucketService: + """Tests for StorageService.describe_bucket().""" + + def test_success(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.set_bucket_metadata.return_value = _META_RESPONSE + service = _make_service(store, mock_client) + + result = service.describe_bucket( + alias="prod", + bucket_id="in.c-sales", + description="Sales data bucket", + ) + + assert result["project_alias"] == "prod" + assert result["bucket_id"] == "in.c-sales" + assert result["description"] == "Sales data bucket" + assert result["result"] == _META_RESPONSE + assert "message" in result + mock_client.set_bucket_metadata.assert_called_once_with( + bucket_id="in.c-sales", + entries=[("KBC.description", "Sales data bucket")], + branch_id=None, + ) + mock_client.close.assert_called_once() + + def test_with_branch(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.set_bucket_metadata.return_value = _META_RESPONSE + service = _make_service(store, mock_client) + + service.describe_bucket( + alias="prod", + bucket_id="in.c-sales", + description="desc", + branch_id=42, + ) + + mock_client.set_bucket_metadata.assert_called_once_with( + bucket_id="in.c-sales", + entries=[("KBC.description", "desc")], + branch_id=42, + ) + + def test_api_error_propagates(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.set_bucket_metadata.side_effect = KeboolaApiError( + message="Bucket not found", + status_code=404, + error_code="BUCKET_NOT_FOUND", + retryable=False, + ) + service = _make_service(store, mock_client) + + with pytest.raises(KeboolaApiError, match="Bucket not found"): + service.describe_bucket(alias="prod", bucket_id="in.c-missing", description="x") + + mock_client.close.assert_called_once() + + +class TestDescribeTableService: + """Tests for StorageService.describe_table().""" + + def test_success(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.set_table_metadata.return_value = _META_RESPONSE + service = _make_service(store, mock_client) + + result = service.describe_table( + alias="prod", + table_id="in.c-sales.orders", + description="All sales orders", + ) + + assert result["project_alias"] == "prod" + assert result["table_id"] == "in.c-sales.orders" + assert result["description"] == "All sales orders" + assert result["result"] == _META_RESPONSE + mock_client.set_table_metadata.assert_called_once_with( + table_id="in.c-sales.orders", + entries=[("KBC.description", "All sales orders")], + branch_id=None, + ) + mock_client.close.assert_called_once() + + def test_with_branch(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.set_table_metadata.return_value = _META_RESPONSE + service = _make_service(store, mock_client) + + service.describe_table( + alias="prod", + table_id="in.c-sales.orders", + description="desc", + branch_id=99, + ) + + mock_client.set_table_metadata.assert_called_once_with( + table_id="in.c-sales.orders", + entries=[("KBC.description", "desc")], + branch_id=99, + ) + + def test_api_error_propagates(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.set_table_metadata.side_effect = KeboolaApiError( + message="Table not found", + status_code=404, + error_code="TABLE_NOT_FOUND", + retryable=False, + ) + service = _make_service(store, mock_client) + + with pytest.raises(KeboolaApiError, match="Table not found"): + service.describe_table(alias="prod", table_id="in.c-missing.t", description="x") + + mock_client.close.assert_called_once() + + +class TestDescribeColumnsService: + """Tests for StorageService.describe_columns().""" + + def test_success_namespaced_keys(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.set_table_metadata.return_value = [] + service = _make_service(store, mock_client) + + result = service.describe_columns( + alias="prod", + table_id="in.c-sales.orders", + columns={"order_id": "Unique order identifier", "total": "Order total in USD"}, + ) + + assert result["project_alias"] == "prod" + assert result["table_id"] == "in.c-sales.orders" + assert result["columns"]["order_id"] == "Unique order identifier" + assert result["columns"]["total"] == "Order total in USD" + mock_client.set_table_metadata.assert_called_once_with( + table_id="in.c-sales.orders", + entries=[ + ("KBC.column.order_id.description", "Unique order identifier"), + ("KBC.column.total.description", "Order total in USD"), + ], + branch_id=None, + ) + mock_client.close.assert_called_once() + + def test_empty_columns_raises_value_error(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + service = _make_service(store, mock_client) + + with pytest.raises(ValueError, match="At least one column"): + service.describe_columns(alias="prod", table_id="in.c-sales.orders", columns={}) + + mock_client.set_table_metadata.assert_not_called() + + def test_with_branch(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.set_table_metadata.return_value = [] + service = _make_service(store, mock_client) + + service.describe_columns( + alias="prod", + table_id="in.c-sales.orders", + columns={"col1": "First column"}, + branch_id=77, + ) + + mock_client.set_table_metadata.assert_called_once_with( + table_id="in.c-sales.orders", + entries=[("KBC.column.col1.description", "First column")], + branch_id=77, + ) + + +class TestDescribeBatchService: + """Tests for StorageService.describe_batch().""" + + def test_success_all_sections(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.set_bucket_metadata.return_value = [] + mock_client.set_table_metadata.return_value = [] + service = _make_service(store, mock_client) + + batch_file = tmp_path / "batch.yaml" + batch_file.write_text( + "buckets:\n" + " in.c-sales: Sales data\n" + "tables:\n" + " in.c-sales.orders: Order data\n" + "columns:\n" + " in.c-sales.orders:\n" + " order_id: Unique order ID\n", + encoding="utf-8", + ) + + result = service.describe_batch(alias="prod", from_file=batch_file) + + assert result["project_alias"] == "prod" + assert len(result["applied"]) == 3 + assert result["errors"] == [] + applied_types = [a["type"] for a in result["applied"]] + assert "bucket" in applied_types + assert "table" in applied_types + assert "columns" in applied_types + # Bucket metadata called once (for the bucket), table metadata called twice + # (once for table description, once for column descriptions) + assert mock_client.set_bucket_metadata.call_count == 1 + assert mock_client.set_table_metadata.call_count == 2 + + def test_file_not_found(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + service = _make_service(store, mock_client) + + with pytest.raises(ValueError, match="Batch file not found"): + service.describe_batch(alias="prod", from_file=tmp_path / "missing.yaml") + + def test_partial_errors_collected(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.set_bucket_metadata.side_effect = [ + [], + KeboolaApiError( + message="Bucket not found", + status_code=404, + error_code="NOT_FOUND", + retryable=False, + ), + ] + service = _make_service(store, mock_client) + + batch_file = tmp_path / "partial.yaml" + batch_file.write_text( + "buckets:\n in.c-good: Good bucket\n in.c-bad: Bad bucket\n", + encoding="utf-8", + ) + + result = service.describe_batch(alias="prod", from_file=batch_file) + + assert len(result["applied"]) == 1 + assert len(result["errors"]) == 1 + assert result["applied"][0]["id"] == "in.c-good" + assert result["errors"][0]["id"] == "in.c-bad" + assert "Bucket not found" in result["errors"][0]["error"] + + def test_empty_yaml(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + service = _make_service(store, mock_client) + + batch_file = tmp_path / "empty.yaml" + batch_file.write_text("", encoding="utf-8") + + result = service.describe_batch(alias="prod", from_file=batch_file) + + assert result["applied"] == [] + assert result["errors"] == [] + mock_client.set_bucket_metadata.assert_not_called() + mock_client.set_table_metadata.assert_not_called() + + def test_invalid_yaml_not_mapping(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + service = _make_service(store, mock_client) + + batch_file = tmp_path / "invalid.yaml" + batch_file.write_text("- item1\n- item2\n", encoding="utf-8") + + with pytest.raises(ValueError, match="must be a YAML mapping"): + service.describe_batch(alias="prod", from_file=batch_file) + + +def _token_info(project_id: int = 258) -> TokenVerifyResponse: + return TokenVerifyResponse( + token_id="12345", + token_description="Test Token", + project_id=project_id, + project_name="Production", + owner_name="Production", + ) + + +class TestGetBucketDetailDescriptionExtraction: + """Verify get_bucket_detail extracts description from metadata and exposes raw metadata.""" + + def test_extracts_description_from_kbc_description_metadata(self, tmp_path: Path) -> None: + """KBC.description (provider=user) in the metadata array is exposed as 'description'.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.verify_token.return_value = _token_info() + mock_client.get_bucket_detail.return_value = { + "id": "in.c-sales", + "displayName": "sales", + "stage": "in", + "description": "", # native field empty + "backend": "snowflake", + "backendPath": ["SAPI_258", "in.c-sales"], + "metadata": [ + { + "id": "9001", + "key": "KBC.description", + "value": "Revenue numbers", + "provider": "user", + "timestamp": "2026-04-22T10:00:00Z", + }, + { + "id": "9002", + "key": "KBC.createdBy.component.id", + "value": "keboola.orchestrator", + "provider": "system", + "timestamp": "2026-04-22T10:00:00Z", + }, + ], + "tables": [], + } + service = _make_service(store, mock_client) + + result = service.get_bucket_detail(alias="prod", bucket_id="in.c-sales") + + assert result["description"] == "Revenue numbers" + # raw_metadata must surface as the 'metadata' field + assert isinstance(result["metadata"], list) + assert len(result["metadata"]) == 2 + assert result["metadata"][0]["key"] == "KBC.description" + + def test_metadata_description_wins_over_native_description(self, tmp_path: Path) -> None: + """Precedence: KBC.description metadata entry overrides the native 'description' field. + + This pins current behavior: when both are present the metadata entry wins, because + the native field is only settable at bucket-create time via the Storage API; any + user-visible description updates flow through the metadata endpoint. + """ + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.verify_token.return_value = _token_info() + mock_client.get_bucket_detail.return_value = { + "id": "in.c-sales", + "displayName": "sales", + "stage": "in", + "description": "Legacy native description", + "backend": "snowflake", + "backendPath": ["SAPI_258", "in.c-sales"], + "metadata": [ + { + "id": "9001", + "key": "KBC.description", + "value": "New metadata description", + "provider": "user", + "timestamp": "2026-04-22T10:00:00Z", + } + ], + "tables": [], + } + service = _make_service(store, mock_client) + + result = service.get_bucket_detail(alias="prod", bucket_id="in.c-sales") + + assert result["description"] == "New metadata description" + + def test_falls_back_to_native_description_when_no_metadata(self, tmp_path: Path) -> None: + """With no KBC.description in metadata, the native 'description' field is used.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.verify_token.return_value = _token_info() + mock_client.get_bucket_detail.return_value = { + "id": "in.c-sales", + "displayName": "sales", + "stage": "in", + "description": "Only native", + "backend": "snowflake", + "backendPath": ["SAPI_258", "in.c-sales"], + "metadata": [], + "tables": [], + } + service = _make_service(store, mock_client) + + result = service.get_bucket_detail(alias="prod", bucket_id="in.c-sales") + + assert result["description"] == "Only native" + assert result["metadata"] == [] + + def test_ignores_non_user_provider_kbc_description(self, tmp_path: Path) -> None: + """A KBC.description entry with provider != 'user' must not be picked up.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.verify_token.return_value = _token_info() + mock_client.get_bucket_detail.return_value = { + "id": "in.c-sales", + "displayName": "sales", + "stage": "in", + "description": "Native wins here", + "backend": "snowflake", + "backendPath": ["SAPI_258", "in.c-sales"], + "metadata": [ + { + "id": "9001", + "key": "KBC.description", + "value": "System-set", + "provider": "system", + "timestamp": "2026-04-22T10:00:00Z", + } + ], + "tables": [], + } + service = _make_service(store, mock_client) + + result = service.get_bucket_detail(alias="prod", bucket_id="in.c-sales") + + assert result["description"] == "Native wins here" + + +class TestGetTableDetailDescriptionExtraction: + """Verify get_table_detail extracts table + per-column descriptions from metadata.""" + + def test_extracts_table_and_column_descriptions(self, tmp_path: Path) -> None: + """KBC.description + KBC.column.{name}.description are surfaced on the response.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.get_table_detail.return_value = { + "id": "in.c-sales.orders", + "name": "orders", + "displayName": "orders", + "bucket": {"id": "in.c-sales"}, + "columns": ["order_id", "total"], + "primaryKey": ["order_id"], + "rowsCount": 42, + "columnMetadata": {}, + "metadata": [ + { + "id": "1", + "key": "KBC.description", + "value": "Sales orders fact table", + "provider": "user", + "timestamp": "2026-04-22T10:00:00Z", + }, + { + "id": "2", + "key": "KBC.column.order_id.description", + "value": "Unique order identifier", + "provider": "user", + "timestamp": "2026-04-22T10:00:00Z", + }, + { + "id": "3", + "key": "KBC.column.total.description", + "value": "Order total in USD", + "provider": "user", + "timestamp": "2026-04-22T10:00:00Z", + }, + ], + } + service = _make_service(store, mock_client) + + result = service.get_table_detail(alias="prod", table_id="in.c-sales.orders") + + assert result["description"] == "Sales orders fact table" + + # column_details must be a list of dicts, one per column, with descriptions + col_map = {c["name"]: c for c in result["column_details"]} + assert col_map["order_id"]["description"] == "Unique order identifier" + assert col_map["total"]["description"] == "Order total in USD" + + # raw_metadata must be exposed as the 'metadata' field + assert isinstance(result["metadata"], list) + assert len(result["metadata"]) == 3 + + def test_columns_without_description_have_no_description_key(self, tmp_path: Path) -> None: + """Columns without a matching KBC.column.{name}.description entry omit 'description'.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.get_table_detail.return_value = { + "id": "in.c-sales.orders", + "name": "orders", + "displayName": "orders", + "bucket": {"id": "in.c-sales"}, + "columns": ["order_id", "total"], + "primaryKey": [], + "rowsCount": 0, + "columnMetadata": {}, + "metadata": [ + { + "id": "1", + "key": "KBC.column.order_id.description", + "value": "Unique order id", + "provider": "user", + "timestamp": "2026-04-22T10:00:00Z", + } + ], + } + service = _make_service(store, mock_client) + + result = service.get_table_detail(alias="prod", table_id="in.c-sales.orders") + + col_map = {c["name"]: c for c in result["column_details"]} + assert col_map["order_id"]["description"] == "Unique order id" + assert "description" not in col_map["total"] + # Table-level description absent when no KBC.description entry + assert result["description"] == "" diff --git a/uv.lock b/uv.lock index 400d03e1..576ed251 100644 --- a/uv.lock +++ b/uv.lock @@ -439,7 +439,7 @@ wheels = [ [[package]] name = "keboola-agent-cli" -version = "0.21.1" +version = "0.22.0" source = { editable = "." } dependencies = [ { name = "httpx" },