From 10f96631565ebcc3370cd6aef6c88e46861fcbbd Mon Sep 17 00:00:00 2001 From: Petr Date: Fri, 21 Aug 2026 23:01:27 +0200 Subject: [PATCH 1/4] chore(agent-prompt): raise the keboola-expert budget to 70 000 B `keboola-expert.md` sat at 61 960 B against a 62 000 B ceiling -- under 100 bytes of headroom. At that margin the budget stopped doing its job and became a tripwire: any PR needing to add a line to the prompt first had to find and justify an unrelated trim, which is review cost with no reviewer benefit. Raise it to 70 000 B. That is still ~12% under the ~80 kB (~20k token) reference point the budget was originally derived from, and the standing guidance is unchanged and restated in the constant's comment: exhaustive per-command detail belongs in `AGENT_CONTEXT`, which loads on demand, and the real answer to sustained growth is splitting the prompt into per-domain specialists rather than another bump. The three prose sites still said "60 KB" -- stale since v0.48.0 moved the ceiling to 62 000 B -- so an author trimming to the documented figure trimmed ~2 kB more than CI required. They now quote the enforced number in the `70 000 B` form and point at the test that asserts it. This overlaps with the open PR #586 (issue #585), which adds a test gating exactly this doc/enforced drift; the wording here is written to satisfy that gate on either merge order. --- CONTRIBUTING.md | 6 +++--- plugins/kbagent/agents/kbagent-pr-reviewer.md | 2 +- tests/test_agent_prompt.py | 21 +++++++++++++------ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 21256511..a379f595 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -378,7 +378,7 @@ before the PR is mergeable. - [ ] **`plugins/kbagent/agents/keboola-expert.md`** -- the subagent system prompt. **Highest silent-drift risk in the repo.** Update at minimum: - [ ] **§1 Rule 6 VERSION GATE** examples (e.g. `flow update needs 0.22.0+`) when adding a command that introduces or relaxes a minimum-version requirement, or when an example version reference is now stale enough to mislead. - - [ ] **§2 Tool Selection Matrix** -- one row **per command GROUP**, not per command. When you add a new write/destructive *group* (e.g. `dev-portal`), give it a single row with `First choice / Fallback / NEVER`. Adding a command to an *existing* group needs no new row. Exhaustive per-command detail belongs in `AGENT_CONTEXT` (`kbagent context`), which is loaded dynamically on demand -- `keboola-expert.md` is a static system prompt loaded into every subagent run and carries a hard 60 KB budget, so it must stay a high-signal decision matrix, not a command catalogue. If a one-row addition would push the file over budget, trim stale content first; do **not** raise the cap. *Severity note:* authors are expected to add the group row, but `/kbagent:review` flags a missing row only **NON-BLOCKING** -- `AGENT_CONTEXT` (a BLOCKING surface above) is the authoritative command catalogue, so a missing matrix row degrades subagent ergonomics without making a command undiscoverable. Don't deprioritize it just because it's non-blocking. + - [ ] **§2 Tool Selection Matrix** -- one row **per command GROUP**, not per command. When you add a new write/destructive *group* (e.g. `dev-portal`), give it a single row with `First choice / Fallback / NEVER`. Adding a command to an *existing* group needs no new row. Exhaustive per-command detail belongs in `AGENT_CONTEXT` (`kbagent context`), which is loaded dynamically on demand -- `keboola-expert.md` is a static system prompt loaded into every subagent run and carries a hard 70 000 B budget, so it must stay a high-signal decision matrix, not a command catalogue. If a one-row addition would push the file over budget, trim stale content first; do **not** raise the cap. The ceiling is enforced by `tests/test_agent_prompt.py::TestPilotAgentFile::test_agent_prompt_under_token_budget`, which is the single source of truth -- check with `wc -c plugins/kbagent/agents/keboola-expert.md` before adding, and remember CI builds the merge commit, so a PR that is individually under budget can still fail once `main` has moved. *Severity note:* authors are expected to add the group row, but `/kbagent:review` flags a missing row only **NON-BLOCKING** -- `AGENT_CONTEXT` (a BLOCKING surface above) is the authoritative command catalogue, so a missing matrix row degrades subagent ergonomics without making a command undiscoverable. Don't deprioritize it just because it's non-blocking. - [ ] **§3 Inline Gotchas** when behavior changed in a way the agent will get wrong by default (e.g. dev-branch auto-materialization, native column-type whitelisting). - [ ] **`plugins/kbagent/skills/kbagent/SKILL.md`** non-table portions -- update the `description:` trigger keywords when introducing a new topic area (so description-matching auto-triggers the skill); add a workflow row to the bottom table if you created a new `references/-workflow.md`. - [ ] **`plugins/kbagent/skills/kbagent/references/commands-reference.md`** -- add the new command bullet under the appropriate section. Hand-maintained, NOT auto-generated. (Yes, this partly duplicates the auto-generated SKILL.md table -- the reference carries denser per-command notes, the table is the at-a-glance picker.) @@ -461,7 +461,7 @@ release checklist below. | `CLAUDE.md` (`## All CLI Commands`) | Adding/removing/renaming commands | NO | | `plugins/kbagent/.claude-plugin/plugin.json` | Every release (auto-synced) | YES (`make version-check`; pre-commit auto-stages) | | `plugins/kbagent/.claude-plugin/CLAUDE.md` | Changing delegation strategy / when-to-delegate rules | NO | -| `plugins/kbagent/agents/keboola-expert.md` | New write/destructive command **group** (one matrix row per group, not per command -- file has a hard 60 KB prompt budget); new minimum-version requirement (Rule 6 VERSION GATE); behavior change (gotchas) | NO -- **highest silent-drift risk** | +| `plugins/kbagent/agents/keboola-expert.md` | New write/destructive command **group** (one matrix row per group, not per command -- file has a hard 70 000 B prompt budget); new minimum-version requirement (Rule 6 VERSION GATE); behavior change (gotchas) | NO -- **highest silent-drift risk** | | `plugins/kbagent/commands/keboola.md` | `/keboola` slash-command UX change (rare) | NO | | `plugins/kbagent/skills/kbagent/SKILL.md` -- table | Auto-generated by `make skill-gen` | YES (`make skill-check`; pre-commit auto-stages) | | `plugins/kbagent/skills/kbagent/SKILL.md` -- description / rules / workflow links | New topic area in `description` triggers; new workflow file added to bottom table | NO | @@ -592,7 +592,7 @@ manual safety net for the silent-drift risks summarized in the 4. **Run `make skill-gen`** -- regenerates the decision table in `SKILL.md`. Idempotent if no commands changed since the previous release. 5. **Manually review `plugins/kbagent/agents/keboola-expert.md`**: - **§1 Rule 6 VERSION GATE examples** -- if any feature this release shipped (or any feature shipped in a previous release that you missed) was previously missing-and-now-present, document it with the right minimum version. Remove stale "since X.Y.Z" mentions that no longer matter to live users. - - **§2 Tool Selection Matrix** -- did you add a new write/destructive command *group* since last release? Is it present with one `First choice / Fallback / NEVER` row (per group, not per command)? Mind the hard 60 KB prompt budget: trim stale content rather than raising the cap. New commands inside an existing group need no new row. + - **§2 Tool Selection Matrix** -- did you add a new write/destructive command *group* since last release? Is it present with one `First choice / Fallback / NEVER` row (per group, not per command)? Mind the hard 70 000 B prompt budget: trim stale content rather than raising the cap. New commands inside an existing group need no new row. - **§3 Inline Gotchas** -- new behavior the agent would get wrong by default? Add it. 6. **Manually review `plugins/kbagent/skills/kbagent/references/gotchas.md`** -- every behavior introduced or changed this release that an AI agent would not infer from `--help` should have its own `(since vX.Y.Z)` entry. The version tag is non-optional. 7. **Manually review `CLAUDE.md` `## All CLI Commands`** -- diff against `kbagent --help` output (and against `kbagent context`). Hand-maintained; CI does not catch drift here. diff --git a/plugins/kbagent/agents/kbagent-pr-reviewer.md b/plugins/kbagent/agents/kbagent-pr-reviewer.md index 74b11a61..4d2114b9 100644 --- a/plugins/kbagent/agents/kbagent-pr-reviewer.md +++ b/plugins/kbagent/agents/kbagent-pr-reviewer.md @@ -135,7 +135,7 @@ verify the file IS updated in the diff. If not, flag it. Specifically: |---|---|---| | `src/keboola_agent_cli/commands/context.py` (`AGENT_CONTEXT`) | Does the new command appear under the right `### ` heading? | BLOCKING | | `CLAUDE.md` `## All CLI Commands` | Does the new command's signature appear in the top-level command list? | BLOCKING | -| `plugins/kbagent/agents/keboola-expert.md` §2 Tool Selection Matrix | One row **per command GROUP**, not per command. If the PR adds a new write/destructive *group*, is there a `\| ... \| First choice \| Fallback \| NEVER \|` row? A new command inside an existing group needs no new row. The file has a hard 60 KB prompt budget; exhaustive per-command coverage lives in `AGENT_CONTEXT` (loaded dynamically), so a missing matrix row is **never BLOCKING** -- flag it NON-BLOCKING only when the new group has zero rows AND `AGENT_CONTEXT` also omits it. | NON-BLOCKING | +| `plugins/kbagent/agents/keboola-expert.md` §2 Tool Selection Matrix | One row **per command GROUP**, not per command. If the PR adds a new write/destructive *group*, is there a `\| ... \| First choice \| Fallback \| NEVER \|` row? A new command inside an existing group needs no new row. The file has a hard 70 000 B prompt budget (enforced by `tests/test_agent_prompt.py`); exhaustive per-command coverage lives in `AGENT_CONTEXT` (loaded dynamically), so a missing matrix row is **never BLOCKING** -- flag it NON-BLOCKING only when the new group has zero rows AND `AGENT_CONTEXT` also omits it. | NON-BLOCKING | | `plugins/kbagent/agents/keboola-expert.md` §1 Rule 6 VERSION GATE | If feature is version-gated, are example version refs (`flow update needs 0.22.0+`) still accurate after this PR? | NON-BLOCKING (informational) | | `plugins/kbagent/agents/keboola-expert.md` §3 Inline Gotchas | If behavior is non-obvious, is there a bullet describing it? | NON-BLOCKING | | `plugins/kbagent/skills/kbagent/references/commands-reference.md` | New command bullet under correct section? | BLOCKING | diff --git a/tests/test_agent_prompt.py b/tests/test_agent_prompt.py index dd3d1287..9cc12984 100644 --- a/tests/test_agent_prompt.py +++ b/tests/test_agent_prompt.py @@ -23,12 +23,21 @@ PLUGIN_CLAUDE_MD = PLUGIN_DIR / ".claude-plugin" / "CLAUDE.md" PLUGIN_JSON = PLUGIN_DIR / ".claude-plugin" / "plugin.json" -# ~20k tokens ≈ 80 kB in typical English markdown (~4 chars/token). -# We target well under that to leave headroom. Bumped 60 kB -> 62 kB in -# v0.48.0 to fit the `feature` command-group matrix row; if this keeps -# creeping up, split keboola-expert into per-domain specialists rather -# than raising the ceiling again. -PROMPT_BYTE_BUDGET = 62_000 +# ~20k tokens ~= 80 kB in typical English markdown (~4 chars/token). The budget +# stays under that so the static prompt never crowds out the task. +# +# History: 60 kB -> 62 000 B in v0.48.0 (the `feature` matrix row); 62 000 B -> +# 70 000 B in v0.88.0. That earlier bump left the file with under 100 bytes of +# headroom, which stopped being a budget and became a tripwire: the next PR to +# touch the prompt paid for an unrelated trim before it could add its own line. +# 70 000 B is a deliberate owner decision to buy that room back, and it still +# sits ~12% under the 80 kB reference point. +# +# It is NOT a licence to grow the file. The standing instruction is unchanged: +# exhaustive per-command detail belongs in `AGENT_CONTEXT` (loaded on demand), +# and the real answer to sustained growth is splitting keboola-expert into +# per-domain specialists, not another bump. Trim before you add. +PROMPT_BYTE_BUDGET = 70_000 @pytest.fixture(scope="module") From 72860c1668d5a04172e89a124a8b4056fdc7844f Mon Sep 17 00:00:00 2001 From: Petr Date: Fri, 21 Aug 2026 23:01:46 +0200 Subject: [PATCH 2/4] fix(storage): surface the table `definition` on `table-detail` (#621) `storage create-table --source-table-id ... --time-partitioning-field ...` followed by `storage swap-tables` is the documented way to repartition a populated BigQuery table. Nothing in kbagent could read the result back. `StorageService.get_table_detail()` assembled its response from an explicit field allowlist and `definition` was never on it, so the one object carrying the registered `timePartitioning` / `rangePartitioning` / `clustering` was dropped on the floor. The write half of the flow was supported, the verify half was not -- confirming a swap meant leaving kbagent for raw Storage API calls or BigQuery metadata access, which is not available at all in a Keboola-managed BigQuery project without `bigquery.jobs.create`. The table ID is identical whether or not the swap happened; the layout is the only thing that tells the two apart. `create-table` is no substitute: its JSON echoes the layout that was REQUESTED, and its `--if-not-exists` skip path nulls the layout keys outright rather than re-deriving the existing table's. The data was already in hand -- `StorageTablesClient.get_table_detail()` returns `GET /v2/storage/tables/{id}` unfiltered, and that response carries `definition` with no `include=` needed. Two upstream details, read off connection's `TableDetailResponseProvider` and `BigqueryDriverConfig::extendTableDefinitionResponse`, shape the rendering: * `definition` is set on EVERY table-detail response, untyped and Snowflake tables included. A null therefore means the stack omitted the key -- never "this table is untyped" -- so the human-mode block keys off the layout fields themselves. Verified live against a Snowflake project: output is byte-identical to before. * When partitioning is set the response also carries `requirePartitionFilter` and `partitions[]`, one entry per physical partition from INFORMATION_SCHEMA.PARTITIONS. That list is unbounded -- thousands of entries on a long-lived daily table -- so human mode prints its length and never its contents. `--json` passes the whole thing through: re-dropping an API field is the bug being fixed here, so it is not repeated one level down. The value is also type-checked before any `.get()` reaches it. The Storage API has really served `definition` as `[]` rather than an object (SUPPORT-16581, pinned in tests/test_storage_empty_definition.py) -- that shape broke the Go CLI's decoder, and `definition.get(...)` would raise on it. `GET /storage/table-detail/...` on `kbagent serve` picks the field up for free via the same service. `storage tables` (the LIST endpoint) is deliberately untouched: the Storage API's `include=` there accepts no `definition` value (`TablesListRequest::ALLOWED_INCLUDE`), so surfacing the layout in a listing would cost one detail request per table -- a different change with a different cost profile. Both files this touches are grandfathered over their file-size ceiling and may only shrink, so the change is made by extracting rather than appending: response assembly moves to the pure, client-free `services/_table_detail.py`, and human rendering to `commands/_storage_table_detail.py`. Net effect is -49 code lines in `storage_service.py` and -25 in `commands/storage.py`, both re-recorded in the size baseline. `create-table` now shares the layout formatters, so both commands print the same string for the same layout -- which is what makes diffing "what I asked for" against "what landed" trustworthy. E2E gains the assertion that matters: after the BigQuery repartition + swap, `table-detail` on the production name must report the clustering that `create-table` applied. --- .claude-plugin/marketplace.json | 2 +- CLAUDE.md | 12 + docs/e2e-scenarios.md | 2 +- plugins/kbagent/.claude-plugin/plugin.json | 2 +- plugins/kbagent/agents/keboola-expert.md | 2 +- plugins/kbagent/skills/kbagent/SKILL.md | 2 +- .../kbagent/references/commands-reference.md | 2 +- .../skills/kbagent/references/gotchas.md | 26 ++ .../references/storage-types-workflow.md | 14 + pyproject.toml | 2 +- scripts/file_size_baseline.json | 4 +- src/keboola_agent_cli/changelog.py | 29 ++ .../commands/_storage_table_detail.py | 113 ++++++++ src/keboola_agent_cli/commands/context.py | 9 + src/keboola_agent_cli/commands/storage.py | 59 ++-- .../services/_table_detail.py | 125 +++++++++ .../services/storage_service.py | 68 +---- tests/test_e2e.py | 30 +++ tests/test_storage_table_definition.py | 253 ++++++++++++++++++ uv.lock | 2 +- 20 files changed, 644 insertions(+), 114 deletions(-) create mode 100644 src/keboola_agent_cli/commands/_storage_table_detail.py create mode 100644 src/keboola_agent_cli/services/_table_detail.py create mode 100644 tests/test_storage_table_definition.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 01aba376..e744f3ac 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "kbagent", - "version": "0.87.0", + "version": "0.88.0", "source": "./plugins/kbagent", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, sync configs as files, manage dev branches, and debug SQL in workspaces", "category": "development" diff --git a/CLAUDE.md b/CLAUDE.md index 07cfb21c..7c93533d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -433,6 +433,18 @@ kbagent storage buckets [--project NAME] [--branch ID] kbagent storage bucket-detail --project NAME --bucket-id ID [--branch ID] kbagent storage tables [--project NAME ...] [--bucket-id ID] [--branch ID] kbagent storage table-detail --project NAME --table-id ID [--branch ID] +# table-detail (0.88.0+, #621): also returns the raw Storage API `definition`. On BigQuery +# that object (timePartitioning / rangePartitioning / clustering / requirePartitionFilter / +# partitions[]) is the ONLY readable record of the registered layout, so it is how a +# `create-table --source-table-id ...` + `swap-tables` repartition is VERIFIED -- the table +# ID is unchanged either way, and `create-table` only echoes the layout you REQUESTED (its +# --if-not-exists skip path nulls the layout keys outright). Human mode adds Time +# partitioning / Range partitioning / Clustering / Partition filter required / Partitions +# (a COUNT) and prints nothing new when there is no layout; --json passes `definition` +# through verbatim, including the unbounded `partitions[]` (one entry per physical +# partition from INFORMATION_SCHEMA.PARTITIONS). `definition` is present on EVERY response +# -- untyped tables get one too -- so null means the stack omitted the key, NOT "untyped". +# `storage tables` (the LIST endpoint) is unaffected: the API has no `definition` include. kbagent storage create-bucket --project NAME --stage STAGE --name NAME [--description D] [--backend B] [--branch ID] kbagent storage create-table --project NAME --bucket-id ID --name NAME [--column COL:TYPE[(length)] ...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--source-table-id ID] [--source-branch-id N] [--time-partitioning-type DAY|HOUR|MONTH|YEAR] [--time-partitioning-field COL] [--time-partitioning-expiration-ms MS] [--range-partitioning-field COL --range-partitioning-start S --range-partitioning-end E --range-partitioning-interval I] [--clustering-field COL ...] [--branch ID] [--if-not-exists] # --column XOR --source-table-id (0.66.0+, BigQuery only): --source-table-id copies an existing table's data into the requested partition/clustering layout (schema derived from source) -> swap into place with swap-tables. Partition/clustering flags work in both modes (BigQuery only); time vs range partitioning are mutually exclusive. A non-BigQuery project fails fast (pre-flight backend check). diff --git a/docs/e2e-scenarios.md b/docs/e2e-scenarios.md index e54927dc..0a7a4976 100644 --- a/docs/e2e-scenarios.md +++ b/docs/e2e-scenarios.md @@ -57,7 +57,7 @@ Resources are prefixed with `e2e-{timestamp}` and cleaned up via yield fixture e | 9 | `storage create-table` | Table created with typed columns (INTEGER, STRING) and primary key | | 10 | `storage upload-table` | 5-row CSV uploaded, `imported_rows: 5` | | 11 | `storage upload-table --incremental` | 3 more rows appended, download verifies 8 total rows | -| 12 | `storage tables`, `storage table-detail` | Table in listing, column details match (id, name, value) | +| 12 | `storage tables`, `storage table-detail` | Table in listing, column details match (id, name, value), `definition` passed through (#621) | | 13 | `storage download-table` | Full download: 8 rows, correct IDs. With `--columns`/`--limit`: subset verified | | 14 | `storage unload-table --download` | Table exported to file storage, file_id > 0, file downloaded | | 15 | `storage load-file` | CSV uploaded as file, then loaded into table via `load-file` | diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index b4fd0bf2..4c98bbc3 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.87.0", + "version": "0.88.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, sync configs as files, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index dff9c053..c53c94ea 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -107,7 +107,7 @@ a critical failure. | Create typed table with native types | `kbagent storage create-table --column pk:VARCHAR(40) --column amount:NUMBER(18,2) --not-null pk --default amount=0` (0.25.0+) | -- | re-creating via raw REST to `/v2/storage/...tables-definition` | | Add one column to an existing table | `kbagent storage add-column --project P --table-id in.c-foo.data --column status:VARCHAR(20) [--not-null] [--default active]` (0.62.0+) -- synchronous Storage endpoint, same `name:TYPE(length)` grammar as `create-table`; the add-side mirror of `delete-column` | -- | re-creating the whole table just to add a field (loses data/PK/dependents); raw `POST /v2/storage/tables/.../columns` | | Promote typed rebuild back into the original name | `kbagent storage swap-tables --project P --table-id in.c-foo.data --target-table-id in.c-foo.data_change_log --branch --yes` (0.28.0+) -- async storage job (`tableSwap`); client polls to completion. Service refuses without a branch; any branch incl. prod | -- | renaming or deleting + re-uploading (loses history; downstream configs need to be rewritten) | -| Repartition / recluster a populated BigQuery table | `kbagent storage create-table --project P --bucket-id in.c-main --name events_repart --source-table-id in.c-main.events --time-partitioning-type DAY --time-partitioning-field created_at --clustering-field tenant_id --primary-key id` (0.66.0+, BigQuery only) to copy the data into the new layout, then `kbagent storage swap-tables --table-id in.c-main.events --target-table-id in.c-main.events_repart --branch --yes` to flip it into place. `--source-table-id` derives the schema from the source so `--column` is forbidden (mutually exclusive); a non-BigQuery project fails fast (pre-flight backend check, exit 2). Range partitioning instead: `--range-partitioning-field/-start/-end/-interval` (all four; bounds are strings; mutually exclusive with time partitioning) | -- | raw `POST /v2/storage/buckets/.../tables-definition` with a `source` object, then manual swap; or a `CREATE TABLE ... AS SELECT` in a workspace (drops NOT NULL + primary key) | +| Repartition / recluster a populated BigQuery table | `kbagent storage create-table --project P --bucket-id in.c-main --name events_repart --source-table-id in.c-main.events --time-partitioning-type DAY --time-partitioning-field created_at --clustering-field tenant_id --primary-key id` (0.66.0+, BigQuery only) to copy the data into the new layout, then `kbagent storage swap-tables --table-id in.c-main.events --target-table-id in.c-main.events_repart --branch --yes` to flip it into place. `--source-table-id` derives the schema from the source so `--column` is forbidden (mutually exclusive); a non-BigQuery project fails fast (pre-flight backend check, exit 2). Range partitioning instead: `--range-partitioning-field/-start/-end/-interval` (all four; bounds are strings; mutually exclusive with time partitioning). VERIFY the swap with `storage table-detail --json` -> `.definition.timePartitioning` / `.clustering` (0.88.0+); `create-table` only echoes the layout you REQUESTED, so it proves nothing | -- | raw `POST /v2/storage/buckets/.../tables-definition` with a `source` object, then manual swap; or a `CREATE TABLE ... AS SELECT` in a workspace (drops NOT NULL + primary key) | | Re-seed a table without losing its schema / PK / dependents | `kbagent storage truncate-table --project P --table-id in.c-foo.data [--branch ID] [--dry-run] [--yes]` (0.32.0+) -- DELETE `/tables/{id}/rows?allowTruncate=1`; endpoint is uniformly async on every branch (returns a queued `tableRowsDelete` job; client polls via `_wait_for_storage_job`). Do NOT pass `async=true` -- the API rejects it. Batch via repeated `--table-id`. Returns `{truncated[], failed[], dry_run, project_alias}` with `truncated[]` entries carrying `{table_id, rows_before, rows_after, branch_id}`. Permission class: `destructive` | -- | drop + recreate the table (loses descriptions, PK, sharing edges, and breaks every downstream config reference); deleting rows via raw SQL in a workspace (bypasses the Storage API audit trail) | | Point-in-time backup of a table before a risky change | `kbagent storage snapshot-create --project P --table-id in.c-foo.data [--description D]` (0.75.0+, #512) -- async `tableSnapshotCreate` job; receipt carries `snapshot_id`. List with `storage snapshots --table-id ...`, inspect with `snapshot-detail --snapshot-id ID` (embeds the source table object) | -- | exporting to CSV as a "backup" (loses column types + PK); relying on the destructive command's `--dry-run` alone | | Restore a snapshot as a NEW table | `kbagent storage table-from-snapshot --project P --snapshot-id ID --bucket-id in.c-foo --name restored [--dry-run]` (0.75.0+, #512) -- classic `tables-async` endpoint; restores data + columns + PK. `--name` REQUIRED (API rejects empty; PHP-client "defaults to snapshot name" docblock is stale). No overwrite: restore under a new name, verify, then `swap-tables`/`delete-table` to promote | -- | trying `create-table --snapshot-id` (not a thing -- `tables-definition` does not accept snapshots); restoring straight onto the production name (duplicate-name error) | diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index a80256c1..4c1ab0a4 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -155,7 +155,7 @@ When working inside a git repository or project directory, run `kbagent init` (o | List storage buckets with sharing/linked bucket information | `kbagent storage buckets` | | Show detailed bucket info including backend-native direct access paths | `kbagent storage bucket-detail --project PROJECT --bucket-id BUCKET-ID` | | List storage tables from one or more projects | `kbagent storage tables` | -| Show detailed table info including columns and types | `kbagent storage table-detail --project PROJECT --table-id TABLE-ID` | +| Show detailed table info including columns, types and physical layout | `kbagent storage table-detail --project PROJECT --table-id TABLE-ID` | | Create a new storage bucket | `kbagent storage create-bucket --project PROJECT --stage STAGE --name NAME` | | Create a new storage table with typed columns | `kbagent storage create-table --project PROJECT --bucket-id BUCKET-ID --name NAME` | | Upload a CSV file into a storage table | `kbagent storage upload-table --project PROJECT --table-id TABLE-ID --file FILE` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index aa9a5e70..c74d1b51 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -159,7 +159,7 @@ Requires a **super-admin** Manage API token (same kind as `org setup`). Same def - `storage buckets [--project NAME] [--branch ID]` -- list buckets with sharing/linked info (branch-aware) - `storage bucket-detail --project NAME --bucket-id ID [--branch ID]` -- bucket detail with backend-native direct-access paths (branch-aware). Output adapts to backend: Snowflake -> `snowflake_database` / `snowflake_schema` / per-table `snowflake_path` quoted with `"..."`. BigQuery -> `bigquery_dataset` (and `bigquery_project` when surfaced via API `databaseName`) / per-table `bigquery_path` quoted with backticks. Always-present backend-agnostic keys: `sql_dialect` (`"snowflake"` / `"bigquery"`) and per-table `sql_path` -- prefer these in agent code instead of branching on backend yourself - `storage tables [--project NAME ...] [--bucket-id ID] [--branch ID]` -- list tables across all connected projects in parallel (multi-project by default, same as `storage buckets`); repeat `--project` to target a subset; `--bucket-id` is applied independently per project (missing buckets become per-project errors); `--branch` requires exactly one `--project` -- `storage table-detail --project NAME --table-id ID [--branch ID]` -- table detail with columns, types, primary key, row count (branch-aware) +- `storage table-detail --project NAME --table-id ID [--branch ID]` -- table detail with columns, types, primary key, row count (branch-aware). Since 0.88.0 (#621) also returns the raw Storage API `definition`: on BigQuery that carries `timePartitioning` / `rangePartitioning` / `clustering` / `requirePartitionFilter` / `partitions[]`, and it is the only way to verify a repartition landed. Human mode prints the layout and a partition COUNT; `--json` passes `definition` through verbatim. Present on every response (untyped tables too), so `null` means the stack omitted the key, not "untyped" - `storage create-bucket --project NAME --stage STAGE --name NAME [--description D] [--backend B] [--branch ID]` -- create bucket (branch-aware). With `--branch ID` on a project lacking the `storage-branches` feature (legacy fake-branch), response carries `legacy_branch_storage: true` and human mode prints a warning -- the runner will create a parallel `out.c--*` bucket at job time. See `storage-types-workflow.md` - `storage create-table --project NAME --bucket-id ID --name NAME [--column col:TYPE[(length)] ...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--source-table-id ID] [--source-branch-id N] [--time-partitioning-type DAY|HOUR|MONTH|YEAR] [--time-partitioning-field COL] [--time-partitioning-expiration-ms MS] [--range-partitioning-field COL --range-partitioning-start S --range-partitioning-end E --range-partitioning-interval I] [--clustering-field COL ...] [--branch ID] [--if-not-exists]` -- create typed table. Base types `STRING/INTEGER/NUMERIC/FLOAT/BOOLEAN/DATE/TIMESTAMP` plus native backend types with length (`VARCHAR(40)`, `NUMBER(18,2)`, `TIMESTAMP_TZ`, `VARIANT`, etc.) -- type/length validation delegated to the Storage API. `--not-null` marks a column `nullable=false`; `--default NAME=VALUE` sets a DEFAULT expression (booleans must be lowercase `true`/`false`). In a dev branch, the target bucket is auto-materialized if it has not yet been written to there -- response surfaces this via `auto_created_bucket: bool`. On legacy fake-branch projects (no `storage-branches` feature), `legacy_branch_storage: true` flags that the runner will use a separate `out.c--*` bucket at job time. `--if-not-exists` (0.47.0+) turns a duplicate-display-name failure into `action: skipped` when the table really exists at the expected id (safe for parallel workers). Since 0.47.1 the skipped envelope reports the EXISTING table's actual `columns`/`primary_key`/`name`, mirrors the request under `requested_columns`/`requested_primary_key`, and sets `schema_drift: true` when they diverge. **`--source-table-id` (0.66.0+, BigQuery only)** copies an existing table's data into the requested partition/clustering layout instead of building from `--column` (schema derived from source -> `--column`/`--not-null`/`--default` forbidden; the two are mutually exclusive). This is the supported way to repartition a populated BigQuery table -- then promote it with `storage swap-tables`. Partition/clustering flags (`--time-partitioning-*`, `--range-partitioning-*`, `--clustering-field`) also work on a plain `--column` create (BigQuery only); time vs range partitioning are mutually exclusive and range bounds are strings. When any source/partition/clustering flag is used, a one-call backend pre-flight rejects non-BigQuery projects (exit 2) before the create. See `storage-types-workflow.md` - `storage upload-table --project NAME --table-id ID --file PATH [--incremental] [--branch ID]` -- upload CSV (branch-aware) diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 4d09fbb6..fdc40340 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -2050,6 +2050,32 @@ unknown -- do not try to parse a fallback message. See [storage-types-workflow.md](storage-types-workflow.md) for the full type inventory and examples. +## `storage table-detail` is the only way to READ a BigQuery partition layout (since v0.88.0, #621) + +- **`create-table` output does not prove anything.** Its `--json` echoes the + `time_partitioning` / `range_partitioning` / `clustering` you *requested*, not + what the server registered, and its `--if-not-exists` skip path sets all three + to `null` on purpose (the existing table's layout is never re-derived). To + confirm a repartition landed, read it back: + `kbagent --json storage table-detail --project P --table-id T` -> + `.definition.timePartitioning` / `.definition.clustering`. +- **Before 0.88.0 the field was dropped entirely.** The service assembled its + response from a field allowlist that omitted `definition`, so on 0.87.0 and + earlier the layout is unreachable from kbagent -- verifying meant raw Storage + API calls or BigQuery metadata access. Version-gate accordingly. +- **`definition` is present on EVERY table-detail response**, untyped and + Snowflake tables included (connection builds one either way). `null` therefore + means the stack omitted the key -- it never means "this table is untyped". + Test for the layout keys, not for `definition`. +- **`.definition.partitions` is unbounded.** It is one entry per physical + partition, straight from `INFORMATION_SCHEMA.PARTITIONS` -- a DAY-partitioned + table with a few years of history returns thousands. `--json` carries the whole + list; human mode prints only the count. Do not pipe the raw JSON of a large + partitioned table into a context window unfiltered. +- **`storage tables` (the LIST endpoint) still has no layout.** The Storage API's + `include=` on the list route accepts no `definition` value, so reading the + layout costs one `table-detail` request per table. + ## `storage create-table --source-table-id` + partition/clustering are BigQuery-only (since 0.66.0) - **`--source-table-id` copies an existing table instead of building from `--column`.** diff --git a/plugins/kbagent/skills/kbagent/references/storage-types-workflow.md b/plugins/kbagent/skills/kbagent/references/storage-types-workflow.md index db33168d..1a2a9a96 100644 --- a/plugins/kbagent/skills/kbagent/references/storage-types-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/storage-types-workflow.md @@ -322,8 +322,22 @@ kbagent storage create-table --project prod --bucket-id in.c-main \ kbagent storage table-detail --project prod --table-id in.c-main.events_repart kbagent storage swap-tables --project prod --table-id in.c-main.events \ --target-table-id in.c-main.events_repart --branch --yes + +# 3. Verify the layout actually landed on the production name (0.88.0+, #621). +# The table ID is identical whether or not the swap happened -- the layout is +# the only thing that tells the two apart. +kbagent --json storage table-detail --project prod --table-id in.c-main.events \ + | jq '.data.definition | {timePartitioning, clustering, requirePartitionFilter}' ``` +On 0.87.0 and earlier `table-detail` dropped `definition`, so step 3 was not +possible from kbagent at all. `create-table`'s own output is not a substitute: +it echoes the layout you *requested*, and its `--if-not-exists` skip path +reports `null` for all three layout keys rather than re-deriving the existing +table's. Note `.definition.partitions` is one entry per physical partition +(thousands on a long-lived daily table) -- select the keys you need, as above, +rather than dumping the whole object. + Rules: - **BigQuery only.** `--source-table-id` and the partition/clustering flags (`--time-partitioning-*`, `--range-partitioning-*`, `--clustering-field`) diff --git a/pyproject.toml b/pyproject.toml index 4daf1f29..f833d98e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-cli" -version = "0.87.0" +version = "0.88.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/scripts/file_size_baseline.json b/scripts/file_size_baseline.json index 2a73b730..49a035e9 100644 --- a/scripts/file_size_baseline.json +++ b/scripts/file_size_baseline.json @@ -3,9 +3,9 @@ "files": { "commands/config.py": 2007, "commands/lineage.py": 1271, - "commands/storage.py": 2246, + "commands/storage.py": 2221, "services/data_app_service.py": 1641, - "services/storage_service.py": 1733, + "services/storage_service.py": 1684, "services/sync_service.py": 1655 } } diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 45744bef..d4e0e40c 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -24,6 +24,35 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.88.0": [ + "Fix (#621): `storage table-detail` now surfaces the Storage API's `definition` " + "object, which for a BigQuery table is the only readable record of the registered " + "`timePartitioning` / `rangePartitioning` / `clustering` layout. The service built " + "its response as a field allowlist and `definition` was never on it, so the layout " + "that `storage create-table --source-table-id ... --time-partitioning-field ...` " + "plus `storage swap-tables` had just applied was invisible to kbagent: the write " + "half of the documented repartition flow was supported, the verify half was not, " + "and confirming a swap meant leaving kbagent for raw Storage API or BigQuery " + "metadata access. `create-table` is no substitute -- its JSON echoes the layout you " + "REQUESTED, and its `--if-not-exists` skip path nulls those keys outright. Human " + "mode prints `Time partitioning:` / `Range partitioning:` / `Clustering:` / " + "`Partition filter required:` / `Partitions:` (a COUNT -- the API returns one entry " + "per physical partition from INFORMATION_SCHEMA.PARTITIONS, thousands on a " + "long-lived daily table) between the existing `Primary key` and `Last import` " + "lines, and prints nothing new when there is no layout, so Snowflake and untyped " + "output is unchanged. `--json` passes `definition` through verbatim, full " + "`partitions[]` included. Note that `definition` is present on EVERY table-detail " + "response, untyped tables included, so a null there means the stack omitted the key " + '-- never "this table is untyped". `GET /storage/table-detail/...` on `kbagent ' + "serve` picks the field up for free. `storage tables` (the LIST endpoint) still has " + "no layout: the API offers no `definition` include value, so it would cost one " + "detail request per table.", + "Internal: the `keboola-expert.md` prompt budget moves 62 000 B -> 70 000 B. The " + "62 000 B ceiling had under 100 bytes of headroom left, which made every PR " + "touching the prompt pay for an unrelated trim first. `AGENT_CONTEXT` remains the " + "place for exhaustive per-command detail, and splitting the prompt into per-domain " + "specialists remains the answer to sustained growth.", + ], "0.87.0": [ "New (#626): `data-app create` gains `--workspace / --no-workspace` and grants " "Storage access BY DEFAULT. The flag writes `runtime.workspace.enabled: true`, which " diff --git a/src/keboola_agent_cli/commands/_storage_table_detail.py b/src/keboola_agent_cli/commands/_storage_table_detail.py new file mode 100644 index 00000000..9ac8946f --- /dev/null +++ b/src/keboola_agent_cli/commands/_storage_table_detail.py @@ -0,0 +1,113 @@ +"""Human-mode rendering for `kbagent storage table-detail` (issue #621). + +Lives in a private module because `commands/storage.py` is past the commands +file-size ceiling and is grandfathered at its recorded size (CONTRIBUTING.md > +"File-size budgets") -- it may shrink, not grow. + +Beyond the columns table this renders the BigQuery *physical layout* the +Storage API reports under `definition`. Two shapes drive the guards here: + +* `definition` is present on every table-detail response, typed or not, so the + layout block keys off the layout fields themselves. A Snowflake or untyped + table prints exactly what it printed before. +* `definition` has really been served as `[]` rather than an object (see + tests/test_storage_empty_definition.py -- it broke the Go CLI's decoder), so + the value is type-checked before any `.get()` reaches it. + +`format_time_partitioning` / `format_range_partitioning` are shared with +`storage create-table`'s result output on purpose: verifying a repartition means +diffing what create-table said it applied against what table-detail reads back, +and that comparison is only trustworthy if both print the same string for the +same layout. +""" + +from __future__ import annotations + +from typing import Any + +from rich.markup import escape +from rich.table import Table + +from ..output import OutputFormatter + + +def format_time_partitioning(time_partitioning: dict[str, Any]) -> str: + """Render a `timePartitioning` block, e.g. `DAY on created_at`.""" + field = time_partitioning.get("field") + suffix = f" on {field}" if field else " (ingestion time)" + return f"{time_partitioning.get('type')}{suffix}" + + +def format_range_partitioning(range_partitioning: dict[str, Any]) -> str: + """Render a `rangePartitioning` block, e.g. `order_id [0, 1000000) step 1000`.""" + bounds = range_partitioning.get("range") or {} + suffix = "" + if bounds: + suffix = f" [{bounds.get('start')}, {bounds.get('end')}) step {bounds.get('interval')}" + return f"{range_partitioning.get('field')}{suffix}" + + +def render_table_layout(formatter: OutputFormatter, definition: Any) -> None: + """Print the physical layout recorded in a table `definition`, if it has one. + + Prints nothing at all when the table has no partitioning or clustering -- + which is every Snowflake and every untyped table. + """ + if not isinstance(definition, dict): + return + + time_partitioning = definition.get("timePartitioning") + if isinstance(time_partitioning, dict): + formatter.console.print( + f" Time partitioning: {format_time_partitioning(time_partitioning)}" + ) + range_partitioning = definition.get("rangePartitioning") + if isinstance(range_partitioning, dict): + formatter.console.print( + f" Range partitioning: {format_range_partitioning(range_partitioning)}" + ) + clustering = definition.get("clustering") + if isinstance(clustering, dict) and clustering.get("fields"): + formatter.console.print(f" Clustering: {', '.join(clustering['fields'])}") + if definition.get("requirePartitionFilter"): + # A query that omits a filter on the partition column fails outright, so + # this is not trivia -- it changes how the table must be read. + formatter.console.print(" Partition filter required: yes") + partitions = definition.get("partitions") + if isinstance(partitions, list) and partitions: + # Count only. The API returns one entry per physical partition straight + # from INFORMATION_SCHEMA.PARTITIONS -- a DAY-partitioned table with a + # few years of history has thousands, which would bury everything above. + # `--json` still carries the full list. + formatter.console.print(f" Partitions: {len(partitions):,}") + + +def render_table_detail(formatter: OutputFormatter, result: dict[str, Any]) -> None: + """Print the full human-mode `storage table-detail` view.""" + formatter.console.print(f"[bold]Table:[/bold] {result['table_id']}") + formatter.console.print(f" Name: {escape(result['display_name'] or result['name'])}") + formatter.console.print(f" Bucket: {result['bucket_id']}") + formatter.console.print(f" Rows: {result['rows_count']:,}") + size_mb = result["data_size_bytes"] / (1024 * 1024) + formatter.console.print(f" Size: {size_mb:.2f} MB") + if result["primary_key"]: + formatter.console.print(f" Primary key: {', '.join(result['primary_key'])}") + render_table_layout(formatter, result.get("definition")) + if result["last_import_date"]: + formatter.console.print(f" Last import: {result['last_import_date']}") + + if result["column_details"]: + formatter.console.print() + table = Table(title="Columns") + table.add_column("Name", style="bold cyan") + table.add_column("Type", style="dim") + table.add_column("Nullable", style="dim") + + for col in result["column_details"]: + table.add_row( + col["name"], + col.get("type", ""), + "yes" if col.get("nullable") else "", + ) + + formatter.console.print(table) diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 3138a3db..b69e7df2 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -593,6 +593,15 @@ kbagent storage table-detail --project NAME --table-id TABLE_ID [--branch ID] Show detailed table info: columns (with types if available), primary key, row count, size, last import date. Uses production by default; pass --branch to query a dev branch explicitly. + Also surfaces the raw Storage API `definition` (0.88.0+, #621) -- for a BigQuery table + that is the ONLY readable record of the registered timePartitioning / rangePartitioning / + clustering layout, so it is how you VERIFY a `create-table --source-table-id` + + `swap-tables` repartition actually landed (`create-table` only echoes what you REQUESTED). + Human mode prints Time partitioning / Range partitioning / Clustering / Partition filter + required / Partitions (a COUNT; --json carries the full partitions[] list, one entry per + physical partition -- thousands on a long-lived daily table). `definition` is present on + EVERY response, untyped tables included, so null means the stack omitted it, NOT "untyped". + `storage tables` (the list endpoint) has no layout: the API offers no `definition` include. kbagent storage create-bucket --project NAME --stage STAGE --name BUCKET_NAME [--description D] [--backend B] [--branch ID] Create a new storage bucket. Stage must be "in" or "out". Branch-aware. diff --git a/src/keboola_agent_cli/commands/storage.py b/src/keboola_agent_cli/commands/storage.py index cf08fcdb..1859263e 100644 --- a/src/keboola_agent_cli/commands/storage.py +++ b/src/keboola_agent_cli/commands/storage.py @@ -20,6 +20,11 @@ map_error_to_exit_code, resolve_branch, ) +from ._storage_table_detail import ( + format_range_partitioning, + format_time_partitioning, + render_table_detail, +) storage_app = typer.Typer(help="Browse and manage storage buckets, tables, and files") @@ -383,7 +388,15 @@ def storage_table_detail( help="Dev branch ID (defaults to active branch if set via 'branch use')", ), ) -> None: - """Show detailed table info including columns and types.""" + """Show detailed table info including columns, types and physical layout. + + The Storage API's `definition` object is returned as-is. On BigQuery it + carries the registered `timePartitioning` / `rangePartitioning` / + `clustering` -- the only way to confirm a `create-table --source-table-id` + + `swap-tables` repartition landed, since the table ID is the same either + way. Human mode prints the layout and a partition count; `--json` carries + the full `definition`. + """ formatter = get_formatter(ctx) service = get_service(ctx, "storage_service") config_store: ConfigStore = ctx.obj["config_store"] @@ -408,34 +421,7 @@ def storage_table_detail( if formatter.json_mode: formatter.output(result) else: - formatter.console.print(f"[bold]Table:[/bold] {result['table_id']}") - formatter.console.print(f" Name: {escape(result['display_name'] or result['name'])}") - formatter.console.print(f" Bucket: {result['bucket_id']}") - formatter.console.print(f" Rows: {result['rows_count']:,}") - size_mb = result["data_size_bytes"] / (1024 * 1024) - formatter.console.print(f" Size: {size_mb:.2f} MB") - if result["primary_key"]: - formatter.console.print(f" Primary key: {', '.join(result['primary_key'])}") - if result["last_import_date"]: - formatter.console.print(f" Last import: {result['last_import_date']}") - - if result["column_details"]: - formatter.console.print() - from rich.table import Table - - table = Table(title="Columns") - table.add_column("Name", style="bold cyan") - table.add_column("Type", style="dim") - table.add_column("Nullable", style="dim") - - for col in result["column_details"]: - table.add_row( - col["name"], - col.get("type", ""), - "yes" if col.get("nullable") else "", - ) - - formatter.console.print(table) + render_table_detail(formatter, result) @storage_app.command("create-bucket", rich_help_panel=_BUCKETS) @@ -730,20 +716,13 @@ def storage_create_table( formatter.console.print(f" Columns: {', '.join(result['columns'])}") time_partitioning = result.get("time_partitioning") if time_partitioning: - field = time_partitioning.get("field") - suffix = f" on {field}" if field else " (ingestion time)" - formatter.console.print(f" Time partitioning: {time_partitioning['type']}{suffix}") + formatter.console.print( + f" Time partitioning: {format_time_partitioning(time_partitioning)}" + ) range_partitioning = result.get("range_partitioning") if range_partitioning: - bounds = range_partitioning.get("range") or {} - bounds_suffix = "" - if bounds: - bounds_suffix = ( - f" [{bounds.get('start')}, {bounds.get('end')})" - f" step {bounds.get('interval')}" - ) formatter.console.print( - f" Range partitioning: {range_partitioning['field']}{bounds_suffix}" + f" Range partitioning: {format_range_partitioning(range_partitioning)}" ) clustering = result.get("clustering") if clustering: diff --git a/src/keboola_agent_cli/services/_table_detail.py b/src/keboola_agent_cli/services/_table_detail.py new file mode 100644 index 00000000..fa8f7e45 --- /dev/null +++ b/src/keboola_agent_cli/services/_table_detail.py @@ -0,0 +1,125 @@ +"""Pure assembly of the `storage table-detail` response payload. + +`StorageService.get_table_detail()` is two things stacked: an I/O step (resolve +the project, fetch `GET /v2/storage/tables/{id}`) and a pure transform of the +resource into kbagent's stable field names. Only the first needs a client, so +the second lives here -- the split CONTRIBUTING.md prescribes for a service that +mixes orchestration with parsing, and it keeps `storage_service.py` shrinking +against its grandfathered file-size ceiling. + +The transform is an explicit allowlist, which is deliberate (the API resource is +large and its legacy corners are not worth re-exporting) but has a failure mode +worth naming: a field the API adds is invisible until someone adds it here. +Issue #621 was exactly that -- `definition`, the only readable record of a +BigQuery table's partition/cluster layout, was dropped for as long as the +command existed. +""" + +from __future__ import annotations + +from typing import Any + + +def _split_descriptions(raw_metadata: list[dict[str, Any]]) -> tuple[str, dict[str, str]]: + """Pull the user table description and per-column descriptions out of metadata. + + Keboola has no user-writable column-metadata endpoint, so column descriptions + are stored as `KBC.column.{name}.description` rows in the *table's* metadata + list rather than alongside the column. + """ + description = "" + col_descriptions: dict[str, str] = {} + for entry in raw_metadata: + key = entry.get("key", "") + if key == "KBC.description" and entry.get("provider") == "user": + description = entry.get("value", "") or "" + elif key.startswith("KBC.column.") and key.endswith(".description"): + col_name = key[len("KBC.column.") : -len(".description")] + col_descriptions[col_name] = entry.get("value", "") or "" + return description, col_descriptions + + +def _column_details( + columns: list[str], + column_metadata: dict[str, list[dict[str, Any]]], + col_descriptions: dict[str, str], +) -> list[dict[str, Any]]: + """Build the per-column view from `columnMetadata`'s KBC.datatype.* rows. + + Types are read from `columnMetadata`, never from `definition` -- the latter + has really been served as `[]` (see tests/test_storage_empty_definition.py). + """ + details: list[dict[str, Any]] = [] + for col in columns: + col_info: dict[str, Any] = {"name": col} + for entry in column_metadata.get(col, []): + key = entry.get("key", "") + value = entry.get("value", "") + if key == "KBC.datatype.basetype": + col_info["type"] = value + elif key == "KBC.datatype.type": + # Native backend type (e.g. "VARCHAR", "NUMBER", "TIMESTAMP_TZ") + # -- distinct from the Keboola basetype it maps to. + col_info["native_type"] = value + elif key == "KBC.datatype.length": + # Length as stored: "40", "18,2", "255", ... + col_info["length"] = value + elif key == "KBC.datatype.nullable": + col_info["nullable"] = value == "1" + elif key == "KBC.datatype.default": + col_info["default"] = value + if col in col_descriptions: + col_info["description"] = col_descriptions[col] + details.append(col_info) + return details + + +def build_table_detail(alias: str, table_id: str, table: dict[str, Any]) -> dict[str, Any]: + """Map a Storage API table resource onto kbagent's `table-detail` payload. + + Args: + alias: Project alias the table was fetched from. + table_id: The requested table ID, used when the resource omits its own. + table: Raw `GET /v2/storage/tables/{id}` resource. + + Returns: + The `storage table-detail` payload, `definition` included. + """ + columns = table.get("columns", []) + raw_metadata: list[dict[str, Any]] = table.get("metadata", []) + description, col_descriptions = _split_descriptions(raw_metadata) + + return { + "project_alias": alias, + "table_id": table.get("id", table_id), + "name": table.get("name", ""), + "display_name": table.get("displayName", ""), + "bucket_id": table.get("bucket", {}).get("id", ""), + # Storage backend of the owning bucket (e.g. "snowflake", + # "bigquery"). Consumers: the web UI keys BigQuery-only features + # (repartition) off it, and type resolution picks the matching + # INFORMATION_SCHEMA dialect for alias / linked tables. + "backend": table.get("bucket", {}).get("backend", ""), + "description": description, + "columns": columns, + "column_details": _column_details( + columns, table.get("columnMetadata", {}), col_descriptions + ), + "primary_key": table.get("primaryKey", []), + # API may return null on empty tables; coerce to 0. + "rows_count": table.get("rowsCount") or 0, + "data_size_bytes": table.get("dataSizeBytes") or 0, + "is_alias": table.get("isAlias", False), + "last_import_date": table.get("lastImportDate", ""), + "last_change_date": table.get("lastChangeDate", ""), + "created": table.get("created", ""), + "metadata": raw_metadata, + # Passed through verbatim (issue #621). Present on EVERY table-detail + # response -- an untyped table gets one too -- so `None` here means the + # stack omitted the key, never "this table is untyped". For a BigQuery + # table this is the only readable record of the registered + # timePartitioning / rangePartitioning / clustering layout, plus + # `requirePartitionFilter` and an unbounded `partitions[]` list. Not + # re-shaped: trimming an API field is the bug this key exists to fix. + "definition": table.get("definition"), + } diff --git a/src/keboola_agent_cli/services/storage_service.py b/src/keboola_agent_cli/services/storage_service.py index ebcc2977..97d1946e 100644 --- a/src/keboola_agent_cli/services/storage_service.py +++ b/src/keboola_agent_cli/services/storage_service.py @@ -14,6 +14,7 @@ from ..constants import STORAGE_BRANCHES_FEATURE from ..errors import ConfigError, ErrorCode, KeboolaApiError from ..models import ProjectConfig +from ._table_detail import build_table_detail from .base import BaseService logger = logging.getLogger(__name__) @@ -639,7 +640,8 @@ def get_table_detail( branch_id: If set, target a specific dev branch. Returns: - Dict with table metadata, columns, and size info. + Dict with table metadata, columns, size info and the raw Storage API + ``definition`` (the BigQuery partition/cluster layout lives there). """ projects = self.resolve_projects([alias]) project = projects[alias] @@ -650,69 +652,7 @@ def get_table_detail( finally: client.close() - columns = table.get("columns", []) - column_metadata = table.get("columnMetadata", {}) - raw_metadata: list[dict[str, Any]] = table.get("metadata", []) - - # Extract description and per-column descriptions from metadata list - description = "" - col_descriptions: dict[str, str] = {} - for m in raw_metadata: - key = m.get("key", "") - if key == "KBC.description" and m.get("provider") == "user": - description = m.get("value", "") or "" - elif key.startswith("KBC.column.") and key.endswith(".description"): - col_name = key[len("KBC.column.") : -len(".description")] - col_descriptions[col_name] = m.get("value", "") or "" - - column_details = [] - for col in columns: - col_info: dict[str, Any] = {"name": col} - meta = column_metadata.get(col, []) - for m in meta: - key = m.get("key", "") - value = m.get("value", "") - if key == "KBC.datatype.basetype": - col_info["type"] = value - elif key == "KBC.datatype.type": - # Native backend type (e.g. "VARCHAR", "NUMBER", "TIMESTAMP_TZ") - # -- distinct from the Keboola basetype it maps to. - col_info["native_type"] = value - elif key == "KBC.datatype.length": - # Length as stored: "40", "18,2", "255", ... - col_info["length"] = value - elif key == "KBC.datatype.nullable": - col_info["nullable"] = value == "1" - elif key == "KBC.datatype.default": - col_info["default"] = value - if col in col_descriptions: - col_info["description"] = col_descriptions[col] - column_details.append(col_info) - - return { - "project_alias": alias, - "table_id": table.get("id", table_id), - "name": table.get("name", ""), - "display_name": table.get("displayName", ""), - "bucket_id": table.get("bucket", {}).get("id", ""), - # Storage backend of the owning bucket (e.g. "snowflake", - # "bigquery"). Consumers: the web UI keys BigQuery-only features - # (repartition) off it, and type resolution picks the matching - # INFORMATION_SCHEMA dialect for alias / linked tables. - "backend": table.get("bucket", {}).get("backend", ""), - "description": description, - "columns": columns, - "column_details": column_details, - "primary_key": table.get("primaryKey", []), - # API may return null on empty tables; coerce to 0. - "rows_count": table.get("rowsCount") or 0, - "data_size_bytes": table.get("dataSizeBytes") or 0, - "is_alias": table.get("isAlias", False), - "last_import_date": table.get("lastImportDate", ""), - "last_change_date": table.get("lastChangeDate", ""), - "created": table.get("created", ""), - "metadata": raw_metadata, - } + return build_table_detail(alias, table_id, table) def list_tables( self, diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 05c54aac..ffc35d02 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -950,6 +950,28 @@ def _test_create_table_from_source(self, bucket_id: str, source_table_id: str) - "--yes", ) + # Verify the swap through `definition` (issue #621). The table ID is + # identical whether or not the swap happened -- the registered layout is + # the only thing that tells the two apart, and before 0.88.0 kbagent + # could not read it at all. `create-table`'s own output is no substitute: + # it echoes the layout that was REQUESTED. + swapped = self._run_ok( + "storage", + "table-detail", + "--project", + self.alias, + "--table-id", + source_table_id, + "--branch", + str(branch_id), + )["data"] + definition = swapped["definition"] + assert isinstance(definition, dict), f"Expected a definition object, got {definition!r}" + assert definition.get("clustering", {}).get("fields") == ["id"], ( + "The clustering layout applied by create-table is not readable on the " + f"production name after the swap: {definition.get('clustering')!r}" + ) + def _test_upload_table(self, table_id: str) -> None: """Upload CSV data to the table.""" csv_path = _create_test_csv(self.data_dir, rows=5) @@ -1299,6 +1321,14 @@ def _test_table_listing(self, bucket_id: str, table_id: str) -> None: assert "id" in col_names assert "name" in col_names assert "value" in col_names + # The raw Storage API `definition` is passed through (issue #621). It is + # present on every table-detail response regardless of backend or typing, + # so the KEY is the contract here; the BigQuery layout inside it is + # asserted in _test_create_table_from_source where a layout is actually set. + assert "definition" in detail, ( + "table-detail dropped `definition` -- the BigQuery partition/cluster " + "layout is unreadable without it (issue #621)" + ) def _test_download_table(self, table_id: str) -> None: """Download table data and verify round-trip integrity.""" diff --git a/tests/test_storage_table_definition.py b/tests/test_storage_table_definition.py new file mode 100644 index 00000000..670bcfaa --- /dev/null +++ b/tests/test_storage_table_definition.py @@ -0,0 +1,253 @@ +"""`storage table-detail` must surface the table's `definition` (issue #621). + +The Storage API's table-detail response (`GET /v2/storage/tables/{id}`) carries a +`definition` object. For a BigQuery table that object is the only readable record +of the registered `timePartitioning` / `rangePartitioning` / `clustering` layout. +`StorageService.get_table_detail()` assembled its return value as an explicit +field allowlist and `definition` was not on it, so the layout that +`storage create-table` + `storage swap-tables` had just applied was invisible: +the write half of the documented repartition flow was supported, the verify half +was not. `create-table` is no substitute -- its JSON echoes the layout the caller +*requested*, and its `--if-not-exists` skip path nulls those keys outright. + +Two properties of the upstream response shape these tests. Both were read off +connection's `TableDetailResponseProvider::getResponseArray()` and +`BigqueryDriverConfig::extendTableDefinitionResponse()`: + +1. `definition` is set on EVERY table-detail response, typed table or not -- an + untyped one gets a `definition` built by + `createUntypedTableDefinitionResponseFromMetadata()`. So a missing/None + `definition` never means "untyped", and the human-mode render must key off the + layout keys themselves, never off the presence of `definition`. +2. When partitioning is set the response also carries `requirePartitionFilter` + and `partitions[]` -- one entry per physical partition, read from + `INFORMATION_SCHEMA.PARTITIONS`. That list is unbounded (a DAY-partitioned + table holding three years of data has ~1,100 entries), so human mode prints + its length and never its contents. JSON mode passes it through unchanged: + re-dropping an API field is the exact bug this issue is about. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.models import AppConfig, ProjectConfig +from keboola_agent_cli.services.storage_service import StorageService + +runner = CliRunner() + +# Canonical fake test token (see tests/helpers conventions). +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + +TABLE_ID = "out.c-my-bucket.my-table" + + +def _make_store(tmp_path: Path) -> ConfigStore: + """Config store with a single project aliased 'test'.""" + config_dir = tmp_path / "config" + config_dir.mkdir(exist_ok=True) + store = ConfigStore(config_dir=config_dir) + store.save( + AppConfig( + projects={ + "test": ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + ) + }, + ) + ) + return store + + +def _table_resource(definition: Any) -> dict[str, Any]: + """A minimal table-detail payload carrying the supplied `definition`.""" + return { + "id": TABLE_ID, + "name": "my-table", + "displayName": "my-table", + "bucket": {"id": "out.c-my-bucket", "backend": "bigquery"}, + "primaryKey": ["id"], + "rowsCount": 6290737, + "dataSizeBytes": 4096, + "isAlias": False, + "lastImportDate": "2026-08-19T17:26:18+0200", + "lastChangeDate": "2026-08-19T17:26:18+0200", + "created": "2026-08-01T00:00:00+0200", + "columns": ["id", "created_at", "tenant_id", "country"], + "columnMetadata": {}, + "metadata": [], + "definition": definition, + } + + +BIGQUERY_LAYOUT: dict[str, Any] = { + "primaryKeysNames": ["id"], + "columns": [{"name": "id", "definition": {"type": "INTEGER"}}], + "timePartitioning": {"type": "DAY", "field": "created_at"}, + "clustering": {"fields": ["tenant_id", "country"]}, + "requirePartitionFilter": True, + "partitions": [ + { + "partitionId": "20260819", + "rowsNumber": "42", + "lastModifiedTime": "1755620778000", + "storageTier": "ACTIVE", + }, + { + "partitionId": "20260820", + "rowsNumber": "43", + "lastModifiedTime": "1755707178000", + "storageTier": "ACTIVE", + }, + ], +} + +RANGE_LAYOUT: dict[str, Any] = { + "primaryKeysNames": ["id"], + "columns": [{"name": "id", "definition": {"type": "INTEGER"}}], + "rangePartitioning": { + "field": "order_id", + "range": {"start": "0", "end": "1000000", "interval": "1000"}, + }, +} + +# What connection returns for a table with no typed definition at all: the key is +# present, the layout keys are not. +UNTYPED_DEFINITION: dict[str, Any] = { + "primaryKeysNames": [], + "columns": [{"name": "id", "definition": {}}], +} + + +def _service(tmp_path: Path, definition: Any) -> StorageService: + client = MagicMock() + client.get_table_detail.return_value = _table_resource(definition) + return StorageService( + config_store=_make_store(tmp_path), + client_factory=lambda _u, _t: client, + ) + + +class TestServicePassthrough: + """The service must stop filtering `definition` out of the response.""" + + def test_bigquery_layout_reaches_the_caller_verbatim(self, tmp_path: Path) -> None: + result = _service(tmp_path, BIGQUERY_LAYOUT).get_table_detail("test", TABLE_ID) + + assert result["definition"] == BIGQUERY_LAYOUT + + def test_partitions_list_is_not_re_dropped(self, tmp_path: Path) -> None: + """`partitions[]` is large but passing it on is the point of the issue.""" + result = _service(tmp_path, BIGQUERY_LAYOUT).get_table_detail("test", TABLE_ID) + + assert result["definition"]["partitions"] == BIGQUERY_LAYOUT["partitions"] + + def test_untyped_table_still_carries_a_definition(self, tmp_path: Path) -> None: + """A missing layout is NOT a missing `definition` -- connection always sends one.""" + result = _service(tmp_path, UNTYPED_DEFINITION).get_table_detail("test", TABLE_ID) + + assert result["definition"] == UNTYPED_DEFINITION + + def test_absent_definition_becomes_none(self, tmp_path: Path) -> None: + """Defensive: an older stack that omits the key must not KeyError.""" + client = MagicMock() + payload = _table_resource(None) + del payload["definition"] + client.get_table_detail.return_value = payload + service = StorageService( + config_store=_make_store(tmp_path), + client_factory=lambda _u, _t: client, + ) + + assert service.get_table_detail("test", TABLE_ID)["definition"] is None + + def test_empty_array_wire_shape_survives(self, tmp_path: Path) -> None: + """The SUPPORT-16581 shape: PHP serialized an empty definition as `[]`. + + See tests/test_storage_empty_definition.py -- a Storage API deploy emitted + `"definition":[]` and broke the Go CLI's strict decoder. Passing the value + through must not raise, and must not coerce it into something else. + """ + result = _service(tmp_path, []).get_table_detail("test", TABLE_ID) + + assert result["definition"] == [] + + +def _invoke_detail(tmp_path: Path, definition: Any, *, json_mode: bool = False) -> Any: + store = _make_store(tmp_path) + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.StorageService") as MockSvc, + ): + MockStore.return_value = store + client = MagicMock() + client.get_table_detail.return_value = _table_resource(definition) + MockSvc.return_value.get_table_detail.side_effect = lambda **kwargs: StorageService( + config_store=store, client_factory=lambda _u, _t: client + ).get_table_detail(**kwargs) + argv = ["storage", "table-detail", "--project", "test", "--table-id", TABLE_ID] + return runner.invoke(app, (["--json"] if json_mode else []) + argv) + + +class TestHumanOutput: + """Human mode renders the layout; without one it prints nothing new.""" + + def test_time_partitioning_and_clustering_are_rendered(self, tmp_path: Path) -> None: + result = _invoke_detail(tmp_path, BIGQUERY_LAYOUT) + + assert result.exit_code == 0, result.output + assert "Time partitioning: DAY on created_at" in result.output + assert "Clustering: tenant_id, country" in result.output + + def test_partitions_are_summarized_not_listed(self, tmp_path: Path) -> None: + """Printing 1,100 partition rows would bury the answer the user asked for.""" + result = _invoke_detail(tmp_path, BIGQUERY_LAYOUT) + + assert "Partitions: 2" in result.output + assert "20260819" not in result.output + + def test_require_partition_filter_is_reported(self, tmp_path: Path) -> None: + """A query without a filter fails outright -- worth one line.""" + result = _invoke_detail(tmp_path, BIGQUERY_LAYOUT) + + assert "Partition filter required: yes" in result.output + + def test_range_partitioning_is_rendered_with_its_bounds(self, tmp_path: Path) -> None: + result = _invoke_detail(tmp_path, RANGE_LAYOUT) + + assert result.exit_code == 0, result.output + assert "Range partitioning: order_id [0, 1000000) step 1000" in result.output + + def test_no_layout_prints_no_layout_lines(self, tmp_path: Path) -> None: + """Snowflake / untyped output stays as it was -- the whole block is guarded.""" + result = _invoke_detail(tmp_path, UNTYPED_DEFINITION) + + assert result.exit_code == 0, result.output + for absent in ("partitioning", "Clustering", "Partitions:"): + assert absent not in result.output + + def test_non_dict_definition_does_not_crash_the_render(self, tmp_path: Path) -> None: + """`definition: []` is a shape the API has really emitted; `.get` would blow up.""" + result = _invoke_detail(tmp_path, []) + + assert result.exit_code == 0, result.output + assert "Clustering" not in result.output + + +class TestJsonOutput: + def test_definition_passes_through_json_mode(self, tmp_path: Path) -> None: + import json + + result = _invoke_detail(tmp_path, BIGQUERY_LAYOUT, json_mode=True) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + data = payload.get("data", payload) + assert data["definition"] == BIGQUERY_LAYOUT diff --git a/uv.lock b/uv.lock index 2d96ab21..6f789a5f 100644 --- a/uv.lock +++ b/uv.lock @@ -581,7 +581,7 @@ wheels = [ [[package]] name = "keboola-cli" -version = "0.87.0" +version = "0.88.0" source = { editable = "." } dependencies = [ { name = "croniter" }, From c0c47d1c042d15271566e6625aa89f29c4291a1a Mon Sep 17 00:00:00 2001 From: Petr Date: Fri, 21 Aug 2026 23:04:56 +0200 Subject: [PATCH 3/4] fix(changelog): lead the #621 note with a self-contained first sentence `kbagent changelog` and the "What's new" banner show only the first sentence, capped at 160 chars; the original opener was cut off mid-clause. --- src/keboola_agent_cli/changelog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index d4e0e40c..c131ba3c 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -25,8 +25,8 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { "0.88.0": [ - "Fix (#621): `storage table-detail` now surfaces the Storage API's `definition` " - "object, which for a BigQuery table is the only readable record of the registered " + "Fix (#621): `storage table-detail` now returns the Storage API's `definition` " + "object. On BigQuery that is the only readable record of a table's registered " "`timePartitioning` / `rangePartitioning` / `clustering` layout. The service built " "its response as a field allowlist and `definition` was never on it, so the layout " "that `storage create-table --source-table-id ... --time-partitioning-field ...` " From 767c8e7c7bcf25af1e557bcb450b9926c44792af Mon Sep 17 00:00:00 2001 From: Petr Date: Fri, 21 Aug 2026 23:14:10 +0200 Subject: [PATCH 4/4] refactor(storage): name the description split with a dataclass (review NB-1) `_split_descriptions` was extracted in the previous commit and returned a bare `tuple[str, dict[str, str]]` -- a scalar table description and a per-column description map, disambiguated only by position. CONTRIBUTING.md's "Return values -- name them with dataclasses, not tuples" calls out exactly this case ("even two-element tuples should use a dataclass when the values are semantically distinct"), and the rule applies since the function is new here, not grandfathered inline code. Returns a frozen `Descriptions(table=..., columns=...)` instead. No behavior change; covered by the existing table-detail and describe-service tests. --- .../services/_table_detail.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/keboola_agent_cli/services/_table_detail.py b/src/keboola_agent_cli/services/_table_detail.py index fa8f7e45..6e389a87 100644 --- a/src/keboola_agent_cli/services/_table_detail.py +++ b/src/keboola_agent_cli/services/_table_detail.py @@ -17,26 +17,35 @@ from __future__ import annotations +from dataclasses import dataclass, field from typing import Any -def _split_descriptions(raw_metadata: list[dict[str, Any]]) -> tuple[str, dict[str, str]]: +@dataclass(frozen=True) +class Descriptions: + """The user-authored descriptions carried in a table's metadata list.""" + + table: str = "" + columns: dict[str, str] = field(default_factory=dict) + + +def _split_descriptions(raw_metadata: list[dict[str, Any]]) -> Descriptions: """Pull the user table description and per-column descriptions out of metadata. Keboola has no user-writable column-metadata endpoint, so column descriptions are stored as `KBC.column.{name}.description` rows in the *table's* metadata list rather than alongside the column. """ - description = "" + table_description = "" col_descriptions: dict[str, str] = {} for entry in raw_metadata: key = entry.get("key", "") if key == "KBC.description" and entry.get("provider") == "user": - description = entry.get("value", "") or "" + table_description = entry.get("value", "") or "" elif key.startswith("KBC.column.") and key.endswith(".description"): col_name = key[len("KBC.column.") : -len(".description")] col_descriptions[col_name] = entry.get("value", "") or "" - return description, col_descriptions + return Descriptions(table=table_description, columns=col_descriptions) def _column_details( @@ -87,7 +96,7 @@ def build_table_detail(alias: str, table_id: str, table: dict[str, Any]) -> dict """ columns = table.get("columns", []) raw_metadata: list[dict[str, Any]] = table.get("metadata", []) - description, col_descriptions = _split_descriptions(raw_metadata) + descriptions = _split_descriptions(raw_metadata) return { "project_alias": alias, @@ -100,10 +109,10 @@ def build_table_detail(alias: str, table_id: str, table: dict[str, Any]) -> dict # (repartition) off it, and type resolution picks the matching # INFORMATION_SCHEMA dialect for alias / linked tables. "backend": table.get("bucket", {}).get("backend", ""), - "description": description, + "description": descriptions.table, "columns": columns, "column_details": _column_details( - columns, table.get("columnMetadata", {}), col_descriptions + columns, table.get("columnMetadata", {}), descriptions.columns ), "primary_key": table.get("primaryKey", []), # API may return null on empty tables; coerce to 0.