From b443a77992313e6c9b91c63e064691bb7ffdc95e Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:15:23 +0200 Subject: [PATCH 01/17] fix(permissions): fail-closed MCP tool classification (#478 phase 0) Unknown tool names now classify as 'destructive' (strictest category) instead of falling through to 'read'. Real catalog tools run_job, run_sync_action, modify_*, deploy_* move from read to write -- they were passing --deny-writes and fanning out to every configured project. classify_mcp_tool in permissions.py is now the single source of truth; mcp_service dispatch derives multi_project from it (unknown tools stay single-project). --- src/keboola_agent_cli/permissions.py | 56 ++++++++++++++++--- src/keboola_agent_cli/services/mcp_service.py | 24 ++++---- tests/test_mcp_service.py | 32 +++++++++-- tests/test_permissions.py | 38 +++++++++++++ 4 files changed, 124 insertions(+), 26 deletions(-) diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index b279aa01..56997d5e 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -328,22 +328,53 @@ "permissions.check": "read", } -# Prefixes for classifying MCP tools (mirrors mcp_service.py WRITE_PREFIXES) -_MCP_WRITE_PREFIXES = ("create_", "update_", "add_", "set_") -_MCP_DESTRUCTIVE_PREFIXES = ("delete_", "remove_") +# Prefixes for classifying MCP tools. Single source of truth -- also drives +# mcp_service.py multi-project dispatch (read tools fan out, others don't). +# Order of evaluation: destructive > read > write > fail-closed default. +_MCP_DESTRUCTIVE_PREFIXES = ("delete_", "remove_", "truncate_", "drop_", "purge_") +# Read markers cover the whole current keboola-mcp-server catalog: +# get_*, docs_query, query_data, search / search_*, find_component_id, +# validate_semantic_query, list_* (future-proof). +_MCP_READ_PREFIXES = ("get_", "list_", "find_", "search_", "docs_", "query_", "validate_", "read_") +_MCP_READ_EXACT = frozenset({"search"}) +_MCP_WRITE_PREFIXES = ( + "create_", + "update_", + "add_", + "set_", + "modify_", + "deploy_", + "run_", + "start_", + "stop_", + "cancel_", + "upload_", + "import_", + "push_", + "refresh_", +) def classify_mcp_tool(tool_name: str) -> str: - """Classify an MCP tool by its name prefix. + """Classify an MCP tool by its name prefix -- FAIL-CLOSED (issue #478). + + A tool that matches no known read or write marker is classified + ``'destructive'`` (the strictest category), so both ``--deny-writes`` + and ``--deny-destructive`` block it and multi-project dispatch never + fans it out. Before 0.73.0 unknown tools fell through to ``'read'``, + which allowed e.g. ``run_job`` or a hypothetical ``truncate_table`` + to pass a write-deny firewall and run on every configured project. Returns: Risk category: 'read', 'write', or 'destructive'. """ if tool_name.startswith(_MCP_DESTRUCTIVE_PREFIXES): return "destructive" + if tool_name in _MCP_READ_EXACT or tool_name.startswith(_MCP_READ_PREFIXES): + return "read" if tool_name.startswith(_MCP_WRITE_PREFIXES): return "write" - return "read" + return "destructive" def _matches_pattern(operation: str, pattern: str) -> bool: @@ -446,12 +477,21 @@ def list_operations(self) -> list[dict[str, str]]: # MCP tool categories (virtual entries for reference) mcp_categories = [ - ("tool:read", "read", "All MCP read tools (get_*, list_*, search, find_*, docs_query)"), - ("tool:write", "write", "All MCP write tools (create_*, update_*, add_*, set_*)"), + ( + "tool:read", + "read", + "All MCP read tools (get_*, list_*, search*, find_*, docs_*, query_*, validate_*)", + ), + ( + "tool:write", + "write", + "All MCP write tools (create_*, update_*, add_*, set_*, modify_*, deploy_*, run_*)", + ), ( "tool:destructive", "destructive", - "All MCP destructive tools (delete_*, remove_*)", + "All MCP destructive tools (delete_*, remove_*, truncate_*, drop_*, purge_*) " + "+ any tool matching no known prefix (fail-closed)", ), ] for name, category, description in mcp_categories: diff --git a/src/keboola_agent_cli/services/mcp_service.py b/src/keboola_agent_cli/services/mcp_service.py index 9cbbcf45..8ef9cae8 100644 --- a/src/keboola_agent_cli/services/mcp_service.py +++ b/src/keboola_agent_cli/services/mcp_service.py @@ -37,7 +37,7 @@ ) from ..errors import ConfigError from ..models import ProjectConfig -from ..permissions import PermissionEngine +from ..permissions import PermissionEngine, classify_mcp_tool from .base import BaseService logger = logging.getLogger(__name__) @@ -64,15 +64,10 @@ async def _semaphored(sem: asyncio.Semaphore, coro: Any) -> Any: return await coro -# Prefixes that indicate write/mutating tools -WRITE_PREFIXES = ( - "create_", - "update_", - "delete_", - "add_", - "remove_", - "set_", -) +# Tool risk classification lives in permissions.classify_mcp_tool -- the single +# source of truth shared with the permission firewall (issue #478). Dispatch is +# fail-closed: only tools classified 'read' fan out across all projects; an +# unknown tool is treated as destructive and targets a single project. # Tools that auto-expand when a required param is missing. # Maps tool_name -> config dict. When the param is absent from user input, @@ -88,8 +83,13 @@ async def _semaphored(sem: asyncio.Semaphore, coro: Any) -> Any: def _is_write_tool(tool_name: str) -> bool: - """Determine if a tool name indicates a write/mutating operation.""" - return tool_name.startswith(WRITE_PREFIXES) + """True when the tool must NOT fan out across projects. + + Everything except a known-read classification counts as a write for + dispatch purposes -- unknown tools are fail-closed to single-project + so a mutating tool can never run on every configured project at once. + """ + return classify_mcp_tool(tool_name) != "read" def detect_mcp_server_command() -> list[str] | None: diff --git a/tests/test_mcp_service.py b/tests/test_mcp_service.py index b7e5f61b..7e12925d 100644 --- a/tests/test_mcp_service.py +++ b/tests/test_mcp_service.py @@ -120,20 +120,40 @@ def test_write_prefixes_detected(self, tool_name: str) -> None: "list_configs", "get_config", "search", + "search_semantic_context", "docs_query", + "query_data", + "validate_semantic_query", "find_component_id", + ], + ) + def test_read_tool_names_not_detected(self, tool_name: str) -> None: + """Known-read tool names fan out (not classified as write for dispatch).""" + assert _is_write_tool(tool_name) is False + + @pytest.mark.parametrize( + "tool_name", + [ + # Real catalog tools that the old prefix list mis-classified as + # reads and fanned out to every project (issue #478). + "run_job", + "run_sync_action", + "deploy_data_app", + "modify_flow", + # Unknown names fail closed to single-project dispatch. "describe_table", "show_bucket", "retrieve_logs", + "truncate_table", ], ) - def test_read_tool_names_not_detected(self, tool_name: str) -> None: - """Tool names that do not start with write prefixes are read tools.""" - assert _is_write_tool(tool_name) is False + def test_unknown_and_mutating_tools_are_write_for_dispatch(self, tool_name: str) -> None: + """Fail-closed (issue #478): anything not known-read stays single-project.""" + assert _is_write_tool(tool_name) is True - def test_empty_tool_name(self) -> None: - """Empty string is not a write tool.""" - assert _is_write_tool("") is False + def test_empty_tool_name_fails_closed(self) -> None: + """Empty string matches no read marker -> treated as write for dispatch.""" + assert _is_write_tool("") is True # --------------------------------------------------------------------------- diff --git a/tests/test_permissions.py b/tests/test_permissions.py index ea082f16..b0ecff0a 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -29,9 +29,47 @@ def test_write_tools(self) -> None: assert classify_mcp_tool("add_tag") == "write" assert classify_mcp_tool("set_metadata") == "write" + def test_mutating_tools_previously_misclassified_as_read(self) -> None: + """Issue #478: run_*/modify_*/deploy_* used to fall through to 'read'.""" + assert classify_mcp_tool("run_job") == "write" + assert classify_mcp_tool("run_sync_action") == "write" + assert classify_mcp_tool("modify_flow") == "write" + assert classify_mcp_tool("modify_streamlit_data_app") == "write" + assert classify_mcp_tool("deploy_data_app") == "write" + def test_destructive_tools(self) -> None: assert classify_mcp_tool("delete_config") == "destructive" assert classify_mcp_tool("remove_tag") == "destructive" + assert classify_mcp_tool("truncate_table") == "destructive" + assert classify_mcp_tool("drop_bucket") == "destructive" + assert classify_mcp_tool("purge_files") == "destructive" + + def test_unknown_tools_fail_closed_to_destructive(self) -> None: + """Issue #478: a tool matching no known prefix must NOT pass as a read. + + 'destructive' is the strictest category: both --deny-writes and + --deny-destructive policies block it. + """ + assert classify_mcp_tool("frobnicate_project") == "destructive" + assert classify_mcp_tool("describe_table") == "destructive" + assert classify_mcp_tool("") == "destructive" + + def test_current_catalog_read_tools(self) -> None: + """Every read tool in today's keboola-mcp-server catalog stays 'read'.""" + for name in ( + "get_project_info", + "get_configs", + "get_flows", + "get_data_apps", + "get_semantic_context", + "search", + "search_semantic_context", + "find_component_id", + "docs_query", + "query_data", + "validate_semantic_query", + ): + assert classify_mcp_tool(name) == "read", name class TestOperationRegistry: From 9d86e9cb959e3e3832d0377bf6c060d1bdc5271e Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:22:29 +0200 Subject: [PATCH 02/17] fix(permissions): per-tool session firewall in tool call + registry entries for parity commands (#478) The tool group callback checks only the coarse 'tool.call' operation and the service-level check reads only the persisted policy, so a session --deny-destructive still allowed 'tool call delete_bucket'. The command now checks the session engine against 'tool:' (fail-closed classifier). Registers operations for the incoming #390 parity commands (docs.query, config.examples, component.sync-action, semantic-layer.schema, transformation.*, flow.examples). --- plugins/kbagent/skills/kbagent/SKILL.md | 2 + src/keboola_agent_cli/commands/tool.py | 17 ++++- src/keboola_agent_cli/permissions.py | 12 ++++ tests/test_tool_call_permissions.py | 82 +++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 tests/test_tool_call_permissions.py diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 520859da..e9400c0e 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -234,6 +234,7 @@ When working inside a git repository or project directory, run `kbagent init` (o | Promote a model from one project to another (NEW + overwrite CHANGED; never deletes) | `kbagent semantic-layer promote --from-project FROM-PROJECT --to-project TO-PROJECT` | | Replay a snapshot into a project. | `kbagent semantic-layer import --project PROJECT --file FILE` | | Show the entities in a semantic-layer model | `kbagent semantic-layer show --project PROJECT` | +| Fetch the server-side JSON Schema of semantic object types | `kbagent semantic-layer schema --project PROJECT` | | Snapshot a semantic-layer model to a self-describing JSON file | `kbagent semantic-layer export --project PROJECT` | | Diff two semantic-layer snapshots (project↔project, project↔file, file↔file) | `kbagent semantic-layer diff` | | Validate a semantic-layer model | `kbagent semantic-layer validate --project PROJECT` | @@ -266,6 +267,7 @@ When working inside a git repository or project directory, run `kbagent init` (o | Promote a model from one project to another (NEW + overwrite CHANGED; never deletes) | `kbagent sl promote --from-project FROM-PROJECT --to-project TO-PROJECT` | | Replay a snapshot into a project. | `kbagent sl import --project PROJECT --file FILE` | | Show the entities in a semantic-layer model | `kbagent sl show --project PROJECT` | +| Fetch the server-side JSON Schema of semantic object types | `kbagent sl schema --project PROJECT` | | Snapshot a semantic-layer model to a self-describing JSON file | `kbagent sl export --project PROJECT` | | Diff two semantic-layer snapshots (project↔project, project↔file, file↔file) | `kbagent sl diff` | | Validate a semantic-layer model | `kbagent sl validate --project PROJECT` | diff --git a/src/keboola_agent_cli/commands/tool.py b/src/keboola_agent_cli/commands/tool.py index b51c3b78..9ed29445 100644 --- a/src/keboola_agent_cli/commands/tool.py +++ b/src/keboola_agent_cli/commands/tool.py @@ -11,9 +11,10 @@ import typer from ..config_store import ConfigStore -from ..errors import ConfigError, ErrorCode +from ..errors import ConfigError, ErrorCode, PermissionDeniedError from ..output import OutputFormatter, format_tool_result, format_tools_table from ._helpers import ( + EXIT_PERMISSION_DENIED, check_cli_permission, emit_project_warnings, get_formatter, @@ -146,6 +147,20 @@ def tool_call( service = get_service(ctx, "mcp_service") config_store: ConfigStore = ctx.obj["config_store"] + # Per-tool session firewall check (issue #478): the group callback only + # checks the coarse 'tool.call' operation, and the service-level check + # sees just the PERSISTED policy -- so without this, a session-only + # --deny-destructive would still let 'tool call delete_bucket' through. + # classify_mcp_tool is fail-closed: unknown tool names count as + # destructive and are blocked by --deny-writes / --deny-destructive. + engine = ctx.obj.get("permission_engine") + if engine is not None and engine.active: + try: + engine.check_or_raise(f"tool:{tool_name}") + except PermissionDeniedError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.PERMISSION_DENIED) + raise typer.Exit(code=EXIT_PERMISSION_DENIED) from None + validate_branch_requires_project(formatter, branch, project) # Auto-resolve active branch from config diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 56997d5e..c1f45886 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -55,6 +55,7 @@ "config.list": "read", "config.detail": "read", "config.search": "read", + "config.examples": "read", "config.update": "write", "config.set-default-bucket": "write", "config.rename": "write", @@ -141,6 +142,15 @@ # Component discovery "component.list": "read", "component.detail": "read", + # sync-action executes component-defined code (testConnection, ...) -- + # freeform action names, so conservatively a write (issue #395). + "component.sync-action": "write", + # Docs Q&A (issue #392) + "docs.query": "read", + # SQL transformation authoring (issue #396) + "transformation.create": "write", + "transformation.show": "read", + "transformation.edit": "write", # Developer Portal (since 0.48.0) # Developer Portal — top-level commands on `dev-portal` (the identity # sub-app's leaves are listed separately below under dev-portal.identity.*). @@ -222,6 +232,7 @@ "encrypt.values": "write", # Semantic layer (metastore) — new in 0.41.0 "semantic-layer.show": "read", + "semantic-layer.schema": "read", "semantic-layer.validate": "read", "semantic-layer.export": "read", "semantic-layer.diff": "read", @@ -300,6 +311,7 @@ "flow.list": "read", "flow.detail": "read", "flow.schema": "read", + "flow.examples": "read", "flow.validate": "read", "flow.new": "write", "flow.update": "write", diff --git a/tests/test_tool_call_permissions.py b/tests/test_tool_call_permissions.py new file mode 100644 index 00000000..a87a2b48 --- /dev/null +++ b/tests/test_tool_call_permissions.py @@ -0,0 +1,82 @@ +"""Per-tool session firewall checks in ``kbagent tool call`` (issue #478). + +The ``tool`` group callback checks only the coarse ``tool.call`` operation, +and the service-level check reads only the PERSISTED policy. The command +itself must therefore run the session engine against ``tool:`` so a +session-only ``--deny-writes`` / ``--deny-destructive`` blocks individual +tools -- with the fail-closed classifier treating unknown names as +destructive. +""" + +import json +from pathlib import Path + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app + +from .helpers import setup_single_project + +runner = CliRunner() + + +def _config_dir(tmp_path: Path) -> Path: + config_dir = tmp_path / "cfg" + config_dir.mkdir() + setup_single_project(config_dir, token="901-55555-fakeTestTokenDoNotUseXXXXXXXX") + return config_dir + + +class TestToolCallSessionFirewall: + def test_deny_destructive_blocks_delete_tool(self, tmp_path: Path) -> None: + result = runner.invoke( + app, + [ + "--json", + "--config-dir", + str(_config_dir(tmp_path)), + "--deny-destructive", + "tool", + "call", + "delete_bucket", + ], + ) + assert result.exit_code != 0 + payload = json.loads(result.stdout) + assert payload["error"]["code"] == "PERMISSION_DENIED" + assert "tool:delete_bucket" in payload["error"]["message"] + + def test_deny_destructive_blocks_unknown_tool_fail_closed(self, tmp_path: Path) -> None: + result = runner.invoke( + app, + [ + "--json", + "--config-dir", + str(_config_dir(tmp_path)), + "--deny-destructive", + "tool", + "call", + "frobnicate_project", + ], + ) + assert result.exit_code != 0 + payload = json.loads(result.stdout) + assert payload["error"]["code"] == "PERMISSION_DENIED" + + def test_deny_writes_blocks_run_job(self, tmp_path: Path) -> None: + """run_job passed --deny-writes before 0.73.0 (classified read).""" + result = runner.invoke( + app, + [ + "--json", + "--config-dir", + str(_config_dir(tmp_path)), + "--deny-writes", + "tool", + "call", + "run_job", + ], + ) + assert result.exit_code != 0 + payload = json.loads(result.stdout) + assert payload["error"]["code"] == "PERMISSION_DENIED" From e613ed4fe1e0e52b592b29ac9209e3508c7b0271 Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:25:31 +0200 Subject: [PATCH 03/17] feat(docs): kbagent docs query -- Keboola documentation Q&A (#392) Ports the docs_query MCP tool: AiServiceClient.docs_question (POST /docs/question), DocsService, thin docs command group. Live-verified against the AI service. Adds the 0.73.0 changelog entry. --- plugins/kbagent/skills/kbagent/SKILL.md | 3 + src/keboola_agent_cli/ai_client.py | 18 ++ src/keboola_agent_cli/changelog.py | 27 ++ src/keboola_agent_cli/cli.py | 5 + src/keboola_agent_cli/commands/docs.py | 81 ++++++ src/keboola_agent_cli/models.py | 21 ++ .../services/docs_service.py | 73 +++++ tests/test_ai_client.py | 49 ++++ tests/test_docs_cli.py | 273 ++++++++++++++++++ 9 files changed, 550 insertions(+) create mode 100644 src/keboola_agent_cli/commands/docs.py create mode 100644 src/keboola_agent_cli/services/docs_service.py create mode 100644 tests/test_docs_cli.py diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index e9400c0e..286c87cf 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -97,8 +97,10 @@ When working inside a git repository or project directory, run `kbagent init` (o | Rotate a token: generate a new value and invalidate the old one (secret shown once) | `kbagent token refresh --project PROJECT --token-id TOKEN-ID` | | List available components from connected projects | `kbagent component list` | | Show detailed information about a specific component | `kbagent component detail --component-id COMPONENT-ID` | +| Run a synchronous component action (e.g. | `kbagent component sync-action --component-id COMPONENT-ID --project PROJECT` | | List configurations from connected projects | `kbagent config list` | | Show detailed information about one or many configurations | `kbagent config detail --component-id COMPONENT-ID` | +| Show sample configuration JSON examples for a component | `kbagent config examples --component-id COMPONENT-ID` | | Search through configuration bodies for a string or pattern | `kbagent config search --query QUERY` | | Update a configuration's metadata and/or content | `kbagent config update --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | | Set or clear ``storage.output.default_bucket`` on a configuration | `kbagent config set-default-bucket --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | @@ -186,6 +188,7 @@ When working inside a git repository or project directory, run `kbagent init` (o | Check whether the configured token can use Kai (master token + AI Agent Chat) | `kbagent kai preflight` | | Fetch the full message history of a single Kai chat | `kbagent kai chat-detail --chat-id CHAT-ID` | | List recent Kai chat sessions | `kbagent kai history` | +| Ask the Keboola documentation a natural language question | `kbagent docs query ` | | List conditional flows (keboola.flow) across projects | `kbagent flow list` | | Show detailed conditional-flow information including phases and tasks | `kbagent flow detail --project PROJECT --flow-id FLOW-ID` | | Print the conditional-flow YAML template, or --full for the live JSON Schema | `kbagent flow schema` | diff --git a/src/keboola_agent_cli/ai_client.py b/src/keboola_agent_cli/ai_client.py index 0237845a..c46deb61 100644 --- a/src/keboola_agent_cli/ai_client.py +++ b/src/keboola_agent_cli/ai_client.py @@ -65,6 +65,24 @@ def get_component_detail(self, component_id: str) -> dict[str, Any]: response = self._do_request("GET", f"/docs/components/{encoded_id}") return response.json() + def docs_question(self, query: str) -> dict[str, Any]: + """Ask the Keboola documentation a natural language question. + + Args: + query: Natural language question about the Keboola platform + (e.g. 'How do I set up incremental loading?'). + + Returns: + Dict with 'text' (Markdown answer) and 'sourceUrls' (list of + documentation URLs the answer is grounded in). + + Raises: + KeboolaApiError: On API errors. + """ + payload = {"query": query} + response = self._do_request("POST", "/docs/question", json=payload) + return response.json() + def suggest_components(self, query: str) -> list[dict[str, Any]]: """Suggest components matching a natural language query. diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 652f7e5e..4ccce8b2 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -24,6 +24,33 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.73.0": [ + "MCP parity + fail-closed firewall (#478 phase 0, epic #390 phase 1): six native " + "commands port the remaining keboola-mcp-server tools, and MCP tool classification " + "fails closed.", + "Security (#478): unknown MCP tool names now classify as `destructive` (strictest) " + "instead of falling through to `read`. Catalog tools `run_job`, `run_sync_action`, " + "`modify_*`, `deploy_*` move from read to write -- they previously passed " + "`--deny-writes` and fanned out to every configured project. Multi-project dispatch " + "is fail-closed too: only known-read tools fan out.", + "Security (#478): `tool call` now enforces the SESSION firewall per tool name -- " + "`--deny-destructive` blocks `tool call delete_bucket` (previously only the persisted " + "policy was checked at tool granularity).", + 'New (#392): `kbagent docs query "QUESTION"` -- answers from the Keboola ' + "documentation via the AI Service (ports `docs_query`).", + "New (#393): `kbagent config examples --component-id ID` -- sample root/row " + "configurations for a component (ports `get_config_examples`).", + "New (#394): `kbagent semantic-layer schema --type metric,dataset,...` -- live JSON " + "schemas of semantic object types from the metastore (ports `get_semantic_schema`).", + "New (#395): `kbagent component sync-action ACTION` -- run synchronous component " + "actions like testConnection (ports `run_sync_action`; shallow root+row config merge " + "identical to the MCP tool).", + "New (#396): `kbagent transformation create|show|edit` -- SQL transformation " + "authoring with the 9-op block/code edit engine (ports create/update_sql_" + "transformation; synthetic b{i}/b{i}.c{j} ids, dialect from project default_backend).", + "New (#397): `kbagent flow examples` + `flow schema` now serves the authoritative " + "bundled conditional-flow schema (ports `get_flow_examples`; fixes schema drift).", + ], "0.72.0": [ "Sync trust cluster (#466, #467, #472, #497): four reliability fixes that make " "`kbagent sync` safe to run against production trees edited by other people.", diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index c6ad1921..71ac7ee8 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -14,6 +14,7 @@ from .commands.context import context_command from .commands.data_app import data_app_app from .commands.dev_portal import dev_portal_app +from .commands.docs import docs_app from .commands.doctor import doctor_command from .commands.encrypt import encrypt_app from .commands.feature import feature_app @@ -52,6 +53,7 @@ from .services.data_app_git_service import DataAppGitService from .services.data_app_service import DataAppService from .services.deep_lineage_service import DeepLineageService +from .services.docs_service import DocsService from .services.doctor_service import DoctorService from .services.encrypt_service import EncryptService from .services.feature_service import FeatureService @@ -119,6 +121,7 @@ app.add_typer(sharing_app, name="sharing", rich_help_panel=_BROWSE) app.add_typer(lineage_app, name="lineage", rich_help_panel=_BROWSE) app.add_typer(kai_app, name="kai", rich_help_panel=_BROWSE) +app.add_typer(docs_app, name="docs", rich_help_panel=_BROWSE) # -- Flows -- _FLOWS = "Flows" @@ -335,6 +338,7 @@ def main( semantic_layer_service = SemanticLayerService(config_store=config_store) repo_validate_service = RepoValidateService(config_store=config_store) kai_service = KaiService(config_store=config_store) + docs_service = DocsService(config_store=config_store) doctor_service = DoctorService(config_store=config_store, mcp_service=mcp_service) version_service = VersionService() http_forwarder_service = HttpForwarderService() @@ -391,6 +395,7 @@ def main( ctx.obj["semantic_layer_service"] = semantic_layer_service ctx.obj["repo_validate_service"] = repo_validate_service ctx.obj["kai_service"] = kai_service + ctx.obj["docs_service"] = docs_service ctx.obj["doctor_service"] = doctor_service ctx.obj["version_service"] = version_service ctx.obj["http_forwarder_service"] = http_forwarder_service diff --git a/src/keboola_agent_cli/commands/docs.py b/src/keboola_agent_cli/commands/docs.py new file mode 100644 index 00000000..18a6a675 --- /dev/null +++ b/src/keboola_agent_cli/commands/docs.py @@ -0,0 +1,81 @@ +"""CLI commands for Keboola documentation Q&A. + +Thin CLI layer: parses arguments, calls DocsService, formats output. +No business logic belongs here. +""" + +import typer +from rich.console import Console +from rich.markup import escape +from rich.panel import Panel + +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ._helpers import ( + check_cli_permission, + get_formatter, + get_service, + map_error_to_exit_code, +) + +docs_app = typer.Typer(help="Ask the Keboola documentation natural-language questions") + + +@docs_app.callback(invoke_without_command=True) +def _docs_permission_check(ctx: typer.Context) -> None: + check_cli_permission(ctx, "docs") + + +def _format_docs_answer(console: Console, data: dict) -> None: + """Render a documentation answer as a Rich panel with a sources list. + + Args: + console: Rich Console instance. + data: Dict with "query", "text", and "source_urls" from DocsService. + """ + text = data.get("text", "") + source_urls = data.get("source_urls", []) + + # Answer text is remote Markdown -- escape it so stray brackets are not + # interpreted as Rich markup (precedent: config.py / storage.py). + lines = [escape(text.strip()) if text.strip() else "[dim](no answer text returned)[/dim]"] + if source_urls: + lines.append("") + lines.append("[bold]Sources:[/bold]") + lines.extend(f" - {escape(url)}" for url in source_urls) + + panel = Panel("\n".join(lines), title="Keboola Docs", expand=False) + console.print(panel) + + +@docs_app.command("query") +def docs_query( + ctx: typer.Context, + question: str = typer.Argument( + ..., + help="Natural language question about the Keboola platform", + ), + project: str | None = typer.Option( + None, + "--project", + help="Project alias (uses first available if not set)", + ), +) -> None: + """Ask the Keboola documentation a natural language question.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "docs_service") + + try: + result = service.ask_docs(alias=project, query=question) + formatter.output(result, _format_docs_answer) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error( + message=exc.message, + error_code=exc.error_code, + project=project or "", + retryable=exc.retryable, + ) + raise typer.Exit(code=exit_code) from None diff --git a/src/keboola_agent_cli/models.py b/src/keboola_agent_cli/models.py index 3edc50ec..96975a57 100644 --- a/src/keboola_agent_cli/models.py +++ b/src/keboola_agent_cli/models.py @@ -306,6 +306,27 @@ class ComponentSuggestion(BaseModel): model_config = {"populate_by_name": True} +class DocsAnswer(BaseModel): + """Answer from Keboola AI Service /docs/question endpoint.""" + + text: str = Field(default="") + source_urls: list[str] = Field(default_factory=list, alias="sourceUrls") + + @field_validator("text", mode="before") + @classmethod + def _none_text_to_empty_string(cls, value: Any) -> Any: + """AI Service may return explicit null for an empty answer.""" + return "" if value is None else value + + @field_validator("source_urls", mode="before") + @classmethod + def _none_sources_to_empty_list(cls, value: Any) -> Any: + """AI Service may return explicit null when no sources matched.""" + return [] if value is None else value + + model_config = {"populate_by_name": True} + + class ErrorResponse(BaseModel): """Structured error response for JSON output mode.""" diff --git a/src/keboola_agent_cli/services/docs_service.py b/src/keboola_agent_cli/services/docs_service.py new file mode 100644 index 00000000..f5b127ce --- /dev/null +++ b/src/keboola_agent_cli/services/docs_service.py @@ -0,0 +1,73 @@ +"""Documentation Q&A service — ask the Keboola docs natural-language questions. + +Bridges the CLI to the AI Service /docs/question endpoint. Resolves the +target project (stack URL + token) from the config store the same way +ComponentService does, then queries the AI Service and normalizes the +response into the CLI's snake_case output contract. +""" + +import logging +from typing import Any + +from ..config_store import ConfigStore +from ..errors import ConfigError +from ..models import DocsAnswer +from .base import BaseService, ClientFactory +from .component_service import AiClientFactory, default_ai_client_factory + +logger = logging.getLogger(__name__) + + +class DocsService(BaseService): + """Business logic for the `kbagent docs` command group. + + Uses AiServiceClient (via injected factory) to answer natural-language + questions grounded in the official Keboola documentation. + """ + + def __init__( + self, + config_store: ConfigStore, + client_factory: ClientFactory | None = None, + ai_client_factory: AiClientFactory | None = None, + ) -> None: + super().__init__(config_store, client_factory) + self._ai_client_factory = ai_client_factory or default_ai_client_factory + + def ask_docs(self, alias: str | None, query: str) -> dict[str, Any]: + """Ask the Keboola documentation a natural language question. + + Args: + alias: Project alias used to derive the stack URL and token. + None means the first configured project. + query: Natural language question about the Keboola platform. + + Returns: + Dict with keys: + - "query": the question as asked + - "text": Markdown answer text + - "source_urls": list of documentation URLs the answer + is grounded in + + Raises: + ConfigError: If no projects are configured or the alias is unknown. + KeboolaApiError: If the AI Service call fails. + """ + projects = self.resolve_projects([alias] if alias else None) + if not projects: + raise ConfigError("No projects configured. Run 'kbagent project add' first.") + first_alias = next(iter(projects)) + project = projects[first_alias] + + ai_client = self._ai_client_factory(project.stack_url, project.token) + try: + raw = ai_client.docs_question(query) + finally: + ai_client.close() + + answer = DocsAnswer(**raw) + return { + "query": query, + "text": answer.text, + "source_urls": answer.source_urls, + } diff --git a/tests/test_ai_client.py b/tests/test_ai_client.py index 592db0f1..4e494813 100644 --- a/tests/test_ai_client.py +++ b/tests/test_ai_client.py @@ -160,6 +160,55 @@ def test_suggest_components_empty(self, httpx_mock) -> None: client.close() +class TestDocsQuestion: + """Verify docs_question() sends the query and returns the raw answer.""" + + def test_docs_question_success(self, httpx_mock) -> None: + """docs_question() POSTs {"query": ...} and returns text + sourceUrls.""" + httpx_mock.add_response( + url=f"{AI_BASE_URL}/docs/question", + json={ + "text": "Incremental loading appends only changed rows.", + "sourceUrls": ["https://help.keboola.com/storage/tables/"], + }, + status_code=200, + ) + + client = AiServiceClient(stack_url=STACK_URL, token=TOKEN) + try: + result = client.docs_question("how does incremental loading work?") + + assert result["text"].startswith("Incremental loading") + assert result["sourceUrls"] == ["https://help.keboola.com/storage/tables/"] + + # Verify the request payload + request = httpx_mock.get_requests()[0] + assert request.method == "POST" + import json + + body = json.loads(request.content) + assert body == {"query": "how does incremental loading work?"} + finally: + client.close() + + def test_docs_question_api_error(self, httpx_mock) -> None: + """docs_question() raises KeboolaApiError on a 4xx response.""" + httpx_mock.add_response( + url=f"{AI_BASE_URL}/docs/question", + json={"error": "Bad request"}, + status_code=400, + ) + + client = AiServiceClient(stack_url=STACK_URL, token=TOKEN) + try: + with pytest.raises(KeboolaApiError) as exc_info: + client.docs_question("") + assert exc_info.value.status_code == 400 + assert exc_info.value.retryable is False + finally: + client.close() + + class TestUrlEncoding: """Verify component IDs with special characters are URL-encoded.""" diff --git a/tests/test_docs_cli.py b/tests/test_docs_cli.py new file mode 100644 index 00000000..bfffb0fc --- /dev/null +++ b/tests/test_docs_cli.py @@ -0,0 +1,273 @@ +"""Tests for the `docs query` command and DocsService. + +The docs_app is not yet wired into cli.py (central wiring happens +separately), so CLI tests mount docs_app on a minimal Typer root app +that provides the same ctx.obj contract (formatter + docs_service + +permission_engine) the real CLI callback builds. +""" + +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import typer +from typer.testing import CliRunner + +from keboola_agent_cli.commands.docs import docs_app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.output import OutputFormatter +from keboola_agent_cli.services.docs_service import DocsService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + +runner = CliRunner() + +ANSWER = { + "query": "how do incremental loads work?", + "text": "Incremental loading appends only changed rows to the table.", + "source_urls": [ + "https://help.keboola.com/storage/tables/#incremental-loading", + "https://help.keboola.com/components/", + ], +} + + +def _build_app(mock_service: MagicMock, json_mode: bool) -> typer.Typer: + """Build a minimal root app mounting docs_app with a stubbed ctx.obj.""" + app = typer.Typer() + + @app.callback() + def _root(ctx: typer.Context) -> None: + ctx.ensure_object(dict) + ctx.obj["formatter"] = OutputFormatter(json_mode=json_mode, no_color=True) + ctx.obj["docs_service"] = mock_service + ctx.obj["permission_engine"] = None + + app.add_typer(docs_app, name="docs") + return app + + +# --------------------------------------------------------------------------- +# docs query (CLI layer) +# --------------------------------------------------------------------------- + + +class TestDocsQueryCli: + """Tests for `kbagent docs query` command.""" + + def test_docs_query_json(self) -> None: + """docs query --json returns {query, text, source_urls}.""" + mock_svc = MagicMock() + mock_svc.ask_docs.return_value = dict(ANSWER) + + app = _build_app(mock_svc, json_mode=True) + result = runner.invoke(app, ["docs", "query", "how do incremental loads work?"]) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"]["query"] == "how do incremental loads work?" + assert output["data"]["text"].startswith("Incremental loading") + assert output["data"]["source_urls"] == ANSWER["source_urls"] + mock_svc.ask_docs.assert_called_once_with( + alias=None, + query="how do incremental loads work?", + ) + + def test_docs_query_passes_project(self) -> None: + """docs query --project forwards the alias to the service.""" + mock_svc = MagicMock() + mock_svc.ask_docs.return_value = dict(ANSWER) + + app = _build_app(mock_svc, json_mode=True) + result = runner.invoke( + app, + ["docs", "query", "how do incremental loads work?", "--project", "prod"], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + mock_svc.ask_docs.assert_called_once_with( + alias="prod", + query="how do incremental loads work?", + ) + + def test_docs_query_human(self) -> None: + """docs query in human mode renders a panel with answer and sources.""" + mock_svc = MagicMock() + mock_svc.ask_docs.return_value = dict(ANSWER) + + app = _build_app(mock_svc, json_mode=False) + result = runner.invoke(app, ["docs", "query", "how do incremental loads work?"]) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "Keboola Docs" in result.output + assert "Incremental loading appends" in result.output + assert "Sources:" in result.output + assert "help.keboola.com" in result.output + + def test_docs_query_human_empty_answer(self) -> None: + """docs query in human mode handles an empty answer gracefully.""" + mock_svc = MagicMock() + mock_svc.ask_docs.return_value = {"query": "q", "text": "", "source_urls": []} + + app = _build_app(mock_svc, json_mode=False) + result = runner.invoke(app, ["docs", "query", "q"]) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "no answer text returned" in result.output + + def test_docs_query_api_error(self) -> None: + """docs query surfaces KeboolaApiError as a structured error envelope.""" + mock_svc = MagicMock() + mock_svc.ask_docs.side_effect = KeboolaApiError( + message="AI service unavailable", + status_code=503, + error_code="RETRY_EXHAUSTED", + retryable=True, + ) + + app = _build_app(mock_svc, json_mode=True) + result = runner.invoke(app, ["docs", "query", "anything"]) + + # RETRY_EXHAUSTED maps to exit code 4 (network/retryable) + assert result.exit_code == 4 + output = json.loads(result.output) + assert output["status"] == "error" + assert output["error"]["code"] == "RETRY_EXHAUSTED" + assert output["error"]["retryable"] is True + + def test_docs_query_config_error(self) -> None: + """docs query maps ConfigError to exit code 5.""" + mock_svc = MagicMock() + mock_svc.ask_docs.side_effect = ConfigError("No projects configured.") + + app = _build_app(mock_svc, json_mode=True) + result = runner.invoke(app, ["docs", "query", "anything"]) + + assert result.exit_code == 5 + output = json.loads(result.output) + assert output["status"] == "error" + assert output["error"]["code"] == "CONFIG_ERROR" + + def test_docs_query_missing_question(self) -> None: + """docs query without the positional QUESTION is a usage error (exit 2).""" + mock_svc = MagicMock() + + app = _build_app(mock_svc, json_mode=True) + result = runner.invoke(app, ["docs", "query"]) + + assert result.exit_code == 2 + mock_svc.ask_docs.assert_not_called() + + +# --------------------------------------------------------------------------- +# DocsService (service layer) +# --------------------------------------------------------------------------- + + +class TestDocsService: + """Unit tests for DocsService.ask_docs alias resolution and normalization.""" + + def _make_store(self, tmp_path: Path, aliases: list[str]) -> ConfigStore: + store = ConfigStore(config_dir=tmp_path / "config") + for i, alias in enumerate(aliases): + store.add_project( + alias, + ProjectConfig( + stack_url=f"https://connection.{alias}.keboola.com", + token=TEST_TOKEN, + project_name=alias, + project_id=1000 + i, + ), + ) + return store + + def _make_service( + self, store: ConfigStore, raw_answer: dict + ) -> tuple[DocsService, MagicMock, MagicMock]: + mock_client = MagicMock() + mock_client.docs_question.return_value = raw_answer + mock_factory = MagicMock(return_value=mock_client) + service = DocsService(config_store=store, ai_client_factory=mock_factory) + return service, mock_client, mock_factory + + def test_ask_docs_explicit_alias(self, tmp_path: Path) -> None: + """Explicit alias resolves that project's stack URL and token.""" + store = self._make_store(tmp_path, ["prod", "dev"]) + raw = {"text": "Answer.", "sourceUrls": ["https://help.keboola.com/x"]} + service, mock_client, mock_factory = self._make_service(store, raw) + + result = service.ask_docs(alias="dev", query="what is a bucket?") + + mock_factory.assert_called_once_with( + "https://connection.dev.keboola.com", + TEST_TOKEN, + ) + mock_client.docs_question.assert_called_once_with("what is a bucket?") + mock_client.close.assert_called_once() + assert result == { + "query": "what is a bucket?", + "text": "Answer.", + "source_urls": ["https://help.keboola.com/x"], + } + + def test_ask_docs_default_alias_uses_first_project(self, tmp_path: Path) -> None: + """alias=None falls back to the first configured project.""" + store = self._make_store(tmp_path, ["prod", "dev"]) + raw = {"text": "Answer.", "sourceUrls": []} + service, _mock_client, mock_factory = self._make_service(store, raw) + + result = service.ask_docs(alias=None, query="q") + + mock_factory.assert_called_once_with( + "https://connection.prod.keboola.com", + TEST_TOKEN, + ) + assert result["source_urls"] == [] + + def test_ask_docs_unknown_alias_raises_config_error(self, tmp_path: Path) -> None: + """Unknown alias raises ConfigError before any HTTP call.""" + store = self._make_store(tmp_path, ["prod"]) + service, mock_client, _mock_factory = self._make_service(store, {}) + + with pytest.raises(ConfigError): + service.ask_docs(alias="nope", query="q") + mock_client.docs_question.assert_not_called() + + def test_ask_docs_no_projects_raises_config_error(self, tmp_path: Path) -> None: + """No configured projects raises an actionable ConfigError.""" + store = self._make_store(tmp_path, []) + service, mock_client, _mock_factory = self._make_service(store, {}) + + with pytest.raises(ConfigError, match="No projects configured"): + service.ask_docs(alias=None, query="q") + mock_client.docs_question.assert_not_called() + + def test_ask_docs_normalizes_null_fields(self, tmp_path: Path) -> None: + """Explicit nulls from the AI Service normalize to '' / [].""" + store = self._make_store(tmp_path, ["prod"]) + raw = {"text": None, "sourceUrls": None} + service, _mock_client, _mock_factory = self._make_service(store, raw) + + result = service.ask_docs(alias="prod", query="q") + + assert result["text"] == "" + assert result["source_urls"] == [] + + def test_ask_docs_closes_client_on_error(self, tmp_path: Path) -> None: + """The AI client is closed even when docs_question raises.""" + store = self._make_store(tmp_path, ["prod"]) + service, mock_client, _mock_factory = self._make_service(store, {}) + mock_client.docs_question.side_effect = KeboolaApiError( + message="boom", + status_code=500, + error_code="RETRY_EXHAUSTED", + retryable=True, + ) + + with pytest.raises(KeboolaApiError): + service.ask_docs(alias="prod", query="q") + mock_client.close.assert_called_once() From 7e1e3e01dfaeaf17483143deab1d0fb6be8b6fc0 Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:26:58 +0200 Subject: [PATCH 04/17] feat(semantic-layer): schema command -- live JSON schemas per object type (#394) Ports get_semantic_schema: MetastoreClient.get_schema (GET /api/v1/schema/{type}, no data-envelope unwrap), service fan-out over requested types with fail-fast validation, thin command with --type CSV / --all. Schemas are server-fetched so they always match the deployed metastore. --- plugins/kbagent/skills/kbagent/SKILL.md | 3 +- .../commands/semantic_layer.py | 73 +++ src/keboola_agent_cli/metastore_client.py | 25 + .../services/semantic_layer_service.py | 62 +++ tests/test_semantic_layer_schema.py | 451 ++++++++++++++++++ 5 files changed, 613 insertions(+), 1 deletion(-) create mode 100644 tests/test_semantic_layer_schema.py diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 286c87cf..3c9fcbdf 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -191,7 +191,8 @@ When working inside a git repository or project directory, run `kbagent init` (o | Ask the Keboola documentation a natural language question | `kbagent docs query ` | | List conditional flows (keboola.flow) across projects | `kbagent flow list` | | Show detailed conditional-flow information including phases and tasks | `kbagent flow detail --project PROJECT --flow-id FLOW-ID` | -| Print the conditional-flow YAML template, or --full for the live JSON Schema | `kbagent flow schema` | +| Print the conditional-flow YAML template, or --full for the JSON Schema | `kbagent flow schema` | +| Show bundled example flow configurations (offline, no project needed) | `kbagent flow examples` | | Validate a conditional-flow definition (schema + semantic checks) | `kbagent flow validate --file FILE` | | Create a new conditional-flow (keboola.flow) configuration | `kbagent flow new --project PROJECT --name NAME` | | Update a flow's name, description, or phases/tasks | `kbagent flow update --project PROJECT --flow-id FLOW-ID` | diff --git a/src/keboola_agent_cli/commands/semantic_layer.py b/src/keboola_agent_cli/commands/semantic_layer.py index 91e73a81..1a3748ab 100644 --- a/src/keboola_agent_cli/commands/semantic_layer.py +++ b/src/keboola_agent_cli/commands/semantic_layer.py @@ -12,9 +12,12 @@ import typer from rich.console import Console +from rich.panel import Panel +from rich.syntax import Syntax from rich.table import Table from ..errors import ErrorCode +from ..services.semantic_layer_service import SCHEMA_TYPE_ALIAS from ._helpers import ( check_cli_permission, get_formatter, @@ -640,6 +643,76 @@ def semantic_layer_show( formatter.output(result, _print_show_detail) +# --------------------------------------------------------------------------- +# semantic-layer schema +# --------------------------------------------------------------------------- + + +def _print_schemas(console: Console, data: dict) -> None: + """Render each requested type's JSON schema in its own panel.""" + project = data.get("project", "") + for entry in data.get("schemas", []): + schema_json = json.dumps(entry.get("schema", {}), indent=2) + console.print( + Panel( + Syntax(schema_json, "json", theme="ansi_dark", word_wrap=True), + title=( + f"[bold cyan]{entry.get('type', '?')}[/bold cyan] schema " + f"([magenta]{project}[/magenta])" + ), + border_style="dim", + ) + ) + + +@semantic_layer_app.command("schema") +def semantic_layer_schema( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + types: str | None = typer.Option( + None, + "--type", + help=( + "Comma-separated semantic type(s): " + "model | dataset | metric | relationship | constraint | glossary." + ), + ), + all_types: bool = typer.Option( + False, "--all", help="Fetch the schema of every known semantic type." + ), +) -> None: + """Fetch the server-side JSON Schema of semantic object types. + + Schemas are fetched live from the project's metastore (never bundled), + so they always match the deployed metastore version. Pass exactly one + of ``--type`` (comma-separated list) or ``--all``. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + + # Mutual exclusion: exactly one of --type / --all. + if all_types == (types is not None): + formatter.error( + message="Specify exactly one of --type or --all.", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) + + if all_types: + type_list = list(SCHEMA_TYPE_ALIAS) + else: + type_list = [t.strip() for t in (types or "").split(",") if t.strip()] + if not type_list: + formatter.error( + message="--type contained no type names.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + + result = _handle_service_call(ctx, service.get_schema, alias=project, types=type_list) + formatter.output(result, _print_schemas) + + # --------------------------------------------------------------------------- # semantic-layer validate [--deep] # --------------------------------------------------------------------------- diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 09fbdb0e..a394d486 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -108,6 +108,31 @@ def list_items( return items return [i for i in items if (i.get("attributes") or {}).get("modelUUID") == model_uuid] + def get_schema(self, item_type: SemanticType) -> dict[str, Any]: + """Fetch the JSON Schema for a semantic object type. + + ``GET /api/v1/schema/{item_type}``. The schema is **server-emitted** + so it always matches the deployed metastore version — never a + hand-rolled static copy (it would drift the moment the metastore + evolves). Unlike the repository verbs, this endpoint returns the + schema document directly with no ``{"data": ...}`` envelope, so the + body is passed through verbatim (mirrors keboola-mcp-server + ``MetastoreClient.get_schema``). + """ + response = self._do_request("GET", f"/api/v1/schema/{item_type}") + body = response.json() + if not isinstance(body, dict): + raise KeboolaApiError( + message=( + f"Unexpected metastore schema response format for " + f"{item_type!r} (expected a JSON object)." + ), + status_code=response.status_code, + error_code=ErrorCode.API_ERROR, + retryable=False, + ) + return body + def get_item(self, item_type: SemanticType, item_id: str) -> dict[str, Any]: """Fetch a single item by its UUID. diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index c9aab1f2..b863e82f 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -171,6 +171,17 @@ def _classify_field_role(name: str, basetype: str) -> str: "glossary": "semantic-glossary", } +# Accepted ``--type`` values for ``semantic-layer schema``: every child type +# from TYPE_ALIAS plus the model envelope itself. Kept as a separate dict +# (rather than adding ``model`` to TYPE_ALIAS in place) because ``show +# --type model`` has no plural payload key in ``_PLURAL_BY_TYPE`` — widening +# TYPE_ALIAS would let ``show`` accept ``model`` and then KeyError while +# rendering. Insertion order is the canonical ``--all`` fetch order. +SCHEMA_TYPE_ALIAS: dict[str, SemanticType] = { + "model": "semantic-model", + **TYPE_ALIAS, +} + # Child-types order used everywhere we fan out per-type fetches. CHILD_TYPES: tuple[SemanticType, ...] = ( "semantic-dataset", @@ -317,6 +328,57 @@ def get_context(self, alias: str, context_id: str) -> dict[str, Any]: context_id=context_id, ) + def get_schema(self, alias: str, types: list[str]) -> dict[str, Any]: + """Fetch the server-side JSON Schema for one or more semantic types. + + ``types`` holds CLI-singular names (``metric``, ``model``, ...) as + accepted by :data:`SCHEMA_TYPE_ALIAS`. Unknown names fail fast + before any network call. Duplicates are collapsed (first occurrence + wins the ordering). Multiple types fan out in parallel, mirroring + :meth:`_fetch_children_parallel`. + + Returns: + ``{"project": alias, "schemas": [{"type": , + "schema": }]}`` in requested order. + """ + requested = list(dict.fromkeys(types)) + unknown = [t for t in requested if t not in SCHEMA_TYPE_ALIAS] + if unknown: + raise ConfigError( + f"Unknown semantic type(s): {', '.join(unknown)}. " + f"Valid types: {', '.join(SCHEMA_TYPE_ALIAS)}." + ) + if not requested: + raise ConfigError( + f"No semantic types requested. Valid types: {', '.join(SCHEMA_TYPE_ALIAS)}." + ) + + project = self._resolve_one_project(alias) + results: dict[str, dict[str, Any]] = {} + with self._new_metastore_client(project) as client: + if len(requested) == 1: + results[requested[0]] = client.get_schema(SCHEMA_TYPE_ALIAS[requested[0]]) + else: + with ThreadPoolExecutor(max_workers=len(requested)) as pool: + future_to_type = { + pool.submit(client.get_schema, SCHEMA_TYPE_ALIAS[t]): t for t in requested + } + errors: list[Exception] = [] + for future in future_to_type: + try: + results[future_to_type[future]] = future.result() + # Future.result() re-raises arbitrary worker exceptions + # (KeboolaApiError, httpx errors, ...); collect and + # surface the first rather than masking the others. + except Exception as exc: + errors.append(exc) + if errors: + raise errors[0] + return { + "project": alias, + "schemas": [{"type": t, "schema": results[t]} for t in requested], + } + # Internal helpers (model-scoped fetches). @staticmethod diff --git a/tests/test_semantic_layer_schema.py b/tests/test_semantic_layer_schema.py new file mode 100644 index 00000000..6d4c9d4c --- /dev/null +++ b/tests/test_semantic_layer_schema.py @@ -0,0 +1,451 @@ +"""Tests for ``kbagent semantic-layer schema`` (issue #394). + +Covers all three layers: + +- L3 ``MetastoreClient.get_schema`` via pytest-httpx (URL, verbatim body + passthrough, non-dict response normalization). +- L2 ``SemanticLayerService.get_schema`` with a MagicMock metastore factory + (single type, multi-type fan-out, dedupe, unknown-type fail-fast, worker + error propagation). +- L1 CLI via CliRunner with the cli.py service factory patched (JSON + envelope, ``--all``, mutual exclusion, error mapping, human panels). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, ErrorCode, KeboolaApiError +from keboola_agent_cli.metastore_client import MetastoreClient +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.config_service import ConfigService +from keboola_agent_cli.services.job_service import JobService +from keboola_agent_cli.services.project_service import ProjectService +from keboola_agent_cli.services.semantic_layer_service import ( + SCHEMA_TYPE_ALIAS, + TYPE_ALIAS, + SemanticLayerService, +) + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" +STACK_URL_US = "https://connection.keboola.com" +METASTORE_URL_US = "https://metastore.keboola.com" + +runner = CliRunner() + + +# --------------------------------------------------------------------------- +# Shared helpers (mirror test_semantic_layer_cli.py / _service.py conventions) +# --------------------------------------------------------------------------- + + +def _make_store(tmp_path: Path, alias: str = "prod") -> ConfigStore: + """Build a ConfigStore with a single project registered.""" + config_dir = tmp_path / "config" + config_dir.mkdir(exist_ok=True) + store = ConfigStore(config_dir=config_dir) + store.add_project( + alias, + ProjectConfig( + stack_url=STACK_URL_US, + token=TEST_TOKEN, + project_name=alias, + project_id=5725, + ), + ) + return store + + +def _make_service( + store: ConfigStore, + *, + metastore_mock: MagicMock | None = None, +) -> tuple[SemanticLayerService, MagicMock]: + """Wire a SemanticLayerService with a mocked metastore client factory.""" + mock = metastore_mock or MagicMock() + mock.__enter__ = MagicMock(return_value=mock) + mock.__exit__ = MagicMock(return_value=False) + service = SemanticLayerService( + config_store=store, + metastore_client_factory=lambda url, token: mock, + ) + return service, mock + + +def _invoke( + args: list[str], + *, + store: ConfigStore, + sl_mock: MagicMock, +): + """Run the CLI with cli.py services patched to mocks.""" + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProj, + patch("keboola_agent_cli.cli.ConfigService") as MockCfg, + patch("keboola_agent_cli.cli.JobService") as MockJob, + patch("keboola_agent_cli.cli.SemanticLayerService") as MockSL, + ): + MockStore.return_value = store + MockProj.return_value = ProjectService(config_store=store) + MockCfg.return_value = ConfigService(config_store=store) + MockJob.return_value = JobService(config_store=store) + MockSL.return_value = sl_mock + return runner.invoke(app, args) + + +@pytest.fixture +def store(tmp_path: Path) -> ConfigStore: + return _make_store(tmp_path) + + +def _schema_for(wire_type: str) -> dict[str, Any]: + """Deterministic fake JSON schema keyed by wire type.""" + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": wire_type, + "type": "object", + "properties": {"name": {"type": "string"}}, + } + + +# --------------------------------------------------------------------------- +# L3 -- MetastoreClient.get_schema +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _no_backoff_sleep(monkeypatch: pytest.MonkeyPatch) -> None: + """Disable retry-backoff sleeps so the suite stays fast.""" + import keboola_agent_cli.http_base as http_base_module + + monkeypatch.setattr(http_base_module.time, "sleep", lambda _x: None) + + +class TestClientGetSchema: + def test_get_schema_url_and_verbatim_body(self, httpx_mock) -> None: + schema = _schema_for("semantic-metric") + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/schema/semantic-metric", + json=schema, + status_code=200, + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TEST_TOKEN) + try: + result = client.get_schema("semantic-metric") + finally: + client.close() + # Verbatim passthrough -- no {"data": ...} unwrapping on this endpoint. + assert result == schema + request = httpx_mock.get_requests()[0] + assert request.headers["X-StorageApi-Token"] == TEST_TOKEN + + def test_get_schema_data_key_not_unwrapped(self, httpx_mock) -> None: + # A schema that legitimately contains a top-level "data" property + # must NOT be unwrapped like the repository endpoints. + schema = {"type": "object", "data": {"nested": True}} + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/schema/semantic-model", + json=schema, + status_code=200, + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TEST_TOKEN) + try: + result = client.get_schema("semantic-model") + finally: + client.close() + assert result == schema + + def test_get_schema_non_dict_body_raises_api_error(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/schema/semantic-dataset", + json=["not", "a", "dict"], + status_code=200, + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TEST_TOKEN) + try: + with pytest.raises(KeboolaApiError) as exc_info: + client.get_schema("semantic-dataset") + finally: + client.close() + assert exc_info.value.error_code == ErrorCode.API_ERROR + assert exc_info.value.retryable is False + + +# --------------------------------------------------------------------------- +# L2 -- SemanticLayerService.get_schema +# --------------------------------------------------------------------------- + + +class TestServiceGetSchema: + def test_single_type(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.return_value = _schema_for("semantic-metric") + + result = service.get_schema("prod", types=["metric"]) + + assert result == { + "project": "prod", + "schemas": [{"type": "metric", "schema": _schema_for("semantic-metric")}], + } + mock.get_schema.assert_called_once_with("semantic-metric") + + def test_model_type_maps_to_semantic_model(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.return_value = _schema_for("semantic-model") + + result = service.get_schema("prod", types=["model"]) + + mock.get_schema.assert_called_once_with("semantic-model") + assert result["schemas"][0]["type"] == "model" + + def test_multi_type_fan_out_preserves_requested_order(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = _schema_for + + result = service.get_schema("prod", types=["metric", "dataset", "model"]) + + assert [s["type"] for s in result["schemas"]] == ["metric", "dataset", "model"] + assert result["schemas"][0]["schema"] == _schema_for("semantic-metric") + assert result["schemas"][1]["schema"] == _schema_for("semantic-dataset") + assert result["schemas"][2]["schema"] == _schema_for("semantic-model") + called_wire_types = {c.args[0] for c in mock.get_schema.call_args_list} + assert called_wire_types == {"semantic-metric", "semantic-dataset", "semantic-model"} + + def test_duplicates_collapsed(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.return_value = _schema_for("semantic-metric") + + result = service.get_schema("prod", types=["metric", "metric"]) + + assert [s["type"] for s in result["schemas"]] == ["metric"] + mock.get_schema.assert_called_once_with("semantic-metric") + + def test_unknown_type_fails_fast_without_network(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + + with pytest.raises(ConfigError) as exc_info: + service.get_schema("prod", types=["metric", "bogus"]) + + assert "bogus" in exc_info.value.message + # The message lists every valid type name. + for valid in SCHEMA_TYPE_ALIAS: + assert valid in exc_info.value.message + mock.get_schema.assert_not_called() + + def test_empty_types_fails_fast(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + + with pytest.raises(ConfigError): + service.get_schema("prod", types=[]) + mock.get_schema.assert_not_called() + + def test_worker_error_propagates(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + + def _raise_for_dataset(wire_type: str) -> dict[str, Any]: + if wire_type == "semantic-dataset": + raise KeboolaApiError( + message="boom", + status_code=500, + error_code=ErrorCode.API_ERROR, + ) + return _schema_for(wire_type) + + mock.get_schema.side_effect = _raise_for_dataset + + with pytest.raises(KeboolaApiError) as exc_info: + service.get_schema("prod", types=["metric", "dataset"]) + assert exc_info.value.error_code == ErrorCode.API_ERROR + + def test_schema_alias_is_superset_of_type_alias_plus_model(self) -> None: + assert set(SCHEMA_TYPE_ALIAS) == set(TYPE_ALIAS) | {"model"} + assert SCHEMA_TYPE_ALIAS["model"] == "semantic-model" + # show --type must NOT accept "model" (no plural payload key). + assert "model" not in TYPE_ALIAS + + def test_operation_registry_classifies_schema_as_read(self) -> None: + """Without this entry the fail-closed default ('write') would deny + `semantic-layer schema` under --deny-writes despite being read-only.""" + from keboola_agent_cli.permissions import OPERATION_REGISTRY + + assert OPERATION_REGISTRY["semantic-layer.schema"] == "read" + + +# --------------------------------------------------------------------------- +# L1 -- CLI `semantic-layer schema` +# --------------------------------------------------------------------------- + + +class TestCliSchema: + def test_single_type_json(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.get_schema.return_value = { + "project": "prod", + "schemas": [{"type": "metric", "schema": _schema_for("semantic-metric")}], + } + result = _invoke( + ["--json", "semantic-layer", "schema", "--project", "prod", "--type", "metric"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["status"] == "ok" + assert body["data"]["schemas"][0]["type"] == "metric" + assert body["data"]["schemas"][0]["schema"]["title"] == "semantic-metric" + mock.get_schema.assert_called_once_with(alias="prod", types=["metric"]) + + def test_multi_type_comma_separated(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.get_schema.return_value = { + "project": "prod", + "schemas": [ + {"type": "metric", "schema": _schema_for("semantic-metric")}, + {"type": "dataset", "schema": _schema_for("semantic-dataset")}, + ], + } + result = _invoke( + [ + "--json", + "semantic-layer", + "schema", + "--project", + "prod", + "--type", + "metric, dataset", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + # Whitespace around commas is stripped. + mock.get_schema.assert_called_once_with(alias="prod", types=["metric", "dataset"]) + + def test_all_fetches_every_known_type(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.get_schema.return_value = { + "project": "prod", + "schemas": [{"type": t, "schema": {}} for t in SCHEMA_TYPE_ALIAS], + } + result = _invoke( + ["--json", "semantic-layer", "schema", "--project", "prod", "--all"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + mock.get_schema.assert_called_once_with(alias="prod", types=list(SCHEMA_TYPE_ALIAS)) + + def test_type_and_all_mutually_exclusive_exit_2(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--json", + "semantic-layer", + "schema", + "--project", + "prod", + "--type", + "metric", + "--all", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2, result.output + body = json.loads(result.output) + assert body["error"]["code"] == "USAGE_ERROR" + mock.get_schema.assert_not_called() + + def test_neither_type_nor_all_exit_2(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + ["--json", "semantic-layer", "schema", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2, result.output + body = json.loads(result.output) + assert body["error"]["code"] == "USAGE_ERROR" + mock.get_schema.assert_not_called() + + def test_empty_type_list_exit_2(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + ["--json", "semantic-layer", "schema", "--project", "prod", "--type", " , "], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2, result.output + body = json.loads(result.output) + assert body["error"]["code"] == "VALIDATION_ERROR" + mock.get_schema.assert_not_called() + + def test_unknown_type_maps_to_config_error_exit_5(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.get_schema.side_effect = ConfigError( + "Unknown semantic type(s): bogus. Valid types: model, dataset, metric, " + "relationship, constraint, glossary." + ) + result = _invoke( + ["--json", "semantic-layer", "schema", "--project", "prod", "--type", "bogus"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 5, result.output + body = json.loads(result.output) + assert body["error"]["code"] == "CONFIG_ERROR" + assert "bogus" in body["error"]["message"] + assert "glossary" in body["error"]["message"] + + def test_api_error_invalid_token_exits_3(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.get_schema.side_effect = KeboolaApiError( + message="bad token", status_code=401, error_code="INVALID_TOKEN" + ) + result = _invoke( + ["--json", "semantic-layer", "schema", "--project", "prod", "--type", "metric"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 3, result.output + body = json.loads(result.output) + assert body["error"]["code"] == "INVALID_TOKEN" + + def test_human_output_renders_per_type_panels(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.get_schema.return_value = { + "project": "prod", + "schemas": [ + {"type": "metric", "schema": _schema_for("semantic-metric")}, + {"type": "dataset", "schema": _schema_for("semantic-dataset")}, + ], + } + result = _invoke( + ["semantic-layer", "schema", "--project", "prod", "--type", "metric,dataset"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + assert "metric" in result.output + assert "dataset" in result.output + assert "schema" in result.output + # The JSON schema body is rendered inside the panels. + assert "semantic-metric" in result.output + + def test_missing_project_arg_exit_2(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + ["--json", "semantic-layer", "schema", "--type", "metric"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2 From b727879966e0a8ba16644c361c9890aa2fd7c580 Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:28:22 +0200 Subject: [PATCH 05/17] docs(context): document docs query + semantic-layer schema (#392, #394) --- CLAUDE.md | 5 +++++ src/keboola_agent_cli/commands/context.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index f62a89ad..ba2cb4dd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -515,6 +515,7 @@ kbagent semantic-layer model create --project P --name N [--description D] [--sq kbagent semantic-layer model delete --project P --model M [--yes] kbagent semantic-layer show --project P [--model M] [--type dataset|metric|relationship|constraint|glossary] kbagent semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N] +kbagent semantic-layer schema --project P (--type model|dataset|metric|relationship|constraint|glossary[,TYPE...] | --all) kbagent semantic-layer get-context --project P --context-id ID kbagent semantic-layer validate --project P [--model M] [--deep] kbagent semantic-layer export --project P [--model M] [--output PATH] @@ -580,6 +581,10 @@ kbagent kai chat --message "msg" [--chat-id ID] [--project NAME] kbagent kai chat-detail --chat-id ID [--project NAME] kbagent kai history [--project NAME] [--limit N] +kbagent docs query "QUESTION" [--project NAME] +# (0.73.0+) Documentation Q&A via the AI Service (server-side RAG). Unlike kai ask it does NOT +# see project data; works with any token. --json emits {query, text, source_urls}. + kbagent flow list [--project NAME] [--branch ID] [--with-schedules] kbagent flow detail --project NAME --flow-id ID [--branch ID] kbagent flow schema [--full --project NAME] diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 3dc607af..615bc498 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -1022,6 +1022,11 @@ Show a model's entities. --type filter: dataset|metric|relationship|constraint|glossary. Without --type prints a per-type count summary. + kbagent semantic-layer schema --project P (--type model|dataset|metric|relationship|constraint|glossary[,TYPE...] | --all) + (since 0.73.0) Live JSON Schema per semantic object type, fetched from the + deployed metastore (never bundled -- cannot drift). Exactly one of + --type/--all. --json emits {{project, schemas: [{{type, schema}}]}}. + kbagent semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N] (since 0.47.0) Project-wide glob search across semantic-layer entity names. Mirrors the upstream keboola-mcp-server search_semantic_context tool so a @@ -1267,6 +1272,15 @@ kbagent kai history [--project NAME] [--limit N] List recent Kai chat sessions. Default limit: 10. +### Documentation Q&A (since v0.73.0) + + kbagent docs query "QUESTION" [--project NAME] + Answer a natural-language question from the Keboola documentation via the + AI Service (server-side RAG; no local corpus). Returns the answer text + plus source URLs. --json emits {{query, text, source_urls}}. Unlike + `kai ask` this does NOT see project data -- it is documentation-only, + works with any token, and is the right tool for "how do I ..." questions. + ### Developer Portal (since v0.49.0) The `dev-portal` command group talks to `apps-api.keboola.com` (the Keboola From 8dbebf5f8c4658634cd19292fd4305110d1cb2b7 Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:29:29 +0200 Subject: [PATCH 06/17] docs(plugin): commands-reference entries for docs query + semantic-layer schema (#392, #394) --- .../kbagent/skills/kbagent/references/commands-reference.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 14c2db5f..ac326b00 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -205,6 +205,9 @@ Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, - `tool list [--project NAME] [--branch ID]` -- list available MCP tools (multi_project annotation) - `tool call TOOL_NAME [--project NAME] [--input JSON|@file|-] [--branch ID]` -- call MCP tool (read = all projects, write = single). `--input` accepts inline JSON, `@file.json`, or `-` (stdin) +## Documentation Q&A (since v0.73.0) +- `docs query "QUESTION" [--project NAME]` -- natural-language answer from the Keboola documentation via the AI Service (server-side RAG, no local corpus). Returns answer text + source URLs; `--json` emits `{query, text, source_urls}`. Unlike `kai ask` it does NOT see project data, works with any token (no master-token / feature-flag requirement), and is the right tool for "how do I ..." questions. Ports the `docs_query` MCP tool. + ## Kai (Keboola AI Assistant) Requires the project to be added with its **master ('owner') Storage API token** -- the auto-generated owner token, not a custom one. Custom tokens cannot access Kai. Also requires the `agent-chat` feature flag on the project. Use `kai preflight` to verify both conditions without raising. - `kai ping [--project NAME]` -- check Kai server health and MCP connection status. Fails with KAI_NOT_ENABLED if the agent-chat feature is missing or the token is not a master token @@ -286,6 +289,7 @@ Manage Keboola metastore models -- datasets, metrics, relationships, constraints - `semantic-layer model delete --project P --model M [--yes]` -- delete a model **and cascade-delete every child entity** (datasets, metrics, relationships, constraints, glossary terms) in `reversed(PUSH_ORDER)` (constraints first, datasets last) before the parent. Confirmation prompt unless `--yes`. **Cascade is unconditional in 0.43.4+** -- before that release the call only DELETEd the parent, silently leaking children pointing at the dead `modelUUID` and breaking subsequent `build` / `import` retries with HTTP 422 name collisions (closes #306). On any child-DELETE failure the parent is **preserved** and the response carries `details.cascade = {attempted, deleted, failures: [{type, id, name, error}], parent_deleted: False, model_uuid}` so the user can re-run after fixing the underlying error. Happy-path envelope adds `cascade.deleted` per-type counts. Legacy `orphaned_children` top-level key kept for back-compat (same shape, meaning flipped from "leaked" to "cascaded") but **deprecated -- removal scheduled for a future minor release**; new callers should read `cascade.deleted` instead. See [gotchas.md](gotchas.md) for the meaning-flip + deprecation note. - `semantic-layer show --project P [--model M] [--type T]` -- show a model's entities. `--type` filters to `dataset | metric | relationship | constraint | glossary`. Without `--type` prints a per-type count summary. `--model` is optional when the project has exactly one model. - `semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N]` (since 0.47.0) -- project-wide glob search across semantic-layer entity names. Mirrors the upstream `keboola-mcp-server search_semantic_context` MCP tool so a downstream caller can drop the MCP dependency for the pre-flight "is the model populated?" check. Patterns are case-sensitive `fnmatch`, repeatable (union); default `*`. Default `--type all` searches every CHILD type (`model` searches semantic models). `--limit N` short-circuits both per-type and outer loops. Envelope: `{project, contexts: [{id, type, name, description, attributes}], total_count}`; the `type` field is the CLI-friendly singular (no `semantic-` prefix). +- `semantic-layer schema --project P (--type model|dataset|metric|relationship|constraint|glossary[,TYPE...] | --all)` (since 0.73.0) -- live JSON Schema per semantic object type, fetched from the deployed metastore (`GET /api/v1/schema/{type}`; never bundled, cannot drift). Exactly one of `--type`/`--all` (usage error otherwise); `--type` takes a comma-separated list, fan-out is parallel. Envelope: `{project, schemas: [{type, schema}]}`. Ports the `get_semantic_schema` MCP tool. - `semantic-layer get-context --project P --context-id ID` (since 0.47.0) -- single-entry fetch by id, irrespective of type. Probes `semantic-model` first then every CHILD type (dataset / metric / relationship / constraint / glossary) until a 200 lands. 404 on any one type is non-terminal; only a full miss raises `NOT_FOUND` (exit 1). Non-404 errors (500, etc.) propagate immediately rather than being swallowed by the next probe. - `semantic-layer validate --project P [--model M] [--deep]` -- structural validation. Basic mode runs local checks: duplicate names, dangling rel/metric refs, SUM-on-PCT (warning), constraint orphans (metrics in `metrics[]` that no longer exist), severity-suffix mismatches between API `severity` and the 4-band name suffix. `--deep` adds parallel Snowflake column-existence probes via the in-process StorageService: phantom dataset fields, phantom column refs in metric SQL, AGG-on-STRING errors. Response: `{valid: bool, deep: bool, errors: [{type, item, detail}], warnings: [...]}`. - `semantic-layer export --project P [--model M] [--output PATH]` -- snapshot the model to a self-describing JSON file (default `./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json`). Schema-versioned for round-trip via `import` / `diff`. From 148cdb5fb10962a17d532559b78d6e7c9d553ceb Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:31:32 +0200 Subject: [PATCH 07/17] feat(component): sync-action + config examples (#393, #395) Ports run_sync_action (dedicated sync-actions.{stack} host, camelCase body, shallow root+row merge identical to the MCP tool) and get_config_examples (surfaces rootConfigurationExamples / rowConfigurationExamples the CLI already fetched but discarded). --- src/keboola_agent_cli/client.py | 62 ++ src/keboola_agent_cli/commands/component.py | 148 ++++ src/keboola_agent_cli/commands/config.py | 87 +++ .../services/component_service.py | 136 +++- tests/test_component_sync_action.py | 712 ++++++++++++++++++ tests/test_config_examples.py | 381 ++++++++++ 6 files changed, 1525 insertions(+), 1 deletion(-) create mode 100644 tests/test_component_sync_action.py create mode 100644 tests/test_config_examples.py diff --git a/src/keboola_agent_cli/client.py b/src/keboola_agent_cli/client.py index 161a4807..e6cc602d 100644 --- a/src/keboola_agent_cli/client.py +++ b/src/keboola_agent_cli/client.py @@ -175,6 +175,7 @@ def __init__(self, stack_url: str, token: str) -> None: self._queue_client: httpx.Client | None = None self._query_client: httpx.Client | None = None self._encrypt_client: httpx.Client | None = None + self._sync_actions_client: httpx.Client | None = None # Lazily built on first Data Streams call (per-device OTLP sources); the # Stream control plane is a sibling host reachable from this stack+token. self._stream_client: StreamClient | None = None @@ -197,6 +198,10 @@ def _query_base_url(self) -> str: def _encrypt_base_url(self) -> str: return self._derive_service_url(self._stack_url, "encryption") + @property + def _sync_actions_base_url(self) -> str: + return self._derive_service_url(self._stack_url, "sync-actions") + def close(self) -> None: """Close the underlying HTTP clients.""" super().close() @@ -206,6 +211,8 @@ def close(self) -> None: self._query_client.close() if self._encrypt_client is not None: self._encrypt_client.close() + if self._sync_actions_client is not None: + self._sync_actions_client.close() if self._stream_client is not None: self._stream_client.close() @@ -292,6 +299,61 @@ def encrypt_values( ) return response.json() + def _sync_actions_request(self, method: str, path: str, **kwargs: Any) -> httpx.Response: + """Execute a Sync Actions API request with retry. + + The Sync Actions service is a sibling host derived from the stack URL + (``sync-actions.{stack-suffix}``); the sub-client inherits the main + client's headers, so the ``X-StorageApi-Token`` auth carries over. + """ + client = self._get_or_create_sub_client("_sync_actions_client", self._sync_actions_base_url) + return self._do_request( + method, path, client=client, base_url=self._sync_actions_base_url, **kwargs + ) + + def run_sync_action( + self, + component_id: str, + action: str, + config_data: dict[str, Any], + branch_id: int | None = None, + timeout: float | None = None, + ) -> Any: + """Run a synchronous component action via the Sync Actions API. + + POSTs to ``/actions`` on the ``sync-actions.{stack-suffix}`` host. + Valid action names are component-defined (surfaced as + ``synchronous_actions`` in component metadata, e.g. ``testConnection``, + ``getTables``); the API validates them server-side. + + Args: + component_id: Component identifier (e.g. 'keboola.ex-db-mysql'). + action: Sync action name (freeform; component-defined). + config_data: The configData payload (typically + ``{"parameters": ..., "storage": ...}``). May carry secrets -- + never log it. + branch_id: If set, sent as ``branchId``; omitted entirely for the + production branch (the API treats an absent key as default). + timeout: Optional per-request timeout in seconds (sync actions can + run long, e.g. ``getTables`` against a large database). + + Returns: + The action result verbatim (opaque dict or list; shape is + action-specific). + """ + body: dict[str, Any] = { + "configData": config_data, + "componentId": component_id, + "action": action, + } + if branch_id is not None: + body["branchId"] = branch_id + request_kwargs: dict[str, Any] = {"json": body} + if timeout is not None: + request_kwargs["timeout"] = timeout + response = self._sync_actions_request("POST", "/actions", **request_kwargs) + return response.json() + def verify_token(self) -> TokenVerifyResponse: """Verify the storage API token and retrieve project information. diff --git a/src/keboola_agent_cli/commands/component.py b/src/keboola_agent_cli/commands/component.py index a911e047..cf2a5d30 100644 --- a/src/keboola_agent_cli/commands/component.py +++ b/src/keboola_agent_cli/commands/component.py @@ -4,11 +4,15 @@ No business logic belongs here. """ +import json + import typer from rich.console import Console from rich.panel import Panel +from rich.syntax import Syntax from rich.table import Table +from ..config_store import ConfigStore from ..constants import VALID_COMPONENT_TYPES from ..errors import ConfigError, ErrorCode, KeboolaApiError from ._helpers import ( @@ -17,7 +21,9 @@ get_formatter, get_service, map_error_to_exit_code, + resolve_branch, ) +from .config import _parse_json_input component_app = typer.Typer(help="Discover and inspect Keboola components") @@ -214,3 +220,145 @@ def component_detail( retryable=exc.retryable, ) raise typer.Exit(code=exit_code) from None + + +def _format_sync_action_result(console: Console, data: dict) -> None: + """Render the sync action result as a JSON syntax panel. + + The result shape is action-specific (opaque dict or list), so a + pretty-printed JSON block is the most honest human rendering. + """ + action = data.get("action", "") + component_id = data.get("component_id", "") + syntax = Syntax( + json.dumps(data.get("result"), indent=2, ensure_ascii=False), + "json", + theme="monokai", + ) + panel = Panel(syntax, title=f"Sync action '{action}' - {component_id}", expand=False) + console.print(panel) + + +@component_app.command("sync-action") +def component_sync_action( + ctx: typer.Context, + action_name: str = typer.Argument( + ..., + help="Sync action name (component-defined, e.g. testConnection, getTables)", + ), + component_id: str = typer.Option( + ..., + "--component-id", + help="Component ID (e.g. keboola.ex-db-mysql)", + ), + config_id: str | None = typer.Option( + None, + "--config-id", + help="Configuration ID whose stored configData to send (required unless --config-data)", + ), + row_id: str | None = typer.Option( + None, + "--row-id", + help="Configuration row ID to shallow-merge over the root configuration", + ), + project: str = typer.Option( + ..., + "--project", + help="Project alias", + ), + branch: int | None = typer.Option( + None, + "--branch", + help="Run in a specific dev branch ID (defaults to active branch)", + ), + config_data: str | None = typer.Option( + None, + "--config-data", + help="Explicit configData JSON: inline, @file.json, or - for stdin (skips config fetch)", + ), + timeout: int | None = typer.Option( + None, + "--timeout", + help="Request timeout in seconds for the action call (long actions e.g. getTables)", + ), +) -> None: + """Run a synchronous component action (e.g. testConnection). + + \b + Valid action names are component-defined -- the API validates them + server-side. By default the stored configuration (--config-id) is sent + as configData; with --row-id the row configuration is shallow-merged + over the root at the top level (row keys replace root keys wholesale, + matching the MCP run_sync_action tool). Use --config-data to send an + explicit payload instead. + + \b + Examples: + # Test a database extractor's stored credentials + kbagent component sync-action testConnection \\ + --component-id keboola.ex-db-mysql --config-id 123456 --project prod + + # Run against a specific row's configuration + kbagent component sync-action getTables \\ + --component-id keboola.ex-db-mysql --config-id 123456 --row-id 654321 --project prod + + # Send an explicit configData payload + kbagent component sync-action testConnection \\ + --component-id keboola.ex-db-mysql --project prod \\ + --config-data '{"parameters": {"db": {"host": "example.com"}}}' + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "component_service") + config_store: ConfigStore = ctx.obj["config_store"] + + if config_id is None and config_data is None: + formatter.error( + message="Either --config-id or --config-data is required.", + error_code=ErrorCode.MISSING_PARAMETER, + ) + raise typer.Exit(code=2) + + if row_id is not None and config_id is None: + formatter.error( + message="--row-id requires --config-id (rows belong to a configuration).", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + + override: dict | None = None + if config_data is not None: + try: + override = _parse_json_input(config_data) + except (json.JSONDecodeError, FileNotFoundError) as exc: + formatter.error( + message=f"Invalid --config-data input: {exc}", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) from None + + _, effective_branch = resolve_branch(config_store, formatter, project, branch) + + try: + result = service.run_sync_action( + alias=project, + component_id=component_id, + action=action_name, + config_id=config_id, + row_id=row_id, + branch_id=effective_branch, + config_data_override=override, + timeout=timeout, + ) + formatter.output(result, _format_sync_action_result) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error( + message=exc.message, + error_code=exc.error_code, + project=project, + retryable=exc.retryable, + ) + raise typer.Exit(code=exit_code) from None diff --git a/src/keboola_agent_cli/commands/config.py b/src/keboola_agent_cli/commands/config.py index 9a29bb3c..f6270f59 100644 --- a/src/keboola_agent_cli/commands/config.py +++ b/src/keboola_agent_cli/commands/config.py @@ -11,6 +11,7 @@ from typing import Any import typer +from rich.console import Console from rich.markup import escape from rich.syntax import Syntax @@ -385,6 +386,92 @@ def _format_config_detail_bulk( console.print() +def _format_config_examples(console: Console, data: dict) -> None: + """Render numbered JSON syntax blocks per example section. + + Sections absent from ``data`` (e.g. root examples filtered out by + ``--row``) are skipped entirely. + """ + component_id = data.get("component_id", "") + console.print( + f"\n[bold]Configuration examples for [cyan]{escape(component_id)}[/cyan][/bold]\n" + ) + + sections = ( + ("Root Configuration Examples", data.get("root_examples")), + ("Row Configuration Examples", data.get("row_examples")), + ) + for heading, examples in sections: + if examples is None: + continue + console.print(f"[bold underline]{heading}[/bold underline]") + if not examples: + console.print("[dim](none)[/dim]\n") + continue + for index, example in enumerate(examples, start=1): + console.print(f"[bold]Example {index}[/bold]") + syntax = Syntax( + json.dumps(example, indent=2, ensure_ascii=False), + "json", + theme="monokai", + ) + console.print(syntax) + console.print() + + +@config_app.command("examples", rich_help_panel="Browse") +def config_examples( + ctx: typer.Context, + component_id: str = typer.Option( + ..., + "--component-id", + help="Component ID (e.g. keboola.ex-google-drive)", + ), + project: str | None = typer.Option( + None, + "--project", + help="Project alias (uses first available if not set)", + ), + row: bool = typer.Option( + False, + "--row", + help="Show row configuration examples only", + ), +) -> None: + """Show sample configuration JSON examples for a component. + + Surfaces the root and row configuration examples published in the + component documentation (the same bodies 'config new' seeds scaffolds + from). Useful as a starting point before 'config update' or + 'config row-create'. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "component_service") + + try: + result = service.get_config_examples(alias=project, component_id=component_id) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error( + message=exc.message, + error_code=exc.error_code, + project=project or "", + retryable=exc.retryable, + ) + raise typer.Exit(code=exit_code) from None + + if row: + result = { + "component_id": result["component_id"], + "row_examples": result["row_examples"], + } + + formatter.output(result, _format_config_examples) + + @config_app.command("search", rich_help_panel="Browse") def config_search( ctx: typer.Context, diff --git a/src/keboola_agent_cli/services/component_service.py b/src/keboola_agent_cli/services/component_service.py index beea1f6d..0df9fd1c 100644 --- a/src/keboola_agent_cli/services/component_service.py +++ b/src/keboola_agent_cli/services/component_service.py @@ -14,7 +14,7 @@ from ..ai_client import AiServiceClient from ..config_store import ConfigStore from ..constants import SECRET_PLACEHOLDER -from ..errors import KeboolaApiError +from ..errors import ConfigError, KeboolaApiError from ..models import ComponentDetail, ComponentSuggestion, ProjectConfig from .base import BaseService, ClientFactory from .org_service import slugify @@ -419,6 +419,140 @@ def get_component_detail(self, alias: str, component_id: str) -> dict[str, Any]: "project_alias": alias, } + def get_config_examples(self, alias: str | None, component_id: str) -> dict[str, Any]: + """Fetch root and row configuration example bodies for a component. + + Ports the MCP ``get_config_examples`` tool (issue #393): the AI Service + component detail already carries ``rootConfigurationExamples`` / + ``rowConfigurationExamples``; this method surfaces the full bodies that + :meth:`get_component_detail` deliberately reduces to counts (its + contract is a summary and stays unchanged). + + Args: + alias: Project alias. When None, the first available project is + used (only the stack URL and token are needed). + component_id: The component identifier (e.g. 'keboola.ex-google-drive'). + + Returns: + Dict with keys ``component_id``, ``root_examples`` (list of dicts), + and ``row_examples`` (list of dicts). + + Raises: + ConfigError: If the alias is not found or no projects are configured. + KeboolaApiError: If the AI Service call fails. + """ + projects = self.resolve_projects([alias] if alias else None) + if not projects: + raise ConfigError( + "No projects configured. Use 'kbagent project add' to connect a project first." + ) + resolved_alias = alias or next(iter(projects)) + project = projects[resolved_alias] + + ai_client = self._ai_client_factory(project.stack_url, project.token) + try: + raw = ai_client.get_component_detail(component_id) + finally: + ai_client.close() + + detail = ComponentDetail(**raw) + return { + "component_id": detail.component_id, + "root_examples": detail.root_configuration_examples, + "row_examples": detail.row_configuration_examples, + } + + def run_sync_action( + self, + alias: str, + component_id: str, + action: str, + config_id: str | None = None, + row_id: str | None = None, + branch_id: int | None = None, + config_data_override: dict[str, Any] | None = None, + timeout: float | None = None, + ) -> dict[str, Any]: + """Run a synchronous component action (issue #395, MCP ``run_sync_action`` port). + + Builds the ``configData`` payload and delegates the POST to + :meth:`KeboolaClient.run_sync_action`. When ``config_data_override`` is + given it is sent verbatim (no config fetch). Otherwise the root + configuration is fetched (honoring ``branch_id``) and, when ``row_id`` + is given, the row configuration is SHALLOW-merged over it at the top + level only -- exactly like the MCP tool: a row-level ``parameters`` or + ``storage`` key REPLACES the root key wholesale (never deep-merged), + so e.g. a row ``storage.input`` replaces the root ``storage.input``. + + Args: + alias: Project alias (resolves stack URL + token). + component_id: Component identifier (e.g. 'keboola.ex-db-mysql'). + action: Sync action name (freeform; component-defined). + config_id: Configuration ID to build configData from. Required + unless ``config_data_override`` is provided. + row_id: Optional configuration row ID to shallow-merge over root. + branch_id: Optional dev branch ID (config fetch + action call). + config_data_override: Explicit configData dict; sent verbatim. + timeout: Optional per-request timeout in seconds for the action call. + + Returns: + Dict with keys ``component_id``, ``action``, and ``result`` (the + opaque action response -- dict or list, action-specific). + + Raises: + ConfigError: If the alias is unknown, or neither ``config_id`` nor + ``config_data_override`` is provided. + KeboolaApiError: If any API call fails. + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + + client = self._client_factory(project.stack_url, project.token) + try: + if config_data_override is not None: + config_data = config_data_override + else: + if config_id is None: + raise ConfigError( + "Either a configuration ID or explicit config data is required " + "to run a sync action." + ) + root = client.get_config_detail(component_id, config_id, branch_id=branch_id) + root_configuration = root.get("configuration") or {} + row_configuration: dict[str, Any] = {} + if row_id is not None: + row = client.get_config_row( + component_id, config_id, row_id, branch_id=branch_id + ) + row_configuration = row.get("configuration") or {} + # SHALLOW top-level merge (MCP parity): row keys replace root + # keys wholesale; do NOT deep-merge. + config_data = { + "parameters": { + **root_configuration.get("parameters", {}), + **row_configuration.get("parameters", {}), + }, + "storage": { + **root_configuration.get("storage", {}), + **row_configuration.get("storage", {}), + }, + } + result = client.run_sync_action( + component_id, + action, + config_data, + branch_id=branch_id, + timeout=timeout, + ) + finally: + client.close() + + return { + "component_id": component_id, + "action": action, + "result": result, + } + def generate_scaffold( self, alias: str, diff --git a/tests/test_component_sync_action.py b/tests/test_component_sync_action.py new file mode 100644 index 00000000..5651656a --- /dev/null +++ b/tests/test_component_sync_action.py @@ -0,0 +1,712 @@ +"""Tests for `kbagent component sync-action` (issue #395, MCP run_sync_action port). + +Covers three layers: +- L3 KeboolaClient.run_sync_action: sync-actions host derivation, camelCase + body keys, branchId omission for production, token header inheritance. +- L2 ComponentService.run_sync_action: override path, root-only configData, + root+row SHALLOW top-level merge semantics (no deep merge), branch + pass-through, config_id validation. +- L1 CLI command: --json envelope, --config-data JSON|@file|- parsing, + usage validation, human render, API error mapping. +""" + +import json +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from helpers import setup_single_project +from keboola_agent_cli.cli import app +from keboola_agent_cli.client import KeboolaClient +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.component_service import ComponentService +from keboola_agent_cli.services.project_service import ProjectService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" +STACK_URL = "https://connection.keboola.com" +SYNC_ACTIONS_URL = "https://sync-actions.keboola.com" + +COMPONENT_ID = "keboola.ex-db-mysql" +CONFIG_ID = "cfg-001" +ROW_ID = "row-001" + +runner = CliRunner() + + +# --------------------------------------------------------------------------- +# L3 client tests +# --------------------------------------------------------------------------- + + +class TestClientRunSyncAction: + """Tests for KeboolaClient.run_sync_action (sync-actions sub-client).""" + + def test_derive_sync_actions_url(self) -> None: + """Host derivation: connection. -> sync-actions..""" + result = KeboolaClient._derive_service_url( + "https://connection.eu-central-1.keboola.com", "sync-actions" + ) + assert result == "https://sync-actions.eu-central-1.keboola.com" + + def test_run_sync_action_posts_camelcase_body(self, httpx_mock) -> None: + """POST /actions carries configData/componentId/action (camelCase).""" + httpx_mock.add_response( + url=f"{SYNC_ACTIONS_URL}/actions", + method="POST", + json={"status": "success"}, + status_code=200, + ) + + config_data = {"parameters": {"db": {"host": "example.com"}}, "storage": {}} + with KeboolaClient(stack_url=STACK_URL, token=TEST_TOKEN) as client: + result = client.run_sync_action(COMPONENT_ID, "testConnection", config_data) + + assert result == {"status": "success"} + requests = httpx_mock.get_requests() + assert len(requests) == 1 + body = json.loads(requests[0].content) + assert body == { + "configData": config_data, + "componentId": COMPONENT_ID, + "action": "testConnection", + } + # branchId must be OMITTED entirely for production + assert "branchId" not in body + # Sub-client inherits the Storage API token header + assert requests[0].headers["X-StorageApi-Token"] == TEST_TOKEN + + def test_run_sync_action_sends_branch_id_when_set(self, httpx_mock) -> None: + """branchId is included in the body when branch_id is given.""" + httpx_mock.add_response( + url=f"{SYNC_ACTIONS_URL}/actions", + method="POST", + json=[{"name": "table1"}], + status_code=200, + ) + + with KeboolaClient(stack_url=STACK_URL, token=TEST_TOKEN) as client: + result = client.run_sync_action( + COMPONENT_ID, "getTables", {"parameters": {}}, branch_id=456 + ) + + # Opaque pass-through: list results survive verbatim + assert result == [{"name": "table1"}] + body = json.loads(httpx_mock.get_requests()[0].content) + assert body["branchId"] == 456 + + def test_run_sync_action_api_error(self, httpx_mock) -> None: + """Non-retryable HTTP errors raise KeboolaApiError.""" + httpx_mock.add_response( + url=f"{SYNC_ACTIONS_URL}/actions", + method="POST", + json={"error": "Action 'nope' not found"}, + status_code=404, + ) + + with ( + KeboolaClient(stack_url=STACK_URL, token=TEST_TOKEN) as client, + pytest.raises(KeboolaApiError) as exc_info, + ): + client.run_sync_action(COMPONENT_ID, "nope", {"parameters": {}}) + + assert exc_info.value.status_code == 404 + + +# --------------------------------------------------------------------------- +# L2 service tests +# --------------------------------------------------------------------------- + + +def _root_config_response( + parameters: dict[str, Any] | None = None, + storage: dict[str, Any] | None = None, +) -> dict[str, Any]: + return { + "id": CONFIG_ID, + "name": "MySQL extractor", + "configuration": { + "parameters": parameters if parameters is not None else {}, + "storage": storage if storage is not None else {}, + }, + } + + +def _row_config_response( + parameters: dict[str, Any] | None = None, + storage: dict[str, Any] | None = None, +) -> dict[str, Any]: + return { + "id": ROW_ID, + "configuration": { + "parameters": parameters if parameters is not None else {}, + "storage": storage if storage is not None else {}, + }, + } + + +def _make_service(tmp_config_dir: Path, client: MagicMock) -> ComponentService: + store = setup_single_project(tmp_config_dir) + return ComponentService( + config_store=store, + client_factory=lambda url, token: client, + ) + + +class TestRunSyncActionService: + """Tests for ComponentService.run_sync_action.""" + + def test_override_path_sends_verbatim(self, tmp_config_dir: Path) -> None: + """config_data_override is sent as-is; no config fetch happens.""" + client = MagicMock() + client.run_sync_action.return_value = {"status": "success"} + service = _make_service(tmp_config_dir, client) + + override = {"parameters": {"db": {"host": "explicit.example.com"}}} + result = service.run_sync_action( + alias="prod", + component_id=COMPONENT_ID, + action="testConnection", + config_data_override=override, + ) + + assert result == { + "component_id": COMPONENT_ID, + "action": "testConnection", + "result": {"status": "success"}, + } + client.get_config_detail.assert_not_called() + client.get_config_row.assert_not_called() + client.run_sync_action.assert_called_once_with( + COMPONENT_ID, + "testConnection", + override, + branch_id=None, + timeout=None, + ) + client.close.assert_called_once() + + def test_root_only_builds_config_data_from_root(self, tmp_config_dir: Path) -> None: + """Without row_id, configData is the root parameters + storage.""" + client = MagicMock() + client.get_config_detail.return_value = _root_config_response( + parameters={"db": {"host": "root.example.com", "port": 3306}}, + storage={"input": {"tables": [{"source": "in.c-main.a"}]}}, + ) + client.run_sync_action.return_value = {"status": "success"} + service = _make_service(tmp_config_dir, client) + + service.run_sync_action( + alias="prod", + component_id=COMPONENT_ID, + action="testConnection", + config_id=CONFIG_ID, + ) + + client.get_config_detail.assert_called_once_with(COMPONENT_ID, CONFIG_ID, branch_id=None) + client.get_config_row.assert_not_called() + sent_config_data = client.run_sync_action.call_args.args[2] + assert sent_config_data == { + "parameters": {"db": {"host": "root.example.com", "port": 3306}}, + "storage": {"input": {"tables": [{"source": "in.c-main.a"}]}}, + } + + def test_root_row_shallow_merge_not_deep(self, tmp_config_dir: Path) -> None: + """Row keys REPLACE root keys at the top level -- never deep-merged. + + The root has parameters.db = {host, port}; the row overrides + parameters.db = {host}. A deep merge would keep root's port; the MCP + parity semantics require the row dict to replace root's wholesale. + """ + client = MagicMock() + client.get_config_detail.return_value = _root_config_response( + parameters={ + "db": {"host": "root.example.com", "port": 3306}, + "rootOnly": "kept", + }, + storage={"input": {"tables": [{"source": "in.c-main.root"}]}}, + ) + client.get_config_row.return_value = _row_config_response( + parameters={"db": {"host": "row.example.com"}}, + storage={"input": {"tables": [{"source": "in.c-main.row"}]}}, + ) + client.run_sync_action.return_value = {"status": "success"} + service = _make_service(tmp_config_dir, client) + + service.run_sync_action( + alias="prod", + component_id=COMPONENT_ID, + action="getTables", + config_id=CONFIG_ID, + row_id=ROW_ID, + ) + + client.get_config_row.assert_called_once_with( + COMPONENT_ID, CONFIG_ID, ROW_ID, branch_id=None + ) + sent_config_data = client.run_sync_action.call_args.args[2] + # Top-level key from row replaces root's dict wholesale: + assert sent_config_data["parameters"]["db"] == {"host": "row.example.com"} + assert "port" not in sent_config_data["parameters"]["db"], ( + "root's nested 'port' must NOT survive -- shallow merge, not deep" + ) + # Root-only top-level keys survive the shallow merge: + assert sent_config_data["parameters"]["rootOnly"] == "kept" + # storage merged independently with the same semantics: + assert sent_config_data["storage"] == {"input": {"tables": [{"source": "in.c-main.row"}]}} + + def test_branch_pass_through(self, tmp_config_dir: Path) -> None: + """branch_id flows to config fetch, row fetch, and the action call.""" + client = MagicMock() + client.get_config_detail.return_value = _root_config_response() + client.get_config_row.return_value = _row_config_response() + client.run_sync_action.return_value = {"status": "success"} + service = _make_service(tmp_config_dir, client) + + service.run_sync_action( + alias="prod", + component_id=COMPONENT_ID, + action="testConnection", + config_id=CONFIG_ID, + row_id=ROW_ID, + branch_id=456, + ) + + client.get_config_detail.assert_called_once_with(COMPONENT_ID, CONFIG_ID, branch_id=456) + client.get_config_row.assert_called_once_with( + COMPONENT_ID, CONFIG_ID, ROW_ID, branch_id=456 + ) + assert client.run_sync_action.call_args.kwargs["branch_id"] == 456 + + def test_timeout_pass_through(self, tmp_config_dir: Path) -> None: + """timeout is forwarded to the client action call.""" + client = MagicMock() + client.run_sync_action.return_value = {"status": "success"} + service = _make_service(tmp_config_dir, client) + + service.run_sync_action( + alias="prod", + component_id=COMPONENT_ID, + action="testConnection", + config_data_override={"parameters": {}}, + timeout=120, + ) + + assert client.run_sync_action.call_args.kwargs["timeout"] == 120 + + def test_missing_config_id_without_override_raises(self, tmp_config_dir: Path) -> None: + """Neither config_id nor override -> ConfigError; client still closed.""" + client = MagicMock() + service = _make_service(tmp_config_dir, client) + + with pytest.raises(ConfigError, match="configuration ID"): + service.run_sync_action( + alias="prod", + component_id=COMPONENT_ID, + action="testConnection", + ) + + client.run_sync_action.assert_not_called() + client.close.assert_called_once() + + def test_api_error_propagates_and_closes_client(self, tmp_config_dir: Path) -> None: + """Errors from the action call bubble up; client is closed.""" + client = MagicMock() + client.get_config_detail.return_value = _root_config_response() + client.run_sync_action.side_effect = KeboolaApiError( + message="Action failed", + status_code=400, + error_code="API_ERROR", + retryable=False, + ) + service = _make_service(tmp_config_dir, client) + + with pytest.raises(KeboolaApiError): + service.run_sync_action( + alias="prod", + component_id=COMPONENT_ID, + action="testConnection", + config_id=CONFIG_ID, + ) + + client.close.assert_called_once() + + +# --------------------------------------------------------------------------- +# L1 CLI tests +# --------------------------------------------------------------------------- + + +def _invoke(tmp_path: Path, mock_svc: MagicMock, args: list[str]): + config_dir = tmp_path / "config" + config_dir.mkdir(exist_ok=True) + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url=STACK_URL, + token=TEST_TOKEN, + project_name="prod", + project_id=1234, + ), + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ComponentService") as MockCompService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCompService.return_value = mock_svc + + return runner.invoke(app, args) + + +def _sync_action_result() -> dict[str, Any]: + return { + "component_id": COMPONENT_ID, + "action": "testConnection", + "result": {"status": "success"}, + } + + +class TestComponentSyncActionCli: + """Tests for `kbagent component sync-action` command.""" + + def test_sync_action_json(self, tmp_path: Path) -> None: + """--json emits the {component_id, action, result} envelope.""" + mock_svc = MagicMock() + mock_svc.run_sync_action.return_value = _sync_action_result() + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "component", + "sync-action", + "testConnection", + "--component-id", + COMPONENT_ID, + "--config-id", + CONFIG_ID, + "--project", + "prod", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"] == _sync_action_result() + mock_svc.run_sync_action.assert_called_once_with( + alias="prod", + component_id=COMPONENT_ID, + action="testConnection", + config_id=CONFIG_ID, + row_id=None, + branch_id=None, + config_data_override=None, + timeout=None, + ) + + def test_sync_action_config_data_inline(self, tmp_path: Path) -> None: + """--config-data inline JSON is parsed and passed as the override.""" + mock_svc = MagicMock() + mock_svc.run_sync_action.return_value = _sync_action_result() + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "component", + "sync-action", + "testConnection", + "--component-id", + COMPONENT_ID, + "--project", + "prod", + "--config-data", + '{"parameters": {"db": {"host": "explicit.example.com"}}}', + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + call_kwargs = mock_svc.run_sync_action.call_args.kwargs + assert call_kwargs["config_data_override"] == { + "parameters": {"db": {"host": "explicit.example.com"}} + } + assert call_kwargs["config_id"] is None + + def test_sync_action_config_data_from_file(self, tmp_path: Path) -> None: + """--config-data @file.json reads the payload from disk.""" + payload = {"parameters": {"token": "from-file"}} + payload_file = tmp_path / "payload.json" + payload_file.write_text(json.dumps(payload), encoding="utf-8") + + mock_svc = MagicMock() + mock_svc.run_sync_action.return_value = _sync_action_result() + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "component", + "sync-action", + "testConnection", + "--component-id", + COMPONENT_ID, + "--project", + "prod", + "--config-data", + f"@{payload_file}", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert mock_svc.run_sync_action.call_args.kwargs["config_data_override"] == payload + + def test_sync_action_invalid_config_data(self, tmp_path: Path) -> None: + """Malformed --config-data JSON is a validation error (exit 2).""" + mock_svc = MagicMock() + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "component", + "sync-action", + "testConnection", + "--component-id", + COMPONENT_ID, + "--project", + "prod", + "--config-data", + "{not json", + ], + ) + + assert result.exit_code == 2 + output = json.loads(result.output) + assert output["status"] == "error" + assert "VALIDATION_ERROR" in output["error"]["code"] + mock_svc.run_sync_action.assert_not_called() + + def test_sync_action_missing_config_id_and_data(self, tmp_path: Path) -> None: + """Neither --config-id nor --config-data -> usage error (exit 2).""" + mock_svc = MagicMock() + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "component", + "sync-action", + "testConnection", + "--component-id", + COMPONENT_ID, + "--project", + "prod", + ], + ) + + assert result.exit_code == 2 + output = json.loads(result.output) + assert output["status"] == "error" + assert "MISSING_PARAMETER" in output["error"]["code"] + mock_svc.run_sync_action.assert_not_called() + + def test_sync_action_row_id_requires_config_id(self, tmp_path: Path) -> None: + """--row-id without --config-id is rejected (exit 2).""" + mock_svc = MagicMock() + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "component", + "sync-action", + "testConnection", + "--component-id", + COMPONENT_ID, + "--project", + "prod", + "--row-id", + ROW_ID, + "--config-data", + "{}", + ], + ) + + assert result.exit_code == 2 + output = json.loads(result.output) + assert output["status"] == "error" + assert "INVALID_ARGUMENT" in output["error"]["code"] + mock_svc.run_sync_action.assert_not_called() + + def test_sync_action_branch_pass_through(self, tmp_path: Path) -> None: + """Explicit --branch is forwarded to the service as branch_id.""" + mock_svc = MagicMock() + mock_svc.run_sync_action.return_value = _sync_action_result() + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "component", + "sync-action", + "getTables", + "--component-id", + COMPONENT_ID, + "--config-id", + CONFIG_ID, + "--row-id", + ROW_ID, + "--project", + "prod", + "--branch", + "456", + "--timeout", + "120", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + mock_svc.run_sync_action.assert_called_once_with( + alias="prod", + component_id=COMPONENT_ID, + action="getTables", + config_id=CONFIG_ID, + row_id=ROW_ID, + branch_id=456, + config_data_override=None, + timeout=120, + ) + + def test_sync_action_human(self, tmp_path: Path) -> None: + """Human mode renders a JSON syntax panel with the action title.""" + mock_svc = MagicMock() + mock_svc.run_sync_action.return_value = _sync_action_result() + + result = _invoke( + tmp_path, + mock_svc, + [ + "component", + "sync-action", + "testConnection", + "--component-id", + COMPONENT_ID, + "--config-id", + CONFIG_ID, + "--project", + "prod", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "testConnection" in result.output + assert "status" in result.output + assert "success" in result.output + + def test_sync_action_api_error_auth(self, tmp_path: Path) -> None: + """INVALID_TOKEN maps to exit code 3 (auth error).""" + mock_svc = MagicMock() + mock_svc.run_sync_action.side_effect = KeboolaApiError( + message="Invalid access token", + status_code=401, + error_code="INVALID_TOKEN", + retryable=False, + ) + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "component", + "sync-action", + "testConnection", + "--component-id", + COMPONENT_ID, + "--config-id", + CONFIG_ID, + "--project", + "prod", + ], + ) + + assert result.exit_code == 3 + output = json.loads(result.output) + assert output["status"] == "error" + assert "INVALID_TOKEN" in output["error"]["code"] + + def test_sync_action_api_error_general(self, tmp_path: Path) -> None: + """Generic API errors map to exit code 1.""" + mock_svc = MagicMock() + mock_svc.run_sync_action.side_effect = KeboolaApiError( + message="Action 'nope' not found", + status_code=404, + error_code="NOT_FOUND", + retryable=False, + ) + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "component", + "sync-action", + "nope", + "--component-id", + COMPONENT_ID, + "--config-id", + CONFIG_ID, + "--project", + "prod", + ], + ) + + assert result.exit_code == 1 + output = json.loads(result.output) + assert output["status"] == "error" + assert "NOT_FOUND" in output["error"]["code"] + + def test_sync_action_config_error(self, tmp_path: Path) -> None: + """ConfigError from the service maps to exit code 5.""" + mock_svc = MagicMock() + mock_svc.run_sync_action.side_effect = ConfigError("Project 'nope' not found") + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "component", + "sync-action", + "testConnection", + "--component-id", + COMPONENT_ID, + "--config-id", + CONFIG_ID, + "--project", + "prod", + ], + ) + + assert result.exit_code == 5 + output = json.loads(result.output) + assert output["status"] == "error" + assert "CONFIG_ERROR" in output["error"]["code"] diff --git a/tests/test_config_examples.py b/tests/test_config_examples.py new file mode 100644 index 00000000..b2b9a936 --- /dev/null +++ b/tests/test_config_examples.py @@ -0,0 +1,381 @@ +"""Tests for `kbagent config examples` (issue #393, MCP get_config_examples port). + +Covers the L2 ComponentService.get_config_examples method (full example +bodies surfaced, alias resolution, error propagation) and the L1 CLI command +(--json structure, --row filter, human render, API error mapping). +""" + +import json +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from helpers import setup_single_project +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.component_service import ComponentService +from keboola_agent_cli.services.project_service import ProjectService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + +runner = CliRunner() + +COMPONENT_DETAIL_RESPONSE: dict[str, Any] = { + "componentId": "keboola.ex-google-drive", + "componentName": "Google Drive", + "componentType": "extractor", + "componentCategories": [], + "componentFlags": [], + "description": "Extract files from Google Drive", + "longDescription": "", + "documentationUrl": "", + "documentation": "", + "configurationSchema": {}, + "configurationRowSchema": {}, + "rootConfigurationExamples": [ + {"parameters": {"baseUrl": "https://example.com", "auth": {"type": "oauth"}}}, + {"parameters": {"baseUrl": "https://other.example.com"}}, + ], + "rowConfigurationExamples": [ + {"parameters": {"fileId": "abc123", "outputTable": "out.c-main.drive"}}, + ], +} + + +def _make_ai_client(detail_response: dict[str, Any]) -> MagicMock: + ai_client = MagicMock() + ai_client.get_component_detail.return_value = detail_response + return ai_client + + +def _make_service(tmp_config_dir: Path, ai_client: MagicMock) -> ComponentService: + store = setup_single_project(tmp_config_dir) + return ComponentService( + config_store=store, + ai_client_factory=lambda url, token: ai_client, + ) + + +# --------------------------------------------------------------------------- +# L2 service tests +# --------------------------------------------------------------------------- + + +class TestGetConfigExamplesService: + """Tests for ComponentService.get_config_examples.""" + + def test_returns_full_example_bodies(self, tmp_config_dir: Path) -> None: + """Both example lists are returned verbatim (not just counts).""" + ai_client = _make_ai_client(COMPONENT_DETAIL_RESPONSE) + service = _make_service(tmp_config_dir, ai_client) + + result = service.get_config_examples(alias="prod", component_id="keboola.ex-google-drive") + + assert result == { + "component_id": "keboola.ex-google-drive", + "root_examples": COMPONENT_DETAIL_RESPONSE["rootConfigurationExamples"], + "row_examples": COMPONENT_DETAIL_RESPONSE["rowConfigurationExamples"], + } + ai_client.get_component_detail.assert_called_once_with("keboola.ex-google-drive") + ai_client.close.assert_called_once() + + def test_alias_none_uses_first_project(self, tmp_config_dir: Path) -> None: + """When alias is None the first configured project is used.""" + ai_client = _make_ai_client(COMPONENT_DETAIL_RESPONSE) + service = _make_service(tmp_config_dir, ai_client) + + result = service.get_config_examples(alias=None, component_id="keboola.ex-google-drive") + + assert result["component_id"] == "keboola.ex-google-drive" + ai_client.get_component_detail.assert_called_once_with("keboola.ex-google-drive") + + def test_no_projects_raises_config_error(self, tmp_config_dir: Path) -> None: + """Without any configured project a ConfigError is raised.""" + store = ConfigStore(config_dir=tmp_config_dir) + ai_client = _make_ai_client(COMPONENT_DETAIL_RESPONSE) + service = ComponentService( + config_store=store, + ai_client_factory=lambda url, token: ai_client, + ) + + with pytest.raises(ConfigError, match="No projects configured"): + service.get_config_examples(alias=None, component_id="keboola.ex-http") + + ai_client.get_component_detail.assert_not_called() + + def test_api_error_propagates_and_closes_client(self, tmp_config_dir: Path) -> None: + """AI Service errors bubble up as KeboolaApiError; client is closed.""" + ai_client = MagicMock() + ai_client.get_component_detail.side_effect = KeboolaApiError( + message="Component not found", + status_code=404, + error_code="NOT_FOUND", + retryable=False, + ) + service = _make_service(tmp_config_dir, ai_client) + + with pytest.raises(KeboolaApiError): + service.get_config_examples(alias="prod", component_id="no.such.component") + + ai_client.close.assert_called_once() + + def test_component_detail_contract_unchanged(self, tmp_config_dir: Path) -> None: + """get_component_detail still returns counts only (no example bodies).""" + ai_client = _make_ai_client(COMPONENT_DETAIL_RESPONSE) + service = _make_service(tmp_config_dir, ai_client) + + result = service.get_component_detail(alias="prod", component_id="keboola.ex-google-drive") + + assert result["examples_count"] == 2 + assert result["row_examples_count"] == 1 + assert "root_examples" not in result + assert "row_examples" not in result + + +# --------------------------------------------------------------------------- +# L1 CLI tests +# --------------------------------------------------------------------------- + + +def _setup_config(config_dir: Path) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=1234, + ), + ) + return store + + +def _invoke(tmp_path: Path, mock_svc: MagicMock, args: list[str]): + config_dir = tmp_path / "config" + config_dir.mkdir(exist_ok=True) + store = _setup_config(config_dir) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ComponentService") as MockCompService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCompService.return_value = mock_svc + + return runner.invoke(app, args) + + +def _examples_result() -> dict[str, Any]: + return { + "component_id": "keboola.ex-google-drive", + "root_examples": [ + {"parameters": {"baseUrl": "https://example.com"}}, + {"parameters": {"baseUrl": "https://other.example.com"}}, + ], + "row_examples": [ + {"parameters": {"fileId": "abc123"}}, + ], + } + + +class TestConfigExamplesCli: + """Tests for `kbagent config examples` command.""" + + def test_examples_json(self, tmp_path: Path) -> None: + """--json emits the structured {component_id, root_examples, row_examples} dict.""" + mock_svc = MagicMock() + mock_svc.get_config_examples.return_value = _examples_result() + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "config", + "examples", + "--component-id", + "keboola.ex-google-drive", + "--project", + "prod", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"]["component_id"] == "keboola.ex-google-drive" + assert len(output["data"]["root_examples"]) == 2 + assert output["data"]["root_examples"][0]["parameters"]["baseUrl"] == ( + "https://example.com" + ) + assert len(output["data"]["row_examples"]) == 1 + mock_svc.get_config_examples.assert_called_once_with( + alias="prod", + component_id="keboola.ex-google-drive", + ) + + def test_examples_row_filter_json(self, tmp_path: Path) -> None: + """--row filters the JSON payload to row examples only.""" + mock_svc = MagicMock() + mock_svc.get_config_examples.return_value = _examples_result() + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "config", + "examples", + "--component-id", + "keboola.ex-google-drive", + "--row", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"]["component_id"] == "keboola.ex-google-drive" + assert "root_examples" not in output["data"] + assert len(output["data"]["row_examples"]) == 1 + # --project omitted -> service receives alias=None (first available) + mock_svc.get_config_examples.assert_called_once_with( + alias=None, + component_id="keboola.ex-google-drive", + ) + + def test_examples_human(self, tmp_path: Path) -> None: + """Human mode renders numbered JSON blocks under both headings.""" + mock_svc = MagicMock() + mock_svc.get_config_examples.return_value = _examples_result() + + result = _invoke( + tmp_path, + mock_svc, + [ + "config", + "examples", + "--component-id", + "keboola.ex-google-drive", + "--project", + "prod", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "Root Configuration Examples" in result.output + assert "Row Configuration Examples" in result.output + assert "Example 1" in result.output + assert "Example 2" in result.output + assert "baseUrl" in result.output + assert "fileId" in result.output + + def test_examples_human_row_only(self, tmp_path: Path) -> None: + """--row in human mode hides the root section entirely.""" + mock_svc = MagicMock() + mock_svc.get_config_examples.return_value = _examples_result() + + result = _invoke( + tmp_path, + mock_svc, + [ + "config", + "examples", + "--component-id", + "keboola.ex-google-drive", + "--row", + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "Root Configuration Examples" not in result.output + assert "Row Configuration Examples" in result.output + assert "fileId" in result.output + + def test_examples_empty_lists_human(self, tmp_path: Path) -> None: + """Components without examples render a (none) placeholder, exit 0.""" + mock_svc = MagicMock() + mock_svc.get_config_examples.return_value = { + "component_id": "keboola.ex-empty", + "root_examples": [], + "row_examples": [], + } + + result = _invoke( + tmp_path, + mock_svc, + ["config", "examples", "--component-id", "keboola.ex-empty"], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "(none)" in result.output + + def test_examples_api_error(self, tmp_path: Path) -> None: + """API NOT_FOUND maps to exit code 1 with structured error output.""" + mock_svc = MagicMock() + mock_svc.get_config_examples.side_effect = KeboolaApiError( + message="Component 'no.such.component' not found", + status_code=404, + error_code="NOT_FOUND", + retryable=False, + ) + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "config", + "examples", + "--component-id", + "no.such.component", + "--project", + "prod", + ], + ) + + assert result.exit_code == 1 + output = json.loads(result.output) + assert output["status"] == "error" + assert "NOT_FOUND" in output["error"]["code"] + + def test_examples_config_error(self, tmp_path: Path) -> None: + """ConfigError (e.g. unknown alias) maps to exit code 5.""" + mock_svc = MagicMock() + mock_svc.get_config_examples.side_effect = ConfigError("Project 'nope' not found") + + result = _invoke( + tmp_path, + mock_svc, + [ + "--json", + "config", + "examples", + "--component-id", + "keboola.ex-http", + "--project", + "nope", + ], + ) + + assert result.exit_code == 5 + output = json.loads(result.output) + assert output["status"] == "error" + assert "CONFIG_ERROR" in output["error"]["code"] + + def test_examples_missing_component_id(self, tmp_path: Path) -> None: + """Missing required --component-id is a usage error (exit 2).""" + mock_svc = MagicMock() + + result = _invoke(tmp_path, mock_svc, ["--json", "config", "examples"]) + + assert result.exit_code == 2 + mock_svc.get_config_examples.assert_not_called() From fe13f2d2f3ed2955999557c251fb9cbe8a36a169 Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:32:32 +0200 Subject: [PATCH 08/17] docs: document component sync-action + config examples (#393, #395) --- CLAUDE.md | 6 ++++++ .../kbagent/references/commands-reference.md | 2 ++ src/keboola_agent_cli/commands/context.py | 16 ++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index ba2cb4dd..e5daf5b0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -468,6 +468,12 @@ kbagent data-app git-credentials-create --project NAME --app-id ID --type ssh_ke kbagent component list [--project NAME] [--type TYPE] [--query QUERY] kbagent component detail --component-id ID [--project NAME] +kbagent component sync-action ACTION_NAME --component-id ID --project ALIAS (--config-id ID [--row-id ID] | --config-data JSON|@file|-) [--branch ID] [--timeout N] +# sync-action (0.73.0+): POST sync-actions.{stack}/actions; ACTION_NAME freeform (component-defined, +# e.g. testConnection/getTables); --row-id shallow-merges row over root at TOP level only (row +# parameters/storage keys replace root wholesale, MCP parity -- NOT deep merge); --config-data +# sends explicit configData verbatim (skips fetch); branchId omitted from body for production. +kbagent config examples --component-id ID [--project NAME] [--row] kbagent config new --component-id ID [--name NAME] [--project NAME] [--output-dir DIR] [--push --no-files --description D --configuration JSON|@file|- --configuration-file PATH --no-validate --branch ID --dry-run --allow-plaintext-on-encrypt-failure] # sync: GitOps -- configs as local files. init/pull/push/diff are filesystem-local (no serve REST surface). diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index ac326b00..2b4c7994 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -63,6 +63,8 @@ Requires a **super-admin** Manage API token (same kind as `org setup`). Same def - `feature user-remove --project ALIAS --email EMAIL --feature NAME [--dry-run] [--yes]` -- disable a feature on a user (`DELETE /manage/users/{email}/features/{name}`). ## Component Discovery +- `component sync-action ACTION_NAME --component-id ID --project ALIAS (--config-id ID [--row-id ID] | --config-data JSON|@file|-) [--branch ID] [--timeout N]` (since 0.73.0) -- run a synchronous component action (`testConnection`, `getTables`, ...) on the `sync-actions.{stack}` service. `ACTION_NAME` is freeform (component-defined; discover via `component detail` `synchronous_actions`). `--row-id` shallow-merges the row over the root config at TOP level only (row `parameters`/`storage` replace root wholesale -- NOT deep merge; MCP `run_sync_action` parity). `--config-data` sends explicit `configData` verbatim. Response is action-specific pass-through. Ports the `run_sync_action` MCP tool. +- `config examples --component-id ID [--project NAME] [--row]` (since 0.73.0) -- sample root/row configurations from the AI-service component detail. `--json` emits `{component_id, root_examples, row_examples}`; `--row` limits to row examples. Ports the `get_config_examples` MCP tool. - `component list [--project NAME] [--type TYPE] [--query "text"]` -- list/search components (AI-powered with `--query`) - `component detail --component-id ID [--project NAME]` -- show component schema, docs URL, examples diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 615bc498..cc5d5aa7 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -139,6 +139,16 @@ kbagent component detail --component-id ID [--project NAME] Show component docs, config schema, and examples count. + kbagent component sync-action ACTION_NAME --component-id ID --project ALIAS (--config-id ID [--row-id ID] | --config-data JSON|@file|-) [--branch ID] [--timeout N] + (since 0.73.0) Run a synchronous component action (testConnection, getTables, + ...) on the dedicated sync-actions service. ACTION_NAME is freeform -- + valid names are component-defined (see component detail synchronous_actions). + --row-id shallow-merges the row over the root config at TOP level only + (row parameters/storage keys replace root wholesale -- NOT a deep merge; + MCP run_sync_action parity). --config-data sends explicit configData + verbatim and skips the config fetch. Response shape is action-specific + (opaque pass-through). + ### Configuration Browsing kbagent config list [--project NAME] [--component-type TYPE] [--component-id ID] [--branch ID] [--include-rows] @@ -208,6 +218,12 @@ kbagent config search --query PATTERN [--project NAME] [--component-type TYPE] [-i] [-r] [--branch ID] Search config bodies for string/regex. Reports match location in JSON tree. Branch-aware. + kbagent config examples --component-id ID [--project NAME] [--row] + (since 0.73.0) Sample root/row configurations for a component, straight from + the AI-service component detail (same data the UI shows). --row limits to + row examples. --json emits {{component_id, root_examples, row_examples}} -- + structured dicts, ideal as a starting point before config new / row-create. + kbagent config variables-set --project NAME --component-id ID --config-id ID --var KEY=VALUE [--var ...] [--replace] [--variables-id ID] [--values-id ID] [--branch ID] [--dry-run] Assign variables to any config. Auto-creates the backing keboola.variables + default row on first call and links the parent; subsequent calls update the same row (merge by default; From 3fd83042a2ebd831fe221b3bcd821c2186cc4416 Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:37:15 +0200 Subject: [PATCH 09/17] fix(semantic-layer): schema resolves the default version to a real JSON Schema (#394) Live metastore returns only a {versions:[...]} listing from the bare /api/v1/schema/{type} endpoint -- no schema body. The upstream MCP tool ships that listing as-is (upstream gap, not mirrored): the service now resolves isDefault (fallback: first entry) and fetches /{version}, returning {type, schema, schema_version}. Live-verified: real $schema/ properties document, version 1.0.0. --- .../kbagent/references/commands-reference.md | 2 +- src/keboola_agent_cli/metastore_client.py | 24 +++++----- .../services/semantic_layer_service.py | 41 +++++++++++++++-- tests/test_semantic_layer_schema.py | 44 ++++++++++++++++++- 4 files changed, 95 insertions(+), 16 deletions(-) diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 2b4c7994..6b302076 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -291,7 +291,7 @@ Manage Keboola metastore models -- datasets, metrics, relationships, constraints - `semantic-layer model delete --project P --model M [--yes]` -- delete a model **and cascade-delete every child entity** (datasets, metrics, relationships, constraints, glossary terms) in `reversed(PUSH_ORDER)` (constraints first, datasets last) before the parent. Confirmation prompt unless `--yes`. **Cascade is unconditional in 0.43.4+** -- before that release the call only DELETEd the parent, silently leaking children pointing at the dead `modelUUID` and breaking subsequent `build` / `import` retries with HTTP 422 name collisions (closes #306). On any child-DELETE failure the parent is **preserved** and the response carries `details.cascade = {attempted, deleted, failures: [{type, id, name, error}], parent_deleted: False, model_uuid}` so the user can re-run after fixing the underlying error. Happy-path envelope adds `cascade.deleted` per-type counts. Legacy `orphaned_children` top-level key kept for back-compat (same shape, meaning flipped from "leaked" to "cascaded") but **deprecated -- removal scheduled for a future minor release**; new callers should read `cascade.deleted` instead. See [gotchas.md](gotchas.md) for the meaning-flip + deprecation note. - `semantic-layer show --project P [--model M] [--type T]` -- show a model's entities. `--type` filters to `dataset | metric | relationship | constraint | glossary`. Without `--type` prints a per-type count summary. `--model` is optional when the project has exactly one model. - `semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N]` (since 0.47.0) -- project-wide glob search across semantic-layer entity names. Mirrors the upstream `keboola-mcp-server search_semantic_context` MCP tool so a downstream caller can drop the MCP dependency for the pre-flight "is the model populated?" check. Patterns are case-sensitive `fnmatch`, repeatable (union); default `*`. Default `--type all` searches every CHILD type (`model` searches semantic models). `--limit N` short-circuits both per-type and outer loops. Envelope: `{project, contexts: [{id, type, name, description, attributes}], total_count}`; the `type` field is the CLI-friendly singular (no `semantic-` prefix). -- `semantic-layer schema --project P (--type model|dataset|metric|relationship|constraint|glossary[,TYPE...] | --all)` (since 0.73.0) -- live JSON Schema per semantic object type, fetched from the deployed metastore (`GET /api/v1/schema/{type}`; never bundled, cannot drift). Exactly one of `--type`/`--all` (usage error otherwise); `--type` takes a comma-separated list, fan-out is parallel. Envelope: `{project, schemas: [{type, schema}]}`. Ports the `get_semantic_schema` MCP tool. +- `semantic-layer schema --project P (--type model|dataset|metric|relationship|constraint|glossary[,TYPE...] | --all)` (since 0.73.0) -- live JSON Schema per semantic object type, fetched from the deployed metastore (never bundled, cannot drift). Exactly one of `--type`/`--all` (usage error otherwise); `--type` takes a comma-separated list, fan-out is parallel. The bare schema endpoint returns only a version LISTING -- the service resolves the `isDefault` version and fetches the real schema (a deliberate improvement over the upstream `get_semantic_schema` MCP tool, which passes the bare listing through). Envelope: `{project, schemas: [{type, schema, schema_version}]}`. - `semantic-layer get-context --project P --context-id ID` (since 0.47.0) -- single-entry fetch by id, irrespective of type. Probes `semantic-model` first then every CHILD type (dataset / metric / relationship / constraint / glossary) until a 200 lands. 404 on any one type is non-terminal; only a full miss raises `NOT_FOUND` (exit 1). Non-404 errors (500, etc.) propagate immediately rather than being swallowed by the next probe. - `semantic-layer validate --project P [--model M] [--deep]` -- structural validation. Basic mode runs local checks: duplicate names, dangling rel/metric refs, SUM-on-PCT (warning), constraint orphans (metrics in `metrics[]` that no longer exist), severity-suffix mismatches between API `severity` and the 4-band name suffix. `--deep` adds parallel Snowflake column-existence probes via the in-process StorageService: phantom dataset fields, phantom column refs in metric SQL, AGG-on-STRING errors. Response: `{valid: bool, deep: bool, errors: [{type, item, detail}], warnings: [...]}`. - `semantic-layer export --project P [--model M] [--output PATH]` -- snapshot the model to a self-describing JSON file (default `./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json`). Schema-versioned for round-trip via `import` / `diff`. diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index a394d486..275af335 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -108,18 +108,20 @@ def list_items( return items return [i for i in items if (i.get("attributes") or {}).get("modelUUID") == model_uuid] - def get_schema(self, item_type: SemanticType) -> dict[str, Any]: - """Fetch the JSON Schema for a semantic object type. - - ``GET /api/v1/schema/{item_type}``. The schema is **server-emitted** - so it always matches the deployed metastore version — never a - hand-rolled static copy (it would drift the moment the metastore - evolves). Unlike the repository verbs, this endpoint returns the - schema document directly with no ``{"data": ...}`` envelope, so the - body is passed through verbatim (mirrors keboola-mcp-server - ``MetastoreClient.get_schema``). + def get_schema(self, item_type: SemanticType, version: str | None = None) -> dict[str, Any]: + """Fetch the JSON Schema (or version listing) for a semantic object type. + + ``GET /api/v1/schema/{item_type}[/{version}]``. The schema is + **server-emitted** so it always matches the deployed metastore + version — never a hand-rolled static copy (it would drift the moment + the metastore evolves). Live behavior (verified 2026-07): the bare + endpoint returns a ``{"versions": [...]}`` listing with NO schema + body; the actual JSON Schema lives at ``/{version}``. The service + layer resolves the default version. No ``{"data": ...}`` envelope on + either form, so the body is passed through verbatim. """ - response = self._do_request("GET", f"/api/v1/schema/{item_type}") + path = f"/api/v1/schema/{item_type}/{version}" if version else f"/api/v1/schema/{item_type}" + response = self._do_request("GET", path) body = response.json() if not isinstance(body, dict): raise KeboolaApiError( diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index b863e82f..f2585581 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -357,11 +357,14 @@ def get_schema(self, alias: str, types: list[str]) -> dict[str, Any]: results: dict[str, dict[str, Any]] = {} with self._new_metastore_client(project) as client: if len(requested) == 1: - results[requested[0]] = client.get_schema(SCHEMA_TYPE_ALIAS[requested[0]]) + results[requested[0]] = self._fetch_resolved_schema( + client, SCHEMA_TYPE_ALIAS[requested[0]] + ) else: with ThreadPoolExecutor(max_workers=len(requested)) as pool: future_to_type = { - pool.submit(client.get_schema, SCHEMA_TYPE_ALIAS[t]): t for t in requested + pool.submit(self._fetch_resolved_schema, client, SCHEMA_TYPE_ALIAS[t]): t + for t in requested } errors: list[Exception] = [] for future in future_to_type: @@ -376,9 +379,41 @@ def get_schema(self, alias: str, types: list[str]) -> dict[str, Any]: raise errors[0] return { "project": alias, - "schemas": [{"type": t, "schema": results[t]} for t in requested], + "schemas": [ + { + "type": t, + "schema": results[t]["schema"], + "schema_version": results[t]["schema_version"], + } + for t in requested + ], } + @staticmethod + def _fetch_resolved_schema(client: MetastoreClient, wire_type: str) -> dict[str, Any]: + """Fetch the actual JSON Schema for a type, resolving the default version. + + Live metastore behavior (2026-07): the bare ``/api/v1/schema/{type}`` + endpoint returns only a ``{"versions": [...]}`` listing (metadata, no + schema body); the real JSON Schema lives at ``/{version}``. The + upstream MCP tool passes the bare listing through -- an upstream gap + we deliberately do NOT mirror: this resolves ``isDefault`` (falling + back to the first entry) and fetches the versioned document. If the + server someday returns the schema directly (no ``versions`` key), + it is passed through unchanged. + """ + body = client.get_schema(wire_type) + versions = body.get("versions") + if not isinstance(versions, list) or not versions: + return {"schema": body, "schema_version": None} + default = next( + (v for v in versions if v.get("isDefault")), + versions[0], + ) + version_id = str(default.get("version", "")) + resolved = client.get_schema(wire_type, version=version_id) if version_id else body + return {"schema": resolved, "schema_version": version_id or None} + # Internal helpers (model-scoped fetches). @staticmethod diff --git a/tests/test_semantic_layer_schema.py b/tests/test_semantic_layer_schema.py index 6d4c9d4c..4f1519dd 100644 --- a/tests/test_semantic_layer_schema.py +++ b/tests/test_semantic_layer_schema.py @@ -194,10 +194,52 @@ def test_single_type(self, tmp_path: Path) -> None: assert result == { "project": "prod", - "schemas": [{"type": "metric", "schema": _schema_for("semantic-metric")}], + "schemas": [ + { + "type": "metric", + "schema": _schema_for("semantic-metric"), + "schema_version": None, + } + ], } mock.get_schema.assert_called_once_with("semantic-metric") + def test_versions_listing_resolves_default_version(self, tmp_path: Path) -> None: + """Live metastore returns a {"versions": [...]} listing from the bare + endpoint; the service must resolve isDefault and fetch the real schema.""" + service, mock = _make_service(_make_store(tmp_path)) + listing = { + "versions": [ + {"version": "0.9.0", "isDefault": False}, + {"version": "1.0.0", "isDefault": True}, + ] + } + real_schema = _schema_for("semantic-metric") + + def _get_schema(wire_type: str, version: str | None = None): + return real_schema if version == "1.0.0" else listing + + mock.get_schema.side_effect = _get_schema + + result = service.get_schema("prod", types=["metric"]) + + assert result["schemas"] == [ + {"type": "metric", "schema": real_schema, "schema_version": "1.0.0"} + ] + + def test_versions_listing_without_default_uses_first(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + listing = {"versions": [{"version": "2.0.0"}, {"version": "1.0.0"}]} + real_schema = {"properties": {"x": {}}} + mock.get_schema.side_effect = lambda wt, version=None: ( + real_schema if version == "2.0.0" else listing + ) + + result = service.get_schema("prod", types=["metric"]) + + assert result["schemas"][0]["schema_version"] == "2.0.0" + assert result["schemas"][0]["schema"] == real_schema + def test_model_type_maps_to_semantic_model(self, tmp_path: Path) -> None: service, mock = _make_service(_make_store(tmp_path)) mock.get_schema.return_value = _schema_for("semantic-model") From 11d02e183a6bff8dd20bffee5f865168250d238c Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:46:10 +0200 Subject: [PATCH 10/17] feat(transformation): create/show/edit command group -- SQL transformation authoring (#396) Ports create_sql_transformation + update_sql_transformation: 9-op block/code edit engine (index walk, no jsonpath; batch-start id semantics; unknown ids list valid ones), synthetic b{i}/b{i}.c{j} ids, dialect from project default_backend (no workspace provisioning), statement split via the existing sync/sql_split.py state machine. Deliberate improvement over MCP: parameters non-blocks keys preserved on edit. Live-verified full lifecycle on project 5946 (create -> show -> str_replace edit -> verify -> sync-push delete). Docs: CLAUDE.md, AGENT_CONTEXT, commands-reference, new transformation-workflow.md, gotchas 0.73.0 section (incl. fail-closed firewall notes). --- CLAUDE.md | 12 + plugins/kbagent/skills/kbagent/SKILL.md | 3 + .../kbagent/references/commands-reference.md | 6 + .../skills/kbagent/references/gotchas.md | 32 + .../references/transformation-workflow.md | 70 ++ src/keboola_agent_cli/cli.py | 2 + src/keboola_agent_cli/commands/context.py | 25 + .../commands/transformation.py | 440 +++++++++++ .../services/_transformation_ops.py | 527 +++++++++++++ .../services/transformation_service.py | 450 +++++++++++ tests/test_transformation_cli.py | 745 ++++++++++++++++++ tests/test_transformation_ops.py | 504 ++++++++++++ 12 files changed, 2816 insertions(+) create mode 100644 plugins/kbagent/skills/kbagent/references/transformation-workflow.md create mode 100644 src/keboola_agent_cli/commands/transformation.py create mode 100644 src/keboola_agent_cli/services/_transformation_ops.py create mode 100644 src/keboola_agent_cli/services/transformation_service.py create mode 100644 tests/test_transformation_cli.py create mode 100644 tests/test_transformation_ops.py diff --git a/CLAUDE.md b/CLAUDE.md index e5daf5b0..27b44924 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -587,6 +587,18 @@ kbagent kai chat --message "msg" [--chat-id ID] [--project NAME] kbagent kai chat-detail --chat-id ID [--project NAME] kbagent kai history [--project NAME] [--limit N] +kbagent transformation create --project NAME --name NAME (--sql 'SELECT ...' | --sql-file PATH) [--created-table NAME ...] [--component-id ID] [--description D] [--branch ID] [--dry-run] +kbagent transformation show --project NAME --config-id ID [--component-id ID] [--branch ID] +kbagent transformation edit --project NAME --config-id ID --change-description TEXT (--op JSON ... | --op-file ops.json) [--storage JSON|@file|-] [--component-id ID] [--branch ID] [--dry-run] +# transformation (0.73.0+): native SQL-transformation editing (port of MCP create/update_sql_transformation, #396). +# create: component derived from the project default_backend (snowflake|bigquery; other backends need +# --component-id); SQL split one-statement-per-script[] element; single block "Blocks"/code "Code"; +# each --created-table T maps to out.c-.. show: synthetic positional ids b{i}/b{i}.c{j}; +# when --component-id omitted, all known SQL transformation components are tried. edit: 9 ops +# (add/remove/rename block+code, set_code, add_script, str_replace) applied sequentially against +# batch-start ids -- ALWAYS `transformation show` first, ids renumber after structural ops; +# --storage REPLACES configuration.storage wholesale; --dry-run previews without PUT. + kbagent docs query "QUESTION" [--project NAME] # (0.73.0+) Documentation Q&A via the AI Service (server-side RAG). Unlike kai ask it does NOT # see project data; works with any token. --json emits {query, text, source_urls}. diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 3c9fcbdf..1d73b1dc 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -189,6 +189,9 @@ When working inside a git repository or project directory, run `kbagent init` (o | Fetch the full message history of a single Kai chat | `kbagent kai chat-detail --chat-id CHAT-ID` | | List recent Kai chat sessions | `kbagent kai history` | | Ask the Keboola documentation a natural language question | `kbagent docs query ` | +| Create a SQL transformation from a SQL script | `kbagent transformation create --name NAME` | +| Show a SQL transformation's block/code tree with positional IDs | `kbagent transformation show --config-id CONFIG-ID` | +| Edit a SQL transformation's blocks/codes with positional operations | `kbagent transformation edit --config-id CONFIG-ID --change-description CHANGE-DESCRIPTION` | | List conditional flows (keboola.flow) across projects | `kbagent flow list` | | Show detailed conditional-flow information including phases and tasks | `kbagent flow detail --project PROJECT --flow-id FLOW-ID` | | Print the conditional-flow YAML template, or --full for the JSON Schema | `kbagent flow schema` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 6b302076..c362e7fb 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -207,6 +207,12 @@ Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, - `tool list [--project NAME] [--branch ID]` -- list available MCP tools (multi_project annotation) - `tool call TOOL_NAME [--project NAME] [--input JSON|@file|-] [--branch ID]` -- call MCP tool (read = all projects, write = single). `--input` accepts inline JSON, `@file.json`, or `-` (stdin) +## SQL Transformations (since v0.73.0) +Ports the MCP `create_sql_transformation` / `update_sql_transformation` tools (#396). See [transformation-workflow.md](transformation-workflow.md) for the show-before-edit recipe. +- `transformation create --project NAME --name NAME (--sql 'SELECT ...' | --sql-file PATH) [--created-table NAME ...] [--component-id ID] [--description D] [--branch ID] [--dry-run]` -- component id from project `default_backend` (snowflake/bigquery; else pass `--component-id`); SQL split one statement per `script[]` element into a single block `Blocks`/code `Code`; each `--created-table T` maps to `out.c-.` (bucket derived from the transformation NAME -- renaming later breaks the match). +- `transformation show --project NAME --config-id ID [--component-id ID] [--branch ID]` -- block/code tree with synthetic positional ids `b{i}`/`b{i}.c{j}` + storage. Probes all SQL transformation components when `--component-id` omitted. **Always show before edit** -- ids renumber after structural ops. +- `transformation edit --project NAME --config-id ID --change-description TEXT (--op JSON ... | --op-file ops.json) [--storage JSON|@file|-] [--component-id ID] [--branch ID] [--dry-run]` -- 9 ops (`add_block`, `remove_block`, `rename_block`, `add_code`, `remove_code`, `rename_code`, `set_code`, `add_script`, `str_replace`) applied sequentially against batch-start ids. `--storage` REPLACES `configuration.storage` wholesale (include ALL mappings you want to keep). Unknown ids error with the list of valid ids. + ## Documentation Q&A (since v0.73.0) - `docs query "QUESTION" [--project NAME]` -- natural-language answer from the Keboola documentation via the AI Service (server-side RAG, no local corpus). Returns answer text + source URLs; `--json` emits `{query, text, source_urls}`. Unlike `kai ask` it does NOT see project data, works with any token (no master-token / feature-flag requirement), and is the right tool for "how do I ..." questions. Ports the `docs_query` MCP tool. diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index e41aba95..6aa5c9b5 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -11,6 +11,38 @@ Versioning convention: behavior; the inline `(updated vX.Y.Z)` records when the refinement landed. --> +## MCP tool classification is FAIL-CLOSED; parity commands replace `tool call` (since v0.73.0) + +- **Unknown MCP tool names classify as `destructive`** -- blocked by BOTH + `--deny-writes` and `--deny-destructive`, and never fanned out multi-project. + Before 0.73.0 anything not matching a write prefix classified as `read`: + `run_job`, `run_sync_action`, `deploy_data_app`, `modify_*` passed + `--deny-writes` AND ran on every configured project in parallel. They are + writes now (single-project dispatch). +- **`tool call` enforces the session firewall per tool name** -- a session-only + `--deny-destructive` now blocks `tool call delete_bucket` (previously only a + PERSISTED `permissions set` policy was checked at tool granularity; session + flags stopped at the coarse `tool.call` operation). +- **Prefer the native parity commands over `tool call`** -- the MCP passthrough + is on a deprecation track (epic #390 / issue #478): `docs query`, + `config examples`, `semantic-layer schema`, `component sync-action`, + `transformation create|show|edit`, `flow examples`. `query_data`'s CLI + answer is `workspace query`. +- **`component sync-action --row-id` merge is SHALLOW** (MCP parity): row + `parameters`/`storage` top-level keys replace the root's wholesale -- a row + that sets `parameters.db` replaces the ENTIRE root `db` object, it does not + deep-merge into it. +- **`transformation edit` ids are positional and renumber** after every + structural op -- always `transformation show` immediately before `edit` + (fresh-fetch rule). `--storage` REPLACES `configuration.storage` wholesale. +- **`semantic-layer schema` resolves the default schema VERSION** -- the bare + metastore endpoint returns only a `{versions: [...]}` listing (the upstream + MCP tool ships that listing as-is; the CLI fetches the real document and + reports `schema_version`). +- **`docs query` vs `kai ask`**: `docs query` is documentation-only RAG (any + token, no feature flag, no project data); `kai ask` sees project data but + needs the master token + `agent-chat` feature. + ## `token` group mints/rotates/revokes SCOPED Storage tokens; secret shown ONCE (since v0.66.0) - **`kbagent token create --project P --description D [--bucket-write B ...] diff --git a/plugins/kbagent/skills/kbagent/references/transformation-workflow.md b/plugins/kbagent/skills/kbagent/references/transformation-workflow.md new file mode 100644 index 00000000..e9beb667 --- /dev/null +++ b/plugins/kbagent/skills/kbagent/references/transformation-workflow.md @@ -0,0 +1,70 @@ +# SQL Transformation Workflow (since v0.73.0) + +Native authoring/editing of SQL transformations -- the CLI port of the MCP +`create_sql_transformation` / `update_sql_transformation` tools (#396). + +## Create + +```bash +kbagent transformation create --project prod \ + --name "Orders Daily Rollup" \ + --sql-file rollup.sql \ + --created-table orders_daily +``` + +- Component id is derived from the project `default_backend` + (snowflake -> `keboola.snowflake-transformation`, bigquery -> + `keboola.google-bigquery-transformation`). Any other backend fails fast -- + pass `--component-id` explicitly. +- The SQL is split one statement per `script[]` element (same splitter the + sync engine uses); everything lands in a single block `Blocks` with one + code `Code` -- identical to what the UI and the MCP tool produce. +- Each `--created-table T` adds an output mapping + `T -> out.c-.`. The bucket name is derived + from the transformation NAME at create time -- renaming the transformation + later does NOT move the bucket. `--created-table` is a declarative mapping + hint, not a guarantee the SQL actually creates that table. +- `--dry-run` prints the would-be payload without POSTing. + +## Inspect (ALWAYS before edit) + +```bash +kbagent --json transformation show --project prod --config-id 123456 +``` + +- Prints the block/code tree with **synthetic positional ids** `b{i}` / + `b{i}.c{j}` (block i, code j) plus storage mappings. +- Ids are NOT persisted anywhere -- they are re-derived from array positions + on every call. **Any structural edit renumbers them.** Fetch fresh ids via + `show` immediately before every `edit` (fresh-fetch rule). +- `--component-id` optional: all known SQL transformation components are + probed until one returns the config. + +## Edit + +```bash +kbagent transformation edit --project prod --config-id 123456 \ + --change-description "split rollup into two blocks" \ + --op '{"op": "add_block", "name": "Cleanup", "position": 1}' \ + --op '{"op": "set_code", "block_id": "b0", "code_id": "b0.c0", "script": "SELECT 1;"}' +``` + +- 9 ops: `add_block`, `remove_block`, `rename_block`, `add_code`, + `remove_code`, `rename_code`, `set_code`, `add_script`, `str_replace`. +- Ops in one invocation apply **sequentially against BATCH-START ids**: an + element added mid-batch has no id until the next `show`; removed elements + invalidate later positional references only on the NEXT invocation. +- Unknown block/code ids fail with the list of currently valid ids. +- `--storage @storage.json` REPLACES `configuration.storage` wholesale -- + include EVERY input/output mapping you want to keep, not just the new ones. +- `--dry-run` previews the resulting tree + op summary without any PUT. + +## Gotchas + +- Show-before-edit is not optional: positional ids drift after every + structural change (same sharp edge as the MCP tool). +- The output bucket is coupled to the create-time name; treat renames as a + new-bucket event and update downstream input mappings accordingly. +- `transformation edit` preserves non-`blocks` keys inside `parameters` + (variables links etc.) -- a deliberate improvement over the MCP tool, which + replaces `parameters` wholesale. diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index 71ac7ee8..7cd7120f 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -38,6 +38,7 @@ from .commands.sync import sync_app from .commands.token import token_app from .commands.tool import tool_app +from .commands.transformation import transformation_app from .commands.version import update_command, version_command from .commands.workspace import workspace_app from .config_store import ConfigStore, resolve_config_dir @@ -122,6 +123,7 @@ app.add_typer(lineage_app, name="lineage", rich_help_panel=_BROWSE) app.add_typer(kai_app, name="kai", rich_help_panel=_BROWSE) app.add_typer(docs_app, name="docs", rich_help_panel=_BROWSE) +app.add_typer(transformation_app, name="transformation", rich_help_panel=_BROWSE) # -- Flows -- _FLOWS = "Flows" diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index cc5d5aa7..1d091381 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -1288,6 +1288,31 @@ kbagent kai history [--project NAME] [--limit N] List recent Kai chat sessions. Default limit: 10. +### SQL Transformations (since v0.73.0) + + kbagent transformation create --project NAME --name NAME (--sql 'SELECT ...' | --sql-file PATH) [--created-table NAME ...] [--component-id ID] [--description D] [--branch ID] [--dry-run] + Create a SQL transformation. Component id derived from the project + default_backend (snowflake -> keboola.snowflake-transformation, + bigquery -> keboola.google-bigquery-transformation; other backends + require --component-id). SQL is split one statement per script element; + a single block "Blocks" with one code "Code" is created (UI/MCP parity). + Each --created-table T adds output mapping T -> out.c-.. + + kbagent transformation show --project NAME --config-id ID [--component-id ID] [--branch ID] + Print the block/code tree with synthetic positional ids b{{i}} / b{{i}}.c{{j}} + plus storage mappings. Without --component-id every known SQL + transformation component is probed (404s skipped). ALWAYS run show + before edit -- ids renumber after every structural change. + + kbagent transformation edit --project NAME --config-id ID --change-description TEXT (--op JSON ... | --op-file ops.json) [--storage JSON|@file|-] [--component-id ID] [--branch ID] [--dry-run] + Apply structured ops to blocks/codes: add_block, remove_block, + rename_block, add_code, remove_code, rename_code, set_code, add_script, + str_replace. Ops in one batch apply sequentially against BATCH-START ids + (mid-batch structural changes do not renumber within the batch). + --storage REPLACES configuration.storage wholesale -- include every + mapping you want to keep. --dry-run previews the resulting tree + op + summary without writing. + ### Documentation Q&A (since v0.73.0) kbagent docs query "QUESTION" [--project NAME] diff --git a/src/keboola_agent_cli/commands/transformation.py b/src/keboola_agent_cli/commands/transformation.py new file mode 100644 index 00000000..1ebbc408 --- /dev/null +++ b/src/keboola_agent_cli/commands/transformation.py @@ -0,0 +1,440 @@ +"""Transformation commands -- create / show / edit SQL transformations. + +Thin CLI layer over +:class:`keboola_agent_cli.services.transformation_service.TransformationService` +(issue #396: native port of the MCP server's create_sql_transformation / +update_sql_transformation tools). No business logic belongs here. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +import typer +from rich.console import Console +from rich.markup import escape + +from ..config_store import ConfigStore +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..services.transformation_service import TransformationService +from ._helpers import ( + check_cli_permission, + get_formatter, + map_error_to_exit_code, + resolve_branch, + resolve_project_alias, +) + +# Max characters of SQL shown per code in human mode (MCP structure_summary parity). +SQL_SNIPPET_MAX_CHARS = 150 + +transformation_app = typer.Typer( + help="SQL transformations - create, inspect, and edit blocks/codes" +) + + +@transformation_app.callback(invoke_without_command=True) +def _transformation_permission_check(ctx: typer.Context) -> None: + check_cli_permission(ctx, "transformation") + + +def _get_transformation_service(ctx: typer.Context) -> TransformationService: + """Fetch the TransformationService from ctx.obj, constructing lazily. + + Falls back to building the service from the shared ConfigStore so the + command group works even before cli.py registers a dedicated + ``transformation_service`` entry. + """ + service = ctx.obj.get("transformation_service") + if service is None: + service = TransformationService(config_store=ctx.obj["config_store"]) + ctx.obj["transformation_service"] = service + return service + + +def _parse_json_arg(raw: str, *, label: str) -> Any: + """Parse a JSON argument: inline JSON, @file, or - for stdin. + + Raises: + ValueError: On missing file or malformed JSON (message names the flag). + """ + try: + if raw == "-": + return json.loads(sys.stdin.read()) + if raw.startswith("@"): + file_path = Path(raw[1:]) + if not file_path.is_file(): + raise ValueError(f"{label}: file not found: {file_path}") + return json.loads(file_path.read_text(encoding="utf-8")) + return json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"{label}: invalid JSON: {exc}") from exc + + +def _render_blocks_human(console: Console, data: dict[str, Any]) -> None: + """Render the block/code tree with synthetic IDs and SQL snippets.""" + name = data.get("name") or "" + header = f"[bold]{escape(name)}[/bold]" if name else "[bold](unnamed)[/bold]" + console.print(f"{header} {data.get('component_id', '')} / {data.get('config_id', '')}") + + blocks = data.get("blocks") or [] + if not blocks: + console.print("[dim]No blocks in this transformation.[/dim]") + for block in blocks: + console.print(f"[cyan]{block['id']}[/cyan] block: {escape(block.get('name', ''))}") + codes = block.get("codes") or [] + if not codes: + console.print(" [dim]no codes[/dim]") + for code in codes: + statement_count = len(code.get("script") or []) + plural = "s" if statement_count != 1 else "" + console.print( + f" [green]{code['id']}[/green] code: {escape(code.get('name', ''))} " + f"({statement_count} statement{plural})" + ) + snippet = (code.get("script_text") or "").strip() + if snippet: + if len(snippet) > SQL_SNIPPET_MAX_CHARS: + truncated = len(snippet) - SQL_SNIPPET_MAX_CHARS + snippet = snippet[:SQL_SNIPPET_MAX_CHARS] + f"... ({truncated} chars truncated)" + console.print(f" [dim]{escape(snippet)}[/dim]") + + storage = data.get("storage") or {} + input_tables = (storage.get("input") or {}).get("tables") or [] + output_tables = (storage.get("output") or {}).get("tables") or [] + if input_tables or output_tables: + console.print( + f"[dim]storage: {len(input_tables)} input table(s), " + f"{len(output_tables)} output table(s)[/dim]" + ) + + +@transformation_app.command("create") +def transformation_create( + ctx: typer.Context, + project: str | None = typer.Option(None, "--project", help="Project alias"), + name: str = typer.Option(..., "--name", help="Transformation name"), + sql: str | None = typer.Option( + None, + "--sql", + help="SQL text (semicolon-separated statements). Mutually exclusive with --sql-file.", + ), + sql_file: Path | None = typer.Option( + None, + "--sql-file", + help="Read SQL from a file. Mutually exclusive with --sql.", + ), + created_table: list[str] | None = typer.Option( + None, + "--created-table", + help=( + "Table name created by the SQL (repeatable). Each is mapped to " + "out.c-. in the output mapping." + ), + ), + component_id: str | None = typer.Option( + None, + "--component-id", + help=( + "SQL transformation component ID (keboola.snowflake-transformation or " + "keboola.google-bigquery-transformation). Default: derived from the " + "project's default backend." + ), + ), + description: str = typer.Option("", "--description", help="Configuration description"), + branch: int | None = typer.Option( + None, "--branch", help="Create in a specific dev branch ID (defaults to active branch)" + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Print the would-be configuration payload without creating" + ), +) -> None: + """Create a SQL transformation from a SQL script. + + The SQL is split into one statement per script element (Keboola runtime + requirement) and stored as a single block "Blocks" with one code "Code". + Each --created-table T is mapped to out.c-., where + is derived from the transformation name (diacritics stripped, spaces + to dashes -- same rule as the Keboola UI and MCP server). + + \b + Examples: + kbagent transformation create --project prod --name "Orders Report" \\ + --sql 'CREATE TABLE "report" AS SELECT * FROM "orders";' --created-table report + kbagent transformation create --project prod --name Cleanup --sql-file ./cleanup.sql --dry-run + """ + formatter = get_formatter(ctx) + service = _get_transformation_service(ctx) + config_store: ConfigStore = ctx.obj["config_store"] + + if (sql is None) == (sql_file is None): + formatter.error( + message="Provide exactly one of --sql or --sql-file.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + + if sql_file is not None: + if not sql_file.is_file(): + formatter.error( + message=f"SQL file not found: {sql_file}", + error_code=ErrorCode.FILE_NOT_FOUND, + ) + raise typer.Exit(code=2) from None + sql = sql_file.read_text(encoding="utf-8") + + alias = resolve_project_alias(ctx, formatter, project) + _, branch_id = resolve_branch(config_store, formatter, alias, branch) + + try: + result = service.create( + alias, + name=name, + sql=sql or "", + created_tables=created_table, + component_id=component_id, + description=description, + branch_id=branch_id, + dry_run=dry_run, + ) + except ValueError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.VALIDATION_ERROR) + raise typer.Exit(code=1) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + def _human(console: Console, data: dict[str, Any]) -> None: + if data.get("dry_run"): + console.print( + "[bold yellow]Dry run[/bold yellow] - configuration that would be created " + f"under [cyan]{data['component_id']}[/cyan]:" + ) + console.print_json(json.dumps(data["configuration"])) + return + console.print( + f"Created transformation [bold]{escape(data['name'])}[/bold] " + f"(config id [cyan]{data['config_id']}[/cyan], " + f"component {data['component_id']}, version {data.get('version')})" + ) + output_tables = data["configuration"]["storage"]["output"]["tables"] + for table in output_tables: + console.print(f" output: {table['source']} -> {table['destination']}") + + formatter.output(result, _human) + + +@transformation_app.command("show") +def transformation_show( + ctx: typer.Context, + project: str | None = typer.Option(None, "--project", help="Project alias"), + config_id: str = typer.Option(..., "--config-id", help="Configuration ID"), + component_id: str | None = typer.Option( + None, + "--component-id", + help=( + "Component ID. When omitted, the known SQL transformation components " + "are tried until the configuration is found." + ), + ), + branch: int | None = typer.Option( + None, "--branch", help="Read from a specific dev branch ID (defaults to active branch)" + ), +) -> None: + """Show a SQL transformation's block/code tree with positional IDs. + + Blocks get synthetic IDs b0, b1, ...; codes get b0.c0, b0.c1, ... + (derived from position, matching the MCP server). Use these IDs with + 'kbagent transformation edit --op'. + """ + formatter = get_formatter(ctx) + service = _get_transformation_service(ctx) + config_store: ConfigStore = ctx.obj["config_store"] + + alias = resolve_project_alias(ctx, formatter, project) + _, branch_id = resolve_branch(config_store, formatter, alias, branch) + + try: + result = service.show( + alias, + config_id=config_id, + component_id=component_id, + branch_id=branch_id, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + formatter.output(result, _render_blocks_human) + + +@transformation_app.command("edit") +def transformation_edit( + ctx: typer.Context, + project: str | None = typer.Option(None, "--project", help="Project alias"), + config_id: str = typer.Option(..., "--config-id", help="Configuration ID"), + component_id: str | None = typer.Option( + None, + "--component-id", + help=( + "Component ID. When omitted, the known SQL transformation components " + "are tried until the configuration is found." + ), + ), + branch: int | None = typer.Option( + None, "--branch", help="Edit in a specific dev branch ID (defaults to active branch)" + ), + change_description: str = typer.Option( + ..., + "--change-description", + help="Human-readable summary of this change (stored in config version history)", + ), + op: list[str] | None = typer.Option( + None, + "--op", + help=( + "Operation as inline JSON (repeatable, applied in order). Ops: " + "add_block, remove_block, rename_block, add_code, remove_code, " + "rename_code, set_code, add_script, str_replace. Example: " + '\'{"op": "set_code", "block_id": "b0", "code_id": "b0.c0", ' + '"script": "SELECT 1;"}\'. IDs come from `transformation show`. ' + "Mutually exclusive with --op-file." + ), + ), + op_file: Path | None = typer.Option( + None, + "--op-file", + help="Read operations from a JSON file containing an array of op objects.", + ), + storage: str | None = typer.Option( + None, + "--storage", + help=( + "FULL REPLACEMENT of configuration.storage (inline JSON, @file, or - " + "for stdin). Include every input/output mapping you want to keep -- " + "the existing storage block is overwritten wholesale." + ), + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Apply ops locally and print the resulting tree without writing", + ), +) -> None: + """Edit a SQL transformation's blocks/codes with positional operations. + + Operations apply sequentially against the structure as fetched: IDs + (b0, b0.c0, ...) refer to positions at the start of the batch, and + elements added within the batch are not addressable until the next + invocation. Run 'kbagent transformation show' first to get IDs. + + \b + Examples: + kbagent transformation edit --project prod --config-id 123 \\ + --change-description "Filter active" \\ + --op '{"op": "str_replace", "search_for": "orders", "replace_with": "orders_active"}' + kbagent transformation edit --project prod --config-id 123 \\ + --change-description "Restructure" --op-file ops.json --dry-run + """ + formatter = get_formatter(ctx) + service = _get_transformation_service(ctx) + config_store: ConfigStore = ctx.obj["config_store"] + + if op and op_file is not None: + formatter.error( + message="Use either --op or --op-file, not both.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + if not op and op_file is None and storage is None: + formatter.error( + message="Nothing to do: provide --op/--op-file and/or --storage.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + + try: + raw_ops = _collect_ops(op, op_file) + storage_payload = ( + _parse_json_arg(storage, label="--storage") if storage is not None else None + ) + except ValueError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.INPUT_ERROR) + raise typer.Exit(code=2) from None + + if storage_payload is not None and not isinstance(storage_payload, dict): + formatter.error( + message="--storage must be a JSON object.", + error_code=ErrorCode.INPUT_ERROR, + ) + raise typer.Exit(code=2) from None + + alias = resolve_project_alias(ctx, formatter, project) + _, branch_id = resolve_branch(config_store, formatter, alias, branch) + + try: + result = service.edit( + alias, + config_id=config_id, + ops=raw_ops, + change_description=change_description, + component_id=component_id, + storage=storage_payload, + branch_id=branch_id, + dry_run=dry_run, + ) + except ValueError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.VALIDATION_ERROR) + raise typer.Exit(code=1) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + def _human(console: Console, data: dict[str, Any]) -> None: + if data.get("dry_run"): + console.print("[bold yellow]Dry run[/bold yellow] - no changes written.") + for message in data.get("operations_applied") or []: + console.print(f" - {escape(message)}") + if data.get("storage_replaced"): + console.print(" - Replaced configuration.storage wholesale") + _render_blocks_human(console, data) + if not data.get("dry_run"): + console.print( + f"Updated config [cyan]{data['config_id']}[/cyan] to version {data.get('version')}" + ) + + formatter.output(result, _human) + + +def _collect_ops(op: list[str] | None, op_file: Path | None) -> list[dict[str, Any]]: + """Collect raw op dicts from repeated --op JSON strings or --op-file. + + Raises: + ValueError: On malformed JSON, missing file, or non-object entries. + """ + raw_ops: list[dict[str, Any]] = [] + if op_file is not None: + parsed = _parse_json_arg(f"@{op_file}", label="--op-file") + if not isinstance(parsed, list): + raise ValueError("--op-file must contain a JSON array of operation objects") + entries = parsed + else: + entries = [_parse_json_arg(item, label="--op") for item in op or []] + + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise ValueError(f"Operation #{index} is not a JSON object: {entry!r}") + raw_ops.append(entry) + return raw_ops diff --git a/src/keboola_agent_cli/services/_transformation_ops.py b/src/keboola_agent_cli/services/_transformation_ops.py new file mode 100644 index 00000000..a085a7f7 --- /dev/null +++ b/src/keboola_agent_cli/services/_transformation_ops.py @@ -0,0 +1,527 @@ +"""Positional block/code update engine for SQL transformations. + +Faithful port of keboola-mcp-server ``tools/components/tf_update.py`` + +``tools/components/model.py`` (Tf* operation models) for issue #396, with +two deliberate implementation differences: + +- No ``jsonpath-ng`` dependency: elements are located by a direct index + walk over the parameters dict (IDs are synthetic and positional, so a + linear scan over ``blocks[]`` / ``codes[]`` is exact). +- Multi-value returns use dataclasses (:class:`OpResult`, + :class:`BatchResult`) instead of bare tuples, per CONTRIBUTING.md. + +The engine operates on the *simplified* parameters shape:: + + {"blocks": [{"id": "b0", "name": ..., "codes": [ + {"id": "b0.c0", "name": ..., "script": ""}]}]} + +where ``script`` is a single SQL text string (statements joined). IDs are +assigned by :func:`add_ids` -- blocks numbered ``b{i}`` from 0, codes +``b{i}.c{j}`` within each block -- and re-derived after every batch, so +they always reflect current positions. Within one batch, operations apply +sequentially against the mutating structure and IDs keep referring to the +structure as it was at batch start (elements added mid-batch carry no ID +until the batch finishes -- same semantics as the MCP server). + +Conversion between this simplified shape and the raw Storage API shape +(``script`` as a list of statements) is provided by +:func:`raw_to_simplified` / :func:`simplified_to_raw`, built on the +existing SQL statement splitter in :mod:`keboola_agent_cli.sync.sql_split`. +""" + +from __future__ import annotations + +import copy +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, Field, TypeAdapter, ValidationError, model_validator + +from ..sync.sql_split import join_statements, split_statements + +# Operations that change the structure of the transformation (trigger ID +# re-derivation and a structure summary). Mirrors tf_update.STRUCTURAL_OPS. +STRUCTURAL_OPS: frozenset[str] = frozenset({"add_block", "add_code", "remove_block", "remove_code"}) + +TfPosition = Literal["start", "end"] + + +class TfCode(BaseModel, frozen=True): + """A code entry inside a transformation block (simplified shape).""" + + name: str = Field(description="A descriptive name for the code block") + script: str = Field(description="The SQL script of the code block") + + +class TfBlock(BaseModel, frozen=True): + """A transformation block (simplified shape).""" + + name: str = Field(description="A descriptive name for the block") + codes: list[TfCode] = Field(default_factory=list, description="SQL code sub-blocks") + + +class TfAddBlock(BaseModel, frozen=True): + """Add a new block to the transformation.""" + + op: Literal["add_block"] + block: TfBlock = Field(description="The block to add") + position: TfPosition = Field(default="end", description="Where to insert the block") + + +class TfRemoveBlock(BaseModel, frozen=True): + """Remove an existing block from the transformation.""" + + op: Literal["remove_block"] + block_id: str = Field(description="The ID of the block to remove") + + +class TfRenameBlock(BaseModel, frozen=True): + """Rename an existing block in the transformation.""" + + op: Literal["rename_block"] + block_id: str = Field(description="The ID of the block to rename") + block_name: str = Field(description="The new name of the block") + + +class TfAddCode(BaseModel, frozen=True): + """Add a new code to an existing block in the transformation.""" + + op: Literal["add_code"] + block_id: str = Field(description="The ID of the block to add the code to") + code: TfCode = Field(description="The code to add") + position: TfPosition = Field(default="end", description="Where to insert the code") + + +class TfRemoveCode(BaseModel, frozen=True): + """Remove an existing code from an existing block in the transformation.""" + + op: Literal["remove_code"] + block_id: str = Field(description="The ID of the block to remove the code from") + code_id: str = Field(description="The ID of the code to remove") + + +class TfRenameCode(BaseModel, frozen=True): + """Rename an existing code in an existing block in the transformation.""" + + op: Literal["rename_code"] + block_id: str = Field(description="The ID of the block containing the code") + code_id: str = Field(description="The ID of the code to rename") + code_name: str = Field(description="The new name of the code") + + +class TfSetCode(BaseModel, frozen=True): + """Set the SQL script of an existing code in an existing block.""" + + op: Literal["set_code"] + block_id: str = Field(description="The ID of the block containing the code") + code_id: str = Field(description="The ID of the code to set") + script: str = Field(description="The SQL script of the code to set") + + +class TfAddScript(BaseModel, frozen=True): + """Append or prepend SQL script text to an existing code.""" + + op: Literal["add_script"] + block_id: str = Field(description="The ID of the block containing the code") + code_id: str = Field(description="The ID of the code to add the script to") + script: str = Field(description="The SQL script to add") + position: TfPosition = Field(default="end", description="Where to add the script") + + +class TfStrReplace(BaseModel, frozen=True): + """Replace a substring in SQL scripts in the transformation.""" + + op: Literal["str_replace"] + block_id: str | None = Field( + default=None, + description=( + "The ID of the block to replace substrings in. If not provided, all blocks are updated." + ), + ) + code_id: str | None = Field( + default=None, + description=( + "The ID of the code to replace substrings in. " + "If not provided, all codes in the block are updated." + ), + ) + search_for: str = Field(description="Substring to search for (non-empty)") + replace_with: str = Field(description="Replacement string (can be empty for deletion)") + + @model_validator(mode="after") + def validate_code_id_requires_block_id(self) -> TfStrReplace: + """code_id can only be specified together with block_id.""" + if self.block_id is None and self.code_id is not None: + raise ValueError("code_id must be None if block_id is None") + return self + + +TfOp = Annotated[ + TfAddBlock + | TfRemoveBlock + | TfRenameBlock + | TfAddCode + | TfRemoveCode + | TfRenameCode + | TfSetCode + | TfAddScript + | TfStrReplace, + Field(discriminator="op"), +] + +_OPS_ADAPTER: TypeAdapter[list[TfOp]] = TypeAdapter(list[TfOp]) + + +def parse_ops(raw_ops: Sequence[dict[str, Any]]) -> list[TfOp]: + """Validate raw op dicts into typed operation models. + + Args: + raw_ops: Sequence of dicts, each with an ``op`` discriminator key + (``add_block``, ``remove_block``, ``rename_block``, ``add_code``, + ``remove_code``, ``rename_code``, ``set_code``, ``add_script``, + ``str_replace``). + + Returns: + List of validated operation models, in input order. + + Raises: + ValueError: With a readable summary when any op fails validation. + """ + try: + return _OPS_ADAPTER.validate_python(list(raw_ops)) + except ValidationError as exc: + parts = [] + for err in exc.errors()[:5]: + loc = ".".join(str(item) for item in err["loc"]) + parts.append(f"{loc}: {err['msg']}" if loc else err["msg"]) + raise ValueError("Invalid operation(s): " + "; ".join(parts)) from exc + + +@dataclass +class OpResult: + """Result of applying a single operation (params mutated in place).""" + + params: dict[str, Any] + message: str + + +@dataclass +class BatchResult: + """Result of applying a batch of operations. + + Attributes: + params: The updated simplified parameters, with positional IDs + re-derived so they reflect the final structure. + messages: Human-readable per-op change summaries, in apply order. + structural: True when any op in the batch changed the block/code + structure (add/remove of a block or code). + """ + + params: dict[str, Any] + messages: list[str] + structural: bool + + +def add_ids(parameters: dict[str, Any]) -> dict[str, Any]: + """Assign synthetic positional IDs to blocks and codes (in place). + + Blocks are numbered sequentially from 0 (``b0``, ``b1``, ...); codes are + numbered from 0 within each block and prefixed with the block ID + (``b0.c0``, ``b0.c1``, ...). Mirrors the MCP server's ``add_ids``. + """ + for bidx, block in enumerate(parameters.get("blocks") or []): + if not isinstance(block, dict): + continue + block["id"] = f"b{bidx}" + for cidx, code in enumerate(block.get("codes") or []): + if not isinstance(code, dict): + continue + code["id"] = f"b{bidx}.c{cidx}" + return parameters + + +def _valid_block_ids(params: dict[str, Any]) -> list[str]: + return [ + block["id"] + for block in params.get("blocks") or [] + if isinstance(block, dict) and "id" in block + ] + + +def _valid_code_ids(block: dict[str, Any]) -> list[str]: + return [ + code["id"] for code in block.get("codes") or [] if isinstance(code, dict) and "id" in code + ] + + +def _find_block(params: dict[str, Any], block_id: str) -> dict[str, Any]: + """Locate a block by its synthetic ID or raise with the valid IDs listed.""" + for block in params.get("blocks") or []: + if isinstance(block, dict) and block.get("id") == block_id: + return block + valid = ", ".join(_valid_block_ids(params)) or "(none)" + raise ValueError(f"Block with id '{block_id}' does not exist. Valid block ids: {valid}") + + +def _find_code(block: dict[str, Any], block_id: str, code_id: str) -> dict[str, Any]: + """Locate a code within a block by ID or raise with the valid IDs listed.""" + for code in block.get("codes") or []: + if isinstance(code, dict) and code.get("id") == code_id: + return code + valid = ", ".join(_valid_code_ids(block)) or "(none)" + raise ValueError( + f"Code with id '{code_id}' in block '{block_id}' does not exist. Valid code ids: {valid}" + ) + + +def add_block(params: dict[str, Any], op: TfAddBlock) -> OpResult: + """Add a new block at the start or end of the transformation.""" + if "blocks" not in params: + raise ValueError("Invalid parameters: must contain 'blocks' key") + if not op.block.name.strip(): + raise ValueError("Invalid operation: block name cannot be empty") + + new_block_dict = op.block.model_dump() + if op.position == "start": + params["blocks"].insert(0, new_block_dict) + else: # "end" + params["blocks"].append(new_block_dict) + + return OpResult(params, f'Added block with name "{op.block.name}"') + + +def remove_block(params: dict[str, Any], op: TfRemoveBlock) -> OpResult: + """Remove an existing block from the transformation.""" + block = _find_block(params, op.block_id) + params["blocks"].remove(block) + return OpResult(params, f'Removed block "{op.block_id}"') + + +def rename_block(params: dict[str, Any], op: TfRenameBlock) -> OpResult: + """Rename an existing block in the transformation.""" + if not op.block_name.strip(): + raise ValueError("Invalid operation: block name cannot be empty") + block = _find_block(params, op.block_id) + block["name"] = op.block_name + return OpResult(params, f'Renamed block "{op.block_id}" to "{op.block_name}"') + + +def add_code(params: dict[str, Any], op: TfAddCode) -> OpResult: + """Add a new code to an existing block.""" + if not op.code.name.strip(): + raise ValueError("Invalid operation: code name cannot be empty") + block = _find_block(params, op.block_id) + codes = block.setdefault("codes", []) + + new_code_dict = op.code.model_dump() + if op.position == "start": + codes.insert(0, new_code_dict) + else: # "end" + codes.append(new_code_dict) + + return OpResult(params, f'Added code with name "{op.code.name}"') + + +def remove_code(params: dict[str, Any], op: TfRemoveCode) -> OpResult: + """Remove an existing code from an existing block.""" + block = _find_block(params, op.block_id) + code = _find_code(block, op.block_id, op.code_id) + block["codes"].remove(code) + return OpResult(params, f'Removed code "{op.code_id}" from block "{op.block_id}"') + + +def rename_code(params: dict[str, Any], op: TfRenameCode) -> OpResult: + """Rename an existing code in an existing block.""" + if not op.code_name.strip(): + raise ValueError("Invalid operation: code name cannot be empty") + block = _find_block(params, op.block_id) + code = _find_code(block, op.block_id, op.code_id) + code["name"] = op.code_name + return OpResult(params, f'Renamed code "{op.code_id}" to "{op.code_name}"') + + +def set_code(params: dict[str, Any], op: TfSetCode) -> OpResult: + """Replace the SQL script of an existing code.""" + if not op.script.strip(): + raise ValueError("Invalid operation: script cannot be empty") + block = _find_block(params, op.block_id) + code = _find_code(block, op.block_id, op.code_id) + code["script"] = op.script + return OpResult(params, f"Changed code with id '{op.code_id}' in block '{op.block_id}'") + + +def add_script(params: dict[str, Any], op: TfAddScript) -> OpResult: + """Append or prepend SQL text to an existing code's script. + + Joins with a single space, matching the MCP server's behavior; the + statement splitter re-segments on push so ``...; SELECT 2;`` still + lands as separate statements in the raw shape. + """ + if not op.script.strip(): + raise ValueError("Invalid operation: script cannot be empty") + block = _find_block(params, op.block_id) + code = _find_code(block, op.block_id, op.code_id) + + current_script = code.get("script") or "" + if op.position == "start": + new_script = f"{op.script} {current_script}" if current_script else op.script + else: # "end" + new_script = f"{current_script} {op.script}" if current_script else op.script + code["script"] = new_script + + return OpResult(params, f"Added script to code with id '{op.code_id}' in block '{op.block_id}'") + + +def str_replace(params: dict[str, Any], op: TfStrReplace) -> OpResult: + """Replace a substring in SQL scripts, scoped by optional block/code ID.""" + if not op.search_for: + raise ValueError("Invalid operation: search string is empty") + if op.search_for == op.replace_with: + raise ValueError( + f'Invalid operation: search string and replace string are the same: "{op.search_for}"' + ) + + if op.block_id is None: + codes = [ + code + for block in params.get("blocks") or [] + if isinstance(block, dict) + for code in block.get("codes") or [] + if isinstance(code, dict) + ] + scope = "the transformation" + elif op.code_id is None: + block = _find_block(params, op.block_id) + codes = [code for code in block.get("codes") or [] if isinstance(code, dict)] + scope = f'block "{op.block_id}"' + else: + block = _find_block(params, op.block_id) + codes = [_find_code(block, op.block_id, op.code_id)] + scope = f'code "{op.code_id}", block "{op.block_id}"' + + if not codes: + raise ValueError(f"No scripts found in {scope}") + + replace_cnt = 0 + for code in codes: + script = code.get("script") + if isinstance(script, str) and op.search_for in script: + replace_cnt += script.count(op.search_for) + code["script"] = script.replace(op.search_for, op.replace_with) + + if replace_cnt == 0: + raise ValueError(f'Search string "{op.search_for}" not found in {scope}') + + occurrence_word = "occurrence" if replace_cnt == 1 else "occurrences" + return OpResult( + params, + f'Replaced {replace_cnt} {occurrence_word} of "{op.search_for}" in {scope}', + ) + + +def _apply_op(params: dict[str, Any], op: Any) -> OpResult: + """Dispatch a single validated op to its applier function.""" + appliers = { + "add_block": add_block, + "remove_block": remove_block, + "rename_block": rename_block, + "add_code": add_code, + "remove_code": remove_code, + "rename_code": rename_code, + "set_code": set_code, + "add_script": add_script, + "str_replace": str_replace, + } + return appliers[op.op](params, op) + + +def apply_ops(parameters: dict[str, Any], ops: Sequence[Any]) -> BatchResult: + """Apply a batch of operations to simplified parameters. + + The input dict is not modified (a deep copy is taken). IDs are derived + at batch start; operations apply sequentially against the mutating + structure (so an ID keeps pointing at the element it identified at + batch start, and elements added mid-batch are not addressable until + the next batch -- MCP-server semantics). After the batch, IDs are + re-derived so the returned parameters carry positionally-correct IDs. + + Args: + parameters: Simplified parameters (``script`` as text string). + ops: Validated operation models (from :func:`parse_ops`). + + Returns: + BatchResult with the updated parameters, per-op messages, and a + structural-change flag. + + Raises: + ValueError: When any op is invalid against the current structure + (unknown ID, empty name/script, search string not found, ...). + """ + params = copy.deepcopy(parameters) + params.setdefault("blocks", []) + add_ids(params) + + structural = any(op.op in STRUCTURAL_OPS for op in ops) + messages: list[str] = [] + for op in ops: + result = _apply_op(params, op) + params = result.params + if result.message: + messages.append(result.message) + + # Re-derive IDs so the output reflects final positions. For + # non-structural batches this is a no-op (structure unchanged). + add_ids(params) + return BatchResult(params=params, messages=messages, structural=structural) + + +def raw_to_simplified(parameters: dict[str, Any]) -> dict[str, Any]: + """Convert raw (Storage API) parameters to the simplified shape. + + ``script`` statement arrays are joined into a single SQL text string + (double-newline separator, Keboola convention). A ``script`` that is + already a plain string (legacy configs; the Storage API accepts it) + is kept as-is, so both shapes round-trip. + """ + blocks_out: list[dict[str, Any]] = [] + for block in parameters.get("blocks") or []: + if not isinstance(block, dict): + continue + codes_out: list[dict[str, Any]] = [] + for code in block.get("codes") or []: + if not isinstance(code, dict): + continue + script = code.get("script") + if isinstance(script, list): + text = join_statements([s for s in script if isinstance(s, str)]) + elif isinstance(script, str): + text = script + else: + text = "" + codes_out.append({"name": code.get("name", ""), "script": text}) + blocks_out.append({"name": block.get("name", ""), "codes": codes_out}) + return {"blocks": blocks_out} + + +def simplified_to_raw(parameters: dict[str, Any]) -> dict[str, Any]: + """Convert simplified parameters back to the raw (Storage API) shape. + + Each SQL text string is split into individual statements via the + state-machine splitter (one statement per ``script[]`` element -- + required by the Keboola runtime). Synthetic ``id`` keys are stripped: + they are positional view-model artifacts and must never be persisted. + """ + blocks_out: list[dict[str, Any]] = [] + for block in parameters.get("blocks") or []: + if not isinstance(block, dict): + continue + codes_out: list[dict[str, Any]] = [] + for code in block.get("codes") or []: + if not isinstance(code, dict): + continue + script = code.get("script") + statements = split_statements(script) if isinstance(script, str) else [] + codes_out.append({"name": code.get("name", ""), "script": statements}) + blocks_out.append({"name": block.get("name", ""), "codes": codes_out}) + return {"blocks": blocks_out} diff --git a/src/keboola_agent_cli/services/transformation_service.py b/src/keboola_agent_cli/services/transformation_service.py new file mode 100644 index 00000000..0dc09687 --- /dev/null +++ b/src/keboola_agent_cli/services/transformation_service.py @@ -0,0 +1,450 @@ +"""SQL transformation service -- create / show / edit block-based SQL configs. + +Native port of keboola-mcp-server's ``create_sql_transformation`` and +``update_sql_transformation`` tools (issue #396). The block/code update +engine lives in :mod:`keboola_agent_cli.services._transformation_ops`; +this module owns: + +- component-ID resolution from the project's default backend + (``verify_token().default_backend``: snowflake / bigquery), +- create-payload shaping (single block "Blocks" with one code "Code", + statements split via the shared SQL splitter, output-table mapping + derived from the transformation name via :func:`clean_bucket_name`), +- config fetch with component-ID fallback across the known SQL + transformation component IDs, +- the edit orchestration (fetch -> normalize -> simplify -> apply ops -> + re-split -> PUT with change description). +""" + +from __future__ import annotations + +import copy +import re +import unicodedata +from dataclasses import dataclass +from typing import Any + +from ..client import KeboolaClient +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..sync.code_extraction import ( + SQL_TRANSFORMATION_COMPONENTS, + normalize_blocks_codes_script, +) +from ..sync.sql_split import join_statements, split_statements +from . import _transformation_ops as tf_ops +from .base import BaseService + +# Backend (verify_token owner.defaultBackend) -> SQL transformation component. +# Mirrors the MCP server's get_sql_transformation_id_from_sql_dialect(). +BACKEND_TO_COMPONENT_ID: dict[str, str] = { + "snowflake": "keboola.snowflake-transformation", + "bigquery": "keboola.google-bigquery-transformation", +} + +# Preference order when --component-id is omitted on show/edit: the two +# backends kbagent can create come first, then the remaining known SQL +# transformation components (sorted for determinism). +COMPONENT_LOOKUP_ORDER: tuple[str, ...] = ( + "keboola.snowflake-transformation", + "keboola.google-bigquery-transformation", + *sorted( + SQL_TRANSFORMATION_COMPONENTS + - {"keboola.snowflake-transformation", "keboola.google-bigquery-transformation"} + ), +) + +# Create-payload shaping: the UI/MCP convention is a single block named +# "Blocks" holding one code named "Code" with the split statements. +DEFAULT_BLOCK_NAME = "Blocks" +DEFAULT_CODE_NAME = "Code" + +# Maximum bucket-name length accepted by Keboola Storage (MCP parity). +MAX_BUCKET_NAME_LENGTH = 96 + +# Output bucket stage prefix for created tables (UI convention). +OUTPUT_BUCKET_PREFIX = "out.c-" + + +def clean_bucket_name(bucket_name: str) -> str: + """Sanitize a transformation name into a Storage bucket name. + + Exact port of the MCP server's ``clean_bucket_name``: + + - Converts to ASCII (diacritics stripped: ``cesky`` from ``český``). + - Replaces all whitespace runs with dashes. + - Removes any character that is not alphanumeric, dash, or underscore. + - Removes leading underscores. + - Caps at :data:`MAX_BUCKET_NAME_LENGTH` characters. + """ + bucket_name = bucket_name.strip() + bucket_name = unicodedata.normalize("NFKD", bucket_name) + bucket_name = bucket_name.encode("ascii", "ignore").decode("ascii") + bucket_name = re.sub(r"\s+", "-", bucket_name) + bucket_name = re.sub(r"[^a-zA-Z0-9_-]", "", bucket_name) + bucket_name = re.sub(r"^_+", "", bucket_name) + return bucket_name[:MAX_BUCKET_NAME_LENGTH] + + +@dataclass +class ResolvedConfig: + """A fetched configuration plus the component ID it was found under.""" + + component_id: str + detail: dict[str, Any] + + +class TransformationService(BaseService): + """Business logic for the ``kbagent transformation`` command group. + + Receives ``ConfigStore`` and a ``client_factory`` via dependency + injection (see :class:`keboola_agent_cli.services.base.BaseService`). + """ + + # ---- create ----------------------------------------------------- + + def create( + self, + alias: str, + *, + name: str, + sql: str, + created_tables: list[str] | None = None, + component_id: str | None = None, + description: str = "", + branch_id: int | None = None, + dry_run: bool = False, + ) -> dict[str, Any]: + """Create a new SQL transformation configuration. + + Args: + alias: Project alias. + name: Transformation name (also drives the output bucket name). + sql: SQL text; split into one statement per ``script[]`` element. + created_tables: Table names created by the SQL (``CREATE TABLE``); + each is mapped to ``out.c-.
``. + component_id: Explicit SQL transformation component ID; when + omitted, derived from the project's default backend. + description: Configuration description. + branch_id: Optional dev-branch ID. + dry_run: When True, return the would-be payload without POSTing. + + Returns: + Result dict with the shaped configuration payload and -- unless + ``dry_run`` -- the created ``config_id`` and ``version``. + + Raises: + ValueError: If the SQL contains no statements. + ConfigError: If the project backend has no SQL transformation + component and no explicit ``component_id`` was given. + KeboolaApiError: On API failure. + """ + statements = split_statements(sql) + if not statements: + raise ValueError("SQL contains no statements (empty input)") + + project = self.resolve_projects([alias])[alias] + client = self._client_factory(project.stack_url, project.token) + try: + resolved_component_id = component_id or self._component_id_from_backend(client, alias) + + configuration = _build_create_configuration( + name=name, + statements=statements, + created_tables=created_tables or [], + ) + + result: dict[str, Any] = { + "project_alias": alias, + "component_id": resolved_component_id, + "name": name, + "description": description, + "branch_id": branch_id, + "configuration": configuration, + "dry_run": dry_run, + } + if dry_run: + return result + + created = client.create_config( + component_id=resolved_component_id, + name=name, + configuration=configuration, + description=description, + branch_id=branch_id, + ) + result["config_id"] = str(created.get("id", "")) + result["version"] = created.get("version") + return result + finally: + client.close() + + # ---- show ------------------------------------------------------- + + def show( + self, + alias: str, + *, + config_id: str, + component_id: str | None = None, + branch_id: int | None = None, + ) -> dict[str, Any]: + """Fetch a SQL transformation and render its block/code tree. + + When ``component_id`` is omitted, the known SQL transformation + component IDs are tried in :data:`COMPONENT_LOOKUP_ORDER` until the + configuration is found. + + Returns: + Dict with ``config_id``, ``component_id``, ``name``, ``blocks`` + (each block ``{id, name, codes:[{id, name, script, script_text}]}`` + with synthetic positional IDs ``b{i}`` / ``b{i}.c{j}``) and + ``storage``. + """ + project = self.resolve_projects([alias])[alias] + client = self._client_factory(project.stack_url, project.token) + try: + resolved = self._fetch_config(client, config_id, component_id, branch_id) + finally: + client.close() + + configuration = copy.deepcopy(resolved.detail.get("configuration") or {}) + # Normalize legacy string-shaped scripts so the view (and the JSON + # contract: script is always a statement array) is stable. + configuration, _ = normalize_blocks_codes_script(resolved.component_id, configuration) + + return { + "project_alias": alias, + "config_id": config_id, + "component_id": resolved.component_id, + "name": resolved.detail.get("name", ""), + "description": resolved.detail.get("description", ""), + "version": resolved.detail.get("version"), + "blocks": _blocks_view(configuration.get("parameters") or {}), + "storage": configuration.get("storage") or {}, + } + + # ---- edit ------------------------------------------------------- + + def edit( + self, + alias: str, + *, + config_id: str, + ops: list[dict[str, Any]], + change_description: str, + component_id: str | None = None, + storage: dict[str, Any] | None = None, + branch_id: int | None = None, + dry_run: bool = False, + ) -> dict[str, Any]: + """Apply a batch of block/code operations to a SQL transformation. + + Operations are validated and applied sequentially against the + simplified structure (see :mod:`._transformation_ops`); the result + is re-split into statement arrays and PUT with the given change + description. ``storage``, when provided, replaces + ``configuration.storage`` wholesale. + + Args: + alias: Project alias. + config_id: Configuration ID to edit. + ops: Raw operation dicts (each with an ``op`` key). May be empty + when only ``storage`` is being replaced. + change_description: Human-readable change summary (required by + the Storage API versioning UX). + component_id: Explicit component ID; auto-detected when omitted. + storage: Full replacement for ``configuration.storage``. + branch_id: Optional dev-branch ID. + dry_run: When True, compute and return the result without PUT. + + Returns: + Result dict with ``operations_applied`` messages, the resulting + ``blocks`` view, and -- unless ``dry_run`` -- the new ``version``. + + Raises: + ValueError: On invalid ops (schema or against current structure). + KeboolaApiError: On API failure (including config not found). + """ + parsed_ops = tf_ops.parse_ops(ops) + + project = self.resolve_projects([alias])[alias] + client = self._client_factory(project.stack_url, project.token) + try: + resolved = self._fetch_config(client, config_id, component_id, branch_id) + + configuration = copy.deepcopy(resolved.detail.get("configuration") or {}) + messages: list[str] = [] + structural = False + + if parsed_ops: + # Normalize legacy string scripts to arrays first so the + # raw -> simplified -> raw round trip is lossless. + configuration, _ = normalize_blocks_codes_script( + resolved.component_id, configuration + ) + existing_params = configuration.get("parameters") or {} + simplified = tf_ops.raw_to_simplified(existing_params) + batch = tf_ops.apply_ops(simplified, parsed_ops) + messages = batch.messages + structural = batch.structural + + # Preserve non-blocks parameter keys; replace blocks with the + # re-split raw shape (synthetic IDs stripped). + new_params = dict(existing_params) + new_params["blocks"] = tf_ops.simplified_to_raw(batch.params)["blocks"] + configuration["parameters"] = new_params + + if storage is not None: + configuration["storage"] = storage + + result: dict[str, Any] = { + "project_alias": alias, + "config_id": config_id, + "component_id": resolved.component_id, + "change_description": change_description, + "operations_applied": messages, + "structural_change": structural, + "storage_replaced": storage is not None, + "blocks": _blocks_view(configuration.get("parameters") or {}), + "dry_run": dry_run, + } + if dry_run: + return result + + updated = client.update_config( + component_id=resolved.component_id, + config_id=config_id, + configuration=configuration, + change_description=change_description, + branch_id=branch_id, + ) + result["version"] = updated.get("version") + return result + finally: + client.close() + + # ---- private helpers --------------------------------------------- + + def _component_id_from_backend(self, client: KeboolaClient, alias: str) -> str: + """Derive the SQL transformation component from the project backend.""" + verify = client.verify_token() + backend = (verify.default_backend or "").lower() + resolved = BACKEND_TO_COMPONENT_ID.get(backend) + if resolved is None: + supported = ", ".join(sorted(BACKEND_TO_COMPONENT_ID)) + raise ConfigError( + f"Project '{alias}' has default backend '{backend}', which has no " + f"SQL transformation component mapping (supported: {supported}). " + "Pass --component-id explicitly." + ) + return resolved + + def _fetch_config( + self, + client: KeboolaClient, + config_id: str, + component_id: str | None, + branch_id: int | None, + ) -> ResolvedConfig: + """Fetch config detail, trying known SQL components when ID omitted.""" + if component_id is not None: + detail = client.get_config_detail(component_id, config_id, branch_id=branch_id) + return ResolvedConfig(component_id=component_id, detail=detail) + + for candidate in COMPONENT_LOOKUP_ORDER: + try: + detail = client.get_config_detail(candidate, config_id, branch_id=branch_id) + return ResolvedConfig(component_id=candidate, detail=detail) + except KeboolaApiError as exc: + if exc.status_code == 404 or exc.error_code == ErrorCode.NOT_FOUND: + continue + raise + + tried = ", ".join(COMPONENT_LOOKUP_ORDER) + raise KeboolaApiError( + message=( + f"Configuration '{config_id}' was not found under any SQL " + f"transformation component (tried: {tried}). If it belongs to a " + "different component, pass --component-id explicitly; for " + "Python/R transformations use 'kbagent config update'." + ), + status_code=404, + error_code=ErrorCode.NOT_FOUND, + ) + + +def _build_create_configuration( + *, + name: str, + statements: list[str], + created_tables: list[str], +) -> dict[str, Any]: + """Shape the create payload (MCP create_transformation_configuration port). + + Single block "Blocks" with one code "Code" carrying the split + statements; each created table maps to + ``out.c-.
`` in the output mapping. + """ + output_tables: list[dict[str, Any]] = [] + if created_tables: + destination_bucket = f"{OUTPUT_BUCKET_PREFIX}{clean_bucket_name(name)}" + output_tables = [ + {"source": table, "destination": f"{destination_bucket}.{table}"} + for table in created_tables + ] + + return { + "parameters": { + "blocks": [ + { + "name": DEFAULT_BLOCK_NAME, + "codes": [{"name": DEFAULT_CODE_NAME, "script": statements}], + } + ] + }, + "storage": { + "input": {"tables": []}, + "output": {"tables": output_tables}, + }, + } + + +def _blocks_view(parameters: dict[str, Any]) -> list[dict[str, Any]]: + """Build the block/code tree with synthetic positional IDs. + + IDs are derived by index walk exactly like the MCP server's + ``add_ids``: blocks ``b{i}``, codes ``b{i}.c{j}``. Each code carries + both the raw statement array (``script``) and the joined SQL text + (``script_text``). + """ + blocks_out: list[dict[str, Any]] = [] + for bidx, block in enumerate(parameters.get("blocks") or []): + if not isinstance(block, dict): + continue + codes_out: list[dict[str, Any]] = [] + for cidx, code in enumerate(block.get("codes") or []): + if not isinstance(code, dict): + continue + script = code.get("script") + if isinstance(script, list): + script_list = [s for s in script if isinstance(s, str)] + elif isinstance(script, str): + script_list = [script] + else: + script_list = [] + codes_out.append( + { + "id": f"b{bidx}.c{cidx}", + "name": code.get("name", ""), + "script": script_list, + "script_text": join_statements(script_list), + } + ) + blocks_out.append( + { + "id": f"b{bidx}", + "name": block.get("name", ""), + "codes": codes_out, + } + ) + return blocks_out diff --git a/tests/test_transformation_cli.py b/tests/test_transformation_cli.py new file mode 100644 index 00000000..9ea51430 --- /dev/null +++ b/tests/test_transformation_cli.py @@ -0,0 +1,745 @@ +"""Tests for the `kbagent transformation` command group (issue #396). + +Exercises the Typer commands through CliRunner against a REAL +TransformationService wired to a mocked KeboolaClient, so payload shaping +(create bucket derivation, dialect default via verify_token, edit batch +re-splitting) is covered end-to-end without HTTP. + +The transformation sub-app is mounted on a standalone root app here (the +group is wired into cli.py separately); ctx.obj mirrors what cli.py +provides (formatter, config_store, project_service). +""" + +import copy +import json +from pathlib import Path +from unittest.mock import MagicMock + +import typer +from typer.testing import CliRunner + +from keboola_agent_cli.commands.transformation import transformation_app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig, TokenVerifyResponse +from keboola_agent_cli.output import OutputFormatter +from keboola_agent_cli.services.project_service import ProjectService +from keboola_agent_cli.services.transformation_service import TransformationService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + +runner = CliRunner() + + +def _make_store(tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir(exist_ok=True) + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=1234, + ), + ) + return store + + +def _verify_response(backend: str) -> TokenVerifyResponse: + return TokenVerifyResponse( + token_id="1", + token_description="test token", + project_id=1234, + project_name="prod", + owner_name="prod", + default_backend=backend, + ) + + +def _make_client(backend: str = "snowflake") -> MagicMock: + client = MagicMock() + client.verify_token.return_value = _verify_response(backend) + client.create_config.return_value = {"id": "9001", "version": 1} + client.update_config.return_value = {"id": "123", "version": 5} + return client + + +def _build_app(store: ConfigStore, client: MagicMock) -> typer.Typer: + """Standalone root app mirroring cli.py's ctx.obj wiring.""" + root = typer.Typer() + + @root.callback() + def _root( + ctx: typer.Context, + json_output: bool = typer.Option(False, "--json"), + ) -> None: + ctx.obj = { + "formatter": OutputFormatter(json_mode=json_output, no_color=True), + "config_store": store, + "project_service": ProjectService(config_store=store), + "transformation_service": TransformationService( + config_store=store, + client_factory=lambda _url, _token: client, + ), + } + + root.add_typer(transformation_app, name="transformation") + return root + + +def _config_detail() -> dict: + """Config detail fixture: one statement-array code + one legacy string code.""" + return { + "id": "123", + "name": "My Transform", + "description": "desc", + "version": 3, + "configuration": { + "parameters": { + "blocks": [ + { + "name": "Main", + "codes": [ + {"name": "load", "script": ["SELECT 1;", "SELECT 2;"]}, + {"name": "clean", "script": "DELETE FROM t;"}, + ], + } + ] + }, + "storage": {"input": {"tables": []}, "output": {"tables": []}}, + }, + } + + +class TestTransformationCreate: + def test_create_payload_shape_and_bucket_derivation(self, tmp_path: Path) -> None: + """Blocks/Code shaping, statement split, and diacritics-stripped bucket.""" + client = _make_client("snowflake") + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "create", + "--project", + "prod", + "--name", + "Můj Report", + "--sql", + 'CREATE TABLE "report" AS SELECT * FROM "src"; DELETE FROM "tmp";', + "--created-table", + "report", + "--description", + "test transform", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["status"] == "ok" + assert payload["data"]["component_id"] == "keboola.snowflake-transformation" + assert payload["data"]["config_id"] == "9001" + + client.create_config.assert_called_once() + kwargs = client.create_config.call_args.kwargs + assert kwargs["component_id"] == "keboola.snowflake-transformation" + assert kwargs["name"] == "Můj Report" + assert kwargs["description"] == "test transform" + configuration = kwargs["configuration"] + assert configuration["parameters"]["blocks"] == [ + { + "name": "Blocks", + "codes": [ + { + "name": "Code", + "script": [ + 'CREATE TABLE "report" AS SELECT * FROM "src";', + 'DELETE FROM "tmp";', + ], + } + ], + } + ] + # Bucket derived from the transformation name, diacritics stripped. + assert configuration["storage"]["output"]["tables"] == [ + {"source": "report", "destination": "out.c-Muj-Report.report"} + ] + assert configuration["storage"]["input"]["tables"] == [] + + def test_create_bigquery_backend_default(self, tmp_path: Path) -> None: + client = _make_client("bigquery") + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "create", + "--project", + "prod", + "--name", + "BQ", + "--sql", + "SELECT 1;", + ], + ) + + assert result.exit_code == 0, result.output + client.verify_token.assert_called_once() + kwargs = client.create_config.call_args.kwargs + assert kwargs["component_id"] == "keboola.google-bigquery-transformation" + + def test_create_unsupported_backend_is_config_error(self, tmp_path: Path) -> None: + client = _make_client("exasol") + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "create", + "--project", + "prod", + "--name", + "X", + "--sql", + "SELECT 1;", + ], + ) + + assert result.exit_code == 5, result.output + payload = json.loads(result.output) + assert payload["status"] == "error" + assert payload["error"]["code"] == "CONFIG_ERROR" + assert "exasol" in payload["error"]["message"] + client.create_config.assert_not_called() + + def test_create_explicit_component_id_skips_verify(self, tmp_path: Path) -> None: + client = _make_client("snowflake") + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "create", + "--project", + "prod", + "--name", + "Explicit", + "--sql", + "SELECT 1;", + "--component-id", + "keboola.google-bigquery-transformation", + ], + ) + + assert result.exit_code == 0, result.output + client.verify_token.assert_not_called() + kwargs = client.create_config.call_args.kwargs + assert kwargs["component_id"] == "keboola.google-bigquery-transformation" + + def test_create_dry_run_makes_no_api_write(self, tmp_path: Path) -> None: + client = _make_client("snowflake") + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "create", + "--project", + "prod", + "--name", + "Dry", + "--sql", + "SELECT 1;", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["data"]["dry_run"] is True + assert "config_id" not in payload["data"] + assert payload["data"]["configuration"]["parameters"]["blocks"][0]["name"] == "Blocks" + client.create_config.assert_not_called() + + def test_create_sql_file(self, tmp_path: Path) -> None: + client = _make_client("snowflake") + app = _build_app(_make_store(tmp_path), client) + sql_file = tmp_path / "query.sql" + sql_file.write_text("SELECT 1;\nSELECT 2;", encoding="utf-8") + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "create", + "--project", + "prod", + "--name", + "FromFile", + "--sql-file", + str(sql_file), + ], + ) + + assert result.exit_code == 0, result.output + kwargs = client.create_config.call_args.kwargs + script = kwargs["configuration"]["parameters"]["blocks"][0]["codes"][0]["script"] + assert script == ["SELECT 1;", "SELECT 2;"] + + def test_create_sql_and_sql_file_conflict(self, tmp_path: Path) -> None: + client = _make_client("snowflake") + app = _build_app(_make_store(tmp_path), client) + sql_file = tmp_path / "query.sql" + sql_file.write_text("SELECT 1;", encoding="utf-8") + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "create", + "--project", + "prod", + "--name", + "X", + "--sql", + "SELECT 1;", + "--sql-file", + str(sql_file), + ], + ) + + assert result.exit_code == 2, result.output + assert json.loads(result.output)["error"]["code"] == "INVALID_ARGUMENT" + + def test_create_neither_sql_nor_file(self, tmp_path: Path) -> None: + client = _make_client("snowflake") + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + ["--json", "transformation", "create", "--project", "prod", "--name", "X"], + ) + + assert result.exit_code == 2, result.output + assert json.loads(result.output)["error"]["code"] == "INVALID_ARGUMENT" + + def test_create_empty_sql_is_validation_error(self, tmp_path: Path) -> None: + client = _make_client("snowflake") + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "create", + "--project", + "prod", + "--name", + "X", + "--sql", + " ", + ], + ) + + assert result.exit_code == 1, result.output + assert json.loads(result.output)["error"]["code"] == "VALIDATION_ERROR" + client.create_config.assert_not_called() + + +class TestTransformationShow: + def test_show_tree_with_synthetic_ids(self, tmp_path: Path) -> None: + client = _make_client() + client.get_config_detail.return_value = _config_detail() + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "show", + "--project", + "prod", + "--config-id", + "123", + "--component-id", + "keboola.snowflake-transformation", + ], + ) + + assert result.exit_code == 0, result.output + data = json.loads(result.output)["data"] + assert data["config_id"] == "123" + assert data["component_id"] == "keboola.snowflake-transformation" + assert data["name"] == "My Transform" + assert data["version"] == 3 + blocks = data["blocks"] + assert blocks[0]["id"] == "b0" + assert blocks[0]["name"] == "Main" + assert blocks[0]["codes"][0]["id"] == "b0.c0" + assert blocks[0]["codes"][0]["script"] == ["SELECT 1;", "SELECT 2;"] + assert blocks[0]["codes"][0]["script_text"] == "SELECT 1;\n\nSELECT 2;" + # Legacy string-shaped script is normalized to a statement array. + assert blocks[0]["codes"][1]["id"] == "b0.c1" + assert blocks[0]["codes"][1]["script"] == ["DELETE FROM t;"] + assert data["storage"] == {"input": {"tables": []}, "output": {"tables": []}} + + def test_show_human_mode_renders_ids(self, tmp_path: Path) -> None: + client = _make_client() + client.get_config_detail.return_value = _config_detail() + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "transformation", + "show", + "--project", + "prod", + "--config-id", + "123", + "--component-id", + "keboola.snowflake-transformation", + ], + ) + + assert result.exit_code == 0, result.output + assert "b0" in result.output + assert "b0.c0" in result.output + assert "My Transform" in result.output + + def test_show_component_fallback(self, tmp_path: Path) -> None: + """Snowflake 404s, BigQuery hits -> component resolved to BigQuery.""" + client = _make_client() + + def _detail(component_id: str, config_id: str, branch_id=None) -> dict: + if component_id == "keboola.google-bigquery-transformation": + return _config_detail() + raise KeboolaApiError( + message="not found", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + + client.get_config_detail.side_effect = _detail + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + ["--json", "transformation", "show", "--project", "prod", "--config-id", "123"], + ) + + assert result.exit_code == 0, result.output + data = json.loads(result.output)["data"] + assert data["component_id"] == "keboola.google-bigquery-transformation" + # Snowflake tried first (preference order), then BigQuery. + tried = [call.args[0] for call in client.get_config_detail.call_args_list] + assert tried[:2] == [ + "keboola.snowflake-transformation", + "keboola.google-bigquery-transformation", + ] + + def test_show_not_found_anywhere(self, tmp_path: Path) -> None: + client = _make_client() + client.get_config_detail.side_effect = KeboolaApiError( + message="not found", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + ["--json", "transformation", "show", "--project", "prod", "--config-id", "999"], + ) + + assert result.exit_code == 1, result.output + payload = json.loads(result.output) + assert payload["error"]["code"] == "NOT_FOUND" + assert "999" in payload["error"]["message"] + + +class TestTransformationEdit: + def test_edit_batch_ops_applied_and_resplit(self, tmp_path: Path) -> None: + client = _make_client() + client.get_config_detail.return_value = _config_detail() + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "edit", + "--project", + "prod", + "--config-id", + "123", + "--component-id", + "keboola.snowflake-transformation", + "--change-description", + "rework load", + "--op", + '{"op": "rename_block", "block_id": "b0", "block_name": "Main Renamed"}', + "--op", + '{"op": "set_code", "block_id": "b0", "code_id": "b0.c0",' + ' "script": "SELECT 100; SELECT 200;"}', + "--op", + '{"op": "str_replace", "search_for": "200", "replace_with": "300"}', + ], + ) + + assert result.exit_code == 0, result.output + data = json.loads(result.output)["data"] + assert data["version"] == 5 + assert data["structural_change"] is False + assert len(data["operations_applied"]) == 3 + assert data["blocks"][0]["name"] == "Main Renamed" + assert data["blocks"][0]["codes"][0]["script"] == ["SELECT 100;", "SELECT 300;"] + + client.update_config.assert_called_once() + kwargs = client.update_config.call_args.kwargs + assert kwargs["component_id"] == "keboola.snowflake-transformation" + assert kwargs["config_id"] == "123" + assert kwargs["change_description"] == "rework load" + blocks = kwargs["configuration"]["parameters"]["blocks"] + assert blocks[0]["name"] == "Main Renamed" + # Multi-statement set_code re-split into one statement per element. + assert blocks[0]["codes"][0]["script"] == ["SELECT 100;", "SELECT 300;"] + # Legacy string script normalized to array on the round trip. + assert blocks[0]["codes"][1]["script"] == ["DELETE FROM t;"] + # No synthetic ids persisted. + assert "id" not in blocks[0] + assert "id" not in blocks[0]["codes"][0] + + def test_edit_dry_run_makes_no_api_write(self, tmp_path: Path) -> None: + client = _make_client() + client.get_config_detail.return_value = _config_detail() + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "edit", + "--project", + "prod", + "--config-id", + "123", + "--component-id", + "keboola.snowflake-transformation", + "--change-description", + "preview", + "--op", + '{"op": "remove_code", "block_id": "b0", "code_id": "b0.c1"}', + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + data = json.loads(result.output)["data"] + assert data["dry_run"] is True + assert data["structural_change"] is True + assert "version" not in data + assert len(data["blocks"][0]["codes"]) == 1 + client.update_config.assert_not_called() + + def test_edit_storage_wholesale_replacement(self, tmp_path: Path) -> None: + client = _make_client() + detail = _config_detail() + client.get_config_detail.return_value = detail + app = _build_app(_make_store(tmp_path), client) + + new_storage = { + "input": {"tables": [{"source": "in.c-main.orders", "destination": "orders"}]}, + "output": {"tables": []}, + } + result = runner.invoke( + app, + [ + "--json", + "transformation", + "edit", + "--project", + "prod", + "--config-id", + "123", + "--component-id", + "keboola.snowflake-transformation", + "--change-description", + "remap inputs", + "--storage", + json.dumps(new_storage), + ], + ) + + assert result.exit_code == 0, result.output + data = json.loads(result.output)["data"] + assert data["storage_replaced"] is True + assert data["operations_applied"] == [] + kwargs = client.update_config.call_args.kwargs + assert kwargs["configuration"]["storage"] == new_storage + # No ops given -> parameters untouched (byte-for-byte as fetched). + assert kwargs["configuration"]["parameters"] == copy.deepcopy( + detail["configuration"]["parameters"] + ) + + def test_edit_op_file(self, tmp_path: Path) -> None: + client = _make_client() + client.get_config_detail.return_value = _config_detail() + app = _build_app(_make_store(tmp_path), client) + ops_file = tmp_path / "ops.json" + ops_file.write_text( + json.dumps( + [ + { + "op": "add_code", + "block_id": "b0", + "code": {"name": "extra", "script": "SELECT 9;"}, + } + ] + ), + encoding="utf-8", + ) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "edit", + "--project", + "prod", + "--config-id", + "123", + "--component-id", + "keboola.snowflake-transformation", + "--change-description", + "add extra code", + "--op-file", + str(ops_file), + ], + ) + + assert result.exit_code == 0, result.output + data = json.loads(result.output)["data"] + assert [c["name"] for c in data["blocks"][0]["codes"]] == ["load", "clean", "extra"] + assert data["blocks"][0]["codes"][2]["id"] == "b0.c2" + + def test_edit_unknown_block_id_lists_valid_ids(self, tmp_path: Path) -> None: + client = _make_client() + client.get_config_detail.return_value = _config_detail() + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "edit", + "--project", + "prod", + "--config-id", + "123", + "--component-id", + "keboola.snowflake-transformation", + "--change-description", + "bad op", + "--op", + '{"op": "rename_block", "block_id": "b9", "block_name": "X"}', + ], + ) + + assert result.exit_code == 1, result.output + payload = json.loads(result.output) + assert payload["error"]["code"] == "VALIDATION_ERROR" + assert "Valid block ids: b0" in payload["error"]["message"] + client.update_config.assert_not_called() + + def test_edit_malformed_op_json(self, tmp_path: Path) -> None: + client = _make_client() + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "edit", + "--project", + "prod", + "--config-id", + "123", + "--change-description", + "bad json", + "--op", + "{not valid json", + ], + ) + + assert result.exit_code == 2, result.output + assert json.loads(result.output)["error"]["code"] == "INPUT_ERROR" + client.get_config_detail.assert_not_called() + client.update_config.assert_not_called() + + def test_edit_op_and_op_file_conflict(self, tmp_path: Path) -> None: + client = _make_client() + app = _build_app(_make_store(tmp_path), client) + ops_file = tmp_path / "ops.json" + ops_file.write_text("[]", encoding="utf-8") + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "edit", + "--project", + "prod", + "--config-id", + "123", + "--change-description", + "conflict", + "--op", + '{"op": "remove_block", "block_id": "b0"}', + "--op-file", + str(ops_file), + ], + ) + + assert result.exit_code == 2, result.output + assert json.loads(result.output)["error"]["code"] == "INVALID_ARGUMENT" + + def test_edit_requires_ops_or_storage(self, tmp_path: Path) -> None: + client = _make_client() + app = _build_app(_make_store(tmp_path), client) + + result = runner.invoke( + app, + [ + "--json", + "transformation", + "edit", + "--project", + "prod", + "--config-id", + "123", + "--change-description", + "noop", + ], + ) + + assert result.exit_code == 2, result.output + assert json.loads(result.output)["error"]["code"] == "INVALID_ARGUMENT" diff --git a/tests/test_transformation_ops.py b/tests/test_transformation_ops.py new file mode 100644 index 00000000..3b4cadcb --- /dev/null +++ b/tests/test_transformation_ops.py @@ -0,0 +1,504 @@ +"""Tests for the transformation block/code ops engine (issue #396). + +Pure-engine tests for services/_transformation_ops.py: every op happy +path, unknown-ID errors (with valid IDs listed), ID renumbering after +structural ops, str_replace scoping/not-found cases, sequential batch +semantics, and raw<->simplified round trips through the SQL statement +splitter. +""" + +import copy + +import pytest + +from keboola_agent_cli.services import _transformation_ops as tf_ops + + +def make_params() -> dict: + """Simplified parameters with 2 blocks / 3 codes (script as SQL text).""" + return { + "blocks": [ + { + "name": "First", + "codes": [ + {"name": "load", "script": "SELECT 1;\n\nSELECT 2;"}, + {"name": "clean", "script": "DELETE FROM t;"}, + ], + }, + { + "name": "Second", + "codes": [ + {"name": "report", "script": 'CREATE TABLE "r" AS SELECT * FROM t;'}, + ], + }, + ] + } + + +def apply(params: dict, raw_ops: list[dict]) -> tf_ops.BatchResult: + """Parse and apply raw op dicts in one go.""" + return tf_ops.apply_ops(params, tf_ops.parse_ops(raw_ops)) + + +class TestParseOps: + def test_parse_valid_ops(self) -> None: + ops = tf_ops.parse_ops( + [ + {"op": "add_block", "block": {"name": "B", "codes": []}}, + {"op": "remove_block", "block_id": "b0"}, + {"op": "str_replace", "search_for": "a", "replace_with": "b"}, + ] + ) + assert [o.op for o in ops] == ["add_block", "remove_block", "str_replace"] + assert ops[0].position == "end" # default + + def test_parse_unknown_op(self) -> None: + with pytest.raises(ValueError, match="Invalid operation"): + tf_ops.parse_ops([{"op": "explode_block", "block_id": "b0"}]) + + def test_parse_missing_field(self) -> None: + with pytest.raises(ValueError, match="Invalid operation"): + tf_ops.parse_ops([{"op": "rename_block", "block_id": "b0"}]) + + def test_parse_str_replace_code_id_requires_block_id(self) -> None: + with pytest.raises(ValueError, match="code_id must be None if block_id is None"): + tf_ops.parse_ops( + [{"op": "str_replace", "code_id": "b0.c0", "search_for": "a", "replace_with": "b"}] + ) + + def test_parse_invalid_position(self) -> None: + with pytest.raises(ValueError, match="Invalid operation"): + tf_ops.parse_ops( + [{"op": "add_block", "block": {"name": "B", "codes": []}, "position": "middle"}] + ) + + +class TestAddIds: + def test_ids_are_positional(self) -> None: + params = make_params() + tf_ops.add_ids(params) + assert params["blocks"][0]["id"] == "b0" + assert params["blocks"][1]["id"] == "b1" + assert params["blocks"][0]["codes"][0]["id"] == "b0.c0" + assert params["blocks"][0]["codes"][1]["id"] == "b0.c1" + assert params["blocks"][1]["codes"][0]["id"] == "b1.c0" + + def test_empty_blocks_ok(self) -> None: + assert tf_ops.add_ids({"blocks": []}) == {"blocks": []} + + +class TestAddBlock: + def test_add_block_end(self) -> None: + result = apply( + make_params(), + [ + { + "op": "add_block", + "block": {"name": "Third", "codes": [{"name": "c", "script": "SELECT 3;"}]}, + } + ], + ) + assert [b["name"] for b in result.params["blocks"]] == ["First", "Second", "Third"] + assert result.params["blocks"][2]["id"] == "b2" + assert result.params["blocks"][2]["codes"][0]["id"] == "b2.c0" + assert result.messages == ['Added block with name "Third"'] + assert result.structural is True + + def test_add_block_start_renumbers(self) -> None: + result = apply( + make_params(), + [{"op": "add_block", "block": {"name": "Zero", "codes": []}, "position": "start"}], + ) + assert [b["name"] for b in result.params["blocks"]] == ["Zero", "First", "Second"] + # IDs re-derived: the new block is b0, the old b0 became b1. + assert result.params["blocks"][0]["id"] == "b0" + assert result.params["blocks"][1]["id"] == "b1" + assert result.params["blocks"][1]["codes"][0]["id"] == "b1.c0" + + def test_add_block_empty_name(self) -> None: + with pytest.raises(ValueError, match="block name cannot be empty"): + apply(make_params(), [{"op": "add_block", "block": {"name": " ", "codes": []}}]) + + +class TestRemoveBlock: + def test_remove_block_renumbers(self) -> None: + result = apply(make_params(), [{"op": "remove_block", "block_id": "b0"}]) + assert [b["name"] for b in result.params["blocks"]] == ["Second"] + # Former b1 is now b0 (IDs re-derived after structural change). + assert result.params["blocks"][0]["id"] == "b0" + assert result.params["blocks"][0]["codes"][0]["id"] == "b0.c0" + assert result.structural is True + + def test_remove_block_unknown_id_lists_valid(self) -> None: + with pytest.raises( + ValueError, match=r"Block with id 'b9' does not exist. Valid block ids: b0, b1" + ): + apply(make_params(), [{"op": "remove_block", "block_id": "b9"}]) + + +class TestRenameBlock: + def test_rename_block(self) -> None: + result = apply( + make_params(), [{"op": "rename_block", "block_id": "b1", "block_name": "Renamed"}] + ) + assert result.params["blocks"][1]["name"] == "Renamed" + assert result.structural is False + + def test_rename_block_empty_name(self) -> None: + with pytest.raises(ValueError, match="block name cannot be empty"): + apply(make_params(), [{"op": "rename_block", "block_id": "b0", "block_name": " "}]) + + def test_rename_block_unknown_id(self) -> None: + with pytest.raises(ValueError, match="Block with id 'b7' does not exist"): + apply(make_params(), [{"op": "rename_block", "block_id": "b7", "block_name": "X"}]) + + +class TestAddCode: + def test_add_code_end(self) -> None: + result = apply( + make_params(), + [ + { + "op": "add_code", + "block_id": "b1", + "code": {"name": "extra", "script": "SELECT 9;"}, + } + ], + ) + codes = result.params["blocks"][1]["codes"] + assert [c["name"] for c in codes] == ["report", "extra"] + assert codes[1]["id"] == "b1.c1" + assert result.messages == ['Added code with name "extra"'] + assert result.structural is True + + def test_add_code_start_renumbers(self) -> None: + result = apply( + make_params(), + [ + { + "op": "add_code", + "block_id": "b0", + "code": {"name": "init", "script": "SET x = 1;"}, + "position": "start", + } + ], + ) + codes = result.params["blocks"][0]["codes"] + assert [c["name"] for c in codes] == ["init", "load", "clean"] + assert [c["id"] for c in codes] == ["b0.c0", "b0.c1", "b0.c2"] + + def test_add_code_unknown_block(self) -> None: + with pytest.raises(ValueError, match="Block with id 'b5' does not exist"): + apply( + make_params(), + [ + { + "op": "add_code", + "block_id": "b5", + "code": {"name": "x", "script": "SELECT 1;"}, + } + ], + ) + + def test_add_code_empty_name(self) -> None: + with pytest.raises(ValueError, match="code name cannot be empty"): + apply( + make_params(), + [{"op": "add_code", "block_id": "b0", "code": {"name": "", "script": "SELECT 1;"}}], + ) + + +class TestRemoveCode: + def test_remove_code_renumbers(self) -> None: + result = apply(make_params(), [{"op": "remove_code", "block_id": "b0", "code_id": "b0.c0"}]) + codes = result.params["blocks"][0]["codes"] + assert [c["name"] for c in codes] == ["clean"] + # Former b0.c1 renumbered to b0.c0. + assert codes[0]["id"] == "b0.c0" + + def test_remove_code_unknown_id_lists_valid(self) -> None: + with pytest.raises( + ValueError, + match=r"Code with id 'b0.c9' in block 'b0' does not exist. Valid code ids: b0.c0, b0.c1", + ): + apply(make_params(), [{"op": "remove_code", "block_id": "b0", "code_id": "b0.c9"}]) + + +class TestRenameCode: + def test_rename_code(self) -> None: + result = apply( + make_params(), + [{"op": "rename_code", "block_id": "b1", "code_id": "b1.c0", "code_name": "summary"}], + ) + assert result.params["blocks"][1]["codes"][0]["name"] == "summary" + + def test_rename_code_empty_name(self) -> None: + with pytest.raises(ValueError, match="code name cannot be empty"): + apply( + make_params(), + [{"op": "rename_code", "block_id": "b0", "code_id": "b0.c0", "code_name": ""}], + ) + + +class TestSetCode: + def test_set_code(self) -> None: + result = apply( + make_params(), + [ + { + "op": "set_code", + "block_id": "b0", + "code_id": "b0.c1", + "script": "TRUNCATE TABLE t;", + } + ], + ) + assert result.params["blocks"][0]["codes"][1]["script"] == "TRUNCATE TABLE t;" + assert result.messages == ["Changed code with id 'b0.c1' in block 'b0'"] + + def test_set_code_empty_script(self) -> None: + with pytest.raises(ValueError, match="script cannot be empty"): + apply( + make_params(), + [{"op": "set_code", "block_id": "b0", "code_id": "b0.c0", "script": " "}], + ) + + def test_set_code_unknown_code(self) -> None: + with pytest.raises(ValueError, match=r"Code with id 'b1\.c4' in block 'b1' does not exist"): + apply( + make_params(), + [{"op": "set_code", "block_id": "b1", "code_id": "b1.c4", "script": "SELECT 1;"}], + ) + + +class TestAddScript: + def test_add_script_end_joins_with_space(self) -> None: + result = apply( + make_params(), + [{"op": "add_script", "block_id": "b0", "code_id": "b0.c1", "script": "SELECT 3;"}], + ) + assert result.params["blocks"][0]["codes"][1]["script"] == "DELETE FROM t; SELECT 3;" + + def test_add_script_start(self) -> None: + result = apply( + make_params(), + [ + { + "op": "add_script", + "block_id": "b0", + "code_id": "b0.c1", + "script": "SET x = 1;", + "position": "start", + } + ], + ) + assert result.params["blocks"][0]["codes"][1]["script"] == "SET x = 1; DELETE FROM t;" + + def test_add_script_to_empty_script(self) -> None: + params = make_params() + params["blocks"][0]["codes"][0]["script"] = "" + result = apply( + params, + [{"op": "add_script", "block_id": "b0", "code_id": "b0.c0", "script": "SELECT 1;"}], + ) + assert result.params["blocks"][0]["codes"][0]["script"] == "SELECT 1;" + + def test_add_script_empty_script(self) -> None: + with pytest.raises(ValueError, match="script cannot be empty"): + apply( + make_params(), + [{"op": "add_script", "block_id": "b0", "code_id": "b0.c0", "script": " "}], + ) + + +class TestStrReplace: + def test_replace_whole_transformation_counts_occurrences(self) -> None: + params = make_params() + result = apply( + params, [{"op": "str_replace", "search_for": "SELECT", "replace_with": "select"}] + ) + # "SELECT 1;\n\nSELECT 2;" has 2 + "CREATE TABLE ... SELECT" has 1. + assert result.messages == ['Replaced 3 occurrences of "SELECT" in the transformation'] + assert result.params["blocks"][0]["codes"][0]["script"] == "select 1;\n\nselect 2;" + + def test_replace_block_scope(self) -> None: + result = apply( + make_params(), + [{"op": "str_replace", "block_id": "b0", "search_for": "t;", "replace_with": "t2;"}], + ) + assert result.params["blocks"][0]["codes"][1]["script"] == "DELETE FROM t2;" + # Block b1 untouched. + assert "FROM t;" in result.params["blocks"][1]["codes"][0]["script"] + assert result.messages == ['Replaced 1 occurrence of "t;" in block "b0"'] + + def test_replace_code_scope(self) -> None: + result = apply( + make_params(), + [ + { + "op": "str_replace", + "block_id": "b0", + "code_id": "b0.c0", + "search_for": "2", + "replace_with": "22", + } + ], + ) + assert result.params["blocks"][0]["codes"][0]["script"] == "SELECT 1;\n\nSELECT 22;" + assert result.messages == ['Replaced 1 occurrence of "2" in code "b0.c0", block "b0"'] + + def test_replace_not_found(self) -> None: + with pytest.raises( + ValueError, match='Search string "nonexistent" not found in the transformation' + ): + apply( + make_params(), + [{"op": "str_replace", "search_for": "nonexistent", "replace_with": "x"}], + ) + + def test_replace_empty_search(self) -> None: + with pytest.raises(ValueError, match="search string is empty"): + apply(make_params(), [{"op": "str_replace", "search_for": "", "replace_with": "x"}]) + + def test_replace_search_equals_replace(self) -> None: + with pytest.raises(ValueError, match="search string and replace string are the same"): + apply( + make_params(), + [{"op": "str_replace", "search_for": "same", "replace_with": "same"}], + ) + + def test_replace_unknown_block_id(self) -> None: + with pytest.raises(ValueError, match="Block with id 'b8' does not exist"): + apply( + make_params(), + [{"op": "str_replace", "block_id": "b8", "search_for": "a", "replace_with": "b"}], + ) + + def test_replace_block_without_codes(self) -> None: + params = {"blocks": [{"name": "Empty", "codes": []}]} + with pytest.raises(ValueError, match='No scripts found in block "b0"'): + apply( + params, + [{"op": "str_replace", "block_id": "b0", "search_for": "a", "replace_with": "b"}], + ) + + +class TestBatchSemantics: + def test_ops_apply_sequentially(self) -> None: + """A later op sees the effect of an earlier op in the same batch.""" + result = apply( + make_params(), + [ + {"op": "set_code", "block_id": "b0", "code_id": "b0.c0", "script": "SELECT 99;"}, + {"op": "str_replace", "search_for": "99", "replace_with": "100"}, + ], + ) + assert result.params["blocks"][0]["codes"][0]["script"] == "SELECT 100;" + + def test_mid_batch_added_block_not_addressable(self) -> None: + """Elements added within a batch carry no ID until the next batch.""" + with pytest.raises(ValueError, match="Block with id 'b2' does not exist"): + apply( + make_params(), + [ + {"op": "add_block", "block": {"name": "New", "codes": []}}, + {"op": "rename_block", "block_id": "b2", "block_name": "Oops"}, + ], + ) + + def test_ids_stick_to_elements_within_batch(self) -> None: + """After removing b0 mid-batch, 'b1' still addresses the original b1.""" + result = apply( + make_params(), + [ + {"op": "remove_block", "block_id": "b0"}, + {"op": "rename_block", "block_id": "b1", "block_name": "Still Second"}, + ], + ) + assert result.params["blocks"][0]["name"] == "Still Second" + assert result.params["blocks"][0]["id"] == "b0" # re-derived at batch end + + def test_input_not_mutated(self) -> None: + params = make_params() + original = copy.deepcopy(params) + apply(params, [{"op": "rename_block", "block_id": "b0", "block_name": "Changed"}]) + assert params == original + + def test_structural_flag_false_for_content_ops(self) -> None: + result = apply( + make_params(), + [{"op": "rename_block", "block_id": "b0", "block_name": "X"}], + ) + assert result.structural is False + + def test_missing_blocks_key_defaults_to_empty(self) -> None: + result = apply({}, [{"op": "add_block", "block": {"name": "B", "codes": []}}]) + assert [b["name"] for b in result.params["blocks"]] == ["B"] + + +class TestRoundTrip: + def test_raw_to_simplified_joins_statements(self) -> None: + raw = { + "blocks": [ + {"name": "B", "codes": [{"name": "c", "script": ["SELECT 1;", "SELECT 2;"]}]} + ] + } + simplified = tf_ops.raw_to_simplified(raw) + assert simplified["blocks"][0]["codes"][0]["script"] == "SELECT 1;\n\nSELECT 2;" + + def test_raw_to_simplified_keeps_string_script(self) -> None: + raw = {"blocks": [{"name": "B", "codes": [{"name": "c", "script": "SELECT 1; SELECT 2;"}]}]} + simplified = tf_ops.raw_to_simplified(raw) + assert simplified["blocks"][0]["codes"][0]["script"] == "SELECT 1; SELECT 2;" + + def test_simplified_to_raw_splits_statements(self) -> None: + simplified = { + "blocks": [ + { + "id": "b0", + "name": "B", + "codes": [{"id": "b0.c0", "name": "c", "script": "SELECT 1;\nSELECT 'a;b';"}], + } + ] + } + raw = tf_ops.simplified_to_raw(simplified) + # Statement split respects string literals; synthetic ids stripped. + assert raw["blocks"][0]["codes"][0]["script"] == ["SELECT 1;", "SELECT 'a;b';"] + assert "id" not in raw["blocks"][0] + assert "id" not in raw["blocks"][0]["codes"][0] + + def test_full_round_trip_preserves_statements(self) -> None: + raw = { + "blocks": [ + { + "name": "B", + "codes": [ + {"name": "c", "script": ["SELECT 1;", "-- note\nSELECT 2;"]}, + ], + } + ] + } + round_tripped = tf_ops.simplified_to_raw(tf_ops.raw_to_simplified(raw)) + assert round_tripped["blocks"][0]["codes"][0]["script"] == [ + "SELECT 1;", + "-- note\nSELECT 2;", + ] + + def test_edit_pipeline_split_integration(self) -> None: + """set_code with multi-statement SQL lands as multiple raw statements.""" + result = apply( + make_params(), + [ + { + "op": "set_code", + "block_id": "b0", + "code_id": "b0.c0", + "script": "CREATE TABLE x AS SELECT 1; INSERT INTO x VALUES (2);", + } + ], + ) + raw = tf_ops.simplified_to_raw(result.params) + assert raw["blocks"][0]["codes"][0]["script"] == [ + "CREATE TABLE x AS SELECT 1;", + "INSERT INTO x VALUES (2);", + ] From 31f5a2c56183164d8746210688f4c85fa7101f99 Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:46:45 +0200 Subject: [PATCH 11/17] feat(flow): examples command + authoritative bundled schema fallback (#397) Ports get_flow_examples (vendored JSONL examples for keboola.flow + legacy keboola.orchestrator, informational-only warning for the latter). flow schema --full without --project now serves a bundled schema snapshot (source=bundled) instead of exit 2; the conditional-flow snapshot is the LIVE keboola.flow configurationSchema from the public component index (upstream deleted their drifted copy -- AJDA-2810). Also fixes real drift in the offline YAML template (retryOn object form, delay/ maxRetries) now pinned by validation tests. --- src/keboola_agent_cli/commands/flow.py | 127 +++- src/keboola_agent_cli/resources/__init__.py | 6 + .../resources/flow/__init__.py | 23 + .../flow/conditional-flow-schema.json | 633 ++++++++++++++++++ .../flow/conditional_flow_examples.jsonl | 3 + .../resources/flow/flow-schema.json | 134 ++++ .../resources/flow/legacy_flow_examples.jsonl | 3 + .../services/flow_service.py | 78 +++ tests/test_flow_cli.py | 19 +- tests/test_flow_examples.py | 159 +++++ 10 files changed, 1158 insertions(+), 27 deletions(-) create mode 100644 src/keboola_agent_cli/resources/__init__.py create mode 100644 src/keboola_agent_cli/resources/flow/__init__.py create mode 100644 src/keboola_agent_cli/resources/flow/conditional-flow-schema.json create mode 100644 src/keboola_agent_cli/resources/flow/conditional_flow_examples.jsonl create mode 100644 src/keboola_agent_cli/resources/flow/flow-schema.json create mode 100644 src/keboola_agent_cli/resources/flow/legacy_flow_examples.jsonl create mode 100644 tests/test_flow_examples.py diff --git a/src/keboola_agent_cli/commands/flow.py b/src/keboola_agent_cli/commands/flow.py index 8cc53a77..15cffc00 100644 --- a/src/keboola_agent_cli/commands/flow.py +++ b/src/keboola_agent_cli/commands/flow.py @@ -19,6 +19,12 @@ from rich.table import Table from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..services.flow_service import ( + FLOW_COMPONENT_ID, + LEGACY_FLOW_COMPONENT_ID, + get_bundled_flow_schema, + get_flow_examples, +) from ..services.flow_validation import find_unreachable_phases, validate_conditional_flow from ._helpers import ( check_cli_permission, @@ -43,6 +49,7 @@ # Update with: kbagent flow update --project ALIAS --flow-id ID --file @flow.yaml # Validate offline: kbagent flow validate --file @flow.yaml # Full JSON schema: kbagent flow schema --full +# Real-world examples: kbagent flow examples # # IDs are STRINGS. goto is a phase id or null (= end the flow). @@ -63,11 +70,13 @@ goto: "transform" - id: "transform" name: "Transform" + # Phase-level retry applies to every job task in the phase. + # Omit retryOn to retry on ANY error. retry: strategy: linear strategyParams: - delaySeconds: 60 - retryOn: ["error"] + maxRetries: 3 + delay: 60 # seconds between attempts next: - id: "done" goto: null @@ -84,11 +93,16 @@ componentId: "keboola.ex-http" configId: "123456789" mode: run + # Task-level retry overrides the phase-level one. retryOn entries are + # objects: type is errorMessageContains or errorMessageExact. retry: strategy: linear strategyParams: - delaySeconds: 30 - retryOn: ["error"] + maxRetries: 2 + delay: 30 + retryOn: + - type: errorMessageContains + value: "timeout" - id: "task-transform" name: "Run transformation" phase: "transform" @@ -363,37 +377,50 @@ def _format_flow_detail(formatter: Any, result: dict[str, Any]) -> None: # --------------------------------------------------------------------------- +def _print_json_schema(formatter: Any, schema: dict[str, Any], source: str) -> None: + """Emit a JSON Schema payload in either output mode, tagged with its source.""" + if formatter.json_mode: + formatter.output({"format": "json-schema", "source": source, "schema": schema}) + else: + formatter.console.print( + Syntax(json.dumps(schema, indent=2), "json", theme="monokai", line_numbers=False) + ) + + @flow_app.command("schema") def flow_schema( ctx: typer.Context, full: bool = typer.Option( False, "--full", - help="Dump the live JSON Schema fetched from the stack (requires --project).", + help=( + "Dump the conditional-flow JSON Schema: live from the stack with " + "--project, bundled snapshot without it." + ), ), project: str | None = typer.Option( None, "--project", - help="Project alias -- required for --full (the schema is served by the stack).", + help="Project alias -- with --full, fetch the live schema from this stack.", ), ) -> None: - """Print the conditional-flow YAML template, or --full for the live JSON Schema. + """Print the conditional-flow YAML template, or --full for the JSON Schema. - The plain template is offline. ``--full`` fetches the real keboola.flow - JSON Schema from the stack's component registry, so it needs ``--project``. + The plain template is offline. ``--full`` dumps the keboola.flow JSON + Schema: with ``--project`` it is fetched live from the stack's component + registry; without it the bundled snapshot (vendored at build time, see + ``kbagent flow examples`` for matching examples) is served offline. """ formatter = get_formatter(ctx) if full: if not project: - formatter.error( - message=( - "--full requires --project: the conditional-flow JSON Schema is " - "served by the stack's component registry, not bundled. " - "Run e.g. 'kbagent flow schema --full --project ALIAS'." - ), - error_code=ErrorCode.VALIDATION_ERROR, - ) - raise typer.Exit(code=2) + if not formatter.json_mode: + formatter.console.print( + "[dim]Bundled conditional-flow JSON Schema snapshot (offline). " + "Pass --project ALIAS to fetch the live schema from the stack.[/dim]" + ) + _print_json_schema(formatter, get_bundled_flow_schema(FLOW_COMPONENT_ID), "bundled") + return service = get_service(ctx, "flow_service") try: @@ -413,12 +440,7 @@ def flow_schema( ) raise typer.Exit(code=4) - if formatter.json_mode: - formatter.output({"format": "json-schema", "schema": schema}) - else: - formatter.console.print( - Syntax(json.dumps(schema, indent=2), "json", theme="monokai", line_numbers=False) - ) + _print_json_schema(formatter, schema, "live") return if formatter.json_mode: @@ -433,6 +455,63 @@ def flow_schema( formatter.console.print(Syntax(_FLOW_SCHEMA, "yaml", theme="monokai", line_numbers=False)) +# --------------------------------------------------------------------------- +# flow examples +# --------------------------------------------------------------------------- + + +@flow_app.command("examples") +def flow_examples( + ctx: typer.Context, + component_id: str = typer.Option( + FLOW_COMPONENT_ID, + "--component-id", + help=( + "Flow component id: keboola.flow (conditional, default) or " + "keboola.orchestrator (legacy, informational only)." + ), + ), +) -> None: + """Show bundled example flow configurations (offline, no project needed). + + Examples are vendored from keboola-mcp-server. The default is + keboola.flow (Conditional Flow); use them as blueprints for + ``kbagent flow new --file @flow.yaml``. + + ``--component-id keboola.orchestrator`` serves the legacy orchestrator + examples for reference only -- kbagent cannot create or edit orchestrator + flows (support was dropped in 0.57.0). + """ + formatter = get_formatter(ctx) + try: + examples = get_flow_examples(component_id) + except ValueError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.INVALID_ARGUMENT) + raise typer.Exit(code=2) from None + + if component_id == LEGACY_FLOW_COMPONENT_ID: + formatter.warning( + "kbagent cannot create or edit keboola.orchestrator flows (legacy " + "orchestrator support was dropped in 0.57.0). These examples are " + "informational only, e.g. for reading flows in foreign projects." + ) + + if formatter.json_mode: + formatter.output(examples) + return + + formatter.console.print( + f"\n[bold]Flow configuration examples for {escape(component_id)}[/bold] " + f"({len(examples)} example{'s' if len(examples) != 1 else ''}):\n" + ) + for index, example in enumerate(examples, 1): + formatter.console.print(f"[cyan bold]{index}. Flow Configuration:[/cyan bold]") + formatter.console.print( + Syntax(json.dumps(example, indent=2), "json", theme="monokai", line_numbers=False) + ) + formatter.console.print() + + # --------------------------------------------------------------------------- # flow validate # --------------------------------------------------------------------------- diff --git a/src/keboola_agent_cli/resources/__init__.py b/src/keboola_agent_cli/resources/__init__.py new file mode 100644 index 00000000..ca0a162b --- /dev/null +++ b/src/keboola_agent_cli/resources/__init__.py @@ -0,0 +1,6 @@ +"""Bundled static resources shipped inside the kbagent wheel. + +Data files live in subpackages (e.g. ``resources.flow``) and are read via +``importlib.resources`` so they resolve identically from a source checkout, +an installed wheel, or a zipapp. Never read them with raw filesystem paths. +""" diff --git a/src/keboola_agent_cli/resources/flow/__init__.py b/src/keboola_agent_cli/resources/flow/__init__.py new file mode 100644 index 00000000..a769be27 --- /dev/null +++ b/src/keboola_agent_cli/resources/flow/__init__.py @@ -0,0 +1,23 @@ +"""Bundled flow resources: example configurations + JSON Schemas. + +Vendored from upstream sources (issue #397 -- port of the keboola-mcp-server +``get_flow_examples`` tool + authoritative schema bundling): + +- ``conditional_flow_examples.jsonl`` / ``legacy_flow_examples.jsonl``: + verbatim copies of ``src/keboola_mcp_server/resources/flow_examples/*`` from + https://github.com/keboola/mcp-server (fetched 2026-07-20). One JSON flow + configuration per line. +- ``conditional-flow-schema.json``: the live ``keboola.flow`` + ``configurationSchema`` snapshot taken from the public Storage component + index (``GET https://connection.keboola.com/v2/storage``) on 2026-07-20. + This is the same document ``kbagent flow schema --full --project ALIAS`` + fetches live; the bundled copy is the offline fallback. It is NEWER than + the (since-removed) mcp-server bundled copy -- upstream deleted theirs in + favour of live fetching precisely because snapshots drift, so refresh this + file from the index when flow features change. +- ``flow-schema.json``: the legacy ``keboola.orchestrator`` schema, verbatim + from ``src/keboola_mcp_server/resources/flow-schema.json`` (still bundled + upstream; orchestrator is frozen so drift risk is nil). kbagent cannot + create or edit orchestrator flows (dropped in 0.57.0) -- this schema and + the legacy examples are informational only. +""" diff --git a/src/keboola_agent_cli/resources/flow/conditional-flow-schema.json b/src/keboola_agent_cli/resources/flow/conditional-flow-schema.json new file mode 100644 index 00000000..bfd4c302 --- /dev/null +++ b/src/keboola_agent_cli/resources/flow/conditional-flow-schema.json @@ -0,0 +1,633 @@ +{ + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema", + "required": [ + "phases", + "tasks" + ], + "properties": { + "tasks": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name", + "task", + "phase" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "task": { + "type": "object", + "oneOf": [ + { + "anyOf": [ + { + "required": [ + "configId" + ] + }, + { + "required": [ + "configData" + ] + } + ], + "required": [ + "type", + "componentId", + "mode" + ], + "properties": { + "mode": { + "enum": [ + "run" + ], + "type": "string" + }, + "type": { + "enum": [ + "job" + ], + "type": "string" + }, + "delay": { + "type": [ + "string", + "number" + ], + "description": "Initial delay in seconds before starting the job" + }, + "retry": { + "$ref": "#/definitions/retryConfiguration", + "description": "Retry configuration specific to this job task. Takes precedence over phase-level retry configuration." + }, + "configId": { + "type": "string", + "description": "ID of the component configuration to run" + }, + "configData": { + "type": "object", + "description": "Inline component configuration parameters" + }, + "componentId": { + "type": "string", + "description": "ID of the Keboola component to execute (e.g., 'keboola.ex-db-snowflake')" + }, + "variableOverrides": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Controls which flow context variables are applied to this job task. Tri-state: omit the field to apply all flow variables, set to [] to apply none, or list specific variable names to apply only those. Flow variables only override values for variable names that already exist in the component's system variables (keboola.variables). This field is consumed by the flow runner and is not passed to the job itself." + } + }, + "description": "Job task that executes a Keboola component configuration" + }, + { + "required": [ + "type", + "recipients", + "title" + ], + "properties": { + "type": { + "enum": [ + "notification" + ], + "type": "string" + }, + "title": { + "type": "string" + }, + "message": { + "type": "string", + "description": "Optional message body. For email: defaults to empty string if not provided. For webhook: defaults to null if not provided." + }, + "recipients": { + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "if": { + "properties": { + "channel": { + "const": "email" + } + } + }, + "then": { + "properties": { + "address": { + "format": "email" + } + } + } + }, + { + "if": { + "properties": { + "channel": { + "const": "webhook" + } + } + }, + "then": { + "properties": { + "address": { + "format": "uri" + } + } + } + } + ], + "required": [ + "channel", + "address" + ], + "properties": { + "address": { + "type": "string", + "description": "Recipient address: email address for email channel, HTTP/HTTPS URL for webhook channel" + }, + "channel": { + "enum": [ + "email", + "webhook" + ], + "type": "string", + "description": "Delivery channel: 'email' for email notifications, 'webhook' for HTTP POST notifications" + } + } + }, + "minItems": 1, + "description": "List of notification recipients. Can mix email and webhook recipients." + } + }, + "description": "Notification task that sends messages via email or webhook" + }, + { + "oneOf": [ + { + "required": [ + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the variable. Will be accessible in conditions throughout the flow. Flow variables are merged with each job task's system variables (from keboola.variables): they override values only for variable names that already exist in the component's variable set. Use variableOverrides on job tasks to control which flow variables are applied to a specific task." + }, + "type": { + "enum": [ + "variable" + ], + "type": "string" + }, + "value": { + "type": "string" + } + }, + "description": "Static variable with a fixed value" + }, + { + "required": [ + "source" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the variable. Will be accessible in conditions throughout the flow. Flow variables are merged with each job task's system variables (from keboola.variables): they override values only for variable names that already exist in the component's variable set. Use variableOverrides on job tasks to control which flow variables are applied to a specific task." + }, + "type": { + "enum": [ + "variable" + ], + "type": "string" + }, + "source": { + "$ref": "#/definitions/variableSourceObject", + "description": "Source definition for computing the variable value dynamically from task results, phase status, constants, or functions" + } + }, + "description": "Dynamic variable computed from other task/phase results" + } + ], + "required": [ + "type", + "name" + ], + "description": "Variable task that defines a variable for use in conditions or job parameters" + } + ] + }, + "phase": { + "type": "string", + "description": "ID of the phase this task belongs to. Must reference an existing phase ID." + }, + "enabled": { + "type": "boolean", + "default": true, + "description": "Whether this task is enabled. Disabled tasks are skipped during execution. Defaults to true if not specified." + } + } + }, + "description": "Array of tasks that perform the actual work. Tasks are executed within phases in this order: notification tasks first, then variable tasks, finally job tasks." + }, + "phases": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "next": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "goto" + ], + "properties": { + "id": { + "type": "string" + }, + "goto": { + "type": [ + "string", + "null" + ], + "description": "Target phase ID to transition to, or null to end the flow. When using conditional transitions, always include a default transition (without a condition) as the last item to prevent flow execution errors." + }, + "name": { + "type": "string", + "description": "Optional descriptive name for the transition - useful for debugging and monitoring" + }, + "condition": { + "$ref": "#/definitions/operatorCondition", + "description": "Condition that must be met for this transition to be taken. If omitted, this serves as the default transition." + } + } + }, + "description": "Array of conditional transitions to other phases. Conditions are evaluated after all tasks in the phase complete. Always include a default transition (without condition) as the last item to prevent execution errors." + }, + "retry": { + "$ref": "#/definitions/retryConfiguration", + "description": "Retry configuration that will be applied to all job tasks in this phase that don't have their own retry configuration. Only applies to job tasks, not notification or variable tasks." + }, + "description": { + "type": "string" + } + }, + "description": "A phase groups related tasks and defines transitions to other phases. Each phase must have at least one active task (enabled: true)." + }, + "description": "Array of phases that group tasks and define execution order. Within each phase, tasks execute in this order: notification tasks first (sequentially), then variable tasks (sequentially), finally job tasks (in parallel). Phases themselves run sequentially based on dependencies and conditions. Phases cannot be empty and must have at least one enabled task." + } + }, + "definitions": { + "taskCondition": { + "type": "object", + "required": [ + "type", + "task", + "value" + ], + "properties": { + "task": { + "type": "string", + "description": "ID of the task to evaluate. The referenced task must have already completed execution. Set '*' when used with phase operators (ALL_TASKS_IN_PHASE, ANY_TASKS_IN_PHASE)" + }, + "type": { + "enum": [ + "task" + ], + "type": "string", + "description": "Returns task-level metadata or associated job metadata from a task that has already completed." + }, + "value": { + "type": "string", + "description": "JMESPath expression evaluated against the task context. Basic top-level keys: 'taskId', 'phaseId', 'status' (success, user_error, application_error, terminated). Job properties available for job tasks: 'job.id', 'job.componentId', 'job.configId', 'job.status', 'job.result', 'job.startTime', 'job.endTime', 'job.duration' (in seconds), 'job.result.output.tables' (array), 'job.result.message' (string). JMESPath filters, projections, fallback (||), and functions (length, keys, etc.) are supported, e.g. \"job.result.output.variables[?name=='environment'] | [0].value\". Missing paths return null; syntactically invalid expressions throw an error." + } + } + }, + "arrayCondition": { + "type": "object", + "required": [ + "type", + "operands" + ], + "properties": { + "type": { + "enum": [ + "array" + ], + "type": "string" + }, + "operands": { + "type": "array", + "items": { + "$ref": "#/definitions/variableSourceObject" + } + } + }, + "description": "A condition that creates an array from multiple operands. Used primarily with the INCLUDES operator." + }, + "phaseCondition": { + "type": "object", + "required": [ + "type", + "phase", + "value" + ], + "properties": { + "type": { + "enum": [ + "phase" + ], + "type": "string", + "description": "Returns phase-level metadata from a phase that has already completed." + }, + "phase": { + "type": "string", + "description": "ID of the phase to evaluate. The referenced phase must have already completed execution." + }, + "value": { + "type": "string", + "description": "JMESPath expression evaluated against the phase context. Available top-level keys: 'phaseId', 'status' (success, user_error, application_error, terminated). Simple dot-notation paths work unchanged; JMESPath filters, projections, fallback (||), and functions (length, keys, etc.) are also supported. Missing paths return null; syntactically invalid expressions throw an error." + } + } + }, + "conditionObject": { + "type": "object", + "oneOf": [ + { + "$ref": "#/definitions/constantCondition" + }, + { + "$ref": "#/definitions/phaseCondition" + }, + { + "$ref": "#/definitions/taskCondition" + }, + { + "$ref": "#/definitions/variableCondition" + }, + { + "$ref": "#/definitions/operatorCondition" + }, + { + "$ref": "#/definitions/functionCondition" + }, + { + "$ref": "#/definitions/arrayCondition" + } + ] + }, + "constantCondition": { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "enum": [ + "const", + "constant" + ], + "type": "string" + }, + "value": { + "type": [ + "string", + "number", + "boolean", + "array" + ] + } + } + }, + "functionCondition": { + "type": "object", + "required": [ + "type", + "function", + "operands" + ], + "properties": { + "type": { + "enum": [ + "function" + ], + "type": "string" + }, + "function": { + "enum": [ + "COUNT", + "DATE" + ], + "type": "string", + "description": "Function type: 'COUNT' (counts elements in array, requires 1 operand), 'DATE' (formats current date/time using PHP DateTime::format, requires 1 operand with format string)" + }, + "operands": { + "type": "array", + "items": { + "$ref": "#/definitions/variableSourceObject" + }, + "description": "Array of conditions that provide inputs to the function. Number of required operands depends on function type." + } + } + }, + "operatorCondition": { + "type": "object", + "oneOf": [ + { + "required": [ + "operator", + "operands" + ], + "properties": { + "type": { + "enum": [ + "operator" + ], + "type": "string" + }, + "operands": { + "type": "array", + "items": { + "$ref": "#/definitions/conditionObject" + }, + "description": "Array of conditions to apply the operator to. Number of required operands depends on operator type." + }, + "operator": { + "enum": [ + "AND", + "OR", + "EQUALS", + "NOT_EQUALS", + "GREATER_THAN", + "LESS_THAN", + "INCLUDES", + "CONTAINS" + ], + "type": "string", + "description": "Operator type: 'AND'/'OR' (logical, requires 1+ operands), 'EQUALS'/'NOT_EQUALS' (equality, requires 2 operands), 'GREATER_THAN'/'LESS_THAN' (comparison, requires 2 operands), 'INCLUDES' (checks if first operand is included in second operand array, requires 2 operands), 'CONTAINS' (checks if first operand string contains second operand string, case-insensitive, requires 2 operands)" + } + } + }, + { + "required": [ + "operator", + "phase", + "operands" + ], + "properties": { + "type": { + "enum": [ + "operator" + ], + "type": "string" + }, + "phase": { + "type": "string", + "description": "ID of the phase to apply the condition to. Must reference an existing phase in the flow." + }, + "operands": { + "type": "array", + "items": { + "$ref": "#/definitions/operatorCondition" + }, + "description": "Array containing the condition to apply to each task in the phase. Use '*' as the task ID in task conditions." + }, + "operator": { + "enum": [ + "ALL_TASKS_IN_PHASE", + "ANY_TASKS_IN_PHASE" + ], + "type": "string", + "description": "Phase operator type: 'ALL_TASKS_IN_PHASE' (condition must be true for all tasks in phase, requires 1 operand), 'ANY_TASKS_IN_PHASE' (condition must be true for at least one task in phase, requires 1 operand)" + } + }, + "description": "Phase-level operators that apply conditions to all or any tasks within a specific phase" + } + ], + "required": [ + "type", + "operator" + ], + "description": "A condition that applies logical or relational operators to other conditions and returns boolean result (true/false)." + }, + "variableCondition": { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "enum": [ + "variable" + ], + "type": "string" + }, + "value": { + "type": "string", + "description": "The name of the variable to evaluate. Must reference a variable defined earlier in the flow." + } + } + }, + "retryConfiguration": { + "type": "object", + "properties": { + "retryOn": { + "type": "array", + "items": { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "enum": [ + "errorMessageContains", + "errorMessageExact" + ], + "type": "string", + "description": "Retry condition type: 'errorMessageContains' (retry if error message contains the value, case-insensitive), 'errorMessageExact' (retry if error message exactly matches the value)" + }, + "value": { + "type": "string" + } + } + }, + "description": "Array of conditions that trigger a retry. Multiple conditions work as OR logic - at least one condition must be met to trigger a retry. If empty or missing, jobs will retry by default for any error." + }, + "strategy": { + "enum": [ + "linear" + ], + "type": "string", + "default": "linear" + }, + "strategyParams": { + "type": "object", + "properties": { + "delay": { + "type": "integer", + "default": 10, + "description": "Delay in seconds between retry attempts" + }, + "maxRetries": { + "type": "integer", + "default": 3, + "description": "Maximum number of retry attempts (default: 3). This is the number of retries after the initial attempt. Set to 0 to disable retry while preserving retry configuration." + } + } + } + }, + "description": "Configuration for automatic retry of failed job tasks. Can be set at phase level (applies to all job tasks in phase) or task level (overrides phase configuration)." + }, + "variableSourceObject": { + "type": "object", + "oneOf": [ + { + "$ref": "#/definitions/constantCondition" + }, + { + "$ref": "#/definitions/phaseCondition" + }, + { + "$ref": "#/definitions/taskCondition" + }, + { + "$ref": "#/definitions/variableCondition" + }, + { + "$ref": "#/definitions/functionCondition" + }, + { + "$ref": "#/definitions/arrayCondition" + } + ], + "description": "Source definition for computing variable values dynamically. Limited subset of condition types that can be used as variable sources (excludes logical operators)." + } + }, + "description": "A Keboola Flow configuration that orchestrates the execution of components. Flows define how tasks are grouped into phases and executed sequentially or in parallel, with conditional transitions between phases." +} diff --git a/src/keboola_agent_cli/resources/flow/conditional_flow_examples.jsonl b/src/keboola_agent_cli/resources/flow/conditional_flow_examples.jsonl new file mode 100644 index 00000000..a492ba77 --- /dev/null +++ b/src/keboola_agent_cli/resources/flow/conditional_flow_examples.jsonl @@ -0,0 +1,3 @@ +{"tasks":[{"id":"40fef978-7092-4d79-a5b4-ea3fb2e38d03","name":"keboola.wr-azure-event-hub-92021091","phase":"6afbf55b-782c-47d7-bf70-f0ef1be6505b","task":{"type":"job","mode":"run","componentId":"keboola.wr-azure-event-hub","configId":"92021091"},"enabled":true},{"id":"43724a3a-7cbe-4d35-9cb9-c6bfb9b940be","name":"keboola.ex-google-drive-92169399","phase":"7dd992b0-9ac5-495b-b277-d8bc0b7e15d5","task":{"type":"job","mode":"run","componentId":"keboola.ex-google-drive","configId":"92169399"},"enabled":true}],"phases":[{"id":"7dd992b0-9ac5-495b-b277-d8bc0b7e15d5","name":"Phase1","next":[{"id":"a25a4e4a-3042-49a2-81d8-fb1103957ebe","goto":"6afbf55b-782c-47d7-bf70-f0ef1be6505b"}]},{"id":"6afbf55b-782c-47d7-bf70-f0ef1be6505b","name":"Phase2"}]} +{"tasks":[{"id":"6bcc72d8-d9a5-4708-b0bd-53c4f6e839f7","name":"keboola.python-transformation-v2-16550","phase":"78c07164-0d1c-41d6-ba48-b821e781d830","task":{"type":"job","mode":"run","componentId":"keboola.python-transformation-v2","configId":"16550"},"enabled":true},{"id":"cd2a4a4f-99c3-4e03-9ece-b8e3e70a7fc7","name":"keboola.python-transformation-v2-17095","phase":"92649482-45d6-475d-aace-33466f37e381","task":{"type":"job","mode":"run","componentId":"keboola.python-transformation-v2","configId":"17095","retry":{"strategy":"linear","strategyParams":{"maxRetries":3,"delay":10}}},"enabled":true},{"id":"639f5bff-20da-442e-b442-fd81879137c3","name":"keboola.python-transformation-v2-16542","phase":"92649482-45d6-475d-aace-33466f37e381","task":{"type":"job","mode":"run","componentId":"keboola.python-transformation-v2","configId":"16542","retry":{"strategy":"linear","strategyParams":{"maxRetries":3,"delay":10}}},"enabled":true},{"id":"ae59ce62-6271-4a42-af65-646c240c27b6","name":"MyNotification","phase":"Error","task":{"type":"notification","title":"MyNotification","message":"Erroroccurred","recipients":[{"channel":"email","address":"john@doe.com"}]},"enabled":true},{"id":"525e2b5a-a9e6-4294-b8c4-eada52c74713","name":"MyNotification","phase":"Success","task":{"type":"notification","title":"MyNotification","message":"Success!!!","recipients":[{"channel":"email","address":"john@doe.com"}]},"enabled":true}],"phases":[{"id":"78c07164-0d1c-41d6-ba48-b821e781d830","name":"Phase1","next":[{"id":"e5dc7c43-d311-4e90-a2ca-6cac8d2eb5f5","goto":"92649482-45d6-475d-aace-33466f37e381"}]},{"id":"92649482-45d6-475d-aace-33466f37e381","name":"Phase2","next":[{"id":"9d35fafe-91c5-4234-b466-cfb0d4edab7f","name":"Condition1","condition":{"type":"operator","operator":"OR","operands":[{"type":"operator","operator":"EQUALS","operands":[{"type":"phase","phase":"78c07164-0d1c-41d6-ba48-b821e781d830","value":"status"},{"type":"const","value":"user_error"}]},{"type":"operator","operator":"EQUALS","operands":[{"type":"phase","phase":"78c07164-0d1c-41d6-ba48-b821e781d830","value":"status"},{"type":"const","value":"application_error"}]}]},"goto":"Error"},{"id":"c2910f57-5db0-460a-837e-6b55858131e2","goto":"Success"}]},{"id":"Error","name":"Error"},{"id":"Success","name":"Success"}]} +{"phases":[{"id":"test-no-retry","name":"TestNoRetry(maxRetries:0)","description":"Testtaskwithretrydisabled(maxRetries:0)","next":[{"id":"check-no-retry-failed","name":"Checkifno-retrytaskfailedasexpected","condition":{"type":"operator","operator":"EQUALS","operands":[{"type":"task","task":"no-retry-user-error","value":"status"},{"type":"const","value":"user_error"}]},"goto":"test-basic-retry"},{"id":"default-to-basic-retry","goto":"test-basic-retry"}]},{"id":"test-basic-retry","name":"TestBasicRetry(3attempts)","description":"Testtaskwithbasicretryconfiguration(3attempts,10sdelay)","next":[{"id":"to-test-message-retry","goto":"test-message-retry"}]},{"id":"test-message-retry","name":"TestErrorMessageRetry","description":"Testretrywithspecificerrormessagematching","next":[{"id":"check-message-retry-success","name":"Checkifmessageretryeventuallysucceeded","condition":{"type":"operator","operator":"EQUALS","operands":[{"type":"task","task":"message-retry-sleep","value":"status"},{"type":"const","value":"success"}]},"goto":"test-extreme-retry"},{"id":"default-to-extreme-retry","goto":"test-extreme-retry"}]},{"id":"test-extreme-retry","name":"TestExtremeRetry(100attempts)","description":"Testwithmaximumretryattemptsandminimaldelay","next":[{"id":"to-notify-results","goto":"notify-results"}]},{"id":"notify-results","name":"NotifyTestResults","description":"Sendnotificationsaboutretrytestingresults"}],"tasks":[{"id":"no-retry-user-error","name":"NoRetryUserErrorTest","phase":"test-no-retry","task":{"type":"job","componentId":"keboola.runner-config-test","configId":"897180330","mode":"run","retry":{"strategy":"linear","strategyParams":{"maxRetries":0,"delay":10}}}},{"id":"basic-retry-app-error","name":"BasicRetryApplicationErrorTest","phase":"test-basic-retry","task":{"type":"job","componentId":"keboola.runner-config-test","configId":"897180386","mode":"run","retry":{"strategy":"linear","strategyParams":{"maxRetries":3,"delay":10}}}},{"id":"basic-retry-success","name":"BasicRetrySuccessTest","phase":"test-basic-retry","task":{"type":"job","componentId":"keboola.python-transformation-v2","configId":"897474983","mode":"run","retry":{"strategy":"linear","strategyParams":{"maxRetries":3,"delay":5}}}},{"id":"message-retry-sleep","name":"MessageRetrySleepTest","phase":"test-message-retry","task":{"type":"job","componentId":"keboola.runner-config-test","configId":"897180272","mode":"run","retry":{"retryOn":[{"type":"errorMessageContains","value":"timeout"},{"type":"errorMessageExact","value":"Connectionrefused"}],"strategy":"linear","strategyParams":{"maxRetries":2,"delay":15}}}},{"id":"extreme-retry-user-error","name":"ExtremeRetryUserErrorTest","phase":"test-extreme-retry","task":{"type":"job","componentId":"keboola.runner-config-test","configId":"897180330","mode":"run","retry":{"retryOn":[{"type":"errorMessageContains","value":"validation"}],"strategy":"linear","strategyParams":{"maxRetries":100,"delay":0}}}},{"id":"extreme-retry-with-delay","name":"ExtremeRetrywithLongDelayTest","phase":"test-extreme-retry","task":{"type":"job","componentId":"keboola.python-transformation-v2","configId":"897474983","mode":"run","delay":60,"retry":{"strategy":"linear","strategyParams":{"maxRetries":5,"delay":120}}}},{"id":"notify-retry-completion","name":"NotifyRetryTestCompletion","phase":"notify-results","task":{"type":"notification","channel":{"type":"email","recipients":["admin@keboola.com","test@example.com"]},"recipients":["admin@keboola.com","test@example.com"],"title":"RetryTestingFlowCompleted","message":"Thecomprehensiveretrytestingflowhascompleted.Pleasechecktheresultsforallretryscenariosincluding:noretry(maxRetries:0),basicretry(3attempts),message-specificretry,andextremeretry(100attempts)configurations."}},{"id":"notify-webhook-results","name":"WebhookNotificationforResults","phase":"notify-results","task":{"type":"notification","channel":{"type":"webhook","recipients":["https://webhook.site/test-retry-results","https://api.example.com/flow-results"]},"recipients":["https://webhook.site/test-retry-results","https://api.example.com/flow-results"],"title":"RetryTestResultsAvailable","message":"Detailedretrytestingresultsarenowavailableforanalysis.Testtimestamp:${test_timestamp}.Maximumretriesconfigured:${max_retries_allowed}."}}]} \ No newline at end of file diff --git a/src/keboola_agent_cli/resources/flow/flow-schema.json b/src/keboola_agent_cli/resources/flow/flow-schema.json new file mode 100644 index 00000000..e01a2c73 --- /dev/null +++ b/src/keboola_agent_cli/resources/flow/flow-schema.json @@ -0,0 +1,134 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "phases": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "integer", + "string" + ], + "minLength": 1, + "description": "Unique identifier of the phase, can by anything just have to be unique within the configuration" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the phase, can be anything" + }, + "description": { + "type": "string", + "description": "Description of the phase, free form, markdown supported" + }, + "dependsOn": { + "type": "array", + "description": "List of phase ids that this phase depends on", + "items": { + "type": [ + "integer", + "string" + ], + "$ref": "#/properties/phases/items/properties/id" + } + } + }, + "required": [ + "id", + "name" + ] + } + }, + "tasks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": [ + "integer", + "string" + ], + "minLength": 1, + "description": "Unique identifier of the task" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "phase": { + "type": [ + "integer", + "string" + ], + "minLength": 1, + "description": "ID of the phase this task belongs to" + }, + "enabled": { + "type": "boolean", + "default": true + }, + "continueOnFailure": { + "type": "boolean", + "default": false + }, + "task": { + "type": "object", + "properties": { + "componentId": { + "type": "string", + "minLength": 1, + "description": "Component id of the task (e.g. keboola.db-ex-mysql) " + }, + "configId": { + "type": "string", + "description": "Configuration id of the task" + }, + "configData": { + "type": "object", + "description": "Configuration data can replace configuration and component ids, but is used seldomly" + }, + "mode": { + "type": "string", + "enum": [ + "run", + "debug" + ], + "default": "run" + }, + "configRowIds": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of configuration row ids" + }, + "tag": { + "type": "string" + }, + "previousJobId": { + "type": "string" + } + }, + "required": [ + "componentId" + ] + } + }, + "required": [ + "id", + "name", + "phase", + "task" + ] + } + } + }, + "required": [ + "phases", + "tasks" + ] +} diff --git a/src/keboola_agent_cli/resources/flow/legacy_flow_examples.jsonl b/src/keboola_agent_cli/resources/flow/legacy_flow_examples.jsonl new file mode 100644 index 00000000..fcb733ea --- /dev/null +++ b/src/keboola_agent_cli/resources/flow/legacy_flow_examples.jsonl @@ -0,0 +1,3 @@ +{"tasks":[{"id":1,"name":"keboola.wr-google-bigquery-v2-28356142","task":{"mode":"run","configId":"28356142","componentId":"keboola.wr-google-bigquery-v2"},"phase":1,"continueOnFailure":false,"enabled":true}],"phases":[{"id":1,"name":"Scheduledconfiguration","dependsOn":[]}]} +{"phases":[{"id":59812,"name":"Extraction","dependsOn":[],"description":"ExtractdatafromWhenIworkandPaychex\n"},{"id":39255,"name":"Step2","dependsOn":[59812]},{"id":10670,"name":"Step3","dependsOn":[39255]}],"tasks":[{"id":36614,"name":"ex-generic-v2-34446855","phase":59812,"task":{"componentId":"ex-generic-v2","configId":"34446855","mode":"run"},"continueOnFailure":false,"enabled":false},{"id":53747,"name":"ex-generic-v2-20064175","phase":59812,"task":{"componentId":"ex-generic-v2","configId":"20064175","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":85065,"name":"keboola.snowflake-transformation-39269975","phase":39255,"task":{"componentId":"keboola.snowflake-transformation","configId":"39269975","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":81023,"name":"ex-generic-v2-39118180","phase":59812,"task":{"componentId":"ex-generic-v2","configId":"39118180","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":15504,"name":"ex-generic-v2-39269678","phase":59812,"task":{"componentId":"ex-generic-v2","configId":"39269678","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":50729,"name":"ex-generic-v2-39269602","phase":59812,"task":{"componentId":"ex-generic-v2","configId":"39269602","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":74851,"name":"ex-generic-v2-39565883","phase":59812,"task":{"componentId":"ex-generic-v2","configId":"39565883","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":73592,"name":"keboola.wr-google-bigquery-v2-39569259","phase":10670,"task":{"componentId":"keboola.wr-google-bigquery-v2","configId":"39569259","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":98648,"name":"keboola.snowflake-transformation-43964834","phase":39255,"task":{"componentId":"keboola.snowflake-transformation","configId":"43964834","mode":"run"},"continueOnFailure":false,"enabled":false}]} +{"phases":[{"id":29854,"name":"PullfromApproachDB","dependsOn":[],"description":"ApproachisthegymmanagementsoftwareforASCEND.Wehavebeengrantedspecificaccesstoaread-replica,whichthispullsfrom."},{"id":11441,"name":"Step2","dependsOn":[29854],"behavior":{"onError":"stop"}},{"id":10722,"name":"Step3","dependsOn":[11441]},{"id":81889,"name":"Step4","dependsOn":[10722]},{"id":39795,"name":"Step5","dependsOn":[81889]},{"id":12446,"name":"Step6","dependsOn":[39795]}],"tasks":[{"id":57032,"name":"keboola.ex-db-mysql-25044458","phase":29854,"task":{"componentId":"keboola.ex-db-mysql","configId":"25044458","mode":"run"},"continueOnFailure":true,"enabled":true},{"id":19763,"name":"keboola.snowflake-transformation-31394325","phase":11441,"task":{"componentId":"keboola.snowflake-transformation","configId":"31394325","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":53948,"name":"keboola.snowflake-transformation-31381886","phase":11441,"task":{"componentId":"keboola.snowflake-transformation","configId":"31381886","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":76686,"name":"keboola.wr-google-bigquery-v2-28356065","phase":39795,"task":{"componentId":"keboola.wr-google-bigquery-v2","configId":"28356065","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":89554,"name":"keboola.snowflake-transformation-31453654","phase":81889,"task":{"componentId":"keboola.snowflake-transformation","configId":"31453654","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":20262,"name":"keboola.snowflake-transformation-32160311","phase":11441,"task":{"componentId":"keboola.snowflake-transformation","configId":"32160311","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":63341,"name":"keboola.wr-snowflake-blob-storage-37677743","phase":12446,"task":{"componentId":"keboola.wr-snowflake-blob-storage","configId":"37677743","mode":"run"},"continueOnFailure":false,"enabled":false},{"id":34828,"name":"keboola.snowflake-transformation-41756770","phase":10722,"task":{"componentId":"keboola.snowflake-transformation","configId":"41756770","mode":"run"},"continueOnFailure":false,"enabled":false},{"id":52790,"name":"keboola.snowflake-transformation-41835563","phase":10722,"task":{"componentId":"keboola.snowflake-transformation","configId":"41835563","mode":"run"},"continueOnFailure":false,"enabled":true},{"id":52449,"name":"keboola.snowflake-transformation-58438178","phase":11441,"task":{"componentId":"keboola.snowflake-transformation","configId":"58438178","mode":"run"},"continueOnFailure":false,"enabled":true}]} \ No newline at end of file diff --git a/src/keboola_agent_cli/services/flow_service.py b/src/keboola_agent_cli/services/flow_service.py index 17fe7589..c7bf97ed 100644 --- a/src/keboola_agent_cli/services/flow_service.py +++ b/src/keboola_agent_cli/services/flow_service.py @@ -20,6 +20,7 @@ import logging from collections.abc import Callable from dataclasses import dataclass +from importlib import resources as importlib_resources from typing import Any from ..ai_client import AiServiceClient @@ -36,6 +37,83 @@ LEGACY_FLOW_COMPONENT_ID = "keboola.orchestrator" SCHEDULER_COMPONENT_ID = "keboola.scheduler" +# --------------------------------------------------------------------------- +# Bundled resources: flow examples + JSON Schemas (issue #397) +# --------------------------------------------------------------------------- + +_FLOW_RESOURCES_PACKAGE = "keboola_agent_cli.resources.flow" + +_FLOW_EXAMPLE_FILES: dict[str, str] = { + FLOW_COMPONENT_ID: "conditional_flow_examples.jsonl", + LEGACY_FLOW_COMPONENT_ID: "legacy_flow_examples.jsonl", +} + +_BUNDLED_FLOW_SCHEMA_FILES: dict[str, str] = { + FLOW_COMPONENT_ID: "conditional-flow-schema.json", + LEGACY_FLOW_COMPONENT_ID: "flow-schema.json", +} + + +def _read_flow_resource(filename: str) -> str: + """Read a bundled flow resource file (works from wheel, sdist, or checkout).""" + return ( + importlib_resources.files(_FLOW_RESOURCES_PACKAGE) + .joinpath(filename) + .read_text(encoding="utf-8") + ) + + +def _known_flow_component_ids() -> str: + """Human-readable list of component ids with bundled resources.""" + return ", ".join(sorted(_FLOW_EXAMPLE_FILES)) + + +def get_flow_examples(component_id: str = FLOW_COMPONENT_ID) -> list[dict[str, Any]]: + """Return the bundled example flow configurations for ``component_id``. + + Examples are vendored verbatim from keboola-mcp-server (JSONL, one flow + configuration object per line). Supported ids: ``keboola.flow`` + (conditional) and ``keboola.orchestrator`` (legacy, informational only -- + kbagent cannot create or edit orchestrator flows since 0.57.0). + + Purely offline -- no project, token, or network access involved. + + :raises ValueError: if ``component_id`` has no bundled examples. + """ + filename = _FLOW_EXAMPLE_FILES.get(component_id) + if filename is None: + raise ValueError( + f"No bundled flow examples for component '{component_id}' " + f"(expected one of: {_known_flow_component_ids()})" + ) + examples: list[dict[str, Any]] = [] + for line in _read_flow_resource(filename).splitlines(): + stripped = line.strip() + if stripped: + examples.append(json.loads(stripped)) + return examples + + +def get_bundled_flow_schema(component_id: str = FLOW_COMPONENT_ID) -> dict[str, Any]: + """Return the bundled JSON Schema for ``component_id`` flow configurations. + + ``keboola.flow`` -> snapshot of the live conditional-flow schema (the same + document ``fetch_flow_schema`` retrieves from the stack; the bundled copy + is the offline fallback). ``keboola.orchestrator`` -> the frozen legacy + schema vendored from keboola-mcp-server. + + :raises ValueError: if ``component_id`` has no bundled schema. + """ + filename = _BUNDLED_FLOW_SCHEMA_FILES.get(component_id) + if filename is None: + raise ValueError( + f"No bundled flow schema for component '{component_id}' " + f"(expected one of: {_known_flow_component_ids()})" + ) + schema: dict[str, Any] = json.loads(_read_flow_resource(filename)) + return schema + + AiClientFactory = Callable[[str, str], AiServiceClient] SchedulerClientFactory = Callable[[str, str], SchedulerClient] diff --git a/tests/test_flow_cli.py b/tests/test_flow_cli.py index 407a852c..8fc16dca 100644 --- a/tests/test_flow_cli.py +++ b/tests/test_flow_cli.py @@ -272,11 +272,24 @@ def test_schema_json(self) -> None: data = json.loads(result.output) assert "phases" in data["data"]["schema"] - def test_schema_full_without_project_errors(self, tmp_path: Path) -> None: + def test_schema_full_without_project_serves_bundled_snapshot(self, tmp_path: Path) -> None: + # Since issue #397 the authoritative conditional-flow JSON Schema is + # bundled: --full without --project serves the offline snapshot + # (previously exit 2) and tags it with source=bundled. + store = _setup_config(tmp_path) + result = _invoke(store, MagicMock(), ["--json", "flow", "schema", "--full"]) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["data"]["format"] == "json-schema" + assert payload["data"]["source"] == "bundled" + assert "phases" in payload["data"]["schema"]["properties"] + + def test_schema_full_without_project_human_mentions_bundled(self, tmp_path: Path) -> None: store = _setup_config(tmp_path) result = _invoke(store, MagicMock(), ["flow", "schema", "--full"]) - assert result.exit_code == 2 - assert "--project" in result.output + assert result.exit_code == 0, result.output + assert "Bundled" in result.output + assert "--project" in result.output # hint how to get the live schema def test_schema_full_with_project_dumps_live_schema(self, tmp_path: Path) -> None: store = _setup_config(tmp_path, {"prod": {}}) diff --git a/tests/test_flow_examples.py b/tests/test_flow_examples.py new file mode 100644 index 00000000..3930919a --- /dev/null +++ b/tests/test_flow_examples.py @@ -0,0 +1,159 @@ +"""Tests for `kbagent flow examples` and the bundled flow resources (issue #397). + +Covers the L2 loaders (``get_flow_examples`` / ``get_bundled_flow_schema``), +the new ``flow examples`` CLI command, and the drift guard that pins the +hand-written ``flow schema`` YAML template to the authoritative bundled +conditional-flow JSON Schema. +""" + +from __future__ import annotations + +import json + +import jsonschema +import pytest +import yaml +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.commands.flow import _FLOW_SCHEMA +from keboola_agent_cli.services.flow_service import ( + FLOW_COMPONENT_ID, + LEGACY_FLOW_COMPONENT_ID, + get_bundled_flow_schema, + get_flow_examples, +) + +runner = CliRunner() + + +def _validator(schema: dict) -> jsonschema.protocols.Validator: + validator_cls = jsonschema.validators.validator_for(schema) + validator_cls.check_schema(schema) + return validator_cls(schema) + + +# --------------------------------------------------------------------------- +# L2: get_flow_examples +# --------------------------------------------------------------------------- + + +class TestGetFlowExamples: + def test_conditional_examples_parse(self) -> None: + examples = get_flow_examples(FLOW_COMPONENT_ID) + assert len(examples) >= 1 + for example in examples: + assert isinstance(example, dict) + assert isinstance(example.get("phases"), list) + assert isinstance(example.get("tasks"), list) + + def test_legacy_examples_parse(self) -> None: + examples = get_flow_examples(LEGACY_FLOW_COMPONENT_ID) + assert len(examples) >= 1 + for example in examples: + assert isinstance(example, dict) + assert isinstance(example.get("phases"), list) + assert isinstance(example.get("tasks"), list) + + def test_default_component_is_conditional(self) -> None: + assert get_flow_examples() == get_flow_examples(FLOW_COMPONENT_ID) + + def test_unknown_component_raises_value_error(self) -> None: + with pytest.raises(ValueError, match=r"keboola\.flow"): + get_flow_examples("keboola.does-not-exist") + + +# --------------------------------------------------------------------------- +# L2: get_bundled_flow_schema +# --------------------------------------------------------------------------- + + +class TestGetBundledFlowSchema: + def test_conditional_schema_is_valid_json_schema(self) -> None: + schema = get_bundled_flow_schema(FLOW_COMPONENT_ID) + _validator(schema) # check_schema raises on an invalid schema + assert "phases" in schema["properties"] + assert "tasks" in schema["properties"] + assert "retryConfiguration" in schema["definitions"] + + def test_legacy_schema_is_valid_json_schema(self) -> None: + schema = get_bundled_flow_schema(LEGACY_FLOW_COMPONENT_ID) + _validator(schema) + assert "tasks" in schema["properties"] + + def test_default_component_is_conditional(self) -> None: + assert get_bundled_flow_schema() == get_bundled_flow_schema(FLOW_COMPONENT_ID) + + def test_unknown_component_raises_value_error(self) -> None: + with pytest.raises(ValueError, match=r"keboola\.orchestrator"): + get_bundled_flow_schema("keboola.does-not-exist") + + def test_yaml_template_validates_against_bundled_schema(self) -> None: + # Drift guard (issue #397): the hand-written `flow schema` authoring + # template must satisfy the authoritative bundled schema. Before the + # fix it drifted: retryOn used bare strings (schema requires + # {type, value} objects) and strategyParams used delaySeconds + # (schema key is delay, plus maxRetries). + schema = get_bundled_flow_schema(FLOW_COMPONENT_ID) + template_doc = yaml.safe_load(_FLOW_SCHEMA) + errors = [e.message for e in _validator(schema).iter_errors(template_doc)] + assert errors == [] + + def test_legacy_examples_validate_against_legacy_schema(self) -> None: + schema = get_bundled_flow_schema(LEGACY_FLOW_COMPONENT_ID) + validator = _validator(schema) + for index, example in enumerate(get_flow_examples(LEGACY_FLOW_COMPONENT_ID)): + errors = [e.message for e in validator.iter_errors(example)] + assert errors == [], f"legacy example {index} does not match the bundled schema" + + +# --------------------------------------------------------------------------- +# CLI: kbagent flow examples +# --------------------------------------------------------------------------- + + +class TestFlowExamplesCommand: + def test_examples_default_json_is_list_of_conditional_examples(self) -> None: + result = runner.invoke(app, ["--json", "flow", "examples"]) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["status"] == "ok" + assert isinstance(payload["data"], list) + assert payload["data"] == get_flow_examples(FLOW_COMPONENT_ID) + + def test_examples_explicit_conditional_json(self) -> None: + result = runner.invoke( + app, ["--json", "flow", "examples", "--component-id", FLOW_COMPONENT_ID] + ) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["data"] == get_flow_examples(FLOW_COMPONENT_ID) + + def test_examples_orchestrator_json(self) -> None: + result = runner.invoke( + app, ["--json", "flow", "examples", "--component-id", LEGACY_FLOW_COMPONENT_ID] + ) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["data"] == get_flow_examples(LEGACY_FLOW_COMPONENT_ID) + + def test_examples_human_output_numbered_blocks(self) -> None: + result = runner.invoke(app, ["flow", "examples"]) + assert result.exit_code == 0, result.output + assert "keboola.flow" in result.output + assert "1. Flow Configuration:" in result.output + # No legacy note on the conditional (default) path. + assert "cannot create or edit" not in result.output + + def test_examples_orchestrator_prints_informational_note(self) -> None: + result = runner.invoke( + app, ["flow", "examples", "--component-id", LEGACY_FLOW_COMPONENT_ID] + ) + assert result.exit_code == 0, result.output + assert "cannot create or edit" in result.output + assert "0.57.0" in result.output + + def test_examples_unknown_component_exits_2(self) -> None: + result = runner.invoke(app, ["flow", "examples", "--component-id", "keboola.wrong"]) + assert result.exit_code == 2 + assert "keboola.wrong" in result.output From ab1c0f2d991de376d3fdcef724ba19123d1e9061 Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:49:41 +0200 Subject: [PATCH 12/17] chore(release): 0.73.0 version bump + doc-sync surfaces (flow examples docs, expert prompt 0.73.0 block, SKILL triggers) --- .claude-plugin/marketplace.json | 2 +- CLAUDE.md | 11 ++++++++--- plugins/kbagent/.claude-plugin/plugin.json | 2 +- plugins/kbagent/agents/keboola-expert.md | 14 ++++++++++++-- plugins/kbagent/skills/kbagent/SKILL.md | 3 ++- .../kbagent/references/commands-reference.md | 2 ++ pyproject.toml | 2 +- src/keboola_agent_cli/commands/context.py | 16 ++++++++++++---- uv.lock | 2 +- 9 files changed, 40 insertions(+), 14 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index afca5695..7292f2f7 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "kbagent", - "version": "0.72.0", + "version": "0.73.0", "source": "./plugins/kbagent", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "category": "development" diff --git a/CLAUDE.md b/CLAUDE.md index 27b44924..3bfea61d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -605,7 +605,8 @@ kbagent docs query "QUESTION" [--project NAME] kbagent flow list [--project NAME] [--branch ID] [--with-schedules] kbagent flow detail --project NAME --flow-id ID [--branch ID] -kbagent flow schema [--full --project NAME] +kbagent flow schema [--full [--project NAME]] +kbagent flow examples [--component-id keboola.flow|keboola.orchestrator] kbagent flow validate --file @flow.yaml|- [--project NAME] kbagent flow new --project NAME --name NAME [--description D] [--file @path.yaml|-|JSON] [--branch ID] kbagent flow update --project NAME --flow-id ID [--name N] [--description D] [--file @path.yaml|-|JSON] [--branch ID] @@ -619,8 +620,12 @@ kbagent flow schedule-remove --project NAME --flow-id ID [--branch ID] [--yes] # Schema-fetch failure (network/empty) does NOT block the write: structural check skipped, # semantic checks still run, a "structural schema validation skipped" warning is surfaced. # flow validate: with --project fetches the live schema (full validation; fetch failure -> -# semantic-only + note); without --project runs semantic-only + a note. flow schema --full -# requires --project (fetches live schema); plain flow schema is the offline YAML template. +# semantic-only + note); without --project runs semantic-only + a note. flow schema --full: +# with --project fetches the live schema (source=live); without --project serves the bundled +# authoritative snapshot (source=bundled, 0.73.0+). Plain flow schema is the offline YAML template. +# flow examples (0.73.0+): bundled example flow configs (vendored from keboola-mcp-server), offline. +# Default keboola.flow; keboola.orchestrator serves legacy examples informational-only (kbagent +# cannot create/edit orchestrator flows). --json emits the bare list of configs. # flow schedule (0.66.1+) also activates the config on the Scheduler Service so the cron fires; # activation failure keeps the config written, sets activated=false + warning, exit stays 0. # flow schedule-remove deregisters from the service before deleting each config. diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index dba4cc21..d284a3ad 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.72.0", + "version": "0.73.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index f1732efe..c991440d 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -85,8 +85,8 @@ a critical failure. |---|---|---|---| | Author / edit a conditional flow (keboola.flow) | `kbagent flow validate --file @flow.yaml --project ALIAS` (fetches live schema; loop until clean) then `kbagent flow new`/`flow update --file` | fetch `flow detail`, merge phases/tasks locally, re-validate, push | `--component-id` (removed 0.57.0); integer ids (ids are STRINGS); `dependsOn` (use `next[].goto` + conditions); `keboola.orchestrator` (dropped 0.57.0); assuming `flow schema --full` works offline (now needs `--project`) | | Schedule flow | `kbagent flow schedule --cron ... [--timezone]` | `tool call create_flow_schedule` | raw REST to `/storage/configurations/keboola.scheduler` | -| Create Snowflake transformation | `kbagent config new --component-id keboola.snowflake-transformation --name N --project P --push --no-files` (0.33.0+; one-shot, no scaffold, body defaults to `{}` and validation auto-skips for empty shell -- then `config update --set ...` to fill in script) **or** `kbagent config new --component-id keboola.snowflake-transformation --project P --output-dir D` + `config update --set ...` (scaffold-then-patch) | `tool call create_sql_transformation` (lower schema, avoids the MCP `create_config` Snowflake refusal) | `tool call create_config` (refuses keboola.snowflake-transformation) -- note: `config new --push` does NOT inherit this refusal because it wraps the raw Storage API directly | -| Update SQL transformation body (script[]) | `kbagent config update --project P --component-id keboola.snowflake-transformation --config-id K --configuration @body.json` (0.28.0+ auto-normalizes string `script` to array; SQL gets statement-level split, Python/R gets `[script]` wrap; envelope's `normalizations: [...]` records every change. 0.31.0+ also re-splits multi-statement LIST elements -- closes the #274 ODBC `statement count 2 vs desired 1` crash that survives the 0.28.0 string fix) | -- | `tool call update_sql_transformation` -- still vulnerable to BOTH the #245 string-vs-array AND #274 list-element runtime crashes because it pushes raw to Storage API; raw `PUT /v2/storage/components/.../configs/...` -- same trap | +| Create SQL transformation | `kbagent transformation create --project P --name N (--sql '...' \| --sql-file F) [--created-table T ...]` (0.73.0+; dialect from project default_backend, statement-split, output mapping derived from name) | `kbagent config new --component-id keboola.snowflake-transformation --name N --project P --push --no-files` then `config update --set ...` (< 0.73.0) | `tool call create_config` (refuses keboola.snowflake-transformation) | +| Edit SQL transformation blocks/codes | `kbagent transformation show` (fresh ids!) then `kbagent transformation edit --config-id K --change-description T --op '{"op":"set_code",...}'` (0.73.0+; 9 ops, batch-start ids b{i}/b{i}.c{j}; --storage REPLACES wholesale) | `kbagent config update --configuration @body.json` (0.28.0+ auto-normalizes string `script` to array; 0.31.0+ re-splits multi-statement LIST elements -- #274) | `tool call update_sql_transformation` -- vulnerable to the #245/#274 runtime crashes (raw push); raw `PUT /v2/storage/...` -- same trap; `transformation edit` without a fresh `show` (positional ids renumber) | | Run a job (and wait) | `kbagent job run --project P --component-id C --config-id K --wait` | `tool call run_component` | `job run` without `--wait` when user expects the result | | Provision / read an OTLP Data Streams endpoint | `kbagent stream create-source -p P --name N --type otlp [--if-not-exists]` (auto-creates logs/metrics/traces sinks) then `stream detail N -p P --reveal` for endpoint+secret (0.50.0+) | `stream list`; `--no-sinks` for a bare source | deriving the `stream-in` URL yourself (use `source.otlp.url`); printing the secret unasked (masked by default) | | Mint / rotate / revoke a scoped Storage token (e.g. a device-enrollment token) | `kbagent token create -p P -d DESC [--bucket-write B ...] [--expires-in N]` / `token refresh --token-id ID` / `token delete --token-id ID` (0.66.0+) -- acting token needs `canManageTokens`; secret shown ONCE (persist only `id`+`expires`). Same ops on the SDK facade: `Client.create_scoped_token / refresh_token / delete_token` (+ `create_stream_source`) | -- | assuming a token upload needs `--component-access`/`--can-read-all-file-uploads` (uploads need `--bucket-write` on the sink bucket; those flags gate READING others' uploads, not uploading); telling the user `stream create-source` needs a master token (it uses the normal Storage token) | @@ -227,6 +227,16 @@ read it when a trigger fires. Each `(X.Y.Z+)` tag is the version floor. dirs; `is_disabled: true` in `_config.yml` = config disabled (absent = enabled); a `never_fetched` warning on diff/push = run `sync pull` first. `sync status` is local-only -- audit real drift with `sync diff`. +- **MCP passthrough is deprecating; firewall is fail-closed** (0.73.0+): + prefer native parity commands over `tool call` -- `docs query`, + `config examples`, `semantic-layer schema`, `component sync-action`, + `transformation create|show|edit`, `flow examples`; `workspace query` + replaces `query_data`. Unknown MCP tool names classify DESTRUCTIVE + (blocked by --deny-writes AND --deny-destructive, never multi-project); + `run_job`/`run_sync_action`/`modify_*`/`deploy_*` are writes now. + **VERSION GATE**: < 0.73.0 these commands do not exist and the firewall + fails OPEN for unknown tools. `component sync-action --row-id` merges + SHALLOW (row top-level keys replace root wholesale). - **Native types** (0.25.0+): `--column amount:NUMBER(18,2)` passes through; `BOOLEAN` defaults must be lowercase; `INTEGER(10)` is invalid (use `NUMBER(3,0)`); `--not-null` / `--default` must name a defined `--column`. diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 1d73b1dc..69095fe2 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -14,7 +14,8 @@ description: > keboola lineage, keboola sync, gitops, dev branch, workspace SQL, data app, streamlit deploy, semantic layer, sl, dev-portal, data stream, OTLP, scoped token, bucket sharing, encrypt secrets, feature flag, flow schedule, - invite member. + invite member, SQL transformation editing, sync action, testConnection, + keboola docs, config examples, flow examples. --- # kbagent -- Keboola Agent CLI diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index c362e7fb..fee77418 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -226,6 +226,8 @@ Requires the project to be added with its **master ('owner') Storage API token** - `kai history [--project NAME] [--limit N]` -- list recent Kai chat sessions (default limit: 10) ## Flows (Conditional Flows -- keboola.flow only) +- `flow examples [--component-id keboola.flow|keboola.orchestrator]` (since 0.73.0) -- bundled example flow configs (vendored from keboola-mcp-server), fully offline; default `keboola.flow`; `keboola.orchestrator` examples are informational-only (kbagent cannot create/edit orchestrator flows). Ports the `get_flow_examples` MCP tool. +- `flow schema --full` without `--project` (since 0.73.0) serves the bundled authoritative snapshot (`source: "bundled"`; previously exit 2); with `--project` fetches the live schema (`source: "live"`). > Since 0.57.0 the `flow` group targets `keboola.flow` (Conditional Flows) ONLY; `keboola.orchestrator` is dropped and `--component-id` is removed from every subcommand. IDs are **strings**; phases use `next[].goto` (a phase id or `null`) + optional `condition`; tasks are typed (`job`/`notification`/`variable`). The old `dependsOn` template is invalid. Execute a flow with `kbagent job run --component-id keboola.flow --config-id ID`. See `flow-workflow.md`. - `flow list [--project NAME] [--branch ID] [--with-schedules]` -- list conditional flows (keboola.flow) across one or all projects. Legacy keboola.orchestrator configs are NOT listed; their total appears as `legacy_orchestrator_count` (+ a warning). `--with-schedules` enriches each row with `schedules: [{schedule_id, cron, timezone, enabled}, ...]` via one extra keboola.scheduler list call per project (not per flow) - `flow detail --project NAME --flow-id ID [--branch ID]` -- full phase/task breakdown; per-phase transitions (`→ goto [condition | default]`), typed-task badges, retry info; JSON is the raw body unchanged diff --git a/pyproject.toml b/pyproject.toml index 22226920..06d32649 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-cli" -version = "0.72.0" +version = "0.73.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 1d091381..f6b4ffe4 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -631,10 +631,18 @@ kbagent flow detail --project NAME --flow-id ID [--branch ID] Show phases, transitions (next[].goto + conditions), typed tasks, and full configuration. - kbagent flow schema [--full --project NAME] - Plain: print the offline conditional-flow YAML template. --full fetches and dumps the - live JSON Schema from the stack (AI Service configurationSchema for keboola.flow) and - REQUIRES --project (the schema is no longer bundled). + kbagent flow schema [--full [--project NAME]] + Plain: print the offline conditional-flow YAML template. --full with --project + fetches the live JSON Schema from the stack (source=live); --full WITHOUT + --project serves the bundled authoritative snapshot (source=bundled, + since 0.73.0 -- previously an error). + + kbagent flow examples [--component-id keboola.flow|keboola.orchestrator] + (since 0.73.0) Bundled example flow configurations (vendored from + keboola-mcp-server), fully offline. Default keboola.flow (conditional); + keboola.orchestrator serves legacy examples with an informational-only + warning (kbagent cannot create or edit orchestrator flows). --json emits + the bare list of example configs. kbagent flow validate --file YAML|@file|- [--project NAME] With --project: fetch the live schema from the stack -> full structural + semantic diff --git a/uv.lock b/uv.lock index 62264b81..f23a9af6 100644 --- a/uv.lock +++ b/uv.lock @@ -590,7 +590,7 @@ wheels = [ [[package]] name = "keboola-cli" -version = "0.72.0" +version = "0.73.0" source = { editable = "." } dependencies = [ { name = "croniter" }, From 7922f5dbcbc6cdd6800cc5a9bd548c7df39d662b Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:54:23 +0200 Subject: [PATCH 13/17] fix(checks): ty narrowing in tf-ops test, SemanticType in schema helper, SKILL description under 1024 chars --- plugins/kbagent/skills/kbagent/SKILL.md | 3 +-- src/keboola_agent_cli/services/semantic_layer_service.py | 2 +- tests/test_transformation_ops.py | 4 +++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 69095fe2..c89efc19 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -14,8 +14,7 @@ description: > keboola lineage, keboola sync, gitops, dev branch, workspace SQL, data app, streamlit deploy, semantic layer, sl, dev-portal, data stream, OTLP, scoped token, bucket sharing, encrypt secrets, feature flag, flow schedule, - invite member, SQL transformation editing, sync action, testConnection, - keboola docs, config examples, flow examples. + invite member, SQL transformation edit, sync action, keboola docs. --- # kbagent -- Keboola Agent CLI diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index f2585581..e59ec5f4 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -390,7 +390,7 @@ def get_schema(self, alias: str, types: list[str]) -> dict[str, Any]: } @staticmethod - def _fetch_resolved_schema(client: MetastoreClient, wire_type: str) -> dict[str, Any]: + def _fetch_resolved_schema(client: MetastoreClient, wire_type: SemanticType) -> dict[str, Any]: """Fetch the actual JSON Schema for a type, resolving the default version. Live metastore behavior (2026-07): the bare ``/api/v1/schema/{type}`` diff --git a/tests/test_transformation_ops.py b/tests/test_transformation_ops.py index 3b4cadcb..43343677 100644 --- a/tests/test_transformation_ops.py +++ b/tests/test_transformation_ops.py @@ -50,7 +50,9 @@ def test_parse_valid_ops(self) -> None: ] ) assert [o.op for o in ops] == ["add_block", "remove_block", "str_replace"] - assert ops[0].position == "end" # default + first = ops[0] + assert isinstance(first, tf_ops.TfAddBlock) + assert first.position == "end" # default def test_parse_unknown_op(self) -> None: with pytest.raises(ValueError, match="Invalid operation"): From 5388346a135728bf8c8145200471a0373920fd63 Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Jul 2026 23:59:34 +0200 Subject: [PATCH 14/17] test(e2e): live coverage for all six MCP parity commands (phase 38.5) --- tests/test_e2e.py | 134 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 1dd18c6a..b71bfe39 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -620,6 +620,17 @@ def test_full_cli_e2e(self) -> None: _step(38, "kai ping / ask / history", "Keboola AI Assistant") self._test_kai_commands() + # ============================================================== + # PHASE 12.6: MCP parity commands (epic #390, 0.73.0) + # ============================================================== + + _step( + 38.5, + "docs/examples/schema/sync-action/transformation/flow examples", + "native ports of the keboola-mcp-server tools", + ) + self._test_mcp_parity_commands() + # ============================================================== # PHASE 13: Job commands (expanded) # ============================================================== @@ -2939,6 +2950,129 @@ def _test_kai_commands(self) -> None: # We just chatted, so there should be at least 1 assert len(data["data"]["chats"]) >= 1 + def _test_mcp_parity_commands(self) -> None: + """MCP parity commands from epic #390 (0.73.0): docs query, config + examples, semantic-layer schema, component sync-action, transformation + lifecycle, flow examples.""" + # docs query — server-side documentation RAG (AI Service) + result = self._run( + "docs", "query", "What is a Keboola Storage bucket?", "--project", self.alias + ) + if result.exit_code != 0: + print(f" {_YELLOW}SKIP: docs query failed (AI Service unavailable?){_RESET}") + else: + data = _json_ok(result) + assert isinstance(data["data"]["text"], str) and data["data"]["text"].strip() + assert isinstance(data["data"]["source_urls"], list) + + # config examples — reformat of AI-service component detail + data = self._run_ok( + "config", + "examples", + "--component-id", + "keboola.ex-google-drive", + "--project", + self.alias, + ) + assert data["data"]["component_id"] == "keboola.ex-google-drive" + assert isinstance(data["data"]["root_examples"], list) + + # semantic-layer schema — live metastore JSON Schema (version-resolved) + result = self._run("semantic-layer", "schema", "--project", self.alias, "--type", "metric") + if result.exit_code != 0: + print(f" {_YELLOW}SKIP: semantic-layer schema (metastore unavailable?){_RESET}") + else: + data = _json_ok(result) + schemas = data["data"]["schemas"] + assert [s["type"] for s in schemas] == ["metric"] + assert isinstance(schemas[0]["schema"], dict) and schemas[0]["schema"] + + # component sync-action — full round-trip to sync-actions.{stack}; + # deliberately bad config: a structured API error PROVES the wiring + # (URL derivation, auth, camelCase body); a 2xx needs live DB creds. + result = self._run( + "component", + "sync-action", + "testConnection", + "--component-id", + "keboola.ex-db-snowflake", + "--project", + self.alias, + "--config-data", + '{"parameters": {"db": {"host": "invalid.example.com"}}}', + ) + assert result.exit_code != 0 + err = _json(result) + assert err["error"]["code"] in ("API_ERROR", "VALIDATION_ERROR") + + # flow examples — bundled, offline + data = self._run_ok("flow", "examples") + assert isinstance(data["data"], list) and data["data"] + assert {"phases", "tasks"} <= set(data["data"][0]) + + # flow schema --full without --project — bundled snapshot fallback + data = self._run_ok("flow", "schema", "--full") + assert data["data"]["source"] == "bundled" + + # transformation lifecycle: create -> show (ids) -> edit -> verify. + # Cleanup: sync-based delete is heavyweight here; the config is + # removed via the recycle-bin-safe Storage API through self.api. + created = self._run_ok( + "transformation", + "create", + "--project", + self.alias, + "--name", + "E2E Parity Transformation", + "--sql", + 'CREATE TABLE "e2e_tf_out" AS SELECT 1 AS "id"; SELECT 2;', + "--created-table", + "e2e_tf_out", + ) + tf_config_id = created["data"]["config_id"] + tf_component_id = created["data"]["component_id"] + try: + shown = self._run_ok( + "transformation", + "show", + "--project", + self.alias, + "--config-id", + tf_config_id, + ) + block = shown["data"]["blocks"][0] + assert block["id"] == "b0" + assert block["codes"][0]["id"] == "b0.c0" + assert len(block["codes"][0]["script"]) == 2 + + self._run_ok( + "transformation", + "edit", + "--project", + self.alias, + "--config-id", + tf_config_id, + "--change-description", + "e2e parity check", + "--op", + '{"op": "str_replace", "search_for": "SELECT 2", "replace_with": "SELECT 3"}', + ) + reshown = self._run_ok( + "transformation", + "show", + "--project", + self.alias, + "--config-id", + tf_config_id, + ) + assert reshown["data"]["blocks"][0]["codes"][0]["script"][1] == "SELECT 3;" + finally: + try: + self.api.delete_config(tf_component_id, tf_config_id) + print(f" Deleted transformation config {tf_config_id}") + except Exception as exc: + print(f" WARN: failed to delete transformation {tf_config_id}: {exc}") + def _test_job_commands(self) -> None: """Verify job listing structure and detail (if jobs exist).""" # job list From acf184343243efb2e4d8b911653658488d91660d Mon Sep 17 00:00:00 2001 From: Petr Date: Tue, 21 Jul 2026 00:08:06 +0200 Subject: [PATCH 15/17] docs(plugin): merge stale flow schema bullet with 0.73.0 bundled-fallback reality (Devin finding 1) --- .../kbagent/skills/kbagent/references/commands-reference.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index fee77418..a6f67bab 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -227,11 +227,10 @@ Requires the project to be added with its **master ('owner') Storage API token** ## Flows (Conditional Flows -- keboola.flow only) - `flow examples [--component-id keboola.flow|keboola.orchestrator]` (since 0.73.0) -- bundled example flow configs (vendored from keboola-mcp-server), fully offline; default `keboola.flow`; `keboola.orchestrator` examples are informational-only (kbagent cannot create/edit orchestrator flows). Ports the `get_flow_examples` MCP tool. -- `flow schema --full` without `--project` (since 0.73.0) serves the bundled authoritative snapshot (`source: "bundled"`; previously exit 2); with `--project` fetches the live schema (`source: "live"`). > Since 0.57.0 the `flow` group targets `keboola.flow` (Conditional Flows) ONLY; `keboola.orchestrator` is dropped and `--component-id` is removed from every subcommand. IDs are **strings**; phases use `next[].goto` (a phase id or `null`) + optional `condition`; tasks are typed (`job`/`notification`/`variable`). The old `dependsOn` template is invalid. Execute a flow with `kbagent job run --component-id keboola.flow --config-id ID`. See `flow-workflow.md`. - `flow list [--project NAME] [--branch ID] [--with-schedules]` -- list conditional flows (keboola.flow) across one or all projects. Legacy keboola.orchestrator configs are NOT listed; their total appears as `legacy_orchestrator_count` (+ a warning). `--with-schedules` enriches each row with `schedules: [{schedule_id, cron, timezone, enabled}, ...]` via one extra keboola.scheduler list call per project (not per flow) - `flow detail --project NAME --flow-id ID [--branch ID]` -- full phase/task breakdown; per-phase transitions (`→ goto [condition | default]`), typed-task badges, retry info; JSON is the raw body unchanged -- `flow schema [--full --project NAME]` -- plain form prints the offline conditional-flow YAML template (string ids, `next[].goto`, typed tasks). `--full` fetches and dumps the **live** JSON Schema from the stack (AI Service `configurationSchema` for `keboola.flow`) and **requires `--project`** -- the schema is no longer bundled +- `flow schema [--full [--project NAME]]` -- plain form prints the offline conditional-flow YAML template (string ids, `next[].goto`, typed tasks). `--full` with `--project` fetches and dumps the **live** JSON Schema from the stack (AI Service `configurationSchema` for `keboola.flow`, `source: "live"`); `--full` without `--project` serves the bundled authoritative snapshot (`source: "bundled"`, since 0.73.0 -- previously exit 2) - `flow validate --file @path.yaml|- [--project NAME]` -- validate a definition. With `--project`: fetch the live schema from the stack for full structural + semantic validation (a fetch failure degrades to semantic-only + a note). Without `--project`: semantic-only validation + a note that structural validation was skipped (no schema source). Exit 0 valid (warnings still printed), exit 2 on errors; `--json` lists `{valid, errors, warnings, notes}` - `flow new --project NAME --name NAME [--description D] [--file @path.yaml|-|JSON] [--branch ID]` -- create a conditional flow; validated against the **live** CF schema fetched from the stack before the API call (`INVALID_FLOW_DEFINITION` on failure). A schema-fetch failure does NOT block the write: structural check skipped, semantic checks still run, a `structural schema validation skipped` warning is surfaced - `flow update --project NAME --flow-id ID [--name N] [--description D] [--file @path.yaml|-|JSON] [--branch ID]` -- update name, description, or phases/tasks; `--file` is a full-replace of phases+tasks; merge-aware validation against the live CF schema (same graceful semantic-only degradation on fetch failure); requires at least one of --name/--description/--file From 6d2f71dc65a03bef6f92d055528b61927544019e Mon Sep 17 00:00:00 2001 From: Petr Date: Tue, 21 Jul 2026 00:21:59 +0200 Subject: [PATCH 16/17] feat(serve): routes for all six parity commands (Devin finding 2, CONTRIBUTING 1:1 convention) POST /documentation/query, GET /configs/examples/{component_id}, POST /components/{id}/actions/{action}, GET /semantic-layer/schema, POST/GET/PATCH /transformations, GET /flows/examples. DocsService + TransformationService registered in the ServiceRegistry. Router is /documentation (NOT /docs): BearerAuthMiddleware exempts the /docs Swagger namespace, a /docs router would ship unauthenticated -- locked by test_docs_query_requires_bearer_auth. --- src/keboola_agent_cli/server/app.py | 28 +- src/keboola_agent_cli/server/dependencies.py | 6 + .../server/routers/components.py | 41 +- .../server/routers/configs.py | 16 + src/keboola_agent_cli/server/routers/docs.py | 34 ++ src/keboola_agent_cli/server/routers/flows.py | 27 +- .../server/routers/semantic_layer.py | 23 + .../server/routers/transformation.py | 115 +++++ tests/test_server_router_calls.py | 483 ++++++++++++++++++ 9 files changed, 767 insertions(+), 6 deletions(-) create mode 100644 src/keboola_agent_cli/server/routers/docs.py create mode 100644 src/keboola_agent_cli/server/routers/transformation.py diff --git a/src/keboola_agent_cli/server/app.py b/src/keboola_agent_cli/server/app.py index 05ebde66..ec40d86c 100644 --- a/src/keboola_agent_cli/server/app.py +++ b/src/keboola_agent_cli/server/app.py @@ -39,6 +39,7 @@ configs, data_apps, dev_portal, + docs, encrypt, feature, flows, @@ -57,6 +58,7 @@ storage, stream, token, + transformation, workspaces, ) @@ -134,6 +136,15 @@ "Mirrors `kbagent component list|detail`." ), }, + { + "name": "transformations", + "description": ( + "**Configurations.** " + "SQL transformations -- create from a SQL script, inspect the " + "block/code tree, and apply positional edit operations. " + "Mirrors `kbagent transformation create|show|edit`." + ), + }, { "name": "encrypt", "description": ( @@ -291,6 +302,17 @@ "Mirrors `kbagent kai *`." ), }, + { + "name": "documentation", + "description": ( + "**AI & Tools.** " + "Ask the official Keboola documentation natural-language " + "questions (AI Service docs Q&A). Served under " + "`/documentation` -- NOT `/docs`, which is the auth-exempt " + "Swagger UI namespace. " + "Mirrors `kbagent docs query`." + ), + }, { "name": "ai-chat", "description": ( @@ -345,11 +367,11 @@ its command tree: - **Project Management** -- projects, members, org, feature flags -- **Configurations** -- configs, components, encrypt +- **Configurations** -- configs, components, transformations, encrypt - **Data** -- storage, search, sharing - **Execution** -- jobs, flows, schedules, data-apps, workspaces - **Development** -- branches, lineage, semantic-layer -- **AI & Tools** -- mcp, kai, ai-chat, agents +- **AI & Tools** -- mcp, kai, documentation, ai-chat, agents - **System** -- health Most endpoints accept a `project` alias either in the body or as a @@ -649,6 +671,8 @@ async def _generic_handler(_request, exc: Exception): app.include_router(encrypt.router) app.include_router(search.router) app.include_router(semantic_layer.router) + app.include_router(transformation.router) + app.include_router(docs.router) app.include_router(org.router) app.include_router(agents.router) diff --git a/src/keboola_agent_cli/server/dependencies.py b/src/keboola_agent_cli/server/dependencies.py index fcf580d8..eb31a582 100644 --- a/src/keboola_agent_cli/server/dependencies.py +++ b/src/keboola_agent_cli/server/dependencies.py @@ -22,6 +22,7 @@ from ..services.data_app_service import DataAppService from ..services.deep_lineage_service import DeepLineageService from ..services.dev_portal_service import DeveloperPortalService +from ..services.docs_service import DocsService from ..services.doctor_service import DoctorService from ..services.encrypt_service import EncryptService from ..services.feature_service import FeatureService @@ -42,6 +43,7 @@ from ..services.stream_service import StreamService from ..services.sync_service import SyncService from ..services.token_service import TokenService +from ..services.transformation_service import TransformationService from ..services.variables_service import VariablesService from ..services.version_service import VersionService from ..services.workspace_service import WorkspaceService @@ -92,6 +94,8 @@ class ServiceRegistry: doctor: DoctorService = field(init=False) version: VersionService = field(init=False) token: TokenService = field(init=False) + docs: DocsService = field(init=False) + transformation: TransformationService = field(init=False) def __post_init__(self) -> None: cs = self.config_store @@ -131,6 +135,8 @@ def __post_init__(self) -> None: self.doctor = DoctorService(config_store=cs, mcp_service=self.mcp) self.version = VersionService() self.token = TokenService(config_store=cs) + self.docs = DocsService(config_store=cs) + self.transformation = TransformationService(config_store=cs) def install_registry(app: FastAPI, registry: ServiceRegistry) -> None: diff --git a/src/keboola_agent_cli/server/routers/components.py b/src/keboola_agent_cli/server/routers/components.py index 70f72239..50c6a937 100644 --- a/src/keboola_agent_cli/server/routers/components.py +++ b/src/keboola_agent_cli/server/routers/components.py @@ -1,16 +1,26 @@ -"""Component discovery (list/detail/scaffold).""" +"""Component discovery (list/detail/scaffold) and synchronous actions.""" from __future__ import annotations from typing import Any from fastapi import APIRouter, Depends +from pydantic import BaseModel from ..dependencies import ServiceRegistry, get_registry router = APIRouter(prefix="/components", tags=["components"]) +class SyncActionRequest(BaseModel): + project: str | None = None + config_id: str | None = None + row_id: str | None = None + branch_id: int | None = None + config_data: dict[str, Any] | None = None + timeout: float | None = None + + @router.get("", summary="List components") def list_components( project: str | None = None, @@ -46,3 +56,32 @@ def scaffold( if project is None: project, _ = registry.project.resolve_pinned_alias(None) return registry.component.generate_scaffold(alias=project, component_id=component_id, name=name) + + +@router.post("/{component_id}/actions/{action}", summary="Run a synchronous component action") +def sync_action( + component_id: str, + action: str, + body: SyncActionRequest, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Run a synchronous component action (e.g. testConnection, getTables). + + Mirrors `kbagent component sync-action`. Either ``config_id`` (stored + configuration, optionally shallow-merged with ``row_id``) or an explicit + ``config_data`` payload is required -- the service enforces this and a + violation surfaces as ConfigError (HTTP 400). + """ + project = body.project + if project is None: + project, _ = registry.project.resolve_pinned_alias(None) + return registry.component.run_sync_action( + alias=project, + component_id=component_id, + action=action, + config_id=body.config_id, + row_id=body.row_id, + branch_id=body.branch_id, + config_data_override=body.config_data, + timeout=body.timeout, + ) diff --git a/src/keboola_agent_cli/server/routers/configs.py b/src/keboola_agent_cli/server/routers/configs.py index 0c5b57f0..dbad15c1 100644 --- a/src/keboola_agent_cli/server/routers/configs.py +++ b/src/keboola_agent_cli/server/routers/configs.py @@ -105,6 +105,22 @@ def search_configs( ) +@router.get("/examples/{component_id}", summary="Get configuration examples for a component") +def config_examples( + component_id: str, + project: str | None = None, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Fetch root and row configuration example bodies for a component. + + Mirrors `kbagent config examples`. The method lives on ComponentService + (the AI Service component detail carries the example bodies); ``project`` + only selects which stack URL + token to use -- omitted means the first + configured project. + """ + return registry.component.get_config_examples(alias=project, component_id=component_id) + + @router.get("/{project}/{component_id}/{config_id}", summary="Get configuration detail") def config_detail( project: str, diff --git a/src/keboola_agent_cli/server/routers/docs.py b/src/keboola_agent_cli/server/routers/docs.py new file mode 100644 index 00000000..838cf1c4 --- /dev/null +++ b/src/keboola_agent_cli/server/routers/docs.py @@ -0,0 +1,34 @@ +"""Documentation Q&A endpoints (Keboola docs natural-language questions). + +Route prefix is ``/documentation`` -- deliberately NOT ``/docs``: the bearer +auth middleware exempts every path starting with ``/docs`` (the Swagger UI +surface, see ``server/auth.py``), so a ``/docs``-prefixed router would ship +its endpoints unauthenticated. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends +from pydantic import BaseModel + +from ..dependencies import ServiceRegistry, get_registry + +router = APIRouter(prefix="/documentation", tags=["documentation"]) + + +class DocsQuery(BaseModel): + query: str + project: str | None = None + + +@router.post("/query", summary="Ask the Keboola documentation a question") +def query(body: DocsQuery, registry: ServiceRegistry = Depends(get_registry)) -> dict[str, Any]: + """Natural-language question answered from the official Keboola docs. + + Mirrors `kbagent docs query`. ``project`` selects which project's stack + URL + token reach the AI Service; omitted means the first configured + project (the answer itself is project-independent). + """ + return registry.docs.ask_docs(alias=body.project, query=body.query) diff --git a/src/keboola_agent_cli/server/routers/flows.py b/src/keboola_agent_cli/server/routers/flows.py index a9129cb5..e5018b26 100644 --- a/src/keboola_agent_cli/server/routers/flows.py +++ b/src/keboola_agent_cli/server/routers/flows.py @@ -7,6 +7,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel +from ...services.flow_service import FLOW_COMPONENT_ID, get_flow_examples from ...services.flow_validation import find_unreachable_phases, validate_conditional_flow from ..dependencies import ServiceRegistry, get_registry @@ -43,9 +44,10 @@ class FlowValidate(BaseModel): project: str | None = None -# NOTE: /validate and /{project}/schema are declared BEFORE the /{project} -# and /{project}/{config_id} routes -- FastAPI matches in declaration order, -# so the literal segments must win over the path parameters. +# NOTE: /validate, /examples, and /{project}/schema are declared BEFORE the +# /{project} and /{project}/{config_id} routes -- FastAPI matches in +# declaration order, so the literal segments must win over the path +# parameters. @router.post("/validate", summary="Validate a conditional-flow definition") @@ -78,6 +80,25 @@ def validate( return {"valid": not errors, "errors": errors, "warnings": warnings, "notes": notes} +@router.get("/examples", summary="Show bundled example flow configurations") +def examples(component_id: str = FLOW_COMPONENT_ID) -> dict[str, Any]: + """Bundled example flow configurations (offline, no project needed). + + Mirrors `kbagent flow examples`. Supported component ids: ``keboola.flow`` + (conditional, default) and ``keboola.orchestrator`` (legacy, informational + only). An unknown component id is a 400. + """ + try: + flow_examples = get_flow_examples(component_id) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return { + "component_id": component_id, + "count": len(flow_examples), + "examples": flow_examples, + } + + @router.get("/{project}/schema", summary="Fetch the live conditional-flow JSON Schema") def get_schema(project: str, registry: ServiceRegistry = Depends(get_registry)) -> dict[str, Any]: """Dump the keboola.flow JSON Schema served by the stack. Mirrors `kbagent flow schema --full`.""" diff --git a/src/keboola_agent_cli/server/routers/semantic_layer.py b/src/keboola_agent_cli/server/routers/semantic_layer.py index b34b8428..cc9d921d 100644 --- a/src/keboola_agent_cli/server/routers/semantic_layer.py +++ b/src/keboola_agent_cli/server/routers/semantic_layer.py @@ -26,6 +26,7 @@ from pydantic import BaseModel, Field, model_validator from ...errors import ErrorCode +from ...services.semantic_layer_service import SCHEMA_TYPE_ALIAS from ..dependencies import ServiceRegistry, get_registry router = APIRouter(prefix="/semantic-layer", tags=["semantic-layer"]) @@ -300,6 +301,28 @@ def get_context( return registry.semantic_layer.get_context(alias=project, context_id=context_id) +@router.get("/schema", summary="Fetch JSON Schemas of semantic object types") +def get_schema( + project: str, + type: list[str] | None = Query( + None, + description=( + "Semantic type(s) to fetch, repeatable " + "(model | dataset | metric | relationship | constraint | glossary). " + "Omitted = every known type (the CLI's --all)." + ), + ), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Fetch the server-side JSON Schema for semantic object types. + + Mirrors `kbagent semantic-layer schema`. Schemas are fetched live from + the project's metastore; unknown type names fail fast (HTTP 400). + """ + types = type if type else list(SCHEMA_TYPE_ALIAS) + return registry.semantic_layer.get_schema(alias=project, types=types) + + @router.get("/export", summary="Export model snapshot") def export( project: str, diff --git a/src/keboola_agent_cli/server/routers/transformation.py b/src/keboola_agent_cli/server/routers/transformation.py new file mode 100644 index 00000000..4ec8fc1d --- /dev/null +++ b/src/keboola_agent_cli/server/routers/transformation.py @@ -0,0 +1,115 @@ +"""SQL transformation endpoints (create / show / edit blocks and codes). + +Mirrors the ``kbagent transformation`` command group backed by +:class:`keboola_agent_cli.services.transformation_service.TransformationService` +(issue #396). ``ValueError`` from the service (empty SQL, invalid ops) maps +to HTTP 400 -- the REST equivalent of the CLI's VALIDATION_ERROR exit path. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from ..dependencies import ServiceRegistry, get_registry + +router = APIRouter(prefix="/transformations", tags=["transformations"]) + + +class TransformationCreate(BaseModel): + name: str + sql: str + created_tables: list[str] | None = None + component_id: str | None = None + description: str = "" + branch_id: int | None = None + dry_run: bool = False + + +class TransformationEdit(BaseModel): + change_description: str + ops: list[dict[str, Any]] = [] + component_id: str | None = None + storage: dict[str, Any] | None = None + branch_id: int | None = None + dry_run: bool = False + + +@router.post("/{project}", summary="Create a SQL transformation") +def create( + project: str, + body: TransformationCreate, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Create a SQL transformation from a SQL script. Mirrors `kbagent transformation create`. + + The SQL is split into one statement per script element; each entry in + ``created_tables`` is mapped to ``out.c-.
`` in the + output mapping. ``component_id`` defaults to the project's backend + (Snowflake / BigQuery). + """ + try: + return registry.transformation.create( + project, + name=body.name, + sql=body.sql, + created_tables=body.created_tables, + component_id=body.component_id, + description=body.description, + branch_id=body.branch_id, + dry_run=body.dry_run, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/{project}/{config_id}", summary="Show a SQL transformation's block tree") +def show( + project: str, + config_id: str, + component_id: str | None = None, + branch_id: int | None = None, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Render the block/code tree with positional IDs (b0, b0.c0, ...). + + Mirrors `kbagent transformation show`. When ``component_id`` is omitted + the known SQL transformation components are tried until the + configuration is found. + """ + return registry.transformation.show( + project, + config_id=config_id, + component_id=component_id, + branch_id=branch_id, + ) + + +@router.patch("/{project}/{config_id}", summary="Edit a SQL transformation with operations") +def edit( + project: str, + config_id: str, + body: TransformationEdit, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Apply a batch of block/code operations. Mirrors `kbagent transformation edit`. + + ``ops`` entries use the IDs from the show route; ``storage``, when set, + replaces ``configuration.storage`` wholesale. ``ops`` may be empty when + only ``storage`` is being replaced. + """ + try: + return registry.transformation.edit( + project, + config_id=config_id, + ops=body.ops, + change_description=body.change_description, + component_id=body.component_id, + storage=body.storage, + branch_id=body.branch_id, + dry_run=body.dry_run, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc diff --git a/tests/test_server_router_calls.py b/tests/test_server_router_calls.py index 8c2970d8..0f7c778d 100644 --- a/tests/test_server_router_calls.py +++ b/tests/test_server_router_calls.py @@ -1081,3 +1081,486 @@ def test_bulk_delete_route_not_shadowed_by_alias_delete(tmp_path: Path) -> None: assert res.status_code == 200, res.text project_svc.bulk_remove_projects.assert_called_once() project_svc.remove_project.assert_not_called() + + +# --------------------------------------------------------------------------- +# docs.py POST /documentation/query +# Service: docs.ask_docs(alias=..., query=...) (mirrors `kbagent docs query`) +# --------------------------------------------------------------------------- + + +def test_docs_query_passes_alias_and_query(tmp_path: Path) -> None: + """POST /documentation/query must call DocsService.ask_docs(alias=, query=).""" + docs_svc = MagicMock() + docs_svc.ask_docs.return_value = {"query": "q", "text": "answer", "source_urls": []} + registry = _mock_registry(docs=docs_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.post( + "/documentation/query", + headers=AUTH, + json={"query": "How do incremental loads work?", "project": PROJECT}, + ) + + assert res.status_code == 200, res.text + docs_svc.ask_docs.assert_called_once_with(alias=PROJECT, query="How do incremental loads work?") + + +def test_docs_query_project_optional_defaults_to_none(tmp_path: Path) -> None: + """Omitting `project` passes alias=None (service picks the first project).""" + docs_svc = MagicMock() + docs_svc.ask_docs.return_value = {"query": "q", "text": "a", "source_urls": []} + registry = _mock_registry(docs=docs_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.post("/documentation/query", headers=AUTH, json={"query": "q"}) + + assert res.status_code == 200, res.text + assert docs_svc.ask_docs.call_args.kwargs["alias"] is None + + +def test_docs_query_config_error_is_400(tmp_path: Path) -> None: + """No projects configured -> ConfigError -> HTTP 400 error envelope.""" + from keboola_agent_cli.errors import ConfigError + + docs_svc = MagicMock() + docs_svc.ask_docs.side_effect = ConfigError("No projects configured.") + registry = _mock_registry(docs=docs_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.post("/documentation/query", headers=AUTH, json={"query": "q"}) + + assert res.status_code == 400, res.text + assert "No projects configured" in res.json()["error"]["message"] + + +def test_docs_query_requires_bearer_auth(tmp_path: Path) -> None: + """/documentation must NOT live in the auth-exempt /docs (Swagger) namespace. + + The auth middleware exempts every path starting with /docs; the docs-QA + router therefore uses /documentation and must reject unauthenticated calls. + """ + docs_svc = MagicMock() + registry = _mock_registry(docs=docs_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.post("/documentation/query", json={"query": "q"}) # no auth header + + assert res.status_code == 401, res.text + docs_svc.ask_docs.assert_not_called() + + +# --------------------------------------------------------------------------- +# configs.py GET /configs/examples/{component_id} +# Service: component.get_config_examples(alias=..., component_id=...) +# (method lives on ComponentService; mirrors `kbagent config examples`) +# --------------------------------------------------------------------------- + + +def test_config_examples_passes_alias_and_component_id(tmp_path: Path) -> None: + """GET /configs/examples/{c} must call ComponentService.get_config_examples.""" + component_svc = MagicMock() + component_svc.get_config_examples.return_value = { + "component_id": COMPONENT, + "root_examples": [{"parameters": {}}], + "row_examples": [], + } + registry = _mock_registry(component=component_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.get( + f"/configs/examples/{COMPONENT}", headers=AUTH, params={"project": PROJECT} + ) + + assert res.status_code == 200, res.text + component_svc.get_config_examples.assert_called_once_with(alias=PROJECT, component_id=COMPONENT) + + +def test_config_examples_project_optional(tmp_path: Path) -> None: + """Without ?project= the router passes alias=None (first configured project).""" + component_svc = MagicMock() + component_svc.get_config_examples.return_value = { + "component_id": COMPONENT, + "root_examples": [], + "row_examples": [], + } + registry = _mock_registry(component=component_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.get(f"/configs/examples/{COMPONENT}", headers=AUTH) + + assert res.status_code == 200, res.text + assert component_svc.get_config_examples.call_args.kwargs["alias"] is None + + +def test_config_examples_api_error_is_502(tmp_path: Path) -> None: + """AI Service failure (KeboolaApiError) -> HTTP 502 error envelope.""" + from keboola_agent_cli.errors import ErrorCode, KeboolaApiError + + component_svc = MagicMock() + component_svc.get_config_examples.side_effect = KeboolaApiError( + message="Component not found", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + registry = _mock_registry(component=component_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.get(f"/configs/examples/{COMPONENT}", headers=AUTH) + + assert res.status_code == 502, res.text + assert "Component not found" in res.json()["error"]["message"] + + +# --------------------------------------------------------------------------- +# components.py POST /components/{component_id}/actions/{action} +# Service: component.run_sync_action(...) (mirrors `kbagent component sync-action`) +# --------------------------------------------------------------------------- + + +def test_component_sync_action_forwards_all_kwargs(tmp_path: Path) -> None: + """POST /components/{c}/actions/{a} must forward every body field by name.""" + component_svc = MagicMock() + component_svc.run_sync_action.return_value = { + "component_id": COMPONENT, + "action": "testConnection", + "result": {"status": "success"}, + } + registry = _mock_registry(component=component_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.post( + f"/components/{COMPONENT}/actions/testConnection", + headers=AUTH, + json={ + "project": PROJECT, + "config_id": CONFIG_ID, + "row_id": ROW_ID, + "branch_id": 123, + "timeout": 60, + }, + ) + + assert res.status_code == 200, res.text + component_svc.run_sync_action.assert_called_once_with( + alias=PROJECT, + component_id=COMPONENT, + action="testConnection", + config_id=CONFIG_ID, + row_id=ROW_ID, + branch_id=123, + config_data_override=None, + timeout=60, + ) + + +def test_component_sync_action_config_data_override(tmp_path: Path) -> None: + """`config_data` in the body reaches the service as config_data_override=.""" + component_svc = MagicMock() + component_svc.run_sync_action.return_value = {"result": {}} + registry = _mock_registry(component=component_svc) + app = _make_app_with_registry(tmp_path, registry) + + payload = {"parameters": {"db": {"host": "example.com"}}} + with TestClient(app) as client: + res = client.post( + f"/components/{COMPONENT}/actions/testConnection", + headers=AUTH, + json={"project": PROJECT, "config_data": payload}, + ) + + assert res.status_code == 200, res.text + kwargs = component_svc.run_sync_action.call_args.kwargs + assert kwargs["config_data_override"] == payload + assert kwargs["config_id"] is None + + +def test_component_sync_action_resolves_pinned_alias(tmp_path: Path) -> None: + """Without `project` in the body, the pinned alias is resolved (like detail/scaffold).""" + component_svc = MagicMock() + component_svc.run_sync_action.return_value = {"result": {}} + project_svc = MagicMock() + project_svc.resolve_pinned_alias.return_value = ("pinned-proj", "config") + registry = _mock_registry(component=component_svc, project=project_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.post( + f"/components/{COMPONENT}/actions/getTables", + headers=AUTH, + json={"config_id": CONFIG_ID}, + ) + + assert res.status_code == 200, res.text + project_svc.resolve_pinned_alias.assert_called_once_with(None) + assert component_svc.run_sync_action.call_args.kwargs["alias"] == "pinned-proj" + + +def test_component_sync_action_missing_inputs_is_400(tmp_path: Path) -> None: + """Service-side ConfigError (no config_id, no config_data) -> HTTP 400.""" + from keboola_agent_cli.errors import ConfigError + + component_svc = MagicMock() + component_svc.run_sync_action.side_effect = ConfigError( + "Either a configuration ID or explicit config data is required to run a sync action." + ) + registry = _mock_registry(component=component_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.post( + f"/components/{COMPONENT}/actions/testConnection", + headers=AUTH, + json={"project": PROJECT}, + ) + + assert res.status_code == 400, res.text + assert "configuration ID" in res.json()["error"]["message"] + + +# --------------------------------------------------------------------------- +# semantic_layer.py GET /semantic-layer/schema +# Service: semantic_layer.get_schema(alias=..., types=[...]) +# (mirrors `kbagent semantic-layer schema`) +# --------------------------------------------------------------------------- + + +def test_semantic_layer_schema_forwards_types(tmp_path: Path) -> None: + """Repeated ?type= params must reach get_schema as an ordered list.""" + sl = MagicMock() + sl.get_schema.return_value = {"project": PROJECT, "schemas": []} + app = _make_app_with_registry(tmp_path, _mock_registry(semantic_layer=sl)) + + with TestClient(app) as client: + res = client.get( + "/semantic-layer/schema", + headers=AUTH, + params=[("project", PROJECT), ("type", "metric"), ("type", "model")], + ) + + assert res.status_code == 200, res.text + sl.get_schema.assert_called_once_with(alias=PROJECT, types=["metric", "model"]) + + +def test_semantic_layer_schema_defaults_to_all_types(tmp_path: Path) -> None: + """Omitting ?type= fetches every known semantic type (the CLI's --all).""" + from keboola_agent_cli.services.semantic_layer_service import SCHEMA_TYPE_ALIAS + + sl = MagicMock() + sl.get_schema.return_value = {"project": PROJECT, "schemas": []} + app = _make_app_with_registry(tmp_path, _mock_registry(semantic_layer=sl)) + + with TestClient(app) as client: + res = client.get("/semantic-layer/schema", headers=AUTH, params={"project": PROJECT}) + + assert res.status_code == 200, res.text + sl.get_schema.assert_called_once_with(alias=PROJECT, types=list(SCHEMA_TYPE_ALIAS)) + + +def test_semantic_layer_schema_unknown_type_is_400(tmp_path: Path) -> None: + """Unknown type name -> service ConfigError -> HTTP 400 (fail fast, no network).""" + from keboola_agent_cli.errors import ConfigError + + sl = MagicMock() + sl.get_schema.side_effect = ConfigError("Unknown semantic type(s): bogus.") + app = _make_app_with_registry(tmp_path, _mock_registry(semantic_layer=sl)) + + with TestClient(app) as client: + res = client.get( + "/semantic-layer/schema", + headers=AUTH, + params={"project": PROJECT, "type": "bogus"}, + ) + + assert res.status_code == 400, res.text + assert "Unknown semantic type" in res.json()["error"]["message"] + + +# --------------------------------------------------------------------------- +# transformation.py POST /{p} + GET /{p}/{cfg} + PATCH /{p}/{cfg} +# Service: transformation.create/show/edit (mirrors `kbagent transformation *`) +# --------------------------------------------------------------------------- + + +def test_transformation_create_forwards_kwargs(tmp_path: Path) -> None: + """POST /transformations/{p} must forward every create field by name.""" + tf = MagicMock() + tf.create.return_value = {"config_id": "77", "name": "Orders"} + app = _make_app_with_registry(tmp_path, _mock_registry(transformation=tf)) + + with TestClient(app) as client: + res = client.post( + f"/transformations/{PROJECT}", + headers=AUTH, + json={ + "name": "Orders", + "sql": 'CREATE TABLE "report" AS SELECT 1;', + "created_tables": ["report"], + "description": "demo", + "branch_id": 5, + "dry_run": True, + }, + ) + + assert res.status_code == 200, res.text + tf.create.assert_called_once_with( + PROJECT, + name="Orders", + sql='CREATE TABLE "report" AS SELECT 1;', + created_tables=["report"], + component_id=None, + description="demo", + branch_id=5, + dry_run=True, + ) + + +def test_transformation_create_empty_sql_is_400(tmp_path: Path) -> None: + """Service ValueError (SQL contains no statements) -> HTTP 400, not 500.""" + tf = MagicMock() + tf.create.side_effect = ValueError("SQL contains no statements (empty input)") + app = _make_app_with_registry(tmp_path, _mock_registry(transformation=tf)) + + with TestClient(app) as client: + res = client.post( + f"/transformations/{PROJECT}", + headers=AUTH, + json={"name": "Empty", "sql": " "}, + ) + + assert res.status_code == 400, res.text + assert "no statements" in res.json()["error"]["message"] + + +def test_transformation_show_forwards_kwargs(tmp_path: Path) -> None: + """GET /transformations/{p}/{cfg} must pass config_id/component_id/branch_id.""" + tf = MagicMock() + tf.show.return_value = {"config_id": CONFIG_ID, "blocks": []} + app = _make_app_with_registry(tmp_path, _mock_registry(transformation=tf)) + + with TestClient(app) as client: + res = client.get( + f"/transformations/{PROJECT}/{CONFIG_ID}", + headers=AUTH, + params={"component_id": "keboola.snowflake-transformation", "branch_id": 9}, + ) + + assert res.status_code == 200, res.text + tf.show.assert_called_once_with( + PROJECT, + config_id=CONFIG_ID, + component_id="keboola.snowflake-transformation", + branch_id=9, + ) + + +def test_transformation_show_not_found_is_502(tmp_path: Path) -> None: + """Config not found under any SQL component -> KeboolaApiError -> HTTP 502.""" + from keboola_agent_cli.errors import ErrorCode, KeboolaApiError + + tf = MagicMock() + tf.show.side_effect = KeboolaApiError( + message="Configuration '42' was not found under any SQL transformation component", + status_code=404, + error_code=ErrorCode.NOT_FOUND, + ) + app = _make_app_with_registry(tmp_path, _mock_registry(transformation=tf)) + + with TestClient(app) as client: + res = client.get(f"/transformations/{PROJECT}/{CONFIG_ID}", headers=AUTH) + + assert res.status_code == 502, res.text + assert "was not found" in res.json()["error"]["message"] + + +def test_transformation_edit_forwards_kwargs(tmp_path: Path) -> None: + """PATCH /transformations/{p}/{cfg} must forward ops + change_description + storage.""" + tf = MagicMock() + tf.edit.return_value = {"config_id": CONFIG_ID, "operations_applied": [], "blocks": []} + app = _make_app_with_registry(tmp_path, _mock_registry(transformation=tf)) + + ops = [{"op": "str_replace", "search_for": "a", "replace_with": "b"}] + storage = {"input": {"tables": []}, "output": {"tables": []}} + with TestClient(app) as client: + res = client.patch( + f"/transformations/{PROJECT}/{CONFIG_ID}", + headers=AUTH, + json={ + "ops": ops, + "change_description": "Rename column", + "storage": storage, + "dry_run": True, + }, + ) + + assert res.status_code == 200, res.text + tf.edit.assert_called_once_with( + PROJECT, + config_id=CONFIG_ID, + ops=ops, + change_description="Rename column", + component_id=None, + storage=storage, + branch_id=None, + dry_run=True, + ) + + +def test_transformation_edit_invalid_op_is_400(tmp_path: Path) -> None: + """Service ValueError (bad op schema) -> HTTP 400, not 500.""" + tf = MagicMock() + tf.edit.side_effect = ValueError("Operation #0 has unknown op 'explode'") + app = _make_app_with_registry(tmp_path, _mock_registry(transformation=tf)) + + with TestClient(app) as client: + res = client.patch( + f"/transformations/{PROJECT}/{CONFIG_ID}", + headers=AUTH, + json={"ops": [{"op": "explode"}], "change_description": "boom"}, + ) + + assert res.status_code == 400, res.text + assert "unknown op" in res.json()["error"]["message"] + + +# --------------------------------------------------------------------------- +# flows.py GET /flows/examples +# Module-level flow_service.get_flow_examples (offline, bundled resources -- +# exercised for real, no mocks; mirrors `kbagent flow examples`). +# --------------------------------------------------------------------------- + + +def test_flow_examples_returns_bundled_conditional_examples(tmp_path: Path) -> None: + """Default component id serves the bundled keboola.flow examples.""" + registry = _mock_registry(flow=MagicMock()) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.get("/flows/examples", headers=AUTH) + + assert res.status_code == 200, res.text + body = res.json() + assert body["component_id"] == "keboola.flow" + assert body["count"] == len(body["examples"]) + assert body["count"] > 0 + assert all(isinstance(example, dict) for example in body["examples"]) + + +def test_flow_examples_unknown_component_is_400(tmp_path: Path) -> None: + """An unknown component id -> ValueError -> HTTP 400 (mirrors CLI exit 2).""" + registry = _mock_registry(flow=MagicMock()) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.get( + "/flows/examples", headers=AUTH, params={"component_id": "keboola.nonsense"} + ) + + assert res.status_code == 400, res.text + assert "No bundled flow examples" in res.json()["error"]["message"] From d82433b982130ea97db862a87e3a8736122bc79a Mon Sep 17 00:00:00 2001 From: Petr Date: Tue, 21 Jul 2026 21:03:10 +0200 Subject: [PATCH 17/17] docs: address kbagent-pr-reviewer findings on #508 NB-2: reword component sync-action docstring first sentence ('e.g. testConnection' -> 'such as testConnection') so generate_skill.py's first-sentence split no longer truncates the SKILL.md decision-table row mid-parenthesis; regenerated SKILL.md. NB-1: add the new transformation-workflow.md row to SKILL.md's Workflow references table. NIT-1: add a docs-query row to keboola-expert.md tool selection matrix. --- plugins/kbagent/agents/keboola-expert.md | 1 + plugins/kbagent/skills/kbagent/SKILL.md | 3 ++- src/keboola_agent_cli/commands/component.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index c991440d..828c2eec 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -93,6 +93,7 @@ a critical failure. | Search items by name across projects | `kbagent search QUERY [--project P] [--type table\|bucket\|config\|flow\|data-app\|transformation] [--search-type textual\|config-based] [--limit N] [--regex]` (0.30.0+); `--regex` (0.67.0+) opts into case-insensitive whole-term regex on entity names — `report` does NOT match `monthly_report`, write `.*report.*`; textual mode marks `table` results matched via a column name with `matched_columns` in `--json` (0.67.0+; always present, `[]` when the name itself matched; always `[]` under `--regex` — regex never matches column names) | `tool call search_tables` / `tool call search_configurations` (one resource-type per call) | chaining multiple `tool call` for different types; `--regex` with `--search-type config-based` (exit 2); `--regex` below 0.67.0 | | Search config JSON bodies | `kbagent search QUERY --search-type config-based [--project P]` (0.30.0+) | `kbagent config search --query Q` (config-body only, no tables/buckets) | repeated `tool call get_config` to grep locally | | Browse configs (exploration) | `kbagent config list` / `kbagent config search --query Q` | `tool call list_configs` | full-project pull via MCP just to grep locally | +| Answer a Keboola-documentation question ("how do I configure incremental loading?") | `kbagent docs query "QUESTION" [--project P]` (0.73.0+; AI-service RAG, returns answer + source URLs) | -- | `kai ask` (project-scoped assistant, not docs Q&A); `tool call docs_query` (deprecated 0.74.0) | | Fetch a specific config | `kbagent config detail --project P --component-id C --config-id K --json` | `tool call get_config` | re-using an earlier JSON dump | | Override the auto-derived output bucket on a config | `kbagent config set-default-bucket --bucket in.c-name` (0.26.0+) -- read-modify-write of `storage.output.default_bucket`, preserves siblings; `--clear` removes it | `kbagent config update --set 'storage.output.default_bucket=in.c-name'` (works pre-0.26.0 but not discoverable) | editing the raw JSON in the UI; full-config replace with `--configuration` (wipes other storage keys) | | Cross-project migration | `kbagent sync pull` + edit files locally + `kbagent sync push --dry-run` | -- | repeated `tool call` loops, one per resource | diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index c89efc19..d388a336 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -97,7 +97,7 @@ When working inside a git repository or project directory, run `kbagent init` (o | Rotate a token: generate a new value and invalidate the old one (secret shown once) | `kbagent token refresh --project PROJECT --token-id TOKEN-ID` | | List available components from connected projects | `kbagent component list` | | Show detailed information about a specific component | `kbagent component detail --component-id COMPONENT-ID` | -| Run a synchronous component action (e.g. | `kbagent component sync-action --component-id COMPONENT-ID --project PROJECT` | +| Run a synchronous component action such as testConnection | `kbagent component sync-action --component-id COMPONENT-ID --project PROJECT` | | List configurations from connected projects | `kbagent config list` | | Show detailed information about one or many configurations | `kbagent config detail --component-id COMPONENT-ID` | | Show sample configuration JSON examples for a component | `kbagent config examples --component-id COMPONENT-ID` | @@ -370,6 +370,7 @@ For detailed response parsing rules and common pitfalls, see [gotchas](reference | All commands cheat sheet | [commands-reference](references/commands-reference.md) | | **Safe config write workflow** (fetch → dry-run → confirm → push) | [safe-write-workflow](references/safe-write-workflow.md) | | Creating new configurations | [scaffold-workflow](references/scaffold-workflow.md) | +| **SQL transformations** (create / show / edit; the show-before-edit rule for positional block/code ids) | [transformation-workflow](references/transformation-workflow.md) | | MCP tools (multi-project read/write) | [mcp-workflow](references/mcp-workflow.md) | | Workspace SQL debugging | [workspace-workflow](references/workspace-workflow.md) | | **Agent Tasks via CLI** (`kbagent agent` CRUD + run + cron-preview + prompt-improve; cron / manual / chained; mcp_tool / cli_command / ai_agent action flavours) | [agent-tasks-cli-workflow](references/agent-tasks-cli-workflow.md) | diff --git a/src/keboola_agent_cli/commands/component.py b/src/keboola_agent_cli/commands/component.py index cf2a5d30..7a0d5dab 100644 --- a/src/keboola_agent_cli/commands/component.py +++ b/src/keboola_agent_cli/commands/component.py @@ -282,7 +282,7 @@ def component_sync_action( help="Request timeout in seconds for the action call (long actions e.g. getTables)", ), ) -> None: - """Run a synchronous component action (e.g. testConnection). + """Run a synchronous component action such as testConnection. \b Valid action names are component-defined -- the API validates them