From fe22df3b12427e023899f3f3ffcf38f505af9473 Mon Sep 17 00:00:00 2001 From: Maxmilian Ottomansky Date: Mon, 20 Apr 2026 15:06:15 +0200 Subject: [PATCH 1/5] feat(job): auto-resolve variableValuesId from linked variables config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `kbagent job run` now reads `configuration.runtime.variables_id` on the parent config and binds the Queue job to a values row automatically. Without this, transformations linked to a keboola.variables config ran against empty `{{ placeholder }}` strings — the most common silent-fail mode in the FIIA pipeline. Resolution order (job_service.resolve_variable_values_id): - explicit --variable-values-id ROW_ID (override) - runtime.variables_values_id on the parent config - first row of the linked keboola.variables config New CLI flags on `job run`: - --variable-values-id ID: hand-pick the values row - --no-variables: skip resolution entirely (mutually exclusive) New error code NO_VARIABLE_ROWS when the linked variables config has zero rows — agents know to run `kbagent config variables-set` first (that command ships with PR #190). Client: `create_job` gained `variable_values_id` parameter. Omitted from the Queue body when unset so existing callers retain wire-level compatibility. Response: `kbagent --json job run` now surfaces `resolvedVariableValuesId` in the JSON payload so callers can verify the binding without a second `job detail` call. Version 0.22.0. Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 2 +- plugins/kbagent/.claude-plugin/plugin.json | 2 +- .../skills/kbagent/references/gotchas.md | 22 +++ pyproject.toml | 2 +- src/keboola_agent_cli/changelog.py | 7 + src/keboola_agent_cli/client.py | 7 + src/keboola_agent_cli/commands/context.py | 6 +- src/keboola_agent_cli/commands/job.py | 32 ++++ .../hints/definitions/job.py | 38 +++- src/keboola_agent_cli/services/job_service.py | 81 ++++++++ tests/test_cli.py | 176 ++++++++++++++++++ tests/test_client.py | 42 +++++ tests/test_e2e.py | 159 ++++++++++++++++ tests/test_services.py | 169 +++++++++++++++++ uv.lock | 2 +- 15 files changed, 741 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b74c591a..291607fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -253,7 +253,7 @@ kbagent config rename --project NAME --component-id ID --config-id ID --name "Ne kbagent job list [--project NAME] [--component-id ID] [--status STATUS] [--limit N] kbagent job detail --project NAME --job-id ID -kbagent job run --project NAME --component-id ID --config-id ID [--row-id ID ...] [--wait] [--timeout N] [--branch ID] +kbagent job run --project NAME --component-id ID --config-id ID [--row-id ID ...] [--wait] [--timeout N] [--branch ID] [--variable-values-id ID] [--no-variables] kbagent job terminate --project NAME (--job-id ID [--job-id ID ...] | --status any|created|waiting|processing [--component-id ID] [--config-id ID] [--branch ID] [--limit N]) [--dry-run] [--yes] kbagent storage buckets [--project NAME] [--branch ID] diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index 61b01f73..d4cd469c 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.20.6", + "version": "0.22.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 3984d7ae..68744062 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -1,5 +1,27 @@ # Gotchas -- Response Parsing and Common Pitfalls +## `job run` now auto-resolves variable values + +Transformations with linked `keboola.variables` used to run against empty strings +unless the caller hand-wired a `variableValuesId` at the HTTP layer. `kbagent job run` +now auto-resolves it: reads `configuration.runtime.variables_id` from the parent +config, picks `variables_values_id` if set, else the first row of the linked +variables config. + +**Override knobs:** +- `--variable-values-id ROW_ID` -- use a specific values row (CI runs, what-if analysis). +- `--no-variables` -- skip resolution entirely (components without variables, or + intentionally running with empty bindings). + +**Error cases:** +- `NO_VARIABLE_ROWS` -- the linked `keboola.variables` config exists but has zero + rows. Fix: `kbagent config variables-set --project X --component-id C --config-id I --var KEY=VALUE`. +- The JSON response now carries `resolvedVariableValuesId` when the resolver fired, + so you can verify the job bound to the right row. + +`--variable-values-id` and `--no-variables` are mutually exclusive; passing both +returns exit 2 / `INVALID_ARGUMENT` before any API call. + ## Response structure varies by command Not all commands return data the same way. Key differences: diff --git a/pyproject.toml b/pyproject.toml index c8d6340e..d622abf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-agent-cli" -version = "0.20.6" +version = "0.22.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index ad66ff10..d3327993 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -8,6 +8,13 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.22.0": [ + "New: `kbagent job run` auto-resolves `variableValuesId` for configs with linked `keboola.variables` -- transformations now run against deployed values instead of empty strings.", + "New: `--variable-values-id ID` to override the auto-resolved values row.", + "New: `--no-variables` to skip resolution entirely (mutually exclusive with `--variable-values-id`).", + "New: `NO_VARIABLE_ROWS` error code when a linked variables config has zero rows (misconfiguration; surface via `--json` error payload or fix via `kbagent config variables-set`).", + "Client: `create_job` gained `variable_values_id` parameter; omitted from body when unset so existing callers retain wire-level compatibility.", + ], "0.20.6": [ "Fix: storage download-table / unload-table no longer OOM on multi-GB tables -- streamed downloads cap RAM at ~1 MiB regardless of table size (#187)", "Fix: _prepend_csv_header() no longer loads the full CSV into RAM (was the second OOM source after slice download)", diff --git a/src/keboola_agent_cli/client.py b/src/keboola_agent_cli/client.py index 8f47cc71..d92b07e3 100644 --- a/src/keboola_agent_cli/client.py +++ b/src/keboola_agent_cli/client.py @@ -1735,6 +1735,7 @@ def create_job( config_row_ids: list[str] | None = None, mode: str = "run", branch_id: int | None = None, + variable_values_id: str | None = None, ) -> dict[str, Any]: """Create and run a Queue API job. @@ -1747,6 +1748,10 @@ def create_job( mode: Job mode (default: run). branch_id: Optional dev branch ID. When set, the job runs on that branch instead of the default (production) branch. + variable_values_id: Optional id of a row in the linked + ``keboola.variables`` config. When set, the Queue API binds + the row's values to the job's `{{ variable }}` placeholders. + Omit for configurations that have no linked variables. Returns: Job dict from the Queue API. @@ -1762,6 +1767,8 @@ def create_job( body["configData"] = config_data if config_row_ids: body["configRowIds"] = config_row_ids + if variable_values_id: + body["variableValuesId"] = variable_values_id response = self._queue_request("POST", "/jobs", json=body) return response.json() diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 64a2f526..9631ca40 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -127,9 +127,13 @@ kbagent job detail --project NAME --job-id ID Full job detail including result message and timing. - kbagent job run --project NAME --component-id ID --config-id ID [--row-id ID ...] [--wait] [--timeout N] [--branch ID] + kbagent job run --project NAME --component-id ID --config-id ID [--row-id ID ...] [--wait] [--timeout N] [--branch ID] [--variable-values-id ID] [--no-variables] Run a Queue API job. --row-id selects specific config rows (repeatable; omit to run entire config). --wait polls until job finishes. --timeout sets max wait in seconds (default 300). Branch-aware. + When the config has linked variables (runtime.variables_id), kbagent auto-resolves a + variableValuesId so the job binds to the deployed values row. --variable-values-id overrides; + --no-variables skips resolution. Error code NO_VARIABLE_ROWS when the linked variables + config has zero rows (run `kbagent config variables-set` to create one). kbagent job terminate --project NAME (--job-id ID [--job-id ID ...] | --status any|created|waiting|processing [--component-id ID] [--config-id ID] [--branch ID] [--limit N]) [--dry-run] [--yes] Kill running jobs via Queue API (POST /jobs/{id}/kill). Use to stop runaway loops or pile-ups. diff --git a/src/keboola_agent_cli/commands/job.py b/src/keboola_agent_cli/commands/job.py index 15183f16..ae1180f0 100644 --- a/src/keboola_agent_cli/commands/job.py +++ b/src/keboola_agent_cli/commands/job.py @@ -191,6 +191,22 @@ def job_run( "--branch", help="Dev branch ID (overrides active branch)", ), + variable_values_id: str | None = typer.Option( + None, + "--variable-values-id", + help=( + "Row id of the linked keboola.variables values row. Overrides " + "auto-resolution. Mutually exclusive with --no-variables." + ), + ), + no_variables: bool = typer.Option( + False, + "--no-variables", + help=( + "Skip variable-values resolution. Use for configs without linked " + "variables or when intentionally running against empty bindings." + ), + ), ) -> None: """Run a job for a component configuration. @@ -199,6 +215,11 @@ def job_run( When a dev branch is active (via 'branch use'), the job automatically runs on that branch. Use --branch to override. + + When the config has linked variables (runtime.variables_id), kbagent + auto-resolves a variableValuesId so the job binds to the deployed + values row. Override with --variable-values-id or skip with + --no-variables. """ if should_hint(ctx): emit_hint( @@ -211,12 +232,21 @@ def job_run( wait=wait, timeout=timeout, branch=branch, + variable_values_id=variable_values_id, + no_variables=no_variables, ) return formatter = get_formatter(ctx) service = get_service(ctx, "job_service") config_store: ConfigStore = ctx.obj["config_store"] + if variable_values_id and no_variables: + formatter.error( + message="--variable-values-id and --no-variables are mutually exclusive.", + error_code="INVALID_ARGUMENT", + ) + raise typer.Exit(code=2) + validate_branch_requires_project(formatter, branch, project) _, effective_branch = resolve_branch(config_store, formatter, project, branch) @@ -240,6 +270,8 @@ def job_run( wait=wait, timeout=timeout, branch_id=effective_branch, + variable_values_id=variable_values_id, + no_variables=no_variables, ) except ConfigError as exc: formatter.error(message=exc.message, error_code="CONFIG_ERROR") diff --git a/src/keboola_agent_cli/hints/definitions/job.py b/src/keboola_agent_cli/hints/definitions/job.py index c50f6bb0..29d01e86 100644 --- a/src/keboola_agent_cli/hints/definitions/job.py +++ b/src/keboola_agent_cli/hints/definitions/job.py @@ -75,6 +75,33 @@ cli_command="job.run", description="Run a component configuration as a job", steps=[ + HintStep( + comment=( + "Resolve linked variables values row (skip when " + "--no-variables; override via --variable-values-id)" + ), + client=ClientCall( + method="get_config_detail", + args={ + "component_id": "{component_id}", + "config_id": "{config_id}", + }, + result_var="detail", + result_hint="dict", + ), + ), + HintStep( + comment="Look up first row of linked variables config if values_id absent", + client=ClientCall( + method="list_config_rows", + args={ + "component_id": '"keboola.variables"', + "config_id": 'detail["configuration"]["runtime"]["variables_id"]', + }, + result_var="var_rows", + result_hint="list", + ), + ), HintStep( comment="Create and submit job to Queue API", client=ClientCall( @@ -83,6 +110,7 @@ "component_id": "{component_id}", "config_id": "{config_id}", "config_row_ids": "{row_id}", + "variable_values_id": 'var_rows[0]["id"] if var_rows else None', }, result_var="job", result_hint="dict", @@ -98,6 +126,8 @@ "config_row_ids": "{row_id}", "wait": "{wait}", "timeout": "{timeout}", + "variable_values_id": "{variable_values_id}", + "no_variables": "{no_variables}", }, ), ), @@ -116,7 +146,13 @@ notes=[ "Uses the Queue API (queue.keboola.com), not Storage API.", "Without --wait, returns immediately after job creation.", - "Service layer handles both create + optional poll in one call.", + "Service layer handles both resolve + create + optional poll in one call.", + ( + "Service auto-resolves variableValuesId from " + "configuration.runtime.variables_id; the client hint shows " + "the underlying two-request pattern." + ), + "Pass --no-variables to skip resolution entirely.", ], ) ) diff --git a/src/keboola_agent_cli/services/job_service.py b/src/keboola_agent_cli/services/job_service.py index 6a27d0c6..14d329e9 100644 --- a/src/keboola_agent_cli/services/job_service.py +++ b/src/keboola_agent_cli/services/job_service.py @@ -157,9 +157,17 @@ def run_job( wait: bool = False, timeout: float = 300.0, branch_id: int | None = None, + variable_values_id: str | None = None, + no_variables: bool = False, ) -> dict[str, Any]: """Create and optionally wait for a Queue API job. + When the config has linked variables (``configuration.runtime.variables_id``), + auto-resolve a ``variableValuesId`` via :meth:`resolve_variable_values_id` + so the job runs against the deployed values row instead of empty + strings. Pass ``variable_values_id`` to override, or ``no_variables=True`` + to skip the resolution entirely. + Args: alias: Project alias. component_id: Component ID to run. @@ -169,6 +177,12 @@ def run_job( timeout: Max seconds to wait (only used when wait=True). branch_id: Optional dev branch ID. When set, the job runs on that branch instead of the default (production) branch. + variable_values_id: Optional explicit values row id. When set, + bypasses auto-resolution and goes straight into the Queue + body. Mutually exclusive with ``no_variables``. + no_variables: If True, skip variable-values resolution entirely + (useful for components that do not support variables, or + when the caller intentionally wants empty-string binding). Returns: Job dict with project_alias. If wait=True, returns the @@ -179,11 +193,21 @@ def run_job( client = self._client_factory(project.stack_url, project.token) try: + resolved_values_id = variable_values_id + if resolved_values_id is None and not no_variables: + resolved_values_id = self.resolve_variable_values_id( + client=client, + component_id=component_id, + config_id=config_id, + branch_id=branch_id, + ) + job = client.create_job( component_id=component_id, config_id=config_id, config_row_ids=config_row_ids, branch_id=branch_id, + variable_values_id=resolved_values_id, ) job_id = str(job.get("id", "")) @@ -193,8 +217,65 @@ def run_job( client.close() job["project_alias"] = alias + if resolved_values_id: + job["resolvedVariableValuesId"] = resolved_values_id return job + @staticmethod + def resolve_variable_values_id( + client: Any, + component_id: str, + config_id: str, + branch_id: int | None = None, + ) -> str | None: + """Resolve the variables values row id to pass to ``create_job``. + + Reads the parent config's ``configuration.runtime.variables_id`` to + find the linked ``keboola.variables`` config. Prefers the explicit + ``configuration.runtime.variables_values_id`` when set; otherwise + falls back to the first row of the linked variables config (the + default-row convention). + + Returns: + The values row id, or ``None`` if the parent config has no + linked variables (no ``runtime.variables_id``). + + Raises: + KeboolaApiError: ``NO_VARIABLE_ROWS`` when the linked variables + config has zero rows (misconfiguration — the config exists + but no values row has been set). + """ + detail = client.get_config_detail( + component_id=component_id, + config_id=config_id, + branch_id=branch_id, + ) + runtime = (detail.get("configuration") or {}).get("runtime") or {} + variables_id = runtime.get("variables_id") + if not variables_id: + return None + + explicit_values_id = runtime.get("variables_values_id") + if explicit_values_id: + return str(explicit_values_id) + + rows = client.list_config_rows( + component_id="keboola.variables", + config_id=str(variables_id), + branch_id=branch_id, + ) + if not rows: + raise KeboolaApiError( + message=( + f"Linked variables config {variables_id} has no rows. " + f"Create one via `kbagent config variables-set` or pass " + f"`--no-variables` to skip resolution." + ), + status_code=0, + error_code="NO_VARIABLE_ROWS", + ) + return str(rows[0].get("id", "")) + def resolve_job_ids_by_filter( self, alias: str, diff --git a/tests/test_cli.py b/tests/test_cli.py index af193b4f..d0350221 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2245,6 +2245,182 @@ def test_job_run_branch_requires_project(self, tmp_path: Path) -> None: # --project is required, so typer returns exit code 2 (usage error) assert result.exit_code == 2 + def test_job_run_explicit_variable_values_id_forwarded(self, tmp_path: Path) -> None: + """--variable-values-id lands in service call as variable_values_id.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config_test( + config_dir, + {"prod": {"token": "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k"}}, + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + ): + MockStore.return_value = store + job_service = MagicMock() + job_service.run_job.return_value = {"id": 700, "status": "waiting"} + MockJobService.return_value = job_service + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + + result = runner.invoke( + app, + [ + "--json", + "job", + "run", + "--project", + "prod", + "--component-id", + "keboola.snowflake-transformation", + "--config-id", + "100", + "--variable-values-id", + "row-user-picked", + ], + ) + + assert result.exit_code == 0, result.output + kwargs = job_service.run_job.call_args.kwargs + assert kwargs["variable_values_id"] == "row-user-picked" + assert kwargs["no_variables"] is False + + def test_job_run_no_variables_flag_forwarded(self, tmp_path: Path) -> None: + """--no-variables sets no_variables=True on the service call.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config_test( + config_dir, + {"prod": {"token": "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k"}}, + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + ): + MockStore.return_value = store + job_service = MagicMock() + job_service.run_job.return_value = {"id": 701, "status": "waiting"} + MockJobService.return_value = job_service + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + + result = runner.invoke( + app, + [ + "--json", + "job", + "run", + "--project", + "prod", + "--component-id", + "keboola.ex-http", + "--config-id", + "42", + "--no-variables", + ], + ) + + assert result.exit_code == 0, result.output + kwargs = job_service.run_job.call_args.kwargs + assert kwargs["no_variables"] is True + assert kwargs["variable_values_id"] is None + + def test_job_run_mutually_exclusive_flags_rejected(self, tmp_path: Path) -> None: + """--variable-values-id + --no-variables is an invalid combination (exit 2).""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config_test( + config_dir, + {"prod": {"token": "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k"}}, + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + ): + MockStore.return_value = store + job_service = MagicMock() + MockJobService.return_value = job_service + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + + result = runner.invoke( + app, + [ + "--json", + "job", + "run", + "--project", + "prod", + "--component-id", + "keboola.ex-http", + "--config-id", + "42", + "--variable-values-id", + "row-1", + "--no-variables", + ], + ) + + assert result.exit_code == 2 + job_service.run_job.assert_not_called() + assert "INVALID_ARGUMENT" in result.output + + def test_job_run_no_variable_rows_error(self, tmp_path: Path) -> None: + """Service raises NO_VARIABLE_ROWS → CLI exits 1 with error_code surfaced.""" + from keboola_agent_cli.errors import KeboolaApiError + + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config_test( + config_dir, + {"prod": {"token": "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k"}}, + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + ): + MockStore.return_value = store + job_service = MagicMock() + job_service.run_job.side_effect = KeboolaApiError( + message="Linked variables config vars-42 has no rows.", + status_code=0, + error_code="NO_VARIABLE_ROWS", + ) + MockJobService.return_value = job_service + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + + result = runner.invoke( + app, + [ + "--json", + "job", + "run", + "--project", + "prod", + "--component-id", + "keboola.snowflake-transformation", + "--config-id", + "100", + ], + ) + + assert result.exit_code != 0 + assert "NO_VARIABLE_ROWS" in result.output + class TestJobTerminate: """Tests for `kbagent job terminate` command.""" diff --git a/tests/test_client.py b/tests/test_client.py index 72cdebbc..0630a0ba 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -2208,6 +2208,48 @@ def test_create_job_with_branch_and_row_ids(self, httpx_mock) -> None: assert body["branchId"] == "123" assert body["configRowIds"] == ["row1", "row2"] + def test_create_job_with_variable_values_id(self, httpx_mock) -> None: + """create_job() with variable_values_id forwards it as ``variableValuesId``.""" + httpx_mock.add_response( + url="https://queue.keboola.com/jobs", + method="POST", + json={"id": 600, "status": "waiting"}, + status_code=201, + ) + + with KeboolaClient(stack_url=_BASE, token=_TOKEN) as client: + client.create_job( + component_id="keboola.snowflake-transformation", + config_id="100", + variable_values_id="row-vars-001", + ) + + import json + + body = json.loads(httpx_mock.get_request().content) + assert body["variableValuesId"] == "row-vars-001" + + def test_create_job_without_variable_values_id_omits_field(self, httpx_mock) -> None: + """Default call path does not include ``variableValuesId`` in the body. + + Locks the "only send when set" contract so configs without linked + variables do not trip validation on the Queue API side. + """ + httpx_mock.add_response( + url="https://queue.keboola.com/jobs", + method="POST", + json={"id": 601, "status": "waiting"}, + status_code=201, + ) + + with KeboolaClient(stack_url=_BASE, token=_TOKEN) as client: + client.create_job(component_id="keboola.ex-http", config_id="42") + + import json + + body = json.loads(httpx_mock.get_request().content) + assert "variableValuesId" not in body + class TestKillJob: """Tests for kill_job() - Queue API POST /jobs/{id}/kill.""" diff --git a/tests/test_e2e.py b/tests/test_e2e.py index ba17f52c..54b816dc 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -2790,3 +2790,162 @@ def test_tool_call_get_buckets(self) -> None: self.alias, ) assert result.exit_code == 0 + + +# --------------------------------------------------------------------------- +# Job run variable values resolution (PR2 / P0-2) +# --------------------------------------------------------------------------- + + +@skip_without_credentials +@pytest.mark.e2e +class TestE2EJobRunVariableValues: + """Prove `kbagent job run` auto-resolves variableValuesId against a live API. + + Sets up a real `keboola.variables` config with one row and a parent + ex-http config whose `configuration.runtime.variables_id` points at it, + then runs `kbagent --json job run --no-wait` and asserts the + response's `resolvedVariableValuesId` matches the created row id. + + Also spot-checks the client path directly via `JobService.resolve_variable_values_id` + (pure resolver, no Queue dispatch) so a Queue outage would not mask a + resolver regression. + """ + + @pytest.fixture(autouse=True) + def setup(self, tmp_path: Path) -> None: + self.token = os.environ[ENV_TOKEN] + raw_url = os.environ.get(ENV_URL, "connection.keboola.com") + self.url = raw_url if raw_url.startswith("https://") else f"https://{raw_url}" + self.alias = f"{RUN_ID}-jobvars" + + self.config_dir = tmp_path / "config" + self.config_dir.mkdir() + + result = _invoke( + self.config_dir, + [ + "--json", + "project", + "add", + "--project", + self.alias, + "--url", + self.url, + "--token", + self.token, + ], + ) + assert result.exit_code == 0, f"project add failed: {result.output}" + + # Client for fixture setup / teardown. + self.client = KeboolaClient(stack_url=self.url, token=self.token) + + # Track created configs so teardown can delete them even on assert fail. + self._created: list[tuple[str, str]] = [] + + yield + + for component_id, config_id in reversed(self._created): + try: + self.client.delete_config(component_id=component_id, config_id=config_id) + except Exception as exc: + print( + f" {_DIM}(teardown) delete_config {component_id}/{config_id} failed: {exc}{_RESET}" + ) + self.client.close() + + def _create_fixture(self) -> tuple[str, str, str]: + """Create variables config + row + linked parent ex-http config. + + Returns ``(variables_config_id, variables_row_id, parent_config_id)``. + """ + vars_cfg = self.client.create_config( + component_id="keboola.variables", + name=f"{RUN_ID}-vars", + description="E2E PR2 fixture", + configuration={ + "variables": [{"name": "year_start", "type": "string"}], + }, + ) + vars_cfg_id = str(vars_cfg["id"]) + self._created.append(("keboola.variables", vars_cfg_id)) + + vars_row = self.client.create_config_row( + component_id="keboola.variables", + config_id=vars_cfg_id, + name="default", + configuration={"values": [{"name": "year_start", "value": "2016"}]}, + ) + vars_row_id = str(vars_row["id"]) + + parent_cfg = self.client.create_config( + component_id="keboola.ex-http", + name=f"{RUN_ID}-http-linked", + description="E2E PR2 linked parent", + configuration={ + "parameters": {"baseUrl": "https://example.com"}, + "runtime": {"variables_id": vars_cfg_id}, + }, + ) + parent_cfg_id = str(parent_cfg["id"]) + self._created.append(("keboola.ex-http", parent_cfg_id)) + + return vars_cfg_id, vars_row_id, parent_cfg_id + + def test_resolve_variable_values_id_live(self) -> None: + """Resolver reads runtime.variables_id + falls back to first row.""" + from keboola_agent_cli.services.job_service import JobService + + _step(1, "create variables + linked parent fixture") + _vars_id, vars_row_id, parent_id = self._create_fixture() + + _step(2, "resolve values row id via JobService") + resolved = JobService.resolve_variable_values_id( + client=self.client, + component_id="keboola.ex-http", + config_id=parent_id, + ) + print(f" {_DIM}resolved={resolved} expected={vars_row_id}{_RESET}") + assert resolved == vars_row_id + + def test_job_run_surfaces_resolved_variable_values_id(self) -> None: + """`kbagent job run --no-wait` returns resolvedVariableValuesId in --json. + + The job itself may fail at execution time (test token may not have + rights to run HTTP jobs or the URL may be unreachable). That is OK: + what we assert is that the resolver picked up the values row and + kbagent surfaced it before/with job submission. + """ + _step(1, "create variables + linked parent fixture") + _vars_id, vars_row_id, parent_id = self._create_fixture() + + _step(2, "kbagent --json job run (no --wait)") + result = _invoke( + self.config_dir, + [ + "--json", + "job", + "run", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + parent_id, + ], + ) + + data = _json(result) + payload = data.get("data", data) + print(f" {_DIM}resolvedVariableValuesId={payload.get('resolvedVariableValuesId')}{_RESET}") + assert payload.get("resolvedVariableValuesId") == vars_row_id + + # Clean up the job we just created (avoid wasted compute) if the + # Queue accepted it. Best-effort: ignore "not killable" transitions. + import contextlib + + job_id = payload.get("id") + if job_id: + with contextlib.suppress(Exception): + self.client.kill_job(str(job_id)) diff --git a/tests/test_services.py b/tests/test_services.py index 5a84e066..ed98e6ff 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -1741,6 +1741,8 @@ def test_run_job_without_branch(self, tmp_config_dir: Path) -> None: "status": "waiting", "component": "keboola.ex-http", } + # Parent config has no linked variables -- resolver returns None + mock_client.get_config_detail.return_value = {} service = JobService( config_store=store, @@ -1760,6 +1762,7 @@ def test_run_job_without_branch(self, tmp_config_dir: Path) -> None: config_id="42", config_row_ids=None, branch_id=None, + variable_values_id=None, ) mock_client.close.assert_called_once() @@ -1782,6 +1785,7 @@ def test_run_job_with_branch(self, tmp_config_dir: Path) -> None: "status": "waiting", "branchId": "789", } + mock_client.get_config_detail.return_value = {} service = JobService( config_store=store, @@ -1802,6 +1806,7 @@ def test_run_job_with_branch(self, tmp_config_dir: Path) -> None: config_id="100", config_row_ids=None, branch_id=789, + variable_values_id=None, ) def test_run_job_with_branch_and_wait(self, tmp_config_dir: Path) -> None: @@ -1824,6 +1829,7 @@ def test_run_job_with_branch_and_wait(self, tmp_config_dir: Path) -> None: "status": "success", "isFinished": True, } + mock_client.get_config_detail.return_value = {} service = JobService( config_store=store, @@ -1846,10 +1852,173 @@ def test_run_job_with_branch_and_wait(self, tmp_config_dir: Path) -> None: config_id="42", config_row_ids=None, branch_id=123, + variable_values_id=None, ) mock_client.wait_for_queue_job.assert_called_once_with("557", max_wait=60.0) +class TestJobServiceVariableValuesResolution: + """Tests for `resolve_variable_values_id` + auto-resolution in `run_job`. + + Locks the P0-2 contract: transformations with linked variables must + run against the deployed values row, not empty strings. + """ + + def _store(self, tmp_config_dir: Path) -> ConfigStore: + store = ConfigStore(config_dir=tmp_config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token="901-abc-defghijklmnopqrst", + project_name="Production", + project_id=1234, + ), + ) + return store + + def _service(self, store: ConfigStore, mock_client: MagicMock) -> JobService: + return JobService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + def test_resolve_uses_explicit_values_id_when_set(self) -> None: + """runtime.variables_values_id wins over first-row fallback.""" + mock_client = MagicMock() + mock_client.get_config_detail.return_value = { + "configuration": { + "runtime": { + "variables_id": "vars-cfg-42", + "variables_values_id": "row-explicit", + } + } + } + + result = JobService.resolve_variable_values_id( + client=mock_client, + component_id="keboola.snowflake-transformation", + config_id="100", + ) + + assert result == "row-explicit" + mock_client.list_config_rows.assert_not_called() + + def test_resolve_falls_back_to_first_row(self) -> None: + """When runtime.variables_id is set but values_id absent, use first row.""" + mock_client = MagicMock() + mock_client.get_config_detail.return_value = { + "configuration": {"runtime": {"variables_id": "vars-cfg-42"}} + } + mock_client.list_config_rows.return_value = [ + {"id": "row-first"}, + {"id": "row-second"}, + ] + + result = JobService.resolve_variable_values_id( + client=mock_client, + component_id="keboola.snowflake-transformation", + config_id="100", + branch_id=789, + ) + + assert result == "row-first" + mock_client.list_config_rows.assert_called_once_with( + component_id="keboola.variables", + config_id="vars-cfg-42", + branch_id=789, + ) + + def test_resolve_returns_none_when_no_variables_link(self) -> None: + """Config without runtime.variables_id → None (skip variableValuesId).""" + mock_client = MagicMock() + mock_client.get_config_detail.return_value = {"configuration": {"runtime": {}}} + + result = JobService.resolve_variable_values_id( + client=mock_client, + component_id="keboola.ex-http", + config_id="42", + ) + + assert result is None + mock_client.list_config_rows.assert_not_called() + + def test_resolve_raises_when_variables_has_zero_rows(self) -> None: + """Linked variables config with no rows → NO_VARIABLE_ROWS (fail fast).""" + mock_client = MagicMock() + mock_client.get_config_detail.return_value = { + "configuration": {"runtime": {"variables_id": "vars-cfg-42"}} + } + mock_client.list_config_rows.return_value = [] + + with pytest.raises(KeboolaApiError) as excinfo: + JobService.resolve_variable_values_id( + client=mock_client, + component_id="keboola.snowflake-transformation", + config_id="100", + ) + assert excinfo.value.error_code == "NO_VARIABLE_ROWS" + + def test_run_job_auto_resolves_values_id(self, tmp_config_dir: Path) -> None: + """run_job dispatches resolver output to create_job's variable_values_id.""" + store = self._store(tmp_config_dir) + mock_client = MagicMock() + mock_client.get_config_detail.return_value = { + "configuration": {"runtime": {"variables_id": "vars-cfg-42"}} + } + mock_client.list_config_rows.return_value = [{"id": "row-first"}] + mock_client.create_job.return_value = {"id": 700, "status": "waiting"} + + result = self._service(store, mock_client).run_job( + alias="prod", + component_id="keboola.snowflake-transformation", + config_id="100", + ) + + assert result["resolvedVariableValuesId"] == "row-first" + mock_client.create_job.assert_called_once_with( + component_id="keboola.snowflake-transformation", + config_id="100", + config_row_ids=None, + branch_id=None, + variable_values_id="row-first", + ) + + def test_run_job_explicit_override_wins(self, tmp_config_dir: Path) -> None: + """User-supplied --variable-values-id bypasses resolution entirely.""" + store = self._store(tmp_config_dir) + mock_client = MagicMock() + mock_client.create_job.return_value = {"id": 701, "status": "waiting"} + + self._service(store, mock_client).run_job( + alias="prod", + component_id="keboola.snowflake-transformation", + config_id="100", + variable_values_id="row-user-picked", + ) + + # Resolver short-circuited; no config detail fetch. + mock_client.get_config_detail.assert_not_called() + mock_client.list_config_rows.assert_not_called() + assert mock_client.create_job.call_args.kwargs["variable_values_id"] == "row-user-picked" + + def test_run_job_no_variables_skips_resolution(self, tmp_config_dir: Path) -> None: + """--no-variables short-circuits the resolver (no detail fetch).""" + store = self._store(tmp_config_dir) + mock_client = MagicMock() + mock_client.create_job.return_value = {"id": 702, "status": "waiting"} + + self._service(store, mock_client).run_job( + alias="prod", + component_id="keboola.snowflake-transformation", + config_id="100", + no_variables=True, + ) + + mock_client.get_config_detail.assert_not_called() + assert mock_client.create_job.call_args.kwargs["variable_values_id"] is None + + def _make_job_store_and_project(tmp_config_dir: Path, alias: str = "prod") -> ConfigStore: """Helper to register a single project so JobService.resolve_projects() works.""" store = ConfigStore(config_dir=tmp_config_dir) diff --git a/uv.lock b/uv.lock index 74824b07..4f2686fa 100644 --- a/uv.lock +++ b/uv.lock @@ -423,7 +423,7 @@ wheels = [ [[package]] name = "keboola-agent-cli" -version = "0.20.6" +version = "0.22.0" source = { editable = "." } dependencies = [ { name = "httpx" }, From ef68d82d0439271f3ea298bace3ca89619c5b0db Mon Sep 17 00:00:00 2001 From: Maxmilian Ottomansky Date: Mon, 20 Apr 2026 15:29:49 +0200 Subject: [PATCH 2/5] fix(job): variables link is root-level, not nested under runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three-agent review pass on PR2 converged on a resolver correctness bug: read path was `configuration.runtime.variables_id`, but real Keboola configs store the link at the root of `configuration` (snake_case, matching the on-disk format written by PR1's VariablesService). The previous E2E passed because its fixtures wrote the same wrong shape — the resolver and the fixture were consistently wrong, not right. Fixed: - Resolver now reads `configuration.variables_id` / `configuration.variables_values_id` at root. - Service/class docstrings call out the snake_case-at-root convention and cross-reference VariablesService. - Hint code sample + note match the actual path. - Service test mocks + E2E fixtures reshaped to root-level keys. - gotchas.md + context.py + commands/job.py docstring updated. Added: - `test_run_job_closes_client_when_resolver_raises` — locks the try/finally close contract on the NO_VARIABLE_ROWS path (best_practices.md §3 gap). - 4 live E2E tests: explicit override wins, --no-variables skips, NO_VARIABLE_ROWS error path, explicit `variables_values_id` pin. CLI/UX polish: - Clearer help strings on --variable-values-id / --no-variables (state WHAT + WHEN to use). - Directive mutual-exclusion error message. - Human-mode banner line when an explicit flag is used. - Hint note mentioning NO_VARIABLE_ROWS exit behavior. Verification: 24/24 targeted tests (client 5 + service 11 + CLI 8), 6/6 live E2E against project 1143, ruff clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../skills/kbagent/references/gotchas.md | 5 +- src/keboola_agent_cli/commands/context.py | 8 +- src/keboola_agent_cli/commands/job.py | 30 ++- .../hints/definitions/job.py | 9 +- src/keboola_agent_cli/services/job_service.py | 34 +-- tests/test_e2e.py | 194 +++++++++++++++++- tests/test_services.py | 45 +++- 7 files changed, 280 insertions(+), 45 deletions(-) diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 68744062..a8928a45 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -4,8 +4,9 @@ Transformations with linked `keboola.variables` used to run against empty strings unless the caller hand-wired a `variableValuesId` at the HTTP layer. `kbagent job run` -now auto-resolves it: reads `configuration.runtime.variables_id` from the parent -config, picks `variables_values_id` if set, else the first row of the linked +now auto-resolves it: reads `configuration.variables_id` from the parent config +(root of the configuration body — same key `VariablesService` writes), picks +`configuration.variables_values_id` if set, else the first row of the linked variables config. **Override knobs:** diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 9631ca40..c61f5f09 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -130,10 +130,10 @@ kbagent job run --project NAME --component-id ID --config-id ID [--row-id ID ...] [--wait] [--timeout N] [--branch ID] [--variable-values-id ID] [--no-variables] Run a Queue API job. --row-id selects specific config rows (repeatable; omit to run entire config). --wait polls until job finishes. --timeout sets max wait in seconds (default 300). Branch-aware. - When the config has linked variables (runtime.variables_id), kbagent auto-resolves a - variableValuesId so the job binds to the deployed values row. --variable-values-id overrides; - --no-variables skips resolution. Error code NO_VARIABLE_ROWS when the linked variables - config has zero rows (run `kbagent config variables-set` to create one). + When the config has linked variables (configuration.variables_id), kbagent auto-resolves + a variableValuesId so the job binds to the deployed values row. --variable-values-id + overrides; --no-variables skips resolution. Error code NO_VARIABLE_ROWS when the linked + variables config has zero rows (run `kbagent config variables-set` to create one). kbagent job terminate --project NAME (--job-id ID [--job-id ID ...] | --status any|created|waiting|processing [--component-id ID] [--config-id ID] [--branch ID] [--limit N]) [--dry-run] [--yes] Kill running jobs via Queue API (POST /jobs/{id}/kill). Use to stop runaway loops or pile-ups. diff --git a/src/keboola_agent_cli/commands/job.py b/src/keboola_agent_cli/commands/job.py index ae1180f0..bbbb8963 100644 --- a/src/keboola_agent_cli/commands/job.py +++ b/src/keboola_agent_cli/commands/job.py @@ -195,16 +195,20 @@ def job_run( None, "--variable-values-id", help=( - "Row id of the linked keboola.variables values row. Overrides " - "auto-resolution. Mutually exclusive with --no-variables." + "Explicit keboola.variables values-row ID to bind. Use when the " + "linked variables config has multiple rows and auto-resolution " + "(first row) picks the wrong one. Mutually exclusive with " + "--no-variables." ), ), no_variables: bool = typer.Option( False, "--no-variables", help=( - "Skip variable-values resolution. Use for configs without linked " - "variables or when intentionally running against empty bindings." + "Skip variable-values resolution entirely. Use for components " + "that do not support variables, or when intentionally running " + "against empty bindings. Mutually exclusive with " + "--variable-values-id." ), ), ) -> None: @@ -216,10 +220,10 @@ def job_run( When a dev branch is active (via 'branch use'), the job automatically runs on that branch. Use --branch to override. - When the config has linked variables (runtime.variables_id), kbagent - auto-resolves a variableValuesId so the job binds to the deployed - values row. Override with --variable-values-id or skip with - --no-variables. + When the config has linked variables (configuration.variables_id), + kbagent auto-resolves a variableValuesId so the job binds to the + deployed values row. Override with --variable-values-id or skip + with --no-variables. """ if should_hint(ctx): emit_hint( @@ -242,7 +246,11 @@ def job_run( if variable_values_id and no_variables: formatter.error( - message="--variable-values-id and --no-variables are mutually exclusive.", + message=( + "--variable-values-id and --no-variables are mutually exclusive. " + "Pass --variable-values-id to bind a specific values row, or " + "--no-variables to skip resolution, but not both." + ), error_code="INVALID_ARGUMENT", ) raise typer.Exit(code=2) @@ -260,6 +268,10 @@ def job_run( msg += f" [dim](waiting up to {timeout:.0f}s)[/dim]" msg += "..." formatter.console.print(msg) + if variable_values_id: + formatter.console.print(f"[dim]Using variable values row: {variable_values_id}[/dim]") + elif no_variables: + formatter.console.print("[dim]Skipping variable-values resolution.[/dim]") try: result = service.run_job( diff --git a/src/keboola_agent_cli/hints/definitions/job.py b/src/keboola_agent_cli/hints/definitions/job.py index 29d01e86..1da64866 100644 --- a/src/keboola_agent_cli/hints/definitions/job.py +++ b/src/keboola_agent_cli/hints/definitions/job.py @@ -96,7 +96,7 @@ method="list_config_rows", args={ "component_id": '"keboola.variables"', - "config_id": 'detail["configuration"]["runtime"]["variables_id"]', + "config_id": 'detail["configuration"]["variables_id"]', }, result_var="var_rows", result_hint="list", @@ -149,10 +149,15 @@ "Service layer handles both resolve + create + optional poll in one call.", ( "Service auto-resolves variableValuesId from " - "configuration.runtime.variables_id; the client hint shows " + "configuration.variables_id; the client hint shows " "the underlying two-request pattern." ), "Pass --no-variables to skip resolution entirely.", + ( + "NO_VARIABLE_ROWS (exit 1) means the linked variables config " + "has zero rows; fix via `kbagent config variables-set` or " + "pass --no-variables." + ), ], ) ) diff --git a/src/keboola_agent_cli/services/job_service.py b/src/keboola_agent_cli/services/job_service.py index 14d329e9..fdf83086 100644 --- a/src/keboola_agent_cli/services/job_service.py +++ b/src/keboola_agent_cli/services/job_service.py @@ -1,7 +1,8 @@ -"""Job listing service - business logic for listing jobs from Queue API. +"""Job service - business logic for Queue API jobs. -Orchestrates multi-project job retrieval in parallel, filtering, -and aggregation without knowing about CLI or HTTP details. +Covers listing (multi-project parallel retrieval + filtering), detail +fetch, creation with optional wait, termination, and variable-values +resolution. Stays agnostic of CLI and HTTP transport details. """ from typing import Any @@ -13,7 +14,7 @@ class JobService(BaseService): - """Business logic for listing Keboola jobs from the Queue API. + """Business logic for Keboola jobs (list, detail, run, terminate). Supports multi-project aggregation: queries multiple projects in parallel using ThreadPoolExecutor, collects results, and reports per-project errors @@ -162,7 +163,7 @@ def run_job( ) -> dict[str, Any]: """Create and optionally wait for a Queue API job. - When the config has linked variables (``configuration.runtime.variables_id``), + When the config has linked variables (``configuration.variables_id``), auto-resolve a ``variableValuesId`` via :meth:`resolve_variable_values_id` so the job runs against the deployed values row instead of empty strings. Pass ``variable_values_id`` to override, or ``no_variables=True`` @@ -230,15 +231,20 @@ def resolve_variable_values_id( ) -> str | None: """Resolve the variables values row id to pass to ``create_job``. - Reads the parent config's ``configuration.runtime.variables_id`` to - find the linked ``keboola.variables`` config. Prefers the explicit - ``configuration.runtime.variables_values_id`` when set; otherwise - falls back to the first row of the linked variables config (the - default-row convention). + Reads the parent config's ``configuration.variables_id`` to find the + linked ``keboola.variables`` config. Prefers the explicit + ``configuration.variables_values_id`` when set; otherwise falls back + to the first row of the linked variables config (the default-row + convention). + + Note the snake_case keys: Keboola stores the variables link at the + root of the ``configuration`` body as ``variables_id`` / + ``variables_values_id`` (matching the on-disk format written by + ``VariablesService``), NOT nested under a ``runtime`` object. Returns: The values row id, or ``None`` if the parent config has no - linked variables (no ``runtime.variables_id``). + linked variables (no ``variables_id``). Raises: KeboolaApiError: ``NO_VARIABLE_ROWS`` when the linked variables @@ -250,12 +256,12 @@ def resolve_variable_values_id( config_id=config_id, branch_id=branch_id, ) - runtime = (detail.get("configuration") or {}).get("runtime") or {} - variables_id = runtime.get("variables_id") + configuration = detail.get("configuration") or {} + variables_id = configuration.get("variables_id") if not variables_id: return None - explicit_values_id = runtime.get("variables_values_id") + explicit_values_id = configuration.get("variables_values_id") if explicit_values_id: return str(explicit_values_id) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 54b816dc..dd522a3e 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -2803,7 +2803,7 @@ class TestE2EJobRunVariableValues: """Prove `kbagent job run` auto-resolves variableValuesId against a live API. Sets up a real `keboola.variables` config with one row and a parent - ex-http config whose `configuration.runtime.variables_id` points at it, + ex-http config whose `configuration.variables_id` points at it, then runs `kbagent --json job run --no-wait` and asserts the response's `resolvedVariableValuesId` matches the created row id. @@ -2885,7 +2885,7 @@ def _create_fixture(self) -> tuple[str, str, str]: description="E2E PR2 linked parent", configuration={ "parameters": {"baseUrl": "https://example.com"}, - "runtime": {"variables_id": vars_cfg_id}, + "variables_id": vars_cfg_id, }, ) parent_cfg_id = str(parent_cfg["id"]) @@ -2894,7 +2894,7 @@ def _create_fixture(self) -> tuple[str, str, str]: return vars_cfg_id, vars_row_id, parent_cfg_id def test_resolve_variable_values_id_live(self) -> None: - """Resolver reads runtime.variables_id + falls back to first row.""" + """Resolver reads configuration.variables_id + falls back to first row.""" from keboola_agent_cli.services.job_service import JobService _step(1, "create variables + linked parent fixture") @@ -2949,3 +2949,191 @@ def test_job_run_surfaces_resolved_variable_values_id(self) -> None: if job_id: with contextlib.suppress(Exception): self.client.kill_job(str(job_id)) + + def test_job_run_explicit_override_wins_over_resolver(self) -> None: + """`--variable-values-id ROW_ID` bypasses the resolver and lands in the job. + + Creates a fixture with TWO values rows (default + alt). Without + --variable-values-id, the resolver picks the first row. With + --variable-values-id set to the SECOND row's id, the service must + use the user's choice, and we assert `resolvedVariableValuesId` + (really: echoed-back) matches the override, not the first row. + """ + import contextlib + + _step(1, "create variables + 2 values rows + linked parent") + vars_cfg_id, default_row_id, parent_id = self._create_fixture() + + # Add a second row and use its id as the override. + alt_row = self.client.create_config_row( + component_id="keboola.variables", + config_id=vars_cfg_id, + name="alt", + configuration={"values": [{"name": "year_start", "value": "2020"}]}, + ) + alt_row_id = str(alt_row["id"]) + assert alt_row_id != default_row_id + + _step(2, "kbagent job run --variable-values-id ") + result = _invoke( + self.config_dir, + [ + "--json", + "job", + "run", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + parent_id, + "--variable-values-id", + alt_row_id, + ], + ) + + data = _json(result) + payload = data.get("data", data) + assert payload.get("resolvedVariableValuesId") == alt_row_id + job_id = payload.get("id") + if job_id: + with contextlib.suppress(Exception): + self.client.kill_job(str(job_id)) + + def test_job_run_no_variables_skips_resolution(self) -> None: + """`--no-variables` suppresses the resolver; no `resolvedVariableValuesId` surfaces. + + Locks the opt-out contract: a component that happens to have a + linked variables config can still be run without variable binding + when the caller explicitly asks (e.g. manual debug runs). + """ + import contextlib + + _step(1, "create variables + linked parent fixture") + _vars_id, _row_id, parent_id = self._create_fixture() + + _step(2, "kbagent job run --no-variables") + result = _invoke( + self.config_dir, + [ + "--json", + "job", + "run", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + parent_id, + "--no-variables", + ], + ) + + data = _json(result) + payload = data.get("data", data) + # Key omitted entirely when resolution was skipped. + assert "resolvedVariableValuesId" not in payload + job_id = payload.get("id") + if job_id: + with contextlib.suppress(Exception): + self.client.kill_job(str(job_id)) + + def test_job_run_no_variable_rows_surfaces_error_code(self) -> None: + """Linked variables config with zero rows exits with `NO_VARIABLE_ROWS`. + + Agent-facing contract: when a transformation is hooked up to a + variables config that has not yet had any row created, kbagent + must fail fast rather than submitting a job that will silently + bind empty strings at runtime. + """ + _step(1, "create empty variables config + linked parent (no rows)") + # Variables config WITHOUT any row. + vars_cfg = self.client.create_config( + component_id="keboola.variables", + name=f"{RUN_ID}-empty-vars", + description="E2E PR2: empty values", + configuration={"variables": [{"name": "year_start", "type": "string"}]}, + ) + vars_cfg_id = str(vars_cfg["id"]) + self._created.append(("keboola.variables", vars_cfg_id)) + + parent_cfg = self.client.create_config( + component_id="keboola.ex-http", + name=f"{RUN_ID}-http-empty-link", + description="E2E PR2: parent with empty-variables link", + configuration={ + "parameters": {"baseUrl": "https://example.com"}, + "variables_id": vars_cfg_id, + }, + ) + parent_id = str(parent_cfg["id"]) + self._created.append(("keboola.ex-http", parent_id)) + + _step(2, "kbagent job run -> expect NO_VARIABLE_ROWS") + result = _invoke( + self.config_dir, + [ + "--json", + "job", + "run", + "--project", + self.alias, + "--component-id", + "keboola.ex-http", + "--config-id", + parent_id, + ], + ) + + assert result.exit_code != 0 + try: + data = json.loads(result.output) + except json.JSONDecodeError: + pytest.fail(f"Expected JSON error output, got: {result.output}") + assert data.get("status") == "error" + assert data.get("error", {}).get("code") == "NO_VARIABLE_ROWS" + + def test_resolver_prefers_explicit_values_id_over_first_row(self) -> None: + """`configuration.variables_values_id` wins over first-row fallback. + + Directly tests the resolver short-circuit path without touching the + Queue API. If a config has pinned a specific values row via the + Keboola UI or sync push, kbagent must honor that selection even + when additional rows exist. + """ + from keboola_agent_cli.services.job_service import JobService + + _step(1, "create variables + 2 rows; parent pins the SECOND row") + vars_cfg_id, first_row_id, _ = self._create_fixture() + + alt_row = self.client.create_config_row( + component_id="keboola.variables", + config_id=vars_cfg_id, + name="pinned", + configuration={"values": [{"name": "year_start", "value": "2025"}]}, + ) + pinned_row_id = str(alt_row["id"]) + assert pinned_row_id != first_row_id + + # Patch the parent to point at the pinned row explicitly. + pinned_parent = self.client.create_config( + component_id="keboola.ex-http", + name=f"{RUN_ID}-http-pinned", + description="E2E PR2: parent pinned to specific values row", + configuration={ + "parameters": {"baseUrl": "https://example.com"}, + "variables_id": vars_cfg_id, + "variables_values_id": pinned_row_id, + }, + ) + pinned_parent_id = str(pinned_parent["id"]) + self._created.append(("keboola.ex-http", pinned_parent_id)) + + _step(2, "resolver returns the pinned row, NOT the first row") + resolved = JobService.resolve_variable_values_id( + client=self.client, + component_id="keboola.ex-http", + config_id=pinned_parent_id, + ) + print(f" {_DIM}resolved={resolved} pinned={pinned_row_id} first={first_row_id}{_RESET}") + assert resolved == pinned_row_id diff --git a/tests/test_services.py b/tests/test_services.py index ed98e6ff..2b7f855d 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -1884,14 +1884,12 @@ def _service(self, store: ConfigStore, mock_client: MagicMock) -> JobService: ) def test_resolve_uses_explicit_values_id_when_set(self) -> None: - """runtime.variables_values_id wins over first-row fallback.""" + """configuration.variables_values_id wins over first-row fallback.""" mock_client = MagicMock() mock_client.get_config_detail.return_value = { "configuration": { - "runtime": { - "variables_id": "vars-cfg-42", - "variables_values_id": "row-explicit", - } + "variables_id": "vars-cfg-42", + "variables_values_id": "row-explicit", } } @@ -1905,10 +1903,10 @@ def test_resolve_uses_explicit_values_id_when_set(self) -> None: mock_client.list_config_rows.assert_not_called() def test_resolve_falls_back_to_first_row(self) -> None: - """When runtime.variables_id is set but values_id absent, use first row.""" + """When configuration.variables_id is set but values_id absent, use first row.""" mock_client = MagicMock() mock_client.get_config_detail.return_value = { - "configuration": {"runtime": {"variables_id": "vars-cfg-42"}} + "configuration": {"variables_id": "vars-cfg-42"} } mock_client.list_config_rows.return_value = [ {"id": "row-first"}, @@ -1930,9 +1928,9 @@ def test_resolve_falls_back_to_first_row(self) -> None: ) def test_resolve_returns_none_when_no_variables_link(self) -> None: - """Config without runtime.variables_id → None (skip variableValuesId).""" + """Config without configuration.variables_id → None (skip variableValuesId).""" mock_client = MagicMock() - mock_client.get_config_detail.return_value = {"configuration": {"runtime": {}}} + mock_client.get_config_detail.return_value = {"configuration": {}} result = JobService.resolve_variable_values_id( client=mock_client, @@ -1947,7 +1945,7 @@ def test_resolve_raises_when_variables_has_zero_rows(self) -> None: """Linked variables config with no rows → NO_VARIABLE_ROWS (fail fast).""" mock_client = MagicMock() mock_client.get_config_detail.return_value = { - "configuration": {"runtime": {"variables_id": "vars-cfg-42"}} + "configuration": {"variables_id": "vars-cfg-42"} } mock_client.list_config_rows.return_value = [] @@ -1964,7 +1962,7 @@ def test_run_job_auto_resolves_values_id(self, tmp_config_dir: Path) -> None: store = self._store(tmp_config_dir) mock_client = MagicMock() mock_client.get_config_detail.return_value = { - "configuration": {"runtime": {"variables_id": "vars-cfg-42"}} + "configuration": {"variables_id": "vars-cfg-42"} } mock_client.list_config_rows.return_value = [{"id": "row-first"}] mock_client.create_job.return_value = {"id": 700, "status": "waiting"} @@ -2018,6 +2016,31 @@ def test_run_job_no_variables_skips_resolution(self, tmp_config_dir: Path) -> No mock_client.get_config_detail.assert_not_called() assert mock_client.create_job.call_args.kwargs["variable_values_id"] is None + def test_run_job_closes_client_when_resolver_raises(self, tmp_config_dir: Path) -> None: + """NO_VARIABLE_ROWS raised by the resolver inside run_job still closes the client. + + Locks the try/finally contract (best_practices.md §3): every error path + that flows out of run_job -- including the resolver raising before + create_job -- must release the HTTP client. + """ + store = self._store(tmp_config_dir) + mock_client = MagicMock() + mock_client.get_config_detail.return_value = { + "configuration": {"variables_id": "vars-cfg-42"} + } + mock_client.list_config_rows.return_value = [] # triggers NO_VARIABLE_ROWS + + with pytest.raises(KeboolaApiError) as excinfo: + self._service(store, mock_client).run_job( + alias="prod", + component_id="keboola.snowflake-transformation", + config_id="100", + ) + + assert excinfo.value.error_code == "NO_VARIABLE_ROWS" + mock_client.create_job.assert_not_called() + mock_client.close.assert_called_once() + def _make_job_store_and_project(tmp_config_dir: Path, alias: str = "prod") -> ConfigStore: """Helper to register a single project so JobService.resolve_projects() works.""" From 6951af8d133b01955baaeb5e91afcef2f7825457 Mon Sep 17 00:00:00 2001 From: Maxmilian Ottomansky Date: Mon, 20 Apr 2026 15:41:08 +0200 Subject: [PATCH 3/5] fix(job): escape() user-controlled id in banner + update commands-reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps caught on a second pass through PR2: 1. `rich.markup.escape()` missing on `variable_values_id` in the new human-mode banner line. Direct violation of best_practices.md §9.1 (the paid-for Rich markup injection bug from the max-project-description review loop). A user passing `--variable-values-id "[red]X[/red]"` would have seen their id rendered as markup. 2. `plugins/kbagent/skills/kbagent/references/commands-reference.md` still listed `job run` with the pre-PR2 signature. Updated with the two new flags + a one-line explanation of the auto-resolve contract and NO_VARIABLE_ROWS error code. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../kbagent/skills/kbagent/references/commands-reference.md | 2 +- src/keboola_agent_cli/commands/job.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index ef6fd86a..60c70d2a 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -39,7 +39,7 @@ All commands support `--json` for structured output. Multi-project flags (`--pro ## Job History - `job list [--project NAME] [--component-id ID] [--config-id ID] [--status STATUS] [--limit N]` -- list jobs (default 50, max 500) - `job detail --project NAME --job-id ID` -- full job detail with timing and result message -- `job run --project NAME --component-id ID --config-id ID [--row-id ID ...] [--wait] [--timeout N] [--branch ID]` -- run a job, optionally wait for completion (branch-aware) +- `job run --project NAME --component-id ID --config-id ID [--row-id ID ...] [--wait] [--timeout N] [--branch ID] [--variable-values-id ID] [--no-variables]` -- run a job, optionally wait for completion (branch-aware). For configs with linked `keboola.variables` (root-level `configuration.variables_id`), kbagent auto-resolves a `variableValuesId` so transformations bind to the deployed values row. `--variable-values-id` overrides; `--no-variables` skips resolution. `NO_VARIABLE_ROWS` when the linked variables config has zero rows -- fix via `kbagent config variables-set`. - `job terminate --project NAME (--job-id ID [--job-id ...] | --status any|created|waiting|processing [--component-id ID] [--config-id ID] [--branch ID] [--limit N]) [--dry-run] [--yes]` -- kill running Queue API jobs. Use to stop runaway loops or clean up pile-ups from repeated `job run` calls. Two modes: by ID (single/batch) or by filter (`--status any` catches every killable state). Response partitions IDs into `killed / already_finished / not_found / failed`; safe to re-run idempotently. Kill is async -- poll `job detail` for `isFinished=true`. ## Storage diff --git a/src/keboola_agent_cli/commands/job.py b/src/keboola_agent_cli/commands/job.py index bbbb8963..0609f9b2 100644 --- a/src/keboola_agent_cli/commands/job.py +++ b/src/keboola_agent_cli/commands/job.py @@ -269,7 +269,11 @@ def job_run( msg += "..." formatter.console.print(msg) if variable_values_id: - formatter.console.print(f"[dim]Using variable values row: {variable_values_id}[/dim]") + from rich.markup import escape + + formatter.console.print( + f"[dim]Using variable values row: {escape(variable_values_id)}[/dim]" + ) elif no_variables: formatter.console.print("[dim]Skipping variable-values resolution.[/dim]") From b91e42246a0ad64fc9825cb6399824aacb9c4853 Mon Sep 17 00:00:00 2001 From: Maxmilian Ottomansky Date: Mon, 20 Apr 2026 17:00:13 +0200 Subject: [PATCH 4/5] harden(job): MALFORMED_VARIABLES_ROW + reject empty --variable-values-id Audited PR2 for the same silent-failure class that PR1's reviewer caught (shape-based skip in encrypt_secrets_in_config). Found two analogs and closed both: 1. If the Storage API returns a first row without a usable `id`, the resolver previously returned `""`, which the client treats as "omit from Queue body" -- silently running the job with empty bindings. Now raises `MALFORMED_VARIABLES_ROW` (fail loud). 2. `--variable-values-id ""` (or whitespace) was falsy for the CLI's truthy check, fell through to the service, and got passed as `""` to `create_job` -- same silent omission. Now rejected at the CLI layer with `INVALID_ARGUMENT` (exit 2). Tests: 2 service (no-id, empty-id-string) + 2 CLI (empty, whitespace). Also: `.gitignore` now matches `.env.*` (was just `.env.local`) so E2E credential files cannot be committed by accident. Previously the credential file slipped into the initial push of this commit; this amend scrubs it from the branch tip. Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 2 +- src/keboola_agent_cli/changelog.py | 2 + src/keboola_agent_cli/commands/job.py | 10 +++ src/keboola_agent_cli/services/job_service.py | 20 ++++- tests/test_cli.py | 90 +++++++++++++++++++ tests/test_services.py | 39 ++++++++ 6 files changed, 161 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8e18444d..c4b6be52 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,7 @@ ENV/ # Environment variables .env -.env.local +.env.* # IDE .idea/ diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index d3327993..bbb22fba 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -13,6 +13,8 @@ "New: `--variable-values-id ID` to override the auto-resolved values row.", "New: `--no-variables` to skip resolution entirely (mutually exclusive with `--variable-values-id`).", "New: `NO_VARIABLE_ROWS` error code when a linked variables config has zero rows (misconfiguration; surface via `--json` error payload or fix via `kbagent config variables-set`).", + "New: `MALFORMED_VARIABLES_ROW` error code when the Storage API returns a first row without a usable `id` -- fails loud rather than submitting a job with empty variable bindings.", + 'Reject: `--variable-values-id ""` (empty or whitespace) returns exit 2 / `INVALID_ARGUMENT` instead of silently skipping the Queue body field.', "Client: `create_job` gained `variable_values_id` parameter; omitted from body when unset so existing callers retain wire-level compatibility.", ], "0.20.6": [ diff --git a/src/keboola_agent_cli/commands/job.py b/src/keboola_agent_cli/commands/job.py index 0609f9b2..15450f46 100644 --- a/src/keboola_agent_cli/commands/job.py +++ b/src/keboola_agent_cli/commands/job.py @@ -244,6 +244,16 @@ def job_run( service = get_service(ctx, "job_service") config_store: ConfigStore = ctx.obj["config_store"] + if variable_values_id is not None and not variable_values_id.strip(): + formatter.error( + message=( + "--variable-values-id cannot be empty or whitespace. " + "Pass a row id, or omit the flag to auto-resolve the default row." + ), + error_code="INVALID_ARGUMENT", + ) + raise typer.Exit(code=2) + if variable_values_id and no_variables: formatter.error( message=( diff --git a/src/keboola_agent_cli/services/job_service.py b/src/keboola_agent_cli/services/job_service.py index fdf83086..8ef781e3 100644 --- a/src/keboola_agent_cli/services/job_service.py +++ b/src/keboola_agent_cli/services/job_service.py @@ -280,7 +280,25 @@ def resolve_variable_values_id( status_code=0, error_code="NO_VARIABLE_ROWS", ) - return str(rows[0].get("id", "")) + + # Defense against a malformed Storage API response: a row without a + # usable `id` would otherwise be returned as `""`, which the client + # treats as "omit from Queue body" -- silently running the job with + # empty variable bindings. Fail loud instead (same silent-skip class + # as the PR1 encryption asymmetry bug). + first_row = rows[0] if isinstance(rows[0], dict) else {} + first_row_id = first_row.get("id") + if not first_row_id: + raise KeboolaApiError( + message=( + f"First row of variables config {variables_id} has no 'id' -- " + f"malformed Storage API response. Refusing to submit a job " + f"with empty variable bindings." + ), + status_code=0, + error_code="MALFORMED_VARIABLES_ROW", + ) + return str(first_row_id) def resolve_job_ids_by_filter( self, diff --git a/tests/test_cli.py b/tests/test_cli.py index d0350221..c3d84fb9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2375,6 +2375,96 @@ def test_job_run_mutually_exclusive_flags_rejected(self, tmp_path: Path) -> None job_service.run_job.assert_not_called() assert "INVALID_ARGUMENT" in result.output + def test_job_run_rejects_empty_variable_values_id(self, tmp_path: Path) -> None: + """`--variable-values-id ""` exits 2 with INVALID_ARGUMENT. + + Locks the fail-loud contract: an empty string must not fall through + to create_job(variable_values_id="") where it would silently be + omitted from the Queue body (same silent-skip class as the PR1 + encryption asymmetry). + """ + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config_test( + config_dir, + {"prod": {"token": "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k"}}, + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + ): + MockStore.return_value = store + job_service = MagicMock() + MockJobService.return_value = job_service + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + + result = runner.invoke( + app, + [ + "--json", + "job", + "run", + "--project", + "prod", + "--component-id", + "keboola.ex-http", + "--config-id", + "42", + "--variable-values-id", + "", + ], + ) + + assert result.exit_code == 2 + job_service.run_job.assert_not_called() + assert "INVALID_ARGUMENT" in result.output + assert "empty" in result.output.lower() + + def test_job_run_rejects_whitespace_variable_values_id(self, tmp_path: Path) -> None: + """Whitespace-only `--variable-values-id " "` also rejected.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config_test( + config_dir, + {"prod": {"token": "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k"}}, + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + ): + MockStore.return_value = store + job_service = MagicMock() + MockJobService.return_value = job_service + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + + result = runner.invoke( + app, + [ + "--json", + "job", + "run", + "--project", + "prod", + "--component-id", + "keboola.ex-http", + "--config-id", + "42", + "--variable-values-id", + " ", + ], + ) + + assert result.exit_code == 2 + job_service.run_job.assert_not_called() + def test_job_run_no_variable_rows_error(self, tmp_path: Path) -> None: """Service raises NO_VARIABLE_ROWS → CLI exits 1 with error_code surfaced.""" from keboola_agent_cli.errors import KeboolaApiError diff --git a/tests/test_services.py b/tests/test_services.py index 2b7f855d..1badbfba 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -1957,6 +1957,45 @@ def test_resolve_raises_when_variables_has_zero_rows(self) -> None: ) assert excinfo.value.error_code == "NO_VARIABLE_ROWS" + def test_resolve_raises_when_first_row_has_no_id(self) -> None: + """Malformed first row (no ``id`` field) -> MALFORMED_VARIABLES_ROW. + + Locks the fail-loud contract: if the Storage API ever returns a row + without a usable ``id``, the resolver must refuse rather than + returning ``""`` and letting the Queue body quietly omit + ``variableValuesId`` -- same silent-skip class as the PR1 + encryption-asymmetry bug. + """ + mock_client = MagicMock() + mock_client.get_config_detail.return_value = { + "configuration": {"variables_id": "vars-cfg-42"} + } + mock_client.list_config_rows.return_value = [{"name": "default"}] + + with pytest.raises(KeboolaApiError) as excinfo: + JobService.resolve_variable_values_id( + client=mock_client, + component_id="keboola.snowflake-transformation", + config_id="100", + ) + assert excinfo.value.error_code == "MALFORMED_VARIABLES_ROW" + + def test_resolve_raises_when_first_row_id_is_empty_string(self) -> None: + """Row with ``id=""`` also triggers MALFORMED_VARIABLES_ROW.""" + mock_client = MagicMock() + mock_client.get_config_detail.return_value = { + "configuration": {"variables_id": "vars-cfg-42"} + } + mock_client.list_config_rows.return_value = [{"id": "", "name": "default"}] + + with pytest.raises(KeboolaApiError) as excinfo: + JobService.resolve_variable_values_id( + client=mock_client, + component_id="keboola.snowflake-transformation", + config_id="100", + ) + assert excinfo.value.error_code == "MALFORMED_VARIABLES_ROW" + def test_run_job_auto_resolves_values_id(self, tmp_config_dir: Path) -> None: """run_job dispatches resolver output to create_job's variable_values_id.""" store = self._store(tmp_config_dir) From c7b7ad5dd22c1c82a7af6edc4704e2b57f52efa9 Mon Sep 17 00:00:00 2001 From: Petr Date: Mon, 20 Apr 2026 23:51:09 +0200 Subject: [PATCH 5/5] docs(plugin): extend variables-workflow with runtime/job-run section Covers the other half of the variables loop: deploying values is only useful if the Queue job binds to them at runtime. Adds an end-to-end example (variables-set -> job run -> inspect resolvedVariableValuesId), the resolution order with root-level convention note, override-knob mutual exclusion, error-code table (NO_VARIABLE_ROWS / MALFORMED_VARIABLES_ROW / INVALID_ARGUMENT), and the JSON response shape. Keeps gotchas.md as the quick-reference; this file is where agents go for the whole mental model. --- .../kbagent/references/variables-workflow.md | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/plugins/kbagent/skills/kbagent/references/variables-workflow.md b/plugins/kbagent/skills/kbagent/references/variables-workflow.md index 573dc8d5..67988823 100644 --- a/plugins/kbagent/skills/kbagent/references/variables-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/variables-workflow.md @@ -184,3 +184,93 @@ Both are supported; pick whichever fits your workflow: They converge on the same API calls; `sync push` just routes through local YAML first. + +## Running jobs against deployed values + +Deploying values is only half the loop. The other half is making sure the +Queue job actually *binds* to them at runtime. `kbagent job run` auto-resolves +a `variableValuesId` and passes it to the Queue API -- without this, a +transformation linked to a `keboola.variables` config runs against empty +`{{ placeholder }}` strings (the most common silent-fail mode in pipelines +that use variables). + +### End-to-end example + +```bash +# 1. Deploy values +kbagent config variables-set --project prod \ + --component-id keboola.snowflake-transformation --config-id 15815157 \ + --var year_start=2025 --var region=eu + +# 2. Run the job -- kbagent auto-resolves values row from the parent's link +kbagent --json job run --project prod \ + --component-id keboola.snowflake-transformation --config-id 15815157 \ + --wait +``` + +Inspect the JSON output for `resolvedVariableValuesId` to verify the binding +without a second `job detail` round-trip. + +### Resolution order + +`JobService.resolve_variable_values_id` picks, in this order: + +1. **Explicit `--variable-values-id ROW_ID`** -- hand-picks a values row. + Use for CI matrix runs ("run this job against each of our 5 environment + values rows") or what-if analysis. +2. **`configuration.variables_values_id`** on the parent config -- if set, + this is the pin that `variables-set` and the Keboola UI both write. +3. **First row** of the linked `keboola.variables` config -- the default-row + convention. + +The link is read from the **root** of the `configuration` body as +`variables_id` / `variables_values_id` (snake_case). NOT nested under a +`runtime` key -- that's a misconception an earlier draft of this feature had. + +### Override knobs + +- `--variable-values-id ROW_ID` -- pin a specific values row (overrides + steps 2 and 3 above). +- `--no-variables` -- skip resolution entirely. Use for components that have + no linked variables config (no-op), or when you intentionally want the job + to run with empty bindings. + +They are **mutually exclusive**. Passing both returns exit 2 / +`INVALID_ARGUMENT` before any API call. + +Empty or whitespace-only `--variable-values-id ""` is rejected at the CLI +layer with the same `INVALID_ARGUMENT`; passing through as `""` would +silently drop `variableValuesId` from the Queue body and reintroduce the +empty-bindings silent failure. + +### Error codes + +| Code | When | Recovery | +|---|---|---| +| `NO_VARIABLE_ROWS` | Linked `keboola.variables` config exists but has zero rows | `kbagent config variables-set --var KEY=VALUE` | +| `MALFORMED_VARIABLES_ROW` | Storage API returned a first row without a usable `id` | Inspect the backing config; fix or pin a specific row via `--variable-values-id` | +| `INVALID_ARGUMENT` | Mutually-exclusive flags, empty `--variable-values-id ""` | Fix CLI args | + +`NO_VARIABLE_ROWS` is the common-case signal that deploy is missing: the +parent was linked (via `sync push` or legacy UI) but the values config was +never populated. The fix is always `variables-set` on the parent. + +### Response shape (`--json`) + +```json +{ + "status": "ok", + "data": { + "project_alias": "prod", + "job_id": "9876543210", + "status": "processing", + "resolvedVariableValuesId": "01kpn7sat48jmhvx20svaqdnf9", + ... + } +} +``` + +`resolvedVariableValuesId` is present only when the resolver actually fired +(i.e., not for `--no-variables`, and not for configs without a +`variables_id` link). Absence of the field in the response is unambiguous +signal that the job ran without variables bindings.