diff --git a/src/keboola_agent_cli/client.py b/src/keboola_agent_cli/client.py index d92b07e3..fa6f4155 100644 --- a/src/keboola_agent_cli/client.py +++ b/src/keboola_agent_cli/client.py @@ -216,25 +216,35 @@ def list_components( response = self._request("GET", f"{prefix}/components", params=params) return response.json() - def list_components_with_configs(self, branch_id: int | None = None) -> list[dict[str, Any]]: + def list_components_with_configs( + self, + branch_id: int | None = None, + component_type: str | None = None, + ) -> list[dict[str, Any]]: """List all components with full configuration bodies and rows. - Makes a single API call to fetch everything needed for sync pull. - Uses the include=configuration,rows parameter to get full config - bodies and config rows in one request. + Makes a single API call to fetch everything needed for sync pull and + for deep search (row-level configuration). Uses the + include=configuration,rows parameter to get full config bodies and + config rows in one request. Args: branch_id: If set, target a specific dev branch. + component_type: Optional filter (extractor, writer, transformation, + application). Passed to the API as ``componentType``. Returns: List of component dicts, each containing a 'configurations' list with full config bodies and nested 'rows'. """ prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + params: dict[str, str] = {"include": "configuration,rows"} + if component_type: + params["componentType"] = component_type resp = self._request( "GET", f"{prefix}/components", - params={"include": "configuration,rows"}, + params=params, ) return resp.json() diff --git a/src/keboola_agent_cli/services/config_service.py b/src/keboola_agent_cli/services/config_service.py index 6ab01835..c5269585 100644 --- a/src/keboola_agent_cli/services/config_service.py +++ b/src/keboola_agent_cli/services/config_service.py @@ -721,13 +721,20 @@ def _search_project_configs( component_id: str | None = None, branch_id: int | None = None, ) -> tuple[str, dict[str, Any], bool] | tuple[str, dict[str, str]]: - """Search configs in a single project (worker thread).""" + """Search configs in a single project (worker thread). + + Uses ``list_components_with_configs`` (``include=configuration,rows``) + so that row-level configuration (Snowflake writer rows, DB extractor + tables, Google Sheets sheets, etc.) is included in the search tree. + Without ``rows``, the API returns only the top-level configuration + body, and searches for row-only properties always miss (see #196). + """ client = self._client_factory(project.stack_url, project.token) try: effective_branch_id = branch_id or project.active_branch_id - components = client.list_components( - component_type=component_type, + components = client.list_components_with_configs( branch_id=effective_branch_id, + component_type=component_type, ) matches: list[dict[str, Any]] = [] configs_searched = 0 diff --git a/tests/test_config_search.py b/tests/test_config_search.py index 5bf06dc2..3c8dfa72 100644 --- a/tests/test_config_search.py +++ b/tests/test_config_search.py @@ -12,9 +12,13 @@ def _make_list_components_client(components: list[dict]) -> MagicMock: - """Create a mock KeboolaClient with list_components returning given data.""" + """Create a mock KeboolaClient with list_components_with_configs returning given data. + + ConfigService.search_configs uses ``list_components_with_configs`` (which + issues ``include=configuration,rows``) so we mock that method. + """ mock_client = MagicMock() - mock_client.list_components.return_value = components + mock_client.list_components_with_configs.return_value = components return mock_client @@ -342,7 +346,135 @@ def test_search_with_component_type_filter(self, tmp_config_dir: Path) -> None: for m in matches: assert m["component_type"] == "extractor" - # The client was called with the component_type filter - mock_client.list_components.assert_called_once_with( - component_type="extractor", branch_id=None + # The client was called with the component_type filter and rows included + mock_client.list_components_with_configs.assert_called_once_with( + branch_id=None, component_type="extractor" + ) + + +# =========================================================================== +# Regression tests for issue #196 -- search must look inside rows[] +# =========================================================================== + + +# Sample data that mirrors how real row-based components (Snowflake writer, +# DB extractors, Google Sheets, etc.) store per-item configuration under +# ``rows[].configuration``. Top-level ``configuration`` is intentionally +# minimal -- the interesting properties live inside rows. +SAMPLE_COMPONENTS_WITH_ROWS = [ + { + "id": "keboola.wr-db-snowflake", + "name": "Snowflake Writer", + "type": "writer", + "configurations": [ + { + "id": "900", + "name": "Write to DWH", + "description": "Writer with per-table rows", + "configuration": { + "parameters": { + "db": {"host": "dwh.snowflakecomputing.com"}, + } + }, + "rows": [ + { + "id": "900-row-1", + "name": "customers table", + "configuration": { + "parameters": { + "dbName": "CUSTOMERS", + "incremental": False, + "primaryKey": ["id"], + } + }, + }, + { + "id": "900-row-2", + "name": "orders table", + "configuration": { + "parameters": { + "dbName": "ORDERS", + "incremental": True, + "primaryKey": ["order_id"], + } + }, + }, + ], + }, + ], + }, +] + + +class TestSearchConfigsRowsCoverage: + """Regression tests ensuring ``config search`` also inspects ``rows[]``. + + Before #196, the service only fetched ``include=configuration`` so row-level + properties were invisible to the search even though the command's help text + promised coverage of "row definitions". + """ + + def test_search_matches_row_configuration(self, tmp_config_dir: Path) -> None: + """A property that only exists in ``rows[].configuration`` is found.""" + store = setup_single_project(tmp_config_dir) + mock_client = _make_list_components_client(SAMPLE_COMPONENTS_WITH_ROWS) + service = ConfigService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + # "CUSTOMERS" appears only inside rows[0].configuration.parameters.dbName + result = service.search_configs(query="CUSTOMERS") + + assert result["errors"] == [] + matches = result["matches"] + assert len(matches) == 1 + assert matches[0]["component_id"] == "keboola.wr-db-snowflake" + assert matches[0]["config_id"] == "900" + + # The match path must point inside rows[0] + assert any( + loc.startswith("rows[0].configuration") and "dbName" in loc + for loc in matches[0]["match_locations"] + ) + + def test_search_matches_boolean_inside_row(self, tmp_config_dir: Path) -> None: + """Scalar values (booleans) inside rows are matched as strings. + + This mirrors the exact query from issue #196 where users searched for + ``"incremental": false`` to audit writer configurations. + """ + store = setup_single_project(tmp_config_dir) + mock_client = _make_list_components_client(SAMPLE_COMPONENTS_WITH_ROWS) + service = ConfigService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + # The recursive walker stringifies booleans; case-insensitive matches "False" + result = service.search_configs(query="false", ignore_case=True) + + matches = result["matches"] + assert len(matches) == 1 + incremental_paths = [ + loc for loc in matches[0]["match_locations"] if loc.endswith("incremental") + ] + # Only row-1 has incremental=False; row-2 has incremental=True + assert incremental_paths == ["rows[0].configuration.parameters.incremental"] + + def test_search_client_receives_rows_include(self, tmp_config_dir: Path) -> None: + """Service must call ``list_components_with_configs`` (include=rows).""" + store = setup_single_project(tmp_config_dir) + mock_client = _make_list_components_client(SAMPLE_COMPONENTS_WITH_ROWS) + service = ConfigService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + service.search_configs(query="anything") + + # The old ``list_components`` (include=configuration only) must NOT be used. + mock_client.list_components.assert_not_called() + mock_client.list_components_with_configs.assert_called_once_with( + branch_id=None, component_type=None ) diff --git a/tests/test_config_search_cli.py b/tests/test_config_search_cli.py index dc73ff39..a99b19b5 100644 --- a/tests/test_config_search_cli.py +++ b/tests/test_config_search_cli.py @@ -93,9 +93,14 @@ def _setup_config_store(config_dir: Path, projects: dict[str, dict] | None = Non def _make_list_components_client(components: list[dict]) -> MagicMock: - """Create a mock KeboolaClient with list_components returning given data.""" + """Create a mock KeboolaClient with list_components_with_configs returning given data. + + ``ConfigService.search_configs`` now uses ``list_components_with_configs`` + (``include=configuration,rows``) so that row-level configuration is part + of the search tree -- see issue #196. + """ mock_client = MagicMock() - mock_client.list_components.return_value = components + mock_client.list_components_with_configs.return_value = components return mock_client @@ -315,3 +320,75 @@ def test_config_search_unknown_project(self, tmp_path: Path) -> None: output = json.loads(result.output) assert output["status"] == "error" assert output["error"]["code"] == "CONFIG_ERROR" + + +# --------------------------------------------------------------------------- +# Row-level search regression (issue #196) +# --------------------------------------------------------------------------- + + +SAMPLE_COMPONENTS_WITH_ROWS = [ + { + "id": "keboola.wr-db-snowflake", + "name": "Snowflake Writer", + "type": "writer", + "configurations": [ + { + "id": "555", + "name": "Warehouse Writer", + "description": "Per-table row configs", + "configuration": {"parameters": {"db": {"host": "dwh.example.com"}}}, + "rows": [ + { + "id": "555-row-1", + "name": "customers", + "configuration": { + "parameters": { + "dbName": "CUSTOMERS", + "incremental": False, + } + }, + }, + ], + }, + ], + }, +] + + +class TestConfigSearchRows: + """End-to-end CLI coverage of the row-level search fix (issue #196).""" + + def test_search_finds_property_inside_row(self, tmp_path: Path) -> None: + """`kbagent config search --query CUSTOMERS` returns the row match.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + mock_client = _make_list_components_client(SAMPLE_COMPONENTS_WITH_ROWS) + store = _setup_config_store(config_dir, {"prod": {"token": TEST_TOKEN}}) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + result = runner.invoke( + app, + ["--json", "config", "search", "--query", "CUSTOMERS"], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + data = output["data"] + assert data["stats"]["matches_found"] == 1 + match = data["matches"][0] + assert match["config_id"] == "555" + assert any(loc.startswith("rows[0].configuration") for loc in match["match_locations"]) diff --git a/tests/test_services.py b/tests/test_services.py index 1badbfba..3f77d593 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -1178,7 +1178,7 @@ class TestConfigServiceSearchConfigs: """Tests for ConfigService.search_configs() with branch_id support.""" def test_search_configs_with_branch_id(self, tmp_config_dir: Path) -> None: - """search_configs passes branch_id to client.list_components.""" + """search_configs passes branch_id to client.list_components_with_configs.""" store = ConfigStore(config_dir=tmp_config_dir) store.add_project( "prod", @@ -1188,7 +1188,10 @@ def test_search_configs_with_branch_id(self, tmp_config_dir: Path) -> None: ), ) - mock_client = _make_list_components_client(SAMPLE_COMPONENTS) + # search_configs uses list_components_with_configs (include=configuration,rows) + # so row-level properties are part of the search tree (see #196). + mock_client = MagicMock() + mock_client.list_components_with_configs.return_value = SAMPLE_COMPONENTS service = ConfigService( config_store=store, client_factory=lambda url, token: mock_client, @@ -1201,7 +1204,9 @@ def test_search_configs_with_branch_id(self, tmp_config_dir: Path) -> None: assert len(matches) == 1 assert matches[0]["config_name"] == "Production Load" - mock_client.list_components.assert_called_once_with(component_type=None, branch_id=123) + mock_client.list_components_with_configs.assert_called_once_with( + branch_id=123, component_type=None + ) mock_client.close.assert_called_once() def test_search_configs_uses_active_branch(self, tmp_config_dir: Path) -> None: @@ -1216,7 +1221,8 @@ def test_search_configs_uses_active_branch(self, tmp_config_dir: Path) -> None: ), ) - mock_client = _make_list_components_client(SAMPLE_COMPONENTS) + mock_client = MagicMock() + mock_client.list_components_with_configs.return_value = SAMPLE_COMPONENTS service = ConfigService( config_store=store, client_factory=lambda url, token: mock_client, @@ -1224,7 +1230,9 @@ def test_search_configs_uses_active_branch(self, tmp_config_dir: Path) -> None: service.search_configs(query="nonexistent-query") - mock_client.list_components.assert_called_once_with(component_type=None, branch_id=88) + mock_client.list_components_with_configs.assert_called_once_with( + branch_id=88, component_type=None + ) mock_client.close.assert_called_once()