Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions src/keboola_agent_cli/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,25 +216,35 @@ def list_components(
response = self._request("GET", f"{prefix}/components", params=params)
return response.json()

def list_components_with_configs(self, branch_id: int | None = None) -> list[dict[str, Any]]:
def list_components_with_configs(
self,
branch_id: int | None = None,
component_type: str | None = None,
) -> list[dict[str, Any]]:
"""List all components with full configuration bodies and rows.

Makes a single API call to fetch everything needed for sync pull.
Uses the include=configuration,rows parameter to get full config
bodies and config rows in one request.
Makes a single API call to fetch everything needed for sync pull and
for deep search (row-level configuration). Uses the
include=configuration,rows parameter to get full config bodies and
config rows in one request.

Args:
branch_id: If set, target a specific dev branch.
component_type: Optional filter (extractor, writer, transformation,
application). Passed to the API as ``componentType``.

Returns:
List of component dicts, each containing a 'configurations' list
with full config bodies and nested 'rows'.
"""
prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage"
params: dict[str, str] = {"include": "configuration,rows"}
if component_type:
params["componentType"] = component_type
resp = self._request(
"GET",
f"{prefix}/components",
params={"include": "configuration,rows"},
params=params,
)
return resp.json()

Expand Down
13 changes: 10 additions & 3 deletions src/keboola_agent_cli/services/config_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -721,13 +721,20 @@ def _search_project_configs(
component_id: str | None = None,
branch_id: int | None = None,
) -> tuple[str, dict[str, Any], bool] | tuple[str, dict[str, str]]:
"""Search configs in a single project (worker thread)."""
"""Search configs in a single project (worker thread).

Uses ``list_components_with_configs`` (``include=configuration,rows``)
so that row-level configuration (Snowflake writer rows, DB extractor
tables, Google Sheets sheets, etc.) is included in the search tree.
Without ``rows``, the API returns only the top-level configuration
body, and searches for row-only properties always miss (see #196).
"""
client = self._client_factory(project.stack_url, project.token)
try:
effective_branch_id = branch_id or project.active_branch_id
components = client.list_components(
component_type=component_type,
components = client.list_components_with_configs(
branch_id=effective_branch_id,
component_type=component_type,
)
matches: list[dict[str, Any]] = []
configs_searched = 0
Expand Down
142 changes: 137 additions & 5 deletions tests/test_config_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,13 @@


def _make_list_components_client(components: list[dict]) -> MagicMock:
"""Create a mock KeboolaClient with list_components returning given data."""
"""Create a mock KeboolaClient with list_components_with_configs returning given data.

ConfigService.search_configs uses ``list_components_with_configs`` (which
issues ``include=configuration,rows``) so we mock that method.
"""
mock_client = MagicMock()
mock_client.list_components.return_value = components
mock_client.list_components_with_configs.return_value = components
return mock_client


Expand Down Expand Up @@ -342,7 +346,135 @@ def test_search_with_component_type_filter(self, tmp_config_dir: Path) -> None:
for m in matches:
assert m["component_type"] == "extractor"

# The client was called with the component_type filter
mock_client.list_components.assert_called_once_with(
component_type="extractor", branch_id=None
# The client was called with the component_type filter and rows included
mock_client.list_components_with_configs.assert_called_once_with(
branch_id=None, component_type="extractor"
)


# ===========================================================================
# Regression tests for issue #196 -- search must look inside rows[]
# ===========================================================================


# Sample data that mirrors how real row-based components (Snowflake writer,
# DB extractors, Google Sheets, etc.) store per-item configuration under
# ``rows[].configuration``. Top-level ``configuration`` is intentionally
# minimal -- the interesting properties live inside rows.
SAMPLE_COMPONENTS_WITH_ROWS = [
{
"id": "keboola.wr-db-snowflake",
"name": "Snowflake Writer",
"type": "writer",
"configurations": [
{
"id": "900",
"name": "Write to DWH",
"description": "Writer with per-table rows",
"configuration": {
"parameters": {
"db": {"host": "dwh.snowflakecomputing.com"},
}
},
"rows": [
{
"id": "900-row-1",
"name": "customers table",
"configuration": {
"parameters": {
"dbName": "CUSTOMERS",
"incremental": False,
"primaryKey": ["id"],
}
},
},
{
"id": "900-row-2",
"name": "orders table",
"configuration": {
"parameters": {
"dbName": "ORDERS",
"incremental": True,
"primaryKey": ["order_id"],
}
},
},
],
},
],
},
]


class TestSearchConfigsRowsCoverage:
"""Regression tests ensuring ``config search`` also inspects ``rows[]``.

Before #196, the service only fetched ``include=configuration`` so row-level
properties were invisible to the search even though the command's help text
promised coverage of "row definitions".
"""

def test_search_matches_row_configuration(self, tmp_config_dir: Path) -> None:
"""A property that only exists in ``rows[].configuration`` is found."""
store = setup_single_project(tmp_config_dir)
mock_client = _make_list_components_client(SAMPLE_COMPONENTS_WITH_ROWS)
service = ConfigService(
config_store=store,
client_factory=lambda url, token: mock_client,
)

# "CUSTOMERS" appears only inside rows[0].configuration.parameters.dbName
result = service.search_configs(query="CUSTOMERS")

assert result["errors"] == []
matches = result["matches"]
assert len(matches) == 1
assert matches[0]["component_id"] == "keboola.wr-db-snowflake"
assert matches[0]["config_id"] == "900"

# The match path must point inside rows[0]
assert any(
loc.startswith("rows[0].configuration") and "dbName" in loc
for loc in matches[0]["match_locations"]
)

def test_search_matches_boolean_inside_row(self, tmp_config_dir: Path) -> None:
"""Scalar values (booleans) inside rows are matched as strings.

This mirrors the exact query from issue #196 where users searched for
``"incremental": false`` to audit writer configurations.
"""
store = setup_single_project(tmp_config_dir)
mock_client = _make_list_components_client(SAMPLE_COMPONENTS_WITH_ROWS)
service = ConfigService(
config_store=store,
client_factory=lambda url, token: mock_client,
)

# The recursive walker stringifies booleans; case-insensitive matches "False"
result = service.search_configs(query="false", ignore_case=True)

matches = result["matches"]
assert len(matches) == 1
incremental_paths = [
loc for loc in matches[0]["match_locations"] if loc.endswith("incremental")
]
# Only row-1 has incremental=False; row-2 has incremental=True
assert incremental_paths == ["rows[0].configuration.parameters.incremental"]

def test_search_client_receives_rows_include(self, tmp_config_dir: Path) -> None:
"""Service must call ``list_components_with_configs`` (include=rows)."""
store = setup_single_project(tmp_config_dir)
mock_client = _make_list_components_client(SAMPLE_COMPONENTS_WITH_ROWS)
service = ConfigService(
config_store=store,
client_factory=lambda url, token: mock_client,
)

service.search_configs(query="anything")

# The old ``list_components`` (include=configuration only) must NOT be used.
mock_client.list_components.assert_not_called()
mock_client.list_components_with_configs.assert_called_once_with(
branch_id=None, component_type=None
)
81 changes: 79 additions & 2 deletions tests/test_config_search_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,14 @@ def _setup_config_store(config_dir: Path, projects: dict[str, dict] | None = Non


def _make_list_components_client(components: list[dict]) -> MagicMock:
"""Create a mock KeboolaClient with list_components returning given data."""
"""Create a mock KeboolaClient with list_components_with_configs returning given data.

``ConfigService.search_configs`` now uses ``list_components_with_configs``
(``include=configuration,rows``) so that row-level configuration is part
of the search tree -- see issue #196.
"""
mock_client = MagicMock()
mock_client.list_components.return_value = components
mock_client.list_components_with_configs.return_value = components
return mock_client


Expand Down Expand Up @@ -315,3 +320,75 @@ def test_config_search_unknown_project(self, tmp_path: Path) -> None:
output = json.loads(result.output)
assert output["status"] == "error"
assert output["error"]["code"] == "CONFIG_ERROR"


# ---------------------------------------------------------------------------
# Row-level search regression (issue #196)
# ---------------------------------------------------------------------------


SAMPLE_COMPONENTS_WITH_ROWS = [
{
"id": "keboola.wr-db-snowflake",
"name": "Snowflake Writer",
"type": "writer",
"configurations": [
{
"id": "555",
"name": "Warehouse Writer",
"description": "Per-table row configs",
"configuration": {"parameters": {"db": {"host": "dwh.example.com"}}},
"rows": [
{
"id": "555-row-1",
"name": "customers",
"configuration": {
"parameters": {
"dbName": "CUSTOMERS",
"incremental": False,
}
},
},
],
},
],
},
]


class TestConfigSearchRows:
"""End-to-end CLI coverage of the row-level search fix (issue #196)."""

def test_search_finds_property_inside_row(self, tmp_path: Path) -> None:
"""`kbagent config search --query CUSTOMERS` returns the row match."""
config_dir = tmp_path / "config"
config_dir.mkdir()

mock_client = _make_list_components_client(SAMPLE_COMPONENTS_WITH_ROWS)
store = _setup_config_store(config_dir, {"prod": {"token": TEST_TOKEN}})

with (
patch("keboola_agent_cli.cli.ConfigStore") as MockStore,
patch("keboola_agent_cli.cli.ProjectService") as MockProjService,
patch("keboola_agent_cli.cli.ConfigService") as MockCfgService,
):
MockStore.return_value = store
MockProjService.return_value = ProjectService(config_store=store)
MockCfgService.return_value = ConfigService(
config_store=store,
client_factory=lambda url, token: mock_client,
)

result = runner.invoke(
app,
["--json", "config", "search", "--query", "CUSTOMERS"],
)

assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}"
output = json.loads(result.output)
assert output["status"] == "ok"
data = output["data"]
assert data["stats"]["matches_found"] == 1
match = data["matches"][0]
assert match["config_id"] == "555"
assert any(loc.startswith("rows[0].configuration") for loc in match["match_locations"])
18 changes: 13 additions & 5 deletions tests/test_services.py
Original file line number Diff line number Diff line change
Expand Up @@ -1178,7 +1178,7 @@ class TestConfigServiceSearchConfigs:
"""Tests for ConfigService.search_configs() with branch_id support."""

def test_search_configs_with_branch_id(self, tmp_config_dir: Path) -> None:
"""search_configs passes branch_id to client.list_components."""
"""search_configs passes branch_id to client.list_components_with_configs."""
store = ConfigStore(config_dir=tmp_config_dir)
store.add_project(
"prod",
Expand All @@ -1188,7 +1188,10 @@ def test_search_configs_with_branch_id(self, tmp_config_dir: Path) -> None:
),
)

mock_client = _make_list_components_client(SAMPLE_COMPONENTS)
# search_configs uses list_components_with_configs (include=configuration,rows)
# so row-level properties are part of the search tree (see #196).
mock_client = MagicMock()
mock_client.list_components_with_configs.return_value = SAMPLE_COMPONENTS
service = ConfigService(
config_store=store,
client_factory=lambda url, token: mock_client,
Expand All @@ -1201,7 +1204,9 @@ def test_search_configs_with_branch_id(self, tmp_config_dir: Path) -> None:
assert len(matches) == 1
assert matches[0]["config_name"] == "Production Load"

mock_client.list_components.assert_called_once_with(component_type=None, branch_id=123)
mock_client.list_components_with_configs.assert_called_once_with(
branch_id=123, component_type=None
)
mock_client.close.assert_called_once()

def test_search_configs_uses_active_branch(self, tmp_config_dir: Path) -> None:
Expand All @@ -1216,15 +1221,18 @@ def test_search_configs_uses_active_branch(self, tmp_config_dir: Path) -> None:
),
)

mock_client = _make_list_components_client(SAMPLE_COMPONENTS)
mock_client = MagicMock()
mock_client.list_components_with_configs.return_value = SAMPLE_COMPONENTS
service = ConfigService(
config_store=store,
client_factory=lambda url, token: mock_client,
)

service.search_configs(query="nonexistent-query")

mock_client.list_components.assert_called_once_with(component_type=None, branch_id=88)
mock_client.list_components_with_configs.assert_called_once_with(
branch_id=88, component_type=None
)
mock_client.close.assert_called_once()


Expand Down