diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index a483d01a..29d74a90 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -67,6 +67,14 @@ This prints all commands, flags, workflows, and tips. Read it fully before proce | Load tables into a workspace | `kbagent workspace load --project PROJECT --workspace-id WORKSPACE-ID --tables TABLES` | | Execute SQL query in a workspace via Query Service | `kbagent workspace query --project PROJECT --workspace-id WORKSPACE-ID` | | Create a workspace from a transformation config | `kbagent workspace from-transformation --project PROJECT --component-id COMPONENT-ID --config-id CONFIG-ID` | +| Initialize a sync working directory for a Keboola project | `kbagent sync init --project PROJECT` | +| Download all configurations from a Keboola project to local files | `kbagent sync pull --project PROJECT` | +| Show which local configurations have been modified, added, or deleted | `kbagent sync status` | +| Show detailed diff between local and remote configurations | `kbagent sync diff --project PROJECT` | +| Push local configuration changes to a Keboola project | `kbagent sync push --project PROJECT` | +| Link the current git branch to a Keboola development branch | `kbagent sync branch-link --project PROJECT` | +| Remove the branch mapping for the current git branch | `kbagent sync branch-unlink` | +| Show the branch mapping status for the current git branch | `kbagent sync branch-status` | ## Response format diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index 064dd7c4..d67ee76b 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -17,6 +17,7 @@ from .commands.org import org_app from .commands.project import project_app from .commands.repl import repl_command +from .commands.sync import sync_app from .commands.tool import tool_app from .commands.version import version_command from .commands.workspace import workspace_app @@ -32,6 +33,7 @@ from .services.mcp_service import McpService from .services.org_service import OrgService from .services.project_service import ProjectService +from .services.sync_service import SyncService from .services.version_service import VersionService from .services.workspace_service import WorkspaceService @@ -51,6 +53,7 @@ app.add_typer(explorer_app, name="explorer") app.add_typer(llm_app, name="llm") app.add_typer(workspace_app, name="workspace") +app.add_typer(sync_app, name="sync") app.command("context")(context_command) app.command("doctor")(doctor_command) app.command("init")(init_command) @@ -125,6 +128,7 @@ def main( org_service = OrgService(config_store=config_store) mcp_service = McpService(config_store=config_store) branch_service = BranchService(config_store=config_store) + sync_service = SyncService(config_store=config_store) workspace_service = WorkspaceService(config_store=config_store) kbc_service = KbcService(config_store=config_store) doctor_service = DoctorService(config_store=config_store, mcp_service=mcp_service) @@ -149,6 +153,7 @@ def main( ctx.obj["org_service"] = org_service ctx.obj["mcp_service"] = mcp_service ctx.obj["branch_service"] = branch_service + ctx.obj["sync_service"] = sync_service ctx.obj["workspace_service"] = workspace_service ctx.obj["kbc_service"] = kbc_service ctx.obj["doctor_service"] = doctor_service diff --git a/src/keboola_agent_cli/client.py b/src/keboola_agent_cli/client.py index d558bf65..9b1d405f 100644 --- a/src/keboola_agent_cli/client.py +++ b/src/keboola_agent_cli/client.py @@ -170,6 +170,51 @@ def list_components(self, component_type: str | None = None) -> list[dict[str, A response = self._request("GET", "/v2/storage/components", params=params) return response.json() + def list_components_with_configs(self, branch_id: int | None = None) -> list[dict[str, Any]]: + """List all components with full configuration bodies and rows. + + Makes a single API call to fetch everything needed for sync pull. + Uses the include=configuration,rows parameter to get full config + bodies and config rows in one request. + + Args: + branch_id: If set, target a specific dev branch. + + Returns: + List of component dicts, each containing a 'configurations' list + with full config bodies and nested 'rows'. + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + resp = self._request( + "GET", + f"{prefix}/components", + params={"include": "configuration,rows"}, + ) + return resp.json() + + def list_config_rows( + self, + component_id: str, + config_id: str, + branch_id: int | None = None, + ) -> list[dict[str, Any]]: + """List all rows for a specific configuration. + + Args: + component_id: Component identifier (e.g. 'keboola.ex-http'). + config_id: Configuration ID. + branch_id: If set, target a specific dev branch. + + Returns: + List of config row dicts. + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + resp = self._request( + "GET", + f"{prefix}/components/{quote(component_id)}/configs/{quote(config_id)}/rows", + ) + return resp.json() + def get_config_detail(self, component_id: str, config_id: str) -> dict[str, Any]: """Get detailed information about a specific configuration. @@ -188,6 +233,153 @@ def get_config_detail(self, component_id: str, config_id: str) -> dict[str, Any] ) return response.json() + def create_config( + self, + component_id: str, + name: str, + configuration: dict[str, Any], + description: str = "", + branch_id: int | None = None, + ) -> dict[str, Any]: + """Create a new configuration for a component. + + POST /v2/storage/[branch/{id}/]components/{comp_id}/configs + + Args: + component_id: Component identifier. + name: Configuration name. + configuration: Configuration body (parameters, storage, etc.). + description: Optional description. + branch_id: If set, target a specific dev branch. + + Returns: + Created configuration dict including the assigned 'id'. + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + resp = self._request( + "POST", + f"{prefix}/components/{quote(component_id)}/configs", + data={ + "name": name, + "description": description, + "configuration": json.dumps(configuration), + }, + ) + return resp.json() + + def update_config( + self, + component_id: str, + config_id: str, + name: str | None = None, + configuration: dict[str, Any] | None = None, + description: str | None = None, + change_description: str = "", + branch_id: int | None = None, + ) -> dict[str, Any]: + """Update an existing configuration. + + PUT /v2/storage/[branch/{id}/]components/{comp_id}/configs/{config_id} + + Only provided (non-None) fields are sent in the request. + + Returns: + Updated configuration dict. + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + data: dict[str, Any] = {} + if name is not None: + data["name"] = name + if description is not None: + data["description"] = description + if configuration is not None: + data["configuration"] = json.dumps(configuration) + if change_description: + data["changeDescription"] = change_description + resp = self._request( + "PUT", + f"{prefix}/components/{quote(component_id)}/configs/{quote(config_id)}", + data=data, + ) + return resp.json() + + def create_config_row( + self, + component_id: str, + config_id: str, + name: str, + configuration: dict[str, Any], + description: str = "", + branch_id: int | None = None, + ) -> dict[str, Any]: + """Create a new configuration row. + + POST /v2/storage/[branch/{id}/]components/{comp_id}/configs/{config_id}/rows + + Returns: + Created row dict including the assigned 'id'. + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + resp = self._request( + "POST", + f"{prefix}/components/{quote(component_id)}/configs/{quote(config_id)}/rows", + data={ + "name": name, + "description": description, + "configuration": json.dumps(configuration), + }, + ) + return resp.json() + + def update_config_row( + self, + component_id: str, + config_id: str, + row_id: str, + name: str | None = None, + configuration: dict[str, Any] | None = None, + description: str | None = None, + change_description: str = "", + branch_id: int | None = None, + ) -> dict[str, Any]: + """Update an existing configuration row. + + PUT /v2/storage/[branch/{id}/]components/{comp_id}/configs/{config_id}/rows/{row_id} + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + data: dict[str, Any] = {} + if name is not None: + data["name"] = name + if description is not None: + data["description"] = description + if configuration is not None: + data["configuration"] = json.dumps(configuration) + if change_description: + data["changeDescription"] = change_description + resp = self._request( + "PUT", + f"{prefix}/components/{quote(component_id)}/configs/{quote(config_id)}/rows/{quote(row_id)}", + data=data, + ) + return resp.json() + + def delete_config_row( + self, + component_id: str, + config_id: str, + row_id: str, + branch_id: int | None = None, + ) -> None: + """Delete a configuration row. + + DELETE /v2/storage/[branch/{id}/]components/{comp_id}/configs/{config_id}/rows/{row_id} + """ + prefix = f"/v2/storage/branch/{branch_id}" if branch_id else "/v2/storage" + self._request( + "DELETE", + f"{prefix}/components/{quote(component_id)}/configs/{quote(config_id)}/rows/{quote(row_id)}", + ) + def _wait_for_storage_job(self, job: dict[str, Any]) -> dict[str, Any]: """Poll a Storage API job until it reaches a terminal state. diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 5fd5aa3a..460ae018 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -337,6 +337,60 @@ Example: kbagent --json workspace from-transformation --project prod --component-id keboola.snowflake-transformation --config-id 22777254 +### Project Sync (GitOps Workflow) + + kbagent sync init --project ALIAS [--directory DIR] [--git-branching] + Initialize a sync working directory for a Keboola project. + Creates .keboola/manifest.json with project metadata and naming conventions. + Use --git-branching to enable git-to-Keboola branch mapping. + Example: + mkdir my-project && cd my-project + kbagent --json sync init --project prod + kbagent --json sync init --project prod --git-branching + + kbagent sync pull --project ALIAS [--directory DIR] [--force] + Download all configurations from Keboola to local files. + Creates a dev-friendly directory structure with _config.yml files. + SQL transformations are extracted into transform.sql with block markers. + Python code is extracted into transform.py/code.py + pyproject.toml. + Example: + kbagent --json sync pull --project prod + + kbagent sync status [--directory DIR] + Show which local configs have been modified, added, or deleted since last pull. + Uses SHA256 hash comparison for reliable change detection. + Example: + kbagent --json sync status + + kbagent sync diff --project ALIAS [--directory DIR] + Show detailed diff between local files and remote Keboola state. + Compares config content (ignoring encrypted value nonces). + Example: + kbagent --json sync diff --project prod + + kbagent sync push --project ALIAS [--directory DIR] [--dry-run] [--force] + Push local changes to Keboola. Creates new configs, updates modified, + deletes removed (with --force). New configs get IDs from API automatically. + --dry-run shows what would change without applying. + Example: + kbagent --json sync push --project prod --dry-run + kbagent --json sync push --project prod + + kbagent sync branch-link --project ALIAS [--branch-id ID] [--branch-name NAME] + Link current git branch to a Keboola development branch. + Auto-creates the Keboola branch if it doesn't exist with the same name. + Requires --git-branching mode enabled via sync init. + Example: + git checkout -b feature/new-etl + kbagent --json sync branch-link --project prod + + kbagent sync branch-unlink [--directory DIR] + Remove the branch mapping for the current git branch. + Does NOT delete the Keboola branch itself. + + kbagent sync branch-status [--directory DIR] + Show the current git branch mapping status. + ### Utility Commands kbagent init [--from-global] @@ -497,6 +551,29 @@ KBC_MANAGE_API_TOKEN=xxx kbagent --json org setup --org-id 123 --url https://connection.keboola.com --yes This creates Storage API tokens for ALL projects in the org and registers them automatically. +17. Sync workflow -- manage configs as local files with GitOps: + # Step 1: Initialize and pull + mkdir my-project && cd my-project + kbagent --json sync init --project prod + kbagent --json sync pull --project prod + + # Step 2: Edit configs locally + # SQL is in transform.sql, Python in code.py, config in _config.yml + # Edit with any IDE, get git diffs, code review, etc. + + # Step 3: Review and push + kbagent --json sync status # local changes + kbagent --json sync diff --project prod # vs remote + kbagent --json sync push --project prod --dry-run # preview + kbagent --json sync push --project prod # apply + + # Git-branching mode (maps git branches to Keboola dev branches): + kbagent --json sync init --project prod --git-branching + git checkout -b feature/new-etl + kbagent --json sync branch-link --project prod # creates Keboola dev branch + kbagent --json sync pull --project prod + # ... edit, push, then merge via PR + Keboola UI + ## Exit Codes 0 Success diff --git a/src/keboola_agent_cli/commands/sync.py b/src/keboola_agent_cli/commands/sync.py new file mode 100644 index 00000000..20c123820 --- /dev/null +++ b/src/keboola_agent_cli/commands/sync.py @@ -0,0 +1,483 @@ +"""Sync commands - init, pull, push, diff, and status for local filesystem sync. + +Thin CLI layer: parses arguments, calls SyncService, formats output. +No business logic belongs here. +""" + +from pathlib import Path + +import typer + +from ..errors import ConfigError, KeboolaApiError +from ._helpers import get_formatter, get_service, map_error_to_exit_code + +sync_app = typer.Typer(help="Sync project configurations with local filesystem") + + +@sync_app.command("init") +def sync_init( + ctx: typer.Context, + project: str = typer.Option( + ..., + "--project", + help="Project alias to initialize sync for", + ), + directory: Path = typer.Option( + Path("."), + "--directory", + "-d", + help="Target directory for the project files", + ), + git_branching: bool = typer.Option( + False, + "--git-branching", + help="Enable git-branching mode (maps git branches to Keboola branches)", + ), +) -> None: + """Initialize a sync working directory for a Keboola project. + + Creates the .keboola/ directory with manifest.json containing + project metadata and naming conventions. Optionally enables + git-branching mode for branch-to-branch mapping. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "sync_service") + project_root = directory.resolve() + + try: + result = service.init_sync( + alias=project, + project_root=project_root, + git_branching=git_branching, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except FileExistsError as exc: + formatter.error(message=str(exc), error_code="ALREADY_EXISTS") + raise typer.Exit(code=1) from None + except KeboolaApiError as exc: + formatter.error( + message=exc.message, + error_code=exc.error_code, + ) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + else: + formatter.success( + f"Initialized sync for project '{result['project_alias']}' (ID: {result['project_id']})" + ) + formatter.console.print(f" API host: {result['api_host']}") + if result["git_branching"]: + formatter.console.print( + f" Git-branching: enabled (default branch: {result['default_branch']})" + ) + for f in result["files_created"]: + formatter.console.print(f" Created: {f}") + + +@sync_app.command("pull") +def sync_pull( + ctx: typer.Context, + project: str = typer.Option( + ..., + "--project", + help="Project alias to pull configurations from", + ), + directory: Path = typer.Option( + Path("."), + "--directory", + "-d", + help="Project root directory (must contain .keboola/)", + ), + force: bool = typer.Option( + False, + "--force", + help="Overwrite local files without checking for modifications", + ), +) -> None: + """Download all configurations from a Keboola project to local files. + + Reads the manifest from .keboola/manifest.json, fetches all + configurations from the API, and writes them as _config.yml files + in the dev-friendly directory structure. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "sync_service") + project_root = directory.resolve() + + try: + result = service.pull( + alias=project, + project_root=project_root, + force=force, + ) + except FileNotFoundError as exc: + formatter.error(message=str(exc), error_code="NOT_INITIALIZED") + raise typer.Exit(code=1) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error( + message=exc.message, + error_code=exc.error_code, + ) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + else: + formatter.success( + f"Pulled {result['configs_pulled']} configurations " + f"({result['rows_pulled']} rows) " + f"into {result['branch_dir']}/" + ) + formatter.console.print(f" Files written: {result['files_written']}") + + +@sync_app.command("status") +def sync_status( + ctx: typer.Context, + directory: Path = typer.Option( + Path("."), + "--directory", + "-d", + help="Project root directory (must contain .keboola/)", + ), +) -> None: + """Show which local configurations have been modified, added, or deleted. + + Compares the local filesystem state against the manifest to detect + changes since the last pull. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "sync_service") + project_root = directory.resolve() + + try: + result = service.status(project_root=project_root) + except FileNotFoundError as exc: + formatter.error(message=str(exc), error_code="NOT_INITIALIZED") + raise typer.Exit(code=1) from None + + if formatter.json_mode: + formatter.output(result) + else: + modified = result["modified"] + added = result["added"] + deleted = result["deleted"] + unchanged = result["unchanged"] + + if not modified and not added and not deleted: + formatter.console.print( + f"[green]No changes detected.[/green] ({unchanged} configurations tracked)" + ) + return + + if modified: + formatter.console.print(f"\n[yellow]Modified ({len(modified)}):[/yellow]") + for m in modified: + formatter.console.print(f" M {m['path']}") + + if added: + formatter.console.print(f"\n[green]Added ({len(added)}):[/green]") + for a in added: + formatter.console.print(f" A {a['path']}") + + if deleted: + formatter.console.print(f"\n[red]Deleted ({len(deleted)}):[/red]") + for d in deleted: + formatter.console.print(f" D {d['path']}") + + formatter.console.print( + f"\n{len(modified)} modified, {len(added)} added, " + f"{len(deleted)} deleted, {unchanged} unchanged" + ) + + +@sync_app.command("diff") +def sync_diff( + ctx: typer.Context, + project: str = typer.Option( + ..., + "--project", + help="Project alias to diff against", + ), + directory: Path = typer.Option( + Path("."), + "--directory", + "-d", + help="Project root directory (must contain .keboola/)", + ), +) -> None: + """Show detailed diff between local and remote configurations. + + Fetches the current remote state and compares each local _config.yml + against it, showing which configs would be created, updated, or deleted. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "sync_service") + project_root = directory.resolve() + + try: + result = service.diff(alias=project, project_root=project_root) + except FileNotFoundError as exc: + formatter.error(message=str(exc), error_code="NOT_INITIALIZED") + raise typer.Exit(code=1) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + else: + changes = result["changes"] + summary = result["summary"] + + if not changes: + formatter.console.print( + "[green]No differences found.[/green] Local and remote are in sync." + ) + return + + for change in changes: + change_type = change["change_type"] + path = change.get("path", change.get("config_name", "")) + prefix = {"added": "[green]+ ", "modified": "[yellow]~ ", "deleted": "[red]- "} + suffix = {"added": "[/green]", "modified": "[/yellow]", "deleted": "[/red]"} + formatter.console.print( + f" {prefix.get(change_type, '')}" + f"{change_type.upper()} {change['component_id']}/{path}" + f"{suffix.get(change_type, '')}" + ) + for detail in change.get("details", []): + formatter.console.print(f" {detail}") + + formatter.console.print( + f"\n{summary['added']} to create, {summary['modified']} to update, " + f"{summary['deleted']} to delete" + ) + + +@sync_app.command("push") +def sync_push( + ctx: typer.Context, + project: str = typer.Option( + ..., + "--project", + help="Project alias to push changes to", + ), + directory: Path = typer.Option( + Path("."), + "--directory", + "-d", + help="Project root directory (must contain .keboola/)", + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Show what would be pushed without actually pushing", + ), + force: bool = typer.Option( + False, + "--force", + help="Allow deletion of remote configs that were removed locally", + ), +) -> None: + """Push local configuration changes to a Keboola project. + + Compares local files against remote state and creates, updates, + or deletes configurations as needed. New configs get IDs assigned + by the API. After push, runs a pull to sync the manifest. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "sync_service") + project_root = directory.resolve() + + try: + result = service.push( + alias=project, + project_root=project_root, + dry_run=dry_run, + force=force, + ) + except FileNotFoundError as exc: + formatter.error(message=str(exc), error_code="NOT_INITIALIZED") + raise typer.Exit(code=1) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + else: + status = result.get("status", "") + + if status == "no_changes": + formatter.console.print("[green]No changes to push.[/green]") + return + + if status == "dry_run": + formatter.console.print("[yellow]Dry run -- no changes applied:[/yellow]") + for change in result.get("changes", []): + ct = change["change_type"] + formatter.console.print( + f" {ct.upper()} {change['component_id']}/{change.get('path', '')}" + ) + summary = result["summary"] + formatter.console.print( + f"\nWould create {summary['added']}, update {summary['modified']}, " + f"delete {summary['deleted']}" + ) + return + + formatter.success( + f"Pushed: {result['created']} created, " + f"{result['updated']} updated, " + f"{result['deleted']} deleted" + ) + for err in result.get("errors", []): + formatter.warning( + f" Error: {err['change_type']} {err['component_id']}/{err['config_id']}: " + f"{err['message']}" + ) + + +@sync_app.command("branch-link") +def sync_branch_link( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + directory: Path = typer.Option(Path("."), "--directory", "-d", help="Project root directory"), + branch_id: int | None = typer.Option( + None, "--branch-id", help="Link to existing Keboola branch by ID" + ), + branch_name: str | None = typer.Option( + None, "--branch-name", help="Create/find branch with this name" + ), +) -> None: + """Link the current git branch to a Keboola development branch. + + Creates a new Keboola dev branch if one doesn't exist with the same name + as the current git branch. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "sync_service") + project_root = directory.resolve() + + try: + result = service.branch_link( + alias=project, + project_root=project_root, + branch_id=branch_id, + branch_name=branch_name, + ) + except FileNotFoundError as exc: + formatter.error(message=str(exc), error_code="NOT_INITIALIZED") + raise typer.Exit(code=1) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + else: + status = result["status"] + if status == "already_linked": + formatter.console.print( + f"Already linked: {result['git_branch']} -> " + f"Keboola branch {result['keboola_branch_id']} ({result['keboola_branch_name']})" + ) + else: + formatter.success( + f"Linked {result['git_branch']} -> " + f"Keboola branch {result['keboola_branch_id']} ({result['keboola_branch_name']})" + ) + + +@sync_app.command("branch-unlink") +def sync_branch_unlink( + ctx: typer.Context, + directory: Path = typer.Option(Path("."), "--directory", "-d", help="Project root directory"), +) -> None: + """Remove the branch mapping for the current git branch. + + Does NOT delete the Keboola branch itself. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "sync_service") + project_root = directory.resolve() + + try: + result = service.branch_unlink(project_root=project_root) + except FileNotFoundError as exc: + formatter.error(message=str(exc), error_code="NOT_INITIALIZED") + raise typer.Exit(code=1) from None + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + + if formatter.json_mode: + formatter.output(result) + else: + if result["status"] == "not_linked": + formatter.console.print(f"Branch '{result['git_branch']}' is not linked.") + else: + formatter.success( + f"Unlinked {result['git_branch']} from Keboola branch {result['keboola_branch_id']}" + ) + + +@sync_app.command("branch-status") +def sync_branch_status( + ctx: typer.Context, + directory: Path = typer.Option(Path("."), "--directory", "-d", help="Project root directory"), +) -> None: + """Show the branch mapping status for the current git branch.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "sync_service") + project_root = directory.resolve() + + try: + result = service.branch_status(project_root=project_root) + except FileNotFoundError as exc: + formatter.error(message=str(exc), error_code="NOT_INITIALIZED") + raise typer.Exit(code=1) from None + + if formatter.json_mode: + formatter.output(result) + else: + if not result.get("git_branching"): + formatter.console.print("Git-branching mode is not enabled.") + return + + git_branch = result.get("git_branch", "unknown") + if result.get("linked"): + if result.get("is_production"): + formatter.console.print( + f"Branch: {git_branch}\nKeboola: production\nStatus: [green]Linked[/green]" + ) + else: + formatter.console.print( + f"Branch: {git_branch}\n" + f"Keboola: {result['keboola_branch_id']} ({result['keboola_branch_name']})\n" + f"Status: [green]Linked[/green]" + ) + else: + formatter.console.print( + f"Branch: {git_branch}\n" + f"Keboola: (none)\n" + f"Status: [red]Not linked[/red]\n\n" + f"Run 'kbagent sync branch-link --project ALIAS' to link." + ) diff --git a/src/keboola_agent_cli/constants.py b/src/keboola_agent_cli/constants.py index ca53a945..18ed5904 100644 --- a/src/keboola_agent_cli/constants.py +++ b/src/keboola_agent_cli/constants.py @@ -86,3 +86,27 @@ # --- Workspace Defaults --- DEFAULT_WORKSPACE_BACKEND: str = "snowflake" + +# --- Sync / Git Workflow --- +KEBOOLA_DIR_NAME: str = ".keboola" +MANIFEST_FILENAME: str = "manifest.json" +BRANCH_MAPPING_FILENAME: str = "branch-mapping.json" +CONFIG_FILENAME: str = "_config.yml" +MANIFEST_VERSION: int = 2 +DEFAULT_NAMING_BRANCH: str = "{branch_name}" +DEFAULT_NAMING_CONFIG: str = "{component_type}/{component_id}/{config_name}" +DEFAULT_NAMING_CONFIG_ROW: str = "rows/{config_row_name}" +DEFAULT_NAMING_SCHEDULER: str = "schedules/{config_name}" +DEFAULT_NAMING_SHARED_CODE: str = "_shared/{target_component_id}" +DEFAULT_NAMING_SHARED_CODE_ROW: str = "codes/{config_row_name}" +DEFAULT_NAMING_VARIABLES: str = "variables" +DEFAULT_NAMING_VARIABLES_VALUES: str = "values/{config_row_name}" +DEFAULT_NAMING_DATA_APP: str = "app/{component_id}/{config_name}" +# Aliases used by sync subsystem +CONFIG_YML_VERSION: int = MANIFEST_VERSION +SANITIZE_NAME_MAX_LENGTH: int = 100 + +# --- Diff Engine --- +DIFF_MAX_DEPTH: int = 3 # max nesting depth for deep_diff detail output +DIFF_MAX_LINES: int = 20 # max number of diff detail lines per config change +ENCRYPTED_PLACEHOLDER: str = "" # placeholder for encrypted values during comparison diff --git a/src/keboola_agent_cli/services/sync_service.py b/src/keboola_agent_cli/services/sync_service.py new file mode 100644 index 00000000..a8f49289 --- /dev/null +++ b/src/keboola_agent_cli/services/sync_service.py @@ -0,0 +1,949 @@ +"""Sync service - business logic for project pull/push/status operations. + +Handles downloading Keboola project configurations to the local filesystem +in a dev-friendly format (YAML configs), and tracking local changes. +""" + +import hashlib +import json +import logging +from pathlib import Path +from typing import Any + +import yaml + +from ..constants import ( + BRANCH_MAPPING_FILENAME, + CONFIG_FILENAME, + KEBOOLA_DIR_NAME, + MANIFEST_VERSION, +) +from ..errors import ConfigError +from ..sync.code_extraction import extract_code_files, merge_code_files +from ..sync.config_format import ( + api_config_to_local, + api_row_to_local, + classify_component_type, + local_config_to_api, +) +from ..sync.diff_engine import compute_changeset +from ..sync.git_utils import get_default_branch, is_git_repo +from ..sync.manifest import ( + Manifest, + ManifestBranch, + ManifestConfigRow, + ManifestConfiguration, + ManifestGitBranching, + ManifestNaming, + ManifestProject, + load_manifest, + save_manifest, +) +from ..sync.naming import config_path, config_row_path +from .base import BaseService + +logger = logging.getLogger(__name__) + + +class SyncService(BaseService): + """Business logic for project sync operations (init, pull, status). + + Single-project operations only. Uses dependency injection for + config_store and client_factory following the BaseService pattern. + """ + + # ------------------------------------------------------------------ + # init + # ------------------------------------------------------------------ + + def init_sync( + self, + alias: str, + project_root: Path, + git_branching: bool = False, + ) -> dict[str, Any]: + """Initialize a sync working directory for a project. + + Creates the ``.keboola/`` directory with ``manifest.json``. + Fetches project metadata from the API to populate the manifest. + + Args: + alias: Project alias from config store. + project_root: Root directory for the sync working tree. + git_branching: Enable git-branching mode. + + Returns: + Dict with initialization stats and created file paths. + + Raises: + ConfigError: If the project alias is not found. + FileExistsError: If manifest already exists (use pull instead). + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + + keboola_dir = project_root / KEBOOLA_DIR_NAME + manifest_path = keboola_dir / "manifest.json" + if manifest_path.exists(): + raise FileExistsError( + f"Manifest already exists at {manifest_path}. " + "Use 'sync pull' to update, or delete .keboola/ to reinitialize." + ) + + # Fetch project info from API + client = self._client_factory(project.stack_url, project.token) + with client: + token_info = client.verify_token() + branches = client.list_dev_branches() + + project_id = token_info.project_id + api_host = project.stack_url.replace("https://", "").rstrip("/") + default_branch_info = next( + (b for b in branches if b.get("isDefault")), + None, + ) + default_branch_id = default_branch_info["id"] if default_branch_info else None + default_branch_name = "main" + + # Git branching setup + git_branching_config = ManifestGitBranching(enabled=False) + if git_branching: + if not is_git_repo(project_root): + raise ConfigError("Git repository not found. Initialize git first: git init") + default_branch_name = get_default_branch(project_root) + git_branching_config = ManifestGitBranching( + enabled=True, + default_branch=default_branch_name, + ) + + # Build manifest + manifest = Manifest( + version=MANIFEST_VERSION, + project=ManifestProject(id=project_id, api_host=api_host), + allow_target_env=True, + git_branching=git_branching_config, + naming=ManifestNaming(), + branches=[ + ManifestBranch( + id=default_branch_id, + path=default_branch_name, + ) + ] + if default_branch_id + else [], + configurations=[], + ) + + # Save manifest + save_manifest(project_root, manifest) + + created_files = [str(manifest_path)] + + # Create branch mapping if git-branching mode + if git_branching: + mapping = { + "version": 1, + "mappings": { + default_branch_name: { + "id": None, + "name": "Main", + } + }, + } + mapping_path = keboola_dir / BRANCH_MAPPING_FILENAME + mapping_path.write_text( + json.dumps(mapping, indent=4, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + created_files.append(str(mapping_path)) + + return { + "status": "initialized", + "project_id": project_id, + "project_alias": alias, + "api_host": api_host, + "git_branching": git_branching, + "default_branch": default_branch_name, + "files_created": created_files, + } + + # ------------------------------------------------------------------ + # pull + # ------------------------------------------------------------------ + + def pull( + self, + alias: str, + project_root: Path, + force: bool = False, + ) -> dict[str, Any]: + """Download all configurations from Keboola to local filesystem. + + Args: + alias: Project alias from config store. + project_root: Root directory of the sync working tree. + force: If True, overwrite existing local files without checking. + + Returns: + Dict with pull statistics (configs, rows, files written). + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + + # Load or verify manifest exists + manifest = load_manifest(project_root) + + # Determine branch to pull from + branch_id = project.active_branch_id + if not branch_id and manifest.branches: + branch_id = manifest.branches[0].id + + # Fetch all components with configs from API + client = self._client_factory(project.stack_url, project.token) + with client: + components = client.list_components_with_configs(branch_id=branch_id) + + # Determine branch directory name + branch_dir_name = "main" + for mb in manifest.branches: + if mb.id == branch_id: + branch_dir_name = mb.path + break + + branch_dir = project_root / branch_dir_name + + # Track stats + configs_pulled = 0 + rows_pulled = 0 + files_written = 0 + new_configurations: list[ManifestConfiguration] = [] + used_paths: set[str] = set() # detect naming collisions + + # Build lookup for existing manifest paths by config ID + # so renames don't cause path changes (stable paths) + existing_paths: dict[str, str] = { + f"{c.component_id}/{c.id}": c.path for c in manifest.configurations + } + + for component in components: + component_id = component.get("id", "") + component_type = classify_component_type(component.get("type", "other")) + configs = component.get("configurations", []) + + for cfg in configs: + config_id = str(cfg.get("id", "")) + config_name = cfg.get("name", "untitled") + + # Reuse existing path if config is already tracked (stable paths) + lookup_key = f"{component_id}/{config_id}" + if lookup_key in existing_paths: + rel_path = existing_paths[lookup_key] + else: + # Generate new filesystem path with collision detection + rel_path = config_path( + manifest.naming.config, + component_type, + component_id, + config_name, + ) + if rel_path in used_paths: + # Append short config ID suffix to resolve collision + suffix = config_id[:8] if len(config_id) > 8 else config_id + rel_path = f"{rel_path}-{suffix}" + used_paths.add(rel_path) + config_dir = branch_dir / rel_path + + # Convert API format to local _config.yml + local_data = api_config_to_local(component_id, cfg, config_id) + + # Extract code files (SQL, Python) if applicable. + # This modifies local_data in place (removes blocks/code) + # and writes separate code files (transform.sql, transform.py, etc.) + extract_code_files(component_id, local_data, config_dir) + + # Write _config.yml (without extracted code) and capture content hash + file_hash = self._write_config_file(config_dir, local_data) + files_written += 1 + configs_pulled += 1 + + # Handle rows + row_manifests: list[ManifestConfigRow] = [] + used_row_paths: set[str] = set() + for row in cfg.get("rows", []): + row_id = str(row.get("id", "")) + row_name = row.get("name", "untitled") + + row_rel_path = config_row_path( + manifest.naming.config_row, + row_name, + ) + if row_rel_path in used_row_paths: + suffix = row_id[:8] if len(row_id) > 8 else row_id + row_rel_path = f"{row_rel_path}-{suffix}" + used_row_paths.add(row_rel_path) + row_dir = config_dir / row_rel_path + + row_local = api_row_to_local(row, component_id) + self._write_config_file(row_dir, row_local) + files_written += 1 + rows_pulled += 1 + + row_manifests.append(ManifestConfigRow(id=row_id, path=row_rel_path)) + + # Record in manifest (store file hash for change detection) + new_configurations.append( + ManifestConfiguration( + branch_id=branch_id or 0, + component_id=component_id, + id=config_id, + path=rel_path, + metadata={"pull_hash": file_hash}, + rows=row_manifests, + ) + ) + + # Update manifest with pulled configurations + manifest.configurations = new_configurations + save_manifest(project_root, manifest) + + return { + "status": "pulled", + "project_alias": alias, + "branch_id": branch_id, + "branch_dir": branch_dir_name, + "configs_pulled": configs_pulled, + "rows_pulled": rows_pulled, + "files_written": files_written, + } + + # ------------------------------------------------------------------ + # status + # ------------------------------------------------------------------ + + def status(self, project_root: Path) -> dict[str, Any]: + """Compare local state against the manifest to detect changes. + + Walks the local filesystem and compares against manifest entries + to classify configurations as modified, added, deleted, or unchanged. + + Args: + project_root: Root directory of the sync working tree. + + Returns: + Dict with lists of modified/added/deleted configs and count of unchanged. + """ + manifest = load_manifest(project_root) + + modified: list[dict[str, str]] = [] + deleted: list[dict[str, str]] = [] + unchanged = 0 + + # Check each manifest entry against local files + for cfg in manifest.configurations: + branch_path = self._find_branch_path(manifest, cfg.branch_id) + config_dir = project_root / branch_path / cfg.path + config_file = config_dir / CONFIG_FILENAME + + if not config_file.exists(): + deleted.append( + { + "component_id": cfg.component_id, + "config_id": cfg.id, + "path": str(cfg.path), + } + ) + continue + + # Compare file hash against the hash stored at pull time + current_hash = self._file_hash(config_file) + pull_hash = cfg.metadata.get("pull_hash", "") + + if pull_hash and current_hash == pull_hash: + unchanged += 1 + else: + modified.append( + { + "component_id": cfg.component_id, + "config_id": cfg.id, + "path": str(cfg.path), + } + ) + + # Scan for added configs (local files without manifest entry) + added = self._find_untracked_configs(project_root, manifest) + + return { + "modified": modified, + "added": added, + "deleted": deleted, + "unchanged": unchanged, + "total_tracked": len(manifest.configurations), + } + + # ------------------------------------------------------------------ + # diff + # ------------------------------------------------------------------ + + def diff( + self, + alias: str, + project_root: Path, + ) -> dict[str, Any]: + """Compare local configs against the remote API state. + + Fetches current state from API, reads local _config.yml files, + and runs the diff engine to produce a detailed changeset. + + Returns: + Dict with 'changes' list and summary counts. + """ + projects = self.resolve_projects([alias]) + project = projects[alias] + manifest = load_manifest(project_root) + + branch_id = project.active_branch_id + if not branch_id and manifest.branches: + branch_id = manifest.branches[0].id + + # Fetch remote state + client = self._client_factory(project.stack_url, project.token) + with client: + components = client.list_components_with_configs(branch_id=branch_id) + + # Build remote configs lookup: "{component_id}/{config_id}" -> API data + remote_configs: dict[str, dict[str, Any]] = {} + for component in components: + component_id = component.get("id", "") + for cfg in component.get("configurations", []): + config_id = str(cfg.get("id", "")) + key = f"{component_id}/{config_id}" + # Convert remote to local format for apples-to-apples comparison + remote_configs[key] = api_config_to_local(component_id, cfg, config_id) + + # Build local configs list from manifest + # Merge code files (transform.sql, code.py, etc.) back into config + # data so comparison with remote is apples-to-apples + local_configs: list[dict[str, Any]] = [] + for cfg in manifest.configurations: + branch_path = self._find_branch_path(manifest, cfg.branch_id) + config_dir = project_root / branch_path / cfg.path + local_data = self._read_config_file(config_dir) + if local_data is None: + continue + merge_code_files(cfg.component_id, local_data, config_dir) + local_configs.append( + { + "component_id": cfg.component_id, + "config_id": cfg.id, + "config_name": local_data.get("name", ""), + "path": cfg.path, + "data": local_data, + } + ) + + # Also add untracked local configs (new files) + for added_cfg in self._find_untracked_configs(project_root, manifest): + branch_path = manifest.branches[0].path if manifest.branches else "main" + config_dir = project_root / branch_path / added_cfg["path"] + local_data = self._read_config_file(config_dir) + if local_data is None: + continue + local_configs.append( + { + "component_id": added_cfg.get("component_id", "unknown"), + "config_id": "", # new config, no ID yet + "config_name": local_data.get("name", ""), + "path": added_cfg["path"], + "data": local_data, + } + ) + + changeset = compute_changeset(local_configs, remote_configs) + + added = [c for c in changeset if c.change_type == "added"] + modified = [c for c in changeset if c.change_type == "modified"] + deleted = [c for c in changeset if c.change_type == "deleted"] + + return { + "changes": [c.to_dict() for c in changeset], + "summary": { + "added": len(added), + "modified": len(modified), + "deleted": len(deleted), + "unchanged": len(local_configs) - len(added) - len(modified), + }, + } + + # ------------------------------------------------------------------ + # push + # ------------------------------------------------------------------ + + def push( + self, + alias: str, + project_root: Path, + dry_run: bool = False, + force: bool = False, + ) -> dict[str, Any]: + """Push local changes to Keboola. + + Computes diff, then creates/updates/deletes configs via API. + New configs get IDs assigned by the API; the manifest is updated. + + Args: + alias: Project alias from config store. + project_root: Root directory of the sync working tree. + dry_run: If True, compute changes but don't execute them. + force: If True, allow deletions without extra confirmation. + + Returns: + Dict with push results (created, updated, deleted, errors). + """ + diff_result = self.diff(alias, project_root) + changes = diff_result["changes"] + + if not changes: + return { + "status": "no_changes", + "created": 0, + "updated": 0, + "deleted": 0, + "errors": [], + } + + if dry_run: + return { + "status": "dry_run", + "changes": changes, + "summary": diff_result["summary"], + } + + projects = self.resolve_projects([alias]) + project = projects[alias] + manifest = load_manifest(project_root) + + branch_id = project.active_branch_id + if not branch_id and manifest.branches: + branch_id = manifest.branches[0].id + + client = self._client_factory(project.stack_url, project.token) + created = 0 + updated = 0 + deleted = 0 + errors: list[dict[str, str]] = [] + + with client: + for change in changes: + change_type = change["change_type"] + component_id = change["component_id"] + config_id = change["config_id"] + config_path_str = change.get("path", "") + + try: + if change_type == "added": + result = self._push_create( + client, + component_id, + config_path_str, + project_root, + manifest, + branch_id, + ) + if result: + created += 1 + + elif change_type == "modified": + self._push_update( + client, + component_id, + config_id, + config_path_str, + project_root, + manifest, + branch_id, + ) + updated += 1 + + elif change_type == "deleted" and force: + client.delete_config( + component_id=component_id, + config_id=config_id, + branch_id=branch_id, + ) + deleted += 1 + + except Exception as exc: + logger.warning( + "Failed to push %s %s/%s: %s", + change_type, + component_id, + config_id, + exc, + ) + errors.append( + { + "change_type": change_type, + "component_id": component_id, + "config_id": config_id, + "message": str(exc), + } + ) + + # Re-pull to update manifest with new IDs and sync state + if created > 0 or updated > 0 or deleted > 0: + self.pull(alias, project_root, force=True) + + return { + "status": "pushed", + "created": created, + "updated": updated, + "deleted": deleted, + "errors": errors, + } + + def _push_create( + self, + client: Any, + component_id: str, + config_path_str: str, + project_root: Path, + manifest: Manifest, + branch_id: int | None, + ) -> dict[str, Any] | None: + """Create a new config from a local _config.yml file.""" + branch_path = manifest.branches[0].path if manifest.branches else "main" + config_dir = project_root / branch_path / config_path_str + local_data = self._read_config_file(config_dir) + if local_data is None: + return None + + # Merge code files (transform.sql, transform.py, code.py) back into config + merge_code_files(component_id, local_data, config_dir) + + name, description, configuration = local_config_to_api(local_data) + result = client.create_config( + component_id=component_id, + name=name, + configuration=configuration, + description=description, + branch_id=branch_id, + ) + logger.info( + "Created config %s/%s (ID: %s)", + component_id, + name, + result.get("id"), + ) + return result + + def _push_update( + self, + client: Any, + component_id: str, + config_id: str, + config_path_str: str, + project_root: Path, + manifest: Manifest, + branch_id: int | None, + ) -> None: + """Update an existing config from a local _config.yml file.""" + branch_path = manifest.branches[0].path if manifest.branches else "main" + config_dir = project_root / branch_path / config_path_str + local_data = self._read_config_file(config_dir) + if local_data is None: + return + + # Merge code files (transform.sql, transform.py, code.py) back into config + merge_code_files(component_id, local_data, config_dir) + + name, description, configuration = local_config_to_api(local_data) + client.update_config( + component_id=component_id, + config_id=config_id, + name=name, + configuration=configuration, + description=description, + change_description="Updated via kbagent sync push", + branch_id=branch_id, + ) + logger.info("Updated config %s/%s", component_id, config_id) + + # ------------------------------------------------------------------ + # branch mapping + # ------------------------------------------------------------------ + + def branch_link( + self, + alias: str, + project_root: Path, + branch_id: int | None = None, + branch_name: str | None = None, + ) -> dict[str, Any]: + """Link the current git branch to a Keboola development branch. + + If no branch_id or branch_name is given: + 1. Get current git branch name + 2. Search for existing Keboola branch with same name + 3. If not found: create a new dev branch + 4. Save mapping to branch-mapping.json + + Args: + alias: Project alias. + project_root: Root directory of the sync working tree. + branch_id: Link to a specific existing Keboola branch. + branch_name: Create/find a branch with this name. + + Returns: + Dict with link result including git branch, Keboola branch ID, name. + """ + from ..sync.branch_mapping import load_branch_mapping, save_branch_mapping + from ..sync.git_utils import get_current_branch + + manifest = load_manifest(project_root) + if not manifest.git_branching.enabled: + raise ConfigError( + "Git-branching mode is not enabled. Run 'sync init --git-branching' first." + ) + + git_branch = get_current_branch(project_root) + if git_branch is None: + raise ConfigError("Cannot determine current git branch.") + + default_branch = manifest.git_branching.default_branch + if git_branch == default_branch: + raise ConfigError( + f"Cannot link the default branch '{default_branch}'. " + "It is automatically linked to Keboola production." + ) + + # Load existing mapping + try: + mapping = load_branch_mapping(project_root) + except FileNotFoundError: + from ..sync.branch_mapping import BranchMapping + + mapping = BranchMapping() + mapping.set(default_branch, None, "Main") + + # Check if already linked + existing = mapping.get(git_branch) + if existing is not None: + return { + "status": "already_linked", + "git_branch": git_branch, + "keboola_branch_id": existing.keboola_id, + "keboola_branch_name": existing.name, + } + + projects = self.resolve_projects([alias]) + project = projects[alias] + client = self._client_factory(project.stack_url, project.token) + + with client: + if branch_id: + # Link to existing branch by ID + branches = client.list_dev_branches() + branch_info = next( + (b for b in branches if b["id"] == branch_id), + None, + ) + if branch_info is None: + raise ConfigError(f"Keboola branch {branch_id} not found.") + kbc_branch_id = str(branch_info["id"]) + kbc_branch_name = branch_info.get("name", "") + elif branch_name: + # Search by name or create + branches = client.list_dev_branches() + branch_info = next( + (b for b in branches if b.get("name") == branch_name), + None, + ) + if branch_info: + kbc_branch_id = str(branch_info["id"]) + kbc_branch_name = branch_info.get("name", "") + else: + result = client.create_dev_branch(name=branch_name) + kbc_branch_id = str(result["id"]) + kbc_branch_name = branch_name + else: + # Default: use git branch name to search/create + branches = client.list_dev_branches() + branch_info = next( + (b for b in branches if b.get("name") == git_branch), + None, + ) + if branch_info: + kbc_branch_id = str(branch_info["id"]) + kbc_branch_name = branch_info.get("name", "") + else: + result = client.create_dev_branch(name=git_branch) + kbc_branch_id = str(result["id"]) + kbc_branch_name = git_branch + + mapping.set(git_branch, kbc_branch_id, kbc_branch_name) + save_branch_mapping(project_root, mapping) + + return { + "status": "linked", + "git_branch": git_branch, + "keboola_branch_id": kbc_branch_id, + "keboola_branch_name": kbc_branch_name, + } + + def branch_unlink( + self, + project_root: Path, + ) -> dict[str, Any]: + """Remove the branch mapping for the current git branch.""" + from ..sync.branch_mapping import load_branch_mapping, save_branch_mapping + from ..sync.git_utils import get_current_branch + + manifest = load_manifest(project_root) + if not manifest.git_branching.enabled: + raise ConfigError("Git-branching mode is not enabled.") + + git_branch = get_current_branch(project_root) + if git_branch is None: + raise ConfigError("Cannot determine current git branch.") + + default_branch = manifest.git_branching.default_branch + if git_branch == default_branch: + raise ConfigError( + f"Cannot unlink the default branch '{default_branch}'. " + "It is permanently linked to Keboola production." + ) + + mapping = load_branch_mapping(project_root) + existing = mapping.get(git_branch) + if existing is None: + return { + "status": "not_linked", + "git_branch": git_branch, + } + + kbc_id = existing.keboola_id + kbc_name = existing.name + mapping.remove(git_branch) + save_branch_mapping(project_root, mapping) + + return { + "status": "unlinked", + "git_branch": git_branch, + "keboola_branch_id": kbc_id, + "keboola_branch_name": kbc_name, + } + + def branch_status( + self, + project_root: Path, + ) -> dict[str, Any]: + """Show the branch mapping status for the current git branch.""" + from ..sync.branch_mapping import load_branch_mapping + from ..sync.git_utils import get_current_branch + + manifest = load_manifest(project_root) + if not manifest.git_branching.enabled: + return {"git_branching": False} + + git_branch = get_current_branch(project_root) + try: + mapping = load_branch_mapping(project_root) + except FileNotFoundError: + return { + "git_branching": True, + "git_branch": git_branch, + "linked": False, + } + + entry = mapping.get(git_branch) if git_branch else None + if entry is None: + return { + "git_branching": True, + "git_branch": git_branch, + "linked": False, + } + + return { + "git_branching": True, + "git_branch": git_branch, + "linked": True, + "keboola_branch_id": entry.keboola_id, + "keboola_branch_name": entry.name, + "is_production": entry.is_production(), + } + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + def _write_config_file(self, config_dir: Path, config_data: dict[str, Any]) -> str: + """Write a ``_config.yml`` file and return its SHA256 hash.""" + config_dir.mkdir(parents=True, exist_ok=True) + config_file = config_dir / CONFIG_FILENAME + content = yaml.dump( + config_data, + default_flow_style=False, + allow_unicode=True, + sort_keys=False, + width=120, + ) + config_file.write_text(content, encoding="utf-8") + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + def _file_hash(self, file_path: Path) -> str: + """Return the SHA256 hex digest of a file's contents.""" + content = file_path.read_bytes() + return hashlib.sha256(content).hexdigest() + + def _read_config_file(self, config_dir: Path) -> dict[str, Any] | None: + """Read and parse a ``_config.yml`` file, returning None if missing.""" + config_file = config_dir / CONFIG_FILENAME + if not config_file.exists(): + return None + try: + return yaml.safe_load(config_file.read_text(encoding="utf-8")) + except yaml.YAMLError: + logger.warning("Failed to parse %s", config_file) + return None + + def _find_branch_path(self, manifest: Manifest, branch_id: int) -> str: + """Find the branch directory name for a given branch ID.""" + for branch in manifest.branches: + if branch.id == branch_id: + return branch.path + return "main" + + def _find_untracked_configs( + self, project_root: Path, manifest: Manifest + ) -> list[dict[str, str]]: + """Scan for _config.yml files that are not tracked in the manifest.""" + tracked_paths: set[str] = set() + for cfg in manifest.configurations: + branch_path = self._find_branch_path(manifest, cfg.branch_id) + tracked_paths.add(str(project_root / branch_path / cfg.path)) + + added: list[dict[str, str]] = [] + for branch in manifest.branches: + branch_dir = project_root / branch.path + if not branch_dir.exists(): + continue + for config_file in branch_dir.rglob(CONFIG_FILENAME): + config_dir = config_file.parent + # Skip row-level configs (they're under rows/ subdirectory) + if "rows" in config_dir.parts: + continue + # Skip branch-level _config.yml + if config_dir == branch_dir: + continue + if str(config_dir) not in tracked_paths: + local_data = self._read_config_file(config_dir) + keboola_meta = local_data.get("_keboola", {}) if local_data else {} + added.append( + { + "component_id": keboola_meta.get("component_id", "unknown"), + "config_id": keboola_meta.get("config_id", ""), + "path": str(config_dir.relative_to(project_root / branch.path)), + } + ) + + return added diff --git a/src/keboola_agent_cli/sync/__init__.py b/src/keboola_agent_cli/sync/__init__.py new file mode 100644 index 00000000..6921cbe1 --- /dev/null +++ b/src/keboola_agent_cli/sync/__init__.py @@ -0,0 +1 @@ +"""Sync package: filesystem serialization for Keboola configurations.""" diff --git a/src/keboola_agent_cli/sync/branch_mapping.py b/src/keboola_agent_cli/sync/branch_mapping.py new file mode 100644 index 00000000..89afac21 --- /dev/null +++ b/src/keboola_agent_cli/sync/branch_mapping.py @@ -0,0 +1,83 @@ +"""Branch mapping for git-to-Keboola branch mapping. + +Manages .keboola/branch-mapping.json which maps git branch names +to Keboola development branch IDs. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from ..constants import BRANCH_MAPPING_FILENAME, KEBOOLA_DIR_NAME + + +class BranchMappingEntry: + """A single git branch -> Keboola branch mapping.""" + + def __init__(self, keboola_id: str | None, name: str): + self.keboola_id = keboola_id # None = production + self.name = name + + def is_production(self) -> bool: + return self.keboola_id is None + + def to_dict(self) -> dict[str, Any]: + return {"id": self.keboola_id, "name": self.name} + + +class BranchMapping: + """Manages git-to-Keboola branch mappings.""" + + def __init__(self) -> None: + self.version: int = 1 + self.mappings: dict[str, BranchMappingEntry] = {} + + def get(self, git_branch: str) -> BranchMappingEntry | None: + return self.mappings.get(git_branch) + + def set(self, git_branch: str, keboola_id: str | None, name: str) -> None: + self.mappings[git_branch] = BranchMappingEntry(keboola_id, name) + + def remove(self, git_branch: str) -> bool: + if git_branch in self.mappings: + del self.mappings[git_branch] + return True + return False + + def to_dict(self) -> dict[str, Any]: + return { + "version": self.version, + "mappings": {k: v.to_dict() for k, v in self.mappings.items()}, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> BranchMapping: + mapping = cls() + mapping.version = data.get("version", 1) + for git_branch, entry in data.get("mappings", {}).items(): + mapping.mappings[git_branch] = BranchMappingEntry( + keboola_id=entry.get("id"), + name=entry.get("name", ""), + ) + return mapping + + +def load_branch_mapping(project_root: Path) -> BranchMapping: + """Load .keboola/branch-mapping.json.""" + path = project_root / KEBOOLA_DIR_NAME / BRANCH_MAPPING_FILENAME + if not path.exists(): + raise FileNotFoundError(f"Branch mapping not found at {path}") + data = json.loads(path.read_text(encoding="utf-8")) + return BranchMapping.from_dict(data) + + +def save_branch_mapping(project_root: Path, mapping: BranchMapping) -> None: + """Save branch mapping to .keboola/branch-mapping.json.""" + path = project_root / KEBOOLA_DIR_NAME / BRANCH_MAPPING_FILENAME + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(mapping.to_dict(), indent=4, ensure_ascii=False) + "\n", + encoding="utf-8", + ) diff --git a/src/keboola_agent_cli/sync/code_extraction.py b/src/keboola_agent_cli/sync/code_extraction.py new file mode 100644 index 00000000..733aed6f --- /dev/null +++ b/src/keboola_agent_cli/sync/code_extraction.py @@ -0,0 +1,397 @@ +"""Extract and merge embedded code from Keboola configurations. + +On pull: extracts SQL/Python code from config parameters into separate files. +On push: reads code files back into config parameters. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +# Component patterns that contain SQL transformations +SQL_TRANSFORMATION_COMPONENTS: set[str] = { + "keboola.snowflake-transformation", + "keboola.synapse-transformation", + "keboola.oracle-transformation", + "keboola.redshift-sql-transformation", +} + +# Component patterns that contain Python transformations +PYTHON_TRANSFORMATION_COMPONENTS: set[str] = { + "keboola.python-transformation-v2", +} + +# Components with embedded Python code (custom apps) +PYTHON_APP_COMPONENTS: set[str] = { + "kds-team.app-custom-python", +} + +SQL_BLOCK_MARKER = "/* ===== BLOCK: {name} ===== */" +SQL_CODE_MARKER = "/* ===== CODE: {name} ===== */" +PYTHON_BLOCK_MARKER = "# ===== BLOCK: {name} =====" +PYTHON_CODE_MARKER = "# ===== CODE: {name} =====" + + +def extract_code_files( + component_id: str, + config_data: dict[str, Any], + config_dir: Path, +) -> dict[str, Any]: + """Extract embedded code from config into separate files. + + Modifies config_data in place to remove extracted code. + Writes code files to config_dir. + Returns the modified config_data. + """ + if component_id in SQL_TRANSFORMATION_COMPONENTS: + return _extract_sql_transformation(config_data, config_dir) + if component_id in PYTHON_TRANSFORMATION_COMPONENTS: + return _extract_python_transformation(config_data, config_dir) + if component_id in PYTHON_APP_COMPONENTS: + return _extract_python_app(config_data, config_dir) + return config_data + + +def merge_code_files( + component_id: str, + config_data: dict[str, Any], + config_dir: Path, +) -> dict[str, Any]: + """Read code files and merge them back into config_data. + + Reverse of extract_code_files. Called before push. + Returns the modified config_data. + """ + if component_id in SQL_TRANSFORMATION_COMPONENTS: + return _merge_sql_transformation(config_data, config_dir) + if component_id in PYTHON_TRANSFORMATION_COMPONENTS: + return _merge_python_transformation(config_data, config_dir) + if component_id in PYTHON_APP_COMPONENTS: + return _merge_python_app(config_data, config_dir) + return config_data + + +# ---- SQL Transformations ---- + + +def _extract_sql_transformation(config_data: dict[str, Any], config_dir: Path) -> dict[str, Any]: + """Extract SQL blocks from parameters.blocks into transform.sql.""" + parameters = config_data.get("parameters", {}) + blocks = parameters.get("blocks", []) + + if not blocks: + return config_data + + lines: list[str] = [] + for block in blocks: + block_name = block.get("name", "unnamed") + lines.append(SQL_BLOCK_MARKER.format(name=block_name)) + lines.append("") + + for code in block.get("codes", []): + code_name = code.get("name", "unnamed") + lines.append(SQL_CODE_MARKER.format(name=code_name)) + + scripts = code.get("script", []) + for script in scripts: + lines.append(script) + lines.append("") + + sql_content = "\n".join(lines).rstrip() + "\n" + + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / "transform.sql").write_text(sql_content, encoding="utf-8") + + # Remove blocks from parameters (they're now in the SQL file) + parameters.pop("blocks", None) + + return config_data + + +def _merge_sql_transformation(config_data: dict[str, Any], config_dir: Path) -> dict[str, Any]: + """Read transform.sql and parse block markers back into parameters.blocks.""" + sql_file = config_dir / "transform.sql" + if not sql_file.exists(): + return config_data + + content = sql_file.read_text(encoding="utf-8") + blocks = _parse_sql_blocks(content) + + parameters = config_data.setdefault("parameters", {}) + parameters["blocks"] = blocks + + return config_data + + +def _parse_sql_blocks(content: str) -> list[dict[str, Any]]: + """Parse SQL content with block/code markers into blocks structure.""" + blocks: list[dict[str, Any]] = [] + current_block: dict[str, Any] | None = None + current_code: dict[str, Any] | None = None + current_script_lines: list[str] = [] + + for line in content.split("\n"): + stripped = line.strip() + + # Check for block marker + if stripped.startswith("/* ===== BLOCK:") and stripped.endswith("===== */"): + # Save previous code if any + if current_code is not None and current_block is not None: + current_code["script"] = ["\n".join(current_script_lines).strip()] + current_block.setdefault("codes", []).append(current_code) + current_code = None + current_script_lines = [] + + block_name = stripped[len("/* ===== BLOCK:") :].rstrip(" =*/").strip() + current_block = {"name": block_name, "codes": []} + blocks.append(current_block) + continue + + # Check for code marker + if stripped.startswith("/* ===== CODE:") and stripped.endswith("===== */"): + # Save previous code if any + if current_code is not None and current_block is not None: + current_code["script"] = ["\n".join(current_script_lines).strip()] + current_block.setdefault("codes", []).append(current_code) + current_script_lines = [] + + code_name = stripped[len("/* ===== CODE:") :].rstrip(" =*/").strip() + current_code = {"name": code_name} + continue + + # Regular line - add to current code's script + if current_code is not None: + current_script_lines.append(line) + + # Don't forget the last code block + if current_code is not None and current_block is not None: + current_code["script"] = ["\n".join(current_script_lines).strip()] + current_block.setdefault("codes", []).append(current_code) + + # If no markers found, treat entire content as single block/code + if not blocks and content.strip(): + blocks = [ + { + "name": "Block 1", + "codes": [{"name": "Code 1", "script": [content.strip()]}], + } + ] + + return blocks + + +# ---- Python Transformations ---- + + +def _extract_python_transformation(config_data: dict[str, Any], config_dir: Path) -> dict[str, Any]: + """Extract Python blocks from parameters.blocks into transform.py, packages into pyproject.toml.""" + parameters = config_data.get("parameters", {}) + blocks = parameters.get("blocks", []) + + if blocks: + lines: list[str] = [] + for block in blocks: + block_name = block.get("name", "unnamed") + lines.append(PYTHON_BLOCK_MARKER.format(name=block_name)) + lines.append("") + + for code in block.get("codes", []): + code_name = code.get("name", "unnamed") + lines.append(PYTHON_CODE_MARKER.format(name=code_name)) + + scripts = code.get("script", []) + for script in scripts: + lines.append(script) + lines.append("") + + py_content = "\n".join(lines).rstrip() + "\n" + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / "transform.py").write_text(py_content, encoding="utf-8") + + # Remove blocks from parameters + parameters.pop("blocks", None) + + # Extract packages to pyproject.toml + packages = parameters.get("packages", []) + if packages: + config_name = config_data.get("name", "transformation") + _write_pyproject_toml(config_dir, config_name, packages, component_id=None, config_id=None) + parameters.pop("packages", None) + + return config_data + + +def _merge_python_transformation(config_data: dict[str, Any], config_dir: Path) -> dict[str, Any]: + """Read transform.py and pyproject.toml back into config_data.""" + py_file = config_dir / "transform.py" + if py_file.exists(): + content = py_file.read_text(encoding="utf-8") + blocks = _parse_python_blocks(content) + parameters = config_data.setdefault("parameters", {}) + parameters["blocks"] = blocks + + # Read packages from pyproject.toml + packages = _read_pyproject_packages(config_dir) + if packages: + parameters = config_data.setdefault("parameters", {}) + parameters["packages"] = packages + + return config_data + + +def _parse_python_blocks(content: str) -> list[dict[str, Any]]: + """Parse Python content with block/code markers into blocks structure.""" + blocks: list[dict[str, Any]] = [] + current_block: dict[str, Any] | None = None + current_code: dict[str, Any] | None = None + current_script_lines: list[str] = [] + + for line in content.split("\n"): + stripped = line.strip() + + if stripped.startswith("# ===== BLOCK:") and stripped.endswith("====="): + if current_code is not None and current_block is not None: + current_code["script"] = ["\n".join(current_script_lines).strip()] + current_block.setdefault("codes", []).append(current_code) + current_code = None + current_script_lines = [] + + block_name = stripped[len("# ===== BLOCK:") :].rstrip(" =").strip() + current_block = {"name": block_name, "codes": []} + blocks.append(current_block) + continue + + if stripped.startswith("# ===== CODE:") and stripped.endswith("====="): + if current_code is not None and current_block is not None: + current_code["script"] = ["\n".join(current_script_lines).strip()] + current_block.setdefault("codes", []).append(current_code) + current_script_lines = [] + + code_name = stripped[len("# ===== CODE:") :].rstrip(" =").strip() + current_code = {"name": code_name} + continue + + if current_code is not None: + current_script_lines.append(line) + + if current_code is not None and current_block is not None: + current_code["script"] = ["\n".join(current_script_lines).strip()] + current_block.setdefault("codes", []).append(current_code) + + if not blocks and content.strip(): + blocks = [ + { + "name": "Block 1", + "codes": [{"name": "Code 1", "script": [content.strip()]}], + } + ] + + return blocks + + +# ---- Python Apps ---- + + +def _extract_python_app(config_data: dict[str, Any], config_dir: Path) -> dict[str, Any]: + """Extract parameters.code into code.py and packages into pyproject.toml.""" + parameters = config_data.get("parameters", {}) + + code = parameters.get("code") + if code and isinstance(code, str): + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / "code.py").write_text(code, encoding="utf-8") + parameters.pop("code", None) + + packages = parameters.get("packages", []) + if packages: + keboola_meta = config_data.get("_keboola", {}) + _write_pyproject_toml( + config_dir, + config_data.get("name", "app"), + packages, + component_id=keboola_meta.get("component_id"), + config_id=keboola_meta.get("config_id"), + ) + parameters.pop("packages", None) + + return config_data + + +def _merge_python_app(config_data: dict[str, Any], config_dir: Path) -> dict[str, Any]: + """Read code.py and pyproject.toml back into config_data.""" + code_file = config_dir / "code.py" + if code_file.exists(): + parameters = config_data.setdefault("parameters", {}) + parameters["code"] = code_file.read_text(encoding="utf-8") + + packages = _read_pyproject_packages(config_dir) + if packages: + parameters = config_data.setdefault("parameters", {}) + parameters["packages"] = packages + + return config_data + + +# ---- pyproject.toml helpers ---- + + +def _write_pyproject_toml( + config_dir: Path, + name: str, + packages: list[str], + component_id: str | None = None, + config_id: str | None = None, +) -> None: + """Write packages to a pyproject.toml file.""" + config_dir.mkdir(parents=True, exist_ok=True) + + # Sanitize name for pyproject + safe_name = name.lower().replace(" ", "-").replace("_", "-") + + lines = [ + "[project]", + f'name = "{safe_name}"', + 'requires-python = ">=3.11"', + "dependencies = [", + ] + for pkg in packages: + lines.append(f' "{pkg}",') + lines.append("]") + + if component_id or config_id: + lines.append("") + lines.append("[tool.keboola]") + if component_id: + lines.append(f'component_id = "{component_id}"') + if config_id: + lines.append(f'config_id = "{config_id}"') + + lines.append("") # trailing newline + (config_dir / "pyproject.toml").write_text("\n".join(lines), encoding="utf-8") + + +def _read_pyproject_packages(config_dir: Path) -> list[str]: + """Read packages from pyproject.toml dependencies.""" + toml_file = config_dir / "pyproject.toml" + if not toml_file.exists(): + return [] + + content = toml_file.read_text(encoding="utf-8") + packages: list[str] = [] + in_deps = False + + for line in content.split("\n"): + stripped = line.strip() + if stripped == "dependencies = [": + in_deps = True + continue + if in_deps: + if stripped == "]": + break + # Strip quotes and trailing comma + pkg = stripped.strip('", ') + if pkg: + packages.append(pkg) + + return packages diff --git a/src/keboola_agent_cli/sync/config_format.py b/src/keboola_agent_cli/sync/config_format.py new file mode 100644 index 00000000..23e4f9c8 --- /dev/null +++ b/src/keboola_agent_cli/sync/config_format.py @@ -0,0 +1,191 @@ +"""Conversion between Keboola API JSON and local _config.yml format. + +The local format is a human-friendly YAML structure that "promotes" deeply +nested configuration keys (parameters, storage.input, storage.output, +processors) to the top level and adds a ``_keboola`` metadata block. +""" + +from __future__ import annotations + +from typing import Any + +from ..constants import CONFIG_YML_VERSION + +# --------------------------------------------------------------------------- +# Component type mapping +# --------------------------------------------------------------------------- + +COMPONENT_TYPE_MAP: dict[str, str] = { + "extractor": "extractor", + "writer": "writer", + "transformation": "transformation", + "application": "application", + "other": "other", +} + +# Orchestrator-like components that have special handling +ORCHESTRATOR_COMPONENTS: set[str] = {"keboola.orchestrator", "keboola.flow"} + + +def classify_component_type(api_type: str) -> str: + """Map an API component type string to its filesystem directory name. + + Falls back to ``"other"`` for unknown types. + """ + return COMPONENT_TYPE_MAP.get(api_type, "other") + + +# --------------------------------------------------------------------------- +# API -> local _config.yml +# --------------------------------------------------------------------------- + + +def api_config_to_local( + component_id: str, config_data: dict[str, Any], config_id: str +) -> dict[str, Any]: + """Convert an API configuration response to the local ``_config.yml`` structure. + + Transformation rules: + - ``version``: always ``CONFIG_YML_VERSION`` + - ``name``, ``description``: taken from the top-level API response + - ``configuration.parameters`` -> ``parameters`` + - ``configuration.storage.input`` -> ``input`` + - ``configuration.storage.output`` -> ``output`` + - ``configuration.processors`` -> ``processors`` + - ``_keboola``: ``{component_id, config_id}`` + + Any remaining keys inside ``configuration`` that are not explicitly + promoted are preserved under a ``_configuration_extra`` key so that + round-tripping does not lose data. + """ + configuration: dict[str, Any] = config_data.get("configuration", {}) + + local: dict[str, Any] = { + "version": CONFIG_YML_VERSION, + "name": config_data.get("name", ""), + "description": config_data.get("description", ""), + } + + # Promote well-known nested keys + if "parameters" in configuration: + local["parameters"] = configuration["parameters"] + + storage: dict[str, Any] = configuration.get("storage", {}) + if "input" in storage: + local["input"] = storage["input"] + if "output" in storage: + local["output"] = storage["output"] + + if "processors" in configuration: + local["processors"] = configuration["processors"] + + # Preserve any extra keys that we do not explicitly promote + promoted_keys = {"parameters", "storage", "processors"} + extras = {k: v for k, v in configuration.items() if k not in promoted_keys} + if extras: + local["_configuration_extra"] = extras + + # Keboola metadata footer + local["_keboola"] = { + "component_id": component_id, + "config_id": config_id, + } + + return local + + +# --------------------------------------------------------------------------- +# local _config.yml -> API +# --------------------------------------------------------------------------- + + +def local_config_to_api( + config_yml: dict[str, Any], +) -> tuple[str, str, dict[str, Any]]: + """Convert a local ``_config.yml`` dict back to API format. + + Returns: + A tuple of ``(name, description, configuration_dict)`` suitable for + an API create/update call. + """ + name: str = config_yml.get("name", "") + description: str = config_yml.get("description", "") + + configuration: dict[str, Any] = {} + + if "parameters" in config_yml: + configuration["parameters"] = config_yml["parameters"] + + # Re-nest input/output under storage + storage: dict[str, Any] = {} + if "input" in config_yml: + storage["input"] = config_yml["input"] + if "output" in config_yml: + storage["output"] = config_yml["output"] + if storage: + configuration["storage"] = storage + + if "processors" in config_yml: + configuration["processors"] = config_yml["processors"] + + # Merge back any extras that were preserved during api->local conversion + extras: dict[str, Any] = config_yml.get("_configuration_extra", {}) + for key, value in extras.items(): + configuration.setdefault(key, value) + + return name, description, configuration + + +# --------------------------------------------------------------------------- +# Row helpers +# --------------------------------------------------------------------------- + + +def api_row_to_local(row_data: dict[str, Any], component_id: str) -> dict[str, Any]: + """Convert an API configuration row to a local row ``_config.yml``. + + Follows the same promotion rules as :func:`api_config_to_local`. + """ + configuration: dict[str, Any] = row_data.get("configuration", {}) + + local: dict[str, Any] = { + "version": CONFIG_YML_VERSION, + "name": row_data.get("name", ""), + "description": row_data.get("description", ""), + } + + if "parameters" in configuration: + local["parameters"] = configuration["parameters"] + + storage: dict[str, Any] = configuration.get("storage", {}) + if "input" in storage: + local["input"] = storage["input"] + if "output" in storage: + local["output"] = storage["output"] + + if "processors" in configuration: + local["processors"] = configuration["processors"] + + promoted_keys = {"parameters", "storage", "processors"} + extras = {k: v for k, v in configuration.items() if k not in promoted_keys} + if extras: + local["_configuration_extra"] = extras + + local["_keboola"] = { + "component_id": component_id, + "row_id": row_data.get("id", ""), + } + + return local + + +def local_row_to_api( + row_yml: dict[str, Any], +) -> tuple[str, str, dict[str, Any]]: + """Convert a local row ``_config.yml`` back to API format. + + Returns: + A tuple of ``(name, description, configuration_dict)``. + """ + # Reuse the same logic -- the structure is identical + return local_config_to_api(row_yml) diff --git a/src/keboola_agent_cli/sync/diff_engine.py b/src/keboola_agent_cli/sync/diff_engine.py new file mode 100644 index 00000000..85361534 --- /dev/null +++ b/src/keboola_agent_cli/sync/diff_engine.py @@ -0,0 +1,368 @@ +"""Diff engine for comparing local configs against remote API state. + +Produces a changeset describing what needs to be created, updated, +or deleted when pushing local changes to Keboola. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +from typing import Any + +from ..constants import DIFF_MAX_DEPTH, DIFF_MAX_LINES, ENCRYPTED_PLACEHOLDER +from .secrets import is_encrypted_value + +# Keys that are internal bookkeeping and should be excluded from comparison. +_IGNORED_KEYS: frozenset[str] = frozenset({"_keboola", "version", "_configuration_extra"}) + + +class ConfigChange: + """Represents a single configuration change.""" + + def __init__( + self, + change_type: str, # "added", "modified", "deleted" + component_id: str, + config_id: str, # empty string for new configs + config_name: str, + path: str, + local_data: dict[str, Any] | None = None, + remote_data: dict[str, Any] | None = None, + details: list[str] | None = None, + ): + self.change_type = change_type + self.component_id = component_id + self.config_id = config_id + self.config_name = config_name + self.path = path + self.local_data = local_data + self.remote_data = remote_data + self.details = details or [] + + def to_dict(self) -> dict[str, Any]: + """Serialize to dict for JSON output.""" + return { + "change_type": self.change_type, + "component_id": self.component_id, + "config_id": self.config_id, + "config_name": self.config_name, + "path": self.path, + "details": self.details, + } + + +# --------------------------------------------------------------------------- +# Normalization helpers +# --------------------------------------------------------------------------- + + +def normalize_for_comparison(obj: Any) -> Any: + """Normalize a config dict for comparison. + + - Replace all encrypted values (KBC::*Secure::*) with a fixed placeholder + to avoid false diffs from encryption nonces. + - Sort dict keys for consistent hashing. + - Strip ``_keboola`` metadata block (internal, not part of config content). + - Strip ``version`` key (local format marker). + - Strip ``_configuration_extra`` key (internal round-trip aid). + + Returns a deep copy -- the original object is never mutated. + """ + return _normalize(copy.deepcopy(obj)) + + +def _normalize(obj: Any) -> Any: + """Recursively normalize *obj* in place (operates on a deep copy).""" + if isinstance(obj, dict): + # Remove ignored keys + for key in _IGNORED_KEYS: + obj.pop(key, None) + + # Recurse into remaining values, replacing encrypted strings + normalized: dict[str, Any] = {} + for key in sorted(obj.keys()): + normalized[key] = _normalize(obj[key]) + return normalized + + if isinstance(obj, list): + return [_normalize(item) for item in obj] + + if is_encrypted_value(obj): + return ENCRYPTED_PLACEHOLDER + + return obj + + +# --------------------------------------------------------------------------- +# Hashing +# --------------------------------------------------------------------------- + + +def config_hash(config_data: dict[str, Any]) -> str: + """Compute a normalized hash of a config for fast change detection.""" + normalized = normalize_for_comparison(config_data) + return hashlib.sha256(json.dumps(normalized, sort_keys=True).encode()).hexdigest() + + +# --------------------------------------------------------------------------- +# Deep diff +# --------------------------------------------------------------------------- + + +def deep_diff( + local: dict[str, Any], + remote: dict[str, Any], + path: str = "", +) -> list[str]: + """Compute human-readable diff descriptions between two config dicts. + + Returns a list of strings like: + + - ``"parameters.api_url changed: 'old' -> 'new'"`` + - ``"output.tables[0].destination added"`` + - ``"description removed"`` + + Skips encrypted values (shows ``'changed (encrypted)'`` instead of + actual values). Limits comparison depth to ``DIFF_MAX_DEPTH`` levels + and caps the output at ``DIFF_MAX_LINES`` entries. + """ + results: list[str] = [] + _deep_diff_recurse( + normalize_for_comparison(local), + normalize_for_comparison(remote), + path=path, + depth=0, + results=results, + ) + return results[:DIFF_MAX_LINES] + + +def _deep_diff_recurse( + local_val: Any, + remote_val: Any, + *, + path: str, + depth: int, + results: list[str], +) -> None: + """Recursive worker for :func:`deep_diff`.""" + # Early exit when we've already collected enough detail lines. + if len(results) >= DIFF_MAX_LINES: + return + + if type(local_val) != type(remote_val): # noqa: E721 + results.append(_format_changed(path, remote_val, local_val)) + return + + if isinstance(local_val, dict) and isinstance(remote_val, dict): + _diff_dicts(local_val, remote_val, path=path, depth=depth, results=results) + return + + if isinstance(local_val, list) and isinstance(remote_val, list): + _diff_lists(local_val, remote_val, path=path, depth=depth, results=results) + return + + # Scalar comparison + if local_val != remote_val: + results.append(_format_changed(path, remote_val, local_val)) + + +def _diff_dicts( + local_dict: dict[str, Any], + remote_dict: dict[str, Any], + *, + path: str, + depth: int, + results: list[str], +) -> None: + """Compare two dicts key by key.""" + all_keys = sorted(set(local_dict.keys()) | set(remote_dict.keys())) + + for key in all_keys: + if len(results) >= DIFF_MAX_LINES: + return + + child_path = f"{path}.{key}" if path else key + in_local = key in local_dict + in_remote = key in remote_dict + + if in_local and not in_remote: + results.append(f"{child_path} added") + continue + + if not in_local and in_remote: + results.append(f"{child_path} removed") + continue + + # Both sides have the key -- recurse if within depth budget. + if depth < DIFF_MAX_DEPTH: + _deep_diff_recurse( + local_dict[key], + remote_dict[key], + path=child_path, + depth=depth + 1, + results=results, + ) + else: + # Beyond depth limit, fall back to equality check. + if local_dict[key] != remote_dict[key]: + results.append(f"{child_path} changed") + + +def _diff_lists( + local_list: list[Any], + remote_list: list[Any], + *, + path: str, + depth: int, + results: list[str], +) -> None: + """Compare two lists element by element.""" + if len(local_list) != len(remote_list): + results.append(f"{path} list length changed: {len(remote_list)} -> {len(local_list)}") + return + + for idx, (local_item, remote_item) in enumerate(zip(local_list, remote_list, strict=True)): + if len(results) >= DIFF_MAX_LINES: + return + + child_path = f"{path}[{idx}]" + + if depth < DIFF_MAX_DEPTH: + _deep_diff_recurse( + local_item, + remote_item, + path=child_path, + depth=depth + 1, + results=results, + ) + else: + if local_item != remote_item: + results.append(f"{child_path} changed") + + +def _format_changed(path: str, old_val: Any, new_val: Any) -> str: + """Format a single scalar change, masking encrypted placeholders.""" + label = path if path else "(root)" + + if old_val == ENCRYPTED_PLACEHOLDER or new_val == ENCRYPTED_PLACEHOLDER: + return f"{label} changed (encrypted)" + + return f"{label} changed: {_repr_short(old_val)} -> {_repr_short(new_val)}" + + +def _repr_short(value: Any, max_length: int = 60) -> str: + """Short repr of a value, truncated if too long.""" + text = repr(value) + if len(text) > max_length: + return text[: max_length - 3] + "..." + return text + + +# --------------------------------------------------------------------------- +# Changeset computation +# --------------------------------------------------------------------------- + + +def compute_changeset( + local_configs: list[dict[str, Any]], + remote_configs: dict[str, dict[str, Any]], +) -> list[ConfigChange]: + """Compare local configs against remote state to produce a changeset. + + Args: + local_configs: List of dicts with keys: + ``component_id``, ``config_id``, ``config_name``, ``path``, ``data``. + remote_configs: Dict keyed by ``"{component_id}/{config_id}"`` with + API config data. + + Returns: + List of :class:`ConfigChange` objects describing what needs to be + created, updated, or deleted. + + Logic: + + - For each local config with a ``config_id`` that exists in remote: + compare hashes; if different -> ``"modified"`` with + :func:`deep_diff` details. + - For each local config with an empty ``config_id`` (or not found in + remote): -> ``"added"``. + - For each remote config not referenced by any local config: + -> ``"deleted"``. + """ + changes: list[ConfigChange] = [] + seen_remote_keys: set[str] = set() + + for entry in local_configs: + component_id: str = entry["component_id"] + config_id: str = entry.get("config_id", "") + config_name: str = entry.get("config_name", "") + path: str = entry.get("path", "") + local_data: dict[str, Any] = entry.get("data", {}) + + remote_key = f"{component_id}/{config_id}" if config_id else "" + + # New config (no id yet, or not in remote) + if not config_id or remote_key not in remote_configs: + changes.append( + ConfigChange( + change_type="added", + component_id=component_id, + config_id=config_id, + config_name=config_name, + path=path, + local_data=local_data, + ) + ) + if remote_key: + seen_remote_keys.add(remote_key) + continue + + # Existing config -- compare + seen_remote_keys.add(remote_key) + remote_data: dict[str, Any] = remote_configs[remote_key] + + local_h = config_hash(local_data) + remote_h = config_hash(remote_data) + + if local_h == remote_h: + # Unchanged -- skip + continue + + details = deep_diff(local_data, remote_data) + changes.append( + ConfigChange( + change_type="modified", + component_id=component_id, + config_id=config_id, + config_name=config_name, + path=path, + local_data=local_data, + remote_data=remote_data, + details=details, + ) + ) + + # Detect deleted configs (in remote but not referenced locally) + for remote_key, remote_data in remote_configs.items(): + if remote_key in seen_remote_keys: + continue + + parts = remote_key.split("/", 1) + component_id = parts[0] if len(parts) > 0 else "" + config_id = parts[1] if len(parts) > 1 else "" + + changes.append( + ConfigChange( + change_type="deleted", + component_id=component_id, + config_id=config_id, + config_name=remote_data.get("name", ""), + path="", + remote_data=remote_data, + ) + ) + + return changes diff --git a/src/keboola_agent_cli/sync/git_utils.py b/src/keboola_agent_cli/sync/git_utils.py new file mode 100644 index 00000000..4170ca5f --- /dev/null +++ b/src/keboola_agent_cli/sync/git_utils.py @@ -0,0 +1,89 @@ +"""Git helper functions for sync operations. + +All functions are safe to call on machines without git installed -- they +return ``None`` or sensible defaults when the ``git`` binary is missing +or the directory is not a repository. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + + +def is_git_repo(path: Path) -> bool: + """Return ``True`` if *path* is inside a git working tree.""" + result = subprocess.run( + ["git", "rev-parse", "--is-inside-work-tree"], + cwd=path, + capture_output=True, + text=True, + check=False, + ) + return result.returncode == 0 and result.stdout.strip() == "true" + + +def get_current_branch(path: Path) -> str | None: + """Return the current git branch name, or ``None`` on failure. + + Uses ``git rev-parse --abbrev-ref HEAD`` which returns the + symbolic branch name (e.g. ``main``, ``feature/foo``). + """ + result = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + cwd=path, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return None + branch = result.stdout.strip() + # Detached HEAD returns "HEAD" + return branch if branch and branch != "HEAD" else None + + +def get_default_branch(path: Path) -> str: + """Detect the default branch name (``main`` or ``master``). + + Strategy: + 1. Check ``git config init.defaultBranch`` (user/repo setting). + 2. Check if ``refs/remotes/origin/main`` exists. + 3. Check if ``refs/remotes/origin/master`` exists. + 4. Fall back to ``"main"``. + """ + # 1. git config + result = subprocess.run( + ["git", "config", "init.defaultBranch"], + cwd=path, + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0 and result.stdout.strip(): + return result.stdout.strip() + + # 2. origin/main + result = subprocess.run( + ["git", "rev-parse", "--verify", "refs/remotes/origin/main"], + cwd=path, + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + return "main" + + # 3. origin/master + result = subprocess.run( + ["git", "rev-parse", "--verify", "refs/remotes/origin/master"], + cwd=path, + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + return "master" + + # 4. Default + return "main" diff --git a/src/keboola_agent_cli/sync/manifest.py b/src/keboola_agent_cli/sync/manifest.py new file mode 100644 index 00000000..49a48058 --- /dev/null +++ b/src/keboola_agent_cli/sync/manifest.py @@ -0,0 +1,152 @@ +"""Pydantic v2 models for .keboola/manifest.json (v2, camelCase via aliases). + +Mirrors the manifest format used by the Keboola Go CLI so that +directories written by kbagent are compatible with `kbc` tooling. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from ..constants import KEBOOLA_DIR_NAME, MANIFEST_FILENAME, MANIFEST_VERSION + +# --------------------------------------------------------------------------- +# Sub-models +# --------------------------------------------------------------------------- + + +class ManifestProject(BaseModel): + """Project identification inside the manifest.""" + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + id: int + api_host: str = Field(alias="apiHost") + + +class ManifestGitBranching(BaseModel): + """Git branching settings.""" + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + enabled: bool = False + default_branch: str = Field(default="main", alias="defaultBranch") + + +class ManifestNaming(BaseModel): + """Naming templates that control the filesystem layout.""" + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + branch: str = "{branch_name}" + config: str = "{component_type}/{component_id}/{config_name}" + config_row: str = Field(default="rows/{config_row_name}", alias="configRow") + scheduler_config: str = Field(default="schedules/{config_name}", alias="schedulerConfig") + shared_code_config: str = Field( + default="_shared/{target_component_id}", alias="sharedCodeConfig" + ) + shared_code_config_row: str = Field( + default="codes/{config_row_name}", alias="sharedCodeConfigRow" + ) + variables_config: str = Field(default="variables", alias="variablesConfig") + variables_values_row: str = Field( + default="values/{config_row_name}", alias="variablesValuesRow" + ) + data_app_config: str = Field(default="app/{component_id}/{config_name}", alias="dataAppConfig") + + +class ManifestBranch(BaseModel): + """A branch entry in the manifest.""" + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + id: int + path: str + metadata: dict[str, Any] = Field(default_factory=dict) + + +class ManifestConfigRow(BaseModel): + """A single configuration row reference.""" + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + id: str + path: str + + +class ManifestConfiguration(BaseModel): + """A single configuration reference inside the manifest.""" + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + branch_id: int = Field(alias="branchId") + component_id: str = Field(alias="componentId") + id: str + path: str + metadata: dict[str, Any] = Field(default_factory=dict) + rows: list[ManifestConfigRow] = Field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Root manifest +# --------------------------------------------------------------------------- + + +class Manifest(BaseModel): + """Root model for .keboola/manifest.json (schema version 2).""" + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + version: int = MANIFEST_VERSION + project: ManifestProject + allow_target_env: bool = Field(default=True, alias="allowTargetEnv") + git_branching: ManifestGitBranching = Field(alias="gitBranching") + sort_by: str = Field(default="id", alias="sortBy") + naming: ManifestNaming + allowed_branches: list[str] = Field(default_factory=list, alias="allowedBranches") + ignored_components: list[str] = Field(default_factory=list, alias="ignoredComponents") + branches: list[ManifestBranch] = Field(default_factory=list) + configurations: list[ManifestConfiguration] = Field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Load / Save helpers +# --------------------------------------------------------------------------- + + +def load_manifest(project_root: Path) -> Manifest: + """Load .keboola/manifest.json from *project_root*. + + Raises: + FileNotFoundError: if the manifest file does not exist. + ValueError: if the JSON cannot be parsed into a valid Manifest. + """ + manifest_path = project_root / KEBOOLA_DIR_NAME / MANIFEST_FILENAME + if not manifest_path.exists(): + raise FileNotFoundError( + f"Manifest not found at {manifest_path}. Is this a Keboola project directory?" + ) + + raw = json.loads(manifest_path.read_text(encoding="utf-8")) + return Manifest.model_validate(raw) + + +def save_manifest(project_root: Path, manifest: Manifest) -> None: + """Save *manifest* to .keboola/manifest.json. + + Uses ``by_alias=True`` so all keys are written in camelCase, + matching the format expected by the Go CLI. + """ + keboola_dir = project_root / KEBOOLA_DIR_NAME + keboola_dir.mkdir(parents=True, exist_ok=True) + + manifest_path = keboola_dir / MANIFEST_FILENAME + payload = manifest.model_dump(mode="json", by_alias=True) + manifest_path.write_text( + json.dumps(payload, indent=4, ensure_ascii=False) + "\n", + encoding="utf-8", + ) diff --git a/src/keboola_agent_cli/sync/naming.py b/src/keboola_agent_cli/sync/naming.py new file mode 100644 index 00000000..c4c41a31 --- /dev/null +++ b/src/keboola_agent_cli/sync/naming.py @@ -0,0 +1,61 @@ +"""Path generation from naming templates. + +The naming templates (stored in ``manifest.json``) use placeholders like +``{component_type}`` and ``{config_name}`` to produce deterministic +filesystem paths for each configuration. +""" + +from __future__ import annotations + +import re + +from ..constants import SANITIZE_NAME_MAX_LENGTH + + +def config_path( + naming_template: str, + component_type: str, + component_id: str, + config_name: str, +) -> str: + """Apply *naming_template* to generate a filesystem path for a configuration. + + Example template: ``"{component_type}/{component_id}/{config_name}"`` + """ + return naming_template.format( + component_type=component_type, + component_id=component_id, + config_name=sanitize_name(config_name), + ) + + +def config_row_path(naming_template: str, row_name: str) -> str: + """Apply *naming_template* to generate a path segment for a config row. + + Example template: ``"rows/{config_row_name}"`` + """ + return naming_template.format( + config_row_name=sanitize_name(row_name), + ) + + +def sanitize_name(name: str) -> str: + """Sanitize *name* for use in filesystem paths. + + Rules: + - Lowercase the string + - Replace spaces and non-alphanumeric characters (except hyphens) + with hyphens + - Collapse consecutive hyphens + - Strip leading and trailing hyphens + - Truncate to ``SANITIZE_NAME_MAX_LENGTH`` characters + """ + result = name.lower() + # Replace anything that is not alphanumeric or a hyphen + result = re.sub(r"[^a-z0-9-]", "-", result) + # Collapse multiple hyphens + result = re.sub(r"-{2,}", "-", result) + # Strip leading/trailing hyphens + result = result.strip("-") + # Enforce max length + return result[:SANITIZE_NAME_MAX_LENGTH] diff --git a/src/keboola_agent_cli/sync/secrets.py b/src/keboola_agent_cli/sync/secrets.py new file mode 100644 index 00000000..3f287738 --- /dev/null +++ b/src/keboola_agent_cli/sync/secrets.py @@ -0,0 +1,62 @@ +"""Encrypted value detection for Keboola configurations. + +Keboola stores secrets as encrypted markers (e.g. +``KBC::ProjectSecure::...``). This module provides helpers to detect +such values and locate them inside arbitrarily nested configuration dicts. +""" + +from __future__ import annotations + +from typing import Any + +ENCRYPTED_PREFIXES: tuple[str, ...] = ( + "KBC::ProjectSecure::", + "KBC::ComponentSecure::", + "KBC::ConfigSecure::", + "KBC::ProjectWideSecure::", +) + + +def is_encrypted_value(value: Any) -> bool: + """Return ``True`` if *value* is a Keboola encrypted marker string.""" + if not isinstance(value, str): + return False + return any(value.startswith(prefix) for prefix in ENCRYPTED_PREFIXES) + + +def is_secret_key(key: str) -> bool: + """Return ``True`` if *key* indicates an encrypted field. + + By Keboola convention, encrypted parameter keys start with ``#``. + """ + return key.startswith("#") + + +def find_encrypted_paths(obj: Any, prefix: str = "") -> list[str]: + """Walk *obj* recursively and return dot-separated paths of all encrypted values. + + Both *encrypted marker values* and *secret keys* (starting with ``#``) + are reported. + + Examples:: + + >>> find_encrypted_paths({"#token": "KBC::ProjectSecure::abc"}) + ['#token'] + >>> find_encrypted_paths({"a": {"#key": "val"}}) + ['a.#key'] + """ + paths: list[str] = [] + + if isinstance(obj, dict): + for key, value in obj.items(): + current = f"{prefix}.{key}" if prefix else key + if is_secret_key(key) or is_encrypted_value(value): + paths.append(current) + else: + paths.extend(find_encrypted_paths(value, prefix=current)) + elif isinstance(obj, list): + for idx, item in enumerate(obj): + current = f"{prefix}[{idx}]" + paths.extend(find_encrypted_paths(item, prefix=current)) + + return paths diff --git a/tests/test_sync_branch_mapping.py b/tests/test_sync_branch_mapping.py new file mode 100644 index 00000000..fb494734 --- /dev/null +++ b/tests/test_sync_branch_mapping.py @@ -0,0 +1,203 @@ +"""Tests for BranchMapping model and I/O (branch_mapping.py). + +Covers the BranchMappingEntry and BranchMapping classes, as well +as load/save filesystem round-trips. +""" + +import json +from pathlib import Path + +import pytest + +from keboola_agent_cli.constants import BRANCH_MAPPING_FILENAME, KEBOOLA_DIR_NAME +from keboola_agent_cli.sync.branch_mapping import ( + BranchMapping, + BranchMappingEntry, + load_branch_mapping, + save_branch_mapping, +) + + +class TestBranchMappingEntry: + """Tests for BranchMappingEntry.""" + + def test_branch_mapping_entry_production(self) -> None: + """None keboola_id indicates production branch.""" + entry = BranchMappingEntry(keboola_id=None, name="Main") + assert entry.is_production() is True + assert entry.keboola_id is None + assert entry.name == "Main" + + def test_branch_mapping_entry_dev_branch(self) -> None: + """Non-None keboola_id indicates development branch.""" + entry = BranchMappingEntry(keboola_id="972851", name="feature/auth") + assert entry.is_production() is False + assert entry.keboola_id == "972851" + assert entry.name == "feature/auth" + + def test_branch_mapping_entry_to_dict(self) -> None: + """to_dict returns the correct JSON-ready structure.""" + entry = BranchMappingEntry(keboola_id="12345", name="my-branch") + assert entry.to_dict() == {"id": "12345", "name": "my-branch"} + + def test_branch_mapping_entry_production_to_dict(self) -> None: + """Production entry serializes id as None.""" + entry = BranchMappingEntry(keboola_id=None, name="Main") + assert entry.to_dict() == {"id": None, "name": "Main"} + + +class TestBranchMapping: + """Tests for BranchMapping.""" + + def test_branch_mapping_set_get(self) -> None: + """set and get work correctly.""" + mapping = BranchMapping() + mapping.set("main", None, "Main") + mapping.set("feature/auth", "972851", "feature/auth") + + main_entry = mapping.get("main") + assert main_entry is not None + assert main_entry.is_production() is True + assert main_entry.name == "Main" + + feature_entry = mapping.get("feature/auth") + assert feature_entry is not None + assert feature_entry.keboola_id == "972851" + assert feature_entry.name == "feature/auth" + + def test_branch_mapping_get_nonexistent(self) -> None: + """get returns None for nonexistent branch.""" + mapping = BranchMapping() + assert mapping.get("nonexistent") is None + + def test_branch_mapping_remove(self) -> None: + """remove deletes an existing mapping and returns True.""" + mapping = BranchMapping() + mapping.set("feature/auth", "972851", "feature/auth") + + assert mapping.remove("feature/auth") is True + assert mapping.get("feature/auth") is None + + def test_branch_mapping_remove_nonexistent(self) -> None: + """remove returns False for nonexistent branch.""" + mapping = BranchMapping() + assert mapping.remove("nonexistent") is False + + def test_branch_mapping_round_trip(self) -> None: + """to_dict/from_dict round-trip preserves data.""" + mapping = BranchMapping() + mapping.set("main", None, "Main") + mapping.set("feature/auth", "972851", "feature/auth") + mapping.set("bugfix/123", "88888", "bugfix/123") + + data = mapping.to_dict() + restored = BranchMapping.from_dict(data) + + assert restored.version == 1 + assert len(restored.mappings) == 3 + + main_entry = restored.get("main") + assert main_entry is not None + assert main_entry.is_production() is True + assert main_entry.name == "Main" + + auth_entry = restored.get("feature/auth") + assert auth_entry is not None + assert auth_entry.keboola_id == "972851" + assert auth_entry.name == "feature/auth" + + bugfix_entry = restored.get("bugfix/123") + assert bugfix_entry is not None + assert bugfix_entry.keboola_id == "88888" + + def test_branch_mapping_to_dict_format(self) -> None: + """to_dict produces the Go CLI compatible format.""" + mapping = BranchMapping() + mapping.set("main", None, "Main") + mapping.set("feature/auth", "972851", "feature/auth") + + data = mapping.to_dict() + assert data["version"] == 1 + assert data["mappings"]["main"] == {"id": None, "name": "Main"} + assert data["mappings"]["feature/auth"] == {"id": "972851", "name": "feature/auth"} + + def test_branch_mapping_from_dict_empty(self) -> None: + """from_dict handles empty mappings.""" + data = {"version": 1, "mappings": {}} + mapping = BranchMapping.from_dict(data) + assert mapping.version == 1 + assert len(mapping.mappings) == 0 + + def test_branch_mapping_from_dict_defaults(self) -> None: + """from_dict uses defaults when fields are missing.""" + data = {} + mapping = BranchMapping.from_dict(data) + assert mapping.version == 1 + assert len(mapping.mappings) == 0 + + +class TestBranchMappingIO: + """Tests for load/save filesystem operations.""" + + def test_load_save_branch_mapping(self, tmp_path: Path) -> None: + """Filesystem round-trip: save then load preserves data.""" + mapping = BranchMapping() + mapping.set("main", None, "Main") + mapping.set("feature/auth", "972851", "feature/auth") + + save_branch_mapping(tmp_path, mapping) + + # Verify file exists + path = tmp_path / KEBOOLA_DIR_NAME / BRANCH_MAPPING_FILENAME + assert path.exists() + + # Verify raw JSON content + raw = json.loads(path.read_text(encoding="utf-8")) + assert raw["version"] == 1 + assert raw["mappings"]["main"]["id"] is None + assert raw["mappings"]["feature/auth"]["id"] == "972851" + + # Load and verify + loaded = load_branch_mapping(tmp_path) + assert loaded.version == 1 + assert len(loaded.mappings) == 2 + + main_entry = loaded.get("main") + assert main_entry is not None + assert main_entry.is_production() is True + + auth_entry = loaded.get("feature/auth") + assert auth_entry is not None + assert auth_entry.keboola_id == "972851" + + def test_load_branch_mapping_not_found(self, tmp_path: Path) -> None: + """load_branch_mapping raises FileNotFoundError when file is missing.""" + with pytest.raises(FileNotFoundError, match="Branch mapping not found"): + load_branch_mapping(tmp_path) + + def test_save_creates_keboola_dir(self, tmp_path: Path) -> None: + """save_branch_mapping creates .keboola/ directory if it doesn't exist.""" + mapping = BranchMapping() + mapping.set("main", None, "Main") + + keboola_dir = tmp_path / KEBOOLA_DIR_NAME + assert not keboola_dir.exists() + + save_branch_mapping(tmp_path, mapping) + + assert keboola_dir.exists() + assert (keboola_dir / BRANCH_MAPPING_FILENAME).exists() + + def test_save_overwrites_existing(self, tmp_path: Path) -> None: + """save_branch_mapping overwrites existing file.""" + mapping1 = BranchMapping() + mapping1.set("main", None, "Main") + save_branch_mapping(tmp_path, mapping1) + + mapping2 = BranchMapping() + mapping2.set("main", None, "Main") + mapping2.set("develop", "99999", "develop") + save_branch_mapping(tmp_path, mapping2) + + loaded = load_branch_mapping(tmp_path) + assert len(loaded.mappings) == 2 diff --git a/tests/test_sync_cli.py b/tests/test_sync_cli.py new file mode 100644 index 00000000..2c690eed --- /dev/null +++ b/tests/test_sync_cli.py @@ -0,0 +1,1336 @@ +"""Tests for sync CLI commands via CliRunner. + +Tests init, pull, and status subcommands. Follows the existing CLI test +pattern from test_cli.py and test_workspace_cli.py with patched services +in ctx.obj. +""" + +import json +import re +from pathlib import Path +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.project_service import ProjectService + + +def _strip_ansi(text: str) -> str: + """Remove ANSI escape sequences from text for assertion matching.""" + return re.sub(r"\x1b\[[0-9;]*m", "", text) + + +TEST_TOKEN = "901-10493007-VDtlEDWDF6Tx5V8jjE8FshFlqM0Hl0c08KHqpt0k" + +runner = CliRunner() + + +def _setup_config(config_dir: Path, projects: dict[str, dict] | None = None) -> ConfigStore: + """Set up a ConfigStore with given projects for CLI sync tests.""" + store = ConfigStore(config_dir=config_dir) + if projects: + for alias, info in projects.items(): + store.add_project( + alias, + ProjectConfig( + stack_url=info.get("stack_url", "https://connection.keboola.com"), + token=info["token"], + project_name=info.get("project_name", alias), + project_id=info.get("project_id", 1234), + ), + ) + return store + + +def _make_sync_service_mock() -> MagicMock: + """Create a fresh MagicMock for SyncService.""" + return MagicMock() + + +# =================================================================== +# Help text tests +# =================================================================== + + +class TestSyncHelp: + """Tests for sync subcommand help output.""" + + def test_sync_init_help(self) -> None: + """sync init --help shows usage text.""" + result = runner.invoke(app, ["sync", "init", "--help"]) + assert result.exit_code == 0 + output = _strip_ansi(result.output) + assert "Initialize" in output or "init" in output + assert "--project" in output + assert "--directory" in output + assert "--git-branching" in output + + def test_sync_pull_help(self) -> None: + """sync pull --help shows usage text.""" + result = runner.invoke(app, ["sync", "pull", "--help"]) + assert result.exit_code == 0 + output = _strip_ansi(result.output) + assert "Download" in output or "pull" in output + assert "--project" in output + assert "--directory" in output + assert "--force" in output + + def test_sync_status_help(self) -> None: + """sync status --help shows usage text.""" + result = runner.invoke(app, ["sync", "status", "--help"]) + assert result.exit_code == 0 + output = _strip_ansi(result.output) + assert "status" in output.lower() + assert "--directory" in output + + +# =================================================================== +# sync init CLI tests +# =================================================================== + + +class TestSyncInitCli: + """Tests for `kbagent sync init` command.""" + + def test_sync_init_json_output(self, tmp_path: Path) -> None: + """sync init --json returns structured JSON with init result.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN, "project_id": 258}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.init_sync.return_value = { + "status": "initialized", + "project_id": 258, + "project_alias": "prod", + "api_host": "connection.keboola.com", + "git_branching": False, + "default_branch": "main", + "files_created": ["/tmp/project/.keboola/manifest.json"], + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "init", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"]["status"] == "initialized" + assert output["data"]["project_id"] == 258 + assert output["data"]["api_host"] == "connection.keboola.com" + assert output["data"]["git_branching"] is False + + def test_sync_init_human_output(self, tmp_path: Path) -> None: + """sync init in human mode shows success message.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN, "project_id": 258}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.init_sync.return_value = { + "status": "initialized", + "project_id": 258, + "project_alias": "prod", + "api_host": "connection.keboola.com", + "git_branching": False, + "default_branch": "main", + "files_created": ["/tmp/project/.keboola/manifest.json"], + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "sync", + "init", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "prod" in result.output + assert "258" in result.output + + def test_sync_init_config_error(self, tmp_path: Path) -> None: + """sync init returns exit code 5 when project alias is not found.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config(config_dir) + + mock_sync = _make_sync_service_mock() + mock_sync.init_sync.side_effect = ConfigError("Project 'missing' not found.") + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "init", + "--project", + "missing", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 5 + + def test_sync_init_already_exists_error(self, tmp_path: Path) -> None: + """sync init returns exit code 1 when manifest already exists.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.init_sync.side_effect = FileExistsError( + "Manifest already exists. Use 'sync pull' to update." + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "init", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 1 + + +# =================================================================== +# sync pull CLI tests +# =================================================================== + + +class TestSyncPullCli: + """Tests for `kbagent sync pull` command.""" + + def test_sync_pull_json_output(self, tmp_path: Path) -> None: + """sync pull --json returns structured JSON with pull stats.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.pull.return_value = { + "status": "pulled", + "project_alias": "prod", + "branch_id": 12345, + "branch_dir": "main", + "configs_pulled": 5, + "rows_pulled": 3, + "files_written": 8, + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "pull", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"]["configs_pulled"] == 5 + assert output["data"]["rows_pulled"] == 3 + assert output["data"]["files_written"] == 8 + + def test_sync_pull_human_output(self, tmp_path: Path) -> None: + """sync pull in human mode shows pulled summary.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.pull.return_value = { + "status": "pulled", + "project_alias": "prod", + "branch_id": 12345, + "branch_dir": "main", + "configs_pulled": 3, + "rows_pulled": 1, + "files_written": 4, + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "sync", + "pull", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "3" in result.output # configs_pulled + assert "1" in result.output # rows_pulled + assert "main" in result.output # branch_dir + + def test_sync_pull_not_initialized_error(self, tmp_path: Path) -> None: + """sync pull returns exit code 1 when project not initialized.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.pull.side_effect = FileNotFoundError( + "Manifest not found. Is this a Keboola project directory?" + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "pull", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 1 + + def test_sync_pull_api_error(self, tmp_path: Path) -> None: + """sync pull returns appropriate exit code on API error.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.pull.side_effect = KeboolaApiError( + message="Invalid token", + status_code=401, + error_code="INVALID_TOKEN", + retryable=False, + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "pull", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 3 # auth error + + +# =================================================================== +# sync status CLI tests +# =================================================================== + + +class TestSyncStatusCli: + """Tests for `kbagent sync status` command.""" + + def test_sync_status_no_changes(self, tmp_path: Path) -> None: + """sync status shows 'No changes detected' when nothing is modified.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.status.return_value = { + "modified": [], + "added": [], + "deleted": [], + "unchanged": 5, + "total_tracked": 5, + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "sync", + "status", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "No changes detected" in result.output + assert "5" in result.output # number of tracked configs + + def test_sync_status_json_output(self, tmp_path: Path) -> None: + """sync status --json returns structured JSON with change lists.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.status.return_value = { + "modified": [ + { + "component_id": "keboola.ex-http", + "config_id": "cfg-001", + "path": "extractor/keboola.ex-http/my-config", + } + ], + "added": [], + "deleted": [ + { + "component_id": "keboola.snowflake-transformation", + "config_id": "cfg-002", + "path": "transformation/keboola.snowflake-transformation/clean-data", + } + ], + "unchanged": 3, + "total_tracked": 5, + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "status", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + data = output["data"] + assert len(data["modified"]) == 1 + assert len(data["deleted"]) == 1 + assert data["unchanged"] == 3 + assert data["total_tracked"] == 5 + + def test_sync_status_with_changes_human(self, tmp_path: Path) -> None: + """sync status in human mode shows M/A/D prefixed entries.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.status.return_value = { + "modified": [ + { + "component_id": "keboola.ex-http", + "config_id": "cfg-001", + "path": "extractor/keboola.ex-http/my-config", + } + ], + "added": [ + { + "component_id": "keboola.ex-db", + "config_id": "cfg-new", + "path": "extractor/keboola.ex-db/new-config", + } + ], + "deleted": [ + { + "component_id": "keboola.snowflake-transformation", + "config_id": "cfg-002", + "path": "transformation/keboola.snowflake-transformation/clean-data", + } + ], + "unchanged": 2, + "total_tracked": 4, + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "sync", + "status", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + # Human output should show M, A, D prefixes + assert "M " in result.output # Modified + assert "A " in result.output # Added + assert "D " in result.output # Deleted + # Should contain summary line + assert "1 modified" in result.output + assert "1 added" in result.output + assert "1 deleted" in result.output + + def test_sync_status_not_initialized_error(self, tmp_path: Path) -> None: + """sync status returns exit code 1 when project not initialized.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config(config_dir) + + mock_sync = _make_sync_service_mock() + mock_sync.status.side_effect = FileNotFoundError( + "Manifest not found. Is this a Keboola project directory?" + ) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "status", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 1 + + +# =================================================================== +# sync diff CLI tests +# =================================================================== + + +class TestSyncDiffCli: + """Tests for `kbagent sync diff` command.""" + + def test_sync_diff_help(self) -> None: + """sync diff --help shows usage text.""" + result = runner.invoke(app, ["sync", "diff", "--help"]) + assert result.exit_code == 0 + output = _strip_ansi(result.output) + assert "--project" in output + assert "--directory" in output + + def test_sync_diff_json_output(self, tmp_path: Path) -> None: + """sync diff --json returns structured JSON with changes and summary.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.diff.return_value = { + "changes": [ + { + "change_type": "modified", + "component_id": "keboola.ex-http", + "config_id": "cfg-001", + "config_name": "My Config", + "path": "extractor/keboola.ex-http/my-config", + "details": ["parameters.url changed: 'old' -> 'new'"], + }, + { + "change_type": "added", + "component_id": "keboola.wr-snowflake", + "config_id": "", + "config_name": "New Writer", + "path": "writer/keboola.wr-snowflake/new-writer", + "details": [], + }, + ], + "summary": { + "added": 1, + "modified": 1, + "deleted": 0, + "unchanged": 3, + }, + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "diff", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + data = output["data"] + assert len(data["changes"]) == 2 + assert data["summary"]["added"] == 1 + assert data["summary"]["modified"] == 1 + assert data["summary"]["deleted"] == 0 + assert data["summary"]["unchanged"] == 3 + + def test_sync_diff_no_changes_human(self, tmp_path: Path) -> None: + """sync diff in human mode shows 'No differences found' when no changes.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.diff.return_value = { + "changes": [], + "summary": { + "added": 0, + "modified": 0, + "deleted": 0, + "unchanged": 5, + }, + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "sync", + "diff", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "No differences found" in result.output + + +# =================================================================== +# sync push CLI tests +# =================================================================== + + +class TestSyncPushCli: + """Tests for `kbagent sync push` command.""" + + def test_sync_push_help(self) -> None: + """sync push --help shows usage text.""" + result = runner.invoke(app, ["sync", "push", "--help"]) + assert result.exit_code == 0 + output = _strip_ansi(result.output) + assert "--project" in output + assert "--directory" in output + assert "--dry-run" in output + assert "--force" in output + + def test_sync_push_json_output(self, tmp_path: Path) -> None: + """sync push --json returns structured JSON with push results.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.push.return_value = { + "status": "pushed", + "created": 1, + "updated": 2, + "deleted": 0, + "errors": [], + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "push", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + data = output["data"] + assert data["status"] == "pushed" + assert data["created"] == 1 + assert data["updated"] == 2 + assert data["deleted"] == 0 + assert data["errors"] == [] + + def test_sync_push_dry_run_human(self, tmp_path: Path) -> None: + """sync push --dry-run in human mode shows dry run output.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.push.return_value = { + "status": "dry_run", + "changes": [ + { + "change_type": "modified", + "component_id": "keboola.ex-http", + "config_id": "cfg-001", + "config_name": "My Config", + "path": "extractor/keboola.ex-http/my-config", + "details": [], + }, + ], + "summary": { + "added": 0, + "modified": 1, + "deleted": 0, + "unchanged": 4, + }, + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "sync", + "push", + "--project", + "prod", + "--dry-run", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "Dry run" in result.output or "dry run" in result.output.lower() + assert "MODIFIED" in result.output + + def test_sync_push_no_changes_human(self, tmp_path: Path) -> None: + """sync push in human mode shows 'No changes to push' when nothing changed.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.push.return_value = { + "status": "no_changes", + "created": 0, + "updated": 0, + "deleted": 0, + "errors": [], + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "sync", + "push", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "No changes to push" in result.output + + +# =================================================================== +# sync branch-link / branch-unlink / branch-status CLI tests +# =================================================================== + + +class TestSyncBranchLinkCli: + """Tests for `kbagent sync branch-link` command.""" + + def test_sync_branch_link_help(self) -> None: + """sync branch-link --help shows usage text.""" + result = runner.invoke(app, ["sync", "branch-link", "--help"]) + assert result.exit_code == 0 + output = _strip_ansi(result.output) + assert "--project" in output + assert "--directory" in output + assert "--branch-id" in output + assert "--branch-name" in output + + def test_sync_branch_link_json_output(self, tmp_path: Path) -> None: + """sync branch-link --json returns structured JSON.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.branch_link.return_value = { + "status": "linked", + "git_branch": "feature/auth", + "keboola_branch_id": "99999", + "keboola_branch_name": "feature/auth", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "branch-link", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"]["status"] == "linked" + assert output["data"]["git_branch"] == "feature/auth" + assert output["data"]["keboola_branch_id"] == "99999" + + def test_sync_branch_link_config_error(self, tmp_path: Path) -> None: + """sync branch-link returns exit code 5 on ConfigError.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.branch_link.side_effect = ConfigError("Git-branching mode is not enabled.") + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "branch-link", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 5 + + def test_sync_branch_link_already_linked_human(self, tmp_path: Path) -> None: + """sync branch-link in human mode shows 'Already linked' for existing mapping.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.branch_link.return_value = { + "status": "already_linked", + "git_branch": "feature/auth", + "keboola_branch_id": "99999", + "keboola_branch_name": "feature/auth", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "sync", + "branch-link", + "--project", + "prod", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "Already linked" in result.output + + +class TestSyncBranchUnlinkCli: + """Tests for `kbagent sync branch-unlink` command.""" + + def test_sync_branch_unlink_help(self) -> None: + """sync branch-unlink --help shows usage text.""" + result = runner.invoke(app, ["sync", "branch-unlink", "--help"]) + assert result.exit_code == 0 + output = _strip_ansi(result.output) + assert "--directory" in output + + def test_sync_branch_unlink_json_output(self, tmp_path: Path) -> None: + """sync branch-unlink --json returns structured JSON.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.branch_unlink.return_value = { + "status": "unlinked", + "git_branch": "feature/auth", + "keboola_branch_id": "99999", + "keboola_branch_name": "feature/auth", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "branch-unlink", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"]["status"] == "unlinked" + + def test_sync_branch_unlink_not_linked_human(self, tmp_path: Path) -> None: + """sync branch-unlink in human mode shows 'not linked' message.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.branch_unlink.return_value = { + "status": "not_linked", + "git_branch": "feature/auth", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "sync", + "branch-unlink", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "not linked" in result.output + + +class TestSyncBranchStatusCli: + """Tests for `kbagent sync branch-status` command.""" + + def test_sync_branch_status_help(self) -> None: + """sync branch-status --help shows usage text.""" + result = runner.invoke(app, ["sync", "branch-status", "--help"]) + assert result.exit_code == 0 + output = _strip_ansi(result.output) + assert "--directory" in output + + def test_sync_branch_status_json_output(self, tmp_path: Path) -> None: + """sync branch-status --json returns structured JSON.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.branch_status.return_value = { + "git_branching": True, + "git_branch": "feature/auth", + "linked": True, + "keboola_branch_id": "99999", + "keboola_branch_name": "feature/auth", + "is_production": False, + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "branch-status", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + output = json.loads(result.output) + assert output["status"] == "ok" + assert output["data"]["linked"] is True + assert output["data"]["keboola_branch_id"] == "99999" + + def test_sync_branch_status_not_linked_human(self, tmp_path: Path) -> None: + """sync branch-status in human mode shows 'Not linked' and hint.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.branch_status.return_value = { + "git_branching": True, + "git_branch": "feature/auth", + "linked": False, + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "sync", + "branch-status", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "Not linked" in result.output + assert "branch-link" in result.output + + def test_sync_branch_status_disabled_human(self, tmp_path: Path) -> None: + """sync branch-status shows 'not enabled' when git branching is off.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config( + config_dir, + {"prod": {"token": TEST_TOKEN}}, + ) + + mock_sync = _make_sync_service_mock() + mock_sync.branch_status.return_value = {"git_branching": False} + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "sync", + "branch-status", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "not enabled" in result.output diff --git a/tests/test_sync_code_extraction.py b/tests/test_sync_code_extraction.py new file mode 100644 index 00000000..e65abe95 --- /dev/null +++ b/tests/test_sync_code_extraction.py @@ -0,0 +1,495 @@ +"""Tests for sync code_extraction module -- SQL/Python code extraction and merging.""" + +import copy +from pathlib import Path + +import pytest + +from keboola_agent_cli.sync.code_extraction import ( + extract_code_files, + merge_code_files, +) + +# --------------------------------------------------------------------------- +# Sample data fixtures +# --------------------------------------------------------------------------- + +SAMPLE_SQL_CONFIG = { + "version": 2, + "name": "Clean Data", + "description": "Cleans raw data", + "parameters": { + "blocks": [ + { + "name": "Preparation", + "codes": [ + { + "name": "Create staging", + "script": ["CREATE TABLE staging AS SELECT * FROM raw;"], + }, + { + "name": "Clean nulls", + "script": ["DELETE FROM staging WHERE id IS NULL;"], + }, + ], + }, + { + "name": "Output", + "codes": [ + { + "name": "Final select", + "script": ["SELECT * FROM staging;"], + }, + ], + }, + ], + }, + "_keboola": { + "component_id": "keboola.snowflake-transformation", + "config_id": "cfg-100", + }, +} + +SAMPLE_PYTHON_TRANSFORM_CONFIG = { + "version": 2, + "name": "Python Analysis", + "description": "Runs Python analysis", + "parameters": { + "blocks": [ + { + "name": "Analysis", + "codes": [ + { + "name": "Load data", + "script": ["import pandas as pd\ndf = pd.read_csv('in/tables/data.csv')"], + }, + { + "name": "Transform", + "script": ["df['total'] = df['price'] * df['qty']"], + }, + ], + }, + ], + "packages": ["pandas==2.1.0", "numpy>=1.24"], + }, + "_keboola": { + "component_id": "keboola.python-transformation-v2", + "config_id": "cfg-200", + }, +} + +SAMPLE_PYTHON_APP_CONFIG = { + "version": 2, + "name": "Custom Script", + "description": "Custom Python app", + "parameters": { + "code": "import json\nresult = {'status': 'ok'}\nprint(json.dumps(result))\n", + "packages": ["requests>=2.31", "beautifulsoup4"], + }, + "_keboola": { + "component_id": "kds-team.app-custom-python", + "config_id": "cfg-300", + }, +} + + +# =================================================================== +# SQL Transformation Tests +# =================================================================== + + +class TestSqlExtraction: + """Tests for SQL transformation code extraction and merging.""" + + def test_extract_sql_blocks(self, tmp_path: Path) -> None: + """Config with blocks produces transform.sql with markers and removes blocks from params.""" + config_data = copy.deepcopy(SAMPLE_SQL_CONFIG) + config_dir = tmp_path / "sql-config" + + result = extract_code_files("keboola.snowflake-transformation", config_data, config_dir) + + # transform.sql should exist + sql_file = config_dir / "transform.sql" + assert sql_file.exists() + + content = sql_file.read_text(encoding="utf-8") + assert "/* ===== BLOCK: Preparation ===== */" in content + assert "/* ===== CODE: Create staging ===== */" in content + assert "CREATE TABLE staging AS SELECT * FROM raw;" in content + assert "/* ===== CODE: Clean nulls ===== */" in content + assert "DELETE FROM staging WHERE id IS NULL;" in content + assert "/* ===== BLOCK: Output ===== */" in content + assert "/* ===== CODE: Final select ===== */" in content + assert "SELECT * FROM staging;" in content + + # Blocks should be removed from parameters + assert "blocks" not in result["parameters"] + + def test_merge_sql_blocks(self, tmp_path: Path) -> None: + """transform.sql with markers is parsed back into blocks structure.""" + config_dir = tmp_path / "sql-config" + config_dir.mkdir(parents=True) + + sql_content = ( + "/* ===== BLOCK: Preparation ===== */\n" + "\n" + "/* ===== CODE: Create staging ===== */\n" + "CREATE TABLE staging AS SELECT * FROM raw;\n" + "\n" + "/* ===== BLOCK: Output ===== */\n" + "\n" + "/* ===== CODE: Final select ===== */\n" + "SELECT * FROM staging;\n" + ) + (config_dir / "transform.sql").write_text(sql_content, encoding="utf-8") + + config_data: dict = {"parameters": {}} + result = merge_code_files("keboola.snowflake-transformation", config_data, config_dir) + + blocks = result["parameters"]["blocks"] + assert len(blocks) == 2 + assert blocks[0]["name"] == "Preparation" + assert len(blocks[0]["codes"]) == 1 + assert blocks[0]["codes"][0]["name"] == "Create staging" + assert "CREATE TABLE staging" in blocks[0]["codes"][0]["script"][0] + + assert blocks[1]["name"] == "Output" + assert blocks[1]["codes"][0]["name"] == "Final select" + assert "SELECT * FROM staging" in blocks[1]["codes"][0]["script"][0] + + def test_sql_round_trip(self, tmp_path: Path) -> None: + """Extract then merge produces equivalent blocks structure.""" + config_data = copy.deepcopy(SAMPLE_SQL_CONFIG) + original_blocks = copy.deepcopy(config_data["parameters"]["blocks"]) + config_dir = tmp_path / "sql-roundtrip" + + # Extract: writes transform.sql, removes blocks + extract_code_files("keboola.snowflake-transformation", config_data, config_dir) + assert "blocks" not in config_data["parameters"] + + # Merge: reads transform.sql, restores blocks + merge_code_files("keboola.snowflake-transformation", config_data, config_dir) + restored_blocks = config_data["parameters"]["blocks"] + + # Compare block/code names and script content + assert len(restored_blocks) == len(original_blocks) + for orig_block, rest_block in zip(original_blocks, restored_blocks, strict=True): + assert orig_block["name"] == rest_block["name"] + assert len(orig_block["codes"]) == len(rest_block["codes"]) + for orig_code, rest_code in zip(orig_block["codes"], rest_block["codes"], strict=True): + assert orig_code["name"] == rest_code["name"] + # Script content should match (whitespace-stripped) + orig_script = orig_code["script"][0].strip() + rest_script = rest_code["script"][0].strip() + assert orig_script == rest_script + + def test_no_blocks_no_file(self, tmp_path: Path) -> None: + """Config without blocks produces no transform.sql file.""" + config_data = { + "version": 2, + "name": "Empty Transform", + "parameters": {}, + "_keboola": { + "component_id": "keboola.snowflake-transformation", + "config_id": "cfg-empty", + }, + } + config_dir = tmp_path / "sql-empty" + + extract_code_files("keboola.snowflake-transformation", config_data, config_dir) + + assert not (config_dir / "transform.sql").exists() + + def test_sql_merge_without_file(self, tmp_path: Path) -> None: + """Merging when transform.sql does not exist leaves config unchanged.""" + config_dir = tmp_path / "sql-nofile" + config_dir.mkdir(parents=True) + + config_data: dict = {"parameters": {"other_key": "value"}} + result = merge_code_files("keboola.snowflake-transformation", config_data, config_dir) + + assert "blocks" not in result["parameters"] + assert result["parameters"]["other_key"] == "value" + + def test_sql_merge_no_markers(self, tmp_path: Path) -> None: + """Plain SQL without markers is treated as a single block/code.""" + config_dir = tmp_path / "sql-plain" + config_dir.mkdir(parents=True) + + (config_dir / "transform.sql").write_text("SELECT 1;\nSELECT 2;\n", encoding="utf-8") + + config_data: dict = {"parameters": {}} + result = merge_code_files("keboola.snowflake-transformation", config_data, config_dir) + + blocks = result["parameters"]["blocks"] + assert len(blocks) == 1 + assert blocks[0]["name"] == "Block 1" + assert blocks[0]["codes"][0]["name"] == "Code 1" + assert "SELECT 1;" in blocks[0]["codes"][0]["script"][0] + + +# =================================================================== +# Python Transformation Tests +# =================================================================== + + +class TestPythonTransformExtraction: + """Tests for Python transformation code extraction and merging.""" + + def test_extract_python_blocks(self, tmp_path: Path) -> None: + """Blocks are extracted to transform.py with Python markers.""" + config_data = copy.deepcopy(SAMPLE_PYTHON_TRANSFORM_CONFIG) + config_dir = tmp_path / "py-transform" + + result = extract_code_files("keboola.python-transformation-v2", config_data, config_dir) + + py_file = config_dir / "transform.py" + assert py_file.exists() + + content = py_file.read_text(encoding="utf-8") + assert "# ===== BLOCK: Analysis =====" in content + assert "# ===== CODE: Load data =====" in content + assert "import pandas as pd" in content + assert "# ===== CODE: Transform =====" in content + assert "df['total'] = df['price'] * df['qty']" in content + + # Blocks should be removed from parameters + assert "blocks" not in result["parameters"] + + def test_extract_python_packages(self, tmp_path: Path) -> None: + """Packages are extracted to pyproject.toml.""" + config_data = copy.deepcopy(SAMPLE_PYTHON_TRANSFORM_CONFIG) + config_dir = tmp_path / "py-packages" + + result = extract_code_files("keboola.python-transformation-v2", config_data, config_dir) + + toml_file = config_dir / "pyproject.toml" + assert toml_file.exists() + + content = toml_file.read_text(encoding="utf-8") + assert '"pandas==2.1.0"' in content + assert '"numpy>=1.24"' in content + assert 'name = "python-analysis"' in content + + # Packages should be removed from parameters + assert "packages" not in result["parameters"] + + def test_merge_python_blocks(self, tmp_path: Path) -> None: + """transform.py with markers is parsed back into blocks.""" + config_dir = tmp_path / "py-merge" + config_dir.mkdir(parents=True) + + py_content = ( + "# ===== BLOCK: Analysis =====\n" + "\n" + "# ===== CODE: Load data =====\n" + "import pandas as pd\n" + "df = pd.read_csv('data.csv')\n" + "\n" + ) + (config_dir / "transform.py").write_text(py_content, encoding="utf-8") + + config_data: dict = {"parameters": {}} + result = merge_code_files("keboola.python-transformation-v2", config_data, config_dir) + + blocks = result["parameters"]["blocks"] + assert len(blocks) == 1 + assert blocks[0]["name"] == "Analysis" + assert blocks[0]["codes"][0]["name"] == "Load data" + assert "import pandas as pd" in blocks[0]["codes"][0]["script"][0] + + def test_merge_python_packages(self, tmp_path: Path) -> None: + """pyproject.toml dependencies are merged back into packages list.""" + config_dir = tmp_path / "py-merge-pkg" + config_dir.mkdir(parents=True) + + toml_content = ( + "[project]\n" + 'name = "my-transform"\n' + 'requires-python = ">=3.11"\n' + "dependencies = [\n" + ' "pandas==2.1.0",\n' + ' "numpy>=1.24",\n' + "]\n" + ) + (config_dir / "pyproject.toml").write_text(toml_content, encoding="utf-8") + + config_data: dict = {"parameters": {}} + result = merge_code_files("keboola.python-transformation-v2", config_data, config_dir) + + assert result["parameters"]["packages"] == ["pandas==2.1.0", "numpy>=1.24"] + + def test_python_transform_round_trip(self, tmp_path: Path) -> None: + """Extract then merge produces equivalent blocks and packages.""" + config_data = copy.deepcopy(SAMPLE_PYTHON_TRANSFORM_CONFIG) + original_blocks = copy.deepcopy(config_data["parameters"]["blocks"]) + original_packages = copy.deepcopy(config_data["parameters"]["packages"]) + config_dir = tmp_path / "py-roundtrip" + + extract_code_files("keboola.python-transformation-v2", config_data, config_dir) + merge_code_files("keboola.python-transformation-v2", config_data, config_dir) + + # Verify packages round-trip + assert config_data["parameters"]["packages"] == original_packages + + # Verify block/code names and content + restored_blocks = config_data["parameters"]["blocks"] + assert len(restored_blocks) == len(original_blocks) + for orig_block, rest_block in zip(original_blocks, restored_blocks, strict=True): + assert orig_block["name"] == rest_block["name"] + for orig_code, rest_code in zip(orig_block["codes"], rest_block["codes"], strict=True): + assert orig_code["name"] == rest_code["name"] + assert orig_code["script"][0].strip() == rest_code["script"][0].strip() + + +# =================================================================== +# Python App Tests +# =================================================================== + + +class TestPythonAppExtraction: + """Tests for Python custom app code extraction and merging.""" + + def test_extract_app_code(self, tmp_path: Path) -> None: + """parameters.code is extracted to code.py.""" + config_data = copy.deepcopy(SAMPLE_PYTHON_APP_CONFIG) + config_dir = tmp_path / "app-code" + + result = extract_code_files("kds-team.app-custom-python", config_data, config_dir) + + code_file = config_dir / "code.py" + assert code_file.exists() + + content = code_file.read_text(encoding="utf-8") + assert "import json" in content + assert "result = {'status': 'ok'}" in content + + # code should be removed from parameters + assert "code" not in result["parameters"] + + def test_extract_app_packages(self, tmp_path: Path) -> None: + """Packages are extracted to pyproject.toml with keboola metadata.""" + config_data = copy.deepcopy(SAMPLE_PYTHON_APP_CONFIG) + config_dir = tmp_path / "app-packages" + + result = extract_code_files("kds-team.app-custom-python", config_data, config_dir) + + toml_file = config_dir / "pyproject.toml" + assert toml_file.exists() + + content = toml_file.read_text(encoding="utf-8") + assert '"requests>=2.31"' in content + assert '"beautifulsoup4"' in content + assert "[tool.keboola]" in content + assert 'component_id = "kds-team.app-custom-python"' in content + assert 'config_id = "cfg-300"' in content + + # packages should be removed from parameters + assert "packages" not in result["parameters"] + + def test_merge_app_code(self, tmp_path: Path) -> None: + """code.py is merged back into parameters.code.""" + config_dir = tmp_path / "app-merge" + config_dir.mkdir(parents=True) + + code_content = "print('hello world')\n" + (config_dir / "code.py").write_text(code_content, encoding="utf-8") + + config_data: dict = {"parameters": {}} + result = merge_code_files("kds-team.app-custom-python", config_data, config_dir) + + assert result["parameters"]["code"] == code_content + + def test_merge_app_packages(self, tmp_path: Path) -> None: + """pyproject.toml is merged back into parameters.packages.""" + config_dir = tmp_path / "app-merge-pkg" + config_dir.mkdir(parents=True) + + toml_content = ( + "[project]\n" + 'name = "my-app"\n' + 'requires-python = ">=3.11"\n' + "dependencies = [\n" + ' "requests>=2.31",\n' + "]\n" + "\n" + "[tool.keboola]\n" + 'component_id = "kds-team.app-custom-python"\n' + ) + (config_dir / "pyproject.toml").write_text(toml_content, encoding="utf-8") + + config_data: dict = {"parameters": {}} + result = merge_code_files("kds-team.app-custom-python", config_data, config_dir) + + assert result["parameters"]["packages"] == ["requests>=2.31"] + + def test_app_round_trip(self, tmp_path: Path) -> None: + """Extract then merge produces equivalent code and packages.""" + config_data = copy.deepcopy(SAMPLE_PYTHON_APP_CONFIG) + original_code = config_data["parameters"]["code"] + original_packages = copy.deepcopy(config_data["parameters"]["packages"]) + config_dir = tmp_path / "app-roundtrip" + + extract_code_files("kds-team.app-custom-python", config_data, config_dir) + merge_code_files("kds-team.app-custom-python", config_data, config_dir) + + assert config_data["parameters"]["code"] == original_code + assert config_data["parameters"]["packages"] == original_packages + + +# =================================================================== +# Non-extractable Components +# =================================================================== + + +class TestNonExtractableComponent: + """Tests for components that should not trigger code extraction.""" + + def test_non_extractable_component(self, tmp_path: Path) -> None: + """Generic components are returned unchanged with no files created.""" + config_data = { + "version": 2, + "name": "My Extractor", + "parameters": {"key": "value"}, + "_keboola": { + "component_id": "keboola.ex-http", + "config_id": "cfg-generic", + }, + } + config_dir = tmp_path / "generic" + + result = extract_code_files("keboola.ex-http", config_data, config_dir) + + # No files should be created + assert not config_dir.exists() + + # Config should be unchanged + assert result["parameters"]["key"] == "value" + + def test_merge_non_extractable_component(self, tmp_path: Path) -> None: + """Merge on generic component is a no-op.""" + config_dir = tmp_path / "generic-merge" + config_dir.mkdir(parents=True) + + config_data: dict = {"parameters": {"key": "value"}} + result = merge_code_files("keboola.ex-http", config_data, config_dir) + + assert result["parameters"]["key"] == "value" + + @pytest.mark.parametrize( + "component_id", + [ + "keboola.snowflake-transformation", + "keboola.synapse-transformation", + "keboola.oracle-transformation", + "keboola.redshift-sql-transformation", + ], + ) + def test_all_sql_components_recognized(self, component_id: str, tmp_path: Path) -> None: + """All SQL transformation component IDs trigger extraction.""" + config_data = copy.deepcopy(SAMPLE_SQL_CONFIG) + config_dir = tmp_path / component_id.replace(".", "-") + + extract_code_files(component_id, config_data, config_dir) + + assert (config_dir / "transform.sql").exists() diff --git a/tests/test_sync_config_format.py b/tests/test_sync_config_format.py new file mode 100644 index 00000000..6cb47478 --- /dev/null +++ b/tests/test_sync_config_format.py @@ -0,0 +1,208 @@ +"""Tests for sync config_format module -- API JSON <-> local YAML conversion.""" + +import pytest + +from keboola_agent_cli.sync.config_format import ( + api_config_to_local, + api_row_to_local, + classify_component_type, + local_config_to_api, + local_row_to_api, +) + +SAMPLE_API_CONFIG = { + "id": "cfg-123", + "name": "My Extractor", + "description": "Extracts data from API", + "configuration": { + "parameters": { + "api_url": "https://example.com", + "#token": "KBC::ProjectSecure::abc", + }, + "storage": { + "input": { + "tables": [{"source": "in.c-main.users", "destination": "users"}], + }, + "output": { + "tables": [{"source": "result", "destination": "out.c-main.result"}], + }, + }, + "processors": { + "after": [{"definition": {"component": "keboola.processor-move-files"}}], + }, + }, +} + +SAMPLE_COMPONENT_ID = "keboola.ex-http" +SAMPLE_CONFIG_ID = "cfg-123" + + +class TestClassifyComponentType: + """Tests for classify_component_type().""" + + @pytest.mark.parametrize( + "api_type,expected", + [ + ("extractor", "extractor"), + ("writer", "writer"), + ("transformation", "transformation"), + ("application", "application"), + ("other", "other"), + ], + ) + def test_classify_component_type_known(self, api_type: str, expected: str) -> None: + """Known component types map to themselves.""" + assert classify_component_type(api_type) == expected + + @pytest.mark.parametrize("api_type", ["unknown", "custom", "orchestrator", ""]) + def test_classify_component_type_fallback(self, api_type: str) -> None: + """Unknown component types fall back to 'other'.""" + assert classify_component_type(api_type) == "other" + + +class TestApiConfigToLocal: + """Tests for api_config_to_local().""" + + def test_api_config_to_local_basic(self) -> None: + """Converted local config has version=2, name, description, and _keboola block.""" + local = api_config_to_local(SAMPLE_COMPONENT_ID, SAMPLE_API_CONFIG, SAMPLE_CONFIG_ID) + + assert local["version"] == 2 + assert local["name"] == "My Extractor" + assert local["description"] == "Extracts data from API" + assert local["_keboola"] == { + "component_id": SAMPLE_COMPONENT_ID, + "config_id": SAMPLE_CONFIG_ID, + } + + def test_api_config_to_local_parameters(self) -> None: + """Parameters are promoted from configuration.parameters to top level.""" + local = api_config_to_local(SAMPLE_COMPONENT_ID, SAMPLE_API_CONFIG, SAMPLE_CONFIG_ID) + + assert "parameters" in local + assert local["parameters"]["api_url"] == "https://example.com" + assert local["parameters"]["#token"] == "KBC::ProjectSecure::abc" + + def test_api_config_to_local_storage(self) -> None: + """Input and output are promoted from configuration.storage.""" + local = api_config_to_local(SAMPLE_COMPONENT_ID, SAMPLE_API_CONFIG, SAMPLE_CONFIG_ID) + + assert "input" in local + assert local["input"]["tables"][0]["source"] == "in.c-main.users" + + assert "output" in local + assert local["output"]["tables"][0]["destination"] == "out.c-main.result" + + def test_api_config_to_local_processors(self) -> None: + """Processors are promoted from configuration.processors.""" + local = api_config_to_local(SAMPLE_COMPONENT_ID, SAMPLE_API_CONFIG, SAMPLE_CONFIG_ID) + + assert "processors" in local + assert local["processors"]["after"][0]["definition"]["component"] == ( + "keboola.processor-move-files" + ) + + def test_api_config_to_local_extras_preserved(self) -> None: + """Unknown keys in configuration are preserved under _configuration_extra.""" + api_config = { + "id": "cfg-1", + "name": "Test", + "description": "", + "configuration": { + "parameters": {"key": "val"}, + "runtime": {"imageTag": "latest"}, + "authorization": {"oauth_api": {"id": "abc"}}, + }, + } + local = api_config_to_local("comp", api_config, "cfg-1") + + assert "_configuration_extra" in local + assert local["_configuration_extra"]["runtime"] == {"imageTag": "latest"} + assert local["_configuration_extra"]["authorization"] == {"oauth_api": {"id": "abc"}} + # Promoted keys must not appear in extras + assert "parameters" not in local["_configuration_extra"] + + def test_api_config_to_local_no_configuration(self) -> None: + """Config with no configuration block produces minimal local structure.""" + api_config = {"id": "cfg-0", "name": "Empty", "description": ""} + local = api_config_to_local("comp", api_config, "cfg-0") + + assert local["name"] == "Empty" + assert "parameters" not in local + assert "input" not in local + assert "output" not in local + assert "processors" not in local + assert "_configuration_extra" not in local + + +class TestLocalConfigToApiRoundTrip: + """Tests for local_config_to_api() and round-trip conversion.""" + + def test_local_config_to_api_round_trip(self) -> None: + """Convert API->local->API and verify the configuration dict matches.""" + local = api_config_to_local(SAMPLE_COMPONENT_ID, SAMPLE_API_CONFIG, SAMPLE_CONFIG_ID) + name, description, configuration = local_config_to_api(local) + + original_config = SAMPLE_API_CONFIG["configuration"] + + assert name == "My Extractor" + assert description == "Extracts data from API" + assert configuration["parameters"] == original_config["parameters"] + assert configuration["storage"] == original_config["storage"] + assert configuration["processors"] == original_config["processors"] + + def test_local_config_to_api_extras_merged_back(self) -> None: + """Extras from _configuration_extra are merged back into API configuration.""" + local = { + "version": 2, + "name": "Test", + "description": "", + "parameters": {"key": "val"}, + "_configuration_extra": {"runtime": {"imageTag": "latest"}}, + "_keboola": {"component_id": "comp", "config_id": "cfg-1"}, + } + _, _, configuration = local_config_to_api(local) + + assert configuration["runtime"] == {"imageTag": "latest"} + assert configuration["parameters"] == {"key": "val"} + + +class TestRowConversion: + """Tests for api_row_to_local() and local_row_to_api().""" + + @pytest.fixture() + def sample_row(self) -> dict: + return { + "id": "row-42", + "name": "First Row", + "description": "A test row", + "configuration": { + "parameters": {"query": "SELECT 1"}, + "storage": { + "output": {"tables": [{"source": "result", "destination": "out.c-main.data"}]}, + }, + }, + } + + def test_api_row_to_local(self, sample_row: dict) -> None: + """Row conversion includes _keboola.row_id and promotes parameters/storage.""" + local = api_row_to_local(sample_row, "keboola.ex-db-snowflake") + + assert local["version"] == 2 + assert local["name"] == "First Row" + assert local["description"] == "A test row" + assert local["parameters"]["query"] == "SELECT 1" + assert "output" in local + assert local["output"]["tables"][0]["destination"] == "out.c-main.data" + assert local["_keboola"]["component_id"] == "keboola.ex-db-snowflake" + assert local["_keboola"]["row_id"] == "row-42" + + def test_local_row_to_api(self, sample_row: dict) -> None: + """Row reverse conversion produces (name, description, configuration) tuple.""" + local = api_row_to_local(sample_row, "keboola.ex-db-snowflake") + name, description, configuration = local_row_to_api(local) + + assert name == "First Row" + assert description == "A test row" + assert configuration["parameters"]["query"] == "SELECT 1" + assert configuration["storage"]["output"]["tables"][0]["destination"] == "out.c-main.data" diff --git a/tests/test_sync_diff_engine.py b/tests/test_sync_diff_engine.py new file mode 100644 index 00000000..413ad8dc --- /dev/null +++ b/tests/test_sync_diff_engine.py @@ -0,0 +1,473 @@ +"""Tests for the sync diff engine module. + +Tests normalize_for_comparison, config_hash, deep_diff, and compute_changeset +functions from keboola_agent_cli.sync.diff_engine. +""" + +from __future__ import annotations + +import copy +from typing import Any + +from keboola_agent_cli.constants import DIFF_MAX_LINES, ENCRYPTED_PLACEHOLDER +from keboola_agent_cli.sync.diff_engine import ( + ConfigChange, + compute_changeset, + config_hash, + deep_diff, + normalize_for_comparison, +) + +# =================================================================== +# TestNormalizeForComparison +# =================================================================== + + +class TestNormalizeForComparison: + """Tests for normalize_for_comparison().""" + + def test_strips_keboola_key(self) -> None: + """_keboola metadata block is removed from output.""" + data = { + "name": "My Config", + "_keboola": {"component_id": "keboola.ex-http", "config_id": "123"}, + "parameters": {"url": "https://example.com"}, + } + result = normalize_for_comparison(data) + assert "_keboola" not in result + assert "name" in result + assert "parameters" in result + + def test_strips_version_key(self) -> None: + """version key is removed from output.""" + data = { + "version": 2, + "name": "My Config", + "parameters": {"key": "value"}, + } + result = normalize_for_comparison(data) + assert "version" not in result + assert "name" in result + + def test_strips_configuration_extra(self) -> None: + """_configuration_extra key is removed from output.""" + data = { + "name": "My Config", + "_configuration_extra": {"runtime": {"backend": "snowflake"}}, + "parameters": {"sql": "SELECT 1"}, + } + result = normalize_for_comparison(data) + assert "_configuration_extra" not in result + assert "name" in result + + def test_replaces_encrypted_values(self) -> None: + """KBC::ProjectSecure::abc is replaced with ENCRYPTED_PLACEHOLDER.""" + data = { + "parameters": { + "#token": "KBC::ProjectSecure::abc123nonce456", + "url": "https://api.example.com", + } + } + result = normalize_for_comparison(data) + assert result["parameters"]["#token"] == ENCRYPTED_PLACEHOLDER + assert result["parameters"]["url"] == "https://api.example.com" + + def test_does_not_mutate_original(self) -> None: + """Original dict is unchanged after normalization.""" + data = { + "version": 2, + "_keboola": {"component_id": "test"}, + "parameters": { + "#secret": "KBC::ProjectSecure::xyz", + "url": "https://example.com", + }, + } + original = copy.deepcopy(data) + normalize_for_comparison(data) + + assert data == original + assert data["version"] == 2 + assert "_keboola" in data + assert data["parameters"]["#secret"] == "KBC::ProjectSecure::xyz" + + def test_nested_encryption(self) -> None: + """Encrypted values deep in nested dicts/lists are replaced.""" + data = { + "parameters": { + "connections": [ + { + "host": "db.example.com", + "#password": "KBC::ComponentSecure::pass123", + "nested": { + "#api_key": "KBC::ConfigSecure::key456", + }, + } + ] + } + } + result = normalize_for_comparison(data) + + connection = result["parameters"]["connections"][0] + assert connection["#password"] == ENCRYPTED_PLACEHOLDER + assert connection["nested"]["#api_key"] == ENCRYPTED_PLACEHOLDER + assert connection["host"] == "db.example.com" + + +# =================================================================== +# TestConfigHash +# =================================================================== + + +class TestConfigHash: + """Tests for config_hash().""" + + def test_same_content_same_hash(self) -> None: + """Identical configs produce the same hash.""" + config = { + "name": "My Config", + "parameters": {"url": "https://api.example.com"}, + } + h1 = config_hash(config) + h2 = config_hash(config) + assert h1 == h2 + + def test_different_content_different_hash(self) -> None: + """Changed parameters produce a different hash.""" + config1 = { + "name": "My Config", + "parameters": {"url": "https://api.example.com"}, + } + config2 = { + "name": "My Config", + "parameters": {"url": "https://api.changed.com"}, + } + assert config_hash(config1) != config_hash(config2) + + def test_encryption_nonces_ignored(self) -> None: + """Two configs with different encrypted markers for same key produce SAME hash.""" + config1 = { + "parameters": { + "#token": "KBC::ProjectSecure::nonce_aaa_111", + "url": "https://example.com", + } + } + config2 = { + "parameters": { + "#token": "KBC::ProjectSecure::nonce_bbb_222", + "url": "https://example.com", + } + } + assert config_hash(config1) == config_hash(config2) + + def test_ordering_independent(self) -> None: + """Dict key order does not affect the hash.""" + config1 = { + "name": "Config", + "parameters": {"a": 1, "b": 2}, + "description": "test", + } + config2 = { + "description": "test", + "parameters": {"b": 2, "a": 1}, + "name": "Config", + } + assert config_hash(config1) == config_hash(config2) + + +# =================================================================== +# TestDeepDiff +# =================================================================== + + +class TestDeepDiff: + """Tests for deep_diff().""" + + def test_changed_scalar(self) -> None: + """Parameter value changed shows old -> new.""" + local = {"parameters": {"url": "https://new.example.com"}} + remote = {"parameters": {"url": "https://old.example.com"}} + + result = deep_diff(local, remote) + + assert len(result) == 1 + assert "parameters.url changed:" in result[0] + assert "'https://old.example.com'" in result[0] + assert "'https://new.example.com'" in result[0] + + def test_added_key(self) -> None: + """Key in local but not remote shows as added.""" + local = {"parameters": {"url": "https://example.com", "timeout": 30}} + remote = {"parameters": {"url": "https://example.com"}} + + result = deep_diff(local, remote) + + assert len(result) == 1 + assert "parameters.timeout added" in result[0] + + def test_removed_key(self) -> None: + """Key in remote but not local shows as removed.""" + local = {"parameters": {"url": "https://example.com"}} + remote = {"parameters": {"url": "https://example.com", "timeout": 30}} + + result = deep_diff(local, remote) + + assert len(result) == 1 + assert "parameters.timeout removed" in result[0] + + def test_encrypted_value_masked(self) -> None: + """Shows 'changed (encrypted)' not actual values.""" + local = {"parameters": {"#token": "KBC::ProjectSecure::new_nonce"}} + remote = {"parameters": {"#token": "KBC::ProjectSecure::old_nonce"}} + + # Both values normalize to ENCRYPTED_PLACEHOLDER, so they are equal + # and should produce no diff. + result = deep_diff(local, remote) + assert result == [] + + def test_encrypted_vs_plaintext_masked(self) -> None: + """When one side is encrypted placeholder, shows 'changed (encrypted)'.""" + local = { + "parameters": { + "#token": "KBC::ProjectSecure::new_nonce", + "url": "https://example.com", + } + } + remote = { + "parameters": { + "url": "https://example.com", + } + } + + result = deep_diff(local, remote) + + assert len(result) == 1 + assert "parameters.#token added" in result[0] + + def test_list_length_changed(self) -> None: + """Lists of different lengths produce a length-change message.""" + local = {"items": [1, 2, 3]} + remote = {"items": [1, 2]} + + result = deep_diff(local, remote) + + assert len(result) == 1 + assert "items list length changed" in result[0] + assert "2 -> 3" in result[0] + + def test_empty_diff_for_identical(self) -> None: + """Identical configs produce empty list.""" + config = { + "name": "Test", + "parameters": {"url": "https://example.com"}, + } + result = deep_diff(config, config) + assert result == [] + + def test_max_lines_respected(self) -> None: + """Diff output is capped at DIFF_MAX_LINES.""" + # Create configs with many differences (more than DIFF_MAX_LINES) + local: dict[str, Any] = {"parameters": {}} + remote: dict[str, Any] = {"parameters": {}} + for i in range(DIFF_MAX_LINES + 10): + local["parameters"][f"key_{i}"] = f"new_value_{i}" + remote["parameters"][f"key_{i}"] = f"old_value_{i}" + + result = deep_diff(local, remote) + + assert len(result) <= DIFF_MAX_LINES + + +# =================================================================== +# TestComputeChangeset +# =================================================================== + + +class TestComputeChangeset: + """Tests for compute_changeset().""" + + def test_added_config(self) -> None: + """Local config with no remote match produces 'added' change.""" + local_configs = [ + { + "component_id": "keboola.ex-http", + "config_id": "", # no ID = new config + "config_name": "New Extractor", + "path": "extractor/keboola.ex-http/new-extractor", + "data": {"name": "New Extractor", "parameters": {"url": "https://example.com"}}, + } + ] + remote_configs: dict[str, dict[str, Any]] = {} + + changes = compute_changeset(local_configs, remote_configs) + + assert len(changes) == 1 + assert changes[0].change_type == "added" + assert changes[0].component_id == "keboola.ex-http" + assert changes[0].config_name == "New Extractor" + + def test_modified_config(self) -> None: + """Local differs from remote produces 'modified' with details.""" + local_configs = [ + { + "component_id": "keboola.ex-http", + "config_id": "cfg-001", + "config_name": "My Extractor", + "path": "extractor/keboola.ex-http/my-extractor", + "data": { + "name": "My Extractor", + "parameters": {"url": "https://new.example.com"}, + }, + } + ] + remote_configs = { + "keboola.ex-http/cfg-001": { + "name": "My Extractor", + "parameters": {"url": "https://old.example.com"}, + }, + } + + changes = compute_changeset(local_configs, remote_configs) + + assert len(changes) == 1 + assert changes[0].change_type == "modified" + assert changes[0].config_id == "cfg-001" + assert len(changes[0].details) > 0 + # Details should mention the URL change + detail_text = " ".join(changes[0].details) + assert "parameters.url" in detail_text + + def test_deleted_config(self) -> None: + """Remote config not in local list produces 'deleted'.""" + local_configs: list[dict[str, Any]] = [] + remote_configs = { + "keboola.ex-http/cfg-001": { + "name": "Old Extractor", + "parameters": {"url": "https://example.com"}, + }, + } + + changes = compute_changeset(local_configs, remote_configs) + + assert len(changes) == 1 + assert changes[0].change_type == "deleted" + assert changes[0].component_id == "keboola.ex-http" + assert changes[0].config_id == "cfg-001" + assert changes[0].config_name == "Old Extractor" + + def test_unchanged_config(self) -> None: + """Identical local and remote produces no changeset entry.""" + config_data = { + "name": "My Extractor", + "parameters": {"url": "https://example.com"}, + } + local_configs = [ + { + "component_id": "keboola.ex-http", + "config_id": "cfg-001", + "config_name": "My Extractor", + "path": "extractor/keboola.ex-http/my-extractor", + "data": config_data, + } + ] + remote_configs = { + "keboola.ex-http/cfg-001": config_data, + } + + changes = compute_changeset(local_configs, remote_configs) + + assert len(changes) == 0 + + def test_mixed_changeset(self) -> None: + """Combination of added, modified, deleted, and unchanged.""" + local_configs = [ + # Unchanged config + { + "component_id": "keboola.ex-http", + "config_id": "cfg-001", + "config_name": "Unchanged", + "path": "extractor/keboola.ex-http/unchanged", + "data": {"name": "Unchanged", "parameters": {"url": "https://example.com"}}, + }, + # Modified config + { + "component_id": "keboola.ex-db", + "config_id": "cfg-002", + "config_name": "Modified", + "path": "extractor/keboola.ex-db/modified", + "data": {"name": "Modified", "parameters": {"host": "new-host.com"}}, + }, + # Added config (no ID) + { + "component_id": "keboola.wr-snowflake", + "config_id": "", + "config_name": "New Writer", + "path": "writer/keboola.wr-snowflake/new-writer", + "data": {"name": "New Writer", "parameters": {"table": "output"}}, + }, + ] + remote_configs = { + # Unchanged - same data + "keboola.ex-http/cfg-001": { + "name": "Unchanged", + "parameters": {"url": "https://example.com"}, + }, + # Modified - different data + "keboola.ex-db/cfg-002": { + "name": "Modified", + "parameters": {"host": "old-host.com"}, + }, + # Deleted - not in local + "keboola.snowflake-transformation/cfg-003": { + "name": "Deleted Transform", + "parameters": {}, + }, + } + + changes = compute_changeset(local_configs, remote_configs) + + change_types = {c.change_type for c in changes} + assert "added" in change_types + assert "modified" in change_types + assert "deleted" in change_types + + added = [c for c in changes if c.change_type == "added"] + modified = [c for c in changes if c.change_type == "modified"] + deleted = [c for c in changes if c.change_type == "deleted"] + + assert len(added) == 1 + assert added[0].component_id == "keboola.wr-snowflake" + + assert len(modified) == 1 + assert modified[0].config_id == "cfg-002" + + assert len(deleted) == 1 + assert deleted[0].config_id == "cfg-003" + assert deleted[0].config_name == "Deleted Transform" + + +# =================================================================== +# TestConfigChange +# =================================================================== + + +class TestConfigChange: + """Tests for ConfigChange.to_dict() serialization.""" + + def test_to_dict_serialization(self) -> None: + """to_dict() includes all expected fields.""" + change = ConfigChange( + change_type="modified", + component_id="keboola.ex-http", + config_id="cfg-001", + config_name="Test Config", + path="extractor/keboola.ex-http/test-config", + details=["parameters.url changed: 'old' -> 'new'"], + ) + d = change.to_dict() + + assert d["change_type"] == "modified" + assert d["component_id"] == "keboola.ex-http" + assert d["config_id"] == "cfg-001" + assert d["config_name"] == "Test Config" + assert d["path"] == "extractor/keboola.ex-http/test-config" + assert len(d["details"]) == 1 diff --git a/tests/test_sync_manifest.py b/tests/test_sync_manifest.py new file mode 100644 index 00000000..e6bff3c2 --- /dev/null +++ b/tests/test_sync_manifest.py @@ -0,0 +1,231 @@ +"""Tests for sync manifest models and load/save functions.""" + +import json + +import pytest + +from keboola_agent_cli.sync.manifest import ( + Manifest, + ManifestBranch, + ManifestConfigRow, + ManifestConfiguration, + ManifestGitBranching, + ManifestNaming, + ManifestProject, + load_manifest, + save_manifest, +) + + +class TestManifestProject: + """Tests for ManifestProject model.""" + + def test_manifest_project_model(self) -> None: + """ManifestProject stores id and apiHost correctly.""" + project = ManifestProject(id=12345, api_host="connection.keboola.com") + assert project.id == 12345 + assert project.api_host == "connection.keboola.com" + + def test_manifest_project_alias(self) -> None: + """ManifestProject can be created with the camelCase alias.""" + project = ManifestProject(id=99, apiHost="connection.eu-central-1.keboola.com") + assert project.api_host == "connection.eu-central-1.keboola.com" + + +class TestManifestGitBranching: + """Tests for ManifestGitBranching model.""" + + def test_manifest_git_branching_defaults(self) -> None: + """Default values: enabled=False, defaultBranch='main'.""" + branching = ManifestGitBranching() + assert branching.enabled is False + assert branching.default_branch == "main" + + def test_manifest_git_branching_custom(self) -> None: + """Custom values override defaults.""" + branching = ManifestGitBranching(enabled=True, default_branch="develop") + assert branching.enabled is True + assert branching.default_branch == "develop" + + +class TestManifestNaming: + """Tests for ManifestNaming model.""" + + def test_manifest_naming_defaults(self) -> None: + """All naming template defaults match the expected patterns.""" + naming = ManifestNaming() + assert naming.branch == "{branch_name}" + assert naming.config == "{component_type}/{component_id}/{config_name}" + assert naming.config_row == "rows/{config_row_name}" + assert naming.scheduler_config == "schedules/{config_name}" + assert naming.shared_code_config == "_shared/{target_component_id}" + assert naming.shared_code_config_row == "codes/{config_row_name}" + assert naming.variables_config == "variables" + assert naming.variables_values_row == "values/{config_row_name}" + assert naming.data_app_config == "app/{component_id}/{config_name}" + + +class TestManifestConfiguration: + """Tests for ManifestConfiguration model.""" + + def test_manifest_configuration_aliases(self) -> None: + """ManifestConfiguration accepts camelCase aliases for branchId and componentId.""" + config = ManifestConfiguration( + branchId=1, + componentId="keboola.ex-db-snowflake", + id="cfg-1", + path="extractor/keboola.ex-db-snowflake/my-config", + ) + assert config.branch_id == 1 + assert config.component_id == "keboola.ex-db-snowflake" + assert config.id == "cfg-1" + assert config.path == "extractor/keboola.ex-db-snowflake/my-config" + assert config.metadata == {} + assert config.rows == [] + + def test_manifest_configuration_with_rows(self) -> None: + """ManifestConfiguration can have row entries.""" + config = ManifestConfiguration( + branchId=1, + componentId="keboola.ex-db-snowflake", + id="cfg-1", + path="extractor/keboola.ex-db-snowflake/my-config", + rows=[ + ManifestConfigRow(id="row-1", path="rows/my-row"), + ], + ) + assert len(config.rows) == 1 + assert config.rows[0].id == "row-1" + assert config.rows[0].path == "rows/my-row" + + +class TestManifestRoundTrip: + """Tests for Manifest load/save round-trip.""" + + def _make_manifest(self) -> Manifest: + """Create a full manifest for testing.""" + return Manifest( + version=2, + project=ManifestProject(id=42, api_host="connection.keboola.com"), + allow_target_env=True, + git_branching=ManifestGitBranching(enabled=False, default_branch="main"), + sort_by="id", + naming=ManifestNaming(), + allowed_branches=["main"], + ignored_components=["keboola.sandboxes"], + branches=[ManifestBranch(id=1, path="main")], + configurations=[ + ManifestConfiguration( + branchId=1, + componentId="keboola.ex-db-snowflake", + id="cfg-123", + path="extractor/keboola.ex-db-snowflake/my-config", + rows=[ManifestConfigRow(id="row-1", path="rows/first-row")], + ) + ], + ) + + def test_manifest_round_trip(self, tmp_path) -> None: + """Save manifest, load it back, verify equality.""" + original = self._make_manifest() + + save_manifest(tmp_path, original) + loaded = load_manifest(tmp_path) + + assert loaded.version == original.version + assert loaded.project.id == original.project.id + assert loaded.project.api_host == original.project.api_host + assert loaded.allow_target_env == original.allow_target_env + assert loaded.git_branching.enabled == original.git_branching.enabled + assert loaded.git_branching.default_branch == original.git_branching.default_branch + assert loaded.sort_by == original.sort_by + assert loaded.naming.config == original.naming.config + assert loaded.allowed_branches == original.allowed_branches + assert loaded.ignored_components == original.ignored_components + assert len(loaded.branches) == 1 + assert loaded.branches[0].id == 1 + assert len(loaded.configurations) == 1 + assert loaded.configurations[0].component_id == "keboola.ex-db-snowflake" + assert loaded.configurations[0].rows[0].id == "row-1" + + def test_manifest_camelcase_output(self, tmp_path) -> None: + """Saved manifest uses camelCase keys in JSON.""" + manifest = self._make_manifest() + save_manifest(tmp_path, manifest) + + raw = json.loads((tmp_path / ".keboola" / "manifest.json").read_text()) + + # Top-level camelCase keys + assert "allowTargetEnv" in raw + assert "gitBranching" in raw + assert "sortBy" in raw + assert "allowedBranches" in raw + assert "ignoredComponents" in raw + + # Nested camelCase keys + assert "apiHost" in raw["project"] + assert "defaultBranch" in raw["gitBranching"] + assert "configRow" in raw["naming"] + assert "schedulerConfig" in raw["naming"] + assert "sharedCodeConfig" in raw["naming"] + assert "sharedCodeConfigRow" in raw["naming"] + assert "variablesConfig" in raw["naming"] + assert "variablesValuesRow" in raw["naming"] + assert "dataAppConfig" in raw["naming"] + + # Configuration entries + assert "branchId" in raw["configurations"][0] + assert "componentId" in raw["configurations"][0] + + def test_save_creates_directory(self, tmp_path) -> None: + """save_manifest creates .keboola/ directory if it does not exist.""" + project_root = tmp_path / "fresh-project" + project_root.mkdir() + + manifest = self._make_manifest() + save_manifest(project_root, manifest) + + keboola_dir = project_root / ".keboola" + assert keboola_dir.exists() + assert keboola_dir.is_dir() + assert (keboola_dir / "manifest.json").exists() + + +class TestLoadManifest: + """Tests for load_manifest error handling.""" + + def test_load_manifest_file_not_found(self, tmp_path) -> None: + """FileNotFoundError raised when manifest.json does not exist.""" + with pytest.raises(FileNotFoundError, match="Manifest not found"): + load_manifest(tmp_path) + + +class TestManifestExtraFields: + """Tests for extra field preservation.""" + + def test_manifest_extra_fields_preserved(self, tmp_path) -> None: + """Unknown fields in manifest JSON are preserved via extra='allow'.""" + keboola_dir = tmp_path / ".keboola" + keboola_dir.mkdir() + manifest_data = { + "version": 2, + "project": {"id": 1, "apiHost": "connection.keboola.com", "unknownField": "kept"}, + "allowTargetEnv": True, + "gitBranching": {"enabled": False, "defaultBranch": "main"}, + "sortBy": "id", + "naming": {"branch": "{branch_name}"}, + "branches": [], + "configurations": [], + "customTopLevel": "preserved", + } + (keboola_dir / "manifest.json").write_text(json.dumps(manifest_data)) + + loaded = load_manifest(tmp_path) + + # Extra field on root model + dumped = loaded.model_dump(mode="json", by_alias=True) + assert dumped["customTopLevel"] == "preserved" + + # Extra field on nested model + project_dumped = loaded.project.model_dump(mode="json", by_alias=True) + assert project_dumped["unknownField"] == "kept" diff --git a/tests/test_sync_naming.py b/tests/test_sync_naming.py new file mode 100644 index 00000000..c53bf0a4 --- /dev/null +++ b/tests/test_sync_naming.py @@ -0,0 +1,82 @@ +"""Tests for sync naming module -- path generation from templates.""" + +import pytest + +from keboola_agent_cli.sync.naming import config_path, config_row_path, sanitize_name + + +class TestConfigPath: + """Tests for config_path().""" + + def test_config_path_basic(self) -> None: + """Standard template produces correct directory path.""" + template = "{component_type}/{component_id}/{config_name}" + result = config_path(template, "extractor", "keboola.ex-http", "My API Source") + + assert result == "extractor/keboola.ex-http/my-api-source" + + def test_config_path_sanitizes_name(self) -> None: + """Config name with spaces and special characters is sanitized.""" + template = "{component_type}/{component_id}/{config_name}" + result = config_path(template, "writer", "keboola.wr-db-snowflake", "Sales Data (v2)!") + + assert result == "writer/keboola.wr-db-snowflake/sales-data-v2" + + +class TestConfigRowPath: + """Tests for config_row_path().""" + + def test_config_row_path(self) -> None: + """Row naming template produces correct path segment.""" + template = "rows/{config_row_name}" + result = config_row_path(template, "First Row") + + assert result == "rows/first-row" + + +class TestSanitizeName: + """Tests for sanitize_name().""" + + def test_sanitize_name_lowercase(self) -> None: + """Name is converted to lowercase.""" + assert sanitize_name("My Config") == "my-config" + + def test_sanitize_name_special_chars(self) -> None: + """Non-alphanumeric characters (except hyphens) become hyphens.""" + assert sanitize_name("API (v2) Config!") == "api-v2-config" + + def test_sanitize_name_collapse_hyphens(self) -> None: + """Multiple consecutive hyphens are collapsed into one.""" + assert sanitize_name("a---b") == "a-b" + + def test_sanitize_name_strip_hyphens(self) -> None: + """Leading and trailing hyphens are stripped.""" + assert sanitize_name("-leading-") == "leading" + + def test_sanitize_name_max_length(self) -> None: + """Names exceeding max length are truncated.""" + long_name = "a" * 200 + result = sanitize_name(long_name) + assert len(result) == 100 + + def test_sanitize_name_already_clean(self) -> None: + """Clean lowercase name passes through unchanged.""" + assert sanitize_name("simple-name") == "simple-name" + + def test_sanitize_name_numbers(self) -> None: + """Numbers are preserved.""" + assert sanitize_name("Config 42 Test") == "config-42-test" + + @pytest.mark.parametrize( + "name,expected", + [ + ("UPPERCASE", "uppercase"), + ("MiXeD CaSe", "mixed-case"), + ("with.dots.here", "with-dots-here"), + ("under_scores", "under-scores"), + (" spaces ", "spaces"), + ], + ) + def test_sanitize_name_various(self, name: str, expected: str) -> None: + """Various input patterns are sanitized correctly.""" + assert sanitize_name(name) == expected diff --git a/tests/test_sync_secrets.py b/tests/test_sync_secrets.py new file mode 100644 index 00000000..3a0ca01f --- /dev/null +++ b/tests/test_sync_secrets.py @@ -0,0 +1,120 @@ +"""Tests for sync secrets module -- encrypted value detection.""" + +import pytest + +from keboola_agent_cli.sync.secrets import ( + find_encrypted_paths, + is_encrypted_value, + is_secret_key, +) + + +class TestIsEncryptedValue: + """Tests for is_encrypted_value().""" + + @pytest.mark.parametrize( + "value", + [ + "KBC::ProjectSecure::abc123", + "KBC::ComponentSecure::xyz", + "KBC::ConfigSecure::secret", + "KBC::ProjectWideSecure::wide", + ], + ) + def test_is_encrypted_value_true(self, value: str) -> None: + """All four encryption prefixes are detected.""" + assert is_encrypted_value(value) is True + + @pytest.mark.parametrize( + "value", + [ + "plain-text-value", + "", + "kbc::projectsecure::lowercase", + "KBC::Unknown::something", + "not-encrypted", + ], + ) + def test_is_encrypted_value_false_strings(self, value: str) -> None: + """Plain strings and wrong prefixes are not encrypted.""" + assert is_encrypted_value(value) is False + + @pytest.mark.parametrize("value", [42, None, True, 3.14, [], {}]) + def test_is_encrypted_value_false_non_strings(self, value: object) -> None: + """Non-string types are never encrypted values.""" + assert is_encrypted_value(value) is False + + +class TestIsSecretKey: + """Tests for is_secret_key().""" + + @pytest.mark.parametrize("key", ["#password", "#token", "#api_key", "#"]) + def test_is_secret_key_true(self, key: str) -> None: + """Keys starting with '#' are secret keys.""" + assert is_secret_key(key) is True + + @pytest.mark.parametrize("key", ["password", "token", "", "api_key", "hash#tag"]) + def test_is_secret_key_false(self, key: str) -> None: + """Keys not starting with '#' are not secret keys.""" + assert is_secret_key(key) is False + + +class TestFindEncryptedPaths: + """Tests for find_encrypted_paths().""" + + def test_find_encrypted_paths_flat(self) -> None: + """Flat dict with encrypted values returns correct paths.""" + data = { + "#token": "KBC::ProjectSecure::abc", + "name": "plain", + "#password": "KBC::ConfigSecure::xyz", + } + paths = find_encrypted_paths(data) + + assert "#token" in paths + assert "#password" in paths + assert "name" not in paths + + def test_find_encrypted_paths_nested(self) -> None: + """Nested dicts and lists with encrypted values are found.""" + data = { + "parameters": { + "#token": "KBC::ProjectSecure::abc", + "nested": { + "#secret": "KBC::ComponentSecure::def", + }, + }, + "plain": "not-secret", + } + paths = find_encrypted_paths(data) + + assert "parameters.#token" in paths + assert "parameters.nested.#secret" in paths + assert len(paths) == 2 + + def test_find_encrypted_paths_empty(self) -> None: + """Empty dict returns empty list.""" + assert find_encrypted_paths({}) == [] + + def test_find_encrypted_paths_list_with_dicts(self) -> None: + """Lists containing dicts with encrypted values are discovered.""" + data = { + "rows": [ + {"#key": "KBC::ProjectSecure::first"}, + {"plain": "ok"}, + {"#key": "KBC::ProjectSecure::third"}, + ], + } + paths = find_encrypted_paths(data) + + assert "rows[0].#key" in paths + assert "rows[2].#key" in paths + assert len(paths) == 2 + + def test_find_encrypted_paths_encrypted_value_on_regular_key(self) -> None: + """An encrypted value on a regular (non-#) key is also detected.""" + data = { + "api_token": "KBC::ProjectSecure::hidden", + } + paths = find_encrypted_paths(data) + assert "api_token" in paths diff --git a/tests/test_sync_service.py b/tests/test_sync_service.py new file mode 100644 index 00000000..47062505 --- /dev/null +++ b/tests/test_sync_service.py @@ -0,0 +1,1391 @@ +"""Tests for SyncService - init, pull, and status business logic. + +Tests use tmp_path for filesystem operations and MagicMock for API client. +""" + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import yaml + +from helpers import setup_single_project +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.constants import ( + BRANCH_MAPPING_FILENAME, + CONFIG_FILENAME, + KEBOOLA_DIR_NAME, + MANIFEST_VERSION, +) +from keboola_agent_cli.errors import ConfigError +from keboola_agent_cli.models import TokenVerifyResponse +from keboola_agent_cli.services.sync_service import SyncService +from keboola_agent_cli.sync.manifest import load_manifest + +# --------------------------------------------------------------------------- +# Sample API data +# --------------------------------------------------------------------------- + +SAMPLE_VERIFY_TOKEN = TokenVerifyResponse( + token_id="tok-001", + token_description="kbagent-cli", + project_id=258, + project_name="Production", + owner_name="My Org", +) + +SAMPLE_BRANCHES = [ + {"id": 12345, "name": "Main", "isDefault": True}, +] + +SAMPLE_BRANCHES_WITH_DEV = [ + {"id": 12345, "name": "Main", "isDefault": True}, + {"id": 99999, "name": "feature-x", "isDefault": False}, +] + +SAMPLE_COMPONENTS = [ + { + "id": "keboola.ex-http", + "type": "extractor", + "configurations": [ + { + "id": "cfg-001", + "name": "My HTTP Extractor", + "description": "Fetches data", + "configuration": { + "parameters": {"baseUrl": "https://api.example.com"}, + }, + "rows": [ + { + "id": "row-001", + "name": "Users Endpoint", + "description": "", + "configuration": { + "parameters": {"path": "/users"}, + }, + } + ], + } + ], + }, + { + "id": "keboola.snowflake-transformation", + "type": "transformation", + "configurations": [ + { + "id": "cfg-002", + "name": "Clean Data", + "description": "Cleans raw data", + "configuration": { + "parameters": {}, + "storage": { + "output": { + "tables": [ + { + "source": "clean", + "destination": "out.c-main.clean", + } + ], + }, + }, + }, + "rows": [], + } + ], + }, +] + +SAMPLE_COMPONENTS_NO_ROWS = [ + { + "id": "keboola.ex-http", + "type": "extractor", + "configurations": [ + { + "id": "cfg-001", + "name": "My HTTP Extractor", + "description": "Fetches data", + "configuration": { + "parameters": {"baseUrl": "https://api.example.com"}, + }, + "rows": [], + } + ], + }, +] + + +# --------------------------------------------------------------------------- +# Mock client factory +# --------------------------------------------------------------------------- + + +def _make_sync_mock_client( + verify_token_response: TokenVerifyResponse | None = None, + components_response: list | None = None, + branches_response: list | None = None, +) -> MagicMock: + """Create a mock KeboolaClient suitable for SyncService tests.""" + client = MagicMock() + # Support context manager usage + client.__enter__ = MagicMock(return_value=client) + client.__exit__ = MagicMock(return_value=False) + + if verify_token_response: + client.verify_token.return_value = verify_token_response + + if components_response is not None: + client.list_components_with_configs.return_value = components_response + + if branches_response is not None: + client.list_dev_branches.return_value = branches_response + + return client + + +# =================================================================== +# init_sync tests +# =================================================================== + + +class TestInitSync: + """Tests for SyncService.init_sync().""" + + def test_init_sync_basic(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """init_sync creates manifest.json with correct project ID, api_host, and branches.""" + mock_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + svc = SyncService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + project_root = tmp_path / "project" + project_root.mkdir() + + result = svc.init_sync(alias="prod", project_root=project_root) + + # Verify result dict + assert result["status"] == "initialized" + assert result["project_id"] == 258 + assert result["project_alias"] == "prod" + assert result["api_host"] == "connection.keboola.com" + assert result["git_branching"] is False + assert result["default_branch"] == "main" + assert len(result["files_created"]) == 1 + + # Verify manifest.json was created + manifest_path = project_root / KEBOOLA_DIR_NAME / "manifest.json" + assert manifest_path.exists() + + manifest = load_manifest(project_root) + assert manifest.version == MANIFEST_VERSION + assert manifest.project.id == 258 + assert manifest.project.api_host == "connection.keboola.com" + assert len(manifest.branches) == 1 + assert manifest.branches[0].id == 12345 + assert manifest.branches[0].path == "main" + assert manifest.configurations == [] + assert manifest.git_branching.enabled is False + + def test_init_sync_git_branching(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """init_sync with git_branching=True creates branch-mapping.json.""" + mock_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + svc = SyncService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + project_root = tmp_path / "project" + project_root.mkdir() + + with ( + patch( + "keboola_agent_cli.services.sync_service.is_git_repo", + return_value=True, + ), + patch( + "keboola_agent_cli.services.sync_service.get_default_branch", + return_value="main", + ), + ): + result = svc.init_sync( + alias="prod", + project_root=project_root, + git_branching=True, + ) + + assert result["git_branching"] is True + assert result["default_branch"] == "main" + assert len(result["files_created"]) == 2 + + # Verify branch-mapping.json was created + mapping_path = project_root / KEBOOLA_DIR_NAME / BRANCH_MAPPING_FILENAME + assert mapping_path.exists() + + mapping = json.loads(mapping_path.read_text(encoding="utf-8")) + assert mapping["version"] == 1 + assert "main" in mapping["mappings"] + assert mapping["mappings"]["main"]["name"] == "Main" + + # Verify manifest has git branching enabled + manifest = load_manifest(project_root) + assert manifest.git_branching.enabled is True + assert manifest.git_branching.default_branch == "main" + + def test_init_sync_already_exists(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """init_sync raises FileExistsError when manifest already exists.""" + mock_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + svc = SyncService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + project_root = tmp_path / "project" + project_root.mkdir() + + # Create manifest first time + svc.init_sync(alias="prod", project_root=project_root) + + # Second time should raise + with pytest.raises(FileExistsError, match="Manifest already exists"): + svc.init_sync(alias="prod", project_root=project_root) + + def test_init_sync_project_not_found(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """init_sync raises ConfigError when alias is not configured.""" + store = setup_single_project(tmp_config_dir) + svc = SyncService(config_store=store) + + project_root = tmp_path / "project" + project_root.mkdir() + + with pytest.raises(ConfigError, match="not found"): + svc.init_sync(alias="nonexistent", project_root=project_root) + + def test_init_sync_git_branching_no_git_repo( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """init_sync with git_branching raises ConfigError when not a git repo.""" + mock_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + svc = SyncService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + project_root = tmp_path / "project" + project_root.mkdir() + + with ( + patch( + "keboola_agent_cli.services.sync_service.is_git_repo", + return_value=False, + ), + pytest.raises(ConfigError, match="Git repository not found"), + ): + svc.init_sync( + alias="prod", + project_root=project_root, + git_branching=True, + ) + + def test_init_sync_strips_https_prefix(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """init_sync strips https:// prefix from stack_url for api_host.""" + mock_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + svc = SyncService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + project_root = tmp_path / "project" + project_root.mkdir() + + result = svc.init_sync(alias="prod", project_root=project_root) + + # https://connection.keboola.com -> connection.keboola.com + assert result["api_host"] == "connection.keboola.com" + assert not result["api_host"].startswith("https://") + + +# =================================================================== +# pull tests +# =================================================================== + + +class TestPull: + """Tests for SyncService.pull().""" + + def _init_project( + self, + tmp_config_dir: Path, + project_root: Path, + branches_response: list | None = None, + ) -> ConfigStore: + """Helper: init a project and return the ConfigStore for reuse.""" + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=branches_response or SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + return store + + def test_pull_basic(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """pull writes _config.yml files for each configuration.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_project(tmp_config_dir, project_root) + + # Create a new service with the pull client + pull_client = _make_sync_mock_client( + components_response=SAMPLE_COMPONENTS_NO_ROWS, + ) + svc = SyncService( + config_store=store, + client_factory=lambda url, token: pull_client, + ) + + result = svc.pull(alias="prod", project_root=project_root) + + assert result["status"] == "pulled" + assert result["project_alias"] == "prod" + assert result["configs_pulled"] == 1 + assert result["rows_pulled"] == 0 + assert result["files_written"] == 1 + assert result["branch_dir"] == "main" + + # Verify _config.yml was written + config_files = list(project_root.rglob(CONFIG_FILENAME)) + assert len(config_files) == 1 + + config_data = yaml.safe_load(config_files[0].read_text(encoding="utf-8")) + assert config_data["name"] == "My HTTP Extractor" + assert config_data["_keboola"]["component_id"] == "keboola.ex-http" + assert config_data["_keboola"]["config_id"] == "cfg-001" + assert config_data["parameters"]["baseUrl"] == "https://api.example.com" + + def test_pull_with_rows(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """pull writes config rows under rows/ subdirectory.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_project(tmp_config_dir, project_root) + + pull_client = _make_sync_mock_client( + components_response=SAMPLE_COMPONENTS, + ) + svc = SyncService( + config_store=store, + client_factory=lambda url, token: pull_client, + ) + + result = svc.pull(alias="prod", project_root=project_root) + + assert result["configs_pulled"] == 2 + assert result["rows_pulled"] == 1 + assert result["files_written"] == 3 # 2 configs + 1 row + + # Verify config files exist + config_files = list(project_root.rglob(CONFIG_FILENAME)) + assert len(config_files) == 3 # 2 configs + 1 row _config.yml + + # Find the row config file (under rows/ subdirectory relative to project_root) + row_config_files = [f for f in config_files if "/rows/" in str(f.relative_to(project_root))] + assert len(row_config_files) == 1 + + row_data = yaml.safe_load(row_config_files[0].read_text(encoding="utf-8")) + assert row_data["name"] == "Users Endpoint" + assert row_data["_keboola"]["component_id"] == "keboola.ex-http" + assert row_data["_keboola"]["row_id"] == "row-001" + assert row_data["parameters"]["path"] == "/users" + + def test_pull_updates_manifest(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """pull updates manifest.configurations after downloading.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_project(tmp_config_dir, project_root) + + # Verify manifest starts with no configurations + manifest_before = load_manifest(project_root) + assert manifest_before.configurations == [] + + pull_client = _make_sync_mock_client( + components_response=SAMPLE_COMPONENTS, + ) + svc = SyncService( + config_store=store, + client_factory=lambda url, token: pull_client, + ) + + svc.pull(alias="prod", project_root=project_root) + + # Verify manifest now has configurations + manifest_after = load_manifest(project_root) + assert len(manifest_after.configurations) == 2 + + # Verify first config entry + cfg1 = manifest_after.configurations[0] + assert cfg1.component_id == "keboola.ex-http" + assert cfg1.id == "cfg-001" + assert cfg1.branch_id == 12345 + assert len(cfg1.rows) == 1 + assert cfg1.rows[0].id == "row-001" + + # Verify second config entry (no rows) + cfg2 = manifest_after.configurations[1] + assert cfg2.component_id == "keboola.snowflake-transformation" + assert cfg2.id == "cfg-002" + assert cfg2.rows == [] + + def test_pull_no_manifest(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """pull raises FileNotFoundError when manifest doesn't exist.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = setup_single_project(tmp_config_dir) + svc = SyncService(config_store=store) + + with pytest.raises(FileNotFoundError, match="Manifest not found"): + svc.pull(alias="prod", project_root=project_root) + + def test_pull_empty_components(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """pull with no components writes zero files and updates manifest.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_project(tmp_config_dir, project_root) + + pull_client = _make_sync_mock_client(components_response=[]) + svc = SyncService( + config_store=store, + client_factory=lambda url, token: pull_client, + ) + + result = svc.pull(alias="prod", project_root=project_root) + + assert result["configs_pulled"] == 0 + assert result["rows_pulled"] == 0 + assert result["files_written"] == 0 + + manifest = load_manifest(project_root) + assert manifest.configurations == [] + + +# =================================================================== +# status tests +# =================================================================== + + +class TestStatus: + """Tests for SyncService.status().""" + + def _init_and_pull( + self, + tmp_config_dir: Path, + project_root: Path, + components: list | None = None, + ) -> SyncService: + """Helper: init + pull to get a working directory with configs.""" + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + pull_client = _make_sync_mock_client( + components_response=components if components is not None else SAMPLE_COMPONENTS, + ) + pull_svc = SyncService( + config_store=store, + client_factory=lambda url, token: pull_client, + ) + pull_svc.pull(alias="prod", project_root=project_root) + return pull_svc + + def test_status_no_changes(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """status after pull shows all configs unchanged.""" + project_root = tmp_path / "project" + project_root.mkdir() + + svc = self._init_and_pull(tmp_config_dir, project_root) + + result = svc.status(project_root=project_root) + + assert result["modified"] == [] + assert result["added"] == [] + assert result["deleted"] == [] + assert result["unchanged"] == 2 # 2 configs from SAMPLE_COMPONENTS + assert result["total_tracked"] == 2 + + def test_status_deleted_config(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """status shows deleted when a _config.yml is removed.""" + project_root = tmp_path / "project" + project_root.mkdir() + + svc = self._init_and_pull(tmp_config_dir, project_root) + + # Delete one config file + config_files = list(project_root.rglob(CONFIG_FILENAME)) + # Find a config file that is NOT under rows/ + top_level_configs = [f for f in config_files if "rows" not in str(f)] + assert len(top_level_configs) >= 1 + + # Delete the first top-level config + deleted_file = top_level_configs[0] + deleted_file.unlink() + + result = svc.status(project_root=project_root) + + assert len(result["deleted"]) == 1 + assert result["deleted"][0]["config_id"] in ("cfg-001", "cfg-002") + # The other config should still be unchanged + assert result["unchanged"] == 1 + + def test_status_no_manifest(self, tmp_path: Path) -> None: + """status raises FileNotFoundError when manifest doesn't exist.""" + project_root = tmp_path / "project" + project_root.mkdir() + + # Use a minimal service (no config store needed for status) + store = MagicMock() + svc = SyncService(config_store=store) + + with pytest.raises(FileNotFoundError, match="Manifest not found"): + svc.status(project_root=project_root) + + def test_status_modified_config(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """status shows modified when _keboola.config_id is changed in a file.""" + project_root = tmp_path / "project" + project_root.mkdir() + + svc = self._init_and_pull(tmp_config_dir, project_root) + + # Modify a config file by changing the _keboola metadata + config_files = list(project_root.rglob(CONFIG_FILENAME)) + top_level_configs = [f for f in config_files if "rows" not in str(f)] + assert len(top_level_configs) >= 1 + + modified_file = top_level_configs[0] + config_data = yaml.safe_load(modified_file.read_text(encoding="utf-8")) + config_data["_keboola"]["config_id"] = "changed-id" + modified_file.write_text( + yaml.dump(config_data, default_flow_style=False), + encoding="utf-8", + ) + + result = svc.status(project_root=project_root) + + assert len(result["modified"]) == 1 + assert result["unchanged"] == 1 + + def test_status_empty_project(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """status with no configurations shows all zeros.""" + project_root = tmp_path / "project" + project_root.mkdir() + + svc = self._init_and_pull(tmp_config_dir, project_root, components=[]) + + result = svc.status(project_root=project_root) + + assert result["modified"] == [] + assert result["added"] == [] + assert result["deleted"] == [] + assert result["unchanged"] == 0 + assert result["total_tracked"] == 0 + + +# =================================================================== +# diff tests +# =================================================================== + + +class TestDiff: + """Tests for SyncService.diff().""" + + def _init_and_pull( + self, + tmp_config_dir: Path, + project_root: Path, + components: list | None = None, + ) -> tuple[ConfigStore, SyncService]: + """Helper: init + pull to get a working directory, return (store, svc).""" + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + pull_client = _make_sync_mock_client( + components_response=components if components is not None else SAMPLE_COMPONENTS_NO_ROWS, + ) + pull_svc = SyncService( + config_store=store, + client_factory=lambda url, token: pull_client, + ) + pull_svc.pull(alias="prod", project_root=project_root) + return store, pull_svc + + def test_diff_no_changes(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """Pull then diff shows no changes when local matches remote.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store, _ = self._init_and_pull(tmp_config_dir, project_root) + + # Create diff service with same components (no changes) + diff_client = _make_sync_mock_client( + components_response=SAMPLE_COMPONENTS_NO_ROWS, + ) + diff_svc = SyncService( + config_store=store, + client_factory=lambda url, token: diff_client, + ) + + result = diff_svc.diff(alias="prod", project_root=project_root) + + assert result["changes"] == [] + assert result["summary"]["added"] == 0 + assert result["summary"]["modified"] == 0 + assert result["summary"]["deleted"] == 0 + + def test_diff_modified_config(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """Modify a local file, diff detects the change.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store, _ = self._init_and_pull(tmp_config_dir, project_root) + + # Modify a local _config.yml file + config_files = list(project_root.rglob(CONFIG_FILENAME)) + assert len(config_files) >= 1 + + modified_file = config_files[0] + config_data = yaml.safe_load(modified_file.read_text(encoding="utf-8")) + config_data["parameters"]["baseUrl"] = "https://changed.example.com" + modified_file.write_text( + yaml.dump(config_data, default_flow_style=False), + encoding="utf-8", + ) + + # Create diff service with original components (remote unchanged) + diff_client = _make_sync_mock_client( + components_response=SAMPLE_COMPONENTS_NO_ROWS, + ) + diff_svc = SyncService( + config_store=store, + client_factory=lambda url, token: diff_client, + ) + + result = diff_svc.diff(alias="prod", project_root=project_root) + + assert result["summary"]["modified"] == 1 + assert len(result["changes"]) == 1 + assert result["changes"][0]["change_type"] == "modified" + + +# =================================================================== +# push tests +# =================================================================== + + +class TestPush: + """Tests for SyncService.push().""" + + def _init_and_pull( + self, + tmp_config_dir: Path, + project_root: Path, + components: list | None = None, + ) -> tuple[ConfigStore, SyncService]: + """Helper: init + pull to get a working directory, return (store, svc).""" + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + pull_client = _make_sync_mock_client( + components_response=components if components is not None else SAMPLE_COMPONENTS_NO_ROWS, + ) + pull_svc = SyncService( + config_store=store, + client_factory=lambda url, token: pull_client, + ) + pull_svc.pull(alias="prod", project_root=project_root) + return store, pull_svc + + def test_push_no_changes(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """Push when no changes returns status 'no_changes'.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store, _ = self._init_and_pull(tmp_config_dir, project_root) + + # Create push service with same components (no changes) + push_client = _make_sync_mock_client( + components_response=SAMPLE_COMPONENTS_NO_ROWS, + ) + push_svc = SyncService( + config_store=store, + client_factory=lambda url, token: push_client, + ) + + result = push_svc.push(alias="prod", project_root=project_root) + + assert result["status"] == "no_changes" + assert result["created"] == 0 + assert result["updated"] == 0 + assert result["deleted"] == 0 + + def test_push_dry_run(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """Push with dry_run returns changes without executing them.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store, _ = self._init_and_pull(tmp_config_dir, project_root) + + # Modify a local file to create a change + config_files = list(project_root.rglob(CONFIG_FILENAME)) + assert len(config_files) >= 1 + modified_file = config_files[0] + config_data = yaml.safe_load(modified_file.read_text(encoding="utf-8")) + config_data["parameters"]["baseUrl"] = "https://changed.example.com" + modified_file.write_text( + yaml.dump(config_data, default_flow_style=False), + encoding="utf-8", + ) + + # Dry run should detect changes but not call API + dry_client = _make_sync_mock_client( + components_response=SAMPLE_COMPONENTS_NO_ROWS, + ) + dry_svc = SyncService( + config_store=store, + client_factory=lambda url, token: dry_client, + ) + + result = dry_svc.push(alias="prod", project_root=project_root, dry_run=True) + + assert result["status"] == "dry_run" + assert "changes" in result + assert "summary" in result + assert result["summary"]["modified"] >= 1 + # Client should NOT have been called for create/update/delete + dry_client.update_config.assert_not_called() + dry_client.create_config.assert_not_called() + + def test_push_update(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """Modify local config, push updates via client.update_config mock.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store, _ = self._init_and_pull(tmp_config_dir, project_root) + + # Modify a local file to create a change + config_files = list(project_root.rglob(CONFIG_FILENAME)) + assert len(config_files) >= 1 + modified_file = config_files[0] + config_data = yaml.safe_load(modified_file.read_text(encoding="utf-8")) + config_data["parameters"]["baseUrl"] = "https://updated.example.com" + modified_file.write_text( + yaml.dump(config_data, default_flow_style=False), + encoding="utf-8", + ) + + # The push service needs a client that: + # 1. Returns original components for diff detection + # 2. Accepts update_config calls + # 3. Returns original components again for the post-push pull + push_client = _make_sync_mock_client( + components_response=SAMPLE_COMPONENTS_NO_ROWS, + ) + push_client.update_config.return_value = {"id": "cfg-001"} + + push_svc = SyncService( + config_store=store, + client_factory=lambda url, token: push_client, + ) + + result = push_svc.push(alias="prod", project_root=project_root) + + assert result["status"] == "pushed" + assert result["updated"] >= 1 + assert result["errors"] == [] + # Verify the client.update_config was actually called + push_client.update_config.assert_called() + + +# =================================================================== +# branch_link tests +# =================================================================== + + +class TestBranchLink: + """Tests for SyncService.branch_link().""" + + def _init_git_branching_project( + self, + tmp_config_dir: Path, + project_root: Path, + ) -> ConfigStore: + """Helper: init a project with git branching enabled.""" + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + with ( + patch( + "keboola_agent_cli.services.sync_service.is_git_repo", + return_value=True, + ), + patch( + "keboola_agent_cli.services.sync_service.get_default_branch", + return_value="main", + ), + ): + init_svc.init_sync( + alias="prod", + project_root=project_root, + git_branching=True, + ) + return store + + def test_branch_link_creates_branch(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_link creates a Keboola branch when none exists with the git branch name.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_git_branching_project(tmp_config_dir, project_root) + + # Mock client that has no existing branch matching "feature/auth", + # so it creates one + link_client = _make_sync_mock_client( + branches_response=[ + {"id": 12345, "name": "Main", "isDefault": True}, + ], + ) + link_client.create_dev_branch.return_value = {"id": 99999} + + svc = SyncService( + config_store=store, + client_factory=lambda url, token: link_client, + ) + + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="feature/auth", + ): + result = svc.branch_link( + alias="prod", + project_root=project_root, + ) + + assert result["status"] == "linked" + assert result["git_branch"] == "feature/auth" + assert result["keboola_branch_id"] == "99999" + assert result["keboola_branch_name"] == "feature/auth" + link_client.create_dev_branch.assert_called_once_with(name="feature/auth") + + # Verify the mapping was saved to disk + from keboola_agent_cli.sync.branch_mapping import load_branch_mapping + + mapping = load_branch_mapping(project_root) + entry = mapping.get("feature/auth") + assert entry is not None + assert entry.keboola_id == "99999" + + def test_branch_link_finds_existing_branch(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_link links to an existing Keboola branch that matches the name.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_git_branching_project(tmp_config_dir, project_root) + + link_client = _make_sync_mock_client( + branches_response=SAMPLE_BRANCHES_WITH_DEV, + ) + + svc = SyncService( + config_store=store, + client_factory=lambda url, token: link_client, + ) + + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="feature-x", + ): + result = svc.branch_link( + alias="prod", + project_root=project_root, + ) + + assert result["status"] == "linked" + assert result["git_branch"] == "feature-x" + assert result["keboola_branch_id"] == "99999" + # Should not have created a new branch + link_client.create_dev_branch.assert_not_called() + + def test_branch_link_default_branch_error(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_link raises ConfigError when on the default (main) branch.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_git_branching_project(tmp_config_dir, project_root) + + svc = SyncService(config_store=store) + + with ( + patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="main", + ), + pytest.raises(ConfigError, match="Cannot link the default branch"), + ): + svc.branch_link( + alias="prod", + project_root=project_root, + ) + + def test_branch_link_git_branching_not_enabled( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """branch_link raises ConfigError when git branching is not enabled.""" + project_root = tmp_path / "project" + project_root.mkdir() + + # Init without git branching + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + svc = SyncService(config_store=store) + + with pytest.raises(ConfigError, match="Git-branching mode is not enabled"): + svc.branch_link( + alias="prod", + project_root=project_root, + ) + + def test_branch_link_already_linked(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_link returns already_linked when mapping already exists.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_git_branching_project(tmp_config_dir, project_root) + + link_client = _make_sync_mock_client( + branches_response=SAMPLE_BRANCHES_WITH_DEV, + ) + + svc = SyncService( + config_store=store, + client_factory=lambda url, token: link_client, + ) + + # First link + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="feature-x", + ): + svc.branch_link(alias="prod", project_root=project_root) + + # Second link should return already_linked + result = svc.branch_link(alias="prod", project_root=project_root) + + assert result["status"] == "already_linked" + assert result["git_branch"] == "feature-x" + assert result["keboola_branch_id"] == "99999" + + def test_branch_link_with_branch_id(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_link with --branch-id links to a specific existing branch.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_git_branching_project(tmp_config_dir, project_root) + + link_client = _make_sync_mock_client( + branches_response=SAMPLE_BRANCHES_WITH_DEV, + ) + + svc = SyncService( + config_store=store, + client_factory=lambda url, token: link_client, + ) + + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="my-feature", + ): + result = svc.branch_link( + alias="prod", + project_root=project_root, + branch_id=99999, + ) + + assert result["status"] == "linked" + assert result["keboola_branch_id"] == "99999" + assert result["keboola_branch_name"] == "feature-x" + + def test_branch_link_with_branch_name_creates( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """branch_link with --branch-name creates a branch with that name.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_git_branching_project(tmp_config_dir, project_root) + + link_client = _make_sync_mock_client( + branches_response=SAMPLE_BRANCHES, # no "custom-name" branch + ) + link_client.create_dev_branch.return_value = {"id": 77777} + + svc = SyncService( + config_store=store, + client_factory=lambda url, token: link_client, + ) + + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="my-feature", + ): + result = svc.branch_link( + alias="prod", + project_root=project_root, + branch_name="custom-name", + ) + + assert result["status"] == "linked" + assert result["keboola_branch_id"] == "77777" + assert result["keboola_branch_name"] == "custom-name" + link_client.create_dev_branch.assert_called_once_with(name="custom-name") + + +# =================================================================== +# branch_unlink tests +# =================================================================== + + +class TestBranchUnlink: + """Tests for SyncService.branch_unlink().""" + + def _init_and_link( + self, + tmp_config_dir: Path, + project_root: Path, + ) -> ConfigStore: + """Helper: init with git branching, then link feature-x.""" + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + with ( + patch( + "keboola_agent_cli.services.sync_service.is_git_repo", + return_value=True, + ), + patch( + "keboola_agent_cli.services.sync_service.get_default_branch", + return_value="main", + ), + ): + init_svc.init_sync( + alias="prod", + project_root=project_root, + git_branching=True, + ) + + # Link feature-x + link_client = _make_sync_mock_client( + branches_response=SAMPLE_BRANCHES_WITH_DEV, + ) + link_svc = SyncService( + config_store=store, + client_factory=lambda url, token: link_client, + ) + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="feature-x", + ): + link_svc.branch_link(alias="prod", project_root=project_root) + + return store + + def test_branch_unlink_success(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_unlink removes the mapping for the current git branch.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_and_link(tmp_config_dir, project_root) + + svc = SyncService(config_store=store) + + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="feature-x", + ): + result = svc.branch_unlink(project_root=project_root) + + assert result["status"] == "unlinked" + assert result["git_branch"] == "feature-x" + assert result["keboola_branch_id"] == "99999" + assert result["keboola_branch_name"] == "feature-x" + + # Verify mapping was removed from disk + from keboola_agent_cli.sync.branch_mapping import load_branch_mapping + + mapping = load_branch_mapping(project_root) + assert mapping.get("feature-x") is None + + def test_branch_unlink_not_linked(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_unlink returns not_linked when branch has no mapping.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_and_link(tmp_config_dir, project_root) + + svc = SyncService(config_store=store) + + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="other-branch", + ): + result = svc.branch_unlink(project_root=project_root) + + assert result["status"] == "not_linked" + assert result["git_branch"] == "other-branch" + + def test_branch_unlink_default_branch_error(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_unlink raises ConfigError when on default branch.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_and_link(tmp_config_dir, project_root) + + svc = SyncService(config_store=store) + + with ( + patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="main", + ), + pytest.raises(ConfigError, match="Cannot unlink the default branch"), + ): + svc.branch_unlink(project_root=project_root) + + +# =================================================================== +# branch_status tests +# =================================================================== + + +class TestBranchStatus: + """Tests for SyncService.branch_status().""" + + def _init_git_branching_project( + self, + tmp_config_dir: Path, + project_root: Path, + ) -> ConfigStore: + """Helper: init a project with git branching enabled.""" + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + with ( + patch( + "keboola_agent_cli.services.sync_service.is_git_repo", + return_value=True, + ), + patch( + "keboola_agent_cli.services.sync_service.get_default_branch", + return_value="main", + ), + ): + init_svc.init_sync( + alias="prod", + project_root=project_root, + git_branching=True, + ) + return store + + def test_branch_status_linked(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_status shows linked status when mapping exists.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_git_branching_project(tmp_config_dir, project_root) + + # Link feature-x first + link_client = _make_sync_mock_client( + branches_response=SAMPLE_BRANCHES_WITH_DEV, + ) + link_svc = SyncService( + config_store=store, + client_factory=lambda url, token: link_client, + ) + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="feature-x", + ): + link_svc.branch_link(alias="prod", project_root=project_root) + + # Now check status + svc = SyncService(config_store=store) + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="feature-x", + ): + result = svc.branch_status(project_root=project_root) + + assert result["git_branching"] is True + assert result["git_branch"] == "feature-x" + assert result["linked"] is True + assert result["keboola_branch_id"] == "99999" + assert result["keboola_branch_name"] == "feature-x" + assert result["is_production"] is False + + def test_branch_status_not_linked(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_status shows not linked when no mapping exists.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_git_branching_project(tmp_config_dir, project_root) + + svc = SyncService(config_store=store) + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="unlinked-branch", + ): + result = svc.branch_status(project_root=project_root) + + assert result["git_branching"] is True + assert result["git_branch"] == "unlinked-branch" + assert result["linked"] is False + + def test_branch_status_production(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_status shows is_production=True for the main branch mapping.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_git_branching_project(tmp_config_dir, project_root) + + svc = SyncService(config_store=store) + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="main", + ): + result = svc.branch_status(project_root=project_root) + + assert result["git_branching"] is True + assert result["git_branch"] == "main" + assert result["linked"] is True + assert result["is_production"] is True + assert result["keboola_branch_id"] is None + + def test_branch_status_git_branching_disabled( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """branch_status returns git_branching=False when not enabled.""" + project_root = tmp_path / "project" + project_root.mkdir() + + # Init without git branching + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + svc = SyncService(config_store=store) + result = svc.branch_status(project_root=project_root) + + assert result == {"git_branching": False} + + def test_branch_status_no_mapping_file(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """branch_status returns linked=False when mapping file is missing.""" + project_root = tmp_path / "project" + project_root.mkdir() + + store = self._init_git_branching_project(tmp_config_dir, project_root) + + # Delete the branch-mapping.json + mapping_path = project_root / KEBOOLA_DIR_NAME / BRANCH_MAPPING_FILENAME + mapping_path.unlink() + + svc = SyncService(config_store=store) + with patch( + "keboola_agent_cli.sync.git_utils.get_current_branch", + return_value="feature-x", + ): + result = svc.branch_status(project_root=project_root) + + assert result["git_branching"] is True + assert result["git_branch"] == "feature-x" + assert result["linked"] is False