From 8a7fb27527e120257dc55828bd41b0f5d5b1b238 Mon Sep 17 00:00:00 2001 From: jordanrburger Date: Sat, 28 Feb 2026 18:06:26 -0500 Subject: [PATCH 1/2] Close must-have gaps in explorer command - Add jsonschema + pyyaml dependencies - Schema validation against kbc-explorer/schema.json (non-blocking) - YAML tier config support (--tiers option) with project-to-tier mapping - UNCLASSIFIED tier warning instead of silent L0 default - Atomic file writes (write .tmp then os.replace) - Change job limit default from 100 to 500 - Add 26 tests covering pure functions and service with mocked deps Closes #15 Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 2 + src/keboola_agent_cli/commands/explorer.py | 86 +++ .../services/explorer_service.py | 624 ++++++++++++++++++ tests/test_explorer_service.py | 402 +++++++++++ uv.lock | 52 +- 5 files changed, 1165 insertions(+), 1 deletion(-) create mode 100644 src/keboola_agent_cli/commands/explorer.py create mode 100644 src/keboola_agent_cli/services/explorer_service.py create mode 100644 tests/test_explorer_service.py diff --git a/pyproject.toml b/pyproject.toml index 982178ba..0a56fdd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,8 @@ dependencies = [ "pydantic>=2.5", "platformdirs>=4", "mcp>=1.0.0,<2.0.0", + "jsonschema>=4.20", + "pyyaml>=6", ] [project.scripts] diff --git a/src/keboola_agent_cli/commands/explorer.py b/src/keboola_agent_cli/commands/explorer.py new file mode 100644 index 00000000..e2f5737d --- /dev/null +++ b/src/keboola_agent_cli/commands/explorer.py @@ -0,0 +1,86 @@ +"""Explorer command - generate data files and open the kbc-explorer dashboard. + +Thin CLI layer: parses arguments, calls ExplorerService, formats output. +No business logic belongs here. +""" + +from pathlib import Path +from typing import Optional + +import typer + +from ..errors import ConfigError +from ._helpers import emit_project_warnings, get_formatter, get_service + +explorer_app = typer.Typer(help="Generate and open the KBC Explorer dashboard") + + +@explorer_app.callback(invoke_without_command=True) +def explorer( + ctx: typer.Context, + project: Optional[list[str]] = typer.Option( + None, + "--project", + help="Project alias(es) to include (repeatable, default: all)", + ), + output_dir: Optional[Path] = typer.Option( + None, + "--output-dir", + help="Directory to write catalog/orchestration files (default: kbc-explorer/)", + ), + job_limit: int = typer.Option( + 500, + "--job-limit", + help="Max jobs per project for statistics (default: 500)", + ), + tiers: Optional[Path] = typer.Option( + None, + "--tiers", + help="Path to YAML tier config file for project tier assignments", + ), + no_open: bool = typer.Option( + False, + "--no-open", + help="Generate files but don't open the browser", + ), +) -> None: + """Generate explorer data from connected projects and open the dashboard.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "explorer_service") + + aliases = project if project else None + + try: + result = service.generate( + aliases=aliases, + output_dir=output_dir, + job_limit=job_limit, + open_browser=not no_open, + tiers_config=tiers, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code="CONFIG_ERROR") + raise typer.Exit(code=5) from None + + if formatter.json_mode: + formatter.output(result) + else: + projects_count = result["projects_count"] + configs_count = result["configs_count"] + jobs_sampled = result["jobs_sampled"] + lineage_edges = result["lineage_edges"] + orch_count = result["orchestrations_count"] + out_dir = result["output_dir"] + + formatter.console.print( + f"[bold green]Explorer generated![/bold green] " + f"{projects_count} projects, {configs_count} configs, " + f"{jobs_sampled} jobs, {lineage_edges} lineage edges, " + f"{orch_count} orchestrations" + ) + formatter.console.print(f"Output: {out_dir}") + + if not no_open: + formatter.console.print("Opening dashboard in browser...") + + emit_project_warnings(formatter, result) diff --git a/src/keboola_agent_cli/services/explorer_service.py b/src/keboola_agent_cli/services/explorer_service.py new file mode 100644 index 00000000..7afa4953 --- /dev/null +++ b/src/keboola_agent_cli/services/explorer_service.py @@ -0,0 +1,624 @@ +"""Explorer service - generates catalog and orchestration data for kbc-explorer. + +Orchestrates data collection across all registered projects by calling +ConfigService, JobService, and LineageService, then assembles the results +into catalog.json/catalog.js and orchestrations.json/orchestrations.js +files that the kbc-explorer HTML app consumes. +""" + +import json +import logging +import os +import webbrowser +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import jsonschema +import yaml + +from .. import __version__ +from ..config_store import ConfigStore +from ..errors import ConfigError, KeboolaApiError +from .base import BaseService, ClientFactory +from .config_service import ConfigService +from .job_service import JobService +from .lineage_service import LineageService + +logger = logging.getLogger(__name__) + +DEFAULT_JOB_LIMIT = 500 +DEFAULT_OUTPUT_DIR = Path(__file__).resolve().parent.parent.parent.parent / "kbc-explorer" + + +def _assign_tier(alias: str, tier_map: dict[str, str] | None = None) -> tuple[str, bool]: + """Assign a tier based on tier config, then alias naming convention. + + Returns: + Tuple of (tier, was_unclassified). was_unclassified is True when + neither the tier map nor the naming convention matched. + """ + if tier_map and alias in tier_map: + return tier_map[alias], False + + lower = alias.lower() + if "-l0-" in lower or lower.startswith("l0-"): + return "L0", False + if "-l1-" in lower or lower.startswith("l1-"): + return "L1", False + if "-l2-" in lower or lower.startswith("l2-"): + return "L2", False + return "L0", True + + +def _compute_job_stats(jobs: list[dict[str, Any]]) -> dict[str, Any]: + """Compute aggregated job statistics from a list of raw job dicts.""" + total = len(jobs) + if total == 0: + return { + "total_jobs": 0, + "status_counts": {}, + "success_rate_pct": 0, + "avg_duration_seconds": 0, + "date_range": {"earliest": None, "latest": None}, + "component_stats": {}, + "failing_configs": [], + } + + status_counts: dict[str, int] = {} + durations: list[float] = [] + timestamps: list[str] = [] + component_stats: dict[str, dict[str, int]] = {} + config_stats: dict[str, dict[str, Any]] = {} + + for job in jobs: + status = job.get("status", "unknown") + status_counts[status] = status_counts.get(status, 0) + 1 + + duration = job.get("durationSeconds") + if duration is not None: + durations.append(float(duration)) + + created = job.get("createdTime") or job.get("startTime") + if created: + timestamps.append(created) + + comp_id = job.get("component", job.get("componentId", "unknown")) + if comp_id not in component_stats: + component_stats[comp_id] = {"success": 0, "error": 0, "other": 0, "total": 0} + component_stats[comp_id]["total"] += 1 + if status == "success": + component_stats[comp_id]["success"] += 1 + elif status == "error": + component_stats[comp_id]["error"] += 1 + else: + component_stats[comp_id]["other"] += 1 + + config_val = job.get("configId") or job.get("config", "") + config_id = str(config_val) if config_val else "" + if config_id: + config_key = f"{comp_id}/{config_id}" + if config_key not in config_stats: + config_stats[config_key] = { + "config_key": config_key, + "component_id": comp_id, + "error_count": 0, + "total_runs": 0, + "last_run": "", + } + config_stats[config_key]["total_runs"] += 1 + if status == "error": + config_stats[config_key]["error_count"] += 1 + if created and created > config_stats[config_key]["last_run"]: + config_stats[config_key]["last_run"] = created + + success_count = status_counts.get("success", 0) + success_rate = round((success_count / total) * 100, 1) if total > 0 else 0 + avg_duration = round(sum(durations) / len(durations), 1) if durations else 0 + + timestamps.sort() + date_range = { + "earliest": timestamps[0] if timestamps else None, + "latest": timestamps[-1] if timestamps else None, + } + + failing_configs = [] + for cs in config_stats.values(): + if cs["error_count"] > 0: + cs["error_rate_pct"] = round( + (cs["error_count"] / cs["total_runs"]) * 100, 1 + ) + failing_configs.append(cs) + failing_configs.sort(key=lambda x: x["error_rate_pct"], reverse=True) + + return { + "total_jobs": total, + "status_counts": status_counts, + "success_rate_pct": success_rate, + "avg_duration_seconds": avg_duration, + "date_range": date_range, + "component_stats": component_stats, + "failing_configs": failing_configs, + } + + +def _build_mermaid(phases: list[dict[str, Any]]) -> str: + """Build a Mermaid graph definition from orchestration phases.""" + lines = ["graph TD"] + for phase in phases: + phase_id = phase.get("id", 0) + phase_name = phase.get("name", f"Phase {phase_id}") + node_id = f"P{phase_id}" + lines.append(f' {node_id}["{phase_name}"]') + for dep_id in phase.get("depends_on", []): + lines.append(f" P{dep_id} --> {node_id}") + for task in phase.get("tasks", []): + task_name = task.get("name", "?") + task_node = f"{node_id}_{task.get('config_id', 'x')}" + icon = task.get("type_icon", "") + label = f"{icon} {task_name}" if icon else task_name + lines.append(f' {task_node}["{label}"]') + lines.append(f" {node_id} --> {task_node}") + return "\n".join(lines) + + +def _type_icon(component_id: str) -> str: + """Map component ID to a short type icon.""" + if ".ex-" in component_id or component_id.startswith("ex-"): + return "EX" + if ".wr-" in component_id or component_id.startswith("wr-"): + return "WR" + if "transformation" in component_id or "snowflake-sql" in component_id: + return "TR" + if "orchestrator" in component_id: + return "OT" + return "AP" + + +class ExplorerService(BaseService): + """Generates catalog and orchestration data for the kbc-explorer HTML app. + + Collects configs, jobs, lineage, and orchestration details from all + registered projects and assembles them into the schema expected by + the explorer's index.html. + """ + + def __init__( + self, + config_store: ConfigStore, + config_service: ConfigService, + job_service: JobService, + lineage_service: LineageService, + client_factory: ClientFactory | None = None, + ) -> None: + super().__init__(config_store=config_store, client_factory=client_factory) + self._config_service = config_service + self._job_service = job_service + self._lineage_service = lineage_service + + def generate( + self, + aliases: list[str] | None = None, + output_dir: Path | None = None, + job_limit: int = DEFAULT_JOB_LIMIT, + open_browser: bool = True, + tiers_config: Path | None = None, + ) -> dict[str, Any]: + """Generate explorer data files and optionally open the browser. + + Args: + aliases: Project aliases to include. None means all. + output_dir: Directory to write files. Defaults to kbc-explorer/. + job_limit: Max jobs per project for stats. + open_browser: Whether to open index.html after generation. + tiers_config: Path to YAML tier config file. + + Returns: + Dict with generation summary and any errors. + """ + if output_dir is None: + output_dir = DEFAULT_OUTPUT_DIR + + projects = self.resolve_projects(aliases) + if not projects: + raise ConfigError("No projects configured. Use 'kbagent project add' first.") + + all_errors: list[dict[str, str]] = [] + + # Step 1: Collect configs + logger.info("Collecting configurations from %d projects...", len(projects)) + config_result = self._config_service.list_configs( + aliases=list(projects.keys()) + ) + all_errors.extend(config_result.get("errors", [])) + + # Step 2: Collect jobs + logger.info("Collecting job history from %d projects...", len(projects)) + job_result = self._job_service.list_jobs( + aliases=list(projects.keys()), limit=job_limit + ) + all_errors.extend(job_result.get("errors", [])) + + # Step 3: Collect lineage + logger.info("Collecting lineage from %d projects...", len(projects)) + lineage_result = self._lineage_service.get_lineage( + aliases=list(projects.keys()) + ) + all_errors.extend(lineage_result.get("errors", [])) + + # Step 4: Group data by project + configs_by_project: dict[str, list[dict[str, Any]]] = {} + for cfg in config_result.get("configs", []): + alias = cfg["project_alias"] + configs_by_project.setdefault(alias, []).append(cfg) + + jobs_by_project: dict[str, list[dict[str, Any]]] = {} + for job in job_result.get("jobs", []): + alias = job["project_alias"] + jobs_by_project.setdefault(alias, []).append(job) + + # Build sharing_out/sharing_in from lineage edges + sharing_out_by_project: dict[str, list[dict[str, Any]]] = {} + sharing_in_by_project: dict[str, list[dict[str, Any]]] = {} + for edge in lineage_result.get("edges", []): + src_alias = edge.get("source_project_alias", "") + tgt_alias = edge.get("target_project_alias", "") + if src_alias: + sharing_out_by_project.setdefault(src_alias, []).append({ + "bucket": edge.get("source_bucket_id", ""), + "target_project": tgt_alias, + "target_project_name": edge.get("target_project_name", ""), + "target_bucket": edge.get("target_bucket_id", ""), + "sharing_type": edge.get("sharing_type", ""), + }) + if tgt_alias: + sharing_in_by_project.setdefault(tgt_alias, []).append({ + "bucket": edge.get("target_bucket_id", ""), + "source_project": src_alias, + "source_project_name": edge.get("source_project_name", ""), + "source_bucket": edge.get("source_bucket_id", ""), + "sharing_type": edge.get("sharing_type", ""), + }) + + # Load tier config if provided + tier_map: dict[str, str] | None = None + tier_descriptions: dict[str, dict[str, str]] | None = None + catalog_description: str | None = None + if tiers_config is not None: + tier_map, tier_descriptions, catalog_description = self._load_tiers_config( + tiers_config, all_errors + ) + + # Step 5: Assemble per-project data + project_data: dict[str, dict[str, Any]] = {} + tiers: dict[str, list[str]] = {"L0": [], "L1": [], "L2": []} + + for alias, project in projects.items(): + tier, was_unclassified = _assign_tier(alias, tier_map) + if was_unclassified: + all_errors.append({ + "project_alias": alias, + "error_code": "TIER_UNCLASSIFIED", + "message": f"Project '{alias}' has no tier mapping — defaulting to L0", + }) + tiers[tier].append(alias) + + # Group configs by type + configs = configs_by_project.get(alias, []) + by_type: dict[str, dict[str, Any]] = {} + for cfg in configs: + ctype = cfg.get("component_type", "other") + if ctype not in by_type: + by_type[ctype] = {"count": 0, "configs": []} + by_type[ctype]["count"] += 1 + by_type[ctype]["configs"].append({ + "config_id": cfg["config_id"], + "config_name": cfg["config_name"], + "config_description": cfg.get("config_description", ""), + "component_id": cfg["component_id"], + "component_name": cfg.get("component_name", ""), + }) + + jobs = jobs_by_project.get(alias, []) + job_stats = _compute_job_stats(jobs) + + project_data[alias] = { + "alias": alias, + "name": project.project_name or alias, + "project_id": project.project_id or 0, + "tier": tier, + "configurations": { + "total_configs": len(configs), + "by_type": by_type, + }, + "job_stats": job_stats, + "sharing_out": sharing_out_by_project.get(alias, []), + "sharing_in": sharing_in_by_project.get(alias, []), + } + + # Step 6: Collect orchestrations + orchestrations = self._collect_orchestrations( + configs_by_project, all_errors + ) + + # Step 7: Build lineage for catalog + lineage_edges = [] + for edge in lineage_result.get("edges", []): + lineage_edges.append({ + "source_project_alias": edge.get("source_project_alias", ""), + "source_project_id": str(edge.get("source_project_id", "")), + "source_project_name": edge.get("source_project_name", ""), + "source_bucket_id": edge.get("source_bucket_id", ""), + "target_project_alias": edge.get("target_project_alias", ""), + "target_project_id": str(edge.get("target_project_id", "")), + "target_project_name": edge.get("target_project_name", ""), + "target_bucket_id": edge.get("target_bucket_id", ""), + "sharing_type": edge.get("sharing_type", ""), + }) + + sharing_out_count = len({ + e.get("source_project_alias") + for e in lineage_result.get("edges", []) + if e.get("source_project_alias") + }) + receiving_in_count = len({ + e.get("target_project_alias") + for e in lineage_result.get("edges", []) + if e.get("target_project_alias") + }) + + # Determine stack_url from first project + first_project = next(iter(projects.values())) + stack_url = first_project.stack_url + + # Step 8: Assemble catalog + default_tier_defs = { + "L0": { + "name": "Data Sources / Extraction", + "description": "Raw data extraction from external systems", + }, + "L1": { + "name": "Processing / Transformation", + "description": "Data processing and transformation", + }, + "L2": { + "name": "Output / Delivery", + "description": "Final data products and delivery", + }, + } + # Merge tier descriptions from config file if provided + if tier_descriptions: + for tier_key, tier_def in tier_descriptions.items(): + if tier_key in default_tier_defs: + default_tier_defs[tier_key].update(tier_def) + + description = catalog_description or f"Project catalog with {len(projects)} projects" + + catalog = { + "metadata": { + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "tool": f"kbagent CLI v{__version__}", + "stack_url": stack_url, + "description": description, + }, + "tiers": { + tier_key: { + **default_tier_defs.get(tier_key, {"name": tier_key, "description": ""}), + "projects": sorted(tier_projects), + } + for tier_key, tier_projects in tiers.items() + }, + "projects": project_data, + "lineage": { + "edges": lineage_edges, + "summary": { + "total_edges": len(lineage_edges), + "projects_sharing_out": sharing_out_count, + "projects_receiving_in": receiving_in_count, + }, + }, + } + + # Step 8b: Schema validation + schema_path = Path(output_dir) / "schema.json" + if schema_path.exists(): + try: + schema = json.loads(schema_path.read_text()) + jsonschema.validate(catalog, schema) + logger.info("Catalog passed schema validation") + except jsonschema.ValidationError as exc: + logger.warning("Catalog schema validation failed: %s", exc.message) + all_errors.append({ + "project_alias": "_schema", + "error_code": "SCHEMA_VALIDATION_ERROR", + "message": f"Schema validation: {exc.message}", + }) + except (json.JSONDecodeError, jsonschema.SchemaError) as exc: + logger.warning("Failed to load/parse schema: %s", exc) + all_errors.append({ + "project_alias": "_schema", + "error_code": "SCHEMA_LOAD_ERROR", + "message": f"Failed to load schema: {exc}", + }) + + # Step 9: Write files (atomic: write .tmp then os.replace) + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + catalog_json_path = output_dir / "catalog.json" + catalog_js_path = output_dir / "catalog.js" + orch_json_path = output_dir / "orchestrations.json" + orch_js_path = output_dir / "orchestrations.js" + + catalog_json_str = json.dumps(catalog, indent=2) + orch_json_str = json.dumps(orchestrations, indent=2) + + for path, content in [ + (catalog_json_path, catalog_json_str), + (catalog_js_path, f"const CATALOG = {catalog_json_str};\n"), + (orch_json_path, orch_json_str), + (orch_js_path, f"const ORCHESTRATIONS = {orch_json_str};\n"), + ]: + tmp_path = path.with_suffix(path.suffix + ".tmp") + tmp_path.write_text(content) + os.replace(tmp_path, path) + + logger.info("Wrote catalog and orchestration files to %s", output_dir) + + # Step 10: Open browser + index_path = output_dir / "index.html" + if open_browser and index_path.exists(): + webbrowser.open(index_path.as_uri()) + + return { + "output_dir": str(output_dir), + "projects_count": len(projects), + "configs_count": sum( + p["configurations"]["total_configs"] for p in project_data.values() + ), + "jobs_sampled": sum( + p["job_stats"]["total_jobs"] for p in project_data.values() + ), + "lineage_edges": len(lineage_edges), + "orchestrations_count": len(orchestrations), + "errors": all_errors, + "files_written": [ + str(catalog_json_path), + str(catalog_js_path), + str(orch_json_path), + str(orch_js_path), + ], + } + + @staticmethod + def _load_tiers_config( + tiers_config: Path, + all_errors: list[dict[str, str]], + ) -> tuple[dict[str, str] | None, dict[str, dict[str, str]] | None, str | None]: + """Load a YAML tier config file. + + Returns: + Tuple of (project_tier_map, tier_descriptions, catalog_description). + Any or all may be None if loading fails. + """ + try: + data = yaml.safe_load(tiers_config.read_text()) + except Exception as exc: + all_errors.append({ + "project_alias": "_tiers", + "error_code": "TIERS_CONFIG_ERROR", + "message": f"Failed to load tiers config: {exc}", + }) + return None, None, None + + if not isinstance(data, dict): + all_errors.append({ + "project_alias": "_tiers", + "error_code": "TIERS_CONFIG_ERROR", + "message": "Tiers config must be a YAML mapping", + }) + return None, None, None + + project_map: dict[str, str] = {} + for alias, tier in data.get("projects", {}).items(): + project_map[str(alias)] = str(tier).upper() + + tier_descriptions: dict[str, dict[str, str]] = {} + for tier_key, tier_def in data.get("tiers", {}).items(): + if isinstance(tier_def, dict): + tier_descriptions[str(tier_key).upper()] = { + k: str(v) for k, v in tier_def.items() + } + + catalog_description = data.get("description") + if catalog_description: + catalog_description = str(catalog_description) + + return project_map, tier_descriptions, catalog_description + + def _collect_orchestrations( + self, + configs_by_project: dict[str, list[dict[str, Any]]], + all_errors: list[dict[str, str]], + ) -> dict[str, Any]: + """Collect orchestration details for all keboola.orchestrator configs.""" + orchestrations: dict[str, Any] = {} + + for alias, configs in configs_by_project.items(): + orch_configs = [ + c for c in configs if c.get("component_id") == "keboola.orchestrator" + ] + for cfg in orch_configs: + config_id = cfg["config_id"] + key = f"{alias}|{config_id}" + try: + detail = self._config_service.get_config_detail( + alias=alias, + component_id="keboola.orchestrator", + config_id=config_id, + ) + orchestrations[key] = self._parse_orchestration( + alias, config_id, cfg, detail + ) + except (KeboolaApiError, ConfigError, Exception) as exc: + logger.warning( + "Failed to fetch orchestration %s/%s: %s", alias, config_id, exc + ) + all_errors.append({ + "project_alias": alias, + "error_code": "ORCHESTRATION_ERROR", + "message": f"Failed to fetch flow {config_id}: {exc}", + }) + + return orchestrations + + @staticmethod + def _parse_orchestration( + alias: str, + config_id: str, + cfg: dict[str, Any], + detail: dict[str, Any], + ) -> dict[str, Any]: + """Parse an orchestration config detail into the explorer format.""" + config_data = detail.get("configuration", {}) + phases_raw = config_data.get("phases", []) + + phases = [] + for phase in phases_raw: + tasks = [] + for task in phase.get("tasks", []): + task_cfg = task.get("task", {}) + comp_id = task_cfg.get("componentId", "") + tasks.append({ + "name": task_cfg.get("name", task.get("name", "")), + "component_id": comp_id, + "component_short": comp_id.split(".")[-1] if comp_id else "", + "config_id": str(task_cfg.get("configId", task_cfg.get("configurationId", ""))), + "enabled": task.get("enabled", True), + "continue_on_failure": task.get("continueOnFailure", False), + "type_icon": _type_icon(comp_id), + }) + phases.append({ + "id": phase.get("id", 0), + "name": phase.get("name", ""), + "depends_on": [d.get("phaseId", d) for d in phase.get("dependsOn", [])], + "tasks": tasks, + }) + + total_tasks = sum(len(p["tasks"]) for p in phases) + + result = { + "project_alias": alias, + "config_id": config_id, + "name": cfg.get("config_name", detail.get("name", "")), + "description": cfg.get("config_description", detail.get("description", "")), + "is_disabled": detail.get("isDisabled", False), + "version": detail.get("version", 0), + "last_modified": detail.get("changeDescription", ""), + "last_modified_by": "", + "phases": phases, + "total_tasks": total_tasks, + "total_phases": len(phases), + "mermaid": _build_mermaid(phases), + } + return result diff --git a/tests/test_explorer_service.py b/tests/test_explorer_service.py new file mode 100644 index 00000000..dee0b58a --- /dev/null +++ b/tests/test_explorer_service.py @@ -0,0 +1,402 @@ +"""Tests for ExplorerService and explorer helper functions.""" + +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.services.explorer_service import ( + ExplorerService, + _assign_tier, + _build_mermaid, + _compute_job_stats, + _type_icon, +) + +from helpers import setup_single_project + + +# --------------------------------------------------------------------------- +# Pure function tests: _assign_tier +# --------------------------------------------------------------------------- + +class TestAssignTier: + + def test_l0_convention(self) -> None: + tier, unclassified = _assign_tier("acme-l0-extract") + assert tier == "L0" + assert not unclassified + + def test_l0_prefix_convention(self) -> None: + tier, unclassified = _assign_tier("l0-extract") + assert tier == "L0" + assert not unclassified + + def test_l1_convention(self) -> None: + tier, unclassified = _assign_tier("acme-l1-transform") + assert tier == "L1" + assert not unclassified + + def test_l1_prefix_convention(self) -> None: + tier, unclassified = _assign_tier("l1-transform") + assert tier == "L1" + assert not unclassified + + def test_l2_convention(self) -> None: + tier, unclassified = _assign_tier("acme-l2-delivery") + assert tier == "L2" + assert not unclassified + + def test_l2_prefix_convention(self) -> None: + tier, unclassified = _assign_tier("l2-delivery") + assert tier == "L2" + assert not unclassified + + def test_default_unclassified(self) -> None: + tier, unclassified = _assign_tier("my-project") + assert tier == "L0" + assert unclassified + + def test_tier_map_override(self) -> None: + tier, unclassified = _assign_tier("my-project", {"my-project": "L2"}) + assert tier == "L2" + assert not unclassified + + def test_tier_map_takes_precedence_over_convention(self) -> None: + tier, unclassified = _assign_tier("acme-l1-transform", {"acme-l1-transform": "L0"}) + assert tier == "L0" + assert not unclassified + + +# --------------------------------------------------------------------------- +# Pure function tests: _compute_job_stats +# --------------------------------------------------------------------------- + +class TestComputeJobStats: + + def test_empty(self) -> None: + stats = _compute_job_stats([]) + assert stats["total_jobs"] == 0 + assert stats["status_counts"] == {} + assert stats["success_rate_pct"] == 0 + assert stats["avg_duration_seconds"] == 0 + assert stats["date_range"]["earliest"] is None + assert stats["failing_configs"] == [] + + def test_mixed(self) -> None: + jobs = [ + {"status": "success", "durationSeconds": 10, "createdTime": "2025-01-01T00:00:00Z", + "component": "keboola.ex-db", "configId": "1"}, + {"status": "success", "durationSeconds": 20, "createdTime": "2025-01-02T00:00:00Z", + "component": "keboola.ex-db", "configId": "1"}, + {"status": "error", "durationSeconds": 5, "createdTime": "2025-01-03T00:00:00Z", + "component": "keboola.wr-db", "configId": "2"}, + ] + stats = _compute_job_stats(jobs) + assert stats["total_jobs"] == 3 + assert stats["status_counts"]["success"] == 2 + assert stats["status_counts"]["error"] == 1 + assert stats["success_rate_pct"] == pytest.approx(66.7, abs=0.1) + assert stats["avg_duration_seconds"] == pytest.approx(11.7, abs=0.1) + assert stats["date_range"]["earliest"] == "2025-01-01T00:00:00Z" + assert stats["date_range"]["latest"] == "2025-01-03T00:00:00Z" + + def test_failing_configs_sorted(self) -> None: + jobs = [ + {"status": "error", "component": "c1", "configId": "a", + "createdTime": "2025-01-01T00:00:00Z"}, + {"status": "success", "component": "c1", "configId": "a", + "createdTime": "2025-01-02T00:00:00Z"}, + {"status": "error", "component": "c2", "configId": "b", + "createdTime": "2025-01-01T00:00:00Z"}, + ] + stats = _compute_job_stats(jobs) + assert len(stats["failing_configs"]) == 2 + # c2/b has 100% error rate, c1/a has 50% + assert stats["failing_configs"][0]["config_key"] == "c2/b" + assert stats["failing_configs"][0]["error_rate_pct"] == 100.0 + assert stats["failing_configs"][1]["config_key"] == "c1/a" + assert stats["failing_configs"][1]["error_rate_pct"] == 50.0 + + +# --------------------------------------------------------------------------- +# Pure function tests: _type_icon +# --------------------------------------------------------------------------- + +class TestTypeIcon: + + def test_extractor(self) -> None: + assert _type_icon("keboola.ex-google-drive") == "EX" + + def test_writer(self) -> None: + assert _type_icon("keboola.wr-snowflake") == "WR" + + def test_transformation(self) -> None: + assert _type_icon("keboola.snowflake-transformation") == "TR" + + def test_snowflake_sql(self) -> None: + assert _type_icon("keboola.snowflake-sql") == "TR" + + def test_orchestrator(self) -> None: + assert _type_icon("keboola.orchestrator") == "OT" + + def test_application(self) -> None: + assert _type_icon("keboola.app-something") == "AP" + + +# --------------------------------------------------------------------------- +# Pure function tests: _build_mermaid +# --------------------------------------------------------------------------- + +class TestBuildMermaid: + + def test_basic_graph(self) -> None: + phases = [ + { + "id": 1, + "name": "Extract", + "depends_on": [], + "tasks": [ + {"name": "Pull data", "config_id": "100", "type_icon": "EX"}, + ], + }, + { + "id": 2, + "name": "Transform", + "depends_on": [1], + "tasks": [], + }, + ] + result = _build_mermaid(phases) + assert result.startswith("graph TD") + assert 'P1["Extract"]' in result + assert 'P2["Transform"]' in result + assert "P1 --> P2" in result + assert 'P1_100["EX Pull data"]' in result + + +# --------------------------------------------------------------------------- +# Service tests with mocked dependencies +# --------------------------------------------------------------------------- + +def _make_mock_services(alias: str = "prod"): + """Create mocked ConfigService, JobService, LineageService.""" + config_svc = MagicMock() + config_svc.list_configs.return_value = { + "configs": [ + { + "project_alias": alias, + "config_id": "1", + "config_name": "My Extractor", + "config_description": "", + "component_id": "keboola.ex-db", + "component_name": "DB Extractor", + "component_type": "extractor", + }, + ], + "errors": [], + } + config_svc.get_config_detail.return_value = { + "configuration": {"phases": []}, + "name": "My Extractor", + "description": "", + "isDisabled": False, + "version": 1, + } + + job_svc = MagicMock() + job_svc.list_jobs.return_value = { + "jobs": [ + { + "project_alias": alias, + "status": "success", + "durationSeconds": 10, + "createdTime": "2025-01-01T00:00:00Z", + "component": "keboola.ex-db", + "configId": "1", + }, + ], + "errors": [], + } + + lineage_svc = MagicMock() + lineage_svc.get_lineage.return_value = {"edges": [], "errors": []} + + return config_svc, job_svc, lineage_svc + + +class TestExplorerServiceGenerate: + + def test_generate_single_project(self, tmp_path: Path) -> None: + store = setup_single_project(tmp_path / "config", alias="l0-prod") + config_svc, job_svc, lineage_svc = _make_mock_services("l0-prod") + output_dir = tmp_path / "output" + + service = ExplorerService( + config_store=store, + config_service=config_svc, + job_service=job_svc, + lineage_service=lineage_svc, + ) + result = service.generate(output_dir=output_dir, open_browser=False) + + assert result["projects_count"] == 1 + assert result["configs_count"] == 1 + assert result["jobs_sampled"] == 1 + assert len(result["files_written"]) == 4 + + # Verify catalog structure + catalog = json.loads((output_dir / "catalog.json").read_text()) + assert "metadata" in catalog + assert "tiers" in catalog + assert "projects" in catalog + assert "lineage" in catalog + assert "l0-prod" in catalog["projects"] + assert catalog["projects"]["l0-prod"]["tier"] == "L0" + + def test_generate_writes_four_files(self, tmp_path: Path) -> None: + store = setup_single_project(tmp_path / "config", alias="l0-prod") + config_svc, job_svc, lineage_svc = _make_mock_services("l0-prod") + output_dir = tmp_path / "output" + + service = ExplorerService( + config_store=store, + config_service=config_svc, + job_service=job_svc, + lineage_service=lineage_svc, + ) + service.generate(output_dir=output_dir, open_browser=False) + + assert (output_dir / "catalog.json").exists() + assert (output_dir / "catalog.js").exists() + assert (output_dir / "orchestrations.json").exists() + assert (output_dir / "orchestrations.js").exists() + + # JS files wrap JSON in a variable assignment + js_content = (output_dir / "catalog.js").read_text() + assert js_content.startswith("const CATALOG = ") + + def test_generate_atomic_writes_no_leftover_tmp(self, tmp_path: Path) -> None: + store = setup_single_project(tmp_path / "config", alias="l0-prod") + config_svc, job_svc, lineage_svc = _make_mock_services("l0-prod") + output_dir = tmp_path / "output" + + service = ExplorerService( + config_store=store, + config_service=config_svc, + job_service=job_svc, + lineage_service=lineage_svc, + ) + service.generate(output_dir=output_dir, open_browser=False) + + # No .tmp files should remain after successful generation + tmp_files = list(output_dir.glob("*.tmp")) + assert tmp_files == [] + + def test_generate_tier_config_override(self, tmp_path: Path) -> None: + store = setup_single_project(tmp_path / "config", alias="my-project") + config_svc, job_svc, lineage_svc = _make_mock_services("my-project") + + output_dir = tmp_path / "output" + tiers_file = tmp_path / "tiers.yaml" + tiers_file.write_text( + "description: My ecosystem\n" + "tiers:\n" + " L0:\n" + " name: Sources\n" + " description: Data sources\n" + "projects:\n" + " my-project: L2\n" + ) + + service = ExplorerService( + config_store=store, + config_service=config_svc, + job_service=job_svc, + lineage_service=lineage_svc, + ) + result = service.generate( + output_dir=output_dir, open_browser=False, tiers_config=tiers_file, + ) + + catalog = json.loads((output_dir / "catalog.json").read_text()) + assert catalog["projects"]["my-project"]["tier"] == "L2" + assert catalog["metadata"]["description"] == "My ecosystem" + # No TIER_UNCLASSIFIED warnings + tier_warnings = [e for e in result["errors"] if e["error_code"] == "TIER_UNCLASSIFIED"] + assert tier_warnings == [] + + def test_generate_unclassified_warning(self, tmp_path: Path) -> None: + store = setup_single_project(tmp_path / "config", alias="my-project") + config_svc, job_svc, lineage_svc = _make_mock_services("my-project") + + output_dir = tmp_path / "output" + + service = ExplorerService( + config_store=store, + config_service=config_svc, + job_service=job_svc, + lineage_service=lineage_svc, + ) + result = service.generate(output_dir=output_dir, open_browser=False) + + tier_warnings = [e for e in result["errors"] if e["error_code"] == "TIER_UNCLASSIFIED"] + assert len(tier_warnings) == 1 + assert "my-project" in tier_warnings[0]["message"] + assert "defaulting to L0" in tier_warnings[0]["message"] + + def test_generate_schema_validation(self, tmp_path: Path) -> None: + store = setup_single_project(tmp_path / "config", alias="l0-prod") + config_svc, job_svc, lineage_svc = _make_mock_services("l0-prod") + output_dir = tmp_path / "output" + output_dir.mkdir(parents=True) + + # Copy schema into output dir + schema_src = Path(__file__).resolve().parent.parent / "kbc-explorer" / "schema.json" + if schema_src.exists(): + (output_dir / "schema.json").write_text(schema_src.read_text()) + + service = ExplorerService( + config_store=store, + config_service=config_svc, + job_service=job_svc, + lineage_service=lineage_svc, + ) + result = service.generate(output_dir=output_dir, open_browser=False) + + # Should pass validation (no SCHEMA_VALIDATION_ERROR) + schema_errors = [ + e for e in result["errors"] if e["error_code"] == "SCHEMA_VALIDATION_ERROR" + ] + assert schema_errors == [] + + def test_generate_schema_validation_catches_invalid(self, tmp_path: Path) -> None: + store = setup_single_project(tmp_path / "config", alias="l0-prod") + config_svc, job_svc, lineage_svc = _make_mock_services("l0-prod") + output_dir = tmp_path / "output" + output_dir.mkdir(parents=True) + + # Write a strict schema that will reject the catalog + strict_schema = { + "type": "object", + "required": ["metadata", "tiers", "projects", "lineage", "nonexistent_field"], + } + (output_dir / "schema.json").write_text(json.dumps(strict_schema)) + + service = ExplorerService( + config_store=store, + config_service=config_svc, + job_service=job_svc, + lineage_service=lineage_svc, + ) + result = service.generate(output_dir=output_dir, open_browser=False) + + schema_errors = [ + e for e in result["errors"] if e["error_code"] == "SCHEMA_VALIDATION_ERROR" + ] + assert len(schema_errors) == 1 + assert "nonexistent_field" in schema_errors[0]["message"] + + # Files should still be written despite validation failure + assert (output_dir / "catalog.json").exists() diff --git a/uv.lock b/uv.lock index aaf2ebdd..2ffef695 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" [[package]] @@ -279,9 +279,11 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "httpx" }, + { name = "jsonschema" }, { name = "mcp" }, { name = "platformdirs" }, { name = "pydantic" }, + { name = "pyyaml" }, { name = "rich" }, { name = "typer" }, ] @@ -297,9 +299,11 @@ dev = [ [package.metadata] requires-dist = [ { name = "httpx", specifier = ">=0.27" }, + { name = "jsonschema", specifier = ">=4.20" }, { name = "mcp", specifier = ">=1.0.0,<2.0.0" }, { name = "platformdirs", specifier = ">=4" }, { name = "pydantic", specifier = ">=2.5" }, + { name = "pyyaml", specifier = ">=6" }, { name = "rich", specifier = ">=13" }, { name = "typer", extras = ["all"], specifier = ">=0.12" }, ] @@ -593,6 +597,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/d2/21af5c535501a7233e734b8af901574572da66fcc254cb35d0609c9080dd/pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42", size = 8932540, upload-time = "2025-07-14T20:13:36.379Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + [[package]] name = "referencing" version = "0.37.0" From b7a90cc1b0f2f069a9ae137b39b1f3238c172b66 Mon Sep 17 00:00:00 2001 From: jordanrburger Date: Sat, 28 Feb 2026 18:12:56 -0500 Subject: [PATCH 2/2] Add explorer command to CLAUDE.md documentation Document explorer.py, explorer_service.py, test_explorer_service.py in project structure, CLI commands list, and dependencies. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index e29d8029..d4c24519 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,6 +51,7 @@ src/keboola_agent_cli/ lineage.py # LAYER 1: CLI commands for cross-project data lineage org.py # LAYER 1: CLI commands for organization bulk onboarding tool.py # LAYER 1: CLI commands for MCP tool list/call + explorer.py # LAYER 1: CLI commands for KBC Explorer dashboard generation context.py # LAYER 1: Agent usage instructions doctor.py # LAYER 1: Health check command services/ @@ -61,6 +62,7 @@ src/keboola_agent_cli/ lineage_service.py # LAYER 2: Cross-project lineage via bucket sharing org_service.py # LAYER 2: Organization setup orchestration mcp_service.py # LAYER 2: MCP tool integration (keboola-mcp-server wrapper) + explorer_service.py # LAYER 2: KBC Explorer catalog/orchestration generation tests/ conftest.py # Shared fixtures (tmp_config_dir, config_store, formatters) @@ -76,6 +78,7 @@ tests/ test_lineage_service.py # Lineage service tests test_mcp_service.py # MCP service tests test_org_service.py # Org service tests (slugify, setup, idempotency) + test_explorer_service.py # Explorer service tests (tier assignment, job stats, generation) test_integration.py # Integration tests (edge cases, linting) ``` @@ -126,7 +129,7 @@ Both share the same retry/backoff pattern (429/5xx, exponential backoff, 3 retri 9. **Tests**: use `typer.testing.CliRunner` for CLI tests, `unittest.mock` for mocking services and clients, `pytest` fixtures from `conftest.py`. -10. **Dependencies**: typer, rich, httpx, pydantic, platformdirs, mcp. Dev: pytest, pytest-httpx. +10. **Dependencies**: typer, rich, httpx, pydantic, platformdirs, mcp, jsonschema, pyyaml. Dev: pytest, pytest-httpx. 11. **Error accumulation**: multi-project operations collect per-project errors without stopping. One project failing doesn't block others (see `lineage_service.py`, `org_service.py`). @@ -156,6 +159,8 @@ kbagent org setup --org-id ID --url URL [--dry-run] [--yes] [--token-description kbagent tool list [--project NAME] kbagent tool call TOOL_NAME [--project NAME] [--input JSON] +kbagent explorer [--project NAME] [--output-dir DIR] [--job-limit N] [--tiers FILE] [--no-open] + kbagent context kbagent doctor ```