From d512c934af82baa4b6e3dc7b89118c38a9a3f249 Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Thu, 11 Jun 2026 13:01:00 -0700 Subject: [PATCH 1/3] Remove ASCII-art logo from CLI welcome screen Drop the `assembly` half-block wordmark from the bare-command welcome screen; `print_banner()` now emits only the one-line version + tagline header. Update the onboard tests to assert on the tagline header instead of the block glyphs. Co-Authored-By: Claude Opus 4.8 (1M context) --- aai_cli/output.py | 14 ++------------ tests/test_onboard_command.py | 13 ++++++------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/aai_cli/output.py b/aai_cli/output.py index c54fca11..96cc84ed 100644 --- a/aai_cli/output.py +++ b/aai_cli/output.py @@ -205,21 +205,13 @@ def emit_error(err: CLIError, *, json_mode: bool) -> None: error_console.print(f"[aai.muted]Suggestion:[/aai.muted] {escape(err.suggestion)}") -# The `assembly` wordmark for the bare-command welcome screen: a compact two-row -# half-block letterform, tinted to the brand accent (see theme.BRAND). Interior -# spaces are load-bearing (they separate the glyphs); trailing spaces are not, so -# they're dropped to survive whitespace-trimming tooling. -_BANNER = """\ -▄▀█ █▀ █▀ █▀▀ █▀▄▀█ █▄▄ █ █▄█ -█▀█ ▄█ ▄█ ██▄ █ ▀ █ █▄█ █▄▄ █""" - # A one-line header: emoji + product + version, then the product tagline. _TAGLINE = "AssemblyAI from your terminal" def print_banner() -> None: - """Print the welcome header — a version + tagline line, then the `assembly` wordmark in - the brand accent (the bare-command welcome screen).""" + """Print the welcome header — a single emoji + product + version + tagline line + in the brand accent (the bare-command welcome screen).""" # highlight=False so Rich's repr-highlighter doesn't recolor the version digits or # the quoted tagline — the line stays a single muted tone behind the brand label. console.print( @@ -227,8 +219,6 @@ def print_banner() -> None: f"[aai.muted]{__version__} — {_TAGLINE}[/aai.muted]", highlight=False, # pragma: no mutate (purely cosmetic: toggles Rich repr coloring, not text) ) - console.print() - console.print(Text(_BANNER, style="aai.brand")) def print_code(code: str, *, language: str = "python") -> None: diff --git a/tests/test_onboard_command.py b/tests/test_onboard_command.py index e54c2052..d6594112 100644 --- a/tests/test_onboard_command.py +++ b/tests/test_onboard_command.py @@ -149,22 +149,21 @@ def test_bare_aai_with_key_shows_help_no_offer(monkeypatch: pytest.MonkeyPatch) assert "Usage" in result.output or "Commands" in result.output -def test_bare_aai_prints_wordmark_banner(monkeypatch: pytest.MonkeyPatch) -> None: - # The welcome screen leads with the `assembly` wordmark; its block glyphs are present. +def test_bare_aai_prints_welcome_header(monkeypatch: pytest.MonkeyPatch) -> None: + # The welcome screen leads with the version + tagline header line. monkeypatch.setenv("ASSEMBLYAI_API_KEY", "sk_test") result = CliRunner().invoke(app, []) assert result.exit_code == 0, result.output - assert "▄▀█ █▀ █▀ █▀▀ █▀▄▀█ █▄▄ █ █▄█" in result.output - assert "█▀█ ▄█ ▄█ ██▄ █ ▀ █ █▄█ █▄▄ █" in result.output + assert "AssemblyAI from your terminal" in result.output def test_bare_aai_quiet_suppresses_banner(monkeypatch: pytest.MonkeyPatch) -> None: - # `--quiet` drops the decorative banner but still prints help. A mutant that - # ignores `quiet` (always banners) would leave the wordmark in the output. + # `--quiet` drops the decorative header but still prints help. A mutant that + # ignores `quiet` (always banners) would leave the tagline in the output. monkeypatch.setenv("ASSEMBLYAI_API_KEY", "sk_test") result = CliRunner().invoke(app, ["--quiet"]) assert result.exit_code == 0, result.output - assert "▄▀█ █▀ █▀" not in result.output + assert "AssemblyAI from your terminal" not in result.output assert "Usage" in result.output or "Commands" in result.output From 7de336ee4cf1645a8a4e7db469fd627f9fbf2684 Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Thu, 11 Jun 2026 13:09:46 -0700 Subject: [PATCH 2/3] Add Datadog dashboard for CLI usage telemetry Importable dashboard (docs/datadog/cli-usage-dashboard.json) + README, built on the source:aai-cli logs from aai_cli/telemetry.py: adoption, reliability/ errors, performance, and environment widgets. No template variables so it renders without facet setup; grouped widgets populate once facets exist. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/datadog/README.md | 41 + docs/datadog/cli-usage-dashboard.json | 1111 +++++++++++++++++++++++++ 2 files changed, 1152 insertions(+) create mode 100644 docs/datadog/README.md create mode 100644 docs/datadog/cli-usage-dashboard.json diff --git a/docs/datadog/README.md b/docs/datadog/README.md new file mode 100644 index 00000000..0becd13b --- /dev/null +++ b/docs/datadog/README.md @@ -0,0 +1,41 @@ +# Datadog dashboard — CLI usage & reliability + +`cli-usage-dashboard.json` is an importable Datadog dashboard for the `aai` CLI's +anonymous usage telemetry (`aai_cli/telemetry.py`). The CLI ships one log per +command run to the Datadog **Logs** intake (`source:aai-cli`) — it is Logs, **not** +RUM. Each log line = one command run. + +## Prerequisite: create the facets + +Log-analytics widgets only work on attributes that are **facets**. Before the +dashboard shows data, add these in **Logs → Configuration → Facets** (or expand any +`source:aai-cli` log and click each attribute → *Create facet*): + +- Facets: `@command`, `@outcome`, `@cli_version`, `@os`, `@python_version`, + `@exit_code`, `@ci`, `@device_id` +- Measure (numeric, not a plain facet): `@duration_ms` + +`@device_id` (unique-device counts) and `@duration_ms` (latency percentiles) are the +two that matter most. + +## Import + +UI: **Dashboards → New Dashboard → ⚙️ → Import dashboard JSON**, paste the file. + +API: + +```sh +curl -X POST "https://api.datadoghq.com/api/v1/dashboard" \ + -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY" \ + -H "Content-Type: application/json" \ + -d @docs/datadog/cli-usage-dashboard.json +``` + +## Notes + +- "Error rate" counts any `@outcome != success` — that includes expected user errors + (e.g. `auth_failure`), not just crashes. Filter to `@outcome:internal_error` for a + crash-only view. +- `@device_id` is a per-machine random UUID, so it approximates installs, not people. +- Events take ~1–2 minutes to index; a fresh `source:aai-cli` search can look empty + for a minute even when delivery succeeded. diff --git a/docs/datadog/cli-usage-dashboard.json b/docs/datadog/cli-usage-dashboard.json new file mode 100644 index 00000000..bdc20f4b --- /dev/null +++ b/docs/datadog/cli-usage-dashboard.json @@ -0,0 +1,1111 @@ +{ + "title": "AssemblyAI CLI — Usage & Reliability", + "description": "Product adoption, reliability, and performance for the `aai` CLI, built on the anonymous usage telemetry shipped to Datadog Logs (source:aai-cli). Each log line = one command run.", + "layout_type": "ordered", + "reflow_type": "fixed", + "notify_list": [], + "template_variables": [], + "widgets": [ + { + "definition": { + "type": "note", + "content": "# AssemblyAI CLI — Usage & Reliability\nBuilt on anonymous usage telemetry (`source:aai-cli`). **Each log line = one command run.** Count tiles work out of the box; the grouped tiles (top commands, outcomes, percentiles, distributions) need facets on `@command`, `@outcome`, `@device_id`, `@duration_ms` (measure), `@os`, `@cli_version`, `@python_version`, `@exit_code`, `@ci`. Events take ~1–2 min to index.", + "background_color": "purple", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 2 + } + }, + { + "definition": { + "type": "group", + "layout_type": "ordered", + "title": "Adoption & Usage", + "background_color": "vivid_blue", + "show_title": true, + "widgets": [ + { + "definition": { + "title": "Total command runs", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "runs", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [] + } + ], + "formulas": [ + { + "formula": "runs" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "definition": { + "title": "Unique devices (installs)", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "devices", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@device_id" + }, + "group_by": [] + } + ], + "formulas": [ + { + "formula": "devices" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 3, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "definition": { + "title": "Avg commands / device", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "runs", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [] + }, + { + "name": "devices", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@device_id" + }, + "group_by": [] + } + ], + "formulas": [ + { + "formula": "runs / devices" + } + ] + } + ], + "autoscale": true, + "precision": 1 + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "definition": { + "title": "Active CLI versions", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "versions", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@cli_version" + }, + "group_by": [] + } + ], + "formulas": [ + { + "formula": "versions" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 9, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "definition": { + "title": "Command runs over time", + "type": "timeseries", + "show_legend": false, + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "name": "runs", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [] + } + ], + "formulas": [ + { + "formula": "runs" + } + ], + "display_type": "bars", + "style": { + "palette": "cool", + "line_type": "solid", + "line_width": "normal" + } + } + ] + }, + "layout": { + "x": 0, + "y": 2, + "width": 6, + "height": 3 + } + }, + { + "definition": { + "title": "Unique devices over time", + "type": "timeseries", + "show_legend": false, + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "name": "devices", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@device_id" + }, + "group_by": [] + } + ], + "formulas": [ + { + "formula": "devices" + } + ], + "display_type": "line", + "style": { + "palette": "purple", + "line_type": "solid", + "line_width": "normal" + } + } + ] + }, + "layout": { + "x": 6, + "y": 2, + "width": 6, + "height": 3 + } + }, + { + "definition": { + "title": "Top commands", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "runs", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@command", + "limit": 15, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "runs" + } + ] + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 5, + "width": 6, + "height": 4 + } + }, + { + "definition": { + "title": "Version adoption over time", + "type": "timeseries", + "show_legend": true, + "legend_layout": "auto", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "name": "runs", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@cli_version", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "runs" + } + ], + "display_type": "area", + "style": { + "palette": "classic", + "line_type": "solid", + "line_width": "normal" + } + } + ] + }, + "layout": { + "x": 6, + "y": 5, + "width": 6, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 2, + "width": 12, + "height": 10 + } + }, + { + "definition": { + "type": "group", + "layout_type": "ordered", + "title": "Reliability & Errors", + "background_color": "vivid_red", + "show_title": true, + "widgets": [ + { + "definition": { + "title": "Error rate", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "errors", + "data_source": "logs", + "search": { + "query": "source:aai-cli -@outcome:success" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [] + }, + { + "name": "total", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [] + } + ], + "formulas": [ + { + "formula": "errors / total * 100" + } + ], + "conditional_formats": [ + { + "comparator": "<", + "value": 2, + "palette": "white_on_green" + }, + { + "comparator": "<", + "value": 5, + "palette": "white_on_yellow" + }, + { + "comparator": ">=", + "value": 5, + "palette": "white_on_red" + } + ] + } + ], + "autoscale": false, + "precision": 2, + "custom_unit": "%" + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "definition": { + "title": "Total errors", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "errors", + "data_source": "logs", + "search": { + "query": "source:aai-cli -@outcome:success" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [] + } + ], + "formulas": [ + { + "formula": "errors" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 2, + "width": 3, + "height": 2 + } + }, + { + "definition": { + "title": "Outcomes over time", + "type": "timeseries", + "show_legend": true, + "legend_layout": "auto", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "name": "runs", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@outcome", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "runs" + } + ], + "display_type": "bars", + "style": { + "palette": "classic", + "line_type": "solid", + "line_width": "normal" + } + } + ] + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 4 + } + }, + { + "definition": { + "title": "Top error types (@outcome)", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "errors", + "data_source": "logs", + "search": { + "query": "source:aai-cli -@outcome:success" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@outcome", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "errors" + } + ] + } + ] + }, + "layout": { + "x": 0, + "y": 4, + "width": 4, + "height": 4 + } + }, + { + "definition": { + "title": "Most error-prone commands", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "errors", + "data_source": "logs", + "search": { + "query": "source:aai-cli -@outcome:success" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@command", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "errors" + } + ] + } + ] + }, + "layout": { + "x": 4, + "y": 4, + "width": 4, + "height": 4 + } + }, + { + "definition": { + "title": "Exit codes", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "runs", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@exit_code", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "runs" + } + ] + } + ] + }, + "layout": { + "x": 8, + "y": 4, + "width": 4, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 12, + "width": 12, + "height": 9 + } + }, + { + "definition": { + "type": "group", + "layout_type": "ordered", + "title": "Performance", + "background_color": "vivid_orange", + "show_title": true, + "widgets": [ + { + "definition": { + "title": "Command duration (p50 / p90 / p95, ms)", + "type": "timeseries", + "show_legend": true, + "legend_layout": "auto", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "name": "p50", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "median", + "metric": "@duration_ms" + }, + "group_by": [] + }, + { + "name": "p90", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "pc90", + "metric": "@duration_ms" + }, + "group_by": [] + }, + { + "name": "p95", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "pc95", + "metric": "@duration_ms" + }, + "group_by": [] + } + ], + "formulas": [ + { + "formula": "p50", + "alias": "p50" + }, + { + "formula": "p90", + "alias": "p90" + }, + { + "formula": "p95", + "alias": "p95" + } + ], + "display_type": "line", + "style": { + "palette": "dog_classic", + "line_type": "solid", + "line_width": "normal" + } + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 8, + "height": 4 + } + }, + { + "definition": { + "title": "Slowest commands (p95 ms)", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "p95", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "pc95", + "metric": "@duration_ms" + }, + "group_by": [ + { + "facet": "@command", + "limit": 10, + "sort": { + "aggregation": "pc95", + "metric": "@duration_ms", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "p95" + } + ] + } + ] + }, + "layout": { + "x": 8, + "y": 0, + "width": 4, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 21, + "width": 12, + "height": 5 + } + }, + { + "definition": { + "type": "group", + "layout_type": "ordered", + "title": "Environment & Reach", + "background_color": "vivid_green", + "show_title": true, + "widgets": [ + { + "definition": { + "title": "Operating system", + "type": "sunburst", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "runs", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@os", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "runs" + } + ] + } + ], + "legend": { + "type": "table" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 4 + } + }, + { + "definition": { + "title": "CI vs interactive", + "type": "sunburst", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "runs", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@ci", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "runs" + } + ] + } + ], + "legend": { + "type": "table" + } + }, + "layout": { + "x": 3, + "y": 0, + "width": 3, + "height": 4 + } + }, + { + "definition": { + "title": "Python version", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "runs", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@python_version", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "runs" + } + ] + } + ] + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 4 + } + }, + { + "definition": { + "title": "CLI version distribution", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "runs", + "data_source": "logs", + "search": { + "query": "source:aai-cli" + }, + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "group_by": [ + { + "facet": "@cli_version", + "limit": 15, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "runs" + } + ] + } + ] + }, + "layout": { + "x": 9, + "y": 0, + "width": 3, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 26, + "width": 12, + "height": 5 + } + } + ] +} From a0dab42f32cd35dff422cef15a018a4d9526b108 Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Thu, 11 Jun 2026 13:09:46 -0700 Subject: [PATCH 3/3] Add JSON validity gate to check.sh scripts/json_lint.py parses every tracked + staged *.json (committed dashboards, init-template vercel.json, recorded API fixtures) so a malformed file fails the gate loudly instead of silently downstream. Validity only -- the recorded tests/fixtures/api/*.json snapshots are not reformatted. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/check.sh | 6 +++++ scripts/json_lint.py | 58 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 scripts/json_lint.py diff --git a/scripts/check.sh b/scripts/check.sh index a57bb5ab..6094697b 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -103,6 +103,12 @@ fi echo "==> markdownlint (docs/ is generated, so excluded)" markdownlint "**/*.md" --ignore docs --ignore node_modules --ignore .pytest_cache +echo "==> json validity (all tracked + staged *.json)" +# Parse every JSON file so a malformed dashboard / vercel.json / fixture fails here +# instead of silently downstream (a bad dashboard just won't import). Validity only — +# recorded tests/fixtures/api/*.json are snapshots and must not be reformatted. +uv run python scripts/json_lint.py + echo "==> prettier (init template JS/CSS)" # CI's runner has prettier on PATH; locally it's skipped with a notice if not # installed, matching how shellcheck/swiftlint self-skip above. diff --git a/scripts/json_lint.py b/scripts/json_lint.py new file mode 100644 index 00000000..b09c8986 --- /dev/null +++ b/scripts/json_lint.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +"""Validate that every JSON file in the repo is well-formed. + +A malformed JSON file (a committed Datadog dashboard, a vercel.json in an init +template, a recorded API fixture) fails silently downstream — a bad dashboard just +won't import, a broken fixture surfaces as a confusing test error. This gate parses +each tracked-or-staged ``*.json`` and fails loudly on the first one that won't parse. + +Scope is git-tracked plus untracked-not-ignored files, so a new JSON file is checked +before it is committed (matching how the gitleaks step scans the working tree). It is +a validity check, not a formatter: the recorded ``tests/fixtures/api/*.json`` are +regenerated snapshots and must not be reflowed. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +def json_files() -> list[str]: + """Tracked + untracked-not-ignored ``*.json`` paths, deduped and sorted.""" + paths: set[str] = set() + for extra in ([], ["--others", "--exclude-standard"]): + out = subprocess.run( + ["git", "ls-files", "-z", *extra, "*.json"], + capture_output=True, + text=True, + check=True, + ).stdout + paths.update(p for p in out.split("\0") if p) + return sorted(paths) + + +def main() -> int: + failures: list[str] = [] + files = json_files() + for path in files: + try: + with Path(path).open(encoding="utf-8") as handle: + json.load(handle) + except (OSError, json.JSONDecodeError) as err: + failures.append(f"{path}: {err}") + + if failures: + sys.stdout.write("Malformed JSON:\n") + for failure in failures: + sys.stdout.write(f" {failure}\n") + return 1 + + sys.stdout.write(f" {len(files)} JSON files OK\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())