diff --git a/.gitleaks.toml b/.gitleaks.toml index b591d675..d1da48c9 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -15,6 +15,11 @@ regexTarget = "match" regexes = [ '''sk_abcdef1234''', '''sk_zzzzzz9999''', + # The shipped telemetry credential (aai_cli/telemetry.py SHIPPED_CLIENT_TOKEN) is a + # Datadog *client token*: write-only and designed to be embedded in client apps, so + # committing it is deliberate (the Supabase-CLI model). Only this exact value is + # allowlisted -- any other real-looking token still fails the gate. + '''pub0d633113b9f7d22faff215fefaf30b43''', ] # `gitleaks dir` scans the working tree regardless of .gitignore, so high-entropy values # in a developer's gitignored `.claude/settings.local.json` (a personal Claude Code file diff --git a/.importlinter b/.importlinter index c8bdbae7..9a4fb79b 100644 --- a/.importlinter +++ b/.importlinter @@ -26,6 +26,7 @@ source_modules = aai_cli.render aai_cli.stdio aai_cli.streaming + aai_cli.telemetry aai_cli.theme aai_cli.transcribe_render aai_cli.ws @@ -51,6 +52,7 @@ modules = aai_cli.commands.setup aai_cli.commands.share aai_cli.commands.stream + aai_cli.commands.telemetry aai_cli.commands.transcribe aai_cli.commands.transcripts @@ -64,5 +66,6 @@ source_modules = aai_cli.environments aai_cli.errors aai_cli.llm + aai_cli.telemetry forbidden_modules = rich diff --git a/AGENTS.md b/AGENTS.md index 59239fe2..75c5f826 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -153,7 +153,7 @@ A Typer CLI. `aai_cli/main.py` builds the `app`, registers each command sub-app, ### Command layer -Each file in `aai_cli/commands/` is a Typer sub-app (`transcribe`, `stream`, `agent`, `speak`, `llm`, `transcripts`, `login` (login/logout/whoami), `doctor`, `init`, `dev`, `share`, `deploy`, `setup`, `onboard`, `account` (balance/usage/limits), `keys`, `sessions`, `audit`). Command bodies run through `context.run_command(ctx, fn, json=...)`, which maps any `CLIError` to clean stderr output + the error's exit code. Commands never print tracebacks for expected failures. +Each file in `aai_cli/commands/` is a Typer sub-app (`transcribe`, `stream`, `agent`, `speak`, `llm`, `transcripts`, `login` (login/logout/whoami), `doctor`, `init`, `dev`, `share`, `deploy`, `setup`, `onboard`, `account` (balance/usage/limits), `keys`, `sessions`, `audit`, `telemetry` (status/enable/disable)). Command bodies run through `context.run_command(ctx, fn, json=...)`, which maps any `CLIError` to clean stderr output + the error's exit code. Commands never print tracebacks for expected failures. ### Cross-cutting state (resolution order matters) @@ -171,6 +171,7 @@ Each file in `aai_cli/commands/` is a Typer sub-app (`transcribe`, `stream`, `ag - **`code_gen/`** — backs `--show-code` on `transcribe`/`stream`/`agent`: builds a ready-to-run Python SDK script from exactly the flags passed (no API key needed; generated code reads `ASSEMBLYAI_API_KEY`). - **`auth/`** — browser-assisted `assembly login` via AMS + **Stytch B2B OAuth discovery** (`discovery.py`, `flow.py`, `loopback.py`, `ams.py`). Not Stytch Connected Apps. - **`init/`** — scaffolds a self-contained FastAPI + HTML starter (`audio-transcription`/`live-captions`/`voice-agent` templates), optionally installs deps and opens the browser; writes the key to a git-ignored `.env`. +- **`telemetry.py`** — anonymous, opt-out usage telemetry (Supabase-CLI model): `context.run_command` wraps each command body in `telemetry.track(ctx.command_path)`, which dispatches one allow-listed event (command path, outcome/exit code, duration, version/OS — never args, paths, or account data) to the Datadog logs intake via a **detached flusher subprocess** (the hidden `assembly telemetry flush`), so commands never wait on telemetry. `SHIPPED_CLIENT_TOKEN` is a committed write-only Datadog *client* token (`pub…`, embeddable by design — never an API key; `AAI_TELEMETRY_CLIENT_TOKEN` overrides). The test suite blanks it via an autouse conftest fixture so no test ever spawns a real flusher. Opt-out: `AAI_TELEMETRY_DISABLED=1` / `DO_NOT_TRACK=1` / `assembly telemetry disable` (persisted as `telemetry_enabled` in config.toml, alongside the random `device_id`). Send-side failures are swallowed (`OSError`/`CLIError`) — telemetry must never break a command. - **`commands/setup.py`** — `assembly setup install/status/remove` wires a coding agent up to AssemblyAI by installing three artifacts: the `assemblyai-docs` docs MCP (via `claude mcp add`), the AssemblyAI skill (via `npx skills add`), and the bundled `aai-cli` skill (copied out of the wheel, no network). Missing `claude`/`npx` is reported and skipped, not an error. ## Conventions diff --git a/README.md b/README.md index 5142f08f..b363d797 100644 --- a/README.md +++ b/README.md @@ -248,6 +248,19 @@ op run -- assembly transcribe call.mp3 # …or wrap the whole In CI, set `ASSEMBLYAI_API_KEY` as a masked secret. `assembly logout` purges the keyring entry; `assembly whoami` / `assembly doctor` confirm the active source without printing the key. +## Telemetry + +`assembly` collects **anonymous** usage telemetry to help improve the CLI: the command name (never its arguments), outcome class and exit code, duration, CLI version, OS, Python version, whether it ran in CI, and a random install id. It never collects arguments, file paths or contents, transcripts, API keys, or account data — and delivery runs in a detached background process, so it never slows a command down. + +Opt out any time, persistently or per-environment: + +```sh +assembly telemetry disable # persisted on this machine (assembly telemetry status to inspect) +export AAI_TELEMETRY_DISABLED=1 # env kill-switch; the cross-tool DO_NOT_TRACK=1 also works +``` + +The ingestion credential in the source is a Datadog **client token** — the write-only, embeddable credential class (it can submit events, read nothing). No account secret ships with the CLI. + ## Account Self-Service These commands use your browser login session (run `assembly login`), not your API key: diff --git a/aai_cli/commands/telemetry.py b/aai_cli/commands/telemetry.py new file mode 100644 index 00000000..2c5ab0f8 --- /dev/null +++ b/aai_cli/commands/telemetry.py @@ -0,0 +1,126 @@ +"""`assembly telemetry` — inspect or change anonymous usage telemetry. + +The collection itself lives in ``aai_cli/telemetry.py``; this command is the +user-facing consent surface: see what would be sent and turn it off (or back +on) persistently. The env kill-switches (``AAI_TELEMETRY_DISABLED=1``, +``DO_NOT_TRACK=1``) always win over the persisted choice. +""" + +from __future__ import annotations + +import typer + +from aai_cli import config, options, output, telemetry +from aai_cli.context import AppState, run_command +from aai_cli.help_text import examples_epilog + +app = typer.Typer(help="Anonymous usage telemetry: status, enable, disable.") + + +def _consent_label() -> str: + return "granted" if telemetry.consent_granted() else "denied" + + +@app.command( + epilog=examples_epilog( + [ + ("Show whether telemetry is active", "assembly telemetry status"), + ("As JSON for scripting", "assembly telemetry status --json"), + ] + ) +) +def status( + ctx: typer.Context, + json_out: bool = options.json_option(), +) -> None: + """Show whether anonymous usage telemetry is active, and why.""" + + def body(_state: AppState, json_mode: bool) -> None: + data: dict[str, object] = { + "enabled": telemetry.is_enabled(), + "consent": _consent_label(), + "token_configured": bool(telemetry.client_token()), + } + + def render(d: dict[str, object]) -> object: + state_line = ( + output.success("Telemetry is enabled.") + if d["enabled"] + else output.muted("Telemetry is disabled.") + ) + detail = output.muted( + f"Consent: {d['consent']}. Intake token configured: " + f"{'yes' if d['token_configured'] else 'no'}." + ) + hint = output.hint( + "Opt out any time: 'assembly telemetry disable' or AAI_TELEMETRY_DISABLED=1." + ) + return output.stack(state_line, detail, hint) + + output.emit(data, render, json_mode=json_mode) + + run_command(ctx, body, json=json_out) + + +@app.command( + epilog=examples_epilog([("Re-enable telemetry", "assembly telemetry enable")]), +) +def enable( + ctx: typer.Context, + json_out: bool = options.json_option(), +) -> None: + """Re-enable anonymous usage telemetry for this machine.""" + + def body(_state: AppState, json_mode: bool) -> None: + config.set_telemetry_enabled(enabled=True) + output.emit( + {"telemetry_enabled": True}, + lambda _d: output.success("Telemetry enabled."), + json_mode=json_mode, + ) + + run_command(ctx, body, json=json_out) + + +@app.command( + hidden=True, + epilog=examples_epilog( + [ + ( + "Internal plumbing, spawned by the CLI itself", + "assembly telemetry flush ''", + ) + ] + ), +) +def flush( + payload: str = typer.Argument(..., help="Serialized telemetry payload (internal)."), +) -> None: + """Deliver one serialized telemetry event to the intake (internal). + + This is the detached flusher `telemetry.dispatch` spawns so user commands never + wait on the network — an explicit, reviewable entry point rather than inline + code. Hidden from help; runs with stdio discarded, so it neither needs nor + produces output. + """ + telemetry.flush_payload(payload) + + +@app.command( + epilog=examples_epilog([("Opt out of telemetry", "assembly telemetry disable")]), +) +def disable( + ctx: typer.Context, + json_out: bool = options.json_option(), +) -> None: + """Opt out of anonymous usage telemetry for this machine.""" + + def body(_state: AppState, json_mode: bool) -> None: + config.set_telemetry_enabled(enabled=False) + output.emit( + {"telemetry_enabled": False}, + lambda _d: output.success("Telemetry disabled."), + json_mode=json_mode, + ) + + run_command(ctx, body, json=json_out) diff --git a/aai_cli/config.py b/aai_cli/config.py index d4409fb8..d201ea8a 100644 --- a/aai_cli/config.py +++ b/aai_cli/config.py @@ -5,6 +5,7 @@ import re import tempfile import tomllib +import uuid from pathlib import Path import keyring @@ -44,6 +45,11 @@ class Config(BaseModel): active_profile: str | None = None profiles: dict[str, Profile] = Field(default_factory=dict) + # Telemetry state (see telemetry.py): a random anonymous install id, and the + # persisted opt-out. None means "never chosen", which the opt-out model reads + # as enabled — distinct from an explicit False written by `assembly telemetry disable`. + device_id: str | None = None + telemetry_enabled: bool | None = None class StoredSession(BaseModel): @@ -348,6 +354,29 @@ def persist_login( _dump(prior_cfg) +def get_device_id() -> str: + """A stable anonymous install id for telemetry: a random UUID minted locally on + first use and persisted in config.toml. Carries nothing derivable from the + machine or account.""" + cfg = _load() + if cfg.device_id is None: + cfg.device_id = str(uuid.uuid4()) + _dump(cfg) + return cfg.device_id + + +def get_telemetry_enabled() -> bool | None: + """The persisted telemetry choice: True/False if the user ran + `aai telemetry enable/disable`, None if they never chose.""" + return _load().telemetry_enabled + + +def set_telemetry_enabled(*, enabled: bool) -> None: + cfg = _load() + cfg.telemetry_enabled = enabled + _dump(cfg) + + def resolve_api_key(*, profile: str | None = None, api_key_flag: str | None = None) -> str: if api_key_flag is not None: if not api_key_flag: diff --git a/aai_cli/context.py b/aai_cli/context.py index a99e10f1..559ee034 100644 --- a/aai_cli/context.py +++ b/aai_cli/context.py @@ -9,7 +9,7 @@ import keyring.errors import typer -from aai_cli import config, environments, output +from aai_cli import config, environments, output, telemetry from aai_cli.auth import run_login_flow from aai_cli.environments import Environment from aai_cli.errors import APIError, CLIError, NotAuthenticated @@ -188,7 +188,10 @@ def run_command( state: AppState = ctx.obj json_mode = output.resolve_json(explicit=json) try: - fn(state, json_mode) + # Inside the try so telemetry sees the raw CLIError (and its error_type) + # before it's folded into a typer.Exit below. + with telemetry.track(ctx.command_path): + fn(state, json_mode) except NotAuthenticated as err: if not auto_login or not _should_auto_login(err): output.emit_error(err, json_mode=json_mode) diff --git a/aai_cli/main.py b/aai_cli/main.py index 3f8269bb..c55bfd12 100644 --- a/aai_cli/main.py +++ b/aai_cli/main.py @@ -36,6 +36,7 @@ share, speak, stream, + telemetry, transcribe, transcripts, ) @@ -66,6 +67,7 @@ # Setup & Tools — get set up & maintain "doctor", "setup", + "telemetry", # History — browse past work "transcripts", "sessions", @@ -326,6 +328,7 @@ def main( app.add_typer(deploy.app) app.add_typer(onboard.app) app.add_typer(setup.app, name="setup", rich_help_panel=help_panels.SETUP) +app.add_typer(telemetry.app, name="telemetry", rich_help_panel=help_panels.SETUP) app.add_typer(keys.app, name="keys", rich_help_panel=help_panels.ACCOUNT) diff --git a/aai_cli/telemetry.py b/aai_cli/telemetry.py new file mode 100644 index 00000000..809a7d29 --- /dev/null +++ b/aai_cli/telemetry.py @@ -0,0 +1,184 @@ +"""Anonymous usage telemetry, modeled on the Supabase CLI's design. + +One allow-listed event per command run (command path, outcome, duration — never +arguments, file paths, ids, or account data) is shipped to the Datadog logs +intake using a write-only *client* token (``pub…``), the credential class +Datadog designs to be embedded in client apps. ``SHIPPED_CLIENT_TOKEN`` carries +it (it is public by design — never put an API key there); +``AAI_TELEMETRY_CLIENT_TOKEN`` overrides it without a release. + +Telemetry is opt-out (``AAI_TELEMETRY_DISABLED=1``, the cross-tool +``DO_NOT_TRACK=1``, or ``assembly telemetry disable``) and must never slow down or +break the command it observes: delivery happens in a detached flusher process, +and every send-side failure is swallowed. +""" + +from __future__ import annotations + +import json +import os +import platform +import subprocess +import sys +import time +from collections.abc import Generator, Mapping +from contextlib import contextmanager + +import typer + +from aai_cli import __version__, config +from aai_cli.errors import CLIError + +ENV_DISABLED = "AAI_TELEMETRY_DISABLED" +ENV_DO_NOT_TRACK = "DO_NOT_TRACK" +ENV_CLIENT_TOKEN = "AAI_TELEMETRY_CLIENT_TOKEN" +ENV_INTAKE_URL = "AAI_TELEMETRY_INTAKE_URL" + +# A Datadog *client token*: the write-only, embeddable credential class (it can +# submit events but read nothing), committed deliberately — the same way PostHog +# `phc_` keys ship in open-source CLIs. An API key (account secret) must never +# appear here. Rotate in Datadog (Organization Settings → Client Tokens) if abused; +# AAI_TELEMETRY_CLIENT_TOKEN overrides without a release. +SHIPPED_CLIENT_TOKEN = "pub0d633113b9f7d22faff215fefaf30b43" + +# Datadog's client-token log intake (US1 site). Orgs on another site override via +# AAI_TELEMETRY_INTAKE_URL without needing a new CLI release. +DEFAULT_INTAKE_URL = "https://browser-intake-datadoghq.com/api/v2/logs" + +_SEND_TIMEOUT_SECONDS = 5.0 + + +def client_token() -> str: + """The write-only intake token: env override first, then the shipped one.""" + return os.environ.get(ENV_CLIENT_TOKEN) or SHIPPED_CLIENT_TOKEN + + +def intake_url() -> str: + return os.environ.get(ENV_INTAKE_URL) or DEFAULT_INTAKE_URL + + +def consent_granted() -> bool: + """Opt-out consent: the env kill-switches win, then the persisted choice. + + Every non-empty value disables — ``DO_NOT_TRACK`` is conventionally ``1`` but + tools commonly export ``true``, and treating those as "still tracking" would + betray the user's stated intent. + """ + if os.environ.get(ENV_DISABLED) or os.environ.get(ENV_DO_NOT_TRACK): + return False + return config.get_telemetry_enabled() is not False + + +def is_enabled() -> bool: + """Telemetry runs only with both a token to send with and consent to send.""" + return bool(client_token()) and consent_granted() + + +def build_event( + command: str, *, outcome: str, exit_code: int, duration_ms: int +) -> dict[str, object]: + """One invocation event, shaped for the Datadog logs intake. + + Every field is allow-listed and anonymous: the command *path* (never its + arguments or options), the outcome class, and coarse machine facts. The + device id is a random UUID minted locally — no account id, email, or + hostname ever rides along. + """ + return { + "ddsource": "aai-cli", + "service": "aai-cli", + "ddtags": f"version:{__version__}", + "message": f"{command} {outcome}", + "command": command, + "outcome": outcome, + "exit_code": exit_code, + "duration_ms": duration_ms, + "cli_version": __version__, + "os": platform.system().lower(), + "python_version": platform.python_version(), + "ci": bool(os.environ.get("CI")), + "device_id": config.get_device_id(), + } + + +def dispatch(event: Mapping[str, object]) -> None: + """Hand one event to a detached `assembly telemetry flush` process; return immediately. + + The child is the CLI's own (hidden) ``telemetry flush`` subcommand — an explicit, + reviewable entry point, the same shape as the Vercel CLI's ``telemetry flush``. + The payload travels via argv and carries only the event + the write-only public + token, so argv visibility is acceptable. Detaching (own session, stdio discarded) + is what keeps the user's command from ever waiting on the network; the child's + env disables telemetry so a flush can never spawn another flusher. + """ + payload = json.dumps({"url": intake_url(), "token": client_token(), "event": event}) + subprocess.Popen( + [sys.executable, "-m", "aai_cli", "telemetry", "flush", payload], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + env={**os.environ, ENV_DISABLED: "1"}, + ) + + +def flush_payload(raw: str) -> None: + """POST one serialized payload to the intake. + + The token rides both as the ``DD-API-KEY`` header (the v2 logs API form) and + the ``dd-api-key`` query param (the browser-intake form the client-token + endpoints expect), so either intake host accepts it. Runs in the detached + flusher (`assembly telemetry flush`) with stdio discarded, so failures need no + handling here. + """ + import httpx2 as httpx + + payload = json.loads(raw) + token = str(payload["token"]) + with httpx.Client(timeout=_SEND_TIMEOUT_SECONDS) as client: + client.post( + str(payload["url"]), + params={"dd-api-key": token}, + headers={"DD-API-KEY": token}, + json=[payload["event"]], + ) + + +def _safe_dispatch(command: str, started: float, *, outcome: str, exit_code: int) -> None: + duration_ms = int((time.monotonic() - started) * 1000) + try: + dispatch( + build_event(command, outcome=outcome, exit_code=exit_code, duration_ms=duration_ms) + ) + except (OSError, CLIError): + # Best-effort by contract: a config/spawn failure while *recording* a command + # must never surface in the command itself. + return + + +@contextmanager +def track(command: str) -> Generator[None]: + """Record one command run, deriving the outcome from whatever escapes the body. + + CLIErrors keep their machine-readable ``error_type`` as the outcome; a + deliberate ``typer.Exit`` maps through its code; anything else is the + catch-all ``internal_error``. The body's exception always re-raises — + tracking observes control flow, never alters it. + """ + if not is_enabled(): + yield + return + started = time.monotonic() + try: + yield + except CLIError as err: + _safe_dispatch(command, started, outcome=err.error_type, exit_code=err.exit_code) + raise + except typer.Exit as exc: + code = exc.exit_code + outcome = "success" if code == 0 else "error" + _safe_dispatch(command, started, outcome=outcome, exit_code=code) + raise + except BaseException: + _safe_dispatch(command, started, outcome="internal_error", exit_code=1) + raise + _safe_dispatch(command, started, outcome="success", exit_code=0) diff --git a/pyproject.toml b/pyproject.toml index 52d32bc3..a0717c9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -245,6 +245,9 @@ max-statements = 40 "aai_cli/auth/loopback.py" = ["A002"] # Template constants include URL path names such as TOKEN_PATH, not credentials. "aai_cli/init/templates/**" = ["S105"] +# ENV_CLIENT_TOKEN holds an env-var *name*; the shipped token constant is empty in +# source (release builds inject the write-only client token). +"aai_cli/telemetry.py" = ["S105"] [tool.vulture] paths = ["aai_cli", "tests"] diff --git a/tests/__snapshots__/test_cli_output_snapshots.ambr b/tests/__snapshots__/test_cli_output_snapshots.ambr index d1250f87..63bd59b8 100644 --- a/tests/__snapshots__/test_cli_output_snapshots.ambr +++ b/tests/__snapshots__/test_cli_output_snapshots.ambr @@ -836,6 +836,96 @@ + ''' +# --- +# name: test_command_help_matches_snapshot[telemetry_disable] + ''' + + Usage: assembly telemetry disable [OPTIONS] + + Opt out of anonymous usage telemetry for this machine. + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Opt out of telemetry + $ assembly telemetry disable + + + + ''' +# --- +# name: test_command_help_matches_snapshot[telemetry_enable] + ''' + + Usage: assembly telemetry enable [OPTIONS] + + Re-enable anonymous usage telemetry for this machine. + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Re-enable telemetry + $ assembly telemetry enable + + + + ''' +# --- +# name: test_command_help_matches_snapshot[telemetry_flush] + ''' + + Usage: assembly telemetry flush [OPTIONS] PAYLOAD + + Deliver one serialized telemetry event to the intake (internal). + + This is the detached flusher `telemetry.dispatch` spawns so user commands + never + wait on the network — an explicit, reviewable entry point rather than inline + code. Hidden from help; runs with stdio discarded, so it neither needs nor + produces output. + + ╭─ Arguments ──────────────────────────────────────────────────────────────────╮ + │ * payload TEXT Serialized telemetry payload (internal). [required] │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Internal plumbing, spawned by the CLI itself + $ assembly telemetry flush '' + + + + ''' +# --- +# name: test_command_help_matches_snapshot[telemetry_status] + ''' + + Usage: assembly telemetry status [OPTIONS] + + Show whether anonymous usage telemetry is active, and why. + + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --json -j Output raw JSON. │ + │ --help Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + Examples + Show whether telemetry is active + $ assembly telemetry status + As JSON for scripting + $ assembly telemetry status --json + + + ''' # --- # name: test_command_help_matches_snapshot[transcribe] diff --git a/tests/conftest.py b/tests/conftest.py index 05d23408..b33d79b3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,6 +64,12 @@ def isolate_env(monkeypatch): "CLAUDECODE", "CLAUDE_CODE_ENTRYPOINT", "NO_COLOR", + # With these cleared (and SHIPPED_CLIENT_TOKEN empty in source), telemetry + # is inert in every test unless one opts in explicitly. + "AAI_TELEMETRY_CLIENT_TOKEN", + "AAI_TELEMETRY_INTAKE_URL", + "AAI_TELEMETRY_DISABLED", + "DO_NOT_TRACK", ): monkeypatch.delenv(var, raising=False) @@ -97,6 +103,20 @@ def memory_keyring(): return backend +@pytest.fixture(autouse=True) +def neutralize_shipped_token(monkeypatch): + # The shipped Datadog client token makes telemetry live by default. In-process + # patches (pytest-socket, mocked boundaries) never reach the detached flusher + # *subprocess* telemetry spawns, so blank the token suite-wide: tests exercise + # telemetry by opting in via AAI_TELEMETRY_CLIENT_TOKEN and patching dispatch. + # Returns the real shipped value so its own tests can still assert its shape. + from aai_cli import telemetry + + original = telemetry.SHIPPED_CLIENT_TOKEN + monkeypatch.setattr(telemetry, "SHIPPED_CLIENT_TOKEN", "") + return original + + @pytest.fixture(autouse=True) def tmp_config(monkeypatch, tmp_path): cfg_dir = tmp_path / "config" diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 010656c5..5d287ea6 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -107,6 +107,7 @@ def test_help_lists_commands_in_workflow_order(): # Setup & Tools "doctor", "setup", + "telemetry", # History "transcripts", "sessions", diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 00000000..2b590cd3 --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,299 @@ +"""Unit tests for the telemetry core (aai_cli/telemetry.py) and its config state.""" + +import json +import subprocess +import sys +import time +import uuid + +import pytest +import typer + +from aai_cli import config, telemetry +from aai_cli.errors import CLIError, UsageError + +# --- token / url resolution ------------------------------------------------- + + +def test_shipped_token_is_a_write_only_client_token(neutralize_shipped_token): + # The committed credential must be a Datadog *client* token (pub…, write-only, + # embeddable by design) — never an API key. The autouse fixture blanks it for + # the suite and hands back the real value for exactly this assertion. + assert neutralize_shipped_token == "pub0d633113b9f7d22faff215fefaf30b43" + # Blanked in-suite, so nothing else accidentally goes live: + assert telemetry.client_token() == "" + + +def test_client_token_env_overrides_shipped(monkeypatch): + monkeypatch.setattr(telemetry, "SHIPPED_CLIENT_TOKEN", "pub_shipped") + assert telemetry.client_token() == "pub_shipped" + monkeypatch.setenv(telemetry.ENV_CLIENT_TOKEN, "pub_env") + assert telemetry.client_token() == "pub_env" + + +def test_intake_url_default_and_override(monkeypatch): + assert telemetry.intake_url() == "https://browser-intake-datadoghq.com/api/v2/logs" + monkeypatch.setenv(telemetry.ENV_INTAKE_URL, "https://example.test/logs") + assert telemetry.intake_url() == "https://example.test/logs" + + +# --- consent ------------------------------------------------------------------ + + +def test_consent_granted_by_default(): + assert telemetry.consent_granted() is True + + +@pytest.mark.parametrize("var", ["AAI_TELEMETRY_DISABLED", "DO_NOT_TRACK"]) +@pytest.mark.parametrize("value", ["1", "true"]) +def test_consent_env_kill_switches(monkeypatch, var, value): + monkeypatch.setenv(var, value) + assert telemetry.consent_granted() is False + + +def test_consent_follows_persisted_choice(): + config.set_telemetry_enabled(enabled=False) + assert telemetry.consent_granted() is False + config.set_telemetry_enabled(enabled=True) + assert telemetry.consent_granted() is True + + +def test_consent_env_wins_over_persisted_enable(monkeypatch): + config.set_telemetry_enabled(enabled=True) + monkeypatch.setenv(telemetry.ENV_DISABLED, "1") + assert telemetry.consent_granted() is False + + +def test_is_enabled_requires_token_and_consent(monkeypatch): + assert telemetry.is_enabled() is False # consent alone is not enough + monkeypatch.setenv(telemetry.ENV_CLIENT_TOKEN, "pub_test") + assert telemetry.is_enabled() is True + config.set_telemetry_enabled(enabled=False) + assert telemetry.is_enabled() is False + + +# --- config-backed telemetry state ------------------------------------------- + + +def test_telemetry_enabled_roundtrip(): + assert config.get_telemetry_enabled() is None + config.set_telemetry_enabled(enabled=False) + assert config.get_telemetry_enabled() is False + config.set_telemetry_enabled(enabled=True) + assert config.get_telemetry_enabled() is True + + +def test_device_id_is_a_stable_random_uuid(tmp_config): + first = config.get_device_id() + # A parseable UUID, not something derived from the machine or account. + assert str(uuid.UUID(first)) == first + assert config.get_device_id() == first + # Persisted: a fresh load (new process semantics) sees the same id. + assert f'device_id = "{first}"' in (tmp_config / "config.toml").read_text() + + +def test_device_id_does_not_clobber_other_settings(): + config.set_profile_env("default", "production") + config.get_device_id() + assert config.get_profile_env("default") == "production" + + +# --- event shape ---------------------------------------------------------------- + + +def test_build_event_is_allowlisted_and_exact(monkeypatch): + import platform + + monkeypatch.setattr(platform, "system", lambda: "Linux") + monkeypatch.setattr(platform, "python_version", lambda: "3.12.9") + event = telemetry.build_event("aai transcribe", outcome="success", exit_code=0, duration_ms=250) + from aai_cli import __version__ + + assert event == { + "ddsource": "aai-cli", + "service": "aai-cli", + "ddtags": f"version:{__version__}", + "message": "aai transcribe success", + "command": "aai transcribe", + "outcome": "success", + "exit_code": 0, + "duration_ms": 250, + "cli_version": __version__, + "os": "linux", + "python_version": "3.12.9", + "ci": False, + "device_id": config.get_device_id(), + } + + +def test_build_event_marks_ci(monkeypatch): + monkeypatch.setenv("CI", "true") + event = telemetry.build_event("aai stream", outcome="api_error", exit_code=1, duration_ms=5) + assert event["ci"] is True + assert event["outcome"] == "api_error" + assert event["exit_code"] == 1 + + +# --- dispatch (detached flusher handoff) ------------------------------------ + + +def test_dispatch_spawns_detached_flusher(monkeypatch): + calls = {} + + def fake_popen(argv, **kwargs): + calls["argv"], calls["kwargs"] = argv, kwargs + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + monkeypatch.setenv(telemetry.ENV_CLIENT_TOKEN, "pub_test") + telemetry.dispatch({"command": "aai doctor"}) + + # The child is the CLI's own (hidden) `telemetry flush` subcommand — explicit + # plumbing, not an inline -c snippet. + assert calls["argv"][:5] == [sys.executable, "-m", "aai_cli", "telemetry", "flush"] + payload = json.loads(calls["argv"][5]) + assert payload == { + "url": "https://browser-intake-datadoghq.com/api/v2/logs", + "token": "pub_test", + "event": {"command": "aai doctor"}, + } + # Detached with stdio discarded: the flusher can never block or pollute the + # command. Telemetry is disabled in the child so a flush never spawns a flusher. + kwargs = calls["kwargs"] + assert kwargs["stdout"] is subprocess.DEVNULL + assert kwargs["stderr"] is subprocess.DEVNULL + assert kwargs["start_new_session"] is True + assert kwargs["env"]["AAI_TELEMETRY_DISABLED"] == "1" + assert sorted(kwargs) == ["env", "start_new_session", "stderr", "stdout"] + + +# --- flusher ------------------------------------------------------------------ + + +class _FakeClient: + def __init__(self, record): + self._record = record + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def post(self, url, **kwargs): + self._record["url"] = url + self._record["kwargs"] = kwargs + + +def test_flush_payload_posts_to_intake(monkeypatch): + record = {} + + def fake_client_factory(*, timeout): + record["timeout"] = timeout + return _FakeClient(record) + + monkeypatch.setattr("httpx2.Client", fake_client_factory) + raw = json.dumps( + {"url": "https://example.test/logs", "token": "pub_x", "event": {"command": "aai llm"}} + ) + telemetry.flush_payload(raw) + + assert record["timeout"] == 5.0 + assert record["url"] == "https://example.test/logs" + assert record["kwargs"] == { + "params": {"dd-api-key": "pub_x"}, + "headers": {"DD-API-KEY": "pub_x"}, + "json": [{"command": "aai llm"}], + } + + +def test_flush_command_delivers_payload(monkeypatch): + # The hidden `assembly telemetry flush` subcommand is what dispatch() spawns; drive it + # through the real CLI so the spawned argv is known to be invocable end to end. + from typer.testing import CliRunner + + from aai_cli.main import app + + seen = [] + monkeypatch.setattr(telemetry, "flush_payload", seen.append) + result = CliRunner().invoke(app, ["telemetry", "flush", '{"the": "payload"}']) + assert result.exit_code == 0 + assert seen == ['{"the": "payload"}'] + + +# --- track -------------------------------------------------------------------- + + +@pytest.fixture +def events(monkeypatch): + """Enable telemetry and capture dispatched events instead of spawning flushers.""" + monkeypatch.setenv(telemetry.ENV_CLIENT_TOKEN, "pub_test") + captured = [] + monkeypatch.setattr(telemetry, "dispatch", captured.append) + return captured + + +def _freeze_duration(monkeypatch, seconds): + ticks = iter([100.0, 100.0 + seconds]) + monkeypatch.setattr(time, "monotonic", lambda: next(ticks)) + + +def test_track_disabled_dispatches_nothing(monkeypatch): + captured = [] + monkeypatch.setattr(telemetry, "dispatch", captured.append) + ran = [] + with telemetry.track("aai doctor"): + ran.append(True) + assert ran == [True] + assert captured == [] + + +def test_track_success(events, monkeypatch): + # 2.0s, not a sub-second value: 2.0 * 1000 = 2000 also catches an off-by-one + # mutation of the ms factor, which int(0.25 * 1001) would round away. + _freeze_duration(monkeypatch, 2.0) + with telemetry.track("aai doctor"): + pass + (event,) = events + assert event["command"] == "aai doctor" + assert event["outcome"] == "success" + assert event["exit_code"] == 0 + assert event["duration_ms"] == 2000 + + +def test_track_cli_error_keeps_error_type_and_reraises(events): + with pytest.raises(UsageError), telemetry.track("aai transcribe"): + raise UsageError("bad flag") + (event,) = events + assert event["outcome"] == "usage_error" + assert event["exit_code"] == 2 + + +@pytest.mark.parametrize( + ("code", "outcome"), [(0, "success"), (3, "error")], ids=["exit-0", "exit-3"] +) +def test_track_typer_exit_maps_code(events, code, outcome): + with pytest.raises(typer.Exit), telemetry.track("aai login"): + raise typer.Exit(code=code) + (event,) = events + assert event["outcome"] == outcome + assert event["exit_code"] == code + + +def test_track_unexpected_exception_is_internal_error(events): + with pytest.raises(RuntimeError), telemetry.track("aai stream"): + raise RuntimeError("boom") + (event,) = events + assert event["outcome"] == "internal_error" + assert event["exit_code"] == 1 + + +@pytest.mark.parametrize("exc", [OSError("spawn failed"), CLIError("corrupt config")]) +def test_track_send_failures_never_break_the_command(monkeypatch, exc): + monkeypatch.setenv(telemetry.ENV_CLIENT_TOKEN, "pub_test") + + def explode(event): + raise exc + + monkeypatch.setattr(telemetry, "dispatch", explode) + with telemetry.track("aai doctor"): + pass # must not raise diff --git a/tests/test_telemetry_command.py b/tests/test_telemetry_command.py new file mode 100644 index 00000000..1cf323a0 --- /dev/null +++ b/tests/test_telemetry_command.py @@ -0,0 +1,138 @@ +"""Tests for `assembly telemetry status/enable/disable` and the run_command integration.""" + +import json + +from typer.testing import CliRunner + +from aai_cli import config, telemetry +from aai_cli.main import app + +runner = CliRunner() + + +def _human(monkeypatch): + monkeypatch.setattr("aai_cli.output.resolve_json", lambda *, explicit: explicit) + + +def _capture_events(monkeypatch, *, token="pub_test"): + monkeypatch.setenv(telemetry.ENV_CLIENT_TOKEN, token) + captured = [] + monkeypatch.setattr(telemetry, "dispatch", captured.append) + return captured + + +def test_status_json_when_inert(): + result = runner.invoke(app, ["telemetry", "status", "--json"]) + assert result.exit_code == 0 + assert json.loads(result.output) == { + "enabled": False, + "consent": "granted", + "token_configured": False, + } + + +def test_status_json_when_enabled(monkeypatch): + _capture_events(monkeypatch) + result = runner.invoke(app, ["telemetry", "status", "--json"]) + assert result.exit_code == 0 + assert json.loads(result.output) == { + "enabled": True, + "consent": "granted", + "token_configured": True, + } + + +def test_status_json_when_opted_out(monkeypatch): + _capture_events(monkeypatch) + config.set_telemetry_enabled(enabled=False) + result = runner.invoke(app, ["telemetry", "status", "--json"]) + assert result.exit_code == 0 + assert json.loads(result.output) == { + "enabled": False, + "consent": "denied", + "token_configured": True, + } + + +def test_status_human_disabled(monkeypatch): + _human(monkeypatch) + result = runner.invoke(app, ["telemetry", "status"]) + assert result.exit_code == 0 + assert "Telemetry is disabled." in result.output + assert "Consent: granted. Intake token configured: no." in result.output + assert "assembly telemetry disable" in result.output + + +def test_status_human_enabled(monkeypatch): + _human(monkeypatch) + _capture_events(monkeypatch) + result = runner.invoke(app, ["telemetry", "status"]) + assert result.exit_code == 0 + assert "Telemetry is enabled." in result.output + assert "Consent: granted. Intake token configured: yes." in result.output + + +def test_disable_persists_and_confirms_json(): + result = runner.invoke(app, ["telemetry", "disable", "--json"]) + assert result.exit_code == 0 + assert json.loads(result.output) == {"telemetry_enabled": False} + assert config.get_telemetry_enabled() is False + + +def test_enable_persists_and_confirms_json(): + config.set_telemetry_enabled(enabled=False) + result = runner.invoke(app, ["telemetry", "enable", "--json"]) + assert result.exit_code == 0 + assert json.loads(result.output) == {"telemetry_enabled": True} + assert config.get_telemetry_enabled() is True + + +def test_enable_and_disable_human(monkeypatch): + _human(monkeypatch) + result = runner.invoke(app, ["telemetry", "disable"]) + assert result.exit_code == 0 + assert "Telemetry disabled." in result.output + result = runner.invoke(app, ["telemetry", "enable"]) + assert result.exit_code == 0 + assert "Telemetry enabled." in result.output + + +def test_flush_is_hidden_plumbing(): + # `flush` exists for dispatch() to spawn, but users shouldn't be steered to it. + result = runner.invoke(app, ["telemetry", "--help"]) + assert result.exit_code == 0 + assert "status" in result.output + assert "flush" not in result.output + + +# --- run_command integration ------------------------------------------------- + + +def test_command_run_is_tracked_with_full_command_path(monkeypatch): + captured = _capture_events(monkeypatch) + result = runner.invoke(app, ["telemetry", "status", "--json"]) + assert result.exit_code == 0 + (event,) = captured + assert event["command"] == "assembly telemetry status" + assert event["outcome"] == "success" + assert event["exit_code"] == 0 + + +def test_failed_command_is_tracked_with_error_type(monkeypatch): + # No stored session and a non-interactive runner: `assembly balance` fails with + # NotAuthenticated before any network call, and telemetry records that class. + captured = _capture_events(monkeypatch) + result = runner.invoke(app, ["balance", "--json"]) + assert result.exit_code == 4 + (event,) = captured + assert event["command"] == "assembly balance" + assert event["outcome"] == "not_authenticated" + assert event["exit_code"] == 4 + + +def test_inert_telemetry_tracks_nothing(monkeypatch): + captured = [] + monkeypatch.setattr(telemetry, "dispatch", captured.append) + result = runner.invoke(app, ["telemetry", "status", "--json"]) + assert result.exit_code == 0 + assert captured == []