From 249a93253840e9c891b6a6fc3b1de6f3620abfa5 Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Fri, 5 Jun 2026 12:04:47 -0700 Subject: [PATCH 1/5] Add macOS system audio streaming --- CLAUDE.md | 2 +- README.md | 14 +- aai_cli/commands/init.py | 13 +- aai_cli/commands/stream.py | 224 +++++++--- aai_cli/init/templates.py | 13 +- aai_cli/init/templates/agent/index.html | 23 -- .../init/templates/agent/static/styles.css | 42 -- .../AGENTS.md | 6 +- .../README.md | 0 .../api/__init__.py | 0 .../api/index.py | 0 .../api/settings.py | 0 .../env.example | 0 .../{agent => audio-transcription}/gitignore | 0 .../templates/audio-transcription/index.html | 45 +++ .../requirements.txt | 0 .../static/app.js | 31 +- .../audio-transcription/static/styles.css | 381 ++++++++++++++++++ .../vercel.json | 0 .../{stream => live-captions}/AGENTS.md | 6 +- .../{stream => live-captions}/README.md | 0 .../{stream => live-captions}/api/__init__.py | 0 .../{stream => live-captions}/api/index.py | 0 .../{stream => live-captions}/api/settings.py | 0 .../{stream => live-captions}/env.example | 0 .../{stream => live-captions}/gitignore | 0 .../init/templates/live-captions/index.html | 29 ++ .../{agent => live-captions}/requirements.txt | 0 .../{stream => live-captions}/static/app.js | 6 +- .../{stream => live-captions}/static/audio.js | 0 .../templates/live-captions/static/styles.css | 179 ++++++++ .../{stream => live-captions}/vercel.json | 0 aai_cli/init/templates/stream/index.html | 23 -- .../init/templates/stream/static/styles.css | 48 --- aai_cli/init/templates/transcribe/index.html | 43 -- .../templates/transcribe/static/styles.css | 117 ------ .../{agent => voice-agent}/AGENTS.md | 6 +- .../{agent => voice-agent}/README.md | 0 .../api/__init__.py | 0 .../{agent => voice-agent}/api/index.py | 0 .../{agent => voice-agent}/api/settings.py | 0 .../{agent => voice-agent}/env.example | 0 .../{transcribe => voice-agent}/gitignore | 0 aai_cli/init/templates/voice-agent/index.html | 27 ++ .../{stream => voice-agent}/requirements.txt | 0 .../{agent => voice-agent}/static/app.js | 13 +- .../{agent => voice-agent}/static/audio.js | 0 .../templates/voice-agent/static/styles.css | 186 +++++++++ .../{transcribe => voice-agent}/vercel.json | 0 aai_cli/streaming/macos.py | 254 ++++++++++++ aai_cli/streaming/macos_system_audio.swift | 347 ++++++++++++++++ aai_cli/streaming/render.py | 134 ++++-- pyproject.toml | 2 +- scripts/check.sh | 33 ++ scripts/template_contract_gate.py | 2 +- .../test_cli_output_snapshots.ambr | 18 +- tests/test_init_command.py | 39 +- tests/test_init_scaffold.py | 12 +- tests/test_init_template_agent.py | 2 +- tests/test_init_template_contract.py | 2 +- tests/test_init_template_stream.py | 2 +- tests/test_init_template_transcribe.py | 2 +- tests/test_init_templates.py | 10 +- tests/test_macos_audio_source.py | 245 +++++++++++ tests/test_stream_command.py | 143 +++++++ tests/test_streaming_render.py | 28 ++ 66 files changed, 2290 insertions(+), 462 deletions(-) delete mode 100644 aai_cli/init/templates/agent/index.html delete mode 100644 aai_cli/init/templates/agent/static/styles.css rename aai_cli/init/templates/{transcribe => audio-transcription}/AGENTS.md (71%) rename aai_cli/init/templates/{transcribe => audio-transcription}/README.md (100%) rename aai_cli/init/templates/{agent => audio-transcription}/api/__init__.py (100%) rename aai_cli/init/templates/{transcribe => audio-transcription}/api/index.py (100%) rename aai_cli/init/templates/{transcribe => audio-transcription}/api/settings.py (100%) rename aai_cli/init/templates/{transcribe => audio-transcription}/env.example (100%) rename aai_cli/init/templates/{agent => audio-transcription}/gitignore (100%) create mode 100644 aai_cli/init/templates/audio-transcription/index.html rename aai_cli/init/templates/{transcribe => audio-transcription}/requirements.txt (100%) rename aai_cli/init/templates/{transcribe => audio-transcription}/static/app.js (83%) create mode 100644 aai_cli/init/templates/audio-transcription/static/styles.css rename aai_cli/init/templates/{agent => audio-transcription}/vercel.json (100%) rename aai_cli/init/templates/{stream => live-captions}/AGENTS.md (73%) rename aai_cli/init/templates/{stream => live-captions}/README.md (100%) rename aai_cli/init/templates/{stream => live-captions}/api/__init__.py (100%) rename aai_cli/init/templates/{stream => live-captions}/api/index.py (100%) rename aai_cli/init/templates/{stream => live-captions}/api/settings.py (100%) rename aai_cli/init/templates/{stream => live-captions}/env.example (100%) rename aai_cli/init/templates/{stream => live-captions}/gitignore (100%) create mode 100644 aai_cli/init/templates/live-captions/index.html rename aai_cli/init/templates/{agent => live-captions}/requirements.txt (100%) rename aai_cli/init/templates/{stream => live-captions}/static/app.js (95%) rename aai_cli/init/templates/{stream => live-captions}/static/audio.js (100%) create mode 100644 aai_cli/init/templates/live-captions/static/styles.css rename aai_cli/init/templates/{stream => live-captions}/vercel.json (100%) delete mode 100644 aai_cli/init/templates/stream/index.html delete mode 100644 aai_cli/init/templates/stream/static/styles.css delete mode 100644 aai_cli/init/templates/transcribe/index.html delete mode 100644 aai_cli/init/templates/transcribe/static/styles.css rename aai_cli/init/templates/{agent => voice-agent}/AGENTS.md (75%) rename aai_cli/init/templates/{agent => voice-agent}/README.md (100%) rename aai_cli/init/templates/{transcribe => voice-agent}/api/__init__.py (100%) rename aai_cli/init/templates/{agent => voice-agent}/api/index.py (100%) rename aai_cli/init/templates/{agent => voice-agent}/api/settings.py (100%) rename aai_cli/init/templates/{agent => voice-agent}/env.example (100%) rename aai_cli/init/templates/{transcribe => voice-agent}/gitignore (100%) create mode 100644 aai_cli/init/templates/voice-agent/index.html rename aai_cli/init/templates/{stream => voice-agent}/requirements.txt (100%) rename aai_cli/init/templates/{agent => voice-agent}/static/app.js (93%) rename aai_cli/init/templates/{agent => voice-agent}/static/audio.js (100%) create mode 100644 aai_cli/init/templates/voice-agent/static/styles.css rename aai_cli/init/templates/{transcribe => voice-agent}/vercel.json (100%) create mode 100644 aai_cli/streaming/macos.py create mode 100644 aai_cli/streaming/macos_system_audio.swift create mode 100644 tests/test_macos_audio_source.py diff --git a/CLAUDE.md b/CLAUDE.md index 0756f0a1..5a9d3558 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,7 +62,7 @@ Each file in `aai_cli/commands/` is a Typer sub-app (`transcribe`, `stream`, `tr - **`agent/`** — full-duplex voice agent (mic in, TTS out via `voices.py`). - **`code_gen/`** — backs `--show-code` on `transcribe`/`stream`/`agent`: builds a ready-to-run Python SDK script from exactly the flags passed (no API key needed; generated code reads `ASSEMBLYAI_API_KEY`). - **`auth/`** — browser-assisted `aai login` via AMS + **Stytch B2B OAuth discovery** (`discovery.py`, `flow.py`, `loopback.py`, `ams.py`). Not Stytch Connected Apps. -- **`init/`** — scaffolds a self-contained FastAPI + HTML starter (`transcribe`/`stream`/`agent` templates), optionally installs deps and opens the browser; writes the key to a git-ignored `.env`. +- **`init/`** — scaffolds a self-contained FastAPI + HTML starter (`audio-transcription`/`live-captions`/`voice-agent` templates), optionally installs deps and opens the browser; writes the key to a git-ignored `.env`. - **`commands/claude.py`** — `aai claude install/status/remove` shells out to `claude mcp add` (the `assemblyai-docs` MCP) and `npx skills add` (the AssemblyAI skill). Missing `claude`/`npx` is reported and skipped, not an error. ## Conventions diff --git a/README.md b/README.md index 70f255fc..7981bf12 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ aai transcribe --sample # transcribe the hosted wildfires.mp3 sample ## Scaffold a starter app ```sh -aai init # pick a template, scaffold it, install deps, open the browser -aai init transcribe myapp # non-interactive: template + directory +aai init # pick a template, scaffold it, install deps, open the browser +aai init audio-transcription myapp # non-interactive: template + directory ``` `aai init` copies a small, self-contained FastAPI + HTML project you can run locally @@ -183,6 +183,8 @@ aai stream path/to/audio.wav # 16 kHz mono WAV streams directly aai stream path/to/audio.mp3 # other formats need ffmpeg on PATH aai stream https://…/clip.mp3 # a URL works too (decoded via ffmpeg) aai stream # from the microphone; Ctrl-C to stop +aai stream --system-audio # macOS: system/app audio + mic as separate sessions +aai stream --system-audio-only # macOS: system/app audio without the mic ``` `aai stream` exposes the full `StreamingParameters` surface as curated flags: @@ -208,6 +210,14 @@ aai stream --sample \ --config vad_threshold=0.7 ``` +On macOS, `--system-audio` uses ScreenCaptureKit to capture system/app audio +without a loopback driver and streams it in a separate Streaming session from +the microphone. The default terminal UI labels finalized turns as `You:` or +`System:`. The first run may ask for Screen & System Audio Recording and +Microphone permissions. The helper does not record screen frames, but macOS +still uses that combined permission label for native system audio capture. +`--system-audio-only` skips the microphone. + ## Live transcript → live LLM `aai stream --llm "PROMPT"` runs a prompt over the live transcript through LLM Gateway, diff --git a/aai_cli/commands/init.py b/aai_cli/commands/init.py index a6dff8cb..b1f0c726 100644 --- a/aai_cli/commands/init.py +++ b/aai_cli/commands/init.py @@ -56,7 +56,7 @@ def _resolve_dir(directory: str | None, template: str, *, here: bool) -> Path: return Path.cwd() if directory: return Path(directory) - return Path.cwd() / f"{template}-app" + return Path.cwd() / template @app.command( @@ -64,8 +64,11 @@ def _resolve_dir(directory: str | None, template: str, *, here: bool) -> Path: epilog=examples_epilog( [ ("Scaffold a new app interactively", "aai init"), - ("Scaffold a transcribe app into ./my-app", "aai init transcribe my-app"), - ("Scaffold into the current directory", "aai init transcribe --here"), + ( + "Scaffold an audio transcription app into ./my-app", + "aai init audio-transcription my-app", + ), + ("Scaffold into the current directory", "aai init audio-transcription --here"), ] ), ) @@ -74,9 +77,7 @@ def init( template: str | None = typer.Argument( None, help="Template to scaffold (omit to pick interactively)." ), - directory: str | None = typer.Argument( - None, help="Target directory (default: