feat(ci): Docker Linux profiling harness for NightDriverStrip builds

.gitignore

@@ -99,3 +99,7 @@ tasks/loop-runs/
 # the captures are local-only and should never be checked in.
 port_scan_stderr*.txt
 port_scan_stdout*.txt
+
+# Profiling harness artifacts (FastLED/fbuild#942) - local-only measurement
+# output: flamegraphs, perf.data, daemon logs, timing tables.
+ci/docker-profile/out/

ci/docker-profile/Dockerfile

@@ -0,0 +1,96 @@
+# Linux profiling harness for fbuild builds of NightDriverStrip
+# (FastLED/fbuild#942 Phase 0).
+#
+# The image carries the *tooling* only: soldr (which bootstraps the
+# pinned Rust toolchain on first run), perf, bpftrace, FlameGraph, and
+# a pinned NightDriverStrip checkout. fbuild itself is built at run
+# time from the bind-mounted source tree so the harness always profiles
+# the current working copy — no COPY of fbuild source here.
+#
+# Caching contract (see README.md):
+#   * PERSISTED across runs via named volumes: /work/target, ~/.cargo,
+#     ~/.rustup, ~/.soldr — these only make the *harness* fast.
+#   * NEVER persisted: ~/.fbuild (and ~/.platformio) — the system under
+#     test. It lives in the container filesystem and dies with the
+#     `--rm` container, so "cold cache" is genuinely cold.
+#
+# Build (from repo root):
+#   docker build -f ci/docker-profile/Dockerfile -t fbuild-profile-linux ci/docker-profile
+#
+# Run: use `uv run python ci/docker-profile/run_profile.py` — it owns
+# the volume flags and privileged-mode setup. Don't hand-type them.
+
+# ubuntu:24.04 for the same reason as ci/docker-mac-cross: the
+# published soldr linux-gnu binary requires glibc 2.39, and 24.04
+# matches the GHA `ubuntu-latest` runners.
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Tooling set = docker-mac-cross baseline + profiling additions:
+#   linux-tools-generic → perf (invoked via /usr/lib/linux-tools-*/perf;
+#                         the /usr/bin/perf wrapper refuses to run when
+#                         `uname -r` doesn't match an installed linux-tools,
+#                         which is always the case on WSL2 kernels)
+#   bpftrace            → off-CPU wait profiling (sched_switch sums)
+#   time                → /usr/bin/time -v resource capture per build
+#   procps              → pkill for daemon teardown between scenarios
+#   perl                → FlameGraph stackcollapse/flamegraph scripts
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+        ca-certificates curl git xz-utils bzip2 zstd \
+        python3 python3-pip python3-venv \
+        build-essential pkg-config file \
+        linux-tools-generic bpftrace time procps perl \
+ && rm -rf /var/lib/apt/lists/*
+
+# uv for python ops (CLAUDE.md mandate).
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh \
+ && cp /root/.local/bin/uv /usr/local/bin/
+
+# esptool: fbuild's ESP32 path shells out to it for elf2image (it is not
+# yet a managed fbuild package — see #942 findings). uv-managed install
+# with the console script linked onto PATH.
+RUN uv tool install esptool \
+ && ln -s /root/.local/bin/esptool* /usr/local/bin/ \
+ && esptool version | head -1
+
+# The running-process broker derives user identity from the machine id;
+# containers have none by default and fbuild logs a fallback warning.
+RUN test -f /etc/machine-id || cat /proc/sys/kernel/random/uuid | tr -d '-\n' > /etc/machine-id
+
+# Soldr binary tarball from GitHub Releases — same rationale and pin as
+# ci/docker-mac-cross/Dockerfile (pip wheel is manylinux_2_39-tagged and
+# falls back to a squatted placeholder on older glibc).
+ARG SOLDR_VERSION=0.7.98
+RUN mkdir -p /opt/soldr-bin \
+ && curl -fsSL \
+        "https://github.com/zackees/soldr/releases/download/v${SOLDR_VERSION}/soldr-v${SOLDR_VERSION}-x86_64-unknown-linux-gnu.tar.zst" \
+    | zstd -d --stdout \
+    | tar -xf - -C /opt/soldr-bin \
+ && for bin in soldr soldr-clang-shim cargo-chef crgx; do \
+        [ -f "/opt/soldr-bin/$bin" ] && cp "/opt/soldr-bin/$bin" "/usr/local/bin/$bin" && chmod +x "/usr/local/bin/$bin"; \
+    done \
+ && soldr --version
+
+# Brendan Gregg's FlameGraph scripts (stackcollapse-perf.pl,
+# stackcollapse-bpftrace.pl, flamegraph.pl). Shallow clone of master —
+# these scripts have been stable for years and the repo is untagged.
+RUN git clone --depth 1 https://github.com/brendangregg/FlameGraph.git /opt/FlameGraph
+
+# Benchmark workload: NightDriverStrip pinned to a fixed upstream sha so
+# every profile run builds the identical source. env:demo is the target
+# (esp32dev / registry espressif32@^6.12.0, WiFi+audio disabled).
+# The checkout is a pristine template; profile_entry.sh copies it to a
+# throwaway per-run directory so build output never contaminates it.
+ARG NIGHTDRIVER_SHA=59113319200a62ffd39c91f7372b47e86df38bcc
+RUN git clone https://github.com/PlummersSoftwareLLC/NightDriverStrip.git /opt/nightdriver \
+ && git -C /opt/nightdriver checkout --detach "${NIGHTDRIVER_SHA}" \
+ && rm -rf /opt/nightdriver/.git
+
+ENV HOME=/root \
+    SOLDR_TRUST_MODE=permissive \
+    CARGO_TERM_COLOR=always
+
+WORKDIR /work
+CMD ["bash", "ci/docker-profile/profile_entry.sh"]

ci/docker-profile/README.md

@@ -0,0 +1,84 @@
+# Docker profiling harness (FastLED/fbuild#942)
+
+Reproducible Linux profiling of `fbuild build` against a pinned
+[NightDriverStrip](https://github.com/PlummersSoftwareLLC/NightDriverStrip)
+checkout (`env:demo`, esp32dev). Produces wall-clock timings, on-CPU and
+off-CPU flamegraphs, and `FBUILD_PERF_LOG` phase timelines for cold-,
+warm-, and hot-cache builds.
+
+## Usage
+
+```bash
+# from repo root — full matrix: cold+warm+hot, 3 iterations each
+uv run python ci/docker-profile/run_profile.py
+
+# quick single cold run
+uv run python ci/docker-profile/run_profile.py -n 1 --scenarios cold
+
+# housekeeping
+uv run python ci/docker-profile/run_profile.py --status   # harness volumes
+uv run python ci/docker-profile/run_profile.py --wipe     # drop them (next run rebuilds fbuild from scratch)
+```
+
+Artifacts land in `ci/docker-profile/out/<timestamp>/` (gitignored):
+`timings.jsonl` + `summary.md` (median wall clock per scenario), and per
+run `oncpu.svg` / `offcpu.svg` flamegraphs, `.folded` stacks, a gzipped
+`perf script` excerpt of the sched capture, CLI logs, `daemon.log`, and
+extracted `perf-log-lines.txt`. Raw `perf.data` files stay inside the
+container (perf cannot write its mmap-backed output onto the 9P Windows
+bind mount, and they run to hundreds of MB — the folded stacks carry
+the analysis-relevant content).
+
+## Scenarios
+
+| scenario | fbuild cache (`~/.fbuild`) | project dir | measures |
+|---|---|---|---|
+| `cold` | wiped (+ `~/.platformio`), daemon killed | fresh copy | downloads, installs, full compile |
+| `warm` | intact, daemon running | fresh copy | first build of a new project with a hot global cache — uncached work shows up here |
+| `hot`  | intact, daemon running | unchanged | no-op rebuild / fast-path overhead |
+
+Scenarios run in the order given per iteration; `warm`/`hot` assume
+`cold` ran first in the same sequence (the default `cold,warm,hot` does
+this per iteration).
+
+## Caching contract
+
+- **Persisted across runs** (named Docker volumes; only make the
+  *harness* fast): `/work/target`, `~/.cargo`, `~/.rustup`, `~/.soldr`.
+  Named volumes, not bind mounts, because Windows/WSL2 bind mounts
+  rewrite mtimes on every container start and bust cargo fingerprints.
+- **Never persisted** (the system under test): `~/.fbuild` and
+  `~/.platformio` live in the container filesystem and die with the
+  `--rm` container. Cold means genuinely cold — fresh downloads.
+
+fbuild + fbuild-daemon are built inside the container from the
+bind-mounted working copy with
+`RUSTFLAGS="-C force-frame-pointers=yes -C debuginfo=2"` (clean perf
+stacks; firmware compile flags are untouched — see the #942 constraint
+that compile settings stay stock).
+
+## Profilers
+
+- **On-CPU**: `perf record -F 99 -e cpu-clock -g --call-graph fp -a`
+  (software clock event — WSL2 exposes no hardware PMU), rendered with
+  [FlameGraph](https://github.com/brendangregg/FlameGraph).
+- **Off-CPU**: `offcpu.bt` (bpftrace, sums blocked µs per stack via
+  `sched:sched_switch`) rendered as `offcpu.svg`; plus a concurrent raw
+  `perf record -e sched:sched_switch` capture as fallback for WSL2
+  kernels where bpftrace stack lookups come back empty.
+- **Phase timeline**: `FBUILD_PERF_LOG=1` — summaries are emitted by
+  the daemon, so the harness harvests
+  `~/.fbuild/prod/daemon/daemon.log` per run.
+
+The container runs `--privileged` (perf_event_open + BPF on Docker
+Desktop) and relaxes `kernel.perf_event_paranoid` best-effort.
+
+## Files
+
+- `Dockerfile` — ubuntu:24.04 + soldr (pinned release tarball, same
+  rationale as `ci/docker-mac-cross`), perf/bpftrace/FlameGraph, and
+  the NightDriverStrip checkout pinned to a fixed sha.
+- `profile_entry.sh` — in-container scenario loop + samplers.
+- `offcpu.bt` — bpftrace off-CPU program.
+- `run_profile.py` — host orchestrator (image build, volumes,
+  `docker run`, summary table).

ci/docker-profile/offcpu.bt

@@ -0,0 +1,44 @@
+// Off-CPU wait-time profiler (FastLED/fbuild#942).
+//
+// Sums microseconds each thread spends blocked (not runnable), keyed by
+// comm + kernel/user stack captured AT THE POINT IT WENT TO SLEEP (at
+// sched_switch the current context is the outgoing prev task, so that
+// is where its sleep-site stack is visible; the wakeup edge only knows
+// the waker's context). Output feeds stackcollapse-bpftrace.pl →
+// offcpu.svg.
+//
+// prev_state != 0 filters to genuine blocking (D/S states); preemptions
+// of still-runnable threads (state 0 / TASK_RUNNING) are excluded so the
+// graph shows waiting, not scheduler churn.
+//
+// Best effort on WSL2 kernels: stack-id lookups can come back empty
+// there, in which case profile_entry.sh falls back to the raw
+// `perf -e sched:sched_switch` recording taken concurrently.
+
+tracepoint:sched:sched_switch
+{
+    if (args->prev_state != 0) {
+        @sleep_start[args->prev_pid] = nsecs;
+        @sleep_comm[args->prev_pid] = comm;
+        @sleep_kstack[args->prev_pid] = kstack;
+        @sleep_ustack[args->prev_pid] = ustack;
+    }
+    $start = @sleep_start[args->next_pid];
+    if ($start) {
+        @usecs[@sleep_comm[args->next_pid],
+               @sleep_kstack[args->next_pid],
+               @sleep_ustack[args->next_pid]] = sum((nsecs - $start) / 1000);
+        delete(@sleep_start[args->next_pid]);
+        delete(@sleep_comm[args->next_pid]);
+        delete(@sleep_kstack[args->next_pid]);
+        delete(@sleep_ustack[args->next_pid]);
+    }
+}
+
+END
+{
+    clear(@sleep_start);
+    clear(@sleep_comm);
+    clear(@sleep_kstack);
+    clear(@sleep_ustack);
+}