diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 69e27fdb..042fda61 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -10,6 +10,10 @@ CI/CD workflows for the fbuild project, covering lint, test, documentation, and - **`loc-gate.yml`** -- Reject `.rs` files over 1000 LOC | **`lint-subprocess.yml`** -- Forbid direct subprocess spawns - **`crate-gate.yml`** -- Reject new workspace crates (monocrate policy, `ci/check_workspace_crates.py`) +## Scheduled Benchmarks + +- **`benchmark-build-comparison.yml`** -- Arduino CLI vs PlatformIO vs fbuild Blink cold/warm benchmark; runs nightly, manually, and for relevant pushes to `main`, then force-publishes the one-commit `benchmark-stats` branch and deploys its site to GitHub Pages + ## Per-Board Builds (push/PR) - **`build-esp32{c2,c3,c5,c6,dev,h2,p4,s2,s3}.yml`** -- ESP32 variants diff --git a/.github/workflows/benchmark-build-comparison.yml b/.github/workflows/benchmark-build-comparison.yml new file mode 100644 index 00000000..6a573d0a --- /dev/null +++ b/.github/workflows/benchmark-build-comparison.yml @@ -0,0 +1,195 @@ +name: Build Comparison Benchmark + +on: + workflow_dispatch: + push: + branches: [main] + paths: + - 'bench/blink/**' + - 'bench/fastled-examples/src/build_comparison.rs' + - 'bench/fastled-examples/Cargo.toml' + - '.github/workflows/benchmark-build-comparison.yml' + schedule: + # Off-hour UTC slot avoids the hosted-runner top-of-hour surge. + - cron: '17 9 * * *' + +concurrency: + group: benchmark-build-comparison-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: '-D warnings' + PLATFORMIO_CORE_VERSION: '6.1.19' + ARDUINO_CLI_VERSION: '1.5.0' + ARDUINO_AVR_CORE_VERSION: '1.8.8' + +jobs: + benchmark: + name: Measure and publish Blink builds + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@v3 + + - name: Setup soldr + uses: zackees/setup-soldr@v0 + with: + version: 0.8.8 + cache: true + build-cache: true + target-cache: true + prebuild-deps: none + linker: platform-default + cache-payload-warn-bytes: 2GiB + + - name: Install Arduino CLI + uses: arduino/setup-arduino-cli@v2 + with: + version: ${{ env.ARDUINO_CLI_VERSION }} + + - name: Install benchmark toolchains + shell: bash + run: | + set -euo pipefail + uv tool install "platformio==${PLATFORMIO_CORE_VERSION}" + arduino-cli core update-index + arduino-cli core install "arduino:avr@${ARDUINO_AVR_CORE_VERSION}" + pio pkg install --project-dir bench/blink --environment uno + + - name: Build fbuild and benchmark runner + shell: bash + run: | + set -euo pipefail + soldr cargo build --release \ + -p fbuild-cli --bin fbuild \ + -p fbuild-daemon --bin fbuild-daemon \ + -p fbuild-bench-fastled-examples --bin bench-build-comparison + + - name: Prime package downloads outside timed measurements + shell: bash + run: | + set -euo pipefail + mkdir -p benchmark-output/bootstrap-arduino + arduino-cli compile \ + --fqbn arduino:avr:uno \ + --build-path benchmark-output/bootstrap-arduino \ + bench/blink + target/release/fbuild build bench/blink -e uno --release + target/release/fbuild clean all bench/blink -e uno --release + rm -rf benchmark-output/bootstrap-arduino + + - name: Restore rolling history + shell: bash + run: | + set -euo pipefail + mkdir -p benchmark-stats + if git ls-remote --exit-code origin refs/heads/benchmark-stats >/dev/null; then + git fetch --depth=1 origin refs/heads/benchmark-stats + git show FETCH_HEAD:history.jsonl > benchmark-stats/history.jsonl + else + status=$? + if [[ "${status}" -eq 2 ]]; then + : > benchmark-stats/history.jsonl + else + exit "${status}" + fi + fi + + - name: Run cold/warm comparison and render site + shell: bash + run: | + set -euo pipefail + generated_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + target/release/bench-build-comparison \ + --project-dir bench/blink \ + --fbuild target/release/fbuild \ + --output-dir benchmark-stats \ + --trials 3 \ + --repository "${GITHUB_REPOSITORY}" \ + --git-sha "${GITHUB_SHA}" \ + --generated-at "${generated_at}" \ + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + --pages-url https://fastled.github.io/fbuild/ \ + --raw-base-url https://raw.githubusercontent.com/FastLED/fbuild/benchmark-stats + + - name: Upload benchmark site snapshot + uses: actions/upload-artifact@v7 + with: + name: build-comparison-site-${{ github.sha }} + path: benchmark-stats + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + + - name: Upload benchmark log + if: always() + uses: actions/upload-artifact@v7 + with: + name: build-comparison-log-${{ github.sha }} + path: benchmark-output/benchmark.log + if-no-files-found: warn + retention-days: 30 + + publish: + name: Publish benchmark artifacts + needs: benchmark + if: ${{ needs.benchmark.result == 'success' && github.ref == 'refs/heads/main' }} + runs-on: ubuntu-24.04 + permissions: + contents: write + pages: write + steps: + - name: Download benchmark site snapshot + uses: actions/download-artifact@v8 + with: + name: build-comparison-site-${{ github.sha }} + path: benchmark-stats + + - name: Publish one-commit benchmark-stats branch + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + publish_dir="$(mktemp -d)" + cp -a benchmark-stats/. "${publish_dir}/" + git -C "${publish_dir}" init + git -C "${publish_dir}" checkout -b benchmark-stats + git -C "${publish_dir}" config user.name 'github-actions[bot]' + git -C "${publish_dir}" config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git -C "${publish_dir}" add . + git -C "${publish_dir}" commit -m "chore: publish build benchmark for ${GITHUB_SHA}" + git -C "${publish_dir}" remote add origin \ + "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git -C "${publish_dir}" push --force origin benchmark-stats + + - name: Configure GitHub Pages + uses: actions/configure-pages@v5 + + - name: Upload GitHub Pages artifact + uses: actions/upload-pages-artifact@v4 + with: + path: benchmark-stats + + deploy-pages: + name: Deploy benchmark report + needs: publish + runs-on: ubuntu-24.04 + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + permissions: + pages: write + id-token: write + steps: + - name: Deploy GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 14af1826..b12c4909 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ libiconv-2.dll # fbuild local build artifacts .fbuild/ +.pio/ +benchmark-output/ +benchmark-stats/ tests/platform/*/build_info*.json # Local cargo cache diff --git a/Cargo.lock b/Cargo.lock index bf1bb578..897ba5d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1113,6 +1113,7 @@ dependencies = [ "fbuild-packages", "fbuild-paths", "fbuild-test-support", + "serde", "serde_json", "tempfile", ] diff --git a/README.md b/README.md index aa279f5f..16cc029d 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,14 @@ that fbuild actively protects in CI. Board descriptions and family deep-dives live in [`docs/BOARD_STATUS.md`](docs/BOARD_STATUS.md). +## Build performance + +[![Arduino CLI vs PlatformIO vs fbuild Blink build benchmark](https://raw.githubusercontent.com/FastLED/fbuild/benchmark-stats/benchmark.svg)](https://fastled.github.io/fbuild/) + +The chart is regenerated nightly from clean-output (cold) and immediate repeat +(warm) Arduino Uno Blink builds. Raw measurements are discoverable through the +[benchmark manifest](https://raw.githubusercontent.com/FastLED/fbuild/benchmark-stats/manifest.json). + ## Installation ```bash diff --git a/bench/README.md b/bench/README.md index 1ffe499e..ac417b29 100644 --- a/bench/README.md +++ b/bench/README.md @@ -19,6 +19,8 @@ soldr cargo bench -p fbuild-header-scan --bench scan_throughput ## Subdirectories +- [`blink/`](blink/README.md) — shared Arduino Uno Blink fixture used by the + nightly Arduino CLI vs PlatformIO vs fbuild whole-build benchmark. - [`fastled-examples/`](fastled-examples/README.md) — real-FastLED warm-cache library-selection matrix (`FastLED/fbuild#205` AC#5, P-01). Discovers examples under `$FASTLED_DIR` (default `~/dev/fastled`), @@ -31,3 +33,7 @@ Other end-to-end matrices (whole-build wall-clock, deploy+flash latency, emulator boot) may join this directory in the future. Each subdirectory must carry its own `README.md` explaining what it measures, how to run it, and which CI gate (if any) it feeds. + +The nightly whole-build comparison is published at +. Its machine-readable discovery document +is `manifest.json` on the one-commit `benchmark-stats` branch. diff --git a/bench/blink/README.md b/bench/blink/README.md new file mode 100644 index 00000000..dce332f2 --- /dev/null +++ b/bench/blink/README.md @@ -0,0 +1,49 @@ +# Blink build comparison + +This directory is one shared Arduino Uno Blink sketch for the nightly +whole-build comparison. Arduino CLI, PlatformIO, and fbuild compile the exact +same `blink.ino` file for the Arduino Uno target. Each ecosystem's framework +distribution is pinned independently: `arduino:avr@1.8.8` for Arduino CLI and +`atmelavr@5.1.0` for PlatformIO/fbuild. + +Each tool is measured for three trials by default: + +- **Cold** removes project outputs and matching compiled framework caches before + compiling. Installed toolchains/framework packages and global + download/compiler caches remain available. +- **Warm** immediately repeats the same build without changing any input. + +The Rust runner records every trial and publishes the median. Cold bars are +drawn behind narrower warm overlays using the same GitHub-dark gray, blue, and +red palette as the zccache and soldr benchmark graphics. + +## Run locally + +Install Arduino CLI, its `arduino:avr` core, and PlatformIO first. Build fbuild +and run the harness through `soldr`: + +```bash +soldr cargo build --release -p fbuild-cli --bin fbuild -p fbuild-daemon --bin fbuild-daemon +soldr cargo run --release -p fbuild-bench-fastled-examples \ + --bin bench-build-comparison -- \ + --project-dir bench/blink \ + --fbuild target/release/fbuild \ + --output-dir benchmark-stats +``` + +The `target/release/fbuild` path above is for Linux and CI. On Windows, use +`target/x86_64-pc-windows-msvc/release/fbuild.exe`. + +The nightly workflow installs and records pinned Arduino CLI and PlatformIO +versions, primes package downloads outside the timed region, carries forward a +365-run `history.jsonl`, and publishes: + +- `manifest.json` — stable discovery index for agents and other clients +- `latest.json` — full metadata, raw trials, and medians for the newest run +- `history.jsonl` — rolling compact history +- `benchmark.svg` — README graphic with cold/warm overlays +- `index.html` — human-facing GitHub Pages report + +The `benchmark-stats` branch is force-published from a fresh repository on +every successful default-branch run, so its Git history always contains one +generated commit. diff --git a/bench/blink/blink.ino b/bench/blink/blink.ino new file mode 100644 index 00000000..fab5a8d4 --- /dev/null +++ b/bench/blink/blink.ino @@ -0,0 +1,10 @@ +void setup() { + pinMode(LED_BUILTIN, OUTPUT); +} + +void loop() { + digitalWrite(LED_BUILTIN, HIGH); + delay(1000); + digitalWrite(LED_BUILTIN, LOW); + delay(1000); +} diff --git a/bench/blink/platformio.ini b/bench/blink/platformio.ini new file mode 100644 index 00000000..1f963b47 --- /dev/null +++ b/bench/blink/platformio.ini @@ -0,0 +1,7 @@ +[platformio] +src_dir = . + +[env:uno] +platform = atmelavr@5.1.0 +board = uno +framework = arduino diff --git a/bench/fastled-examples/Cargo.toml b/bench/fastled-examples/Cargo.toml index de4eb18f..272e8976 100644 --- a/bench/fastled-examples/Cargo.toml +++ b/bench/fastled-examples/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fbuild-bench-fastled-examples" -description = "Warm-cache library-selection bench across the FastLED examples matrix (#205 Phase 7)" +description = "End-to-end benchmark harnesses for fbuild" version.workspace = true edition.workspace = true rust-version.workspace = true @@ -11,10 +11,15 @@ publish = false name = "bench-fastled-examples" path = "src/main.rs" +[[bin]] +name = "bench-build-comparison" +path = "src/build_comparison.rs" + [dependencies] fbuild-library-select = { path = "../../crates/fbuild-library-select" } fbuild-packages = { path = "../../crates/fbuild-packages" } fbuild-paths = { path = "../../crates/fbuild-paths" } fbuild-test-support = { path = "../../crates/fbuild-test-support" } tempfile = { workspace = true } +serde = { workspace = true } serde_json = { workspace = true } diff --git a/bench/fastled-examples/src/README.md b/bench/fastled-examples/src/README.md index a9e5304a..20da48f1 100644 --- a/bench/fastled-examples/src/README.md +++ b/bench/fastled-examples/src/README.md @@ -1,4 +1,10 @@ # bench/fastled-examples/src -Source for the `bench-fastled-examples` binary. See the parent -[`README.md`](../README.md) for what the harness does and how to run it. +Sources for the repository's end-to-end benchmark binaries: + +- `main.rs` implements `bench-fastled-examples`. +- `build_comparison.rs` implements the nightly Arduino CLI vs PlatformIO vs + fbuild Blink build comparison and static-site renderer. + +See the parent [`README.md`](../README.md) for the FastLED harness and +[`../../blink/README.md`](../../blink/README.md) for the whole-build benchmark. diff --git a/bench/fastled-examples/src/build_comparison.rs b/bench/fastled-examples/src/build_comparison.rs new file mode 100644 index 00000000..a0b5f372 --- /dev/null +++ b/bench/fastled-examples/src/build_comparison.rs @@ -0,0 +1,992 @@ +//! Nightly Arduino CLI vs PlatformIO vs fbuild whole-build benchmark. +//! +//! The harness measures the same Arduino Uno Blink sketch with each real CLI, +//! then renders the one-commit benchmark site's JSON, SVG, and HTML artifacts. + +use serde::Serialize; +use serde_json::{Value, json}; +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +const HISTORY_MAX_LINES: usize = 365; +const DEFAULT_REPOSITORY: &str = "FastLED/fbuild"; +const DEFAULT_PAGES_URL: &str = "https://fastled.github.io/fbuild/"; +const DEFAULT_RAW_BASE_URL: &str = + "https://raw.githubusercontent.com/FastLED/fbuild/benchmark-stats"; + +type AppResult = Result>; + +#[derive(Debug)] +struct Options { + output_dir: PathBuf, + project_dir: PathBuf, + fbuild: PathBuf, + arduino_cli: OsString, + platformio: OsString, + trials: usize, + repository: String, + git_sha: String, + generated_at: String, + run_url: String, + pages_url: String, + raw_base_url: String, +} + +#[derive(Clone, Copy, Debug)] +enum ToolKind { + Arduino, + PlatformIo, + Fbuild, +} + +#[derive(Clone, Copy, Debug)] +struct ToolStyle { + key: &'static str, + label: &'static str, + cold_color: &'static str, + warm_color: &'static str, +} + +impl ToolKind { + fn style(self) -> ToolStyle { + match self { + Self::Arduino => ToolStyle { + key: "arduino", + label: "Arduino CLI", + cold_color: "#3b4046", + warm_color: "#8b949e", + }, + Self::PlatformIo => ToolStyle { + key: "platformio", + label: "PlatformIO", + cold_color: "#1f3a7a", + warm_color: "#79c0ff", + }, + Self::Fbuild => ToolStyle { + key: "fbuild", + label: "fbuild", + cold_color: "#5b1f1c", + warm_color: "#f85149", + }, + } + } +} + +#[derive(Clone, Debug, Serialize)] +struct ToolResult { + tool: String, + display_name: String, + version: String, + cold_ms: f64, + warm_ms: f64, + speedup: f64, + cold_trials_ms: Vec, + warm_trials_ms: Vec, +} + +#[derive(Clone, Debug)] +struct Metadata { + generated_at: String, + git_sha: String, + repository: String, + run_url: String, + project: String, + trials: usize, +} + +fn main() { + if let Err(error) = run() { + eprintln!("build comparison benchmark failed: {error}"); + std::process::exit(1); + } +} + +fn run() -> AppResult<()> { + let options = parse_options()?; + let repo_root = env::current_dir()?; + let project_dir = absolute_from(&repo_root, &options.project_dir); + let output_dir = absolute_from(&repo_root, &options.output_dir); + let fbuild = absolute_from(&repo_root, &options.fbuild); + + if !project_dir.join("blink.ino").is_file() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("{} does not contain blink.ino", project_dir.display()), + ) + .into()); + } + if !fbuild.is_file() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("fbuild binary not found at {}", fbuild.display()), + ) + .into()); + } + + fs::create_dir_all(&output_dir)?; + let log_dir = repo_root.join("benchmark-output"); + fs::create_dir_all(&log_dir)?; + let log_path = log_dir.join("benchmark.log"); + let mut log = File::create(&log_path)?; + + let versions = [ + ( + ToolKind::Arduino, + command_version(&options.arduino_cli, &["version"])?, + ), + ( + ToolKind::PlatformIo, + command_version(&options.platformio, &["--version"])?, + ), + ( + ToolKind::Fbuild, + command_version(fbuild.as_os_str(), &["--version"])?, + ), + ]; + + let arduino_build_dir = repo_root.join("benchmark-output/arduino-build"); + let mut results = Vec::with_capacity(versions.len()); + for (kind, version) in versions { + let result = measure_tool( + kind, + &version, + &options, + &repo_root, + &project_dir, + &fbuild, + &arduino_build_dir, + &mut log, + )?; + println!( + "{:<12} cold {:>10.3} ms | warm {:>10.3} ms | {:>7.2}x", + result.display_name, result.cold_ms, result.warm_ms, result.speedup + ); + results.push(result); + } + + let metadata = Metadata { + generated_at: options.generated_at.clone(), + git_sha: options.git_sha.clone(), + repository: options.repository.clone(), + run_url: options.run_url.clone(), + project: options.project_dir.to_string_lossy().replace('\\', "/"), + trials: options.trials, + }; + write_outputs( + &output_dir, + &metadata, + &results, + &options.pages_url, + &options.raw_base_url, + )?; + println!("Published benchmark artifacts to {}", output_dir.display()); + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn measure_tool( + kind: ToolKind, + version: &str, + options: &Options, + repo_root: &Path, + project_dir: &Path, + fbuild: &Path, + arduino_build_dir: &Path, + log: &mut File, +) -> AppResult { + let mut cold_trials_ms = Vec::with_capacity(options.trials); + let mut warm_trials_ms = Vec::with_capacity(options.trials); + + if matches!(kind, ToolKind::Fbuild) { + writeln!(log, "\n===== fbuild daemon preflight =====")?; + run_logged( + fbuild.as_os_str(), + &os_args(&["daemon", "restart"]), + repo_root, + log, + )?; + } + + for trial in 1..=options.trials { + writeln!( + log, + "\n===== {} trial {trial}/{} =====", + kind.style().label, + options.trials + )?; + clean_tool( + kind, + options, + repo_root, + project_dir, + fbuild, + arduino_build_dir, + log, + )?; + let cold = timed_build( + kind, + options, + repo_root, + project_dir, + fbuild, + arduino_build_dir, + log, + )?; + let warm = timed_build( + kind, + options, + repo_root, + project_dir, + fbuild, + arduino_build_dir, + log, + )?; + cold_trials_ms.push(round_millis(cold)); + warm_trials_ms.push(round_millis(warm)); + } + + let cold_ms = round_millis(median(&cold_trials_ms)); + let warm_ms = round_millis(median(&warm_trials_ms)); + let speedup = if warm_ms > 0.0 { + round_to(cold_ms / warm_ms, 3) + } else { + 0.0 + }; + let style = kind.style(); + Ok(ToolResult { + tool: style.key.to_string(), + display_name: style.label.to_string(), + version: version.to_string(), + cold_ms, + warm_ms, + speedup, + cold_trials_ms, + warm_trials_ms, + }) +} + +fn clean_tool( + kind: ToolKind, + options: &Options, + repo_root: &Path, + project_dir: &Path, + fbuild: &Path, + arduino_build_dir: &Path, + log: &mut File, +) -> AppResult<()> { + match kind { + ToolKind::Arduino => remove_dir_within(repo_root, arduino_build_dir), + ToolKind::PlatformIo => { + let args = os_args(&[ + "run", + "--project-dir", + &project_dir.to_string_lossy(), + "--environment", + "uno", + "--target", + "clean", + ]); + run_logged(&options.platformio, &args, repo_root, log).map(|_| ()) + } + ToolKind::Fbuild => { + let args = os_args(&[ + "clean", + "all", + &project_dir.to_string_lossy(), + "--environment", + "uno", + "--release", + ]); + run_logged(fbuild.as_os_str(), &args, repo_root, log).map(|_| ()) + } + } +} + +fn timed_build( + kind: ToolKind, + options: &Options, + repo_root: &Path, + project_dir: &Path, + fbuild: &Path, + arduino_build_dir: &Path, + log: &mut File, +) -> AppResult { + let (program, args) = match kind { + ToolKind::Arduino => ( + options.arduino_cli.as_os_str(), + os_args(&[ + "compile", + "--fqbn", + "arduino:avr:uno", + "--build-path", + &arduino_build_dir.to_string_lossy(), + &project_dir.to_string_lossy(), + ]), + ), + ToolKind::PlatformIo => ( + options.platformio.as_os_str(), + os_args(&[ + "run", + "--project-dir", + &project_dir.to_string_lossy(), + "--environment", + "uno", + ]), + ), + ToolKind::Fbuild => ( + fbuild.as_os_str(), + os_args(&[ + "build", + &project_dir.to_string_lossy(), + "--environment", + "uno", + "--release", + ]), + ), + }; + + let started = Instant::now(); + run_logged(program, &args, repo_root, log)?; + Ok(started.elapsed().as_secs_f64() * 1000.0) +} + +fn os_args(values: &[&str]) -> Vec { + values.iter().map(OsString::from).collect() +} + +fn run_logged(program: &OsStr, args: &[OsString], cwd: &Path, log: &mut File) -> AppResult { + writeln!(log, "$ {}", display_command(program, args))?; + log.flush()?; + let output = Command::new(program) + .args(args) + .current_dir(cwd) + .env("CI", "true") + .env("PLATFORMIO_SETTING_ENABLE_TELEMETRY", "no") + .output()?; + log.write_all(&output.stdout)?; + log.write_all(&output.stderr)?; + log.flush()?; + if !output.status.success() { + return Err(io::Error::other(format!( + "command failed with {}: {}; see benchmark-output/benchmark.log", + output.status, + display_command(program, args) + )) + .into()); + } + Ok(output) +} + +fn display_command(program: &OsStr, args: &[OsString]) -> String { + std::iter::once(program) + .chain(args.iter().map(OsString::as_os_str)) + .map(|part| format!("{:?}", part.to_string_lossy())) + .collect::>() + .join(" ") +} + +fn command_version(program: &OsStr, args: &[&str]) -> AppResult { + let output = Command::new(program).args(args).output()?; + if !output.status.success() { + return Err(io::Error::other(format!("failed to query version from {:?}", program)).into()); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let version = if stdout.trim().is_empty() { + stderr.trim() + } else { + stdout.trim() + }; + Ok(version.lines().next().unwrap_or("unknown").to_string()) +} + +fn remove_dir_within(root: &Path, target: &Path) -> AppResult<()> { + if !target.starts_with(root) || target == root { + return Err(io::Error::other(format!( + "refusing to remove benchmark path outside repository: {}", + target.display() + )) + .into()); + } + match fs::remove_dir_all(target) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +fn absolute_from(root: &Path, path: &Path) -> PathBuf { + if path.is_absolute() { + path.to_path_buf() + } else { + root.join(path) + } +} + +fn median(values: &[f64]) -> f64 { + let mut sorted = values.to_vec(); + sorted.sort_by(f64::total_cmp); + let middle = sorted.len() / 2; + if sorted.len() % 2 == 0 { + (sorted[middle - 1] + sorted[middle]) / 2.0 + } else { + sorted[middle] + } +} + +fn round_millis(value: f64) -> f64 { + round_to(value, 3) +} + +fn round_to(value: f64, places: i32) -> f64 { + let factor = 10_f64.powi(places); + (value * factor).round() / factor +} + +fn write_outputs( + output_dir: &Path, + metadata: &Metadata, + results: &[ToolResult], + pages_url: &str, + raw_base_url: &str, +) -> AppResult<()> { + fs::create_dir_all(output_dir)?; + let latest = latest_payload(metadata, results); + write_json(output_dir.join("latest.json"), &latest)?; + write_history(output_dir.join("history.jsonl"), metadata, results)?; + write_json( + output_dir.join("manifest.json"), + &manifest_payload(metadata, pages_url, raw_base_url), + )?; + fs::write( + output_dir.join("benchmark.svg"), + render_svg(metadata, results), + )?; + fs::write( + output_dir.join("index.html"), + render_html(metadata, results), + )?; + fs::write(output_dir.join(".nojekyll"), "")?; + Ok(()) +} + +fn latest_payload(metadata: &Metadata, results: &[ToolResult]) -> Value { + json!({ + "schema_version": 1, + "metadata": { + "generated_at": metadata.generated_at, + "git_sha": metadata.git_sha, + "repository": metadata.repository, + "run_url": metadata.run_url, + "runner": { + "os": env::consts::OS, + "arch": env::consts::ARCH, + }, + "fixture": metadata.project, + "board": "Arduino Uno", + "fqbn": "arduino:avr:uno", + "toolchain_pins": { + "arduino_core": "arduino:avr@1.8.8", + "platformio_platform": "atmelavr@5.1.0", + "note": "ecosystem framework distributions are pinned independently", + }, + "trials": metadata.trials, + "statistic": "median", + "cold_definition": "project outputs and matching compiled framework caches removed; installed packages and global download/compiler caches retained", + "warm_definition": "immediate no-change rebuild after the cold build", + }, + "results": results, + }) +} + +fn manifest_payload(metadata: &Metadata, pages_url: &str, raw_base_url: &str) -> Value { + let raw = raw_base_url.trim_end_matches('/'); + json!({ + "schema_version": 1, + "generated_at": metadata.generated_at, + "git_sha": metadata.git_sha, + "branch": "benchmark-stats", + "repository": metadata.repository, + "artifacts": { + "manifest": { + "description": "Discovery index regenerated by every benchmark run.", + "url": format!("{raw}/manifest.json"), + "content_type": "application/json", + "schema_version": 1, + }, + "latest": { + "description": "Full metadata, raw trials, and median cold/warm timings for the newest run.", + "url": format!("{raw}/latest.json"), + "content_type": "application/json", + "schema_version": 1, + }, + "history": { + "description": "Rolling compact history, one JSON object per benchmark run.", + "url": format!("{raw}/history.jsonl"), + "content_type": "application/x-ndjson", + "schema_version": 1, + "max_lines": HISTORY_MAX_LINES, + }, + "image": { + "description": "GitHub-dark cold bars with narrower warm timing overlays.", + "url": format!("{raw}/benchmark.svg"), + "content_type": "image/svg+xml", + }, + "site": { + "description": "Human-facing benchmark report.", + "url": pages_url, + "content_type": "text/html", + }, + }, + }) +} + +fn write_json(path: PathBuf, payload: &Value) -> AppResult<()> { + fs::write(path, serde_json::to_string_pretty(payload)? + "\n")?; + Ok(()) +} + +fn write_history(path: PathBuf, metadata: &Metadata, results: &[ToolResult]) -> AppResult<()> { + let mut prior = if path.is_file() { + fs::read_to_string(&path)? + .lines() + .filter(|line| !line.trim().is_empty()) + .map(str::to_string) + .collect::>() + } else { + Vec::new() + }; + if prior.len() >= HISTORY_MAX_LINES { + let discard = prior.len() - (HISTORY_MAX_LINES - 1); + prior.drain(..discard); + } + let compact_results = results + .iter() + .map(|result| { + json!({ + "tool": result.tool, + "cold_ms": result.cold_ms, + "warm_ms": result.warm_ms, + "speedup": result.speedup, + }) + }) + .collect::>(); + prior.push(serde_json::to_string(&json!({ + "ts": metadata.generated_at, + "sha": metadata.git_sha, + "results": compact_results, + }))?); + fs::write(path, prior.join("\n") + "\n")?; + Ok(()) +} + +fn render_svg(metadata: &Metadata, results: &[ToolResult]) -> String { + let width = 960.0; + let height = 410.0; + let bar_x = 190.0; + let bar_width = 480.0; + let max_ms = results + .iter() + .flat_map(|result| [result.cold_ms, result.warm_ms]) + .fold(0.0_f64, f64::max) + .max(1.0); + let mut rows = String::new(); + for (index, result) in results.iter().enumerate() { + let kind = match result.tool.as_str() { + "arduino" => ToolKind::Arduino, + "platformio" => ToolKind::PlatformIo, + _ => ToolKind::Fbuild, + }; + let style = kind.style(); + let y = 174.0 + index as f64 * 68.0; + let cold_width = (result.cold_ms / max_ms * bar_width).max(3.0); + let warm_width = (result.warm_ms / max_ms * bar_width).max(3.0); + rows.push_str(&format!( + r##" + {label} + + + {label} cold median: {cold_ms:.3} ms + + + {label} warm median: {warm_ms:.3} ms + + cold {cold_ms:.1} ms | warm {warm_ms:.1} ms + {speedup:.2}x cold / warm + +"##, + label_y = y + 21.0, + label = xml_escape(&result.display_name), + warm = style.warm_color, + bar_x = bar_x, + cold_y = y, + bar_width = bar_width, + cold_width = cold_width, + cold = style.cold_color, + cold_ms = result.cold_ms, + warm_y = y + 7.0, + warm_width = warm_width, + warm_ms = result.warm_ms, + value_y = y + 13.0, + speed_y = y + 34.0, + speedup = result.speedup, + )); + } + let short_sha = metadata.git_sha.chars().take(12).collect::(); + format!( + r##" + Arduino CLI vs PlatformIO vs fbuild Blink build benchmark + Cold build bars with narrower warm build timing overlays for an Arduino Uno Blink sketch. + + + + Arduino Uno Blink build times + Arduino CLI vs PlatformIO vs fbuild | median of {trials} trials + Generated {generated_at} | sha {sha} + + + cold (back) + warm (front overlay) + scale: slowest median = {max_ms:.1} ms +{rows} + Machine data: manifest.json | latest.json | history.jsonl + +"##, + width = width, + height = height, + trials = metadata.trials, + generated_at = xml_escape(&metadata.generated_at), + sha = xml_escape(&short_sha), + max_ms = max_ms, + rows = rows, + ) +} + +fn render_html(metadata: &Metadata, results: &[ToolResult]) -> String { + let rows = results + .iter() + .map(|result| { + format!( + "{}{:.3} ms{:.3} ms{:.2}x{}", + html_escape(&result.display_name), + result.cold_ms, + result.warm_ms, + result.speedup, + html_escape(&result.version) + ) + }) + .collect::>() + .join("\n"); + format!( + r#" + + + + + fbuild Blink build benchmark + + + +
+

fbuild Blink build benchmark

+

Generated {generated_at} from {sha}. Median of {trials} trials on {os}/{arch}.

+

All three tools compile the same Arduino Uno bench/blink/blink.ino. Cold removes project outputs and matching compiled framework caches while retaining installed packages and global download/compiler caches. Warm is the immediate no-change rebuild. The narrower warm bar overlays the cold bar.

+ Arduino CLI vs PlatformIO vs fbuild cold and warm Blink build timings +
+ + + {rows} +
ToolCold medianWarm medianCold / warmVersion
+
+

Machine-readable data

+ +

Benchmark workflow run | one-commit publication branch

+
+ + +"#, + generated_at = html_escape(&metadata.generated_at), + sha = html_escape(&metadata.git_sha), + trials = metadata.trials, + os = env::consts::OS, + arch = env::consts::ARCH, + rows = rows, + run_url = html_escape(&metadata.run_url), + repository = html_escape(&metadata.repository), + ) +} + +fn xml_escape(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + +fn html_escape(value: &str) -> String { + xml_escape(value) +} + +fn parse_options() -> AppResult { + let mut options = Options { + output_dir: PathBuf::from("benchmark-stats"), + project_dir: PathBuf::from("bench/blink"), + fbuild: PathBuf::from(if cfg!(windows) { + "target/release/fbuild.exe" + } else { + "target/release/fbuild" + }), + arduino_cli: OsString::from(if cfg!(windows) { + "arduino-cli.exe" + } else { + "arduino-cli" + }), + platformio: OsString::from(if cfg!(windows) { "pio.exe" } else { "pio" }), + trials: 3, + repository: DEFAULT_REPOSITORY.to_string(), + git_sha: env::var("GITHUB_SHA").unwrap_or_else(|_| "local".to_string()), + generated_at: default_generated_at(), + run_url: String::new(), + pages_url: DEFAULT_PAGES_URL.to_string(), + raw_base_url: DEFAULT_RAW_BASE_URL.to_string(), + }; + + let mut args = env::args_os().skip(1); + while let Some(flag) = args.next() { + let flag_text = flag.to_string_lossy(); + if flag_text == "--help" || flag_text == "-h" { + print_help(); + std::process::exit(0); + } + let value = args.next().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("missing value for {flag_text}"), + ) + })?; + match flag_text.as_ref() { + "--output-dir" => options.output_dir = PathBuf::from(value), + "--project-dir" => options.project_dir = PathBuf::from(value), + "--fbuild" => options.fbuild = PathBuf::from(value), + "--arduino-cli" => options.arduino_cli = value, + "--platformio" => options.platformio = value, + "--trials" => { + options.trials = value.to_string_lossy().parse()?; + if options.trials == 0 || options.trials > 20 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "--trials must be between 1 and 20", + ) + .into()); + } + } + "--repository" => options.repository = value.to_string_lossy().into_owned(), + "--git-sha" => options.git_sha = value.to_string_lossy().into_owned(), + "--generated-at" => options.generated_at = value.to_string_lossy().into_owned(), + "--run-url" => options.run_url = value.to_string_lossy().into_owned(), + "--pages-url" => options.pages_url = value.to_string_lossy().into_owned(), + "--raw-base-url" => { + options.raw_base_url = value.to_string_lossy().into_owned(); + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown option {flag_text}"), + ) + .into()); + } + } + } + Ok(options) +} + +fn default_generated_at() -> String { + let seconds = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + format!("unix:{seconds}") +} + +fn print_help() { + println!( + "bench-build-comparison [options]\n\ + \n\ + --output-dir PATH Static-site output (default: benchmark-stats)\n\ + --project-dir PATH Shared Blink project (default: bench/blink)\n\ + --fbuild PATH fbuild binary (default: target/release/fbuild)\n\ + --arduino-cli COMMAND Arduino CLI executable\n\ + --platformio COMMAND PlatformIO executable\n\ + --trials N Cold/warm trial pairs (default: 3)\n\ + --repository OWNER/REPO Metadata repository\n\ + --git-sha SHA Commit measured\n\ + --generated-at ISO8601 Run timestamp\n\ + --run-url URL GitHub Actions run URL\n\ + --pages-url URL Published HTML URL\n\ + --raw-base-url URL Raw benchmark-stats branch base URL" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_results() -> Vec { + vec![ + ToolResult { + tool: "arduino".into(), + display_name: "Arduino CLI".into(), + version: "arduino-cli 1.5.0".into(), + cold_ms: 1200.0, + warm_ms: 800.0, + speedup: 1.5, + cold_trials_ms: vec![1100.0, 1200.0, 1300.0], + warm_trials_ms: vec![750.0, 800.0, 850.0], + }, + ToolResult { + tool: "platformio".into(), + display_name: "PlatformIO".into(), + version: "PlatformIO Core 6.1.19".into(), + cold_ms: 900.0, + warm_ms: 300.0, + speedup: 3.0, + cold_trials_ms: vec![850.0, 900.0, 950.0], + warm_trials_ms: vec![280.0, 300.0, 320.0], + }, + ToolResult { + tool: "fbuild".into(), + display_name: "fbuild".into(), + version: "fbuild 0.1.0".into(), + cold_ms: 600.0, + warm_ms: 40.0, + speedup: 15.0, + cold_trials_ms: vec![580.0, 600.0, 620.0], + warm_trials_ms: vec![38.0, 40.0, 42.0], + }, + ] + } + + fn sample_metadata() -> Metadata { + Metadata { + generated_at: "2026-07-22T12:00:00Z".into(), + git_sha: "0123456789abcdef".into(), + repository: DEFAULT_REPOSITORY.into(), + run_url: "https://github.com/FastLED/fbuild/actions/runs/1".into(), + project: "bench/blink".into(), + trials: 3, + } + } + + #[test] + fn median_handles_odd_and_even_trial_counts() { + assert_eq!(median(&[9.0, 1.0, 5.0]), 5.0); + assert_eq!(median(&[9.0, 1.0, 7.0, 3.0]), 5.0); + } + + #[test] + fn remove_dir_within_guards_boundaries() { + let sandbox = tempfile::tempdir().unwrap(); + let root = sandbox.path().join("root"); + let nested = root.join("nested"); + let sibling = sandbox.path().join("sibling"); + fs::create_dir_all(&nested).unwrap(); + fs::create_dir_all(&sibling).unwrap(); + + assert!(remove_dir_within(&root, &root).is_err()); + assert!(remove_dir_within(&root, &sibling).is_err()); + assert!(root.is_dir()); + assert!(sibling.is_dir()); + + remove_dir_within(&root, &nested).unwrap(); + assert!(!nested.exists()); + } + + #[test] + fn svg_uses_reference_palette_and_warm_overlay() { + let svg = render_svg(&sample_metadata(), &sample_results()); + for color in [ + "#3b4046", "#8b949e", "#1f3a7a", "#79c0ff", "#5b1f1c", "#f85149", + ] { + assert!(svg.contains(color), "missing {color}"); + } + assert!(svg.contains("height=\"28\"")); + assert!(svg.contains("height=\"14\"")); + assert!(svg.contains("cold (back) + warm (front overlay)")); + } + + #[test] + fn outputs_include_agent_discovery_and_bounded_history() { + let temp = tempfile::tempdir().unwrap(); + let history = (0..HISTORY_MAX_LINES) + .map(|index| format!(r#"{{"old":{index}}}"#)) + .collect::>() + .join("\n") + + "\n"; + fs::write(temp.path().join("history.jsonl"), history).unwrap(); + write_outputs( + temp.path(), + &sample_metadata(), + &sample_results(), + DEFAULT_PAGES_URL, + DEFAULT_RAW_BASE_URL, + ) + .unwrap(); + + for file in [ + "manifest.json", + "latest.json", + "history.jsonl", + "benchmark.svg", + "index.html", + ".nojekyll", + ] { + assert!(temp.path().join(file).is_file(), "missing {file}"); + } + let manifest: Value = + serde_json::from_str(&fs::read_to_string(temp.path().join("manifest.json")).unwrap()) + .unwrap(); + assert_eq!(manifest["branch"], "benchmark-stats"); + assert_eq!( + manifest["artifacts"]["history"]["max_lines"], + HISTORY_MAX_LINES + ); + let history = fs::read_to_string(temp.path().join("history.jsonl")).unwrap(); + assert_eq!(history.lines().count(), HISTORY_MAX_LINES); + assert!(history.lines().last().unwrap().contains("0123456789abcdef")); + let html = fs::read_to_string(temp.path().join("index.html")).unwrap(); + assert!(html.contains("stable discovery index for agents")); + } +}