diff --git a/.github/workflows/loc-gate.yml b/.github/workflows/loc-gate.yml new file mode 100644 index 00000000..9d441401 --- /dev/null +++ b/.github/workflows/loc-gate.yml @@ -0,0 +1,39 @@ +name: LOC Gate + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + loc-gate: + name: Reject .rs files over 1000 LOC + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check .rs file line counts + shell: bash + run: | + set -euo pipefail + THRESHOLD=1000 + violations=0 + while IFS= read -r -d '' file; do + lines=$(wc -l < "$file") + if [ "$lines" -gt "$THRESHOLD" ]; then + echo "::error file=${file}::${file} has ${lines} LOC (limit: ${THRESHOLD})" + echo "FAIL ${lines} ${file}" + violations=$((violations + 1)) + fi + done < <(find . -type f -name '*.rs' \ + -not -path './target/*' \ + -not -path './.git/*' \ + -print0) + if [ "$violations" -gt 0 ]; then + echo "" + echo "Found ${violations} .rs file(s) over ${THRESHOLD} LOC." + echo "Split them into smaller modules before merging." + exit 1 + fi + echo "All .rs files are within the ${THRESHOLD} LOC limit." diff --git a/README.md b/README.md index a55cb872..7e1da2ba 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ [![Build Native Binaries](https://github.com/fastled/fbuild/actions/workflows/build.yml/badge.svg)](https://github.com/fastled/fbuild/actions/workflows/build.yml) [![Library Selection Acceptance (#205)](https://github.com/fastled/fbuild/actions/workflows/acceptance-205.yml/badge.svg)](https://github.com/fastled/fbuild/actions/workflows/acceptance-205.yml) [![Library Selection Perf (#205)](https://github.com/fastled/fbuild/actions/workflows/bench-205.yml/badge.svg)](https://github.com/fastled/fbuild/actions/workflows/bench-205.yml) +[![LOC Gate](https://github.com/fastled/fbuild/actions/workflows/loc-gate.yml/badge.svg)](https://github.com/fastled/fbuild/actions/workflows/loc-gate.yml)
Per-platform board build badges (click to expand) diff --git a/crates/fbuild-build/src/compile_database.rs b/crates/fbuild-build/src/compile_database.rs deleted file mode 100644 index c19db7ae..00000000 --- a/crates/fbuild-build/src/compile_database.rs +++ /dev/null @@ -1,1693 +0,0 @@ -//! Compile database (`compile_commands.json`) generation for IDE support. -//! -//! Generates a JSON compilation database (clangd-compatible) so that -//! "Go to Definition" and other IDE features work with real include paths -//! instead of response file (`@file`) references. - -use std::path::{Path, PathBuf}; - -use fbuild_core::Result; - -use crate::flag_overlay::LanguageExtraFlags; - -/// A single entry in the compile database. -#[derive(Debug, Clone, serde::Serialize)] -pub struct CompileEntry { - /// The compiler invocation as an argument list (preferred by clangd). - pub arguments: Vec, - /// The working directory for the compilation. - pub directory: String, - /// The source file being compiled. - pub file: String, - /// The output object file (optional per spec). - #[serde(skip_serializing_if = "Option::is_none")] - pub output: Option, -} - -/// Container for compile database entries. -pub struct CompileDatabase { - entries: Vec, -} - -impl Default for CompileDatabase { - fn default() -> Self { - Self::new() - } -} - -impl CompileDatabase { - /// Create an empty compile database. - pub fn new() -> Self { - Self { - entries: Vec::new(), - } - } - - /// Add an entry to the database. - pub fn add_entry(&mut self, entry: CompileEntry) { - self.entries.push(entry); - } - - /// Add multiple entries to the database. - pub fn extend(&mut self, entries: Vec) { - self.entries.extend(entries); - } - - /// Whether the database has any entries. - pub fn has_entries(&self) -> bool { - !self.entries.is_empty() - } - - /// Write `compile_commands.json` to the given directory. - pub fn write(&self, dir: &Path) -> Result { - std::fs::create_dir_all(dir).map_err(|e| { - fbuild_core::FbuildError::BuildFailed(format!( - "failed to create directory {}: {}", - dir.display(), - e - )) - })?; - - let path = dir.join("compile_commands.json"); - let json = serde_json::to_string_pretty(&self.entries).map_err(|e| { - fbuild_core::FbuildError::BuildFailed(format!( - "failed to serialize compile database: {}", - e - )) - })?; - - write_if_changed(&path, json.as_bytes())?; - - Ok(path) - } - - /// Write to `build_dir` and copy to `project_dir` (matching Python fbuild behavior). - /// - /// If the project has a `library.json` at its root (indicating it IS a library, - /// e.g. FastLED), the copy to project root is suppressed to avoid overwriting - /// a meson/cmake-generated `compile_commands.json` that uses correct source paths. - pub fn write_and_copy(&self, build_dir: &Path, project_dir: &Path) -> Result { - let build_path = self.write(build_dir)?; - - if is_library_project(project_dir) { - tracing::info!( - "library.json detected — skipping compile_commands.json copy to project root" - ); - return Ok(build_path); - } - - let project_path = self.write(project_dir)?; - Ok(project_path) - } - - /// Path callers should report as the effective compile database output. - pub fn expected_output_path(build_dir: &Path, project_dir: &Path) -> PathBuf { - if is_library_project(project_dir) { - build_dir.join("compile_commands.json") - } else { - project_dir.join("compile_commands.json") - } - } -} - -fn write_if_changed(path: &Path, contents: &[u8]) -> Result<()> { - if let Ok(existing) = std::fs::read(path) { - if existing == contents { - return Ok(()); - } - } - - std::fs::write(path, contents).map_err(|e| { - fbuild_core::FbuildError::BuildFailed(format!("failed to write {}: {}", path.display(), e)) - }) -} - -/// Strip cache wrapper (sccache/zccache/ccache) from compiler arguments. -/// -/// If the first element of `args` is a known cache wrapper, returns args -/// without it (the real compiler is the second element). Otherwise returns -/// args unchanged. -pub fn strip_cache_wrapper(args: &[String]) -> Vec { - if args.len() < 2 { - return args.to_vec(); - } - - // Extract the file stem manually so Windows paths (with `\`) work on Unix. - // `Path::file_stem` only splits on the platform's native separator, so - // `C:\...\sccache.exe` is treated as one component on Linux/macOS. - let filename = args[0].rsplit(['/', '\\']).next().unwrap_or(&args[0]); - let stem = filename - .strip_suffix(".exe") - .or_else(|| filename.strip_suffix(".EXE")) - .unwrap_or(filename) - .to_lowercase(); - - if stem == "sccache" || stem == "ccache" || stem == "zccache" { - if stem == "zccache" && args.get(1).is_some_and(|arg| arg == "wrap") { - if args.len() < 3 { - return args.to_vec(); - } - return args[2..].to_vec(); - } - args[1..].to_vec() - } else { - args.to_vec() - } -} - -/// Target architecture for clang flag translation. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TargetArchitecture { - Xtensa, - Riscv32, - Avr, - Arm, -} - -impl TargetArchitecture { - pub fn target_triple(&self) -> &'static str { - match self { - Self::Xtensa => "xtensa-esp-elf", - Self::Riscv32 => "riscv32-esp-elf", - Self::Avr => "avr", - Self::Arm => "arm-none-eabi", - } - } -} - -/// Check whether a GCC-specific flag should be removed for clang. -fn should_remove_flag(flag: &str, arch: TargetArchitecture) -> bool { - // Common GCC-only flags unsupported by clang / IWYU - match flag { - "-flto=auto" - | "-flto" - | "-fno-fat-lto-objects" - | "-fuse-linker-plugin" - | "-ffat-lto-objects" - | "-freorder-blocks" - | "-fno-jump-tables" => return true, - _ => {} - } - - match arch { - TargetArchitecture::Xtensa => { - matches!( - flag, - "-mlongcalls" - | "-mdisable-hardware-atomics" - | "-mfix-esp32-psram-cache-issue" - | "-fstrict-volatile-bitfields" - | "-mtext-section-literals" - | "-fno-tree-switch-conversion" - ) || flag.starts_with("-mfix-esp32-psram-cache-strategy=") - } - TargetArchitecture::Riscv32 => matches!(flag, "-mabi=ilp32" | "-mno-fdiv"), - TargetArchitecture::Arm => flag == "-mthumb-interwork", - TargetArchitecture::Avr => false, - } -} - -/// Translate compiler arguments from GCC to clang-compatible equivalents. -/// -/// - Replaces the GCC/G++ compiler path with `clang`/`clang++` -/// - Inserts `--target=` as the second argument -/// - Removes architecture-specific flags that clang doesn't understand -pub fn translate_flags_for_clang(args: &[String], arch: TargetArchitecture) -> Vec { - if args.is_empty() { - return Vec::new(); - } - - let mut result = Vec::with_capacity(args.len() + 1); - - // Replace compiler path: detect g++ vs gcc by checking the normalized path - let compiler_path = args[0].to_lowercase().replace('\\', "/"); - let clang_name = if compiler_path.ends_with("g++") || compiler_path.ends_with("g++.exe") { - "clang++" - } else { - "clang" - }; - result.push(clang_name.to_string()); - - // Add target triple as second argument - result.push(format!("--target={}", arch.target_triple())); - - // Filter remaining args - for arg in &args[1..] { - if !should_remove_flag(arg, arch) { - result.push(arg.clone()); - } - } - - result -} - -impl CompileDatabase { - /// Create a new compile database with GCC flags translated to clang equivalents. - pub fn translate_for_clang(&self, arch: TargetArchitecture) -> CompileDatabase { - let entries = self - .entries - .iter() - .map(|entry| CompileEntry { - arguments: translate_flags_for_clang(&entry.arguments, arch), - directory: entry.directory.clone(), - file: entry.file.clone(), - output: entry.output.clone(), - }) - .collect(); - CompileDatabase { entries } - } - - /// Prepare compile database for IWYU (include-what-you-use) analysis. - /// - /// Transforms the existing (already clang-translated) compile database so that - /// IWYU can process cross-compiled embedded code: - /// - /// - Removes `--target=` flags (IWYU doesn't need code generation support) - /// - Deduplicates `-D` defines (keeps first occurrence of each key) - /// - Converts non-project `-I` paths to `-isystem` (suppresses IWYU suggestions) - /// - Adds extra `-isystem` paths (e.g. GCC toolchain builtin includes) - pub fn prepare_for_iwyu( - &self, - project_src_dir: &Path, - extra_system_includes: &[PathBuf], - ) -> CompileDatabase { - let src_prefix = project_src_dir.to_string_lossy().to_lowercase(); - let entries = self - .entries - .iter() - .map(|entry| { - let mut args = - Vec::with_capacity(entry.arguments.len() + extra_system_includes.len() * 2); - let mut seen_defines = std::collections::HashSet::new(); - - for arg in &entry.arguments { - // Remove --target= flags - if arg.starts_with("--target=") { - continue; - } - - // Deduplicate -D flags (keep first occurrence by key) - if arg.starts_with("-D") { - let key = if let Some(eq_pos) = arg.find('=') { - &arg[..eq_pos] - } else { - arg.as_str() - }; - if !seen_defines.insert(key.to_string()) { - continue; - } - } - - // Convert non-project -I to -isystem (suppresses IWYU analysis) - if let Some(path) = arg.strip_prefix("-I") { - let normalized = path.replace('\\', "/").to_lowercase(); - if normalized.starts_with(&src_prefix) { - args.push(arg.clone()); - } else { - args.push("-isystem".to_string()); - args.push(path.to_string()); - } - continue; - } - - args.push(arg.clone()); - } - - // Append GCC toolchain builtin include dirs as -isystem - for inc in extra_system_includes { - args.push("-isystem".to_string()); - args.push(inc.to_string_lossy().to_string()); - } - - CompileEntry { - arguments: args, - directory: entry.directory.clone(), - file: entry.file.clone(), - output: entry.output.clone(), - } - }) - .collect(); - CompileDatabase { entries } - } -} - -/// Check if a project is a library (has `library.json` at the root). -/// -/// Library projects (e.g. FastLED) often have their own build system that -/// generates a correct `compile_commands.json`. We skip overwriting the -/// project root file to avoid clobbering it. -pub fn is_library_project(project_dir: &Path) -> bool { - project_dir.join("library.json").exists() -} - -/// Generate compile database entries for a set of source files. -/// -/// # Arguments -/// - `gcc_path` / `gxx_path` — real compiler paths (not cache wrappers) -/// - `c_flags` / `cpp_flags` — language-specific flags -/// - `include_flags` — separate `-I` flags (for ESP32; empty for AVR/Teensy where they're in c/cpp_flags) -/// - `extra_flags` — user/src flags -/// - `sources` — source files to generate entries for -/// - `build_dir` — where object files go (for `-o` path) -/// - `project_dir` — used as the `directory` field -#[allow(clippy::too_many_arguments)] -pub fn generate_entries( - gcc_path: &Path, - gxx_path: &Path, - c_flags: &[String], - cpp_flags: &[String], - include_flags: &[String], - extra_flags: &LanguageExtraFlags, - sources: &[PathBuf], - build_dir: &Path, - project_dir: &Path, -) -> Vec { - let directory = project_dir.to_string_lossy().to_string(); - - sources - .iter() - .map(|source| { - let ext = source - .extension() - .unwrap_or_default() - .to_string_lossy() - .to_lowercase(); - - let (compiler, flags) = match ext.as_str() { - "c" | "s" => (gcc_path, c_flags), - _ => (gxx_path, cpp_flags), - }; - - let obj = crate::compiler::CompilerBase::object_path(source, build_dir); - let source_extra_flags = extra_flags.for_source(source); - - let mut arguments = Vec::with_capacity( - 1 + flags.len() + include_flags.len() + source_extra_flags.len() + 4, - ); - arguments.push(compiler.to_string_lossy().to_string()); - arguments.extend(flags.iter().cloned()); - arguments.extend(include_flags.iter().cloned()); - arguments.extend(source_extra_flags); - arguments.push("-c".to_string()); - arguments.push(source.to_string_lossy().to_string()); - arguments.push("-o".to_string()); - arguments.push(obj.to_string_lossy().to_string()); - - CompileEntry { - arguments, - directory: directory.clone(), - file: source.to_string_lossy().to_string(), - output: Some(obj.to_string_lossy().to_string()), - } - }) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[allow(clippy::too_many_arguments)] - fn generate_entries( - gcc_path: &Path, - gxx_path: &Path, - c_flags: &[String], - cpp_flags: &[String], - include_flags: &[String], - extra_flags: &[String], - sources: &[PathBuf], - build_dir: &Path, - project_dir: &Path, - ) -> Vec { - super::generate_entries( - gcc_path, - gxx_path, - c_flags, - cpp_flags, - include_flags, - &LanguageExtraFlags { - common: extra_flags.to_vec(), - c: Vec::new(), - cxx: Vec::new(), - asm: Vec::new(), - }, - sources, - build_dir, - project_dir, - ) - } - - // --- Serialization tests --- - - #[test] - fn test_compile_entry_serialization() { - let entry = CompileEntry { - arguments: vec![ - "/usr/bin/gcc".to_string(), - "-c".to_string(), - "main.c".to_string(), - ], - directory: "/project".to_string(), - file: "main.c".to_string(), - output: Some("main.o".to_string()), - }; - - let json = serde_json::to_value(&entry).unwrap(); - assert_eq!(json["directory"], "/project"); - assert_eq!(json["file"], "main.c"); - assert_eq!(json["output"], "main.o"); - assert!(json["arguments"].is_array()); - } - - #[test] - fn test_compile_entry_output_none_omitted() { - let entry = CompileEntry { - arguments: vec!["/usr/bin/gcc".to_string()], - directory: "/project".to_string(), - file: "main.c".to_string(), - output: None, - }; - - let json = serde_json::to_string(&entry).unwrap(); - assert!(!json.contains("output")); - } - - // --- CompileDatabase container tests --- - - #[test] - fn test_database_empty() { - let db = CompileDatabase::new(); - assert!(!db.has_entries()); - } - - #[test] - fn test_database_add_entry() { - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec![], - directory: String::new(), - file: "test.c".to_string(), - output: None, - }); - assert!(db.has_entries()); - } - - #[test] - fn test_database_write_valid_json() { - let tmp = tempfile::TempDir::new().unwrap(); - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec!["/usr/bin/gcc".to_string(), "-c".to_string()], - directory: "/project".to_string(), - file: "main.c".to_string(), - output: None, - }); - - let path = db.write(tmp.path()).unwrap(); - assert!(path.exists()); - - let content = std::fs::read_to_string(&path).unwrap(); - let parsed: serde_json::Value = serde_json::from_str(&content).unwrap(); - assert!(parsed.is_array()); - assert_eq!(parsed.as_array().unwrap().len(), 1); - assert_eq!(parsed[0]["file"], "main.c"); - } - - #[test] - fn test_database_write_creates_parent_dirs() { - let tmp = tempfile::TempDir::new().unwrap(); - let nested = tmp.path().join("a").join("b").join("c"); - let db = CompileDatabase::new(); - let path = db.write(&nested).unwrap(); - assert!(path.exists()); - } - - #[test] - fn test_database_write_and_copy() { - let tmp = tempfile::TempDir::new().unwrap(); - let build_dir = tmp.path().join("build"); - let project_dir = tmp.path().join("project"); - - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec![], - directory: String::new(), - file: "test.c".to_string(), - output: None, - }); - - db.write_and_copy(&build_dir, &project_dir).unwrap(); - assert!(build_dir.join("compile_commands.json").exists()); - assert!(project_dir.join("compile_commands.json").exists()); - } - - #[test] - fn test_database_write_does_not_rewrite_unchanged_contents() { - let tmp = tempfile::TempDir::new().unwrap(); - let dir = tmp.path().join("build"); - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec!["/usr/bin/gcc".to_string(), "-c".to_string()], - directory: "/project".to_string(), - file: "main.c".to_string(), - output: None, - }); - - let path = db.write(&dir).unwrap(); - let first_mtime = std::fs::metadata(&path).unwrap().modified().unwrap(); - std::thread::sleep(std::time::Duration::from_millis(20)); - - let path_again = db.write(&dir).unwrap(); - let second_mtime = std::fs::metadata(&path_again).unwrap().modified().unwrap(); - - assert_eq!(path, path_again); - assert_eq!(first_mtime, second_mtime); - } - - #[test] - fn test_expected_output_path_prefers_project_root_for_normal_projects() { - let tmp = tempfile::TempDir::new().unwrap(); - let build_dir = tmp.path().join("build"); - let project_dir = tmp.path().join("project"); - std::fs::create_dir_all(&project_dir).unwrap(); - - assert_eq!( - CompileDatabase::expected_output_path(&build_dir, &project_dir), - project_dir.join("compile_commands.json") - ); - } - - #[test] - fn test_expected_output_path_prefers_build_dir_for_library_projects() { - let tmp = tempfile::TempDir::new().unwrap(); - let build_dir = tmp.path().join("build"); - let project_dir = tmp.path().join("project"); - std::fs::create_dir_all(&project_dir).unwrap(); - std::fs::write(project_dir.join("library.json"), r#"{"name":"test"}"#).unwrap(); - - assert_eq!( - CompileDatabase::expected_output_path(&build_dir, &project_dir), - build_dir.join("compile_commands.json") - ); - } - - // --- Cache wrapper stripping tests --- - - #[test] - fn test_strip_sccache() { - let args = vec![ - "sccache".to_string(), - "/usr/bin/gcc".to_string(), - "-c".to_string(), - ]; - let stripped = strip_cache_wrapper(&args); - assert_eq!(stripped[0], "/usr/bin/gcc"); - assert_eq!(stripped.len(), 2); - } - - #[test] - fn test_strip_zccache() { - let args = vec![ - "/path/to/zccache".to_string(), - "/usr/bin/gcc".to_string(), - "-c".to_string(), - ]; - let stripped = strip_cache_wrapper(&args); - assert_eq!(stripped[0], "/usr/bin/gcc"); - } - - #[test] - fn test_strip_zccache_wrap_mode() { - let args = vec![ - "C:\\tools\\zccache.exe".to_string(), - "wrap".to_string(), - "C:\\tc\\bin\\xtensa-esp32-elf-g++.exe".to_string(), - "-c".to_string(), - ]; - let stripped = strip_cache_wrapper(&args); - assert_eq!(stripped[0], "C:\\tc\\bin\\xtensa-esp32-elf-g++.exe"); - assert_eq!(stripped[1], "-c"); - } - - #[test] - fn test_strip_ccache() { - let args = vec![ - "ccache".to_string(), - "/usr/bin/gcc".to_string(), - "-c".to_string(), - ]; - let stripped = strip_cache_wrapper(&args); - assert_eq!(stripped[0], "/usr/bin/gcc"); - } - - #[test] - fn test_strip_no_wrapper() { - let args = vec!["/usr/bin/gcc".to_string(), "-c".to_string()]; - let stripped = strip_cache_wrapper(&args); - assert_eq!(stripped, args); - } - - #[test] - fn test_strip_empty() { - let args: Vec = vec![]; - let stripped = strip_cache_wrapper(&args); - assert!(stripped.is_empty()); - } - - // --- Entry generation tests --- - - #[test] - fn test_generate_entries_c_uses_gcc() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &["-std=c11".to_string()], - &["-std=c++17".to_string()], - &[], - &[], - &[PathBuf::from("main.c")], - Path::new("/build"), - Path::new("/project"), - ); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].arguments[0], "/usr/bin/gcc"); - assert!(entries[0].arguments.contains(&"-std=c11".to_string())); - } - - #[test] - fn test_generate_entries_cpp_uses_gxx() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &["-std=c11".to_string()], - &["-std=c++17".to_string()], - &[], - &[], - &[PathBuf::from("main.cpp")], - Path::new("/build"), - Path::new("/project"), - ); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); - assert!(entries[0].arguments.contains(&"-std=c++17".to_string())); - } - - #[test] - fn test_generate_entries_s_uses_gcc() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &["-std=c11".to_string()], - &["-std=c++17".to_string()], - &[], - &[], - &[PathBuf::from("startup.s")], - Path::new("/build"), - Path::new("/project"), - ); - assert_eq!(entries[0].arguments[0], "/usr/bin/gcc"); - } - - #[test] - fn test_generate_entries_empty_sources() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &[], - &[], - Path::new("/build"), - Path::new("/project"), - ); - assert!(entries.is_empty()); - } - - #[test] - fn test_generate_entries_include_flags_in_args() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &["-I/sdk/include".to_string(), "-I/core/include".to_string()], - &[], - &[PathBuf::from("main.cpp")], - Path::new("/build"), - Path::new("/project"), - ); - assert!(entries[0].arguments.contains(&"-I/sdk/include".to_string())); - assert!(entries[0] - .arguments - .contains(&"-I/core/include".to_string())); - } - - #[test] - fn test_generate_entries_extra_flags_in_args() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &["-DUSER_FLAG=1".to_string()], - &[PathBuf::from("main.cpp")], - Path::new("/build"), - Path::new("/project"), - ); - assert!(entries[0].arguments.contains(&"-DUSER_FLAG=1".to_string())); - } - - #[test] - fn test_generate_entries_directory_is_project_dir() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &[], - &[PathBuf::from("main.cpp")], - Path::new("/build"), - Path::new("/my/project"), - ); - assert_eq!(entries[0].directory, "/my/project"); - } - - #[test] - fn test_generate_entries_arguments_structure() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &["-Os".to_string()], - &["-Os".to_string()], - &["-I/inc".to_string()], - &["-DFOO".to_string()], - &[PathBuf::from("main.c")], - Path::new("/build"), - Path::new("/project"), - ); - - let args = &entries[0].arguments; - // Starts with compiler - assert_eq!(args[0], "/usr/bin/gcc"); - // Ends with -c source -o object - let len = args.len(); - assert_eq!(args[len - 4], "-c"); - assert_eq!(args[len - 3], "main.c"); - assert_eq!(args[len - 2], "-o"); - } - - // ========================================================================= - // Adversarial tests — designed to expose edge-case bugs - // ========================================================================= - - // --- Cache wrapper stripping edge cases --- - - #[test] - fn test_strip_cache_wrapper_windows_exe() { - let args = vec![ - "C:\\Users\\user\\.cargo\\bin\\sccache.exe".to_string(), - "C:\\tools\\gcc.exe".to_string(), - "-c".to_string(), - ]; - let stripped = strip_cache_wrapper(&args); - assert_eq!(stripped[0], "C:\\tools\\gcc.exe"); - assert_eq!(stripped.len(), 2); - } - - #[test] - fn test_strip_cache_wrapper_case_insensitive() { - // Windows file systems are case-insensitive - for name in &[ - "SCCACHE", "Sccache", "ZCCACHE", "Zccache", "CCACHE", "Ccache", - ] { - let args = vec![ - name.to_string(), - "/usr/bin/gcc".to_string(), - "-c".to_string(), - ]; - let stripped = strip_cache_wrapper(&args); - assert_eq!( - stripped[0], "/usr/bin/gcc", - "failed to strip cache wrapper: {}", - name - ); - } - } - - #[test] - fn test_strip_cache_wrapper_single_element_wrapper() { - // Only the wrapper, no actual compiler — should return as-is - let args = vec!["sccache".to_string()]; - let stripped = strip_cache_wrapper(&args); - assert_eq!(stripped, args); - } - - #[test] - fn test_strip_cache_wrapper_not_a_wrapper() { - // File named "sccache-stats" shouldn't be stripped - let args = vec!["sccache-stats".to_string(), "/usr/bin/gcc".to_string()]; - let stripped = strip_cache_wrapper(&args); - // file_stem of "sccache-stats" is "sccache-stats", not "sccache" - assert_eq!(stripped.len(), 2); - assert_eq!(stripped[0], "sccache-stats"); - } - - // --- Extension classification adversarial tests --- - - #[test] - fn test_generate_entries_uppercase_c_extension() { - // .C is treated as C++ on some systems, but our lowercase normalization - // maps it to "c" → gcc. This matches GCC behavior. - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &[], - &[PathBuf::from("main.C")], - Path::new("/build"), - Path::new("/project"), - ); - // After to_lowercase(), ".C" becomes "c" → uses gcc - assert_eq!(entries[0].arguments[0], "/usr/bin/gcc"); - } - - #[test] - fn test_generate_entries_cc_extension_uses_gxx() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &[], - &[PathBuf::from("module.cc")], - Path::new("/build"), - Path::new("/project"), - ); - assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); - } - - #[test] - fn test_generate_entries_cxx_extension_uses_gxx() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &[], - &[PathBuf::from("module.cxx")], - Path::new("/build"), - Path::new("/project"), - ); - assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); - } - - #[test] - fn test_generate_entries_ino_cpp_uses_gxx() { - // Preprocessed .ino files become .ino.cpp — extension is "cpp" - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &[], - &[PathBuf::from("sketch.ino.cpp")], - Path::new("/build"), - Path::new("/project"), - ); - assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); - } - - #[test] - fn test_generate_entries_no_extension_uses_gxx() { - // Files without extension fall through to g++ (the default branch) - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &[], - &[PathBuf::from("Makefile")], - Path::new("/build"), - Path::new("/project"), - ); - assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); - } - - #[test] - fn test_generate_entries_uppercase_s_assembly_uses_gcc() { - // .S (uppercase) is GCC-preprocessed assembly, should use gcc - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &[], - &[PathBuf::from("boot.S")], - Path::new("/build"), - Path::new("/project"), - ); - // to_lowercase() → "s" → matches gcc branch - assert_eq!(entries[0].arguments[0], "/usr/bin/gcc"); - } - - // --- Path handling adversarial tests --- - - #[test] - fn test_generate_entries_paths_with_spaces() { - let entries = generate_entries( - Path::new("/usr/bin/my gcc"), - Path::new("/usr/bin/my g++"), - &[], - &[], - &["-I/path with spaces/include".to_string()], - &[], - &[PathBuf::from("/my project/src/main.cpp")], - Path::new("/my build"), - Path::new("/my project"), - ); - assert_eq!(entries[0].directory, "/my project"); - assert_eq!(entries[0].file, "/my project/src/main.cpp"); - assert!(entries[0] - .arguments - .contains(&"-I/path with spaces/include".to_string())); - } - - #[test] - fn test_generate_entries_windows_backslash_paths() { - let entries = generate_entries( - Path::new("C:\\tools\\gcc.exe"), - Path::new("C:\\tools\\g++.exe"), - &[], - &[], - &[], - &[], - &[PathBuf::from("C:\\Users\\user\\project\\src\\main.cpp")], - Path::new("C:\\Users\\user\\build"), - Path::new("C:\\Users\\user\\project"), - ); - // to_string_lossy preserves original path separators - assert!(!entries[0].file.is_empty()); - assert!(!entries[0].directory.is_empty()); - // The output field should point to the build dir - assert!( - entries[0].output.as_ref().unwrap().contains("build") - || entries[0].output.as_ref().unwrap().contains("Users") - ); - } - - // --- Arguments must never contain @file (response file) references --- - - #[test] - fn test_generate_entries_no_response_file_in_args() { - // Even with many include flags, generate_entries should produce - // individual -I flags, never @file references (those are for GCC only). - let include_flags: Vec = - (0..300).map(|i| format!("-I/sdk/include/{}", i)).collect(); - - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &include_flags, - &[], - &[PathBuf::from("main.cpp")], - Path::new("/build"), - Path::new("/project"), - ); - - for arg in &entries[0].arguments { - assert!( - !arg.starts_with('@'), - "compile_commands.json must not contain @file references: {}", - arg - ); - } - // All 300 include flags should be present individually - assert!(entries[0] - .arguments - .contains(&"-I/sdk/include/0".to_string())); - assert!(entries[0] - .arguments - .contains(&"-I/sdk/include/299".to_string())); - } - - // --- File field must be the source path, not the build path --- - - #[test] - fn test_generate_entries_file_is_source_not_build() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &[], - &[PathBuf::from("/project/src/main.cpp")], - Path::new("/project/.fbuild/build/esp32/src"), - Path::new("/project"), - ); - assert_eq!(entries[0].file, "/project/src/main.cpp"); - // Output should be in the build dir - assert!( - entries[0].output.as_ref().unwrap().contains(".fbuild"), - "output should be in build dir: {:?}", - entries[0].output - ); - } - - // --- write_and_copy: both files must have identical content --- - - #[test] - fn test_write_and_copy_identical_content() { - let tmp = tempfile::TempDir::new().unwrap(); - let build_dir = tmp.path().join("build"); - let project_dir = tmp.path().join("project"); - - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec![ - "/usr/bin/g++".to_string(), - "-c".to_string(), - "main.cpp".to_string(), - ], - directory: "/project".to_string(), - file: "main.cpp".to_string(), - output: Some("main.cpp.o".to_string()), - }); - - db.write_and_copy(&build_dir, &project_dir).unwrap(); - - let build_content = - std::fs::read_to_string(build_dir.join("compile_commands.json")).unwrap(); - let project_content = - std::fs::read_to_string(project_dir.join("compile_commands.json")).unwrap(); - assert_eq!(build_content, project_content); - } - - // --- write_and_copy: suppressed when library.json exists --- - - #[test] - fn test_write_and_copy_suppressed_for_library_project() { - let tmp = tempfile::TempDir::new().unwrap(); - let build_dir = tmp.path().join("build"); - let project_dir = tmp.path().join("project"); - std::fs::create_dir_all(&project_dir).unwrap(); - - // Create library.json to simulate a library project (like FastLED) - std::fs::write( - project_dir.join("library.json"), - r#"{"name": "FastLED", "version": "3.10.3"}"#, - ) - .unwrap(); - - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec!["/usr/bin/g++".to_string()], - directory: project_dir.to_string_lossy().to_string(), - file: "main.cpp".to_string(), - output: None, - }); - - let result_path = db.write_and_copy(&build_dir, &project_dir).unwrap(); - - // Build dir should have the file - assert!(build_dir.join("compile_commands.json").exists()); - // Project dir should NOT have compile_commands.json (suppressed) - assert!( - !project_dir.join("compile_commands.json").exists(), - "compile_commands.json should NOT be copied to project root for library projects" - ); - // The returned path should be the build dir path - assert_eq!(result_path, build_dir.join("compile_commands.json")); - } - - #[test] - fn test_write_and_copy_not_suppressed_for_sketch_project() { - let tmp = tempfile::TempDir::new().unwrap(); - let build_dir = tmp.path().join("build"); - let project_dir = tmp.path().join("project"); - std::fs::create_dir_all(&project_dir).unwrap(); - - // No library.json — this is a normal sketch project - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec!["/usr/bin/g++".to_string()], - directory: project_dir.to_string_lossy().to_string(), - file: "main.cpp".to_string(), - output: None, - }); - - db.write_and_copy(&build_dir, &project_dir).unwrap(); - - // Both should exist - assert!(build_dir.join("compile_commands.json").exists()); - assert!(project_dir.join("compile_commands.json").exists()); - } - - // --- is_library_project detection --- - - #[test] - fn test_is_library_project_with_library_json() { - let tmp = tempfile::TempDir::new().unwrap(); - std::fs::write(tmp.path().join("library.json"), r#"{"name": "MyLib"}"#).unwrap(); - assert!(is_library_project(tmp.path())); - } - - #[test] - fn test_is_library_project_without_library_json() { - let tmp = tempfile::TempDir::new().unwrap(); - assert!(!is_library_project(tmp.path())); - } - - // --- Empty database produces valid JSON --- - - #[test] - fn test_write_empty_database_valid_json() { - let tmp = tempfile::TempDir::new().unwrap(); - let db = CompileDatabase::new(); - let path = db.write(tmp.path()).unwrap(); - - let content = std::fs::read_to_string(&path).unwrap(); - let parsed: serde_json::Value = serde_json::from_str(&content).unwrap(); - assert!(parsed.is_array()); - assert!(parsed.as_array().unwrap().is_empty()); - } - - // --- Mixed source types in a single call --- - - #[test] - fn test_generate_entries_mixed_sources() { - let sources = vec![ - PathBuf::from("main.cpp"), - PathBuf::from("util.c"), - PathBuf::from("boot.S"), - PathBuf::from("driver.cc"), - PathBuf::from("algo.cxx"), - ]; - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &["-std=c11".to_string()], - &["-std=c++17".to_string()], - &[], - &[], - &sources, - Path::new("/build"), - Path::new("/project"), - ); - assert_eq!(entries.len(), 5); - - // main.cpp → g++ - assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); - assert!(entries[0].arguments.contains(&"-std=c++17".to_string())); - - // util.c → gcc - assert_eq!(entries[1].arguments[0], "/usr/bin/gcc"); - assert!(entries[1].arguments.contains(&"-std=c11".to_string())); - - // boot.S → gcc (assembly) - assert_eq!(entries[2].arguments[0], "/usr/bin/gcc"); - - // driver.cc → g++ - assert_eq!(entries[3].arguments[0], "/usr/bin/g++"); - - // algo.cxx → g++ - assert_eq!(entries[4].arguments[0], "/usr/bin/g++"); - } - - // --- Duplicate sources don't panic --- - - #[test] - fn test_generate_entries_duplicate_sources() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &[], - &[PathBuf::from("main.cpp"), PathBuf::from("main.cpp")], - Path::new("/build"), - Path::new("/project"), - ); - // Both entries should exist (dedup is the caller's responsibility) - assert_eq!(entries.len(), 2); - } - - // --- Include flags with build dir paths (the clangd navigation issue) --- - - #[test] - fn test_generate_entries_include_flags_preserved_verbatim() { - // The compile database should faithfully reproduce whatever include - // flags it receives. The ORCHESTRATOR is responsible for passing - // source-tree paths, not build-dir paths. - let include_flags = vec![ - "-I/project/src".to_string(), // source tree ✓ - "-I/home/user/.fbuild/build/esp32/libs/fastled/src".to_string(), // cache path ✗ - "-I/framework/cores/esp32".to_string(), // framework ✓ - ]; - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &include_flags, - &[], - &[PathBuf::from("main.cpp")], - Path::new("/build"), - Path::new("/project"), - ); - // All include flags should be present, unmodified - for flag in &include_flags { - assert!( - entries[0].arguments.contains(flag), - "missing include flag: {}", - flag - ); - } - } - - // --- Output path uses build_dir, not project source dir --- - - #[test] - fn test_generate_entries_output_in_build_dir() { - let entries = generate_entries( - Path::new("/usr/bin/gcc"), - Path::new("/usr/bin/g++"), - &[], - &[], - &[], - &[], - &[PathBuf::from("/project/src/main.cpp")], - Path::new("/build/obj"), - Path::new("/project"), - ); - let output = entries[0].output.as_ref().unwrap(); - assert!( - output.starts_with("/build/obj"), - "output should start with build dir: {}", - output - ); - } - - // --- Extend adds all entries --- - - #[test] - fn test_database_extend_accumulates() { - let mut db = CompileDatabase::new(); - let entries1 = vec![CompileEntry { - arguments: vec![], - directory: String::new(), - file: "a.c".to_string(), - output: None, - }]; - let entries2 = vec![ - CompileEntry { - arguments: vec![], - directory: String::new(), - file: "b.c".to_string(), - output: None, - }, - CompileEntry { - arguments: vec![], - directory: String::new(), - file: "c.c".to_string(), - output: None, - }, - ]; - db.extend(entries1); - db.extend(entries2); - // Should have all 3 entries - let tmp = tempfile::TempDir::new().unwrap(); - let path = db.write(tmp.path()).unwrap(); - let content = std::fs::read_to_string(&path).unwrap(); - let parsed: serde_json::Value = serde_json::from_str(&content).unwrap(); - assert_eq!(parsed.as_array().unwrap().len(), 3); - } - - // ========================================================================= - // Clang flag translation tests - // ========================================================================= - - #[test] - fn test_target_triples() { - assert_eq!(TargetArchitecture::Xtensa.target_triple(), "xtensa-esp-elf"); - assert_eq!( - TargetArchitecture::Riscv32.target_triple(), - "riscv32-esp-elf" - ); - assert_eq!(TargetArchitecture::Avr.target_triple(), "avr"); - assert_eq!(TargetArchitecture::Arm.target_triple(), "arm-none-eabi"); - } - - #[test] - fn test_translate_gcc_to_clang() { - let args = vec![ - "/usr/bin/avr-gcc".to_string(), - "-Os".to_string(), - "-c".to_string(), - ]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Avr); - assert_eq!(result[0], "clang"); - } - - #[test] - fn test_translate_gxx_to_clangxx() { - let args = vec!["/usr/bin/arm-none-eabi-g++".to_string(), "-Os".to_string()]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Arm); - assert_eq!(result[0], "clang++"); - } - - #[test] - fn test_translate_windows_compiler_path() { - let args = vec![ - "C:\\tools\\xtensa-esp32-elf-g++.exe".to_string(), - "-Os".to_string(), - ]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Xtensa); - assert_eq!(result[0], "clang++"); - } - - #[test] - fn test_translate_adds_target() { - let args = vec!["/usr/bin/gcc".to_string(), "-c".to_string()]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Xtensa); - assert_eq!(result[1], "--target=xtensa-esp-elf"); - } - - #[test] - fn test_translate_removes_common_lto_flags() { - let args = vec![ - "/usr/bin/gcc".to_string(), - "-flto=auto".to_string(), - "-flto".to_string(), - "-fno-fat-lto-objects".to_string(), - "-fuse-linker-plugin".to_string(), - "-ffat-lto-objects".to_string(), - "-Os".to_string(), - ]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Avr); - assert!(!result.contains(&"-flto=auto".to_string())); - assert!(!result.contains(&"-flto".to_string())); - assert!(!result.contains(&"-fno-fat-lto-objects".to_string())); - assert!(!result.contains(&"-fuse-linker-plugin".to_string())); - assert!(!result.contains(&"-ffat-lto-objects".to_string())); - assert!(result.contains(&"-Os".to_string())); - } - - #[test] - fn test_translate_xtensa_removals() { - let args = vec![ - "/usr/bin/xtensa-esp32-elf-gcc".to_string(), - "-mlongcalls".to_string(), - "-mdisable-hardware-atomics".to_string(), - "-mfix-esp32-psram-cache-issue".to_string(), - "-fstrict-volatile-bitfields".to_string(), - "-mtext-section-literals".to_string(), - "-fno-tree-switch-conversion".to_string(), - "-Os".to_string(), - ]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Xtensa); - assert!(!result.contains(&"-mlongcalls".to_string())); - assert!(!result.contains(&"-mdisable-hardware-atomics".to_string())); - assert!(!result.contains(&"-mfix-esp32-psram-cache-issue".to_string())); - assert!(!result.contains(&"-fstrict-volatile-bitfields".to_string())); - assert!(!result.contains(&"-mtext-section-literals".to_string())); - assert!(!result.contains(&"-fno-tree-switch-conversion".to_string())); - assert!(result.contains(&"-Os".to_string())); - } - - #[test] - fn test_translate_xtensa_psram_strategy_prefix() { - let args = vec![ - "/usr/bin/gcc".to_string(), - "-mfix-esp32-psram-cache-strategy=memw".to_string(), - "-Os".to_string(), - ]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Xtensa); - assert!(!result.contains(&"-mfix-esp32-psram-cache-strategy=memw".to_string())); - assert!(result.contains(&"-Os".to_string())); - } - - #[test] - fn test_translate_riscv_removals() { - let args = vec![ - "/usr/bin/riscv32-esp-elf-gcc".to_string(), - "-mabi=ilp32".to_string(), - "-mno-fdiv".to_string(), - "-march=rv32imac".to_string(), - "-Os".to_string(), - ]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Riscv32); - assert!(!result.contains(&"-mabi=ilp32".to_string())); - assert!(!result.contains(&"-mno-fdiv".to_string())); - assert!(result.contains(&"-march=rv32imac".to_string())); - } - - #[test] - fn test_translate_arm_removals() { - let args = vec![ - "/usr/bin/arm-none-eabi-g++".to_string(), - "-mthumb-interwork".to_string(), - "-mcpu=cortex-m7".to_string(), - "-Os".to_string(), - ]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Arm); - assert!(!result.contains(&"-mthumb-interwork".to_string())); - assert!(result.contains(&"-mcpu=cortex-m7".to_string())); - } - - #[test] - fn test_translate_avr_no_extra_removals() { - let args = vec![ - "/usr/bin/avr-gcc".to_string(), - "-mmcu=atmega328p".to_string(), - "-Os".to_string(), - ]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Avr); - assert!(result.contains(&"-mmcu=atmega328p".to_string())); - assert!(result.contains(&"-Os".to_string())); - } - - #[test] - fn test_translate_preserves_includes_and_defines() { - let args = vec![ - "/usr/bin/gcc".to_string(), - "-I/path/to/include".to_string(), - "-DFOO=1".to_string(), - "-c".to_string(), - ]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Avr); - assert!(result.contains(&"-I/path/to/include".to_string())); - assert!(result.contains(&"-DFOO=1".to_string())); - } - - #[test] - fn test_translate_empty_args() { - let args: Vec = vec![]; - let result = translate_flags_for_clang(&args, TargetArchitecture::Avr); - assert!(result.is_empty()); - } - - #[test] - fn test_database_translate_for_clang() { - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec![ - "/usr/bin/xtensa-esp32-elf-gcc".to_string(), - "-mlongcalls".to_string(), - "-Os".to_string(), - "-c".to_string(), - "main.c".to_string(), - ], - directory: "/project".to_string(), - file: "main.c".to_string(), - output: Some("main.o".to_string()), - }); - db.add_entry(CompileEntry { - arguments: vec![ - "/usr/bin/xtensa-esp32-elf-g++".to_string(), - "-mlongcalls".to_string(), - "-std=c++17".to_string(), - "-c".to_string(), - "app.cpp".to_string(), - ], - directory: "/project".to_string(), - file: "app.cpp".to_string(), - output: Some("app.o".to_string()), - }); - - let translated = db.translate_for_clang(TargetArchitecture::Xtensa); - - // First entry: gcc → clang - assert_eq!(translated.entries[0].arguments[0], "clang"); - assert_eq!( - translated.entries[0].arguments[1], - "--target=xtensa-esp-elf" - ); - assert!(!translated.entries[0] - .arguments - .contains(&"-mlongcalls".to_string())); - assert!(translated.entries[0].arguments.contains(&"-Os".to_string())); - assert_eq!(translated.entries[0].file, "main.c"); - - // Second entry: g++ → clang++ - assert_eq!(translated.entries[1].arguments[0], "clang++"); - assert!(!translated.entries[1] - .arguments - .contains(&"-mlongcalls".to_string())); - assert!(translated.entries[1] - .arguments - .contains(&"-std=c++17".to_string())); - } - - #[test] - fn test_translate_does_not_modify_original() { - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec![ - "/usr/bin/gcc".to_string(), - "-mlongcalls".to_string(), - "-Os".to_string(), - ], - directory: "/project".to_string(), - file: "main.c".to_string(), - output: None, - }); - - let _translated = db.translate_for_clang(TargetArchitecture::Xtensa); - - // Original should still have -mlongcalls - assert!(db.entries[0].arguments.contains(&"-mlongcalls".to_string())); - assert_eq!(db.entries[0].arguments[0], "/usr/bin/gcc"); - } - - // ========================================================================= - // IWYU preparation tests - // ========================================================================= - - #[test] - fn test_should_remove_freorder_blocks() { - assert!(should_remove_flag( - "-freorder-blocks", - TargetArchitecture::Xtensa - )); - assert!(should_remove_flag( - "-freorder-blocks", - TargetArchitecture::Avr - )); - } - - #[test] - fn test_should_remove_fno_jump_tables() { - assert!(should_remove_flag( - "-fno-jump-tables", - TargetArchitecture::Xtensa - )); - } - - #[test] - fn test_fstack_protector_preserved() { - // -fstack-protector is supported by clang — keep it - assert!(!should_remove_flag( - "-fstack-protector", - TargetArchitecture::Xtensa - )); - } - - #[test] - fn test_prepare_for_iwyu_removes_target() { - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec![ - "clang++".into(), - "--target=xtensa-esp-elf".into(), - "-Os".into(), - "-c".into(), - "src/main.cpp".into(), - ], - directory: "/project".into(), - file: "src/main.cpp".into(), - output: None, - }); - let result = db.prepare_for_iwyu(Path::new("/project/src"), &[]); - assert!(!result.entries[0] - .arguments - .iter() - .any(|a| a.starts_with("--target="))); - } - - #[test] - fn test_prepare_for_iwyu_dedup_defines() { - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec![ - "clang".into(), - "-DFOO=1".into(), - "-DBAR".into(), - "-DFOO=2".into(), // duplicate, should be dropped - ], - directory: "/project".into(), - file: "src/main.c".into(), - output: None, - }); - let result = db.prepare_for_iwyu(Path::new("/project/src"), &[]); - let defines: Vec<&str> = result.entries[0] - .arguments - .iter() - .filter(|a| a.starts_with("-D")) - .map(|a| a.as_str()) - .collect(); - assert_eq!(defines, vec!["-DFOO=1", "-DBAR"]); - } - - #[test] - fn test_prepare_for_iwyu_converts_system_includes() { - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec![ - "clang".into(), - "-I/project/src/mylib".into(), - "-I/usr/include/esp32".into(), - ], - directory: "/project".into(), - file: "src/main.c".into(), - output: None, - }); - let result = db.prepare_for_iwyu(Path::new("/project/src"), &[]); - let args = &result.entries[0].arguments; - // Project include kept as -I - assert!(args.contains(&"-I/project/src/mylib".to_string())); - // System include converted to -isystem - assert!(args.contains(&"-isystem".to_string())); - assert!(args.contains(&"/usr/include/esp32".to_string())); - assert!(!args.contains(&"-I/usr/include/esp32".to_string())); - } - - #[test] - fn test_prepare_for_iwyu_adds_extra_system_includes() { - let mut db = CompileDatabase::new(); - db.add_entry(CompileEntry { - arguments: vec!["clang".into(), "-c".into(), "src/main.c".into()], - directory: "/project".into(), - file: "src/main.c".into(), - output: None, - }); - let extras = vec![PathBuf::from("/toolchain/lib/gcc/xtensa/14/include")]; - let result = db.prepare_for_iwyu(Path::new("/project/src"), &extras); - let args = &result.entries[0].arguments; - assert!(args.contains(&"-isystem".to_string())); - assert!(args.contains(&"/toolchain/lib/gcc/xtensa/14/include".to_string())); - } -} diff --git a/crates/fbuild-build/src/compile_database/README.md b/crates/fbuild-build/src/compile_database/README.md new file mode 100644 index 00000000..e0ef54d3 --- /dev/null +++ b/crates/fbuild-build/src/compile_database/README.md @@ -0,0 +1,17 @@ +# compile_database + +Generates `compile_commands.json` (clangd-compatible JSON compilation database) +so IDE features like "Go to Definition" work with real include paths instead of +GCC response files (`@file`). + +## Modules + +- **`types.rs`** -- `CompileEntry`, `CompileDatabase` struct, `TargetArchitecture` enum. +- **`database.rs`** -- File IO (`write`, `write_and_copy`, `expected_output_path`) and `is_library_project` detection. +- **`cache_wrapper.rs`** -- `strip_cache_wrapper`: strips sccache/zccache/ccache wrappers from arg lists. +- **`clang.rs`** -- GCC-to-clang flag translation and IWYU (include-what-you-use) preparation. +- **`generate.rs`** -- `generate_entries`: builds entries from compiler flags and a list of sources. +- **`tests.rs`** -- Unit and adversarial tests for the whole module. + +This module was split out of a single `compile_database.rs` file to satisfy the +1000-LOC-per-file gate enforced in CI. diff --git a/crates/fbuild-build/src/compile_database/cache_wrapper.rs b/crates/fbuild-build/src/compile_database/cache_wrapper.rs new file mode 100644 index 00000000..3cf0db77 --- /dev/null +++ b/crates/fbuild-build/src/compile_database/cache_wrapper.rs @@ -0,0 +1,34 @@ +//! Strip compiler-cache wrappers (sccache/zccache/ccache) from argument lists. + +/// Strip cache wrapper (sccache/zccache/ccache) from compiler arguments. +/// +/// If the first element of `args` is a known cache wrapper, returns args +/// without it (the real compiler is the second element). Otherwise returns +/// args unchanged. +pub fn strip_cache_wrapper(args: &[String]) -> Vec { + if args.len() < 2 { + return args.to_vec(); + } + + // Extract the file stem manually so Windows paths (with `\`) work on Unix. + // `Path::file_stem` only splits on the platform's native separator, so + // `C:\...\sccache.exe` is treated as one component on Linux/macOS. + let filename = args[0].rsplit(['/', '\\']).next().unwrap_or(&args[0]); + let stem = filename + .strip_suffix(".exe") + .or_else(|| filename.strip_suffix(".EXE")) + .unwrap_or(filename) + .to_lowercase(); + + if stem == "sccache" || stem == "ccache" || stem == "zccache" { + if stem == "zccache" && args.get(1).is_some_and(|arg| arg == "wrap") { + if args.len() < 3 { + return args.to_vec(); + } + return args[2..].to_vec(); + } + args[1..].to_vec() + } else { + args.to_vec() + } +} diff --git a/crates/fbuild-build/src/compile_database/clang.rs b/crates/fbuild-build/src/compile_database/clang.rs new file mode 100644 index 00000000..df7efdd0 --- /dev/null +++ b/crates/fbuild-build/src/compile_database/clang.rs @@ -0,0 +1,161 @@ +//! Clang flag translation and IWYU preparation. + +use std::path::{Path, PathBuf}; + +use super::types::{CompileDatabase, CompileEntry, TargetArchitecture}; + +/// Check whether a GCC-specific flag should be removed for clang. +pub(super) fn should_remove_flag(flag: &str, arch: TargetArchitecture) -> bool { + // Common GCC-only flags unsupported by clang / IWYU + match flag { + "-flto=auto" + | "-flto" + | "-fno-fat-lto-objects" + | "-fuse-linker-plugin" + | "-ffat-lto-objects" + | "-freorder-blocks" + | "-fno-jump-tables" => return true, + _ => {} + } + + match arch { + TargetArchitecture::Xtensa => { + matches!( + flag, + "-mlongcalls" + | "-mdisable-hardware-atomics" + | "-mfix-esp32-psram-cache-issue" + | "-fstrict-volatile-bitfields" + | "-mtext-section-literals" + | "-fno-tree-switch-conversion" + ) || flag.starts_with("-mfix-esp32-psram-cache-strategy=") + } + TargetArchitecture::Riscv32 => matches!(flag, "-mabi=ilp32" | "-mno-fdiv"), + TargetArchitecture::Arm => flag == "-mthumb-interwork", + TargetArchitecture::Avr => false, + } +} + +/// Translate compiler arguments from GCC to clang-compatible equivalents. +/// +/// - Replaces the GCC/G++ compiler path with `clang`/`clang++` +/// - Inserts `--target=` as the second argument +/// - Removes architecture-specific flags that clang doesn't understand +pub fn translate_flags_for_clang(args: &[String], arch: TargetArchitecture) -> Vec { + if args.is_empty() { + return Vec::new(); + } + + let mut result = Vec::with_capacity(args.len() + 1); + + // Replace compiler path: detect g++ vs gcc by checking the normalized path + let compiler_path = args[0].to_lowercase().replace('\\', "/"); + let clang_name = if compiler_path.ends_with("g++") || compiler_path.ends_with("g++.exe") { + "clang++" + } else { + "clang" + }; + result.push(clang_name.to_string()); + + // Add target triple as second argument + result.push(format!("--target={}", arch.target_triple())); + + // Filter remaining args + for arg in &args[1..] { + if !should_remove_flag(arg, arch) { + result.push(arg.clone()); + } + } + + result +} + +impl CompileDatabase { + /// Create a new compile database with GCC flags translated to clang equivalents. + pub fn translate_for_clang(&self, arch: TargetArchitecture) -> CompileDatabase { + let entries = self + .entries + .iter() + .map(|entry| CompileEntry { + arguments: translate_flags_for_clang(&entry.arguments, arch), + directory: entry.directory.clone(), + file: entry.file.clone(), + output: entry.output.clone(), + }) + .collect(); + CompileDatabase { entries } + } + + /// Prepare compile database for IWYU (include-what-you-use) analysis. + /// + /// Transforms the existing (already clang-translated) compile database so that + /// IWYU can process cross-compiled embedded code: + /// + /// - Removes `--target=` flags (IWYU doesn't need code generation support) + /// - Deduplicates `-D` defines (keeps first occurrence of each key) + /// - Converts non-project `-I` paths to `-isystem` (suppresses IWYU suggestions) + /// - Adds extra `-isystem` paths (e.g. GCC toolchain builtin includes) + pub fn prepare_for_iwyu( + &self, + project_src_dir: &Path, + extra_system_includes: &[PathBuf], + ) -> CompileDatabase { + let src_prefix = project_src_dir.to_string_lossy().to_lowercase(); + let entries = self + .entries + .iter() + .map(|entry| { + let mut args = + Vec::with_capacity(entry.arguments.len() + extra_system_includes.len() * 2); + let mut seen_defines = std::collections::HashSet::new(); + + for arg in &entry.arguments { + // Remove --target= flags + if arg.starts_with("--target=") { + continue; + } + + // Deduplicate -D flags (keep first occurrence by key) + if arg.starts_with("-D") { + let key = if let Some(eq_pos) = arg.find('=') { + &arg[..eq_pos] + } else { + arg.as_str() + }; + if !seen_defines.insert(key.to_string()) { + continue; + } + } + + // Convert non-project -I to -isystem (suppresses IWYU analysis) + if let Some(path) = arg.strip_prefix("-I") { + let normalized = path.replace('\\', "/").to_lowercase(); + if normalized.starts_with(&src_prefix) { + args.push(arg.clone()); + } else { + args.push("-isystem".to_string()); + args.push(path.to_string()); + } + continue; + } + + args.push(arg.clone()); + } + + // Append GCC toolchain builtin include dirs as -isystem + for inc in extra_system_includes { + args.push("-isystem".to_string()); + args.push(inc.to_string_lossy().to_string()); + } + + CompileEntry { + arguments: args, + directory: entry.directory.clone(), + file: entry.file.clone(), + output: entry.output.clone(), + } + }) + .collect(); + CompileDatabase { entries } + } +} diff --git a/crates/fbuild-build/src/compile_database/database.rs b/crates/fbuild-build/src/compile_database/database.rs new file mode 100644 index 00000000..9fa192a9 --- /dev/null +++ b/crates/fbuild-build/src/compile_database/database.rs @@ -0,0 +1,81 @@ +//! File IO and library-project detection for `CompileDatabase`. + +use std::path::{Path, PathBuf}; + +use fbuild_core::Result; + +use super::types::CompileDatabase; + +impl CompileDatabase { + /// Write `compile_commands.json` to the given directory. + pub fn write(&self, dir: &Path) -> Result { + std::fs::create_dir_all(dir).map_err(|e| { + fbuild_core::FbuildError::BuildFailed(format!( + "failed to create directory {}: {}", + dir.display(), + e + )) + })?; + + let path = dir.join("compile_commands.json"); + let json = serde_json::to_string_pretty(&self.entries).map_err(|e| { + fbuild_core::FbuildError::BuildFailed(format!( + "failed to serialize compile database: {}", + e + )) + })?; + + write_if_changed(&path, json.as_bytes())?; + + Ok(path) + } + + /// Write to `build_dir` and copy to `project_dir` (matching Python fbuild behavior). + /// + /// If the project has a `library.json` at its root (indicating it IS a library, + /// e.g. FastLED), the copy to project root is suppressed to avoid overwriting + /// a meson/cmake-generated `compile_commands.json` that uses correct source paths. + pub fn write_and_copy(&self, build_dir: &Path, project_dir: &Path) -> Result { + let build_path = self.write(build_dir)?; + + if is_library_project(project_dir) { + tracing::info!( + "library.json detected — skipping compile_commands.json copy to project root" + ); + return Ok(build_path); + } + + let project_path = self.write(project_dir)?; + Ok(project_path) + } + + /// Path callers should report as the effective compile database output. + pub fn expected_output_path(build_dir: &Path, project_dir: &Path) -> PathBuf { + if is_library_project(project_dir) { + build_dir.join("compile_commands.json") + } else { + project_dir.join("compile_commands.json") + } + } +} + +fn write_if_changed(path: &Path, contents: &[u8]) -> Result<()> { + if let Ok(existing) = std::fs::read(path) { + if existing == contents { + return Ok(()); + } + } + + std::fs::write(path, contents).map_err(|e| { + fbuild_core::FbuildError::BuildFailed(format!("failed to write {}: {}", path.display(), e)) + }) +} + +/// Check if a project is a library (has `library.json` at the root). +/// +/// Library projects (e.g. FastLED) often have their own build system that +/// generates a correct `compile_commands.json`. We skip overwriting the +/// project root file to avoid clobbering it. +pub fn is_library_project(project_dir: &Path) -> bool { + project_dir.join("library.json").exists() +} diff --git a/crates/fbuild-build/src/compile_database/generate.rs b/crates/fbuild-build/src/compile_database/generate.rs new file mode 100644 index 00000000..0a9e627e --- /dev/null +++ b/crates/fbuild-build/src/compile_database/generate.rs @@ -0,0 +1,70 @@ +//! Generate compile database entries from compiler flags and source lists. + +use std::path::{Path, PathBuf}; + +use crate::flag_overlay::LanguageExtraFlags; + +use super::types::CompileEntry; + +/// Generate compile database entries for a set of source files. +/// +/// # Arguments +/// - `gcc_path` / `gxx_path` — real compiler paths (not cache wrappers) +/// - `c_flags` / `cpp_flags` — language-specific flags +/// - `include_flags` — separate `-I` flags (for ESP32; empty for AVR/Teensy where they're in c/cpp_flags) +/// - `extra_flags` — user/src flags +/// - `sources` — source files to generate entries for +/// - `build_dir` — where object files go (for `-o` path) +/// - `project_dir` — used as the `directory` field +#[allow(clippy::too_many_arguments)] +pub fn generate_entries( + gcc_path: &Path, + gxx_path: &Path, + c_flags: &[String], + cpp_flags: &[String], + include_flags: &[String], + extra_flags: &LanguageExtraFlags, + sources: &[PathBuf], + build_dir: &Path, + project_dir: &Path, +) -> Vec { + let directory = project_dir.to_string_lossy().to_string(); + + sources + .iter() + .map(|source| { + let ext = source + .extension() + .unwrap_or_default() + .to_string_lossy() + .to_lowercase(); + + let (compiler, flags) = match ext.as_str() { + "c" | "s" => (gcc_path, c_flags), + _ => (gxx_path, cpp_flags), + }; + + let obj = crate::compiler::CompilerBase::object_path(source, build_dir); + let source_extra_flags = extra_flags.for_source(source); + + let mut arguments = Vec::with_capacity( + 1 + flags.len() + include_flags.len() + source_extra_flags.len() + 4, + ); + arguments.push(compiler.to_string_lossy().to_string()); + arguments.extend(flags.iter().cloned()); + arguments.extend(include_flags.iter().cloned()); + arguments.extend(source_extra_flags); + arguments.push("-c".to_string()); + arguments.push(source.to_string_lossy().to_string()); + arguments.push("-o".to_string()); + arguments.push(obj.to_string_lossy().to_string()); + + CompileEntry { + arguments, + directory: directory.clone(), + file: source.to_string_lossy().to_string(), + output: Some(obj.to_string_lossy().to_string()), + } + }) + .collect() +} diff --git a/crates/fbuild-build/src/compile_database/mod.rs b/crates/fbuild-build/src/compile_database/mod.rs new file mode 100644 index 00000000..362d4a3b --- /dev/null +++ b/crates/fbuild-build/src/compile_database/mod.rs @@ -0,0 +1,20 @@ +//! Compile database (`compile_commands.json`) generation for IDE support. +//! +//! Generates a JSON compilation database (clangd-compatible) so that +//! "Go to Definition" and other IDE features work with real include paths +//! instead of response file (`@file`) references. + +mod cache_wrapper; +mod clang; +mod database; +mod generate; +mod types; + +pub use cache_wrapper::strip_cache_wrapper; +pub use clang::translate_flags_for_clang; +pub use database::is_library_project; +pub use generate::generate_entries; +pub use types::{CompileDatabase, CompileEntry, TargetArchitecture}; + +#[cfg(test)] +mod tests; diff --git a/crates/fbuild-build/src/compile_database/tests/README.md b/crates/fbuild-build/src/compile_database/tests/README.md new file mode 100644 index 00000000..de5d1730 --- /dev/null +++ b/crates/fbuild-build/src/compile_database/tests/README.md @@ -0,0 +1,18 @@ +# compile_database tests + +Unit and adversarial tests for the `compile_database` module, split across +several files so no single file exceeds the project's 1000-LOC gate. + +## Files + +- **`mod.rs`** -- Declares the test submodules and exposes a shared + `generate_entries` shim that wraps `LanguageExtraFlags` for tests that only + care about the common-flag path. +- **`serialization_and_write.rs`** -- `CompileEntry` JSON serialization, + `CompileDatabase` container behavior, `write` / `write_and_copy`, + `expected_output_path`, and `is_library_project` detection. +- **`cache_wrapper.rs`** -- `strip_cache_wrapper` (sccache / zccache / ccache). +- **`generate.rs`** -- `generate_entries`: extension-based compiler routing, + path handling, argument structure, adversarial edge cases. +- **`clang.rs`** -- `translate_flags_for_clang`, `should_remove_flag`, + `CompileDatabase::translate_for_clang`, and `prepare_for_iwyu` (IWYU prep). diff --git a/crates/fbuild-build/src/compile_database/tests/cache_wrapper.rs b/crates/fbuild-build/src/compile_database/tests/cache_wrapper.rs new file mode 100644 index 00000000..f3550061 --- /dev/null +++ b/crates/fbuild-build/src/compile_database/tests/cache_wrapper.rs @@ -0,0 +1,114 @@ +//! Tests for `strip_cache_wrapper`. + +use crate::compile_database::strip_cache_wrapper; + +#[test] +fn test_strip_sccache() { + let args = vec![ + "sccache".to_string(), + "/usr/bin/gcc".to_string(), + "-c".to_string(), + ]; + let stripped = strip_cache_wrapper(&args); + assert_eq!(stripped[0], "/usr/bin/gcc"); + assert_eq!(stripped.len(), 2); +} + +#[test] +fn test_strip_zccache() { + let args = vec![ + "/path/to/zccache".to_string(), + "/usr/bin/gcc".to_string(), + "-c".to_string(), + ]; + let stripped = strip_cache_wrapper(&args); + assert_eq!(stripped[0], "/usr/bin/gcc"); +} + +#[test] +fn test_strip_zccache_wrap_mode() { + let args = vec![ + "C:\\tools\\zccache.exe".to_string(), + "wrap".to_string(), + "C:\\tc\\bin\\xtensa-esp32-elf-g++.exe".to_string(), + "-c".to_string(), + ]; + let stripped = strip_cache_wrapper(&args); + assert_eq!(stripped[0], "C:\\tc\\bin\\xtensa-esp32-elf-g++.exe"); + assert_eq!(stripped[1], "-c"); +} + +#[test] +fn test_strip_ccache() { + let args = vec![ + "ccache".to_string(), + "/usr/bin/gcc".to_string(), + "-c".to_string(), + ]; + let stripped = strip_cache_wrapper(&args); + assert_eq!(stripped[0], "/usr/bin/gcc"); +} + +#[test] +fn test_strip_no_wrapper() { + let args = vec!["/usr/bin/gcc".to_string(), "-c".to_string()]; + let stripped = strip_cache_wrapper(&args); + assert_eq!(stripped, args); +} + +#[test] +fn test_strip_empty() { + let args: Vec = vec![]; + let stripped = strip_cache_wrapper(&args); + assert!(stripped.is_empty()); +} + +#[test] +fn test_strip_cache_wrapper_windows_exe() { + let args = vec![ + "C:\\Users\\user\\.cargo\\bin\\sccache.exe".to_string(), + "C:\\tools\\gcc.exe".to_string(), + "-c".to_string(), + ]; + let stripped = strip_cache_wrapper(&args); + assert_eq!(stripped[0], "C:\\tools\\gcc.exe"); + assert_eq!(stripped.len(), 2); +} + +#[test] +fn test_strip_cache_wrapper_case_insensitive() { + // Windows file systems are case-insensitive + for name in &[ + "SCCACHE", "Sccache", "ZCCACHE", "Zccache", "CCACHE", "Ccache", + ] { + let args = vec![ + name.to_string(), + "/usr/bin/gcc".to_string(), + "-c".to_string(), + ]; + let stripped = strip_cache_wrapper(&args); + assert_eq!( + stripped[0], "/usr/bin/gcc", + "failed to strip cache wrapper: {}", + name + ); + } +} + +#[test] +fn test_strip_cache_wrapper_single_element_wrapper() { + // Only the wrapper, no actual compiler — should return as-is + let args = vec!["sccache".to_string()]; + let stripped = strip_cache_wrapper(&args); + assert_eq!(stripped, args); +} + +#[test] +fn test_strip_cache_wrapper_not_a_wrapper() { + // File named "sccache-stats" shouldn't be stripped + let args = vec!["sccache-stats".to_string(), "/usr/bin/gcc".to_string()]; + let stripped = strip_cache_wrapper(&args); + // file_stem of "sccache-stats" is "sccache-stats", not "sccache" + assert_eq!(stripped.len(), 2); + assert_eq!(stripped[0], "sccache-stats"); +} diff --git a/crates/fbuild-build/src/compile_database/tests/clang.rs b/crates/fbuild-build/src/compile_database/tests/clang.rs new file mode 100644 index 00000000..a4f5b81b --- /dev/null +++ b/crates/fbuild-build/src/compile_database/tests/clang.rs @@ -0,0 +1,359 @@ +//! Clang flag translation and IWYU preparation tests. + +use std::path::{Path, PathBuf}; + +use super::super::clang::should_remove_flag; +use crate::compile_database::{ + translate_flags_for_clang, CompileDatabase, CompileEntry, TargetArchitecture, +}; + +#[test] +fn test_target_triples() { + assert_eq!(TargetArchitecture::Xtensa.target_triple(), "xtensa-esp-elf"); + assert_eq!( + TargetArchitecture::Riscv32.target_triple(), + "riscv32-esp-elf" + ); + assert_eq!(TargetArchitecture::Avr.target_triple(), "avr"); + assert_eq!(TargetArchitecture::Arm.target_triple(), "arm-none-eabi"); +} + +#[test] +fn test_translate_gcc_to_clang() { + let args = vec![ + "/usr/bin/avr-gcc".to_string(), + "-Os".to_string(), + "-c".to_string(), + ]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Avr); + assert_eq!(result[0], "clang"); +} + +#[test] +fn test_translate_gxx_to_clangxx() { + let args = vec!["/usr/bin/arm-none-eabi-g++".to_string(), "-Os".to_string()]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Arm); + assert_eq!(result[0], "clang++"); +} + +#[test] +fn test_translate_windows_compiler_path() { + let args = vec![ + "C:\\tools\\xtensa-esp32-elf-g++.exe".to_string(), + "-Os".to_string(), + ]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Xtensa); + assert_eq!(result[0], "clang++"); +} + +#[test] +fn test_translate_adds_target() { + let args = vec!["/usr/bin/gcc".to_string(), "-c".to_string()]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Xtensa); + assert_eq!(result[1], "--target=xtensa-esp-elf"); +} + +#[test] +fn test_translate_removes_common_lto_flags() { + let args = vec![ + "/usr/bin/gcc".to_string(), + "-flto=auto".to_string(), + "-flto".to_string(), + "-fno-fat-lto-objects".to_string(), + "-fuse-linker-plugin".to_string(), + "-ffat-lto-objects".to_string(), + "-Os".to_string(), + ]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Avr); + assert!(!result.contains(&"-flto=auto".to_string())); + assert!(!result.contains(&"-flto".to_string())); + assert!(!result.contains(&"-fno-fat-lto-objects".to_string())); + assert!(!result.contains(&"-fuse-linker-plugin".to_string())); + assert!(!result.contains(&"-ffat-lto-objects".to_string())); + assert!(result.contains(&"-Os".to_string())); +} + +#[test] +fn test_translate_xtensa_removals() { + let args = vec![ + "/usr/bin/xtensa-esp32-elf-gcc".to_string(), + "-mlongcalls".to_string(), + "-mdisable-hardware-atomics".to_string(), + "-mfix-esp32-psram-cache-issue".to_string(), + "-fstrict-volatile-bitfields".to_string(), + "-mtext-section-literals".to_string(), + "-fno-tree-switch-conversion".to_string(), + "-Os".to_string(), + ]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Xtensa); + assert!(!result.contains(&"-mlongcalls".to_string())); + assert!(!result.contains(&"-mdisable-hardware-atomics".to_string())); + assert!(!result.contains(&"-mfix-esp32-psram-cache-issue".to_string())); + assert!(!result.contains(&"-fstrict-volatile-bitfields".to_string())); + assert!(!result.contains(&"-mtext-section-literals".to_string())); + assert!(!result.contains(&"-fno-tree-switch-conversion".to_string())); + assert!(result.contains(&"-Os".to_string())); +} + +#[test] +fn test_translate_xtensa_psram_strategy_prefix() { + let args = vec![ + "/usr/bin/gcc".to_string(), + "-mfix-esp32-psram-cache-strategy=memw".to_string(), + "-Os".to_string(), + ]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Xtensa); + assert!(!result.contains(&"-mfix-esp32-psram-cache-strategy=memw".to_string())); + assert!(result.contains(&"-Os".to_string())); +} + +#[test] +fn test_translate_riscv_removals() { + let args = vec![ + "/usr/bin/riscv32-esp-elf-gcc".to_string(), + "-mabi=ilp32".to_string(), + "-mno-fdiv".to_string(), + "-march=rv32imac".to_string(), + "-Os".to_string(), + ]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Riscv32); + assert!(!result.contains(&"-mabi=ilp32".to_string())); + assert!(!result.contains(&"-mno-fdiv".to_string())); + assert!(result.contains(&"-march=rv32imac".to_string())); +} + +#[test] +fn test_translate_arm_removals() { + let args = vec![ + "/usr/bin/arm-none-eabi-g++".to_string(), + "-mthumb-interwork".to_string(), + "-mcpu=cortex-m7".to_string(), + "-Os".to_string(), + ]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Arm); + assert!(!result.contains(&"-mthumb-interwork".to_string())); + assert!(result.contains(&"-mcpu=cortex-m7".to_string())); +} + +#[test] +fn test_translate_avr_no_extra_removals() { + let args = vec![ + "/usr/bin/avr-gcc".to_string(), + "-mmcu=atmega328p".to_string(), + "-Os".to_string(), + ]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Avr); + assert!(result.contains(&"-mmcu=atmega328p".to_string())); + assert!(result.contains(&"-Os".to_string())); +} + +#[test] +fn test_translate_preserves_includes_and_defines() { + let args = vec![ + "/usr/bin/gcc".to_string(), + "-I/path/to/include".to_string(), + "-DFOO=1".to_string(), + "-c".to_string(), + ]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Avr); + assert!(result.contains(&"-I/path/to/include".to_string())); + assert!(result.contains(&"-DFOO=1".to_string())); +} + +#[test] +fn test_translate_empty_args() { + let args: Vec = vec![]; + let result = translate_flags_for_clang(&args, TargetArchitecture::Avr); + assert!(result.is_empty()); +} + +#[test] +fn test_database_translate_for_clang() { + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec![ + "/usr/bin/xtensa-esp32-elf-gcc".to_string(), + "-mlongcalls".to_string(), + "-Os".to_string(), + "-c".to_string(), + "main.c".to_string(), + ], + directory: "/project".to_string(), + file: "main.c".to_string(), + output: Some("main.o".to_string()), + }); + db.add_entry(CompileEntry { + arguments: vec![ + "/usr/bin/xtensa-esp32-elf-g++".to_string(), + "-mlongcalls".to_string(), + "-std=c++17".to_string(), + "-c".to_string(), + "app.cpp".to_string(), + ], + directory: "/project".to_string(), + file: "app.cpp".to_string(), + output: Some("app.o".to_string()), + }); + + let translated = db.translate_for_clang(TargetArchitecture::Xtensa); + + // First entry: gcc → clang + assert_eq!(translated.entries[0].arguments[0], "clang"); + assert_eq!( + translated.entries[0].arguments[1], + "--target=xtensa-esp-elf" + ); + assert!(!translated.entries[0] + .arguments + .contains(&"-mlongcalls".to_string())); + assert!(translated.entries[0].arguments.contains(&"-Os".to_string())); + assert_eq!(translated.entries[0].file, "main.c"); + + // Second entry: g++ → clang++ + assert_eq!(translated.entries[1].arguments[0], "clang++"); + assert!(!translated.entries[1] + .arguments + .contains(&"-mlongcalls".to_string())); + assert!(translated.entries[1] + .arguments + .contains(&"-std=c++17".to_string())); +} + +#[test] +fn test_translate_does_not_modify_original() { + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec![ + "/usr/bin/gcc".to_string(), + "-mlongcalls".to_string(), + "-Os".to_string(), + ], + directory: "/project".to_string(), + file: "main.c".to_string(), + output: None, + }); + + let _translated = db.translate_for_clang(TargetArchitecture::Xtensa); + + // Original should still have -mlongcalls + assert!(db.entries[0].arguments.contains(&"-mlongcalls".to_string())); + assert_eq!(db.entries[0].arguments[0], "/usr/bin/gcc"); +} + +// ========================================================================= +// IWYU preparation tests +// ========================================================================= + +#[test] +fn test_should_remove_freorder_blocks() { + assert!(should_remove_flag( + "-freorder-blocks", + TargetArchitecture::Xtensa + )); + assert!(should_remove_flag( + "-freorder-blocks", + TargetArchitecture::Avr + )); +} + +#[test] +fn test_should_remove_fno_jump_tables() { + assert!(should_remove_flag( + "-fno-jump-tables", + TargetArchitecture::Xtensa + )); +} + +#[test] +fn test_fstack_protector_preserved() { + // -fstack-protector is supported by clang — keep it + assert!(!should_remove_flag( + "-fstack-protector", + TargetArchitecture::Xtensa + )); +} + +#[test] +fn test_prepare_for_iwyu_removes_target() { + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec![ + "clang++".into(), + "--target=xtensa-esp-elf".into(), + "-Os".into(), + "-c".into(), + "src/main.cpp".into(), + ], + directory: "/project".into(), + file: "src/main.cpp".into(), + output: None, + }); + let result = db.prepare_for_iwyu(Path::new("/project/src"), &[]); + assert!(!result.entries[0] + .arguments + .iter() + .any(|a| a.starts_with("--target="))); +} + +#[test] +fn test_prepare_for_iwyu_dedup_defines() { + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec![ + "clang".into(), + "-DFOO=1".into(), + "-DBAR".into(), + "-DFOO=2".into(), // duplicate, should be dropped + ], + directory: "/project".into(), + file: "src/main.c".into(), + output: None, + }); + let result = db.prepare_for_iwyu(Path::new("/project/src"), &[]); + let defines: Vec<&str> = result.entries[0] + .arguments + .iter() + .filter(|a| a.starts_with("-D")) + .map(|a| a.as_str()) + .collect(); + assert_eq!(defines, vec!["-DFOO=1", "-DBAR"]); +} + +#[test] +fn test_prepare_for_iwyu_converts_system_includes() { + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec![ + "clang".into(), + "-I/project/src/mylib".into(), + "-I/usr/include/esp32".into(), + ], + directory: "/project".into(), + file: "src/main.c".into(), + output: None, + }); + let result = db.prepare_for_iwyu(Path::new("/project/src"), &[]); + let args = &result.entries[0].arguments; + // Project include kept as -I + assert!(args.contains(&"-I/project/src/mylib".to_string())); + // System include converted to -isystem + assert!(args.contains(&"-isystem".to_string())); + assert!(args.contains(&"/usr/include/esp32".to_string())); + assert!(!args.contains(&"-I/usr/include/esp32".to_string())); +} + +#[test] +fn test_prepare_for_iwyu_adds_extra_system_includes() { + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec!["clang".into(), "-c".into(), "src/main.c".into()], + directory: "/project".into(), + file: "src/main.c".into(), + output: None, + }); + let extras = vec![PathBuf::from("/toolchain/lib/gcc/xtensa/14/include")]; + let result = db.prepare_for_iwyu(Path::new("/project/src"), &extras); + let args = &result.entries[0].arguments; + assert!(args.contains(&"-isystem".to_string())); + assert!(args.contains(&"/toolchain/lib/gcc/xtensa/14/include".to_string())); +} diff --git a/crates/fbuild-build/src/compile_database/tests/generate.rs b/crates/fbuild-build/src/compile_database/tests/generate.rs new file mode 100644 index 00000000..02507185 --- /dev/null +++ b/crates/fbuild-build/src/compile_database/tests/generate.rs @@ -0,0 +1,478 @@ +//! Tests for `generate_entries`. + +use std::path::{Path, PathBuf}; + +use super::generate_entries; + +// --- Entry generation tests --- + +#[test] +fn test_generate_entries_c_uses_gcc() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &["-std=c11".to_string()], + &["-std=c++17".to_string()], + &[], + &[], + &[PathBuf::from("main.c")], + Path::new("/build"), + Path::new("/project"), + ); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].arguments[0], "/usr/bin/gcc"); + assert!(entries[0].arguments.contains(&"-std=c11".to_string())); +} + +#[test] +fn test_generate_entries_cpp_uses_gxx() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &["-std=c11".to_string()], + &["-std=c++17".to_string()], + &[], + &[], + &[PathBuf::from("main.cpp")], + Path::new("/build"), + Path::new("/project"), + ); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); + assert!(entries[0].arguments.contains(&"-std=c++17".to_string())); +} + +#[test] +fn test_generate_entries_s_uses_gcc() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &["-std=c11".to_string()], + &["-std=c++17".to_string()], + &[], + &[], + &[PathBuf::from("startup.s")], + Path::new("/build"), + Path::new("/project"), + ); + assert_eq!(entries[0].arguments[0], "/usr/bin/gcc"); +} + +#[test] +fn test_generate_entries_empty_sources() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &[], + &[], + Path::new("/build"), + Path::new("/project"), + ); + assert!(entries.is_empty()); +} + +#[test] +fn test_generate_entries_include_flags_in_args() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &["-I/sdk/include".to_string(), "-I/core/include".to_string()], + &[], + &[PathBuf::from("main.cpp")], + Path::new("/build"), + Path::new("/project"), + ); + assert!(entries[0].arguments.contains(&"-I/sdk/include".to_string())); + assert!(entries[0] + .arguments + .contains(&"-I/core/include".to_string())); +} + +#[test] +fn test_generate_entries_extra_flags_in_args() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &["-DUSER_FLAG=1".to_string()], + &[PathBuf::from("main.cpp")], + Path::new("/build"), + Path::new("/project"), + ); + assert!(entries[0].arguments.contains(&"-DUSER_FLAG=1".to_string())); +} + +#[test] +fn test_generate_entries_directory_is_project_dir() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &[], + &[PathBuf::from("main.cpp")], + Path::new("/build"), + Path::new("/my/project"), + ); + assert_eq!(entries[0].directory, "/my/project"); +} + +#[test] +fn test_generate_entries_arguments_structure() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &["-Os".to_string()], + &["-Os".to_string()], + &["-I/inc".to_string()], + &["-DFOO".to_string()], + &[PathBuf::from("main.c")], + Path::new("/build"), + Path::new("/project"), + ); + + let args = &entries[0].arguments; + // Starts with compiler + assert_eq!(args[0], "/usr/bin/gcc"); + // Ends with -c source -o object + let len = args.len(); + assert_eq!(args[len - 4], "-c"); + assert_eq!(args[len - 3], "main.c"); + assert_eq!(args[len - 2], "-o"); +} + +// --- Extension classification adversarial tests --- + +#[test] +fn test_generate_entries_uppercase_c_extension() { + // .C is treated as C++ on some systems, but our lowercase normalization + // maps it to "c" → gcc. This matches GCC behavior. + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &[], + &[PathBuf::from("main.C")], + Path::new("/build"), + Path::new("/project"), + ); + // After to_lowercase(), ".C" becomes "c" → uses gcc + assert_eq!(entries[0].arguments[0], "/usr/bin/gcc"); +} + +#[test] +fn test_generate_entries_cc_extension_uses_gxx() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &[], + &[PathBuf::from("module.cc")], + Path::new("/build"), + Path::new("/project"), + ); + assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); +} + +#[test] +fn test_generate_entries_cxx_extension_uses_gxx() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &[], + &[PathBuf::from("module.cxx")], + Path::new("/build"), + Path::new("/project"), + ); + assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); +} + +#[test] +fn test_generate_entries_ino_cpp_uses_gxx() { + // Preprocessed .ino files become .ino.cpp — extension is "cpp" + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &[], + &[PathBuf::from("sketch.ino.cpp")], + Path::new("/build"), + Path::new("/project"), + ); + assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); +} + +#[test] +fn test_generate_entries_no_extension_uses_gxx() { + // Files without extension fall through to g++ (the default branch) + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &[], + &[PathBuf::from("Makefile")], + Path::new("/build"), + Path::new("/project"), + ); + assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); +} + +#[test] +fn test_generate_entries_uppercase_s_assembly_uses_gcc() { + // .S (uppercase) is GCC-preprocessed assembly, should use gcc + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &[], + &[PathBuf::from("boot.S")], + Path::new("/build"), + Path::new("/project"), + ); + // to_lowercase() → "s" → matches gcc branch + assert_eq!(entries[0].arguments[0], "/usr/bin/gcc"); +} + +// --- Path handling adversarial tests --- + +#[test] +fn test_generate_entries_paths_with_spaces() { + let entries = generate_entries( + Path::new("/usr/bin/my gcc"), + Path::new("/usr/bin/my g++"), + &[], + &[], + &["-I/path with spaces/include".to_string()], + &[], + &[PathBuf::from("/my project/src/main.cpp")], + Path::new("/my build"), + Path::new("/my project"), + ); + assert_eq!(entries[0].directory, "/my project"); + assert_eq!(entries[0].file, "/my project/src/main.cpp"); + assert!(entries[0] + .arguments + .contains(&"-I/path with spaces/include".to_string())); +} + +#[test] +fn test_generate_entries_windows_backslash_paths() { + let entries = generate_entries( + Path::new("C:\\tools\\gcc.exe"), + Path::new("C:\\tools\\g++.exe"), + &[], + &[], + &[], + &[], + &[PathBuf::from("C:\\Users\\user\\project\\src\\main.cpp")], + Path::new("C:\\Users\\user\\build"), + Path::new("C:\\Users\\user\\project"), + ); + // to_string_lossy preserves original path separators + assert!(!entries[0].file.is_empty()); + assert!(!entries[0].directory.is_empty()); + // The output field should point to the build dir + assert!( + entries[0].output.as_ref().unwrap().contains("build") + || entries[0].output.as_ref().unwrap().contains("Users") + ); +} + +// --- Arguments must never contain @file (response file) references --- + +#[test] +fn test_generate_entries_no_response_file_in_args() { + // Even with many include flags, generate_entries should produce + // individual -I flags, never @file references (those are for GCC only). + let include_flags: Vec = + (0..300).map(|i| format!("-I/sdk/include/{}", i)).collect(); + + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &include_flags, + &[], + &[PathBuf::from("main.cpp")], + Path::new("/build"), + Path::new("/project"), + ); + + for arg in &entries[0].arguments { + assert!( + !arg.starts_with('@'), + "compile_commands.json must not contain @file references: {}", + arg + ); + } + // All 300 include flags should be present individually + assert!(entries[0] + .arguments + .contains(&"-I/sdk/include/0".to_string())); + assert!(entries[0] + .arguments + .contains(&"-I/sdk/include/299".to_string())); +} + +// --- File field must be the source path, not the build path --- + +#[test] +fn test_generate_entries_file_is_source_not_build() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &[], + &[PathBuf::from("/project/src/main.cpp")], + Path::new("/project/.fbuild/build/esp32/src"), + Path::new("/project"), + ); + assert_eq!(entries[0].file, "/project/src/main.cpp"); + // Output should be in the build dir + assert!( + entries[0].output.as_ref().unwrap().contains(".fbuild"), + "output should be in build dir: {:?}", + entries[0].output + ); +} + +// --- Mixed source types in a single call --- + +#[test] +fn test_generate_entries_mixed_sources() { + let sources = vec![ + PathBuf::from("main.cpp"), + PathBuf::from("util.c"), + PathBuf::from("boot.S"), + PathBuf::from("driver.cc"), + PathBuf::from("algo.cxx"), + ]; + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &["-std=c11".to_string()], + &["-std=c++17".to_string()], + &[], + &[], + &sources, + Path::new("/build"), + Path::new("/project"), + ); + assert_eq!(entries.len(), 5); + + // main.cpp → g++ + assert_eq!(entries[0].arguments[0], "/usr/bin/g++"); + assert!(entries[0].arguments.contains(&"-std=c++17".to_string())); + + // util.c → gcc + assert_eq!(entries[1].arguments[0], "/usr/bin/gcc"); + assert!(entries[1].arguments.contains(&"-std=c11".to_string())); + + // boot.S → gcc (assembly) + assert_eq!(entries[2].arguments[0], "/usr/bin/gcc"); + + // driver.cc → g++ + assert_eq!(entries[3].arguments[0], "/usr/bin/g++"); + + // algo.cxx → g++ + assert_eq!(entries[4].arguments[0], "/usr/bin/g++"); +} + +// --- Duplicate sources don't panic --- + +#[test] +fn test_generate_entries_duplicate_sources() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &[], + &[PathBuf::from("main.cpp"), PathBuf::from("main.cpp")], + Path::new("/build"), + Path::new("/project"), + ); + // Both entries should exist (dedup is the caller's responsibility) + assert_eq!(entries.len(), 2); +} + +// --- Include flags with build dir paths (the clangd navigation issue) --- + +#[test] +fn test_generate_entries_include_flags_preserved_verbatim() { + // The compile database should faithfully reproduce whatever include + // flags it receives. The ORCHESTRATOR is responsible for passing + // source-tree paths, not build-dir paths. + let include_flags = vec![ + "-I/project/src".to_string(), // source tree ✓ + "-I/home/user/.fbuild/build/esp32/libs/fastled/src".to_string(), // cache path ✗ + "-I/framework/cores/esp32".to_string(), // framework ✓ + ]; + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &include_flags, + &[], + &[PathBuf::from("main.cpp")], + Path::new("/build"), + Path::new("/project"), + ); + // All include flags should be present, unmodified + for flag in &include_flags { + assert!( + entries[0].arguments.contains(flag), + "missing include flag: {}", + flag + ); + } +} + +// --- Output path uses build_dir, not project source dir --- + +#[test] +fn test_generate_entries_output_in_build_dir() { + let entries = generate_entries( + Path::new("/usr/bin/gcc"), + Path::new("/usr/bin/g++"), + &[], + &[], + &[], + &[], + &[PathBuf::from("/project/src/main.cpp")], + Path::new("/build/obj"), + Path::new("/project"), + ); + let output = entries[0].output.as_ref().unwrap(); + assert!( + output.starts_with("/build/obj"), + "output should start with build dir: {}", + output + ); +} diff --git a/crates/fbuild-build/src/compile_database/tests/mod.rs b/crates/fbuild-build/src/compile_database/tests/mod.rs new file mode 100644 index 00000000..70108072 --- /dev/null +++ b/crates/fbuild-build/src/compile_database/tests/mod.rs @@ -0,0 +1,46 @@ +//! Test suite for the compile_database module, split across multiple files +//! to satisfy the per-file LOC gate. + +use std::path::{Path, PathBuf}; + +use crate::flag_overlay::LanguageExtraFlags; + +use super::CompileEntry; + +/// Test-only shim around `super::generate_entries` that accepts a flat +/// `&[String]` of extra flags (instead of `LanguageExtraFlags`), since +/// most tests only exercise the common-flag path. +#[allow(clippy::too_many_arguments)] +pub(super) fn generate_entries( + gcc_path: &Path, + gxx_path: &Path, + c_flags: &[String], + cpp_flags: &[String], + include_flags: &[String], + extra_flags: &[String], + sources: &[PathBuf], + build_dir: &Path, + project_dir: &Path, +) -> Vec { + super::generate_entries( + gcc_path, + gxx_path, + c_flags, + cpp_flags, + include_flags, + &LanguageExtraFlags { + common: extra_flags.to_vec(), + c: Vec::new(), + cxx: Vec::new(), + asm: Vec::new(), + }, + sources, + build_dir, + project_dir, + ) +} + +mod cache_wrapper; +mod clang; +mod generate; +mod serialization_and_write; diff --git a/crates/fbuild-build/src/compile_database/tests/serialization_and_write.rs b/crates/fbuild-build/src/compile_database/tests/serialization_and_write.rs new file mode 100644 index 00000000..b83b13c7 --- /dev/null +++ b/crates/fbuild-build/src/compile_database/tests/serialization_and_write.rs @@ -0,0 +1,310 @@ +//! Serialization, container, and on-disk write tests. + +use crate::compile_database::{is_library_project, CompileDatabase, CompileEntry}; + +// --- Serialization tests --- + +#[test] +fn test_compile_entry_serialization() { + let entry = CompileEntry { + arguments: vec![ + "/usr/bin/gcc".to_string(), + "-c".to_string(), + "main.c".to_string(), + ], + directory: "/project".to_string(), + file: "main.c".to_string(), + output: Some("main.o".to_string()), + }; + + let json = serde_json::to_value(&entry).unwrap(); + assert_eq!(json["directory"], "/project"); + assert_eq!(json["file"], "main.c"); + assert_eq!(json["output"], "main.o"); + assert!(json["arguments"].is_array()); +} + +#[test] +fn test_compile_entry_output_none_omitted() { + let entry = CompileEntry { + arguments: vec!["/usr/bin/gcc".to_string()], + directory: "/project".to_string(), + file: "main.c".to_string(), + output: None, + }; + + let json = serde_json::to_string(&entry).unwrap(); + assert!(!json.contains("output")); +} + +// --- CompileDatabase container tests --- + +#[test] +fn test_database_empty() { + let db = CompileDatabase::new(); + assert!(!db.has_entries()); +} + +#[test] +fn test_database_add_entry() { + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec![], + directory: String::new(), + file: "test.c".to_string(), + output: None, + }); + assert!(db.has_entries()); +} + +#[test] +fn test_database_write_valid_json() { + let tmp = tempfile::TempDir::new().unwrap(); + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec!["/usr/bin/gcc".to_string(), "-c".to_string()], + directory: "/project".to_string(), + file: "main.c".to_string(), + output: None, + }); + + let path = db.write(tmp.path()).unwrap(); + assert!(path.exists()); + + let content = std::fs::read_to_string(&path).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&content).unwrap(); + assert!(parsed.is_array()); + assert_eq!(parsed.as_array().unwrap().len(), 1); + assert_eq!(parsed[0]["file"], "main.c"); +} + +#[test] +fn test_database_write_creates_parent_dirs() { + let tmp = tempfile::TempDir::new().unwrap(); + let nested = tmp.path().join("a").join("b").join("c"); + let db = CompileDatabase::new(); + let path = db.write(&nested).unwrap(); + assert!(path.exists()); +} + +#[test] +fn test_database_write_and_copy() { + let tmp = tempfile::TempDir::new().unwrap(); + let build_dir = tmp.path().join("build"); + let project_dir = tmp.path().join("project"); + + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec![], + directory: String::new(), + file: "test.c".to_string(), + output: None, + }); + + db.write_and_copy(&build_dir, &project_dir).unwrap(); + assert!(build_dir.join("compile_commands.json").exists()); + assert!(project_dir.join("compile_commands.json").exists()); +} + +#[test] +fn test_database_write_does_not_rewrite_unchanged_contents() { + let tmp = tempfile::TempDir::new().unwrap(); + let dir = tmp.path().join("build"); + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec!["/usr/bin/gcc".to_string(), "-c".to_string()], + directory: "/project".to_string(), + file: "main.c".to_string(), + output: None, + }); + + let path = db.write(&dir).unwrap(); + let first_mtime = std::fs::metadata(&path).unwrap().modified().unwrap(); + std::thread::sleep(std::time::Duration::from_millis(20)); + + let path_again = db.write(&dir).unwrap(); + let second_mtime = std::fs::metadata(&path_again).unwrap().modified().unwrap(); + + assert_eq!(path, path_again); + assert_eq!(first_mtime, second_mtime); +} + +#[test] +fn test_expected_output_path_prefers_project_root_for_normal_projects() { + let tmp = tempfile::TempDir::new().unwrap(); + let build_dir = tmp.path().join("build"); + let project_dir = tmp.path().join("project"); + std::fs::create_dir_all(&project_dir).unwrap(); + + assert_eq!( + CompileDatabase::expected_output_path(&build_dir, &project_dir), + project_dir.join("compile_commands.json") + ); +} + +#[test] +fn test_expected_output_path_prefers_build_dir_for_library_projects() { + let tmp = tempfile::TempDir::new().unwrap(); + let build_dir = tmp.path().join("build"); + let project_dir = tmp.path().join("project"); + std::fs::create_dir_all(&project_dir).unwrap(); + std::fs::write(project_dir.join("library.json"), r#"{"name":"test"}"#).unwrap(); + + assert_eq!( + CompileDatabase::expected_output_path(&build_dir, &project_dir), + build_dir.join("compile_commands.json") + ); +} + +// --- write_and_copy: both files must have identical content --- + +#[test] +fn test_write_and_copy_identical_content() { + let tmp = tempfile::TempDir::new().unwrap(); + let build_dir = tmp.path().join("build"); + let project_dir = tmp.path().join("project"); + + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec![ + "/usr/bin/g++".to_string(), + "-c".to_string(), + "main.cpp".to_string(), + ], + directory: "/project".to_string(), + file: "main.cpp".to_string(), + output: Some("main.cpp.o".to_string()), + }); + + db.write_and_copy(&build_dir, &project_dir).unwrap(); + + let build_content = + std::fs::read_to_string(build_dir.join("compile_commands.json")).unwrap(); + let project_content = + std::fs::read_to_string(project_dir.join("compile_commands.json")).unwrap(); + assert_eq!(build_content, project_content); +} + +// --- write_and_copy: suppressed when library.json exists --- + +#[test] +fn test_write_and_copy_suppressed_for_library_project() { + let tmp = tempfile::TempDir::new().unwrap(); + let build_dir = tmp.path().join("build"); + let project_dir = tmp.path().join("project"); + std::fs::create_dir_all(&project_dir).unwrap(); + + // Create library.json to simulate a library project (like FastLED) + std::fs::write( + project_dir.join("library.json"), + r#"{"name": "FastLED", "version": "3.10.3"}"#, + ) + .unwrap(); + + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec!["/usr/bin/g++".to_string()], + directory: project_dir.to_string_lossy().to_string(), + file: "main.cpp".to_string(), + output: None, + }); + + let result_path = db.write_and_copy(&build_dir, &project_dir).unwrap(); + + // Build dir should have the file + assert!(build_dir.join("compile_commands.json").exists()); + // Project dir should NOT have compile_commands.json (suppressed) + assert!( + !project_dir.join("compile_commands.json").exists(), + "compile_commands.json should NOT be copied to project root for library projects" + ); + // The returned path should be the build dir path + assert_eq!(result_path, build_dir.join("compile_commands.json")); +} + +#[test] +fn test_write_and_copy_not_suppressed_for_sketch_project() { + let tmp = tempfile::TempDir::new().unwrap(); + let build_dir = tmp.path().join("build"); + let project_dir = tmp.path().join("project"); + std::fs::create_dir_all(&project_dir).unwrap(); + + // No library.json — this is a normal sketch project + let mut db = CompileDatabase::new(); + db.add_entry(CompileEntry { + arguments: vec!["/usr/bin/g++".to_string()], + directory: project_dir.to_string_lossy().to_string(), + file: "main.cpp".to_string(), + output: None, + }); + + db.write_and_copy(&build_dir, &project_dir).unwrap(); + + // Both should exist + assert!(build_dir.join("compile_commands.json").exists()); + assert!(project_dir.join("compile_commands.json").exists()); +} + +// --- is_library_project detection --- + +#[test] +fn test_is_library_project_with_library_json() { + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::write(tmp.path().join("library.json"), r#"{"name": "MyLib"}"#).unwrap(); + assert!(is_library_project(tmp.path())); +} + +#[test] +fn test_is_library_project_without_library_json() { + let tmp = tempfile::TempDir::new().unwrap(); + assert!(!is_library_project(tmp.path())); +} + +// --- Empty database produces valid JSON --- + +#[test] +fn test_write_empty_database_valid_json() { + let tmp = tempfile::TempDir::new().unwrap(); + let db = CompileDatabase::new(); + let path = db.write(tmp.path()).unwrap(); + + let content = std::fs::read_to_string(&path).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&content).unwrap(); + assert!(parsed.is_array()); + assert!(parsed.as_array().unwrap().is_empty()); +} + +// --- Extend adds all entries --- + +#[test] +fn test_database_extend_accumulates() { + let mut db = CompileDatabase::new(); + let entries1 = vec![CompileEntry { + arguments: vec![], + directory: String::new(), + file: "a.c".to_string(), + output: None, + }]; + let entries2 = vec![ + CompileEntry { + arguments: vec![], + directory: String::new(), + file: "b.c".to_string(), + output: None, + }, + CompileEntry { + arguments: vec![], + directory: String::new(), + file: "c.c".to_string(), + output: None, + }, + ]; + db.extend(entries1); + db.extend(entries2); + // Should have all 3 entries + let tmp = tempfile::TempDir::new().unwrap(); + let path = db.write(tmp.path()).unwrap(); + let content = std::fs::read_to_string(&path).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&content).unwrap(); + assert_eq!(parsed.as_array().unwrap().len(), 3); +} diff --git a/crates/fbuild-build/src/compile_database/types.rs b/crates/fbuild-build/src/compile_database/types.rs new file mode 100644 index 00000000..2e4cba3b --- /dev/null +++ b/crates/fbuild-build/src/compile_database/types.rs @@ -0,0 +1,70 @@ +//! Core types for the compile database: entries, container, and target arch enum. + +/// A single entry in the compile database. +#[derive(Debug, Clone, serde::Serialize)] +pub struct CompileEntry { + /// The compiler invocation as an argument list (preferred by clangd). + pub arguments: Vec, + /// The working directory for the compilation. + pub directory: String, + /// The source file being compiled. + pub file: String, + /// The output object file (optional per spec). + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, +} + +/// Container for compile database entries. +pub struct CompileDatabase { + pub(in crate::compile_database) entries: Vec, +} + +impl Default for CompileDatabase { + fn default() -> Self { + Self::new() + } +} + +impl CompileDatabase { + /// Create an empty compile database. + pub fn new() -> Self { + Self { + entries: Vec::new(), + } + } + + /// Add an entry to the database. + pub fn add_entry(&mut self, entry: CompileEntry) { + self.entries.push(entry); + } + + /// Add multiple entries to the database. + pub fn extend(&mut self, entries: Vec) { + self.entries.extend(entries); + } + + /// Whether the database has any entries. + pub fn has_entries(&self) -> bool { + !self.entries.is_empty() + } +} + +/// Target architecture for clang flag translation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetArchitecture { + Xtensa, + Riscv32, + Avr, + Arm, +} + +impl TargetArchitecture { + pub fn target_triple(&self) -> &'static str { + match self { + Self::Xtensa => "xtensa-esp-elf", + Self::Riscv32 => "riscv32-esp-elf", + Self::Avr => "avr", + Self::Arm => "arm-none-eabi", + } + } +} diff --git a/crates/fbuild-build/src/esp32/orchestrator.rs b/crates/fbuild-build/src/esp32/orchestrator.rs deleted file mode 100644 index e130db44..00000000 --- a/crates/fbuild-build/src/esp32/orchestrator.rs +++ /dev/null @@ -1,1934 +0,0 @@ -//! ESP32 build orchestrator — wires together config, packages, compiler, linker. -//! -//! Build phases: -//! 1. Parse platformio.ini -//! 2. Load board config (esp32dev/esp32c6/etc.) -//! 3. Load MCU config from embedded JSON -//! 4. Ensure ESP32 platform (pioarduino) -//! 5. Resolve + ensure ESP32 toolchain via metadata -//! 6. Ensure ESP32 framework (Arduino core + ESP-IDF SDK libs) -//! 7. Setup build directories -//! 8. Collect include paths: core + variant + SDK (305+) + user src -//! 9. Download + compile library dependencies -//! 10. Scan sources (sketch + core) -//! 11. Compile core sources -//! 12. Compile sketch sources -//! 13. Link (with linker scripts + SDK libs + library archives) -//! 14. Convert to .bin -//! 15. Copy bootloader.bin + partitions.bin -//! 16. Size reporting - -use std::path::{Path, PathBuf}; -use std::time::Instant; - -use fbuild_core::{Platform, Result}; -use fbuild_packages::Framework; -use serde::Serialize; - -use crate::build_fingerprint::{ - expected_fast_path_artifacts, stable_hash_json, FastPathCheckInputs, FastPathContract, - FastPathPersistInputs, BUILD_FINGERPRINT_VERSION, -}; -use crate::flag_overlay::LanguageExtraFlags; -use crate::linker::LinkerScripts; - -use crate::compiler::Compiler as _; -use crate::{BuildOrchestrator, BuildParams, BuildResult, SourceScanner}; - -use super::esp32_compiler::Esp32Compiler; -use super::esp32_linker::Esp32Linker; -use super::mcu_config::get_mcu_config; - -/// ESP32 platform build orchestrator. -pub struct Esp32Orchestrator; - -#[derive(Debug, Serialize)] -struct Esp32FingerprintMetadata { - version: u32, - env_name: String, - profile: String, - board_name: String, - board_mcu: String, - board_define: String, - board_core: String, - board_variant: String, - board_variant_h: Option, - board_extra_flags: Option, - board_upload_protocol: Option, - board_upload_speed: Option, - board_partitions: Option, - board_ldscript: Option, - board_platform: Option, - architecture: String, - platform: String, - flash_mode: String, - flash_freq: String, - flash_size: String, - max_flash: Option, - max_ram: Option, - eh_frame_policy: &'static str, -} - -fn framework_failure_marker(build_dir: &Path, lib_name: &str) -> PathBuf { - build_dir.join(format!(".{lib_name}.failed")) -} - -fn framework_signature( - include_dirs: &[PathBuf], - c_flags: &[String], - cpp_flags: &[String], -) -> String { - let mut parts = Vec::with_capacity(include_dirs.len() + c_flags.len() + cpp_flags.len() + 2); - parts.push("i".to_string()); - parts.extend( - include_dirs - .iter() - .map(|p| p.to_string_lossy().into_owned()), - ); - parts.push("c".to_string()); - parts.extend(c_flags.iter().cloned()); - parts.push("cxx".to_string()); - parts.extend(cpp_flags.iter().cloned()); - parts.join("\x1f") -} - -fn latest_mtime(paths: &[PathBuf]) -> Result> { - let mut latest = None; - for path in paths { - let modified = std::fs::metadata(path)?.modified()?; - latest = Some(match latest { - Some(current) if current > modified => current, - _ => modified, - }); - } - Ok(latest) -} - -fn should_skip_failed_framework_lib( - marker_path: &Path, - signature: &str, - sources: &[PathBuf], -) -> Result { - if !marker_path.exists() { - return Ok(false); - } - - let marker_text = std::fs::read_to_string(marker_path)?; - let recorded_signature = marker_text.lines().next().unwrap_or_default(); - if recorded_signature != signature { - return Ok(false); - } - - let Some(latest_source_time) = latest_mtime(sources)? else { - return Ok(false); - }; - let marker_time = std::fs::metadata(marker_path)?.modified()?; - Ok(marker_time >= latest_source_time) -} - -fn record_failed_framework_lib(marker_path: &Path, signature: &str, error: &str) { - let _ = std::fs::write(marker_path, format!("{signature}\n{error}\n")); -} - -fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { - match profile { - fbuild_core::BuildProfile::Release => "release", - fbuild_core::BuildProfile::Quick => "quick", - } -} - -fn compile_db_is_current(build_dir: &Path, project_dir: &Path) -> bool { - let build_copy = build_dir.join("compile_commands.json"); - if !build_copy.exists() { - return false; - } - crate::compile_database::CompileDatabase::expected_output_path(build_dir, project_dir).exists() -} - -impl BuildOrchestrator for Esp32Orchestrator { - fn platform(&self) -> Platform { - Platform::Espressif32 - } - - fn build(&self, params: &BuildParams) -> Result { - let start = Instant::now(); - // Env-gated per-phase timer (FBUILD_PERF_LOG=1); zero overhead when unset. - let mut perf = crate::perf_log::PerfTimer::new("esp32-orchestrator"); - - // 0. Discover zccache compiler cache (startup is deferred until compile work begins) - let compiler_cache = { - let _g = perf.phase("zccache-discover"); - crate::zccache::find_zccache().map(std::path::Path::to_path_buf) - }; - - // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = crate::pipeline::BuildContext::new_with_perf(params, Some(&mut perf))?; - - // Compute eh_frame strip policy once per build (FastLED/fbuild#243). - // Reads sdkconfig from the project dir (ESP32 only) so panic-backtrace - // / gdbstub users automatically Preserve. - let sdkconfig = - fbuild_config::sdkconfig::SdkConfigSummary::from_project_dir(¶ms.project_dir); - let eh_frame_policy = crate::eh_frame_policy_compute::compute_eh_frame_policy( - &ctx, - params.profile, - Some(&sdkconfig), - ); - - // 3. Load MCU config from embedded JSON - let mut mcu_config = get_mcu_config(&ctx.board.mcu)?; - - tracing::info!( - "ESP32 build: {} ({}, {})", - ctx.board.name, - ctx.board.mcu, - mcu_config.architecture - ); - - // 4-6. Resolve platform, toolchain, and framework - let _resolve_phase = perf.phase("pioarduino-resolve"); - let (toolchain, framework) = - resolve_pioarduino_packages(¶ms.project_dir, &ctx.board.mcu, &mcu_config)?; - drop(_resolve_phase); - let _toolchain_cache_dir = fbuild_packages::Package::get_info(&toolchain).install_path; - let _framework_cache_dir = fbuild_packages::Package::get_info(&framework).install_path; - - // Aliases for build dirs (already set up by BuildContext::new()) - let build_dir = &ctx.build_dir; - let core_build_dir = &ctx.core_build_dir; - let src_build_dir = &ctx.src_build_dir; - - // Read link-affecting config before the expensive include/library/source discovery steps - // so the no-op fast path can return early on warm builds. - let sdk_ld_flags = framework.get_sdk_ld_flags(&ctx.board.mcu); - let sdk_defines = framework.get_sdk_defines(&ctx.board.mcu); - - if sdk_ld_flags.iter().any(|f| f == "-fno-lto") { - mcu_config.disable_lto(); - } - - let mut user_flags = sdk_defines; - let user_build_flags = ctx.config.get_build_flags(¶ms.env_name)?; - user_flags.extend(user_build_flags.clone()); - let embed_files = ctx.config.get_embed_files(¶ms.env_name)?; - let embed_txtfiles = ctx.config.get_embed_txtfiles(¶ms.env_name)?; - - let f_for_image = ctx - .board - .f_image - .as_deref() - .or(ctx.board.f_flash.as_deref()); - let flash_freq = crate::esp32::esp32_linker::f_flash_to_esptool_freq( - f_for_image, - mcu_config.default_flash_freq(), - ); - let flash_mode = ctx - .board - .flash_mode - .clone() - .unwrap_or_else(|| mcu_config.default_flash_mode().to_string()); - let flash_size = crate::esp32::mcu_config::bytes_to_flash_size( - ctx.board.max_flash, - mcu_config.default_flash_size(), - ) - .to_string(); - let metadata_hash = stable_hash_json(&Esp32FingerprintMetadata { - version: BUILD_FINGERPRINT_VERSION, - env_name: params.env_name.clone(), - profile: profile_label(params.profile).to_string(), - board_name: ctx.board.name.clone(), - board_mcu: ctx.board.mcu.clone(), - board_define: ctx.board.board.clone(), - board_core: ctx.board.core.clone(), - board_variant: ctx.board.variant.clone(), - board_variant_h: ctx.board.variant_h.clone(), - board_extra_flags: ctx.board.extra_flags.clone(), - board_upload_protocol: ctx.board.upload_protocol.clone(), - board_upload_speed: ctx.board.upload_speed.clone(), - board_partitions: ctx.board.partitions.clone(), - board_ldscript: ctx.board.ldscript.clone(), - board_platform: ctx.board.platform_str.clone(), - architecture: mcu_config.architecture.clone(), - platform: "espressif32".to_string(), - flash_mode: flash_mode.clone(), - flash_freq: flash_freq.clone(), - flash_size: flash_size.clone(), - max_flash: ctx.board.max_flash, - max_ram: ctx.board.max_ram, - eh_frame_policy: match eh_frame_policy { - crate::eh_frame_policy::EhFramePolicy::Strip => "strip", - crate::eh_frame_policy::EhFramePolicy::Preserve => "preserve", - }, - })?; - let (fast_elf, [fast_bin, fast_boot, fast_parts, fast_app0], fast_compile_db) = - expected_fast_path_artifacts( - build_dir, - ¶ms.project_dir, - [ - "firmware.bin", - "bootloader.bin", - "partitions.bin", - "boot_app0.bin", - ], - ); - let fast_path = { - let _g = perf.phase("fp-watches-collect"); - FastPathContract::for_project_outputs( - build_dir, - ¶ms.project_dir, - [ - fast_elf.clone(), - fast_bin.clone(), - fast_boot.clone(), - fast_parts.clone(), - fast_app0.clone(), - fast_compile_db.clone(), - ], - ) - }; - - if !params.compiledb_only - && !params.symbol_analysis - && params.symbol_analysis_path.is_none() - { - let _fast_path_phase = perf.phase("fast-path-check"); - // ESP32 also requires the project-root copy of compile_commands.json - // to be in sync with the build-dir copy. That's platform-specific, - // so it rides on the shared helper via `extra_artifact_ok`. - let compile_db_fresh = || compile_db_is_current(build_dir, ¶ms.project_dir); - let inputs = FastPathCheckInputs { - metadata_hash: &metadata_hash, - extra_artifact_ok: Some(&compile_db_fresh), - watch_set_cache: params.watch_set_cache.as_deref(), - compiler_cache: compiler_cache.as_deref(), - }; - if let Some(hit) = crate::build_fingerprint::fast_path_check(&fast_path, &inputs)? { - ctx.build_log.push( - "No-op fingerprint matched; reusing existing ESP32 artifacts.".to_string(), - ); - let elapsed = start.elapsed().as_secs_f64(); - return Ok(BuildResult { - success: true, - firmware_path: Some(fast_bin), - elf_path: Some(fast_elf), - size_info: hit.size_info, - symbol_map: None, - build_time_secs: elapsed, - message: format!( - "ESP32 ({}) build for {} reused cached artifacts", - ctx.board.mcu, params.env_name - ), - compile_database_path: Some(fast_compile_db), - build_log: ctx.build_log, - }); - } - } - - if let Some(ref zcc) = compiler_cache { - crate::zccache::ensure_running(zcc); - } - - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; - tracing::info!( - "ESP32 {} toolchain at {}", - if mcu_config.is_riscv() { - "RISC-V" - } else { - "Xtensa" - }, - toolchain_dir.display() - ); - - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; - tracing::info!("ESP32 framework at {}", framework_dir.display()); - - let tc_label = if mcu_config.is_riscv() { - "riscv32-esp-elf-gcc" - } else { - "xtensa-esp-elf-gcc" - }; - crate::pipeline::log_toolchain_version( - &toolchain.get_gcc_path(), - tc_label, - &mut ctx.build_log, - ); - - let core_dir = framework.get_core_dir(&ctx.board.core); - let variant_dir = framework.get_variant_dir(&ctx.board.variant); - let sdk_memory_type = ctx - .board - .effective_esp32_memory_type(mcu_config.default_flash_mode()); - - let mut include_dirs = vec![core_dir.clone()]; - if variant_dir.exists() { - include_dirs.push(variant_dir.clone()); - } - // Add SDK include paths (294+ paths from ESP-IDF) - include_dirs - .extend(framework.get_sdk_include_dirs(&ctx.board.mcu, sdk_memory_type.as_deref())); - - // Add built-in Arduino library includes (Wire, SPI, WiFi, etc.) - let builtin_libs_dir = framework.get_libraries_dir(); - if builtin_libs_dir.is_dir() { - if let Ok(entries) = std::fs::read_dir(&builtin_libs_dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - let lib_src = path.join("src"); - if lib_src.is_dir() { - include_dirs.push(lib_src); - } - } - } - } - } - - include_dirs.push(ctx.src_dir.clone()); - crate::pipeline::discover_project_includes(¶ms.project_dir, &mut include_dirs); - // Toolchain sysroot includes (xtensa headers, etc.) - include_dirs.extend(toolchain.get_include_dirs()); - - // Read SDK flags early — needed to check LTO before compiling. - let sdk_ld_flags = framework.get_sdk_ld_flags(&ctx.board.mcu); - let sdk_lib_flags = framework.get_sdk_lib_flags(&ctx.board.mcu, sdk_memory_type.as_deref()); - let sdk_ld_scripts = - LinkerScripts::from_raw_flags(&framework.get_sdk_ld_scripts(&ctx.board.mcu)); - let sdk_defines = framework.get_sdk_defines(&ctx.board.mcu); - - // If SDK specifies -fno-lto, disable LTO in MCU config profiles to avoid - // compiling objects with LTO that the linker can't handle. - if sdk_ld_flags.iter().any(|f| f == "-fno-lto") { - mcu_config.disable_lto(); - } - - // 8.5. Library dependencies - let lib_deps = ctx.config.get_lib_deps(¶ms.env_name)?; - let lib_ignore = ctx.config.get_lib_ignore(¶ms.env_name)?; - - use fbuild_packages::Toolchain; - let mut library_archives = Vec::new(); - - // Read user build_flags early — needed for both library and sketch compilation. - // SDK defines (from flags/defines) are prepended so user flags can override them. - let mut user_flags = sdk_defines; - let mut user_build_flags = ctx.config.get_build_flags(¶ms.env_name)?; - user_build_flags.extend(params.extra_build_flags.clone()); - user_flags.extend(user_build_flags.clone()); - let user_overlay = LanguageExtraFlags { - common: user_flags - .iter() - .cloned() - .chain(ctx.global_compile_overlay.common.iter().cloned()) - .collect(), - c: ctx.global_compile_overlay.c.clone(), - cxx: ctx.global_compile_overlay.cxx.clone(), - asm: ctx.global_compile_overlay.asm.clone(), - }; - let src_overlay = LanguageExtraFlags::combined(&[ - &user_overlay, - &LanguageExtraFlags { - common: ctx.src_flags.clone(), - c: Vec::new(), - cxx: Vec::new(), - asm: Vec::new(), - }, - &ctx.project_compile_overlay, - ]); - - // Emit a warning if CDC on boot is effectively enabled (may cause Serial to block - // when no USB host is connected). - warn_if_cdc_on_boot( - &ctx.board.name, - ctx.board.extra_flags.as_deref(), - &user_build_flags, - ); - crate::warn_debug_build_flags(&user_build_flags); - - if !lib_deps.is_empty() { - let libs_dir = build_dir.join("libs"); - - // Build compiler to get flags for library compilation - let mut defines = ctx.board.get_defines(); - defines.extend(mcu_config.defines_map()); - - let temp_compiler = Esp32Compiler::with_temp_dir( - toolchain.get_gcc_path(), - toolchain.get_gxx_path(), - mcu_config.clone(), - &ctx.board.f_cpu, - defines.clone(), - include_dirs.clone(), - params.profile, - params.verbose, - build_dir.join("tmp"), - ) - .with_build_unflags(ctx.build_unflags.clone()) - .with_eh_frame_policy(eh_frame_policy); - // Apply user build_flags to library compilation (matching PlatformIO behavior). - // User flags like -std=gnu++2a replace the MCU config's -std=gnu++2b. - let c_flags = apply_overlay_flags(&temp_compiler.c_flags(), &user_overlay, "dummy.c"); - let cpp_flags = - apply_overlay_flags(&temp_compiler.cpp_flags(), &user_overlay, "dummy.cpp"); - - let jobs = crate::parallel::effective_jobs(params.jobs); - // Use gcc-ar for LTO archives so the linker-plugin index is written. - let dep_ar_path = toolchain.get_ar_path(); - let dep_gcc_ar_path = toolchain.get_gcc_ar_path(); - let dep_lib_ar_path = crate::pipeline::pick_archiver( - &dep_ar_path, - &dep_gcc_ar_path, - &c_flags, - &cpp_flags, - ); - let lib_result = fbuild_packages::library::library_manager::ensure_libraries_sync( - &lib_deps, - &lib_ignore, - &toolchain.get_gcc_path(), - &toolchain.get_gxx_path(), - dep_lib_ar_path, - &c_flags, - &cpp_flags, - &include_dirs, - &libs_dir, - params.verbose, - jobs, - compiler_cache.as_deref(), - )?; - - // Add library include dirs to the main include list - include_dirs.extend(lib_result.include_dirs); - library_archives = lib_result.archives; - - tracing::info!( - "libraries: {} archives, {} new include dirs", - library_archives.len(), - include_dirs.len() - ); - } - - // 8.5b. Project-as-library compilation — shared with sequential pipeline. - // When the project root contains library.json or library.properties (e.g., FastLED), - // the project's own src/ directory is compiled as a library archive so that example - // sketches can link against it. Centralized in pipeline::compile_project_as_library - // so every orchestrator gets this behavior architecturally. - if !params.compiledb_only { - // Build temp compiler to get the actual c_flags/cpp_flags ESP32 uses for - // library compilation. SDK defines + user flags must be applied so the - // archive matches what sketch sources see. - let mut p_defines = ctx.board.get_defines(); - p_defines.extend(mcu_config.defines_map()); - let p_compiler = Esp32Compiler::with_temp_dir( - toolchain.get_gcc_path(), - toolchain.get_gxx_path(), - mcu_config.clone(), - &ctx.board.f_cpu, - p_defines, - include_dirs.clone(), - params.profile, - params.verbose, - build_dir.join("tmp"), - ) - .with_build_unflags(ctx.build_unflags.clone()) - .with_eh_frame_policy(eh_frame_policy); - let p_c_flags = apply_overlay_flags(&p_compiler.c_flags(), &src_overlay, "dummy.c"); - let p_cpp_flags = - apply_overlay_flags(&p_compiler.cpp_flags(), &src_overlay, "dummy.cpp"); - - // Collect lib/* names so the helper can detect collisions with project-as-library. - let mut existing_lib_names = std::collections::HashSet::new(); - let local_lib_dir = params.project_dir.join("lib"); - if local_lib_dir.is_dir() { - if let Ok(entries) = std::fs::read_dir(&local_lib_dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - existing_lib_names.insert(name.to_lowercase()); - } - } - } - } - } - - let gcc_path = toolchain.get_gcc_path(); - let gxx_path = toolchain.get_gxx_path(); - let ar_path = toolchain.get_ar_path(); - let gcc_ar_path = toolchain.get_gcc_ar_path(); - // Use gcc-ar for LTO archives so the linker-plugin index is written. - let lib_ar_path = - crate::pipeline::pick_archiver(&ar_path, &gcc_ar_path, &p_c_flags, &p_cpp_flags); - let lib_env = crate::pipeline::LibraryBuildEnv { - gcc_path: &gcc_path, - gxx_path: &gxx_path, - ar_path: lib_ar_path, - c_flags: &p_c_flags, - cpp_flags: &p_cpp_flags, - include_dirs: &include_dirs, - verbose: params.verbose, - jobs: crate::parallel::effective_jobs(params.jobs), - compiler_cache: compiler_cache.as_deref(), - }; - if let Some(archive) = crate::pipeline::compile_project_as_library( - ¶ms.project_dir, - &ctx.src_dir, - build_dir, - &lib_env, - &existing_lib_names, - )? { - library_archives.push(archive); - } - } - - tracing::info!("include paths: {} total", include_dirs.len()); - - // 8.6. Compile framework built-in libraries (WiFi, FS, SPIFFS, Network, etc.) - // The linker's --gc-sections will strip any unused code. - // Skip when only generating compile_commands.json. - if !params.compiledb_only { - let fw_libs_started = Instant::now(); - perf.checkpoint("fw-libs-start"); - let builtin_libs_dir = framework.get_libraries_dir(); - if builtin_libs_dir.is_dir() { - let fw_libs_build_dir = build_dir.join("fw_libs"); - std::fs::create_dir_all(&fw_libs_build_dir)?; - - // Build set of already-compiled library names - let already_compiled: std::collections::HashSet = library_archives - .iter() - .filter_map(|p| p.file_stem()) - .filter_map(|s| s.to_str()) - .filter_map(|s| s.strip_prefix("lib")) - .map(|s| s.to_string()) - .collect(); - - // Get compiler flags for framework library compilation - let mut fw_defines = ctx.board.get_defines(); - fw_defines.extend(mcu_config.defines_map()); - - let fw_compiler = Esp32Compiler::with_temp_dir( - toolchain.get_gcc_path(), - toolchain.get_gxx_path(), - mcu_config.clone(), - &ctx.board.f_cpu, - fw_defines, - include_dirs.clone(), - params.profile, - params.verbose, - build_dir.join("tmp"), - ) - .with_build_unflags(ctx.build_unflags.clone()) - .with_eh_frame_policy(eh_frame_policy); - let fw_c_flags = - apply_overlay_flags(&fw_compiler.c_flags(), &user_overlay, "dummy.c"); - let fw_cpp_flags = - apply_overlay_flags(&fw_compiler.cpp_flags(), &user_overlay, "dummy.cpp"); - let fw_signature = framework_signature(&include_dirs, &fw_c_flags, &fw_cpp_flags); - - let mut fw_lib_count = 0; - let mut fw_lib_seen = 0; - if let Ok(entries) = std::fs::read_dir(&builtin_libs_dir) { - for entry in entries.flatten() { - let path = entry.path(); - if !path.is_dir() { - continue; - } - let lib_name = path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .to_lowercase(); - if lib_name.starts_with('.') || already_compiled.contains(&lib_name) { - continue; - } - - let lib_src = path.join("src"); - if !lib_src.is_dir() { - continue; - } - - fw_lib_seen += 1; - - // Check if archive already exists - let archive_path = fw_libs_build_dir.join(format!("lib{}.a", lib_name)); - if archive_path.exists() { - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-cache-hit name={} index={}", - lib_name, fw_lib_seen - )); - } - library_archives.push(archive_path); - fw_lib_count += 1; - continue; - } - - // Collect source files - let lib_info = - fbuild_packages::library::library_info::InstalledLibrary::new( - &path, &lib_name, - ); - let sources = lib_info.get_source_files(); - if sources.is_empty() { - continue; - } - let failure_marker = - framework_failure_marker(&fw_libs_build_dir, &lib_name); - if should_skip_failed_framework_lib( - &failure_marker, - &fw_signature, - &sources, - )? { - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-skip-failed name={} index={} sources={}", - lib_name, - fw_lib_seen, - sources.len() - )); - } - tracing::debug!( - "skipping previously failed framework library '{}'", - lib_name - ); - continue; - } - - let fw_jobs = crate::parallel::effective_jobs(params.jobs); - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-compile-start name={} index={} sources={} jobs={}", - lib_name, - fw_lib_seen, - sources.len(), - fw_jobs - )); - } - // Use gcc-ar for LTO archives so the linker-plugin index is written. - let fw_ar_path = toolchain.get_ar_path(); - let fw_gcc_ar_path = toolchain.get_gcc_ar_path(); - let fw_lib_ar_path = crate::pipeline::pick_archiver( - &fw_ar_path, - &fw_gcc_ar_path, - &fw_c_flags, - &fw_cpp_flags, - ); - match fbuild_packages::library::library_compiler::compile_library_with_jobs( - &lib_name, - &sources, - &include_dirs, - &toolchain.get_gcc_path(), - &toolchain.get_gxx_path(), - fw_lib_ar_path, - &fw_c_flags, - &fw_cpp_flags, - &fw_libs_build_dir, - params.verbose, - fw_jobs, - compiler_cache.as_deref(), - ) { - Ok(Some(archive)) => { - let _ = std::fs::remove_file(&failure_marker); - library_archives.push(archive); - fw_lib_count += 1; - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-compile-finish name={} index={} count={}", - lib_name, fw_lib_seen, fw_lib_count - )); - } - } - Ok(None) => { - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-header-only name={} index={}", - lib_name, fw_lib_seen - )); - } - } - Err(e) => { - // Non-fatal: some framework libs may fail to compile - // (e.g., platform-specific ones). The linker will report - // if any actually-needed symbols are missing. - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-compile-error name={} index={}", - lib_name, fw_lib_seen - )); - } - tracing::debug!( - "framework library {} failed to compile: {}", - lib_name, - e - ); - record_failed_framework_lib( - &failure_marker, - &fw_signature, - &e.to_string(), - ); - } - } - } - } - - if fw_lib_count > 0 { - tracing::info!("compiled {} framework built-in libraries", fw_lib_count); - } - } - perf.record("fw-libs", fw_libs_started.elapsed()); - perf.checkpoint("fw-libs-finish"); - } - - // 9. Scan sources - let sources = { - let _g = perf.phase("scan-sources"); - let scanner = SourceScanner::new(&ctx.src_dir, src_build_dir); - let variant_dir_opt = if variant_dir.exists() { - Some(variant_dir.as_path()) - } else { - None - }; - scanner.scan_all_filtered( - Some(&core_dir), - variant_dir_opt, - ctx.source_filter.as_deref(), - )? - }; - - tracing::info!( - "sources: {} sketch, {} core, {} variant", - sources.sketch_sources.len(), - sources.core_sources.len(), - sources.variant_sources.len(), - ); - - // 10-11. Compile - let mut defines = ctx.board.get_defines(); - defines.extend(mcu_config.defines_map()); - - // Defines required by the new framework (3.3.7+). - // Use \" escapes for GCC response file compatibility (see ctx.board.rs). - defines - .entry("ARDUINO_BOARD".to_string()) - .or_insert_with(|| format!("\\\"{}\\\"", ctx.board.name)); - defines - .entry("ARDUINO_VARIANT".to_string()) - .or_insert_with(|| format!("\\\"{}\\\"", ctx.board.variant)); - - let compiler = Esp32Compiler::with_temp_dir( - toolchain.get_gcc_path(), - toolchain.get_gxx_path(), - mcu_config.clone(), - &ctx.board.f_cpu, - defines, - include_dirs.clone(), - params.profile, - params.verbose, - build_dir.join("tmp"), - ) - .with_build_unflags(ctx.build_unflags.clone()) - .with_eh_frame_policy(eh_frame_policy); - let jobs = crate::parallel::effective_jobs(params.jobs); - tracing::info!("parallel compilation: {} jobs", jobs); - - // Build source lists and flags needed for compile_commands.json - let mut all_core_sources: Vec = Vec::new(); - all_core_sources.extend(sources.core_sources.iter().cloned()); - all_core_sources.extend(sources.variant_sources.iter().cloned()); - - // Precompute values needed for compile_commands.json in both paths - let include_flags = compiler.base.build_include_flags(); - let arch = if mcu_config.is_xtensa() { - crate::compile_database::TargetArchitecture::Xtensa - } else { - crate::compile_database::TargetArchitecture::Riscv32 - }; - - // compiledb_only: generate compile_commands.json without compiling - if params.compiledb_only { - let compile_database_path = { - let _g = perf.phase("compile-db"); - crate::pipeline::generate_compile_db( - compiler.gcc_path(), - compiler.gxx_path(), - &compiler.c_flags(), - &compiler.cpp_flags(), - &include_flags, - &user_overlay, - &src_overlay, - &all_core_sources, - &sources.sketch_sources, - core_build_dir, - src_build_dir, - build_dir, - ¶ms.project_dir, - arch, - )? - }; - let elapsed = start.elapsed().as_secs_f64(); - return Ok(BuildResult { - success: true, - firmware_path: None, - elf_path: None, - size_info: None, - symbol_map: None, - build_time_secs: elapsed, - message: format!( - "compile_commands.json generated for {} ({})", - params.env_name, ctx.board.mcu - ), - compile_database_path, - build_log: ctx.build_log, - }); - } - - // Compile core + variant sources in parallel - let build_log_mutex = std::sync::Mutex::new(ctx.build_log); - let core_result = { - let _g = perf.phase("compile-core-variant"); - crate::parallel::compile_sources_parallel( - &compiler, - &all_core_sources, - core_build_dir, - &user_overlay, - jobs, - Some(&build_log_mutex), - )? - }; - - // Compile sketch sources in parallel - let sketch_result = { - let _g = perf.phase("compile-sketch"); - crate::parallel::compile_sources_parallel( - &compiler, - &sources.sketch_sources, - src_build_dir, - &src_overlay, - jobs, - Some(&build_log_mutex), - )? - }; - - // Unwrap build log and flush collected warnings - let mut build_log = build_log_mutex.into_inner().unwrap(); - for w in core_result.warnings.iter().chain(&sketch_result.warnings) { - crate::build_output::collect_warnings(w, &mut build_log); - } - - let core_objects = core_result.objects; - let mut sketch_objects = sketch_result.objects; - - // Compile local libraries from the project's lib/ directory. - // PlatformIO discovers and compiles these automatically. - { - let _g = perf.phase("compile-local-libs"); - let local_lib_dir = params.project_dir.join("lib"); - if local_lib_dir.is_dir() { - if let Ok(entries) = std::fs::read_dir(&local_lib_dir) { - for entry in entries.flatten() { - let lib_path = entry.path(); - if !lib_path.is_dir() { - continue; - } - let lib_name = lib_path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .to_string(); - - let lib_info = - fbuild_packages::library::library_info::InstalledLibrary::new( - &lib_path, &lib_name, - ); - let lib_sources = lib_info.get_source_files(); - if lib_sources.is_empty() { - continue; - } - - let lib_build_dir = build_dir.join("lib").join(&lib_name); - tracing::info!( - "compiling local library '{}': {} source files", - lib_name, - lib_sources.len() - ); - - // Use gcc-ar for LTO archives so the linker-plugin index is written. - let local_ar_path = toolchain.get_ar_path(); - let local_gcc_ar_path = toolchain.get_gcc_ar_path(); - let local_c_flags = - apply_overlay_flags(&compiler.c_flags(), &src_overlay, "dummy.c"); - let local_cpp_flags = - apply_overlay_flags(&compiler.cpp_flags(), &src_overlay, "dummy.cpp"); - let local_lib_ar_path = crate::pipeline::pick_archiver( - &local_ar_path, - &local_gcc_ar_path, - &local_c_flags, - &local_cpp_flags, - ); - match fbuild_packages::library::library_compiler::compile_library_with_jobs( - &lib_name, - &lib_sources, - &include_dirs, - &toolchain.get_gcc_path(), - &toolchain.get_gxx_path(), - local_lib_ar_path, - &local_c_flags, - &local_cpp_flags, - &lib_build_dir, - params.verbose, - jobs, - compiler_cache.as_deref(), - ) { - Ok(Some(archive)) => { - library_archives.push(archive); - } - Ok(None) => {} // header-only - Err(e) => { - return Err(fbuild_core::FbuildError::BuildFailed(format!( - "local library '{}' failed to compile: {}", - lib_name, e - ))); - } - } - } - } - } - } - - // 11.5. Process embedded files (board_build.embed_files + embed_txtfiles) - // - // `.lnk` entries are pre-resolved: each `.lnk` is parsed, its blob is - // fetched (or pulled from the disk cache), and the materialized path - // is substituted in place before objcopy sees it. The `_lnk_leases` - // vector keeps cache leases alive until we leave this scope, so the - // disk-cache GC can't reap a blob mid-build. - if !embed_files.is_empty() || !embed_txtfiles.is_empty() { - let _g = perf.phase("embed-files"); - let embed_dir = build_dir.join("embed"); - std::fs::create_dir_all(&embed_dir)?; - - let lnk_dir = embed_dir.join("lnk"); - let mut _lnk_leases: Vec = Vec::new(); - let lnk_cache = fbuild_packages::DiskCache::open().ok(); - - let expand = |entries: &[String]| -> Result> { - let mut out = Vec::with_capacity(entries.len()); - for entry in entries { - let p = if Path::new(entry).is_absolute() { - std::path::PathBuf::from(entry) - } else { - params.project_dir.join(entry) - }; - if fbuild_packages::lnk::has_lnk_extension(&p) { - let cache = lnk_cache.as_ref().ok_or_else(|| { - fbuild_core::FbuildError::PackageError( - "disk cache unavailable; cannot resolve .lnk entries".to_string(), - ) - })?; - let m = fbuild_packages::lnk::materialize_lnk_entry(&p, &lnk_dir, cache)?; - out.push(m.target_path.to_string_lossy().into_owned()); - } else { - out.push(entry.clone()); - } - } - Ok(out) - }; - let resolved_embed_files = expand(&embed_files)?; - let resolved_embed_txtfiles = expand(&embed_txtfiles)?; - - let objcopy_path = toolchain.get_objcopy_path(); - let (output_target, binary_arch) = if mcu_config.is_riscv() { - ("elf32-littleriscv", "riscv") - } else { - ("elf32-xtensa-le", "xtensa") - }; - - let embed_objects = process_embed_files( - &resolved_embed_files, - &resolved_embed_txtfiles, - ¶ms.project_dir, - &embed_dir, - &objcopy_path, - output_target, - binary_arch, - params.verbose, - )?; - - sketch_objects.extend(embed_objects); - } - - // 11.6. Generate compile_commands.json - let compile_database_path = { - let _g = perf.phase("compile-db"); - crate::pipeline::generate_compile_db( - compiler.gcc_path(), - compiler.gxx_path(), - &compiler.c_flags(), - &compiler.cpp_flags(), - &include_flags, - &user_overlay, - &src_overlay, - &all_core_sources, - &sources.sketch_sources, - core_build_dir, - src_build_dir, - build_dir, - ¶ms.project_dir, - arch, - )? - }; - - // 12-13. Link + convert - // Library archives join core_objects in the archives parameter - let mut all_archives: Vec = core_objects; - all_archives.extend(library_archives); - - let linker = Esp32Linker::new( - toolchain.get_gcc_path(), - toolchain.get_ar_path(), - toolchain.get_objcopy_path(), - toolchain.get_size_path(), - mcu_config.clone(), - sdk_ld_flags, - sdk_lib_flags, - sdk_ld_scripts, - params.profile, - Some(flash_mode.clone()), - &flash_freq, - ctx.board.max_flash, - ctx.board.max_ram, - params.verbose, - ); - - let link_result = { - let _g = perf.phase("link-convert-size"); - crate::linker::Linker::link_all( - &linker, - &sketch_objects, - &all_archives, - build_dir, - &crate::linker::LinkExtraArgs { - flags: ctx.overlay_link_flags.clone(), - libs: ctx.overlay_link_libs.clone(), - }, - params.symbol_analysis, - )? - }; - - // 14. Prepare boot artifacts for deployment / emulation - let boot_artifacts_started = Instant::now(); - perf.checkpoint("boot-artifacts-start"); - let boot_dst = build_dir.join("bootloader.bin"); - let boot_bin_src = framework.get_bootloader_bin(&ctx.board.mcu); - if boot_bin_src.exists() { - // Pre-built bootloader.bin available — just copy - std::fs::copy(&boot_bin_src, &boot_dst)?; - tracing::info!("copied bootloader.bin"); - } else { - // Convert bootloader ELF to BIN using esptool elf2image. - // - // CRITICAL: The ESP32 ROM bootloader can only fetch the second-stage - // bootloader from flash in DIO mode (or OPI mode for octal-SPI - // boards). Even if the application is QIO, the bootloader itself - // must be DIO — otherwise the ROM bootloader cannot read it and - // the chip enters a watchdog reset loop with `Saved PC` pointing - // into ROM (e.g. `0x400454d5` on ESP32-S3). - // - // The Arduino-ESP32 framework ships pre-built ELFs named - // `bootloader__.elf` for this exact reason; we pick - // `bootloader_dio_80m.elf` for non-OPI boards and pass - // `--flash-mode dio` to esptool so the resulting BIN header - // (byte 0x02) has the correct mode. The application's flash - // mode (which may be QIO/QOUT/etc) is unaffected — that mode - // is encoded in the firmware.bin and applied later by the - // second-stage bootloader. - // - // We treat the app's `flash_mode` as OPI iff it equals "opi"; - // every other value (qio, qout, dio, dout, undefined) maps to - // a DIO bootloader. This matches Arduino-ESP32 / PlatformIO - // behaviour. Frequency is taken as-is because the bootloader - // ELFs only exist at 80m (and 120m for QIO chips), and the - // ROM bootloader runs at the boot frequency anyway. - let app_flash_mode = ctx - .board - .flash_mode - .as_deref() - .unwrap_or(mcu_config.default_flash_mode()); - let boot_flash_mode = if app_flash_mode == "opi" { - "opi" - } else { - "dio" - }; - let boot_elf = - framework.get_bootloader_elf(&ctx.board.mcu, boot_flash_mode, &flash_freq); - if boot_elf.exists() { - let boot_elf_str = boot_elf.to_string_lossy(); - let boot_dst_str = boot_dst.to_string_lossy(); - let flash_size = crate::esp32::mcu_config::bytes_to_flash_size( - ctx.board.max_flash, - mcu_config.default_flash_size(), - ); - let args = [ - "esptool", - "--chip", - &ctx.board.mcu, - "elf2image", - "--flash-mode", - boot_flash_mode, - "--flash-freq", - &flash_freq, - "--flash-size", - flash_size, - &boot_elf_str, - "-o", - &boot_dst_str, - ]; - match fbuild_core::subprocess::run_command( - &args, - None, - None, - Some(std::time::Duration::from_secs(30)), - ) { - Ok(result) if result.success() => { - tracing::info!("converted bootloader ELF → bootloader.bin"); - } - Ok(result) => { - tracing::warn!( - "bootloader elf2image failed: {}{}", - result.stderr, - result.stdout - ); - } - Err(e) => { - tracing::warn!("esptool not found for bootloader conversion: {}", e); - } - } - } else { - tracing::warn!( - "no bootloader found at {} or {}", - boot_bin_src.display(), - boot_elf.display() - ); - } - } - - let parts_dst = build_dir.join("partitions.bin"); - let parts_bin_src = framework.get_partitions_bin(&ctx.board.mcu); - if parts_bin_src.exists() { - // Pre-built partitions.bin available — just copy - std::fs::copy(&parts_bin_src, &parts_dst)?; - tracing::info!("copied partitions.bin"); - } else { - // Generate partitions.bin from CSV using gen_esp32part.py - let partitions_name = ctx.board.partitions.as_deref().unwrap_or("default.csv"); - let parts_csv = framework.get_partitions_csv(partitions_name); - let gen_tool = framework.get_gen_esp32part(); - if parts_csv.exists() && gen_tool.exists() { - let gen_tool_str = gen_tool.to_string_lossy(); - let parts_csv_str = parts_csv.to_string_lossy(); - let parts_dst_str = parts_dst.to_string_lossy(); - let args = [ - "python", - &gen_tool_str, - "-q", - &parts_csv_str, - &parts_dst_str, - ]; - match fbuild_core::subprocess::run_command( - &args, - None, - None, - Some(std::time::Duration::from_secs(10)), - ) { - Ok(result) if result.success() => { - tracing::info!("generated partitions.bin from {}", partitions_name); - } - Ok(result) => { - tracing::warn!("gen_esp32part.py failed: {}", result.stderr); - } - Err(e) => { - tracing::warn!("python not found for partitions generation: {}", e); - } - } - } else { - tracing::warn!( - "no partitions source: csv={} gen_tool={}", - parts_csv.display(), - gen_tool.display() - ); - } - } - - let boot_app0_src = framework.get_boot_app0_bin(); - let boot_app0_dst = build_dir.join("boot_app0.bin"); - if boot_app0_src.exists() { - std::fs::copy(&boot_app0_src, &boot_app0_dst)?; - tracing::info!("copied boot_app0.bin"); - } - perf.record("boot-artifacts", boot_artifacts_started.elapsed()); - perf.checkpoint("boot-artifacts-finish"); - - // 15. Size reporting + result assembly - let fingerprint_started = Instant::now(); - perf.checkpoint("fingerprint-save-start"); - let fast_path_ready = fast_path - .required_artifacts() - .iter() - .all(|path| path.exists()) - && compile_db_is_current(build_dir, ¶ms.project_dir); - if fast_path_ready { - crate::build_fingerprint::persist_fast_path_success( - &fast_path, - &FastPathPersistInputs { - metadata_hash: &metadata_hash, - size_info: link_result.size_info.clone(), - watch_set_cache: params.watch_set_cache.as_deref(), - compiler_cache: compiler_cache.as_deref(), - }, - ); - } else { - tracing::warn!( - "skipping ESP32 fast-path persistence because final artifacts are incomplete" - ); - } - perf.record("fingerprint-save", fingerprint_started.elapsed()); - perf.checkpoint("fingerprint-save-finish"); - - crate::pipeline::handle_link_result( - &link_result, - &mut build_log, - params.symbol_analysis_path.as_deref(), - params.verbose, - ); - let elapsed = start.elapsed().as_secs_f64(); - let platform_label = format!("ESP32 ({})", ctx.board.mcu); - Ok(crate::pipeline::assemble_build_result( - link_result, - elapsed, - &platform_label, - ¶ms.env_name, - compile_database_path, - build_log, - )) - } -} - -/// Apply user build_flags from platformio.ini onto base compiler flags. -/// -/// Matches PlatformIO behavior: user flags are appended to common flags, -/// but `-std=` flags replace the existing standard (not stack). `-D` flags are -/// deduplicated by macro name so later values override earlier defaults without -/// tripping GCC redefinition warnings. -fn apply_user_flags(base_flags: &[String], user_flags: &[String]) -> Vec { - let mut result = base_flags.to_vec(); - for flag in user_flags { - if flag.starts_with("-std=") { - // Replace any existing -std= flag - result.retain(|f| !f.starts_with("-std=")); - } else if let Some(define_name) = define_flag_name(flag) { - // Replace any existing -DNAME / -DNAME=value flag with the same macro name. - result.retain(|f| define_flag_name(f) != Some(define_name)); - } - result.push(flag.clone()); - } - result -} - -fn apply_overlay_flags( - base_flags: &[String], - overlay: &LanguageExtraFlags, - probe_name: &str, -) -> Vec { - apply_user_flags(base_flags, &overlay.for_source(Path::new(probe_name))) -} - -fn define_flag_name(flag: &str) -> Option<&str> { - let define = flag.strip_prefix("-D")?; - let name = define - .split_once('=') - .map_or(define, |(name, _)| name) - .trim(); - if name.is_empty() { - None - } else { - Some(name) - } -} - -/// Resolve framework + toolchain for pioarduino mode (GCC 14 + ESP-IDF 5.x). -/// -/// Downloads pioarduino platform.json, resolves toolchain via metadata, -/// and downloads the split framework + libs packages. -fn resolve_pioarduino_packages( - project_dir: &Path, - mcu: &str, - mcu_config: &super::mcu_config::Esp32McuConfig, -) -> Result<( - fbuild_packages::toolchain::Esp32Toolchain, - fbuild_packages::library::Esp32Framework, -)> { - // Ensure pioarduino platform (contains platform.json with metadata URLs) - let platform = fbuild_packages::library::Esp32Platform::new(project_dir); - fbuild_packages::Package::ensure_installed(&platform)?; - - // Resolve toolchain via metadata - let toolchain = resolve_and_create_toolchain(&platform, project_dir, mcu_config)?; - - // Resolve framework - let framework = match platform.get_package_url("framework-arduinoespressif32") { - Ok(url) => { - tracing::info!("resolved framework URL from platform.json"); - fbuild_packages::library::Esp32Framework::from_url(project_dir, &url) - } - Err(e) => { - tracing::warn!("could not resolve framework URL, using legacy: {}", e); - fbuild_packages::library::Esp32Framework::new(project_dir, mcu) - } - }; - - // Ensure framework is installed before trying to install libs - let _ = fbuild_packages::Package::ensure_installed(&framework)?; - - // Ensure SDK libs (split package in pioarduino 3.3.7+) - if let Ok(libs_url) = platform.get_package_url("framework-arduinoespressif32-libs") { - framework.ensure_libs(&libs_url)?; - } - - // Ensure MCU-specific skeleton libs (e.g. ESP32-C2, ESP32-C61). - // Some MCUs ship their SDK in a separate skeleton package. - let mcu_suffix = mcu.strip_prefix("esp32").unwrap_or(""); - if !mcu_suffix.is_empty() { - let skeleton_name = format!("framework-arduino-{}-skeleton-lib", mcu_suffix); - if let Ok(skeleton_url) = platform.get_package_url(&skeleton_name) { - framework.ensure_mcu_libs(&skeleton_url, mcu)?; - } - } - - Ok((toolchain, framework)) -} - -fn resolve_and_create_toolchain( - platform: &fbuild_packages::library::Esp32Platform, - project_dir: &Path, - mcu_config: &super::mcu_config::Esp32McuConfig, -) -> Result { - let is_riscv = mcu_config.is_riscv(); - let prefix = mcu_config.toolchain_prefix(); - - // Try metadata-based resolution - match platform.get_toolchain_metadata_url(is_riscv) { - Ok(metadata_url) => { - let toolchain_name = if is_riscv { - "toolchain-riscv32-esp" - } else { - "toolchain-xtensa-esp-elf" - }; - - let cache = fbuild_packages::Cache::new(project_dir); - let cache_dir = cache.toolchains_dir().join(toolchain_name); - - match fbuild_packages::toolchain::esp32_metadata::resolve_toolchain_url_sync( - &metadata_url, - toolchain_name, - &cache_dir, - ) { - Ok(resolved) => { - tracing::info!("resolved {} toolchain URL from metadata", toolchain_name); - Ok(fbuild_packages::toolchain::Esp32Toolchain::from_resolved( - project_dir, - &resolved.url, - resolved.sha256.as_deref(), - is_riscv, - &prefix, - )) - } - Err(e) => { - tracing::warn!("metadata resolution failed, using legacy URLs: {}", e); - Ok(fbuild_packages::toolchain::Esp32Toolchain::new( - project_dir, - is_riscv, - &prefix, - )) - } - } - } - Err(e) => { - tracing::warn!( - "could not read platform.json, using legacy toolchain URLs: {}", - e - ); - Ok(fbuild_packages::toolchain::Esp32Toolchain::new( - project_dir, - is_riscv, - &prefix, - )) - } - } -} - -/// Process `board_build.embed_files` and `board_build.embed_txtfiles`. -/// -/// Converts data files into linkable ELF objects using `objcopy --input-target binary`. -/// This generates `_binary__start`, `_binary__end`, and `_binary__size` -/// symbols that the firmware can reference. -/// -/// - `embed_files`: embedded as-is (binary) -/// - `embed_txtfiles`: a null-terminated copy is created first, then embedded -#[allow(clippy::too_many_arguments)] -fn process_embed_files( - embed_files: &[String], - embed_txtfiles: &[String], - project_dir: &Path, - embed_dir: &Path, - objcopy_path: &Path, - output_target: &str, - binary_arch: &str, - verbose: bool, -) -> Result> { - use fbuild_core::subprocess::run_command; - - let mut objects = Vec::new(); - - // Helper: convert a relative file path to the object file name. - // e.g. "config/timezones.json" → "config_timezones_json.o" - let to_obj_name = |path: &str| -> String { - let sanitized = path.replace(['/', '\\', '.', '-'], "_"); - format!("{}.o", sanitized) - }; - - // Process binary embed files (embed as-is, cwd=project_dir) - for file in embed_files { - let src_path = project_dir.join(file); - if !src_path.exists() { - tracing::warn!("embed_files: {} not found, skipping", src_path.display()); - continue; - } - - let obj_name = to_obj_name(file); - let obj_path = embed_dir.join(&obj_name); - - if obj_path.exists() { - objects.push(obj_path); - continue; - } - - let args = [ - objcopy_path.to_string_lossy().to_string(), - "--input-target".to_string(), - "binary".to_string(), - "--output-target".to_string(), - output_target.to_string(), - "--binary-architecture".to_string(), - binary_arch.to_string(), - "--rename-section".to_string(), - ".data=.rodata.embedded".to_string(), - file.replace('\\', "/"), - obj_path.to_string_lossy().to_string(), - ]; - - if verbose { - tracing::info!("embed: {}", args.join(" ")); - } - - let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, Some(project_dir), None, None)?; - - if !result.success() { - return Err(fbuild_core::FbuildError::BuildFailed(format!( - "objcopy failed for embed file {}:\n{}", - file, result.stderr - ))); - } - - tracing::info!("embedded binary file: {}", file); - objects.push(obj_path); - } - - // Process text embed files (null-terminated copy, then objcopy from embed_dir) - for file in embed_txtfiles { - let src_path = project_dir.join(file); - if !src_path.exists() { - tracing::warn!("embed_txtfiles: {} not found, skipping", src_path.display()); - continue; - } - - let obj_name = to_obj_name(file); - let obj_path = embed_dir.join(&obj_name); - - if obj_path.exists() { - objects.push(obj_path); - continue; - } - - // Create null-terminated copy in embed_dir preserving relative path - let rel_dest = embed_dir.join(file); - if let Some(parent) = rel_dest.parent() { - std::fs::create_dir_all(parent)?; - } - let mut content = std::fs::read(&src_path)?; - if content.last() != Some(&0) { - content.push(0); - } - std::fs::write(&rel_dest, &content)?; - - let args = [ - objcopy_path.to_string_lossy().to_string(), - "--input-target".to_string(), - "binary".to_string(), - "--output-target".to_string(), - output_target.to_string(), - "--binary-architecture".to_string(), - binary_arch.to_string(), - "--rename-section".to_string(), - ".data=.rodata.embedded".to_string(), - file.replace('\\', "/"), - obj_path.to_string_lossy().to_string(), - ]; - - if verbose { - tracing::info!("embed txt: {}", args.join(" ")); - } - - // Run from embed_dir so objcopy generates symbols from the relative path - let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, Some(embed_dir), None, None)?; - - if !result.success() { - return Err(fbuild_core::FbuildError::BuildFailed(format!( - "objcopy failed for embed txtfile {}:\n{}", - file, result.stderr - ))); - } - - tracing::info!("embedded text file: {}", file); - objects.push(obj_path); - } - - if !objects.is_empty() { - tracing::info!("processed {} embedded files", objects.len()); - } - - Ok(objects) -} - -/// Create an ESP32 orchestrator (convenience for get_orchestrator dispatch). -pub fn create() -> Box { - Box::new(Esp32Orchestrator) -} - -/// Determine whether ARDUINO_USB_CDC_ON_BOOT is effectively enabled. -/// -/// Combines `board_extra_flags` (a space-separated string from the board JSON) with -/// `user_build_flags` (from platformio.ini `build_flags`). Board flags are applied -/// first; user flags can override them. The **last** definition of -/// `-DARDUINO_USB_CDC_ON_BOOT=N` wins, matching C preprocessor semantics. -/// -/// Returns `true` only if the final effective value is `1`. -pub fn cdc_on_boot_enabled(board_extra_flags: Option<&str>, user_build_flags: &[String]) -> bool { - // Collect all flags in application order: board first, then user. - let board_tokens: Vec = board_extra_flags - .unwrap_or("") - .split_whitespace() - .map(|s| s.to_string()) - .collect(); - - let all_flags: Vec<&str> = board_tokens - .iter() - .map(|s| s.as_str()) - .chain(user_build_flags.iter().map(|s| s.as_str())) - .collect(); - - let mut effective: Option = None; - - for flag in &all_flags { - // Normalise: strip leading whitespace and optional `-D` prefix added by some tools. - let stripped = flag.trim(); - // Match `-DARDUINO_USB_CDC_ON_BOOT=VALUE` or `ARDUINO_USB_CDC_ON_BOOT=VALUE` - let without_d = stripped.strip_prefix("-D").unwrap_or(stripped); - - if let Some(value) = without_d.strip_prefix("ARDUINO_USB_CDC_ON_BOOT=") { - effective = Some(value.trim() == "1"); - } - } - - effective.unwrap_or(false) -} - -/// Emit a `tracing::warn!` if CDC on boot is effectively enabled. -/// -/// `ARDUINO_USB_CDC_ON_BOOT=1` initialises the USB CDC port during boot via native -/// USB (ESP32-S3, C3, C6, S2, …). When no USB host is connected at power-on any -/// call to `Serial.print()` will block indefinitely because the CDC TX buffer has no -/// consumer to drain it. -pub fn warn_if_cdc_on_boot( - board_name: &str, - board_extra_flags: Option<&str>, - user_build_flags: &[String], -) { - if cdc_on_boot_enabled(board_extra_flags, user_build_flags) { - tracing::warn!( - "Board '{}' has ARDUINO_USB_CDC_ON_BOOT=1. \ - If no USB host is connected at power-on, Serial.print() will block \ - indefinitely. Add -DARDUINO_USB_CDC_ON_BOOT=0 to build_flags to suppress this warning.", - board_name - ); - } -} - -/// Check if a project is configured for ESP32 by reading its platformio.ini. -pub fn is_esp32_project(project_dir: &Path, env_name: &str) -> bool { - crate::pipeline::is_platform_project(project_dir, env_name, fbuild_core::Platform::Espressif32) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::time::Duration; - - #[test] - fn test_apply_user_flags_replaces_std_flag() { - let base = vec!["-Os".to_string(), "-std=gnu++2b".to_string()]; - let user = vec!["-std=gnu++20".to_string()]; - - let result = apply_user_flags(&base, &user); - - assert_eq!(result, vec!["-Os", "-std=gnu++20"]); - } - - #[test] - fn test_apply_user_flags_replaces_define_with_same_name() { - let base = vec![ - r#"-DIDF_VER=\"v5.5.1-710-g8410210c9a\""#.to_string(), - r#"-DESP_MDNS_VERSION_NUMBER=\"1.9.0\""#.to_string(), - "-Os".to_string(), - ]; - let user = vec![ - r#"-DESP_MDNS_VERSION_NUMBER=\"1.9.1\""#.to_string(), - r#"-DIDF_VER=\"v5.5.2-729-g87912cd291\""#.to_string(), - ]; - - let result = apply_user_flags(&base, &user); - - assert_eq!( - result, - vec![ - "-Os", - r#"-DESP_MDNS_VERSION_NUMBER=\"1.9.1\""#, - r#"-DIDF_VER=\"v5.5.2-729-g87912cd291\""#, - ] - ); - } - - #[test] - fn test_apply_user_flags_replaces_bare_define_with_value_define() { - let base = vec!["-DFOO".to_string(), "-DBAR=1".to_string()]; - let user = vec!["-DFOO=2".to_string()]; - - let result = apply_user_flags(&base, &user); - - assert_eq!(result, vec!["-DBAR=1", "-DFOO=2"]); - } - - #[test] - fn test_esp32_orchestrator_platform() { - let orch = Esp32Orchestrator; - assert_eq!(orch.platform(), Platform::Espressif32); - } - - #[test] - fn test_is_esp32_project() { - let tmp = tempfile::TempDir::new().unwrap(); - std::fs::write( - tmp.path().join("platformio.ini"), - "[env:esp32c6]\nplatform = espressif32\nboard = esp32-c6\nframework = arduino\n", - ) - .unwrap(); - assert!(is_esp32_project(tmp.path(), "esp32c6")); - assert!(!is_esp32_project(tmp.path(), "uno")); - } - - #[test] - fn test_is_not_esp32_project() { - let tmp = tempfile::TempDir::new().unwrap(); - std::fs::write( - tmp.path().join("platformio.ini"), - "[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n", - ) - .unwrap(); - assert!(!is_esp32_project(tmp.path(), "uno")); - } - - // --- CDC on boot warning tests --- - - /// Board that enables CDC on boot via extra_flags (e.g. Adafruit Feather ESP32-S3). - #[test] - fn test_cdc_enabled_by_board_extra_flags() { - let board_flags = Some( - "-DARDUINO_ADAFRUIT_FEATHER_ESP32S3 -DARDUINO_USB_CDC_ON_BOOT=1 -DARDUINO_RUNNING_CORE=1" - ); - assert!(cdc_on_boot_enabled(board_flags, &[])); - } - - /// Board that explicitly disables CDC on boot. - #[test] - fn test_cdc_disabled_by_board_extra_flags() { - let board_flags = Some("-DARDUINO_FREENOVE_ESP32_S3_WROOM -DARDUINO_USB_CDC_ON_BOOT=0"); - assert!(!cdc_on_boot_enabled(board_flags, &[])); - } - - /// Plain ESP32 dev board with no CDC flag at all — not enabled. - #[test] - fn test_no_cdc_flag_returns_false() { - let board_flags = Some("-DARDUINO_ESP32_DEV"); - assert!(!cdc_on_boot_enabled(board_flags, &[])); - } - - /// No board flags at all — not enabled. - #[test] - fn test_no_flags_at_all_returns_false() { - assert!(!cdc_on_boot_enabled(None, &[])); - } - - /// User build_flags override a board-level enable (last definition wins). - #[test] - fn test_user_flag_overrides_board_enable() { - let board_flags = Some("-DARDUINO_USB_CDC_ON_BOOT=1"); - let user_flags = vec!["-DARDUINO_USB_CDC_ON_BOOT=0".to_string()]; - assert!(!cdc_on_boot_enabled(board_flags, &user_flags)); - } - - /// User build_flags can enable CDC that the board left unconfigured. - #[test] - fn test_user_flag_enables_cdc() { - let board_flags = Some("-DARDUINO_ESP32_DEV"); - let user_flags = vec!["-DARDUINO_USB_CDC_ON_BOOT=1".to_string()]; - assert!(cdc_on_boot_enabled(board_flags, &user_flags)); - } - - /// Multiple user flags — last one wins. - #[test] - fn test_last_user_flag_wins() { - let board_flags = Some("-DARDUINO_USB_CDC_ON_BOOT=1"); - let user_flags = vec![ - "-DARDUINO_USB_CDC_ON_BOOT=0".to_string(), - "-DARDUINO_USB_CDC_ON_BOOT=1".to_string(), - ]; - assert!(cdc_on_boot_enabled(board_flags, &user_flags)); - } - - /// Flags provided as whitespace-separated string should be parsed correctly. - #[test] - fn test_multi_flag_string_parsed_correctly() { - // Board flags: the enable flag appears after another flag. - let board_flags = Some("-DSOME_DEFINE -DARDUINO_USB_CDC_ON_BOOT=1 -DANOTHER=1"); - assert!(cdc_on_boot_enabled(board_flags, &[])); - } - - /// `warn_if_cdc_on_boot` should not panic for any combination of inputs. - #[test] - fn test_warn_if_cdc_on_boot_no_panic() { - // CDC enabled — triggers warning path - warn_if_cdc_on_boot( - "Adafruit Feather ESP32-S3", - Some("-DARDUINO_USB_CDC_ON_BOOT=1"), - &[], - ); - // CDC disabled — no warning - warn_if_cdc_on_boot( - "Freenove ESP32-S3-WROOM", - Some("-DARDUINO_USB_CDC_ON_BOOT=0"), - &[], - ); - // No flag at all — no warning - warn_if_cdc_on_boot("ESP32 Dev Module", Some("-DARDUINO_ESP32_DEV"), &[]); - // No board flags — no warning - warn_if_cdc_on_boot("Some Board", None, &[]); - // User override suppresses board enable - warn_if_cdc_on_boot( - "Some Board", - Some("-DARDUINO_USB_CDC_ON_BOOT=1"), - &["-DARDUINO_USB_CDC_ON_BOOT=0".to_string()], - ); - } - - #[test] - fn test_framework_signature_changes_with_flags() { - let includes = vec![PathBuf::from("C:/sdk/include")]; - let sig_a = framework_signature( - &includes, - &["-O2".to_string()], - &["-std=gnu++17".to_string()], - ); - let sig_b = framework_signature( - &includes, - &["-O0".to_string()], - &["-std=gnu++17".to_string()], - ); - assert_ne!(sig_a, sig_b); - } - - #[test] - fn test_apply_user_flags_replaces_existing_define_by_key() { - let merged = apply_user_flags( - &[r#"-DIDF_VER=\"old\""#.to_string(), "-O2".to_string()], - &[r#"-DIDF_VER=\"new\""#.to_string()], - ); - assert_eq!( - merged, - vec![r#"-O2"#.to_string(), r#"-DIDF_VER=\"new\""#.to_string()] - ); - } - - #[test] - fn test_apply_user_flags_keeps_last_user_define() { - let merged = apply_user_flags( - &[], - &[ - r#"-DMBEDTLS_CONFIG_FILE=\"a.h\""#.to_string(), - r#"-DMBEDTLS_CONFIG_FILE=\"b.h\""#.to_string(), - ], - ); - assert_eq!(merged, vec![r#"-DMBEDTLS_CONFIG_FILE=\"b.h\""#.to_string()]); - } - - #[test] - fn test_skip_failed_framework_lib_when_marker_matches_and_is_current() { - let tmp = tempfile::TempDir::new().unwrap(); - let source = tmp.path().join("Matter.cpp"); - std::fs::write(&source, "int x;").unwrap(); - let marker = framework_failure_marker(tmp.path(), "matter"); - let sig = framework_signature(&[], &["-O2".to_string()], &["-std=gnu++2b".to_string()]); - std::thread::sleep(Duration::from_millis(20)); - record_failed_framework_lib(&marker, &sig, "compile failed"); - - assert!(should_skip_failed_framework_lib(&marker, &sig, &[source]).unwrap()); - } - - #[test] - fn test_retry_failed_framework_lib_after_source_change() { - let tmp = tempfile::TempDir::new().unwrap(); - let source = tmp.path().join("Matter.cpp"); - std::fs::write(&source, "int x;").unwrap(); - let marker = framework_failure_marker(tmp.path(), "matter"); - let sig = framework_signature(&[], &["-O2".to_string()], &["-std=gnu++2b".to_string()]); - std::thread::sleep(Duration::from_millis(20)); - record_failed_framework_lib(&marker, &sig, "compile failed"); - std::thread::sleep(Duration::from_millis(20)); - std::fs::write(&source, "int y;").unwrap(); - - assert!(!should_skip_failed_framework_lib(&marker, &sig, &[source]).unwrap()); - } - - #[test] - fn test_retry_failed_framework_lib_after_signature_change() { - let tmp = tempfile::TempDir::new().unwrap(); - let source = tmp.path().join("Matter.cpp"); - std::fs::write(&source, "int x;").unwrap(); - let marker = framework_failure_marker(tmp.path(), "matter"); - let sig_a = framework_signature(&[], &["-O2".to_string()], &["-std=gnu++2b".to_string()]); - let sig_b = framework_signature(&[], &["-O0".to_string()], &["-std=gnu++2b".to_string()]); - std::thread::sleep(Duration::from_millis(20)); - record_failed_framework_lib(&marker, &sig_a, "compile failed"); - - assert!(!should_skip_failed_framework_lib(&marker, &sig_b, &[source]).unwrap()); - } -} diff --git a/crates/fbuild-build/src/esp32/orchestrator/README.md b/crates/fbuild-build/src/esp32/orchestrator/README.md new file mode 100644 index 00000000..916eaeb6 --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/README.md @@ -0,0 +1,23 @@ +# esp32::orchestrator + +ESP32 build orchestrator split into focused submodules so no single file +exceeds the 1000-LOC gate. + +| File | Responsibility | +|---|---| +| `mod.rs` | Module root; exposes `Esp32Orchestrator` and the small public helpers (`create`, `is_esp32_project`, `cdc_on_boot_enabled`, `warn_if_cdc_on_boot`). | +| `build.rs` | `impl BuildOrchestrator for Esp32Orchestrator`. Top-level phase wiring. | +| `packages.rs` | pioarduino platform / framework / toolchain resolution. | +| `framework_libs.rs` | Compiles built-in Arduino libraries shipped with the framework. | +| `local_libs.rs` | Compiles libraries from the project's `lib/` directory. | +| `embed.rs` | `objcopy --input-target binary` conversion of embedded files. | +| `embed_stage.rs` | `.lnk` resolution and target selection wrapper around `embed`. | +| `boot_artifacts.rs` | Produces `bootloader.bin`, `partitions.bin`, `boot_app0.bin`. | +| `fingerprint.rs` | Serialised metadata struct used for the fast-path hash. | +| `helpers.rs` | Flag merging (`apply_user_flags`), failure markers, signature, etc. | +| `cdc.rs` | USB-CDC-on-boot warning + small public convenience helpers. | +| `tests.rs` | Unit tests for the helpers + the public surface. | + +External crates continue to reference items at the original path +`fbuild_build::esp32::orchestrator::Esp32Orchestrator`; that public API is +preserved by re-exports in `mod.rs`. diff --git a/crates/fbuild-build/src/esp32/orchestrator/boot_artifacts.rs b/crates/fbuild-build/src/esp32/orchestrator/boot_artifacts.rs new file mode 100644 index 00000000..7aeba170 --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/boot_artifacts.rs @@ -0,0 +1,170 @@ +//! Prepare bootloader.bin, partitions.bin and boot_app0.bin for deployment / emulation. + +use std::path::Path; +use std::time::Instant; + +use fbuild_core::Result; + +use super::super::mcu_config::Esp32McuConfig; + +/// Stage boot/partition/boot_app0 binaries into `build_dir`. Logs warnings on +/// missing inputs but does not error — the linker output remains usable for +/// in-tree flows even when the boot artifacts cannot be produced. +pub(super) fn prepare_boot_artifacts( + build_dir: &Path, + framework: &fbuild_packages::library::Esp32Framework, + board: &fbuild_config::BoardConfig, + mcu_config: &Esp32McuConfig, + flash_freq: &str, + perf: &mut crate::perf_log::PerfTimer, +) -> Result<()> { + let boot_artifacts_started = Instant::now(); + perf.checkpoint("boot-artifacts-start"); + let boot_dst = build_dir.join("bootloader.bin"); + let boot_bin_src = framework.get_bootloader_bin(&board.mcu); + if boot_bin_src.exists() { + // Pre-built bootloader.bin available — just copy + std::fs::copy(&boot_bin_src, &boot_dst)?; + tracing::info!("copied bootloader.bin"); + } else { + // Convert bootloader ELF to BIN using esptool elf2image. + // + // CRITICAL: The ESP32 ROM bootloader can only fetch the second-stage + // bootloader from flash in DIO mode (or OPI mode for octal-SPI + // boards). Even if the application is QIO, the bootloader itself + // must be DIO — otherwise the ROM bootloader cannot read it and + // the chip enters a watchdog reset loop with `Saved PC` pointing + // into ROM (e.g. `0x400454d5` on ESP32-S3). + // + // The Arduino-ESP32 framework ships pre-built ELFs named + // `bootloader__.elf` for this exact reason; we pick + // `bootloader_dio_80m.elf` for non-OPI boards and pass + // `--flash-mode dio` to esptool so the resulting BIN header + // (byte 0x02) has the correct mode. The application's flash + // mode (which may be QIO/QOUT/etc) is unaffected — that mode + // is encoded in the firmware.bin and applied later by the + // second-stage bootloader. + // + // We treat the app's `flash_mode` as OPI iff it equals "opi"; + // every other value (qio, qout, dio, dout, undefined) maps to + // a DIO bootloader. Frequency is taken as-is because the bootloader + // ELFs only exist at 80m (and 120m for QIO chips), and the + // ROM bootloader runs at the boot frequency anyway. + let app_flash_mode = board + .flash_mode + .as_deref() + .unwrap_or(mcu_config.default_flash_mode()); + let boot_flash_mode = if app_flash_mode == "opi" { + "opi" + } else { + "dio" + }; + let boot_elf = framework.get_bootloader_elf(&board.mcu, boot_flash_mode, flash_freq); + if boot_elf.exists() { + let boot_elf_str = boot_elf.to_string_lossy(); + let boot_dst_str = boot_dst.to_string_lossy(); + let flash_size = crate::esp32::mcu_config::bytes_to_flash_size( + board.max_flash, + mcu_config.default_flash_size(), + ); + let args = [ + "esptool", + "--chip", + &board.mcu, + "elf2image", + "--flash-mode", + boot_flash_mode, + "--flash-freq", + flash_freq, + "--flash-size", + flash_size, + &boot_elf_str, + "-o", + &boot_dst_str, + ]; + match fbuild_core::subprocess::run_command( + &args, + None, + None, + Some(std::time::Duration::from_secs(30)), + ) { + Ok(result) if result.success() => { + tracing::info!("converted bootloader ELF → bootloader.bin"); + } + Ok(result) => { + tracing::warn!( + "bootloader elf2image failed: {}{}", + result.stderr, + result.stdout + ); + } + Err(e) => { + tracing::warn!("esptool not found for bootloader conversion: {}", e); + } + } + } else { + tracing::warn!( + "no bootloader found at {} or {}", + boot_bin_src.display(), + boot_elf.display() + ); + } + } + + let parts_dst = build_dir.join("partitions.bin"); + let parts_bin_src = framework.get_partitions_bin(&board.mcu); + if parts_bin_src.exists() { + // Pre-built partitions.bin available — just copy + std::fs::copy(&parts_bin_src, &parts_dst)?; + tracing::info!("copied partitions.bin"); + } else { + // Generate partitions.bin from CSV using gen_esp32part.py + let partitions_name = board.partitions.as_deref().unwrap_or("default.csv"); + let parts_csv = framework.get_partitions_csv(partitions_name); + let gen_tool = framework.get_gen_esp32part(); + if parts_csv.exists() && gen_tool.exists() { + let gen_tool_str = gen_tool.to_string_lossy(); + let parts_csv_str = parts_csv.to_string_lossy(); + let parts_dst_str = parts_dst.to_string_lossy(); + let args = [ + "python", + &gen_tool_str, + "-q", + &parts_csv_str, + &parts_dst_str, + ]; + match fbuild_core::subprocess::run_command( + &args, + None, + None, + Some(std::time::Duration::from_secs(10)), + ) { + Ok(result) if result.success() => { + tracing::info!("generated partitions.bin from {}", partitions_name); + } + Ok(result) => { + tracing::warn!("gen_esp32part.py failed: {}", result.stderr); + } + Err(e) => { + tracing::warn!("python not found for partitions generation: {}", e); + } + } + } else { + tracing::warn!( + "no partitions source: csv={} gen_tool={}", + parts_csv.display(), + gen_tool.display() + ); + } + } + + let boot_app0_src = framework.get_boot_app0_bin(); + let boot_app0_dst = build_dir.join("boot_app0.bin"); + if boot_app0_src.exists() { + std::fs::copy(&boot_app0_src, &boot_app0_dst)?; + tracing::info!("copied boot_app0.bin"); + } + perf.record("boot-artifacts", boot_artifacts_started.elapsed()); + perf.checkpoint("boot-artifacts-finish"); + Ok(()) +} diff --git a/crates/fbuild-build/src/esp32/orchestrator/build.rs b/crates/fbuild-build/src/esp32/orchestrator/build.rs new file mode 100644 index 00000000..c733d3f7 --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/build.rs @@ -0,0 +1,782 @@ +//! `impl BuildOrchestrator for Esp32Orchestrator` — the high-level build flow. +//! +//! Most heavy work delegates to sibling submodules (`packages`, `framework_libs`, +//! `local_libs`, `boot_artifacts`, `embed_stage`, `helpers`). + +use std::time::Instant; + +use fbuild_core::{Platform, Result}; +use fbuild_packages::Framework; + +use super::super::esp32_compiler::Esp32Compiler; +use super::super::esp32_linker::Esp32Linker; +use super::super::mcu_config::get_mcu_config; +use super::boot_artifacts::prepare_boot_artifacts; +use super::cdc::warn_if_cdc_on_boot; +use super::embed_stage::stage_embed_files; +use super::fingerprint::Esp32FingerprintMetadata; +use super::framework_libs::compile_framework_builtin_libs; +use super::helpers::{ + apply_overlay_flags, compile_db_is_current, profile_label, +}; +use super::local_libs::compile_local_libraries; +use super::packages::resolve_pioarduino_packages; +use super::Esp32Orchestrator; + +use crate::build_fingerprint::{ + expected_fast_path_artifacts, stable_hash_json, FastPathCheckInputs, FastPathContract, + FastPathPersistInputs, BUILD_FINGERPRINT_VERSION, +}; +use crate::compiler::Compiler as _; +use crate::flag_overlay::LanguageExtraFlags; +use crate::linker::LinkerScripts; +use crate::{BuildOrchestrator, BuildParams, BuildResult, SourceScanner}; + +impl BuildOrchestrator for Esp32Orchestrator { + fn platform(&self) -> Platform { + Platform::Espressif32 + } + + fn build(&self, params: &BuildParams) -> Result { + let start = Instant::now(); + // Env-gated per-phase timer (FBUILD_PERF_LOG=1); zero overhead when unset. + let mut perf = crate::perf_log::PerfTimer::new("esp32-orchestrator"); + + // 0. Discover zccache compiler cache (startup is deferred until compile work begins) + let compiler_cache = { + let _g = perf.phase("zccache-discover"); + crate::zccache::find_zccache().map(std::path::Path::to_path_buf) + }; + + // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags + let mut ctx = crate::pipeline::BuildContext::new_with_perf(params, Some(&mut perf))?; + + // Compute eh_frame strip policy once per build (FastLED/fbuild#243). + // Reads sdkconfig from the project dir (ESP32 only) so panic-backtrace + // / gdbstub users automatically Preserve. + let sdkconfig = + fbuild_config::sdkconfig::SdkConfigSummary::from_project_dir(¶ms.project_dir); + let eh_frame_policy = crate::eh_frame_policy_compute::compute_eh_frame_policy( + &ctx, + params.profile, + Some(&sdkconfig), + ); + + // 3. Load MCU config from embedded JSON + let mut mcu_config = get_mcu_config(&ctx.board.mcu)?; + + tracing::info!( + "ESP32 build: {} ({}, {})", + ctx.board.name, + ctx.board.mcu, + mcu_config.architecture + ); + + // 4-6. Resolve platform, toolchain, and framework + let _resolve_phase = perf.phase("pioarduino-resolve"); + let (toolchain, framework) = + resolve_pioarduino_packages(¶ms.project_dir, &ctx.board.mcu, &mcu_config)?; + drop(_resolve_phase); + let _toolchain_cache_dir = fbuild_packages::Package::get_info(&toolchain).install_path; + let _framework_cache_dir = fbuild_packages::Package::get_info(&framework).install_path; + + // Aliases for build dirs (already set up by BuildContext::new()) + let build_dir = &ctx.build_dir; + let core_build_dir = &ctx.core_build_dir; + let src_build_dir = &ctx.src_build_dir; + + // Read link-affecting config before the expensive include/library/source discovery steps + // so the no-op fast path can return early on warm builds. + let sdk_ld_flags = framework.get_sdk_ld_flags(&ctx.board.mcu); + let sdk_defines = framework.get_sdk_defines(&ctx.board.mcu); + + if sdk_ld_flags.iter().any(|f| f == "-fno-lto") { + mcu_config.disable_lto(); + } + + let mut user_flags = sdk_defines; + let user_build_flags = ctx.config.get_build_flags(¶ms.env_name)?; + user_flags.extend(user_build_flags.clone()); + let embed_files = ctx.config.get_embed_files(¶ms.env_name)?; + let embed_txtfiles = ctx.config.get_embed_txtfiles(¶ms.env_name)?; + + let f_for_image = ctx + .board + .f_image + .as_deref() + .or(ctx.board.f_flash.as_deref()); + let flash_freq = crate::esp32::esp32_linker::f_flash_to_esptool_freq( + f_for_image, + mcu_config.default_flash_freq(), + ); + let flash_mode = ctx + .board + .flash_mode + .clone() + .unwrap_or_else(|| mcu_config.default_flash_mode().to_string()); + let flash_size = crate::esp32::mcu_config::bytes_to_flash_size( + ctx.board.max_flash, + mcu_config.default_flash_size(), + ) + .to_string(); + let metadata_hash = stable_hash_json(&Esp32FingerprintMetadata { + version: BUILD_FINGERPRINT_VERSION, + env_name: params.env_name.clone(), + profile: profile_label(params.profile).to_string(), + board_name: ctx.board.name.clone(), + board_mcu: ctx.board.mcu.clone(), + board_define: ctx.board.board.clone(), + board_core: ctx.board.core.clone(), + board_variant: ctx.board.variant.clone(), + board_variant_h: ctx.board.variant_h.clone(), + board_extra_flags: ctx.board.extra_flags.clone(), + board_upload_protocol: ctx.board.upload_protocol.clone(), + board_upload_speed: ctx.board.upload_speed.clone(), + board_partitions: ctx.board.partitions.clone(), + board_ldscript: ctx.board.ldscript.clone(), + board_platform: ctx.board.platform_str.clone(), + architecture: mcu_config.architecture.clone(), + platform: "espressif32".to_string(), + flash_mode: flash_mode.clone(), + flash_freq: flash_freq.clone(), + flash_size: flash_size.clone(), + max_flash: ctx.board.max_flash, + max_ram: ctx.board.max_ram, + eh_frame_policy: match eh_frame_policy { + crate::eh_frame_policy::EhFramePolicy::Strip => "strip", + crate::eh_frame_policy::EhFramePolicy::Preserve => "preserve", + }, + })?; + let (fast_elf, [fast_bin, fast_boot, fast_parts, fast_app0], fast_compile_db) = + expected_fast_path_artifacts( + build_dir, + ¶ms.project_dir, + [ + "firmware.bin", + "bootloader.bin", + "partitions.bin", + "boot_app0.bin", + ], + ); + let fast_path = { + let _g = perf.phase("fp-watches-collect"); + FastPathContract::for_project_outputs( + build_dir, + ¶ms.project_dir, + [ + fast_elf.clone(), + fast_bin.clone(), + fast_boot.clone(), + fast_parts.clone(), + fast_app0.clone(), + fast_compile_db.clone(), + ], + ) + }; + + if !params.compiledb_only + && !params.symbol_analysis + && params.symbol_analysis_path.is_none() + { + let _fast_path_phase = perf.phase("fast-path-check"); + // ESP32 also requires the project-root copy of compile_commands.json + // to be in sync with the build-dir copy. That's platform-specific, + // so it rides on the shared helper via `extra_artifact_ok`. + let compile_db_fresh = || compile_db_is_current(build_dir, ¶ms.project_dir); + let inputs = FastPathCheckInputs { + metadata_hash: &metadata_hash, + extra_artifact_ok: Some(&compile_db_fresh), + watch_set_cache: params.watch_set_cache.as_deref(), + compiler_cache: compiler_cache.as_deref(), + }; + if let Some(hit) = crate::build_fingerprint::fast_path_check(&fast_path, &inputs)? { + ctx.build_log.push( + "No-op fingerprint matched; reusing existing ESP32 artifacts.".to_string(), + ); + let elapsed = start.elapsed().as_secs_f64(); + return Ok(BuildResult { + success: true, + firmware_path: Some(fast_bin), + elf_path: Some(fast_elf), + size_info: hit.size_info, + symbol_map: None, + build_time_secs: elapsed, + message: format!( + "ESP32 ({}) build for {} reused cached artifacts", + ctx.board.mcu, params.env_name + ), + compile_database_path: Some(fast_compile_db), + build_log: ctx.build_log, + }); + } + } + + if let Some(ref zcc) = compiler_cache { + crate::zccache::ensure_running(zcc); + } + + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + tracing::info!( + "ESP32 {} toolchain at {}", + if mcu_config.is_riscv() { + "RISC-V" + } else { + "Xtensa" + }, + toolchain_dir.display() + ); + + let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + tracing::info!("ESP32 framework at {}", framework_dir.display()); + + let tc_label = if mcu_config.is_riscv() { + "riscv32-esp-elf-gcc" + } else { + "xtensa-esp-elf-gcc" + }; + crate::pipeline::log_toolchain_version( + &toolchain.get_gcc_path(), + tc_label, + &mut ctx.build_log, + ); + + let core_dir = framework.get_core_dir(&ctx.board.core); + let variant_dir = framework.get_variant_dir(&ctx.board.variant); + let sdk_memory_type = ctx + .board + .effective_esp32_memory_type(mcu_config.default_flash_mode()); + + let mut include_dirs = vec![core_dir.clone()]; + if variant_dir.exists() { + include_dirs.push(variant_dir.clone()); + } + // Add SDK include paths (294+ paths from ESP-IDF) + include_dirs + .extend(framework.get_sdk_include_dirs(&ctx.board.mcu, sdk_memory_type.as_deref())); + + // Add built-in Arduino library includes (Wire, SPI, WiFi, etc.) + let builtin_libs_dir = framework.get_libraries_dir(); + if builtin_libs_dir.is_dir() { + if let Ok(entries) = std::fs::read_dir(&builtin_libs_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + let lib_src = path.join("src"); + if lib_src.is_dir() { + include_dirs.push(lib_src); + } + } + } + } + } + + include_dirs.push(ctx.src_dir.clone()); + crate::pipeline::discover_project_includes(¶ms.project_dir, &mut include_dirs); + // Toolchain sysroot includes (xtensa headers, etc.) + include_dirs.extend(toolchain.get_include_dirs()); + + // Read SDK flags early — needed to check LTO before compiling. + let sdk_ld_flags = framework.get_sdk_ld_flags(&ctx.board.mcu); + let sdk_lib_flags = framework.get_sdk_lib_flags(&ctx.board.mcu, sdk_memory_type.as_deref()); + let sdk_ld_scripts = + LinkerScripts::from_raw_flags(&framework.get_sdk_ld_scripts(&ctx.board.mcu)); + let sdk_defines = framework.get_sdk_defines(&ctx.board.mcu); + + // If SDK specifies -fno-lto, disable LTO in MCU config profiles to avoid + // compiling objects with LTO that the linker can't handle. + if sdk_ld_flags.iter().any(|f| f == "-fno-lto") { + mcu_config.disable_lto(); + } + + // 8.5. Library dependencies + let lib_deps = ctx.config.get_lib_deps(¶ms.env_name)?; + let lib_ignore = ctx.config.get_lib_ignore(¶ms.env_name)?; + + use fbuild_packages::Toolchain; + let mut library_archives = Vec::new(); + + // Read user build_flags early — needed for both library and sketch compilation. + // SDK defines (from flags/defines) are prepended so user flags can override them. + let mut user_flags = sdk_defines; + let mut user_build_flags = ctx.config.get_build_flags(¶ms.env_name)?; + user_build_flags.extend(params.extra_build_flags.clone()); + user_flags.extend(user_build_flags.clone()); + let user_overlay = LanguageExtraFlags { + common: user_flags + .iter() + .cloned() + .chain(ctx.global_compile_overlay.common.iter().cloned()) + .collect(), + c: ctx.global_compile_overlay.c.clone(), + cxx: ctx.global_compile_overlay.cxx.clone(), + asm: ctx.global_compile_overlay.asm.clone(), + }; + let src_overlay = LanguageExtraFlags::combined(&[ + &user_overlay, + &LanguageExtraFlags { + common: ctx.src_flags.clone(), + c: Vec::new(), + cxx: Vec::new(), + asm: Vec::new(), + }, + &ctx.project_compile_overlay, + ]); + + // Emit a warning if CDC on boot is effectively enabled (may cause Serial to block + // when no USB host is connected). + warn_if_cdc_on_boot( + &ctx.board.name, + ctx.board.extra_flags.as_deref(), + &user_build_flags, + ); + crate::warn_debug_build_flags(&user_build_flags); + + if !lib_deps.is_empty() { + let libs_dir = build_dir.join("libs"); + + // Build compiler to get flags for library compilation + let mut defines = ctx.board.get_defines(); + defines.extend(mcu_config.defines_map()); + + let temp_compiler = Esp32Compiler::with_temp_dir( + toolchain.get_gcc_path(), + toolchain.get_gxx_path(), + mcu_config.clone(), + &ctx.board.f_cpu, + defines.clone(), + include_dirs.clone(), + params.profile, + params.verbose, + build_dir.join("tmp"), + ) + .with_build_unflags(ctx.build_unflags.clone()) + .with_eh_frame_policy(eh_frame_policy); + // Apply user build_flags to library compilation (matching PlatformIO behavior). + // User flags like -std=gnu++2a replace the MCU config's -std=gnu++2b. + let c_flags = apply_overlay_flags(&temp_compiler.c_flags(), &user_overlay, "dummy.c"); + let cpp_flags = + apply_overlay_flags(&temp_compiler.cpp_flags(), &user_overlay, "dummy.cpp"); + + let jobs = crate::parallel::effective_jobs(params.jobs); + // Use gcc-ar for LTO archives so the linker-plugin index is written. + let dep_ar_path = toolchain.get_ar_path(); + let dep_gcc_ar_path = toolchain.get_gcc_ar_path(); + let dep_lib_ar_path = crate::pipeline::pick_archiver( + &dep_ar_path, + &dep_gcc_ar_path, + &c_flags, + &cpp_flags, + ); + let lib_result = fbuild_packages::library::library_manager::ensure_libraries_sync( + &lib_deps, + &lib_ignore, + &toolchain.get_gcc_path(), + &toolchain.get_gxx_path(), + dep_lib_ar_path, + &c_flags, + &cpp_flags, + &include_dirs, + &libs_dir, + params.verbose, + jobs, + compiler_cache.as_deref(), + )?; + + // Add library include dirs to the main include list + include_dirs.extend(lib_result.include_dirs); + library_archives = lib_result.archives; + + tracing::info!( + "libraries: {} archives, {} new include dirs", + library_archives.len(), + include_dirs.len() + ); + } + + // 8.5b. Project-as-library compilation — shared with sequential pipeline. + // When the project root contains library.json or library.properties (e.g., FastLED), + // the project's own src/ directory is compiled as a library archive so that example + // sketches can link against it. Centralized in pipeline::compile_project_as_library + // so every orchestrator gets this behavior architecturally. + if !params.compiledb_only { + // Build temp compiler to get the actual c_flags/cpp_flags ESP32 uses for + // library compilation. SDK defines + user flags must be applied so the + // archive matches what sketch sources see. + let mut p_defines = ctx.board.get_defines(); + p_defines.extend(mcu_config.defines_map()); + let p_compiler = Esp32Compiler::with_temp_dir( + toolchain.get_gcc_path(), + toolchain.get_gxx_path(), + mcu_config.clone(), + &ctx.board.f_cpu, + p_defines, + include_dirs.clone(), + params.profile, + params.verbose, + build_dir.join("tmp"), + ) + .with_build_unflags(ctx.build_unflags.clone()) + .with_eh_frame_policy(eh_frame_policy); + let p_c_flags = apply_overlay_flags(&p_compiler.c_flags(), &src_overlay, "dummy.c"); + let p_cpp_flags = + apply_overlay_flags(&p_compiler.cpp_flags(), &src_overlay, "dummy.cpp"); + + // Collect lib/* names so the helper can detect collisions with project-as-library. + let mut existing_lib_names = std::collections::HashSet::new(); + let local_lib_dir = params.project_dir.join("lib"); + if local_lib_dir.is_dir() { + if let Ok(entries) = std::fs::read_dir(&local_lib_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + existing_lib_names.insert(name.to_lowercase()); + } + } + } + } + } + + let gcc_path = toolchain.get_gcc_path(); + let gxx_path = toolchain.get_gxx_path(); + let ar_path = toolchain.get_ar_path(); + let gcc_ar_path = toolchain.get_gcc_ar_path(); + // Use gcc-ar for LTO archives so the linker-plugin index is written. + let lib_ar_path = + crate::pipeline::pick_archiver(&ar_path, &gcc_ar_path, &p_c_flags, &p_cpp_flags); + let lib_env = crate::pipeline::LibraryBuildEnv { + gcc_path: &gcc_path, + gxx_path: &gxx_path, + ar_path: lib_ar_path, + c_flags: &p_c_flags, + cpp_flags: &p_cpp_flags, + include_dirs: &include_dirs, + verbose: params.verbose, + jobs: crate::parallel::effective_jobs(params.jobs), + compiler_cache: compiler_cache.as_deref(), + }; + if let Some(archive) = crate::pipeline::compile_project_as_library( + ¶ms.project_dir, + &ctx.src_dir, + build_dir, + &lib_env, + &existing_lib_names, + )? { + library_archives.push(archive); + } + } + + tracing::info!("include paths: {} total", include_dirs.len()); + + // 8.6. Compile framework built-in libraries (WiFi, FS, SPIFFS, Network, etc.) + // The linker's --gc-sections will strip any unused code. + // Skip when only generating compile_commands.json. + if !params.compiledb_only { + compile_framework_builtin_libs( + params, + &mut perf, + &framework, + &toolchain, + &mcu_config, + &ctx.board, + &ctx.build_unflags, + eh_frame_policy, + &include_dirs, + &user_overlay, + build_dir, + compiler_cache.as_deref(), + &mut library_archives, + )?; + } + + // 9. Scan sources + let sources = { + let _g = perf.phase("scan-sources"); + let scanner = SourceScanner::new(&ctx.src_dir, src_build_dir); + let variant_dir_opt = if variant_dir.exists() { + Some(variant_dir.as_path()) + } else { + None + }; + scanner.scan_all_filtered( + Some(&core_dir), + variant_dir_opt, + ctx.source_filter.as_deref(), + )? + }; + + tracing::info!( + "sources: {} sketch, {} core, {} variant", + sources.sketch_sources.len(), + sources.core_sources.len(), + sources.variant_sources.len(), + ); + + // 10-11. Compile + let mut defines = ctx.board.get_defines(); + defines.extend(mcu_config.defines_map()); + + // Defines required by the new framework (3.3.7+). + // Use \" escapes for GCC response file compatibility (see ctx.board.rs). + defines + .entry("ARDUINO_BOARD".to_string()) + .or_insert_with(|| format!("\\\"{}\\\"", ctx.board.name)); + defines + .entry("ARDUINO_VARIANT".to_string()) + .or_insert_with(|| format!("\\\"{}\\\"", ctx.board.variant)); + + let compiler = Esp32Compiler::with_temp_dir( + toolchain.get_gcc_path(), + toolchain.get_gxx_path(), + mcu_config.clone(), + &ctx.board.f_cpu, + defines, + include_dirs.clone(), + params.profile, + params.verbose, + build_dir.join("tmp"), + ) + .with_build_unflags(ctx.build_unflags.clone()) + .with_eh_frame_policy(eh_frame_policy); + let jobs = crate::parallel::effective_jobs(params.jobs); + tracing::info!("parallel compilation: {} jobs", jobs); + + // Build source lists and flags needed for compile_commands.json + let mut all_core_sources: Vec = Vec::new(); + all_core_sources.extend(sources.core_sources.iter().cloned()); + all_core_sources.extend(sources.variant_sources.iter().cloned()); + + // Precompute values needed for compile_commands.json in both paths + let include_flags = compiler.base.build_include_flags(); + let arch = if mcu_config.is_xtensa() { + crate::compile_database::TargetArchitecture::Xtensa + } else { + crate::compile_database::TargetArchitecture::Riscv32 + }; + + // compiledb_only: generate compile_commands.json without compiling + if params.compiledb_only { + let compile_database_path = { + let _g = perf.phase("compile-db"); + crate::pipeline::generate_compile_db( + compiler.gcc_path(), + compiler.gxx_path(), + &compiler.c_flags(), + &compiler.cpp_flags(), + &include_flags, + &user_overlay, + &src_overlay, + &all_core_sources, + &sources.sketch_sources, + core_build_dir, + src_build_dir, + build_dir, + ¶ms.project_dir, + arch, + )? + }; + let elapsed = start.elapsed().as_secs_f64(); + return Ok(BuildResult { + success: true, + firmware_path: None, + elf_path: None, + size_info: None, + symbol_map: None, + build_time_secs: elapsed, + message: format!( + "compile_commands.json generated for {} ({})", + params.env_name, ctx.board.mcu + ), + compile_database_path, + build_log: ctx.build_log, + }); + } + + // Compile core + variant sources in parallel + let build_log_mutex = std::sync::Mutex::new(ctx.build_log); + let core_result = { + let _g = perf.phase("compile-core-variant"); + crate::parallel::compile_sources_parallel( + &compiler, + &all_core_sources, + core_build_dir, + &user_overlay, + jobs, + Some(&build_log_mutex), + )? + }; + + // Compile sketch sources in parallel + let sketch_result = { + let _g = perf.phase("compile-sketch"); + crate::parallel::compile_sources_parallel( + &compiler, + &sources.sketch_sources, + src_build_dir, + &src_overlay, + jobs, + Some(&build_log_mutex), + )? + }; + + // Unwrap build log and flush collected warnings + let mut build_log = build_log_mutex.into_inner().unwrap(); + for w in core_result.warnings.iter().chain(&sketch_result.warnings) { + crate::build_output::collect_warnings(w, &mut build_log); + } + + let core_objects = core_result.objects; + let mut sketch_objects = sketch_result.objects; + + // Compile local libraries from the project's lib/ directory. + // PlatformIO discovers and compiles these automatically. + { + let _g = perf.phase("compile-local-libs"); + compile_local_libraries( + ¶ms.project_dir, + build_dir, + &compiler, + &toolchain, + &include_dirs, + &src_overlay, + jobs, + params.verbose, + compiler_cache.as_deref(), + &mut library_archives, + )?; + } + + // 11.5. Process embedded files (board_build.embed_files + embed_txtfiles) + // + // `.lnk` entries are pre-resolved: each `.lnk` is parsed, its blob is + // fetched (or pulled from the disk cache), and the materialized path + // is substituted in place before objcopy sees it. The `_lnk_leases` + // vector keeps cache leases alive until we leave this scope, so the + // disk-cache GC can't reap a blob mid-build. + if !embed_files.is_empty() || !embed_txtfiles.is_empty() { + let _g = perf.phase("embed-files"); + let objcopy_path = toolchain.get_objcopy_path(); + let embed_objects = stage_embed_files( + &embed_files, + &embed_txtfiles, + ¶ms.project_dir, + build_dir, + &objcopy_path, + &mcu_config, + params.verbose, + )?; + sketch_objects.extend(embed_objects); + } + + // 11.6. Generate compile_commands.json + let compile_database_path = { + let _g = perf.phase("compile-db"); + crate::pipeline::generate_compile_db( + compiler.gcc_path(), + compiler.gxx_path(), + &compiler.c_flags(), + &compiler.cpp_flags(), + &include_flags, + &user_overlay, + &src_overlay, + &all_core_sources, + &sources.sketch_sources, + core_build_dir, + src_build_dir, + build_dir, + ¶ms.project_dir, + arch, + )? + }; + + // 12-13. Link + convert + // Library archives join core_objects in the archives parameter + let mut all_archives: Vec = core_objects; + all_archives.extend(library_archives); + + let linker = Esp32Linker::new( + toolchain.get_gcc_path(), + toolchain.get_ar_path(), + toolchain.get_objcopy_path(), + toolchain.get_size_path(), + mcu_config.clone(), + sdk_ld_flags, + sdk_lib_flags, + sdk_ld_scripts, + params.profile, + Some(flash_mode.clone()), + &flash_freq, + ctx.board.max_flash, + ctx.board.max_ram, + params.verbose, + ); + + let link_result = { + let _g = perf.phase("link-convert-size"); + crate::linker::Linker::link_all( + &linker, + &sketch_objects, + &all_archives, + build_dir, + &crate::linker::LinkExtraArgs { + flags: ctx.overlay_link_flags.clone(), + libs: ctx.overlay_link_libs.clone(), + }, + params.symbol_analysis, + )? + }; + + // 14. Prepare boot artifacts for deployment / emulation + prepare_boot_artifacts( + build_dir, + &framework, + &ctx.board, + &mcu_config, + &flash_freq, + &mut perf, + )?; + + // 15. Size reporting + result assembly + let fingerprint_started = Instant::now(); + perf.checkpoint("fingerprint-save-start"); + let fast_path_ready = fast_path + .required_artifacts() + .iter() + .all(|path| path.exists()) + && compile_db_is_current(build_dir, ¶ms.project_dir); + if fast_path_ready { + crate::build_fingerprint::persist_fast_path_success( + &fast_path, + &FastPathPersistInputs { + metadata_hash: &metadata_hash, + size_info: link_result.size_info.clone(), + watch_set_cache: params.watch_set_cache.as_deref(), + compiler_cache: compiler_cache.as_deref(), + }, + ); + } else { + tracing::warn!( + "skipping ESP32 fast-path persistence because final artifacts are incomplete" + ); + } + perf.record("fingerprint-save", fingerprint_started.elapsed()); + perf.checkpoint("fingerprint-save-finish"); + + crate::pipeline::handle_link_result( + &link_result, + &mut build_log, + params.symbol_analysis_path.as_deref(), + params.verbose, + ); + let elapsed = start.elapsed().as_secs_f64(); + let platform_label = format!("ESP32 ({})", ctx.board.mcu); + Ok(crate::pipeline::assemble_build_result( + link_result, + elapsed, + &platform_label, + ¶ms.env_name, + compile_database_path, + build_log, + )) + } +} diff --git a/crates/fbuild-build/src/esp32/orchestrator/cdc.rs b/crates/fbuild-build/src/esp32/orchestrator/cdc.rs new file mode 100644 index 00000000..c330263c --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/cdc.rs @@ -0,0 +1,74 @@ +//! USB-CDC-on-boot warning logic and small public helpers re-exported from `orchestrator`. + +use std::path::Path; + +use crate::BuildOrchestrator; + +/// Create an ESP32 orchestrator (convenience for get_orchestrator dispatch). +pub fn create() -> Box { + Box::new(super::Esp32Orchestrator) +} + +/// Determine whether ARDUINO_USB_CDC_ON_BOOT is effectively enabled. +/// +/// Combines `board_extra_flags` (a space-separated string from the board JSON) with +/// `user_build_flags` (from platformio.ini `build_flags`). Board flags are applied +/// first; user flags can override them. The **last** definition of +/// `-DARDUINO_USB_CDC_ON_BOOT=N` wins, matching C preprocessor semantics. +/// +/// Returns `true` only if the final effective value is `1`. +pub fn cdc_on_boot_enabled(board_extra_flags: Option<&str>, user_build_flags: &[String]) -> bool { + // Collect all flags in application order: board first, then user. + let board_tokens: Vec = board_extra_flags + .unwrap_or("") + .split_whitespace() + .map(|s| s.to_string()) + .collect(); + + let all_flags: Vec<&str> = board_tokens + .iter() + .map(|s| s.as_str()) + .chain(user_build_flags.iter().map(|s| s.as_str())) + .collect(); + + let mut effective: Option = None; + + for flag in &all_flags { + // Normalise: strip leading whitespace and optional `-D` prefix added by some tools. + let stripped = flag.trim(); + // Match `-DARDUINO_USB_CDC_ON_BOOT=VALUE` or `ARDUINO_USB_CDC_ON_BOOT=VALUE` + let without_d = stripped.strip_prefix("-D").unwrap_or(stripped); + + if let Some(value) = without_d.strip_prefix("ARDUINO_USB_CDC_ON_BOOT=") { + effective = Some(value.trim() == "1"); + } + } + + effective.unwrap_or(false) +} + +/// Emit a `tracing::warn!` if CDC on boot is effectively enabled. +/// +/// `ARDUINO_USB_CDC_ON_BOOT=1` initialises the USB CDC port during boot via native +/// USB (ESP32-S3, C3, C6, S2, …). When no USB host is connected at power-on any +/// call to `Serial.print()` will block indefinitely because the CDC TX buffer has no +/// consumer to drain it. +pub fn warn_if_cdc_on_boot( + board_name: &str, + board_extra_flags: Option<&str>, + user_build_flags: &[String], +) { + if cdc_on_boot_enabled(board_extra_flags, user_build_flags) { + tracing::warn!( + "Board '{}' has ARDUINO_USB_CDC_ON_BOOT=1. \ + If no USB host is connected at power-on, Serial.print() will block \ + indefinitely. Add -DARDUINO_USB_CDC_ON_BOOT=0 to build_flags to suppress this warning.", + board_name + ); + } +} + +/// Check if a project is configured for ESP32 by reading its platformio.ini. +pub fn is_esp32_project(project_dir: &Path, env_name: &str) -> bool { + crate::pipeline::is_platform_project(project_dir, env_name, fbuild_core::Platform::Espressif32) +} diff --git a/crates/fbuild-build/src/esp32/orchestrator/embed.rs b/crates/fbuild-build/src/esp32/orchestrator/embed.rs new file mode 100644 index 00000000..aafacf74 --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/embed.rs @@ -0,0 +1,150 @@ +//! Convert `board_build.embed_files` / `embed_txtfiles` into linkable ELF objects. + +use std::path::Path; + +use fbuild_core::Result; + +/// Process `board_build.embed_files` and `board_build.embed_txtfiles`. +/// +/// Converts data files into linkable ELF objects using `objcopy --input-target binary`. +/// This generates `_binary__start`, `_binary__end`, and `_binary__size` +/// symbols that the firmware can reference. +/// +/// - `embed_files`: embedded as-is (binary) +/// - `embed_txtfiles`: a null-terminated copy is created first, then embedded +#[allow(clippy::too_many_arguments)] +pub(super) fn process_embed_files( + embed_files: &[String], + embed_txtfiles: &[String], + project_dir: &Path, + embed_dir: &Path, + objcopy_path: &Path, + output_target: &str, + binary_arch: &str, + verbose: bool, +) -> Result> { + use fbuild_core::subprocess::run_command; + + let mut objects = Vec::new(); + + // Helper: convert a relative file path to the object file name. + // e.g. "config/timezones.json" → "config_timezones_json.o" + let to_obj_name = |path: &str| -> String { + let sanitized = path.replace(['/', '\\', '.', '-'], "_"); + format!("{}.o", sanitized) + }; + + // Process binary embed files (embed as-is, cwd=project_dir) + for file in embed_files { + let src_path = project_dir.join(file); + if !src_path.exists() { + tracing::warn!("embed_files: {} not found, skipping", src_path.display()); + continue; + } + + let obj_name = to_obj_name(file); + let obj_path = embed_dir.join(&obj_name); + + if obj_path.exists() { + objects.push(obj_path); + continue; + } + + let args = [ + objcopy_path.to_string_lossy().to_string(), + "--input-target".to_string(), + "binary".to_string(), + "--output-target".to_string(), + output_target.to_string(), + "--binary-architecture".to_string(), + binary_arch.to_string(), + "--rename-section".to_string(), + ".data=.rodata.embedded".to_string(), + file.replace('\\', "/"), + obj_path.to_string_lossy().to_string(), + ]; + + if verbose { + tracing::info!("embed: {}", args.join(" ")); + } + + let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + let result = run_command(&args_ref, Some(project_dir), None, None)?; + + if !result.success() { + return Err(fbuild_core::FbuildError::BuildFailed(format!( + "objcopy failed for embed file {}:\n{}", + file, result.stderr + ))); + } + + tracing::info!("embedded binary file: {}", file); + objects.push(obj_path); + } + + // Process text embed files (null-terminated copy, then objcopy from embed_dir) + for file in embed_txtfiles { + let src_path = project_dir.join(file); + if !src_path.exists() { + tracing::warn!("embed_txtfiles: {} not found, skipping", src_path.display()); + continue; + } + + let obj_name = to_obj_name(file); + let obj_path = embed_dir.join(&obj_name); + + if obj_path.exists() { + objects.push(obj_path); + continue; + } + + // Create null-terminated copy in embed_dir preserving relative path + let rel_dest = embed_dir.join(file); + if let Some(parent) = rel_dest.parent() { + std::fs::create_dir_all(parent)?; + } + let mut content = std::fs::read(&src_path)?; + if content.last() != Some(&0) { + content.push(0); + } + std::fs::write(&rel_dest, &content)?; + + let args = [ + objcopy_path.to_string_lossy().to_string(), + "--input-target".to_string(), + "binary".to_string(), + "--output-target".to_string(), + output_target.to_string(), + "--binary-architecture".to_string(), + binary_arch.to_string(), + "--rename-section".to_string(), + ".data=.rodata.embedded".to_string(), + file.replace('\\', "/"), + obj_path.to_string_lossy().to_string(), + ]; + + if verbose { + tracing::info!("embed txt: {}", args.join(" ")); + } + + // Run from embed_dir so objcopy generates symbols from the relative path + let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + let result = run_command(&args_ref, Some(embed_dir), None, None)?; + + if !result.success() { + return Err(fbuild_core::FbuildError::BuildFailed(format!( + "objcopy failed for embed txtfile {}:\n{}", + file, result.stderr + ))); + } + + tracing::info!("embedded text file: {}", file); + objects.push(obj_path); + } + + if !objects.is_empty() { + tracing::info!("processed {} embedded files", objects.len()); + } + + Ok(objects) +} diff --git a/crates/fbuild-build/src/esp32/orchestrator/embed_stage.rs b/crates/fbuild-build/src/esp32/orchestrator/embed_stage.rs new file mode 100644 index 00000000..381fad81 --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/embed_stage.rs @@ -0,0 +1,71 @@ +//! Wrap `process_embed_files` with `.lnk` resolution + objcopy target selection. + +use std::path::{Path, PathBuf}; + +use fbuild_core::Result; + +use super::super::mcu_config::Esp32McuConfig; +use super::embed::process_embed_files; + +/// Resolve `.lnk` entries in `embed_files`/`embed_txtfiles` against the disk +/// cache, then convert each entry into a linkable ELF object. Returns the +/// list of object files to be appended to the sketch link set. +#[allow(clippy::too_many_arguments)] +pub(super) fn stage_embed_files( + embed_files: &[String], + embed_txtfiles: &[String], + project_dir: &Path, + build_dir: &Path, + objcopy_path: &Path, + mcu_config: &Esp32McuConfig, + verbose: bool, +) -> Result> { + let embed_dir = build_dir.join("embed"); + std::fs::create_dir_all(&embed_dir)?; + + let lnk_dir = embed_dir.join("lnk"); + let mut _lnk_leases: Vec = Vec::new(); + let lnk_cache = fbuild_packages::DiskCache::open().ok(); + + let expand = |entries: &[String]| -> Result> { + let mut out = Vec::with_capacity(entries.len()); + for entry in entries { + let p = if Path::new(entry).is_absolute() { + PathBuf::from(entry) + } else { + project_dir.join(entry) + }; + if fbuild_packages::lnk::has_lnk_extension(&p) { + let cache = lnk_cache.as_ref().ok_or_else(|| { + fbuild_core::FbuildError::PackageError( + "disk cache unavailable; cannot resolve .lnk entries".to_string(), + ) + })?; + let m = fbuild_packages::lnk::materialize_lnk_entry(&p, &lnk_dir, cache)?; + out.push(m.target_path.to_string_lossy().into_owned()); + } else { + out.push(entry.clone()); + } + } + Ok(out) + }; + let resolved_embed_files = expand(embed_files)?; + let resolved_embed_txtfiles = expand(embed_txtfiles)?; + + let (output_target, binary_arch) = if mcu_config.is_riscv() { + ("elf32-littleriscv", "riscv") + } else { + ("elf32-xtensa-le", "xtensa") + }; + + process_embed_files( + &resolved_embed_files, + &resolved_embed_txtfiles, + project_dir, + &embed_dir, + objcopy_path, + output_target, + binary_arch, + verbose, + ) +} diff --git a/crates/fbuild-build/src/esp32/orchestrator/fingerprint.rs b/crates/fbuild-build/src/esp32/orchestrator/fingerprint.rs new file mode 100644 index 00000000..dfdf8a8a --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/fingerprint.rs @@ -0,0 +1,30 @@ +//! Esp32 fast-path fingerprint metadata struct (serialised via stable JSON hash). + +use serde::Serialize; + +#[derive(Debug, Serialize)] +pub(super) struct Esp32FingerprintMetadata { + pub version: u32, + pub env_name: String, + pub profile: String, + pub board_name: String, + pub board_mcu: String, + pub board_define: String, + pub board_core: String, + pub board_variant: String, + pub board_variant_h: Option, + pub board_extra_flags: Option, + pub board_upload_protocol: Option, + pub board_upload_speed: Option, + pub board_partitions: Option, + pub board_ldscript: Option, + pub board_platform: Option, + pub architecture: String, + pub platform: String, + pub flash_mode: String, + pub flash_freq: String, + pub flash_size: String, + pub max_flash: Option, + pub max_ram: Option, + pub eh_frame_policy: &'static str, +} diff --git a/crates/fbuild-build/src/esp32/orchestrator/framework_libs.rs b/crates/fbuild-build/src/esp32/orchestrator/framework_libs.rs new file mode 100644 index 00000000..f480ac72 --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/framework_libs.rs @@ -0,0 +1,220 @@ +//! Compile framework built-in libraries (WiFi, FS, SPIFFS, Network, etc.) +//! shipped under `framework/libraries//src/`. Linker `--gc-sections` +//! strips unused code, so we err on the side of compiling everything. + +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use fbuild_core::Result; +use fbuild_packages::Framework; + +use super::super::esp32_compiler::Esp32Compiler; +use super::super::mcu_config::Esp32McuConfig; +use super::helpers::{ + apply_overlay_flags, framework_failure_marker, framework_signature, + record_failed_framework_lib, should_skip_failed_framework_lib, +}; +use crate::flag_overlay::LanguageExtraFlags; +use crate::BuildParams; +use crate::compiler::Compiler as _; + +/// Compile every Arduino built-in library shipped with the ESP32 framework. +/// Library archives are appended to `library_archives`. +#[allow(clippy::too_many_arguments)] +pub(super) fn compile_framework_builtin_libs( + params: &BuildParams, + perf: &mut crate::perf_log::PerfTimer, + framework: &fbuild_packages::library::Esp32Framework, + toolchain: &fbuild_packages::toolchain::Esp32Toolchain, + mcu_config: &Esp32McuConfig, + board: &fbuild_config::BoardConfig, + build_unflags: &[String], + eh_frame_policy: crate::eh_frame_policy::EhFramePolicy, + include_dirs: &[PathBuf], + user_overlay: &LanguageExtraFlags, + build_dir: &Path, + compiler_cache: Option<&Path>, + library_archives: &mut Vec, +) -> Result<()> { + use fbuild_packages::Toolchain; + + let fw_libs_started = Instant::now(); + perf.checkpoint("fw-libs-start"); + let builtin_libs_dir = framework.get_libraries_dir(); + if !builtin_libs_dir.is_dir() { + perf.record("fw-libs", fw_libs_started.elapsed()); + perf.checkpoint("fw-libs-finish"); + return Ok(()); + } + + let fw_libs_build_dir = build_dir.join("fw_libs"); + std::fs::create_dir_all(&fw_libs_build_dir)?; + + // Build set of already-compiled library names + let already_compiled: std::collections::HashSet = library_archives + .iter() + .filter_map(|p| p.file_stem()) + .filter_map(|s| s.to_str()) + .filter_map(|s| s.strip_prefix("lib")) + .map(|s| s.to_string()) + .collect(); + + // Get compiler flags for framework library compilation + let mut fw_defines = board.get_defines(); + fw_defines.extend(mcu_config.defines_map()); + + let fw_compiler = Esp32Compiler::with_temp_dir( + toolchain.get_gcc_path(), + toolchain.get_gxx_path(), + mcu_config.clone(), + &board.f_cpu, + fw_defines, + include_dirs.to_vec(), + params.profile, + params.verbose, + build_dir.join("tmp"), + ) + .with_build_unflags(build_unflags.to_vec()) + .with_eh_frame_policy(eh_frame_policy); + let fw_c_flags = apply_overlay_flags(&fw_compiler.c_flags(), user_overlay, "dummy.c"); + let fw_cpp_flags = apply_overlay_flags(&fw_compiler.cpp_flags(), user_overlay, "dummy.cpp"); + let fw_signature = framework_signature(include_dirs, &fw_c_flags, &fw_cpp_flags); + + let mut fw_lib_count = 0; + let mut fw_lib_seen = 0; + if let Ok(entries) = std::fs::read_dir(&builtin_libs_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if !path.is_dir() { + continue; + } + let lib_name = path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_lowercase(); + if lib_name.starts_with('.') || already_compiled.contains(&lib_name) { + continue; + } + + let lib_src = path.join("src"); + if !lib_src.is_dir() { + continue; + } + + fw_lib_seen += 1; + + // Check if archive already exists + let archive_path = fw_libs_build_dir.join(format!("lib{}.a", lib_name)); + if archive_path.exists() { + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-cache-hit name={} index={}", + lib_name, fw_lib_seen + )); + } + library_archives.push(archive_path); + fw_lib_count += 1; + continue; + } + + // Collect source files + let lib_info = + fbuild_packages::library::library_info::InstalledLibrary::new(&path, &lib_name); + let sources = lib_info.get_source_files(); + if sources.is_empty() { + continue; + } + let failure_marker = framework_failure_marker(&fw_libs_build_dir, &lib_name); + if should_skip_failed_framework_lib(&failure_marker, &fw_signature, &sources)? { + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-skip-failed name={} index={} sources={}", + lib_name, + fw_lib_seen, + sources.len() + )); + } + tracing::debug!( + "skipping previously failed framework library '{}'", + lib_name + ); + continue; + } + + let fw_jobs = crate::parallel::effective_jobs(params.jobs); + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-compile-start name={} index={} sources={} jobs={}", + lib_name, + fw_lib_seen, + sources.len(), + fw_jobs + )); + } + // Use gcc-ar for LTO archives so the linker-plugin index is written. + let fw_ar_path = toolchain.get_ar_path(); + let fw_gcc_ar_path = toolchain.get_gcc_ar_path(); + let fw_lib_ar_path = crate::pipeline::pick_archiver( + &fw_ar_path, + &fw_gcc_ar_path, + &fw_c_flags, + &fw_cpp_flags, + ); + match fbuild_packages::library::library_compiler::compile_library_with_jobs( + &lib_name, + &sources, + include_dirs, + &toolchain.get_gcc_path(), + &toolchain.get_gxx_path(), + fw_lib_ar_path, + &fw_c_flags, + &fw_cpp_flags, + &fw_libs_build_dir, + params.verbose, + fw_jobs, + compiler_cache, + ) { + Ok(Some(archive)) => { + let _ = std::fs::remove_file(&failure_marker); + library_archives.push(archive); + fw_lib_count += 1; + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-compile-finish name={} index={} count={}", + lib_name, fw_lib_seen, fw_lib_count + )); + } + } + Ok(None) => { + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-header-only name={} index={}", + lib_name, fw_lib_seen + )); + } + } + Err(e) => { + // Non-fatal: some framework libs may fail to compile + // (e.g., platform-specific ones). The linker will report + // if any actually-needed symbols are missing. + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-compile-error name={} index={}", + lib_name, fw_lib_seen + )); + } + tracing::debug!("framework library {} failed to compile: {}", lib_name, e); + record_failed_framework_lib(&failure_marker, &fw_signature, &e.to_string()); + } + } + } + } + + if fw_lib_count > 0 { + tracing::info!("compiled {} framework built-in libraries", fw_lib_count); + } + perf.record("fw-libs", fw_libs_started.elapsed()); + perf.checkpoint("fw-libs-finish"); + Ok(()) +} diff --git a/crates/fbuild-build/src/esp32/orchestrator/helpers.rs b/crates/fbuild-build/src/esp32/orchestrator/helpers.rs new file mode 100644 index 00000000..d94dbfb8 --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/helpers.rs @@ -0,0 +1,126 @@ +//! Helper functions for the ESP32 orchestrator: failure markers, fingerprinting, +//! flag merging, and small utilities used across orchestration phases. + +use std::path::{Path, PathBuf}; + +use fbuild_core::Result; + +use crate::flag_overlay::LanguageExtraFlags; + +pub(super) fn framework_failure_marker(build_dir: &Path, lib_name: &str) -> PathBuf { + build_dir.join(format!(".{lib_name}.failed")) +} + +pub(super) fn framework_signature( + include_dirs: &[PathBuf], + c_flags: &[String], + cpp_flags: &[String], +) -> String { + let mut parts = Vec::with_capacity(include_dirs.len() + c_flags.len() + cpp_flags.len() + 2); + parts.push("i".to_string()); + parts.extend( + include_dirs + .iter() + .map(|p| p.to_string_lossy().into_owned()), + ); + parts.push("c".to_string()); + parts.extend(c_flags.iter().cloned()); + parts.push("cxx".to_string()); + parts.extend(cpp_flags.iter().cloned()); + parts.join("\x1f") +} + +pub(super) fn latest_mtime(paths: &[PathBuf]) -> Result> { + let mut latest = None; + for path in paths { + let modified = std::fs::metadata(path)?.modified()?; + latest = Some(match latest { + Some(current) if current > modified => current, + _ => modified, + }); + } + Ok(latest) +} + +pub(super) fn should_skip_failed_framework_lib( + marker_path: &Path, + signature: &str, + sources: &[PathBuf], +) -> Result { + if !marker_path.exists() { + return Ok(false); + } + + let marker_text = std::fs::read_to_string(marker_path)?; + let recorded_signature = marker_text.lines().next().unwrap_or_default(); + if recorded_signature != signature { + return Ok(false); + } + + let Some(latest_source_time) = latest_mtime(sources)? else { + return Ok(false); + }; + let marker_time = std::fs::metadata(marker_path)?.modified()?; + Ok(marker_time >= latest_source_time) +} + +pub(super) fn record_failed_framework_lib(marker_path: &Path, signature: &str, error: &str) { + let _ = std::fs::write(marker_path, format!("{signature}\n{error}\n")); +} + +pub(super) fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { + match profile { + fbuild_core::BuildProfile::Release => "release", + fbuild_core::BuildProfile::Quick => "quick", + } +} + +pub(super) fn compile_db_is_current(build_dir: &Path, project_dir: &Path) -> bool { + let build_copy = build_dir.join("compile_commands.json"); + if !build_copy.exists() { + return false; + } + crate::compile_database::CompileDatabase::expected_output_path(build_dir, project_dir).exists() +} + +/// Apply user build_flags from platformio.ini onto base compiler flags. +/// +/// Matches PlatformIO behavior: user flags are appended to common flags, +/// but `-std=` flags replace the existing standard (not stack). `-D` flags are +/// deduplicated by macro name so later values override earlier defaults without +/// tripping GCC redefinition warnings. +pub(super) fn apply_user_flags(base_flags: &[String], user_flags: &[String]) -> Vec { + let mut result = base_flags.to_vec(); + for flag in user_flags { + if flag.starts_with("-std=") { + // Replace any existing -std= flag + result.retain(|f| !f.starts_with("-std=")); + } else if let Some(define_name) = define_flag_name(flag) { + // Replace any existing -DNAME / -DNAME=value flag with the same macro name. + result.retain(|f| define_flag_name(f) != Some(define_name)); + } + result.push(flag.clone()); + } + result +} + +pub(super) fn apply_overlay_flags( + base_flags: &[String], + overlay: &LanguageExtraFlags, + probe_name: &str, +) -> Vec { + apply_user_flags(base_flags, &overlay.for_source(Path::new(probe_name))) +} + +pub(super) fn define_flag_name(flag: &str) -> Option<&str> { + let define = flag.strip_prefix("-D")?; + let name = define + .split_once('=') + .map_or(define, |(name, _)| name) + .trim(); + if name.is_empty() { + None + } else { + Some(name) + } +} diff --git a/crates/fbuild-build/src/esp32/orchestrator/local_libs.rs b/crates/fbuild-build/src/esp32/orchestrator/local_libs.rs new file mode 100644 index 00000000..c47d2151 --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/local_libs.rs @@ -0,0 +1,103 @@ +//! Compile local libraries from the project's `lib/` directory (PlatformIO convention). + +use std::path::{Path, PathBuf}; + +use fbuild_core::Result; + +use super::super::esp32_compiler::Esp32Compiler; +use super::helpers::apply_overlay_flags; +use crate::flag_overlay::LanguageExtraFlags; +use crate::compiler::Compiler as _; + +/// Walk `project_dir/lib/*` and compile each subdirectory as a library archive. +/// Archives are appended to `library_archives`. +#[allow(clippy::too_many_arguments)] +pub(super) fn compile_local_libraries( + project_dir: &Path, + build_dir: &Path, + compiler: &Esp32Compiler, + toolchain: &fbuild_packages::toolchain::Esp32Toolchain, + include_dirs: &[PathBuf], + src_overlay: &LanguageExtraFlags, + jobs: usize, + verbose: bool, + compiler_cache: Option<&Path>, + library_archives: &mut Vec, +) -> Result<()> { + use fbuild_packages::Toolchain; + + let local_lib_dir = project_dir.join("lib"); + if !local_lib_dir.is_dir() { + return Ok(()); + } + let entries = match std::fs::read_dir(&local_lib_dir) { + Ok(it) => it, + Err(_) => return Ok(()), + }; + + for entry in entries.flatten() { + let lib_path = entry.path(); + if !lib_path.is_dir() { + continue; + } + let lib_name = lib_path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_string(); + + let lib_info = fbuild_packages::library::library_info::InstalledLibrary::new( + &lib_path, &lib_name, + ); + let lib_sources = lib_info.get_source_files(); + if lib_sources.is_empty() { + continue; + } + + let lib_build_dir = build_dir.join("lib").join(&lib_name); + tracing::info!( + "compiling local library '{}': {} source files", + lib_name, + lib_sources.len() + ); + + // Use gcc-ar for LTO archives so the linker-plugin index is written. + let local_ar_path = toolchain.get_ar_path(); + let local_gcc_ar_path = toolchain.get_gcc_ar_path(); + let local_c_flags = apply_overlay_flags(&compiler.c_flags(), src_overlay, "dummy.c"); + let local_cpp_flags = + apply_overlay_flags(&compiler.cpp_flags(), src_overlay, "dummy.cpp"); + let local_lib_ar_path = crate::pipeline::pick_archiver( + &local_ar_path, + &local_gcc_ar_path, + &local_c_flags, + &local_cpp_flags, + ); + match fbuild_packages::library::library_compiler::compile_library_with_jobs( + &lib_name, + &lib_sources, + include_dirs, + &toolchain.get_gcc_path(), + &toolchain.get_gxx_path(), + local_lib_ar_path, + &local_c_flags, + &local_cpp_flags, + &lib_build_dir, + verbose, + jobs, + compiler_cache, + ) { + Ok(Some(archive)) => { + library_archives.push(archive); + } + Ok(None) => {} // header-only + Err(e) => { + return Err(fbuild_core::FbuildError::BuildFailed(format!( + "local library '{}' failed to compile: {}", + lib_name, e + ))); + } + } + } + Ok(()) +} diff --git a/crates/fbuild-build/src/esp32/orchestrator/mod.rs b/crates/fbuild-build/src/esp32/orchestrator/mod.rs new file mode 100644 index 00000000..a69eccb0 --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/mod.rs @@ -0,0 +1,42 @@ +//! ESP32 build orchestrator — wires together config, packages, compiler, linker. +//! +//! Build phases: +//! 1. Parse platformio.ini +//! 2. Load board config (esp32dev/esp32c6/etc.) +//! 3. Load MCU config from embedded JSON +//! 4. Ensure ESP32 platform (pioarduino) +//! 5. Resolve + ensure ESP32 toolchain via metadata +//! 6. Ensure ESP32 framework (Arduino core + ESP-IDF SDK libs) +//! 7. Setup build directories +//! 8. Collect include paths: core + variant + SDK (305+) + user src +//! 9. Download + compile library dependencies +//! 10. Scan sources (sketch + core) +//! 11. Compile core sources +//! 12. Compile sketch sources +//! 13. Link (with linker scripts + SDK libs + library archives) +//! 14. Convert to .bin +//! 15. Copy bootloader.bin + partitions.bin +//! 16. Size reporting +//! +//! The orchestrator is split across sibling files in this directory to keep +//! each one under the 1000-LOC gate. `build.rs` contains the top-level +//! `impl BuildOrchestrator`; everything else is helper modules. + +mod boot_artifacts; +mod build; +mod cdc; +mod embed; +mod embed_stage; +mod fingerprint; +mod framework_libs; +mod helpers; +mod local_libs; +mod packages; + +#[cfg(test)] +mod tests; + +/// ESP32 platform build orchestrator. +pub struct Esp32Orchestrator; + +pub use cdc::{cdc_on_boot_enabled, create, is_esp32_project, warn_if_cdc_on_boot}; diff --git a/crates/fbuild-build/src/esp32/orchestrator/packages.rs b/crates/fbuild-build/src/esp32/orchestrator/packages.rs new file mode 100644 index 00000000..17ec1155 --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/packages.rs @@ -0,0 +1,116 @@ +//! Package resolution for pioarduino (platform.json, framework, toolchain). + +use std::path::Path; + +use fbuild_core::Result; + +/// Resolve framework + toolchain for pioarduino mode (GCC 14 + ESP-IDF 5.x). +/// +/// Downloads pioarduino platform.json, resolves toolchain via metadata, +/// and downloads the split framework + libs packages. +pub(super) fn resolve_pioarduino_packages( + project_dir: &Path, + mcu: &str, + mcu_config: &super::super::mcu_config::Esp32McuConfig, +) -> Result<( + fbuild_packages::toolchain::Esp32Toolchain, + fbuild_packages::library::Esp32Framework, +)> { + // Ensure pioarduino platform (contains platform.json with metadata URLs) + let platform = fbuild_packages::library::Esp32Platform::new(project_dir); + fbuild_packages::Package::ensure_installed(&platform)?; + + // Resolve toolchain via metadata + let toolchain = resolve_and_create_toolchain(&platform, project_dir, mcu_config)?; + + // Resolve framework + let framework = match platform.get_package_url("framework-arduinoespressif32") { + Ok(url) => { + tracing::info!("resolved framework URL from platform.json"); + fbuild_packages::library::Esp32Framework::from_url(project_dir, &url) + } + Err(e) => { + tracing::warn!("could not resolve framework URL, using legacy: {}", e); + fbuild_packages::library::Esp32Framework::new(project_dir, mcu) + } + }; + + // Ensure framework is installed before trying to install libs + let _ = fbuild_packages::Package::ensure_installed(&framework)?; + + // Ensure SDK libs (split package in pioarduino 3.3.7+) + if let Ok(libs_url) = platform.get_package_url("framework-arduinoespressif32-libs") { + framework.ensure_libs(&libs_url)?; + } + + // Ensure MCU-specific skeleton libs (e.g. ESP32-C2, ESP32-C61). + // Some MCUs ship their SDK in a separate skeleton package. + let mcu_suffix = mcu.strip_prefix("esp32").unwrap_or(""); + if !mcu_suffix.is_empty() { + let skeleton_name = format!("framework-arduino-{}-skeleton-lib", mcu_suffix); + if let Ok(skeleton_url) = platform.get_package_url(&skeleton_name) { + framework.ensure_mcu_libs(&skeleton_url, mcu)?; + } + } + + Ok((toolchain, framework)) +} + +fn resolve_and_create_toolchain( + platform: &fbuild_packages::library::Esp32Platform, + project_dir: &Path, + mcu_config: &super::super::mcu_config::Esp32McuConfig, +) -> Result { + let is_riscv = mcu_config.is_riscv(); + let prefix = mcu_config.toolchain_prefix(); + + // Try metadata-based resolution + match platform.get_toolchain_metadata_url(is_riscv) { + Ok(metadata_url) => { + let toolchain_name = if is_riscv { + "toolchain-riscv32-esp" + } else { + "toolchain-xtensa-esp-elf" + }; + + let cache = fbuild_packages::Cache::new(project_dir); + let cache_dir = cache.toolchains_dir().join(toolchain_name); + + match fbuild_packages::toolchain::esp32_metadata::resolve_toolchain_url_sync( + &metadata_url, + toolchain_name, + &cache_dir, + ) { + Ok(resolved) => { + tracing::info!("resolved {} toolchain URL from metadata", toolchain_name); + Ok(fbuild_packages::toolchain::Esp32Toolchain::from_resolved( + project_dir, + &resolved.url, + resolved.sha256.as_deref(), + is_riscv, + &prefix, + )) + } + Err(e) => { + tracing::warn!("metadata resolution failed, using legacy URLs: {}", e); + Ok(fbuild_packages::toolchain::Esp32Toolchain::new( + project_dir, + is_riscv, + &prefix, + )) + } + } + } + Err(e) => { + tracing::warn!( + "could not read platform.json, using legacy toolchain URLs: {}", + e + ); + Ok(fbuild_packages::toolchain::Esp32Toolchain::new( + project_dir, + is_riscv, + &prefix, + )) + } + } +} diff --git a/crates/fbuild-build/src/esp32/orchestrator/tests.rs b/crates/fbuild-build/src/esp32/orchestrator/tests.rs new file mode 100644 index 00000000..2c951f38 --- /dev/null +++ b/crates/fbuild-build/src/esp32/orchestrator/tests.rs @@ -0,0 +1,260 @@ +//! Unit tests for the ESP32 orchestrator's helpers and public API. + +use super::cdc::{cdc_on_boot_enabled, is_esp32_project, warn_if_cdc_on_boot}; +use super::helpers::{ + apply_user_flags, framework_failure_marker, framework_signature, record_failed_framework_lib, + should_skip_failed_framework_lib, +}; +use super::Esp32Orchestrator; +use crate::BuildOrchestrator; +use fbuild_core::Platform; +use std::path::PathBuf; +use std::time::Duration; + +#[test] +fn test_apply_user_flags_replaces_std_flag() { + let base = vec!["-Os".to_string(), "-std=gnu++2b".to_string()]; + let user = vec!["-std=gnu++20".to_string()]; + + let result = apply_user_flags(&base, &user); + + assert_eq!(result, vec!["-Os", "-std=gnu++20"]); +} + +#[test] +fn test_apply_user_flags_replaces_define_with_same_name() { + let base = vec![ + r#"-DIDF_VER=\"v5.5.1-710-g8410210c9a\""#.to_string(), + r#"-DESP_MDNS_VERSION_NUMBER=\"1.9.0\""#.to_string(), + "-Os".to_string(), + ]; + let user = vec![ + r#"-DESP_MDNS_VERSION_NUMBER=\"1.9.1\""#.to_string(), + r#"-DIDF_VER=\"v5.5.2-729-g87912cd291\""#.to_string(), + ]; + + let result = apply_user_flags(&base, &user); + + assert_eq!( + result, + vec![ + "-Os", + r#"-DESP_MDNS_VERSION_NUMBER=\"1.9.1\""#, + r#"-DIDF_VER=\"v5.5.2-729-g87912cd291\""#, + ] + ); +} + +#[test] +fn test_apply_user_flags_replaces_bare_define_with_value_define() { + let base = vec!["-DFOO".to_string(), "-DBAR=1".to_string()]; + let user = vec!["-DFOO=2".to_string()]; + + let result = apply_user_flags(&base, &user); + + assert_eq!(result, vec!["-DBAR=1", "-DFOO=2"]); +} + +#[test] +fn test_esp32_orchestrator_platform() { + let orch = Esp32Orchestrator; + assert_eq!(orch.platform(), Platform::Espressif32); +} + +#[test] +fn test_is_esp32_project() { + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::write( + tmp.path().join("platformio.ini"), + "[env:esp32c6]\nplatform = espressif32\nboard = esp32-c6\nframework = arduino\n", + ) + .unwrap(); + assert!(is_esp32_project(tmp.path(), "esp32c6")); + assert!(!is_esp32_project(tmp.path(), "uno")); +} + +#[test] +fn test_is_not_esp32_project() { + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::write( + tmp.path().join("platformio.ini"), + "[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n", + ) + .unwrap(); + assert!(!is_esp32_project(tmp.path(), "uno")); +} + +// --- CDC on boot warning tests --- + +/// Board that enables CDC on boot via extra_flags (e.g. Adafruit Feather ESP32-S3). +#[test] +fn test_cdc_enabled_by_board_extra_flags() { + let board_flags = Some( + "-DARDUINO_ADAFRUIT_FEATHER_ESP32S3 -DARDUINO_USB_CDC_ON_BOOT=1 -DARDUINO_RUNNING_CORE=1", + ); + assert!(cdc_on_boot_enabled(board_flags, &[])); +} + +/// Board that explicitly disables CDC on boot. +#[test] +fn test_cdc_disabled_by_board_extra_flags() { + let board_flags = Some("-DARDUINO_FREENOVE_ESP32_S3_WROOM -DARDUINO_USB_CDC_ON_BOOT=0"); + assert!(!cdc_on_boot_enabled(board_flags, &[])); +} + +/// Plain ESP32 dev board with no CDC flag at all — not enabled. +#[test] +fn test_no_cdc_flag_returns_false() { + let board_flags = Some("-DARDUINO_ESP32_DEV"); + assert!(!cdc_on_boot_enabled(board_flags, &[])); +} + +/// No board flags at all — not enabled. +#[test] +fn test_no_flags_at_all_returns_false() { + assert!(!cdc_on_boot_enabled(None, &[])); +} + +/// User build_flags override a board-level enable (last definition wins). +#[test] +fn test_user_flag_overrides_board_enable() { + let board_flags = Some("-DARDUINO_USB_CDC_ON_BOOT=1"); + let user_flags = vec!["-DARDUINO_USB_CDC_ON_BOOT=0".to_string()]; + assert!(!cdc_on_boot_enabled(board_flags, &user_flags)); +} + +/// User build_flags can enable CDC that the board left unconfigured. +#[test] +fn test_user_flag_enables_cdc() { + let board_flags = Some("-DARDUINO_ESP32_DEV"); + let user_flags = vec!["-DARDUINO_USB_CDC_ON_BOOT=1".to_string()]; + assert!(cdc_on_boot_enabled(board_flags, &user_flags)); +} + +/// Multiple user flags — last one wins. +#[test] +fn test_last_user_flag_wins() { + let board_flags = Some("-DARDUINO_USB_CDC_ON_BOOT=1"); + let user_flags = vec![ + "-DARDUINO_USB_CDC_ON_BOOT=0".to_string(), + "-DARDUINO_USB_CDC_ON_BOOT=1".to_string(), + ]; + assert!(cdc_on_boot_enabled(board_flags, &user_flags)); +} + +/// Flags provided as whitespace-separated string should be parsed correctly. +#[test] +fn test_multi_flag_string_parsed_correctly() { + // Board flags: the enable flag appears after another flag. + let board_flags = Some("-DSOME_DEFINE -DARDUINO_USB_CDC_ON_BOOT=1 -DANOTHER=1"); + assert!(cdc_on_boot_enabled(board_flags, &[])); +} + +/// `warn_if_cdc_on_boot` should not panic for any combination of inputs. +#[test] +fn test_warn_if_cdc_on_boot_no_panic() { + // CDC enabled — triggers warning path + warn_if_cdc_on_boot( + "Adafruit Feather ESP32-S3", + Some("-DARDUINO_USB_CDC_ON_BOOT=1"), + &[], + ); + // CDC disabled — no warning + warn_if_cdc_on_boot( + "Freenove ESP32-S3-WROOM", + Some("-DARDUINO_USB_CDC_ON_BOOT=0"), + &[], + ); + // No flag at all — no warning + warn_if_cdc_on_boot("ESP32 Dev Module", Some("-DARDUINO_ESP32_DEV"), &[]); + // No board flags — no warning + warn_if_cdc_on_boot("Some Board", None, &[]); + // User override suppresses board enable + warn_if_cdc_on_boot( + "Some Board", + Some("-DARDUINO_USB_CDC_ON_BOOT=1"), + &["-DARDUINO_USB_CDC_ON_BOOT=0".to_string()], + ); +} + +#[test] +fn test_framework_signature_changes_with_flags() { + let includes = vec![PathBuf::from("C:/sdk/include")]; + let sig_a = framework_signature( + &includes, + &["-O2".to_string()], + &["-std=gnu++17".to_string()], + ); + let sig_b = framework_signature( + &includes, + &["-O0".to_string()], + &["-std=gnu++17".to_string()], + ); + assert_ne!(sig_a, sig_b); +} + +#[test] +fn test_apply_user_flags_replaces_existing_define_by_key() { + let merged = apply_user_flags( + &[r#"-DIDF_VER=\"old\""#.to_string(), "-O2".to_string()], + &[r#"-DIDF_VER=\"new\""#.to_string()], + ); + assert_eq!( + merged, + vec![r#"-O2"#.to_string(), r#"-DIDF_VER=\"new\""#.to_string()] + ); +} + +#[test] +fn test_apply_user_flags_keeps_last_user_define() { + let merged = apply_user_flags( + &[], + &[ + r#"-DMBEDTLS_CONFIG_FILE=\"a.h\""#.to_string(), + r#"-DMBEDTLS_CONFIG_FILE=\"b.h\""#.to_string(), + ], + ); + assert_eq!(merged, vec![r#"-DMBEDTLS_CONFIG_FILE=\"b.h\""#.to_string()]); +} + +#[test] +fn test_skip_failed_framework_lib_when_marker_matches_and_is_current() { + let tmp = tempfile::TempDir::new().unwrap(); + let source = tmp.path().join("Matter.cpp"); + std::fs::write(&source, "int x;").unwrap(); + let marker = framework_failure_marker(tmp.path(), "matter"); + let sig = framework_signature(&[], &["-O2".to_string()], &["-std=gnu++2b".to_string()]); + std::thread::sleep(Duration::from_millis(20)); + record_failed_framework_lib(&marker, &sig, "compile failed"); + + assert!(should_skip_failed_framework_lib(&marker, &sig, &[source]).unwrap()); +} + +#[test] +fn test_retry_failed_framework_lib_after_source_change() { + let tmp = tempfile::TempDir::new().unwrap(); + let source = tmp.path().join("Matter.cpp"); + std::fs::write(&source, "int x;").unwrap(); + let marker = framework_failure_marker(tmp.path(), "matter"); + let sig = framework_signature(&[], &["-O2".to_string()], &["-std=gnu++2b".to_string()]); + std::thread::sleep(Duration::from_millis(20)); + record_failed_framework_lib(&marker, &sig, "compile failed"); + std::thread::sleep(Duration::from_millis(20)); + std::fs::write(&source, "int y;").unwrap(); + + assert!(!should_skip_failed_framework_lib(&marker, &sig, &[source]).unwrap()); +} + +#[test] +fn test_retry_failed_framework_lib_after_signature_change() { + let tmp = tempfile::TempDir::new().unwrap(); + let source = tmp.path().join("Matter.cpp"); + std::fs::write(&source, "int x;").unwrap(); + let marker = framework_failure_marker(tmp.path(), "matter"); + let sig_a = framework_signature(&[], &["-O2".to_string()], &["-std=gnu++2b".to_string()]); + let sig_b = framework_signature(&[], &["-O0".to_string()], &["-std=gnu++2b".to_string()]); + std::thread::sleep(Duration::from_millis(20)); + record_failed_framework_lib(&marker, &sig_a, "compile failed"); + + assert!(!should_skip_failed_framework_lib(&marker, &sig_b, &[source]).unwrap()); +} diff --git a/crates/fbuild-build/src/pipeline.rs b/crates/fbuild-build/src/pipeline.rs deleted file mode 100644 index 5b30a7ea..00000000 --- a/crates/fbuild-build/src/pipeline.rs +++ /dev/null @@ -1,1424 +0,0 @@ -//! Shared build pipeline helpers used by all platform orchestrators. -//! -//! Extracts the duplicated config-parse → board-load → build-dir-setup → compile → link -//! sequence that was copy-pasted across AVR, Teensy, and ESP32 orchestrators. - -use std::path::{Path, PathBuf}; -use std::time::Instant; - -use fbuild_core::{BuildLog, Result}; - -use crate::compile_database::{self, CompileDatabase, TargetArchitecture}; -use crate::compiler::Compiler; -use crate::flag_overlay::LanguageExtraFlags; -use crate::linker::LinkResult; -use crate::source_scanner::SourceCollection; -use crate::{BuildParams, BuildResult}; - -/// Common build state initialized at the start of every platform's `build()` method. -/// -/// Created by [`BuildContext::new()`], which handles config parsing, board loading, -/// build directory setup, source directory resolution, and user flag collection. -pub struct BuildContext { - pub config: fbuild_config::PlatformIOConfig, - pub board: fbuild_config::BoardConfig, - pub build_log: BuildLog, - pub build_dir: PathBuf, - pub core_build_dir: PathBuf, - pub src_build_dir: PathBuf, - pub src_dir: PathBuf, - pub source_filter: Option, - pub user_flags: Vec, - pub src_flags: Vec, - pub all_src_flags: Vec, - pub global_compile_overlay: LanguageExtraFlags, - pub project_compile_overlay: LanguageExtraFlags, - pub overlay_link_flags: Vec, - pub overlay_link_libs: Vec, - /// Tokens from PlatformIO `build_unflags` to strip from the effective - /// compile command. Already applied to `user_flags` / `src_flags` / - /// `overlay_link_flags` by `BuildContext::new`; orchestrators can pass - /// this to their platform compiler (via e.g. `with_build_unflags`) to - /// also filter framework/toolchain-contributed flags. See - /// FastLED/fbuild#37. - pub build_unflags: Vec, -} - -impl BuildContext { - /// Parse platformio.ini, load board config, setup build directories, - /// resolve source directory, and collect user flags. - /// - /// Takes `&BuildParams` so that new fields (e.g. `src_dir`) flow through - /// automatically — orchestrators just pass `params` without listing every field. - pub fn new(params: &BuildParams) -> Result { - Self::new_with_perf(params, None) - } - - /// Variant that records phase timings into an optional `PerfTimer`. - /// - /// Orchestrators that want per-phase visibility (see [`crate::perf_log`]) - /// pass in a shared timer. Callers that don't care get zero overhead by - /// passing `None`. - pub fn new_with_perf( - params: &BuildParams, - mut perf: Option<&mut crate::perf_log::PerfTimer>, - ) -> Result { - let project_dir = ¶ms.project_dir; - let env_name = ¶ms.env_name; - - // 1. Parse platformio.ini, attaching any forwarded `PLATFORMIO_*` env - // var overrides from the CLI caller (the daemon does not inherit - // caller env vars). - let t0 = std::time::Instant::now(); - let ini_path = project_dir.join("platformio.ini"); - let pio_overrides = fbuild_config::PioEnvOverrides::from_map(params.pio_env.clone()); - let config = - fbuild_config::PlatformIOConfig::from_path_with_overrides(&ini_path, pio_overrides)?; - let env_config = config.get_env_config(env_name)?; - let overlay = - crate::script_runtime::resolve_extra_script_overlay(project_dir, env_name, &config)?; - if let Some(p) = perf.as_mut() { - p.record("config-parse", t0.elapsed()); - } - - // 2. Load board config - let t0 = std::time::Instant::now(); - let board_id = env_config.get("board").ok_or_else(|| { - fbuild_core::FbuildError::ConfigError("missing 'board' in environment config".into()) - })?; - let overrides = config.get_board_overrides(env_name)?; - let board = fbuild_config::BoardConfig::from_board_id(board_id, &overrides)?; - if let Some(p) = perf.as_mut() { - p.record("board-load", t0.elapsed()); - } - - // 3. Build log initialization - let mut build_log = if params.no_timestamp { - crate::build_output::create_build_log(params.log_sender.clone()) - } else { - crate::build_output::create_build_log_with_epoch( - params.log_sender.clone(), - std::time::Instant::now(), - ) - }; - crate::build_output::log_build_banner(&mut build_log, env_name); - crate::build_output::log_board_info( - &mut build_log, - &board.name, - &board.mcu, - &board.f_cpu, - board.max_flash, - board.max_ram, - ); - for note in &overlay.notes { - build_log.push(format!("extra_scripts: {}", note)); - } - - // 4. Setup build directories - let t0 = std::time::Instant::now(); - let cache = fbuild_packages::Cache::new(project_dir); - if params.clean { - cache.clean_build(env_name, params.profile)?; - } - cache.ensure_build_directories(env_name, params.profile)?; - - let build_dir = cache.get_build_dir(env_name, params.profile); - let core_build_dir = cache.get_core_build_dir(env_name, params.profile); - let src_build_dir = cache.get_src_build_dir(env_name, params.profile); - if let Some(p) = perf.as_mut() { - p.record("build-dirs", t0.elapsed()); - } - - // 5. Resolve source directory (Arduino IDE convention: fall back to project root) - // Priority: explicit override (from HTTP request) > env var > INI config > "src" - let src_dir = project_dir.join( - params - .src_dir - .as_deref() - .map(|s| s.to_string()) - .or_else(|| config.get_src_dir(env_name).ok().flatten()) - .unwrap_or_else(|| "src".to_string()), - ); - let src_dir = if src_dir.exists() { - src_dir - } else { - project_dir.to_path_buf() - }; - let source_filter = config.get_source_filter(env_name)?; - - // 6. Collect user flags - let t0 = std::time::Instant::now(); - let build_type = config.get_build_type(env_name)?; - let user_flags = config.get_build_flags(env_name)?; - crate::warn_debug_build_flags(&user_flags); - let src_flags = config.get_build_src_flags(env_name)?; - let overlay_link_flags = overlay.link.flags.clone(); - let (user_flags, src_flags, mut overlay_link_flags) = if build_type == "debug" { - let debug_build_flags = config.get_debug_build_flags(env_name)?; - apply_debug_build_type( - user_flags, - src_flags, - overlay_link_flags, - &debug_build_flags, - ) - } else { - (user_flags, src_flags, overlay_link_flags) - }; - let build_unflags = config.get_build_unflags(env_name)?; - let (user_flags, src_flags, all_src_flags) = - apply_build_unflags(user_flags, src_flags, &build_unflags); - remove_unflagged_tokens(&mut overlay_link_flags, &build_unflags); - if let Some(p) = perf.as_mut() { - p.record("flag-collect", t0.elapsed()); - } - - Ok(Self { - config, - board, - build_log, - build_dir, - core_build_dir, - src_build_dir, - src_dir, - source_filter, - user_flags, - src_flags, - all_src_flags, - global_compile_overlay: overlay.global_compile, - project_compile_overlay: overlay.project_compile, - overlay_link_flags, - overlay_link_libs: overlay.link.libs, - build_unflags, - }) - } -} - -fn apply_build_unflags( - mut user_flags: Vec, - mut src_flags: Vec, - build_unflags: &[String], -) -> (Vec, Vec, Vec) { - if build_unflags.is_empty() { - let all_src_flags = user_flags.iter().chain(src_flags.iter()).cloned().collect(); - return (user_flags, src_flags, all_src_flags); - } - - remove_unflagged_tokens(&mut user_flags, build_unflags); - remove_unflagged_tokens(&mut src_flags, build_unflags); - let all_src_flags = user_flags.iter().chain(src_flags.iter()).cloned().collect(); - (user_flags, src_flags, all_src_flags) -} - -fn apply_debug_build_type( - mut user_flags: Vec, - mut src_flags: Vec, - mut link_flags: Vec, - debug_build_flags: &[String], -) -> (Vec, Vec, Vec) { - cleanup_platformio_debug_scope(&mut user_flags); - cleanup_platformio_debug_scope(&mut src_flags); - cleanup_platformio_debug_scope(&mut link_flags); - - let mut compile_debug_flags = vec!["-D__PLATFORMIO_BUILD_DEBUG__".to_string()]; - compile_debug_flags.extend(debug_build_flags.iter().cloned()); - - user_flags.extend(compile_debug_flags.iter().cloned()); - src_flags.extend(compile_debug_flags); - - let link_debug_flags: Vec = debug_build_flags - .iter() - .filter(|flag| is_platformio_debug_link_flag(flag)) - .cloned() - .collect(); - link_flags.extend(link_debug_flags); - - (user_flags, src_flags, link_flags) -} - -/// Remove tokens listed in `build_unflags` from `flags` in place, using -/// PlatformIO-compatible semantics: exact token matches and flag-value -/// pair matches for options that take values (like `-include`, `-D`). -/// Public so platform compilers can apply it to the full effective flag -/// set — framework + toolchain + user — not just the user-facing scopes -/// already handled by `apply_build_unflags` in `BuildContext::new`. -/// See FastLED/fbuild#37. -pub fn remove_unflagged_tokens(flags: &mut Vec, build_unflags: &[String]) { - let mut i = 0; - while i < build_unflags.len() { - let token = &build_unflags[i]; - if flag_takes_value(token) && i + 1 < build_unflags.len() { - remove_flag_value_pair(flags, token, &build_unflags[i + 1]); - i += 2; - } else { - flags.retain(|flag| flag != token); - i += 1; - } - } -} - -fn remove_flag_value_pair(flags: &mut Vec, option: &str, value: &str) { - let mut filtered = Vec::with_capacity(flags.len()); - let mut i = 0; - while i < flags.len() { - let current = &flags[i]; - if current == option && i + 1 < flags.len() && flags[i + 1] == value { - i += 2; - continue; - } - filtered.push(current.clone()); - i += 1; - } - *flags = filtered; -} - -fn cleanup_platformio_debug_scope(flags: &mut Vec) { - flags.retain(|flag| !is_platformio_debug_cleanup_flag(flag)); -} - -fn is_platformio_debug_cleanup_flag(flag: &str) -> bool { - if flag == "-Os" || flag == "-g" { - return true; - } - if flag.len() == 3 { - let bytes = flag.as_bytes(); - if bytes[0] == b'-' && matches!(bytes[2], b'0' | b'1' | b'2' | b'3') { - return matches!(bytes[1], b'O' | b'g'); - } - } - if flag.len() == 6 && flag.starts_with("-ggdb") { - return matches!(flag.as_bytes()[5], b'0' | b'1' | b'2' | b'3'); - } - false -} - -fn is_platformio_debug_link_flag(flag: &str) -> bool { - flag.starts_with("-O") || flag == "-g" || flag.starts_with("-g") -} - -fn flag_takes_value(flag: &str) -> bool { - matches!( - flag, - "-include" - | "-imacros" - | "-isystem" - | "-iquote" - | "-iprefix" - | "-iwithprefix" - | "-iwithprefixbefore" - | "-Xlinker" - | "-Wa" - | "-Wl" - | "-Wp" - | "-L" - | "-T" - ) -} - -/// Add the project's `include/` directory and `lib/` subdirectories to include paths. -/// -/// PlatformIO automatically adds these — replicate that behavior. -pub fn discover_project_includes(project_dir: &Path, include_dirs: &mut Vec) { - // PlatformIO automatically includes the project's include/ directory - let include_dir = project_dir.join("include"); - if include_dir.is_dir() { - include_dirs.push(include_dir); - } - - // PlatformIO automatically discovers libraries placed in the project's lib/ directory. - // Each subdirectory is treated as a library — add its root (and src/ if present). - let local_lib_dir = project_dir.join("lib"); - if local_lib_dir.is_dir() { - if let Ok(entries) = std::fs::read_dir(&local_lib_dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - let lib_src = path.join("src"); - if lib_src.is_dir() { - include_dirs.push(lib_src); - } - // Always add the root too (some libraries have headers at top level) - include_dirs.push(path); - } - } - } - } - - // Project-as-library detection (PlatformIO convention). - // When a project root contains library.json or library.properties, the project - // itself is a library and its src/ directory is automatically added to include - // paths for any sketch built within the project. This allows building example - // sketches against the library being developed (e.g., FastLED examples). - let library_json = project_dir.join("library.json"); - let library_props = project_dir.join("library.properties"); - if library_json.exists() || library_props.exists() { - let project_src = project_dir.join("src"); - if project_src.is_dir() && !include_dirs.contains(&project_src) { - include_dirs.push(project_src); - } - } -} - -/// Returns true if the project is a PlatformIO library (has library.json or library.properties). -pub fn is_project_a_library(project_dir: &Path) -> bool { - project_dir.join("library.json").exists() || project_dir.join("library.properties").exists() -} - -/// Compile a list of sources in parallel with incremental rebuild detection. -/// -/// Thin wrapper over [`crate::parallel::compile_sources_parallel`] that flushes -/// collected warnings into the shared build log Mutex. Used by -/// [`run_sequential_build_with_libs`]; ESP32 calls `compile_sources_parallel` -/// directly because it interleaves multiple compile phases through the same -/// log Mutex. -pub fn compile_sources( - compiler: &dyn Compiler, - sources: &[PathBuf], - build_dir: &Path, - extra_flags: &LanguageExtraFlags, - jobs: usize, - build_log: &std::sync::Mutex, -) -> Result> { - let result = crate::parallel::compile_sources_parallel( - compiler, - sources, - build_dir, - extra_flags, - jobs, - Some(build_log), - )?; - if !result.warnings.is_empty() { - let mut log = build_log.lock().unwrap(); - for w in &result.warnings { - crate::build_output::collect_warnings(w, &mut log); - } - } - Ok(result.objects) -} - -/// Compile all libraries in the project's `lib/` directory. -/// -/// Each library's source files are compiled in parallel via -/// [`crate::parallel::compile_sources_parallel`]. Libraries themselves are -/// processed one after another so the per-lib `jobs` budget isn't oversubscribed. -pub fn compile_local_libraries( - compiler: &dyn Compiler, - project_dir: &Path, - build_dir: &Path, - extra_flags: &LanguageExtraFlags, - jobs: usize, - build_log: &std::sync::Mutex, -) -> Result> { - let mut library_objects = Vec::new(); - let local_lib_dir = project_dir.join("lib"); - if !local_lib_dir.is_dir() { - return Ok(library_objects); - } - let entries = match std::fs::read_dir(&local_lib_dir) { - Ok(e) => e, - Err(_) => return Ok(library_objects), - }; - for entry in entries.flatten() { - let lib_path = entry.path(); - if !lib_path.is_dir() { - continue; - } - let lib_name = lib_path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .to_string(); - - let lib_info = - fbuild_packages::library::library_info::InstalledLibrary::new(&lib_path, &lib_name); - let lib_sources = lib_info.get_source_files(); - if lib_sources.is_empty() { - continue; - } - - let lib_build_dir = build_dir.join("lib").join(&lib_name); - std::fs::create_dir_all(&lib_build_dir)?; - tracing::info!( - "compiling local library '{}': {} source files", - lib_name, - lib_sources.len() - ); - - let result = crate::parallel::compile_sources_parallel( - compiler, - &lib_sources, - &lib_build_dir, - extra_flags, - jobs, - Some(build_log), - ) - .map_err(|e| { - fbuild_core::FbuildError::BuildFailed(format!( - "local library '{}' compilation failed: {}", - lib_name, e - )) - })?; - library_objects.extend(result.objects); - if !result.warnings.is_empty() { - let mut log = build_log.lock().unwrap(); - for w in &result.warnings { - crate::build_output::collect_warnings(w, &mut log); - } - } - } - Ok(library_objects) -} - -/// Generate `compile_commands.json` from core/variant and sketch sources. -#[allow(clippy::too_many_arguments)] -pub fn generate_compile_db( - gcc_path: &Path, - gxx_path: &Path, - c_flags: &[String], - cpp_flags: &[String], - include_flags: &[String], - user_flags: &LanguageExtraFlags, - all_src_flags: &LanguageExtraFlags, - core_sources: &[PathBuf], - sketch_sources: &[PathBuf], - core_build_dir: &Path, - src_build_dir: &Path, - build_dir: &Path, - project_dir: &Path, - arch: TargetArchitecture, -) -> Result> { - let mut compile_db = CompileDatabase::new(); - compile_db.extend(compile_database::generate_entries( - gcc_path, - gxx_path, - c_flags, - cpp_flags, - include_flags, - user_flags, - core_sources, - core_build_dir, - project_dir, - )); - compile_db.extend(compile_database::generate_entries( - gcc_path, - gxx_path, - c_flags, - cpp_flags, - include_flags, - all_src_flags, - sketch_sources, - src_build_dir, - project_dir, - )); - let compile_db = compile_db.translate_for_clang(arch); - if compile_db.has_entries() { - Ok(Some(compile_db.write_and_copy(build_dir, project_dir)?)) - } else { - Ok(None) - } -} - -/// Log size report and artifacts from a link result. -/// -/// When `symbol_analysis_path` is `Some`, the report is written to that path -/// and only a one-liner is logged (unless `verbose` is true, which also streams -/// the full report). When `None`, the report is written to `symbol_analysis.txt` -/// in the build artifacts directory and streamed to the build log. -pub fn handle_link_result( - link_result: &LinkResult, - build_log: &mut BuildLog, - symbol_analysis_path: Option<&Path>, - verbose: bool, -) { - if link_result.hex_path.is_some() { - crate::build_output::log_linking(build_log, "Building firmware.hex"); - } else if link_result.bin_path.is_some() { - crate::build_output::log_linking(build_log, "Building firmware.bin"); - } - - if let Some(ref size) = link_result.size_info { - tracing::info!( - "size: text={} data={} bss={} | flash={}/{} ({:.1}%) ram={}/{} ({:.1}%)", - size.text, - size.data, - size.bss, - size.total_flash, - size.max_flash.unwrap_or(0), - size.flash_percent().unwrap_or(0.0), - size.total_ram, - size.max_ram.unwrap_or(0), - size.ram_percent().unwrap_or(0.0), - ); - crate::build_output::log_size_report(build_log, size); - } - - if let Some(ref symbols) = link_result.symbol_map { - let report = crate::build_output::format_symbol_report(symbols); - - if let Some(path) = symbol_analysis_path { - // User gave an explicit path — write there, log a one-liner - if let Err(e) = std::fs::write(path, &report) { - tracing::warn!("failed to write symbol analysis: {e}"); - } else { - build_log.push(format!("Symbol analysis written to {}", path.display())); - } - // Also stream full report when --verbose - if verbose { - crate::build_output::log_symbol_report(build_log, symbols); - } - } else { - // No path — stream to console and write to artifacts dir - crate::build_output::log_symbol_report(build_log, symbols); - if let Some(ref elf) = link_result.elf_path { - if let Some(build_dir) = elf.parent() { - let txt_path = build_dir.join("symbol_analysis.txt"); - if let Err(e) = std::fs::write(&txt_path, &report) { - tracing::warn!("failed to write symbol_analysis.txt: {e}"); - } else { - build_log.push(format!("Symbol analysis: {}", txt_path.display())); - } - } - } - } - } - - if let Some(ref elf) = link_result.elf_path { - crate::build_output::log_artifact(build_log, elf); - } - let firmware = link_result - .hex_path - .as_ref() - .or(link_result.bin_path.as_ref()); - if let Some(fw) = firmware { - crate::build_output::log_artifact(build_log, fw); - } -} - -/// Assemble the final `BuildResult` from link output and build metadata. -pub fn assemble_build_result( - link_result: LinkResult, - elapsed: f64, - platform_label: &str, - env_name: &str, - compile_database_path: Option, - build_log: BuildLog, -) -> BuildResult { - tracing::info!("build completed in {:.1}s", elapsed); - BuildResult { - success: true, - firmware_path: link_result.bin_path.or(link_result.hex_path), - elf_path: link_result.elf_path, - size_info: link_result.size_info, - symbol_map: link_result.symbol_map, - build_time_secs: elapsed, - message: format!("{} build for {} completed", platform_label, env_name), - compile_database_path, - build_log, - } -} - -/// Run the sequential compile → link → result pipeline used by AVR, Teensy, -/// RP2040, STM32, ESP8266, CH32V, NRF52, SAM, Renesas, and Apollo3. -/// -/// Handles: compiledb_only early return, sequential compilation of -/// core/variant/sketch/libs, compile database generation, linking, and result -/// assembly. -/// -/// ESP32 cannot use this because it uses parallel compilation and has -/// additional hooks (SDK libs, embed files, bootloader prep). It calls -/// [`compile_project_as_library`] directly. -/// -/// When `lib_env` is `Some`, the project's own `src/` is compiled as a library -/// archive (matching PlatformIO's project-as-library convention) and linked -/// with the rest of the build. See [`compile_project_as_library`] and -/// ISSUES.md Issue 1. -#[allow(clippy::too_many_arguments)] -pub fn run_sequential_build_with_libs( - compiler: &dyn Compiler, - linker: &dyn crate::linker::Linker, - mut ctx: BuildContext, - params: &BuildParams, - sources: &SourceCollection, - extra_link_inputs: &[PathBuf], - lib_env: Option<&LibraryBuildEnv<'_>>, - arch: TargetArchitecture, - platform_label: &str, - start: Instant, -) -> Result { - // Env-gated per-phase timer (FBUILD_PERF_LOG=1). Emits summary on drop. - // Zero-overhead when the env var is unset — phase guards become no-ops. - let mut perf = crate::perf_log::PerfTimer::new("pipeline"); - let core_and_variant: Vec = sources - .core_sources - .iter() - .chain(sources.variant_sources.iter()) - .cloned() - .collect(); - let user_overlay = LanguageExtraFlags { - common: ctx - .user_flags - .iter() - .cloned() - .chain(ctx.global_compile_overlay.common.iter().cloned()) - .collect(), - c: ctx.global_compile_overlay.c.clone(), - cxx: ctx.global_compile_overlay.cxx.clone(), - asm: ctx.global_compile_overlay.asm.clone(), - }; - let src_overlay = LanguageExtraFlags::combined(&[ - &user_overlay, - &LanguageExtraFlags { - common: ctx.src_flags.clone(), - c: Vec::new(), - cxx: Vec::new(), - asm: Vec::new(), - }, - &ctx.project_compile_overlay, - ]); - - // compiledb_only: generate compile_commands.json without compiling - if params.compiledb_only { - let compile_database_path = generate_compile_db( - compiler.gcc_path(), - compiler.gxx_path(), - &compiler.c_flags(), - &compiler.cpp_flags(), - &[], - &user_overlay, - &src_overlay, - &core_and_variant, - &sources.sketch_sources, - &ctx.core_build_dir, - &ctx.src_build_dir, - &ctx.build_dir, - ¶ms.project_dir, - arch, - )?; - let elapsed = start.elapsed().as_secs_f64(); - return Ok(BuildResult { - success: true, - firmware_path: None, - elf_path: None, - size_info: None, - symbol_map: None, - build_time_secs: elapsed, - message: format!("compile_commands.json generated for {}", params.env_name), - compile_database_path, - build_log: ctx.build_log, - }); - } - - // Wrap the build log so it can be shared across parallel compile phases. - // Phases still run one after another (compile core → variant → sketch → - // libs → link), but each phase fans out file compilation across `jobs` - // threads via `compile_sources_parallel`. - let jobs = crate::parallel::effective_jobs(params.jobs); - let build_log_mutex = std::sync::Mutex::new(ctx.build_log); - - // Compile core + variant - let mut core_objects = { - let _g = perf.phase("compile-core"); - compile_sources( - compiler, - &sources.core_sources, - &ctx.core_build_dir, - &user_overlay, - jobs, - &build_log_mutex, - )? - }; - let variant_objects = { - let _g = perf.phase("compile-variant"); - compile_sources( - compiler, - &sources.variant_sources, - &ctx.core_build_dir, - &user_overlay, - jobs, - &build_log_mutex, - )? - }; - core_objects.extend(variant_objects); - - // Compile sketch - let sketch_objects = { - let _g = perf.phase("compile-sketch"); - compile_sources( - compiler, - &sources.sketch_sources, - &ctx.src_build_dir, - &src_overlay, - jobs, - &build_log_mutex, - )? - }; - - // Compile local libraries (lib/* — loose objects, LTO-safe; per-lib parallel) - let library_objects = { - let _g = perf.phase("compile-local-libs"); - compile_local_libraries( - compiler, - ¶ms.project_dir, - &ctx.build_dir, - &src_overlay, - jobs, - &build_log_mutex, - )? - }; - - // Unwrap the build log Mutex back into the context for the remaining - // single-threaded phases (link, result assembly). - ctx.build_log = build_log_mutex.into_inner().unwrap(); - - // Project-as-library: compile project root's src/ as an archive when - // building an example sketch from a library project (e.g. FastLED examples). - // Only runs when caller provided a LibraryBuildEnv with toolchain paths. - let project_as_lib_archive: Option = { - let _g = perf.phase("project-as-lib"); - if let Some(env) = lib_env { - // Collect existing lib/* names so the helper can detect collisions. - let mut existing_lib_names = std::collections::HashSet::new(); - let local_lib_dir = params.project_dir.join("lib"); - if local_lib_dir.is_dir() { - if let Ok(entries) = std::fs::read_dir(&local_lib_dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - existing_lib_names.insert(name.to_lowercase()); - } - } - } - } - } - compile_project_as_library( - ¶ms.project_dir, - &ctx.src_dir, - &ctx.build_dir, - env, - &existing_lib_names, - )? - } else { - None - } - }; - - // Generate compile_commands.json - let compile_database_path = { - let _g = perf.phase("compile-db"); - generate_compile_db( - compiler.gcc_path(), - compiler.gxx_path(), - &compiler.c_flags(), - &compiler.cpp_flags(), - &[], - &user_overlay, - &src_overlay, - &core_and_variant, - &sources.sketch_sources, - &ctx.core_build_dir, - &ctx.src_build_dir, - &ctx.build_dir, - ¶ms.project_dir, - arch, - )? - }; - - // Link - crate::build_output::log_linking(&mut ctx.build_log, "Linking firmware.elf"); - core_objects.extend(library_objects); - core_objects.extend(extra_link_inputs.iter().cloned()); - if let Some(archive) = project_as_lib_archive { - // GCC accepts .a in the same positional slot as .o files. - core_objects.push(archive); - } - let link_result = { - let _g = perf.phase("link"); - crate::linker::Linker::link_all( - linker, - &sketch_objects, - &core_objects, - &ctx.build_dir, - &crate::linker::LinkExtraArgs { - flags: ctx.overlay_link_flags.clone(), - libs: ctx.overlay_link_libs.clone(), - }, - params.symbol_analysis, - )? - }; - - // Result - handle_link_result( - &link_result, - &mut ctx.build_log, - params.symbol_analysis_path.as_deref(), - params.verbose, - ); - let elapsed = start.elapsed().as_secs_f64(); - Ok(assemble_build_result( - link_result, - elapsed, - platform_label, - ¶ms.env_name, - compile_database_path, - ctx.build_log, - )) -} - -/// Log the version of a GCC toolchain by running `gcc -dumpversion`. -pub fn log_toolchain_version(gcc_path: &Path, label: &str, build_log: &mut BuildLog) { - if let Ok(ver_out) = fbuild_core::subprocess::run_command( - &[gcc_path.to_string_lossy().as_ref(), "-dumpversion"], - None, - None, - None, - ) { - let version = ver_out.stdout.trim().to_string(); - if !version.is_empty() { - crate::build_output::log_toolchain_version(build_log, label, &version); - } - } -} - -/// Tool paths and flag sets needed to compile and archive a standalone library. -/// -/// Bundles parameters that flow together through library-compilation helpers -/// (replaces several `#[allow(clippy::too_many_arguments)]` sites). -#[derive(Debug, Clone)] -pub struct LibraryBuildEnv<'a> { - pub gcc_path: &'a Path, - pub gxx_path: &'a Path, - /// Archiver path. For LTO-enabled builds, callers should pass the - /// toolchain's `gcc-ar` (`Toolchain::get_gcc_ar_path()`) so the - /// linker-plugin index gets written into the archive. See ISSUES.md - /// Issue 8. - pub ar_path: &'a Path, - pub c_flags: &'a [String], - pub cpp_flags: &'a [String], - pub include_dirs: &'a [PathBuf], - pub verbose: bool, - pub jobs: usize, - pub compiler_cache: Option<&'a Path>, -} - -/// Pick the LTO-aware archiver when any compile flag enables LTO. -/// -/// Plain `ar` doesn't insert the LTO linker-plugin index, so on toolchains -/// where the plugin path isn't auto-discovered, the linker silently drops -/// LTO-only symbols. The `gcc-ar` wrapper writes the index — use it whenever -/// `-flto` (or `-flto=auto`) is in the compile flags. -/// -/// `gcc_ar_path` should come from `Toolchain::get_gcc_ar_path()`, which -/// already falls back to `ar` when `gcc-ar` isn't available on disk. -pub fn pick_archiver<'a>( - ar_path: &'a Path, - gcc_ar_path: &'a Path, - c_flags: &[String], - cpp_flags: &[String], -) -> &'a Path { - let has_lto = c_flags.iter().any(|f| f.starts_with("-flto")) - || cpp_flags.iter().any(|f| f.starts_with("-flto")); - if has_lto { - gcc_ar_path - } else { - ar_path - } -} - -#[cfg(test)] -mod tests { - #[test] - fn test_apply_build_unflags_removes_exact_tokens_from_global_and_src_flags() { - let (user_flags, src_flags, all_src_flags) = super::apply_build_unflags( - vec![ - "-Os".to_string(), - "-DDEBUG".to_string(), - "-Wall".to_string(), - ], - vec!["-DDEBUG".to_string(), "-Winvalid-pch".to_string()], - &["-DDEBUG".to_string(), "-Os".to_string()], - ); - - assert_eq!(user_flags, vec!["-Wall"]); - assert_eq!(src_flags, vec!["-Winvalid-pch"]); - assert_eq!(all_src_flags, vec!["-Wall", "-Winvalid-pch"]); - } - - #[test] - fn test_apply_build_unflags_removes_common_option_value_pair() { - let (user_flags, src_flags, all_src_flags) = super::apply_build_unflags( - vec![ - "-include".to_string(), - "config/common.h".to_string(), - "-Wall".to_string(), - ], - vec![ - "-include".to_string(), - "config/common.h".to_string(), - "-Winvalid-pch".to_string(), - ], - &["-include".to_string(), "config/common.h".to_string()], - ); - - assert_eq!(user_flags, vec!["-Wall"]); - assert_eq!(src_flags, vec!["-Winvalid-pch"]); - assert_eq!(all_src_flags, vec!["-Wall", "-Winvalid-pch"]); - } - - #[test] - fn test_apply_debug_build_type_replaces_opt_flags_and_adds_debug_define() { - let (user_flags, src_flags, link_flags) = super::apply_debug_build_type( - vec!["-Os".to_string(), "-Wall".to_string()], - vec!["-O2".to_string(), "-Winvalid-pch".to_string()], - Vec::new(), - &["-Og".to_string(), "-g2".to_string(), "-ggdb2".to_string()], - ); - - assert_eq!( - user_flags, - vec![ - "-Wall", - "-D__PLATFORMIO_BUILD_DEBUG__", - "-Og", - "-g2", - "-ggdb2" - ] - ); - assert_eq!( - src_flags, - vec![ - "-Winvalid-pch", - "-D__PLATFORMIO_BUILD_DEBUG__", - "-Og", - "-g2", - "-ggdb2" - ] - ); - assert_eq!(link_flags, vec!["-Og", "-g2", "-ggdb2"]); - } - - #[test] - fn test_debug_mode_then_build_unflags_can_remove_debug_flags_again() { - let (user_flags, src_flags, mut link_flags) = super::apply_debug_build_type( - vec!["-Os".to_string(), "-Wall".to_string()], - vec!["-Winvalid-pch".to_string()], - Vec::new(), - &["-Og".to_string(), "-g2".to_string(), "-ggdb2".to_string()], - ); - let (user_flags, src_flags, all_src_flags) = - super::apply_build_unflags(user_flags, src_flags, &["-g2".to_string()]); - super::remove_unflagged_tokens(&mut link_flags, &["-g2".to_string()]); - - assert_eq!( - user_flags, - vec!["-Wall", "-D__PLATFORMIO_BUILD_DEBUG__", "-Og", "-ggdb2"] - ); - assert_eq!( - src_flags, - vec![ - "-Winvalid-pch", - "-D__PLATFORMIO_BUILD_DEBUG__", - "-Og", - "-ggdb2" - ] - ); - assert_eq!( - all_src_flags, - vec![ - "-Wall", - "-D__PLATFORMIO_BUILD_DEBUG__", - "-Og", - "-ggdb2", - "-Winvalid-pch", - "-D__PLATFORMIO_BUILD_DEBUG__", - "-Og", - "-ggdb2" - ] - ); - assert_eq!(link_flags, vec!["-Og", "-ggdb2"]); - } -} - -/// Compile the project's own `src/` as a library archive, when the project -/// root contains `library.json`/`library.properties` and we're building an -/// example sketch (i.e. `src_dir` points elsewhere). -/// -/// Returns `Ok(None)` when not applicable (not a library project, normal -/// build, header-only, no src dir, or name collides with a `lib/` -/// subdirectory). Returns `Ok(Some(archive_path))` when the project-as- -/// library archive was produced. -/// -/// Matches PlatformIO's project-as-library convention; see ISSUES.md Issue 1. -pub fn compile_project_as_library( - project_dir: &Path, - src_dir: &Path, - build_dir: &Path, - env: &LibraryBuildEnv<'_>, - existing_lib_names: &std::collections::HashSet, -) -> Result> { - // Guard 1: must be a library project (library.json or library.properties at root). - if !is_project_a_library(project_dir) { - return Ok(None); - } - - // Guard 2: project must have a src/ dir. - let project_src = project_dir.join("src"); - if !project_src.is_dir() { - return Ok(None); - } - - // Guard 3: must be building an example. If src_dir IS the project's own - // src/, we're doing a normal library self-build and the sketch scanner - // is already compiling these sources — don't double-compile. - // Also guard the BuildContext fallback where src_dir collapses to - // project_dir (would cause the scanner to recursively pick up library - // sources, leading to multiply-defined symbols). - if src_dir == project_src || src_dir == project_dir { - return Ok(None); - } - - // Compute lib name from project dir basename. - let lib_name = project_dir - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("project") - .to_lowercase(); - - // Guard 4: collision with a lib// subdirectory — lib/ wins - // (matches PlatformIO behavior). - if existing_lib_names.contains(&lib_name) { - tracing::warn!( - "project-as-library '{}' collides with lib/{} — skipping project root", - lib_name, - lib_name - ); - return Ok(None); - } - - // Discover sources via the same helper used for installed libraries. - let lib_info = - fbuild_packages::library::library_info::InstalledLibrary::new(project_dir, &lib_name); - let sources = lib_info.get_source_files(); - if sources.is_empty() { - tracing::info!("project-as-library '{}' is header-only", lib_name); - return Ok(None); - } - - tracing::info!( - "compiling project-as-library: {} ({} sources from {})", - lib_name, - sources.len(), - project_src.display() - ); - - let project_libs_dir = build_dir.join("project_lib"); - std::fs::create_dir_all(&project_libs_dir)?; - - match fbuild_packages::library::library_compiler::compile_library_with_jobs( - &lib_name, - &sources, - env.include_dirs, - env.gcc_path, - env.gxx_path, - env.ar_path, - env.c_flags, - env.cpp_flags, - &project_libs_dir, - env.verbose, - env.jobs, - env.compiler_cache, - ) { - Ok(Some(archive)) => { - tracing::info!( - "project-as-library compiled: {} sources -> {}", - sources.len(), - archive.display() - ); - Ok(Some(archive)) - } - Ok(None) => Ok(None), // unreachable when sources is non-empty, but safe - Err(e) => Err(fbuild_core::FbuildError::BuildFailed(format!( - "project-as-library '{}' compilation failed: {}", - lib_name, e - ))), - } -} - -/// Check if a project is configured for a specific platform by reading its platformio.ini. -pub fn is_platform_project( - project_dir: &Path, - env_name: &str, - platform: fbuild_core::Platform, -) -> bool { - let ini_path = project_dir.join("platformio.ini"); - if let Ok(config) = fbuild_config::PlatformIOConfig::from_path(&ini_path) { - if let Ok(env) = config.get_env_config(env_name) { - if let Some(platform_str) = env.get("platform") { - return platform.matches_str(platform_str); - } - } - } - false -} - -#[cfg(test)] -mod pick_archiver_tests { - use super::*; - - #[test] - fn test_picks_plain_ar_without_lto() { - let ar = Path::new("/tc/bin/avr-ar"); - let gcc_ar = Path::new("/tc/bin/avr-gcc-ar"); - let c_flags = vec!["-Os".to_string()]; - let cpp_flags = vec!["-std=gnu++17".to_string()]; - assert_eq!(pick_archiver(ar, gcc_ar, &c_flags, &cpp_flags), ar); - } - - #[test] - fn test_picks_gcc_ar_when_c_flags_have_lto() { - let ar = Path::new("/tc/bin/avr-ar"); - let gcc_ar = Path::new("/tc/bin/avr-gcc-ar"); - let c_flags = vec!["-Os".to_string(), "-flto".to_string()]; - let cpp_flags: Vec = vec![]; - assert_eq!(pick_archiver(ar, gcc_ar, &c_flags, &cpp_flags), gcc_ar); - } - - #[test] - fn test_picks_gcc_ar_when_cpp_flags_have_lto_auto() { - let ar = Path::new("/tc/bin/xtensa-esp-elf-ar"); - let gcc_ar = Path::new("/tc/bin/xtensa-esp-elf-gcc-ar"); - let c_flags: Vec = vec![]; - let cpp_flags = vec!["-flto=auto".to_string()]; - assert_eq!(pick_archiver(ar, gcc_ar, &c_flags, &cpp_flags), gcc_ar); - } -} - -#[cfg(test)] -mod project_as_library_tests { - use super::*; - use std::collections::HashSet; - - /// Helper: build a `LibraryBuildEnv` with bogus tool paths. - /// - /// Safe to use whenever the guard logic is expected to short-circuit BEFORE - /// any tool invocation. If the function actually tries to compile, the - /// bogus paths force an error so the test fails loudly. - fn bogus_env<'a>( - include_dirs: &'a [PathBuf], - c_flags: &'a [String], - cpp_flags: &'a [String], - ) -> LibraryBuildEnv<'a> { - // Use empty paths so any subprocess invocation would fail fast. - // We rely on the test's tempdir scope to keep references alive. - LibraryBuildEnv { - gcc_path: Path::new("/__bogus__/gcc"), - gxx_path: Path::new("/__bogus__/g++"), - ar_path: Path::new("/__bogus__/ar"), - c_flags, - cpp_flags, - include_dirs, - verbose: false, - jobs: 1, - compiler_cache: None, - } - } - - #[test] - fn test_returns_none_when_not_a_library() { - let tmp = tempfile::TempDir::new().unwrap(); - let project_dir = tmp.path(); - // No library.json or library.properties - std::fs::create_dir_all(project_dir.join("src")).unwrap(); - std::fs::write(project_dir.join("src").join("lib.cpp"), "").unwrap(); - - let src_dir = project_dir.join("examples").join("Demo"); - let include_dirs: Vec = vec![]; - let c_flags: Vec = vec![]; - let cpp_flags: Vec = vec![]; - let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); - - let result = compile_project_as_library( - project_dir, - &src_dir, - &project_dir.join("build"), - &env, - &HashSet::new(), - ); - assert!(matches!(result, Ok(None))); - } - - #[test] - fn test_returns_none_when_src_dir_equals_project_src() { - // Library project being built normally (not as an example) — must - // NOT compile project-as-library or we'd double-compile sketch sources. - let tmp = tempfile::TempDir::new().unwrap(); - let project_dir = tmp.path(); - std::fs::write(project_dir.join("library.json"), r#"{"name": "test"}"#).unwrap(); - let project_src = project_dir.join("src"); - std::fs::create_dir_all(&project_src).unwrap(); - std::fs::write(project_src.join("lib.cpp"), "").unwrap(); - - let include_dirs: Vec = vec![]; - let c_flags: Vec = vec![]; - let cpp_flags: Vec = vec![]; - let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); - - let result = compile_project_as_library( - project_dir, - &project_src, - &project_dir.join("build"), - &env, - &HashSet::new(), - ); - assert!(matches!(result, Ok(None))); - } - - #[test] - fn test_returns_none_when_src_dir_equals_project_dir() { - // BuildContext::new falls back to project_dir when the resolved src - // dir doesn't exist. In that fallback, the sketch scanner walks - // project_dir recursively and would pick up library sources — so we - // must skip project-as-library to avoid multiply-defined symbols. - let tmp = tempfile::TempDir::new().unwrap(); - let project_dir = tmp.path(); - std::fs::write(project_dir.join("library.json"), r#"{"name": "test"}"#).unwrap(); - std::fs::create_dir_all(project_dir.join("src")).unwrap(); - std::fs::write(project_dir.join("src").join("lib.cpp"), "").unwrap(); - - let include_dirs: Vec = vec![]; - let c_flags: Vec = vec![]; - let cpp_flags: Vec = vec![]; - let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); - - let result = compile_project_as_library( - project_dir, - project_dir, // src_dir == project_dir (fallback case) - &project_dir.join("build"), - &env, - &HashSet::new(), - ); - assert!(matches!(result, Ok(None))); - } - - #[test] - fn test_returns_none_when_no_src_dir() { - // library.properties exists but no src/ directory. - let tmp = tempfile::TempDir::new().unwrap(); - let project_dir = tmp.path(); - std::fs::write(project_dir.join("library.properties"), "name=Test\n").unwrap(); - - let src_dir = project_dir.join("examples").join("Demo"); - let include_dirs: Vec = vec![]; - let c_flags: Vec = vec![]; - let cpp_flags: Vec = vec![]; - let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); - - let result = compile_project_as_library( - project_dir, - &src_dir, - &project_dir.join("build"), - &env, - &HashSet::new(), - ); - assert!(matches!(result, Ok(None))); - } - - #[test] - fn test_returns_none_when_header_only() { - // library.json + src/ but only headers — header-only library, not - // an error, just nothing to compile. - let tmp = tempfile::TempDir::new().unwrap(); - let project_dir = tmp.path(); - std::fs::write(project_dir.join("library.json"), r#"{"name": "test"}"#).unwrap(); - let project_src = project_dir.join("src"); - std::fs::create_dir_all(&project_src).unwrap(); - std::fs::write(project_src.join("lib.h"), "").unwrap(); - std::fs::write(project_src.join("inline.hpp"), "").unwrap(); - - let src_dir = project_dir.join("examples").join("Demo"); - let include_dirs: Vec = vec![]; - let c_flags: Vec = vec![]; - let cpp_flags: Vec = vec![]; - let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); - - let result = compile_project_as_library( - project_dir, - &src_dir, - &project_dir.join("build"), - &env, - &HashSet::new(), - ); - assert!(matches!(result, Ok(None))); - } - - #[test] - fn test_returns_none_on_collision_with_lib_dir() { - // If a user has both library.json AND lib//, the lib/ - // version wins (matches PlatformIO behavior). Must skip project-as- - // library to prevent two libfastled.a archives at link time. - let tmp = tempfile::TempDir::new().unwrap(); - // Create a project dir with a known basename to control lib_name. - let project_dir = tmp.path().join("FastLED"); - std::fs::create_dir_all(&project_dir).unwrap(); - std::fs::write(project_dir.join("library.json"), r#"{"name": "FastLED"}"#).unwrap(); - let project_src = project_dir.join("src"); - std::fs::create_dir_all(&project_src).unwrap(); - std::fs::write(project_src.join("FastLED.cpp"), "").unwrap(); - - let src_dir = project_dir.join("examples").join("Blink"); - - let include_dirs: Vec = vec![]; - let c_flags: Vec = vec![]; - let cpp_flags: Vec = vec![]; - let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); - - let mut existing = HashSet::new(); - existing.insert("fastled".to_string()); // lowercased project basename - - let result = compile_project_as_library( - &project_dir, - &src_dir, - &project_dir.join("build"), - &env, - &existing, - ); - assert!(matches!(result, Ok(None))); - } - - #[test] - fn test_attempts_compile_when_building_example() { - // The positive case: library project + sketch lives elsewhere + has - // sources + no name collision → must reach the compile path. We - // verify this by passing a bogus gcc path and asserting the function - // ERRORS (not Ok(None)). An Ok(None) here would mean a guard - // incorrectly skipped the compile. - let tmp = tempfile::TempDir::new().unwrap(); - let project_dir = tmp.path().join("FastLED"); - std::fs::create_dir_all(&project_dir).unwrap(); - std::fs::write(project_dir.join("library.json"), r#"{"name": "FastLED"}"#).unwrap(); - let project_src = project_dir.join("src"); - std::fs::create_dir_all(&project_src).unwrap(); - std::fs::write(project_src.join("FastLED.cpp"), "// stub").unwrap(); - - let src_dir = project_dir.join("examples").join("Blink"); - std::fs::create_dir_all(&src_dir).unwrap(); - - let include_dirs: Vec = vec![]; - let c_flags: Vec = vec![]; - let cpp_flags: Vec = vec![]; - let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); - - let result = compile_project_as_library( - &project_dir, - &src_dir, - &project_dir.join("build"), - &env, - &HashSet::new(), - ); - // Must NOT be Ok(None) — that would mean a guard skipped compile. - // Either Err (bogus tool failed) or Ok(Some(_)) (impossible without - // a real toolchain) is acceptable. - if let Ok(None) = result { - panic!("expected compile to be attempted, but a guard returned Ok(None)"); - } - } -} diff --git a/crates/fbuild-build/src/pipeline/README.md b/crates/fbuild-build/src/pipeline/README.md new file mode 100644 index 00000000..e4649959 --- /dev/null +++ b/crates/fbuild-build/src/pipeline/README.md @@ -0,0 +1,20 @@ +# pipeline + +Shared build pipeline helpers used by all platform orchestrators. + +This module was split out of a single `pipeline.rs` file to satisfy the +per-file LOC gate. The public API surface is unchanged — every item +previously exported from `crate::pipeline` is re-exported here. + +Submodules: + +- `build_unflags` — `apply_build_unflags`, `apply_debug_build_type`, + `remove_unflagged_tokens`, and related flag-cleanup helpers. +- `project_discovery` — include-path discovery and platform/library detection. +- `compile` — source compilation helpers, local library compilation, + `compile_commands.json` generation, toolchain version logging. +- `link` — link result logging and final `BuildResult` assembly. +- `library` — `LibraryBuildEnv`, `pick_archiver`, and project-as-library + compilation. +- `sequential` — the sequential compile -> link -> result pipeline used by + most non-ESP32 platforms. diff --git a/crates/fbuild-build/src/pipeline/build_unflags.rs b/crates/fbuild-build/src/pipeline/build_unflags.rs new file mode 100644 index 00000000..53269939 --- /dev/null +++ b/crates/fbuild-build/src/pipeline/build_unflags.rs @@ -0,0 +1,242 @@ +//! `build_unflags` and debug-mode flag-cleanup helpers. +//! +//! Implements PlatformIO-compatible semantics for removing tokens from the +//! effective compile / link command, and the `build_type = debug` flag +//! transformation. + +pub(super) fn apply_build_unflags( + mut user_flags: Vec, + mut src_flags: Vec, + build_unflags: &[String], +) -> (Vec, Vec, Vec) { + if build_unflags.is_empty() { + let all_src_flags = user_flags.iter().chain(src_flags.iter()).cloned().collect(); + return (user_flags, src_flags, all_src_flags); + } + + remove_unflagged_tokens(&mut user_flags, build_unflags); + remove_unflagged_tokens(&mut src_flags, build_unflags); + let all_src_flags = user_flags.iter().chain(src_flags.iter()).cloned().collect(); + (user_flags, src_flags, all_src_flags) +} + +pub(super) fn apply_debug_build_type( + mut user_flags: Vec, + mut src_flags: Vec, + mut link_flags: Vec, + debug_build_flags: &[String], +) -> (Vec, Vec, Vec) { + cleanup_platformio_debug_scope(&mut user_flags); + cleanup_platformio_debug_scope(&mut src_flags); + cleanup_platformio_debug_scope(&mut link_flags); + + let mut compile_debug_flags = vec!["-D__PLATFORMIO_BUILD_DEBUG__".to_string()]; + compile_debug_flags.extend(debug_build_flags.iter().cloned()); + + user_flags.extend(compile_debug_flags.iter().cloned()); + src_flags.extend(compile_debug_flags); + + let link_debug_flags: Vec = debug_build_flags + .iter() + .filter(|flag| is_platformio_debug_link_flag(flag)) + .cloned() + .collect(); + link_flags.extend(link_debug_flags); + + (user_flags, src_flags, link_flags) +} + +/// Remove tokens listed in `build_unflags` from `flags` in place, using +/// PlatformIO-compatible semantics: exact token matches and flag-value +/// pair matches for options that take values (like `-include`, `-D`). +/// Public so platform compilers can apply it to the full effective flag +/// set — framework + toolchain + user — not just the user-facing scopes +/// already handled by `apply_build_unflags` in `BuildContext::new`. +/// See FastLED/fbuild#37. +pub fn remove_unflagged_tokens(flags: &mut Vec, build_unflags: &[String]) { + let mut i = 0; + while i < build_unflags.len() { + let token = &build_unflags[i]; + if flag_takes_value(token) && i + 1 < build_unflags.len() { + remove_flag_value_pair(flags, token, &build_unflags[i + 1]); + i += 2; + } else { + flags.retain(|flag| flag != token); + i += 1; + } + } +} + +fn remove_flag_value_pair(flags: &mut Vec, option: &str, value: &str) { + let mut filtered = Vec::with_capacity(flags.len()); + let mut i = 0; + while i < flags.len() { + let current = &flags[i]; + if current == option && i + 1 < flags.len() && flags[i + 1] == value { + i += 2; + continue; + } + filtered.push(current.clone()); + i += 1; + } + *flags = filtered; +} + +fn cleanup_platformio_debug_scope(flags: &mut Vec) { + flags.retain(|flag| !is_platformio_debug_cleanup_flag(flag)); +} + +fn is_platformio_debug_cleanup_flag(flag: &str) -> bool { + if flag == "-Os" || flag == "-g" { + return true; + } + if flag.len() == 3 { + let bytes = flag.as_bytes(); + if bytes[0] == b'-' && matches!(bytes[2], b'0' | b'1' | b'2' | b'3') { + return matches!(bytes[1], b'O' | b'g'); + } + } + if flag.len() == 6 && flag.starts_with("-ggdb") { + return matches!(flag.as_bytes()[5], b'0' | b'1' | b'2' | b'3'); + } + false +} + +fn is_platformio_debug_link_flag(flag: &str) -> bool { + flag.starts_with("-O") || flag == "-g" || flag.starts_with("-g") +} + +fn flag_takes_value(flag: &str) -> bool { + matches!( + flag, + "-include" + | "-imacros" + | "-isystem" + | "-iquote" + | "-iprefix" + | "-iwithprefix" + | "-iwithprefixbefore" + | "-Xlinker" + | "-Wa" + | "-Wl" + | "-Wp" + | "-L" + | "-T" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_apply_build_unflags_removes_exact_tokens_from_global_and_src_flags() { + let (user_flags, src_flags, all_src_flags) = apply_build_unflags( + vec![ + "-Os".to_string(), + "-DDEBUG".to_string(), + "-Wall".to_string(), + ], + vec!["-DDEBUG".to_string(), "-Winvalid-pch".to_string()], + &["-DDEBUG".to_string(), "-Os".to_string()], + ); + + assert_eq!(user_flags, vec!["-Wall"]); + assert_eq!(src_flags, vec!["-Winvalid-pch"]); + assert_eq!(all_src_flags, vec!["-Wall", "-Winvalid-pch"]); + } + + #[test] + fn test_apply_build_unflags_removes_common_option_value_pair() { + let (user_flags, src_flags, all_src_flags) = apply_build_unflags( + vec![ + "-include".to_string(), + "config/common.h".to_string(), + "-Wall".to_string(), + ], + vec![ + "-include".to_string(), + "config/common.h".to_string(), + "-Winvalid-pch".to_string(), + ], + &["-include".to_string(), "config/common.h".to_string()], + ); + + assert_eq!(user_flags, vec!["-Wall"]); + assert_eq!(src_flags, vec!["-Winvalid-pch"]); + assert_eq!(all_src_flags, vec!["-Wall", "-Winvalid-pch"]); + } + + #[test] + fn test_apply_debug_build_type_replaces_opt_flags_and_adds_debug_define() { + let (user_flags, src_flags, link_flags) = apply_debug_build_type( + vec!["-Os".to_string(), "-Wall".to_string()], + vec!["-O2".to_string(), "-Winvalid-pch".to_string()], + Vec::new(), + &["-Og".to_string(), "-g2".to_string(), "-ggdb2".to_string()], + ); + + assert_eq!( + user_flags, + vec![ + "-Wall", + "-D__PLATFORMIO_BUILD_DEBUG__", + "-Og", + "-g2", + "-ggdb2" + ] + ); + assert_eq!( + src_flags, + vec![ + "-Winvalid-pch", + "-D__PLATFORMIO_BUILD_DEBUG__", + "-Og", + "-g2", + "-ggdb2" + ] + ); + assert_eq!(link_flags, vec!["-Og", "-g2", "-ggdb2"]); + } + + #[test] + fn test_debug_mode_then_build_unflags_can_remove_debug_flags_again() { + let (user_flags, src_flags, mut link_flags) = apply_debug_build_type( + vec!["-Os".to_string(), "-Wall".to_string()], + vec!["-Winvalid-pch".to_string()], + Vec::new(), + &["-Og".to_string(), "-g2".to_string(), "-ggdb2".to_string()], + ); + let (user_flags, src_flags, all_src_flags) = + apply_build_unflags(user_flags, src_flags, &["-g2".to_string()]); + remove_unflagged_tokens(&mut link_flags, &["-g2".to_string()]); + + assert_eq!( + user_flags, + vec!["-Wall", "-D__PLATFORMIO_BUILD_DEBUG__", "-Og", "-ggdb2"] + ); + assert_eq!( + src_flags, + vec![ + "-Winvalid-pch", + "-D__PLATFORMIO_BUILD_DEBUG__", + "-Og", + "-ggdb2" + ] + ); + assert_eq!( + all_src_flags, + vec![ + "-Wall", + "-D__PLATFORMIO_BUILD_DEBUG__", + "-Og", + "-ggdb2", + "-Winvalid-pch", + "-D__PLATFORMIO_BUILD_DEBUG__", + "-Og", + "-ggdb2" + ] + ); + assert_eq!(link_flags, vec!["-Og", "-ggdb2"]); + } +} diff --git a/crates/fbuild-build/src/pipeline/compile.rs b/crates/fbuild-build/src/pipeline/compile.rs new file mode 100644 index 00000000..18ba7efb --- /dev/null +++ b/crates/fbuild-build/src/pipeline/compile.rs @@ -0,0 +1,178 @@ +//! Source compilation helpers and `compile_commands.json` generation. + +use std::path::{Path, PathBuf}; + +use fbuild_core::{BuildLog, Result}; + +use crate::compile_database::{self, CompileDatabase, TargetArchitecture}; +use crate::compiler::Compiler; +use crate::flag_overlay::LanguageExtraFlags; + +/// Compile a list of sources in parallel with incremental rebuild detection. +/// +/// Thin wrapper over [`crate::parallel::compile_sources_parallel`] that flushes +/// collected warnings into the shared build log Mutex. Used by +/// [`super::run_sequential_build_with_libs`]; ESP32 calls `compile_sources_parallel` +/// directly because it interleaves multiple compile phases through the same +/// log Mutex. +pub fn compile_sources( + compiler: &dyn Compiler, + sources: &[PathBuf], + build_dir: &Path, + extra_flags: &LanguageExtraFlags, + jobs: usize, + build_log: &std::sync::Mutex, +) -> Result> { + let result = crate::parallel::compile_sources_parallel( + compiler, + sources, + build_dir, + extra_flags, + jobs, + Some(build_log), + )?; + if !result.warnings.is_empty() { + let mut log = build_log.lock().unwrap(); + for w in &result.warnings { + crate::build_output::collect_warnings(w, &mut log); + } + } + Ok(result.objects) +} + +/// Compile all libraries in the project's `lib/` directory. +/// +/// Each library's source files are compiled in parallel via +/// [`crate::parallel::compile_sources_parallel`]. Libraries themselves are +/// processed one after another so the per-lib `jobs` budget isn't oversubscribed. +pub fn compile_local_libraries( + compiler: &dyn Compiler, + project_dir: &Path, + build_dir: &Path, + extra_flags: &LanguageExtraFlags, + jobs: usize, + build_log: &std::sync::Mutex, +) -> Result> { + let mut library_objects = Vec::new(); + let local_lib_dir = project_dir.join("lib"); + if !local_lib_dir.is_dir() { + return Ok(library_objects); + } + let entries = match std::fs::read_dir(&local_lib_dir) { + Ok(e) => e, + Err(_) => return Ok(library_objects), + }; + for entry in entries.flatten() { + let lib_path = entry.path(); + if !lib_path.is_dir() { + continue; + } + let lib_name = lib_path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_string(); + + let lib_info = + fbuild_packages::library::library_info::InstalledLibrary::new(&lib_path, &lib_name); + let lib_sources = lib_info.get_source_files(); + if lib_sources.is_empty() { + continue; + } + + let lib_build_dir = build_dir.join("lib").join(&lib_name); + std::fs::create_dir_all(&lib_build_dir)?; + tracing::info!( + "compiling local library '{}': {} source files", + lib_name, + lib_sources.len() + ); + + let result = crate::parallel::compile_sources_parallel( + compiler, + &lib_sources, + &lib_build_dir, + extra_flags, + jobs, + Some(build_log), + ) + .map_err(|e| { + fbuild_core::FbuildError::BuildFailed(format!( + "local library '{}' compilation failed: {}", + lib_name, e + )) + })?; + library_objects.extend(result.objects); + if !result.warnings.is_empty() { + let mut log = build_log.lock().unwrap(); + for w in &result.warnings { + crate::build_output::collect_warnings(w, &mut log); + } + } + } + Ok(library_objects) +} + +/// Generate `compile_commands.json` from core/variant and sketch sources. +#[allow(clippy::too_many_arguments)] +pub fn generate_compile_db( + gcc_path: &Path, + gxx_path: &Path, + c_flags: &[String], + cpp_flags: &[String], + include_flags: &[String], + user_flags: &LanguageExtraFlags, + all_src_flags: &LanguageExtraFlags, + core_sources: &[PathBuf], + sketch_sources: &[PathBuf], + core_build_dir: &Path, + src_build_dir: &Path, + build_dir: &Path, + project_dir: &Path, + arch: TargetArchitecture, +) -> Result> { + let mut compile_db = CompileDatabase::new(); + compile_db.extend(compile_database::generate_entries( + gcc_path, + gxx_path, + c_flags, + cpp_flags, + include_flags, + user_flags, + core_sources, + core_build_dir, + project_dir, + )); + compile_db.extend(compile_database::generate_entries( + gcc_path, + gxx_path, + c_flags, + cpp_flags, + include_flags, + all_src_flags, + sketch_sources, + src_build_dir, + project_dir, + )); + let compile_db = compile_db.translate_for_clang(arch); + if compile_db.has_entries() { + Ok(Some(compile_db.write_and_copy(build_dir, project_dir)?)) + } else { + Ok(None) + } +} + +/// Log the version of a GCC toolchain by running `gcc -dumpversion`. +pub fn log_toolchain_version(gcc_path: &Path, label: &str, build_log: &mut BuildLog) { + if let Ok(ver_out) = fbuild_core::subprocess::run_command( + &[gcc_path.to_string_lossy().as_ref(), "-dumpversion"], + None, + None, + None, + ) { + let version = ver_out.stdout.trim().to_string(); + if !version.is_empty() { + crate::build_output::log_toolchain_version(build_log, label, &version); + } + } +} diff --git a/crates/fbuild-build/src/pipeline/context.rs b/crates/fbuild-build/src/pipeline/context.rs new file mode 100644 index 00000000..1dbc3e58 --- /dev/null +++ b/crates/fbuild-build/src/pipeline/context.rs @@ -0,0 +1,189 @@ +//! `BuildContext`: common build state initialized at the start of every +//! platform's `build()` method. + +use std::path::PathBuf; + +use fbuild_core::{BuildLog, Result}; + +use crate::flag_overlay::LanguageExtraFlags; +use crate::BuildParams; + +use super::build_unflags::{apply_build_unflags, apply_debug_build_type, remove_unflagged_tokens}; + +/// Common build state initialized at the start of every platform's `build()` method. +/// +/// Created by [`BuildContext::new()`], which handles config parsing, board loading, +/// build directory setup, source directory resolution, and user flag collection. +pub struct BuildContext { + pub config: fbuild_config::PlatformIOConfig, + pub board: fbuild_config::BoardConfig, + pub build_log: BuildLog, + pub build_dir: PathBuf, + pub core_build_dir: PathBuf, + pub src_build_dir: PathBuf, + pub src_dir: PathBuf, + pub source_filter: Option, + pub user_flags: Vec, + pub src_flags: Vec, + pub all_src_flags: Vec, + pub global_compile_overlay: LanguageExtraFlags, + pub project_compile_overlay: LanguageExtraFlags, + pub overlay_link_flags: Vec, + pub overlay_link_libs: Vec, + /// Tokens from PlatformIO `build_unflags` to strip from the effective + /// compile command. Already applied to `user_flags` / `src_flags` / + /// `overlay_link_flags` by `BuildContext::new`; orchestrators can pass + /// this to their platform compiler (via e.g. `with_build_unflags`) to + /// also filter framework/toolchain-contributed flags. See + /// FastLED/fbuild#37. + pub build_unflags: Vec, +} + +impl BuildContext { + /// Parse platformio.ini, load board config, setup build directories, + /// resolve source directory, and collect user flags. + /// + /// Takes `&BuildParams` so that new fields (e.g. `src_dir`) flow through + /// automatically — orchestrators just pass `params` without listing every field. + pub fn new(params: &BuildParams) -> Result { + Self::new_with_perf(params, None) + } + + /// Variant that records phase timings into an optional `PerfTimer`. + /// + /// Orchestrators that want per-phase visibility (see [`crate::perf_log`]) + /// pass in a shared timer. Callers that don't care get zero overhead by + /// passing `None`. + pub fn new_with_perf( + params: &BuildParams, + mut perf: Option<&mut crate::perf_log::PerfTimer>, + ) -> Result { + let project_dir = ¶ms.project_dir; + let env_name = ¶ms.env_name; + + // 1. Parse platformio.ini, attaching any forwarded `PLATFORMIO_*` env + // var overrides from the CLI caller (the daemon does not inherit + // caller env vars). + let t0 = std::time::Instant::now(); + let ini_path = project_dir.join("platformio.ini"); + let pio_overrides = fbuild_config::PioEnvOverrides::from_map(params.pio_env.clone()); + let config = + fbuild_config::PlatformIOConfig::from_path_with_overrides(&ini_path, pio_overrides)?; + let env_config = config.get_env_config(env_name)?; + let overlay = + crate::script_runtime::resolve_extra_script_overlay(project_dir, env_name, &config)?; + if let Some(p) = perf.as_mut() { + p.record("config-parse", t0.elapsed()); + } + + // 2. Load board config + let t0 = std::time::Instant::now(); + let board_id = env_config.get("board").ok_or_else(|| { + fbuild_core::FbuildError::ConfigError("missing 'board' in environment config".into()) + })?; + let overrides = config.get_board_overrides(env_name)?; + let board = fbuild_config::BoardConfig::from_board_id(board_id, &overrides)?; + if let Some(p) = perf.as_mut() { + p.record("board-load", t0.elapsed()); + } + + // 3. Build log initialization + let mut build_log = if params.no_timestamp { + crate::build_output::create_build_log(params.log_sender.clone()) + } else { + crate::build_output::create_build_log_with_epoch( + params.log_sender.clone(), + std::time::Instant::now(), + ) + }; + crate::build_output::log_build_banner(&mut build_log, env_name); + crate::build_output::log_board_info( + &mut build_log, + &board.name, + &board.mcu, + &board.f_cpu, + board.max_flash, + board.max_ram, + ); + for note in &overlay.notes { + build_log.push(format!("extra_scripts: {}", note)); + } + + // 4. Setup build directories + let t0 = std::time::Instant::now(); + let cache = fbuild_packages::Cache::new(project_dir); + if params.clean { + cache.clean_build(env_name, params.profile)?; + } + cache.ensure_build_directories(env_name, params.profile)?; + + let build_dir = cache.get_build_dir(env_name, params.profile); + let core_build_dir = cache.get_core_build_dir(env_name, params.profile); + let src_build_dir = cache.get_src_build_dir(env_name, params.profile); + if let Some(p) = perf.as_mut() { + p.record("build-dirs", t0.elapsed()); + } + + // 5. Resolve source directory (Arduino IDE convention: fall back to project root) + // Priority: explicit override (from HTTP request) > env var > INI config > "src" + let src_dir = project_dir.join( + params + .src_dir + .as_deref() + .map(|s| s.to_string()) + .or_else(|| config.get_src_dir(env_name).ok().flatten()) + .unwrap_or_else(|| "src".to_string()), + ); + let src_dir = if src_dir.exists() { + src_dir + } else { + project_dir.to_path_buf() + }; + let source_filter = config.get_source_filter(env_name)?; + + // 6. Collect user flags + let t0 = std::time::Instant::now(); + let build_type = config.get_build_type(env_name)?; + let user_flags = config.get_build_flags(env_name)?; + crate::warn_debug_build_flags(&user_flags); + let src_flags = config.get_build_src_flags(env_name)?; + let overlay_link_flags = overlay.link.flags.clone(); + let (user_flags, src_flags, mut overlay_link_flags) = if build_type == "debug" { + let debug_build_flags = config.get_debug_build_flags(env_name)?; + apply_debug_build_type( + user_flags, + src_flags, + overlay_link_flags, + &debug_build_flags, + ) + } else { + (user_flags, src_flags, overlay_link_flags) + }; + let build_unflags = config.get_build_unflags(env_name)?; + let (user_flags, src_flags, all_src_flags) = + apply_build_unflags(user_flags, src_flags, &build_unflags); + remove_unflagged_tokens(&mut overlay_link_flags, &build_unflags); + if let Some(p) = perf.as_mut() { + p.record("flag-collect", t0.elapsed()); + } + + Ok(Self { + config, + board, + build_log, + build_dir, + core_build_dir, + src_build_dir, + src_dir, + source_filter, + user_flags, + src_flags, + all_src_flags, + global_compile_overlay: overlay.global_compile, + project_compile_overlay: overlay.project_compile, + overlay_link_flags, + overlay_link_libs: overlay.link.libs, + build_unflags, + }) + } +} diff --git a/crates/fbuild-build/src/pipeline/library.rs b/crates/fbuild-build/src/pipeline/library.rs new file mode 100644 index 00000000..91295743 --- /dev/null +++ b/crates/fbuild-build/src/pipeline/library.rs @@ -0,0 +1,421 @@ +//! Library compilation helpers: `LibraryBuildEnv`, archiver selection, +//! and the "project-as-library" compile path. + +use std::path::{Path, PathBuf}; + +use fbuild_core::Result; + +use super::project_discovery::is_project_a_library; + +/// Tool paths and flag sets needed to compile and archive a standalone library. +/// +/// Bundles parameters that flow together through library-compilation helpers +/// (replaces several `#[allow(clippy::too_many_arguments)]` sites). +#[derive(Debug, Clone)] +pub struct LibraryBuildEnv<'a> { + pub gcc_path: &'a Path, + pub gxx_path: &'a Path, + /// Archiver path. For LTO-enabled builds, callers should pass the + /// toolchain's `gcc-ar` (`Toolchain::get_gcc_ar_path()`) so the + /// linker-plugin index gets written into the archive. See ISSUES.md + /// Issue 8. + pub ar_path: &'a Path, + pub c_flags: &'a [String], + pub cpp_flags: &'a [String], + pub include_dirs: &'a [PathBuf], + pub verbose: bool, + pub jobs: usize, + pub compiler_cache: Option<&'a Path>, +} + +/// Pick the LTO-aware archiver when any compile flag enables LTO. +/// +/// Plain `ar` doesn't insert the LTO linker-plugin index, so on toolchains +/// where the plugin path isn't auto-discovered, the linker silently drops +/// LTO-only symbols. The `gcc-ar` wrapper writes the index — use it whenever +/// `-flto` (or `-flto=auto`) is in the compile flags. +/// +/// `gcc_ar_path` should come from `Toolchain::get_gcc_ar_path()`, which +/// already falls back to `ar` when `gcc-ar` isn't available on disk. +pub fn pick_archiver<'a>( + ar_path: &'a Path, + gcc_ar_path: &'a Path, + c_flags: &[String], + cpp_flags: &[String], +) -> &'a Path { + let has_lto = c_flags.iter().any(|f| f.starts_with("-flto")) + || cpp_flags.iter().any(|f| f.starts_with("-flto")); + if has_lto { + gcc_ar_path + } else { + ar_path + } +} + +/// Compile the project's own `src/` as a library archive, when the project +/// root contains `library.json`/`library.properties` and we're building an +/// example sketch (i.e. `src_dir` points elsewhere). +/// +/// Returns `Ok(None)` when not applicable (not a library project, normal +/// build, header-only, no src dir, or name collides with a `lib/` +/// subdirectory). Returns `Ok(Some(archive_path))` when the project-as- +/// library archive was produced. +/// +/// Matches PlatformIO's project-as-library convention; see ISSUES.md Issue 1. +pub fn compile_project_as_library( + project_dir: &Path, + src_dir: &Path, + build_dir: &Path, + env: &LibraryBuildEnv<'_>, + existing_lib_names: &std::collections::HashSet, +) -> Result> { + // Guard 1: must be a library project (library.json or library.properties at root). + if !is_project_a_library(project_dir) { + return Ok(None); + } + + // Guard 2: project must have a src/ dir. + let project_src = project_dir.join("src"); + if !project_src.is_dir() { + return Ok(None); + } + + // Guard 3: must be building an example. If src_dir IS the project's own + // src/, we're doing a normal library self-build and the sketch scanner + // is already compiling these sources — don't double-compile. + // Also guard the BuildContext fallback where src_dir collapses to + // project_dir (would cause the scanner to recursively pick up library + // sources, leading to multiply-defined symbols). + if src_dir == project_src || src_dir == project_dir { + return Ok(None); + } + + // Compute lib name from project dir basename. + let lib_name = project_dir + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("project") + .to_lowercase(); + + // Guard 4: collision with a lib// subdirectory — lib/ wins + // (matches PlatformIO behavior). + if existing_lib_names.contains(&lib_name) { + tracing::warn!( + "project-as-library '{}' collides with lib/{} — skipping project root", + lib_name, + lib_name + ); + return Ok(None); + } + + // Discover sources via the same helper used for installed libraries. + let lib_info = + fbuild_packages::library::library_info::InstalledLibrary::new(project_dir, &lib_name); + let sources = lib_info.get_source_files(); + if sources.is_empty() { + tracing::info!("project-as-library '{}' is header-only", lib_name); + return Ok(None); + } + + tracing::info!( + "compiling project-as-library: {} ({} sources from {})", + lib_name, + sources.len(), + project_src.display() + ); + + let project_libs_dir = build_dir.join("project_lib"); + std::fs::create_dir_all(&project_libs_dir)?; + + match fbuild_packages::library::library_compiler::compile_library_with_jobs( + &lib_name, + &sources, + env.include_dirs, + env.gcc_path, + env.gxx_path, + env.ar_path, + env.c_flags, + env.cpp_flags, + &project_libs_dir, + env.verbose, + env.jobs, + env.compiler_cache, + ) { + Ok(Some(archive)) => { + tracing::info!( + "project-as-library compiled: {} sources -> {}", + sources.len(), + archive.display() + ); + Ok(Some(archive)) + } + Ok(None) => Ok(None), // unreachable when sources is non-empty, but safe + Err(e) => Err(fbuild_core::FbuildError::BuildFailed(format!( + "project-as-library '{}' compilation failed: {}", + lib_name, e + ))), + } +} + +#[cfg(test)] +mod pick_archiver_tests { + use super::*; + + #[test] + fn test_picks_plain_ar_without_lto() { + let ar = Path::new("/tc/bin/avr-ar"); + let gcc_ar = Path::new("/tc/bin/avr-gcc-ar"); + let c_flags = vec!["-Os".to_string()]; + let cpp_flags = vec!["-std=gnu++17".to_string()]; + assert_eq!(pick_archiver(ar, gcc_ar, &c_flags, &cpp_flags), ar); + } + + #[test] + fn test_picks_gcc_ar_when_c_flags_have_lto() { + let ar = Path::new("/tc/bin/avr-ar"); + let gcc_ar = Path::new("/tc/bin/avr-gcc-ar"); + let c_flags = vec!["-Os".to_string(), "-flto".to_string()]; + let cpp_flags: Vec = vec![]; + assert_eq!(pick_archiver(ar, gcc_ar, &c_flags, &cpp_flags), gcc_ar); + } + + #[test] + fn test_picks_gcc_ar_when_cpp_flags_have_lto_auto() { + let ar = Path::new("/tc/bin/xtensa-esp-elf-ar"); + let gcc_ar = Path::new("/tc/bin/xtensa-esp-elf-gcc-ar"); + let c_flags: Vec = vec![]; + let cpp_flags = vec!["-flto=auto".to_string()]; + assert_eq!(pick_archiver(ar, gcc_ar, &c_flags, &cpp_flags), gcc_ar); + } +} + +#[cfg(test)] +mod project_as_library_tests { + use super::*; + use std::collections::HashSet; + + /// Helper: build a `LibraryBuildEnv` with bogus tool paths. + /// + /// Safe to use whenever the guard logic is expected to short-circuit BEFORE + /// any tool invocation. If the function actually tries to compile, the + /// bogus paths force an error so the test fails loudly. + fn bogus_env<'a>( + include_dirs: &'a [PathBuf], + c_flags: &'a [String], + cpp_flags: &'a [String], + ) -> LibraryBuildEnv<'a> { + // Use empty paths so any subprocess invocation would fail fast. + // We rely on the test's tempdir scope to keep references alive. + LibraryBuildEnv { + gcc_path: Path::new("/__bogus__/gcc"), + gxx_path: Path::new("/__bogus__/g++"), + ar_path: Path::new("/__bogus__/ar"), + c_flags, + cpp_flags, + include_dirs, + verbose: false, + jobs: 1, + compiler_cache: None, + } + } + + #[test] + fn test_returns_none_when_not_a_library() { + let tmp = tempfile::TempDir::new().unwrap(); + let project_dir = tmp.path(); + // No library.json or library.properties + std::fs::create_dir_all(project_dir.join("src")).unwrap(); + std::fs::write(project_dir.join("src").join("lib.cpp"), "").unwrap(); + + let src_dir = project_dir.join("examples").join("Demo"); + let include_dirs: Vec = vec![]; + let c_flags: Vec = vec![]; + let cpp_flags: Vec = vec![]; + let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); + + let result = compile_project_as_library( + project_dir, + &src_dir, + &project_dir.join("build"), + &env, + &HashSet::new(), + ); + assert!(matches!(result, Ok(None))); + } + + #[test] + fn test_returns_none_when_src_dir_equals_project_src() { + // Library project being built normally (not as an example) — must + // NOT compile project-as-library or we'd double-compile sketch sources. + let tmp = tempfile::TempDir::new().unwrap(); + let project_dir = tmp.path(); + std::fs::write(project_dir.join("library.json"), r#"{"name": "test"}"#).unwrap(); + let project_src = project_dir.join("src"); + std::fs::create_dir_all(&project_src).unwrap(); + std::fs::write(project_src.join("lib.cpp"), "").unwrap(); + + let include_dirs: Vec = vec![]; + let c_flags: Vec = vec![]; + let cpp_flags: Vec = vec![]; + let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); + + let result = compile_project_as_library( + project_dir, + &project_src, + &project_dir.join("build"), + &env, + &HashSet::new(), + ); + assert!(matches!(result, Ok(None))); + } + + #[test] + fn test_returns_none_when_src_dir_equals_project_dir() { + // BuildContext::new falls back to project_dir when the resolved src + // dir doesn't exist. In that fallback, the sketch scanner walks + // project_dir recursively and would pick up library sources — so we + // must skip project-as-library to avoid multiply-defined symbols. + let tmp = tempfile::TempDir::new().unwrap(); + let project_dir = tmp.path(); + std::fs::write(project_dir.join("library.json"), r#"{"name": "test"}"#).unwrap(); + std::fs::create_dir_all(project_dir.join("src")).unwrap(); + std::fs::write(project_dir.join("src").join("lib.cpp"), "").unwrap(); + + let include_dirs: Vec = vec![]; + let c_flags: Vec = vec![]; + let cpp_flags: Vec = vec![]; + let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); + + let result = compile_project_as_library( + project_dir, + project_dir, // src_dir == project_dir (fallback case) + &project_dir.join("build"), + &env, + &HashSet::new(), + ); + assert!(matches!(result, Ok(None))); + } + + #[test] + fn test_returns_none_when_no_src_dir() { + // library.properties exists but no src/ directory. + let tmp = tempfile::TempDir::new().unwrap(); + let project_dir = tmp.path(); + std::fs::write(project_dir.join("library.properties"), "name=Test\n").unwrap(); + + let src_dir = project_dir.join("examples").join("Demo"); + let include_dirs: Vec = vec![]; + let c_flags: Vec = vec![]; + let cpp_flags: Vec = vec![]; + let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); + + let result = compile_project_as_library( + project_dir, + &src_dir, + &project_dir.join("build"), + &env, + &HashSet::new(), + ); + assert!(matches!(result, Ok(None))); + } + + #[test] + fn test_returns_none_when_header_only() { + // library.json + src/ but only headers — header-only library, not + // an error, just nothing to compile. + let tmp = tempfile::TempDir::new().unwrap(); + let project_dir = tmp.path(); + std::fs::write(project_dir.join("library.json"), r#"{"name": "test"}"#).unwrap(); + let project_src = project_dir.join("src"); + std::fs::create_dir_all(&project_src).unwrap(); + std::fs::write(project_src.join("lib.h"), "").unwrap(); + std::fs::write(project_src.join("inline.hpp"), "").unwrap(); + + let src_dir = project_dir.join("examples").join("Demo"); + let include_dirs: Vec = vec![]; + let c_flags: Vec = vec![]; + let cpp_flags: Vec = vec![]; + let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); + + let result = compile_project_as_library( + project_dir, + &src_dir, + &project_dir.join("build"), + &env, + &HashSet::new(), + ); + assert!(matches!(result, Ok(None))); + } + + #[test] + fn test_returns_none_on_collision_with_lib_dir() { + // If a user has both library.json AND lib//, the lib/ + // version wins (matches PlatformIO behavior). Must skip project-as- + // library to prevent two libfastled.a archives at link time. + let tmp = tempfile::TempDir::new().unwrap(); + // Create a project dir with a known basename to control lib_name. + let project_dir = tmp.path().join("FastLED"); + std::fs::create_dir_all(&project_dir).unwrap(); + std::fs::write(project_dir.join("library.json"), r#"{"name": "FastLED"}"#).unwrap(); + let project_src = project_dir.join("src"); + std::fs::create_dir_all(&project_src).unwrap(); + std::fs::write(project_src.join("FastLED.cpp"), "").unwrap(); + + let src_dir = project_dir.join("examples").join("Blink"); + + let include_dirs: Vec = vec![]; + let c_flags: Vec = vec![]; + let cpp_flags: Vec = vec![]; + let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); + + let mut existing = HashSet::new(); + existing.insert("fastled".to_string()); // lowercased project basename + + let result = compile_project_as_library( + &project_dir, + &src_dir, + &project_dir.join("build"), + &env, + &existing, + ); + assert!(matches!(result, Ok(None))); + } + + #[test] + fn test_attempts_compile_when_building_example() { + // The positive case: library project + sketch lives elsewhere + has + // sources + no name collision → must reach the compile path. We + // verify this by passing a bogus gcc path and asserting the function + // ERRORS (not Ok(None)). An Ok(None) here would mean a guard + // incorrectly skipped the compile. + let tmp = tempfile::TempDir::new().unwrap(); + let project_dir = tmp.path().join("FastLED"); + std::fs::create_dir_all(&project_dir).unwrap(); + std::fs::write(project_dir.join("library.json"), r#"{"name": "FastLED"}"#).unwrap(); + let project_src = project_dir.join("src"); + std::fs::create_dir_all(&project_src).unwrap(); + std::fs::write(project_src.join("FastLED.cpp"), "// stub").unwrap(); + + let src_dir = project_dir.join("examples").join("Blink"); + std::fs::create_dir_all(&src_dir).unwrap(); + + let include_dirs: Vec = vec![]; + let c_flags: Vec = vec![]; + let cpp_flags: Vec = vec![]; + let env = bogus_env(&include_dirs, &c_flags, &cpp_flags); + + let result = compile_project_as_library( + &project_dir, + &src_dir, + &project_dir.join("build"), + &env, + &HashSet::new(), + ); + // Must NOT be Ok(None) — that would mean a guard skipped compile. + // Either Err (bogus tool failed) or Ok(Some(_)) (impossible without + // a real toolchain) is acceptable. + if let Ok(None) = result { + panic!("expected compile to be attempted, but a guard returned Ok(None)"); + } + } +} diff --git a/crates/fbuild-build/src/pipeline/link.rs b/crates/fbuild-build/src/pipeline/link.rs new file mode 100644 index 00000000..3487181d --- /dev/null +++ b/crates/fbuild-build/src/pipeline/link.rs @@ -0,0 +1,107 @@ +//! Link-result reporting and final `BuildResult` assembly. + +use std::path::{Path, PathBuf}; + +use fbuild_core::BuildLog; + +use crate::linker::LinkResult; +use crate::BuildResult; + +/// Log size report and artifacts from a link result. +/// +/// When `symbol_analysis_path` is `Some`, the report is written to that path +/// and only a one-liner is logged (unless `verbose` is true, which also streams +/// the full report). When `None`, the report is written to `symbol_analysis.txt` +/// in the build artifacts directory and streamed to the build log. +pub fn handle_link_result( + link_result: &LinkResult, + build_log: &mut BuildLog, + symbol_analysis_path: Option<&Path>, + verbose: bool, +) { + if link_result.hex_path.is_some() { + crate::build_output::log_linking(build_log, "Building firmware.hex"); + } else if link_result.bin_path.is_some() { + crate::build_output::log_linking(build_log, "Building firmware.bin"); + } + + if let Some(ref size) = link_result.size_info { + tracing::info!( + "size: text={} data={} bss={} | flash={}/{} ({:.1}%) ram={}/{} ({:.1}%)", + size.text, + size.data, + size.bss, + size.total_flash, + size.max_flash.unwrap_or(0), + size.flash_percent().unwrap_or(0.0), + size.total_ram, + size.max_ram.unwrap_or(0), + size.ram_percent().unwrap_or(0.0), + ); + crate::build_output::log_size_report(build_log, size); + } + + if let Some(ref symbols) = link_result.symbol_map { + let report = crate::build_output::format_symbol_report(symbols); + + if let Some(path) = symbol_analysis_path { + // User gave an explicit path — write there, log a one-liner + if let Err(e) = std::fs::write(path, &report) { + tracing::warn!("failed to write symbol analysis: {e}"); + } else { + build_log.push(format!("Symbol analysis written to {}", path.display())); + } + // Also stream full report when --verbose + if verbose { + crate::build_output::log_symbol_report(build_log, symbols); + } + } else { + // No path — stream to console and write to artifacts dir + crate::build_output::log_symbol_report(build_log, symbols); + if let Some(ref elf) = link_result.elf_path { + if let Some(build_dir) = elf.parent() { + let txt_path = build_dir.join("symbol_analysis.txt"); + if let Err(e) = std::fs::write(&txt_path, &report) { + tracing::warn!("failed to write symbol_analysis.txt: {e}"); + } else { + build_log.push(format!("Symbol analysis: {}", txt_path.display())); + } + } + } + } + } + + if let Some(ref elf) = link_result.elf_path { + crate::build_output::log_artifact(build_log, elf); + } + let firmware = link_result + .hex_path + .as_ref() + .or(link_result.bin_path.as_ref()); + if let Some(fw) = firmware { + crate::build_output::log_artifact(build_log, fw); + } +} + +/// Assemble the final `BuildResult` from link output and build metadata. +pub fn assemble_build_result( + link_result: LinkResult, + elapsed: f64, + platform_label: &str, + env_name: &str, + compile_database_path: Option, + build_log: BuildLog, +) -> BuildResult { + tracing::info!("build completed in {:.1}s", elapsed); + BuildResult { + success: true, + firmware_path: link_result.bin_path.or(link_result.hex_path), + elf_path: link_result.elf_path, + size_info: link_result.size_info, + symbol_map: link_result.symbol_map, + build_time_secs: elapsed, + message: format!("{} build for {} completed", platform_label, env_name), + compile_database_path, + build_log, + } +} diff --git a/crates/fbuild-build/src/pipeline/mod.rs b/crates/fbuild-build/src/pipeline/mod.rs new file mode 100644 index 00000000..d2fccdde --- /dev/null +++ b/crates/fbuild-build/src/pipeline/mod.rs @@ -0,0 +1,26 @@ +//! Shared build pipeline helpers used by all platform orchestrators. +//! +//! Extracts the duplicated config-parse → board-load → build-dir-setup → compile → link +//! sequence that was copy-pasted across AVR, Teensy, and ESP32 orchestrators. +//! +//! This module is split into submodules for maintainability; the public API +//! is preserved by re-exporting every previously top-level item here so that +//! `crate::pipeline::Foo` continues to resolve unchanged. + +mod build_unflags; +mod compile; +mod context; +mod library; +mod link; +mod project_discovery; +mod sequential; + +pub use build_unflags::remove_unflagged_tokens; +pub use compile::{ + compile_local_libraries, compile_sources, generate_compile_db, log_toolchain_version, +}; +pub use context::BuildContext; +pub use library::{compile_project_as_library, pick_archiver, LibraryBuildEnv}; +pub use link::{assemble_build_result, handle_link_result}; +pub use project_discovery::{discover_project_includes, is_platform_project, is_project_a_library}; +pub use sequential::run_sequential_build_with_libs; diff --git a/crates/fbuild-build/src/pipeline/project_discovery.rs b/crates/fbuild-build/src/pipeline/project_discovery.rs new file mode 100644 index 00000000..2d8902ad --- /dev/null +++ b/crates/fbuild-build/src/pipeline/project_discovery.rs @@ -0,0 +1,70 @@ +//! Project-level discovery helpers: include paths, library detection, +//! and platform-config matching. + +use std::path::{Path, PathBuf}; + +/// Add the project's `include/` directory and `lib/` subdirectories to include paths. +/// +/// PlatformIO automatically adds these — replicate that behavior. +pub fn discover_project_includes(project_dir: &Path, include_dirs: &mut Vec) { + // PlatformIO automatically includes the project's include/ directory + let include_dir = project_dir.join("include"); + if include_dir.is_dir() { + include_dirs.push(include_dir); + } + + // PlatformIO automatically discovers libraries placed in the project's lib/ directory. + // Each subdirectory is treated as a library — add its root (and src/ if present). + let local_lib_dir = project_dir.join("lib"); + if local_lib_dir.is_dir() { + if let Ok(entries) = std::fs::read_dir(&local_lib_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + let lib_src = path.join("src"); + if lib_src.is_dir() { + include_dirs.push(lib_src); + } + // Always add the root too (some libraries have headers at top level) + include_dirs.push(path); + } + } + } + } + + // Project-as-library detection (PlatformIO convention). + // When a project root contains library.json or library.properties, the project + // itself is a library and its src/ directory is automatically added to include + // paths for any sketch built within the project. This allows building example + // sketches against the library being developed (e.g., FastLED examples). + let library_json = project_dir.join("library.json"); + let library_props = project_dir.join("library.properties"); + if library_json.exists() || library_props.exists() { + let project_src = project_dir.join("src"); + if project_src.is_dir() && !include_dirs.contains(&project_src) { + include_dirs.push(project_src); + } + } +} + +/// Returns true if the project is a PlatformIO library (has library.json or library.properties). +pub fn is_project_a_library(project_dir: &Path) -> bool { + project_dir.join("library.json").exists() || project_dir.join("library.properties").exists() +} + +/// Check if a project is configured for a specific platform by reading its platformio.ini. +pub fn is_platform_project( + project_dir: &Path, + env_name: &str, + platform: fbuild_core::Platform, +) -> bool { + let ini_path = project_dir.join("platformio.ini"); + if let Ok(config) = fbuild_config::PlatformIOConfig::from_path(&ini_path) { + if let Ok(env) = config.get_env_config(env_name) { + if let Some(platform_str) = env.get("platform") { + return platform.matches_str(platform_str); + } + } + } + false +} diff --git a/crates/fbuild-build/src/pipeline/sequential.rs b/crates/fbuild-build/src/pipeline/sequential.rs new file mode 100644 index 00000000..0894f272 --- /dev/null +++ b/crates/fbuild-build/src/pipeline/sequential.rs @@ -0,0 +1,266 @@ +//! The sequential compile → link → result pipeline used by AVR, Teensy, +//! RP2040, STM32, ESP8266, CH32V, NRF52, SAM, Renesas, and Apollo3. + +use std::path::PathBuf; +use std::time::Instant; + +use fbuild_core::Result; + +use crate::compile_database::TargetArchitecture; +use crate::compiler::Compiler; +use crate::flag_overlay::LanguageExtraFlags; +use crate::source_scanner::SourceCollection; +use crate::{BuildParams, BuildResult}; + +use super::compile::{compile_local_libraries, compile_sources, generate_compile_db}; +use super::context::BuildContext; +use super::library::{compile_project_as_library, LibraryBuildEnv}; +use super::link::{assemble_build_result, handle_link_result}; + +/// Run the sequential compile → link → result pipeline used by AVR, Teensy, +/// RP2040, STM32, ESP8266, CH32V, NRF52, SAM, Renesas, and Apollo3. +/// +/// Handles: compiledb_only early return, sequential compilation of +/// core/variant/sketch/libs, compile database generation, linking, and result +/// assembly. +/// +/// ESP32 cannot use this because it uses parallel compilation and has +/// additional hooks (SDK libs, embed files, bootloader prep). It calls +/// [`compile_project_as_library`] directly. +/// +/// When `lib_env` is `Some`, the project's own `src/` is compiled as a library +/// archive (matching PlatformIO's project-as-library convention) and linked +/// with the rest of the build. See [`compile_project_as_library`] and +/// ISSUES.md Issue 1. +#[allow(clippy::too_many_arguments)] +pub fn run_sequential_build_with_libs( + compiler: &dyn Compiler, + linker: &dyn crate::linker::Linker, + mut ctx: BuildContext, + params: &BuildParams, + sources: &SourceCollection, + extra_link_inputs: &[PathBuf], + lib_env: Option<&LibraryBuildEnv<'_>>, + arch: TargetArchitecture, + platform_label: &str, + start: Instant, +) -> Result { + // Env-gated per-phase timer (FBUILD_PERF_LOG=1). Emits summary on drop. + // Zero-overhead when the env var is unset — phase guards become no-ops. + let mut perf = crate::perf_log::PerfTimer::new("pipeline"); + let core_and_variant: Vec = sources + .core_sources + .iter() + .chain(sources.variant_sources.iter()) + .cloned() + .collect(); + let user_overlay = LanguageExtraFlags { + common: ctx + .user_flags + .iter() + .cloned() + .chain(ctx.global_compile_overlay.common.iter().cloned()) + .collect(), + c: ctx.global_compile_overlay.c.clone(), + cxx: ctx.global_compile_overlay.cxx.clone(), + asm: ctx.global_compile_overlay.asm.clone(), + }; + let src_overlay = LanguageExtraFlags::combined(&[ + &user_overlay, + &LanguageExtraFlags { + common: ctx.src_flags.clone(), + c: Vec::new(), + cxx: Vec::new(), + asm: Vec::new(), + }, + &ctx.project_compile_overlay, + ]); + + // compiledb_only: generate compile_commands.json without compiling + if params.compiledb_only { + let compile_database_path = generate_compile_db( + compiler.gcc_path(), + compiler.gxx_path(), + &compiler.c_flags(), + &compiler.cpp_flags(), + &[], + &user_overlay, + &src_overlay, + &core_and_variant, + &sources.sketch_sources, + &ctx.core_build_dir, + &ctx.src_build_dir, + &ctx.build_dir, + ¶ms.project_dir, + arch, + )?; + let elapsed = start.elapsed().as_secs_f64(); + return Ok(BuildResult { + success: true, + firmware_path: None, + elf_path: None, + size_info: None, + symbol_map: None, + build_time_secs: elapsed, + message: format!("compile_commands.json generated for {}", params.env_name), + compile_database_path, + build_log: ctx.build_log, + }); + } + + // Wrap the build log so it can be shared across parallel compile phases. + // Phases still run one after another (compile core → variant → sketch → + // libs → link), but each phase fans out file compilation across `jobs` + // threads via `compile_sources_parallel`. + let jobs = crate::parallel::effective_jobs(params.jobs); + let build_log_mutex = std::sync::Mutex::new(ctx.build_log); + + // Compile core + variant + let mut core_objects = { + let _g = perf.phase("compile-core"); + compile_sources( + compiler, + &sources.core_sources, + &ctx.core_build_dir, + &user_overlay, + jobs, + &build_log_mutex, + )? + }; + let variant_objects = { + let _g = perf.phase("compile-variant"); + compile_sources( + compiler, + &sources.variant_sources, + &ctx.core_build_dir, + &user_overlay, + jobs, + &build_log_mutex, + )? + }; + core_objects.extend(variant_objects); + + // Compile sketch + let sketch_objects = { + let _g = perf.phase("compile-sketch"); + compile_sources( + compiler, + &sources.sketch_sources, + &ctx.src_build_dir, + &src_overlay, + jobs, + &build_log_mutex, + )? + }; + + // Compile local libraries (lib/* — loose objects, LTO-safe; per-lib parallel) + let library_objects = { + let _g = perf.phase("compile-local-libs"); + compile_local_libraries( + compiler, + ¶ms.project_dir, + &ctx.build_dir, + &src_overlay, + jobs, + &build_log_mutex, + )? + }; + + // Unwrap the build log Mutex back into the context for the remaining + // single-threaded phases (link, result assembly). + ctx.build_log = build_log_mutex.into_inner().unwrap(); + + // Project-as-library: compile project root's src/ as an archive when + // building an example sketch from a library project (e.g. FastLED examples). + // Only runs when caller provided a LibraryBuildEnv with toolchain paths. + let project_as_lib_archive: Option = { + let _g = perf.phase("project-as-lib"); + if let Some(env) = lib_env { + // Collect existing lib/* names so the helper can detect collisions. + let mut existing_lib_names = std::collections::HashSet::new(); + let local_lib_dir = params.project_dir.join("lib"); + if local_lib_dir.is_dir() { + if let Ok(entries) = std::fs::read_dir(&local_lib_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + existing_lib_names.insert(name.to_lowercase()); + } + } + } + } + } + compile_project_as_library( + ¶ms.project_dir, + &ctx.src_dir, + &ctx.build_dir, + env, + &existing_lib_names, + )? + } else { + None + } + }; + + // Generate compile_commands.json + let compile_database_path = { + let _g = perf.phase("compile-db"); + generate_compile_db( + compiler.gcc_path(), + compiler.gxx_path(), + &compiler.c_flags(), + &compiler.cpp_flags(), + &[], + &user_overlay, + &src_overlay, + &core_and_variant, + &sources.sketch_sources, + &ctx.core_build_dir, + &ctx.src_build_dir, + &ctx.build_dir, + ¶ms.project_dir, + arch, + )? + }; + + // Link + crate::build_output::log_linking(&mut ctx.build_log, "Linking firmware.elf"); + core_objects.extend(library_objects); + core_objects.extend(extra_link_inputs.iter().cloned()); + if let Some(archive) = project_as_lib_archive { + // GCC accepts .a in the same positional slot as .o files. + core_objects.push(archive); + } + let link_result = { + let _g = perf.phase("link"); + crate::linker::Linker::link_all( + linker, + &sketch_objects, + &core_objects, + &ctx.build_dir, + &crate::linker::LinkExtraArgs { + flags: ctx.overlay_link_flags.clone(), + libs: ctx.overlay_link_libs.clone(), + }, + params.symbol_analysis, + )? + }; + + // Result + handle_link_result( + &link_result, + &mut ctx.build_log, + params.symbol_analysis_path.as_deref(), + params.verbose, + ); + let elapsed = start.elapsed().as_secs_f64(); + Ok(assemble_build_result( + link_result, + elapsed, + platform_label, + ¶ms.env_name, + compile_database_path, + ctx.build_log, + )) +} diff --git a/crates/fbuild-build/src/stm32/orchestrator.rs b/crates/fbuild-build/src/stm32/orchestrator.rs deleted file mode 100644 index d1ad9e39..00000000 --- a/crates/fbuild-build/src/stm32/orchestrator.rs +++ /dev/null @@ -1,1004 +0,0 @@ -//! STM32 build orchestrator — wires together config, packages, compiler, linker. -//! -//! Build phases: -//! 1. Parse platformio.ini -//! 2. Load board config (bluepill_f103c8, blackpill_f411ce, nucleo_*, etc.) -//! 3. Ensure ARM GCC toolchain -//! 4. Ensure STM32duino cores (Arduino_Core_STM32) -//! 5. Setup build directories -//! 6. Scan source files -//! 7. Compile core + variant sources -//! 8. Compile sketch sources -//! 9. Link (with linker script from variant dir) -//! 10. Convert to hex + report size - -use std::collections::{HashMap, HashSet}; -use std::path::{Path, PathBuf}; -use std::time::Instant; - -use fbuild_core::{Platform, Result}; -use fbuild_packages::{Framework, Toolchain}; - -use crate::compile_database::TargetArchitecture; -use crate::framework_libs::{ - library_select_kv_store, resolve_framework_library_sources, - resolve_framework_library_sources_cached, -}; -use crate::generic_arm::{ArmCompiler, ArmLinker}; -use crate::pipeline; -use crate::source_scanner::SourceCollection; -use crate::{BuildOrchestrator, BuildParams, BuildResult, SourceScanner}; - -/// STM32 platform build orchestrator. -pub struct Stm32Orchestrator; - -impl BuildOrchestrator for Stm32Orchestrator { - fn platform(&self) -> Platform { - Platform::Ststm32 - } - - fn build(&self, params: &BuildParams) -> Result { - let start = Instant::now(); - - // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = pipeline::BuildContext::new(params)?; - - // Compute eh_frame strip policy once per build (FastLED/fbuild#244). - let eh_frame_policy = - crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); - - // 3. Ensure ARM GCC toolchain - let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; - tracing::info!("arm-gcc toolchain at {}", toolchain_dir.display()); - - pipeline::log_toolchain_version( - &toolchain.get_gcc_path(), - "arm-none-eabi-gcc", - &mut ctx.build_log, - ); - - if is_arduino_mbed_stm32_variant(&ctx.board.variant) { - return build_arduino_mbed_stm32(params, ctx, &toolchain, start); - } - - // 4. Ensure STM32duino cores - let framework = fbuild_packages::library::Stm32Cores::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; - tracing::info!("STM32 cores at {}", framework_dir.display()); - - // 5. Scan sources (core + variant) - // STM32duino uses "arduino" as its core directory name, even though - // the board JSON says core = "stm32". Map it here. - let core_dir = framework.get_core_dir("arduino"); - let framework_props = - load_stm32_framework_props(&ctx.board.variant, &framework.get_boards_txt()); - let resolved_variant = framework_props - .as_ref() - .and_then(|props| props.get("variant").cloned()) - .unwrap_or_else(|| ctx.board.variant.clone()); - let variant_dir = framework.get_variant_dir(&resolved_variant); - let selected_variant = select_variant_files( - &variant_dir, - &resolved_variant, - framework_props - .as_ref() - .and_then(|props| props.get("variant_h").map(String::as_str)) - .or(ctx.board.variant_h.as_deref()), - ); - - let scanner = SourceScanner::new(&ctx.src_dir, &ctx.src_build_dir); - // Scan core + variant, but pass None for variant — we'll filter variant - // sources manually because the variant dir contains files for multiple - // board variants (MALYAN, AFROFLIGHT, etc.) and startup files that - // conflict with the generic one in cores/arduino/stm32/. - let mut sources = - scanner.scan_all_filtered(Some(&core_dir), None, ctx.source_filter.as_deref())?; - sources.variant_sources = scanner - .scan_variant_sources(&variant_dir) - .into_iter() - .filter(|p| keep_variant_source(p, &selected_variant)) - .collect(); - - // SrcWrapper is a core library in STM32duino — its sources must be - // compiled alongside the Arduino core (HAL wrappers, syscalls, etc.) - // scan_core_sources is recursive, so one call covers all subdirs. - let libs_dir = framework.get_libraries_dir(); - let srcwrapper_src = libs_dir.join("SrcWrapper").join("src"); - if srcwrapper_src.exists() { - sources - .core_sources - .extend(scanner.scan_core_sources(&srcwrapper_src)); - } - - // Walk Arduino_Core_STM32's libraries/ (SPI, Wire, EEPROM, ...) and - // pull in any the sketch transitively #includes. Without this, sketches - // that include fail with "No such file or directory" because - // STM32duino only exposes bundled libraries via this framework-level - // discovery (PlatformIO's LDF does the same for `framework = arduino`). - let framework_libs = framework.get_framework_libraries(); - // WHY: STM32duino targets every Cortex-M family from M0 (F0xx) up - // through M7 (H7xx) but the toolchain triple is constant - // (`arm-none-eabi`). The cache key already includes - // `framework_install_path` + `framework_version`, so per-MCU drift - // is handled there — this string only needs to disambiguate stm32 - // from teensy etc. so cross-platform key collisions are impossible. - let framework_info = fbuild_packages::Package::get_info(&framework); - let framework_library_sources = match library_select_kv_store() { - Some(store) => { - let key_inputs = fbuild_library_select::cache::CacheKeyInputs { - toolchain_triple: "stm32-arm-none-eabi", - framework_install_path: &framework_info.install_path, - framework_version: &framework_info.version, - }; - resolve_framework_library_sources_cached( - &framework_libs, - ¶ms.project_dir, - &ctx.src_dir, - &key_inputs, - store, - ) - } - None => resolve_framework_library_sources( - &framework_libs, - ¶ms.project_dir, - &ctx.src_dir, - ), - }; - if !framework_library_sources.is_empty() { - tracing::info!( - "STM32 framework library sources added: {}", - framework_library_sources.len() - ); - sources.core_sources.extend(framework_library_sources); - } - - tracing::info!( - "sources: {} sketch, {} core, {} variant", - sources.sketch_sources.len(), - sources.core_sources.len(), - sources.variant_sources.len(), - ); - - // 6. Build include dirs + compiler - // Extract MCU family from variant path (e.g. "STM32F1xx" from "STM32F1xx/F103C8T_...") - let family = resolved_variant.split('/').next().unwrap_or("STM32F1xx"); - - let mut mcu_config = - super::mcu_config::get_stm32_config_for_mcu(&ctx.board.mcu.to_lowercase())?; - // STM32duino linker scripts reference LD_MAX_SIZE, LD_MAX_DATA_SIZE, and - // LD_FLASH_OFFSET as symbols. Provide them via --defsym from board config. - let max_flash = ctx.board.max_flash.unwrap_or(65536); - let max_ram = ctx.board.max_ram.unwrap_or(20480); - mcu_config - .linker_flags - .push(format!("-Wl,--defsym=LD_MAX_SIZE={max_flash}")); - mcu_config - .linker_flags - .push(format!("-Wl,--defsym=LD_MAX_DATA_SIZE={max_ram}")); - mcu_config - .linker_flags - .push("-Wl,--defsym=LD_FLASH_OFFSET=0".to_string()); - let mut defines = ctx.board.get_defines(); - defines.extend(mcu_config.defines_map()); - if let Some(board_define) = framework_props - .as_ref() - .and_then(|props| props.get("board")) - { - defines.insert(format!("ARDUINO_{board_define}"), "1".to_string()); - } - // STM32duino's stm32_def.h checks for STM32YYxx (e.g. STM32F1xx). The board - // JSON extra_flags may only have STM32F1, so ensure the full family define. - defines.insert(family.to_string(), "1".to_string()); - // STM32duino variant sources are guarded by ARDUINO_GENERIC_ defines. - // Derive from the MCU name: stm32f103c8t6 → ARDUINO_GENERIC_F103C8TX - let generic_board = stm32_generic_board_define(&ctx.board.mcu); - defines.insert(format!("ARDUINO_{generic_board}"), "1".to_string()); - // STM32duino requires these defines for HAL/LL and variant header resolution. - defines.insert("USE_HAL_DRIVER".to_string(), "1".to_string()); - defines.insert("USE_FULL_LL_DRIVER".to_string(), "1".to_string()); - defines.insert( - "VARIANT_H".to_string(), - format!("\\\"{}\\\"", selected_variant.header), - ); - // UART HAL module is disabled by default in stm32yyxx_hal_conf.h — enable it - // so WSerial.h can create the Serial instance. - defines.insert("HAL_UART_MODULE_ENABLED".to_string(), "1".to_string()); - defines.insert("HAL_PCD_MODULE_ENABLED".to_string(), "1".to_string()); - - // Build include dirs manually (can't use get_include_paths because - // STM32duino core dir is "arduino", not the board JSON's "stm32") - let mut include_dirs = vec![core_dir.clone(), variant_dir.clone()]; - // Core subdirectories (AVR compat, STM32 HAL wrapper) - include_dirs.push(core_dir.join("avr")); - include_dirs.push(core_dir.join("stm32")); - // SrcWrapper include dirs (clock.h, analog.h, LL/stm32yyxx_ll_*.h, etc.) - let srcwrapper_inc = libs_dir.join("SrcWrapper").join("inc"); - if srcwrapper_inc.exists() { - include_dirs.push(srcwrapper_inc.clone()); - let ll_inc = srcwrapper_inc.join("LL"); - if ll_inc.exists() { - include_dirs.push(ll_inc); - } - } - include_dirs.push(ctx.src_dir.clone()); - pipeline::discover_project_includes(¶ms.project_dir, &mut include_dirs); - // Bundled framework library headers (SPI, Wire, EEPROM, ...) so - // sketches can `#include ` etc. - include_dirs.extend(framework.get_framework_library_include_dirs()); - - // STM32duino system includes (CMSIS device, HAL drivers, etc.) - let system_dir = framework.get_system_dir(); - add_stm32_system_includes(&system_dir, family, &mut include_dirs); - - // CMSIS Core includes (core_cm3.h, core_cm4.h, etc.) — not bundled in STM32duino - let cmsis = fbuild_packages::library::CmsisFramework::new(¶ms.project_dir); - let _cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis)?; - tracing::info!("CMSIS framework installed"); - include_dirs.push(cmsis.get_core_include_dir()); - - // Toolchain sysroot includes (ARM CMSIS headers, etc.) - include_dirs.extend(toolchain.get_include_dirs()); - - let compiler = ArmCompiler::new( - toolchain.get_gcc_path(), - toolchain.get_gxx_path(), - &ctx.board.mcu, - &ctx.board.f_cpu, - defines, - include_dirs.clone(), - mcu_config.clone(), - params.profile, - params.verbose, - ) - .with_build_unflags(ctx.build_unflags.clone()) - .with_eh_frame_policy(eh_frame_policy); - - // 7. Create linker (linker script from variant dir) - let linker_script = match ctx.board.ldscript.as_deref() { - Some(name) => variant_dir.join(name), - None => framework.get_linker_script(&resolved_variant), - }; - let linker = ArmLinker::new( - toolchain.get_gcc_path(), - toolchain.get_ar_path(), - toolchain.get_objcopy_path(), - toolchain.get_size_path(), - linker_script, - mcu_config, - params.profile, - ctx.board.max_flash, - ctx.board.max_ram, - params.verbose, - ); - - // 8. Build LibraryBuildEnv for project-as-library compilation - let gcc_path = toolchain.get_gcc_path(); - let gxx_path = toolchain.get_gxx_path(); - let ar_path = toolchain.get_ar_path(); - let gcc_ar_path = toolchain.get_gcc_ar_path(); - let c_flags = crate::compiler::Compiler::c_flags(&compiler); - let cpp_flags = crate::compiler::Compiler::cpp_flags(&compiler); - // Use gcc-ar for LTO archives so the linker-plugin index is written. - let lib_ar_path = pipeline::pick_archiver(&ar_path, &gcc_ar_path, &c_flags, &cpp_flags); - let lib_env = pipeline::LibraryBuildEnv { - gcc_path: &gcc_path, - gxx_path: &gxx_path, - ar_path: lib_ar_path, - c_flags: &c_flags, - cpp_flags: &cpp_flags, - include_dirs: &include_dirs, - verbose: params.verbose, - jobs: crate::parallel::effective_jobs(params.jobs), - compiler_cache: None, - }; - - // 9. Run shared sequential build pipeline - pipeline::run_sequential_build_with_libs( - &compiler, - &linker, - ctx, - params, - &sources, - &[], - Some(&lib_env), - TargetArchitecture::Arm, - "STM32", - start, - ) - } -} - -fn build_arduino_mbed_stm32( - params: &BuildParams, - ctx: pipeline::BuildContext, - toolchain: &fbuild_packages::toolchain::ArmToolchain, - start: Instant, -) -> Result { - // Compute eh_frame strip policy once per build (FastLED/fbuild#244). - let eh_frame_policy = - crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); - - let framework = fbuild_packages::library::ArduinoMbedCore::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; - tracing::info!("Arduino mbed core at {}", framework_dir.display()); - - let core_dir = framework.get_core_dir("arduino"); - let variant_dir = framework.get_variant_dir(&ctx.board.variant); - - let scanner = SourceScanner::new(&ctx.src_dir, &ctx.src_build_dir); - let sources = SourceCollection { - sketch_sources: scanner.scan_sketch_sources_filtered(ctx.source_filter.as_deref())?, - core_sources: framework.get_core_sources(), - variant_sources: framework.get_variant_sources(&ctx.board.variant), - headers: Vec::new(), - }; - - tracing::info!( - "sources: {} sketch, {} core, {} variant", - sources.sketch_sources.len(), - sources.core_sources.len(), - sources.variant_sources.len(), - ); - - let mut defines = ctx.board.get_defines(); - defines.insert("ARDUINO".to_string(), "10810".to_string()); - defines.insert("ARDUINO_ARCH_MBED".to_string(), "1".to_string()); - for token in fbuild_core::shell_split::split( - &framework.read_variant_file(&ctx.board.variant, "defines.txt"), - ) { - if let Some(def) = token.strip_prefix("-D") { - if let Some((key, val)) = def.split_once('=') { - defines.insert(key.to_string(), val.to_string()); - } else { - defines.insert(def.to_string(), "1".to_string()); - } - } - } - let board_ldflags = load_stm32_framework_props(&ctx.board.variant, &framework.get_boards_txt()) - .and_then(|props| props.get("extra_ldflags").cloned()) - .map(|flags| fbuild_core::shell_split::split(&flags)) - .unwrap_or_default(); - apply_define_flags(&board_ldflags, &mut defines); - - let mut include_dirs = vec![ - core_dir.clone(), - core_dir.join("api").join("deprecated"), - core_dir.join("api").join("deprecated-avr-comp"), - variant_dir.clone(), - ]; - include_dirs.extend(framework.get_variant_includes(&ctx.board.variant)); - include_dirs.push(ctx.src_dir.clone()); - pipeline::discover_project_includes(¶ms.project_dir, &mut include_dirs); - dedupe_paths(&mut include_dirs); - - let mut variant_ldflags = fbuild_core::shell_split::split( - &framework.read_variant_file(&ctx.board.variant, "ldflags.txt"), - ); - variant_ldflags.extend(board_ldflags.iter().cloned()); - dedupe_strings(&mut variant_ldflags); - let linker_script = preprocess_linker_script( - toolchain.get_gxx_path(), - &variant_dir, - &ctx.board.variant, - &ctx.build_dir, - &variant_ldflags, - )?; - - let mcu_config = build_arduino_mbed_mcu_config( - &framework, - &ctx.board.variant, - framework.get_mbed_lib(&ctx.board.variant), - &board_ldflags, - ); - - let compiler = ArmCompiler::new( - toolchain.get_gcc_path(), - toolchain.get_gxx_path(), - &ctx.board.mcu, - &ctx.board.f_cpu, - defines, - include_dirs.clone(), - mcu_config.clone(), - params.profile, - params.verbose, - ) - .with_build_unflags(ctx.build_unflags.clone()) - .with_eh_frame_policy(eh_frame_policy); - - let linker = ArmLinker::new( - toolchain.get_gcc_path(), - toolchain.get_ar_path(), - toolchain.get_objcopy_path(), - toolchain.get_size_path(), - linker_script, - mcu_config, - params.profile, - ctx.board.max_flash, - ctx.board.max_ram, - params.verbose, - ); - - let gcc_path = toolchain.get_gcc_path(); - let gxx_path = toolchain.get_gxx_path(); - let ar_path = toolchain.get_ar_path(); - let gcc_ar_path = toolchain.get_gcc_ar_path(); - let c_flags = crate::compiler::Compiler::c_flags(&compiler); - let cpp_flags = crate::compiler::Compiler::cpp_flags(&compiler); - let lib_ar_path = pipeline::pick_archiver(&ar_path, &gcc_ar_path, &c_flags, &cpp_flags); - let lib_env = pipeline::LibraryBuildEnv { - gcc_path: &gcc_path, - gxx_path: &gxx_path, - ar_path: lib_ar_path, - c_flags: &c_flags, - cpp_flags: &cpp_flags, - include_dirs: &include_dirs, - verbose: params.verbose, - jobs: crate::parallel::effective_jobs(params.jobs), - compiler_cache: None, - }; - - pipeline::run_sequential_build_with_libs( - &compiler, - &linker, - ctx, - params, - &sources, - &[], - Some(&lib_env), - TargetArchitecture::Arm, - "STM32", - start, - ) -} - -/// Add STM32duino system include directories for CMSIS and HAL. -/// -/// The STM32duino core bundles CMSIS and HAL drivers under `system/`: -/// - `system/Drivers/CMSIS/Device/ST//Include/` — MCU device headers -/// - `system/Drivers/CMSIS/Core/Include/` — ARM CMSIS core headers -/// - `system/Drivers/_HAL_Driver/Inc/` — STM32 HAL headers -/// - `system//` — System startup headers -fn add_stm32_system_includes( - system_dir: &Path, - family: &str, - include_dirs: &mut Vec, -) { - let drivers = system_dir.join("Drivers"); - - // CMSIS Device headers (stm32f1xx.h, stm32f103xb.h, etc.) - let cmsis_device = drivers - .join("CMSIS") - .join("Device") - .join("ST") - .join(family) - .join("Include"); - if cmsis_device.exists() { - include_dirs.push(cmsis_device); - } - - // CMSIS Core headers (core_cm3.h, core_cm4.h, etc.) - let cmsis_core = drivers.join("CMSIS").join("Core").join("Include"); - if cmsis_core.exists() { - include_dirs.push(cmsis_core); - } - - // CMSIS startup templates (startup_stm32f103xb.s, etc.) - let cmsis_startup = drivers - .join("CMSIS") - .join("Device") - .join("ST") - .join(family) - .join("Source") - .join("Templates") - .join("gcc"); - if cmsis_startup.exists() { - include_dirs.push(cmsis_startup); - } - - // HAL Driver headers (stm32f1xx_hal.h, etc.) - let hal_driver = drivers.join(format!("{family}_HAL_Driver")); - let hal_inc = hal_driver.join("Inc"); - if hal_inc.exists() { - include_dirs.push(hal_inc); - } - // HAL Driver sources — SrcWrapper's stm32yyxx_hal.c does #include "stm32f1xx_hal.c" - let hal_src = hal_driver.join("Src"); - if hal_src.exists() { - include_dirs.push(hal_src); - } - - // System family directory (startup and system config headers) - let system_family = system_dir.join(family); - if system_family.exists() { - include_dirs.push(system_family); - } -} - -fn is_arduino_mbed_stm32_variant(variant: &str) -> bool { - matches!( - variant, - "GIGA" | "PORTENTA_H7_M7" | "GENERIC_STM32H747_M4" | "NICLA_VISION" | "OPTA" - ) -} - -fn build_arduino_mbed_mcu_config( - framework: &fbuild_packages::library::ArduinoMbedCore, - variant_name: &str, - mbed_lib: PathBuf, - board_ldflags: &[String], -) -> crate::generic_arm::ArmMcuConfig { - let cflags = - fbuild_core::shell_split::split(&framework.read_variant_file(variant_name, "cflags.txt")); - let cxxflags = - fbuild_core::shell_split::split(&framework.read_variant_file(variant_name, "cxxflags.txt")); - let ldflags = - fbuild_core::shell_split::split(&framework.read_variant_file(variant_name, "ldflags.txt")); - - let mut common_flags: Vec = cflags - .into_iter() - .filter(|f| f != "-c" && !f.starts_with("-std=")) - .collect(); - common_flags.push("-nostdlib".to_string()); - dedupe_strings(&mut common_flags); - - let c_flags = vec!["-std=gnu11".to_string()]; - let mut cxx_only = cxxflags - .into_iter() - .filter(|f| f != "-c" && !common_flags.contains(f)) - .collect::>(); - dedupe_strings(&mut cxx_only); - - let mut linker_flags: Vec = ldflags - .into_iter() - .filter(|f| !f.starts_with("-D")) - .collect(); - linker_flags.extend( - board_ldflags - .iter() - .filter(|f| !f.starts_with("-D")) - .cloned(), - ); - linker_flags.push("--specs=nano.specs".to_string()); - linker_flags.push("--specs=nosys.specs".to_string()); - dedupe_strings(&mut linker_flags); - - crate::generic_arm::ArmMcuConfig { - name: "ArduinoCore-mbed".to_string(), - description: format!("Arduino mbed variant {variant_name}"), - architecture: "arm-cortex-m".to_string(), - compiler_flags: crate::compiler::CompilerFlags { - common: common_flags, - c: c_flags, - cxx: cxx_only, - }, - linker_flags, - linker_libs: vec![ - "-Wl,--whole-archive".to_string(), - mbed_lib.to_string_lossy().to_string(), - "-Wl,--no-whole-archive".to_string(), - "-Wl,--start-group".to_string(), - "-lstdc++".to_string(), - "-lsupc++".to_string(), - "-lm".to_string(), - "-lc".to_string(), - "-lgcc".to_string(), - "-lnosys".to_string(), - "-Wl,--end-group".to_string(), - ], - objcopy: crate::compiler::ObjcopyConfig { - output_format: "binary".to_string(), - remove_sections: Vec::new(), - }, - profiles: HashMap::new(), - defines: Vec::new(), - } -} - -fn preprocess_linker_script( - gxx_path: PathBuf, - variant_dir: &Path, - variant_name: &str, - build_dir: &Path, - ldflags: &[String], -) -> Result { - let input = variant_dir.join("linker_script.ld"); - let output = build_dir.join("cpp.linker_script.ld"); - let mut args = vec![ - gxx_path.to_string_lossy().to_string(), - "-E".to_string(), - "-P".to_string(), - "-x".to_string(), - "c".to_string(), - ]; - args.extend(ldflags.iter().filter(|f| f.starts_with("-D")).cloned()); - args.push(input.to_string_lossy().to_string()); - args.push("-o".to_string()); - args.push(output.to_string_lossy().to_string()); - - let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = fbuild_core::subprocess::run_command(&args_ref, None, None, None)?; - if !result.success() { - return Err(fbuild_core::FbuildError::BuildFailed(format!( - "failed to preprocess Arduino mbed linker script for {}:\n{}", - variant_name, result.stderr - ))); - } - - Ok(output) -} - -fn dedupe_paths(paths: &mut Vec) { - let mut seen = HashSet::new(); - paths.retain(|path| seen.insert(path.clone())); -} - -fn dedupe_strings(flags: &mut Vec) { - let mut seen = HashSet::new(); - flags.retain(|flag| seen.insert(flag.clone())); -} - -fn apply_define_flags(flags: &[String], defines: &mut HashMap) { - for flag in flags { - if let Some(def) = flag.strip_prefix("-D") { - if let Some((key, val)) = def.split_once('=') { - defines.insert(key.to_string(), val.to_string()); - } else { - defines.insert(def.to_string(), "1".to_string()); - } - } - } -} - -/// Derive the STM32duino generic board define from the MCU name. -/// -/// `stm32f103c8t6` → `GENERIC_F103C8TX` -/// `stm32f411ceu6` → `GENERIC_F411CEUX` -/// -/// Pattern: strip `stm32` prefix, uppercase, replace last char with `X`. -fn stm32_generic_board_define(mcu: &str) -> String { - let suffix = mcu - .to_lowercase() - .strip_prefix("stm32") - .unwrap_or(&mcu.to_lowercase()) - .to_uppercase(); - // Replace last character (pin-count digit) with X - let mut chars: Vec = suffix.chars().collect(); - if let Some(last) = chars.last_mut() { - *last = 'X'; - } - let trimmed: String = chars.into_iter().collect(); - format!("GENERIC_{trimmed}") -} - -#[derive(Debug, Clone)] -struct SelectedVariantFiles { - header: String, - source_stem: Option, - peripheral_stem: Option, -} - -fn select_variant_files( - variant_dir: &Path, - variant_name: &str, - preferred_header: Option<&str>, -) -> SelectedVariantFiles { - let entries = std::fs::read_dir(variant_dir) - .ok() - .into_iter() - .flatten() - .flatten() - .filter_map(|entry| entry.file_name().into_string().ok()) - .collect::>(); - - let header = preferred_header - .and_then(|name| find_entry_case_insensitive(&entries, name)) - .or_else(|| pick_variant_file(&entries, variant_name, "variant_", ".h")) - .unwrap_or_else(|| "variant_generic.h".to_string()); - - let header_suffix = Path::new(&header) - .file_stem() - .and_then(|stem| stem.to_str()) - .and_then(|stem| stem.strip_prefix("variant_")); - let source_stem = header_suffix - .and_then(|suffix| { - find_entry_case_insensitive(&entries, &format!("variant_{suffix}.cpp")) - .map(|name| stem_lower(&name)) - }) - .or_else(|| { - pick_variant_file(&entries, variant_name, "variant_", ".cpp") - .map(|name| stem_lower(&name)) - }); - let peripheral_stem = header_suffix - .and_then(|suffix| { - find_entry_case_insensitive(&entries, &format!("PeripheralPins_{suffix}.c")) - .or_else(|| { - find_entry_case_insensitive(&entries, &format!("peripheralpins_{suffix}.c")) - }) - .map(|name| stem_lower(&name)) - }) - .or_else(|| { - pick_variant_file(&entries, variant_name, "peripheralpins_", ".c") - .map(|name| stem_lower(&name)) - }); - - SelectedVariantFiles { - header, - source_stem, - peripheral_stem, - } -} - -fn pick_variant_file( - entries: &[String], - variant_name: &str, - prefix: &str, - suffix: &str, -) -> Option { - let normalized = normalize_variant_name(variant_name); - let exact = format!("{prefix}{normalized}{suffix}"); - if let Some(name) = find_entry_case_insensitive(entries, &exact) { - return Some(name); - } - - let generic = format!("{prefix}generic{suffix}"); - if let Some(name) = find_entry_case_insensitive(entries, &generic) { - return Some(name); - } - - let mut matches = entries - .iter() - .filter(|name| { - let lower = name.to_lowercase(); - lower.starts_with(prefix) && lower.ends_with(suffix) - }) - .cloned() - .collect::>(); - matches.sort_by_key(|name| name.to_lowercase()); - matches.into_iter().next() -} - -fn find_entry_case_insensitive(entries: &[String], target: &str) -> Option { - entries - .iter() - .find(|name| name.eq_ignore_ascii_case(target)) - .cloned() -} - -fn keep_variant_source(path: &Path, selected: &SelectedVariantFiles) -> bool { - let name = path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .to_lowercase(); - let stem = stem_lower(&name); - - if name.starts_with("startup_") { - return false; - } - if name.starts_with("variant_") { - return selected - .source_stem - .as_ref() - .is_some_and(|wanted| &stem == wanted); - } - if name.starts_with("peripheralpins_") { - return selected - .peripheral_stem - .as_ref() - .is_some_and(|wanted| &stem == wanted); - } - - true -} - -fn normalize_variant_name(name: &str) -> String { - name.to_lowercase() - .replace(['/', '\\', '-', ' '], "_") - .replace("__", "_") -} - -fn stem_lower(name: &str) -> String { - Path::new(name) - .file_stem() - .unwrap_or_default() - .to_string_lossy() - .to_lowercase() -} - -fn load_stm32_framework_props( - board_or_variant: &str, - boards_txt: &Path, -) -> Option> { - let content = std::fs::read_to_string(boards_txt).ok()?; - let preferred_key = if board_or_variant.contains('/') { - ".build.variant" - } else { - ".build.board" - }; - let prefix = find_stm32_prop_prefix(&content, preferred_key, board_or_variant) - .or_else(|| find_stm32_prop_prefix(&content, ".build.variant", board_or_variant))?; - - let mut props = HashMap::new(); - for scope in stm32_property_scopes(&prefix) { - let line_prefix = format!("{scope}."); - for line in content.lines() { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - continue; - } - let Some(rest) = trimmed.strip_prefix(&line_prefix) else { - continue; - }; - let Some((key, value)) = rest.split_once('=') else { - continue; - }; - let key = key.trim(); - let normalized = key - .strip_prefix("build.") - .or_else(|| key.strip_prefix("upload.")) - .unwrap_or(key); - props.insert(normalized.to_string(), value.trim().to_string()); - if normalized != key { - props.insert(key.to_string(), value.trim().to_string()); - } - } - } - - let substitutions = [ - ( - "{build.board}", - props.get("board").cloned().unwrap_or_default(), - ), - ( - "{build.variant}", - props.get("variant").cloned().unwrap_or_default(), - ), - ]; - for value in props.values_mut() { - for (needle, replacement) in &substitutions { - if !replacement.is_empty() { - *value = value.replace(needle, replacement); - } - } - } - - Some(props) -} - -fn find_stm32_prop_prefix(content: &str, suffix: &str, value: &str) -> Option { - content.lines().find_map(|line| { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - return None; - } - let (key, actual) = trimmed.split_once('=')?; - if key.ends_with(suffix) && actual.trim() == value { - Some(key.trim_end_matches(suffix).to_string()) - } else { - None - } - }) -} - -fn stm32_property_scopes(prefix: &str) -> Vec { - let segments = prefix.split('.').collect::>(); - if segments.is_empty() { - return Vec::new(); - } - - let mut scopes = vec![segments[0].to_string()]; - let mut idx = 1; - while idx + 2 < segments.len() { - if segments[idx] != "menu" { - break; - } - idx += 3; - scopes.push(segments[..idx].join(".")); - } - - if scopes.last().map_or(true, |scope| scope != prefix) { - scopes.push(prefix.to_string()); - } - - scopes -} - -/// Create an STM32 orchestrator. -pub fn create() -> Box { - Box::new(Stm32Orchestrator) -} - -/// Check if a project is configured for STM32. -pub fn is_stm32_project(project_dir: &Path, env_name: &str) -> bool { - crate::pipeline::is_platform_project(project_dir, env_name, fbuild_core::Platform::Ststm32) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fs; - - #[test] - fn test_stm32_orchestrator_platform() { - let orch = Stm32Orchestrator; - assert_eq!(orch.platform(), Platform::Ststm32); - } - - #[test] - fn test_is_stm32_project() { - let tmp = tempfile::TempDir::new().unwrap(); - std::fs::write( - tmp.path().join("platformio.ini"), - "[env:bluepill]\nplatform = ststm32\nboard = bluepill_f103c8\nframework = arduino\n", - ) - .unwrap(); - assert!(is_stm32_project(tmp.path(), "bluepill")); - assert!(!is_stm32_project(tmp.path(), "uno")); - } - - #[test] - fn test_load_stm32_framework_props_resolves_variant_h_template() { - let tmp = tempfile::TempDir::new().unwrap(); - let boards_txt = tmp.path().join("boards.txt"); - fs::write( - &boards_txt, - "\ -GenF1.build.variant_h=variant_{build.board}.h -GenF1.menu.pnum.MAPLEMINI_F103CB.build.board=MAPLEMINI_F103CB -GenF1.menu.pnum.MAPLEMINI_F103CB.build.variant=STM32F1xx/F103C8T_F103CB(T-U) -giga.menu.target_core.cm7.build.variant=GIGA -giga.build.extra_ldflags=-DCM4_BINARY_START=0x08180000 -", - ) - .unwrap(); - - let maple = load_stm32_framework_props("MAPLEMINI_F103CB", &boards_txt).unwrap(); - assert_eq!( - maple.get("variant_h").map(String::as_str), - Some("variant_MAPLEMINI_F103CB.h") - ); - - let giga = load_stm32_framework_props("GIGA", &boards_txt).unwrap(); - assert_eq!( - giga.get("extra_ldflags").map(String::as_str), - Some("-DCM4_BINARY_START=0x08180000") - ); - } - - #[test] - fn test_stm32_property_scopes_include_parent_menu_levels() { - assert_eq!( - stm32_property_scopes("giga.menu.target_core.cm7"), - vec!["giga", "giga.menu.target_core.cm7"] - ); - assert_eq!( - stm32_property_scopes("foo.menu.cpu.atmega328.menu.speed.fast"), - vec![ - "foo", - "foo.menu.cpu.atmega328", - "foo.menu.cpu.atmega328.menu.speed.fast" - ] - ); - } - - #[test] - fn test_select_variant_files_prefers_framework_header() { - let tmp = tempfile::TempDir::new().unwrap(); - fs::write(tmp.path().join("variant_MAPLEMINI_F103CB.h"), "").unwrap(); - fs::write(tmp.path().join("variant_MAPLEMINI_F103CB.cpp"), "").unwrap(); - fs::write(tmp.path().join("variant_generic.cpp"), "").unwrap(); - - let selected = select_variant_files( - tmp.path(), - "STM32F1xx/F103C8T_F103CB(T-U)", - Some("variant_MAPLEMINI_F103CB.h"), - ); - - assert_eq!(selected.header, "variant_MAPLEMINI_F103CB.h"); - assert_eq!( - selected.source_stem.as_deref(), - Some("variant_maplemini_f103cb") - ); - } -} diff --git a/crates/fbuild-build/src/stm32/orchestrator/README.md b/crates/fbuild-build/src/stm32/orchestrator/README.md new file mode 100644 index 00000000..ec072232 --- /dev/null +++ b/crates/fbuild-build/src/stm32/orchestrator/README.md @@ -0,0 +1,21 @@ +# STM32 orchestrator + +`Stm32Orchestrator` plus the support modules that drive the STM32 build. + +The orchestrator was split into a module directory to keep every `.rs` file +under the 1000-LOC CI gate. The public API is unchanged — re-exports from +`stm32/mod.rs` and the `stm32::orchestrator::{Stm32Orchestrator, create, +is_stm32_project}` paths still resolve as before. + +## Submodules + +- `arduino_mbed.rs` — Arduino mbed-core build path (GIGA, PORTENTA_H7_M7, + NICLA_VISION, OPTA, GENERIC_STM32H747_M4). +- `framework_props.rs` — STM32duino `boards.txt` parser with `menu.*` scope + resolution and `{build.*}` template substitution. +- `includes.rs` — CMSIS/HAL include-path assembly and small dedupe / define + helpers shared by both build paths. +- `variant_files.rs` — pick the right `variant_*.{h,cpp}` and + `PeripheralPins_*.c` so the linker sees one variant per build. +- `mod.rs` — primary STM32duino flow (phases 1-10) and the `Stm32Orchestrator` + trait impl. Holds the unit tests. diff --git a/crates/fbuild-build/src/stm32/orchestrator/arduino_mbed.rs b/crates/fbuild-build/src/stm32/orchestrator/arduino_mbed.rs new file mode 100644 index 00000000..fd027d58 --- /dev/null +++ b/crates/fbuild-build/src/stm32/orchestrator/arduino_mbed.rs @@ -0,0 +1,278 @@ +//! Arduino-mbed build path for STM32-based mbed variants (GIGA, PORTENTA, ...). +//! +//! Extracted from `orchestrator.rs` (see [`super`]). The STM32duino core +//! doesn't cover these boards — Arduino ships its own pre-built mbed library +//! plus per-variant `cflags.txt`, `cxxflags.txt`, and `ldflags.txt` files +//! that we replay verbatim. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use fbuild_core::Result; +use fbuild_packages::Toolchain; + +use crate::compile_database::TargetArchitecture; +use crate::generic_arm::{ArmCompiler, ArmLinker}; +use crate::pipeline; +use crate::source_scanner::SourceCollection; +use crate::{BuildParams, BuildResult, SourceScanner}; + +use super::framework_props::load_stm32_framework_props; +use super::includes::{apply_define_flags, dedupe_paths, dedupe_strings}; + +pub(super) fn is_arduino_mbed_stm32_variant(variant: &str) -> bool { + matches!( + variant, + "GIGA" | "PORTENTA_H7_M7" | "GENERIC_STM32H747_M4" | "NICLA_VISION" | "OPTA" + ) +} + +pub(super) fn build_arduino_mbed_stm32( + params: &BuildParams, + ctx: pipeline::BuildContext, + toolchain: &fbuild_packages::toolchain::ArmToolchain, + start: Instant, +) -> Result { + // Compute eh_frame strip policy once per build (FastLED/fbuild#244). + let eh_frame_policy = + crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); + + let framework = fbuild_packages::library::ArduinoMbedCore::new(¶ms.project_dir); + let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + tracing::info!("Arduino mbed core at {}", framework_dir.display()); + + let core_dir = framework.get_core_dir("arduino"); + let variant_dir = framework.get_variant_dir(&ctx.board.variant); + + let scanner = SourceScanner::new(&ctx.src_dir, &ctx.src_build_dir); + let sources = SourceCollection { + sketch_sources: scanner.scan_sketch_sources_filtered(ctx.source_filter.as_deref())?, + core_sources: framework.get_core_sources(), + variant_sources: framework.get_variant_sources(&ctx.board.variant), + headers: Vec::new(), + }; + + tracing::info!( + "sources: {} sketch, {} core, {} variant", + sources.sketch_sources.len(), + sources.core_sources.len(), + sources.variant_sources.len(), + ); + + let mut defines = ctx.board.get_defines(); + defines.insert("ARDUINO".to_string(), "10810".to_string()); + defines.insert("ARDUINO_ARCH_MBED".to_string(), "1".to_string()); + for token in fbuild_core::shell_split::split( + &framework.read_variant_file(&ctx.board.variant, "defines.txt"), + ) { + if let Some(def) = token.strip_prefix("-D") { + if let Some((key, val)) = def.split_once('=') { + defines.insert(key.to_string(), val.to_string()); + } else { + defines.insert(def.to_string(), "1".to_string()); + } + } + } + let board_ldflags = load_stm32_framework_props(&ctx.board.variant, &framework.get_boards_txt()) + .and_then(|props| props.get("extra_ldflags").cloned()) + .map(|flags| fbuild_core::shell_split::split(&flags)) + .unwrap_or_default(); + apply_define_flags(&board_ldflags, &mut defines); + + let mut include_dirs = vec![ + core_dir.clone(), + core_dir.join("api").join("deprecated"), + core_dir.join("api").join("deprecated-avr-comp"), + variant_dir.clone(), + ]; + include_dirs.extend(framework.get_variant_includes(&ctx.board.variant)); + include_dirs.push(ctx.src_dir.clone()); + pipeline::discover_project_includes(¶ms.project_dir, &mut include_dirs); + dedupe_paths(&mut include_dirs); + + let mut variant_ldflags = fbuild_core::shell_split::split( + &framework.read_variant_file(&ctx.board.variant, "ldflags.txt"), + ); + variant_ldflags.extend(board_ldflags.iter().cloned()); + dedupe_strings(&mut variant_ldflags); + let linker_script = preprocess_linker_script( + toolchain.get_gxx_path(), + &variant_dir, + &ctx.board.variant, + &ctx.build_dir, + &variant_ldflags, + )?; + + let mcu_config = build_arduino_mbed_mcu_config( + &framework, + &ctx.board.variant, + framework.get_mbed_lib(&ctx.board.variant), + &board_ldflags, + ); + + let compiler = ArmCompiler::new( + toolchain.get_gcc_path(), + toolchain.get_gxx_path(), + &ctx.board.mcu, + &ctx.board.f_cpu, + defines, + include_dirs.clone(), + mcu_config.clone(), + params.profile, + params.verbose, + ) + .with_build_unflags(ctx.build_unflags.clone()) + .with_eh_frame_policy(eh_frame_policy); + + let linker = ArmLinker::new( + toolchain.get_gcc_path(), + toolchain.get_ar_path(), + toolchain.get_objcopy_path(), + toolchain.get_size_path(), + linker_script, + mcu_config, + params.profile, + ctx.board.max_flash, + ctx.board.max_ram, + params.verbose, + ); + + let gcc_path = toolchain.get_gcc_path(); + let gxx_path = toolchain.get_gxx_path(); + let ar_path = toolchain.get_ar_path(); + let gcc_ar_path = toolchain.get_gcc_ar_path(); + let c_flags = crate::compiler::Compiler::c_flags(&compiler); + let cpp_flags = crate::compiler::Compiler::cpp_flags(&compiler); + let lib_ar_path = pipeline::pick_archiver(&ar_path, &gcc_ar_path, &c_flags, &cpp_flags); + let lib_env = pipeline::LibraryBuildEnv { + gcc_path: &gcc_path, + gxx_path: &gxx_path, + ar_path: lib_ar_path, + c_flags: &c_flags, + cpp_flags: &cpp_flags, + include_dirs: &include_dirs, + verbose: params.verbose, + jobs: crate::parallel::effective_jobs(params.jobs), + compiler_cache: None, + }; + + pipeline::run_sequential_build_with_libs( + &compiler, + &linker, + ctx, + params, + &sources, + &[], + Some(&lib_env), + TargetArchitecture::Arm, + "STM32", + start, + ) +} + +fn build_arduino_mbed_mcu_config( + framework: &fbuild_packages::library::ArduinoMbedCore, + variant_name: &str, + mbed_lib: PathBuf, + board_ldflags: &[String], +) -> crate::generic_arm::ArmMcuConfig { + let cflags = + fbuild_core::shell_split::split(&framework.read_variant_file(variant_name, "cflags.txt")); + let cxxflags = + fbuild_core::shell_split::split(&framework.read_variant_file(variant_name, "cxxflags.txt")); + let ldflags = + fbuild_core::shell_split::split(&framework.read_variant_file(variant_name, "ldflags.txt")); + + let mut common_flags: Vec = cflags + .into_iter() + .filter(|f| f != "-c" && !f.starts_with("-std=")) + .collect(); + common_flags.push("-nostdlib".to_string()); + dedupe_strings(&mut common_flags); + + let c_flags = vec!["-std=gnu11".to_string()]; + let mut cxx_only = cxxflags + .into_iter() + .filter(|f| f != "-c" && !common_flags.contains(f)) + .collect::>(); + dedupe_strings(&mut cxx_only); + + let mut linker_flags: Vec = ldflags + .into_iter() + .filter(|f| !f.starts_with("-D")) + .collect(); + linker_flags.extend( + board_ldflags + .iter() + .filter(|f| !f.starts_with("-D")) + .cloned(), + ); + linker_flags.push("--specs=nano.specs".to_string()); + linker_flags.push("--specs=nosys.specs".to_string()); + dedupe_strings(&mut linker_flags); + + crate::generic_arm::ArmMcuConfig { + name: "ArduinoCore-mbed".to_string(), + description: format!("Arduino mbed variant {variant_name}"), + architecture: "arm-cortex-m".to_string(), + compiler_flags: crate::compiler::CompilerFlags { + common: common_flags, + c: c_flags, + cxx: cxx_only, + }, + linker_flags, + linker_libs: vec![ + "-Wl,--whole-archive".to_string(), + mbed_lib.to_string_lossy().to_string(), + "-Wl,--no-whole-archive".to_string(), + "-Wl,--start-group".to_string(), + "-lstdc++".to_string(), + "-lsupc++".to_string(), + "-lm".to_string(), + "-lc".to_string(), + "-lgcc".to_string(), + "-lnosys".to_string(), + "-Wl,--end-group".to_string(), + ], + objcopy: crate::compiler::ObjcopyConfig { + output_format: "binary".to_string(), + remove_sections: Vec::new(), + }, + profiles: HashMap::new(), + defines: Vec::new(), + } +} + +fn preprocess_linker_script( + gxx_path: PathBuf, + variant_dir: &Path, + variant_name: &str, + build_dir: &Path, + ldflags: &[String], +) -> Result { + let input = variant_dir.join("linker_script.ld"); + let output = build_dir.join("cpp.linker_script.ld"); + let mut args = vec![ + gxx_path.to_string_lossy().to_string(), + "-E".to_string(), + "-P".to_string(), + "-x".to_string(), + "c".to_string(), + ]; + args.extend(ldflags.iter().filter(|f| f.starts_with("-D")).cloned()); + args.push(input.to_string_lossy().to_string()); + args.push("-o".to_string()); + args.push(output.to_string_lossy().to_string()); + + let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + let result = fbuild_core::subprocess::run_command(&args_ref, None, None, None)?; + if !result.success() { + return Err(fbuild_core::FbuildError::BuildFailed(format!( + "failed to preprocess Arduino mbed linker script for {}:\n{}", + variant_name, result.stderr + ))); + } + + Ok(output) +} diff --git a/crates/fbuild-build/src/stm32/orchestrator/framework_props.rs b/crates/fbuild-build/src/stm32/orchestrator/framework_props.rs new file mode 100644 index 00000000..e5a7c188 --- /dev/null +++ b/crates/fbuild-build/src/stm32/orchestrator/framework_props.rs @@ -0,0 +1,108 @@ +//! STM32duino `boards.txt` property loader. +//! +//! Extracted from `orchestrator.rs` (see [`super`]). +//! +//! Parses the Arduino-flavored `boards.txt` shipped with STM32duino so the +//! orchestrator can resolve a board id or variant path to a flat property map +//! (variant header, extra ldflags, etc.) while respecting parent `menu.*` scopes. + +use std::collections::HashMap; +use std::path::Path; + +pub(super) fn load_stm32_framework_props( + board_or_variant: &str, + boards_txt: &Path, +) -> Option> { + let content = std::fs::read_to_string(boards_txt).ok()?; + let preferred_key = if board_or_variant.contains('/') { + ".build.variant" + } else { + ".build.board" + }; + let prefix = find_stm32_prop_prefix(&content, preferred_key, board_or_variant) + .or_else(|| find_stm32_prop_prefix(&content, ".build.variant", board_or_variant))?; + + let mut props = HashMap::new(); + for scope in stm32_property_scopes(&prefix) { + let line_prefix = format!("{scope}."); + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + let Some(rest) = trimmed.strip_prefix(&line_prefix) else { + continue; + }; + let Some((key, value)) = rest.split_once('=') else { + continue; + }; + let key = key.trim(); + let normalized = key + .strip_prefix("build.") + .or_else(|| key.strip_prefix("upload.")) + .unwrap_or(key); + props.insert(normalized.to_string(), value.trim().to_string()); + if normalized != key { + props.insert(key.to_string(), value.trim().to_string()); + } + } + } + + let substitutions = [ + ( + "{build.board}", + props.get("board").cloned().unwrap_or_default(), + ), + ( + "{build.variant}", + props.get("variant").cloned().unwrap_or_default(), + ), + ]; + for value in props.values_mut() { + for (needle, replacement) in &substitutions { + if !replacement.is_empty() { + *value = value.replace(needle, replacement); + } + } + } + + Some(props) +} + +fn find_stm32_prop_prefix(content: &str, suffix: &str, value: &str) -> Option { + content.lines().find_map(|line| { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + return None; + } + let (key, actual) = trimmed.split_once('=')?; + if key.ends_with(suffix) && actual.trim() == value { + Some(key.trim_end_matches(suffix).to_string()) + } else { + None + } + }) +} + +pub(super) fn stm32_property_scopes(prefix: &str) -> Vec { + let segments = prefix.split('.').collect::>(); + if segments.is_empty() { + return Vec::new(); + } + + let mut scopes = vec![segments[0].to_string()]; + let mut idx = 1; + while idx + 2 < segments.len() { + if segments[idx] != "menu" { + break; + } + idx += 3; + scopes.push(segments[..idx].join(".")); + } + + if scopes.last().map_or(true, |scope| scope != prefix) { + scopes.push(prefix.to_string()); + } + + scopes +} diff --git a/crates/fbuild-build/src/stm32/orchestrator/includes.rs b/crates/fbuild-build/src/stm32/orchestrator/includes.rs new file mode 100644 index 00000000..ad8ef2e6 --- /dev/null +++ b/crates/fbuild-build/src/stm32/orchestrator/includes.rs @@ -0,0 +1,112 @@ +//! STM32 include-path and flag-handling helpers. +//! +//! Extracted from `orchestrator.rs` (see [`super`]). + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +/// Add STM32duino system include directories for CMSIS and HAL. +/// +/// The STM32duino core bundles CMSIS and HAL drivers under `system/`: +/// - `system/Drivers/CMSIS/Device/ST//Include/` — MCU device headers +/// - `system/Drivers/CMSIS/Core/Include/` — ARM CMSIS core headers +/// - `system/Drivers/_HAL_Driver/Inc/` — STM32 HAL headers +/// - `system//` — System startup headers +pub(super) fn add_stm32_system_includes( + system_dir: &Path, + family: &str, + include_dirs: &mut Vec, +) { + let drivers = system_dir.join("Drivers"); + + // CMSIS Device headers (stm32f1xx.h, stm32f103xb.h, etc.) + let cmsis_device = drivers + .join("CMSIS") + .join("Device") + .join("ST") + .join(family) + .join("Include"); + if cmsis_device.exists() { + include_dirs.push(cmsis_device); + } + + // CMSIS Core headers (core_cm3.h, core_cm4.h, etc.) + let cmsis_core = drivers.join("CMSIS").join("Core").join("Include"); + if cmsis_core.exists() { + include_dirs.push(cmsis_core); + } + + // CMSIS startup templates (startup_stm32f103xb.s, etc.) + let cmsis_startup = drivers + .join("CMSIS") + .join("Device") + .join("ST") + .join(family) + .join("Source") + .join("Templates") + .join("gcc"); + if cmsis_startup.exists() { + include_dirs.push(cmsis_startup); + } + + // HAL Driver headers (stm32f1xx_hal.h, etc.) + let hal_driver = drivers.join(format!("{family}_HAL_Driver")); + let hal_inc = hal_driver.join("Inc"); + if hal_inc.exists() { + include_dirs.push(hal_inc); + } + // HAL Driver sources — SrcWrapper's stm32yyxx_hal.c does #include "stm32f1xx_hal.c" + let hal_src = hal_driver.join("Src"); + if hal_src.exists() { + include_dirs.push(hal_src); + } + + // System family directory (startup and system config headers) + let system_family = system_dir.join(family); + if system_family.exists() { + include_dirs.push(system_family); + } +} + +pub(super) fn dedupe_paths(paths: &mut Vec) { + let mut seen = HashSet::new(); + paths.retain(|path| seen.insert(path.clone())); +} + +pub(super) fn dedupe_strings(flags: &mut Vec) { + let mut seen = HashSet::new(); + flags.retain(|flag| seen.insert(flag.clone())); +} + +pub(super) fn apply_define_flags(flags: &[String], defines: &mut HashMap) { + for flag in flags { + if let Some(def) = flag.strip_prefix("-D") { + if let Some((key, val)) = def.split_once('=') { + defines.insert(key.to_string(), val.to_string()); + } else { + defines.insert(def.to_string(), "1".to_string()); + } + } + } +} + +/// Derive the STM32duino generic board define from the MCU name. +/// +/// `stm32f103c8t6` → `GENERIC_F103C8TX` +/// `stm32f411ceu6` → `GENERIC_F411CEUX` +/// +/// Pattern: strip `stm32` prefix, uppercase, replace last char with `X`. +pub(super) fn stm32_generic_board_define(mcu: &str) -> String { + let suffix = mcu + .to_lowercase() + .strip_prefix("stm32") + .unwrap_or(&mcu.to_lowercase()) + .to_uppercase(); + // Replace last character (pin-count digit) with X + let mut chars: Vec = suffix.chars().collect(); + if let Some(last) = chars.last_mut() { + *last = 'X'; + } + let trimmed: String = chars.into_iter().collect(); + format!("GENERIC_{trimmed}") +} diff --git a/crates/fbuild-build/src/stm32/orchestrator/mod.rs b/crates/fbuild-build/src/stm32/orchestrator/mod.rs new file mode 100644 index 00000000..7dc2f244 --- /dev/null +++ b/crates/fbuild-build/src/stm32/orchestrator/mod.rs @@ -0,0 +1,425 @@ +//! STM32 build orchestrator — wires together config, packages, compiler, linker. +//! +//! Build phases: +//! 1. Parse platformio.ini +//! 2. Load board config (bluepill_f103c8, blackpill_f411ce, nucleo_*, etc.) +//! 3. Ensure ARM GCC toolchain +//! 4. Ensure STM32duino cores (Arduino_Core_STM32) +//! 5. Setup build directories +//! 6. Scan source files +//! 7. Compile core + variant sources +//! 8. Compile sketch sources +//! 9. Link (with linker script from variant dir) +//! 10. Convert to hex + report size +//! +//! Module layout (refactored to keep each .rs file under the 1000-LOC gate): +//! - [`arduino_mbed`] — Arduino mbed-core build path (GIGA, PORTENTA, ...) +//! - [`framework_props`] — STM32duino `boards.txt` parser +//! - [`includes`] — include-path/define helpers and small shared utilities +//! - [`variant_files`] — variant_*.{h,cpp} / PeripheralPins_*.c selection + +mod arduino_mbed; +mod framework_props; +mod includes; +mod variant_files; + +use std::path::Path; +use std::time::Instant; + +use fbuild_core::{Platform, Result}; +use fbuild_packages::{Framework, Toolchain}; + +use crate::compile_database::TargetArchitecture; +use crate::framework_libs::{ + library_select_kv_store, resolve_framework_library_sources, + resolve_framework_library_sources_cached, +}; +use crate::generic_arm::{ArmCompiler, ArmLinker}; +use crate::pipeline; +use crate::{BuildOrchestrator, BuildParams, BuildResult, SourceScanner}; + +use self::arduino_mbed::{build_arduino_mbed_stm32, is_arduino_mbed_stm32_variant}; +use self::framework_props::load_stm32_framework_props; +use self::includes::{add_stm32_system_includes, stm32_generic_board_define}; +use self::variant_files::{keep_variant_source, select_variant_files}; + +/// STM32 platform build orchestrator. +pub struct Stm32Orchestrator; + +impl BuildOrchestrator for Stm32Orchestrator { + fn platform(&self) -> Platform { + Platform::Ststm32 + } + + fn build(&self, params: &BuildParams) -> Result { + let start = Instant::now(); + + // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags + let mut ctx = pipeline::BuildContext::new(params)?; + + // Compute eh_frame strip policy once per build (FastLED/fbuild#244). + let eh_frame_policy = + crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); + + // 3. Ensure ARM GCC toolchain + let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + tracing::info!("arm-gcc toolchain at {}", toolchain_dir.display()); + + pipeline::log_toolchain_version( + &toolchain.get_gcc_path(), + "arm-none-eabi-gcc", + &mut ctx.build_log, + ); + + if is_arduino_mbed_stm32_variant(&ctx.board.variant) { + return build_arduino_mbed_stm32(params, ctx, &toolchain, start); + } + + // 4. Ensure STM32duino cores + let framework = fbuild_packages::library::Stm32Cores::new(¶ms.project_dir); + let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + tracing::info!("STM32 cores at {}", framework_dir.display()); + + // 5. Scan sources (core + variant) + // STM32duino uses "arduino" as its core directory name, even though + // the board JSON says core = "stm32". Map it here. + let core_dir = framework.get_core_dir("arduino"); + let framework_props = + load_stm32_framework_props(&ctx.board.variant, &framework.get_boards_txt()); + let resolved_variant = framework_props + .as_ref() + .and_then(|props| props.get("variant").cloned()) + .unwrap_or_else(|| ctx.board.variant.clone()); + let variant_dir = framework.get_variant_dir(&resolved_variant); + let selected_variant = select_variant_files( + &variant_dir, + &resolved_variant, + framework_props + .as_ref() + .and_then(|props| props.get("variant_h").map(String::as_str)) + .or(ctx.board.variant_h.as_deref()), + ); + + let scanner = SourceScanner::new(&ctx.src_dir, &ctx.src_build_dir); + // Scan core + variant, but pass None for variant — we'll filter variant + // sources manually because the variant dir contains files for multiple + // board variants (MALYAN, AFROFLIGHT, etc.) and startup files that + // conflict with the generic one in cores/arduino/stm32/. + let mut sources = + scanner.scan_all_filtered(Some(&core_dir), None, ctx.source_filter.as_deref())?; + sources.variant_sources = scanner + .scan_variant_sources(&variant_dir) + .into_iter() + .filter(|p| keep_variant_source(p, &selected_variant)) + .collect(); + + // SrcWrapper is a core library in STM32duino — its sources must be + // compiled alongside the Arduino core (HAL wrappers, syscalls, etc.) + // scan_core_sources is recursive, so one call covers all subdirs. + let libs_dir = framework.get_libraries_dir(); + let srcwrapper_src = libs_dir.join("SrcWrapper").join("src"); + if srcwrapper_src.exists() { + sources + .core_sources + .extend(scanner.scan_core_sources(&srcwrapper_src)); + } + + // Walk Arduino_Core_STM32's libraries/ (SPI, Wire, EEPROM, ...) and + // pull in any the sketch transitively #includes. Without this, sketches + // that include fail with "No such file or directory" because + // STM32duino only exposes bundled libraries via this framework-level + // discovery (PlatformIO's LDF does the same for `framework = arduino`). + let framework_libs = framework.get_framework_libraries(); + // WHY: STM32duino targets every Cortex-M family from M0 (F0xx) up + // through M7 (H7xx) but the toolchain triple is constant + // (`arm-none-eabi`). The cache key already includes + // `framework_install_path` + `framework_version`, so per-MCU drift + // is handled there — this string only needs to disambiguate stm32 + // from teensy etc. so cross-platform key collisions are impossible. + let framework_info = fbuild_packages::Package::get_info(&framework); + let framework_library_sources = match library_select_kv_store() { + Some(store) => { + let key_inputs = fbuild_library_select::cache::CacheKeyInputs { + toolchain_triple: "stm32-arm-none-eabi", + framework_install_path: &framework_info.install_path, + framework_version: &framework_info.version, + }; + resolve_framework_library_sources_cached( + &framework_libs, + ¶ms.project_dir, + &ctx.src_dir, + &key_inputs, + store, + ) + } + None => resolve_framework_library_sources( + &framework_libs, + ¶ms.project_dir, + &ctx.src_dir, + ), + }; + if !framework_library_sources.is_empty() { + tracing::info!( + "STM32 framework library sources added: {}", + framework_library_sources.len() + ); + sources.core_sources.extend(framework_library_sources); + } + + tracing::info!( + "sources: {} sketch, {} core, {} variant", + sources.sketch_sources.len(), + sources.core_sources.len(), + sources.variant_sources.len(), + ); + + // 6. Build include dirs + compiler + // Extract MCU family from variant path (e.g. "STM32F1xx" from "STM32F1xx/F103C8T_...") + let family = resolved_variant.split('/').next().unwrap_or("STM32F1xx"); + + let mut mcu_config = + super::mcu_config::get_stm32_config_for_mcu(&ctx.board.mcu.to_lowercase())?; + // STM32duino linker scripts reference LD_MAX_SIZE, LD_MAX_DATA_SIZE, and + // LD_FLASH_OFFSET as symbols. Provide them via --defsym from board config. + let max_flash = ctx.board.max_flash.unwrap_or(65536); + let max_ram = ctx.board.max_ram.unwrap_or(20480); + mcu_config + .linker_flags + .push(format!("-Wl,--defsym=LD_MAX_SIZE={max_flash}")); + mcu_config + .linker_flags + .push(format!("-Wl,--defsym=LD_MAX_DATA_SIZE={max_ram}")); + mcu_config + .linker_flags + .push("-Wl,--defsym=LD_FLASH_OFFSET=0".to_string()); + let mut defines = ctx.board.get_defines(); + defines.extend(mcu_config.defines_map()); + if let Some(board_define) = framework_props + .as_ref() + .and_then(|props| props.get("board")) + { + defines.insert(format!("ARDUINO_{board_define}"), "1".to_string()); + } + // STM32duino's stm32_def.h checks for STM32YYxx (e.g. STM32F1xx). The board + // JSON extra_flags may only have STM32F1, so ensure the full family define. + defines.insert(family.to_string(), "1".to_string()); + // STM32duino variant sources are guarded by ARDUINO_GENERIC_ defines. + // Derive from the MCU name: stm32f103c8t6 → ARDUINO_GENERIC_F103C8TX + let generic_board = stm32_generic_board_define(&ctx.board.mcu); + defines.insert(format!("ARDUINO_{generic_board}"), "1".to_string()); + // STM32duino requires these defines for HAL/LL and variant header resolution. + defines.insert("USE_HAL_DRIVER".to_string(), "1".to_string()); + defines.insert("USE_FULL_LL_DRIVER".to_string(), "1".to_string()); + defines.insert( + "VARIANT_H".to_string(), + format!("\\\"{}\\\"", selected_variant.header), + ); + // UART HAL module is disabled by default in stm32yyxx_hal_conf.h — enable it + // so WSerial.h can create the Serial instance. + defines.insert("HAL_UART_MODULE_ENABLED".to_string(), "1".to_string()); + defines.insert("HAL_PCD_MODULE_ENABLED".to_string(), "1".to_string()); + + // Build include dirs manually (can't use get_include_paths because + // STM32duino core dir is "arduino", not the board JSON's "stm32") + let mut include_dirs = vec![core_dir.clone(), variant_dir.clone()]; + // Core subdirectories (AVR compat, STM32 HAL wrapper) + include_dirs.push(core_dir.join("avr")); + include_dirs.push(core_dir.join("stm32")); + // SrcWrapper include dirs (clock.h, analog.h, LL/stm32yyxx_ll_*.h, etc.) + let srcwrapper_inc = libs_dir.join("SrcWrapper").join("inc"); + if srcwrapper_inc.exists() { + include_dirs.push(srcwrapper_inc.clone()); + let ll_inc = srcwrapper_inc.join("LL"); + if ll_inc.exists() { + include_dirs.push(ll_inc); + } + } + include_dirs.push(ctx.src_dir.clone()); + pipeline::discover_project_includes(¶ms.project_dir, &mut include_dirs); + // Bundled framework library headers (SPI, Wire, EEPROM, ...) so + // sketches can `#include ` etc. + include_dirs.extend(framework.get_framework_library_include_dirs()); + + // STM32duino system includes (CMSIS device, HAL drivers, etc.) + let system_dir = framework.get_system_dir(); + add_stm32_system_includes(&system_dir, family, &mut include_dirs); + + // CMSIS Core includes (core_cm3.h, core_cm4.h, etc.) — not bundled in STM32duino + let cmsis = fbuild_packages::library::CmsisFramework::new(¶ms.project_dir); + let _cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis)?; + tracing::info!("CMSIS framework installed"); + include_dirs.push(cmsis.get_core_include_dir()); + + // Toolchain sysroot includes (ARM CMSIS headers, etc.) + include_dirs.extend(toolchain.get_include_dirs()); + + let compiler = ArmCompiler::new( + toolchain.get_gcc_path(), + toolchain.get_gxx_path(), + &ctx.board.mcu, + &ctx.board.f_cpu, + defines, + include_dirs.clone(), + mcu_config.clone(), + params.profile, + params.verbose, + ) + .with_build_unflags(ctx.build_unflags.clone()) + .with_eh_frame_policy(eh_frame_policy); + + // 7. Create linker (linker script from variant dir) + let linker_script = match ctx.board.ldscript.as_deref() { + Some(name) => variant_dir.join(name), + None => framework.get_linker_script(&resolved_variant), + }; + let linker = ArmLinker::new( + toolchain.get_gcc_path(), + toolchain.get_ar_path(), + toolchain.get_objcopy_path(), + toolchain.get_size_path(), + linker_script, + mcu_config, + params.profile, + ctx.board.max_flash, + ctx.board.max_ram, + params.verbose, + ); + + // 8. Build LibraryBuildEnv for project-as-library compilation + let gcc_path = toolchain.get_gcc_path(); + let gxx_path = toolchain.get_gxx_path(); + let ar_path = toolchain.get_ar_path(); + let gcc_ar_path = toolchain.get_gcc_ar_path(); + let c_flags = crate::compiler::Compiler::c_flags(&compiler); + let cpp_flags = crate::compiler::Compiler::cpp_flags(&compiler); + // Use gcc-ar for LTO archives so the linker-plugin index is written. + let lib_ar_path = pipeline::pick_archiver(&ar_path, &gcc_ar_path, &c_flags, &cpp_flags); + let lib_env = pipeline::LibraryBuildEnv { + gcc_path: &gcc_path, + gxx_path: &gxx_path, + ar_path: lib_ar_path, + c_flags: &c_flags, + cpp_flags: &cpp_flags, + include_dirs: &include_dirs, + verbose: params.verbose, + jobs: crate::parallel::effective_jobs(params.jobs), + compiler_cache: None, + }; + + // 9. Run shared sequential build pipeline + pipeline::run_sequential_build_with_libs( + &compiler, + &linker, + ctx, + params, + &sources, + &[], + Some(&lib_env), + TargetArchitecture::Arm, + "STM32", + start, + ) + } +} + +/// Create an STM32 orchestrator. +pub fn create() -> Box { + Box::new(Stm32Orchestrator) +} + +/// Check if a project is configured for STM32. +pub fn is_stm32_project(project_dir: &Path, env_name: &str) -> bool { + crate::pipeline::is_platform_project(project_dir, env_name, fbuild_core::Platform::Ststm32) +} + +#[cfg(test)] +mod tests { + use super::framework_props::{load_stm32_framework_props, stm32_property_scopes}; + use super::variant_files::select_variant_files; + use super::*; + use std::fs; + + #[test] + fn test_stm32_orchestrator_platform() { + let orch = Stm32Orchestrator; + assert_eq!(orch.platform(), Platform::Ststm32); + } + + #[test] + fn test_is_stm32_project() { + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::write( + tmp.path().join("platformio.ini"), + "[env:bluepill]\nplatform = ststm32\nboard = bluepill_f103c8\nframework = arduino\n", + ) + .unwrap(); + assert!(is_stm32_project(tmp.path(), "bluepill")); + assert!(!is_stm32_project(tmp.path(), "uno")); + } + + #[test] + fn test_load_stm32_framework_props_resolves_variant_h_template() { + let tmp = tempfile::TempDir::new().unwrap(); + let boards_txt = tmp.path().join("boards.txt"); + fs::write( + &boards_txt, + "\ +GenF1.build.variant_h=variant_{build.board}.h +GenF1.menu.pnum.MAPLEMINI_F103CB.build.board=MAPLEMINI_F103CB +GenF1.menu.pnum.MAPLEMINI_F103CB.build.variant=STM32F1xx/F103C8T_F103CB(T-U) +giga.menu.target_core.cm7.build.variant=GIGA +giga.build.extra_ldflags=-DCM4_BINARY_START=0x08180000 +", + ) + .unwrap(); + + let maple = load_stm32_framework_props("MAPLEMINI_F103CB", &boards_txt).unwrap(); + assert_eq!( + maple.get("variant_h").map(String::as_str), + Some("variant_MAPLEMINI_F103CB.h") + ); + + let giga = load_stm32_framework_props("GIGA", &boards_txt).unwrap(); + assert_eq!( + giga.get("extra_ldflags").map(String::as_str), + Some("-DCM4_BINARY_START=0x08180000") + ); + } + + #[test] + fn test_stm32_property_scopes_include_parent_menu_levels() { + assert_eq!( + stm32_property_scopes("giga.menu.target_core.cm7"), + vec!["giga", "giga.menu.target_core.cm7"] + ); + assert_eq!( + stm32_property_scopes("foo.menu.cpu.atmega328.menu.speed.fast"), + vec![ + "foo", + "foo.menu.cpu.atmega328", + "foo.menu.cpu.atmega328.menu.speed.fast" + ] + ); + } + + #[test] + fn test_select_variant_files_prefers_framework_header() { + let tmp = tempfile::TempDir::new().unwrap(); + fs::write(tmp.path().join("variant_MAPLEMINI_F103CB.h"), "").unwrap(); + fs::write(tmp.path().join("variant_MAPLEMINI_F103CB.cpp"), "").unwrap(); + fs::write(tmp.path().join("variant_generic.cpp"), "").unwrap(); + + let selected = select_variant_files( + tmp.path(), + "STM32F1xx/F103C8T_F103CB(T-U)", + Some("variant_MAPLEMINI_F103CB.h"), + ); + + assert_eq!(selected.header, "variant_MAPLEMINI_F103CB.h"); + assert_eq!( + selected.source_stem.as_deref(), + Some("variant_maplemini_f103cb") + ); + } +} diff --git a/crates/fbuild-build/src/stm32/orchestrator/variant_files.rs b/crates/fbuild-build/src/stm32/orchestrator/variant_files.rs new file mode 100644 index 00000000..0adb982c --- /dev/null +++ b/crates/fbuild-build/src/stm32/orchestrator/variant_files.rs @@ -0,0 +1,143 @@ +//! STM32duino variant file selection. +//! +//! Extracted from `orchestrator.rs` (see [`super`]). The variant directory +//! ships sources for many boards (MAPLEMINI, AFROFLIGHT, MALYAN, ...) plus +//! several startup files. We pick exactly one `variant_*.{h,cpp}` and one +//! `PeripheralPins_*.c` per build so the linker doesn't see duplicate symbols. + +use std::path::Path; + +#[derive(Debug, Clone)] +pub(super) struct SelectedVariantFiles { + pub(super) header: String, + pub(super) source_stem: Option, + pub(super) peripheral_stem: Option, +} + +pub(super) fn select_variant_files( + variant_dir: &Path, + variant_name: &str, + preferred_header: Option<&str>, +) -> SelectedVariantFiles { + let entries = std::fs::read_dir(variant_dir) + .ok() + .into_iter() + .flatten() + .flatten() + .filter_map(|entry| entry.file_name().into_string().ok()) + .collect::>(); + + let header = preferred_header + .and_then(|name| find_entry_case_insensitive(&entries, name)) + .or_else(|| pick_variant_file(&entries, variant_name, "variant_", ".h")) + .unwrap_or_else(|| "variant_generic.h".to_string()); + + let header_suffix = Path::new(&header) + .file_stem() + .and_then(|stem| stem.to_str()) + .and_then(|stem| stem.strip_prefix("variant_")); + let source_stem = header_suffix + .and_then(|suffix| { + find_entry_case_insensitive(&entries, &format!("variant_{suffix}.cpp")) + .map(|name| stem_lower(&name)) + }) + .or_else(|| { + pick_variant_file(&entries, variant_name, "variant_", ".cpp") + .map(|name| stem_lower(&name)) + }); + let peripheral_stem = header_suffix + .and_then(|suffix| { + find_entry_case_insensitive(&entries, &format!("PeripheralPins_{suffix}.c")) + .or_else(|| { + find_entry_case_insensitive(&entries, &format!("peripheralpins_{suffix}.c")) + }) + .map(|name| stem_lower(&name)) + }) + .or_else(|| { + pick_variant_file(&entries, variant_name, "peripheralpins_", ".c") + .map(|name| stem_lower(&name)) + }); + + SelectedVariantFiles { + header, + source_stem, + peripheral_stem, + } +} + +fn pick_variant_file( + entries: &[String], + variant_name: &str, + prefix: &str, + suffix: &str, +) -> Option { + let normalized = normalize_variant_name(variant_name); + let exact = format!("{prefix}{normalized}{suffix}"); + if let Some(name) = find_entry_case_insensitive(entries, &exact) { + return Some(name); + } + + let generic = format!("{prefix}generic{suffix}"); + if let Some(name) = find_entry_case_insensitive(entries, &generic) { + return Some(name); + } + + let mut matches = entries + .iter() + .filter(|name| { + let lower = name.to_lowercase(); + lower.starts_with(prefix) && lower.ends_with(suffix) + }) + .cloned() + .collect::>(); + matches.sort_by_key(|name| name.to_lowercase()); + matches.into_iter().next() +} + +fn find_entry_case_insensitive(entries: &[String], target: &str) -> Option { + entries + .iter() + .find(|name| name.eq_ignore_ascii_case(target)) + .cloned() +} + +pub(super) fn keep_variant_source(path: &Path, selected: &SelectedVariantFiles) -> bool { + let name = path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_lowercase(); + let stem = stem_lower(&name); + + if name.starts_with("startup_") { + return false; + } + if name.starts_with("variant_") { + return selected + .source_stem + .as_ref() + .is_some_and(|wanted| &stem == wanted); + } + if name.starts_with("peripheralpins_") { + return selected + .peripheral_stem + .as_ref() + .is_some_and(|wanted| &stem == wanted); + } + + true +} + +fn normalize_variant_name(name: &str) -> String { + name.to_lowercase() + .replace(['/', '\\', '-', ' '], "_") + .replace("__", "_") +} + +fn stem_lower(name: &str) -> String { + Path::new(name) + .file_stem() + .unwrap_or_default() + .to_string_lossy() + .to_lowercase() +} diff --git a/crates/fbuild-cli/src/README.md b/crates/fbuild-cli/src/README.md index 0c857851..7befdd14 100644 --- a/crates/fbuild-cli/src/README.md +++ b/crates/fbuild-cli/src/README.md @@ -2,6 +2,8 @@ ## Modules -- **`main.rs`** -- CLI entry point; Clap parser with subcommands (build, deploy, monitor, reset, purge, daemon, device, show, mcp, clang-tidy, iwyu, clang-query), dispatches to daemon client +- **`main.rs`** -- CLI entry point; spawns the larger-stack `fbuild-main` thread that hands control to `cli::async_main` +- **`cli/`** -- Clap parser, subcommand argument types, and per-subcommand handlers (see `cli/README.md`); split out of `main.rs` to keep each `.rs` under the 900 LOC gate +- **`mcp/`** -- MCP (Model Context Protocol) stdio JSON-RPC server (see `mcp/README.md`); split out of a flat `mcp.rs` to keep each `.rs` under the 900 LOC gate - **`daemon_client.rs`** -- `DaemonClient` HTTP client, request/response types, daemon spawn with stale binary detection, NDJSON streaming, compact status display -- **`mcp.rs`** -- MCP (Model Context Protocol) stdio JSON-RPC server; translates AI assistant tool/resource calls into daemon HTTP requests +- **`lib_select.rs`** -- diagnostic LDF-style library selection resolver (`fbuild lib-select`) diff --git a/crates/fbuild-cli/src/cli/README.md b/crates/fbuild-cli/src/cli/README.md new file mode 100644 index 00000000..cc6343ca --- /dev/null +++ b/crates/fbuild-cli/src/cli/README.md @@ -0,0 +1,27 @@ +# CLI + +Subcommand handlers and argument types for the `fbuild` binary. + +The original `main.rs` exceeded the 900 LOC gate (~3687 lines), so it was +split into this `cli/` module directory. `main.rs` now contains only the +process entry point and the larger-stack trampoline; everything else lives +here and is dispatched from `cli::async_main`. + +## Modules + +- **`mod.rs`** -- module declarations + re-exports of the public CLI surface +- **`args.rs`** -- Clap-derived `Cli` / `Commands` / `DaemonAction` / `DeviceAction` / `LnkAction`, `KNOWN_SUBCOMMANDS`, `rewrite_args`, `resolve_project_dir` +- **`dispatch.rs`** -- `async_main`: parse argv, set up tracing + Ctrl+C, fan each subcommand out to its handler +- **`monitor_parse.rs`** -- `ParsedMonitorFlags`, `parse_jobs`, `parse_monitor_flags`, `shell_tokenize` +- **`pio.rs`** -- PlatformIO passthrough: `find_pio`, `run_pio_command`, `pio_build`, `pio_deploy`, `pio_monitor` +- **`build.rs`** -- `run_build`, `normalize_path`, `open_in_browser` +- **`deploy.rs`** -- `run_deploy`, `run_test_emu`, `run_monitor`, deploy-route resolution (`CliDeployRoute`, `CliEmulatorKind`) +- **`compile_many.rs`** -- `run_compile_many` (#238) and the `fbuild ci` (#242) adapter helpers +- **`clang_tools.rs`** -- `run_clang_tool`, `run_iwyu`, IWYU cache key + output filtering +- **`purge.rs`** -- `run_purge`, `run_purge_gc`, size formatting, cache listing +- **`daemon_cmd.rs`** -- `run_daemon` and friends (status / restart / kill / locks / cache-stats / gc), process-management helpers +- **`device.rs`** -- `run_device` (list / status / lease / release / take) +- **`show.rs`** -- `run_show`, `show_daemon_logs` +- **`reset.rs`** -- `run_reset` +- **`lnk.rs`** -- `run_lnk` (pull / check / add) +- **`tests.rs`** -- unit tests for argument normalization and `fbuild ci` parsing diff --git a/crates/fbuild-cli/src/cli/args.rs b/crates/fbuild-cli/src/cli/args.rs new file mode 100644 index 00000000..0ce3967f --- /dev/null +++ b/crates/fbuild-cli/src/cli/args.rs @@ -0,0 +1,552 @@ +//! Clap-derived CLI parser, subcommand enums, and small argv helpers. +//! +//! Carved out of `main.rs` so the parser definition can live alongside +//! `KNOWN_SUBCOMMANDS` / `rewrite_args` / `resolve_project_dir` without +//! pulling in the handler bodies. + +use clap::{Parser, Subcommand}; + +use super::monitor_parse::parse_jobs; + +#[derive(Parser)] +#[command( + name = "fbuild", + version, + about = "PlatformIO-compatible embedded build tool" +)] +pub struct Cli { + #[command(subcommand)] + pub command: Option, + + /// Project directory (positional, for `fbuild `) + pub project_dir: Option, + + /// Target environment + #[arg(short = 'e', long)] + pub environment: Option, + + /// Verbose output + #[arg(short, long)] + pub verbose: bool, + + /// Serial port (e.g., COM5, /dev/ttyUSB0) + #[arg(short = 'p', long)] + pub port: Option, + + /// Clean build before deploy + #[arg(short = 'c', long)] + pub clean: bool, + + /// Monitor after deploy; optionally pass flags as a string + #[arg(long, num_args = 0..=1, default_missing_value = "")] + pub monitor: Option, + + /// Use PlatformIO compatibility mode + #[arg(long)] + pub platformio: bool, + + /// Monitor timeout in seconds + #[arg(long)] + pub timeout: Option, + + /// Halt monitor on error pattern + #[arg(long)] + pub halt_on_error: Option, + + /// Halt monitor on success pattern + #[arg(long)] + pub halt_on_success: Option, + + /// Expected output pattern for monitor + #[arg(long)] + pub expect: Option, +} + +#[derive(Subcommand)] +pub enum Commands { + /// Build firmware + Build { + project_dir: Option, + #[arg(short = 'e', long)] + environment: Option, + #[arg(short = 'c', long)] + clean: bool, + #[arg(short, long)] + verbose: bool, + #[arg(short = 'j', long, value_parser = parse_jobs)] + jobs: Option, + #[arg(long, group = "build_profile")] + quick: bool, + #[arg(long, group = "build_profile")] + release: bool, + #[arg(long)] + platformio: bool, + /// Verify daemon starts and environment resolves, but skip the actual build + #[arg(long)] + dry_run: bool, + /// Build target: 'compiledb' generates compile_commands.json without compiling + #[arg(short = 't', long, value_parser = ["compiledb"])] + target: Option, + /// Run per-symbol memory analysis after building; optionally write report to PATH + /// instead of streaming to console + #[arg(long, num_args = 0..=1, default_missing_value = "")] + symbol_analysis: Option, + /// Disable elapsed-time prefix on build output lines + #[arg(long)] + no_timestamp: bool, + /// Export build artifacts to a tooling-friendly directory + #[arg(long)] + output_dir: Option, + }, + /// Deploy firmware to device + Deploy { + project_dir: Option, + #[arg(short = 'e', long)] + environment: Option, + #[arg(short = 'p', long)] + port: Option, + #[arg(short = 'c', long)] + clean: bool, + /// Monitor after deploy; optionally pass flags as a string + /// e.g., --monitor="--timeout 60 --halt-on-success \"TEST PASSED\"" + #[arg(long, num_args = 0..=1, default_missing_value = "")] + monitor: Option, + #[arg(short, long)] + verbose: bool, + #[arg(long)] + platformio: bool, + #[arg(long)] + timeout: Option, + #[arg(long)] + halt_on_error: Option, + #[arg(long)] + halt_on_success: Option, + #[arg(long)] + expect: Option, + /// Disable timestamp prefix on monitor output lines + #[arg(long)] + no_timestamp: bool, + /// Skip the build step and deploy existing firmware (upload-only mode) + #[arg(long)] + skip_build: bool, + /// Deploy to the native QEMU emulator instead of a physical device + #[arg(long)] + qemu: bool, + /// Timeout in seconds for QEMU execution (default: 30) + #[arg(long, default_value = "30")] + qemu_timeout: u32, + /// Override the board's default upload baud rate + #[arg(short = 'b', long = "baud", alias = "baud-rate")] + baud_rate: Option, + /// Deploy destination: device (default) or emulator + #[arg(long = "to", value_parser = ["device", "emu", "emulator"])] + to: Option, + /// Emulator backend when deploying to `emu` + #[arg(long, value_parser = ["avr8js", "qemu", "simavr"])] + emulator: Option, + /// Legacy deploy target alias: device, qemu, or avr8js + #[arg(long, value_parser = ["device", "qemu", "avr8js"], hide = true)] + target: Option, + /// Export build artifacts to a tooling-friendly directory + #[arg(long)] + output_dir: Option, + }, + /// Monitor serial output + Monitor { + project_dir: Option, + #[arg(short = 'e', long)] + environment: Option, + #[arg(short = 'p', long)] + port: Option, + #[arg(short = 'b', long = "baud", alias = "baud-rate")] + baud_rate: Option, + #[arg(short, long)] + verbose: bool, + #[arg(long)] + platformio: bool, + #[arg(long)] + timeout: Option, + #[arg(long)] + halt_on_error: Option, + #[arg(long)] + halt_on_success: Option, + #[arg(long)] + expect: Option, + /// Disable timestamp prefix on each output line + #[arg(long)] + no_timestamp: bool, + }, + /// Reset device without re-flashing + Reset { + /// Project directory + #[arg(default_value = ".")] + project_dir: String, + /// Target environment + #[arg(short = 'e', long)] + environment: Option, + /// Serial port (e.g., COM5, /dev/ttyUSB0) + #[arg(short = 'p', long)] + port: Option, + /// Verbose output + #[arg(short, long)] + verbose: bool, + }, + /// Purge cached packages + Purge { + target: Option, + #[arg(long)] + dry_run: bool, + #[arg(long)] + project_dir: Option, + /// Run LRU garbage collection instead of full purge + #[arg(long)] + gc: bool, + }, + /// Manage the fbuild daemon + Daemon { + #[command(subcommand)] + action: DaemonAction, + }, + /// Show daemon logs or other information + Show { + /// What to show (currently only 'daemon' for daemon logs) + target: String, + /// Don't follow the log file (just print last lines and exit) + #[arg(long)] + no_follow: bool, + /// Number of lines to show initially (default: 50) + #[arg(long, default_value = "50")] + lines: usize, + }, + /// Manage connected devices + Device { + #[command(subcommand)] + action: DeviceAction, + }, + /// Start MCP (Model Context Protocol) server for AI assistant integration + Mcp, + /// Run clang-tidy static analysis on project sources + ClangTidy { + project_dir: Option, + #[arg(short = 'e', long)] + environment: Option, + #[arg(short, long)] + verbose: bool, + }, + /// Run include-what-you-use analysis on project sources + Iwyu { + project_dir: Option, + #[arg(short = 'e', long)] + environment: Option, + #[arg(short, long)] + verbose: bool, + }, + /// Build firmware and run it in an emulator for testing + TestEmu { + project_dir: Option, + #[arg(short = 'e', long)] + environment: Option, + #[arg(short, long)] + verbose: bool, + /// Timeout in seconds for the emulator run + #[arg(long)] + timeout: Option, + /// Halt on error pattern (regex) + #[arg(long)] + halt_on_error: Option, + /// Halt on success pattern (regex) + #[arg(long)] + halt_on_success: Option, + /// Expected output pattern (regex) + #[arg(long)] + expect: Option, + /// Disable timestamp prefix on output lines + #[arg(long)] + no_timestamp: bool, + /// Emulator backend: "qemu", "avr8js", or "simavr" (auto-detected if omitted) + #[arg(long, value_parser = ["avr8js", "qemu", "simavr"])] + emulator: Option, + }, + /// Run clang-query on project sources + ClangQuery { + project_dir: Option, + #[arg(short = 'e', long)] + environment: Option, + #[arg(short, long)] + verbose: bool, + /// clang-query matcher expression + #[arg(short = 'm', long)] + matcher: Option, + }, + /// Manage `.lnk` resource pointers (fetch / verify / add). + /// + /// `.lnk` files are tiny JSON manifests checked into source control + /// that point at remote binary blobs (sha256-verified). At build time + /// fbuild downloads + caches them; this command lets you operate on + /// them outside of a build. + Lnk { + #[command(subcommand)] + action: LnkAction, + }, + /// Diagnostic: drive the LDF-style library-selection resolver and print + /// the selected library set. Useful for debugging FastLED/fbuild#202 / + /// `#204`-style "library not found" issues without running a full build. + LibSelect { + /// Project directory (defaults to "."). + project_dir: Option, + /// Target environment. + #[arg(short = 'e', long)] + environment: Option, + /// Show selection origin per library, unresolved headers, etc. + #[arg(long, conflicts_with = "json")] + explain: bool, + /// Emit machine-readable JSON instead of plain text. + #[arg(long, conflicts_with = "explain")] + json: bool, + }, + /// Two-stage compile of many sketches against the same board + /// (FastLED/fbuild#238). Builds the framework + library archives once + /// (stage 1) and fans out per-sketch compile + link in parallel + /// (stage 2). Independent parallelism knobs for each stage so memory- + /// heavy framework work stays modest while sketch work saturates cores. + CompileMany { + /// Board id (e.g. "uno", "teensy41"). Used to dispatch to the + /// right platform orchestrator and to pick the matching + /// `[env:]` (or first env with `board = `) inside + /// each sketch's `platformio.ini`. + #[arg(long)] + board: String, + /// Parallelism for stage 1 (framework + library compile). When + /// omitted, defaults to `min(cores, 2)`. + #[arg(long)] + framework_jobs: Option, + /// Parallelism for stage 2 (per-sketch compile + link). When + /// omitted, defaults to `cores`. + #[arg(long)] + sketch_jobs: Option, + /// Build profile. + #[arg(long, group = "compile_many_profile")] + quick: bool, + #[arg(long, group = "compile_many_profile")] + release: bool, + /// Verbose compiler output. + #[arg(short, long)] + verbose: bool, + /// Sketch project directories (each must contain `platformio.ini`). + #[arg(required = true)] + sketches: Vec, + }, + /// PlatformIO-compatible CI command (FastLED/fbuild#242). Drop-in + /// replacement for `pio ci` that delegates to the `compile-many` + /// two-stage primitive (FastLED/fbuild#241). + Ci { + /// Board id (e.g. "uno", "teensy41"). Matches `pio ci --board`. + #[arg(short = 'b', long)] + board: String, + /// Extra library search directory (repeatable). Mapped to the + /// `PLATFORMIO_LIB_EXTRA_DIRS` env var, ';' separated on Windows + /// and ':' separated elsewhere. Matches `pio ci --lib`. + #[arg(short = 'l', long = "lib")] + libs: Vec, + /// Path to a `platformio.ini` to use instead of the per-sketch + /// one. Mapped to `PLATFORMIO_PROJECT_CONFIG`. Matches + /// `pio ci --project-conf`. + #[arg(short = 'c', long = "project-conf")] + project_conf: Option, + /// Accepted for compatibility with `pio ci`; build directories + /// are always kept under `.fbuild/build/...` (no-op). + #[arg(long)] + keep_build_dir: bool, + /// Accepted for compatibility with `pio ci`. Not yet honored; + /// emits a warning when set. + #[arg(long)] + build_dir: Option, + /// Parallelism for stage 1 (framework + library compile). + #[arg(long)] + framework_jobs: Option, + /// Parallelism for stage 2 (per-sketch compile + link). + #[arg(long)] + sketch_jobs: Option, + /// Build profile. + #[arg(long, group = "ci_profile")] + quick: bool, + #[arg(long, group = "ci_profile")] + release: bool, + /// Verbose compiler output. + #[arg(short, long)] + verbose: bool, + /// Sketches to build. Each entry is either a project directory + /// containing `platformio.ini` or a `.ino` file whose parent + /// directory is the project. Matches `pio ci` positional args. + #[arg(required = true)] + sketches: Vec, + }, +} + +/// Subcommands for `fbuild lnk`. +#[derive(Subcommand)] +pub enum LnkAction { + /// Walk the current dir (or a project root) and fetch every `.lnk` + /// referenced blob into the disk cache. Cache hits are no-ops. + Pull { + /// Project root to scan. Defaults to the current directory. + project_dir: Option, + }, + /// Verify every `.lnk` blob in the cache matches its sha256, without + /// touching the network. Reports mismatches; exits non-zero on any. + Check { + /// Project root to scan. Defaults to the current directory. + project_dir: Option, + }, + /// Download a URL once, compute its sha256, and write a new `.lnk` + /// JSON pointing at it. Useful for adding new resources without + /// hand-editing JSON. + Add { + /// URL to download. + url: String, + /// Where to write the `.lnk` file. Defaults to the URL's basename + /// + `.lnk` in the current directory. + #[arg(short = 'o', long)] + output: Option, + }, +} + +#[derive(Subcommand)] +pub enum DeviceAction { + /// List all connected devices + List { + /// Refresh device discovery before listing + #[arg(long)] + refresh: bool, + }, + /// Show detailed status of a device + Status { + /// Serial port (e.g. COM3, /dev/ttyUSB0) + port: String, + }, + /// Acquire a lease on a device + Lease { + /// Serial port (e.g. COM3, /dev/ttyUSB0) + port: String, + /// Lease type: "exclusive" (default) or "monitor" + #[arg(short = 't', long, default_value = "exclusive")] + lease_type: String, + /// Description for the lease + #[arg(short, long, default_value = "")] + description: String, + }, + /// Release a lease on a device + Release { + /// Serial port (e.g. COM3, /dev/ttyUSB0) + port: String, + /// Specific lease ID to release (releases all if omitted) + #[arg(long)] + lease_id: Option, + }, + /// Forcibly take a device from the current holder + Take { + /// Serial port (e.g. COM3, /dev/ttyUSB0) + port: String, + /// Mandatory reason for preemption + #[arg(short, long)] + reason: String, + }, +} + +#[derive(Subcommand)] +pub enum DaemonAction { + /// Stop the daemon gracefully + Stop, + /// Show daemon status + Status, + /// Restart the daemon (stop then start) + Restart, + /// List running daemon instances + List, + /// Kill a daemon process (bypasses graceful shutdown) + Kill { + /// PID of the daemon to kill (auto-detected if omitted) + #[arg(long)] + pid: Option, + /// Force kill (SIGKILL/TerminateProcess) instead of graceful termination + #[arg(short, long)] + force: bool, + }, + /// Kill all fbuild-daemon processes + KillAll { + /// Force kill (SIGKILL/TerminateProcess) instead of graceful termination + #[arg(short, long)] + force: bool, + }, + /// Show lock status (project locks, serial sessions) + Locks, + /// Clear stale locks + ClearLocks, + /// Show disk cache statistics + CacheStats, + /// Run disk cache garbage collection + Gc, + /// Tail daemon logs (alias for `fbuild show daemon`) + Monitor { + /// Don't follow the log file (just print last lines and exit) + #[arg(long)] + no_follow: bool, + /// Number of lines to show initially + #[arg(long, default_value = "50")] + lines: usize, + }, +} + +/// Resolve project_dir: prefer the subcommand's value, fall back to the top-level positional arg, +/// then default to ".". This lets callers write either `fbuild build ` or `fbuild build`. +pub fn resolve_project_dir( + subcommand_dir: Option, + top_level_dir: &Option, +) -> String { + subcommand_dir + .or_else(|| top_level_dir.clone()) + .unwrap_or_else(|| ".".to_string()) +} + +/// Known subcommand names for arg rewriting. +pub const KNOWN_SUBCOMMANDS: &[&str] = &[ + "build", + "deploy", + "monitor", + "reset", + "purge", + "show", + "daemon", + "device", + "mcp", + "clang-tidy", + "iwyu", + "clang-query", + "test-emu", + "lib-select", + "compile-many", + "ci", +]; + +/// Rewrite `fbuild ...` → `fbuild ...` +/// so that both `fbuild build ` and `fbuild build` work. +pub fn rewrite_args() -> Vec { + let args: Vec = std::env::args().collect(); + if args.len() >= 3 { + let first = &args[1]; + let second = &args[2]; + // If first arg is NOT a subcommand and second IS, swap them + if !first.starts_with('-') + && !KNOWN_SUBCOMMANDS.contains(&first.as_str()) + && KNOWN_SUBCOMMANDS.contains(&second.as_str()) + { + let mut rewritten = Vec::with_capacity(args.len()); + rewritten.push(args[0].clone()); + rewritten.push(second.clone()); // subcommand first + rewritten.push(first.clone()); // project_dir second + rewritten.extend(args[3..].iter().cloned()); + return rewritten; + } + } + args +} diff --git a/crates/fbuild-cli/src/cli/build.rs b/crates/fbuild-cli/src/cli/build.rs new file mode 100644 index 00000000..074f1f7f --- /dev/null +++ b/crates/fbuild-cli/src/cli/build.rs @@ -0,0 +1,158 @@ +//! `fbuild build` handler and a couple of path / browser helpers. + +use crate::daemon_client::{self, BuildRequest, DaemonClient}; + +pub fn open_in_browser(url: &str) -> fbuild_core::Result<()> { + let args: Vec<&str> = if cfg!(target_os = "windows") { + vec!["cmd", "/c", "start", "", url] + } else if cfg!(target_os = "macos") { + vec!["open", url] + } else { + vec!["xdg-open", url] + }; + let output = fbuild_core::subprocess::run_command(&args, None, None, None) + .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to launch browser: {}", e)))?; + + if output.success() { + Ok(()) + } else { + Err(fbuild_core::FbuildError::Other(format!( + "browser launcher exited with status {}", + output.exit_code + ))) + } +} + +#[allow(clippy::too_many_arguments)] +pub async fn run_build( + project_dir: String, + environment: Option, + clean: bool, + verbose: bool, + jobs: Option, + quick: bool, + release: bool, + dry_run: bool, + target: Option, + symbol_analysis: Option, + no_timestamp: bool, + output_dir: Option, +) -> fbuild_core::Result<()> { + // FBUILD_PERF_LOG=1 enables coarse CLI-side timing (daemon handshake + + // round-trip). Zero-overhead when unset. + let perf_enabled = std::env::var("FBUILD_PERF_LOG") + .map(|v| !v.is_empty() && v != "0") + .unwrap_or(false); + let cli_start = std::time::Instant::now(); + let handshake_start = std::time::Instant::now(); + daemon_client::ensure_daemon_running().await?; + let handshake_elapsed = handshake_start.elapsed(); + + // Dry-run: verify daemon starts and environment resolves, then exit + if dry_run { + let project_path = std::path::Path::new(&project_dir); + let ini_path = project_path.join("platformio.ini"); + let config = fbuild_config::PlatformIOConfig::from_path(&ini_path)?; + let env_name = environment + .as_deref() + .or_else(|| config.get_default_environment()) + .unwrap_or("default"); + println!("Environment: {}", env_name); + println!("Daemon is running. Dry-run complete."); + return Ok(()); + } + + let client = DaemonClient::new(); + + let profile = if release { + Some("release".to_string()) + } else if quick { + Some("quick".to_string()) + } else { + None + }; + let generate_compiledb = target.as_deref() == Some("compiledb"); + if generate_compiledb { + let env_label = environment.as_deref().unwrap_or("default"); + println!( + "Generating compile_commands.json for environment: {}...", + env_label + ); + } + let (caller_pid, caller_cwd) = daemon_client::caller_info(); + let req = BuildRequest { + project_dir: project_dir.clone(), + environment, + clean_build: clean, + verbose, + jobs, + profile, + generate_compiledb, + compiledb_only: generate_compiledb, + request_id: None, + caller_pid, + caller_cwd, + stream: true, + symbol_analysis: symbol_analysis.is_some(), + symbol_analysis_path: symbol_analysis.filter(|s| !s.is_empty()), + no_timestamp, + src_dir: std::env::var("PLATFORMIO_SRC_DIR") + .ok() + .filter(|s| !s.is_empty()), + output_dir, + pio_env: daemon_client::capture_pio_env(), + }; + + let stream_start = std::time::Instant::now(); + let resp = client.build_streaming(&req).await?; + let stream_elapsed = stream_start.elapsed(); + if !resp.message.is_empty() { + println!("{}", resp.message); + } + if perf_enabled { + let summary = format!( + "[perf-log cli-build] daemon-handshake={} ms, server-roundtrip={} ms, total={} ms", + handshake_elapsed.as_millis(), + stream_elapsed.as_millis(), + cli_start.elapsed().as_millis(), + ); + tracing::info!(target: "fbuild_cli::perf_log", "{}", summary); + eprintln!("{}", summary); + } + if !resp.success { + std::process::exit(resp.exit_code); + } + if generate_compiledb { + let db_path = std::path::Path::new(&project_dir).join("compile_commands.json"); + if db_path.exists() { + println!("compile_commands.json written to {}", db_path.display()); + } + } + Ok(()) +} + +/// Convert MSYS/Git-Bash paths (/c/Users/...) to native Windows paths and canonicalize. +pub fn normalize_path(path: &str) -> fbuild_core::Result { + let converted = if cfg!(windows) { + // /c/foo → C:\foo + let bytes = path.as_bytes(); + if bytes.len() >= 3 + && bytes[0] == b'/' + && bytes[2] == b'/' + && bytes[1].is_ascii_alphabetic() + { + let drive = (bytes[1] as char).to_ascii_uppercase(); + format!("{}:{}", drive, path[2..].replace('/', "\\")) + } else { + path.to_string() + } + } else { + path.to_string() + }; + let canon = std::fs::canonicalize(&converted).map_err(|e| { + fbuild_core::FbuildError::Other(format!("cannot resolve path '{}': {}", path, e)) + })?; + let s = canon.to_string_lossy().to_string(); + // Strip \\?\ prefix that canonicalize adds on Windows + Ok(s.strip_prefix(r"\\?\").unwrap_or(&s).to_string()) +} diff --git a/crates/fbuild-cli/src/cli/clang_tools.rs b/crates/fbuild-cli/src/cli/clang_tools.rs new file mode 100644 index 00000000..f1dae956 --- /dev/null +++ b/crates/fbuild-cli/src/cli/clang_tools.rs @@ -0,0 +1,597 @@ +//! Clang-family analysis subcommands: `clang-tidy`, `clang-query`, and +//! `iwyu` (include-what-you-use), plus a couple of cache + filter helpers +//! used by them. + +use super::build::{normalize_path, run_build}; + +/// Run IWYU (include-what-you-use) analysis with ESP32 cross-compilation support. +/// +/// Unlike the generic `run_clang_tool()`, this: +/// 1. Downloads IWYU via ClangComponent +/// 2. Generates compile_commands.json via the fbuild daemon +/// 3. Preprocesses the database: adds GCC builtin includes, converts framework +/// `-I` to `-isystem`, removes `--target=` flags, deduplicates `-D` defines +/// 4. Writes a modified compile_commands.json to a temp dir +/// 5. Runs IWYU per-source-file with `-p ` +/// 6. Filters output to only show suggestions for files under `src/` +pub async fn run_iwyu( + project_dir: String, + environment: Option, + verbose: bool, +) -> fbuild_core::Result<()> { + let project_dir = normalize_path(&project_dir)?; + + // Step 1: Ensure IWYU is installed + let component = fbuild_packages::toolchain::ClangComponent::new( + fbuild_packages::toolchain::ClangComponentKind::Iwyu, + ); + let tool_path = component.get_binary("include-what-you-use").await?; + println!("Using include-what-you-use: {}", tool_path.display()); + + // Step 2: Generate compile_commands.json via fbuild daemon (skip if it already exists) + let project_path = std::path::Path::new(&project_dir); + let db_path = project_path.join("compile_commands.json"); + if db_path.exists() { + println!("Using existing compile_commands.json"); + } else { + println!("Generating compile_commands.json..."); + run_build( + project_dir.clone(), + environment.clone(), + false, + verbose, + None, + false, + false, + false, + Some("compiledb".to_string()), + None, + true, // no_timestamp: compiledb generation doesn't need timestamps + None, + ) + .await?; + if !db_path.exists() { + return Err(fbuild_core::FbuildError::Other( + "compile_commands.json was not generated".into(), + )); + } + } + let db_content = std::fs::read_to_string(&db_path).map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to read compile_commands.json: {}", e)) + })?; + let entries: Vec = serde_json::from_str(&db_content).map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to parse compile_commands.json: {}", e)) + })?; + + // Filter to project source files only. + // Source files can be under /src/ (original) or + // /.fbuild/build//*/src/ (build copies with Arduino preprocessing). + // Exclude framework/SDK files (paths containing cache/platforms/ or cache/toolchains/). + let src_dir = project_path.join("src"); + let build_src_suffix = format!("{}src", std::path::MAIN_SEPARATOR_STR); + let project_prefix = project_path + .to_string_lossy() + .replace('/', std::path::MAIN_SEPARATOR_STR); + let source_entries: Vec<&serde_json::Value> = entries + .iter() + .filter(|e| { + e.get("file") + .and_then(|f| f.as_str()) + .map(|f| { + let p = std::path::Path::new(f); + // Direct match: file is under /src/ + if p.starts_with(&src_dir) { + return true; + } + // Build copy: file is under /.fbuild/build/.../src/ + let f_normalized = f.replace('/', std::path::MAIN_SEPARATOR_STR); + f_normalized.starts_with(&project_prefix) + && f_normalized.contains(&build_src_suffix) + && !f_normalized.contains("cache") + }) + .unwrap_or(false) + }) + .collect(); + + if source_entries.is_empty() { + println!("No source files found in compile_commands.json under src/"); + return Ok(()); + } + + // Step 4: Find GCC toolchain builtin include dirs + let gcc_includes = fbuild_packages::toolchain::clang::find_gcc_builtin_include_dirs(); + if !gcc_includes.is_empty() { + println!("Found {} GCC builtin include dir(s)", gcc_includes.len()); + if verbose { + for inc in &gcc_includes { + println!(" {}", inc.display()); + } + } + } + + // Step 5: Preprocess compile_commands.json for IWYU + // Transform entries directly as JSON: remove --target=, dedup -D, convert -I to -isystem + let src_prefix = src_dir.to_string_lossy().replace('\\', "/").to_lowercase(); + let iwyu_entries: Vec = entries + .iter() + .map(|entry| { + let mut new_entry = entry.clone(); + if let Some(args) = entry.get("arguments").and_then(|a| a.as_array()) { + let mut new_args: Vec = + Vec::with_capacity(args.len() + gcc_includes.len() * 2); + let mut seen_defines = std::collections::HashSet::new(); + + for arg_val in args { + let arg = arg_val.as_str().unwrap_or(""); + + // Remove --target= flags + if arg.starts_with("--target=") { + continue; + } + + // Remove GCC-only flags unsupported by IWYU's clang + if matches!( + arg, + "-freorder-blocks" + | "-fno-jump-tables" + | "-flto" + | "-flto=auto" + | "-fno-fat-lto-objects" + | "-fuse-linker-plugin" + | "-ffat-lto-objects" + | "-mlongcalls" + | "-mdisable-hardware-atomics" + | "-fstrict-volatile-bitfields" + | "-mtext-section-literals" + | "-fno-tree-switch-conversion" + | "-mthumb-interwork" + ) || arg.starts_with("-mfix-esp32-psram-cache-strategy=") + { + continue; + } + + // Deduplicate -D flags (keep first occurrence by key) + if arg.starts_with("-D") { + let key = if let Some(eq_pos) = arg.find('=') { + &arg[..eq_pos] + } else { + arg + }; + if !seen_defines.insert(key.to_string()) { + continue; + } + } + + // Convert non-project -I to -isystem + if let Some(path) = arg.strip_prefix("-I") { + let normalized = path.replace('\\', "/").to_lowercase(); + if normalized.starts_with(&src_prefix) { + new_args.push(arg_val.clone()); + } else { + new_args.push(serde_json::Value::String("-isystem".into())); + new_args.push(serde_json::Value::String(path.to_string())); + } + continue; + } + + new_args.push(arg_val.clone()); + } + + // Append GCC toolchain builtin include dirs as -isystem + for inc in &gcc_includes { + new_args.push(serde_json::Value::String("-isystem".into())); + new_args.push(serde_json::Value::String(inc.to_string_lossy().to_string())); + } + + new_entry["arguments"] = serde_json::Value::Array(new_args); + } + new_entry + }) + .collect(); + + // Write modified compile_commands.json to .fbuild/iwyu/ for IWYU to read via -p + let iwyu_dir_path = fbuild_paths::get_project_fbuild_dir(project_path).join("iwyu"); + std::fs::create_dir_all(&iwyu_dir_path).map_err(|e| { + fbuild_core::FbuildError::Other(format!( + "failed to create {}: {}", + iwyu_dir_path.display(), + e + )) + })?; + let iwyu_db_path = iwyu_dir_path.join("compile_commands.json"); + let iwyu_json = serde_json::to_string_pretty(&iwyu_entries).map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to serialize IWYU compile database: {}", e)) + })?; + std::fs::write(&iwyu_db_path, iwyu_json).map_err(|e| { + fbuild_core::FbuildError::Other(format!( + "failed to write {}: {}", + iwyu_db_path.display(), + e + )) + })?; + // Step 6: Set up zccache-style content-addressed cache for IWYU results. + // Cache key = blake3(source_content + iwyu_entry_json) per file. + let cache_dir = iwyu_dir_path.join("cache"); + std::fs::create_dir_all(&cache_dir).map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to create {}: {}", cache_dir.display(), e)) + })?; + + // Build a lookup from file path → preprocessed IWYU entry JSON for cache keying. + let iwyu_entry_map: std::collections::HashMap = iwyu_entries + .iter() + .filter_map(|e| { + let file = e.get("file")?.as_str()?.to_string(); + let json = serde_json::to_string(e).ok()?; + Some((file, json)) + }) + .collect(); + + println!( + "Running include-what-you-use on {} source file(s)...", + source_entries.len() + ); + + // Step 7: Run IWYU in parallel with caching + let jobs = std::thread::available_parallelism() + .map(|n| n.get() * 2) + .unwrap_or(4); + let semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(jobs)); + let tool_arc = std::sync::Arc::new(tool_path); + let iwyu_dir = std::sync::Arc::new(iwyu_dir_path); + let src_dir_arc = std::sync::Arc::new(src_dir.clone()); + let cache_dir_arc = std::sync::Arc::new(cache_dir); + let entry_map_arc = std::sync::Arc::new(iwyu_entry_map); + + // Collect source file paths from the filtered entries + let source_files: Vec = source_entries + .iter() + .filter_map(|e| e.get("file").and_then(|f| f.as_str()).map(String::from)) + .collect(); + + let mut handles = Vec::new(); + for file in source_files { + let sem = semaphore.clone(); + let tool = tool_arc.clone(); + let p_dir = iwyu_dir.clone(); + let src = src_dir_arc.clone(); + let cache_d = cache_dir_arc.clone(); + let emap = entry_map_arc.clone(); + let verbose_flag = verbose; + let handle = tokio::spawn(async move { + let _permit = sem.acquire().await.unwrap(); + let src_path = src.as_ref().clone(); + + // Compute blake3 cache key from source content + compile entry + let cache_key = iwyu_cache_key(&file, &emap); + let cache_path: Option = cache_key + .as_ref() + .map(|k| cache_d.join(format!("{}.txt", k))); + + // Check cache + if let Some(ref cp) = cache_path { + if cp.exists() { + if let Ok(cached) = std::fs::read_to_string(cp) { + return (file, Ok(cached), true, src_path); + } + } + } + + // Cache miss — run IWYU. Parallel async fan-out inside the CLI binary + // (no daemon containment group in this process). + // allow-direct-spawn: parallel async fan-out in CLI; no containment group here. + let mut cmd = tokio::process::Command::new(tool.as_ref()); + cmd.arg("-p").arg(p_dir.as_ref()); + cmd.arg("-Xiwyu").arg("--no_comments"); + cmd.arg("-Xiwyu").arg("--quoted_includes_first"); + cmd.arg("-Xiwyu").arg("--max_line_length=100"); + if verbose_flag { + cmd.arg("-Xiwyu").arg("--verbose=3"); + } + cmd.arg(&file); + let output = cmd.output().await; + + match output { + Ok(out) => { + let stderr = String::from_utf8_lossy(&out.stderr).to_string(); + // Store in cache on success + if let Some(ref cp) = cache_path { + let _ = std::fs::write(cp, &stderr); + } + (file, Ok(stderr), false, src_path) + } + Err(e) => (file, Err(format!("{}", e)), false, src_path), + } + }); + handles.push(handle); + } + + let mut total_suggestions = 0usize; + let mut failed_files = Vec::new(); + let mut cache_hits = 0usize; + let mut cache_misses = 0usize; + + for handle in handles { + let (file, result, cached, src_path) = handle + .await + .map_err(|e| fbuild_core::FbuildError::Other(format!("task join error: {}", e)))?; + if cached { + cache_hits += 1; + } else { + cache_misses += 1; + } + match result { + Ok(stderr) => { + let filtered = filter_iwyu_output(&stderr, &src_path); + if !filtered.trim().is_empty() { + print!("{}", filtered); + total_suggestions += filtered + .lines() + .filter(|l| l.contains("should add") || l.contains("should remove")) + .count(); + } + } + Err(e) => { + eprintln!("failed to run include-what-you-use on {}: {}", file, e); + failed_files.push(file); + } + } + } + + println!("\n--- include-what-you-use summary ---"); + println!("Suggestions: {}", total_suggestions); + println!( + "Cache: {} hit(s), {} miss(es)", + cache_hits, cache_misses + ); + if !failed_files.is_empty() { + println!("Failed: {} file(s)", failed_files.len()); + } + + if !failed_files.is_empty() { + Err(fbuild_core::FbuildError::BuildFailed( + "include-what-you-use failed on some files".into(), + )) + } else { + Ok(()) + } +} + +/// Compute a blake3 cache key for an IWYU analysis of a source file. +/// +/// The key is derived from the source file content and the preprocessed +/// compile_commands.json entry for that file (which includes all flags). +/// This mirrors zccache's content-addressed hashing strategy. +pub fn iwyu_cache_key( + file: &str, + entry_map: &std::collections::HashMap, +) -> Option { + let source_content = std::fs::read(file).ok()?; + let entry_json = entry_map.get(file)?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"fbuild-iwyu-cache-v1"); + hasher.update(&source_content); + hasher.update(entry_json.as_bytes()); + Some(hasher.finalize().to_hex().to_string()) +} + +/// Filter IWYU output to show only suggestions for files under `src_dir`. +/// +/// IWYU outputs blocks like: +/// ```text +/// /path/to/file.h should add these lines: +/// #include +/// +/// /path/to/file.h should remove these lines: +/// - #include +/// +/// The full include-list for /path/to/file.h: +/// #include +/// --- +/// ``` +/// +/// We only keep blocks whose file path is under `src_dir`. +pub fn filter_iwyu_output(output: &str, src_dir: &std::path::Path) -> String { + let src_prefix = src_dir.to_string_lossy().replace('\\', "/").to_lowercase(); + let mut result = String::new(); + let mut current_block = String::new(); + let mut block_is_user_file = false; + + for line in output.lines() { + // Detect block headers: "path/to/file should add/remove these lines:" + // or "The full include-list for path/to/file:" + let is_header = line.contains(" should add these lines") + || line.contains(" should remove these lines") + || line.starts_with("The full include-list for "); + + if is_header { + // Flush previous block if it was a user file + if block_is_user_file && !current_block.trim().is_empty() { + result.push_str(¤t_block); + result.push('\n'); + } + current_block.clear(); + + // Check if this new block's file is under src_dir + let file_path = line + .split(" should ") + .next() + .or_else(|| { + line.strip_prefix("The full include-list for ") + .and_then(|s| s.strip_suffix(':')) + }) + .unwrap_or(""); + let normalized = file_path.replace('\\', "/").to_lowercase(); + block_is_user_file = normalized.starts_with(&src_prefix); + } + + current_block.push_str(line); + current_block.push('\n'); + } + + // Flush last block + if block_is_user_file && !current_block.trim().is_empty() { + result.push_str(¤t_block); + } + + result +} + +/// Generic runner for clang-based analysis tools (clang-tidy, clang-query). +/// +/// 1. Ensure tool binary is installed via ClangComponent (downloads on demand) +/// 2. Generate compile_commands.json via fbuild daemon (build -t compiledb) +/// 3. Run tool on each source file in parallel (ncpus * 2) +#[allow(clippy::too_many_arguments)] +pub async fn run_clang_tool( + kind: fbuild_packages::toolchain::ClangComponentKind, + binary_name: &str, + project_dir: String, + environment: Option, + verbose: bool, + extra_args: &[&str], +) -> fbuild_core::Result<()> { + let project_dir = normalize_path(&project_dir)?; + + // Step 1: Ensure tool is installed + let component = fbuild_packages::toolchain::ClangComponent::new(kind); + let tool_path = component.get_binary(binary_name).await?; + println!("Using {}: {}", binary_name, tool_path.display()); + + // Step 2: Generate compile_commands.json via fbuild daemon + println!("Generating compile_commands.json..."); + run_build( + project_dir.clone(), + environment.clone(), + false, // clean + verbose, + None, // jobs + false, // quick + false, // release + false, // dry_run + Some("compiledb".to_string()), + None, + true, // no_timestamp: compiledb generation doesn't need timestamps + None, + ) + .await?; + + // Step 3: Read compile_commands.json to get source files + let project_path = std::path::Path::new(&project_dir); + let db_path = project_path.join("compile_commands.json"); + if !db_path.exists() { + return Err(fbuild_core::FbuildError::Other( + "compile_commands.json was not generated".into(), + )); + } + let db_content = std::fs::read_to_string(&db_path).map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to read compile_commands.json: {}", e)) + })?; + let entries: Vec = serde_json::from_str(&db_content).map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to parse compile_commands.json: {}", e)) + })?; + + // Filter to project source files only (under src/) + let src_dir = project_path.join("src"); + let source_files: Vec = entries + .iter() + .filter_map(|e| e.get("file").and_then(|f| f.as_str()).map(String::from)) + .filter(|f| { + let p = std::path::Path::new(f); + p.starts_with(&src_dir) + }) + .collect(); + + if source_files.is_empty() { + println!("No source files found in compile_commands.json under src/"); + return Ok(()); + } + println!( + "Running {} on {} source file(s)...", + binary_name, + source_files.len() + ); + + // Step 4: Run tool in parallel with ncpus * 2 + let jobs = std::thread::available_parallelism() + .map(|n| n.get() * 2) + .unwrap_or(4); + println!("Using {} parallel jobs", jobs); + + let semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(jobs)); + let project_dir_arc = std::sync::Arc::new(project_dir.clone()); + let tool_arc = std::sync::Arc::new(tool_path); + let extra_owned: Vec = extra_args.iter().map(|s| s.to_string()).collect(); + let verbose_flag = verbose; + + let mut handles = Vec::new(); + for file in source_files { + let sem = semaphore.clone(); + let tool = tool_arc.clone(); + let pd = project_dir_arc.clone(); + let extra = extra_owned.clone(); + let handle = tokio::spawn(async move { + let _permit = sem.acquire().await.unwrap(); + // allow-direct-spawn: parallel async fan-out (clang-tidy) in CLI binary. + let mut cmd = tokio::process::Command::new(tool.as_ref()); + cmd.arg("-p").arg(pd.as_ref()); + for arg in &extra { + cmd.arg(arg); + } + cmd.arg(&file); + if !verbose_flag { + cmd.arg("--quiet"); + } + let output = cmd.output().await; + (file, output) + }); + handles.push(handle); + } + + let mut total_warnings = 0usize; + let mut total_errors = 0usize; + let mut failed_files = Vec::new(); + + for handle in handles { + let (file, result) = handle + .await + .map_err(|e| fbuild_core::FbuildError::Other(format!("task join error: {}", e)))?; + match result { + Ok(output) => { + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let combined = format!("{}{}", stdout, stderr); + if !combined.trim().is_empty() { + print!("{}", combined); + } + for line in combined.lines() { + if line.contains("warning:") { + total_warnings += 1; + } + if line.contains("error:") { + total_errors += 1; + } + } + } + Err(e) => { + eprintln!("failed to run {} on {}: {}", binary_name, file, e); + failed_files.push(file); + } + } + } + + println!("\n--- {} summary ---", binary_name); + println!("Warnings: {}", total_warnings); + println!("Errors: {}", total_errors); + if !failed_files.is_empty() { + println!("Failed: {} file(s)", failed_files.len()); + } + + if total_errors > 0 || !failed_files.is_empty() { + Err(fbuild_core::FbuildError::BuildFailed(format!( + "{} found errors", + binary_name + ))) + } else { + Ok(()) + } +} diff --git a/crates/fbuild-cli/src/cli/compile_many.rs b/crates/fbuild-cli/src/cli/compile_many.rs new file mode 100644 index 00000000..9a17e60f --- /dev/null +++ b/crates/fbuild-cli/src/cli/compile_many.rs @@ -0,0 +1,192 @@ +//! `fbuild compile-many` (FastLED/fbuild#238) and the thin `fbuild ci` +//! adapter that maps `pio ci` flags onto it. + +/// Separator used to join `PLATFORMIO_LIB_EXTRA_DIRS` entries. +/// +/// PlatformIO follows `PATH`-style conventions: ';' on Windows, ':' elsewhere. +/// Centralized here so the CLI handler and the unit tests agree. +pub fn ci_lib_extra_dirs_sep() -> &'static str { + if cfg!(windows) { + ";" + } else { + ":" + } +} + +/// Map a single `pio ci` positional argument to a project directory. +/// +/// `pio ci` lets callers point at either a sketch dir or directly at the +/// `.ino` file. fbuild's `compile-many` only takes project dirs, so a `.ino` +/// path is rewritten to its parent. Case-insensitive on the extension so +/// `Blink.INO` works on Windows where casing isn't preserved. +pub fn normalize_ci_sketch_entry(entry: &str) -> String { + let path = std::path::Path::new(entry); + let is_ino = path + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.eq_ignore_ascii_case("ino")) + .unwrap_or(false); + if is_ino { + if let Some(parent) = path.parent() { + let p = parent.to_string_lossy().to_string(); + if p.is_empty() { + return ".".to_string(); + } + return p; + } + } + entry.to_string() +} + +/// Normalize every `pio ci` positional argument. +pub fn normalize_ci_sketches(entries: &[String]) -> Vec { + entries + .iter() + .map(|e| normalize_ci_sketch_entry(e)) + .collect() +} + +/// Build the `PLATFORMIO_*` env overlay for `fbuild ci` from `--lib` and +/// `--project-conf`. Returns an empty map when neither flag was set. +pub fn build_ci_pio_env( + libs: &[String], + project_conf: Option<&str>, +) -> std::collections::HashMap { + let mut env = std::collections::HashMap::new(); + if !libs.is_empty() { + env.insert( + "PLATFORMIO_LIB_EXTRA_DIRS".to_string(), + libs.join(ci_lib_extra_dirs_sep()), + ); + } + if let Some(conf) = project_conf { + let canonical = std::fs::canonicalize(conf) + .ok() + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_else(|| conf.to_string()); + env.insert("PLATFORMIO_PROJECT_CONFIG".to_string(), canonical); + } + env +} + +/// Handler for `fbuild compile-many` (FastLED/fbuild#238). +/// +/// Runs the two-stage compile-many primitive in-process via the existing +/// `fbuild-build` orchestrator. We bypass the daemon here on purpose: the +/// design goal of #238 is "one process, one toolchain load, one LDF run, +/// one framework build" — so all stage-1 + stage-2 work happens in this +/// process, parallelism is driven by the `compile_many` thread pool, and +/// no per-sketch daemon round-trip is incurred. +pub struct CompileManyArgs { + pub board: String, + pub framework_jobs: Option, + pub sketch_jobs: Option, + pub quick: bool, + pub release: bool, + pub verbose: bool, + pub sketches: Vec, + pub pio_env: std::collections::HashMap, +} + +pub async fn run_compile_many(args: CompileManyArgs) -> fbuild_core::Result<()> { + use fbuild_build::compile_many::{compile_many, CompileManyRequest, Stage}; + + let CompileManyArgs { + board, + framework_jobs, + sketch_jobs, + quick, + release, + verbose, + sketches, + pio_env, + } = args; + + let profile = if release { + fbuild_core::BuildProfile::Release + } else if quick { + fbuild_core::BuildProfile::Quick + } else { + // Default to release: matches `fbuild build`'s default profile so + // CI builds aren't silently dropped into quick mode. + fbuild_core::BuildProfile::Release + }; + + let sketches: Vec = + sketches.into_iter().map(std::path::PathBuf::from).collect(); + + let req = CompileManyRequest { + board: board.clone(), + sketches: sketches.clone(), + framework_jobs, + sketch_jobs, + profile, + verbose, + pio_env, + }; + + let effective_framework = req + .framework_jobs + .unwrap_or_else(fbuild_build::compile_many::default_framework_jobs); + let effective_sketch = req + .sketch_jobs + .unwrap_or_else(fbuild_build::compile_many::default_sketch_jobs); + println!( + "compile-many: board={} sketches={} framework_jobs={} sketch_jobs={}", + board, + sketches.len(), + effective_framework, + effective_sketch, + ); + + // `compile_many` is fully synchronous (CPU-bound). Run it on a + // blocking pool so we don't tie up the tokio runtime thread. + let result = tokio::task::spawn_blocking(move || compile_many(req)) + .await + .map_err(|e| { + fbuild_core::FbuildError::Other(format!("compile-many task panicked: {e}")) + })??; + + // Per-sketch result map suitable for the bench summary. + println!(); + println!("compile-many results:"); + for r in &result.results { + let stage_label = match r.stage { + Stage::Stage1Framework => "stage1", + Stage::Stage2Sketch => "stage2", + }; + let status = if r.success { "OK" } else { "FAIL" }; + let log_str = r + .log_path + .as_ref() + .map(|p| p.display().to_string()) + .unwrap_or_else(|| "-".to_string()); + println!( + " [{}] {} ({:.2}s) {} log={} {}", + stage_label, + status, + r.build_time_secs, + r.sketch.display(), + log_str, + r.message, + ); + } + println!(); + println!( + "compile-many summary: stage1={}/{:.2}s stage2={}/{:.2}s total={:.2}s", + result.stage1_count, + result.stage1_secs, + result.stage2_count, + result.stage2_secs, + result.total_secs, + ); + + if !result.all_success { + return Err(fbuild_core::FbuildError::BuildFailed(format!( + "compile-many: {}/{} sketches failed", + result.results.iter().filter(|r| !r.success).count(), + result.results.len(), + ))); + } + Ok(()) +} diff --git a/crates/fbuild-cli/src/cli/daemon_cmd.rs b/crates/fbuild-cli/src/cli/daemon_cmd.rs new file mode 100644 index 00000000..ad6e0a42 --- /dev/null +++ b/crates/fbuild-cli/src/cli/daemon_cmd.rs @@ -0,0 +1,474 @@ +//! `fbuild daemon ...` subcommand handlers (status, restart, kill, locks, +//! cache-stats, gc, monitor-tail) plus process-management helpers shared +//! with the purge subcommand. + +use crate::daemon_client::{self, DaemonClient}; + +use super::args::DaemonAction; +use super::purge::format_size; +use super::show::run_show; + +pub async fn run_daemon(action: DaemonAction) -> fbuild_core::Result<()> { + let client = DaemonClient::new(); + match action { + DaemonAction::Stop => { + if !client.health().await { + println!("daemon is not running"); + return Ok(()); + } + client.shutdown().await?; + // Wait for it to actually stop + for _ in 0..50 { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + if !client.health().await { + println!("daemon stopped"); + return Ok(()); + } + } + println!("daemon stop requested (may still be shutting down)"); + } + DaemonAction::Status => { + if client.health().await { + match client.daemon_info().await { + Ok(info) => { + let uptime = format_uptime(info.uptime_seconds); + println!("daemon is running at {}", fbuild_paths::get_daemon_url()); + println!(" PID: {}", info.pid); + println!(" Port: {}", info.port); + println!(" Uptime: {}", uptime); + println!(" Version: {}", info.version); + println!(" Mode: {}", if info.dev_mode { "dev" } else { "prod" }); + println!(" State: {}", info.daemon_state); + if info.operation_in_progress { + if let Some(ref op) = info.current_operation { + println!(" Operation: {}", op); + } else { + println!(" Operation: (in progress)"); + } + } + if info.client_count > 0 { + println!(" Clients: {}", info.client_count); + } + if let Some(ref cwd) = info.spawner_cwd { + println!(" Spawned from: {}", cwd); + } + if let Some(mtime) = info.source_mtime { + println!(" Binary mtime: {:.0}", mtime); + } + } + Err(_) => { + println!("daemon is running at {}", fbuild_paths::get_daemon_url()); + } + } + } else { + println!("daemon is not running"); + } + } + DaemonAction::Restart => { + // Stop if running + if client.health().await { + client.shutdown().await?; + for _ in 0..50 { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + if !client.health().await { + break; + } + } + } + // Start fresh + daemon_client::ensure_daemon_running().await?; + println!("daemon restarted"); + } + DaemonAction::List => { + run_daemon_list(&client).await?; + } + DaemonAction::Kill { pid, force } => { + run_daemon_kill(&client, pid, force).await?; + } + DaemonAction::KillAll { force } => { + run_daemon_kill_all(force)?; + } + DaemonAction::Locks => { + run_daemon_locks(&client).await?; + } + DaemonAction::ClearLocks => { + run_daemon_clear_locks(&client).await?; + } + DaemonAction::CacheStats => { + run_daemon_cache_stats(&client).await?; + } + DaemonAction::Gc => { + run_daemon_gc(&client).await?; + } + DaemonAction::Monitor { no_follow, lines } => { + return run_show("daemon", !no_follow, lines); + } + } + Ok(()) +} + +pub async fn run_daemon_list(client: &DaemonClient) -> fbuild_core::Result<()> { + if client.health().await { + match client.daemon_info().await { + Ok(info) => { + let uptime = format_uptime(info.uptime_seconds); + println!("fbuild daemon (running)"); + println!(" PID: {}", info.pid); + println!(" Port: {}", info.port); + println!(" Uptime: {}", uptime); + println!(" Version: {}", info.version); + println!(" Mode: {}", if info.dev_mode { "dev" } else { "prod" }); + } + Err(e) => { + println!("daemon is running but info unavailable: {}", e); + } + } + } else { + println!("no daemon is running"); + // Check for stale PID file + let pid_file = fbuild_paths::get_daemon_pid_file(); + if pid_file.exists() { + if let Ok(contents) = std::fs::read_to_string(&pid_file) { + println!( + " (stale PID file: {} — PID {})", + pid_file.display(), + contents.trim() + ); + } + } + } + + // Also scan for orphan processes + let pids = find_daemon_pids()?; + if pids.len() > 1 { + println!("\nwarning: multiple fbuild-daemon processes detected:"); + for pid in &pids { + println!(" PID {}", pid); + } + println!("use 'fbuild daemon kill-all' to clean up"); + } + Ok(()) +} + +pub async fn run_daemon_locks(client: &DaemonClient) -> fbuild_core::Result<()> { + if !client.health().await { + println!("daemon is not running"); + return Ok(()); + } + + let status = client.lock_status().await?; + + // Display port locks + if status.port_locks.is_empty() { + println!("Port Locks: (none)"); + } else { + println!("Port Locks:"); + for lock in &status.port_locks { + let state = if lock.is_held { "HELD" } else { "FREE" }; + let writer = lock.writer_client_id.as_deref().unwrap_or("none"); + println!( + " {} [{}] open={} writer={} readers={}", + lock.port, state, lock.is_open, writer, lock.reader_count + ); + } + } + + // Display project locks + if status.project_locks.is_empty() { + println!("Project Locks: (none)"); + } else { + println!("Project Locks:"); + for lock in &status.project_locks { + let state = if lock.is_held { "HELD" } else { "FREE" }; + println!(" {} [{}]", lock.project_dir, state); + } + } + + if !status.stale_locks.is_empty() { + println!( + "\nWarning: {} stale lock(s) detected. Use 'fbuild daemon clear-locks' to clear.", + status.stale_locks.len() + ); + } + + Ok(()) +} + +pub async fn run_daemon_clear_locks(client: &DaemonClient) -> fbuild_core::Result<()> { + if !client.health().await { + println!("daemon is not running"); + return Ok(()); + } + + let result = client.clear_locks().await?; + println!("{}", result.message); + if result.cleared_count > 0 { + println!("Cleared {} lock(s)", result.cleared_count); + } + Ok(()) +} + +pub async fn run_daemon_cache_stats(client: &DaemonClient) -> fbuild_core::Result<()> { + if !client.health().await { + // Fall back to local cache stats if daemon isn't running + match fbuild_packages::DiskCache::open() { + Ok(dc) => { + let stats = dc.stats().map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to read cache stats: {}", e)) + })?; + println!("{}", stats); + } + Err(e) => { + return Err(fbuild_core::FbuildError::Other(format!( + "failed to open disk cache: {}", + e + ))); + } + } + return Ok(()); + } + + let stats = client.cache_stats().await?; + if !stats.success { + return Err(fbuild_core::FbuildError::Other(format!( + "failed to get cache stats: {}", + stats.message.as_deref().unwrap_or("unknown error") + ))); + } + println!("Disk Cache Statistics:"); + println!(" Entries: {}", stats.entry_count); + println!(" Installed: {}", format_size(stats.installed_bytes)); + println!(" Archives: {}", format_size(stats.archive_bytes)); + println!(" Total: {}", format_size(stats.total_bytes)); + println!( + " Watermarks: {} high / {} low", + format_size(stats.high_watermark), + format_size(stats.low_watermark) + ); + println!(" Archive budget: {}", format_size(stats.archive_budget)); + Ok(()) +} + +pub async fn run_daemon_gc(client: &DaemonClient) -> fbuild_core::Result<()> { + if !client.health().await { + // Fall back to local GC if daemon isn't running + let dc = fbuild_packages::DiskCache::open().map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to open disk cache: {}", e)) + })?; + let report = dc + .run_gc() + .map_err(|e| fbuild_core::FbuildError::Other(format!("GC failed: {}", e)))?; + print_gc_report(&report); + return Ok(()); + } + + let result = client.run_gc().await?; + if !result.success { + return Err(fbuild_core::FbuildError::Other(format!( + "GC failed: {}", + result.message.as_deref().unwrap_or("unknown error") + ))); + } + println!("GC complete:"); + println!( + " Installed evicted: {} ({})", + result.installed_evicted, + format_size(result.installed_bytes_freed) + ); + println!( + " Archives evicted: {} ({})", + result.archives_evicted, + format_size(result.archive_bytes_freed) + ); + println!( + " Total freed: {}", + format_size(result.total_bytes_freed) + ); + if result.orphan_files_removed > 0 { + println!(" Orphan files removed: {}", result.orphan_files_removed); + } + if result.orphan_rows_cleaned > 0 { + println!(" Orphan rows cleaned: {}", result.orphan_rows_cleaned); + } + Ok(()) +} + +pub fn print_gc_report(report: &fbuild_packages::disk_cache::GcReport) { + if report.total_bytes_freed() == 0 + && report.orphan_files_removed == 0 + && report.orphan_rows_cleaned == 0 + { + println!("GC: nothing to clean up"); + return; + } + println!("GC complete:"); + println!( + " Installed evicted: {} ({})", + report.installed_evicted, + format_size(report.installed_bytes_freed) + ); + println!( + " Archives evicted: {} ({})", + report.archives_evicted, + format_size(report.archive_bytes_freed) + ); + println!( + " Total freed: {}", + format_size(report.total_bytes_freed()) + ); + if report.orphan_files_removed > 0 { + println!(" Orphan files removed: {}", report.orphan_files_removed); + } + if report.orphan_rows_cleaned > 0 { + println!(" Orphan rows cleaned: {}", report.orphan_rows_cleaned); + } +} + +pub async fn run_daemon_kill( + client: &DaemonClient, + pid: Option, + force: bool, +) -> fbuild_core::Result<()> { + let target_pid = if let Some(p) = pid { + p + } else if client.health().await { + match client.daemon_info().await { + Ok(info) => info.pid, + Err(_) => read_pid_from_file()?, + } + } else { + read_pid_from_file()? + }; + + kill_process(target_pid, force)?; + println!("killed daemon (PID {})", target_pid); + let _ = std::fs::remove_file(fbuild_paths::get_daemon_pid_file()); + Ok(()) +} + +pub fn read_pid_from_file() -> fbuild_core::Result { + let pid_file = fbuild_paths::get_daemon_pid_file(); + if pid_file.exists() { + std::fs::read_to_string(&pid_file) + .ok() + .and_then(|s| s.trim().parse().ok()) + .ok_or_else(|| { + fbuild_core::FbuildError::DaemonError( + "could not parse PID from PID file".to_string(), + ) + }) + } else { + Err(fbuild_core::FbuildError::DaemonError( + "no daemon running and no PID file found".to_string(), + )) + } +} + +pub fn run_daemon_kill_all(force: bool) -> fbuild_core::Result<()> { + let pids = find_daemon_pids()?; + if pids.is_empty() { + println!("no fbuild-daemon processes found"); + return Ok(()); + } + + let mut killed = 0; + for pid in &pids { + match kill_process(*pid, force) { + Ok(()) => { + println!("killed daemon (PID {})", pid); + killed += 1; + } + Err(e) => { + eprintln!("failed to kill PID {}: {}", pid, e); + } + } + } + + let _ = std::fs::remove_file(fbuild_paths::get_daemon_pid_file()); + println!("killed {} daemon(s)", killed); + Ok(()) +} + +pub fn kill_process(pid: u32, force: bool) -> fbuild_core::Result<()> { + let pid_str = pid.to_string(); + let argv: Vec<&str> = if cfg!(windows) { + if force { + vec!["taskkill", "/F", "/PID", &pid_str] + } else { + vec!["taskkill", "/PID", &pid_str] + } + } else { + let signal = if force { "-9" } else { "-TERM" }; + vec!["kill", signal, &pid_str] + }; + + let output = fbuild_core::subprocess::run_command(&argv, None, None, None).map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to execute kill command: {}", e)) + })?; + + if !output.success() { + return Err(fbuild_core::FbuildError::Other(format!( + "kill failed: {}", + output.stderr.trim() + ))); + } + Ok(()) +} + +pub fn find_daemon_pids() -> fbuild_core::Result> { + if cfg!(windows) { + let output = fbuild_core::subprocess::run_command( + &[ + "tasklist", + "/FI", + "IMAGENAME eq fbuild-daemon.exe", + "/FO", + "CSV", + "/NH", + ], + None, + None, + None, + ) + .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to run tasklist: {}", e)))?; + let mut pids = Vec::new(); + for line in output.stdout.lines() { + // CSV format: "image name","PID","session name","session#","mem usage" + if line.contains("fbuild-daemon") { + let fields: Vec<&str> = line.split(',').collect(); + if fields.len() >= 2 { + let pid_str = fields[1].trim_matches('"').trim(); + if let Ok(pid) = pid_str.parse::() { + pids.push(pid); + } + } + } + } + Ok(pids) + } else { + let output = fbuild_core::subprocess::run_command( + &["pgrep", "-f", "fbuild-daemon"], + None, + None, + None, + ) + .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to run pgrep: {}", e)))?; + let pids: Vec = output + .stdout + .lines() + .filter_map(|line| line.trim().parse().ok()) + .collect(); + Ok(pids) + } +} + +pub fn format_uptime(seconds: f64) -> String { + let secs = seconds as u64; + if secs < 60 { + format!("{}s", secs) + } else if secs < 3600 { + format!("{}m {}s", secs / 60, secs % 60) + } else { + format!("{}h {}m", secs / 3600, (secs % 3600) / 60) + } +} diff --git a/crates/fbuild-cli/src/cli/deploy.rs b/crates/fbuild-cli/src/cli/deploy.rs new file mode 100644 index 00000000..f3a2a5bb --- /dev/null +++ b/crates/fbuild-cli/src/cli/deploy.rs @@ -0,0 +1,345 @@ +//! `fbuild deploy`, `fbuild monitor`, and `fbuild test-emu` handlers, +//! plus the destination/emulator resolution helpers they share. + +use super::build::open_in_browser; +use crate::daemon_client::{ + self, DaemonClient, DeployRequest, MonitorRequest, OperationResponse, TestEmuRequest, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CliEmulatorKind { + Qemu, + Avr8js, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CliDeployRoute { + Device, + Emulator(CliEmulatorKind), +} + +pub fn infer_cli_default_emulator_kind( + project_dir: &str, + environment: Option<&str>, +) -> fbuild_core::Result> { + let project_dir = std::path::Path::new(project_dir); + let config = fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")) + .map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to parse platformio.ini: {}", e)) + })?; + let env_name = environment + .map(|s| s.to_string()) + .or_else(|| config.get_default_environment().map(|s| s.to_string())) + .unwrap_or_else(|| "default".to_string()); + let env_config = config.get_env_config(&env_name).map_err(|e| { + fbuild_core::FbuildError::Other(format!("invalid environment '{}': {}", env_name, e)) + })?; + let platform_str = env_config.get("platform").cloned().unwrap_or_default(); + let Some(platform) = fbuild_core::Platform::from_platform_str(&platform_str) else { + return Ok(None); + }; + let Some(board_id) = env_config.get("board").cloned() else { + return Ok(None); + }; + let board_overrides = config.get_board_overrides(&env_name).unwrap_or_default(); + let board = fbuild_config::BoardConfig::from_board_id(&board_id, &board_overrides) + .or_else(|_| { + fbuild_config::BoardConfig::from_board_id(&board_id, &std::collections::HashMap::new()) + }) + .ok(); + Ok( + match (platform, board.as_ref().map(|board| board.mcu.as_str())) { + (fbuild_core::Platform::AtmelAvr, _) | (fbuild_core::Platform::AtmelMegaAvr, _) => { + Some(CliEmulatorKind::Avr8js) + } + (fbuild_core::Platform::Espressif32, Some(mcu)) + if mcu.eq_ignore_ascii_case("esp32s3") => + { + Some(CliEmulatorKind::Qemu) + } + _ => None, + }, + ) +} + +pub fn resolve_cli_deploy_route( + to: Option<&str>, + emulator: Option<&str>, + target: Option<&str>, + qemu: bool, + default_emulator: Option, +) -> fbuild_core::Result { + if let Some(target) = target { + return match target { + "device" => Ok(CliDeployRoute::Device), + "qemu" => Ok(CliDeployRoute::Emulator(CliEmulatorKind::Qemu)), + "avr8js" => Ok(CliDeployRoute::Emulator(CliEmulatorKind::Avr8js)), + other => Err(fbuild_core::FbuildError::Other(format!( + "unsupported deploy target '{}'", + other + ))), + }; + } + + match to.unwrap_or("device") { + "device" => { + if qemu { + return Err(fbuild_core::FbuildError::Other( + "--qemu cannot be combined with --to device".to_string(), + )); + } + if let Some(emulator) = emulator { + return Err(fbuild_core::FbuildError::Other(format!( + "--emulator {} requires --to emu", + emulator + ))); + } + Ok(CliDeployRoute::Device) + } + "emu" | "emulator" => { + let emulator = if qemu { + if let Some(explicit) = emulator { + if explicit != "qemu" { + return Err(fbuild_core::FbuildError::Other( + "--qemu cannot be combined with a different --emulator".to_string(), + )); + } + } + "qemu" + } else { + match emulator { + Some(explicit) => explicit, + None => match default_emulator { + Some(CliEmulatorKind::Qemu) => "qemu", + Some(CliEmulatorKind::Avr8js) => "avr8js", + None => { + return Err(fbuild_core::FbuildError::Other( + "--to emu requires an explicit --emulator for this board" + .to_string(), + )) + } + }, + } + }; + match emulator { + "qemu" => Ok(CliDeployRoute::Emulator(CliEmulatorKind::Qemu)), + "avr8js" => Ok(CliDeployRoute::Emulator(CliEmulatorKind::Avr8js)), + other => Err(fbuild_core::FbuildError::Other(format!( + "unsupported emulator '{}'", + other + ))), + } + } + other => Err(fbuild_core::FbuildError::Other(format!( + "unsupported deploy destination '{}'", + other + ))), + } +} + +#[allow(clippy::too_many_arguments)] +pub async fn run_deploy( + project_dir: String, + environment: Option, + port: Option, + clean: bool, + monitor_after: bool, + verbose: bool, + timeout: Option, + halt_on_error: Option, + halt_on_success: Option, + expect: Option, + no_timestamp: bool, + skip_build: bool, + qemu: bool, + qemu_timeout: u32, + baud_rate: Option, + to: Option, + emulator: Option, + target: Option, + output_dir: Option, +) -> fbuild_core::Result<()> { + daemon_client::ensure_daemon_running().await?; + let client = DaemonClient::new(); + + let default_emulator = if matches!(to.as_deref(), Some("emu" | "emulator")) + && emulator.is_none() + && target.is_none() + && !qemu + { + infer_cli_default_emulator_kind(&project_dir, environment.as_deref())? + } else { + None + }; + let deploy_route = resolve_cli_deploy_route( + to.as_deref(), + emulator.as_deref(), + target.as_deref(), + qemu, + default_emulator, + )?; + + let (caller_pid, caller_cwd) = daemon_client::caller_info(); + let req = DeployRequest { + project_dir, + environment, + port, + monitor_after, + skip_build, + clean_build: clean, + verbose, + monitor_timeout: timeout, + monitor_halt_on_error: halt_on_error, + monitor_halt_on_success: halt_on_success, + monitor_expect: expect, + monitor_show_timestamp: !no_timestamp, + baud_rate, + to, + emulator, + target, + qemu, + qemu_timeout, + request_id: None, + caller_pid, + caller_cwd, + src_dir: std::env::var("PLATFORMIO_SRC_DIR") + .ok() + .filter(|s| !s.is_empty()), + output_dir, + pio_env: daemon_client::capture_pio_env(), + }; + + let resp = client.deploy(&req).await?; + if deploy_route == CliDeployRoute::Emulator(CliEmulatorKind::Qemu) + || deploy_route == CliDeployRoute::Emulator(CliEmulatorKind::Avr8js) + { + print_operation_streams(&resp); + } + println!("{}", resp.message); + if !resp.success { + std::process::exit(resp.exit_code); + } + // Open browser for avr8js only when daemon returned a launch URL (non-headless mode) + if deploy_route == CliDeployRoute::Emulator(CliEmulatorKind::Avr8js) { + if let Some(url) = resp.launch_url.as_deref() { + if let Err(e) = open_in_browser(url) { + eprintln!("warning: failed to open browser: {}", e); + eprintln!("open this URL manually: {}", url); + } + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub async fn run_test_emu( + project_dir: String, + environment: Option, + verbose: bool, + timeout: Option, + halt_on_error: Option, + halt_on_success: Option, + expect: Option, + no_timestamp: bool, + emulator: Option, +) -> fbuild_core::Result<()> { + daemon_client::ensure_daemon_running().await?; + let client = DaemonClient::new(); + + let (caller_pid, caller_cwd) = daemon_client::caller_info(); + let req = TestEmuRequest { + project_dir, + environment, + verbose, + timeout, + halt_on_error, + halt_on_success, + expect, + emulator, + show_timestamp: !no_timestamp, + request_id: None, + caller_pid, + caller_cwd, + pio_env: daemon_client::capture_pio_env(), + }; + + let resp = client.test_emu(&req).await?; + print_operation_streams(&resp); + println!("{}", resp.message); + if !resp.success { + // Guarantee a non-zero exit when the daemon reports failure. A + // structured error response carries `exit_code`, but if the + // daemon handler or an intermediate proxy returns 0 alongside + // success=false (issue #130), we must still surface failure to + // the shell rather than silently exiting 0. + let code = if resp.exit_code == 0 { + 1 + } else { + resp.exit_code + }; + std::process::exit(code); + } + Ok(()) +} + +pub fn print_operation_streams(resp: &OperationResponse) { + if let Some(stdout) = resp + .stdout + .as_deref() + .filter(|text| !text.trim().is_empty()) + { + print!("{}", stdout); + if !stdout.ends_with('\n') { + println!(); + } + } + if let Some(stderr) = resp + .stderr + .as_deref() + .filter(|text| !text.trim().is_empty()) + { + eprint!("{}", stderr); + if !stderr.ends_with('\n') { + eprintln!(); + } + } +} + +#[allow(clippy::too_many_arguments)] +pub async fn run_monitor( + project_dir: String, + environment: Option, + port: Option, + baud_rate: Option, + timeout: Option, + halt_on_error: Option, + halt_on_success: Option, + expect: Option, + no_timestamp: bool, +) -> fbuild_core::Result<()> { + daemon_client::ensure_daemon_running().await?; + let client = DaemonClient::new(); + + let (caller_pid, caller_cwd) = daemon_client::caller_info(); + let req = MonitorRequest { + project_dir, + environment, + port, + baud_rate, + halt_on_error, + halt_on_success, + expect, + timeout, + show_timestamp: !no_timestamp, + request_id: None, + caller_pid, + caller_cwd, + }; + + let resp = client.monitor(&req).await?; + println!("{}", resp.message); + if !resp.success { + std::process::exit(resp.exit_code); + } + Ok(()) +} diff --git a/crates/fbuild-cli/src/cli/device.rs b/crates/fbuild-cli/src/cli/device.rs new file mode 100644 index 00000000..0f6b3265 --- /dev/null +++ b/crates/fbuild-cli/src/cli/device.rs @@ -0,0 +1,91 @@ +//! `fbuild device` subcommand: list / status / lease / release / take. + +use crate::daemon_client::{self, DaemonClient}; + +use super::args::DeviceAction; + +pub async fn run_device(action: DeviceAction) -> fbuild_core::Result<()> { + daemon_client::ensure_daemon_running().await?; + let client = DaemonClient::new(); + + match action { + DeviceAction::List { refresh } => { + let resp = client.list_devices(refresh).await?; + if resp.devices.is_empty() { + println!("no devices found"); + return Ok(()); + } + println!("{:<20} {:<12} {:<20}", "PORT", "DEVICE ID", "DESCRIPTION"); + println!("{}", "-".repeat(52)); + for dev in &resp.devices { + let id = dev.device_id.as_deref().unwrap_or("-"); + println!("{:<20} {:<12} {:<20}", dev.port, id, dev.description); + } + println!("\n{} device(s) found", resp.devices.len()); + } + DeviceAction::Status { port } => { + let resp = client.device_status(&port).await?; + if !resp.success { + eprintln!("error: {}", resp.description); + return Ok(()); + } + let connected = if resp.is_connected { + "connected" + } else { + "disconnected" + }; + println!(" {}", resp.port); + println!(" Device ID: {}", resp.device_id); + println!(" Description: {}", resp.description); + println!(" Status: {}", connected); + println!( + " Available: {}", + if resp.available_for_exclusive { + "yes" + } else { + "no" + } + ); + if let Some(ref holder) = resp.exclusive_holder { + println!(" Exclusive holder: {}", holder); + } + if resp.monitor_count > 0 { + println!(" Monitor sessions: {}", resp.monitor_count); + } + } + DeviceAction::Lease { + port, + lease_type, + description, + } => { + let resp = client + .device_lease(&port, &lease_type, &description) + .await?; + if resp.success { + println!("lease acquired on '{}'", port); + if let Some(ref id) = resp.lease_id { + println!(" lease_id: {}", id); + } + } else { + eprintln!("error: {}", resp.message); + } + } + DeviceAction::Release { port, lease_id } => { + let resp = client.device_release(&port, lease_id.as_deref()).await?; + if resp.success { + println!("{}", resp.message); + } else { + eprintln!("error: {}", resp.message); + } + } + DeviceAction::Take { port, reason } => { + let resp = client.device_preempt(&port, &reason).await?; + if resp.success { + println!("{}", resp.message); + } else { + eprintln!("error: {}", resp.message); + } + } + } + Ok(()) +} diff --git a/crates/fbuild-cli/src/cli/dispatch.rs b/crates/fbuild-cli/src/cli/dispatch.rs new file mode 100644 index 00000000..126a22d9 --- /dev/null +++ b/crates/fbuild-cli/src/cli/dispatch.rs @@ -0,0 +1,416 @@ +//! Top-level async dispatcher: parse argv, set up tracing + Ctrl+C, then +//! fan out to per-subcommand handlers in the topic modules. + +use clap::Parser; + +use crate::{daemon_client, lib_select, mcp}; + +use super::args::{rewrite_args, resolve_project_dir, Cli, Commands}; +use super::build::run_build; +use super::clang_tools::{run_clang_tool, run_iwyu}; +use super::compile_many::{ + build_ci_pio_env, normalize_ci_sketches, run_compile_many, CompileManyArgs, +}; +use super::daemon_cmd::run_daemon; +use super::deploy::{run_deploy, run_monitor, run_test_emu}; +use super::device::run_device; +use super::lnk::run_lnk; +use super::monitor_parse::parse_monitor_flags; +use super::pio::{pio_build, pio_deploy, pio_monitor}; +use super::purge::{run_purge, run_purge_gc}; +use super::reset::run_reset; +use super::show::run_show; + +pub async fn async_main() { + let cli = Cli::parse_from(rewrite_args()); + + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + + // Scan caller's environment for `PLATFORMIO_*` vars and warn about any + // that fbuild does not act on, so users aren't bitten by silent + // mis-builds. The captured map is forwarded to the daemon per request. + let pio_env = daemon_client::capture_pio_env(); + for var in fbuild_config::scan_unsupported(&pio_env) { + tracing::warn!("{} is set but not supported by fbuild (ignored)", var); + } + for var in fbuild_config::scan_warn_only(&pio_env) { + tracing::warn!("{} is set but fbuild does not act on it", var); + } + + // Handle Ctrl+C with exit code 130 (standard POSIX SIGINT behavior, matches Python) + ctrlc::set_handler(move || { + eprintln!("\nInterrupted"); + std::process::exit(130); + }) + .ok(); + + // Notify when running in dev mode (matches Python behavior) + if std::env::var("FBUILD_DEV_MODE").is_ok_and(|v| v == "1") { + eprintln!("FBUILD_DEV_MODE=1 (dev mode: port 8865, ~/.fbuild/dev/)"); + } + + // Extract top-level project_dir before matching (since match partially moves cli) + let top_level_project_dir = cli.project_dir.clone(); + + let result = match cli.command { + Some(Commands::Build { + project_dir, + environment, + clean, + verbose, + jobs, + quick, + release, + platformio, + dry_run, + target, + symbol_analysis, + no_timestamp, + output_dir, + }) => { + let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); + if platformio { + pio_build(&project_dir, environment.as_deref(), clean, verbose) + } else { + run_build( + project_dir, + environment, + clean, + verbose, + jobs, + quick, + release, + dry_run, + target, + symbol_analysis, + no_timestamp, + output_dir, + ) + .await + } + } + Some(Commands::Deploy { + project_dir, + environment, + port, + clean, + monitor, + verbose, + platformio, + timeout, + halt_on_error, + halt_on_success, + expect, + no_timestamp, + skip_build, + qemu, + qemu_timeout, + baud_rate, + to, + emulator, + target, + output_dir, + }) => { + let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); + if platformio { + pio_deploy( + &project_dir, + environment.as_deref(), + port.as_deref(), + clean, + verbose, + ) + } else { + let monitor_after = monitor.is_some(); + let parsed = monitor + .as_deref() + .filter(|s| !s.is_empty()) + .map(parse_monitor_flags) + .unwrap_or_default(); + run_deploy( + project_dir, + environment, + port, + clean, + monitor_after, + verbose, + timeout.or(parsed.timeout), + halt_on_error.or(parsed.halt_on_error), + halt_on_success.or(parsed.halt_on_success), + expect.or(parsed.expect), + no_timestamp, + skip_build, + qemu, + qemu_timeout, + baud_rate, + to, + emulator, + target, + output_dir, + ) + .await + } + } + Some(Commands::Monitor { + project_dir, + environment, + port, + baud_rate, + verbose: _, + platformio, + timeout, + halt_on_error, + halt_on_success, + expect, + no_timestamp, + }) => { + let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); + if platformio { + pio_monitor( + &project_dir, + environment.as_deref(), + port.as_deref(), + baud_rate, + ) + } else { + run_monitor( + project_dir, + environment, + port, + baud_rate, + timeout, + halt_on_error, + halt_on_success, + expect, + no_timestamp, + ) + .await + } + } + Some(Commands::Reset { + project_dir, + environment, + port, + verbose, + }) => run_reset(project_dir, environment, port, verbose), + Some(Commands::Purge { + target, + dry_run, + project_dir, + gc, + }) => { + if gc { + run_purge_gc().await + } else { + run_purge(target, dry_run, project_dir) + } + } + Some(Commands::Daemon { action }) => run_daemon(action).await, + Some(Commands::Show { + target, + no_follow, + lines, + }) => run_show(&target, !no_follow, lines), + Some(Commands::Device { action }) => run_device(action).await, + Some(Commands::Mcp) => { + let code = mcp::run_mcp_server().await; + if code == 0 { + Ok(()) + } else { + Err(fbuild_core::FbuildError::BuildFailed( + "MCP server exited with error".to_string(), + )) + } + } + Some(Commands::ClangTidy { + project_dir, + environment, + verbose, + }) => { + let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); + run_clang_tool( + fbuild_packages::toolchain::ClangComponentKind::ClangExtra, + "clang-tidy", + project_dir, + environment, + verbose, + &[], + ) + .await + } + Some(Commands::Iwyu { + project_dir, + environment, + verbose, + }) => { + let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); + run_iwyu(project_dir, environment, verbose).await + } + Some(Commands::TestEmu { + project_dir, + environment, + verbose, + timeout, + halt_on_error, + halt_on_success, + expect, + no_timestamp, + emulator, + }) => { + let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); + run_test_emu( + project_dir, + environment, + verbose, + timeout, + halt_on_error, + halt_on_success, + expect, + no_timestamp, + emulator, + ) + .await + } + Some(Commands::ClangQuery { + project_dir, + environment, + verbose, + matcher, + }) => { + let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); + let extra: Vec = matcher + .map(|m| vec!["-c".to_string(), m]) + .unwrap_or_default(); + let extra_refs: Vec<&str> = extra.iter().map(|s| s.as_str()).collect(); + run_clang_tool( + fbuild_packages::toolchain::ClangComponentKind::ClangExtra, + "clang-query", + project_dir, + environment, + verbose, + &extra_refs, + ) + .await + } + Some(Commands::Lnk { action }) => run_lnk(action, &top_level_project_dir).await, + Some(Commands::LibSelect { + project_dir, + environment, + explain, + json, + }) => { + let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); + let exit = lib_select::run( + std::path::Path::new(&project_dir), + environment.as_deref(), + explain, + json, + ); + std::process::exit(exit); + } + Some(Commands::CompileMany { + board, + framework_jobs, + sketch_jobs, + quick, + release, + verbose, + sketches, + }) => { + run_compile_many(CompileManyArgs { + board, + framework_jobs, + sketch_jobs, + quick, + release, + verbose, + sketches, + pio_env: std::collections::HashMap::new(), + }) + .await + } + Some(Commands::Ci { + board, + libs, + project_conf, + keep_build_dir: _keep_build_dir, + build_dir, + framework_jobs, + sketch_jobs, + quick, + release, + verbose, + sketches, + }) => { + if let Some(bd) = &build_dir { + eprintln!( + "warning: --build-dir {} is accepted for pio ci compatibility but not yet honored; outputs go to .fbuild/build/...", + bd + ); + } + let normalized = normalize_ci_sketches(&sketches); + let pio_env = build_ci_pio_env(&libs, project_conf.as_deref()); + run_compile_many(CompileManyArgs { + board, + framework_jobs, + sketch_jobs, + quick, + release, + verbose, + sketches: normalized, + pio_env, + }) + .await + } + None => { + // Default action: deploy with monitor (like Python fbuild) + let project_dir = cli.project_dir.unwrap_or_else(|| ".".to_string()); + if cli.platformio { + pio_deploy( + &project_dir, + cli.environment.as_deref(), + cli.port.as_deref(), + cli.clean, + cli.verbose, + ) + } else { + let monitor_after = true; + let parsed = cli + .monitor + .as_deref() + .filter(|s| !s.is_empty()) + .map(parse_monitor_flags) + .unwrap_or_default(); + run_deploy( + project_dir, + cli.environment, + cli.port, + cli.clean, + monitor_after, + cli.verbose, + cli.timeout.or(parsed.timeout), + cli.halt_on_error.or(parsed.halt_on_error), + cli.halt_on_success.or(parsed.halt_on_success), + cli.expect.or(parsed.expect), + false, + false, + false, + 30, + None, + None, + None, + None, + None, + ) + .await + } + } + }; + + if let Err(e) = result { + eprintln!("error: {}", e); + std::process::exit(1); + } +} diff --git a/crates/fbuild-cli/src/cli/lnk.rs b/crates/fbuild-cli/src/cli/lnk.rs new file mode 100644 index 00000000..ddd7e669 --- /dev/null +++ b/crates/fbuild-cli/src/cli/lnk.rs @@ -0,0 +1,220 @@ +//! `fbuild lnk` subcommands. +//! +//! - `pull` — scan + fetch every .lnk's blob into the disk cache +//! - `check` — verify every cached blob's sha256 (no network) +//! - `add` — fetch a URL once, hash it, write a new .lnk pointing at it + +use super::args::LnkAction; + +pub async fn run_lnk( + action: LnkAction, + top_level_project_dir: &Option, +) -> fbuild_core::Result<()> { + use std::io::Write; + use std::path::PathBuf; + + use fbuild_packages::lnk::{scan_for_lnk, ExtractMode, LnkFile}; + use sha2::{Digest, Sha256}; + + fn open_cache() -> fbuild_core::Result { + fbuild_packages::DiskCache::open().map_err(|e| { + fbuild_core::FbuildError::PackageError(format!("failed to open lnk disk cache: {e}")) + }) + } + + fn resolve_root(explicit: Option, fallback: &Option) -> PathBuf { + let chosen = explicit + .or_else(|| fallback.clone()) + .unwrap_or_else(|| ".".to_string()); + PathBuf::from(chosen) + } + + match action { + LnkAction::Pull { project_dir } => { + let root = resolve_root(project_dir, top_level_project_dir); + let discovered = scan_for_lnk(&root)?; + if discovered.is_empty() { + println!("no .lnk files found under {}", root.display()); + return Ok(()); + } + let cache = open_cache()?; + let mut ok = 0usize; + let mut failed = 0usize; + for d in &discovered { + match fbuild_packages::lnk::resolve(&d.lnk, &cache) { + Ok(r) => { + ok += 1; + println!( + "ok {} → {} ({})", + d.path.display(), + r.path.display(), + d.lnk.sha256 + ); + } + Err(e) => { + failed += 1; + eprintln!("FAIL {}: {}", d.path.display(), e); + } + } + } + println!( + "\nlnk pull: {ok} ok, {failed} failed (of {})", + discovered.len() + ); + if failed > 0 { + std::process::exit(1); + } + Ok(()) + } + + LnkAction::Check { project_dir } => { + let root = resolve_root(project_dir, top_level_project_dir); + let discovered = scan_for_lnk(&root)?; + if discovered.is_empty() { + println!("no .lnk files found under {}", root.display()); + return Ok(()); + } + let cache = open_cache()?; + let mut ok = 0usize; + let mut missing = 0usize; + let mut mismatched = 0usize; + for d in &discovered { + let entry = cache + .lookup( + fbuild_packages::disk_cache::Kind::LnkBlobs, + &d.lnk.url, + &d.lnk.sha256, + ) + .map_err(|e| { + fbuild_core::FbuildError::PackageError(format!( + "lnk cache lookup failed for {}: {e}", + d.path.display() + )) + })?; + let Some(entry) = entry else { + missing += 1; + println!( + "MISSING {} (run `fbuild lnk pull` to fetch)", + d.path.display() + ); + continue; + }; + let blob_path = PathBuf::from(entry.archive_path.unwrap_or_default()); + if !blob_path.exists() { + missing += 1; + println!( + "MISSING {} (cache index points at {} which is gone)", + d.path.display(), + blob_path.display() + ); + continue; + } + let bytes = std::fs::read(&blob_path).map_err(|e| { + fbuild_core::FbuildError::PackageError(format!( + "failed to read {}: {e}", + blob_path.display() + )) + })?; + let mut h = Sha256::new(); + h.update(&bytes); + let actual = format!("{:x}", h.finalize()); + if actual == d.lnk.sha256 { + ok += 1; + println!("ok {}", d.path.display()); + } else { + mismatched += 1; + println!( + "BAD {} (expected {}, got {})", + d.path.display(), + d.lnk.sha256, + actual + ); + } + } + println!( + "\nlnk check: {ok} ok, {missing} missing, {mismatched} mismatched (of {})", + discovered.len() + ); + if mismatched > 0 || missing > 0 { + std::process::exit(1); + } + Ok(()) + } + + LnkAction::Add { url, output } => { + // Determine output path before downloading so we fail early on a + // bad output spec. + let basename = url.rsplit('/').next().unwrap_or("blob"); + let output_path = match output { + Some(p) => PathBuf::from(p), + None => PathBuf::from(format!("{basename}.lnk")), + }; + if let Some(parent) = output_path.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent).map_err(|e| { + fbuild_core::FbuildError::PackageError(format!( + "failed to create {}: {e}", + parent.display() + )) + })?; + } + } + + // Download to a temp dir, hash it, then write the .lnk. + let tmp = tempfile::tempdir().map_err(|e| { + fbuild_core::FbuildError::PackageError(format!("failed to create temp dir: {e}")) + })?; + let downloaded = fbuild_packages::downloader::download_file(&url, tmp.path()).await?; + let bytes = std::fs::read(&downloaded).map_err(|e| { + fbuild_core::FbuildError::PackageError(format!( + "failed to read downloaded file: {e}" + )) + })?; + let mut h = Sha256::new(); + h.update(&bytes); + let sha = format!("{:x}", h.finalize()); + + // Round-trip through serde so the format matches what the + // parser accepts. Also ensures a v=1 wrapper. + let lnk = LnkFile { + version: 1, + url: url.clone(), + sha256: sha.clone(), + size: Some(bytes.len() as u64), + extract: ExtractMode::File, + }; + let json = serde_json::json!({ + "v": lnk.version, + "url": lnk.url, + "sha256": lnk.sha256, + "size": lnk.size, + }); + let pretty = serde_json::to_string_pretty(&json).map_err(|e| { + fbuild_core::FbuildError::PackageError(format!( + "failed to serialize .lnk JSON: {e}" + )) + })?; + let mut f = std::fs::File::create(&output_path).map_err(|e| { + fbuild_core::FbuildError::PackageError(format!( + "failed to create {}: {e}", + output_path.display() + )) + })?; + f.write_all(pretty.as_bytes()).map_err(|e| { + fbuild_core::FbuildError::PackageError(format!( + "failed to write {}: {e}", + output_path.display() + )) + })?; + f.write_all(b"\n").ok(); + + println!( + "wrote {} ({} bytes, sha256={})", + output_path.display(), + bytes.len(), + sha + ); + Ok(()) + } + } +} diff --git a/crates/fbuild-cli/src/cli/mod.rs b/crates/fbuild-cli/src/cli/mod.rs new file mode 100644 index 00000000..4b35f416 --- /dev/null +++ b/crates/fbuild-cli/src/cli/mod.rs @@ -0,0 +1,33 @@ +//! CLI argument types and dispatch. +//! +//! This module hosts the Clap-derived `Cli` / `Commands` / subaction enums, +//! the small helpers that wire argv into them, and the `async_main` +//! dispatcher that fans each parsed subcommand out to a handler in one of +//! the topic submodules. +//! +//! The split exists purely to keep every `.rs` file in this crate under the +//! 900 LOC ceiling enforced by the LOC gate — public behavior is preserved +//! byte-for-byte. Items remain reachable at `crate::cli::::`. + +pub mod args; +pub mod build; +pub mod clang_tools; +pub mod compile_many; +pub mod daemon_cmd; +pub mod deploy; +pub mod device; +pub mod dispatch; +pub mod lnk; +pub mod monitor_parse; +pub mod pio; +pub mod purge; +pub mod reset; +pub mod show; + +#[cfg(test)] +mod tests; + +// Single re-export so `main.rs` can write `cli::async_main()` without +// pulling in the dispatcher's submodule path. All other items stay +// addressable via `crate::cli::::`. +pub use dispatch::async_main; diff --git a/crates/fbuild-cli/src/cli/monitor_parse.rs b/crates/fbuild-cli/src/cli/monitor_parse.rs new file mode 100644 index 00000000..bf9b079a --- /dev/null +++ b/crates/fbuild-cli/src/cli/monitor_parse.rs @@ -0,0 +1,109 @@ +//! Small parsers shared by the `--monitor "..."` flag, `--jobs`, and +//! anything else that needs shell-style tokenization. + +/// Parsed monitor flags extracted from a `--monitor="..."` string. +#[derive(Default)] +pub struct ParsedMonitorFlags { + pub timeout: Option, + pub halt_on_error: Option, + pub halt_on_success: Option, + pub expect: Option, +} + +/// Validate that jobs count is >= 1 (matches Python behavior). +pub fn parse_jobs(s: &str) -> Result { + let n: usize = s.parse().map_err(|e| format!("{e}"))?; + if n == 0 { + return Err("jobs must be >= 1".to_string()); + } + Ok(n) +} + +/// Parse monitor flags from a string like `--timeout 60 --halt-on-success "TEST PASSED"`. +pub fn parse_monitor_flags(s: &str) -> ParsedMonitorFlags { + let mut result = ParsedMonitorFlags::default(); + let tokens = shell_tokenize(s); + let mut i = 0; + while i < tokens.len() { + match tokens[i].as_str() { + "--timeout" | "-t" => { + if let Some(val) = tokens.get(i + 1) { + result.timeout = val.parse().ok(); + i += 1; + } + } + "--halt-on-error" => { + if let Some(val) = tokens.get(i + 1) { + result.halt_on_error = Some(val.clone()); + i += 1; + } + } + "--halt-on-success" => { + if let Some(val) = tokens.get(i + 1) { + result.halt_on_success = Some(val.clone()); + i += 1; + } + } + "--expect" => { + if let Some(val) = tokens.get(i + 1) { + result.expect = Some(val.clone()); + i += 1; + } + } + other => { + // Handle --key=value form + if let Some(rest) = other.strip_prefix("--timeout=") { + result.timeout = rest.parse().ok(); + } else if let Some(rest) = other.strip_prefix("--halt-on-error=") { + result.halt_on_error = Some(rest.to_string()); + } else if let Some(rest) = other.strip_prefix("--halt-on-success=") { + result.halt_on_success = Some(rest.to_string()); + } else if let Some(rest) = other.strip_prefix("--expect=") { + result.expect = Some(rest.to_string()); + } + } + } + i += 1; + } + result +} + +/// Simple shell-style tokenizer that handles quoted strings. +pub fn shell_tokenize(s: &str) -> Vec { + let mut tokens = Vec::new(); + let mut current = String::new(); + let mut in_single_quote = false; + let mut in_double_quote = false; + let mut escape_next = false; + + for ch in s.chars() { + if escape_next { + current.push(ch); + escape_next = false; + continue; + } + match ch { + '\\' if !in_single_quote => { + escape_next = true; + } + '\'' if !in_double_quote => { + in_single_quote = !in_single_quote; + } + '"' if !in_single_quote => { + in_double_quote = !in_double_quote; + } + ' ' | '\t' if !in_single_quote && !in_double_quote => { + if !current.is_empty() { + tokens.push(std::mem::take(&mut current)); + } + } + _ => { + current.push(ch); + } + } + } + if !current.is_empty() { + tokens.push(current); + } + tokens +} diff --git a/crates/fbuild-cli/src/cli/pio.rs b/crates/fbuild-cli/src/cli/pio.rs new file mode 100644 index 00000000..dc48aa7b --- /dev/null +++ b/crates/fbuild-cli/src/cli/pio.rs @@ -0,0 +1,131 @@ +//! PlatformIO passthrough: delegates to `pio` CLI instead of fbuild daemon. +//! +//! Used when callers pass `--platformio` to the top-level `build`, `deploy`, +//! or `monitor` subcommands so a single fbuild binary can drive either the +//! fbuild daemon or the upstream `pio` CLI for A/B comparisons. + +/// Find the `pio` binary. Checks PATH first, then the fbuild cache. +pub fn find_pio() -> fbuild_core::Result { + // Check PATH + let locator = if cfg!(windows) { "where" } else { "which" }; + if let Ok(output) = fbuild_core::subprocess::run_command(&[locator, "pio"], None, None, None) { + if output.success() { + let path = output + .stdout + .lines() + .next() + .unwrap_or("") + .trim() + .to_string(); + if !path.is_empty() { + return Ok(std::path::PathBuf::from(path)); + } + } + } + + // Check fbuild cache (PlatformIO installed via iso_env) + let cache = fbuild_paths::get_cache_root().join("platform"); + let candidates = if cfg!(windows) { + vec![ + cache.join("Scripts").join("pio.exe"), + cache.join("Scripts").join("pio"), + ] + } else { + vec![cache.join("bin").join("pio")] + }; + for c in candidates { + if c.exists() { + return Ok(c); + } + } + + Err(fbuild_core::FbuildError::Other( + "PlatformIO not found. Install it with: pip install platformio".to_string(), + )) +} + +/// Run a PlatformIO command with real-time output streaming. +pub fn run_pio_command(args: &[&str]) -> fbuild_core::Result<()> { + let pio = find_pio()?; + let pio_str = pio.to_string_lossy(); + let mut argv: Vec<&str> = vec![pio_str.as_ref()]; + argv.extend_from_slice(args); + let code = fbuild_core::subprocess::run_command_passthrough(&argv, None, None, None) + .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to run pio: {}", e)))?; + + if code != 0 { + std::process::exit(code); + } + Ok(()) +} + +pub fn pio_build( + project_dir: &str, + environment: Option<&str>, + clean: bool, + verbose: bool, +) -> fbuild_core::Result<()> { + if clean { + let mut args = vec!["run", "--target", "clean", "-d", project_dir]; + if let Some(env) = environment { + args.extend(["-e", env]); + } + let _ = run_pio_command(&args); + } + let mut args = vec!["run", "-d", project_dir]; + if let Some(env) = environment { + args.extend(["-e", env]); + } + if verbose { + args.push("-v"); + } + run_pio_command(&args) +} + +pub fn pio_deploy( + project_dir: &str, + environment: Option<&str>, + port: Option<&str>, + clean: bool, + verbose: bool, +) -> fbuild_core::Result<()> { + if clean { + let mut args = vec!["run", "--target", "clean", "-d", project_dir]; + if let Some(env) = environment { + args.extend(["-e", env]); + } + let _ = run_pio_command(&args); + } + let mut args = vec!["run", "--target", "upload", "-d", project_dir]; + if let Some(env) = environment { + args.extend(["-e", env]); + } + if let Some(p) = port { + args.extend(["--upload-port", p]); + } + if verbose { + args.push("-v"); + } + run_pio_command(&args) +} + +pub fn pio_monitor( + project_dir: &str, + environment: Option<&str>, + port: Option<&str>, + baud_rate: Option, +) -> fbuild_core::Result<()> { + let baud_str; + let mut args = vec!["device", "monitor", "-d", project_dir]; + if let Some(env) = environment { + args.extend(["-e", env]); + } + if let Some(p) = port { + args.extend(["--port", p]); + } + if let Some(b) = baud_rate { + baud_str = b.to_string(); + args.extend(["--baud", &baud_str]); + } + run_pio_command(&args) +} diff --git a/crates/fbuild-cli/src/cli/purge.rs b/crates/fbuild-cli/src/cli/purge.rs new file mode 100644 index 00000000..1326dab8 --- /dev/null +++ b/crates/fbuild-cli/src/cli/purge.rs @@ -0,0 +1,197 @@ +//! `fbuild purge` and `fbuild purge --gc` handlers plus the byte/size +//! formatting helpers they share with the daemon subcommands. + +use crate::daemon_client::DaemonClient; + +use super::daemon_cmd::print_gc_report; + +pub async fn run_purge_gc() -> fbuild_core::Result<()> { + // Try to route GC through the daemon to respect its gc_mutex. + let client = DaemonClient::new(); + if client.health().await { + let result = client.run_gc().await?; + if !result.success { + return Err(fbuild_core::FbuildError::Other(format!( + "GC failed: {}", + result.message.as_deref().unwrap_or("unknown error") + ))); + } + println!("GC complete (via daemon):"); + println!( + " Installed evicted: {} ({})", + result.installed_evicted, + format_size(result.installed_bytes_freed) + ); + println!( + " Archives evicted: {} ({})", + result.archives_evicted, + format_size(result.archive_bytes_freed) + ); + println!( + " Total freed: {}", + format_size(result.total_bytes_freed) + ); + if result.orphan_files_removed > 0 { + println!(" Orphan files removed: {}", result.orphan_files_removed); + } + if result.orphan_rows_cleaned > 0 { + println!(" Orphan rows cleaned: {}", result.orphan_rows_cleaned); + } + return Ok(()); + } + + // No daemon running — safe to run GC locally. + match fbuild_packages::DiskCache::open() { + Ok(dc) => match dc.run_gc() { + Ok(report) => { + print_gc_report(&report); + Ok(()) + } + Err(e) => Err(fbuild_core::FbuildError::Other(format!("GC failed: {}", e))), + }, + Err(e) => Err(fbuild_core::FbuildError::Other(format!( + "failed to open disk cache: {}", + e + ))), + } +} + +pub fn run_purge( + target: Option, + dry_run: bool, + project_dir: Option, +) -> fbuild_core::Result<()> { + let cache_root = fbuild_paths::get_cache_root(); + + match target.as_deref() { + None => { + // No target: list cached packages (matches Python behavior) + list_cached_packages(&cache_root)?; + std::process::exit(1); + } + Some("all") => { + // Purge entire global cache + purge_dir(&cache_root, dry_run)?; + } + Some("project") => { + // Purge project-local .fbuild/ directory + let pd = project_dir.as_deref().unwrap_or("."); + let fbuild_dir = fbuild_paths::get_project_fbuild_dir(std::path::Path::new(pd)); + purge_dir(&fbuild_dir, dry_run)?; + } + Some(t) => { + // Purge specific cache subdirectory (e.g., environment name) + let path = cache_root.join(t); + if !path.exists() { + eprintln!("target not found: {}", path.display()); + return Ok(()); + } + purge_dir(&path, dry_run)?; + } + } + Ok(()) +} + +pub fn purge_dir(path: &std::path::Path, dry_run: bool) -> fbuild_core::Result<()> { + if !path.exists() { + println!("nothing to purge: {}", path.display()); + return Ok(()); + } + let size = dir_size(path); + if dry_run { + println!("would remove: {} ({})", path.display(), format_size(size)); + } else { + std::fs::remove_dir_all(path).map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to remove {}: {}", path.display(), e)) + })?; + println!("removed: {} ({})", path.display(), format_size(size)); + } + Ok(()) +} + +pub fn list_cached_packages(cache_root: &std::path::Path) -> fbuild_core::Result<()> { + if !cache_root.exists() { + println!("No cached packages found at {}", cache_root.display()); + println!("\nUsage:"); + println!(" fbuild purge all Remove all cached packages"); + println!(" fbuild purge project Remove project build artifacts (.fbuild/)"); + println!(" fbuild purge Remove specific cache subdirectory"); + println!(" fbuild purge ... --dry-run Show what would be removed"); + return Ok(()); + } + + let mut total_size: u64 = 0; + let mut total_count: usize = 0; + + // Walk top-level type directories (toolchains, platforms, frameworks, etc.) + let mut entries: Vec<_> = std::fs::read_dir(cache_root) + .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to read cache dir: {}", e)))? + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false)) + .collect(); + entries.sort_by_key(|e| e.file_name()); + + for type_entry in entries { + let type_name = type_entry.file_name(); + let type_path = type_entry.path(); + + // Collect packages within this type directory + let mut packages: Vec<(String, u64)> = Vec::new(); + if let Ok(subdirs) = std::fs::read_dir(&type_path) { + for sub in subdirs.filter_map(|e| e.ok()) { + let sub_path = sub.path(); + if sub_path.is_dir() { + let name = sub.file_name().to_string_lossy().to_string(); + let size = dir_size(&sub_path); + packages.push((name, size)); + } + } + } + packages.sort_by(|a, b| a.0.cmp(&b.0)); + + if !packages.is_empty() { + println!("{}:", type_name.to_string_lossy().to_uppercase()); + for (name, size) in &packages { + println!(" {} ({})", name, format_size(*size)); + total_size += size; + total_count += 1; + } + println!(); + } + } + + println!( + "Total: {} package(s), {}", + total_count, + format_size(total_size) + ); + println!("\nUse 'fbuild purge all' to remove all, or 'fbuild purge ' for specific."); + Ok(()) +} + +pub fn dir_size(path: &std::path::Path) -> u64 { + let mut size = 0u64; + if let Ok(entries) = std::fs::read_dir(path) { + for entry in entries.filter_map(|e| e.ok()) { + let p = entry.path(); + if p.is_dir() { + size += dir_size(&p); + } else if let Ok(meta) = p.metadata() { + size += meta.len(); + } + } + } + size +} + +pub fn format_size(bytes: u64) -> String { + if bytes < 1024 { + format!("{} B", bytes) + } else if bytes < 1024 * 1024 { + format!("{:.1} KB", bytes as f64 / 1024.0) + } else if bytes < 1024 * 1024 * 1024 { + format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)) + } else { + format!("{:.2} GB", bytes as f64 / (1024.0 * 1024.0 * 1024.0)) + } +} diff --git a/crates/fbuild-cli/src/cli/reset.rs b/crates/fbuild-cli/src/cli/reset.rs new file mode 100644 index 00000000..428bbd5f --- /dev/null +++ b/crates/fbuild-cli/src/cli/reset.rs @@ -0,0 +1,53 @@ +//! `fbuild reset` — reboot a device without re-flashing. + +pub fn run_reset( + project_dir: String, + environment: Option, + port: Option, + verbose: bool, +) -> fbuild_core::Result<()> { + let project_path = std::path::Path::new(&project_dir); + let ini_path = project_path.join("platformio.ini"); + + // Read config to detect board/platform + let config = fbuild_config::PlatformIOConfig::from_path(&ini_path)?; + let env_name = if let Some(ref e) = environment { + e.clone() + } else { + config + .get_default_environment() + .ok_or_else(|| { + fbuild_core::FbuildError::ConfigError( + "no environment found in platformio.ini".to_string(), + ) + })? + .to_string() + }; + + let env_config = config.get_env_config(&env_name)?; + let board = env_config.get("board").ok_or_else(|| { + fbuild_core::FbuildError::ConfigError(format!( + "no 'board' key in environment '{}'", + env_name + )) + })?; + + let platform = fbuild_deploy::reset::detect_platform_for_reset(board); + + // Determine port + let port = port.ok_or_else(|| { + fbuild_core::FbuildError::SerialError("no serial port specified (use --port)".to_string()) + })?; + + println!("resetting {} device on {}...", platform, port); + match fbuild_deploy::reset::reset_device(platform, &port, verbose)? { + true => { + println!("device reset successful"); + Ok(()) + } + false => { + eprintln!("device reset failed"); + std::process::exit(1); + } + } +} diff --git a/crates/fbuild-cli/src/cli/show.rs b/crates/fbuild-cli/src/cli/show.rs new file mode 100644 index 00000000..b6caf94f --- /dev/null +++ b/crates/fbuild-cli/src/cli/show.rs @@ -0,0 +1,58 @@ +//! `fbuild show ` plus the daemon log tail used by both `show +//! daemon` and `daemon monitor`. + +pub fn run_show(target: &str, follow: bool, lines: usize) -> fbuild_core::Result<()> { + match target { + "daemon" => show_daemon_logs(follow, lines), + other => { + eprintln!("unknown show target: '{}' (available: daemon)", other); + std::process::exit(1); + } + } +} + +pub fn show_daemon_logs(follow: bool, initial_lines: usize) -> fbuild_core::Result<()> { + let log_path = fbuild_paths::get_daemon_log_file(); + if !log_path.exists() { + eprintln!("daemon log file not found: {}", log_path.display()); + eprintln!("the daemon may not have been started yet"); + return Ok(()); + } + + let content = std::fs::read_to_string(&log_path) + .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to read log file: {}", e)))?; + + // Show last N lines + let all_lines: Vec<&str> = content.lines().collect(); + let start = all_lines.len().saturating_sub(initial_lines); + for line in &all_lines[start..] { + println!("{}", line); + } + + if !follow { + return Ok(()); + } + + // Follow mode: poll for new content + println!("--- following {} (Ctrl+C to stop) ---", log_path.display()); + let mut pos = content.len() as u64; + loop { + std::thread::sleep(std::time::Duration::from_millis(100)); + let current_len = std::fs::metadata(&log_path).map(|m| m.len()).unwrap_or(pos); + + if current_len > pos { + use std::io::{Read, Seek}; + if let Ok(mut file) = std::fs::File::open(&log_path) { + let _ = file.seek(std::io::SeekFrom::Start(pos)); + let mut buf = String::new(); + if file.read_to_string(&mut buf).is_ok() && !buf.is_empty() { + print!("{}", buf); + } + pos = current_len; + } + } else if current_len < pos { + // Log file was truncated/rotated — re-read from start + pos = 0; + } + } +} diff --git a/crates/fbuild-cli/src/cli/tests.rs b/crates/fbuild-cli/src/cli/tests.rs new file mode 100644 index 00000000..e3347850 --- /dev/null +++ b/crates/fbuild-cli/src/cli/tests.rs @@ -0,0 +1,140 @@ +//! Unit tests for CLI argument normalization and `fbuild ci` parsing. + +use super::args::{Cli, Commands}; +use super::compile_many::{build_ci_pio_env, normalize_ci_sketch_entry, normalize_ci_sketches}; +use clap::Parser; + +#[test] +fn normalize_ino_path_strips_to_parent_dir() { + let entry = "examples/Blink/Blink.ino"; + let got = normalize_ci_sketch_entry(entry); + assert_eq!( + got, + std::path::Path::new("examples/Blink").to_string_lossy() + ); +} + +#[test] +fn normalize_ino_path_is_case_insensitive() { + let entry = "examples/Blink/Blink.INO"; + let got = normalize_ci_sketch_entry(entry); + assert_eq!( + got, + std::path::Path::new("examples/Blink").to_string_lossy() + ); +} + +#[test] +fn normalize_passes_through_project_dirs() { + let entry = "examples/Blink"; + assert_eq!(normalize_ci_sketch_entry(entry), "examples/Blink"); +} + +#[test] +fn normalize_bare_ino_becomes_dot() { + let entry = "Blink.ino"; + assert_eq!(normalize_ci_sketch_entry(entry), "."); +} + +#[test] +fn normalize_batch_preserves_order() { + let entries = vec![ + "examples/Blink/Blink.ino".to_string(), + "examples/Fire2012".to_string(), + ]; + let got = normalize_ci_sketches(&entries); + assert_eq!(got.len(), 2); + assert_eq!( + got[0], + std::path::Path::new("examples/Blink").to_string_lossy() + ); + assert_eq!(got[1], "examples/Fire2012"); +} + +#[test] +fn build_pio_env_joins_libs_with_platform_separator() { + let libs = vec!["a".to_string(), "b".to_string()]; + let env = build_ci_pio_env(&libs, None); + let expected = if cfg!(windows) { "a;b" } else { "a:b" }; + assert_eq!( + env.get("PLATFORMIO_LIB_EXTRA_DIRS").map(String::as_str), + Some(expected) + ); + assert!(!env.contains_key("PLATFORMIO_PROJECT_CONFIG")); +} + +#[test] +fn build_pio_env_omits_libs_key_when_empty() { + let env = build_ci_pio_env(&[], None); + assert!(env.is_empty()); +} + +#[test] +fn build_pio_env_falls_back_to_as_given_when_canonicalize_fails() { + let bogus = "/this/path/does/not/exist/conf.ini"; + let env = build_ci_pio_env(&[], Some(bogus)); + assert_eq!( + env.get("PLATFORMIO_PROJECT_CONFIG").map(String::as_str), + Some(bogus) + ); +} + +#[test] +fn ci_subcommand_round_trips_through_clap() { + let argv = [ + "fbuild", + "ci", + "--board", + "uno", + "--lib", + "./libs", + "--lib", + "./more", + "-c", + "custom.ini", + "examples/Blink/Blink.ino", + ]; + let cli = Cli::try_parse_from(argv).expect("parse"); + match cli.command { + Some(Commands::Ci { + board, + libs, + project_conf, + sketches, + .. + }) => { + assert_eq!(board, "uno"); + assert_eq!(libs, vec!["./libs".to_string(), "./more".to_string()]); + assert_eq!(project_conf.as_deref(), Some("custom.ini")); + assert_eq!(sketches, vec!["examples/Blink/Blink.ino".to_string()]); + } + _ => panic!("expected Ci subcommand"), + } +} + +#[test] +fn ci_short_board_flag_b_is_accepted() { + let argv = ["fbuild", "ci", "-b", "uno", "examples/Blink"]; + let cli = Cli::try_parse_from(argv).expect("parse"); + match cli.command { + Some(Commands::Ci { + board, sketches, .. + }) => { + assert_eq!(board, "uno"); + assert_eq!(sketches, vec!["examples/Blink".to_string()]); + } + _ => panic!("expected Ci subcommand"), + } +} + +#[test] +fn ci_requires_at_least_one_sketch() { + let argv = ["fbuild", "ci", "--board", "uno"]; + assert!(Cli::try_parse_from(argv).is_err()); +} + +#[test] +fn ci_quick_and_release_are_mutually_exclusive() { + let argv = ["fbuild", "ci", "-b", "uno", "--quick", "--release", "."]; + assert!(Cli::try_parse_from(argv).is_err()); +} diff --git a/crates/fbuild-cli/src/main.rs b/crates/fbuild-cli/src/main.rs index 267f1b1f..b53f3402 100644 --- a/crates/fbuild-cli/src/main.rs +++ b/crates/fbuild-cli/src/main.rs @@ -1,550 +1,8 @@ +mod cli; mod daemon_client; mod lib_select; mod mcp; -use clap::{Parser, Subcommand}; -use daemon_client::{BuildRequest, DaemonClient, DeployRequest, MonitorRequest, TestEmuRequest}; - -#[derive(Parser)] -#[command( - name = "fbuild", - version, - about = "PlatformIO-compatible embedded build tool" -)] -struct Cli { - #[command(subcommand)] - command: Option, - - /// Project directory (positional, for `fbuild `) - project_dir: Option, - - /// Target environment - #[arg(short = 'e', long)] - environment: Option, - - /// Verbose output - #[arg(short, long)] - verbose: bool, - - /// Serial port (e.g., COM5, /dev/ttyUSB0) - #[arg(short = 'p', long)] - port: Option, - - /// Clean build before deploy - #[arg(short = 'c', long)] - clean: bool, - - /// Monitor after deploy; optionally pass flags as a string - #[arg(long, num_args = 0..=1, default_missing_value = "")] - monitor: Option, - - /// Use PlatformIO compatibility mode - #[arg(long)] - platformio: bool, - - /// Monitor timeout in seconds - #[arg(long)] - timeout: Option, - - /// Halt monitor on error pattern - #[arg(long)] - halt_on_error: Option, - - /// Halt monitor on success pattern - #[arg(long)] - halt_on_success: Option, - - /// Expected output pattern for monitor - #[arg(long)] - expect: Option, -} - -#[derive(Subcommand)] -enum Commands { - /// Build firmware - Build { - project_dir: Option, - #[arg(short = 'e', long)] - environment: Option, - #[arg(short = 'c', long)] - clean: bool, - #[arg(short, long)] - verbose: bool, - #[arg(short = 'j', long, value_parser = parse_jobs)] - jobs: Option, - #[arg(long, group = "build_profile")] - quick: bool, - #[arg(long, group = "build_profile")] - release: bool, - #[arg(long)] - platformio: bool, - /// Verify daemon starts and environment resolves, but skip the actual build - #[arg(long)] - dry_run: bool, - /// Build target: 'compiledb' generates compile_commands.json without compiling - #[arg(short = 't', long, value_parser = ["compiledb"])] - target: Option, - /// Run per-symbol memory analysis after building; optionally write report to PATH - /// instead of streaming to console - #[arg(long, num_args = 0..=1, default_missing_value = "")] - symbol_analysis: Option, - /// Disable elapsed-time prefix on build output lines - #[arg(long)] - no_timestamp: bool, - /// Export build artifacts to a tooling-friendly directory - #[arg(long)] - output_dir: Option, - }, - /// Deploy firmware to device - Deploy { - project_dir: Option, - #[arg(short = 'e', long)] - environment: Option, - #[arg(short = 'p', long)] - port: Option, - #[arg(short = 'c', long)] - clean: bool, - /// Monitor after deploy; optionally pass flags as a string - /// e.g., --monitor="--timeout 60 --halt-on-success \"TEST PASSED\"" - #[arg(long, num_args = 0..=1, default_missing_value = "")] - monitor: Option, - #[arg(short, long)] - verbose: bool, - #[arg(long)] - platformio: bool, - #[arg(long)] - timeout: Option, - #[arg(long)] - halt_on_error: Option, - #[arg(long)] - halt_on_success: Option, - #[arg(long)] - expect: Option, - /// Disable timestamp prefix on monitor output lines - #[arg(long)] - no_timestamp: bool, - /// Skip the build step and deploy existing firmware (upload-only mode) - #[arg(long)] - skip_build: bool, - /// Deploy to the native QEMU emulator instead of a physical device - #[arg(long)] - qemu: bool, - /// Timeout in seconds for QEMU execution (default: 30) - #[arg(long, default_value = "30")] - qemu_timeout: u32, - /// Override the board's default upload baud rate - #[arg(short = 'b', long = "baud", alias = "baud-rate")] - baud_rate: Option, - /// Deploy destination: device (default) or emulator - #[arg(long = "to", value_parser = ["device", "emu", "emulator"])] - to: Option, - /// Emulator backend when deploying to `emu` - #[arg(long, value_parser = ["avr8js", "qemu", "simavr"])] - emulator: Option, - /// Legacy deploy target alias: device, qemu, or avr8js - #[arg(long, value_parser = ["device", "qemu", "avr8js"], hide = true)] - target: Option, - /// Export build artifacts to a tooling-friendly directory - #[arg(long)] - output_dir: Option, - }, - /// Monitor serial output - Monitor { - project_dir: Option, - #[arg(short = 'e', long)] - environment: Option, - #[arg(short = 'p', long)] - port: Option, - #[arg(short = 'b', long = "baud", alias = "baud-rate")] - baud_rate: Option, - #[arg(short, long)] - verbose: bool, - #[arg(long)] - platformio: bool, - #[arg(long)] - timeout: Option, - #[arg(long)] - halt_on_error: Option, - #[arg(long)] - halt_on_success: Option, - #[arg(long)] - expect: Option, - /// Disable timestamp prefix on each output line - #[arg(long)] - no_timestamp: bool, - }, - /// Reset device without re-flashing - Reset { - /// Project directory - #[arg(default_value = ".")] - project_dir: String, - /// Target environment - #[arg(short = 'e', long)] - environment: Option, - /// Serial port (e.g., COM5, /dev/ttyUSB0) - #[arg(short = 'p', long)] - port: Option, - /// Verbose output - #[arg(short, long)] - verbose: bool, - }, - /// Purge cached packages - Purge { - target: Option, - #[arg(long)] - dry_run: bool, - #[arg(long)] - project_dir: Option, - /// Run LRU garbage collection instead of full purge - #[arg(long)] - gc: bool, - }, - /// Manage the fbuild daemon - Daemon { - #[command(subcommand)] - action: DaemonAction, - }, - /// Show daemon logs or other information - Show { - /// What to show (currently only 'daemon' for daemon logs) - target: String, - /// Don't follow the log file (just print last lines and exit) - #[arg(long)] - no_follow: bool, - /// Number of lines to show initially (default: 50) - #[arg(long, default_value = "50")] - lines: usize, - }, - /// Manage connected devices - Device { - #[command(subcommand)] - action: DeviceAction, - }, - /// Start MCP (Model Context Protocol) server for AI assistant integration - Mcp, - /// Run clang-tidy static analysis on project sources - ClangTidy { - project_dir: Option, - #[arg(short = 'e', long)] - environment: Option, - #[arg(short, long)] - verbose: bool, - }, - /// Run include-what-you-use analysis on project sources - Iwyu { - project_dir: Option, - #[arg(short = 'e', long)] - environment: Option, - #[arg(short, long)] - verbose: bool, - }, - /// Build firmware and run it in an emulator for testing - TestEmu { - project_dir: Option, - #[arg(short = 'e', long)] - environment: Option, - #[arg(short, long)] - verbose: bool, - /// Timeout in seconds for the emulator run - #[arg(long)] - timeout: Option, - /// Halt on error pattern (regex) - #[arg(long)] - halt_on_error: Option, - /// Halt on success pattern (regex) - #[arg(long)] - halt_on_success: Option, - /// Expected output pattern (regex) - #[arg(long)] - expect: Option, - /// Disable timestamp prefix on output lines - #[arg(long)] - no_timestamp: bool, - /// Emulator backend: "qemu", "avr8js", or "simavr" (auto-detected if omitted) - #[arg(long, value_parser = ["avr8js", "qemu", "simavr"])] - emulator: Option, - }, - /// Run clang-query on project sources - ClangQuery { - project_dir: Option, - #[arg(short = 'e', long)] - environment: Option, - #[arg(short, long)] - verbose: bool, - /// clang-query matcher expression - #[arg(short = 'm', long)] - matcher: Option, - }, - /// Manage `.lnk` resource pointers (fetch / verify / add). - /// - /// `.lnk` files are tiny JSON manifests checked into source control - /// that point at remote binary blobs (sha256-verified). At build time - /// fbuild downloads + caches them; this command lets you operate on - /// them outside of a build. - Lnk { - #[command(subcommand)] - action: LnkAction, - }, - /// Diagnostic: drive the LDF-style library-selection resolver and print - /// the selected library set. Useful for debugging FastLED/fbuild#202 / - /// `#204`-style "library not found" issues without running a full build. - LibSelect { - /// Project directory (defaults to "."). - project_dir: Option, - /// Target environment. - #[arg(short = 'e', long)] - environment: Option, - /// Show selection origin per library, unresolved headers, etc. - #[arg(long, conflicts_with = "json")] - explain: bool, - /// Emit machine-readable JSON instead of plain text. - #[arg(long, conflicts_with = "explain")] - json: bool, - }, - /// Two-stage compile of many sketches against the same board - /// (FastLED/fbuild#238). Builds the framework + library archives once - /// (stage 1) and fans out per-sketch compile + link in parallel - /// (stage 2). Independent parallelism knobs for each stage so memory- - /// heavy framework work stays modest while sketch work saturates cores. - CompileMany { - /// Board id (e.g. "uno", "teensy41"). Used to dispatch to the - /// right platform orchestrator and to pick the matching - /// `[env:]` (or first env with `board = `) inside - /// each sketch's `platformio.ini`. - #[arg(long)] - board: String, - /// Parallelism for stage 1 (framework + library compile). When - /// omitted, defaults to `min(cores, 2)`. - #[arg(long)] - framework_jobs: Option, - /// Parallelism for stage 2 (per-sketch compile + link). When - /// omitted, defaults to `cores`. - #[arg(long)] - sketch_jobs: Option, - /// Build profile. - #[arg(long, group = "compile_many_profile")] - quick: bool, - #[arg(long, group = "compile_many_profile")] - release: bool, - /// Verbose compiler output. - #[arg(short, long)] - verbose: bool, - /// Sketch project directories (each must contain `platformio.ini`). - #[arg(required = true)] - sketches: Vec, - }, - /// PlatformIO-compatible CI command (FastLED/fbuild#242). Drop-in - /// replacement for `pio ci` that delegates to the `compile-many` - /// two-stage primitive (FastLED/fbuild#241). - Ci { - /// Board id (e.g. "uno", "teensy41"). Matches `pio ci --board`. - #[arg(short = 'b', long)] - board: String, - /// Extra library search directory (repeatable). Mapped to the - /// `PLATFORMIO_LIB_EXTRA_DIRS` env var, ';' separated on Windows - /// and ':' separated elsewhere. Matches `pio ci --lib`. - #[arg(short = 'l', long = "lib")] - libs: Vec, - /// Path to a `platformio.ini` to use instead of the per-sketch - /// one. Mapped to `PLATFORMIO_PROJECT_CONFIG`. Matches - /// `pio ci --project-conf`. - #[arg(short = 'c', long = "project-conf")] - project_conf: Option, - /// Accepted for compatibility with `pio ci`; build directories - /// are always kept under `.fbuild/build/...` (no-op). - #[arg(long)] - keep_build_dir: bool, - /// Accepted for compatibility with `pio ci`. Not yet honored; - /// emits a warning when set. - #[arg(long)] - build_dir: Option, - /// Parallelism for stage 1 (framework + library compile). - #[arg(long)] - framework_jobs: Option, - /// Parallelism for stage 2 (per-sketch compile + link). - #[arg(long)] - sketch_jobs: Option, - /// Build profile. - #[arg(long, group = "ci_profile")] - quick: bool, - #[arg(long, group = "ci_profile")] - release: bool, - /// Verbose compiler output. - #[arg(short, long)] - verbose: bool, - /// Sketches to build. Each entry is either a project directory - /// containing `platformio.ini` or a `.ino` file whose parent - /// directory is the project. Matches `pio ci` positional args. - #[arg(required = true)] - sketches: Vec, - }, -} - -/// Subcommands for `fbuild lnk`. -#[derive(Subcommand)] -enum LnkAction { - /// Walk the current dir (or a project root) and fetch every `.lnk` - /// referenced blob into the disk cache. Cache hits are no-ops. - Pull { - /// Project root to scan. Defaults to the current directory. - project_dir: Option, - }, - /// Verify every `.lnk` blob in the cache matches its sha256, without - /// touching the network. Reports mismatches; exits non-zero on any. - Check { - /// Project root to scan. Defaults to the current directory. - project_dir: Option, - }, - /// Download a URL once, compute its sha256, and write a new `.lnk` - /// JSON pointing at it. Useful for adding new resources without - /// hand-editing JSON. - Add { - /// URL to download. - url: String, - /// Where to write the `.lnk` file. Defaults to the URL's basename - /// + `.lnk` in the current directory. - #[arg(short = 'o', long)] - output: Option, - }, -} - -#[derive(Subcommand)] -enum DeviceAction { - /// List all connected devices - List { - /// Refresh device discovery before listing - #[arg(long)] - refresh: bool, - }, - /// Show detailed status of a device - Status { - /// Serial port (e.g. COM3, /dev/ttyUSB0) - port: String, - }, - /// Acquire a lease on a device - Lease { - /// Serial port (e.g. COM3, /dev/ttyUSB0) - port: String, - /// Lease type: "exclusive" (default) or "monitor" - #[arg(short = 't', long, default_value = "exclusive")] - lease_type: String, - /// Description for the lease - #[arg(short, long, default_value = "")] - description: String, - }, - /// Release a lease on a device - Release { - /// Serial port (e.g. COM3, /dev/ttyUSB0) - port: String, - /// Specific lease ID to release (releases all if omitted) - #[arg(long)] - lease_id: Option, - }, - /// Forcibly take a device from the current holder - Take { - /// Serial port (e.g. COM3, /dev/ttyUSB0) - port: String, - /// Mandatory reason for preemption - #[arg(short, long)] - reason: String, - }, -} - -#[derive(Subcommand)] -enum DaemonAction { - /// Stop the daemon gracefully - Stop, - /// Show daemon status - Status, - /// Restart the daemon (stop then start) - Restart, - /// List running daemon instances - List, - /// Kill a daemon process (bypasses graceful shutdown) - Kill { - /// PID of the daemon to kill (auto-detected if omitted) - #[arg(long)] - pid: Option, - /// Force kill (SIGKILL/TerminateProcess) instead of graceful termination - #[arg(short, long)] - force: bool, - }, - /// Kill all fbuild-daemon processes - KillAll { - /// Force kill (SIGKILL/TerminateProcess) instead of graceful termination - #[arg(short, long)] - force: bool, - }, - /// Show lock status (project locks, serial sessions) - Locks, - /// Clear stale locks - ClearLocks, - /// Show disk cache statistics - CacheStats, - /// Run disk cache garbage collection - Gc, - /// Tail daemon logs (alias for `fbuild show daemon`) - Monitor { - /// Don't follow the log file (just print last lines and exit) - #[arg(long)] - no_follow: bool, - /// Number of lines to show initially - #[arg(long, default_value = "50")] - lines: usize, - }, -} - -/// Resolve project_dir: prefer the subcommand's value, fall back to the top-level positional arg, -/// then default to ".". This lets callers write either `fbuild build ` or `fbuild build`. -fn resolve_project_dir(subcommand_dir: Option, top_level_dir: &Option) -> String { - subcommand_dir - .or_else(|| top_level_dir.clone()) - .unwrap_or_else(|| ".".to_string()) -} - -/// Known subcommand names for arg rewriting. -const KNOWN_SUBCOMMANDS: &[&str] = &[ - "build", - "deploy", - "monitor", - "reset", - "purge", - "show", - "daemon", - "device", - "mcp", - "clang-tidy", - "iwyu", - "clang-query", - "test-emu", - "lib-select", - "compile-many", - "ci", -]; - -/// Rewrite `fbuild ...` → `fbuild ...` -/// so that both `fbuild build ` and `fbuild build` work. -fn rewrite_args() -> Vec { - let args: Vec = std::env::args().collect(); - if args.len() >= 3 { - let first = &args[1]; - let second = &args[2]; - // If first arg is NOT a subcommand and second IS, swap them - if !first.starts_with('-') - && !KNOWN_SUBCOMMANDS.contains(&first.as_str()) - && KNOWN_SUBCOMMANDS.contains(&second.as_str()) - { - let mut rewritten = Vec::with_capacity(args.len()); - rewritten.push(args[0].clone()); - rewritten.push(second.clone()); // subcommand first - rewritten.push(first.clone()); // project_dir second - rewritten.extend(args[3..].iter().cloned()); - return rewritten; - } - } - args -} - fn main() { // Trampoline through a larger-stack thread: Windows' default 1 MB main-thread // stack is not enough for clap's `--help` formatting across fbuild's full @@ -565,3123 +23,5 @@ fn async_main_entry() { .enable_all() .build() .expect("failed to build tokio runtime"); - rt.block_on(async_main()); -} - -async fn async_main() { - let cli = Cli::parse_from(rewrite_args()); - - tracing_subscriber::fmt() - .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) - .init(); - - // Scan caller's environment for `PLATFORMIO_*` vars and warn about any - // that fbuild does not act on, so users aren't bitten by silent - // mis-builds. The captured map is forwarded to the daemon per request. - let pio_env = daemon_client::capture_pio_env(); - for var in fbuild_config::scan_unsupported(&pio_env) { - tracing::warn!("{} is set but not supported by fbuild (ignored)", var); - } - for var in fbuild_config::scan_warn_only(&pio_env) { - tracing::warn!("{} is set but fbuild does not act on it", var); - } - - // Handle Ctrl+C with exit code 130 (standard POSIX SIGINT behavior, matches Python) - ctrlc::set_handler(move || { - eprintln!("\nInterrupted"); - std::process::exit(130); - }) - .ok(); - - // Notify when running in dev mode (matches Python behavior) - if std::env::var("FBUILD_DEV_MODE").is_ok_and(|v| v == "1") { - eprintln!("FBUILD_DEV_MODE=1 (dev mode: port 8865, ~/.fbuild/dev/)"); - } - - // Extract top-level project_dir before matching (since match partially moves cli) - let top_level_project_dir = cli.project_dir.clone(); - - let result = match cli.command { - Some(Commands::Build { - project_dir, - environment, - clean, - verbose, - jobs, - quick, - release, - platformio, - dry_run, - target, - symbol_analysis, - no_timestamp, - output_dir, - }) => { - let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); - if platformio { - pio_build(&project_dir, environment.as_deref(), clean, verbose) - } else { - run_build( - project_dir, - environment, - clean, - verbose, - jobs, - quick, - release, - dry_run, - target, - symbol_analysis, - no_timestamp, - output_dir, - ) - .await - } - } - Some(Commands::Deploy { - project_dir, - environment, - port, - clean, - monitor, - verbose, - platformio, - timeout, - halt_on_error, - halt_on_success, - expect, - no_timestamp, - skip_build, - qemu, - qemu_timeout, - baud_rate, - to, - emulator, - target, - output_dir, - }) => { - let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); - if platformio { - pio_deploy( - &project_dir, - environment.as_deref(), - port.as_deref(), - clean, - verbose, - ) - } else { - let monitor_after = monitor.is_some(); - let parsed = monitor - .as_deref() - .filter(|s| !s.is_empty()) - .map(parse_monitor_flags) - .unwrap_or_default(); - run_deploy( - project_dir, - environment, - port, - clean, - monitor_after, - verbose, - timeout.or(parsed.timeout), - halt_on_error.or(parsed.halt_on_error), - halt_on_success.or(parsed.halt_on_success), - expect.or(parsed.expect), - no_timestamp, - skip_build, - qemu, - qemu_timeout, - baud_rate, - to, - emulator, - target, - output_dir, - ) - .await - } - } - Some(Commands::Monitor { - project_dir, - environment, - port, - baud_rate, - verbose: _, - platformio, - timeout, - halt_on_error, - halt_on_success, - expect, - no_timestamp, - }) => { - let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); - if platformio { - pio_monitor( - &project_dir, - environment.as_deref(), - port.as_deref(), - baud_rate, - ) - } else { - run_monitor( - project_dir, - environment, - port, - baud_rate, - timeout, - halt_on_error, - halt_on_success, - expect, - no_timestamp, - ) - .await - } - } - Some(Commands::Reset { - project_dir, - environment, - port, - verbose, - }) => run_reset(project_dir, environment, port, verbose), - Some(Commands::Purge { - target, - dry_run, - project_dir, - gc, - }) => { - if gc { - run_purge_gc().await - } else { - run_purge(target, dry_run, project_dir) - } - } - Some(Commands::Daemon { action }) => run_daemon(action).await, - Some(Commands::Show { - target, - no_follow, - lines, - }) => run_show(&target, !no_follow, lines), - Some(Commands::Device { action }) => run_device(action).await, - Some(Commands::Mcp) => { - let code = mcp::run_mcp_server().await; - if code == 0 { - Ok(()) - } else { - Err(fbuild_core::FbuildError::BuildFailed( - "MCP server exited with error".to_string(), - )) - } - } - Some(Commands::ClangTidy { - project_dir, - environment, - verbose, - }) => { - let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); - run_clang_tool( - fbuild_packages::toolchain::ClangComponentKind::ClangExtra, - "clang-tidy", - project_dir, - environment, - verbose, - &[], - ) - .await - } - Some(Commands::Iwyu { - project_dir, - environment, - verbose, - }) => { - let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); - run_iwyu(project_dir, environment, verbose).await - } - Some(Commands::TestEmu { - project_dir, - environment, - verbose, - timeout, - halt_on_error, - halt_on_success, - expect, - no_timestamp, - emulator, - }) => { - let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); - run_test_emu( - project_dir, - environment, - verbose, - timeout, - halt_on_error, - halt_on_success, - expect, - no_timestamp, - emulator, - ) - .await - } - Some(Commands::ClangQuery { - project_dir, - environment, - verbose, - matcher, - }) => { - let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); - let extra: Vec = matcher - .map(|m| vec!["-c".to_string(), m]) - .unwrap_or_default(); - let extra_refs: Vec<&str> = extra.iter().map(|s| s.as_str()).collect(); - run_clang_tool( - fbuild_packages::toolchain::ClangComponentKind::ClangExtra, - "clang-query", - project_dir, - environment, - verbose, - &extra_refs, - ) - .await - } - Some(Commands::Lnk { action }) => run_lnk(action, &top_level_project_dir).await, - Some(Commands::LibSelect { - project_dir, - environment, - explain, - json, - }) => { - let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); - let exit = lib_select::run( - std::path::Path::new(&project_dir), - environment.as_deref(), - explain, - json, - ); - std::process::exit(exit); - } - Some(Commands::CompileMany { - board, - framework_jobs, - sketch_jobs, - quick, - release, - verbose, - sketches, - }) => { - run_compile_many(CompileManyArgs { - board, - framework_jobs, - sketch_jobs, - quick, - release, - verbose, - sketches, - pio_env: std::collections::HashMap::new(), - }) - .await - } - Some(Commands::Ci { - board, - libs, - project_conf, - keep_build_dir: _keep_build_dir, - build_dir, - framework_jobs, - sketch_jobs, - quick, - release, - verbose, - sketches, - }) => { - if let Some(bd) = &build_dir { - eprintln!( - "warning: --build-dir {} is accepted for pio ci compatibility but not yet honored; outputs go to .fbuild/build/...", - bd - ); - } - let normalized = normalize_ci_sketches(&sketches); - let pio_env = build_ci_pio_env(&libs, project_conf.as_deref()); - run_compile_many(CompileManyArgs { - board, - framework_jobs, - sketch_jobs, - quick, - release, - verbose, - sketches: normalized, - pio_env, - }) - .await - } - None => { - // Default action: deploy with monitor (like Python fbuild) - let project_dir = cli.project_dir.unwrap_or_else(|| ".".to_string()); - if cli.platformio { - pio_deploy( - &project_dir, - cli.environment.as_deref(), - cli.port.as_deref(), - cli.clean, - cli.verbose, - ) - } else { - let monitor_after = true; - let parsed = cli - .monitor - .as_deref() - .filter(|s| !s.is_empty()) - .map(parse_monitor_flags) - .unwrap_or_default(); - run_deploy( - project_dir, - cli.environment, - cli.port, - cli.clean, - monitor_after, - cli.verbose, - cli.timeout.or(parsed.timeout), - cli.halt_on_error.or(parsed.halt_on_error), - cli.halt_on_success.or(parsed.halt_on_success), - cli.expect.or(parsed.expect), - false, - false, - false, - 30, - None, - None, - None, - None, - None, - ) - .await - } - } - }; - - if let Err(e) = result { - eprintln!("error: {}", e); - std::process::exit(1); - } -} - -/// Parsed monitor flags extracted from a `--monitor="..."` string. -#[derive(Default)] -struct ParsedMonitorFlags { - timeout: Option, - halt_on_error: Option, - halt_on_success: Option, - expect: Option, -} - -/// Validate that jobs count is >= 1 (matches Python behavior). -fn parse_jobs(s: &str) -> Result { - let n: usize = s.parse().map_err(|e| format!("{e}"))?; - if n == 0 { - return Err("jobs must be >= 1".to_string()); - } - Ok(n) -} - -/// Parse monitor flags from a string like `--timeout 60 --halt-on-success "TEST PASSED"`. -fn parse_monitor_flags(s: &str) -> ParsedMonitorFlags { - let mut result = ParsedMonitorFlags::default(); - let tokens = shell_tokenize(s); - let mut i = 0; - while i < tokens.len() { - match tokens[i].as_str() { - "--timeout" | "-t" => { - if let Some(val) = tokens.get(i + 1) { - result.timeout = val.parse().ok(); - i += 1; - } - } - "--halt-on-error" => { - if let Some(val) = tokens.get(i + 1) { - result.halt_on_error = Some(val.clone()); - i += 1; - } - } - "--halt-on-success" => { - if let Some(val) = tokens.get(i + 1) { - result.halt_on_success = Some(val.clone()); - i += 1; - } - } - "--expect" => { - if let Some(val) = tokens.get(i + 1) { - result.expect = Some(val.clone()); - i += 1; - } - } - other => { - // Handle --key=value form - if let Some(rest) = other.strip_prefix("--timeout=") { - result.timeout = rest.parse().ok(); - } else if let Some(rest) = other.strip_prefix("--halt-on-error=") { - result.halt_on_error = Some(rest.to_string()); - } else if let Some(rest) = other.strip_prefix("--halt-on-success=") { - result.halt_on_success = Some(rest.to_string()); - } else if let Some(rest) = other.strip_prefix("--expect=") { - result.expect = Some(rest.to_string()); - } - } - } - i += 1; - } - result -} - -/// Simple shell-style tokenizer that handles quoted strings. -fn shell_tokenize(s: &str) -> Vec { - let mut tokens = Vec::new(); - let mut current = String::new(); - let mut in_single_quote = false; - let mut in_double_quote = false; - let mut escape_next = false; - - for ch in s.chars() { - if escape_next { - current.push(ch); - escape_next = false; - continue; - } - match ch { - '\\' if !in_single_quote => { - escape_next = true; - } - '\'' if !in_double_quote => { - in_single_quote = !in_single_quote; - } - '"' if !in_single_quote => { - in_double_quote = !in_double_quote; - } - ' ' | '\t' if !in_single_quote && !in_double_quote => { - if !current.is_empty() { - tokens.push(std::mem::take(&mut current)); - } - } - _ => { - current.push(ch); - } - } - } - if !current.is_empty() { - tokens.push(current); - } - tokens -} - -// --------------------------------------------------------------------------- -// PlatformIO passthrough: delegates to `pio` CLI instead of fbuild daemon -// --------------------------------------------------------------------------- - -/// Find the `pio` binary. Checks PATH first, then the fbuild cache. -fn find_pio() -> fbuild_core::Result { - // Check PATH - let locator = if cfg!(windows) { "where" } else { "which" }; - if let Ok(output) = fbuild_core::subprocess::run_command(&[locator, "pio"], None, None, None) { - if output.success() { - let path = output - .stdout - .lines() - .next() - .unwrap_or("") - .trim() - .to_string(); - if !path.is_empty() { - return Ok(std::path::PathBuf::from(path)); - } - } - } - - // Check fbuild cache (PlatformIO installed via iso_env) - let cache = fbuild_paths::get_cache_root().join("platform"); - let candidates = if cfg!(windows) { - vec![ - cache.join("Scripts").join("pio.exe"), - cache.join("Scripts").join("pio"), - ] - } else { - vec![cache.join("bin").join("pio")] - }; - for c in candidates { - if c.exists() { - return Ok(c); - } - } - - Err(fbuild_core::FbuildError::Other( - "PlatformIO not found. Install it with: pip install platformio".to_string(), - )) -} - -/// Run a PlatformIO command with real-time output streaming. -fn run_pio_command(args: &[&str]) -> fbuild_core::Result<()> { - let pio = find_pio()?; - let pio_str = pio.to_string_lossy(); - let mut argv: Vec<&str> = vec![pio_str.as_ref()]; - argv.extend_from_slice(args); - let code = fbuild_core::subprocess::run_command_passthrough(&argv, None, None, None) - .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to run pio: {}", e)))?; - - if code != 0 { - std::process::exit(code); - } - Ok(()) -} - -fn pio_build( - project_dir: &str, - environment: Option<&str>, - clean: bool, - verbose: bool, -) -> fbuild_core::Result<()> { - if clean { - let mut args = vec!["run", "--target", "clean", "-d", project_dir]; - if let Some(env) = environment { - args.extend(["-e", env]); - } - let _ = run_pio_command(&args); - } - let mut args = vec!["run", "-d", project_dir]; - if let Some(env) = environment { - args.extend(["-e", env]); - } - if verbose { - args.push("-v"); - } - run_pio_command(&args) -} - -fn pio_deploy( - project_dir: &str, - environment: Option<&str>, - port: Option<&str>, - clean: bool, - verbose: bool, -) -> fbuild_core::Result<()> { - if clean { - let mut args = vec!["run", "--target", "clean", "-d", project_dir]; - if let Some(env) = environment { - args.extend(["-e", env]); - } - let _ = run_pio_command(&args); - } - let mut args = vec!["run", "--target", "upload", "-d", project_dir]; - if let Some(env) = environment { - args.extend(["-e", env]); - } - if let Some(p) = port { - args.extend(["--upload-port", p]); - } - if verbose { - args.push("-v"); - } - run_pio_command(&args) -} - -fn pio_monitor( - project_dir: &str, - environment: Option<&str>, - port: Option<&str>, - baud_rate: Option, -) -> fbuild_core::Result<()> { - let baud_str; - let mut args = vec!["device", "monitor", "-d", project_dir]; - if let Some(env) = environment { - args.extend(["-e", env]); - } - if let Some(p) = port { - args.extend(["--port", p]); - } - if let Some(b) = baud_rate { - baud_str = b.to_string(); - args.extend(["--baud", &baud_str]); - } - run_pio_command(&args) -} - -fn open_in_browser(url: &str) -> fbuild_core::Result<()> { - let args: Vec<&str> = if cfg!(target_os = "windows") { - vec!["cmd", "/c", "start", "", url] - } else if cfg!(target_os = "macos") { - vec!["open", url] - } else { - vec!["xdg-open", url] - }; - let output = fbuild_core::subprocess::run_command(&args, None, None, None) - .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to launch browser: {}", e)))?; - - if output.success() { - Ok(()) - } else { - Err(fbuild_core::FbuildError::Other(format!( - "browser launcher exited with status {}", - output.exit_code - ))) - } -} - -#[allow(clippy::too_many_arguments)] -async fn run_build( - project_dir: String, - environment: Option, - clean: bool, - verbose: bool, - jobs: Option, - quick: bool, - release: bool, - dry_run: bool, - target: Option, - symbol_analysis: Option, - no_timestamp: bool, - output_dir: Option, -) -> fbuild_core::Result<()> { - // FBUILD_PERF_LOG=1 enables coarse CLI-side timing (daemon handshake + - // round-trip). Zero-overhead when unset. - let perf_enabled = std::env::var("FBUILD_PERF_LOG") - .map(|v| !v.is_empty() && v != "0") - .unwrap_or(false); - let cli_start = std::time::Instant::now(); - let handshake_start = std::time::Instant::now(); - daemon_client::ensure_daemon_running().await?; - let handshake_elapsed = handshake_start.elapsed(); - - // Dry-run: verify daemon starts and environment resolves, then exit - if dry_run { - let project_path = std::path::Path::new(&project_dir); - let ini_path = project_path.join("platformio.ini"); - let config = fbuild_config::PlatformIOConfig::from_path(&ini_path)?; - let env_name = environment - .as_deref() - .or_else(|| config.get_default_environment()) - .unwrap_or("default"); - println!("Environment: {}", env_name); - println!("Daemon is running. Dry-run complete."); - return Ok(()); - } - - let client = DaemonClient::new(); - - let profile = if release { - Some("release".to_string()) - } else if quick { - Some("quick".to_string()) - } else { - None - }; - let generate_compiledb = target.as_deref() == Some("compiledb"); - if generate_compiledb { - let env_label = environment.as_deref().unwrap_or("default"); - println!( - "Generating compile_commands.json for environment: {}...", - env_label - ); - } - let (caller_pid, caller_cwd) = daemon_client::caller_info(); - let req = BuildRequest { - project_dir: project_dir.clone(), - environment, - clean_build: clean, - verbose, - jobs, - profile, - generate_compiledb, - compiledb_only: generate_compiledb, - request_id: None, - caller_pid, - caller_cwd, - stream: true, - symbol_analysis: symbol_analysis.is_some(), - symbol_analysis_path: symbol_analysis.filter(|s| !s.is_empty()), - no_timestamp, - src_dir: std::env::var("PLATFORMIO_SRC_DIR") - .ok() - .filter(|s| !s.is_empty()), - output_dir, - pio_env: daemon_client::capture_pio_env(), - }; - - let stream_start = std::time::Instant::now(); - let resp = client.build_streaming(&req).await?; - let stream_elapsed = stream_start.elapsed(); - if !resp.message.is_empty() { - println!("{}", resp.message); - } - if perf_enabled { - let summary = format!( - "[perf-log cli-build] daemon-handshake={} ms, server-roundtrip={} ms, total={} ms", - handshake_elapsed.as_millis(), - stream_elapsed.as_millis(), - cli_start.elapsed().as_millis(), - ); - tracing::info!(target: "fbuild_cli::perf_log", "{}", summary); - eprintln!("{}", summary); - } - if !resp.success { - std::process::exit(resp.exit_code); - } - if generate_compiledb { - let db_path = std::path::Path::new(&project_dir).join("compile_commands.json"); - if db_path.exists() { - println!("compile_commands.json written to {}", db_path.display()); - } - } - Ok(()) -} - -/// Separator used to join `PLATFORMIO_LIB_EXTRA_DIRS` entries. -/// -/// PlatformIO follows `PATH`-style conventions: ';' on Windows, ':' elsewhere. -/// Centralized here so the CLI handler and the unit tests agree. -fn ci_lib_extra_dirs_sep() -> &'static str { - if cfg!(windows) { - ";" - } else { - ":" - } -} - -/// Map a single `pio ci` positional argument to a project directory. -/// -/// `pio ci` lets callers point at either a sketch dir or directly at the -/// `.ino` file. fbuild's `compile-many` only takes project dirs, so a `.ino` -/// path is rewritten to its parent. Case-insensitive on the extension so -/// `Blink.INO` works on Windows where casing isn't preserved. -fn normalize_ci_sketch_entry(entry: &str) -> String { - let path = std::path::Path::new(entry); - let is_ino = path - .extension() - .and_then(|e| e.to_str()) - .map(|e| e.eq_ignore_ascii_case("ino")) - .unwrap_or(false); - if is_ino { - if let Some(parent) = path.parent() { - let p = parent.to_string_lossy().to_string(); - if p.is_empty() { - return ".".to_string(); - } - return p; - } - } - entry.to_string() -} - -/// Normalize every `pio ci` positional argument. -fn normalize_ci_sketches(entries: &[String]) -> Vec { - entries - .iter() - .map(|e| normalize_ci_sketch_entry(e)) - .collect() -} - -/// Build the `PLATFORMIO_*` env overlay for `fbuild ci` from `--lib` and -/// `--project-conf`. Returns an empty map when neither flag was set. -fn build_ci_pio_env( - libs: &[String], - project_conf: Option<&str>, -) -> std::collections::HashMap { - let mut env = std::collections::HashMap::new(); - if !libs.is_empty() { - env.insert( - "PLATFORMIO_LIB_EXTRA_DIRS".to_string(), - libs.join(ci_lib_extra_dirs_sep()), - ); - } - if let Some(conf) = project_conf { - let canonical = std::fs::canonicalize(conf) - .ok() - .map(|p| p.to_string_lossy().to_string()) - .unwrap_or_else(|| conf.to_string()); - env.insert("PLATFORMIO_PROJECT_CONFIG".to_string(), canonical); - } - env -} - -/// Handler for `fbuild compile-many` (FastLED/fbuild#238). -/// -/// Runs the two-stage compile-many primitive in-process via the existing -/// `fbuild-build` orchestrator. We bypass the daemon here on purpose: the -/// design goal of #238 is "one process, one toolchain load, one LDF run, -/// one framework build" — so all stage-1 + stage-2 work happens in this -/// process, parallelism is driven by the `compile_many` thread pool, and -/// no per-sketch daemon round-trip is incurred. -struct CompileManyArgs { - board: String, - framework_jobs: Option, - sketch_jobs: Option, - quick: bool, - release: bool, - verbose: bool, - sketches: Vec, - pio_env: std::collections::HashMap, -} - -async fn run_compile_many(args: CompileManyArgs) -> fbuild_core::Result<()> { - use fbuild_build::compile_many::{compile_many, CompileManyRequest, Stage}; - - let CompileManyArgs { - board, - framework_jobs, - sketch_jobs, - quick, - release, - verbose, - sketches, - pio_env, - } = args; - - let profile = if release { - fbuild_core::BuildProfile::Release - } else if quick { - fbuild_core::BuildProfile::Quick - } else { - // Default to release: matches `fbuild build`'s default profile so - // CI builds aren't silently dropped into quick mode. - fbuild_core::BuildProfile::Release - }; - - let sketches: Vec = - sketches.into_iter().map(std::path::PathBuf::from).collect(); - - let req = CompileManyRequest { - board: board.clone(), - sketches: sketches.clone(), - framework_jobs, - sketch_jobs, - profile, - verbose, - pio_env, - }; - - let effective_framework = req - .framework_jobs - .unwrap_or_else(fbuild_build::compile_many::default_framework_jobs); - let effective_sketch = req - .sketch_jobs - .unwrap_or_else(fbuild_build::compile_many::default_sketch_jobs); - println!( - "compile-many: board={} sketches={} framework_jobs={} sketch_jobs={}", - board, - sketches.len(), - effective_framework, - effective_sketch, - ); - - // `compile_many` is fully synchronous (CPU-bound). Run it on a - // blocking pool so we don't tie up the tokio runtime thread. - let result = tokio::task::spawn_blocking(move || compile_many(req)) - .await - .map_err(|e| { - fbuild_core::FbuildError::Other(format!("compile-many task panicked: {e}")) - })??; - - // Per-sketch result map suitable for the bench summary. - println!(); - println!("compile-many results:"); - for r in &result.results { - let stage_label = match r.stage { - Stage::Stage1Framework => "stage1", - Stage::Stage2Sketch => "stage2", - }; - let status = if r.success { "OK" } else { "FAIL" }; - let log_str = r - .log_path - .as_ref() - .map(|p| p.display().to_string()) - .unwrap_or_else(|| "-".to_string()); - println!( - " [{}] {} ({:.2}s) {} log={} {}", - stage_label, - status, - r.build_time_secs, - r.sketch.display(), - log_str, - r.message, - ); - } - println!(); - println!( - "compile-many summary: stage1={}/{:.2}s stage2={}/{:.2}s total={:.2}s", - result.stage1_count, - result.stage1_secs, - result.stage2_count, - result.stage2_secs, - result.total_secs, - ); - - if !result.all_success { - return Err(fbuild_core::FbuildError::BuildFailed(format!( - "compile-many: {}/{} sketches failed", - result.results.iter().filter(|r| !r.success).count(), - result.results.len(), - ))); - } - Ok(()) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum CliEmulatorKind { - Qemu, - Avr8js, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum CliDeployRoute { - Device, - Emulator(CliEmulatorKind), -} - -fn infer_cli_default_emulator_kind( - project_dir: &str, - environment: Option<&str>, -) -> fbuild_core::Result> { - let project_dir = std::path::Path::new(project_dir); - let config = fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")) - .map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to parse platformio.ini: {}", e)) - })?; - let env_name = environment - .map(|s| s.to_string()) - .or_else(|| config.get_default_environment().map(|s| s.to_string())) - .unwrap_or_else(|| "default".to_string()); - let env_config = config.get_env_config(&env_name).map_err(|e| { - fbuild_core::FbuildError::Other(format!("invalid environment '{}': {}", env_name, e)) - })?; - let platform_str = env_config.get("platform").cloned().unwrap_or_default(); - let Some(platform) = fbuild_core::Platform::from_platform_str(&platform_str) else { - return Ok(None); - }; - let Some(board_id) = env_config.get("board").cloned() else { - return Ok(None); - }; - let board_overrides = config.get_board_overrides(&env_name).unwrap_or_default(); - let board = fbuild_config::BoardConfig::from_board_id(&board_id, &board_overrides) - .or_else(|_| { - fbuild_config::BoardConfig::from_board_id(&board_id, &std::collections::HashMap::new()) - }) - .ok(); - Ok( - match (platform, board.as_ref().map(|board| board.mcu.as_str())) { - (fbuild_core::Platform::AtmelAvr, _) | (fbuild_core::Platform::AtmelMegaAvr, _) => { - Some(CliEmulatorKind::Avr8js) - } - (fbuild_core::Platform::Espressif32, Some(mcu)) - if mcu.eq_ignore_ascii_case("esp32s3") => - { - Some(CliEmulatorKind::Qemu) - } - _ => None, - }, - ) -} - -fn resolve_cli_deploy_route( - to: Option<&str>, - emulator: Option<&str>, - target: Option<&str>, - qemu: bool, - default_emulator: Option, -) -> fbuild_core::Result { - if let Some(target) = target { - return match target { - "device" => Ok(CliDeployRoute::Device), - "qemu" => Ok(CliDeployRoute::Emulator(CliEmulatorKind::Qemu)), - "avr8js" => Ok(CliDeployRoute::Emulator(CliEmulatorKind::Avr8js)), - other => Err(fbuild_core::FbuildError::Other(format!( - "unsupported deploy target '{}'", - other - ))), - }; - } - - match to.unwrap_or("device") { - "device" => { - if qemu { - return Err(fbuild_core::FbuildError::Other( - "--qemu cannot be combined with --to device".to_string(), - )); - } - if let Some(emulator) = emulator { - return Err(fbuild_core::FbuildError::Other(format!( - "--emulator {} requires --to emu", - emulator - ))); - } - Ok(CliDeployRoute::Device) - } - "emu" | "emulator" => { - let emulator = if qemu { - if let Some(explicit) = emulator { - if explicit != "qemu" { - return Err(fbuild_core::FbuildError::Other( - "--qemu cannot be combined with a different --emulator".to_string(), - )); - } - } - "qemu" - } else { - match emulator { - Some(explicit) => explicit, - None => match default_emulator { - Some(CliEmulatorKind::Qemu) => "qemu", - Some(CliEmulatorKind::Avr8js) => "avr8js", - None => { - return Err(fbuild_core::FbuildError::Other( - "--to emu requires an explicit --emulator for this board" - .to_string(), - )) - } - }, - } - }; - match emulator { - "qemu" => Ok(CliDeployRoute::Emulator(CliEmulatorKind::Qemu)), - "avr8js" => Ok(CliDeployRoute::Emulator(CliEmulatorKind::Avr8js)), - other => Err(fbuild_core::FbuildError::Other(format!( - "unsupported emulator '{}'", - other - ))), - } - } - other => Err(fbuild_core::FbuildError::Other(format!( - "unsupported deploy destination '{}'", - other - ))), - } -} - -#[allow(clippy::too_many_arguments)] -async fn run_deploy( - project_dir: String, - environment: Option, - port: Option, - clean: bool, - monitor_after: bool, - verbose: bool, - timeout: Option, - halt_on_error: Option, - halt_on_success: Option, - expect: Option, - no_timestamp: bool, - skip_build: bool, - qemu: bool, - qemu_timeout: u32, - baud_rate: Option, - to: Option, - emulator: Option, - target: Option, - output_dir: Option, -) -> fbuild_core::Result<()> { - daemon_client::ensure_daemon_running().await?; - let client = DaemonClient::new(); - - let default_emulator = if matches!(to.as_deref(), Some("emu" | "emulator")) - && emulator.is_none() - && target.is_none() - && !qemu - { - infer_cli_default_emulator_kind(&project_dir, environment.as_deref())? - } else { - None - }; - let deploy_route = resolve_cli_deploy_route( - to.as_deref(), - emulator.as_deref(), - target.as_deref(), - qemu, - default_emulator, - )?; - - let (caller_pid, caller_cwd) = daemon_client::caller_info(); - let req = DeployRequest { - project_dir, - environment, - port, - monitor_after, - skip_build, - clean_build: clean, - verbose, - monitor_timeout: timeout, - monitor_halt_on_error: halt_on_error, - monitor_halt_on_success: halt_on_success, - monitor_expect: expect, - monitor_show_timestamp: !no_timestamp, - baud_rate, - to, - emulator, - target, - qemu, - qemu_timeout, - request_id: None, - caller_pid, - caller_cwd, - src_dir: std::env::var("PLATFORMIO_SRC_DIR") - .ok() - .filter(|s| !s.is_empty()), - output_dir, - pio_env: daemon_client::capture_pio_env(), - }; - - let resp = client.deploy(&req).await?; - if deploy_route == CliDeployRoute::Emulator(CliEmulatorKind::Qemu) - || deploy_route == CliDeployRoute::Emulator(CliEmulatorKind::Avr8js) - { - print_operation_streams(&resp); - } - println!("{}", resp.message); - if !resp.success { - std::process::exit(resp.exit_code); - } - // Open browser for avr8js only when daemon returned a launch URL (non-headless mode) - if deploy_route == CliDeployRoute::Emulator(CliEmulatorKind::Avr8js) { - if let Some(url) = resp.launch_url.as_deref() { - if let Err(e) = open_in_browser(url) { - eprintln!("warning: failed to open browser: {}", e); - eprintln!("open this URL manually: {}", url); - } - } - } - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -async fn run_test_emu( - project_dir: String, - environment: Option, - verbose: bool, - timeout: Option, - halt_on_error: Option, - halt_on_success: Option, - expect: Option, - no_timestamp: bool, - emulator: Option, -) -> fbuild_core::Result<()> { - daemon_client::ensure_daemon_running().await?; - let client = DaemonClient::new(); - - let (caller_pid, caller_cwd) = daemon_client::caller_info(); - let req = TestEmuRequest { - project_dir, - environment, - verbose, - timeout, - halt_on_error, - halt_on_success, - expect, - emulator, - show_timestamp: !no_timestamp, - request_id: None, - caller_pid, - caller_cwd, - pio_env: daemon_client::capture_pio_env(), - }; - - let resp = client.test_emu(&req).await?; - print_operation_streams(&resp); - println!("{}", resp.message); - if !resp.success { - // Guarantee a non-zero exit when the daemon reports failure. A - // structured error response carries `exit_code`, but if the - // daemon handler or an intermediate proxy returns 0 alongside - // success=false (issue #130), we must still surface failure to - // the shell rather than silently exiting 0. - let code = if resp.exit_code == 0 { - 1 - } else { - resp.exit_code - }; - std::process::exit(code); - } - Ok(()) -} - -fn print_operation_streams(resp: &daemon_client::OperationResponse) { - if let Some(stdout) = resp - .stdout - .as_deref() - .filter(|text| !text.trim().is_empty()) - { - print!("{}", stdout); - if !stdout.ends_with('\n') { - println!(); - } - } - if let Some(stderr) = resp - .stderr - .as_deref() - .filter(|text| !text.trim().is_empty()) - { - eprint!("{}", stderr); - if !stderr.ends_with('\n') { - eprintln!(); - } - } -} - -#[allow(clippy::too_many_arguments)] -async fn run_monitor( - project_dir: String, - environment: Option, - port: Option, - baud_rate: Option, - timeout: Option, - halt_on_error: Option, - halt_on_success: Option, - expect: Option, - no_timestamp: bool, -) -> fbuild_core::Result<()> { - daemon_client::ensure_daemon_running().await?; - let client = DaemonClient::new(); - - let (caller_pid, caller_cwd) = daemon_client::caller_info(); - let req = MonitorRequest { - project_dir, - environment, - port, - baud_rate, - halt_on_error, - halt_on_success, - expect, - timeout, - show_timestamp: !no_timestamp, - request_id: None, - caller_pid, - caller_cwd, - }; - - let resp = client.monitor(&req).await?; - println!("{}", resp.message); - if !resp.success { - std::process::exit(resp.exit_code); - } - Ok(()) -} - -/// Convert MSYS/Git-Bash paths (/c/Users/...) to native Windows paths and canonicalize. -fn normalize_path(path: &str) -> fbuild_core::Result { - let converted = if cfg!(windows) { - // /c/foo → C:\foo - let bytes = path.as_bytes(); - if bytes.len() >= 3 - && bytes[0] == b'/' - && bytes[2] == b'/' - && bytes[1].is_ascii_alphabetic() - { - let drive = (bytes[1] as char).to_ascii_uppercase(); - format!("{}:{}", drive, path[2..].replace('/', "\\")) - } else { - path.to_string() - } - } else { - path.to_string() - }; - let canon = std::fs::canonicalize(&converted).map_err(|e| { - fbuild_core::FbuildError::Other(format!("cannot resolve path '{}': {}", path, e)) - })?; - let s = canon.to_string_lossy().to_string(); - // Strip \\?\ prefix that canonicalize adds on Windows - Ok(s.strip_prefix(r"\\?\").unwrap_or(&s).to_string()) -} - -/// Run IWYU (include-what-you-use) analysis with ESP32 cross-compilation support. -/// -/// Unlike the generic `run_clang_tool()`, this: -/// 1. Downloads IWYU via ClangComponent -/// 2. Generates compile_commands.json via the fbuild daemon -/// 3. Preprocesses the database: adds GCC builtin includes, converts framework -/// `-I` to `-isystem`, removes `--target=` flags, deduplicates `-D` defines -/// 4. Writes a modified compile_commands.json to a temp dir -/// 5. Runs IWYU per-source-file with `-p ` -/// 6. Filters output to only show suggestions for files under `src/` -async fn run_iwyu( - project_dir: String, - environment: Option, - verbose: bool, -) -> fbuild_core::Result<()> { - let project_dir = normalize_path(&project_dir)?; - - // Step 1: Ensure IWYU is installed - let component = fbuild_packages::toolchain::ClangComponent::new( - fbuild_packages::toolchain::ClangComponentKind::Iwyu, - ); - let tool_path = component.get_binary("include-what-you-use").await?; - println!("Using include-what-you-use: {}", tool_path.display()); - - // Step 2: Generate compile_commands.json via fbuild daemon (skip if it already exists) - let project_path = std::path::Path::new(&project_dir); - let db_path = project_path.join("compile_commands.json"); - if db_path.exists() { - println!("Using existing compile_commands.json"); - } else { - println!("Generating compile_commands.json..."); - run_build( - project_dir.clone(), - environment.clone(), - false, - verbose, - None, - false, - false, - false, - Some("compiledb".to_string()), - None, - true, // no_timestamp: compiledb generation doesn't need timestamps - None, - ) - .await?; - if !db_path.exists() { - return Err(fbuild_core::FbuildError::Other( - "compile_commands.json was not generated".into(), - )); - } - } - let db_content = std::fs::read_to_string(&db_path).map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to read compile_commands.json: {}", e)) - })?; - let entries: Vec = serde_json::from_str(&db_content).map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to parse compile_commands.json: {}", e)) - })?; - - // Filter to project source files only. - // Source files can be under /src/ (original) or - // /.fbuild/build//*/src/ (build copies with Arduino preprocessing). - // Exclude framework/SDK files (paths containing cache/platforms/ or cache/toolchains/). - let src_dir = project_path.join("src"); - let build_src_suffix = format!("{}src", std::path::MAIN_SEPARATOR_STR); - let project_prefix = project_path - .to_string_lossy() - .replace('/', std::path::MAIN_SEPARATOR_STR); - let source_entries: Vec<&serde_json::Value> = entries - .iter() - .filter(|e| { - e.get("file") - .and_then(|f| f.as_str()) - .map(|f| { - let p = std::path::Path::new(f); - // Direct match: file is under /src/ - if p.starts_with(&src_dir) { - return true; - } - // Build copy: file is under /.fbuild/build/.../src/ - let f_normalized = f.replace('/', std::path::MAIN_SEPARATOR_STR); - f_normalized.starts_with(&project_prefix) - && f_normalized.contains(&build_src_suffix) - && !f_normalized.contains("cache") - }) - .unwrap_or(false) - }) - .collect(); - - if source_entries.is_empty() { - println!("No source files found in compile_commands.json under src/"); - return Ok(()); - } - - // Step 4: Find GCC toolchain builtin include dirs - let gcc_includes = fbuild_packages::toolchain::clang::find_gcc_builtin_include_dirs(); - if !gcc_includes.is_empty() { - println!("Found {} GCC builtin include dir(s)", gcc_includes.len()); - if verbose { - for inc in &gcc_includes { - println!(" {}", inc.display()); - } - } - } - - // Step 5: Preprocess compile_commands.json for IWYU - // Transform entries directly as JSON: remove --target=, dedup -D, convert -I to -isystem - let src_prefix = src_dir.to_string_lossy().replace('\\', "/").to_lowercase(); - let iwyu_entries: Vec = entries - .iter() - .map(|entry| { - let mut new_entry = entry.clone(); - if let Some(args) = entry.get("arguments").and_then(|a| a.as_array()) { - let mut new_args: Vec = - Vec::with_capacity(args.len() + gcc_includes.len() * 2); - let mut seen_defines = std::collections::HashSet::new(); - - for arg_val in args { - let arg = arg_val.as_str().unwrap_or(""); - - // Remove --target= flags - if arg.starts_with("--target=") { - continue; - } - - // Remove GCC-only flags unsupported by IWYU's clang - if matches!( - arg, - "-freorder-blocks" - | "-fno-jump-tables" - | "-flto" - | "-flto=auto" - | "-fno-fat-lto-objects" - | "-fuse-linker-plugin" - | "-ffat-lto-objects" - | "-mlongcalls" - | "-mdisable-hardware-atomics" - | "-fstrict-volatile-bitfields" - | "-mtext-section-literals" - | "-fno-tree-switch-conversion" - | "-mthumb-interwork" - ) || arg.starts_with("-mfix-esp32-psram-cache-strategy=") - { - continue; - } - - // Deduplicate -D flags (keep first occurrence by key) - if arg.starts_with("-D") { - let key = if let Some(eq_pos) = arg.find('=') { - &arg[..eq_pos] - } else { - arg - }; - if !seen_defines.insert(key.to_string()) { - continue; - } - } - - // Convert non-project -I to -isystem - if let Some(path) = arg.strip_prefix("-I") { - let normalized = path.replace('\\', "/").to_lowercase(); - if normalized.starts_with(&src_prefix) { - new_args.push(arg_val.clone()); - } else { - new_args.push(serde_json::Value::String("-isystem".into())); - new_args.push(serde_json::Value::String(path.to_string())); - } - continue; - } - - new_args.push(arg_val.clone()); - } - - // Append GCC toolchain builtin include dirs as -isystem - for inc in &gcc_includes { - new_args.push(serde_json::Value::String("-isystem".into())); - new_args.push(serde_json::Value::String(inc.to_string_lossy().to_string())); - } - - new_entry["arguments"] = serde_json::Value::Array(new_args); - } - new_entry - }) - .collect(); - - // Write modified compile_commands.json to .fbuild/iwyu/ for IWYU to read via -p - let iwyu_dir_path = fbuild_paths::get_project_fbuild_dir(project_path).join("iwyu"); - std::fs::create_dir_all(&iwyu_dir_path).map_err(|e| { - fbuild_core::FbuildError::Other(format!( - "failed to create {}: {}", - iwyu_dir_path.display(), - e - )) - })?; - let iwyu_db_path = iwyu_dir_path.join("compile_commands.json"); - let iwyu_json = serde_json::to_string_pretty(&iwyu_entries).map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to serialize IWYU compile database: {}", e)) - })?; - std::fs::write(&iwyu_db_path, iwyu_json).map_err(|e| { - fbuild_core::FbuildError::Other(format!( - "failed to write {}: {}", - iwyu_db_path.display(), - e - )) - })?; - // Step 6: Set up zccache-style content-addressed cache for IWYU results. - // Cache key = blake3(source_content + iwyu_entry_json) per file. - let cache_dir = iwyu_dir_path.join("cache"); - std::fs::create_dir_all(&cache_dir).map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to create {}: {}", cache_dir.display(), e)) - })?; - - // Build a lookup from file path → preprocessed IWYU entry JSON for cache keying. - let iwyu_entry_map: std::collections::HashMap = iwyu_entries - .iter() - .filter_map(|e| { - let file = e.get("file")?.as_str()?.to_string(); - let json = serde_json::to_string(e).ok()?; - Some((file, json)) - }) - .collect(); - - println!( - "Running include-what-you-use on {} source file(s)...", - source_entries.len() - ); - - // Step 7: Run IWYU in parallel with caching - let jobs = std::thread::available_parallelism() - .map(|n| n.get() * 2) - .unwrap_or(4); - let semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(jobs)); - let tool_arc = std::sync::Arc::new(tool_path); - let iwyu_dir = std::sync::Arc::new(iwyu_dir_path); - let src_dir_arc = std::sync::Arc::new(src_dir.clone()); - let cache_dir_arc = std::sync::Arc::new(cache_dir); - let entry_map_arc = std::sync::Arc::new(iwyu_entry_map); - - // Collect source file paths from the filtered entries - let source_files: Vec = source_entries - .iter() - .filter_map(|e| e.get("file").and_then(|f| f.as_str()).map(String::from)) - .collect(); - - let mut handles = Vec::new(); - for file in source_files { - let sem = semaphore.clone(); - let tool = tool_arc.clone(); - let p_dir = iwyu_dir.clone(); - let src = src_dir_arc.clone(); - let cache_d = cache_dir_arc.clone(); - let emap = entry_map_arc.clone(); - let verbose_flag = verbose; - let handle = tokio::spawn(async move { - let _permit = sem.acquire().await.unwrap(); - let src_path = src.as_ref().clone(); - - // Compute blake3 cache key from source content + compile entry - let cache_key = iwyu_cache_key(&file, &emap); - let cache_path: Option = cache_key - .as_ref() - .map(|k| cache_d.join(format!("{}.txt", k))); - - // Check cache - if let Some(ref cp) = cache_path { - if cp.exists() { - if let Ok(cached) = std::fs::read_to_string(cp) { - return (file, Ok(cached), true, src_path); - } - } - } - - // Cache miss — run IWYU. Parallel async fan-out inside the CLI binary - // (no daemon containment group in this process). - // allow-direct-spawn: parallel async fan-out in CLI; no containment group here. - let mut cmd = tokio::process::Command::new(tool.as_ref()); - cmd.arg("-p").arg(p_dir.as_ref()); - cmd.arg("-Xiwyu").arg("--no_comments"); - cmd.arg("-Xiwyu").arg("--quoted_includes_first"); - cmd.arg("-Xiwyu").arg("--max_line_length=100"); - if verbose_flag { - cmd.arg("-Xiwyu").arg("--verbose=3"); - } - cmd.arg(&file); - let output = cmd.output().await; - - match output { - Ok(out) => { - let stderr = String::from_utf8_lossy(&out.stderr).to_string(); - // Store in cache on success - if let Some(ref cp) = cache_path { - let _ = std::fs::write(cp, &stderr); - } - (file, Ok(stderr), false, src_path) - } - Err(e) => (file, Err(format!("{}", e)), false, src_path), - } - }); - handles.push(handle); - } - - let mut total_suggestions = 0usize; - let mut failed_files = Vec::new(); - let mut cache_hits = 0usize; - let mut cache_misses = 0usize; - - for handle in handles { - let (file, result, cached, src_path) = handle - .await - .map_err(|e| fbuild_core::FbuildError::Other(format!("task join error: {}", e)))?; - if cached { - cache_hits += 1; - } else { - cache_misses += 1; - } - match result { - Ok(stderr) => { - let filtered = filter_iwyu_output(&stderr, &src_path); - if !filtered.trim().is_empty() { - print!("{}", filtered); - total_suggestions += filtered - .lines() - .filter(|l| l.contains("should add") || l.contains("should remove")) - .count(); - } - } - Err(e) => { - eprintln!("failed to run include-what-you-use on {}: {}", file, e); - failed_files.push(file); - } - } - } - - println!("\n--- include-what-you-use summary ---"); - println!("Suggestions: {}", total_suggestions); - println!( - "Cache: {} hit(s), {} miss(es)", - cache_hits, cache_misses - ); - if !failed_files.is_empty() { - println!("Failed: {} file(s)", failed_files.len()); - } - - if !failed_files.is_empty() { - Err(fbuild_core::FbuildError::BuildFailed( - "include-what-you-use failed on some files".into(), - )) - } else { - Ok(()) - } -} - -/// Compute a blake3 cache key for an IWYU analysis of a source file. -/// -/// The key is derived from the source file content and the preprocessed -/// compile_commands.json entry for that file (which includes all flags). -/// This mirrors zccache's content-addressed hashing strategy. -fn iwyu_cache_key( - file: &str, - entry_map: &std::collections::HashMap, -) -> Option { - let source_content = std::fs::read(file).ok()?; - let entry_json = entry_map.get(file)?; - let mut hasher = blake3::Hasher::new(); - hasher.update(b"fbuild-iwyu-cache-v1"); - hasher.update(&source_content); - hasher.update(entry_json.as_bytes()); - Some(hasher.finalize().to_hex().to_string()) -} - -/// Filter IWYU output to show only suggestions for files under `src_dir`. -/// -/// IWYU outputs blocks like: -/// ```text -/// /path/to/file.h should add these lines: -/// #include -/// -/// /path/to/file.h should remove these lines: -/// - #include -/// -/// The full include-list for /path/to/file.h: -/// #include -/// --- -/// ``` -/// -/// We only keep blocks whose file path is under `src_dir`. -fn filter_iwyu_output(output: &str, src_dir: &std::path::Path) -> String { - let src_prefix = src_dir.to_string_lossy().replace('\\', "/").to_lowercase(); - let mut result = String::new(); - let mut current_block = String::new(); - let mut block_is_user_file = false; - - for line in output.lines() { - // Detect block headers: "path/to/file should add/remove these lines:" - // or "The full include-list for path/to/file:" - let is_header = line.contains(" should add these lines") - || line.contains(" should remove these lines") - || line.starts_with("The full include-list for "); - - if is_header { - // Flush previous block if it was a user file - if block_is_user_file && !current_block.trim().is_empty() { - result.push_str(¤t_block); - result.push('\n'); - } - current_block.clear(); - - // Check if this new block's file is under src_dir - let file_path = line - .split(" should ") - .next() - .or_else(|| { - line.strip_prefix("The full include-list for ") - .and_then(|s| s.strip_suffix(':')) - }) - .unwrap_or(""); - let normalized = file_path.replace('\\', "/").to_lowercase(); - block_is_user_file = normalized.starts_with(&src_prefix); - } - - current_block.push_str(line); - current_block.push('\n'); - } - - // Flush last block - if block_is_user_file && !current_block.trim().is_empty() { - result.push_str(¤t_block); - } - - result -} - -/// Generic runner for clang-based analysis tools (clang-tidy, clang-query). -/// -/// 1. Ensure tool binary is installed via ClangComponent (downloads on demand) -/// 2. Generate compile_commands.json via fbuild daemon (build -t compiledb) -/// 3. Run tool on each source file in parallel (ncpus * 2) -#[allow(clippy::too_many_arguments)] -async fn run_clang_tool( - kind: fbuild_packages::toolchain::ClangComponentKind, - binary_name: &str, - project_dir: String, - environment: Option, - verbose: bool, - extra_args: &[&str], -) -> fbuild_core::Result<()> { - let project_dir = normalize_path(&project_dir)?; - - // Step 1: Ensure tool is installed - let component = fbuild_packages::toolchain::ClangComponent::new(kind); - let tool_path = component.get_binary(binary_name).await?; - println!("Using {}: {}", binary_name, tool_path.display()); - - // Step 2: Generate compile_commands.json via fbuild daemon - println!("Generating compile_commands.json..."); - run_build( - project_dir.clone(), - environment.clone(), - false, // clean - verbose, - None, // jobs - false, // quick - false, // release - false, // dry_run - Some("compiledb".to_string()), - None, - true, // no_timestamp: compiledb generation doesn't need timestamps - None, - ) - .await?; - - // Step 3: Read compile_commands.json to get source files - let project_path = std::path::Path::new(&project_dir); - let db_path = project_path.join("compile_commands.json"); - if !db_path.exists() { - return Err(fbuild_core::FbuildError::Other( - "compile_commands.json was not generated".into(), - )); - } - let db_content = std::fs::read_to_string(&db_path).map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to read compile_commands.json: {}", e)) - })?; - let entries: Vec = serde_json::from_str(&db_content).map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to parse compile_commands.json: {}", e)) - })?; - - // Filter to project source files only (under src/) - let src_dir = project_path.join("src"); - let source_files: Vec = entries - .iter() - .filter_map(|e| e.get("file").and_then(|f| f.as_str()).map(String::from)) - .filter(|f| { - let p = std::path::Path::new(f); - p.starts_with(&src_dir) - }) - .collect(); - - if source_files.is_empty() { - println!("No source files found in compile_commands.json under src/"); - return Ok(()); - } - println!( - "Running {} on {} source file(s)...", - binary_name, - source_files.len() - ); - - // Step 4: Run tool in parallel with ncpus * 2 - let jobs = std::thread::available_parallelism() - .map(|n| n.get() * 2) - .unwrap_or(4); - println!("Using {} parallel jobs", jobs); - - let semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(jobs)); - let project_dir_arc = std::sync::Arc::new(project_dir.clone()); - let tool_arc = std::sync::Arc::new(tool_path); - let extra_owned: Vec = extra_args.iter().map(|s| s.to_string()).collect(); - let verbose_flag = verbose; - - let mut handles = Vec::new(); - for file in source_files { - let sem = semaphore.clone(); - let tool = tool_arc.clone(); - let pd = project_dir_arc.clone(); - let extra = extra_owned.clone(); - let handle = tokio::spawn(async move { - let _permit = sem.acquire().await.unwrap(); - // allow-direct-spawn: parallel async fan-out (clang-tidy) in CLI binary. - let mut cmd = tokio::process::Command::new(tool.as_ref()); - cmd.arg("-p").arg(pd.as_ref()); - for arg in &extra { - cmd.arg(arg); - } - cmd.arg(&file); - if !verbose_flag { - cmd.arg("--quiet"); - } - let output = cmd.output().await; - (file, output) - }); - handles.push(handle); - } - - let mut total_warnings = 0usize; - let mut total_errors = 0usize; - let mut failed_files = Vec::new(); - - for handle in handles { - let (file, result) = handle - .await - .map_err(|e| fbuild_core::FbuildError::Other(format!("task join error: {}", e)))?; - match result { - Ok(output) => { - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - let combined = format!("{}{}", stdout, stderr); - if !combined.trim().is_empty() { - print!("{}", combined); - } - for line in combined.lines() { - if line.contains("warning:") { - total_warnings += 1; - } - if line.contains("error:") { - total_errors += 1; - } - } - } - Err(e) => { - eprintln!("failed to run {} on {}: {}", binary_name, file, e); - failed_files.push(file); - } - } - } - - println!("\n--- {} summary ---", binary_name); - println!("Warnings: {}", total_warnings); - println!("Errors: {}", total_errors); - if !failed_files.is_empty() { - println!("Failed: {} file(s)", failed_files.len()); - } - - if total_errors > 0 || !failed_files.is_empty() { - Err(fbuild_core::FbuildError::BuildFailed(format!( - "{} found errors", - binary_name - ))) - } else { - Ok(()) - } -} - -async fn run_purge_gc() -> fbuild_core::Result<()> { - // Try to route GC through the daemon to respect its gc_mutex. - let client = DaemonClient::new(); - if client.health().await { - let result = client.run_gc().await?; - if !result.success { - return Err(fbuild_core::FbuildError::Other(format!( - "GC failed: {}", - result.message.as_deref().unwrap_or("unknown error") - ))); - } - println!("GC complete (via daemon):"); - println!( - " Installed evicted: {} ({})", - result.installed_evicted, - format_size(result.installed_bytes_freed) - ); - println!( - " Archives evicted: {} ({})", - result.archives_evicted, - format_size(result.archive_bytes_freed) - ); - println!( - " Total freed: {}", - format_size(result.total_bytes_freed) - ); - if result.orphan_files_removed > 0 { - println!(" Orphan files removed: {}", result.orphan_files_removed); - } - if result.orphan_rows_cleaned > 0 { - println!(" Orphan rows cleaned: {}", result.orphan_rows_cleaned); - } - return Ok(()); - } - - // No daemon running — safe to run GC locally. - match fbuild_packages::DiskCache::open() { - Ok(dc) => match dc.run_gc() { - Ok(report) => { - print_gc_report(&report); - Ok(()) - } - Err(e) => Err(fbuild_core::FbuildError::Other(format!("GC failed: {}", e))), - }, - Err(e) => Err(fbuild_core::FbuildError::Other(format!( - "failed to open disk cache: {}", - e - ))), - } -} - -fn run_purge( - target: Option, - dry_run: bool, - project_dir: Option, -) -> fbuild_core::Result<()> { - let cache_root = fbuild_paths::get_cache_root(); - - match target.as_deref() { - None => { - // No target: list cached packages (matches Python behavior) - list_cached_packages(&cache_root)?; - std::process::exit(1); - } - Some("all") => { - // Purge entire global cache - purge_dir(&cache_root, dry_run)?; - } - Some("project") => { - // Purge project-local .fbuild/ directory - let pd = project_dir.as_deref().unwrap_or("."); - let fbuild_dir = fbuild_paths::get_project_fbuild_dir(std::path::Path::new(pd)); - purge_dir(&fbuild_dir, dry_run)?; - } - Some(t) => { - // Purge specific cache subdirectory (e.g., environment name) - let path = cache_root.join(t); - if !path.exists() { - eprintln!("target not found: {}", path.display()); - return Ok(()); - } - purge_dir(&path, dry_run)?; - } - } - Ok(()) -} - -fn purge_dir(path: &std::path::Path, dry_run: bool) -> fbuild_core::Result<()> { - if !path.exists() { - println!("nothing to purge: {}", path.display()); - return Ok(()); - } - let size = dir_size(path); - if dry_run { - println!("would remove: {} ({})", path.display(), format_size(size)); - } else { - std::fs::remove_dir_all(path).map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to remove {}: {}", path.display(), e)) - })?; - println!("removed: {} ({})", path.display(), format_size(size)); - } - Ok(()) -} - -fn list_cached_packages(cache_root: &std::path::Path) -> fbuild_core::Result<()> { - if !cache_root.exists() { - println!("No cached packages found at {}", cache_root.display()); - println!("\nUsage:"); - println!(" fbuild purge all Remove all cached packages"); - println!(" fbuild purge project Remove project build artifacts (.fbuild/)"); - println!(" fbuild purge Remove specific cache subdirectory"); - println!(" fbuild purge ... --dry-run Show what would be removed"); - return Ok(()); - } - - let mut total_size: u64 = 0; - let mut total_count: usize = 0; - - // Walk top-level type directories (toolchains, platforms, frameworks, etc.) - let mut entries: Vec<_> = std::fs::read_dir(cache_root) - .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to read cache dir: {}", e)))? - .filter_map(|e| e.ok()) - .filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false)) - .collect(); - entries.sort_by_key(|e| e.file_name()); - - for type_entry in entries { - let type_name = type_entry.file_name(); - let type_path = type_entry.path(); - - // Collect packages within this type directory - let mut packages: Vec<(String, u64)> = Vec::new(); - if let Ok(subdirs) = std::fs::read_dir(&type_path) { - for sub in subdirs.filter_map(|e| e.ok()) { - let sub_path = sub.path(); - if sub_path.is_dir() { - let name = sub.file_name().to_string_lossy().to_string(); - let size = dir_size(&sub_path); - packages.push((name, size)); - } - } - } - packages.sort_by(|a, b| a.0.cmp(&b.0)); - - if !packages.is_empty() { - println!("{}:", type_name.to_string_lossy().to_uppercase()); - for (name, size) in &packages { - println!(" {} ({})", name, format_size(*size)); - total_size += size; - total_count += 1; - } - println!(); - } - } - - println!( - "Total: {} package(s), {}", - total_count, - format_size(total_size) - ); - println!("\nUse 'fbuild purge all' to remove all, or 'fbuild purge ' for specific."); - Ok(()) -} - -fn dir_size(path: &std::path::Path) -> u64 { - let mut size = 0u64; - if let Ok(entries) = std::fs::read_dir(path) { - for entry in entries.filter_map(|e| e.ok()) { - let p = entry.path(); - if p.is_dir() { - size += dir_size(&p); - } else if let Ok(meta) = p.metadata() { - size += meta.len(); - } - } - } - size -} - -fn format_size(bytes: u64) -> String { - if bytes < 1024 { - format!("{} B", bytes) - } else if bytes < 1024 * 1024 { - format!("{:.1} KB", bytes as f64 / 1024.0) - } else if bytes < 1024 * 1024 * 1024 { - format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)) - } else { - format!("{:.2} GB", bytes as f64 / (1024.0 * 1024.0 * 1024.0)) - } -} - -async fn run_device(action: DeviceAction) -> fbuild_core::Result<()> { - daemon_client::ensure_daemon_running().await?; - let client = DaemonClient::new(); - - match action { - DeviceAction::List { refresh } => { - let resp = client.list_devices(refresh).await?; - if resp.devices.is_empty() { - println!("no devices found"); - return Ok(()); - } - println!("{:<20} {:<12} {:<20}", "PORT", "DEVICE ID", "DESCRIPTION"); - println!("{}", "-".repeat(52)); - for dev in &resp.devices { - let id = dev.device_id.as_deref().unwrap_or("-"); - println!("{:<20} {:<12} {:<20}", dev.port, id, dev.description); - } - println!("\n{} device(s) found", resp.devices.len()); - } - DeviceAction::Status { port } => { - let resp = client.device_status(&port).await?; - if !resp.success { - eprintln!("error: {}", resp.description); - return Ok(()); - } - let connected = if resp.is_connected { - "connected" - } else { - "disconnected" - }; - println!(" {}", resp.port); - println!(" Device ID: {}", resp.device_id); - println!(" Description: {}", resp.description); - println!(" Status: {}", connected); - println!( - " Available: {}", - if resp.available_for_exclusive { - "yes" - } else { - "no" - } - ); - if let Some(ref holder) = resp.exclusive_holder { - println!(" Exclusive holder: {}", holder); - } - if resp.monitor_count > 0 { - println!(" Monitor sessions: {}", resp.monitor_count); - } - } - DeviceAction::Lease { - port, - lease_type, - description, - } => { - let resp = client - .device_lease(&port, &lease_type, &description) - .await?; - if resp.success { - println!("lease acquired on '{}'", port); - if let Some(ref id) = resp.lease_id { - println!(" lease_id: {}", id); - } - } else { - eprintln!("error: {}", resp.message); - } - } - DeviceAction::Release { port, lease_id } => { - let resp = client.device_release(&port, lease_id.as_deref()).await?; - if resp.success { - println!("{}", resp.message); - } else { - eprintln!("error: {}", resp.message); - } - } - DeviceAction::Take { port, reason } => { - let resp = client.device_preempt(&port, &reason).await?; - if resp.success { - println!("{}", resp.message); - } else { - eprintln!("error: {}", resp.message); - } - } - } - Ok(()) -} - -async fn run_daemon(action: DaemonAction) -> fbuild_core::Result<()> { - let client = DaemonClient::new(); - match action { - DaemonAction::Stop => { - if !client.health().await { - println!("daemon is not running"); - return Ok(()); - } - client.shutdown().await?; - // Wait for it to actually stop - for _ in 0..50 { - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - if !client.health().await { - println!("daemon stopped"); - return Ok(()); - } - } - println!("daemon stop requested (may still be shutting down)"); - } - DaemonAction::Status => { - if client.health().await { - match client.daemon_info().await { - Ok(info) => { - let uptime = format_uptime(info.uptime_seconds); - println!("daemon is running at {}", fbuild_paths::get_daemon_url()); - println!(" PID: {}", info.pid); - println!(" Port: {}", info.port); - println!(" Uptime: {}", uptime); - println!(" Version: {}", info.version); - println!(" Mode: {}", if info.dev_mode { "dev" } else { "prod" }); - println!(" State: {}", info.daemon_state); - if info.operation_in_progress { - if let Some(ref op) = info.current_operation { - println!(" Operation: {}", op); - } else { - println!(" Operation: (in progress)"); - } - } - if info.client_count > 0 { - println!(" Clients: {}", info.client_count); - } - if let Some(ref cwd) = info.spawner_cwd { - println!(" Spawned from: {}", cwd); - } - if let Some(mtime) = info.source_mtime { - println!(" Binary mtime: {:.0}", mtime); - } - } - Err(_) => { - println!("daemon is running at {}", fbuild_paths::get_daemon_url()); - } - } - } else { - println!("daemon is not running"); - } - } - DaemonAction::Restart => { - // Stop if running - if client.health().await { - client.shutdown().await?; - for _ in 0..50 { - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - if !client.health().await { - break; - } - } - } - // Start fresh - daemon_client::ensure_daemon_running().await?; - println!("daemon restarted"); - } - DaemonAction::List => { - run_daemon_list(&client).await?; - } - DaemonAction::Kill { pid, force } => { - run_daemon_kill(&client, pid, force).await?; - } - DaemonAction::KillAll { force } => { - run_daemon_kill_all(force)?; - } - DaemonAction::Locks => { - run_daemon_locks(&client).await?; - } - DaemonAction::ClearLocks => { - run_daemon_clear_locks(&client).await?; - } - DaemonAction::CacheStats => { - run_daemon_cache_stats(&client).await?; - } - DaemonAction::Gc => { - run_daemon_gc(&client).await?; - } - DaemonAction::Monitor { no_follow, lines } => { - return run_show("daemon", !no_follow, lines); - } - } - Ok(()) -} - -async fn run_daemon_list(client: &DaemonClient) -> fbuild_core::Result<()> { - if client.health().await { - match client.daemon_info().await { - Ok(info) => { - let uptime = format_uptime(info.uptime_seconds); - println!("fbuild daemon (running)"); - println!(" PID: {}", info.pid); - println!(" Port: {}", info.port); - println!(" Uptime: {}", uptime); - println!(" Version: {}", info.version); - println!(" Mode: {}", if info.dev_mode { "dev" } else { "prod" }); - } - Err(e) => { - println!("daemon is running but info unavailable: {}", e); - } - } - } else { - println!("no daemon is running"); - // Check for stale PID file - let pid_file = fbuild_paths::get_daemon_pid_file(); - if pid_file.exists() { - if let Ok(contents) = std::fs::read_to_string(&pid_file) { - println!( - " (stale PID file: {} — PID {})", - pid_file.display(), - contents.trim() - ); - } - } - } - - // Also scan for orphan processes - let pids = find_daemon_pids()?; - if pids.len() > 1 { - println!("\nwarning: multiple fbuild-daemon processes detected:"); - for pid in &pids { - println!(" PID {}", pid); - } - println!("use 'fbuild daemon kill-all' to clean up"); - } - Ok(()) -} - -async fn run_daemon_locks(client: &DaemonClient) -> fbuild_core::Result<()> { - if !client.health().await { - println!("daemon is not running"); - return Ok(()); - } - - let status = client.lock_status().await?; - - // Display port locks - if status.port_locks.is_empty() { - println!("Port Locks: (none)"); - } else { - println!("Port Locks:"); - for lock in &status.port_locks { - let state = if lock.is_held { "HELD" } else { "FREE" }; - let writer = lock.writer_client_id.as_deref().unwrap_or("none"); - println!( - " {} [{}] open={} writer={} readers={}", - lock.port, state, lock.is_open, writer, lock.reader_count - ); - } - } - - // Display project locks - if status.project_locks.is_empty() { - println!("Project Locks: (none)"); - } else { - println!("Project Locks:"); - for lock in &status.project_locks { - let state = if lock.is_held { "HELD" } else { "FREE" }; - println!(" {} [{}]", lock.project_dir, state); - } - } - - if !status.stale_locks.is_empty() { - println!( - "\nWarning: {} stale lock(s) detected. Use 'fbuild daemon clear-locks' to clear.", - status.stale_locks.len() - ); - } - - Ok(()) -} - -async fn run_daemon_clear_locks(client: &DaemonClient) -> fbuild_core::Result<()> { - if !client.health().await { - println!("daemon is not running"); - return Ok(()); - } - - let result = client.clear_locks().await?; - println!("{}", result.message); - if result.cleared_count > 0 { - println!("Cleared {} lock(s)", result.cleared_count); - } - Ok(()) -} - -async fn run_daemon_cache_stats(client: &DaemonClient) -> fbuild_core::Result<()> { - if !client.health().await { - // Fall back to local cache stats if daemon isn't running - match fbuild_packages::DiskCache::open() { - Ok(dc) => { - let stats = dc.stats().map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to read cache stats: {}", e)) - })?; - println!("{}", stats); - } - Err(e) => { - return Err(fbuild_core::FbuildError::Other(format!( - "failed to open disk cache: {}", - e - ))); - } - } - return Ok(()); - } - - let stats = client.cache_stats().await?; - if !stats.success { - return Err(fbuild_core::FbuildError::Other(format!( - "failed to get cache stats: {}", - stats.message.as_deref().unwrap_or("unknown error") - ))); - } - println!("Disk Cache Statistics:"); - println!(" Entries: {}", stats.entry_count); - println!(" Installed: {}", format_size(stats.installed_bytes)); - println!(" Archives: {}", format_size(stats.archive_bytes)); - println!(" Total: {}", format_size(stats.total_bytes)); - println!( - " Watermarks: {} high / {} low", - format_size(stats.high_watermark), - format_size(stats.low_watermark) - ); - println!(" Archive budget: {}", format_size(stats.archive_budget)); - Ok(()) -} - -async fn run_daemon_gc(client: &DaemonClient) -> fbuild_core::Result<()> { - if !client.health().await { - // Fall back to local GC if daemon isn't running - let dc = fbuild_packages::DiskCache::open().map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to open disk cache: {}", e)) - })?; - let report = dc - .run_gc() - .map_err(|e| fbuild_core::FbuildError::Other(format!("GC failed: {}", e)))?; - print_gc_report(&report); - return Ok(()); - } - - let result = client.run_gc().await?; - if !result.success { - return Err(fbuild_core::FbuildError::Other(format!( - "GC failed: {}", - result.message.as_deref().unwrap_or("unknown error") - ))); - } - println!("GC complete:"); - println!( - " Installed evicted: {} ({})", - result.installed_evicted, - format_size(result.installed_bytes_freed) - ); - println!( - " Archives evicted: {} ({})", - result.archives_evicted, - format_size(result.archive_bytes_freed) - ); - println!( - " Total freed: {}", - format_size(result.total_bytes_freed) - ); - if result.orphan_files_removed > 0 { - println!(" Orphan files removed: {}", result.orphan_files_removed); - } - if result.orphan_rows_cleaned > 0 { - println!(" Orphan rows cleaned: {}", result.orphan_rows_cleaned); - } - Ok(()) -} - -fn print_gc_report(report: &fbuild_packages::disk_cache::GcReport) { - if report.total_bytes_freed() == 0 - && report.orphan_files_removed == 0 - && report.orphan_rows_cleaned == 0 - { - println!("GC: nothing to clean up"); - return; - } - println!("GC complete:"); - println!( - " Installed evicted: {} ({})", - report.installed_evicted, - format_size(report.installed_bytes_freed) - ); - println!( - " Archives evicted: {} ({})", - report.archives_evicted, - format_size(report.archive_bytes_freed) - ); - println!( - " Total freed: {}", - format_size(report.total_bytes_freed()) - ); - if report.orphan_files_removed > 0 { - println!(" Orphan files removed: {}", report.orphan_files_removed); - } - if report.orphan_rows_cleaned > 0 { - println!(" Orphan rows cleaned: {}", report.orphan_rows_cleaned); - } -} - -async fn run_daemon_kill( - client: &DaemonClient, - pid: Option, - force: bool, -) -> fbuild_core::Result<()> { - let target_pid = if let Some(p) = pid { - p - } else if client.health().await { - match client.daemon_info().await { - Ok(info) => info.pid, - Err(_) => read_pid_from_file()?, - } - } else { - read_pid_from_file()? - }; - - kill_process(target_pid, force)?; - println!("killed daemon (PID {})", target_pid); - let _ = std::fs::remove_file(fbuild_paths::get_daemon_pid_file()); - Ok(()) -} - -fn read_pid_from_file() -> fbuild_core::Result { - let pid_file = fbuild_paths::get_daemon_pid_file(); - if pid_file.exists() { - std::fs::read_to_string(&pid_file) - .ok() - .and_then(|s| s.trim().parse().ok()) - .ok_or_else(|| { - fbuild_core::FbuildError::DaemonError( - "could not parse PID from PID file".to_string(), - ) - }) - } else { - Err(fbuild_core::FbuildError::DaemonError( - "no daemon running and no PID file found".to_string(), - )) - } -} - -fn run_daemon_kill_all(force: bool) -> fbuild_core::Result<()> { - let pids = find_daemon_pids()?; - if pids.is_empty() { - println!("no fbuild-daemon processes found"); - return Ok(()); - } - - let mut killed = 0; - for pid in &pids { - match kill_process(*pid, force) { - Ok(()) => { - println!("killed daemon (PID {})", pid); - killed += 1; - } - Err(e) => { - eprintln!("failed to kill PID {}: {}", pid, e); - } - } - } - - let _ = std::fs::remove_file(fbuild_paths::get_daemon_pid_file()); - println!("killed {} daemon(s)", killed); - Ok(()) -} - -fn kill_process(pid: u32, force: bool) -> fbuild_core::Result<()> { - let pid_str = pid.to_string(); - let argv: Vec<&str> = if cfg!(windows) { - if force { - vec!["taskkill", "/F", "/PID", &pid_str] - } else { - vec!["taskkill", "/PID", &pid_str] - } - } else { - let signal = if force { "-9" } else { "-TERM" }; - vec!["kill", signal, &pid_str] - }; - - let output = fbuild_core::subprocess::run_command(&argv, None, None, None).map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to execute kill command: {}", e)) - })?; - - if !output.success() { - return Err(fbuild_core::FbuildError::Other(format!( - "kill failed: {}", - output.stderr.trim() - ))); - } - Ok(()) -} - -fn find_daemon_pids() -> fbuild_core::Result> { - if cfg!(windows) { - let output = fbuild_core::subprocess::run_command( - &[ - "tasklist", - "/FI", - "IMAGENAME eq fbuild-daemon.exe", - "/FO", - "CSV", - "/NH", - ], - None, - None, - None, - ) - .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to run tasklist: {}", e)))?; - let mut pids = Vec::new(); - for line in output.stdout.lines() { - // CSV format: "image name","PID","session name","session#","mem usage" - if line.contains("fbuild-daemon") { - let fields: Vec<&str> = line.split(',').collect(); - if fields.len() >= 2 { - let pid_str = fields[1].trim_matches('"').trim(); - if let Ok(pid) = pid_str.parse::() { - pids.push(pid); - } - } - } - } - Ok(pids) - } else { - let output = fbuild_core::subprocess::run_command( - &["pgrep", "-f", "fbuild-daemon"], - None, - None, - None, - ) - .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to run pgrep: {}", e)))?; - let pids: Vec = output - .stdout - .lines() - .filter_map(|line| line.trim().parse().ok()) - .collect(); - Ok(pids) - } -} - -fn format_uptime(seconds: f64) -> String { - let secs = seconds as u64; - if secs < 60 { - format!("{}s", secs) - } else if secs < 3600 { - format!("{}m {}s", secs / 60, secs % 60) - } else { - format!("{}h {}m", secs / 3600, (secs % 3600) / 60) - } -} - -fn run_show(target: &str, follow: bool, lines: usize) -> fbuild_core::Result<()> { - match target { - "daemon" => show_daemon_logs(follow, lines), - other => { - eprintln!("unknown show target: '{}' (available: daemon)", other); - std::process::exit(1); - } - } -} - -fn show_daemon_logs(follow: bool, initial_lines: usize) -> fbuild_core::Result<()> { - let log_path = fbuild_paths::get_daemon_log_file(); - if !log_path.exists() { - eprintln!("daemon log file not found: {}", log_path.display()); - eprintln!("the daemon may not have been started yet"); - return Ok(()); - } - - let content = std::fs::read_to_string(&log_path) - .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to read log file: {}", e)))?; - - // Show last N lines - let all_lines: Vec<&str> = content.lines().collect(); - let start = all_lines.len().saturating_sub(initial_lines); - for line in &all_lines[start..] { - println!("{}", line); - } - - if !follow { - return Ok(()); - } - - // Follow mode: poll for new content - println!("--- following {} (Ctrl+C to stop) ---", log_path.display()); - let mut pos = content.len() as u64; - loop { - std::thread::sleep(std::time::Duration::from_millis(100)); - let current_len = std::fs::metadata(&log_path).map(|m| m.len()).unwrap_or(pos); - - if current_len > pos { - use std::io::{Read, Seek}; - if let Ok(mut file) = std::fs::File::open(&log_path) { - let _ = file.seek(std::io::SeekFrom::Start(pos)); - let mut buf = String::new(); - if file.read_to_string(&mut buf).is_ok() && !buf.is_empty() { - print!("{}", buf); - } - pos = current_len; - } - } else if current_len < pos { - // Log file was truncated/rotated — re-read from start - pos = 0; - } - } -} - -fn run_reset( - project_dir: String, - environment: Option, - port: Option, - verbose: bool, -) -> fbuild_core::Result<()> { - let project_path = std::path::Path::new(&project_dir); - let ini_path = project_path.join("platformio.ini"); - - // Read config to detect board/platform - let config = fbuild_config::PlatformIOConfig::from_path(&ini_path)?; - let env_name = if let Some(ref e) = environment { - e.clone() - } else { - config - .get_default_environment() - .ok_or_else(|| { - fbuild_core::FbuildError::ConfigError( - "no environment found in platformio.ini".to_string(), - ) - })? - .to_string() - }; - - let env_config = config.get_env_config(&env_name)?; - let board = env_config.get("board").ok_or_else(|| { - fbuild_core::FbuildError::ConfigError(format!( - "no 'board' key in environment '{}'", - env_name - )) - })?; - - let platform = fbuild_deploy::reset::detect_platform_for_reset(board); - - // Determine port - let port = port.ok_or_else(|| { - fbuild_core::FbuildError::SerialError("no serial port specified (use --port)".to_string()) - })?; - - println!("resetting {} device on {}...", platform, port); - match fbuild_deploy::reset::reset_device(platform, &port, verbose)? { - true => { - println!("device reset successful"); - Ok(()) - } - false => { - eprintln!("device reset failed"); - std::process::exit(1); - } - } -} - -// ============================================================================ -// `fbuild lnk` subcommands -// ============================================================================ -// -// `pull` — scan + fetch every .lnk's blob into the disk cache -// `check` — verify every cached blob's sha256 (no network) -// `add` — fetch a URL once, hash it, write a new .lnk pointing at it - -async fn run_lnk( - action: LnkAction, - top_level_project_dir: &Option, -) -> fbuild_core::Result<()> { - use std::io::Write; - use std::path::PathBuf; - - use fbuild_packages::lnk::{scan_for_lnk, ExtractMode, LnkFile}; - use sha2::{Digest, Sha256}; - - fn open_cache() -> fbuild_core::Result { - fbuild_packages::DiskCache::open().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!("failed to open lnk disk cache: {e}")) - }) - } - - fn resolve_root(explicit: Option, fallback: &Option) -> PathBuf { - let chosen = explicit - .or_else(|| fallback.clone()) - .unwrap_or_else(|| ".".to_string()); - PathBuf::from(chosen) - } - - match action { - LnkAction::Pull { project_dir } => { - let root = resolve_root(project_dir, top_level_project_dir); - let discovered = scan_for_lnk(&root)?; - if discovered.is_empty() { - println!("no .lnk files found under {}", root.display()); - return Ok(()); - } - let cache = open_cache()?; - let mut ok = 0usize; - let mut failed = 0usize; - for d in &discovered { - match fbuild_packages::lnk::resolve(&d.lnk, &cache) { - Ok(r) => { - ok += 1; - println!( - "ok {} → {} ({})", - d.path.display(), - r.path.display(), - d.lnk.sha256 - ); - } - Err(e) => { - failed += 1; - eprintln!("FAIL {}: {}", d.path.display(), e); - } - } - } - println!( - "\nlnk pull: {ok} ok, {failed} failed (of {})", - discovered.len() - ); - if failed > 0 { - std::process::exit(1); - } - Ok(()) - } - - LnkAction::Check { project_dir } => { - let root = resolve_root(project_dir, top_level_project_dir); - let discovered = scan_for_lnk(&root)?; - if discovered.is_empty() { - println!("no .lnk files found under {}", root.display()); - return Ok(()); - } - let cache = open_cache()?; - let mut ok = 0usize; - let mut missing = 0usize; - let mut mismatched = 0usize; - for d in &discovered { - let entry = cache - .lookup( - fbuild_packages::disk_cache::Kind::LnkBlobs, - &d.lnk.url, - &d.lnk.sha256, - ) - .map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "lnk cache lookup failed for {}: {e}", - d.path.display() - )) - })?; - let Some(entry) = entry else { - missing += 1; - println!( - "MISSING {} (run `fbuild lnk pull` to fetch)", - d.path.display() - ); - continue; - }; - let blob_path = PathBuf::from(entry.archive_path.unwrap_or_default()); - if !blob_path.exists() { - missing += 1; - println!( - "MISSING {} (cache index points at {} which is gone)", - d.path.display(), - blob_path.display() - ); - continue; - } - let bytes = std::fs::read(&blob_path).map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to read {}: {e}", - blob_path.display() - )) - })?; - let mut h = Sha256::new(); - h.update(&bytes); - let actual = format!("{:x}", h.finalize()); - if actual == d.lnk.sha256 { - ok += 1; - println!("ok {}", d.path.display()); - } else { - mismatched += 1; - println!( - "BAD {} (expected {}, got {})", - d.path.display(), - d.lnk.sha256, - actual - ); - } - } - println!( - "\nlnk check: {ok} ok, {missing} missing, {mismatched} mismatched (of {})", - discovered.len() - ); - if mismatched > 0 || missing > 0 { - std::process::exit(1); - } - Ok(()) - } - - LnkAction::Add { url, output } => { - // Determine output path before downloading so we fail early on a - // bad output spec. - let basename = url.rsplit('/').next().unwrap_or("blob"); - let output_path = match output { - Some(p) => PathBuf::from(p), - None => PathBuf::from(format!("{basename}.lnk")), - }; - if let Some(parent) = output_path.parent() { - if !parent.as_os_str().is_empty() { - std::fs::create_dir_all(parent).map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create {}: {e}", - parent.display() - )) - })?; - } - } - - // Download to a temp dir, hash it, then write the .lnk. - let tmp = tempfile::tempdir().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!("failed to create temp dir: {e}")) - })?; - let downloaded = fbuild_packages::downloader::download_file(&url, tmp.path()).await?; - let bytes = std::fs::read(&downloaded).map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to read downloaded file: {e}" - )) - })?; - let mut h = Sha256::new(); - h.update(&bytes); - let sha = format!("{:x}", h.finalize()); - - // Round-trip through serde so the format matches what the - // parser accepts. Also ensures a v=1 wrapper. - let lnk = LnkFile { - version: 1, - url: url.clone(), - sha256: sha.clone(), - size: Some(bytes.len() as u64), - extract: ExtractMode::File, - }; - let json = serde_json::json!({ - "v": lnk.version, - "url": lnk.url, - "sha256": lnk.sha256, - "size": lnk.size, - }); - let pretty = serde_json::to_string_pretty(&json).map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to serialize .lnk JSON: {e}" - )) - })?; - let mut f = std::fs::File::create(&output_path).map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create {}: {e}", - output_path.display() - )) - })?; - f.write_all(pretty.as_bytes()).map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to write {}: {e}", - output_path.display() - )) - })?; - f.write_all(b"\n").ok(); - - println!( - "wrote {} ({} bytes, sha256={})", - output_path.display(), - bytes.len(), - sha - ); - Ok(()) - } - } -} - -#[cfg(test)] -mod ci_tests { - use super::*; - use clap::Parser; - - #[test] - fn normalize_ino_path_strips_to_parent_dir() { - let entry = "examples/Blink/Blink.ino"; - let got = normalize_ci_sketch_entry(entry); - assert_eq!( - got, - std::path::Path::new("examples/Blink").to_string_lossy() - ); - } - - #[test] - fn normalize_ino_path_is_case_insensitive() { - let entry = "examples/Blink/Blink.INO"; - let got = normalize_ci_sketch_entry(entry); - assert_eq!( - got, - std::path::Path::new("examples/Blink").to_string_lossy() - ); - } - - #[test] - fn normalize_passes_through_project_dirs() { - let entry = "examples/Blink"; - assert_eq!(normalize_ci_sketch_entry(entry), "examples/Blink"); - } - - #[test] - fn normalize_bare_ino_becomes_dot() { - let entry = "Blink.ino"; - assert_eq!(normalize_ci_sketch_entry(entry), "."); - } - - #[test] - fn normalize_batch_preserves_order() { - let entries = vec![ - "examples/Blink/Blink.ino".to_string(), - "examples/Fire2012".to_string(), - ]; - let got = normalize_ci_sketches(&entries); - assert_eq!(got.len(), 2); - assert_eq!( - got[0], - std::path::Path::new("examples/Blink").to_string_lossy() - ); - assert_eq!(got[1], "examples/Fire2012"); - } - - #[test] - fn build_pio_env_joins_libs_with_platform_separator() { - let libs = vec!["a".to_string(), "b".to_string()]; - let env = build_ci_pio_env(&libs, None); - let expected = if cfg!(windows) { "a;b" } else { "a:b" }; - assert_eq!( - env.get("PLATFORMIO_LIB_EXTRA_DIRS").map(String::as_str), - Some(expected) - ); - assert!(!env.contains_key("PLATFORMIO_PROJECT_CONFIG")); - } - - #[test] - fn build_pio_env_omits_libs_key_when_empty() { - let env = build_ci_pio_env(&[], None); - assert!(env.is_empty()); - } - - #[test] - fn build_pio_env_falls_back_to_as_given_when_canonicalize_fails() { - let bogus = "/this/path/does/not/exist/conf.ini"; - let env = build_ci_pio_env(&[], Some(bogus)); - assert_eq!( - env.get("PLATFORMIO_PROJECT_CONFIG").map(String::as_str), - Some(bogus) - ); - } - - #[test] - fn ci_subcommand_round_trips_through_clap() { - let argv = [ - "fbuild", - "ci", - "--board", - "uno", - "--lib", - "./libs", - "--lib", - "./more", - "-c", - "custom.ini", - "examples/Blink/Blink.ino", - ]; - let cli = Cli::try_parse_from(argv).expect("parse"); - match cli.command { - Some(Commands::Ci { - board, - libs, - project_conf, - sketches, - .. - }) => { - assert_eq!(board, "uno"); - assert_eq!(libs, vec!["./libs".to_string(), "./more".to_string()]); - assert_eq!(project_conf.as_deref(), Some("custom.ini")); - assert_eq!(sketches, vec!["examples/Blink/Blink.ino".to_string()]); - } - _ => panic!("expected Ci subcommand"), - } - } - - #[test] - fn ci_short_board_flag_b_is_accepted() { - let argv = ["fbuild", "ci", "-b", "uno", "examples/Blink"]; - let cli = Cli::try_parse_from(argv).expect("parse"); - match cli.command { - Some(Commands::Ci { - board, sketches, .. - }) => { - assert_eq!(board, "uno"); - assert_eq!(sketches, vec!["examples/Blink".to_string()]); - } - _ => panic!("expected Ci subcommand"), - } - } - - #[test] - fn ci_requires_at_least_one_sketch() { - let argv = ["fbuild", "ci", "--board", "uno"]; - assert!(Cli::try_parse_from(argv).is_err()); - } - - #[test] - fn ci_quick_and_release_are_mutually_exclusive() { - let argv = ["fbuild", "ci", "-b", "uno", "--quick", "--release", "."]; - assert!(Cli::try_parse_from(argv).is_err()); - } + rt.block_on(cli::async_main()); } diff --git a/crates/fbuild-cli/src/mcp.rs b/crates/fbuild-cli/src/mcp.rs deleted file mode 100644 index 5df2ed53..00000000 --- a/crates/fbuild-cli/src/mcp.rs +++ /dev/null @@ -1,1086 +0,0 @@ -//! MCP (Model Context Protocol) server for fbuild. -//! -//! Runs as a stdio-based JSON-RPC server that AI assistants (Claude Desktop, -//! Cursor, VS Code) can connect to for querying and controlling the fbuild -//! daemon. Translates MCP tool/resource/prompt calls into HTTP requests to -//! the running daemon. - -use crate::daemon_client::DaemonClient; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::io::{self, BufRead, Write}; - -// --------------------------------------------------------------------------- -// JSON-RPC types -// --------------------------------------------------------------------------- - -#[derive(Debug, Deserialize)] -struct JsonRpcRequest { - jsonrpc: String, - id: Option, - method: String, - #[serde(default)] - params: Option, -} - -#[derive(Debug, Serialize)] -struct JsonRpcResponse { - jsonrpc: String, - id: Value, - #[serde(skip_serializing_if = "Option::is_none")] - result: Option, - #[serde(skip_serializing_if = "Option::is_none")] - error: Option, -} - -#[derive(Debug, Serialize)] -struct JsonRpcError { - code: i32, - message: String, - #[serde(skip_serializing_if = "Option::is_none")] - data: Option, -} - -impl JsonRpcResponse { - fn success(id: Value, result: Value) -> Self { - Self { - jsonrpc: "2.0".to_string(), - id, - result: Some(result), - error: None, - } - } - - fn error(id: Value, code: i32, message: String) -> Self { - Self { - jsonrpc: "2.0".to_string(), - id, - result: None, - error: Some(JsonRpcError { - code, - message, - data: None, - }), - } - } -} - -// --------------------------------------------------------------------------- -// MCP protocol types -// --------------------------------------------------------------------------- - -#[derive(Debug, Serialize)] -struct ToolDefinition { - name: String, - description: String, - #[serde(rename = "inputSchema")] - input_schema: Value, - #[serde(skip_serializing_if = "Option::is_none")] - annotations: Option, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct ToolAnnotations { - #[serde(skip_serializing_if = "Option::is_none")] - read_only_hint: Option, - #[serde(skip_serializing_if = "Option::is_none")] - destructive_hint: Option, - #[serde(skip_serializing_if = "Option::is_none")] - idempotent_hint: Option, -} - -#[derive(Debug, Serialize)] -struct ResourceDefinition { - uri: String, - name: String, - description: String, - #[serde(rename = "mimeType")] - mime_type: String, -} - -#[derive(Debug, Serialize)] -struct PromptDefinition { - name: String, - description: String, - #[serde(skip_serializing_if = "Option::is_none")] - arguments: Option>, -} - -#[derive(Debug, Serialize)] -struct PromptArgument { - name: String, - description: String, - required: bool, -} - -#[derive(Debug, Serialize)] -struct TextContent { - #[serde(rename = "type")] - content_type: String, - text: String, -} - -#[derive(Debug, Serialize)] -// ResourceContent is used by the MCP resources/read response format. -#[allow(dead_code)] -struct ResourceContent { - uri: String, - #[serde(rename = "mimeType")] - mime_type: String, - text: String, -} - -#[derive(Debug, Serialize)] -struct PromptMessage { - role: String, - content: TextContent, -} - -// --------------------------------------------------------------------------- -// Tool definitions -// --------------------------------------------------------------------------- - -fn tool_definitions() -> Vec { - vec![ - ToolDefinition { - name: "get_daemon_status".to_string(), - description: "Get daemon status including PID, uptime, version, port, and current operation state.".to_string(), - input_schema: serde_json::json!({ - "type": "object", - "properties": {}, - "required": [] - }), - annotations: Some(ToolAnnotations { - read_only_hint: Some(true), - destructive_hint: None, - idempotent_hint: None, - }), - }, - ToolDefinition { - name: "list_devices".to_string(), - description: "List all serial devices known to the daemon, with connection and lease information.".to_string(), - input_schema: serde_json::json!({ - "type": "object", - "properties": {}, - "required": [] - }), - annotations: Some(ToolAnnotations { - read_only_hint: Some(true), - destructive_hint: None, - idempotent_hint: None, - }), - }, - ToolDefinition { - name: "get_lock_status".to_string(), - description: "Get active and stale lock information from the daemon.".to_string(), - input_schema: serde_json::json!({ - "type": "object", - "properties": {}, - "required": [] - }), - annotations: Some(ToolAnnotations { - read_only_hint: Some(true), - destructive_hint: None, - idempotent_hint: None, - }), - }, - ToolDefinition { - name: "trigger_build".to_string(), - description: "Trigger a firmware build for a project. Blocks until the build completes.".to_string(), - input_schema: serde_json::json!({ - "type": "object", - "properties": { - "project_dir": { - "type": "string", - "description": "Absolute path to the project directory." - }, - "environment": { - "type": "string", - "description": "Build environment name (e.g. 'uno', 'esp32c6')." - }, - "clean": { - "type": "boolean", - "description": "Whether to perform a clean build.", - "default": false - }, - "verbose": { - "type": "boolean", - "description": "Enable verbose compiler output.", - "default": false - }, - "jobs": { - "type": ["integer", "null"], - "description": "Number of parallel compilation workers (null = auto)." - } - }, - "required": ["project_dir", "environment"] - }), - annotations: Some(ToolAnnotations { - read_only_hint: Some(false), - destructive_hint: Some(false), - idempotent_hint: None, - }), - }, - ToolDefinition { - name: "trigger_deploy".to_string(), - description: "Trigger a firmware deploy (build + flash) for a project. Blocks until the deploy completes.".to_string(), - input_schema: serde_json::json!({ - "type": "object", - "properties": { - "project_dir": { - "type": "string", - "description": "Absolute path to the project directory." - }, - "environment": { - "type": "string", - "description": "Build environment name." - }, - "port": { - "type": ["string", "null"], - "description": "Serial port (e.g. 'COM3'). Null for auto-detect." - }, - "skip_build": { - "type": "boolean", - "description": "Skip the build step and flash existing firmware.", - "default": false - } - }, - "required": ["project_dir", "environment"] - }), - annotations: Some(ToolAnnotations { - read_only_hint: Some(false), - destructive_hint: Some(true), - idempotent_hint: None, - }), - }, - ToolDefinition { - name: "refresh_devices".to_string(), - description: "Re-scan serial ports and update the device inventory. Returns the list of currently connected devices.".to_string(), - input_schema: serde_json::json!({ - "type": "object", - "properties": {}, - "required": [] - }), - annotations: Some(ToolAnnotations { - read_only_hint: Some(false), - destructive_hint: Some(false), - idempotent_hint: Some(true), - }), - }, - ToolDefinition { - name: "clear_stale_locks".to_string(), - description: "Force-release any stale (stuck) locks in the daemon.".to_string(), - input_schema: serde_json::json!({ - "type": "object", - "properties": {}, - "required": [] - }), - annotations: Some(ToolAnnotations { - read_only_hint: Some(false), - destructive_hint: Some(true), - idempotent_hint: Some(true), - }), - }, - ToolDefinition { - name: "get_firmware_status".to_string(), - description: "Get firmware deployment information for a serial port (hash, source, staleness).".to_string(), - input_schema: serde_json::json!({ - "type": "object", - "properties": { - "port": { - "type": "string", - "description": "Serial port (e.g. 'COM3', '/dev/ttyUSB0')." - } - }, - "required": ["port"] - }), - annotations: Some(ToolAnnotations { - read_only_hint: Some(true), - destructive_hint: None, - idempotent_hint: None, - }), - }, - ] -} - -fn resource_definitions() -> Vec { - vec![ - ResourceDefinition { - uri: "fbuild://daemon/log".to_string(), - name: "Daemon Log".to_string(), - description: "Last 200 lines of the daemon log file.".to_string(), - mime_type: "text/plain".to_string(), - }, - ResourceDefinition { - uri: "fbuild://project/{project_dir}/config".to_string(), - name: "Project Config".to_string(), - description: "Parsed platformio.ini configuration for a project.".to_string(), - mime_type: "application/json".to_string(), - }, - ResourceDefinition { - uri: "fbuild://firmware/{port}".to_string(), - name: "Firmware Status".to_string(), - description: "Firmware deployment information for a serial port (connection, lease, availability).".to_string(), - mime_type: "application/json".to_string(), - }, - ] -} - -fn prompt_definitions() -> Vec { - vec![ - PromptDefinition { - name: "diagnose_build_failure".to_string(), - description: "Gather diagnostic information for a build failure (errors, recent ops, stale locks).".to_string(), - arguments: Some(vec![PromptArgument { - name: "project_dir".to_string(), - description: "Project directory to filter diagnostics (optional).".to_string(), - required: false, - }]), - }, - PromptDefinition { - name: "recommend_deploy_target".to_string(), - description: "Recommend which device to deploy firmware to based on device inventory and lease status.".to_string(), - arguments: Some(vec![PromptArgument { - name: "environment".to_string(), - description: "Target build environment (optional).".to_string(), - required: false, - }]), - }, - ] -} - -// --------------------------------------------------------------------------- -// Tool execution -// --------------------------------------------------------------------------- - -async fn execute_tool(client: &DaemonClient, name: &str, args: &Value) -> Result { - match name { - "get_daemon_status" => { - let info = client - .daemon_info() - .await - .map_err(|e| format!("Failed to get daemon info: {}", e))?; - - Ok(serde_json::json!({ - "pid": info.pid, - "uptime_seconds": info.uptime_seconds, - "version": info.version, - "port": info.port, - "dev_mode": info.dev_mode, - "state": format!("{:?}", info.daemon_state).to_lowercase(), - "operation_in_progress": info.operation_in_progress, - "current_operation": info.current_operation, - "client_count": info.client_count, - "spawner_cwd": info.spawner_cwd, - })) - } - "list_devices" => { - let devices = client - .list_devices(false) - .await - .map_err(|e| format!("Failed to list devices: {}", e))?; - - let device_list: Vec = devices - .devices - .iter() - .map(|d| { - serde_json::json!({ - "port": d.port, - "device_id": d.device_id, - "description": d.description, - "vid": d.vid, - "pid": d.pid, - }) - }) - .collect(); - - Ok(serde_json::json!({ - "device_count": device_list.len(), - "devices": device_list, - })) - } - "get_lock_status" => { - let locks = client - .lock_status() - .await - .map_err(|e| format!("Failed to get lock status: {}", e))?; - - Ok(serde_json::json!({ - "port_locks": locks.port_locks.iter().map(|l| serde_json::json!({ - "port": l.port, - "is_held": l.is_held, - "is_open": l.is_open, - "writer_client_id": l.writer_client_id, - "reader_count": l.reader_count, - })).collect::>(), - "project_locks": locks.project_locks.iter().map(|l| serde_json::json!({ - "project_dir": l.project_dir, - "is_held": l.is_held, - })).collect::>(), - "stale_locks": locks.stale_locks, - "active_port_lock_count": locks.port_locks.iter().filter(|l| l.is_held).count(), - "active_project_lock_count": locks.project_locks.iter().filter(|l| l.is_held).count(), - })) - } - "trigger_build" => { - let project_dir = args - .get("project_dir") - .and_then(|v| v.as_str()) - .ok_or("project_dir is required")? - .to_string(); - let environment = args - .get("environment") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - let clean = args.get("clean").and_then(|v| v.as_bool()).unwrap_or(false); - let verbose = args - .get("verbose") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - let jobs = args - .get("jobs") - .and_then(|v| v.as_u64()) - .map(|n| n as usize); - - let (caller_pid, caller_cwd) = crate::daemon_client::caller_info(); - let req = crate::daemon_client::BuildRequest { - project_dir, - environment, - clean_build: clean, - verbose, - jobs, - profile: None, - generate_compiledb: false, - compiledb_only: false, - request_id: Some(uuid_v4()), - caller_pid, - caller_cwd, - stream: false, - symbol_analysis: false, - symbol_analysis_path: None, - no_timestamp: false, - src_dir: std::env::var("PLATFORMIO_SRC_DIR") - .ok() - .filter(|s| !s.is_empty()), - output_dir: None, - pio_env: crate::daemon_client::capture_pio_env(), - }; - - let resp = client - .build(&req) - .await - .map_err(|e| format!("Build request failed: {}", e))?; - - Ok(serde_json::json!({ - "success": resp.success, - "message": resp.message, - "exit_code": resp.exit_code, - "request_id": resp.request_id, - })) - } - "trigger_deploy" => { - let project_dir = args - .get("project_dir") - .and_then(|v| v.as_str()) - .ok_or("project_dir is required")? - .to_string(); - let environment = args - .get("environment") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - let port = args - .get("port") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - let skip_build = args - .get("skip_build") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - - let (caller_pid, caller_cwd) = crate::daemon_client::caller_info(); - let req = crate::daemon_client::DeployRequest { - project_dir, - environment, - port, - monitor_after: false, - skip_build, - clean_build: false, - verbose: false, - monitor_timeout: None, - monitor_halt_on_error: None, - monitor_halt_on_success: None, - monitor_expect: None, - monitor_show_timestamp: true, - baud_rate: None, - to: None, - emulator: None, - target: None, - qemu: false, - qemu_timeout: 30, - request_id: Some(uuid_v4()), - caller_pid, - caller_cwd, - src_dir: std::env::var("PLATFORMIO_SRC_DIR") - .ok() - .filter(|s| !s.is_empty()), - output_dir: None, - pio_env: crate::daemon_client::capture_pio_env(), - }; - - let resp = client - .deploy(&req) - .await - .map_err(|e| format!("Deploy request failed: {}", e))?; - - Ok(serde_json::json!({ - "success": resp.success, - "message": resp.message, - "exit_code": resp.exit_code, - "request_id": resp.request_id, - })) - } - "refresh_devices" => { - let devices = client - .list_devices(true) - .await - .map_err(|e| format!("Failed to refresh devices: {}", e))?; - - let device_list: Vec = devices - .devices - .iter() - .map(|d| { - serde_json::json!({ - "port": d.port, - "device_id": d.device_id, - "description": d.description, - }) - }) - .collect(); - - Ok(serde_json::json!({ - "device_count": device_list.len(), - "devices": device_list, - })) - } - "clear_stale_locks" => { - let resp = client - .clear_locks() - .await - .map_err(|e| format!("Failed to clear locks: {}", e))?; - - Ok(serde_json::json!({ - "released_count": resp.cleared_count, - "message": resp.message, - })) - } - "get_firmware_status" => { - let port = args - .get("port") - .and_then(|v| v.as_str()) - .ok_or("port is required")?; - - let status = client - .device_status(port) - .await - .map_err(|e| format!("Failed to get device status: {}", e))?; - - Ok(serde_json::json!({ - "port": status.port, - "device_id": status.device_id, - "description": status.description, - "is_connected": status.is_connected, - "available_for_exclusive": status.available_for_exclusive, - "exclusive_holder": status.exclusive_holder, - "monitor_count": status.monitor_count, - })) - } - _ => Err(format!("Unknown tool: {}", name)), - } -} - -// --------------------------------------------------------------------------- -// Resource reading -// --------------------------------------------------------------------------- - -fn read_resource(uri: &str) -> Result<(String, String), String> { - if uri == "fbuild://daemon/log" { - let log_file = fbuild_paths::get_daemon_log_file(); - let text = std::fs::read_to_string(&log_file) - .unwrap_or_else(|_| "(daemon log not available)".to_string()); - let lines: Vec<&str> = text.lines().collect(); - let tail = if lines.len() > 200 { - &lines[lines.len() - 200..] - } else { - &lines - }; - Ok(("text/plain".to_string(), tail.join("\n"))) - } else if uri.starts_with("fbuild://project/") && uri.ends_with("/config") { - let path_part = uri - .strip_prefix("fbuild://project/") - .and_then(|s| s.strip_suffix("/config")) - .ok_or("Invalid project config URI")?; - - let decoded = urlencoding_decode(path_part); - let ini_path = std::path::Path::new(&decoded).join("platformio.ini"); - - if !ini_path.exists() { - return Ok(( - "application/json".to_string(), - serde_json::json!({"error": format!("platformio.ini not found at {}", ini_path.display())}).to_string(), - )); - } - - let content = std::fs::read_to_string(&ini_path) - .map_err(|e| format!("Failed to read {}: {}", ini_path.display(), e))?; - - Ok(( - "application/json".to_string(), - serde_json::json!({ - "project_dir": decoded, - "raw_ini": content, - }) - .to_string(), - )) - } else if uri.starts_with("fbuild://firmware/") { - let port = uri - .strip_prefix("fbuild://firmware/") - .ok_or("Invalid firmware URI")?; - let port = urlencoding_decode(port); - - // This is a synchronous context, but we need the device_status endpoint. - // Return a JSON pointer that tells the client how to fetch it. - Ok(( - "application/json".to_string(), - serde_json::json!({ - "port": port, - "note": "Use the get_firmware_status tool for live device status.", - "endpoint": format!("/api/devices/{}/status", port) - }) - .to_string(), - )) - } else { - Err(format!("Unknown resource URI: {}", uri)) - } -} - -// --------------------------------------------------------------------------- -// Prompt execution -// --------------------------------------------------------------------------- - -async fn execute_prompt( - client: &DaemonClient, - name: &str, - args: &Value, -) -> Result, String> { - match name { - "diagnose_build_failure" => { - let mut sections = vec!["# Build Failure Diagnostic Report\n".to_string()]; - - // Daemon status - match client.daemon_info().await { - Ok(info) => { - sections.push("## Daemon Status\n".to_string()); - sections.push(format!( - "- State: {:?}\n- PID: {}\n- Uptime: {:.1}s\n- Operation in progress: {}\n", - info.daemon_state, - info.pid, - info.uptime_seconds, - info.operation_in_progress - )); - if let Some(op) = &info.current_operation { - sections.push(format!("- Current operation: {}\n", op)); - } - } - Err(e) => { - sections.push(format!("## Daemon Status\n\nDaemon unreachable: {}\n", e)); - } - } - - // Stale locks - if let Ok(locks) = client.lock_status().await { - if !locks.stale_locks.is_empty() { - sections.push("## Stale Lock Warning\n".to_string()); - for lock in &locks.stale_locks { - sections.push(format!("- Stale lock: `{}`", lock)); - } - sections.push( - "\nConsider running the `clear_stale_locks` tool to release these.\n" - .to_string(), - ); - } - } - - // Devices - if let Ok(devices) = client.list_devices(false).await { - sections.push("## Connected Devices\n".to_string()); - if devices.devices.is_empty() { - sections.push("No devices connected.\n".to_string()); - } else { - for d in &devices.devices { - sections.push(format!("- **{}** ({})", d.port, d.description)); - } - sections.push(String::new()); - } - } - - let _project_dir = args - .get("project_dir") - .and_then(|v| v.as_str()) - .unwrap_or("(not specified)"); - - Ok(vec![PromptMessage { - role: "user".to_string(), - content: TextContent { - content_type: "text".to_string(), - text: sections.join("\n"), - }, - }]) - } - "recommend_deploy_target" => { - let mut sections = vec!["# Deploy Target Recommendation\n".to_string()]; - - match client.list_devices(true).await { - Ok(devices) => { - sections.push("## Device Inventory\n".to_string()); - sections.push(format!("- Total devices: {}\n", devices.devices.len())); - - if devices.devices.is_empty() { - sections.push( - "**No devices connected.** Plug in a board and run `refresh_devices`.\n" - .to_string(), - ); - } else { - sections.push("## Connected Devices\n".to_string()); - for d in &devices.devices { - sections.push(format!("- **{}** ({})", d.port, d.description)); - } - sections.push(String::new()); - - sections.push("## Recommendation\n".to_string()); - let first = &devices.devices[0]; - sections.push(format!( - "Deploy to **{}** ({}) - first available device.\n", - first.port, first.description - )); - } - } - Err(e) => { - sections.push(format!("Cannot list devices (daemon error): {}\n", e)); - } - } - - Ok(vec![PromptMessage { - role: "user".to_string(), - content: TextContent { - content_type: "text".to_string(), - text: sections.join("\n"), - }, - }]) - } - _ => Err(format!("Unknown prompt: {}", name)), - } -} - -// --------------------------------------------------------------------------- -// MCP server main loop -// --------------------------------------------------------------------------- - -pub async fn run_mcp_server() -> i32 { - let client = DaemonClient::new(); - - // Ensure daemon is running before starting MCP server - if let Err(e) = crate::daemon_client::ensure_daemon_running().await { - let err = serde_json::json!({ - "error": format!("Failed to start daemon: {}", e) - }); - eprintln!("MCP server: {}", err); - return 1; - } - - let stdin = io::stdin(); - let stdout = io::stdout(); - let reader = stdin.lock(); - - for line in reader.lines() { - let line = match line { - Ok(l) => l, - Err(_) => break, // EOF or read error - }; - - let line = line.trim().to_string(); - if line.is_empty() { - continue; - } - - let request: JsonRpcRequest = match serde_json::from_str(&line) { - Ok(r) => r, - Err(e) => { - // Parse error — send JSON-RPC error - let resp = - JsonRpcResponse::error(Value::Null, -32700, format!("Parse error: {}", e)); - send_response(&stdout, &resp); - continue; - } - }; - - if request.jsonrpc != "2.0" { - if let Some(id) = request.id { - let resp = - JsonRpcResponse::error(id, -32600, "Invalid JSON-RPC version".to_string()); - send_response(&stdout, &resp); - } - continue; - } - - // Notifications (no id) — handle silently - if request.id.is_none() { - // "initialized", "notifications/cancelled", etc. — just acknowledge - continue; - } - - let id = request.id.unwrap(); - let params = request.params.unwrap_or(Value::Null); - - let response = match request.method.as_str() { - "initialize" => JsonRpcResponse::success( - id, - serde_json::json!({ - "protocolVersion": "2024-11-05", - "capabilities": { - "tools": {}, - "resources": {}, - "prompts": {} - }, - "serverInfo": { - "name": "fbuild", - "version": env!("CARGO_PKG_VERSION") - } - }), - ), - "ping" => JsonRpcResponse::success(id, serde_json::json!({})), - "tools/list" => { - let tools = tool_definitions(); - JsonRpcResponse::success(id, serde_json::json!({ "tools": tools })) - } - "tools/call" => { - let tool_name = params.get("name").and_then(|v| v.as_str()).unwrap_or(""); - let tool_args = params - .get("arguments") - .cloned() - .unwrap_or(serde_json::json!({})); - - match execute_tool(&client, tool_name, &tool_args).await { - Ok(result) => JsonRpcResponse::success( - id, - serde_json::json!({ - "content": [{ - "type": "text", - "text": serde_json::to_string_pretty(&result).unwrap_or_default() - }] - }), - ), - Err(e) => JsonRpcResponse::success( - id, - serde_json::json!({ - "content": [{ - "type": "text", - "text": format!("Error: {}", e) - }], - "isError": true - }), - ), - } - } - "resources/list" => { - let resources = resource_definitions(); - JsonRpcResponse::success(id, serde_json::json!({ "resources": resources })) - } - "resources/read" => { - let uri = params.get("uri").and_then(|v| v.as_str()).unwrap_or(""); - - match read_resource(uri) { - Ok((mime_type, text)) => JsonRpcResponse::success( - id, - serde_json::json!({ - "contents": [{ - "uri": uri, - "mimeType": mime_type, - "text": text - }] - }), - ), - Err(e) => JsonRpcResponse::error(id, -32602, format!("Resource error: {}", e)), - } - } - "prompts/list" => { - let prompts = prompt_definitions(); - JsonRpcResponse::success(id, serde_json::json!({ "prompts": prompts })) - } - "prompts/get" => { - let prompt_name = params.get("name").and_then(|v| v.as_str()).unwrap_or(""); - let prompt_args = params - .get("arguments") - .cloned() - .unwrap_or(serde_json::json!({})); - - match execute_prompt(&client, prompt_name, &prompt_args).await { - Ok(messages) => JsonRpcResponse::success( - id, - serde_json::json!({ - "description": format!("Results for prompt '{}'", prompt_name), - "messages": messages - }), - ), - Err(e) => JsonRpcResponse::error(id, -32602, format!("Prompt error: {}", e)), - } - } - _ => { - JsonRpcResponse::error(id, -32601, format!("Method not found: {}", request.method)) - } - }; - - send_response(&stdout, &response); - } - - 0 -} - -fn send_response(stdout: &io::Stdout, response: &JsonRpcResponse) { - let json = serde_json::to_string(response).unwrap_or_default(); - let mut out = stdout.lock(); - let _ = writeln!(out, "{}", json); - let _ = out.flush(); -} - -/// Simple percent-decoding for URI path segments. -fn urlencoding_decode(input: &str) -> String { - let mut result = String::with_capacity(input.len()); - let mut chars = input.bytes(); - while let Some(b) = chars.next() { - if b == b'%' { - let hi = chars.next().unwrap_or(b'0'); - let lo = chars.next().unwrap_or(b'0'); - let val = hex_val(hi) * 16 + hex_val(lo); - result.push(val as char); - } else { - result.push(b as char); - } - } - result -} - -fn hex_val(b: u8) -> u8 { - match b { - b'0'..=b'9' => b - b'0', - b'a'..=b'f' => b - b'a' + 10, - b'A'..=b'F' => b - b'A' + 10, - _ => 0, - } -} - -/// Generate a UUID v4 string (simple implementation without extra deps). -fn uuid_v4() -> String { - use std::time::{SystemTime, UNIX_EPOCH}; - let seed = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - // Use time-based pseudo-random (good enough for request IDs, not crypto) - let pid = std::process::id() as u128; - let val = seed ^ (pid << 32) ^ (seed >> 16); - format!( - "{:08x}-{:04x}-4{:03x}-{:04x}-{:012x}", - (val >> 96) as u32, - (val >> 80) as u16, - (val >> 64) as u16 & 0x0fff, - ((val >> 48) as u16 & 0x3fff) | 0x8000, - val as u64 & 0xffffffffffff - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn tool_definitions_are_valid_json() { - let tools = tool_definitions(); - assert!(tools.len() >= 7); - for tool in &tools { - let json = serde_json::to_value(tool).unwrap(); - assert!(json.get("name").is_some()); - assert!(json.get("inputSchema").is_some()); - } - } - - #[test] - fn resource_definitions_are_valid() { - let resources = resource_definitions(); - assert_eq!(resources.len(), 3); - assert_eq!(resources[0].uri, "fbuild://daemon/log"); - assert_eq!(resources[2].uri, "fbuild://firmware/{port}"); - } - - #[test] - fn prompt_definitions_are_valid() { - let prompts = prompt_definitions(); - assert_eq!(prompts.len(), 2); - assert_eq!(prompts[0].name, "diagnose_build_failure"); - assert_eq!(prompts[1].name, "recommend_deploy_target"); - } - - #[test] - fn urlencoding_decode_works() { - assert_eq!(urlencoding_decode("hello%20world"), "hello world"); - assert_eq!(urlencoding_decode("C%3A%5Cdev"), "C:\\dev"); - assert_eq!(urlencoding_decode("no-encoding"), "no-encoding"); - } - - #[test] - fn uuid_v4_has_correct_format() { - let id = uuid_v4(); - assert_eq!(id.len(), 36); - assert_eq!(id.chars().nth(8), Some('-')); - assert_eq!(id.chars().nth(13), Some('-')); - assert_eq!(id.chars().nth(14), Some('4')); // version 4 - assert_eq!(id.chars().nth(18), Some('-')); - } - - #[test] - fn json_rpc_response_serialization() { - let resp = - JsonRpcResponse::success(Value::Number(1.into()), serde_json::json!({"key": "value"})); - let json = serde_json::to_string(&resp).unwrap(); - assert!(json.contains("\"jsonrpc\":\"2.0\"")); - assert!(json.contains("\"result\"")); - assert!(!json.contains("\"error\"")); - } - - #[test] - fn json_rpc_error_serialization() { - let resp = JsonRpcResponse::error( - Value::Number(2.into()), - -32601, - "Method not found".to_string(), - ); - let json = serde_json::to_string(&resp).unwrap(); - assert!(json.contains("\"error\"")); - assert!(json.contains("-32601")); - assert!(!json.contains("\"result\"")); - } - - #[test] - fn read_resource_unknown_uri_returns_error() { - let result = read_resource("fbuild://unknown/thing"); - assert!(result.is_err()); - } - - #[test] - fn read_resource_firmware_returns_json() { - let (mime, text) = read_resource("fbuild://firmware/COM3").unwrap(); - assert_eq!(mime, "application/json"); - let parsed: serde_json::Value = serde_json::from_str(&text).unwrap(); - assert_eq!(parsed["port"], "COM3"); - } -} diff --git a/crates/fbuild-cli/src/mcp/README.md b/crates/fbuild-cli/src/mcp/README.md new file mode 100644 index 00000000..0be13738 --- /dev/null +++ b/crates/fbuild-cli/src/mcp/README.md @@ -0,0 +1,20 @@ +# MCP Module + +The MCP (Model Context Protocol) stdio JSON-RPC server. Translates AI assistant +tool/resource/prompt calls into HTTP requests against the fbuild daemon. + +## Files + +- **`mod.rs`** -- Module root. Wires submodules together and re-exports `run_mcp_server` (the only public item). +- **`jsonrpc.rs`** -- JSON-RPC 2.0 envelope types (`JsonRpcRequest`, `JsonRpcResponse`, `JsonRpcError`). +- **`types.rs`** -- MCP protocol value types (`ToolDefinition`, `ToolAnnotations`, `ResourceDefinition`, `PromptDefinition`, `PromptArgument`, `TextContent`, `ResourceContent`, `PromptMessage`). +- **`definitions.rs`** -- Static lists of tools, resources, and prompts advertised by the server. +- **`tools.rs`** -- `tools/call` dispatch; one match arm per tool, each translating into a `DaemonClient` HTTP call. +- **`resources.rs`** -- `resources/read` dispatch for `fbuild://daemon/log`, `fbuild://project/{dir}/config`, and `fbuild://firmware/{port}`. +- **`prompts.rs`** -- `prompts/get` dispatch for `diagnose_build_failure` and `recommend_deploy_target`. +- **`util.rs`** -- Helpers: `urlencoding_decode`, `uuid_v4` (no extra dependencies). +- **`server.rs`** -- Stdio loop: reads JSON-RPC requests from stdin, dispatches to the relevant submodule, writes responses to stdout. Exports `run_mcp_server`. + +## Public API + +Only `run_mcp_server` is re-exported through `mcp::`; everything else is crate-private. The original `mcp.rs` was split into submodules to keep each file under the 1000-LOC gate. diff --git a/crates/fbuild-cli/src/mcp/definitions.rs b/crates/fbuild-cli/src/mcp/definitions.rs new file mode 100644 index 00000000..838c2774 --- /dev/null +++ b/crates/fbuild-cli/src/mcp/definitions.rs @@ -0,0 +1,246 @@ +//! Static tool, resource, and prompt definitions advertised by the MCP server. + +use super::types::{ + PromptArgument, PromptDefinition, ResourceDefinition, ToolAnnotations, ToolDefinition, +}; + +pub(super) fn tool_definitions() -> Vec { + vec![ + ToolDefinition { + name: "get_daemon_status".to_string(), + description: "Get daemon status including PID, uptime, version, port, and current operation state.".to_string(), + input_schema: serde_json::json!({ + "type": "object", + "properties": {}, + "required": [] + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(true), + destructive_hint: None, + idempotent_hint: None, + }), + }, + ToolDefinition { + name: "list_devices".to_string(), + description: "List all serial devices known to the daemon, with connection and lease information.".to_string(), + input_schema: serde_json::json!({ + "type": "object", + "properties": {}, + "required": [] + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(true), + destructive_hint: None, + idempotent_hint: None, + }), + }, + ToolDefinition { + name: "get_lock_status".to_string(), + description: "Get active and stale lock information from the daemon.".to_string(), + input_schema: serde_json::json!({ + "type": "object", + "properties": {}, + "required": [] + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(true), + destructive_hint: None, + idempotent_hint: None, + }), + }, + ToolDefinition { + name: "trigger_build".to_string(), + description: "Trigger a firmware build for a project. Blocks until the build completes.".to_string(), + input_schema: serde_json::json!({ + "type": "object", + "properties": { + "project_dir": { + "type": "string", + "description": "Absolute path to the project directory." + }, + "environment": { + "type": "string", + "description": "Build environment name (e.g. 'uno', 'esp32c6')." + }, + "clean": { + "type": "boolean", + "description": "Whether to perform a clean build.", + "default": false + }, + "verbose": { + "type": "boolean", + "description": "Enable verbose compiler output.", + "default": false + }, + "jobs": { + "type": ["integer", "null"], + "description": "Number of parallel compilation workers (null = auto)." + } + }, + "required": ["project_dir", "environment"] + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(false), + destructive_hint: Some(false), + idempotent_hint: None, + }), + }, + ToolDefinition { + name: "trigger_deploy".to_string(), + description: "Trigger a firmware deploy (build + flash) for a project. Blocks until the deploy completes.".to_string(), + input_schema: serde_json::json!({ + "type": "object", + "properties": { + "project_dir": { + "type": "string", + "description": "Absolute path to the project directory." + }, + "environment": { + "type": "string", + "description": "Build environment name." + }, + "port": { + "type": ["string", "null"], + "description": "Serial port (e.g. 'COM3'). Null for auto-detect." + }, + "skip_build": { + "type": "boolean", + "description": "Skip the build step and flash existing firmware.", + "default": false + } + }, + "required": ["project_dir", "environment"] + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(false), + destructive_hint: Some(true), + idempotent_hint: None, + }), + }, + ToolDefinition { + name: "refresh_devices".to_string(), + description: "Re-scan serial ports and update the device inventory. Returns the list of currently connected devices.".to_string(), + input_schema: serde_json::json!({ + "type": "object", + "properties": {}, + "required": [] + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(false), + destructive_hint: Some(false), + idempotent_hint: Some(true), + }), + }, + ToolDefinition { + name: "clear_stale_locks".to_string(), + description: "Force-release any stale (stuck) locks in the daemon.".to_string(), + input_schema: serde_json::json!({ + "type": "object", + "properties": {}, + "required": [] + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(false), + destructive_hint: Some(true), + idempotent_hint: Some(true), + }), + }, + ToolDefinition { + name: "get_firmware_status".to_string(), + description: "Get firmware deployment information for a serial port (hash, source, staleness).".to_string(), + input_schema: serde_json::json!({ + "type": "object", + "properties": { + "port": { + "type": "string", + "description": "Serial port (e.g. 'COM3', '/dev/ttyUSB0')." + } + }, + "required": ["port"] + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(true), + destructive_hint: None, + idempotent_hint: None, + }), + }, + ] +} + +pub(super) fn resource_definitions() -> Vec { + vec![ + ResourceDefinition { + uri: "fbuild://daemon/log".to_string(), + name: "Daemon Log".to_string(), + description: "Last 200 lines of the daemon log file.".to_string(), + mime_type: "text/plain".to_string(), + }, + ResourceDefinition { + uri: "fbuild://project/{project_dir}/config".to_string(), + name: "Project Config".to_string(), + description: "Parsed platformio.ini configuration for a project.".to_string(), + mime_type: "application/json".to_string(), + }, + ResourceDefinition { + uri: "fbuild://firmware/{port}".to_string(), + name: "Firmware Status".to_string(), + description: "Firmware deployment information for a serial port (connection, lease, availability).".to_string(), + mime_type: "application/json".to_string(), + }, + ] +} + +pub(super) fn prompt_definitions() -> Vec { + vec![ + PromptDefinition { + name: "diagnose_build_failure".to_string(), + description: "Gather diagnostic information for a build failure (errors, recent ops, stale locks).".to_string(), + arguments: Some(vec![PromptArgument { + name: "project_dir".to_string(), + description: "Project directory to filter diagnostics (optional).".to_string(), + required: false, + }]), + }, + PromptDefinition { + name: "recommend_deploy_target".to_string(), + description: "Recommend which device to deploy firmware to based on device inventory and lease status.".to_string(), + arguments: Some(vec![PromptArgument { + name: "environment".to_string(), + description: "Target build environment (optional).".to_string(), + required: false, + }]), + }, + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tool_definitions_are_valid_json() { + let tools = tool_definitions(); + assert!(tools.len() >= 7); + for tool in &tools { + let json = serde_json::to_value(tool).unwrap(); + assert!(json.get("name").is_some()); + assert!(json.get("inputSchema").is_some()); + } + } + + #[test] + fn resource_definitions_are_valid() { + let resources = resource_definitions(); + assert_eq!(resources.len(), 3); + assert_eq!(resources[0].uri, "fbuild://daemon/log"); + assert_eq!(resources[2].uri, "fbuild://firmware/{port}"); + } + + #[test] + fn prompt_definitions_are_valid() { + let prompts = prompt_definitions(); + assert_eq!(prompts.len(), 2); + assert_eq!(prompts[0].name, "diagnose_build_failure"); + assert_eq!(prompts[1].name, "recommend_deploy_target"); + } +} diff --git a/crates/fbuild-cli/src/mcp/jsonrpc.rs b/crates/fbuild-cli/src/mcp/jsonrpc.rs new file mode 100644 index 00000000..8af8aaaa --- /dev/null +++ b/crates/fbuild-cli/src/mcp/jsonrpc.rs @@ -0,0 +1,83 @@ +//! JSON-RPC 2.0 envelope types used by the MCP stdio server. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Debug, Deserialize)] +pub(super) struct JsonRpcRequest { + pub(super) jsonrpc: String, + pub(super) id: Option, + pub(super) method: String, + #[serde(default)] + pub(super) params: Option, +} + +#[derive(Debug, Serialize)] +pub(super) struct JsonRpcResponse { + jsonrpc: String, + id: Value, + #[serde(skip_serializing_if = "Option::is_none")] + result: Option, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +#[derive(Debug, Serialize)] +pub(super) struct JsonRpcError { + code: i32, + message: String, + #[serde(skip_serializing_if = "Option::is_none")] + data: Option, +} + +impl JsonRpcResponse { + pub(super) fn success(id: Value, result: Value) -> Self { + Self { + jsonrpc: "2.0".to_string(), + id, + result: Some(result), + error: None, + } + } + + pub(super) fn error(id: Value, code: i32, message: String) -> Self { + Self { + jsonrpc: "2.0".to_string(), + id, + result: None, + error: Some(JsonRpcError { + code, + message, + data: None, + }), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_rpc_response_serialization() { + let resp = + JsonRpcResponse::success(Value::Number(1.into()), serde_json::json!({"key": "value"})); + let json = serde_json::to_string(&resp).unwrap(); + assert!(json.contains("\"jsonrpc\":\"2.0\"")); + assert!(json.contains("\"result\"")); + assert!(!json.contains("\"error\"")); + } + + #[test] + fn json_rpc_error_serialization() { + let resp = JsonRpcResponse::error( + Value::Number(2.into()), + -32601, + "Method not found".to_string(), + ); + let json = serde_json::to_string(&resp).unwrap(); + assert!(json.contains("\"error\"")); + assert!(json.contains("-32601")); + assert!(!json.contains("\"result\"")); + } +} diff --git a/crates/fbuild-cli/src/mcp/mod.rs b/crates/fbuild-cli/src/mcp/mod.rs new file mode 100644 index 00000000..bf6fee01 --- /dev/null +++ b/crates/fbuild-cli/src/mcp/mod.rs @@ -0,0 +1,28 @@ +//! MCP (Model Context Protocol) server for fbuild. +//! +//! Runs as a stdio-based JSON-RPC server that AI assistants (Claude Desktop, +//! Cursor, VS Code) can connect to for querying and controlling the fbuild +//! daemon. Translates MCP tool/resource/prompt calls into HTTP requests to +//! the running daemon. +//! +//! Submodules: +//! +//! - [`jsonrpc`] - JSON-RPC 2.0 envelope types. +//! - [`types`] - MCP protocol value types (tool/resource/prompt definitions). +//! - [`definitions`] - Static tool, resource, and prompt advertisements. +//! - [`tools`] - `tools/call` dispatch and per-tool implementations. +//! - [`resources`] - `resources/read` dispatch. +//! - [`prompts`] - `prompts/get` dispatch. +//! - [`util`] - Small standalone helpers (URI decoding, UUID generation). +//! - [`server`] - Stdio loop tying everything together. + +mod definitions; +mod jsonrpc; +mod prompts; +mod resources; +mod server; +mod tools; +mod types; +mod util; + +pub use server::run_mcp_server; diff --git a/crates/fbuild-cli/src/mcp/prompts.rs b/crates/fbuild-cli/src/mcp/prompts.rs new file mode 100644 index 00000000..f8078e6c --- /dev/null +++ b/crates/fbuild-cli/src/mcp/prompts.rs @@ -0,0 +1,119 @@ +//! Implementation of MCP `prompts/get` dispatch. + +use super::types::{PromptMessage, TextContent}; +use crate::daemon_client::DaemonClient; +use serde_json::Value; + +pub(super) async fn execute_prompt( + client: &DaemonClient, + name: &str, + args: &Value, +) -> Result, String> { + match name { + "diagnose_build_failure" => { + let mut sections = vec!["# Build Failure Diagnostic Report\n".to_string()]; + + // Daemon status + match client.daemon_info().await { + Ok(info) => { + sections.push("## Daemon Status\n".to_string()); + sections.push(format!( + "- State: {:?}\n- PID: {}\n- Uptime: {:.1}s\n- Operation in progress: {}\n", + info.daemon_state, + info.pid, + info.uptime_seconds, + info.operation_in_progress + )); + if let Some(op) = &info.current_operation { + sections.push(format!("- Current operation: {}\n", op)); + } + } + Err(e) => { + sections.push(format!("## Daemon Status\n\nDaemon unreachable: {}\n", e)); + } + } + + // Stale locks + if let Ok(locks) = client.lock_status().await { + if !locks.stale_locks.is_empty() { + sections.push("## Stale Lock Warning\n".to_string()); + for lock in &locks.stale_locks { + sections.push(format!("- Stale lock: `{}`", lock)); + } + sections.push( + "\nConsider running the `clear_stale_locks` tool to release these.\n" + .to_string(), + ); + } + } + + // Devices + if let Ok(devices) = client.list_devices(false).await { + sections.push("## Connected Devices\n".to_string()); + if devices.devices.is_empty() { + sections.push("No devices connected.\n".to_string()); + } else { + for d in &devices.devices { + sections.push(format!("- **{}** ({})", d.port, d.description)); + } + sections.push(String::new()); + } + } + + let _project_dir = args + .get("project_dir") + .and_then(|v| v.as_str()) + .unwrap_or("(not specified)"); + + Ok(vec![PromptMessage { + role: "user".to_string(), + content: TextContent { + content_type: "text".to_string(), + text: sections.join("\n"), + }, + }]) + } + "recommend_deploy_target" => { + let mut sections = vec!["# Deploy Target Recommendation\n".to_string()]; + + match client.list_devices(true).await { + Ok(devices) => { + sections.push("## Device Inventory\n".to_string()); + sections.push(format!("- Total devices: {}\n", devices.devices.len())); + + if devices.devices.is_empty() { + sections.push( + "**No devices connected.** Plug in a board and run `refresh_devices`.\n" + .to_string(), + ); + } else { + sections.push("## Connected Devices\n".to_string()); + for d in &devices.devices { + sections.push(format!("- **{}** ({})", d.port, d.description)); + } + sections.push(String::new()); + + sections.push("## Recommendation\n".to_string()); + let first = &devices.devices[0]; + sections.push(format!( + "Deploy to **{}** ({}) - first available device.\n", + first.port, first.description + )); + } + } + Err(e) => { + sections.push(format!("Cannot list devices (daemon error): {}\n", e)); + } + } + + Ok(vec![PromptMessage { + role: "user".to_string(), + content: TextContent { + content_type: "text".to_string(), + text: sections.join("\n"), + }, + }]) + } + _ => Err(format!("Unknown prompt: {}", name)), + } +} diff --git a/crates/fbuild-cli/src/mcp/resources.rs b/crates/fbuild-cli/src/mcp/resources.rs new file mode 100644 index 00000000..1f940537 --- /dev/null +++ b/crates/fbuild-cli/src/mcp/resources.rs @@ -0,0 +1,83 @@ +//! Implementation of MCP `resources/read` dispatch. + +use super::util::urlencoding_decode; + +pub(super) fn read_resource(uri: &str) -> Result<(String, String), String> { + if uri == "fbuild://daemon/log" { + let log_file = fbuild_paths::get_daemon_log_file(); + let text = std::fs::read_to_string(&log_file) + .unwrap_or_else(|_| "(daemon log not available)".to_string()); + let lines: Vec<&str> = text.lines().collect(); + let tail = if lines.len() > 200 { + &lines[lines.len() - 200..] + } else { + &lines + }; + Ok(("text/plain".to_string(), tail.join("\n"))) + } else if uri.starts_with("fbuild://project/") && uri.ends_with("/config") { + let path_part = uri + .strip_prefix("fbuild://project/") + .and_then(|s| s.strip_suffix("/config")) + .ok_or("Invalid project config URI")?; + + let decoded = urlencoding_decode(path_part); + let ini_path = std::path::Path::new(&decoded).join("platformio.ini"); + + if !ini_path.exists() { + return Ok(( + "application/json".to_string(), + serde_json::json!({"error": format!("platformio.ini not found at {}", ini_path.display())}).to_string(), + )); + } + + let content = std::fs::read_to_string(&ini_path) + .map_err(|e| format!("Failed to read {}: {}", ini_path.display(), e))?; + + Ok(( + "application/json".to_string(), + serde_json::json!({ + "project_dir": decoded, + "raw_ini": content, + }) + .to_string(), + )) + } else if uri.starts_with("fbuild://firmware/") { + let port = uri + .strip_prefix("fbuild://firmware/") + .ok_or("Invalid firmware URI")?; + let port = urlencoding_decode(port); + + // This is a synchronous context, but we need the device_status endpoint. + // Return a JSON pointer that tells the client how to fetch it. + Ok(( + "application/json".to_string(), + serde_json::json!({ + "port": port, + "note": "Use the get_firmware_status tool for live device status.", + "endpoint": format!("/api/devices/{}/status", port) + }) + .to_string(), + )) + } else { + Err(format!("Unknown resource URI: {}", uri)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn read_resource_unknown_uri_returns_error() { + let result = read_resource("fbuild://unknown/thing"); + assert!(result.is_err()); + } + + #[test] + fn read_resource_firmware_returns_json() { + let (mime, text) = read_resource("fbuild://firmware/COM3").unwrap(); + assert_eq!(mime, "application/json"); + let parsed: serde_json::Value = serde_json::from_str(&text).unwrap(); + assert_eq!(parsed["port"], "COM3"); + } +} diff --git a/crates/fbuild-cli/src/mcp/server.rs b/crates/fbuild-cli/src/mcp/server.rs new file mode 100644 index 00000000..31efa3da --- /dev/null +++ b/crates/fbuild-cli/src/mcp/server.rs @@ -0,0 +1,177 @@ +//! MCP stdio server main loop: reads JSON-RPC requests from stdin, writes responses to stdout. + +use super::definitions::{prompt_definitions, resource_definitions, tool_definitions}; +use super::jsonrpc::{JsonRpcRequest, JsonRpcResponse}; +use super::prompts::execute_prompt; +use super::resources::read_resource; +use super::tools::execute_tool; +use crate::daemon_client::DaemonClient; +use serde_json::Value; +use std::io::{self, BufRead, Write}; + +pub async fn run_mcp_server() -> i32 { + let client = DaemonClient::new(); + + // Ensure daemon is running before starting MCP server + if let Err(e) = crate::daemon_client::ensure_daemon_running().await { + let err = serde_json::json!({ + "error": format!("Failed to start daemon: {}", e) + }); + eprintln!("MCP server: {}", err); + return 1; + } + + let stdin = io::stdin(); + let stdout = io::stdout(); + let reader = stdin.lock(); + + for line in reader.lines() { + let line = match line { + Ok(l) => l, + Err(_) => break, // EOF or read error + }; + + let line = line.trim().to_string(); + if line.is_empty() { + continue; + } + + let request: JsonRpcRequest = match serde_json::from_str(&line) { + Ok(r) => r, + Err(e) => { + // Parse error — send JSON-RPC error + let resp = + JsonRpcResponse::error(Value::Null, -32700, format!("Parse error: {}", e)); + send_response(&stdout, &resp); + continue; + } + }; + + if request.jsonrpc != "2.0" { + if let Some(id) = request.id { + let resp = + JsonRpcResponse::error(id, -32600, "Invalid JSON-RPC version".to_string()); + send_response(&stdout, &resp); + } + continue; + } + + // Notifications (no id) — handle silently + if request.id.is_none() { + // "initialized", "notifications/cancelled", etc. — just acknowledge + continue; + } + + let id = request.id.unwrap(); + let params = request.params.unwrap_or(Value::Null); + + let response = match request.method.as_str() { + "initialize" => JsonRpcResponse::success( + id, + serde_json::json!({ + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": {}, + "resources": {}, + "prompts": {} + }, + "serverInfo": { + "name": "fbuild", + "version": env!("CARGO_PKG_VERSION") + } + }), + ), + "ping" => JsonRpcResponse::success(id, serde_json::json!({})), + "tools/list" => { + let tools = tool_definitions(); + JsonRpcResponse::success(id, serde_json::json!({ "tools": tools })) + } + "tools/call" => { + let tool_name = params.get("name").and_then(|v| v.as_str()).unwrap_or(""); + let tool_args = params + .get("arguments") + .cloned() + .unwrap_or(serde_json::json!({})); + + match execute_tool(&client, tool_name, &tool_args).await { + Ok(result) => JsonRpcResponse::success( + id, + serde_json::json!({ + "content": [{ + "type": "text", + "text": serde_json::to_string_pretty(&result).unwrap_or_default() + }] + }), + ), + Err(e) => JsonRpcResponse::success( + id, + serde_json::json!({ + "content": [{ + "type": "text", + "text": format!("Error: {}", e) + }], + "isError": true + }), + ), + } + } + "resources/list" => { + let resources = resource_definitions(); + JsonRpcResponse::success(id, serde_json::json!({ "resources": resources })) + } + "resources/read" => { + let uri = params.get("uri").and_then(|v| v.as_str()).unwrap_or(""); + + match read_resource(uri) { + Ok((mime_type, text)) => JsonRpcResponse::success( + id, + serde_json::json!({ + "contents": [{ + "uri": uri, + "mimeType": mime_type, + "text": text + }] + }), + ), + Err(e) => JsonRpcResponse::error(id, -32602, format!("Resource error: {}", e)), + } + } + "prompts/list" => { + let prompts = prompt_definitions(); + JsonRpcResponse::success(id, serde_json::json!({ "prompts": prompts })) + } + "prompts/get" => { + let prompt_name = params.get("name").and_then(|v| v.as_str()).unwrap_or(""); + let prompt_args = params + .get("arguments") + .cloned() + .unwrap_or(serde_json::json!({})); + + match execute_prompt(&client, prompt_name, &prompt_args).await { + Ok(messages) => JsonRpcResponse::success( + id, + serde_json::json!({ + "description": format!("Results for prompt '{}'", prompt_name), + "messages": messages + }), + ), + Err(e) => JsonRpcResponse::error(id, -32602, format!("Prompt error: {}", e)), + } + } + _ => { + JsonRpcResponse::error(id, -32601, format!("Method not found: {}", request.method)) + } + }; + + send_response(&stdout, &response); + } + + 0 +} + +fn send_response(stdout: &io::Stdout, response: &JsonRpcResponse) { + let json = serde_json::to_string(response).unwrap_or_default(); + let mut out = stdout.lock(); + let _ = writeln!(out, "{}", json); + let _ = out.flush(); +} diff --git a/crates/fbuild-cli/src/mcp/tools.rs b/crates/fbuild-cli/src/mcp/tools.rs new file mode 100644 index 00000000..47b4141b --- /dev/null +++ b/crates/fbuild-cli/src/mcp/tools.rs @@ -0,0 +1,254 @@ +//! Implementation of MCP `tools/call` dispatch. + +use super::util::uuid_v4; +use crate::daemon_client::DaemonClient; +use serde_json::Value; + +pub(super) async fn execute_tool( + client: &DaemonClient, + name: &str, + args: &Value, +) -> Result { + match name { + "get_daemon_status" => { + let info = client + .daemon_info() + .await + .map_err(|e| format!("Failed to get daemon info: {}", e))?; + + Ok(serde_json::json!({ + "pid": info.pid, + "uptime_seconds": info.uptime_seconds, + "version": info.version, + "port": info.port, + "dev_mode": info.dev_mode, + "state": format!("{:?}", info.daemon_state).to_lowercase(), + "operation_in_progress": info.operation_in_progress, + "current_operation": info.current_operation, + "client_count": info.client_count, + "spawner_cwd": info.spawner_cwd, + })) + } + "list_devices" => { + let devices = client + .list_devices(false) + .await + .map_err(|e| format!("Failed to list devices: {}", e))?; + + let device_list: Vec = devices + .devices + .iter() + .map(|d| { + serde_json::json!({ + "port": d.port, + "device_id": d.device_id, + "description": d.description, + "vid": d.vid, + "pid": d.pid, + }) + }) + .collect(); + + Ok(serde_json::json!({ + "device_count": device_list.len(), + "devices": device_list, + })) + } + "get_lock_status" => { + let locks = client + .lock_status() + .await + .map_err(|e| format!("Failed to get lock status: {}", e))?; + + Ok(serde_json::json!({ + "port_locks": locks.port_locks.iter().map(|l| serde_json::json!({ + "port": l.port, + "is_held": l.is_held, + "is_open": l.is_open, + "writer_client_id": l.writer_client_id, + "reader_count": l.reader_count, + })).collect::>(), + "project_locks": locks.project_locks.iter().map(|l| serde_json::json!({ + "project_dir": l.project_dir, + "is_held": l.is_held, + })).collect::>(), + "stale_locks": locks.stale_locks, + "active_port_lock_count": locks.port_locks.iter().filter(|l| l.is_held).count(), + "active_project_lock_count": locks.project_locks.iter().filter(|l| l.is_held).count(), + })) + } + "trigger_build" => { + let project_dir = args + .get("project_dir") + .and_then(|v| v.as_str()) + .ok_or("project_dir is required")? + .to_string(); + let environment = args + .get("environment") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let clean = args.get("clean").and_then(|v| v.as_bool()).unwrap_or(false); + let verbose = args + .get("verbose") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let jobs = args + .get("jobs") + .and_then(|v| v.as_u64()) + .map(|n| n as usize); + + let (caller_pid, caller_cwd) = crate::daemon_client::caller_info(); + let req = crate::daemon_client::BuildRequest { + project_dir, + environment, + clean_build: clean, + verbose, + jobs, + profile: None, + generate_compiledb: false, + compiledb_only: false, + request_id: Some(uuid_v4()), + caller_pid, + caller_cwd, + stream: false, + symbol_analysis: false, + symbol_analysis_path: None, + no_timestamp: false, + src_dir: std::env::var("PLATFORMIO_SRC_DIR") + .ok() + .filter(|s| !s.is_empty()), + output_dir: None, + pio_env: crate::daemon_client::capture_pio_env(), + }; + + let resp = client + .build(&req) + .await + .map_err(|e| format!("Build request failed: {}", e))?; + + Ok(serde_json::json!({ + "success": resp.success, + "message": resp.message, + "exit_code": resp.exit_code, + "request_id": resp.request_id, + })) + } + "trigger_deploy" => { + let project_dir = args + .get("project_dir") + .and_then(|v| v.as_str()) + .ok_or("project_dir is required")? + .to_string(); + let environment = args + .get("environment") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let port = args + .get("port") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let skip_build = args + .get("skip_build") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + let (caller_pid, caller_cwd) = crate::daemon_client::caller_info(); + let req = crate::daemon_client::DeployRequest { + project_dir, + environment, + port, + monitor_after: false, + skip_build, + clean_build: false, + verbose: false, + monitor_timeout: None, + monitor_halt_on_error: None, + monitor_halt_on_success: None, + monitor_expect: None, + monitor_show_timestamp: true, + baud_rate: None, + to: None, + emulator: None, + target: None, + qemu: false, + qemu_timeout: 30, + request_id: Some(uuid_v4()), + caller_pid, + caller_cwd, + src_dir: std::env::var("PLATFORMIO_SRC_DIR") + .ok() + .filter(|s| !s.is_empty()), + output_dir: None, + pio_env: crate::daemon_client::capture_pio_env(), + }; + + let resp = client + .deploy(&req) + .await + .map_err(|e| format!("Deploy request failed: {}", e))?; + + Ok(serde_json::json!({ + "success": resp.success, + "message": resp.message, + "exit_code": resp.exit_code, + "request_id": resp.request_id, + })) + } + "refresh_devices" => { + let devices = client + .list_devices(true) + .await + .map_err(|e| format!("Failed to refresh devices: {}", e))?; + + let device_list: Vec = devices + .devices + .iter() + .map(|d| { + serde_json::json!({ + "port": d.port, + "device_id": d.device_id, + "description": d.description, + }) + }) + .collect(); + + Ok(serde_json::json!({ + "device_count": device_list.len(), + "devices": device_list, + })) + } + "clear_stale_locks" => { + let resp = client + .clear_locks() + .await + .map_err(|e| format!("Failed to clear locks: {}", e))?; + + Ok(serde_json::json!({ + "released_count": resp.cleared_count, + "message": resp.message, + })) + } + "get_firmware_status" => { + let port = args + .get("port") + .and_then(|v| v.as_str()) + .ok_or("port is required")?; + + let status = client + .device_status(port) + .await + .map_err(|e| format!("Failed to get device status: {}", e))?; + + Ok(serde_json::json!({ + "port": status.port, + "device_id": status.device_id, + "description": status.description, + "is_connected": status.is_connected, + "available_for_exclusive": status.available_for_exclusive, + "exclusive_holder": status.exclusive_holder, + "monitor_count": status.monitor_count, + })) + } + _ => Err(format!("Unknown tool: {}", name)), + } +} diff --git a/crates/fbuild-cli/src/mcp/types.rs b/crates/fbuild-cli/src/mcp/types.rs new file mode 100644 index 00000000..f7f05a25 --- /dev/null +++ b/crates/fbuild-cli/src/mcp/types.rs @@ -0,0 +1,72 @@ +//! MCP protocol value types: tools, resources, prompts, content blocks. + +use serde::Serialize; +use serde_json::Value; + +#[derive(Debug, Serialize)] +pub(super) struct ToolDefinition { + pub(super) name: String, + pub(super) description: String, + #[serde(rename = "inputSchema")] + pub(super) input_schema: Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) annotations: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct ToolAnnotations { + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) read_only_hint: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) destructive_hint: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) idempotent_hint: Option, +} + +#[derive(Debug, Serialize)] +pub(super) struct ResourceDefinition { + pub(super) uri: String, + pub(super) name: String, + pub(super) description: String, + #[serde(rename = "mimeType")] + pub(super) mime_type: String, +} + +#[derive(Debug, Serialize)] +pub(super) struct PromptDefinition { + pub(super) name: String, + pub(super) description: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) arguments: Option>, +} + +#[derive(Debug, Serialize)] +pub(super) struct PromptArgument { + pub(super) name: String, + pub(super) description: String, + pub(super) required: bool, +} + +#[derive(Debug, Serialize)] +pub(super) struct TextContent { + #[serde(rename = "type")] + pub(super) content_type: String, + pub(super) text: String, +} + +#[derive(Debug, Serialize)] +// ResourceContent is used by the MCP resources/read response format. +#[allow(dead_code)] +pub(super) struct ResourceContent { + pub(super) uri: String, + #[serde(rename = "mimeType")] + pub(super) mime_type: String, + pub(super) text: String, +} + +#[derive(Debug, Serialize)] +pub(super) struct PromptMessage { + pub(super) role: String, + pub(super) content: TextContent, +} diff --git a/crates/fbuild-cli/src/mcp/util.rs b/crates/fbuild-cli/src/mcp/util.rs new file mode 100644 index 00000000..8c14fe6a --- /dev/null +++ b/crates/fbuild-cli/src/mcp/util.rs @@ -0,0 +1,69 @@ +//! Small standalone helpers for the MCP server (URI decoding, UUID generation). + +/// Simple percent-decoding for URI path segments. +pub(super) fn urlencoding_decode(input: &str) -> String { + let mut result = String::with_capacity(input.len()); + let mut chars = input.bytes(); + while let Some(b) = chars.next() { + if b == b'%' { + let hi = chars.next().unwrap_or(b'0'); + let lo = chars.next().unwrap_or(b'0'); + let val = hex_val(hi) * 16 + hex_val(lo); + result.push(val as char); + } else { + result.push(b as char); + } + } + result +} + +fn hex_val(b: u8) -> u8 { + match b { + b'0'..=b'9' => b - b'0', + b'a'..=b'f' => b - b'a' + 10, + b'A'..=b'F' => b - b'A' + 10, + _ => 0, + } +} + +/// Generate a UUID v4 string (simple implementation without extra deps). +pub(super) fn uuid_v4() -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + let seed = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + // Use time-based pseudo-random (good enough for request IDs, not crypto) + let pid = std::process::id() as u128; + let val = seed ^ (pid << 32) ^ (seed >> 16); + format!( + "{:08x}-{:04x}-4{:03x}-{:04x}-{:012x}", + (val >> 96) as u32, + (val >> 80) as u16, + (val >> 64) as u16 & 0x0fff, + ((val >> 48) as u16 & 0x3fff) | 0x8000, + val as u64 & 0xffffffffffff + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn urlencoding_decode_works() { + assert_eq!(urlencoding_decode("hello%20world"), "hello world"); + assert_eq!(urlencoding_decode("C%3A%5Cdev"), "C:\\dev"); + assert_eq!(urlencoding_decode("no-encoding"), "no-encoding"); + } + + #[test] + fn uuid_v4_has_correct_format() { + let id = uuid_v4(); + assert_eq!(id.len(), 36); + assert_eq!(id.chars().nth(8), Some('-')); + assert_eq!(id.chars().nth(13), Some('-')); + assert_eq!(id.chars().nth(14), Some('4')); // version 4 + assert_eq!(id.chars().nth(18), Some('-')); + } +} diff --git a/crates/fbuild-config/src/README.md b/crates/fbuild-config/src/README.md index 4e28fe7c..04fb4265 100644 --- a/crates/fbuild-config/src/README.md +++ b/crates/fbuild-config/src/README.md @@ -3,6 +3,6 @@ ## Modules - **`lib.rs`** -- Crate root; re-exports `PlatformIOConfig`, `BoardConfig`, `McuSpec` -- **`ini_parser.rs`** -- PlatformIO INI parser with `extends` inheritance and `${section.key}` variable substitution +- **`ini_parser/`** -- PlatformIO INI parser with `extends` inheritance and `${section.key}` variable substitution (split into `mod.rs`, `parser.rs`, `variables.rs`, `values.rs`, `tests.rs`) - **`board.rs`** -- Board configuration from built-in JSON assets, boards.txt, and platformio.ini overrides - **`mcu.rs`** -- `McuSpec` struct defining MCU flash and RAM limits diff --git a/crates/fbuild-config/src/board.rs b/crates/fbuild-config/src/board.rs deleted file mode 100644 index e64f70e4..00000000 --- a/crates/fbuild-config/src/board.rs +++ /dev/null @@ -1,1478 +0,0 @@ -//! Board configuration from boards.txt and built-in defaults. -//! -//! Supports: -//! - Loading from Arduino boards.txt format -//! - Built-in defaults for common boards -//! - Field overrides from platformio.ini board_build.* keys -//! - Preprocessor defines generation -//! - Include path resolution - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Esp32QemuPsramConfig { - pub size_mib: u32, - pub is_octal: bool, -} - -/// Metadata for a single debug tool entry from the board JSON `debug.tools` section. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct DebugToolMeta { - /// Whether the tool is built into the board (no external hardware needed). - #[serde(default)] - pub onboard: bool, - /// Whether this is the board's default debug tool. - #[serde(default)] - pub default: bool, -} - -/// Known emulator/simulator tool names that can run firmware without hardware. -const EMULATOR_TOOL_NAMES: &[&str] = &["simavr", "qemu", "renode", "ovpsim", "verilator"]; - -/// Board configuration loaded from boards.txt or built-in defaults. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BoardConfig { - pub name: String, - pub mcu: String, - pub f_cpu: String, - pub board: String, - pub core: String, - pub variant: String, - /// Variant header override for frameworks that use `#include VARIANT_H` - pub variant_h: Option, - /// USB vendor ID (optional) - pub vid: Option, - /// USB product ID (optional) - pub pid: Option, - /// Extra build flags from board definition - pub extra_flags: Option, - /// Upload protocol (e.g. "arduino", "esptool", "teensy-gui") - pub upload_protocol: Option, - /// Upload speed - pub upload_speed: Option, - /// Maximum flash size in bytes - pub max_flash: Option, - /// Maximum RAM size in bytes - pub max_ram: Option, - /// Flash mode (e.g. "dio", "qio") — ESP32 boards - pub flash_mode: Option, - /// Memory profile (e.g. "qio_qspi", "qio_opi") - ESP32 boards - pub memory_type: Option, - /// PSRAM type (e.g. "qspi", "opi") - ESP32 boards - pub psram_type: Option, - /// Flash frequency (e.g. "80000000L") — ESP32 boards - pub f_flash: Option, - /// Image flash frequency override (e.g. "48000000L") — used by esptool when - /// the board's actual SPI clock (`f_flash`) doesn't match a valid esptool frequency. - /// PlatformIO calls this `build.f_image`. When present, this takes priority over - /// `f_flash` for esptool's `--flash-freq` argument. - pub f_image: Option, - /// Partition table file (e.g. "default_8MB.csv") — ESP32 boards - pub partitions: Option, - /// Linker script (e.g. "esp32s3_out.ld") - pub ldscript: Option, - /// Platform string from board JSON (e.g. "atmelmegaavr", "atmelavr") - pub platform_str: Option, - /// Debug tools from board JSON `debug.tools` section. - /// Maps tool name (e.g. "simavr", "qemu", "renode") to its metadata. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub debug_tools: Option>, -} - -impl BoardConfig { - /// Load board config from a boards.txt file. - /// - /// Format: `uno.build.mcu=atmega328p`, `uno.name=Arduino Uno` - pub fn from_boards_txt( - path: &Path, - board_id: &str, - overrides: &HashMap, - ) -> fbuild_core::Result { - let content = std::fs::read_to_string(path).map_err(|e| { - fbuild_core::FbuildError::ConfigError(format!( - "failed to read boards.txt at {}: {}", - path.display(), - e - )) - })?; - - let props = parse_boards_txt(&content, board_id); - if props.is_empty() { - return Err(fbuild_core::FbuildError::ConfigError(format!( - "board '{}' not found in {}", - board_id, - path.display() - ))); - } - - let get = |key: &str| -> Option { - overrides - .get(key) - .cloned() - .or_else(|| props.get(key).cloned()) - }; - - let name = get("name").unwrap_or_else(|| board_id.to_string()); - let mcu = get("mcu").ok_or_else(|| { - fbuild_core::FbuildError::ConfigError(format!( - "board '{}' missing required field 'mcu'", - board_id - )) - })?; - - // For ESP32 chips we deliberately drop the boards.txt `flash_mode` - // field and let downstream consumers fall back to the per-MCU - // default ("dio"). See the equivalent comment in `from_board_id` - // for the rationale (ESP32-S3 QIE-bit unreliability + bootloader - // ROM that requires DIO). - let is_esp32_family = mcu.starts_with("esp32"); - - Ok(Self { - name, - mcu, - f_cpu: get("f_cpu").unwrap_or_else(|| "16000000L".to_string()), - board: get("board") - .or_else(|| props.get("board").cloned()) - .unwrap_or_else(|| board_id_to_board_define(board_id)), - core: get("core").unwrap_or_else(|| "arduino".to_string()), - variant: get("variant").unwrap_or_else(|| "standard".to_string()), - variant_h: get("variant_h"), - vid: get("vid"), - pid: get("pid"), - extra_flags: get("extra_flags"), - upload_protocol: get("upload.protocol") - .or_else(|| props.get("upload.protocol").cloned()), - upload_speed: get("upload.speed").or_else(|| props.get("upload.speed").cloned()), - max_flash: get("maximum_size") - .or_else(|| props.get("maximum_size").cloned()) - .and_then(|s| s.parse().ok()), - max_ram: get("maximum_data_size") - .or_else(|| props.get("maximum_data_size").cloned()) - .and_then(|s| s.parse().ok()), - flash_mode: if is_esp32_family { - overrides.get("flash_mode").cloned() - } else { - get("flash_mode") - }, - memory_type: if is_esp32_family { - overrides - .get("memory_type") - .cloned() - .or_else(|| get("memory_type")) - } else { - get("memory_type") - }, - psram_type: if is_esp32_family { - overrides - .get("psram_type") - .cloned() - .or_else(|| get("psram_type")) - } else { - get("psram_type") - }, - f_flash: get("f_flash"), - f_image: get("f_image"), - partitions: get("partitions"), - ldscript: get("ldscript"), - platform_str: get("platform_str"), - debug_tools: None, // boards.txt format does not contain debug metadata - }) - } - - /// Load board config from built-in defaults. - pub fn from_board_id( - board_id: &str, - overrides: &HashMap, - ) -> fbuild_core::Result { - let defaults = get_board_defaults(board_id).ok_or_else(|| { - fbuild_core::FbuildError::ConfigError(format!( - "unknown board '{}' (no built-in defaults)", - board_id - )) - })?; - - let get = |key: &str, default: &str| -> String { - overrides - .get(key) - .cloned() - .unwrap_or_else(|| defaults.get(key).cloned().unwrap_or(default.to_string())) - }; - - // Determine if this is an ESP32-family chip — used to ignore the - // board JSON's `flash_mode` field below. We have to compute this - // here (before constructing Self) because the resolution of - // `flash_mode` happens inside the struct literal. - let resolved_mcu = get("mcu", "unknown"); - let is_esp32_family = resolved_mcu.starts_with("esp32"); - - Ok(Self { - name: get("name", board_id), - mcu: get("mcu", "unknown"), - f_cpu: get("f_cpu", "16000000L"), - board: get("board", &board_id_to_board_define(board_id)), - core: get("core", "arduino"), - variant: get("variant", "standard"), - variant_h: overrides - .get("variant_h") - .cloned() - .or_else(|| defaults.get("variant_h").cloned()), - vid: overrides - .get("vid") - .cloned() - .or_else(|| defaults.get("vid").cloned()), - pid: overrides - .get("pid") - .cloned() - .or_else(|| defaults.get("pid").cloned()), - extra_flags: overrides - .get("extra_flags") - .cloned() - .or_else(|| defaults.get("extra_flags").cloned()), - upload_protocol: overrides - .get("upload.protocol") - .cloned() - .or_else(|| defaults.get("upload.protocol").cloned()), - upload_speed: overrides - .get("upload.speed") - .cloned() - .or_else(|| defaults.get("upload.speed").cloned()), - max_flash: overrides - .get("maximum_size") - .and_then(|s| s.parse().ok()) - .or_else(|| defaults.get("maximum_size").and_then(|s| s.parse().ok())), - max_ram: overrides - .get("maximum_data_size") - .and_then(|s| s.parse().ok()) - .or_else(|| { - defaults - .get("maximum_data_size") - .and_then(|s| s.parse().ok()) - }), - // Flash mode resolution: - // - For ESP32 chips, IGNORE the board JSON's flash_mode field. - // Many board JSONs ship `flash_mode: qio` because the flash - // chip *supports* QIO, but ESP32-S3's QIE-bit init is - // unreliable on real hardware. The MCU-level default - // (`default_flash_mode` in fbuild-build's esp32 configs) is - // "dio" for the entire ESP32 family — that's the safe value - // to use unless the user explicitly opts in via the env - // section's `board_build.flash_mode = qio`. - // - For non-ESP32 chips, the existing behaviour is preserved - // (env override → board JSON → None). - // - Either way, downstream code that needs an effective value - // when this is `None` should fall back to - // `mcu_config.default_flash_mode()`. - flash_mode: if is_esp32_family { - overrides.get("flash_mode").cloned() - } else { - overrides - .get("flash_mode") - .cloned() - .or_else(|| defaults.get("flash_mode").cloned()) - }, - memory_type: overrides - .get("memory_type") - .cloned() - .or_else(|| defaults.get("memory_type").cloned()), - psram_type: overrides - .get("psram_type") - .cloned() - .or_else(|| defaults.get("psram_type").cloned()), - f_flash: overrides - .get("f_flash") - .cloned() - .or_else(|| defaults.get("f_flash").cloned()), - f_image: overrides - .get("f_image") - .cloned() - .or_else(|| defaults.get("f_image").cloned()), - partitions: overrides - .get("partitions") - .cloned() - .or_else(|| defaults.get("partitions").cloned()), - ldscript: overrides - .get("ldscript") - .cloned() - .or_else(|| defaults.get("ldscript").cloned()), - platform_str: defaults.get("platform_str").cloned(), - debug_tools: get_board_debug_tools(board_id), - }) - } - - /// Returns emulator/simulator tools available for this board. - /// - /// Filters `debug_tools` to only include known software emulators - /// (simavr, qemu, renode, ovpsim, verilator), excluding hardware debug probes. - pub fn emulators(&self) -> HashMap<&str, &DebugToolMeta> { - let Some(ref tools) = self.debug_tools else { - return HashMap::new(); - }; - tools - .iter() - .filter(|(name, _)| EMULATOR_TOOL_NAMES.contains(&name.as_str())) - .map(|(name, meta)| (name.as_str(), meta)) - .collect() - } - - /// Check whether this board supports a specific emulator tool. - pub fn has_emulator(&self, tool_name: &str) -> bool { - self.debug_tools - .as_ref() - .is_some_and(|tools| tools.contains_key(tool_name)) - && EMULATOR_TOOL_NAMES.contains(&tool_name) - } - - /// Resolve the effective ESP32 SDK memory profile used for variant headers/libs. - /// - /// This keeps the SDK `sdkconfig.h` and memory-profile libraries aligned - /// with the repo's effective flash-mode policy. Boards that explicitly use - /// OPI flash keep the `opi` flash-half because that represents a distinct - /// bus type rather than an optional fast-read mode. - pub fn effective_esp32_memory_type(&self, default_flash_mode: &str) -> Option { - if !self.mcu.starts_with("esp32") { - return None; - } - - let effective_flash_mode = self - .flash_mode - .as_deref() - .unwrap_or(default_flash_mode) - .to_ascii_lowercase(); - - let (flash_half, psram_half) = if let Some(memory_type) = self.memory_type.as_deref() { - if let Some((flash, psram)) = memory_type.split_once('_') { - ( - Some(flash.to_ascii_lowercase()), - Some(psram.to_ascii_lowercase()), - ) - } else { - (Some(memory_type.to_ascii_lowercase()), None) - } - } else { - (None, None) - }; - - let resolved_flash = match flash_half.as_deref() { - Some("opi") => "opi".to_string(), - _ => effective_flash_mode, - }; - let resolved_psram = psram_half - .or_else(|| self.psram_type.as_deref().map(|s| s.to_ascii_lowercase())) - .unwrap_or_else(|| "qspi".to_string()); - - Some(format!("{}_{}", resolved_flash, resolved_psram)) - } - - pub fn qemu_esp32_psram_config(&self) -> Option { - let has_psram = self - .extra_flags - .as_deref() - .is_some_and(|flags| extra_flags_contain_define(flags, "BOARD_HAS_PSRAM")) - || self.psram_type.is_some(); - if !has_psram { - return None; - } - - let is_octal = self - .psram_type - .as_deref() - .is_some_and(|psram| psram.eq_ignore_ascii_case("opi")) - || self - .memory_type - .as_deref() - .is_some_and(|memory| memory.ends_with("_opi")); - let size_mib = infer_psram_size_mib(&self.name).unwrap_or(if is_octal { 8 } else { 2 }); - - Some(Esp32QemuPsramConfig { size_mib, is_octal }) - } - - /// Detect the platform from the board JSON's platform field, or fall back to MCU heuristic. - pub fn platform(&self) -> Option { - // Prefer explicit platform from board JSON (distinguishes AtmelMegaAvr from AtmelAvr) - if let Some(ref p) = self.platform_str { - if let Some(platform) = fbuild_core::Platform::from_platform_str(p) { - return Some(platform); - } - } - let mcu = self.mcu.to_lowercase(); - if mcu.starts_with("atmega") || mcu.starts_with("attiny") || mcu.starts_with("at90") { - Some(fbuild_core::Platform::AtmelAvr) - } else if mcu.starts_with("esp32") { - Some(fbuild_core::Platform::Espressif32) - } else if mcu.starts_with("esp8266") || mcu.starts_with("esp8285") { - Some(fbuild_core::Platform::Espressif8266) - } else if mcu.starts_with("imxrt") || mcu.starts_with("mk") { - Some(fbuild_core::Platform::Teensy) - } else if mcu.starts_with("rp2040") || mcu.starts_with("rp2350") { - Some(fbuild_core::Platform::RaspberryPi) - } else if mcu.starts_with("stm32") { - Some(fbuild_core::Platform::Ststm32) - } else if mcu.starts_with("nrf52") { - Some(fbuild_core::Platform::NordicNrf52) - } else if mcu.starts_with("at91sam") || mcu.starts_with("sam") { - Some(fbuild_core::Platform::AtmelSam) - } else if mcu.starts_with("ra4") || mcu.starts_with("ra6") { - Some(fbuild_core::Platform::RenesasRa) - } else if mcu.starts_with("ch32") { - Some(fbuild_core::Platform::Ch32v) - } else if mcu.starts_with("apollo3") || mcu.starts_with("ama3b") { - Some(fbuild_core::Platform::Apollo3) - } else { - None - } - } - - /// Generate preprocessor defines for this board. - /// - /// Returns defines like: PLATFORMIO, F_CPU, ARDUINO, `ARDUINO_`, `ARDUINO_ARCH_` - pub fn get_defines(&self) -> HashMap { - let mut defines = HashMap::new(); - - defines.insert("PLATFORMIO".to_string(), "1".to_string()); - defines.insert("F_CPU".to_string(), self.f_cpu.clone()); - - // Default Arduino version. Platform-specific overrides (e.g. Teensy=10819) - // are in MCU config JSON defines, merged by the orchestrator after this. - defines.insert("ARDUINO".to_string(), "10808".to_string()); - - defines.insert( - format!("ARDUINO_{}", self.board.to_uppercase()), - "1".to_string(), - ); - // ARDUINO_BOARD and ARDUINO_VARIANT as quoted string defines. - // Use \" escapes so GCC response files on Windows preserve the quotes - // (bare " is treated as a word delimiter by GCC's response file parser). - defines.insert( - "ARDUINO_BOARD".to_string(), - format!("\\\"{}\\\"", self.board), - ); - defines.insert( - "ARDUINO_VARIANT".to_string(), - format!("\\\"{}\\\"", self.variant), - ); - - // Architecture define - let arch = self.arch_define(); - if !arch.is_empty() { - defines.insert(format!("ARDUINO_ARCH_{}", arch), "1".to_string()); - } - - // MCU-specific define for AVR - let mcu_upper = self.mcu.to_uppercase(); - if mcu_upper.starts_with("ATMEGA") || mcu_upper.starts_with("ATTINY") { - defines.insert(format!("__AVR_{}__", mcu_upper), "1".to_string()); - } - - // Teensy __MCU__ define (MCU detection, not a versioned constant) - if matches!(self.platform(), Some(fbuild_core::Platform::Teensy)) - && mcu_upper.starts_with("IMXRT") - { - defines.insert(format!("__{}__", mcu_upper), "1".to_string()); - } - - // USB VID/PID defines for USB-native boards (Leonardo, Micro, etc.) - if let Some(ref vid) = self.vid { - defines.insert("USB_VID".to_string(), vid.clone()); - } - if let Some(ref pid) = self.pid { - defines.insert("USB_PID".to_string(), pid.clone()); - } - - // Extra flags - if let Some(ref flags) = self.extra_flags { - for flag in fbuild_core::shell_split::split(flags) { - if let Some(define) = flag.strip_prefix("-D") { - if let Some(eq_pos) = define.find('=') { - defines.insert( - define[..eq_pos].to_string(), - define[eq_pos + 1..].to_string(), - ); - } else { - defines.insert(define.to_string(), "1".to_string()); - } - } - } - } - - defines - } - - /// Get include paths relative to a framework root directory. - /// - /// Returns: `[cores/, variants/]` - pub fn get_include_paths(&self, framework_root: &Path) -> Vec { - vec![ - framework_root.join("cores").join(&self.core), - framework_root.join("variants").join(&self.variant), - ] - } - - fn arch_define(&self) -> String { - match self.platform() { - Some(fbuild_core::Platform::AtmelAvr) => "AVR".to_string(), - Some(fbuild_core::Platform::AtmelMegaAvr) => "MEGAAVR".to_string(), - Some(fbuild_core::Platform::Espressif32) => "ESP32".to_string(), - Some(fbuild_core::Platform::Espressif8266) => "ESP8266".to_string(), - Some(fbuild_core::Platform::NordicNrf52) => "NRF52".to_string(), - Some(fbuild_core::Platform::RaspberryPi) => "RP2040".to_string(), - Some(fbuild_core::Platform::RenesasRa) => "RENESAS".to_string(), - Some(fbuild_core::Platform::SiliconLabs) => "SILABS".to_string(), - Some(fbuild_core::Platform::Ststm32) => "STM32".to_string(), - Some(fbuild_core::Platform::AtmelSam) => "SAM".to_string(), - Some(fbuild_core::Platform::Teensy) => "TEENSY".to_string(), - Some(fbuild_core::Platform::Ch32v) => "CH32V".to_string(), - Some(fbuild_core::Platform::Apollo3) => "APOLLO3".to_string(), - Some(fbuild_core::Platform::Wasm) | None => String::new(), - } - } -} - -/// Parse boards.txt content for a specific board_id. -/// -/// Format: `board_id.key=value`, with `build.` and `upload.` prefixes. -fn parse_boards_txt(content: &str, board_id: &str) -> HashMap { - let mut props = HashMap::new(); - let prefix = format!("{}.", board_id); - - for line in content.lines() { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - continue; - } - - if let Some(rest) = trimmed.strip_prefix(&prefix) { - if let Some(eq_pos) = rest.find('=') { - let key = rest[..eq_pos].trim(); - let value = rest[eq_pos + 1..].trim(); - - // Strip build. and upload. prefixes for convenience - let normalized_key = key - .strip_prefix("build.") - .or_else(|| key.strip_prefix("upload.")) - .unwrap_or(key); - - // Keep both the original and stripped key - props.insert(normalized_key.to_string(), value.to_string()); - if normalized_key != key { - props.insert(key.to_string(), value.to_string()); - } - } - } - } - - props -} - -/// Convert a board_id like "uno" to a board define like "UNO". -fn board_id_to_board_define(board_id: &str) -> String { - board_id.to_uppercase().replace('-', "_") -} - -fn extra_flags_contain_define(extra_flags: &str, define: &str) -> bool { - extra_flags.split_whitespace().any(|flag| { - let Some(raw) = flag.strip_prefix("-D") else { - return false; - }; - raw.split_once('=').map_or(raw, |(name, _)| name) == define - }) -} - -fn infer_psram_size_mib(name: &str) -> Option { - let upper = name.to_ascii_uppercase(); - - for size in [32_u32, 16, 8, 4, 2] { - if upper.contains(&format!("{size} MB PSRAM")) { - return Some(size); - } - } - - for (marker, size) in [ - ("R32", 32_u32), - ("R16", 16), - ("R8", 8), - ("R4", 4), - ("R2", 2), - ] { - if upper.contains(marker) { - return Some(size); - } - } - - None -} - -/// Embedded board database — 1609 boards from PlatformIO registry JSON files. -/// -/// Loaded once on first access via `OnceLock`. Each entry maps board_id → JSON object -/// with fields: id, name, mcu, platform, fcpu, ram, rom, etc. -static BOARD_DB: std::sync::OnceLock> = - std::sync::OnceLock::new(); - -static BOARDS_DIR: include_dir::Dir = - include_dir::include_dir!("$CARGO_MANIFEST_DIR/assets/boards/json"); - -fn get_board_db() -> &'static HashMap { - BOARD_DB.get_or_init(|| { - let mut db = HashMap::new(); - for file in BOARDS_DIR.files() { - let Some(stem) = file.path().file_stem().and_then(|s| s.to_str()) else { - continue; - }; - if file.path().extension().and_then(|e| e.to_str()) != Some("json") { - continue; - } - let Some(contents) = file.contents_utf8() else { - continue; - }; - match serde_json::from_str(contents) { - Ok(value) => { - db.insert(stem.to_string(), value); - } - Err(e) => { - tracing::error!("failed to parse board {}: {}", stem, e); - } - } - } - db - }) -} - -/// Common board aliases mapping short names to JSON board_ids. -fn resolve_board_alias(board_id: &str) -> &str { - match board_id { - "mega" => "megaatmega2560", - "nano" | "nanoatmega328" => "nanoatmega328", - "pico" | "rpipico" => "rpipico", - "picow" | "rpipicow" => "rpipicow", - "pico2" | "rpipico2" => "rpipico2", - "esp32c3" => "esp32-c3-devkitm-1", - "esp32c6" => "esp32-c6-devkitm-1", - "esp32s3" => "esp32-s3-devkitc-1", - "ch32l103" => "genericCH32L103C8T6", - "ch32v003" => "genericCH32V003F4P6", - "ch32v006" => "genericCH32V006K8U6", - "ch32v103" => "genericCH32V103C8T6", - "ch32v203" => "genericCH32V203C8T6", - "ch32v208" => "genericCH32V208WBU6", - "ch32v303" => "genericCH32V303VCT6", - "ch32v307" => "genericCH32V307VCT6", - "ch32x035" => "genericCH32X035C8T6", - "adafruit_grand_central_m4" => "adafruit_grandcentral_m4", - other => other, - } -} - -/// Extract debug tools from a board JSON entry's `debug.tools` section. -fn get_board_debug_tools(board_id: &str) -> Option> { - let db = get_board_db(); - let resolved = resolve_board_alias(board_id); - let entry = db.get(board_id).or_else(|| db.get(resolved))?; - - let tools = entry - .get("debug") - .and_then(|d| d.get("tools")) - .and_then(|t| t.as_object())?; - - if tools.is_empty() { - return None; - } - - let mut result = HashMap::new(); - for (name, meta) in tools { - let onboard = meta - .get("onboard") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - let default = meta - .get("default") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - result.insert(name.clone(), DebugToolMeta { onboard, default }); - } - - Some(result) -} - -fn get_board_defaults(board_id: &str) -> Option> { - let db = get_board_db(); - let resolved = resolve_board_alias(board_id); - let entry = db.get(board_id).or_else(|| db.get(resolved))?; - - let mut d = HashMap::new(); - - if let Some(name) = entry.get("name").and_then(|v| v.as_str()) { - d.insert("name".into(), name.to_string()); - } - - let mcu = entry - .get("mcu") - .and_then(|v| v.as_str()) - .unwrap_or("unknown") - .to_lowercase(); - d.insert("mcu".into(), mcu.clone()); - - if let Some(fcpu) = entry.get("fcpu").and_then(|v| v.as_u64()) { - d.insert("f_cpu".into(), format!("{}L", fcpu)); - } - - if let Some(ram) = entry.get("ram").and_then(|v| v.as_u64()) { - d.insert("maximum_data_size".into(), ram.to_string()); - } - - if let Some(rom) = entry.get("rom").and_then(|v| v.as_u64()) { - d.insert("maximum_size".into(), rom.to_string()); - } - - if let Some(platform) = entry.get("platform").and_then(|v| v.as_str()) { - d.insert("platform_str".into(), platform.to_string()); - } - - // Data-driven: read build section from enriched JSON - if let Some(build) = entry.get("build").and_then(|v| v.as_object()) { - if let Some(core) = build.get("core").and_then(|v| v.as_str()) { - d.insert("core".into(), core.to_string()); - } - if let Some(variant) = build.get("variant").and_then(|v| v.as_str()) { - d.insert("variant".into(), variant.to_string()); - } - if let Some(variant_h) = build.get("variant_h").and_then(|v| v.as_str()) { - d.insert("variant_h".into(), variant_h.to_string()); - } - if let Some(flags) = build.get("extra_flags").and_then(|v| v.as_str()) { - d.insert("extra_flags".into(), flags.to_string()); - } - if let Some(vid) = build.get("vid").and_then(|v| v.as_str()) { - d.insert("vid".into(), vid.to_string()); - } - if let Some(pid) = build.get("pid").and_then(|v| v.as_str()) { - d.insert("pid".into(), pid.to_string()); - } - if let Some(flash_mode) = build.get("flash_mode").and_then(|v| v.as_str()) { - d.insert("flash_mode".into(), flash_mode.to_string()); - } - if let Some(memory_type) = build.get("memory_type").and_then(|v| v.as_str()) { - d.insert("memory_type".into(), memory_type.to_string()); - } - if let Some(psram_type) = build.get("psram_type").and_then(|v| v.as_str()) { - d.insert("psram_type".into(), psram_type.to_string()); - } - if let Some(f_flash) = build.get("f_flash").and_then(|v| v.as_str()) { - d.insert("f_flash".into(), f_flash.to_string()); - } - if let Some(f_image) = build.get("f_image").and_then(|v| v.as_str()) { - d.insert("f_image".into(), f_image.to_string()); - } - // Arduino sub-fields - if let Some(arduino) = build.get("arduino").and_then(|v| v.as_object()) { - if let Some(ldscript) = arduino.get("ldscript").and_then(|v| v.as_str()) { - d.insert("ldscript".into(), ldscript.to_string()); - } - if let Some(partitions) = arduino.get("partitions").and_then(|v| v.as_str()) { - d.insert("partitions".into(), partitions.to_string()); - } - if let Some(memory_type) = arduino.get("memory_type").and_then(|v| v.as_str()) { - d.insert("memory_type".into(), memory_type.to_string()); - } - // Core-specific overrides: build.arduino..variant - // e.g. build.arduino.openwch.variant = "CH32V00x/CH32V003F4" - if let Some(core_name) = build.get("core").and_then(|v| v.as_str()) { - if let Some(core_obj) = arduino.get(core_name).and_then(|v| v.as_object()) { - if let Some(variant) = core_obj.get("variant").and_then(|v| v.as_str()) { - d.insert("variant".into(), variant.to_string()); - } - if let Some(vh) = core_obj.get("variant_h").and_then(|v| v.as_str()) { - d.insert("variant_h".into(), vh.to_string()); - } - } - } - } - } - - // Data-driven: read upload section from enriched JSON - if let Some(upload) = entry.get("upload").and_then(|v| v.as_object()) { - if let Some(protocol) = upload.get("protocol").and_then(|v| v.as_str()) { - d.insert("upload.protocol".into(), protocol.to_string()); - } - if let Some(speed) = upload.get("speed").and_then(|v| v.as_u64()) { - d.insert("upload.speed".into(), speed.to_string()); - } - } - - // Fallback for unenriched boards: derive from platform - if !d.contains_key("core") { - d.insert("core".into(), "arduino".into()); - } - if !d.contains_key("variant") { - // For ESP32 boards, the Arduino framework uses the MCU name as the - // variant directory name (e.g., variants/esp32c6/). Fall back to - // MCU rather than board_id so builds find pins_arduino.h. - let fallback = if d.get("core").is_some_and(|c| c == "esp32") { - d.get("mcu") - .cloned() - .unwrap_or_else(|| board_id.to_string()) - } else { - board_id.to_string() - }; - d.insert("variant".into(), fallback); - } - d.entry("board".into()) - .or_insert_with(|| board_id_to_board_define(board_id)); - - Some(d) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - use tempfile::NamedTempFile; - - fn write_boards_txt(content: &str) -> NamedTempFile { - let mut f = NamedTempFile::new().unwrap(); - f.write_all(content.as_bytes()).unwrap(); - f.flush().unwrap(); - f - } - - #[test] - fn test_from_board_id_uno() { - let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - assert_eq!(config.name, "Arduino Uno"); - assert_eq!(config.mcu, "atmega328p"); - assert_eq!(config.f_cpu, "16000000L"); - assert_eq!(config.board, "UNO"); - assert_eq!(config.core, "arduino"); - assert_eq!(config.variant, "standard"); - assert_eq!(config.max_flash, Some(32256)); - assert_eq!(config.max_ram, Some(2048)); - // extra_flags from enriched JSON - assert_eq!(config.extra_flags, Some("-DARDUINO_AVR_UNO".to_string())); - assert_eq!(config.upload_protocol, Some("arduino".to_string())); - assert_eq!(config.upload_speed, Some("115200".to_string())); - } - - #[test] - fn test_from_board_id_mega() { - let config = BoardConfig::from_board_id("mega", &HashMap::new()).unwrap(); - assert_eq!(config.mcu, "atmega2560"); - assert_eq!(config.variant, "mega"); - } - - #[test] - fn test_from_board_id_megaatmega2560_alias() { - let config = BoardConfig::from_board_id("megaatmega2560", &HashMap::new()).unwrap(); - assert_eq!(config.mcu, "atmega2560"); - } - - #[test] - fn test_ch32v_aliases() { - let cases = [ - ("ch32l103", "ch32l103c8t6", Some("variant_CH32L103C8T6.h")), - ("ch32v003", "ch32v003f4p6", Some("variant_CH32V003F4.h")), - ("ch32v006", "ch32v006k8u6", Some("variant_CH32V006K8.h")), - ("ch32v103", "ch32v103c8t6", Some("variant_CH32V103R8T6.h")), - ("ch32v203", "ch32v203c8t6", Some("variant_CH32V203C8.h")), - ("ch32v208", "ch32v208wbu6", Some("variant_CH32V203C8.h")), - ("ch32v303", "ch32v303vct6", Some("variant_CH32V307VCT6.h")), - ("ch32v307", "ch32v307vct6", Some("variant_CH32V307VCT6.h")), - ("ch32x035", "ch32x035c8t6", Some("variant_CH32X035G8U.h")), - ]; - for (alias, expected_mcu, expected_variant_h) in cases { - let config = BoardConfig::from_board_id(alias, &HashMap::new()).unwrap(); - assert_eq!( - config.mcu, expected_mcu, - "alias '{}' should resolve to mcu '{}'", - alias, expected_mcu - ); - assert_eq!( - config.variant_h.as_deref(), - expected_variant_h, - "alias '{}' should resolve to variant_h '{:?}'", - alias, - expected_variant_h - ); - } - } - - #[test] - fn test_from_board_id_unknown() { - let result = BoardConfig::from_board_id("nonexistent_board", &HashMap::new()); - assert!(result.is_err()); - } - - #[test] - fn test_from_board_id_with_overrides() { - let mut overrides = HashMap::new(); - overrides.insert("f_cpu".to_string(), "8000000L".to_string()); - let config = BoardConfig::from_board_id("uno", &overrides).unwrap(); - assert_eq!(config.f_cpu, "8000000L"); - assert_eq!(config.mcu, "atmega328p"); // not overridden - } - - #[test] - fn test_from_boards_txt_valid() { - let f = write_boards_txt( - "\ -uno.name=Arduino Uno -uno.build.mcu=atmega328p -uno.build.f_cpu=16000000L -uno.build.board=AVR_UNO -uno.build.core=arduino -uno.build.variant=standard -uno.upload.maximum_size=32256 -uno.upload.maximum_data_size=2048 -", - ); - let config = BoardConfig::from_boards_txt(f.path(), "uno", &HashMap::new()).unwrap(); - assert_eq!(config.name, "Arduino Uno"); - assert_eq!(config.mcu, "atmega328p"); - assert_eq!(config.max_flash, Some(32256)); - } - - #[test] - fn test_from_boards_txt_nonexistent_board() { - let f = write_boards_txt("uno.name=Arduino Uno\nuno.build.mcu=atmega328p\n"); - let result = BoardConfig::from_boards_txt(f.path(), "mega", &HashMap::new()); - assert!(result.is_err()); - } - - #[test] - fn test_from_boards_txt_with_overrides() { - let f = write_boards_txt( - "\ -uno.name=Arduino Uno -uno.build.mcu=atmega328p -uno.build.f_cpu=16000000L -uno.build.board=AVR_UNO -uno.build.core=arduino -uno.build.variant=standard -", - ); - let mut overrides = HashMap::new(); - overrides.insert("f_cpu".to_string(), "8000000L".to_string()); - let config = BoardConfig::from_boards_txt(f.path(), "uno", &overrides).unwrap(); - assert_eq!(config.f_cpu, "8000000L"); - } - - #[test] - fn test_from_boards_txt_multiple_boards() { - let f = write_boards_txt( - "\ -uno.name=Arduino Uno -uno.build.mcu=atmega328p -mega.name=Arduino Mega 2560 -mega.build.mcu=atmega2560 -", - ); - let uno = BoardConfig::from_boards_txt(f.path(), "uno", &HashMap::new()).unwrap(); - let mega = BoardConfig::from_boards_txt(f.path(), "mega", &HashMap::new()).unwrap(); - assert_eq!(uno.mcu, "atmega328p"); - assert_eq!(mega.mcu, "atmega2560"); - } - - #[test] - fn test_from_boards_txt_with_upload_info() { - let f = write_boards_txt( - "\ -leonardo.name=Arduino Leonardo -leonardo.build.mcu=atmega32u4 -leonardo.build.vid=0x2341 -leonardo.build.pid=0x8036 -leonardo.upload.protocol=avr109 -leonardo.upload.speed=57600 -", - ); - let config = BoardConfig::from_boards_txt(f.path(), "leonardo", &HashMap::new()).unwrap(); - assert_eq!(config.vid, Some("0x2341".to_string())); - assert_eq!(config.pid, Some("0x8036".to_string())); - assert_eq!(config.upload_protocol, Some("avr109".to_string())); - assert_eq!(config.upload_speed, Some("57600".to_string())); - } - - #[test] - fn test_get_defines_basic() { - let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - let defines = config.get_defines(); - assert_eq!(defines.get("PLATFORMIO"), Some(&"1".to_string())); - assert_eq!(defines.get("F_CPU"), Some(&"16000000L".to_string())); - assert_eq!(defines.get("ARDUINO"), Some(&"10808".to_string())); - assert_eq!(defines.get("ARDUINO_AVR_UNO"), Some(&"1".to_string())); - assert_eq!(defines.get("ARDUINO_ARCH_AVR"), Some(&"1".to_string())); - assert_eq!(defines.get("__AVR_ATMEGA328P__"), Some(&"1".to_string())); - } - - #[test] - fn test_get_defines_with_extra_flags() { - let mut config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - config.extra_flags = Some("-DCUSTOM_FLAG -DVALUE=42".to_string()); - let defines = config.get_defines(); - assert_eq!(defines.get("CUSTOM_FLAG"), Some(&"1".to_string())); - assert_eq!(defines.get("VALUE"), Some(&"42".to_string())); - } - - #[test] - fn test_get_include_paths() { - let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - let paths = config.get_include_paths(Path::new("/framework")); - assert_eq!(paths.len(), 2); - assert_eq!(paths[0], PathBuf::from("/framework/cores/arduino")); - assert_eq!(paths[1], PathBuf::from("/framework/variants/standard")); - } - - #[test] - fn test_platform_detection() { - let avr = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - assert_eq!(avr.platform(), Some(fbuild_core::Platform::AtmelAvr)); - - let esp = BoardConfig::from_board_id("esp32dev", &HashMap::new()).unwrap(); - assert_eq!(esp.platform(), Some(fbuild_core::Platform::Espressif32)); - - let teensy = BoardConfig::from_board_id("teensy41", &HashMap::new()).unwrap(); - assert_eq!(teensy.platform(), Some(fbuild_core::Platform::Teensy)); - - let rpi = BoardConfig::from_board_id("rpipico", &HashMap::new()).unwrap(); - assert_eq!(rpi.platform(), Some(fbuild_core::Platform::RaspberryPi)); - } - - #[test] - fn test_parse_boards_txt_with_comments() { - let props = parse_boards_txt( - "# This is a comment\nuno.name=Arduino Uno\n# Another comment\nuno.build.mcu=atmega328p\n", - "uno", - ); - assert_eq!(props.get("name"), Some(&"Arduino Uno".to_string())); - assert_eq!(props.get("mcu"), Some(&"atmega328p".to_string())); - } - - #[test] - fn test_parse_boards_txt_empty_lines() { - let props = parse_boards_txt( - "\nuno.name=Arduino Uno\n\nuno.build.mcu=atmega328p\n\n", - "uno", - ); - assert_eq!(props.get("name"), Some(&"Arduino Uno".to_string())); - } - - #[test] - fn test_parse_boards_txt_ignores_other_boards() { - let props = parse_boards_txt( - "uno.name=Arduino Uno\nmega.name=Arduino Mega\nuno.build.mcu=atmega328p\n", - "uno", - ); - assert_eq!(props.get("name"), Some(&"Arduino Uno".to_string())); - assert!(!props.values().any(|v| v == "Arduino Mega")); - } - - #[test] - fn test_repr() { - let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - let repr = format!("{:?}", config); - assert!(repr.contains("Arduino Uno")); - assert!(repr.contains("atmega328p")); - } - - // --- Data-driven enriched JSON tests --- - - #[test] - fn test_mega_upload_protocol_wiring() { - // Bug fix: mega needs protocol=wiring, not arduino - let config = BoardConfig::from_board_id("mega", &HashMap::new()).unwrap(); - assert_eq!(config.upload_protocol, Some("wiring".to_string())); - assert_eq!(config.core, "arduino"); - assert_eq!(config.variant, "mega"); - } - - #[test] - fn test_nano_upload_speed_57600() { - // Bug fix: nano uses 57600, not 115200 - let config = BoardConfig::from_board_id("nano", &HashMap::new()).unwrap(); - assert_eq!(config.upload_speed, Some("57600".to_string())); - assert_eq!(config.upload_protocol, Some("arduino".to_string())); - assert_eq!(config.variant, "eightanaloginputs"); - } - - #[test] - fn test_teensy36_core_teensy3() { - // Bug fix: teensy30-36 use core=teensy3, not teensy4 - let config = BoardConfig::from_board_id("teensy36", &HashMap::new()).unwrap(); - assert_eq!(config.core, "teensy3"); - assert_eq!(config.upload_protocol, Some("teensy-gui".to_string())); - } - - #[test] - fn test_teensy41_core_teensy4() { - let config = BoardConfig::from_board_id("teensy41", &HashMap::new()).unwrap(); - assert_eq!(config.core, "teensy4"); - assert_eq!(config.upload_protocol, Some("teensy-gui".to_string())); - } - - #[test] - fn test_esp32dev_enriched_fields() { - let config = BoardConfig::from_board_id("esp32dev", &HashMap::new()).unwrap(); - assert_eq!(config.core, "esp32"); - assert_eq!(config.variant, "esp32"); - // ESP32 boards now intentionally drop the JSON-shipped flash_mode - // (see comments in `from_board_id`). Downstream consumers fall back - // to mcu_config.default_flash_mode() which is "dio". - assert_eq!(config.flash_mode, None); - assert_eq!(config.memory_type, None); - assert_eq!(config.f_flash, Some("40000000L".to_string())); - assert_eq!(config.ldscript, Some("esp32_out.ld".to_string())); - assert_eq!(config.upload_speed, Some("460800".to_string())); - } - - #[test] - fn test_esp32_flash_mode_env_override_honoured() { - // The user can opt back into QIO via `board_build.flash_mode = qio` - // in their [env:X] section, which the daemon translates into a - // `flash_mode` override key. - let mut overrides = HashMap::new(); - overrides.insert("flash_mode".to_string(), "qio".to_string()); - let config = BoardConfig::from_board_id("esp32dev", &overrides).unwrap(); - assert_eq!(config.flash_mode, Some("qio".to_string())); - } - - #[test] - fn test_pico_enriched_fields() { - let config = BoardConfig::from_board_id("rpipico", &HashMap::new()).unwrap(); - assert_eq!(config.core, "earlephilhower"); - assert_eq!(config.variant, "rpipico"); - assert_eq!(config.upload_protocol, Some("picotool".to_string())); - } - - #[test] - fn test_extra_flags_produce_defines() { - // Enriched extra_flags should produce correct ARDUINO_* defines - let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - let defines = config.get_defines(); - assert_eq!(defines.get("ARDUINO_AVR_UNO"), Some(&"1".to_string())); - } - - #[test] - fn test_esp32c3_board_config() { - let config = BoardConfig::from_board_id("esp32c3", &HashMap::new()).unwrap(); - assert_eq!(config.mcu, "esp32c3"); - assert_eq!(config.core, "esp32"); - // ESP32 boards drop the JSON-shipped flash_mode (see - // `test_esp32dev_enriched_fields`); fall back is "dio" from MCU config. - assert_eq!(config.flash_mode, None); - assert_eq!(config.memory_type, None); - assert_eq!(config.ldscript, Some("esp32c3_out.ld".to_string())); - // ESP32-C3 DevKit runs at 160 MHz - assert_eq!(config.f_cpu, "160000000L"); - } - - #[test] - fn test_esp32_effective_memory_type_tracks_effective_flash_mode() { - let config = BoardConfig::from_board_id("esp32c3", &HashMap::new()).unwrap(); - assert_eq!( - config.effective_esp32_memory_type("dio"), - Some("dio_qspi".to_string()) - ); - } - - #[test] - fn test_esp32_effective_memory_type_preserves_opi_flash_profiles() { - let config = - BoardConfig::from_board_id("esp32-s3-devkitc-1-n32r8v", &HashMap::new()).unwrap(); - assert_eq!( - config.effective_esp32_memory_type("dio"), - Some("opi_opi".to_string()) - ); - } - - #[test] - fn test_qemu_psram_config_absent_for_non_psram_board() { - let config = BoardConfig::from_board_id("esp32-s3-devkitc-1", &HashMap::new()).unwrap(); - assert_eq!(config.qemu_esp32_psram_config(), None); - } - - #[test] - fn test_qemu_psram_config_detects_quad_psram_board() { - let config = BoardConfig::from_board_id("esp32-s3-devkitc1-n8r2", &HashMap::new()).unwrap(); - assert_eq!( - config.qemu_esp32_psram_config(), - Some(Esp32QemuPsramConfig { - size_mib: 2, - is_octal: false, - }) - ); - } - - #[test] - fn test_qemu_psram_config_detects_octal_psram_board() { - let config = BoardConfig::from_board_id("esp32-s3-devkitc1-n8r8", &HashMap::new()).unwrap(); - assert_eq!( - config.qemu_esp32_psram_config(), - Some(Esp32QemuPsramConfig { - size_mib: 8, - is_octal: true, - }) - ); - } - - #[test] - fn test_esp32c3_devkitm1_board_config() { - // Direct look up by full board ID (same underlying JSON) - let config = BoardConfig::from_board_id("esp32-c3-devkitm-1", &HashMap::new()).unwrap(); - assert_eq!(config.mcu, "esp32c3"); - let flags = config.extra_flags.unwrap_or_default(); - assert!( - flags.contains("ARDUINO_ESP32C3_DEV"), - "expected ARDUINO_ESP32C3_DEV in extra_flags, got: {flags}" - ); - } - - #[test] - fn test_esp32c3_no_psram() { - // The plain C3 DevKit has no PSRAM - let config = BoardConfig::from_board_id("esp32c3", &HashMap::new()).unwrap(); - let flags = config.extra_flags.clone().unwrap_or_default(); - assert!( - !flags.contains("BOARD_HAS_PSRAM"), - "ESP32-C3 DevKit should not have PSRAM flag, got: {flags}" - ); - } - - // --- USB VID/PID defines --- - - #[test] - fn test_get_defines_usb_vid_pid() { - let mut config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - config.vid = Some("0x2341".to_string()); - config.pid = Some("0x8036".to_string()); - let defines = config.get_defines(); - assert_eq!(defines.get("USB_VID"), Some(&"0x2341".to_string())); - assert_eq!(defines.get("USB_PID"), Some(&"0x8036".to_string())); - } - - #[test] - fn test_get_defines_no_usb_when_absent() { - let config = BoardConfig::from_board_id("esp32dev", &HashMap::new()).unwrap(); - assert!(config.vid.is_none()); - let defines = config.get_defines(); - assert!(!defines.contains_key("USB_VID")); - assert!(!defines.contains_key("USB_PID")); - } - - #[test] - fn test_leonardo_board_has_vid_pid() { - let config = BoardConfig::from_board_id("leonardo", &HashMap::new()).unwrap(); - assert_eq!(config.vid, Some("0x2341".to_string())); - assert_eq!(config.pid, Some("0x8036".to_string())); - let defines = config.get_defines(); - assert_eq!(defines.get("USB_VID"), Some(&"0x2341".to_string())); - assert_eq!(defines.get("USB_PID"), Some(&"0x8036".to_string())); - } - - #[test] - fn test_attiny1604_board_config() { - let config = BoardConfig::from_board_id("ATtiny1604", &HashMap::new()).unwrap(); - assert_eq!(config.mcu, "attiny1604"); - assert_eq!(config.core, "megatinycore"); - assert_eq!(config.variant, "txy4"); - assert_eq!(config.platform(), Some(fbuild_core::Platform::AtmelMegaAvr)); - let defines = config.get_defines(); - assert_eq!(defines.get("ARDUINO_ARCH_MEGAAVR"), Some(&"1".to_string())); - } - - #[test] - fn test_nano_every_board_config() { - let config = BoardConfig::from_board_id("nano_every", &HashMap::new()).unwrap(); - assert_eq!(config.mcu, "atmega4809"); - assert_eq!(config.core, "arduino"); - assert_eq!(config.variant, "nona4809"); - assert_eq!(config.platform(), Some(fbuild_core::Platform::AtmelMegaAvr)); - } - - /// Validate that ALL megatinycore boards have the required framework-injected - /// defines in extra_flags. PlatformIO's builder script injects these at build - /// time, but fbuild must carry them in the board JSON since we don't run - /// PlatformIO's SCons scripts. - #[test] - fn test_megatinycore_boards_have_required_defines() { - let db = get_board_db(); - let required = &[ - "MEGATINYCORE=", - "MEGATINYCORE_MAJOR=", - "MEGATINYCORE_MINOR=", - "MEGATINYCORE_PATCH=", - "MEGATINYCORE_RELEASED=", - "CORE_ATTACH_ALL", - "TWI_MORS", - "CLOCK_SOURCE=", - ]; - let mut failures = Vec::new(); - for (board_id, value) in db.iter() { - let core = value - .pointer("/build/core") - .and_then(|v| v.as_str()) - .unwrap_or(""); - if core != "megatinycore" { - continue; - } - let flags = value - .pointer("/build/extra_flags") - .and_then(|v| v.as_str()) - .unwrap_or(""); - for &define in required { - if !flags.contains(&format!("-D{define}")) { - failures.push(format!("{board_id}: missing -D{define}")); - } - } - } - assert!( - failures.is_empty(), - "megatinycore boards missing required framework defines:\n{}", - failures.join("\n") - ); - } - - /// Validate that ALL dxcore boards have the required framework-injected - /// defines in extra_flags. - #[test] - fn test_dxcore_boards_have_required_defines() { - let db = get_board_db(); - let required = &[ - "DXCORE=", - "DXCORE_MAJOR=", - "DXCORE_MINOR=", - "DXCORE_PATCH=", - "DXCORE_RELEASED=", - "CORE_ATTACH_ALL", - "TWI_MORS_SINGLE", - "MILLIS_USE_TIMER", - "CLOCK_SOURCE=", - ]; - let mut failures = Vec::new(); - for (board_id, value) in db.iter() { - let core = value - .pointer("/build/core") - .and_then(|v| v.as_str()) - .unwrap_or(""); - if core != "dxcore" { - continue; - } - let flags = value - .pointer("/build/extra_flags") - .and_then(|v| v.as_str()) - .unwrap_or(""); - for &define in required { - if !flags.contains(&format!("-D{define}")) { - failures.push(format!("{board_id}: missing -D{define}")); - } - } - } - assert!( - failures.is_empty(), - "dxcore boards missing required framework defines:\n{}", - failures.join("\n") - ); - } - - #[test] - fn test_uno_debug_tools_has_simavr() { - let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - let tools = config - .debug_tools - .as_ref() - .expect("uno should have debug tools"); - assert!(tools.contains_key("simavr"), "uno should have simavr"); - assert!(tools.contains_key("avr-stub"), "uno should have avr-stub"); - // simavr is not marked as onboard in the board JSON - assert!(!tools["simavr"].onboard); - } - - #[test] - fn test_emulators_filters_hardware_probes() { - let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - let emus = config.emulators(); - // simavr is an emulator, avr-stub is not in EMULATOR_TOOL_NAMES - assert!(emus.contains_key("simavr"), "simavr should be in emulators"); - assert!( - !emus.contains_key("avr-stub"), - "avr-stub is not an emulator" - ); - } - - #[test] - fn test_has_emulator() { - let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - assert!(config.has_emulator("simavr")); - assert!(!config.has_emulator("qemu")); - assert!(!config.has_emulator("avr-stub")); // not in EMULATOR_TOOL_NAMES - } - - #[test] - fn test_debug_tools_round_trip_serde() { - let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); - let json = serde_json::to_string(&config).unwrap(); - let restored: BoardConfig = serde_json::from_str(&json).unwrap(); - assert_eq!(config.debug_tools, restored.debug_tools); - } - - #[test] - fn test_boards_txt_has_no_debug_tools() { - let f = write_boards_txt( - "\ -uno.name=Arduino Uno -uno.build.mcu=atmega328p -uno.build.f_cpu=16000000L -", - ); - let config = BoardConfig::from_boards_txt(f.path(), "uno", &HashMap::new()).unwrap(); - assert!( - config.debug_tools.is_none(), - "boards.txt should not have debug tools" - ); - } - - #[test] - fn test_esp32c6_devkitc1_has_variant() { - // Regression: esp32-c6-devkitc-1.json was missing build.variant, - // causing fbuild to look for variants/esp32-c6-devkitc-1/ instead - // of variants/esp32c6/, which broke compilation (pins_arduino.h - // not found). See https://github.com/FastLED/fbuild/issues/46 - let config = BoardConfig::from_board_id("esp32-c6-devkitc-1", &HashMap::new()).unwrap(); - assert_eq!(config.mcu, "esp32c6"); - assert_eq!(config.core, "esp32"); - assert_eq!( - config.variant, "esp32c6", - "esp32-c6-devkitc-1 must have variant=esp32c6, not the board ID fallback" - ); - } - - #[test] - fn test_esp32c6_alias_has_variant() { - // The 'esp32c6' alias resolves to esp32-c6-devkitm-1 which has - // the variant field. Verify both paths produce correct variant. - let config = BoardConfig::from_board_id("esp32c6", &HashMap::new()).unwrap(); - assert_eq!(config.mcu, "esp32c6"); - assert_eq!( - config.variant, "esp32c6", - "esp32c6 alias must resolve to variant=esp32c6" - ); - } - - #[test] - fn test_board_without_debug_section() { - // Find a board that has no debug section (if any), or verify graceful handling - // by checking that debug_tools is populated from the JSON when present - let config = BoardConfig::from_board_id("esp32dev", &HashMap::new()).unwrap(); - // esp32dev has debug tools (hardware probes only, no emulators) - if let Some(ref tools) = config.debug_tools { - let emus = config.emulators(); - // esp32dev has no software emulators, only hardware probes - assert!( - emus.is_empty(), - "esp32dev should have no emulators, got: {:?}", - emus - ); - // But it should still have hardware debug tools - assert!(!tools.is_empty(), "esp32dev should have debug tools"); - } - } -} diff --git a/crates/fbuild-config/src/board/README.md b/crates/fbuild-config/src/board/README.md new file mode 100644 index 00000000..d4610542 --- /dev/null +++ b/crates/fbuild-config/src/board/README.md @@ -0,0 +1,24 @@ +# board module + +Board configuration loaded from Arduino `boards.txt` files or the embedded +PlatformIO board JSON database. Re-exported from the parent crate as +`fbuild_config::BoardConfig`, `DebugToolMeta`, and `Esp32QemuPsramConfig`. + +## Files + +- `mod.rs` — Module root and public re-exports. +- `types.rs` — `BoardConfig`, `DebugToolMeta`, `Esp32QemuPsramConfig`, and the + internal `EMULATOR_TOOL_NAMES` list. +- `loaders.rs` — `BoardConfig::from_boards_txt`, `BoardConfig::from_board_id`, + and the shared `parse_boards_txt` line parser. +- `methods.rs` — Accessor / derivation methods: `emulators`, `has_emulator`, + `effective_esp32_memory_type`, `qemu_esp32_psram_config`, `platform`, + `get_defines`, `get_include_paths`. +- `db.rs` — Embedded JSON board database (`include_dir!`), board-id alias + resolution, debug-tool extraction, and the flat defaults map consumed by + `from_board_id`. +- `tests.rs` — Unit tests covering loaders, defaults, defines, ESP32 flash / + PSRAM heuristics, and database invariants. + +This split was introduced to satisfy the CI LOC gate (no `.rs` file may +exceed 1000 lines); behaviour and the public API are unchanged. diff --git a/crates/fbuild-config/src/board/db.rs b/crates/fbuild-config/src/board/db.rs new file mode 100644 index 00000000..615e1923 --- /dev/null +++ b/crates/fbuild-config/src/board/db.rs @@ -0,0 +1,235 @@ +//! Embedded board database and default-extraction helpers. +//! +//! Loads PlatformIO registry JSON files baked into the binary via +//! [`include_dir`], resolves common board-id aliases, and projects the +//! relevant fields into a flat `HashMap` of defaults +//! consumed by [`BoardConfig::from_board_id`]. + +use std::collections::HashMap; + +use super::types::DebugToolMeta; + +/// Embedded board database — 1609 boards from PlatformIO registry JSON files. +/// +/// Loaded once on first access via `OnceLock`. Each entry maps board_id → JSON object +/// with fields: id, name, mcu, platform, fcpu, ram, rom, etc. +static BOARD_DB: std::sync::OnceLock> = + std::sync::OnceLock::new(); + +static BOARDS_DIR: include_dir::Dir = + include_dir::include_dir!("$CARGO_MANIFEST_DIR/assets/boards/json"); + +pub(super) fn get_board_db() -> &'static HashMap { + BOARD_DB.get_or_init(|| { + let mut db = HashMap::new(); + for file in BOARDS_DIR.files() { + let Some(stem) = file.path().file_stem().and_then(|s| s.to_str()) else { + continue; + }; + if file.path().extension().and_then(|e| e.to_str()) != Some("json") { + continue; + } + let Some(contents) = file.contents_utf8() else { + continue; + }; + match serde_json::from_str(contents) { + Ok(value) => { + db.insert(stem.to_string(), value); + } + Err(e) => { + tracing::error!("failed to parse board {}: {}", stem, e); + } + } + } + db + }) +} + +/// Convert a board_id like "uno" to a board define like "UNO". +pub(super) fn board_id_to_board_define(board_id: &str) -> String { + board_id.to_uppercase().replace('-', "_") +} + +/// Common board aliases mapping short names to JSON board_ids. +fn resolve_board_alias(board_id: &str) -> &str { + match board_id { + "mega" => "megaatmega2560", + "nano" | "nanoatmega328" => "nanoatmega328", + "pico" | "rpipico" => "rpipico", + "picow" | "rpipicow" => "rpipicow", + "pico2" | "rpipico2" => "rpipico2", + "esp32c3" => "esp32-c3-devkitm-1", + "esp32c6" => "esp32-c6-devkitm-1", + "esp32s3" => "esp32-s3-devkitc-1", + "ch32l103" => "genericCH32L103C8T6", + "ch32v003" => "genericCH32V003F4P6", + "ch32v006" => "genericCH32V006K8U6", + "ch32v103" => "genericCH32V103C8T6", + "ch32v203" => "genericCH32V203C8T6", + "ch32v208" => "genericCH32V208WBU6", + "ch32v303" => "genericCH32V303VCT6", + "ch32v307" => "genericCH32V307VCT6", + "ch32x035" => "genericCH32X035C8T6", + "adafruit_grand_central_m4" => "adafruit_grandcentral_m4", + other => other, + } +} + +/// Extract debug tools from a board JSON entry's `debug.tools` section. +pub(super) fn get_board_debug_tools(board_id: &str) -> Option> { + let db = get_board_db(); + let resolved = resolve_board_alias(board_id); + let entry = db.get(board_id).or_else(|| db.get(resolved))?; + + let tools = entry + .get("debug") + .and_then(|d| d.get("tools")) + .and_then(|t| t.as_object())?; + + if tools.is_empty() { + return None; + } + + let mut result = HashMap::new(); + for (name, meta) in tools { + let onboard = meta + .get("onboard") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let default = meta + .get("default") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + result.insert(name.clone(), DebugToolMeta { onboard, default }); + } + + Some(result) +} + +pub(super) fn get_board_defaults(board_id: &str) -> Option> { + let db = get_board_db(); + let resolved = resolve_board_alias(board_id); + let entry = db.get(board_id).or_else(|| db.get(resolved))?; + + let mut d = HashMap::new(); + + if let Some(name) = entry.get("name").and_then(|v| v.as_str()) { + d.insert("name".into(), name.to_string()); + } + + let mcu = entry + .get("mcu") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_lowercase(); + d.insert("mcu".into(), mcu.clone()); + + if let Some(fcpu) = entry.get("fcpu").and_then(|v| v.as_u64()) { + d.insert("f_cpu".into(), format!("{}L", fcpu)); + } + + if let Some(ram) = entry.get("ram").and_then(|v| v.as_u64()) { + d.insert("maximum_data_size".into(), ram.to_string()); + } + + if let Some(rom) = entry.get("rom").and_then(|v| v.as_u64()) { + d.insert("maximum_size".into(), rom.to_string()); + } + + if let Some(platform) = entry.get("platform").and_then(|v| v.as_str()) { + d.insert("platform_str".into(), platform.to_string()); + } + + // Data-driven: read build section from enriched JSON + if let Some(build) = entry.get("build").and_then(|v| v.as_object()) { + if let Some(core) = build.get("core").and_then(|v| v.as_str()) { + d.insert("core".into(), core.to_string()); + } + if let Some(variant) = build.get("variant").and_then(|v| v.as_str()) { + d.insert("variant".into(), variant.to_string()); + } + if let Some(variant_h) = build.get("variant_h").and_then(|v| v.as_str()) { + d.insert("variant_h".into(), variant_h.to_string()); + } + if let Some(flags) = build.get("extra_flags").and_then(|v| v.as_str()) { + d.insert("extra_flags".into(), flags.to_string()); + } + if let Some(vid) = build.get("vid").and_then(|v| v.as_str()) { + d.insert("vid".into(), vid.to_string()); + } + if let Some(pid) = build.get("pid").and_then(|v| v.as_str()) { + d.insert("pid".into(), pid.to_string()); + } + if let Some(flash_mode) = build.get("flash_mode").and_then(|v| v.as_str()) { + d.insert("flash_mode".into(), flash_mode.to_string()); + } + if let Some(memory_type) = build.get("memory_type").and_then(|v| v.as_str()) { + d.insert("memory_type".into(), memory_type.to_string()); + } + if let Some(psram_type) = build.get("psram_type").and_then(|v| v.as_str()) { + d.insert("psram_type".into(), psram_type.to_string()); + } + if let Some(f_flash) = build.get("f_flash").and_then(|v| v.as_str()) { + d.insert("f_flash".into(), f_flash.to_string()); + } + if let Some(f_image) = build.get("f_image").and_then(|v| v.as_str()) { + d.insert("f_image".into(), f_image.to_string()); + } + // Arduino sub-fields + if let Some(arduino) = build.get("arduino").and_then(|v| v.as_object()) { + if let Some(ldscript) = arduino.get("ldscript").and_then(|v| v.as_str()) { + d.insert("ldscript".into(), ldscript.to_string()); + } + if let Some(partitions) = arduino.get("partitions").and_then(|v| v.as_str()) { + d.insert("partitions".into(), partitions.to_string()); + } + if let Some(memory_type) = arduino.get("memory_type").and_then(|v| v.as_str()) { + d.insert("memory_type".into(), memory_type.to_string()); + } + // Core-specific overrides: build.arduino..variant + // e.g. build.arduino.openwch.variant = "CH32V00x/CH32V003F4" + if let Some(core_name) = build.get("core").and_then(|v| v.as_str()) { + if let Some(core_obj) = arduino.get(core_name).and_then(|v| v.as_object()) { + if let Some(variant) = core_obj.get("variant").and_then(|v| v.as_str()) { + d.insert("variant".into(), variant.to_string()); + } + if let Some(vh) = core_obj.get("variant_h").and_then(|v| v.as_str()) { + d.insert("variant_h".into(), vh.to_string()); + } + } + } + } + } + + // Data-driven: read upload section from enriched JSON + if let Some(upload) = entry.get("upload").and_then(|v| v.as_object()) { + if let Some(protocol) = upload.get("protocol").and_then(|v| v.as_str()) { + d.insert("upload.protocol".into(), protocol.to_string()); + } + if let Some(speed) = upload.get("speed").and_then(|v| v.as_u64()) { + d.insert("upload.speed".into(), speed.to_string()); + } + } + + // Fallback for unenriched boards: derive from platform + if !d.contains_key("core") { + d.insert("core".into(), "arduino".into()); + } + if !d.contains_key("variant") { + // For ESP32 boards, the Arduino framework uses the MCU name as the + // variant directory name (e.g., variants/esp32c6/). Fall back to + // MCU rather than board_id so builds find pins_arduino.h. + let fallback = if d.get("core").is_some_and(|c| c == "esp32") { + d.get("mcu") + .cloned() + .unwrap_or_else(|| board_id.to_string()) + } else { + board_id.to_string() + }; + d.insert("variant".into(), fallback); + } + d.entry("board".into()) + .or_insert_with(|| board_id_to_board_define(board_id)); + + Some(d) +} diff --git a/crates/fbuild-config/src/board/loaders.rs b/crates/fbuild-config/src/board/loaders.rs new file mode 100644 index 00000000..ea2e82c6 --- /dev/null +++ b/crates/fbuild-config/src/board/loaders.rs @@ -0,0 +1,268 @@ +//! Board configuration loaders. +//! +//! Implements [`BoardConfig::from_boards_txt`] (Arduino boards.txt format) and +//! [`BoardConfig::from_board_id`] (built-in JSON database), along with the +//! `boards.txt` line parser they share. + +use std::collections::HashMap; +use std::path::Path; + +use super::db::{board_id_to_board_define, get_board_debug_tools, get_board_defaults}; +use super::types::BoardConfig; + +impl BoardConfig { + /// Load board config from a boards.txt file. + /// + /// Format: `uno.build.mcu=atmega328p`, `uno.name=Arduino Uno` + pub fn from_boards_txt( + path: &Path, + board_id: &str, + overrides: &HashMap, + ) -> fbuild_core::Result { + let content = std::fs::read_to_string(path).map_err(|e| { + fbuild_core::FbuildError::ConfigError(format!( + "failed to read boards.txt at {}: {}", + path.display(), + e + )) + })?; + + let props = parse_boards_txt(&content, board_id); + if props.is_empty() { + return Err(fbuild_core::FbuildError::ConfigError(format!( + "board '{}' not found in {}", + board_id, + path.display() + ))); + } + + let get = |key: &str| -> Option { + overrides + .get(key) + .cloned() + .or_else(|| props.get(key).cloned()) + }; + + let name = get("name").unwrap_or_else(|| board_id.to_string()); + let mcu = get("mcu").ok_or_else(|| { + fbuild_core::FbuildError::ConfigError(format!( + "board '{}' missing required field 'mcu'", + board_id + )) + })?; + + // For ESP32 chips we deliberately drop the boards.txt `flash_mode` + // field and let downstream consumers fall back to the per-MCU + // default ("dio"). See the equivalent comment in `from_board_id` + // for the rationale (ESP32-S3 QIE-bit unreliability + bootloader + // ROM that requires DIO). + let is_esp32_family = mcu.starts_with("esp32"); + + Ok(Self { + name, + mcu, + f_cpu: get("f_cpu").unwrap_or_else(|| "16000000L".to_string()), + board: get("board") + .or_else(|| props.get("board").cloned()) + .unwrap_or_else(|| board_id_to_board_define(board_id)), + core: get("core").unwrap_or_else(|| "arduino".to_string()), + variant: get("variant").unwrap_or_else(|| "standard".to_string()), + variant_h: get("variant_h"), + vid: get("vid"), + pid: get("pid"), + extra_flags: get("extra_flags"), + upload_protocol: get("upload.protocol") + .or_else(|| props.get("upload.protocol").cloned()), + upload_speed: get("upload.speed").or_else(|| props.get("upload.speed").cloned()), + max_flash: get("maximum_size") + .or_else(|| props.get("maximum_size").cloned()) + .and_then(|s| s.parse().ok()), + max_ram: get("maximum_data_size") + .or_else(|| props.get("maximum_data_size").cloned()) + .and_then(|s| s.parse().ok()), + flash_mode: if is_esp32_family { + overrides.get("flash_mode").cloned() + } else { + get("flash_mode") + }, + memory_type: if is_esp32_family { + overrides + .get("memory_type") + .cloned() + .or_else(|| get("memory_type")) + } else { + get("memory_type") + }, + psram_type: if is_esp32_family { + overrides + .get("psram_type") + .cloned() + .or_else(|| get("psram_type")) + } else { + get("psram_type") + }, + f_flash: get("f_flash"), + f_image: get("f_image"), + partitions: get("partitions"), + ldscript: get("ldscript"), + platform_str: get("platform_str"), + debug_tools: None, // boards.txt format does not contain debug metadata + }) + } + + /// Load board config from built-in defaults. + pub fn from_board_id( + board_id: &str, + overrides: &HashMap, + ) -> fbuild_core::Result { + let defaults = get_board_defaults(board_id).ok_or_else(|| { + fbuild_core::FbuildError::ConfigError(format!( + "unknown board '{}' (no built-in defaults)", + board_id + )) + })?; + + let get = |key: &str, default: &str| -> String { + overrides + .get(key) + .cloned() + .unwrap_or_else(|| defaults.get(key).cloned().unwrap_or(default.to_string())) + }; + + // Determine if this is an ESP32-family chip — used to ignore the + // board JSON's `flash_mode` field below. We have to compute this + // here (before constructing Self) because the resolution of + // `flash_mode` happens inside the struct literal. + let resolved_mcu = get("mcu", "unknown"); + let is_esp32_family = resolved_mcu.starts_with("esp32"); + + Ok(Self { + name: get("name", board_id), + mcu: get("mcu", "unknown"), + f_cpu: get("f_cpu", "16000000L"), + board: get("board", &board_id_to_board_define(board_id)), + core: get("core", "arduino"), + variant: get("variant", "standard"), + variant_h: overrides + .get("variant_h") + .cloned() + .or_else(|| defaults.get("variant_h").cloned()), + vid: overrides + .get("vid") + .cloned() + .or_else(|| defaults.get("vid").cloned()), + pid: overrides + .get("pid") + .cloned() + .or_else(|| defaults.get("pid").cloned()), + extra_flags: overrides + .get("extra_flags") + .cloned() + .or_else(|| defaults.get("extra_flags").cloned()), + upload_protocol: overrides + .get("upload.protocol") + .cloned() + .or_else(|| defaults.get("upload.protocol").cloned()), + upload_speed: overrides + .get("upload.speed") + .cloned() + .or_else(|| defaults.get("upload.speed").cloned()), + max_flash: overrides + .get("maximum_size") + .and_then(|s| s.parse().ok()) + .or_else(|| defaults.get("maximum_size").and_then(|s| s.parse().ok())), + max_ram: overrides + .get("maximum_data_size") + .and_then(|s| s.parse().ok()) + .or_else(|| { + defaults + .get("maximum_data_size") + .and_then(|s| s.parse().ok()) + }), + // Flash mode resolution: + // - For ESP32 chips, IGNORE the board JSON's flash_mode field. + // Many board JSONs ship `flash_mode: qio` because the flash + // chip *supports* QIO, but ESP32-S3's QIE-bit init is + // unreliable on real hardware. The MCU-level default + // (`default_flash_mode` in fbuild-build's esp32 configs) is + // "dio" for the entire ESP32 family — that's the safe value + // to use unless the user explicitly opts in via the env + // section's `board_build.flash_mode = qio`. + // - For non-ESP32 chips, the existing behaviour is preserved + // (env override → board JSON → None). + // - Either way, downstream code that needs an effective value + // when this is `None` should fall back to + // `mcu_config.default_flash_mode()`. + flash_mode: if is_esp32_family { + overrides.get("flash_mode").cloned() + } else { + overrides + .get("flash_mode") + .cloned() + .or_else(|| defaults.get("flash_mode").cloned()) + }, + memory_type: overrides + .get("memory_type") + .cloned() + .or_else(|| defaults.get("memory_type").cloned()), + psram_type: overrides + .get("psram_type") + .cloned() + .or_else(|| defaults.get("psram_type").cloned()), + f_flash: overrides + .get("f_flash") + .cloned() + .or_else(|| defaults.get("f_flash").cloned()), + f_image: overrides + .get("f_image") + .cloned() + .or_else(|| defaults.get("f_image").cloned()), + partitions: overrides + .get("partitions") + .cloned() + .or_else(|| defaults.get("partitions").cloned()), + ldscript: overrides + .get("ldscript") + .cloned() + .or_else(|| defaults.get("ldscript").cloned()), + platform_str: defaults.get("platform_str").cloned(), + debug_tools: get_board_debug_tools(board_id), + }) + } +} + +/// Parse boards.txt content for a specific board_id. +/// +/// Format: `board_id.key=value`, with `build.` and `upload.` prefixes. +pub(super) fn parse_boards_txt(content: &str, board_id: &str) -> HashMap { + let mut props = HashMap::new(); + let prefix = format!("{}.", board_id); + + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + if let Some(rest) = trimmed.strip_prefix(&prefix) { + if let Some(eq_pos) = rest.find('=') { + let key = rest[..eq_pos].trim(); + let value = rest[eq_pos + 1..].trim(); + + // Strip build. and upload. prefixes for convenience + let normalized_key = key + .strip_prefix("build.") + .or_else(|| key.strip_prefix("upload.")) + .unwrap_or(key); + + // Keep both the original and stripped key + props.insert(normalized_key.to_string(), value.to_string()); + if normalized_key != key { + props.insert(key.to_string(), value.to_string()); + } + } + } + } + + props +} diff --git a/crates/fbuild-config/src/board/methods.rs b/crates/fbuild-config/src/board/methods.rs new file mode 100644 index 00000000..10e9f749 --- /dev/null +++ b/crates/fbuild-config/src/board/methods.rs @@ -0,0 +1,271 @@ +//! Accessor / derivation methods on [`BoardConfig`]. +//! +//! Includes emulator queries, ESP32 flash/PSRAM heuristics, platform +//! detection, preprocessor defines generation, and include-path resolution. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use super::types::{BoardConfig, DebugToolMeta, Esp32QemuPsramConfig, EMULATOR_TOOL_NAMES}; + +impl BoardConfig { + /// Returns emulator/simulator tools available for this board. + /// + /// Filters `debug_tools` to only include known software emulators + /// (simavr, qemu, renode, ovpsim, verilator), excluding hardware debug probes. + pub fn emulators(&self) -> HashMap<&str, &DebugToolMeta> { + let Some(ref tools) = self.debug_tools else { + return HashMap::new(); + }; + tools + .iter() + .filter(|(name, _)| EMULATOR_TOOL_NAMES.contains(&name.as_str())) + .map(|(name, meta)| (name.as_str(), meta)) + .collect() + } + + /// Check whether this board supports a specific emulator tool. + pub fn has_emulator(&self, tool_name: &str) -> bool { + self.debug_tools + .as_ref() + .is_some_and(|tools| tools.contains_key(tool_name)) + && EMULATOR_TOOL_NAMES.contains(&tool_name) + } + + /// Resolve the effective ESP32 SDK memory profile used for variant headers/libs. + /// + /// This keeps the SDK `sdkconfig.h` and memory-profile libraries aligned + /// with the repo's effective flash-mode policy. Boards that explicitly use + /// OPI flash keep the `opi` flash-half because that represents a distinct + /// bus type rather than an optional fast-read mode. + pub fn effective_esp32_memory_type(&self, default_flash_mode: &str) -> Option { + if !self.mcu.starts_with("esp32") { + return None; + } + + let effective_flash_mode = self + .flash_mode + .as_deref() + .unwrap_or(default_flash_mode) + .to_ascii_lowercase(); + + let (flash_half, psram_half) = if let Some(memory_type) = self.memory_type.as_deref() { + if let Some((flash, psram)) = memory_type.split_once('_') { + ( + Some(flash.to_ascii_lowercase()), + Some(psram.to_ascii_lowercase()), + ) + } else { + (Some(memory_type.to_ascii_lowercase()), None) + } + } else { + (None, None) + }; + + let resolved_flash = match flash_half.as_deref() { + Some("opi") => "opi".to_string(), + _ => effective_flash_mode, + }; + let resolved_psram = psram_half + .or_else(|| self.psram_type.as_deref().map(|s| s.to_ascii_lowercase())) + .unwrap_or_else(|| "qspi".to_string()); + + Some(format!("{}_{}", resolved_flash, resolved_psram)) + } + + pub fn qemu_esp32_psram_config(&self) -> Option { + let has_psram = self + .extra_flags + .as_deref() + .is_some_and(|flags| extra_flags_contain_define(flags, "BOARD_HAS_PSRAM")) + || self.psram_type.is_some(); + if !has_psram { + return None; + } + + let is_octal = self + .psram_type + .as_deref() + .is_some_and(|psram| psram.eq_ignore_ascii_case("opi")) + || self + .memory_type + .as_deref() + .is_some_and(|memory| memory.ends_with("_opi")); + let size_mib = infer_psram_size_mib(&self.name).unwrap_or(if is_octal { 8 } else { 2 }); + + Some(Esp32QemuPsramConfig { size_mib, is_octal }) + } + + /// Detect the platform from the board JSON's platform field, or fall back to MCU heuristic. + pub fn platform(&self) -> Option { + // Prefer explicit platform from board JSON (distinguishes AtmelMegaAvr from AtmelAvr) + if let Some(ref p) = self.platform_str { + if let Some(platform) = fbuild_core::Platform::from_platform_str(p) { + return Some(platform); + } + } + let mcu = self.mcu.to_lowercase(); + if mcu.starts_with("atmega") || mcu.starts_with("attiny") || mcu.starts_with("at90") { + Some(fbuild_core::Platform::AtmelAvr) + } else if mcu.starts_with("esp32") { + Some(fbuild_core::Platform::Espressif32) + } else if mcu.starts_with("esp8266") || mcu.starts_with("esp8285") { + Some(fbuild_core::Platform::Espressif8266) + } else if mcu.starts_with("imxrt") || mcu.starts_with("mk") { + Some(fbuild_core::Platform::Teensy) + } else if mcu.starts_with("rp2040") || mcu.starts_with("rp2350") { + Some(fbuild_core::Platform::RaspberryPi) + } else if mcu.starts_with("stm32") { + Some(fbuild_core::Platform::Ststm32) + } else if mcu.starts_with("nrf52") { + Some(fbuild_core::Platform::NordicNrf52) + } else if mcu.starts_with("at91sam") || mcu.starts_with("sam") { + Some(fbuild_core::Platform::AtmelSam) + } else if mcu.starts_with("ra4") || mcu.starts_with("ra6") { + Some(fbuild_core::Platform::RenesasRa) + } else if mcu.starts_with("ch32") { + Some(fbuild_core::Platform::Ch32v) + } else if mcu.starts_with("apollo3") || mcu.starts_with("ama3b") { + Some(fbuild_core::Platform::Apollo3) + } else { + None + } + } + + /// Generate preprocessor defines for this board. + /// + /// Returns defines like: PLATFORMIO, F_CPU, ARDUINO, `ARDUINO_`, `ARDUINO_ARCH_` + pub fn get_defines(&self) -> HashMap { + let mut defines = HashMap::new(); + + defines.insert("PLATFORMIO".to_string(), "1".to_string()); + defines.insert("F_CPU".to_string(), self.f_cpu.clone()); + + // Default Arduino version. Platform-specific overrides (e.g. Teensy=10819) + // are in MCU config JSON defines, merged by the orchestrator after this. + defines.insert("ARDUINO".to_string(), "10808".to_string()); + + defines.insert( + format!("ARDUINO_{}", self.board.to_uppercase()), + "1".to_string(), + ); + // ARDUINO_BOARD and ARDUINO_VARIANT as quoted string defines. + // Use \" escapes so GCC response files on Windows preserve the quotes + // (bare " is treated as a word delimiter by GCC's response file parser). + defines.insert( + "ARDUINO_BOARD".to_string(), + format!("\\\"{}\\\"", self.board), + ); + defines.insert( + "ARDUINO_VARIANT".to_string(), + format!("\\\"{}\\\"", self.variant), + ); + + // Architecture define + let arch = self.arch_define(); + if !arch.is_empty() { + defines.insert(format!("ARDUINO_ARCH_{}", arch), "1".to_string()); + } + + // MCU-specific define for AVR + let mcu_upper = self.mcu.to_uppercase(); + if mcu_upper.starts_with("ATMEGA") || mcu_upper.starts_with("ATTINY") { + defines.insert(format!("__AVR_{}__", mcu_upper), "1".to_string()); + } + + // Teensy __MCU__ define (MCU detection, not a versioned constant) + if matches!(self.platform(), Some(fbuild_core::Platform::Teensy)) + && mcu_upper.starts_with("IMXRT") + { + defines.insert(format!("__{}__", mcu_upper), "1".to_string()); + } + + // USB VID/PID defines for USB-native boards (Leonardo, Micro, etc.) + if let Some(ref vid) = self.vid { + defines.insert("USB_VID".to_string(), vid.clone()); + } + if let Some(ref pid) = self.pid { + defines.insert("USB_PID".to_string(), pid.clone()); + } + + // Extra flags + if let Some(ref flags) = self.extra_flags { + for flag in fbuild_core::shell_split::split(flags) { + if let Some(define) = flag.strip_prefix("-D") { + if let Some(eq_pos) = define.find('=') { + defines.insert( + define[..eq_pos].to_string(), + define[eq_pos + 1..].to_string(), + ); + } else { + defines.insert(define.to_string(), "1".to_string()); + } + } + } + } + + defines + } + + /// Get include paths relative to a framework root directory. + /// + /// Returns: `[cores/, variants/]` + pub fn get_include_paths(&self, framework_root: &Path) -> Vec { + vec![ + framework_root.join("cores").join(&self.core), + framework_root.join("variants").join(&self.variant), + ] + } + + fn arch_define(&self) -> String { + match self.platform() { + Some(fbuild_core::Platform::AtmelAvr) => "AVR".to_string(), + Some(fbuild_core::Platform::AtmelMegaAvr) => "MEGAAVR".to_string(), + Some(fbuild_core::Platform::Espressif32) => "ESP32".to_string(), + Some(fbuild_core::Platform::Espressif8266) => "ESP8266".to_string(), + Some(fbuild_core::Platform::NordicNrf52) => "NRF52".to_string(), + Some(fbuild_core::Platform::RaspberryPi) => "RP2040".to_string(), + Some(fbuild_core::Platform::RenesasRa) => "RENESAS".to_string(), + Some(fbuild_core::Platform::SiliconLabs) => "SILABS".to_string(), + Some(fbuild_core::Platform::Ststm32) => "STM32".to_string(), + Some(fbuild_core::Platform::AtmelSam) => "SAM".to_string(), + Some(fbuild_core::Platform::Teensy) => "TEENSY".to_string(), + Some(fbuild_core::Platform::Ch32v) => "CH32V".to_string(), + Some(fbuild_core::Platform::Apollo3) => "APOLLO3".to_string(), + Some(fbuild_core::Platform::Wasm) | None => String::new(), + } + } +} + +fn extra_flags_contain_define(extra_flags: &str, define: &str) -> bool { + extra_flags.split_whitespace().any(|flag| { + let Some(raw) = flag.strip_prefix("-D") else { + return false; + }; + raw.split_once('=').map_or(raw, |(name, _)| name) == define + }) +} + +fn infer_psram_size_mib(name: &str) -> Option { + let upper = name.to_ascii_uppercase(); + + for size in [32_u32, 16, 8, 4, 2] { + if upper.contains(&format!("{size} MB PSRAM")) { + return Some(size); + } + } + + for (marker, size) in [ + ("R32", 32_u32), + ("R16", 16), + ("R8", 8), + ("R4", 4), + ("R2", 2), + ] { + if upper.contains(marker) { + return Some(size); + } + } + + None +} diff --git a/crates/fbuild-config/src/board/mod.rs b/crates/fbuild-config/src/board/mod.rs new file mode 100644 index 00000000..e0c40d93 --- /dev/null +++ b/crates/fbuild-config/src/board/mod.rs @@ -0,0 +1,25 @@ +//! Board configuration from boards.txt and built-in defaults. +//! +//! Supports: +//! - Loading from Arduino boards.txt format +//! - Built-in defaults for common boards +//! - Field overrides from platformio.ini board_build.* keys +//! - Preprocessor defines generation +//! - Include path resolution +//! +//! The module is split into: +//! - [`types`] – the public [`BoardConfig`] struct and supporting types +//! - [`loaders`] – `from_boards_txt` / `from_board_id` constructors and the +//! `boards.txt` line parser +//! - [`methods`] – accessor / derivation methods on `BoardConfig` +//! - [`db`] – embedded JSON board database, alias resolution, default extraction + +mod db; +mod loaders; +mod methods; +mod types; + +#[cfg(test)] +mod tests; + +pub use types::{BoardConfig, DebugToolMeta, Esp32QemuPsramConfig}; diff --git a/crates/fbuild-config/src/board/tests.rs b/crates/fbuild-config/src/board/tests.rs new file mode 100644 index 00000000..e61e5a25 --- /dev/null +++ b/crates/fbuild-config/src/board/tests.rs @@ -0,0 +1,657 @@ +//! Unit tests for the board module. + +use std::collections::HashMap; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use tempfile::NamedTempFile; + +use super::db::get_board_db; +use super::loaders::parse_boards_txt; +use super::{BoardConfig, Esp32QemuPsramConfig}; + +fn write_boards_txt(content: &str) -> NamedTempFile { + let mut f = NamedTempFile::new().unwrap(); + f.write_all(content.as_bytes()).unwrap(); + f.flush().unwrap(); + f +} + +#[test] +fn test_from_board_id_uno() { + let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + assert_eq!(config.name, "Arduino Uno"); + assert_eq!(config.mcu, "atmega328p"); + assert_eq!(config.f_cpu, "16000000L"); + assert_eq!(config.board, "UNO"); + assert_eq!(config.core, "arduino"); + assert_eq!(config.variant, "standard"); + assert_eq!(config.max_flash, Some(32256)); + assert_eq!(config.max_ram, Some(2048)); + // extra_flags from enriched JSON + assert_eq!(config.extra_flags, Some("-DARDUINO_AVR_UNO".to_string())); + assert_eq!(config.upload_protocol, Some("arduino".to_string())); + assert_eq!(config.upload_speed, Some("115200".to_string())); +} + +#[test] +fn test_from_board_id_mega() { + let config = BoardConfig::from_board_id("mega", &HashMap::new()).unwrap(); + assert_eq!(config.mcu, "atmega2560"); + assert_eq!(config.variant, "mega"); +} + +#[test] +fn test_from_board_id_megaatmega2560_alias() { + let config = BoardConfig::from_board_id("megaatmega2560", &HashMap::new()).unwrap(); + assert_eq!(config.mcu, "atmega2560"); +} + +#[test] +fn test_ch32v_aliases() { + let cases = [ + ("ch32l103", "ch32l103c8t6", Some("variant_CH32L103C8T6.h")), + ("ch32v003", "ch32v003f4p6", Some("variant_CH32V003F4.h")), + ("ch32v006", "ch32v006k8u6", Some("variant_CH32V006K8.h")), + ("ch32v103", "ch32v103c8t6", Some("variant_CH32V103R8T6.h")), + ("ch32v203", "ch32v203c8t6", Some("variant_CH32V203C8.h")), + ("ch32v208", "ch32v208wbu6", Some("variant_CH32V203C8.h")), + ("ch32v303", "ch32v303vct6", Some("variant_CH32V307VCT6.h")), + ("ch32v307", "ch32v307vct6", Some("variant_CH32V307VCT6.h")), + ("ch32x035", "ch32x035c8t6", Some("variant_CH32X035G8U.h")), + ]; + for (alias, expected_mcu, expected_variant_h) in cases { + let config = BoardConfig::from_board_id(alias, &HashMap::new()).unwrap(); + assert_eq!( + config.mcu, expected_mcu, + "alias '{}' should resolve to mcu '{}'", + alias, expected_mcu + ); + assert_eq!( + config.variant_h.as_deref(), + expected_variant_h, + "alias '{}' should resolve to variant_h '{:?}'", + alias, + expected_variant_h + ); + } +} + +#[test] +fn test_from_board_id_unknown() { + let result = BoardConfig::from_board_id("nonexistent_board", &HashMap::new()); + assert!(result.is_err()); +} + +#[test] +fn test_from_board_id_with_overrides() { + let mut overrides = HashMap::new(); + overrides.insert("f_cpu".to_string(), "8000000L".to_string()); + let config = BoardConfig::from_board_id("uno", &overrides).unwrap(); + assert_eq!(config.f_cpu, "8000000L"); + assert_eq!(config.mcu, "atmega328p"); // not overridden +} + +#[test] +fn test_from_boards_txt_valid() { + let f = write_boards_txt( + "\ +uno.name=Arduino Uno +uno.build.mcu=atmega328p +uno.build.f_cpu=16000000L +uno.build.board=AVR_UNO +uno.build.core=arduino +uno.build.variant=standard +uno.upload.maximum_size=32256 +uno.upload.maximum_data_size=2048 +", + ); + let config = BoardConfig::from_boards_txt(f.path(), "uno", &HashMap::new()).unwrap(); + assert_eq!(config.name, "Arduino Uno"); + assert_eq!(config.mcu, "atmega328p"); + assert_eq!(config.max_flash, Some(32256)); +} + +#[test] +fn test_from_boards_txt_nonexistent_board() { + let f = write_boards_txt("uno.name=Arduino Uno\nuno.build.mcu=atmega328p\n"); + let result = BoardConfig::from_boards_txt(f.path(), "mega", &HashMap::new()); + assert!(result.is_err()); +} + +#[test] +fn test_from_boards_txt_with_overrides() { + let f = write_boards_txt( + "\ +uno.name=Arduino Uno +uno.build.mcu=atmega328p +uno.build.f_cpu=16000000L +uno.build.board=AVR_UNO +uno.build.core=arduino +uno.build.variant=standard +", + ); + let mut overrides = HashMap::new(); + overrides.insert("f_cpu".to_string(), "8000000L".to_string()); + let config = BoardConfig::from_boards_txt(f.path(), "uno", &overrides).unwrap(); + assert_eq!(config.f_cpu, "8000000L"); +} + +#[test] +fn test_from_boards_txt_multiple_boards() { + let f = write_boards_txt( + "\ +uno.name=Arduino Uno +uno.build.mcu=atmega328p +mega.name=Arduino Mega 2560 +mega.build.mcu=atmega2560 +", + ); + let uno = BoardConfig::from_boards_txt(f.path(), "uno", &HashMap::new()).unwrap(); + let mega = BoardConfig::from_boards_txt(f.path(), "mega", &HashMap::new()).unwrap(); + assert_eq!(uno.mcu, "atmega328p"); + assert_eq!(mega.mcu, "atmega2560"); +} + +#[test] +fn test_from_boards_txt_with_upload_info() { + let f = write_boards_txt( + "\ +leonardo.name=Arduino Leonardo +leonardo.build.mcu=atmega32u4 +leonardo.build.vid=0x2341 +leonardo.build.pid=0x8036 +leonardo.upload.protocol=avr109 +leonardo.upload.speed=57600 +", + ); + let config = BoardConfig::from_boards_txt(f.path(), "leonardo", &HashMap::new()).unwrap(); + assert_eq!(config.vid, Some("0x2341".to_string())); + assert_eq!(config.pid, Some("0x8036".to_string())); + assert_eq!(config.upload_protocol, Some("avr109".to_string())); + assert_eq!(config.upload_speed, Some("57600".to_string())); +} + +#[test] +fn test_get_defines_basic() { + let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + let defines = config.get_defines(); + assert_eq!(defines.get("PLATFORMIO"), Some(&"1".to_string())); + assert_eq!(defines.get("F_CPU"), Some(&"16000000L".to_string())); + assert_eq!(defines.get("ARDUINO"), Some(&"10808".to_string())); + assert_eq!(defines.get("ARDUINO_AVR_UNO"), Some(&"1".to_string())); + assert_eq!(defines.get("ARDUINO_ARCH_AVR"), Some(&"1".to_string())); + assert_eq!(defines.get("__AVR_ATMEGA328P__"), Some(&"1".to_string())); +} + +#[test] +fn test_get_defines_with_extra_flags() { + let mut config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + config.extra_flags = Some("-DCUSTOM_FLAG -DVALUE=42".to_string()); + let defines = config.get_defines(); + assert_eq!(defines.get("CUSTOM_FLAG"), Some(&"1".to_string())); + assert_eq!(defines.get("VALUE"), Some(&"42".to_string())); +} + +#[test] +fn test_get_include_paths() { + let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + let paths = config.get_include_paths(Path::new("/framework")); + assert_eq!(paths.len(), 2); + assert_eq!(paths[0], PathBuf::from("/framework/cores/arduino")); + assert_eq!(paths[1], PathBuf::from("/framework/variants/standard")); +} + +#[test] +fn test_platform_detection() { + let avr = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + assert_eq!(avr.platform(), Some(fbuild_core::Platform::AtmelAvr)); + + let esp = BoardConfig::from_board_id("esp32dev", &HashMap::new()).unwrap(); + assert_eq!(esp.platform(), Some(fbuild_core::Platform::Espressif32)); + + let teensy = BoardConfig::from_board_id("teensy41", &HashMap::new()).unwrap(); + assert_eq!(teensy.platform(), Some(fbuild_core::Platform::Teensy)); + + let rpi = BoardConfig::from_board_id("rpipico", &HashMap::new()).unwrap(); + assert_eq!(rpi.platform(), Some(fbuild_core::Platform::RaspberryPi)); +} + +#[test] +fn test_parse_boards_txt_with_comments() { + let props = parse_boards_txt( + "# This is a comment\nuno.name=Arduino Uno\n# Another comment\nuno.build.mcu=atmega328p\n", + "uno", + ); + assert_eq!(props.get("name"), Some(&"Arduino Uno".to_string())); + assert_eq!(props.get("mcu"), Some(&"atmega328p".to_string())); +} + +#[test] +fn test_parse_boards_txt_empty_lines() { + let props = parse_boards_txt( + "\nuno.name=Arduino Uno\n\nuno.build.mcu=atmega328p\n\n", + "uno", + ); + assert_eq!(props.get("name"), Some(&"Arduino Uno".to_string())); +} + +#[test] +fn test_parse_boards_txt_ignores_other_boards() { + let props = parse_boards_txt( + "uno.name=Arduino Uno\nmega.name=Arduino Mega\nuno.build.mcu=atmega328p\n", + "uno", + ); + assert_eq!(props.get("name"), Some(&"Arduino Uno".to_string())); + assert!(!props.values().any(|v| v == "Arduino Mega")); +} + +#[test] +fn test_repr() { + let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + let repr = format!("{:?}", config); + assert!(repr.contains("Arduino Uno")); + assert!(repr.contains("atmega328p")); +} + +// --- Data-driven enriched JSON tests --- + +#[test] +fn test_mega_upload_protocol_wiring() { + // Bug fix: mega needs protocol=wiring, not arduino + let config = BoardConfig::from_board_id("mega", &HashMap::new()).unwrap(); + assert_eq!(config.upload_protocol, Some("wiring".to_string())); + assert_eq!(config.core, "arduino"); + assert_eq!(config.variant, "mega"); +} + +#[test] +fn test_nano_upload_speed_57600() { + // Bug fix: nano uses 57600, not 115200 + let config = BoardConfig::from_board_id("nano", &HashMap::new()).unwrap(); + assert_eq!(config.upload_speed, Some("57600".to_string())); + assert_eq!(config.upload_protocol, Some("arduino".to_string())); + assert_eq!(config.variant, "eightanaloginputs"); +} + +#[test] +fn test_teensy36_core_teensy3() { + // Bug fix: teensy30-36 use core=teensy3, not teensy4 + let config = BoardConfig::from_board_id("teensy36", &HashMap::new()).unwrap(); + assert_eq!(config.core, "teensy3"); + assert_eq!(config.upload_protocol, Some("teensy-gui".to_string())); +} + +#[test] +fn test_teensy41_core_teensy4() { + let config = BoardConfig::from_board_id("teensy41", &HashMap::new()).unwrap(); + assert_eq!(config.core, "teensy4"); + assert_eq!(config.upload_protocol, Some("teensy-gui".to_string())); +} + +#[test] +fn test_esp32dev_enriched_fields() { + let config = BoardConfig::from_board_id("esp32dev", &HashMap::new()).unwrap(); + assert_eq!(config.core, "esp32"); + assert_eq!(config.variant, "esp32"); + // ESP32 boards now intentionally drop the JSON-shipped flash_mode + // (see comments in `from_board_id`). Downstream consumers fall back + // to mcu_config.default_flash_mode() which is "dio". + assert_eq!(config.flash_mode, None); + assert_eq!(config.memory_type, None); + assert_eq!(config.f_flash, Some("40000000L".to_string())); + assert_eq!(config.ldscript, Some("esp32_out.ld".to_string())); + assert_eq!(config.upload_speed, Some("460800".to_string())); +} + +#[test] +fn test_esp32_flash_mode_env_override_honoured() { + // The user can opt back into QIO via `board_build.flash_mode = qio` + // in their [env:X] section, which the daemon translates into a + // `flash_mode` override key. + let mut overrides = HashMap::new(); + overrides.insert("flash_mode".to_string(), "qio".to_string()); + let config = BoardConfig::from_board_id("esp32dev", &overrides).unwrap(); + assert_eq!(config.flash_mode, Some("qio".to_string())); +} + +#[test] +fn test_pico_enriched_fields() { + let config = BoardConfig::from_board_id("rpipico", &HashMap::new()).unwrap(); + assert_eq!(config.core, "earlephilhower"); + assert_eq!(config.variant, "rpipico"); + assert_eq!(config.upload_protocol, Some("picotool".to_string())); +} + +#[test] +fn test_extra_flags_produce_defines() { + // Enriched extra_flags should produce correct ARDUINO_* defines + let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + let defines = config.get_defines(); + assert_eq!(defines.get("ARDUINO_AVR_UNO"), Some(&"1".to_string())); +} + +#[test] +fn test_esp32c3_board_config() { + let config = BoardConfig::from_board_id("esp32c3", &HashMap::new()).unwrap(); + assert_eq!(config.mcu, "esp32c3"); + assert_eq!(config.core, "esp32"); + // ESP32 boards drop the JSON-shipped flash_mode (see + // `test_esp32dev_enriched_fields`); fall back is "dio" from MCU config. + assert_eq!(config.flash_mode, None); + assert_eq!(config.memory_type, None); + assert_eq!(config.ldscript, Some("esp32c3_out.ld".to_string())); + // ESP32-C3 DevKit runs at 160 MHz + assert_eq!(config.f_cpu, "160000000L"); +} + +#[test] +fn test_esp32_effective_memory_type_tracks_effective_flash_mode() { + let config = BoardConfig::from_board_id("esp32c3", &HashMap::new()).unwrap(); + assert_eq!( + config.effective_esp32_memory_type("dio"), + Some("dio_qspi".to_string()) + ); +} + +#[test] +fn test_esp32_effective_memory_type_preserves_opi_flash_profiles() { + let config = + BoardConfig::from_board_id("esp32-s3-devkitc-1-n32r8v", &HashMap::new()).unwrap(); + assert_eq!( + config.effective_esp32_memory_type("dio"), + Some("opi_opi".to_string()) + ); +} + +#[test] +fn test_qemu_psram_config_absent_for_non_psram_board() { + let config = BoardConfig::from_board_id("esp32-s3-devkitc-1", &HashMap::new()).unwrap(); + assert_eq!(config.qemu_esp32_psram_config(), None); +} + +#[test] +fn test_qemu_psram_config_detects_quad_psram_board() { + let config = BoardConfig::from_board_id("esp32-s3-devkitc1-n8r2", &HashMap::new()).unwrap(); + assert_eq!( + config.qemu_esp32_psram_config(), + Some(Esp32QemuPsramConfig { + size_mib: 2, + is_octal: false, + }) + ); +} + +#[test] +fn test_qemu_psram_config_detects_octal_psram_board() { + let config = BoardConfig::from_board_id("esp32-s3-devkitc1-n8r8", &HashMap::new()).unwrap(); + assert_eq!( + config.qemu_esp32_psram_config(), + Some(Esp32QemuPsramConfig { + size_mib: 8, + is_octal: true, + }) + ); +} + +#[test] +fn test_esp32c3_devkitm1_board_config() { + // Direct look up by full board ID (same underlying JSON) + let config = BoardConfig::from_board_id("esp32-c3-devkitm-1", &HashMap::new()).unwrap(); + assert_eq!(config.mcu, "esp32c3"); + let flags = config.extra_flags.unwrap_or_default(); + assert!( + flags.contains("ARDUINO_ESP32C3_DEV"), + "expected ARDUINO_ESP32C3_DEV in extra_flags, got: {flags}" + ); +} + +#[test] +fn test_esp32c3_no_psram() { + // The plain C3 DevKit has no PSRAM + let config = BoardConfig::from_board_id("esp32c3", &HashMap::new()).unwrap(); + let flags = config.extra_flags.clone().unwrap_or_default(); + assert!( + !flags.contains("BOARD_HAS_PSRAM"), + "ESP32-C3 DevKit should not have PSRAM flag, got: {flags}" + ); +} + +// --- USB VID/PID defines --- + +#[test] +fn test_get_defines_usb_vid_pid() { + let mut config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + config.vid = Some("0x2341".to_string()); + config.pid = Some("0x8036".to_string()); + let defines = config.get_defines(); + assert_eq!(defines.get("USB_VID"), Some(&"0x2341".to_string())); + assert_eq!(defines.get("USB_PID"), Some(&"0x8036".to_string())); +} + +#[test] +fn test_get_defines_no_usb_when_absent() { + let config = BoardConfig::from_board_id("esp32dev", &HashMap::new()).unwrap(); + assert!(config.vid.is_none()); + let defines = config.get_defines(); + assert!(!defines.contains_key("USB_VID")); + assert!(!defines.contains_key("USB_PID")); +} + +#[test] +fn test_leonardo_board_has_vid_pid() { + let config = BoardConfig::from_board_id("leonardo", &HashMap::new()).unwrap(); + assert_eq!(config.vid, Some("0x2341".to_string())); + assert_eq!(config.pid, Some("0x8036".to_string())); + let defines = config.get_defines(); + assert_eq!(defines.get("USB_VID"), Some(&"0x2341".to_string())); + assert_eq!(defines.get("USB_PID"), Some(&"0x8036".to_string())); +} + +#[test] +fn test_attiny1604_board_config() { + let config = BoardConfig::from_board_id("ATtiny1604", &HashMap::new()).unwrap(); + assert_eq!(config.mcu, "attiny1604"); + assert_eq!(config.core, "megatinycore"); + assert_eq!(config.variant, "txy4"); + assert_eq!(config.platform(), Some(fbuild_core::Platform::AtmelMegaAvr)); + let defines = config.get_defines(); + assert_eq!(defines.get("ARDUINO_ARCH_MEGAAVR"), Some(&"1".to_string())); +} + +#[test] +fn test_nano_every_board_config() { + let config = BoardConfig::from_board_id("nano_every", &HashMap::new()).unwrap(); + assert_eq!(config.mcu, "atmega4809"); + assert_eq!(config.core, "arduino"); + assert_eq!(config.variant, "nona4809"); + assert_eq!(config.platform(), Some(fbuild_core::Platform::AtmelMegaAvr)); +} + +/// Validate that ALL megatinycore boards have the required framework-injected +/// defines in extra_flags. PlatformIO's builder script injects these at build +/// time, but fbuild must carry them in the board JSON since we don't run +/// PlatformIO's SCons scripts. +#[test] +fn test_megatinycore_boards_have_required_defines() { + let db = get_board_db(); + let required = &[ + "MEGATINYCORE=", + "MEGATINYCORE_MAJOR=", + "MEGATINYCORE_MINOR=", + "MEGATINYCORE_PATCH=", + "MEGATINYCORE_RELEASED=", + "CORE_ATTACH_ALL", + "TWI_MORS", + "CLOCK_SOURCE=", + ]; + let mut failures = Vec::new(); + for (board_id, value) in db.iter() { + let core = value + .pointer("/build/core") + .and_then(|v| v.as_str()) + .unwrap_or(""); + if core != "megatinycore" { + continue; + } + let flags = value + .pointer("/build/extra_flags") + .and_then(|v| v.as_str()) + .unwrap_or(""); + for &define in required { + if !flags.contains(&format!("-D{define}")) { + failures.push(format!("{board_id}: missing -D{define}")); + } + } + } + assert!( + failures.is_empty(), + "megatinycore boards missing required framework defines:\n{}", + failures.join("\n") + ); +} + +/// Validate that ALL dxcore boards have the required framework-injected +/// defines in extra_flags. +#[test] +fn test_dxcore_boards_have_required_defines() { + let db = get_board_db(); + let required = &[ + "DXCORE=", + "DXCORE_MAJOR=", + "DXCORE_MINOR=", + "DXCORE_PATCH=", + "DXCORE_RELEASED=", + "CORE_ATTACH_ALL", + "TWI_MORS_SINGLE", + "MILLIS_USE_TIMER", + "CLOCK_SOURCE=", + ]; + let mut failures = Vec::new(); + for (board_id, value) in db.iter() { + let core = value + .pointer("/build/core") + .and_then(|v| v.as_str()) + .unwrap_or(""); + if core != "dxcore" { + continue; + } + let flags = value + .pointer("/build/extra_flags") + .and_then(|v| v.as_str()) + .unwrap_or(""); + for &define in required { + if !flags.contains(&format!("-D{define}")) { + failures.push(format!("{board_id}: missing -D{define}")); + } + } + } + assert!( + failures.is_empty(), + "dxcore boards missing required framework defines:\n{}", + failures.join("\n") + ); +} + +#[test] +fn test_uno_debug_tools_has_simavr() { + let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + let tools = config + .debug_tools + .as_ref() + .expect("uno should have debug tools"); + assert!(tools.contains_key("simavr"), "uno should have simavr"); + assert!(tools.contains_key("avr-stub"), "uno should have avr-stub"); + // simavr is not marked as onboard in the board JSON + assert!(!tools["simavr"].onboard); +} + +#[test] +fn test_emulators_filters_hardware_probes() { + let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + let emus = config.emulators(); + // simavr is an emulator, avr-stub is not in EMULATOR_TOOL_NAMES + assert!(emus.contains_key("simavr"), "simavr should be in emulators"); + assert!( + !emus.contains_key("avr-stub"), + "avr-stub is not an emulator" + ); +} + +#[test] +fn test_has_emulator() { + let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + assert!(config.has_emulator("simavr")); + assert!(!config.has_emulator("qemu")); + assert!(!config.has_emulator("avr-stub")); // not in EMULATOR_TOOL_NAMES +} + +#[test] +fn test_debug_tools_round_trip_serde() { + let config = BoardConfig::from_board_id("uno", &HashMap::new()).unwrap(); + let json = serde_json::to_string(&config).unwrap(); + let restored: BoardConfig = serde_json::from_str(&json).unwrap(); + assert_eq!(config.debug_tools, restored.debug_tools); +} + +#[test] +fn test_boards_txt_has_no_debug_tools() { + let f = write_boards_txt( + "\ +uno.name=Arduino Uno +uno.build.mcu=atmega328p +uno.build.f_cpu=16000000L +", + ); + let config = BoardConfig::from_boards_txt(f.path(), "uno", &HashMap::new()).unwrap(); + assert!( + config.debug_tools.is_none(), + "boards.txt should not have debug tools" + ); +} + +#[test] +fn test_esp32c6_devkitc1_has_variant() { + // Regression: esp32-c6-devkitc-1.json was missing build.variant, + // causing fbuild to look for variants/esp32-c6-devkitc-1/ instead + // of variants/esp32c6/, which broke compilation (pins_arduino.h + // not found). See https://github.com/FastLED/fbuild/issues/46 + let config = BoardConfig::from_board_id("esp32-c6-devkitc-1", &HashMap::new()).unwrap(); + assert_eq!(config.mcu, "esp32c6"); + assert_eq!(config.core, "esp32"); + assert_eq!( + config.variant, "esp32c6", + "esp32-c6-devkitc-1 must have variant=esp32c6, not the board ID fallback" + ); +} + +#[test] +fn test_esp32c6_alias_has_variant() { + // The 'esp32c6' alias resolves to esp32-c6-devkitm-1 which has + // the variant field. Verify both paths produce correct variant. + let config = BoardConfig::from_board_id("esp32c6", &HashMap::new()).unwrap(); + assert_eq!(config.mcu, "esp32c6"); + assert_eq!( + config.variant, "esp32c6", + "esp32c6 alias must resolve to variant=esp32c6" + ); +} + +#[test] +fn test_board_without_debug_section() { + // Find a board that has no debug section (if any), or verify graceful handling + // by checking that debug_tools is populated from the JSON when present + let config = BoardConfig::from_board_id("esp32dev", &HashMap::new()).unwrap(); + // esp32dev has debug tools (hardware probes only, no emulators) + if let Some(ref tools) = config.debug_tools { + let emus = config.emulators(); + // esp32dev has no software emulators, only hardware probes + assert!( + emus.is_empty(), + "esp32dev should have no emulators, got: {:?}", + emus + ); + // But it should still have hardware debug tools + assert!(!tools.is_empty(), "esp32dev should have debug tools"); + } +} diff --git a/crates/fbuild-config/src/board/types.rs b/crates/fbuild-config/src/board/types.rs new file mode 100644 index 00000000..4652255f --- /dev/null +++ b/crates/fbuild-config/src/board/types.rs @@ -0,0 +1,80 @@ +//! Board configuration data types. +//! +//! Defines the [`BoardConfig`] struct loaded from boards.txt and built-in +//! defaults, along with supporting metadata types ([`DebugToolMeta`], +//! [`Esp32QemuPsramConfig`]) and module-private constants. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Esp32QemuPsramConfig { + pub size_mib: u32, + pub is_octal: bool, +} + +/// Metadata for a single debug tool entry from the board JSON `debug.tools` section. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct DebugToolMeta { + /// Whether the tool is built into the board (no external hardware needed). + #[serde(default)] + pub onboard: bool, + /// Whether this is the board's default debug tool. + #[serde(default)] + pub default: bool, +} + +/// Known emulator/simulator tool names that can run firmware without hardware. +pub(super) const EMULATOR_TOOL_NAMES: &[&str] = + &["simavr", "qemu", "renode", "ovpsim", "verilator"]; + +/// Board configuration loaded from boards.txt or built-in defaults. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BoardConfig { + pub name: String, + pub mcu: String, + pub f_cpu: String, + pub board: String, + pub core: String, + pub variant: String, + /// Variant header override for frameworks that use `#include VARIANT_H` + pub variant_h: Option, + /// USB vendor ID (optional) + pub vid: Option, + /// USB product ID (optional) + pub pid: Option, + /// Extra build flags from board definition + pub extra_flags: Option, + /// Upload protocol (e.g. "arduino", "esptool", "teensy-gui") + pub upload_protocol: Option, + /// Upload speed + pub upload_speed: Option, + /// Maximum flash size in bytes + pub max_flash: Option, + /// Maximum RAM size in bytes + pub max_ram: Option, + /// Flash mode (e.g. "dio", "qio") — ESP32 boards + pub flash_mode: Option, + /// Memory profile (e.g. "qio_qspi", "qio_opi") - ESP32 boards + pub memory_type: Option, + /// PSRAM type (e.g. "qspi", "opi") - ESP32 boards + pub psram_type: Option, + /// Flash frequency (e.g. "80000000L") — ESP32 boards + pub f_flash: Option, + /// Image flash frequency override (e.g. "48000000L") — used by esptool when + /// the board's actual SPI clock (`f_flash`) doesn't match a valid esptool frequency. + /// PlatformIO calls this `build.f_image`. When present, this takes priority over + /// `f_flash` for esptool's `--flash-freq` argument. + pub f_image: Option, + /// Partition table file (e.g. "default_8MB.csv") — ESP32 boards + pub partitions: Option, + /// Linker script (e.g. "esp32s3_out.ld") + pub ldscript: Option, + /// Platform string from board JSON (e.g. "atmelmegaavr", "atmelavr") + pub platform_str: Option, + /// Debug tools from board JSON `debug.tools` section. + /// Maps tool name (e.g. "simavr", "qemu", "renode") to its metadata. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub debug_tools: Option>, +} diff --git a/crates/fbuild-config/src/ini_parser.rs b/crates/fbuild-config/src/ini_parser.rs deleted file mode 100644 index a75c955f..00000000 --- a/crates/fbuild-config/src/ini_parser.rs +++ /dev/null @@ -1,1603 +0,0 @@ -//! PlatformIO INI parser with environment inheritance and variable substitution. -//! -//! Features: -//! - Sections: `[env:name]`, `[platformio]`, custom sections -//! - `extends` directive for environment inheritance -//! - Variable substitution: `${section.key}` or `${env:section.key}` -//! - Multi-line values (continuation lines starting with whitespace) -//! - Inline comments (` ; comment` or ` # comment`) -//! - Base `[env]` section merged into all `[env:*]` sections - -use std::collections::HashMap; -use std::collections::HashSet; -use std::path::{Path, PathBuf}; - -use regex::Regex; - -use crate::pio_env::PioEnvOverrides; - -/// Parsed platformio.ini configuration. -pub struct PlatformIOConfig { - /// Raw sections: section_name -> key -> value - sections: HashMap>, - /// Resolved environment configs (with inheritance applied) - resolved_envs: HashMap>, - /// Path to the platformio.ini file - path: PathBuf, - /// Per-request `PLATFORMIO_*` env var overrides forwarded from the caller. - /// - /// Used by getters to honor env-driven overrides without reading - /// `std::env::var` directly. The daemon does not inherit caller env vars, - /// so all `PLATFORMIO_*` config must flow through this struct. - overrides: PioEnvOverrides, -} - -impl PlatformIOConfig { - /// Parse a platformio.ini file with no env var overrides. - /// - /// Equivalent to `from_path_with_overrides(path, PioEnvOverrides::empty())`. - pub fn from_path(path: &Path) -> fbuild_core::Result { - Self::from_path_with_overrides(path, PioEnvOverrides::empty()) - } - - /// Parse a platformio.ini file and attach `PLATFORMIO_*` env var overrides. - /// - /// The overrides are consulted by getters before falling back to INI values, - /// allowing CLI callers to forward env vars to the daemon over HTTP without - /// the daemon process needing to inherit them. - pub fn from_path_with_overrides( - path: &Path, - overrides: PioEnvOverrides, - ) -> fbuild_core::Result { - let content = std::fs::read_to_string(path).map_err(|e| { - fbuild_core::FbuildError::ConfigError(format!( - "failed to read {}: {}", - path.display(), - e - )) - })?; - - let sections = Self::parse_ini(&content)?; - let resolved_envs = Self::resolve_all_envs(§ions)?; - - Ok(Self { - sections, - resolved_envs, - path: path.to_path_buf(), - overrides, - }) - } - - /// Borrow the env var overrides attached to this config. - pub fn overrides(&self) -> &PioEnvOverrides { - &self.overrides - } - - /// List all environment names (from `[env:name]` sections). - pub fn get_environments(&self) -> Vec<&str> { - let mut envs: Vec<&str> = self.resolved_envs.keys().map(|s| s.as_str()).collect(); - envs.sort(); - envs - } - - /// Check if an environment exists. - pub fn has_environment(&self, env_name: &str) -> bool { - self.resolved_envs.contains_key(env_name) - } - - /// Get resolved config for an environment (with inheritance applied). - pub fn get_env_config(&self, env_name: &str) -> fbuild_core::Result<&HashMap> { - self.resolved_envs.get(env_name).ok_or_else(|| { - fbuild_core::FbuildError::ConfigError(format!("environment '{}' not found", env_name)) - }) - } - - /// Get the default environment name. - /// - /// Priority: - /// 1. forwarded `PLATFORMIO_DEFAULT_ENVS` override (first value) - /// 2. `[platformio]` section `default_envs` key (first value) - /// 3. First environment in file order - /// 4. None if no environments - pub fn get_default_environment(&self) -> Option<&str> { - // Check forwarded PLATFORMIO_DEFAULT_ENVS first. - if let Some(defaults) = self.overrides.get_default_envs() { - let first = defaults.split(',').next().unwrap_or("").trim(); - if !first.is_empty() && self.resolved_envs.contains_key(first) { - return Some( - self.resolved_envs - .keys() - .find(|k| k.as_str() == first) - .map(|k| k.as_str()) - .unwrap(), - ); - } - } - - // Fall back to [platformio].default_envs. - if let Some(pio) = self.sections.get("platformio") { - if let Some(defaults) = pio.get("default_envs") { - let first = defaults.split(',').next().unwrap_or("").trim(); - if !first.is_empty() && self.resolved_envs.contains_key(first) { - return Some( - self.resolved_envs - .keys() - .find(|k| k.as_str() == first) - .map(|k| k.as_str()) - .unwrap(), - ); - } - } - } - // Fall back to first environment - let mut envs: Vec<&str> = self.resolved_envs.keys().map(|s| s.as_str()).collect(); - envs.sort(); - envs.into_iter().next() - } - - /// Get build flags for an environment, parsed into a list. - /// - /// Handles: - /// - Space-separated flags on one line - /// - Multi-line flags (one per line) - /// - `-D FLAG` → `-DFLAG` normalization - pub fn get_build_flags(&self, env_name: &str) -> fbuild_core::Result> { - if let Some(flags) = self.overrides.get_build_flags() { - return Ok(parse_flags(flags)); - } - - let config = self.get_env_config(env_name)?; - match config.get("build_flags") { - Some(flags) => Ok(parse_flags(flags)), - None => Ok(Vec::new()), - } - } - - /// Get build_src_flags for an environment (sketch-only flags). - pub fn get_build_src_flags(&self, env_name: &str) -> fbuild_core::Result> { - if let Some(flags) = self.overrides.get_build_src_flags() { - return Ok(parse_flags(flags)); - } - - let config = self.get_env_config(env_name)?; - match config.get("build_src_flags") { - Some(flags) => Ok(parse_flags(flags)), - None => Ok(Vec::new()), - } - } - - /// Get build_unflags for an environment. - /// - /// Priority: - /// 1. `PLATFORMIO_BUILD_UNFLAGS` forwarded override - /// 2. `build_unflags` - pub fn get_build_unflags(&self, env_name: &str) -> fbuild_core::Result> { - if let Some(flags) = self.overrides.get_build_unflags() { - return Ok(parse_flags(flags)); - } - - let config = self.get_env_config(env_name)?; - match config.get("build_unflags") { - Some(flags) => Ok(parse_flags(flags)), - None => Ok(Vec::new()), - } - } - - /// Get build_type for an environment. - /// - /// PlatformIO defaults this to `release`. - pub fn get_build_type(&self, env_name: &str) -> fbuild_core::Result { - let config = self.get_env_config(env_name)?; - Ok(config - .get("build_type") - .map(|value| strip_inline_comment(value)) - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| "release".to_string())) - } - - /// Get debug_build_flags for an environment. - /// - /// PlatformIO defaults to `-Og -g2 -ggdb2` when the option is not set. - pub fn get_debug_build_flags(&self, env_name: &str) -> fbuild_core::Result> { - let config = self.get_env_config(env_name)?; - match config.get("debug_build_flags") { - Some(flags) => Ok(parse_flags(flags)), - None => Ok(vec![ - "-Og".to_string(), - "-g2".to_string(), - "-ggdb2".to_string(), - ]), - } - } - - /// Get source filter rules for an environment. - /// - /// Priority: - /// 1. `PLATFORMIO_BUILD_SRC_FILTER` forwarded override - /// 2. `build_src_filter` - /// 3. legacy `src_filter` - pub fn get_source_filter(&self, env_name: &str) -> fbuild_core::Result> { - if let Some(value) = self.overrides.get_build_src_filter() { - let cleaned = strip_inline_comment(value); - if !cleaned.is_empty() { - return Ok(Some(cleaned.to_string())); - } - } - - let config = self.get_env_config(env_name)?; - for key in ["build_src_filter", "src_filter"] { - if let Some(value) = config.get(key) { - let cleaned = strip_inline_comment(value); - if !cleaned.is_empty() { - return Ok(Some(cleaned.to_string())); - } - } - } - - Ok(None) - } - - /// Get library dependencies for an environment. - pub fn get_lib_deps(&self, env_name: &str) -> fbuild_core::Result> { - let config = self.get_env_config(env_name)?; - match config.get("lib_deps") { - Some(deps) => Ok(parse_lib_deps(deps)), - None => Ok(Vec::new()), - } - } - - /// Get lib_ignore for an environment (libraries to skip). - pub fn get_lib_ignore(&self, env_name: &str) -> fbuild_core::Result> { - let config = self.get_env_config(env_name)?; - match config.get("lib_ignore") { - Some(deps) => Ok(parse_lib_deps(deps)), - None => Ok(Vec::new()), - } - } - - /// Get `extra_scripts` for an environment. - /// - /// PlatformIO treats entries without an explicit prefix as POST scripts. - /// Values may be provided comma-separated or as a multi-line list. - pub fn get_extra_scripts(&self, env_name: &str) -> fbuild_core::Result> { - let config = self.get_env_config(env_name)?; - match config.get("extra_scripts") { - Some(scripts) => Ok(parse_list_values(scripts)), - None => Ok(Vec::new()), - } - } - - /// Get `board_build.embed_files` for an environment (binary files to embed). - pub fn get_embed_files(&self, env_name: &str) -> fbuild_core::Result> { - let overrides = self.get_board_overrides(env_name)?; - match overrides.get("embed_files") { - Some(files) => Ok(parse_lib_deps(files)), - None => Ok(Vec::new()), - } - } - - /// Get `board_build.embed_txtfiles` for an environment (text files to embed with null terminator). - pub fn get_embed_txtfiles(&self, env_name: &str) -> fbuild_core::Result> { - let overrides = self.get_board_overrides(env_name)?; - match overrides.get("embed_txtfiles") { - Some(files) => Ok(parse_lib_deps(files)), - None => Ok(Vec::new()), - } - } - - /// Get src_dir setting, checking forwarded override and ini config. - pub fn get_src_dir(&self, env_name: &str) -> fbuild_core::Result> { - // Forwarded PLATFORMIO_SRC_DIR takes precedence. - if let Some(env_val) = self.overrides.get_src_dir() { - return Ok(Some(env_val.to_string())); - } - - let config = self.get_env_config(env_name)?; - if let Some(src_dir) = config.get("src_dir") { - // Strip inline comments - let cleaned = strip_inline_comment(src_dir); - if !cleaned.is_empty() { - return Ok(Some(cleaned)); - } - } - - // Check [platformio] section - if let Some(pio) = self.sections.get("platformio") { - if let Some(src_dir) = pio.get("src_dir") { - let cleaned = strip_inline_comment(src_dir); - if !cleaned.is_empty() { - return Ok(Some(cleaned)); - } - } - } - - Ok(None) - } - - /// Get board_build.* and board_upload.* overrides from the environment config. - pub fn get_board_overrides( - &self, - env_name: &str, - ) -> fbuild_core::Result> { - let config = self.get_env_config(env_name)?; - let mut overrides = HashMap::new(); - - for (key, value) in config { - if let Some(stripped) = key.strip_prefix("board_build.") { - overrides.insert(stripped.to_string(), value.clone()); - } else if let Some(stripped) = key.strip_prefix("board_upload.") { - overrides.insert(format!("upload.{}", stripped), value.clone()); - } - } - - Ok(overrides) - } - - /// Get the file path this config was loaded from. - pub fn path(&self) -> &Path { - &self.path - } - - // --- Private implementation --- - - /// Parse raw INI content into sections. - fn parse_ini(content: &str) -> fbuild_core::Result>> { - let mut sections: HashMap> = HashMap::new(); - let mut current_section: Option = None; - let mut current_key: Option = None; - let mut current_value = String::new(); - - for line in content.lines() { - let trimmed = line.trim(); - - // Skip empty lines and pure comment lines - if trimmed.is_empty() || trimmed.starts_with('#') || trimmed.starts_with(';') { - // If we're accumulating a multi-line value and hit a blank line, - // that ends the multi-line value - if trimmed.is_empty() && current_key.is_some() { - if let (Some(ref section), Some(ref key)) = (¤t_section, ¤t_key) { - let section_map = sections.entry(section.clone()).or_default(); - section_map.insert(key.clone(), current_value.trim().to_string()); - current_key = None; - current_value.clear(); - } - } - continue; - } - - // Section header: [section_name] - if trimmed.starts_with('[') && trimmed.ends_with(']') { - // Flush previous key-value - if let (Some(ref section), Some(ref key)) = (¤t_section, ¤t_key) { - let section_map = sections.entry(section.clone()).or_default(); - section_map.insert(key.clone(), current_value.trim().to_string()); - current_key = None; - current_value.clear(); - } - - let name = trimmed[1..trimmed.len() - 1].trim().to_string(); - current_section = Some(name); - continue; - } - - // Continuation line (starts with whitespace) — append to current value - if (line.starts_with(' ') || line.starts_with('\t')) && current_key.is_some() { - let val = strip_inline_comment(trimmed); - if !current_value.is_empty() { - current_value.push('\n'); - } - current_value.push_str(&val); - continue; - } - - // Key = value line - if let Some(eq_pos) = trimmed.find('=') { - // Flush previous key-value - if let (Some(ref section), Some(ref key)) = (¤t_section, ¤t_key) { - let section_map = sections.entry(section.clone()).or_default(); - section_map.insert(key.clone(), current_value.trim().to_string()); - } - - let key = trimmed[..eq_pos].trim().to_string(); - let value = strip_inline_comment(trimmed[eq_pos + 1..].trim()); - - current_key = Some(key); - current_value = value; - } - } - - // Flush last key-value - if let (Some(ref section), Some(ref key)) = (¤t_section, ¤t_key) { - let section_map = sections.entry(section.clone()).or_default(); - section_map.insert(key.clone(), current_value.trim().to_string()); - } - - Ok(sections) - } - - /// Resolve all `[env:*]` sections with inheritance. - fn resolve_all_envs( - sections: &HashMap>, - ) -> fbuild_core::Result>> { - let mut resolved = HashMap::new(); - - // Find all env:* sections - let env_names: Vec = sections - .keys() - .filter_map(|k| k.strip_prefix("env:").map(|s| s.to_string())) - .collect(); - - for env_name in &env_names { - let config = Self::resolve_env(sections, env_name, &mut HashSet::new())?; - // Apply variable substitution - let substituted = Self::substitute_vars(sections, &config); - resolved.insert(env_name.clone(), substituted); - } - - Ok(resolved) - } - - /// Resolve a single environment's config, following `extends` chains. - fn resolve_env( - sections: &HashMap>, - env_name: &str, - visited: &mut HashSet, - ) -> fbuild_core::Result> { - if !visited.insert(env_name.to_string()) { - return Err(fbuild_core::FbuildError::ConfigError(format!( - "circular extends dependency for env:{}", - env_name - ))); - } - - let section_key = format!("env:{}", env_name); - let section = sections.get(§ion_key).ok_or_else(|| { - fbuild_core::FbuildError::ConfigError(format!( - "environment '{}' not found in config", - env_name - )) - })?; - - // Start with base [env] section if it exists - let mut config: HashMap = sections.get("env").cloned().unwrap_or_default(); - - // Apply extends (parent values, then child overrides) - if let Some(extends) = section.get("extends") { - for parent_ref in extends.split(',') { - let parent_ref = parent_ref.trim(); - // extends can reference "env:name" or just "name" (treated as section name) - let parent_name = parent_ref.strip_prefix("env:").unwrap_or(parent_ref); - - if sections.contains_key(&format!("env:{}", parent_name)) { - let parent_config = Self::resolve_env(sections, parent_name, visited)?; - // Parent values go first, child overrides later - for (k, v) in parent_config { - config.insert(k, v); - } - } else if sections.contains_key(parent_name) { - // Direct section reference (non-env section) — resolve its - // extends chain so inherited keys like lib_deps propagate. - let parent_config = Self::resolve_section(sections, parent_name, visited)?; - for (k, v) in parent_config { - config.insert(k, v); - } - } - } - } - - // Apply this environment's values (override parents) - for (k, v) in section { - if k != "extends" { - config.insert(k.clone(), v.clone()); - } - } - - Ok(config) - } - - /// Resolve a non-env section's config, following `extends` chains. - fn resolve_section( - sections: &HashMap>, - section_name: &str, - visited: &mut HashSet, - ) -> fbuild_core::Result> { - if !visited.insert(format!("section:{}", section_name)) { - return Err(fbuild_core::FbuildError::ConfigError(format!( - "circular extends dependency for section '{}'", - section_name - ))); - } - - let section = sections.get(section_name).cloned().unwrap_or_default(); - let mut config = HashMap::new(); - - // Follow extends chain - if let Some(extends) = section.get("extends") { - for parent_ref in extends.split(',') { - let parent_ref = parent_ref.trim(); - if sections.contains_key(&format!("env:{}", parent_ref)) { - let parent_config = Self::resolve_env(sections, parent_ref, visited)?; - for (k, v) in parent_config { - config.insert(k, v); - } - } else if sections.contains_key(parent_ref) { - let parent_config = Self::resolve_section(sections, parent_ref, visited)?; - for (k, v) in parent_config { - config.insert(k, v); - } - } - } - } - - // Apply this section's own values (override parents) - for (k, v) in §ion { - if k != "extends" { - config.insert(k.clone(), v.clone()); - } - } - - Ok(config) - } - - /// Apply `${section.key}` variable substitution. - fn substitute_vars( - sections: &HashMap>, - config: &HashMap, - ) -> HashMap { - let re = Regex::new(r"\$\{([^}]+)\}").unwrap(); - let max_depth = 10; - let mut result = config.clone(); - - for (_key, value) in result.iter_mut() { - let mut current = value.clone(); - for _ in 0..max_depth { - let next = re - .replace_all(¤t, |caps: ®ex::Captures| { - let reference = &caps[1]; - resolve_variable(sections, config, reference) - }) - .to_string(); - - if next == current { - break; - } - current = next; - } - *value = current; - } - - result - } -} - -/// Resolve a variable reference like "section.key" or "env:name.key". -fn resolve_variable( - sections: &HashMap>, - current_config: &HashMap, - reference: &str, -) -> String { - // Try "section.key" format - if let Some(dot_pos) = reference.find('.') { - let section_ref = &reference[..dot_pos]; - let key = &reference[dot_pos + 1..]; - - // Try env:name format - let section_name = if section_ref.starts_with("env:") { - section_ref.to_string() - } else { - // Could be env:name or just a section name - let as_env = format!("env:{}", section_ref); - if sections.contains_key(&as_env) { - as_env - } else { - section_ref.to_string() - } - }; - - // Look up the key, following the `extends` chain if needed - if let Some(val) = resolve_section_key(sections, §ion_name, key, &mut HashSet::new()) { - return val; - } - } - - // Try as a key in the current config - if let Some(val) = current_config.get(reference) { - return val.clone(); - } - - // Unresolved — return as-is - format!("${{{}}}", reference) -} - -/// Look up a key in a section, following `extends` chains. -fn resolve_section_key( - sections: &HashMap>, - section_name: &str, - key: &str, - visited: &mut HashSet, -) -> Option { - if !visited.insert(section_name.to_string()) { - return None; // circular extends - } - - let section = sections.get(section_name)?; - - // Direct lookup - if let Some(val) = section.get(key) { - return Some(val.clone()); - } - - // Follow extends chain - if let Some(extends) = section.get("extends") { - for parent_ref in extends.split(',') { - let parent_ref = parent_ref.trim(); - // Resolve parent section name (could be "env:name" or bare name) - let parent_name = if parent_ref.starts_with("env:") || sections.contains_key(parent_ref) - { - parent_ref.to_string() - } else { - let as_env = format!("env:{}", parent_ref); - if sections.contains_key(&as_env) { - as_env - } else { - parent_ref.to_string() - } - }; - - if let Some(val) = resolve_section_key(sections, &parent_name, key, visited) { - return Some(val); - } - } - } - - None -} - -/// Strip inline comments (` ; comment` or ` # comment`). -/// Be careful not to strip hash/semicolons that are part of values. -fn strip_inline_comment(s: &str) -> String { - // Only strip comments that are preceded by whitespace - // This avoids stripping "#include" or URLs with "#" - let bytes = s.as_bytes(); - for i in 1..bytes.len() { - if bytes[i - 1] == b' ' && (bytes[i] == b';' || bytes[i] == b'#') { - return s[..i].trim().to_string(); - } - } - s.trim().to_string() -} - -/// Parse build flags string into a list. -/// -/// Handles: -/// - Space-separated flags: `-DFOO -DBAR` -/// - Multi-line: one flag per line -/// - `-D FLAG` → `-DFLAG` normalization -/// - Preserves arguments for `-include`, `-I`, `-L`, etc. -fn parse_flags(flags_str: &str) -> Vec { - let mut result = Vec::new(); - - for line in flags_str.lines() { - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - - let chars = trimmed.chars(); - let mut current = String::new(); - let mut in_quotes = false; - let mut quote_char = ' '; - - for c in chars { - match c { - '"' | '\'' if !in_quotes => { - in_quotes = true; - quote_char = c; - current.push(c); - } - c if in_quotes && c == quote_char => { - in_quotes = false; - current.push(c); - } - ' ' | '\t' if !in_quotes => { - if !current.is_empty() { - result.push(current.clone()); - current.clear(); - } - } - _ => { - current.push(c); - } - } - } - - if !current.is_empty() { - result.push(current); - } - } - - // Normalize `-D FLAG` → `-DFLAG` - let mut normalized = Vec::new(); - let mut i = 0; - while i < result.len() { - if result[i] == "-D" && i + 1 < result.len() { - normalized.push(format!("-D{}", result[i + 1])); - i += 2; - } else { - normalized.push(result[i].clone()); - i += 1; - } - } - - normalized -} - -/// Parse library dependencies from a multi-line or comma-separated string. -fn parse_lib_deps(deps_str: &str) -> Vec { - let mut result = Vec::new(); - - for line in deps_str.lines() { - for dep in line.split(',') { - let trimmed = dep.trim(); - if !trimmed.is_empty() { - result.push(trimmed.to_string()); - } - } - } - - result -} - -/// Parse a generic multi-value option from a multi-line or comma-separated string. -fn parse_list_values(value: &str) -> Vec { - let mut result = Vec::new(); - - for line in value.lines() { - for item in line.split(',') { - let trimmed = item.trim(); - if !trimmed.is_empty() { - result.push(trimmed.to_string()); - } - } - } - - result -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::BTreeMap; - use std::io::Write; - use tempfile::NamedTempFile; - - fn write_ini(content: &str) -> NamedTempFile { - let mut f = NamedTempFile::new().unwrap(); - f.write_all(content.as_bytes()).unwrap(); - f.flush().unwrap(); - f - } - - fn overrides(pairs: &[(&str, &str)]) -> crate::pio_env::PioEnvOverrides { - crate::pio_env::PioEnvOverrides::from_map( - pairs - .iter() - .map(|(k, v)| ((*k).to_string(), (*v).to_string())) - .collect::>(), - ) - } - - #[test] - fn test_init_with_valid_file() { - let f = write_ini( - "\ -[env:uno] -platform = atmelavr -board = uno -framework = arduino -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!(config.get_environments(), vec!["uno"]); - } - - #[test] - fn test_init_with_nonexistent_file() { - let result = PlatformIOConfig::from_path(Path::new("/nonexistent/platformio.ini")); - assert!(result.is_err()); - } - - #[test] - fn test_get_environments_multiple() { - let f = write_ini( - "\ -[env:uno] -platform = atmelavr -board = uno -framework = arduino - -[env:esp32] -platform = espressif32 -board = esp32dev -framework = arduino -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let envs = config.get_environments(); - assert_eq!(envs.len(), 2); - assert!(envs.contains(&"uno")); - assert!(envs.contains(&"esp32")); - } - - #[test] - fn test_get_environments_empty() { - let f = write_ini("[platformio]\ndefault_envs = \n"); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert!(config.get_environments().is_empty()); - } - - #[test] - fn test_get_env_config_valid() { - let f = write_ini( - "\ -[env:uno] -platform = atmelavr -board = uno -framework = arduino -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let env = config.get_env_config("uno").unwrap(); - assert_eq!(env.get("platform").unwrap(), "atmelavr"); - assert_eq!(env.get("board").unwrap(), "uno"); - } - - #[test] - fn test_get_env_config_nonexistent() { - let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert!(config.get_env_config("nonexistent").is_err()); - } - - #[test] - fn test_get_env_config_with_base_env_inheritance() { - let f = write_ini( - "\ -[env] -framework = arduino - -[env:uno] -platform = atmelavr -board = uno -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let env = config.get_env_config("uno").unwrap(); - assert_eq!(env.get("framework").unwrap(), "arduino"); - assert_eq!(env.get("platform").unwrap(), "atmelavr"); - } - - #[test] - fn test_get_build_flags_present() { - let f = write_ini( - "\ -[env:uno] -platform = atmelavr -board = uno -framework = arduino -build_flags = -DFOO -DBAR -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let flags = config.get_build_flags("uno").unwrap(); - assert_eq!(flags, vec!["-DFOO", "-DBAR"]); - } - - #[test] - fn test_get_build_flags_absent() { - let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let flags = config.get_build_flags("uno").unwrap(); - assert!(flags.is_empty()); - } - - #[test] - fn test_get_build_flags_prefers_forwarded_override() { - let f = write_ini( - "\ -[env:uno] -platform = atmelavr -board = uno -framework = arduino -build_flags = -DINI=1 -", - ); - let config = PlatformIOConfig::from_path_with_overrides( - f.path(), - overrides(&[("PLATFORMIO_BUILD_FLAGS", "-DOVERRIDE=1 -DFAST=1")]), - ) - .unwrap(); - assert_eq!( - config.get_build_flags("uno").unwrap(), - vec!["-DOVERRIDE=1", "-DFAST=1"] - ); - } - - #[test] - fn test_get_build_flags_multiline() { - let f = write_ini( - "\ -[env:uno] -platform = atmelavr -board = uno -framework = arduino -build_flags = - -DFOO - -DBAR - -DBAZ -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let flags = config.get_build_flags("uno").unwrap(); - assert_eq!(flags, vec!["-DFOO", "-DBAR", "-DBAZ"]); - } - - #[test] - fn test_get_build_flags_d_space_normalization() { - let f = write_ini( - "\ -[env:uno] -platform = atmelavr -board = uno -framework = arduino -build_flags = -D FOO -D BAR -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let flags = config.get_build_flags("uno").unwrap(); - assert_eq!(flags, vec!["-DFOO", "-DBAR"]); - } - - #[test] - fn test_get_build_src_flags_prefers_forwarded_override() { - let f = write_ini( - "\ -[env:uno] -platform = atmelavr -board = uno -framework = arduino -build_src_flags = -DINI_SRC -", - ); - let config = PlatformIOConfig::from_path_with_overrides( - f.path(), - overrides(&[("PLATFORMIO_BUILD_SRC_FLAGS", "-DOVERRIDE_SRC=1")]), - ) - .unwrap(); - assert_eq!( - config.get_build_src_flags("uno").unwrap(), - vec!["-DOVERRIDE_SRC=1"] - ); - } - - #[test] - fn test_get_extra_scripts_multiline_and_comma_separated() { - let f = write_ini( - "\ -[env:uno] -platform = atmelavr -board = uno -framework = arduino -extra_scripts = - pre:scripts/pre.py, scripts/post.py - post:scripts/after.py -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let scripts = config.get_extra_scripts("uno").unwrap(); - assert_eq!( - scripts, - vec![ - "pre:scripts/pre.py", - "scripts/post.py", - "post:scripts/after.py" - ] - ); - } - - #[test] - fn test_get_lib_deps_present() { - let f = write_ini( - "\ -[env:uno] -platform = atmelavr -board = uno -framework = arduino -lib_deps = - FastLED - ArduinoJson -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let deps = config.get_lib_deps("uno").unwrap(); - assert_eq!(deps, vec!["FastLED", "ArduinoJson"]); - } - - #[test] - fn test_get_lib_deps_comma_separated() { - let f = write_ini( - "\ -[env:uno] -platform = atmelavr -board = uno -framework = arduino -lib_deps = FastLED, ArduinoJson -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let deps = config.get_lib_deps("uno").unwrap(); - assert_eq!(deps, vec!["FastLED", "ArduinoJson"]); - } - - #[test] - fn test_get_lib_deps_absent() { - let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let deps = config.get_lib_deps("uno").unwrap(); - assert!(deps.is_empty()); - } - - #[test] - fn test_has_environment() { - let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert!(config.has_environment("uno")); - assert!(!config.has_environment("esp32")); - } - - #[test] - fn test_get_default_environment_explicit() { - let f = write_ini( - "\ -[platformio] -default_envs = esp32 - -[env:uno] -platform = atmelavr -board = uno -framework = arduino - -[env:esp32] -platform = espressif32 -board = esp32dev -framework = arduino -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!(config.get_default_environment(), Some("esp32")); - } - - #[test] - fn test_get_default_environment_prefers_forwarded_override() { - let f = write_ini( - "\ -[platformio] -default_envs = esp32 - -[env:uno] -platform = atmelavr -board = uno -framework = arduino - -[env:esp32] -platform = espressif32 -board = esp32dev -framework = arduino -", - ); - let config = PlatformIOConfig::from_path_with_overrides( - f.path(), - overrides(&[("PLATFORMIO_DEFAULT_ENVS", "uno")]), - ) - .unwrap(); - assert_eq!(config.get_default_environment(), Some("uno")); - } - - #[test] - fn test_get_default_environment_first_fallback() { - let f = write_ini( - "\ -[env:alpha] -platform = atmelavr -board = uno -framework = arduino - -[env:beta] -platform = espressif32 -board = esp32dev -framework = arduino -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - // Should return first alphabetically - assert_eq!(config.get_default_environment(), Some("alpha")); - } - - #[test] - fn test_get_default_environment_none() { - let f = write_ini("[platformio]\n"); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!(config.get_default_environment(), None); - } - - #[test] - fn test_extends_inheritance() { - let f = write_ini( - "\ -[env:base] -platform = atmelavr -framework = arduino -build_flags = -DBASE - -[env:child] -extends = env:base -board = uno -build_flags = -DCHILD -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let env = config.get_env_config("child").unwrap(); - assert_eq!(env.get("platform").unwrap(), "atmelavr"); - assert_eq!(env.get("framework").unwrap(), "arduino"); - assert_eq!(env.get("board").unwrap(), "uno"); - // Child overrides parent's build_flags - assert_eq!(env.get("build_flags").unwrap(), "-DCHILD"); - } - - #[test] - fn test_get_src_dir_from_ini() { - let f = write_ini( - "\ -[platformio] -src_dir = custom_src - -[env:uno] -platform = atmelavr -board = uno -framework = arduino -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!( - config.get_src_dir("uno").unwrap(), - Some("custom_src".to_string()) - ); - } - - #[test] - fn test_get_src_dir_returns_none_when_not_set() { - let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!(config.get_src_dir("uno").unwrap(), None); - } - - #[test] - fn test_get_src_dir_with_inline_comment() { - let f = write_ini( - "\ -[platformio] -src_dir = custom_src ; this is the source dir - -[env:uno] -platform = atmelavr -board = uno -framework = arduino -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!( - config.get_src_dir("uno").unwrap(), - Some("custom_src".to_string()) - ); - } - - #[test] - fn test_get_src_dir_prefers_forwarded_override() { - let f = write_ini( - "\ -[platformio] -src_dir = ini_src - -[env:uno] -platform = atmelavr -board = uno -framework = arduino -", - ); - let config = PlatformIOConfig::from_path_with_overrides( - f.path(), - overrides(&[("PLATFORMIO_SRC_DIR", "override_src")]), - ) - .unwrap(); - assert_eq!( - config.get_src_dir("uno").unwrap(), - Some("override_src".to_string()) - ); - } - - #[test] - fn test_real_world_config() { - let f = write_ini( - "\ -[platformio] -default_envs = esp32dev - -[env] -framework = arduino - -[env:esp32dev] -platform = espressif32 -board = esp32dev -build_flags = - -DFASTLED_ESP32 - -DCORE_DEBUG_LEVEL=0 -lib_deps = - FastLED - ArduinoJson - -[env:uno] -platform = atmelavr -board = uno -build_flags = -DFASTLED_AVR -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - - // Default env - assert_eq!(config.get_default_environment(), Some("esp32dev")); - - // ESP32 config - let esp = config.get_env_config("esp32dev").unwrap(); - assert_eq!(esp.get("platform").unwrap(), "espressif32"); - assert_eq!(esp.get("framework").unwrap(), "arduino"); // inherited from [env] - - let esp_flags = config.get_build_flags("esp32dev").unwrap(); - assert_eq!(esp_flags, vec!["-DFASTLED_ESP32", "-DCORE_DEBUG_LEVEL=0"]); - - let esp_deps = config.get_lib_deps("esp32dev").unwrap(); - assert_eq!(esp_deps, vec!["FastLED", "ArduinoJson"]); - - // Uno config - let uno = config.get_env_config("uno").unwrap(); - assert_eq!(uno.get("platform").unwrap(), "atmelavr"); - assert_eq!(uno.get("framework").unwrap(), "arduino"); // inherited from [env] - - let uno_flags = config.get_build_flags("uno").unwrap(); - assert_eq!(uno_flags, vec!["-DFASTLED_AVR"]); - } - - #[test] - fn test_strip_inline_comment() { - assert_eq!(strip_inline_comment("value ; comment"), "value"); - assert_eq!(strip_inline_comment("value # comment"), "value"); - assert_eq!(strip_inline_comment("value"), "value"); - assert_eq!(strip_inline_comment("#include "), "#include "); - } - - #[test] - fn test_parse_flags() { - assert_eq!(parse_flags("-DFOO -DBAR"), vec!["-DFOO", "-DBAR"]); - assert_eq!(parse_flags("-D FOO -D BAR"), vec!["-DFOO", "-DBAR"]); - assert_eq!( - parse_flags("-DFOO\n-DBAR\n-DBAZ"), - vec!["-DFOO", "-DBAR", "-DBAZ"] - ); - } - - #[test] - fn test_parse_lib_deps() { - assert_eq!( - parse_lib_deps("FastLED, ArduinoJson"), - vec!["FastLED", "ArduinoJson"] - ); - assert_eq!( - parse_lib_deps("FastLED\nArduinoJson"), - vec!["FastLED", "ArduinoJson"] - ); - } - - #[test] - fn test_get_lib_ignore_present() { - let f = write_ini( - "\ -[env:esp32dev] -platform = espressif32 -board = esp32dev -framework = arduino -lib_ignore = - WiFi - FS -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let ignore = config.get_lib_ignore("esp32dev").unwrap(); - assert_eq!(ignore, vec!["WiFi", "FS"]); - } - - #[test] - fn test_get_lib_ignore_absent() { - let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let ignore = config.get_lib_ignore("uno").unwrap(); - assert!(ignore.is_empty()); - } - - #[test] - fn test_variable_substitution_follows_extends_chain() { - // Mirrors NightDriverStrip pattern: [base] defines build_flags, - // [dev_esp32] extends base (no build_flags), [env:demo] references - // ${dev_esp32.build_flags} — must resolve through the extends chain. - let f = write_ini( - "\ -[base] -build_flags = -std=gnu++2a - -Ofast - -[dev_esp32] -extends = base -board = esp32dev - -[env:demo] -extends = dev_esp32 -platform = espressif32 -framework = arduino -build_flags = -DDEMO=1 - ${dev_esp32.build_flags} -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let flags = config.get_build_flags("demo").unwrap(); - assert!( - flags.contains(&"-DDEMO=1".to_string()), - "should contain -DDEMO=1, got: {:?}", - flags - ); - assert!( - flags.contains(&"-std=gnu++2a".to_string()), - "should resolve ${{dev_esp32.build_flags}} through extends chain to [base], got: {:?}", - flags - ); - assert!( - flags.contains(&"-Ofast".to_string()), - "should contain -Ofast from [base], got: {:?}", - flags - ); - // Must NOT contain the literal unresolved variable - let raw = config.get_env_config("demo").unwrap(); - let raw_flags = raw.get("build_flags").unwrap(); - assert!( - !raw_flags.contains("${"), - "build_flags should not contain unresolved variables, got: {}", - raw_flags - ); - } - - #[test] - fn test_non_env_extends_inherits_lib_deps() { - // [env:demo] extends [dev_esp32] extends [base]. - // lib_deps is only on [base] — must propagate through. - let f = write_ini( - "\ -[base] -lib_deps = fastled/FastLED@^3.7.8 - bblanchon/ArduinoJson@^7.0.0 - -[dev_esp32] -extends = base -board = esp32dev - -[env:demo] -extends = dev_esp32 -platform = espressif32 -framework = arduino -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let deps = config.get_lib_deps("demo").unwrap(); - assert!( - deps.iter().any(|d| d.contains("FastLED")), - "should inherit lib_deps from [base] via [dev_esp32], got: {:?}", - deps - ); - assert!( - deps.iter().any(|d| d.contains("ArduinoJson")), - "should inherit ArduinoJson from [base], got: {:?}", - deps - ); - } - - #[test] - fn test_get_embed_files() { - let f = write_ini( - "\ -[env:demo] -platform = espressif32 -board = esp32dev -framework = arduino -board_build.embed_files = site/dist/index.html.gz - site/dist/favicon.ico.gz -board_build.embed_txtfiles = config/timezones.json -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - let embed = config.get_embed_files("demo").unwrap(); - assert_eq!(embed.len(), 2); - assert!(embed.contains(&"site/dist/index.html.gz".to_string())); - assert!(embed.contains(&"site/dist/favicon.ico.gz".to_string())); - - let txt = config.get_embed_txtfiles("demo").unwrap(); - assert_eq!(txt.len(), 1); - assert_eq!(txt[0], "config/timezones.json"); - } - - #[test] - fn test_get_embed_files_empty() { - let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert!(config.get_embed_files("uno").unwrap().is_empty()); - assert!(config.get_embed_txtfiles("uno").unwrap().is_empty()); - } - - #[test] - fn test_get_source_filter_prefers_build_src_filter() { - let f = write_ini( - "\ -[env:demo] -platform = espressif32 -board = esp32dev -framework = arduino -src_filter = + -build_src_filter = - +<*> - - -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!( - config.get_source_filter("demo").unwrap(), - Some("+<*>\n-".to_string()) - ); - } - - #[test] - fn test_get_source_filter_falls_back_to_src_filter() { - let f = write_ini( - "\ -[env:demo] -platform = ststm32 -board = bluepill_f103c8 -framework = stm32cube -src_filter = - + - - -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!( - config.get_source_filter("demo").unwrap(), - Some("+\n-".to_string()) - ); - } - - #[test] - fn test_get_build_unflags_prefers_env_override() { - let f = write_ini( - "\ -[env:demo] -platform = espressif32 -board = esp32dev -framework = arduino -build_unflags = -std=gnu++11 -", - ); - let overrides = crate::pio_env::PioEnvOverrides::from_map( - [( - "PLATFORMIO_BUILD_UNFLAGS".to_string(), - "-std=gnu++17 -DDEBUG".to_string(), - )] - .into_iter() - .collect(), - ); - let config = PlatformIOConfig::from_path_with_overrides(f.path(), overrides).unwrap(); - assert_eq!( - config.get_build_unflags("demo").unwrap(), - vec!["-std=gnu++17", "-DDEBUG"] - ); - } - - #[test] - fn test_get_build_unflags_from_ini() { - let f = write_ini( - "\ -[env:demo] -platform = ststm32 -board = bluepill_f103c8 -framework = stm32cube -build_unflags = - -Os - -DDEBUG -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!( - config.get_build_unflags("demo").unwrap(), - vec!["-Os", "-DDEBUG"] - ); - } - - #[test] - fn test_get_build_type_defaults_to_release() { - let f = write_ini( - "\ -[env:demo] -platform = atmelavr -board = uno -framework = arduino -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!(config.get_build_type("demo").unwrap(), "release"); - } - - #[test] - fn test_get_build_type_reads_debug() { - let f = write_ini( - "\ -[env:demo] -platform = espressif32 -board = esp32dev -framework = arduino -build_type = debug -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!(config.get_build_type("demo").unwrap(), "debug"); - } - - #[test] - fn test_get_debug_build_flags_uses_platformio_defaults() { - let f = write_ini( - "\ -[env:demo] -platform = espressif32 -board = esp32dev -framework = arduino -build_type = debug -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!( - config.get_debug_build_flags("demo").unwrap(), - vec!["-Og", "-g2", "-ggdb2"] - ); - } - - #[test] - fn test_get_debug_build_flags_reads_ini_override() { - let f = write_ini( - "\ -[env:demo] -platform = espressif32 -board = esp32dev -framework = arduino -build_type = debug -debug_build_flags = -Og -g3 -", - ); - let config = PlatformIOConfig::from_path(f.path()).unwrap(); - assert_eq!( - config.get_debug_build_flags("demo").unwrap(), - vec!["-Og", "-g3"] - ); - } -} diff --git a/crates/fbuild-config/src/ini_parser/README.md b/crates/fbuild-config/src/ini_parser/README.md new file mode 100644 index 00000000..71dbde57 --- /dev/null +++ b/crates/fbuild-config/src/ini_parser/README.md @@ -0,0 +1,17 @@ +# `ini_parser` module + +PlatformIO INI parser, split into focused submodules to stay below the per-file +LOC gate. + +## Submodules + +- **`mod.rs`** -- Public API: `PlatformIOConfig` struct, constructors, and + getters (`get_env_config`, `get_build_flags`, `get_lib_deps`, etc.). +- **`parser.rs`** -- Raw INI tokenizer (`parse_ini`), `[env:*]` inheritance + resolution (`resolve_all_envs`, `resolve_env`, `resolve_section`), and + `${section.key}` variable substitution driver (`substitute_vars`). +- **`variables.rs`** -- Variable-reference lookup helpers + (`resolve_variable`, `resolve_section_key`) that follow `extends` chains. +- **`values.rs`** -- Value-string helpers: `strip_inline_comment`, + `parse_flags`, `parse_lib_deps`, `parse_list_values`. +- **`tests.rs`** -- Unit tests for the public API and the parsing helpers. diff --git a/crates/fbuild-config/src/ini_parser/mod.rs b/crates/fbuild-config/src/ini_parser/mod.rs new file mode 100644 index 00000000..70af7f03 --- /dev/null +++ b/crates/fbuild-config/src/ini_parser/mod.rs @@ -0,0 +1,346 @@ +//! PlatformIO INI parser with environment inheritance and variable substitution. +//! +//! Features: +//! - Sections: `[env:name]`, `[platformio]`, custom sections +//! - `extends` directive for environment inheritance +//! - Variable substitution: `${section.key}` or `${env:section.key}` +//! - Multi-line values (continuation lines starting with whitespace) +//! - Inline comments (` ; comment` or ` # comment`) +//! - Base `[env]` section merged into all `[env:*]` sections + +mod parser; +#[cfg(test)] +mod tests; +mod values; +mod variables; + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use crate::pio_env::PioEnvOverrides; + +use self::parser::{parse_ini, resolve_all_envs}; +use self::values::{parse_flags, parse_lib_deps, parse_list_values, strip_inline_comment}; + +/// Parsed platformio.ini configuration. +pub struct PlatformIOConfig { + /// Raw sections: section_name -> key -> value + sections: HashMap>, + /// Resolved environment configs (with inheritance applied) + resolved_envs: HashMap>, + /// Path to the platformio.ini file + path: PathBuf, + /// Per-request `PLATFORMIO_*` env var overrides forwarded from the caller. + /// + /// Used by getters to honor env-driven overrides without reading + /// `std::env::var` directly. The daemon does not inherit caller env vars, + /// so all `PLATFORMIO_*` config must flow through this struct. + overrides: PioEnvOverrides, +} + +impl PlatformIOConfig { + /// Parse a platformio.ini file with no env var overrides. + /// + /// Equivalent to `from_path_with_overrides(path, PioEnvOverrides::empty())`. + pub fn from_path(path: &Path) -> fbuild_core::Result { + Self::from_path_with_overrides(path, PioEnvOverrides::empty()) + } + + /// Parse a platformio.ini file and attach `PLATFORMIO_*` env var overrides. + /// + /// The overrides are consulted by getters before falling back to INI values, + /// allowing CLI callers to forward env vars to the daemon over HTTP without + /// the daemon process needing to inherit them. + pub fn from_path_with_overrides( + path: &Path, + overrides: PioEnvOverrides, + ) -> fbuild_core::Result { + let content = std::fs::read_to_string(path).map_err(|e| { + fbuild_core::FbuildError::ConfigError(format!( + "failed to read {}: {}", + path.display(), + e + )) + })?; + + let sections = parse_ini(&content)?; + let resolved_envs = resolve_all_envs(§ions)?; + + Ok(Self { + sections, + resolved_envs, + path: path.to_path_buf(), + overrides, + }) + } + + /// Borrow the env var overrides attached to this config. + pub fn overrides(&self) -> &PioEnvOverrides { + &self.overrides + } + + /// List all environment names (from `[env:name]` sections). + pub fn get_environments(&self) -> Vec<&str> { + let mut envs: Vec<&str> = self.resolved_envs.keys().map(|s| s.as_str()).collect(); + envs.sort(); + envs + } + + /// Check if an environment exists. + pub fn has_environment(&self, env_name: &str) -> bool { + self.resolved_envs.contains_key(env_name) + } + + /// Get resolved config for an environment (with inheritance applied). + pub fn get_env_config(&self, env_name: &str) -> fbuild_core::Result<&HashMap> { + self.resolved_envs.get(env_name).ok_or_else(|| { + fbuild_core::FbuildError::ConfigError(format!("environment '{}' not found", env_name)) + }) + } + + /// Get the default environment name. + /// + /// Priority: + /// 1. forwarded `PLATFORMIO_DEFAULT_ENVS` override (first value) + /// 2. `[platformio]` section `default_envs` key (first value) + /// 3. First environment in file order + /// 4. None if no environments + pub fn get_default_environment(&self) -> Option<&str> { + // Check forwarded PLATFORMIO_DEFAULT_ENVS first. + if let Some(defaults) = self.overrides.get_default_envs() { + let first = defaults.split(',').next().unwrap_or("").trim(); + if !first.is_empty() && self.resolved_envs.contains_key(first) { + return Some( + self.resolved_envs + .keys() + .find(|k| k.as_str() == first) + .map(|k| k.as_str()) + .unwrap(), + ); + } + } + + // Fall back to [platformio].default_envs. + if let Some(pio) = self.sections.get("platformio") { + if let Some(defaults) = pio.get("default_envs") { + let first = defaults.split(',').next().unwrap_or("").trim(); + if !first.is_empty() && self.resolved_envs.contains_key(first) { + return Some( + self.resolved_envs + .keys() + .find(|k| k.as_str() == first) + .map(|k| k.as_str()) + .unwrap(), + ); + } + } + } + // Fall back to first environment + let mut envs: Vec<&str> = self.resolved_envs.keys().map(|s| s.as_str()).collect(); + envs.sort(); + envs.into_iter().next() + } + + /// Get build flags for an environment, parsed into a list. + /// + /// Handles: + /// - Space-separated flags on one line + /// - Multi-line flags (one per line) + /// - `-D FLAG` → `-DFLAG` normalization + pub fn get_build_flags(&self, env_name: &str) -> fbuild_core::Result> { + if let Some(flags) = self.overrides.get_build_flags() { + return Ok(parse_flags(flags)); + } + + let config = self.get_env_config(env_name)?; + match config.get("build_flags") { + Some(flags) => Ok(parse_flags(flags)), + None => Ok(Vec::new()), + } + } + + /// Get build_src_flags for an environment (sketch-only flags). + pub fn get_build_src_flags(&self, env_name: &str) -> fbuild_core::Result> { + if let Some(flags) = self.overrides.get_build_src_flags() { + return Ok(parse_flags(flags)); + } + + let config = self.get_env_config(env_name)?; + match config.get("build_src_flags") { + Some(flags) => Ok(parse_flags(flags)), + None => Ok(Vec::new()), + } + } + + /// Get build_unflags for an environment. + /// + /// Priority: + /// 1. `PLATFORMIO_BUILD_UNFLAGS` forwarded override + /// 2. `build_unflags` + pub fn get_build_unflags(&self, env_name: &str) -> fbuild_core::Result> { + if let Some(flags) = self.overrides.get_build_unflags() { + return Ok(parse_flags(flags)); + } + + let config = self.get_env_config(env_name)?; + match config.get("build_unflags") { + Some(flags) => Ok(parse_flags(flags)), + None => Ok(Vec::new()), + } + } + + /// Get build_type for an environment. + /// + /// PlatformIO defaults this to `release`. + pub fn get_build_type(&self, env_name: &str) -> fbuild_core::Result { + let config = self.get_env_config(env_name)?; + Ok(config + .get("build_type") + .map(|value| strip_inline_comment(value)) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "release".to_string())) + } + + /// Get debug_build_flags for an environment. + /// + /// PlatformIO defaults to `-Og -g2 -ggdb2` when the option is not set. + pub fn get_debug_build_flags(&self, env_name: &str) -> fbuild_core::Result> { + let config = self.get_env_config(env_name)?; + match config.get("debug_build_flags") { + Some(flags) => Ok(parse_flags(flags)), + None => Ok(vec![ + "-Og".to_string(), + "-g2".to_string(), + "-ggdb2".to_string(), + ]), + } + } + + /// Get source filter rules for an environment. + /// + /// Priority: + /// 1. `PLATFORMIO_BUILD_SRC_FILTER` forwarded override + /// 2. `build_src_filter` + /// 3. legacy `src_filter` + pub fn get_source_filter(&self, env_name: &str) -> fbuild_core::Result> { + if let Some(value) = self.overrides.get_build_src_filter() { + let cleaned = strip_inline_comment(value); + if !cleaned.is_empty() { + return Ok(Some(cleaned.to_string())); + } + } + + let config = self.get_env_config(env_name)?; + for key in ["build_src_filter", "src_filter"] { + if let Some(value) = config.get(key) { + let cleaned = strip_inline_comment(value); + if !cleaned.is_empty() { + return Ok(Some(cleaned.to_string())); + } + } + } + + Ok(None) + } + + /// Get library dependencies for an environment. + pub fn get_lib_deps(&self, env_name: &str) -> fbuild_core::Result> { + let config = self.get_env_config(env_name)?; + match config.get("lib_deps") { + Some(deps) => Ok(parse_lib_deps(deps)), + None => Ok(Vec::new()), + } + } + + /// Get lib_ignore for an environment (libraries to skip). + pub fn get_lib_ignore(&self, env_name: &str) -> fbuild_core::Result> { + let config = self.get_env_config(env_name)?; + match config.get("lib_ignore") { + Some(deps) => Ok(parse_lib_deps(deps)), + None => Ok(Vec::new()), + } + } + + /// Get `extra_scripts` for an environment. + /// + /// PlatformIO treats entries without an explicit prefix as POST scripts. + /// Values may be provided comma-separated or as a multi-line list. + pub fn get_extra_scripts(&self, env_name: &str) -> fbuild_core::Result> { + let config = self.get_env_config(env_name)?; + match config.get("extra_scripts") { + Some(scripts) => Ok(parse_list_values(scripts)), + None => Ok(Vec::new()), + } + } + + /// Get `board_build.embed_files` for an environment (binary files to embed). + pub fn get_embed_files(&self, env_name: &str) -> fbuild_core::Result> { + let overrides = self.get_board_overrides(env_name)?; + match overrides.get("embed_files") { + Some(files) => Ok(parse_lib_deps(files)), + None => Ok(Vec::new()), + } + } + + /// Get `board_build.embed_txtfiles` for an environment (text files to embed with null terminator). + pub fn get_embed_txtfiles(&self, env_name: &str) -> fbuild_core::Result> { + let overrides = self.get_board_overrides(env_name)?; + match overrides.get("embed_txtfiles") { + Some(files) => Ok(parse_lib_deps(files)), + None => Ok(Vec::new()), + } + } + + /// Get src_dir setting, checking forwarded override and ini config. + pub fn get_src_dir(&self, env_name: &str) -> fbuild_core::Result> { + // Forwarded PLATFORMIO_SRC_DIR takes precedence. + if let Some(env_val) = self.overrides.get_src_dir() { + return Ok(Some(env_val.to_string())); + } + + let config = self.get_env_config(env_name)?; + if let Some(src_dir) = config.get("src_dir") { + // Strip inline comments + let cleaned = strip_inline_comment(src_dir); + if !cleaned.is_empty() { + return Ok(Some(cleaned)); + } + } + + // Check [platformio] section + if let Some(pio) = self.sections.get("platformio") { + if let Some(src_dir) = pio.get("src_dir") { + let cleaned = strip_inline_comment(src_dir); + if !cleaned.is_empty() { + return Ok(Some(cleaned)); + } + } + } + + Ok(None) + } + + /// Get board_build.* and board_upload.* overrides from the environment config. + pub fn get_board_overrides( + &self, + env_name: &str, + ) -> fbuild_core::Result> { + let config = self.get_env_config(env_name)?; + let mut overrides = HashMap::new(); + + for (key, value) in config { + if let Some(stripped) = key.strip_prefix("board_build.") { + overrides.insert(stripped.to_string(), value.clone()); + } else if let Some(stripped) = key.strip_prefix("board_upload.") { + overrides.insert(format!("upload.{}", stripped), value.clone()); + } + } + + Ok(overrides) + } + + /// Get the file path this config was loaded from. + pub fn path(&self) -> &Path { + &self.path + } +} diff --git a/crates/fbuild-config/src/ini_parser/parser.rs b/crates/fbuild-config/src/ini_parser/parser.rs new file mode 100644 index 00000000..7680ba4f --- /dev/null +++ b/crates/fbuild-config/src/ini_parser/parser.rs @@ -0,0 +1,240 @@ +//! Raw INI parsing and `[env:*]` inheritance resolution. + +use std::collections::HashMap; +use std::collections::HashSet; + +use regex::Regex; + +use super::values::strip_inline_comment; +use super::variables::resolve_variable; + +/// Parse raw INI content into sections. +pub(super) fn parse_ini( + content: &str, +) -> fbuild_core::Result>> { + let mut sections: HashMap> = HashMap::new(); + let mut current_section: Option = None; + let mut current_key: Option = None; + let mut current_value = String::new(); + + for line in content.lines() { + let trimmed = line.trim(); + + // Skip empty lines and pure comment lines + if trimmed.is_empty() || trimmed.starts_with('#') || trimmed.starts_with(';') { + // If we're accumulating a multi-line value and hit a blank line, + // that ends the multi-line value + if trimmed.is_empty() && current_key.is_some() { + if let (Some(ref section), Some(ref key)) = (¤t_section, ¤t_key) { + let section_map = sections.entry(section.clone()).or_default(); + section_map.insert(key.clone(), current_value.trim().to_string()); + current_key = None; + current_value.clear(); + } + } + continue; + } + + // Section header: [section_name] + if trimmed.starts_with('[') && trimmed.ends_with(']') { + // Flush previous key-value + if let (Some(ref section), Some(ref key)) = (¤t_section, ¤t_key) { + let section_map = sections.entry(section.clone()).or_default(); + section_map.insert(key.clone(), current_value.trim().to_string()); + current_key = None; + current_value.clear(); + } + + let name = trimmed[1..trimmed.len() - 1].trim().to_string(); + current_section = Some(name); + continue; + } + + // Continuation line (starts with whitespace) — append to current value + if (line.starts_with(' ') || line.starts_with('\t')) && current_key.is_some() { + let val = strip_inline_comment(trimmed); + if !current_value.is_empty() { + current_value.push('\n'); + } + current_value.push_str(&val); + continue; + } + + // Key = value line + if let Some(eq_pos) = trimmed.find('=') { + // Flush previous key-value + if let (Some(ref section), Some(ref key)) = (¤t_section, ¤t_key) { + let section_map = sections.entry(section.clone()).or_default(); + section_map.insert(key.clone(), current_value.trim().to_string()); + } + + let key = trimmed[..eq_pos].trim().to_string(); + let value = strip_inline_comment(trimmed[eq_pos + 1..].trim()); + + current_key = Some(key); + current_value = value; + } + } + + // Flush last key-value + if let (Some(ref section), Some(ref key)) = (¤t_section, ¤t_key) { + let section_map = sections.entry(section.clone()).or_default(); + section_map.insert(key.clone(), current_value.trim().to_string()); + } + + Ok(sections) +} + +/// Resolve all `[env:*]` sections with inheritance. +pub(super) fn resolve_all_envs( + sections: &HashMap>, +) -> fbuild_core::Result>> { + let mut resolved = HashMap::new(); + + // Find all env:* sections + let env_names: Vec = sections + .keys() + .filter_map(|k| k.strip_prefix("env:").map(|s| s.to_string())) + .collect(); + + for env_name in &env_names { + let config = resolve_env(sections, env_name, &mut HashSet::new())?; + // Apply variable substitution + let substituted = substitute_vars(sections, &config); + resolved.insert(env_name.clone(), substituted); + } + + Ok(resolved) +} + +/// Resolve a single environment's config, following `extends` chains. +pub(super) fn resolve_env( + sections: &HashMap>, + env_name: &str, + visited: &mut HashSet, +) -> fbuild_core::Result> { + if !visited.insert(env_name.to_string()) { + return Err(fbuild_core::FbuildError::ConfigError(format!( + "circular extends dependency for env:{}", + env_name + ))); + } + + let section_key = format!("env:{}", env_name); + let section = sections.get(§ion_key).ok_or_else(|| { + fbuild_core::FbuildError::ConfigError(format!( + "environment '{}' not found in config", + env_name + )) + })?; + + // Start with base [env] section if it exists + let mut config: HashMap = sections.get("env").cloned().unwrap_or_default(); + + // Apply extends (parent values, then child overrides) + if let Some(extends) = section.get("extends") { + for parent_ref in extends.split(',') { + let parent_ref = parent_ref.trim(); + // extends can reference "env:name" or just "name" (treated as section name) + let parent_name = parent_ref.strip_prefix("env:").unwrap_or(parent_ref); + + if sections.contains_key(&format!("env:{}", parent_name)) { + let parent_config = resolve_env(sections, parent_name, visited)?; + // Parent values go first, child overrides later + for (k, v) in parent_config { + config.insert(k, v); + } + } else if sections.contains_key(parent_name) { + // Direct section reference (non-env section) — resolve its + // extends chain so inherited keys like lib_deps propagate. + let parent_config = resolve_section(sections, parent_name, visited)?; + for (k, v) in parent_config { + config.insert(k, v); + } + } + } + } + + // Apply this environment's values (override parents) + for (k, v) in section { + if k != "extends" { + config.insert(k.clone(), v.clone()); + } + } + + Ok(config) +} + +/// Resolve a non-env section's config, following `extends` chains. +pub(super) fn resolve_section( + sections: &HashMap>, + section_name: &str, + visited: &mut HashSet, +) -> fbuild_core::Result> { + if !visited.insert(format!("section:{}", section_name)) { + return Err(fbuild_core::FbuildError::ConfigError(format!( + "circular extends dependency for section '{}'", + section_name + ))); + } + + let section = sections.get(section_name).cloned().unwrap_or_default(); + let mut config = HashMap::new(); + + // Follow extends chain + if let Some(extends) = section.get("extends") { + for parent_ref in extends.split(',') { + let parent_ref = parent_ref.trim(); + if sections.contains_key(&format!("env:{}", parent_ref)) { + let parent_config = resolve_env(sections, parent_ref, visited)?; + for (k, v) in parent_config { + config.insert(k, v); + } + } else if sections.contains_key(parent_ref) { + let parent_config = resolve_section(sections, parent_ref, visited)?; + for (k, v) in parent_config { + config.insert(k, v); + } + } + } + } + + // Apply this section's own values (override parents) + for (k, v) in §ion { + if k != "extends" { + config.insert(k.clone(), v.clone()); + } + } + + Ok(config) +} + +/// Apply `${section.key}` variable substitution. +pub(super) fn substitute_vars( + sections: &HashMap>, + config: &HashMap, +) -> HashMap { + let re = Regex::new(r"\$\{([^}]+)\}").unwrap(); + let max_depth = 10; + let mut result = config.clone(); + + for (_key, value) in result.iter_mut() { + let mut current = value.clone(); + for _ in 0..max_depth { + let next = re + .replace_all(¤t, |caps: ®ex::Captures| { + let reference = &caps[1]; + resolve_variable(sections, config, reference) + }) + .to_string(); + + if next == current { + break; + } + current = next; + } + *value = current; + } + + result +} diff --git a/crates/fbuild-config/src/ini_parser/tests.rs b/crates/fbuild-config/src/ini_parser/tests.rs new file mode 100644 index 00000000..103b3028 --- /dev/null +++ b/crates/fbuild-config/src/ini_parser/tests.rs @@ -0,0 +1,838 @@ +//! Tests for `ini_parser`. + +use super::values::{parse_flags, parse_lib_deps, strip_inline_comment}; +use super::PlatformIOConfig; +use std::collections::BTreeMap; +use std::io::Write; +use std::path::Path; +use tempfile::NamedTempFile; + +fn write_ini(content: &str) -> NamedTempFile { + let mut f = NamedTempFile::new().unwrap(); + f.write_all(content.as_bytes()).unwrap(); + f.flush().unwrap(); + f +} + +fn overrides(pairs: &[(&str, &str)]) -> crate::pio_env::PioEnvOverrides { + crate::pio_env::PioEnvOverrides::from_map( + pairs + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect::>(), + ) +} + +#[test] +fn test_init_with_valid_file() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!(config.get_environments(), vec!["uno"]); +} + +#[test] +fn test_init_with_nonexistent_file() { + let result = PlatformIOConfig::from_path(Path::new("/nonexistent/platformio.ini")); + assert!(result.is_err()); +} + +#[test] +fn test_get_environments_multiple() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino + +[env:esp32] +platform = espressif32 +board = esp32dev +framework = arduino +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let envs = config.get_environments(); + assert_eq!(envs.len(), 2); + assert!(envs.contains(&"uno")); + assert!(envs.contains(&"esp32")); +} + +#[test] +fn test_get_environments_empty() { + let f = write_ini("[platformio]\ndefault_envs = \n"); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert!(config.get_environments().is_empty()); +} + +#[test] +fn test_get_env_config_valid() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let env = config.get_env_config("uno").unwrap(); + assert_eq!(env.get("platform").unwrap(), "atmelavr"); + assert_eq!(env.get("board").unwrap(), "uno"); +} + +#[test] +fn test_get_env_config_nonexistent() { + let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert!(config.get_env_config("nonexistent").is_err()); +} + +#[test] +fn test_get_env_config_with_base_env_inheritance() { + let f = write_ini( + "\ +[env] +framework = arduino + +[env:uno] +platform = atmelavr +board = uno +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let env = config.get_env_config("uno").unwrap(); + assert_eq!(env.get("framework").unwrap(), "arduino"); + assert_eq!(env.get("platform").unwrap(), "atmelavr"); +} + +#[test] +fn test_get_build_flags_present() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino +build_flags = -DFOO -DBAR +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let flags = config.get_build_flags("uno").unwrap(); + assert_eq!(flags, vec!["-DFOO", "-DBAR"]); +} + +#[test] +fn test_get_build_flags_absent() { + let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let flags = config.get_build_flags("uno").unwrap(); + assert!(flags.is_empty()); +} + +#[test] +fn test_get_build_flags_prefers_forwarded_override() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino +build_flags = -DINI=1 +", + ); + let config = PlatformIOConfig::from_path_with_overrides( + f.path(), + overrides(&[("PLATFORMIO_BUILD_FLAGS", "-DOVERRIDE=1 -DFAST=1")]), + ) + .unwrap(); + assert_eq!( + config.get_build_flags("uno").unwrap(), + vec!["-DOVERRIDE=1", "-DFAST=1"] + ); +} + +#[test] +fn test_get_build_flags_multiline() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino +build_flags = + -DFOO + -DBAR + -DBAZ +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let flags = config.get_build_flags("uno").unwrap(); + assert_eq!(flags, vec!["-DFOO", "-DBAR", "-DBAZ"]); +} + +#[test] +fn test_get_build_flags_d_space_normalization() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino +build_flags = -D FOO -D BAR +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let flags = config.get_build_flags("uno").unwrap(); + assert_eq!(flags, vec!["-DFOO", "-DBAR"]); +} + +#[test] +fn test_get_build_src_flags_prefers_forwarded_override() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino +build_src_flags = -DINI_SRC +", + ); + let config = PlatformIOConfig::from_path_with_overrides( + f.path(), + overrides(&[("PLATFORMIO_BUILD_SRC_FLAGS", "-DOVERRIDE_SRC=1")]), + ) + .unwrap(); + assert_eq!( + config.get_build_src_flags("uno").unwrap(), + vec!["-DOVERRIDE_SRC=1"] + ); +} + +#[test] +fn test_get_extra_scripts_multiline_and_comma_separated() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino +extra_scripts = + pre:scripts/pre.py, scripts/post.py + post:scripts/after.py +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let scripts = config.get_extra_scripts("uno").unwrap(); + assert_eq!( + scripts, + vec![ + "pre:scripts/pre.py", + "scripts/post.py", + "post:scripts/after.py" + ] + ); +} + +#[test] +fn test_get_lib_deps_present() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino +lib_deps = + FastLED + ArduinoJson +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let deps = config.get_lib_deps("uno").unwrap(); + assert_eq!(deps, vec!["FastLED", "ArduinoJson"]); +} + +#[test] +fn test_get_lib_deps_comma_separated() { + let f = write_ini( + "\ +[env:uno] +platform = atmelavr +board = uno +framework = arduino +lib_deps = FastLED, ArduinoJson +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let deps = config.get_lib_deps("uno").unwrap(); + assert_eq!(deps, vec!["FastLED", "ArduinoJson"]); +} + +#[test] +fn test_get_lib_deps_absent() { + let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let deps = config.get_lib_deps("uno").unwrap(); + assert!(deps.is_empty()); +} + +#[test] +fn test_has_environment() { + let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert!(config.has_environment("uno")); + assert!(!config.has_environment("esp32")); +} + +#[test] +fn test_get_default_environment_explicit() { + let f = write_ini( + "\ +[platformio] +default_envs = esp32 + +[env:uno] +platform = atmelavr +board = uno +framework = arduino + +[env:esp32] +platform = espressif32 +board = esp32dev +framework = arduino +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!(config.get_default_environment(), Some("esp32")); +} + +#[test] +fn test_get_default_environment_prefers_forwarded_override() { + let f = write_ini( + "\ +[platformio] +default_envs = esp32 + +[env:uno] +platform = atmelavr +board = uno +framework = arduino + +[env:esp32] +platform = espressif32 +board = esp32dev +framework = arduino +", + ); + let config = PlatformIOConfig::from_path_with_overrides( + f.path(), + overrides(&[("PLATFORMIO_DEFAULT_ENVS", "uno")]), + ) + .unwrap(); + assert_eq!(config.get_default_environment(), Some("uno")); +} + +#[test] +fn test_get_default_environment_first_fallback() { + let f = write_ini( + "\ +[env:alpha] +platform = atmelavr +board = uno +framework = arduino + +[env:beta] +platform = espressif32 +board = esp32dev +framework = arduino +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + // Should return first alphabetically + assert_eq!(config.get_default_environment(), Some("alpha")); +} + +#[test] +fn test_get_default_environment_none() { + let f = write_ini("[platformio]\n"); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!(config.get_default_environment(), None); +} + +#[test] +fn test_extends_inheritance() { + let f = write_ini( + "\ +[env:base] +platform = atmelavr +framework = arduino +build_flags = -DBASE + +[env:child] +extends = env:base +board = uno +build_flags = -DCHILD +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let env = config.get_env_config("child").unwrap(); + assert_eq!(env.get("platform").unwrap(), "atmelavr"); + assert_eq!(env.get("framework").unwrap(), "arduino"); + assert_eq!(env.get("board").unwrap(), "uno"); + // Child overrides parent's build_flags + assert_eq!(env.get("build_flags").unwrap(), "-DCHILD"); +} + +#[test] +fn test_get_src_dir_from_ini() { + let f = write_ini( + "\ +[platformio] +src_dir = custom_src + +[env:uno] +platform = atmelavr +board = uno +framework = arduino +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!( + config.get_src_dir("uno").unwrap(), + Some("custom_src".to_string()) + ); +} + +#[test] +fn test_get_src_dir_returns_none_when_not_set() { + let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!(config.get_src_dir("uno").unwrap(), None); +} + +#[test] +fn test_get_src_dir_with_inline_comment() { + let f = write_ini( + "\ +[platformio] +src_dir = custom_src ; this is the source dir + +[env:uno] +platform = atmelavr +board = uno +framework = arduino +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!( + config.get_src_dir("uno").unwrap(), + Some("custom_src".to_string()) + ); +} + +#[test] +fn test_get_src_dir_prefers_forwarded_override() { + let f = write_ini( + "\ +[platformio] +src_dir = ini_src + +[env:uno] +platform = atmelavr +board = uno +framework = arduino +", + ); + let config = PlatformIOConfig::from_path_with_overrides( + f.path(), + overrides(&[("PLATFORMIO_SRC_DIR", "override_src")]), + ) + .unwrap(); + assert_eq!( + config.get_src_dir("uno").unwrap(), + Some("override_src".to_string()) + ); +} + +#[test] +fn test_real_world_config() { + let f = write_ini( + "\ +[platformio] +default_envs = esp32dev + +[env] +framework = arduino + +[env:esp32dev] +platform = espressif32 +board = esp32dev +build_flags = + -DFASTLED_ESP32 + -DCORE_DEBUG_LEVEL=0 +lib_deps = + FastLED + ArduinoJson + +[env:uno] +platform = atmelavr +board = uno +build_flags = -DFASTLED_AVR +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + + // Default env + assert_eq!(config.get_default_environment(), Some("esp32dev")); + + // ESP32 config + let esp = config.get_env_config("esp32dev").unwrap(); + assert_eq!(esp.get("platform").unwrap(), "espressif32"); + assert_eq!(esp.get("framework").unwrap(), "arduino"); // inherited from [env] + + let esp_flags = config.get_build_flags("esp32dev").unwrap(); + assert_eq!(esp_flags, vec!["-DFASTLED_ESP32", "-DCORE_DEBUG_LEVEL=0"]); + + let esp_deps = config.get_lib_deps("esp32dev").unwrap(); + assert_eq!(esp_deps, vec!["FastLED", "ArduinoJson"]); + + // Uno config + let uno = config.get_env_config("uno").unwrap(); + assert_eq!(uno.get("platform").unwrap(), "atmelavr"); + assert_eq!(uno.get("framework").unwrap(), "arduino"); // inherited from [env] + + let uno_flags = config.get_build_flags("uno").unwrap(); + assert_eq!(uno_flags, vec!["-DFASTLED_AVR"]); +} + +#[test] +fn test_strip_inline_comment() { + assert_eq!(strip_inline_comment("value ; comment"), "value"); + assert_eq!(strip_inline_comment("value # comment"), "value"); + assert_eq!(strip_inline_comment("value"), "value"); + assert_eq!(strip_inline_comment("#include "), "#include "); +} + +#[test] +fn test_parse_flags() { + assert_eq!(parse_flags("-DFOO -DBAR"), vec!["-DFOO", "-DBAR"]); + assert_eq!(parse_flags("-D FOO -D BAR"), vec!["-DFOO", "-DBAR"]); + assert_eq!( + parse_flags("-DFOO\n-DBAR\n-DBAZ"), + vec!["-DFOO", "-DBAR", "-DBAZ"] + ); +} + +#[test] +fn test_parse_lib_deps() { + assert_eq!( + parse_lib_deps("FastLED, ArduinoJson"), + vec!["FastLED", "ArduinoJson"] + ); + assert_eq!( + parse_lib_deps("FastLED\nArduinoJson"), + vec!["FastLED", "ArduinoJson"] + ); +} + +#[test] +fn test_get_lib_ignore_present() { + let f = write_ini( + "\ +[env:esp32dev] +platform = espressif32 +board = esp32dev +framework = arduino +lib_ignore = + WiFi + FS +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let ignore = config.get_lib_ignore("esp32dev").unwrap(); + assert_eq!(ignore, vec!["WiFi", "FS"]); +} + +#[test] +fn test_get_lib_ignore_absent() { + let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let ignore = config.get_lib_ignore("uno").unwrap(); + assert!(ignore.is_empty()); +} + +#[test] +fn test_variable_substitution_follows_extends_chain() { + // Mirrors NightDriverStrip pattern: [base] defines build_flags, + // [dev_esp32] extends base (no build_flags), [env:demo] references + // ${dev_esp32.build_flags} — must resolve through the extends chain. + let f = write_ini( + "\ +[base] +build_flags = -std=gnu++2a + -Ofast + +[dev_esp32] +extends = base +board = esp32dev + +[env:demo] +extends = dev_esp32 +platform = espressif32 +framework = arduino +build_flags = -DDEMO=1 + ${dev_esp32.build_flags} +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let flags = config.get_build_flags("demo").unwrap(); + assert!( + flags.contains(&"-DDEMO=1".to_string()), + "should contain -DDEMO=1, got: {:?}", + flags + ); + assert!( + flags.contains(&"-std=gnu++2a".to_string()), + "should resolve ${{dev_esp32.build_flags}} through extends chain to [base], got: {:?}", + flags + ); + assert!( + flags.contains(&"-Ofast".to_string()), + "should contain -Ofast from [base], got: {:?}", + flags + ); + // Must NOT contain the literal unresolved variable + let raw = config.get_env_config("demo").unwrap(); + let raw_flags = raw.get("build_flags").unwrap(); + assert!( + !raw_flags.contains("${"), + "build_flags should not contain unresolved variables, got: {}", + raw_flags + ); +} + +#[test] +fn test_non_env_extends_inherits_lib_deps() { + // [env:demo] extends [dev_esp32] extends [base]. + // lib_deps is only on [base] — must propagate through. + let f = write_ini( + "\ +[base] +lib_deps = fastled/FastLED@^3.7.8 + bblanchon/ArduinoJson@^7.0.0 + +[dev_esp32] +extends = base +board = esp32dev + +[env:demo] +extends = dev_esp32 +platform = espressif32 +framework = arduino +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let deps = config.get_lib_deps("demo").unwrap(); + assert!( + deps.iter().any(|d| d.contains("FastLED")), + "should inherit lib_deps from [base] via [dev_esp32], got: {:?}", + deps + ); + assert!( + deps.iter().any(|d| d.contains("ArduinoJson")), + "should inherit ArduinoJson from [base], got: {:?}", + deps + ); +} + +#[test] +fn test_get_embed_files() { + let f = write_ini( + "\ +[env:demo] +platform = espressif32 +board = esp32dev +framework = arduino +board_build.embed_files = site/dist/index.html.gz + site/dist/favicon.ico.gz +board_build.embed_txtfiles = config/timezones.json +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + let embed = config.get_embed_files("demo").unwrap(); + assert_eq!(embed.len(), 2); + assert!(embed.contains(&"site/dist/index.html.gz".to_string())); + assert!(embed.contains(&"site/dist/favicon.ico.gz".to_string())); + + let txt = config.get_embed_txtfiles("demo").unwrap(); + assert_eq!(txt.len(), 1); + assert_eq!(txt[0], "config/timezones.json"); +} + +#[test] +fn test_get_embed_files_empty() { + let f = write_ini("[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n"); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert!(config.get_embed_files("uno").unwrap().is_empty()); + assert!(config.get_embed_txtfiles("uno").unwrap().is_empty()); +} + +#[test] +fn test_get_source_filter_prefers_build_src_filter() { + let f = write_ini( + "\ +[env:demo] +platform = espressif32 +board = esp32dev +framework = arduino +src_filter = + +build_src_filter = + +<*> + - +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!( + config.get_source_filter("demo").unwrap(), + Some("+<*>\n-".to_string()) + ); +} + +#[test] +fn test_get_source_filter_falls_back_to_src_filter() { + let f = write_ini( + "\ +[env:demo] +platform = ststm32 +board = bluepill_f103c8 +framework = stm32cube +src_filter = + + + - +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!( + config.get_source_filter("demo").unwrap(), + Some("+\n-".to_string()) + ); +} + +#[test] +fn test_get_build_unflags_prefers_env_override() { + let f = write_ini( + "\ +[env:demo] +platform = espressif32 +board = esp32dev +framework = arduino +build_unflags = -std=gnu++11 +", + ); + let overrides = crate::pio_env::PioEnvOverrides::from_map( + [( + "PLATFORMIO_BUILD_UNFLAGS".to_string(), + "-std=gnu++17 -DDEBUG".to_string(), + )] + .into_iter() + .collect(), + ); + let config = PlatformIOConfig::from_path_with_overrides(f.path(), overrides).unwrap(); + assert_eq!( + config.get_build_unflags("demo").unwrap(), + vec!["-std=gnu++17", "-DDEBUG"] + ); +} + +#[test] +fn test_get_build_unflags_from_ini() { + let f = write_ini( + "\ +[env:demo] +platform = ststm32 +board = bluepill_f103c8 +framework = stm32cube +build_unflags = + -Os + -DDEBUG +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!( + config.get_build_unflags("demo").unwrap(), + vec!["-Os", "-DDEBUG"] + ); +} + +#[test] +fn test_get_build_type_defaults_to_release() { + let f = write_ini( + "\ +[env:demo] +platform = atmelavr +board = uno +framework = arduino +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!(config.get_build_type("demo").unwrap(), "release"); +} + +#[test] +fn test_get_build_type_reads_debug() { + let f = write_ini( + "\ +[env:demo] +platform = espressif32 +board = esp32dev +framework = arduino +build_type = debug +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!(config.get_build_type("demo").unwrap(), "debug"); +} + +#[test] +fn test_get_debug_build_flags_uses_platformio_defaults() { + let f = write_ini( + "\ +[env:demo] +platform = espressif32 +board = esp32dev +framework = arduino +build_type = debug +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!( + config.get_debug_build_flags("demo").unwrap(), + vec!["-Og", "-g2", "-ggdb2"] + ); +} + +#[test] +fn test_get_debug_build_flags_reads_ini_override() { + let f = write_ini( + "\ +[env:demo] +platform = espressif32 +board = esp32dev +framework = arduino +build_type = debug +debug_build_flags = -Og -g3 +", + ); + let config = PlatformIOConfig::from_path(f.path()).unwrap(); + assert_eq!( + config.get_debug_build_flags("demo").unwrap(), + vec!["-Og", "-g3"] + ); +} diff --git a/crates/fbuild-config/src/ini_parser/values.rs b/crates/fbuild-config/src/ini_parser/values.rs new file mode 100644 index 00000000..7e101fc5 --- /dev/null +++ b/crates/fbuild-config/src/ini_parser/values.rs @@ -0,0 +1,113 @@ +//! Value-string helpers: inline-comment stripping, flag tokenization, +//! and multi-value list parsing. + +/// Strip inline comments (` ; comment` or ` # comment`). +/// Be careful not to strip hash/semicolons that are part of values. +pub(super) fn strip_inline_comment(s: &str) -> String { + // Only strip comments that are preceded by whitespace + // This avoids stripping "#include" or URLs with "#" + let bytes = s.as_bytes(); + for i in 1..bytes.len() { + if bytes[i - 1] == b' ' && (bytes[i] == b';' || bytes[i] == b'#') { + return s[..i].trim().to_string(); + } + } + s.trim().to_string() +} + +/// Parse build flags string into a list. +/// +/// Handles: +/// - Space-separated flags: `-DFOO -DBAR` +/// - Multi-line: one flag per line +/// - `-D FLAG` → `-DFLAG` normalization +/// - Preserves arguments for `-include`, `-I`, `-L`, etc. +pub(super) fn parse_flags(flags_str: &str) -> Vec { + let mut result = Vec::new(); + + for line in flags_str.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + + let chars = trimmed.chars(); + let mut current = String::new(); + let mut in_quotes = false; + let mut quote_char = ' '; + + for c in chars { + match c { + '"' | '\'' if !in_quotes => { + in_quotes = true; + quote_char = c; + current.push(c); + } + c if in_quotes && c == quote_char => { + in_quotes = false; + current.push(c); + } + ' ' | '\t' if !in_quotes => { + if !current.is_empty() { + result.push(current.clone()); + current.clear(); + } + } + _ => { + current.push(c); + } + } + } + + if !current.is_empty() { + result.push(current); + } + } + + // Normalize `-D FLAG` → `-DFLAG` + let mut normalized = Vec::new(); + let mut i = 0; + while i < result.len() { + if result[i] == "-D" && i + 1 < result.len() { + normalized.push(format!("-D{}", result[i + 1])); + i += 2; + } else { + normalized.push(result[i].clone()); + i += 1; + } + } + + normalized +} + +/// Parse library dependencies from a multi-line or comma-separated string. +pub(super) fn parse_lib_deps(deps_str: &str) -> Vec { + let mut result = Vec::new(); + + for line in deps_str.lines() { + for dep in line.split(',') { + let trimmed = dep.trim(); + if !trimmed.is_empty() { + result.push(trimmed.to_string()); + } + } + } + + result +} + +/// Parse a generic multi-value option from a multi-line or comma-separated string. +pub(super) fn parse_list_values(value: &str) -> Vec { + let mut result = Vec::new(); + + for line in value.lines() { + for item in line.split(',') { + let trimmed = item.trim(); + if !trimmed.is_empty() { + result.push(trimmed.to_string()); + } + } + } + + result +} diff --git a/crates/fbuild-config/src/ini_parser/variables.rs b/crates/fbuild-config/src/ini_parser/variables.rs new file mode 100644 index 00000000..0b70f4a1 --- /dev/null +++ b/crates/fbuild-config/src/ini_parser/variables.rs @@ -0,0 +1,87 @@ +//! `${section.key}` variable reference resolution. + +use std::collections::HashMap; +use std::collections::HashSet; + +/// Resolve a variable reference like "section.key" or "env:name.key". +pub(super) fn resolve_variable( + sections: &HashMap>, + current_config: &HashMap, + reference: &str, +) -> String { + // Try "section.key" format + if let Some(dot_pos) = reference.find('.') { + let section_ref = &reference[..dot_pos]; + let key = &reference[dot_pos + 1..]; + + // Try env:name format + let section_name = if section_ref.starts_with("env:") { + section_ref.to_string() + } else { + // Could be env:name or just a section name + let as_env = format!("env:{}", section_ref); + if sections.contains_key(&as_env) { + as_env + } else { + section_ref.to_string() + } + }; + + // Look up the key, following the `extends` chain if needed + if let Some(val) = resolve_section_key(sections, §ion_name, key, &mut HashSet::new()) { + return val; + } + } + + // Try as a key in the current config + if let Some(val) = current_config.get(reference) { + return val.clone(); + } + + // Unresolved — return as-is + format!("${{{}}}", reference) +} + +/// Look up a key in a section, following `extends` chains. +fn resolve_section_key( + sections: &HashMap>, + section_name: &str, + key: &str, + visited: &mut HashSet, +) -> Option { + if !visited.insert(section_name.to_string()) { + return None; // circular extends + } + + let section = sections.get(section_name)?; + + // Direct lookup + if let Some(val) = section.get(key) { + return Some(val.clone()); + } + + // Follow extends chain + if let Some(extends) = section.get("extends") { + for parent_ref in extends.split(',') { + let parent_ref = parent_ref.trim(); + // Resolve parent section name (could be "env:name" or bare name) + let parent_name = if parent_ref.starts_with("env:") || sections.contains_key(parent_ref) + { + parent_ref.to_string() + } else { + let as_env = format!("env:{}", parent_ref); + if sections.contains_key(&as_env) { + as_env + } else { + parent_ref.to_string() + } + }; + + if let Some(val) = resolve_section_key(sections, &parent_name, key, visited) { + return Some(val); + } + } + } + + None +} diff --git a/crates/fbuild-daemon/src/handlers/README.md b/crates/fbuild-daemon/src/handlers/README.md index 985843d1..4188d8f3 100644 --- a/crates/fbuild-daemon/src/handlers/README.md +++ b/crates/fbuild-daemon/src/handlers/README.md @@ -4,8 +4,8 @@ HTTP and WebSocket route handlers for the fbuild daemon. - **`mod.rs`** -- Module declarations - **`health.rs`** -- `GET /`, `/health`, `/api/daemon/info`, `POST /api/daemon/shutdown` -- **`operations.rs`** -- `POST /api/build`, `/api/deploy`, `/api/monitor`, `/api/install-deps`, `/api/reset` with RAII `OperationGuard` for state tracking +- **`operations/`** -- `POST /api/build`, `/api/deploy`, `/api/monitor`, `/api/install-deps`, `/api/reset` with RAII `OperationGuard` for state tracking (split into submodules, see `operations/README.md`) - **`devices.rs`** -- Device discovery, lease acquire/release/preempt handlers for `/api/devices/` endpoints - **`locks.rs`** -- `GET /api/locks/status` and `POST /api/locks/clear` for project and serial port locks -- **`emulator.rs`** -- Emulator deploy handlers (AVR8js, QEMU), `EmulatorRunner` trait abstraction, `POST /api/test-emu` build-then-emulate flow +- **`emulator/`** -- Emulator deploy handlers (AVR8js, QEMU, simavr), `EmulatorRunner` trait abstraction, `POST /api/test-emu` build-then-emulate flow. See `emulator/README.md` for the submodule layout. - **`websockets.rs`** -- WebSocket upgrade handlers: serial monitor (`/ws/serial-monitor`), status streaming (`/ws/status`), log streaming (`/ws/logs`), named monitor sessions (`/ws/monitor/{session_id}`) diff --git a/crates/fbuild-daemon/src/handlers/emulator.rs b/crates/fbuild-daemon/src/handlers/emulator.rs deleted file mode 100644 index 713ed4f7..00000000 --- a/crates/fbuild-daemon/src/handlers/emulator.rs +++ /dev/null @@ -1,3245 +0,0 @@ -use crate::context::DaemonContext; -use crate::handlers::operations::{MonitorOutcome, MonitorState}; -use crate::models::OperationResponse; -use axum::extract::{Path as AxumPath, State}; -use axum::http::{header, HeaderValue, StatusCode}; -use axum::response::{Html, IntoResponse}; -use axum::Json; -use fbuild_packages::{Package, Toolchain}; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::process::Stdio; -use std::sync::Arc; -use tokio::io::{AsyncBufReadExt, BufReader}; - -const AVR8JS_APP_JS: &str = include_str!("../../web/avr8js/app.js"); -const AVR8JS_HEADLESS_MJS: &str = include_str!("../../web/avr8js/headless.mjs"); - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct Avr8jsSessionManifest { - session_id: String, - project_dir: String, - env_name: String, - board_id: String, - platform: String, - mcu: String, - f_cpu_hz: u32, - firmware_hex: String, - firmware_elf: Option, - created_at_unix: f64, -} - -#[derive(Debug, Serialize)] -struct Avr8jsSessionResponse { - session_id: String, - env_name: String, - board_id: String, - platform: String, - mcu: String, - f_cpu_hz: u32, - firmware_hex_url: String, -} - -fn now_unix() -> f64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs_f64() -} - -fn render_page(session_id: &str) -> String { - format!( - r#" - - - - - AVR8js Monitor - - - -
-
-
AVR8js Emulator
-
Loading session...
-
- -
-
Session {}
-
Browser-side ATmega328P serial monitor
-
-
- - - - -"#, - session_id, - serde_json::to_string(session_id).unwrap_or_else(|_| "\"\"".to_string()), - ) -} - -fn load_session_manifest( - ctx: &DaemonContext, - session_id: &str, -) -> fbuild_core::Result { - let manifest_path = ctx - .avr8js_sessions - .get(session_id) - .map(|entry| entry.value().clone()) - .ok_or_else(|| { - fbuild_core::FbuildError::Other(format!("unknown AVR8js session '{}'", session_id)) - })?; - let raw = std::fs::read_to_string(&manifest_path)?; - serde_json::from_str(&raw).map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to parse AVR8js session manifest: {}", e)) - }) -} - -pub struct DeployAvr8jsRequest { - pub request_id: String, - pub project_dir: PathBuf, - pub env_name: String, - pub board_id: String, - pub platform: fbuild_core::Platform, - pub firmware_path: PathBuf, - pub elf_path: Option, - pub monitor_after: bool, - pub output_file: String, - pub output_dir: Option, - pub monitor_timeout: Option, - pub halt_on_error: Option, - pub halt_on_success: Option, - pub expect: Option, - pub show_timestamp: bool, - pub verbose: bool, -} - -pub struct DeployQemuRequest { - pub request_id: String, - pub project_dir: PathBuf, - pub env_name: String, - pub board_id: String, - pub platform: fbuild_core::Platform, - pub firmware_path: PathBuf, - pub elf_path: Option, - pub output_file: String, - pub output_dir: Option, - pub monitor_timeout: Option, - pub qemu_timeout_secs: u32, - pub halt_on_error: Option, - pub halt_on_success: Option, - pub expect: Option, - pub show_timestamp: bool, - pub verbose: bool, - pub board_overrides: HashMap, -} - -struct ProcessLine { - is_stderr: bool, - line: String, -} - -enum ProcessEvent { - Line(ProcessLine), - StreamClosed, -} - -struct QemuRunResult { - outcome: MonitorOutcome, - stdout: String, - stderr: String, - exit_code: Option, -} - -struct RunQemuOptions<'a> { - elf_path: Option, - addr2line_path: Option, - timeout_secs: Option, - halt_on_error: Option<&'a str>, - halt_on_success: Option<&'a str>, - expect: Option<&'a str>, - show_timestamp: bool, - verbose: bool, - /// Label used in user-visible messages (e.g. "QEMU", "simavr"). - process_label: &'a str, -} - -pub async fn deploy_avr8js( - ctx: Arc, - req: DeployAvr8jsRequest, -) -> (StatusCode, Json) { - let DeployAvr8jsRequest { - request_id, - project_dir, - env_name, - board_id, - platform, - firmware_path, - elf_path, - monitor_after, - output_file, - output_dir, - monitor_timeout, - halt_on_error, - halt_on_success, - expect, - show_timestamp, - verbose, - } = req; - - if !matches!( - platform, - fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr - ) { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - "avr8js deploy target is only supported for AVR boards".to_string(), - )), - ); - } - if board_id != "uno" { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!( - "avr8js deploy target currently supports only board 'uno' (got '{}')", - board_id - ), - )), - ); - } - if firmware_path.extension().and_then(|ext| ext.to_str()) != Some("hex") { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!( - "avr8js deploy target requires firmware.hex, got '{}'", - firmware_path.display() - ), - )), - ); - } - - let board = match fbuild_config::BoardConfig::from_board_id(&board_id, &HashMap::new()) { - Ok(board) => board, - Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("failed to load AVR8js board config: {}", e), - )), - ); - } - }; - if !board.mcu.eq_ignore_ascii_case("atmega328p") { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!( - "avr8js deploy target currently supports only ATmega328P, got '{}'", - board.mcu - ), - )), - ); - } - - let session_id = uuid::Uuid::new_v4().to_string(); - let session_dir = fbuild_paths::get_project_fbuild_dir(&project_dir) - .join("emulators") - .join("avr8js") - .join(&env_name) - .join(&session_id); - if let Err(e) = std::fs::create_dir_all(&session_dir) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("failed to create AVR8js session dir: {}", e), - )), - ); - } - - let staged_hex = session_dir.join("firmware.hex"); - if let Err(e) = std::fs::copy(&firmware_path, &staged_hex) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("failed to stage AVR8js firmware.hex: {}", e), - )), - ); - } - - let staged_elf = if let Some(ref elf) = elf_path { - let dest = session_dir.join("firmware.elf"); - match std::fs::copy(elf, &dest) { - Ok(_) => Some(dest), - Err(_) => None, - } - } else { - None - }; - - let manifest = Avr8jsSessionManifest { - session_id: session_id.clone(), - project_dir: project_dir.to_string_lossy().to_string(), - env_name: env_name.clone(), - board_id, - platform: format!("{:?}", platform), - mcu: board.mcu.clone(), - f_cpu_hz: board - .f_cpu - .trim_end_matches('L') - .parse::() - .unwrap_or(16_000_000), - firmware_hex: staged_hex.to_string_lossy().to_string(), - firmware_elf: staged_elf.map(|p| p.to_string_lossy().to_string()), - created_at_unix: now_unix(), - }; - let manifest_path = session_dir.join("session.json"); - if let Err(e) = std::fs::write( - &manifest_path, - serde_json::to_vec_pretty(&manifest).unwrap_or_default(), - ) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("failed to write AVR8js session manifest: {}", e), - )), - ); - } - ctx.avr8js_sessions - .insert(session_id.clone(), manifest_path); - - if monitor_after { - // Headless path: run avr8js in Node.js subprocess, capture UART on stdout - let node_path = match find_node() { - Ok(p) => p, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail(request_id, e.to_string())), - ); - } - }; - let avr8js_cache = match ensure_avr8js_npm() { - Ok(p) => p, - Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail(request_id, e.to_string())), - ); - } - }; - - let script_path = session_dir.join("headless.mjs"); - if let Err(e) = std::fs::write(&script_path, AVR8JS_HEADLESS_MJS) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("failed to write headless.mjs: {}", e), - )), - ); - } - - let avr8js_result = match run_avr8js_headless( - &node_path, - &script_path, - &staged_hex, - manifest.f_cpu_hz, - &avr8js_cache, - RunAvr8jsHeadlessOptions { - timeout_secs: monitor_timeout, - halt_on_error: halt_on_error.as_deref(), - halt_on_success: halt_on_success.as_deref(), - expect: expect.as_deref(), - show_timestamp, - verbose, - }, - ) - .await - { - Ok(r) => r, - Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail(request_id, e.to_string())), - ); - } - }; - - match avr8js_result.outcome { - MonitorOutcome::Success(message) => ( - StatusCode::OK, - Json(OperationResponse { - success: true, - request_id, - message: format!("avr8js run succeeded: {}", message), - exit_code: 0, - output_file: Some(output_file), - output_dir, - launch_url: None, - stdout: Some(avr8js_result.stdout), - stderr: Some(avr8js_result.stderr), - }), - ), - MonitorOutcome::Error(message) => ( - StatusCode::OK, - Json(OperationResponse { - success: false, - request_id, - message: format!("avr8js run failed: {}", message), - exit_code: 1, - output_file: Some(output_file), - output_dir, - launch_url: None, - stdout: Some(avr8js_result.stdout), - stderr: Some(avr8js_result.stderr), - }), - ), - MonitorOutcome::Timeout { expect_found } => { - let success = expect.is_none() || expect_found; - let exit_code = if success { 0 } else { 1 }; - ( - StatusCode::OK, - Json(OperationResponse { - success, - request_id, - message: if success { - "avr8js run completed (timeout)".to_string() - } else { - "avr8js run timed out (expected pattern not found)".to_string() - }, - exit_code, - output_file: Some(output_file), - output_dir, - launch_url: None, - stdout: Some(avr8js_result.stdout), - stderr: Some(avr8js_result.stderr), - }), - ) - } - } - } else { - // Browser path: return URL for the avr8js web UI - let launch_url = Some(format!( - "http://127.0.0.1:{}/emulator/avr8js/{}", - ctx.port, session_id - )); - ( - StatusCode::OK, - Json(OperationResponse { - success: true, - request_id, - message: "deploy complete".to_string(), - exit_code: 0, - output_file: Some(output_file), - output_dir, - launch_url, - stdout: None, - stderr: None, - }), - ) - } -} - -// --------------------------------------------------------------------------- -// Headless avr8js helpers -// --------------------------------------------------------------------------- - -fn find_node() -> fbuild_core::Result { - let node = if cfg!(windows) { "node.exe" } else { "node" }; - // Route through fbuild-core's `run_command` so the probe spawn is - // captured by the daemon's containment group (issue #32). The probe - // is short-lived (`node --version`) but a missing binary should - // still bubble up the same way. - match fbuild_core::subprocess::run_command(&[node, "--version"], None, None, None) { - Ok(output) if output.success() => Ok(PathBuf::from(node)), - _ => Err(fbuild_core::FbuildError::DeployFailed( - "Node.js is required for headless avr8js emulation but 'node' was not found on PATH. \ - Install Node.js 18+ from https://nodejs.org/" - .to_string(), - )), - } -} - -/// Env-var that forces `ensure_avr8js_npm` to wipe and reinstall the cache -/// regardless of the integrity marker. Set `FBUILD_REFRESH_EMU_CACHE=1` before -/// invoking `fbuild test-emu` (or restart the daemon with it in the env) to -/// recover from a corrupt or partial avr8js install. -const REFRESH_EMU_CACHE_ENV: &str = "FBUILD_REFRESH_EMU_CACHE"; - -fn refresh_emu_cache_requested() -> bool { - matches!( - std::env::var(REFRESH_EMU_CACHE_ENV) - .ok() - .as_deref() - .map(str::trim), - Some("1") | Some("true") | Some("TRUE") | Some("yes") | Some("YES") - ) -} - -fn avr8js_cache_is_intact(cache_dir: &Path) -> bool { - // Cheapest proof of a non-corrupt install: both the `avr8js` module dir - // *and* its `package.json` must exist. A stale/partial `npm install` - // sometimes leaves the directory without the manifest and causes - // Node to fail with `ERR_MODULE_NOT_FOUND` at runtime. - let module_dir = cache_dir.join("node_modules").join("avr8js"); - let marker = module_dir.join("package.json"); - module_dir.is_dir() && marker.is_file() -} - -/// Describes what `prepare_avr8js_cache_for_install` did to the cache dir -/// before a reinstall attempt. Exposed for unit testing; production code -/// only needs the side-effect on disk. -#[derive(Debug, PartialEq, Eq)] -enum Avr8jsCachePrep { - /// Cache was already intact — no-op, reinstall not required. - AlreadyIntact, - /// No preexisting cache tree; reinstall will populate a fresh dir. - NothingToClean, - /// `node_modules/` was present but corrupt (missing marker); wiped. - WipedNodeModules, - /// Force-refresh was requested; entire `cache_dir` was wiped. - ForceWiped, -} - -/// Inspect `cache_dir`, wipe corrupt/partial installs, and return what was -/// done. Does NOT run `npm install`. -fn prepare_avr8js_cache_for_install(cache_dir: &Path, force_refresh: bool) -> Avr8jsCachePrep { - if !force_refresh && avr8js_cache_is_intact(cache_dir) { - return Avr8jsCachePrep::AlreadyIntact; - } - - if force_refresh && cache_dir.exists() { - tracing::info!( - "{}=1 set; wiping avr8js cache at {}", - REFRESH_EMU_CACHE_ENV, - cache_dir.display() - ); - if let Err(e) = std::fs::remove_dir_all(cache_dir) { - tracing::warn!( - "failed to wipe avr8js cache at {}: {} (continuing with reinstall)", - cache_dir.display(), - e - ); - } - return Avr8jsCachePrep::ForceWiped; - } - - if cache_dir.exists() { - let node_modules = cache_dir.join("node_modules"); - if node_modules.exists() { - tracing::warn!( - "avr8js cache at {} is corrupt (missing node_modules/avr8js/package.json); reinstalling", - cache_dir.display() - ); - if let Err(e) = std::fs::remove_dir_all(&node_modules) { - tracing::warn!( - "failed to wipe avr8js node_modules at {}: {} (continuing with reinstall)", - node_modules.display(), - e - ); - } - return Avr8jsCachePrep::WipedNodeModules; - } - } - - Avr8jsCachePrep::NothingToClean -} - -fn ensure_avr8js_npm() -> fbuild_core::Result { - let cache_dir = fbuild_paths::get_cache_root().join("avr8js-node"); - ensure_avr8js_npm_in(&cache_dir, refresh_emu_cache_requested())?; - Ok(cache_dir) -} - -/// Populate `cache_dir` with a fresh `node_modules/avr8js` install, wiping -/// a corrupt or partial install as needed. Split out from `ensure_avr8js_npm` -/// so unit tests can inject a temporary cache dir without touching env vars. -fn ensure_avr8js_npm_in(cache_dir: &Path, force_refresh: bool) -> fbuild_core::Result<()> { - if prepare_avr8js_cache_for_install(cache_dir, force_refresh) == Avr8jsCachePrep::AlreadyIntact - { - return Ok(()); - } - - std::fs::create_dir_all(cache_dir).map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!( - "failed to create avr8js cache dir at {}: {}", - cache_dir.display(), - e - )) - })?; - - let npm = if cfg!(windows) { "npm.cmd" } else { "npm" }; - // Route through `run_command` (which spawns via the daemon's - // containment group) so an `npm install` killed mid-flight doesn't - // leak node processes after the daemon dies. See FastLED/fbuild#32. - let cache_dir_str = cache_dir.to_string_lossy().to_string(); - let output = fbuild_core::subprocess::run_command( - &[ - npm, - "install", - "--save", - "avr8js@0.21.0", - "--prefix", - &cache_dir_str, - ], - None, - None, - None, - ) - .map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!( - "failed to launch 'npm' (for `npm install avr8js@0.21.0 --prefix {}`): {}. \ - Ensure `npm` is installed alongside Node.js and on PATH \ - (https://nodejs.org/). If npm is installed, set \ - {}=1 to force a clean reinstall.", - cache_dir.display(), - e, - REFRESH_EMU_CACHE_ENV - )) - })?; - if !output.success() { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "`npm install avr8js@0.21.0 --prefix {}` exited with status {}.\n\ - --- stdout ---\n{}\n--- stderr ---\n{}", - cache_dir.display(), - output.exit_code, - output.stdout.trim_end(), - output.stderr.trim_end() - ))); - } - - // Post-install integrity check: guard against npm exiting 0 without - // actually extracting the package (rare, but has been seen on Windows - // under antivirus interference). - if !avr8js_cache_is_intact(cache_dir) { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "avr8js npm install at {} reported success but the cache is still \ - incomplete (missing node_modules/avr8js/package.json). \ - Try rerunning with {}=1.", - cache_dir.display(), - REFRESH_EMU_CACHE_ENV - ))); - } - - Ok(()) -} - -struct Avr8jsRunResult { - outcome: MonitorOutcome, - stdout: String, - stderr: String, - exit_code: Option, -} - -struct RunAvr8jsHeadlessOptions<'a> { - timeout_secs: Option, - halt_on_error: Option<&'a str>, - halt_on_success: Option<&'a str>, - expect: Option<&'a str>, - show_timestamp: bool, - verbose: bool, -} - -async fn run_avr8js_headless( - node_path: &Path, - script_path: &Path, - hex_path: &Path, - f_cpu_hz: u32, - avr8js_cache_dir: &Path, - options: RunAvr8jsHeadlessOptions<'_>, -) -> fbuild_core::Result { - // Fail-fast: re-verify the cache is intact before we spawn Node. This - // guards against race conditions (cache wiped between `ensure_avr8js_npm` - // and this call) and makes the error actionable instead of a cryptic - // Node `ERR_MODULE_NOT_FOUND` stack trace (issue #86). - if !avr8js_cache_is_intact(avr8js_cache_dir) { - tracing::error!( - "avr8js cache not populated at {}; aborting (expected \ - node_modules/avr8js/package.json). Set {}=1 to force reinstall.", - avr8js_cache_dir.display(), - REFRESH_EMU_CACHE_ENV - ); - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "avr8js cache not populated at {} (missing \ - node_modules/avr8js/package.json). \ - Set {}=1 to force reinstall.", - avr8js_cache_dir.display(), - REFRESH_EMU_CACHE_ENV - ))); - } - - // allow-direct-spawn: tokio streaming emulator; blocking NativeProcess unsuitable. - let mut cmd = tokio::process::Command::new(node_path); - cmd.arg(script_path) - .arg("--hex") - .arg(hex_path) - .arg("--f-cpu") - .arg(f_cpu_hz.to_string()) - .env("NODE_PATH", avr8js_cache_dir.join("node_modules")) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - #[cfg(windows)] - { - const CREATE_NO_WINDOW: u32 = 0x08000000; - cmd.creation_flags(CREATE_NO_WINDOW); - } - - if options.verbose { - tracing::info!( - "avr8js headless: {} {} --hex {} --f-cpu {}", - node_path.display(), - script_path.display(), - hex_path.display(), - f_cpu_hz - ); - } - - // Route through containment (#32) so a daemon crash mid-emulation - // takes node.exe and any helper processes it spawned with it. - let mut child = - fbuild_core::containment::tokio_spawn::spawn_contained(&mut cmd).map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!( - "failed to launch Node.js for avr8js: {}", - e - )) - })?; - - let stdout = child.stdout.take().ok_or_else(|| { - fbuild_core::FbuildError::DeployFailed("failed to capture avr8js stdout".to_string()) - })?; - let stderr = child.stderr.take().ok_or_else(|| { - fbuild_core::FbuildError::DeployFailed("failed to capture avr8js stderr".to_string()) - })?; - - let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); - let stdout_task = tokio::spawn(spawn_line_reader(stdout, false, tx.clone())); - let stderr_task = tokio::spawn(spawn_line_reader(stderr, true, tx)); - - let mut monitor = MonitorState::new( - options.timeout_secs, - options.halt_on_error, - options.halt_on_success, - options.expect, - options.show_timestamp, - ); - let mut stdout_buf = String::new(); - let mut stderr_buf = String::new(); - let mut streams_open = 2usize; - let mut child_exit: Option = None; - let mut final_outcome: Option = None; - - loop { - if monitor.timed_out() { - final_outcome = Some(monitor.timeout_outcome()); - let _ = child.kill().await; - break; - } - - let recv_timeout = monitor - .remaining() - .unwrap_or(std::time::Duration::from_secs(1)); - - tokio::select! { - status = child.wait(), if child_exit.is_none() => { - child_exit = Some(status.map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!("avr8js wait failed: {}", e)) - })?); - if streams_open == 0 { - break; - } - } - maybe_event = tokio::time::timeout(recv_timeout, rx.recv()) => { - match maybe_event { - Ok(Some(ProcessEvent::Line(line))) => { - let target = if line.is_stderr { &mut stderr_buf } else { &mut stdout_buf }; - target.push_str(&line.line); - target.push('\n'); - - if let Some(outcome) = monitor.process_line(&line.line) { - final_outcome = Some(outcome); - let _ = child.kill().await; - break; - } - } - Ok(Some(ProcessEvent::StreamClosed)) => { - streams_open = streams_open.saturating_sub(1); - if streams_open == 0 && child_exit.is_some() { - break; - } - } - Ok(None) => { - if child_exit.is_some() { - break; - } - } - Err(_) => { - final_outcome = Some(monitor.timeout_outcome()); - let _ = child.kill().await; - break; - } - } - } - } - } - - if child_exit.is_none() { - child_exit = Some(child.wait().await.map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!("avr8js wait failed: {}", e)) - })?); - } - - let _ = stdout_task.await; - let _ = stderr_task.await; - - let outcome = if let Some(outcome) = final_outcome { - outcome - } else if let Some(status) = child_exit { - if status.success() { - if options.expect.is_some() && !monitor.expect_found() { - MonitorOutcome::Error( - "avr8js exited before the expected pattern was found".to_string(), - ) - } else { - MonitorOutcome::Success("avr8js exited normally".to_string()) - } - } else { - MonitorOutcome::Error(format!( - "avr8js exited with code {}", - status.code().unwrap_or(-1) - )) - } - } else { - MonitorOutcome::Error("avr8js exited unexpectedly".to_string()) - }; - - Ok(Avr8jsRunResult { - outcome, - stdout: stdout_buf, - stderr: stderr_buf, - exit_code: child_exit.and_then(|s| s.code()), - }) -} - -fn qemu_session_dir(project_dir: &Path, env_name: &str) -> PathBuf { - fbuild_paths::get_project_fbuild_dir(project_dir) - .join("emulators") - .join("qemu") - .join(env_name) - .join(uuid::Uuid::new_v4().to_string()) -} - -fn build_linux_macos_qemu_hint(err: &str) -> String { - if cfg!(any(target_os = "linux", target_os = "macos")) { - format!( - "{}. On Linux/macOS, ensure QEMU runtime deps are installed: libgcrypt, glib2, pixman, SDL2, and libslirp.", - err - ) - } else { - err.to_string() - } -} - -fn resolve_esp32_toolchain_gcc_path( - project_dir: &Path, - mcu_config: &fbuild_build::esp32::mcu_config::Esp32McuConfig, -) -> fbuild_core::Result { - let platform = fbuild_packages::library::Esp32Platform::new(project_dir); - Package::ensure_installed(&platform)?; - - let is_riscv = mcu_config.is_riscv(); - let prefix = mcu_config.toolchain_prefix(); - let toolchain_name = if is_riscv { - "toolchain-riscv32-esp" - } else { - "toolchain-xtensa-esp-elf" - }; - - let toolchain = match platform.get_toolchain_metadata_url(is_riscv) { - Ok(metadata_url) => { - let cache = fbuild_packages::Cache::new(project_dir); - let cache_dir = cache.toolchains_dir().join(toolchain_name); - match fbuild_packages::toolchain::esp32_metadata::resolve_toolchain_url_sync( - &metadata_url, - toolchain_name, - &cache_dir, - ) { - Ok(resolved) => fbuild_packages::toolchain::Esp32Toolchain::from_resolved( - project_dir, - &resolved.url, - resolved.sha256.as_deref(), - is_riscv, - &prefix, - ), - Err(_) => { - fbuild_packages::toolchain::Esp32Toolchain::new(project_dir, is_riscv, &prefix) - } - } - } - Err(_) => fbuild_packages::toolchain::Esp32Toolchain::new(project_dir, is_riscv, &prefix), - }; - - let _ = Package::ensure_installed(&toolchain)?; - Ok(toolchain.get_gcc_path()) -} - -#[cfg(windows)] -fn apply_windows_process_flags(cmd: &mut tokio::process::Command, exe_path: &Path) { - const CREATE_NO_WINDOW: u32 = 0x08000000; - cmd.creation_flags(CREATE_NO_WINDOW); - - let current_path = std::env::var("PATH").unwrap_or_default(); - if let Ok(path_env) = - fbuild_packages::toolchain::build_windows_qemu_path_env(exe_path, ¤t_path) - { - cmd.env("PATH", path_env); - } -} - -#[cfg(not(windows))] -fn apply_windows_process_flags(_cmd: &mut tokio::process::Command, _exe_path: &Path) {} - -async fn spawn_line_reader( - stream: impl tokio::io::AsyncRead + Unpin + Send + 'static, - is_stderr: bool, - tx: tokio::sync::mpsc::UnboundedSender, -) { - let mut lines = BufReader::new(stream).lines(); - while let Ok(Some(line)) = lines.next_line().await { - let _ = tx.send(ProcessEvent::Line(ProcessLine { is_stderr, line })); - } - let _ = tx.send(ProcessEvent::StreamClosed); -} - -async fn run_qemu_process( - qemu_path: &Path, - args: &[String], - options: RunQemuOptions<'_>, -) -> fbuild_core::Result { - // allow-direct-spawn: tokio streaming QEMU emulator; blocking NativeProcess unsuitable. - let mut cmd = tokio::process::Command::new(qemu_path); - cmd.args(args) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - apply_windows_process_flags(&mut cmd, qemu_path); - - let label = options.process_label; - if options.verbose { - tracing::info!("{}: {} {}", label, qemu_path.display(), args.join(" ")); - } - - // Route through containment (#32). QEMU is a long-running process - // — a daemon hard-kill mid-emulation must not leave `qemu-system-*` - // or its `conhost.exe` wrapper behind. - let mut child = - fbuild_core::containment::tokio_spawn::spawn_contained(&mut cmd).map_err(|e| { - fbuild_core::FbuildError::DeployFailed(build_linux_macos_qemu_hint(&format!( - "failed to launch {} at {}: {}", - label, - qemu_path.display(), - e - ))) - })?; - - let stdout = child.stdout.take().ok_or_else(|| { - fbuild_core::FbuildError::DeployFailed(format!("failed to capture {} stdout", label)) - })?; - let stderr = child.stderr.take().ok_or_else(|| { - fbuild_core::FbuildError::DeployFailed(format!("failed to capture {} stderr", label)) - })?; - - let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); - let stdout_task = tokio::spawn(spawn_line_reader(stdout, false, tx.clone())); - let stderr_task = tokio::spawn(spawn_line_reader(stderr, true, tx)); - - let mut monitor = MonitorState::new( - options.timeout_secs, - options.halt_on_error, - options.halt_on_success, - options.expect, - options.show_timestamp, - ); - let mut crash_decoder = - fbuild_serial::crash_decoder::CrashDecoder::new(options.elf_path, options.addr2line_path); - let mut stdout_buf = String::new(); - let mut stderr_buf = String::new(); - let mut synthetic_buf = String::new(); - let mut streams_open = 2usize; - let mut child_exit: Option = None; - let mut final_outcome: Option = None; - - loop { - if monitor.timed_out() { - final_outcome = Some(monitor.timeout_outcome()); - let _ = child.kill().await; - break; - } - - let recv_timeout = monitor - .remaining() - .unwrap_or(std::time::Duration::from_secs(1)); - - tokio::select! { - status = child.wait(), if child_exit.is_none() => { - child_exit = Some(status.map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!("{} wait failed: {}", label, e)) - })?); - if streams_open == 0 { - break; - } - } - maybe_event = tokio::time::timeout(recv_timeout, rx.recv()) => { - match maybe_event { - Ok(Some(ProcessEvent::Line(line))) => { - let target = if line.is_stderr { &mut stderr_buf } else { &mut stdout_buf }; - target.push_str(&line.line); - target.push('\n'); - - if let Some(outcome) = monitor.process_line(&line.line) { - final_outcome = Some(outcome); - let _ = child.kill().await; - break; - } - - if let Some(decoded_lines) = crash_decoder.process_line(&line.line) { - for decoded in decoded_lines { - synthetic_buf.push_str(&decoded); - synthetic_buf.push('\n'); - if let Some(outcome) = monitor.process_line(&decoded) { - final_outcome = Some(outcome); - let _ = child.kill().await; - break; - } - } - if final_outcome.is_some() { - break; - } - } - } - Ok(Some(ProcessEvent::StreamClosed)) => { - streams_open = streams_open.saturating_sub(1); - if streams_open == 0 && child_exit.is_some() { - break; - } - } - Ok(None) => { - if child_exit.is_some() { - break; - } - } - Err(_) => { - final_outcome = Some(monitor.timeout_outcome()); - let _ = child.kill().await; - break; - } - } - } - } - } - - if child_exit.is_none() { - child_exit = Some(child.wait().await.map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!("{} wait failed: {}", label, e)) - })?); - } - - let _ = stdout_task.await; - let _ = stderr_task.await; - - if !synthetic_buf.is_empty() { - stdout_buf.push_str(&synthetic_buf); - } - - let outcome = if let Some(outcome) = final_outcome { - outcome - } else if let Some(status) = child_exit { - if status.success() { - if options.expect.is_some() && !monitor.expect_found() { - MonitorOutcome::Error(format!( - "{} exited before the expected pattern was found", - label - )) - } else { - MonitorOutcome::Success(format!("{} exited normally", label)) - } - } else { - MonitorOutcome::Error(format!( - "{} exited with code {}", - label, - status.code().unwrap_or(-1) - )) - } - } else { - MonitorOutcome::Error(format!("{} exited unexpectedly", label)) - }; - - Ok(QemuRunResult { - outcome, - stdout: stdout_buf, - stderr: stderr_buf, - exit_code: child_exit.and_then(|s| s.code()), - }) -} - -pub async fn deploy_qemu( - _ctx: Arc, - req: DeployQemuRequest, -) -> (StatusCode, Json) { - let DeployQemuRequest { - request_id, - project_dir, - env_name, - board_id, - platform, - firmware_path, - elf_path, - output_file, - output_dir, - monitor_timeout, - qemu_timeout_secs, - halt_on_error, - halt_on_success, - expect, - show_timestamp, - verbose, - board_overrides, - } = req; - - if platform != fbuild_core::Platform::Espressif32 { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - "QEMU deploy target is currently supported only for ESP32-family boards" - .to_string(), - )), - ); - } - if firmware_path.extension().and_then(|ext| ext.to_str()) != Some("bin") { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!( - "QEMU deploy target requires firmware.bin, got '{}'", - firmware_path.display() - ), - )), - ); - } - - let board = match fbuild_config::BoardConfig::from_board_id(&board_id, &board_overrides) { - Ok(board) => board, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("failed to load board config for QEMU: {}", e), - )), - ); - } - }; - if !is_qemu_supported_esp32_mcu(&board.mcu) { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!( - "native QEMU deploy currently supports ESP32, ESP32-S3 (Xtensa) and \ - ESP32-C3, ESP32-C6, ESP32-H2 (RISC-V) boards, got '{}'", - board.mcu - ), - )), - ); - } - - let mcu_config = match fbuild_build::esp32::mcu_config::get_mcu_config(&board.mcu) { - Ok(cfg) => cfg, - Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("failed to load MCU config for QEMU: {}", e), - )), - ); - } - }; - - let effective_flash_mode = board - .flash_mode - .as_deref() - .unwrap_or(mcu_config.default_flash_mode()); - if !effective_flash_mode.eq_ignore_ascii_case("dio") { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!( - "QEMU requires a DIO-compatible flash image; effective flash mode is '{}'", - effective_flash_mode - ), - )), - ); - } - - let flash_size_bytes = match fbuild_deploy::esp32::resolve_qemu_flash_size_bytes( - &board, - mcu_config.default_flash_size(), - ) { - Ok(size) => size, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail(request_id, e.to_string())), - ); - } - }; - - let qemu = match resolve_esp_qemu_for_mcu(&project_dir, &board.mcu) { - Ok(path) => path, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - build_linux_macos_qemu_hint(&e.to_string()), - )), - ); - } - }; - - let session_dir = qemu_session_dir(&project_dir, &env_name); - if let Err(e) = std::fs::create_dir_all(&session_dir) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("failed to create QEMU session dir: {}", e), - )), - ); - } - let flash_image = session_dir.join("qemu_flash.bin"); - // Only apply the ESP32-S3 ADC calibration patch for S3 variants. - let elf_for_adc_patch = if board.mcu.eq_ignore_ascii_case("esp32s3") { - elf_path.as_deref() - } else { - None - }; - if let Err(e) = fbuild_deploy::esp32::create_qemu_flash_image( - &firmware_path, - &flash_image, - flash_size_bytes, - mcu_config.bootloader_offset(), - mcu_config.partitions_offset(), - mcu_config.firmware_offset(), - elf_for_adc_patch, - ) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("failed to create QEMU flash image: {}", e), - )), - ); - } - - let args = fbuild_deploy::esp32::build_qemu_args( - &board.mcu, - &flash_image, - board.qemu_esp32_psram_config(), - ); - let addr2line_path = elf_path.as_ref().and_then(|_| { - resolve_esp32_toolchain_gcc_path(&project_dir, &mcu_config) - .ok() - .and_then(|gcc| fbuild_serial::crash_decoder::derive_addr2line_path(&gcc)) - }); - - let timeout_secs = monitor_timeout.or(Some(qemu_timeout_secs as f64)); - let qemu_result = match run_qemu_process( - &qemu, - &args, - RunQemuOptions { - elf_path, - addr2line_path, - timeout_secs, - halt_on_error: halt_on_error.as_deref(), - halt_on_success: halt_on_success.as_deref(), - expect: expect.as_deref(), - show_timestamp, - verbose, - process_label: "QEMU", - }, - ) - .await - { - Ok(result) => result, - Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail(request_id, e.to_string())), - ); - } - }; - - let real_exit_code = qemu_result.exit_code; - match qemu_result.outcome { - MonitorOutcome::Success(message) => ( - StatusCode::OK, - Json(OperationResponse { - success: true, - request_id, - message: format!("QEMU run succeeded: {}", message), - exit_code: real_exit_code.unwrap_or(0), - output_file: Some(output_file), - output_dir, - launch_url: None, - stdout: Some(qemu_result.stdout), - stderr: Some(qemu_result.stderr), - }), - ), - MonitorOutcome::Error(message) => ( - StatusCode::OK, - Json(OperationResponse { - success: false, - request_id, - message: format!("QEMU run failed: {}", message), - exit_code: real_exit_code.unwrap_or(1), - output_file: Some(output_file), - output_dir, - launch_url: None, - stdout: Some(qemu_result.stdout), - stderr: Some(qemu_result.stderr), - }), - ), - MonitorOutcome::Timeout { expect_found } => { - let success = expect.is_none() || expect_found; - let exit_code = real_exit_code.unwrap_or(if success { 0 } else { 1 }); - ( - StatusCode::OK, - Json(OperationResponse { - success, - request_id, - message: if success { - "QEMU run completed (timeout)".to_string() - } else { - "QEMU run timed out (expected pattern not found)".to_string() - }, - exit_code, - output_file: Some(output_file), - output_dir, - launch_url: None, - stdout: Some(qemu_result.stdout), - stderr: Some(qemu_result.stderr), - }), - ) - } - } -} - -pub async fn avr8js_page( - AxumPath(session_id): AxumPath, - State(ctx): State>, -) -> impl IntoResponse { - match load_session_manifest(&ctx, &session_id) { - Ok(_) => Html(render_page(&session_id)).into_response(), - Err(e) => (StatusCode::NOT_FOUND, e.to_string()).into_response(), - } -} - -pub async fn avr8js_app_js() -> impl IntoResponse { - ( - [( - header::CONTENT_TYPE, - HeaderValue::from_static("application/javascript; charset=utf-8"), - )], - AVR8JS_APP_JS, - ) -} - -pub async fn avr8js_session_json( - AxumPath(session_id): AxumPath, - State(ctx): State>, -) -> impl IntoResponse { - match load_session_manifest(&ctx, &session_id) { - Ok(manifest) => ( - StatusCode::OK, - Json(Avr8jsSessionResponse { - session_id: manifest.session_id.clone(), - env_name: manifest.env_name, - board_id: manifest.board_id, - platform: manifest.platform, - mcu: manifest.mcu, - f_cpu_hz: manifest.f_cpu_hz, - firmware_hex_url: format!("/api/emulator/avr8js/{}/firmware.hex", session_id), - }), - ) - .into_response(), - Err(e) => (StatusCode::NOT_FOUND, e.to_string()).into_response(), - } -} - -pub async fn avr8js_firmware_hex( - AxumPath(session_id): AxumPath, - State(ctx): State>, -) -> impl IntoResponse { - match load_session_manifest(&ctx, &session_id) { - Ok(manifest) => match std::fs::read_to_string(&manifest.firmware_hex) { - Ok(hex) => ( - [( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - )], - hex, - ) - .into_response(), - Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), - }, - Err(e) => (StatusCode::NOT_FOUND, e.to_string()).into_response(), - } -} - -// --------------------------------------------------------------------------- -// EmulatorRunner abstraction (Issue #23) -// --------------------------------------------------------------------------- - -use fbuild_core::emulator::{EmulatorArtifactBundle, EmulatorOutcome, EmulatorRunResult}; - -/// Configuration for an emulator test run (user-facing options). -pub struct EmulatorRunConfig { - pub firmware_path: PathBuf, - pub elf_path: Option, - /// Structured artifact bundle for runner validation. - pub artifact_bundle: EmulatorArtifactBundle, - pub timeout: Option, - pub halt_on_error: Option, - pub halt_on_success: Option, - pub expect: Option, - pub show_timestamp: bool, - pub verbose: bool, -} - -/// Convert a `MonitorOutcome` into an `EmulatorOutcome`. -fn monitor_outcome_to_emulator(outcome: MonitorOutcome, exit_code: Option) -> EmulatorOutcome { - match outcome { - MonitorOutcome::Success(msg) => EmulatorOutcome::Passed(msg), - MonitorOutcome::Error(msg) => { - // Heuristic: if the process crashed (non-zero exit with crash signature) - if let Some(code) = exit_code { - if code != 0 && (msg.contains("abort()") || msg.contains("Guru Meditation")) { - return EmulatorOutcome::Crashed(msg); - } - } - EmulatorOutcome::Failed(msg) - } - MonitorOutcome::Timeout { expect_found } => EmulatorOutcome::TimedOut { expect_found }, - } -} - -/// Abstraction over emulator backends. Each implementation knows how to set up -/// and execute a specific emulator (QEMU, avr8js, etc.). -#[async_trait::async_trait] -pub trait EmulatorRunner: Send + Sync { - /// Human-readable name of this runner (e.g. "QEMU ESP32-S3", "avr8js ATmega328P"). - fn name(&self) -> &str; - - /// Run the emulator with the given configuration. - async fn run(&self, config: &EmulatorRunConfig) -> fbuild_core::Result; -} - -/// QEMU-based emulator runner for ESP32-family boards. -pub struct QemuRunner { - project_dir: PathBuf, - env_name: String, - board: fbuild_config::BoardConfig, - display_name: String, -} - -impl QemuRunner { - pub fn new(project_dir: PathBuf, env_name: String, board: fbuild_config::BoardConfig) -> Self { - let display_name = format!("QEMU {}", board.mcu.to_uppercase()); - Self { - project_dir, - env_name, - board, - display_name, - } - } -} - -#[async_trait::async_trait] -impl EmulatorRunner for QemuRunner { - fn name(&self) -> &str { - &self.display_name - } - - async fn run(&self, config: &EmulatorRunConfig) -> fbuild_core::Result { - if let Err(msg) = config - .artifact_bundle - .validate_for(fbuild_core::emulator::RunnerKind::QemuEsp32) - { - return Err(fbuild_core::FbuildError::DeployFailed(msg)); - } - let mcu_config = fbuild_build::esp32::mcu_config::get_mcu_config(&self.board.mcu)?; - - let effective_flash_mode = self - .board - .flash_mode - .as_deref() - .unwrap_or(mcu_config.default_flash_mode()); - if !effective_flash_mode.eq_ignore_ascii_case("dio") { - return Ok(EmulatorRunResult { - outcome: EmulatorOutcome::Unsupported(format!( - "QEMU requires DIO flash mode; effective mode is '{}'", - effective_flash_mode - )), - stdout: String::new(), - stderr: String::new(), - command_line: String::new(), - exit_code: None, - }); - } - - let flash_size_bytes = fbuild_deploy::esp32::resolve_qemu_flash_size_bytes( - &self.board, - mcu_config.default_flash_size(), - )?; - - let qemu = resolve_esp_qemu_for_mcu(&self.project_dir, &self.board.mcu)?; - - let session_dir = qemu_session_dir(&self.project_dir, &self.env_name); - std::fs::create_dir_all(&session_dir)?; - - let flash_image = session_dir.join("qemu_flash.bin"); - - // Only apply the ESP32-S3 ADC calibration patch for S3 variants. - let elf_for_adc_patch = if self.board.mcu.eq_ignore_ascii_case("esp32s3") { - config.elf_path.as_deref() - } else { - None - }; - fbuild_deploy::esp32::create_qemu_flash_image( - &config.firmware_path, - &flash_image, - flash_size_bytes, - mcu_config.bootloader_offset(), - mcu_config.partitions_offset(), - mcu_config.firmware_offset(), - elf_for_adc_patch, - )?; - - let args = fbuild_deploy::esp32::build_qemu_args( - &self.board.mcu, - &flash_image, - self.board.qemu_esp32_psram_config(), - ); - let addr2line_path = config.elf_path.as_ref().and_then(|_| { - resolve_esp32_toolchain_gcc_path(&self.project_dir, &mcu_config) - .ok() - .and_then(|gcc| fbuild_serial::crash_decoder::derive_addr2line_path(&gcc)) - }); - - let command_line = format!("{} {}", qemu.display(), args.join(" ")); - - let qemu_result = run_qemu_process( - &qemu, - &args, - RunQemuOptions { - elf_path: config.elf_path.clone(), - addr2line_path, - timeout_secs: config.timeout, - halt_on_error: config.halt_on_error.as_deref(), - halt_on_success: config.halt_on_success.as_deref(), - expect: config.expect.as_deref(), - show_timestamp: config.show_timestamp, - verbose: config.verbose, - process_label: "QEMU", - }, - ) - .await?; - - let exit_code = qemu_result.exit_code; - let outcome = monitor_outcome_to_emulator(qemu_result.outcome, exit_code); - - Ok(EmulatorRunResult { - outcome, - stdout: qemu_result.stdout, - stderr: qemu_result.stderr, - command_line, - exit_code, - }) - } -} - -/// AVR8js-based emulator runner for ATmega328P (headless Node.js). -pub struct Avr8jsRunner { - board: fbuild_config::BoardConfig, -} - -impl Avr8jsRunner { - pub fn new(board: fbuild_config::BoardConfig) -> Self { - Self { board } - } -} - -#[async_trait::async_trait] -impl EmulatorRunner for Avr8jsRunner { - fn name(&self) -> &str { - "avr8js ATmega328P" - } - - async fn run(&self, config: &EmulatorRunConfig) -> fbuild_core::Result { - if let Err(msg) = config - .artifact_bundle - .validate_for(fbuild_core::emulator::RunnerKind::Avr8js) - { - return Err(fbuild_core::FbuildError::DeployFailed(msg)); - } - let node_path = find_node()?; - let avr8js_cache = ensure_avr8js_npm()?; - - let session_dir = tempfile::TempDir::new()?; - let script_path = session_dir.path().join("headless.mjs"); - std::fs::write(&script_path, AVR8JS_HEADLESS_MJS)?; - - let f_cpu_hz: u32 = self - .board - .f_cpu - .trim_end_matches('L') - .parse() - .unwrap_or(16_000_000); - - let command_line = format!( - "{} {} --hex {} --f-cpu {}", - node_path.display(), - script_path.display(), - config.firmware_path.display(), - f_cpu_hz - ); - - let avr8js_result = run_avr8js_headless( - &node_path, - &script_path, - &config.firmware_path, - f_cpu_hz, - &avr8js_cache, - RunAvr8jsHeadlessOptions { - timeout_secs: config.timeout, - halt_on_error: config.halt_on_error.as_deref(), - halt_on_success: config.halt_on_success.as_deref(), - expect: config.expect.as_deref(), - show_timestamp: config.show_timestamp, - verbose: config.verbose, - }, - ) - .await?; - - let exit_code = avr8js_result.exit_code; - let outcome = monitor_outcome_to_emulator(avr8js_result.outcome, exit_code); - - Ok(EmulatorRunResult { - outcome, - stdout: avr8js_result.stdout, - stderr: avr8js_result.stderr, - command_line, - exit_code, - }) - } -} - -// --------------------------------------------------------------------------- -// SimAVR native runner (Issue #24) -// --------------------------------------------------------------------------- - -/// Find the `simavr` binary on PATH. -/// -/// SimAVR is a native AVR simulator. Install via: -/// - Linux: `apt install simavr` or build from source -/// - macOS: `brew install simavr` -/// - Windows: build from source (MSYS2/MinGW) — limited support -fn find_simavr() -> fbuild_core::Result { - let simavr = if cfg!(windows) { - "simavr.exe" - } else { - "simavr" - }; - // Try running simavr to verify it exists; route through containment - // (issue #32). This is a short-lived probe so the containment - // difference is purely consistency. - match fbuild_core::subprocess::run_command(&[simavr, "--help"], None, None, None) { - Ok(_) => Ok(PathBuf::from(simavr)), - Err(_) => { - let install_hint = if cfg!(target_os = "linux") { - "Install via: apt install simavr (Debian/Ubuntu) or your distro's package manager" - } else if cfg!(target_os = "macos") { - "Install via: brew install simavr" - } else { - "SimAVR has limited Windows support. Build from source via MSYS2/MinGW, \ - or use --emulator avr8js for ATmega328P boards instead" - }; - Err(fbuild_core::FbuildError::DeployFailed(format!( - "simavr is required but '{}' was not found on PATH. {}", - simavr, install_hint - ))) - } - } -} - -/// SimAVR-based native emulator runner for AVR boards. -/// -/// Supports any AVR board that advertises `simavr` in its debug_tools. -/// Primary targets: ATmega328P (Uno), ATmega32U4 (Leonardo), ATmega2560 (Mega). -/// Consumes `firmware.elf` directly. -pub struct SimavrRunner { - board: fbuild_config::BoardConfig, -} - -impl SimavrRunner { - pub fn new(board: fbuild_config::BoardConfig) -> Self { - Self { board } - } -} - -#[async_trait::async_trait] -impl EmulatorRunner for SimavrRunner { - fn name(&self) -> &str { - "simavr" - } - - async fn run(&self, config: &EmulatorRunConfig) -> fbuild_core::Result { - if let Err(msg) = config - .artifact_bundle - .validate_for(fbuild_core::emulator::RunnerKind::Simavr) - { - return Err(fbuild_core::FbuildError::DeployFailed(msg)); - } - let simavr_path = find_simavr()?; - - // simavr requires an ELF file - let elf_path = config.elf_path.as_ref().ok_or_else(|| { - fbuild_core::FbuildError::DeployFailed( - "simavr requires firmware.elf but no ELF path was produced by the build" - .to_string(), - ) - })?; - - let f_cpu_hz: u32 = self - .board - .f_cpu - .trim_end_matches('L') - .parse() - .unwrap_or(16_000_000); - - let mcu = self.board.mcu.to_lowercase(); - - let args: Vec = vec![ - "-m".to_string(), - mcu.clone(), - "-f".to_string(), - f_cpu_hz.to_string(), - elf_path.display().to_string(), - ]; - - let command_line = format!("{} {}", simavr_path.display(), args.join(" ")); - - // Reuse the generic process runner (run_qemu_process) with no crash decoder - let result = run_qemu_process( - &simavr_path, - &args, - RunQemuOptions { - elf_path: None, // no ESP32 crash decoder needed - addr2line_path: None, // no addr2line for AVR in this path - timeout_secs: config.timeout, - halt_on_error: config.halt_on_error.as_deref(), - halt_on_success: config.halt_on_success.as_deref(), - expect: config.expect.as_deref(), - show_timestamp: config.show_timestamp, - verbose: config.verbose, - process_label: "simavr", - }, - ) - .await?; - - let exit_code = result.exit_code; - let outcome = monitor_outcome_to_emulator(result.outcome, exit_code); - - Ok(EmulatorRunResult { - outcome, - stdout: result.stdout, - stderr: result.stderr, - command_line, - exit_code, - }) - } -} - -/// Check whether a given MCU is supported by the QEMU runner. -/// -/// Supported MCUs: -/// - Xtensa (`qemu-system-xtensa`): `esp32`, `esp32s3` -/// - RISC-V (`qemu-system-riscv32`): `esp32c3`, `esp32c6`, `esp32h2` -fn is_qemu_supported_esp32_mcu(mcu: &str) -> bool { - fbuild_packages::toolchain::EspQemuArch::for_mcu(mcu).is_some() -} - -/// Resolve the Espressif QEMU executable appropriate for the given MCU. -/// -/// Picks `qemu-system-xtensa` for ESP32/ESP32-S3 and `qemu-system-riscv32` -/// for ESP32-C3/C6/H2. Returns the resolved binary path (downloading into -/// the managed fbuild cache if required). -fn resolve_esp_qemu_for_mcu(project_dir: &Path, mcu: &str) -> fbuild_core::Result { - let arch = fbuild_packages::toolchain::EspQemuArch::for_mcu(mcu).ok_or_else(|| { - fbuild_core::FbuildError::DeployFailed(format!( - "no QEMU backend available for MCU '{}'", - mcu - )) - })?; - let pkg = fbuild_packages::toolchain::EspQemu::new(project_dir, arch)?; - pkg.resolve_executable() -} - -/// Fail fast if the board's flash mode is incompatible with QEMU (DIO only). -fn check_qemu_flash_mode(board: &fbuild_config::BoardConfig) -> fbuild_core::Result<()> { - let mcu_config = fbuild_build::esp32::mcu_config::get_mcu_config(&board.mcu)?; - let effective_flash_mode = board - .flash_mode - .as_deref() - .unwrap_or(mcu_config.default_flash_mode()); - if !effective_flash_mode.eq_ignore_ascii_case("dio") { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "QEMU requires DIO flash mode; board '{}' uses '{}'", - board.name, effective_flash_mode - ))); - } - Ok(()) -} - -/// Select the appropriate emulator runner based on platform, MCU, and optional -/// explicit emulator choice. -/// -/// Returns `Err` with `EmulatorOutcome::Unsupported` information if no runner -/// matches. -pub fn select_runner( - project_dir: &Path, - env_name: &str, - platform: fbuild_core::Platform, - board_id: &str, - board_overrides: &HashMap, - emulator: Option<&str>, -) -> fbuild_core::Result> { - let board = fbuild_config::BoardConfig::from_board_id(board_id, board_overrides)?; - - if let Some(explicit) = emulator { - return match explicit { - "qemu" => { - if platform != fbuild_core::Platform::Espressif32 { - return Err(fbuild_core::FbuildError::DeployFailed( - "QEMU runner is only supported for ESP32-family boards".to_string(), - )); - } - if !is_qemu_supported_esp32_mcu(&board.mcu) { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "QEMU runner currently supports ESP32, ESP32-S3 (Xtensa) and \ - ESP32-C3, ESP32-C6, ESP32-H2 (RISC-V), got '{}'", - board.mcu - ))); - } - check_qemu_flash_mode(&board)?; - Ok(Box::new(QemuRunner::new( - project_dir.to_path_buf(), - env_name.to_string(), - board, - ))) - } - "avr8js" => { - if !matches!( - platform, - fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr - ) { - return Err(fbuild_core::FbuildError::DeployFailed( - "avr8js runner is only supported for AVR boards".to_string(), - )); - } - if !board.mcu.eq_ignore_ascii_case("atmega328p") { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "avr8js runner currently supports only ATmega328P, got '{}'", - board.mcu - ))); - } - Ok(Box::new(Avr8jsRunner::new(board))) - } - "simavr" => { - if !matches!( - platform, - fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr - ) { - return Err(fbuild_core::FbuildError::DeployFailed( - "simavr runner is only supported for AVR boards".to_string(), - )); - } - if !board.has_emulator("simavr") { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "board '{}' does not advertise simavr support in its debug_tools", - board_id - ))); - } - Ok(Box::new(SimavrRunner::new(board))) - } - other => Err(fbuild_core::FbuildError::DeployFailed(format!( - "unsupported emulator '{}'; available: qemu, avr8js, simavr", - other - ))), - }; - } - - // Auto-detect based on platform and MCU - match platform { - fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr => { - if board.mcu.eq_ignore_ascii_case("atmega328p") { - // Default to avr8js for ATmega328P (no external binary needed) - Ok(Box::new(Avr8jsRunner::new(board))) - } else if board.has_emulator("simavr") { - // Use simavr for other AVR MCUs that advertise it - Ok(Box::new(SimavrRunner::new(board))) - } else { - Err(fbuild_core::FbuildError::DeployFailed(format!( - "no emulator runner available for AVR MCU '{}'; \ - ATmega328P is supported via avr8js, other AVR boards require simavr in debug_tools", - board.mcu - ))) - } - } - fbuild_core::Platform::Espressif32 => { - if is_qemu_supported_esp32_mcu(&board.mcu) { - check_qemu_flash_mode(&board)?; - Ok(Box::new(QemuRunner::new( - project_dir.to_path_buf(), - env_name.to_string(), - board, - ))) - } else { - Err(fbuild_core::FbuildError::DeployFailed(format!( - "no emulator runner available for ESP32 MCU '{}'; \ - ESP32, ESP32-S3 (Xtensa) and ESP32-C3, ESP32-C6, ESP32-H2 (RISC-V) \ - are supported via QEMU", - board.mcu - ))) - } - } - _ => Err(fbuild_core::FbuildError::DeployFailed(format!( - "no emulator runner available for platform {:?}", - platform - ))), - } -} - -/// POST /api/test-emu handler — build firmware then run it in an emulator. -pub async fn test_emu( - State(ctx): State>, - Json(req): Json, -) -> (StatusCode, Json) { - let request_id = req - .request_id - .clone() - .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); - let project_dir = PathBuf::from(&req.project_dir); - - // Mark the daemon as busy for the full build + emulate lifecycle. - // Without this guard the 30 s self-eviction loop sees an "empty" - // daemon during long (>30 s) ESP32/QEMU builds and triggers graceful - // shutdown, which closes the in-flight HTTP connection and surfaces - // as `error sending request for url (.../api/test-emu)` on the CLI - // side. See issue #130. - let _op_guard = crate::handlers::operations::OperationGuard::new( - &ctx, - fbuild_core::DaemonState::Building, - Some(format!("test-emu {}", req.project_dir)), - ); - - if !project_dir.exists() { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("project directory does not exist: {}", req.project_dir), - )), - ); - } - - // Parse config - let config = - match fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")) { - Ok(c) => c, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("failed to parse platformio.ini: {}", e), - )), - ); - } - }; - - let env_name = req - .environment - .clone() - .or_else(|| config.get_default_environment().map(|s| s.to_string())) - .unwrap_or_else(|| "default".to_string()); - - let env_config = match config.get_env_config(&env_name) { - Ok(c) => c, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("invalid environment '{}': {}", env_name, e), - )), - ); - } - }; - - let platform_str = env_config.get("platform").cloned().unwrap_or_default(); - let platform = match fbuild_core::Platform::from_platform_str(&platform_str) { - Some(p) => p, - None => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("unsupported platform: {}", platform_str), - )), - ); - } - }; - - let board_id = env_config.get("board").cloned().unwrap_or_else(|| { - fbuild_build::get_platform_support(platform) - .map(|s| s.default_board_id().to_string()) - .unwrap_or_else(|_| "unknown".to_string()) - }); - let board_overrides = config.get_board_overrides(&env_name).unwrap_or_default(); - - // Select the emulator runner before building (fail fast on unsupported boards) - let runner = match select_runner( - &project_dir, - &env_name, - platform, - &board_id, - &board_overrides, - req.emulator.as_deref(), - ) { - Ok(r) => r, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail(request_id, e.to_string())), - ); - } - }; - - // Build firmware (hold project lock only during build, not the emulator phase) - let build_result = { - let lock = ctx.project_lock(&project_dir); - let _guard = lock.lock().await; - - let needs_qemu_flags = platform == fbuild_core::Platform::Espressif32 - && req.emulator.as_deref() != Some("avr8js"); - let board_for_flags = if needs_qemu_flags { - fbuild_config::BoardConfig::from_board_id(&board_id, &board_overrides).ok() - } else { - None - }; - - let build_dir = fbuild_paths::get_project_build_root(&project_dir); - let params = fbuild_build::BuildParams { - project_dir: project_dir.clone(), - env_name: env_name.clone(), - clean: false, - profile: fbuild_core::BuildProfile::Release, - build_dir, - verbose: req.verbose, - jobs: None, - generate_compiledb: false, - compiledb_only: false, - log_sender: None, - symbol_analysis: false, - symbol_analysis_path: None, - no_timestamp: false, - src_dir: None, - pio_env: req.pio_env.clone(), - extra_build_flags: if needs_qemu_flags { - board_for_flags - .as_ref() - .map(|b| crate::handlers::operations::qemu_extra_build_flags(platform, &b.mcu)) - .unwrap_or_default() - } else { - Vec::new() - }, - watch_set_cache: Some(std::sync::Arc::clone(&ctx.watch_set_cache) as std::sync::Arc<_>), - }; - - let p = platform; - tokio::task::spawn_blocking(move || { - let orchestrator = fbuild_build::get_orchestrator(p)?; - orchestrator.build(¶ms) - }) - .await - }; - - let (firmware_path, elf_path) = match build_result { - Ok(Ok(r)) if r.success => { - let fw = r.firmware_path.clone().unwrap_or_else(|| { - r.elf_path - .clone() - .unwrap_or_else(|| PathBuf::from("firmware.bin")) - }); - (fw, r.elf_path) - } - Ok(Ok(r)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("build failed: {}", r.message), - )), - ); - } - Ok(Err(e)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("build error: {}", e), - )), - ); - } - Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("build task panicked: {}", e), - )), - ); - } - }; - - // Run the emulator - let artifact_bundle = EmulatorArtifactBundle::from_paths(&firmware_path, elf_path.as_deref()); - let run_config = EmulatorRunConfig { - firmware_path, - elf_path, - artifact_bundle, - timeout: req.timeout, - halt_on_error: req.halt_on_error.clone(), - halt_on_success: req.halt_on_success.clone(), - expect: req.expect.clone(), - show_timestamp: req.show_timestamp, - verbose: req.verbose, - }; - - let emu_result = match runner.run(&run_config).await { - Ok(r) => r, - Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("emulator error: {}", e), - )), - ); - } - }; - - let success = emu_result.is_success(); - let exit_code = emu_result.exit_code.unwrap_or(if success { 0 } else { 1 }); - let message = format!( - "{} test-emu {}: {}", - runner.name(), - if success { "passed" } else { "failed" }, - emu_result.outcome - ); - - ( - StatusCode::OK, - Json(OperationResponse { - success, - request_id, - message, - exit_code, - output_file: None, - output_dir: None, - launch_url: None, - stdout: Some(emu_result.stdout), - stderr: Some(emu_result.stderr), - }), - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use fbuild_build::{BuildOrchestrator, BuildParams}; - use fbuild_core::BuildProfile; - - #[test] - fn monitor_outcome_to_emulator_maps_success() { - let outcome = monitor_outcome_to_emulator(MonitorOutcome::Success("ok".into()), Some(0)); - assert_eq!(outcome, EmulatorOutcome::Passed("ok".into())); - } - - #[test] - fn monitor_outcome_to_emulator_maps_error() { - let outcome = monitor_outcome_to_emulator(MonitorOutcome::Error("bad".into()), Some(1)); - assert_eq!(outcome, EmulatorOutcome::Failed("bad".into())); - } - - #[test] - fn monitor_outcome_to_emulator_maps_crash() { - let outcome = monitor_outcome_to_emulator( - MonitorOutcome::Error("abort() was called at PC 0x4200".into()), - Some(134), - ); - assert_eq!( - outcome, - EmulatorOutcome::Crashed("abort() was called at PC 0x4200".into()) - ); - } - - #[test] - fn monitor_outcome_to_emulator_maps_timeout() { - let outcome = - monitor_outcome_to_emulator(MonitorOutcome::Timeout { expect_found: true }, None); - assert_eq!(outcome, EmulatorOutcome::TimedOut { expect_found: true }); - } - - fn test_process_command(lines: &[&str]) -> (PathBuf, Vec) { - #[cfg(windows)] - { - let system_root = - std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string()); - let exe = PathBuf::from(system_root).join(r"System32\cmd.exe"); - let script = lines - .iter() - .map(|line| format!("echo {}", line)) - .collect::>() - .join(" & "); - (exe, vec!["/C".to_string(), script]) - } - - #[cfg(not(windows))] - { - let script = lines - .iter() - .map(|line| format!("printf '%s\\n' '{}'", line.replace('\'', "'\"'\"'"))) - .collect::>() - .join("; "); - (PathBuf::from("sh"), vec!["-c".to_string(), script]) - } - } - - #[tokio::test] - async fn run_qemu_process_reports_expected_success_output() { - let (exe, args) = test_process_command(&["Hello from ESP32-S3!"]); - let result = run_qemu_process( - &exe, - &args, - RunQemuOptions { - elf_path: None, - addr2line_path: None, - timeout_secs: Some(2.0), - halt_on_error: None, - halt_on_success: None, - expect: Some("Hello from ESP32-S3"), - show_timestamp: false, - verbose: false, - process_label: "QEMU", - }, - ) - .await - .unwrap(); - - assert!(result.stdout.contains("Hello from ESP32-S3!")); - match result.outcome { - MonitorOutcome::Success(message) => { - assert!(message.contains("QEMU exited normally")); - } - other => panic!("expected success outcome, got {:?}", other), - } - } - - #[tokio::test] - async fn run_qemu_process_surfaces_crash_decoder_output() { - let (exe, args) = - test_process_command(&["abort() was called at PC 0x42002a3c", "Rebooting..."]); - let result = run_qemu_process( - &exe, - &args, - RunQemuOptions { - elf_path: None, - addr2line_path: None, - timeout_secs: Some(2.0), - halt_on_error: Some("no firmware\\.elf found"), - halt_on_success: None, - expect: None, - show_timestamp: false, - verbose: false, - process_label: "QEMU", - }, - ) - .await - .unwrap(); - - assert!(result - .stdout - .contains("abort() was called at PC 0x42002a3c")); - assert!(result.stdout.contains("no firmware.elf found")); - match result.outcome { - MonitorOutcome::Error(message) => { - assert!(message.contains("halt-on-error pattern matched")); - } - other => panic!("expected error outcome, got {:?}", other), - } - } - - #[test] - #[ignore] - fn run_real_esp32s3_fixture_in_qemu() { - let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .to_path_buf(); - let project_dir = repo_root.join("tests/platform/esp32s3"); - if !project_dir.exists() { - eprintln!("SKIP: {} does not exist", project_dir.display()); - return; - } - - let build_dir = project_dir.join(".fbuild/build-qemu"); - let params = BuildParams { - project_dir: project_dir.clone(), - env_name: "esp32s3".to_string(), - clean: true, - profile: BuildProfile::Release, - build_dir, - verbose: true, - jobs: None, - generate_compiledb: false, - compiledb_only: false, - log_sender: None, - symbol_analysis: false, - symbol_analysis_path: None, - no_timestamp: false, - src_dir: None, - pio_env: Default::default(), - extra_build_flags: vec![ - "-DARDUINO_USB_MODE=0".to_string(), - "-DARDUINO_USB_CDC_ON_BOOT=0".to_string(), - ], - watch_set_cache: None, - }; - - let orchestrator = fbuild_build::esp32::orchestrator::Esp32Orchestrator; - let build_result = orchestrator - .build(¶ms) - .expect("ESP32-S3 fixture build should succeed"); - assert!(build_result.success); - - let firmware_path = build_result - .firmware_path - .clone() - .expect("should produce firmware.bin"); - let elf_path = build_result.elf_path.clone(); - - let board = - fbuild_config::BoardConfig::from_board_id("esp32-s3-devkitc-1", &Default::default()) - .unwrap(); - let mcu_config = fbuild_build::esp32::mcu_config::get_mcu_config("esp32s3").unwrap(); - let flash_size_bytes = fbuild_deploy::esp32::resolve_qemu_flash_size_bytes( - &board, - mcu_config.default_flash_size(), - ) - .unwrap(); - - let session_dir = tempfile::TempDir::new().unwrap(); - let flash_image = session_dir.path().join("flash.bin"); - fbuild_deploy::esp32::create_qemu_flash_image( - &firmware_path, - &flash_image, - flash_size_bytes, - mcu_config.bootloader_offset(), - mcu_config.partitions_offset(), - mcu_config.firmware_offset(), - elf_path.as_deref(), - ) - .unwrap(); - - let qemu = fbuild_packages::toolchain::EspQemuXtensa::new(&project_dir) - .and_then(|pkg| pkg.resolve_executable()) - .expect("native QEMU should resolve for ignored integration test"); - let args = fbuild_deploy::esp32::build_qemu_args( - &board.mcu, - &flash_image, - board.qemu_esp32_psram_config(), - ); - let addr2line_path = resolve_esp32_toolchain_gcc_path(&project_dir, &mcu_config) - .ok() - .and_then(|gcc| fbuild_serial::crash_decoder::derive_addr2line_path(&gcc)); - - let rt = tokio::runtime::Runtime::new().unwrap(); - let result = rt - .block_on(run_qemu_process( - &qemu, - &args, - RunQemuOptions { - elf_path, - addr2line_path, - timeout_secs: Some(15.0), - halt_on_error: None, - halt_on_success: Some("Hello from ESP32-S3!"), - expect: Some("Hello from ESP32-S3!"), - show_timestamp: false, - verbose: true, - process_label: "QEMU", - }, - )) - .unwrap(); - - assert!(result.stdout.contains("Hello from ESP32-S3!")); - match result.outcome { - MonitorOutcome::Success(_) => {} - other => panic!("expected success outcome, got {:?}", other), - } - } - - // ----------------------------------------------------------------------- - // avr8js headless tests (use fake process, no real Node.js needed) - // ----------------------------------------------------------------------- - - #[tokio::test] - async fn run_avr8js_headless_captures_stdout() { - let (exe, args) = test_process_command(&["Hello from AVR!"]); - // run_avr8js_headless expects (node, script, hex, f_cpu, cache_dir, options). - // We bypass that by calling the lower-level run with the fake exe directly. - // Since run_avr8js_headless builds its own command, we test the same - // subprocess loop via run_qemu_process which shares identical logic. - let result = run_qemu_process( - &exe, - &args, - RunQemuOptions { - elf_path: None, - addr2line_path: None, - timeout_secs: Some(2.0), - halt_on_error: None, - halt_on_success: None, - expect: Some("Hello from AVR"), - show_timestamp: false, - verbose: false, - process_label: "QEMU", - }, - ) - .await - .unwrap(); - - assert!(result.stdout.contains("Hello from AVR!")); - match result.outcome { - MonitorOutcome::Success(msg) => { - assert!(msg.contains("QEMU exited normally")); - } - other => panic!("expected success, got {:?}", other), - } - } - - #[tokio::test] - async fn run_avr8js_headless_halt_on_success() { - let (exe, args) = - test_process_command(&["booting...", "PASS: all tests passed", "more output"]); - let result = run_qemu_process( - &exe, - &args, - RunQemuOptions { - elf_path: None, - addr2line_path: None, - timeout_secs: Some(2.0), - halt_on_error: None, - halt_on_success: Some("PASS:"), - expect: None, - show_timestamp: false, - verbose: false, - process_label: "QEMU", - }, - ) - .await - .unwrap(); - - match result.outcome { - MonitorOutcome::Success(msg) => { - assert!(msg.contains("halt-on-success pattern matched")); - } - other => panic!("expected success, got {:?}", other), - } - } - - #[tokio::test] - async fn run_avr8js_headless_halt_on_error() { - let (exe, args) = test_process_command(&["booting...", "FAIL: assertion failed"]); - let result = run_qemu_process( - &exe, - &args, - RunQemuOptions { - elf_path: None, - addr2line_path: None, - timeout_secs: Some(2.0), - halt_on_error: Some("FAIL:"), - halt_on_success: None, - expect: None, - show_timestamp: false, - verbose: false, - process_label: "QEMU", - }, - ) - .await - .unwrap(); - - match result.outcome { - MonitorOutcome::Error(msg) => { - assert!(msg.contains("halt-on-error pattern matched")); - } - other => panic!("expected error, got {:?}", other), - } - } - - // ----------------------------------------------------------------------- - // SimAVR runner tests (use fake process, no real simavr needed) - // ----------------------------------------------------------------------- - - #[tokio::test] - async fn simavr_runner_captures_stdout_via_process_runner() { - // SimavrRunner delegates to run_qemu_process, so we verify the same - // subprocess monitoring path works for simavr-style output. - let (exe, args) = test_process_command(&["Hello from ATmega2560!"]); - let result = run_qemu_process( - &exe, - &args, - RunQemuOptions { - elf_path: None, - addr2line_path: None, - timeout_secs: Some(2.0), - halt_on_error: None, - halt_on_success: None, - expect: Some("Hello from ATmega2560"), - show_timestamp: false, - verbose: false, - process_label: "simavr", - }, - ) - .await - .unwrap(); - - assert!(result.stdout.contains("Hello from ATmega2560!")); - match result.outcome { - MonitorOutcome::Success(_) => {} - other => panic!("expected success, got {:?}", other), - } - } - - #[tokio::test] - async fn simavr_runner_halt_on_success() { - let (exe, args) = test_process_command(&["simavr: init", "PASS: all tests passed", "done"]); - let result = run_qemu_process( - &exe, - &args, - RunQemuOptions { - elf_path: None, - addr2line_path: None, - timeout_secs: Some(2.0), - halt_on_error: None, - halt_on_success: Some("PASS:"), - expect: None, - show_timestamp: false, - verbose: false, - process_label: "simavr", - }, - ) - .await - .unwrap(); - - match result.outcome { - MonitorOutcome::Success(msg) => { - assert!(msg.contains("halt-on-success pattern matched")); - } - other => panic!("expected success, got {:?}", other), - } - } - - #[tokio::test] - async fn simavr_runner_halt_on_error() { - let (exe, args) = test_process_command(&["simavr: init", "FAIL: assertion failed"]); - let result = run_qemu_process( - &exe, - &args, - RunQemuOptions { - elf_path: None, - addr2line_path: None, - timeout_secs: Some(2.0), - halt_on_error: Some("FAIL:"), - halt_on_success: None, - expect: None, - show_timestamp: false, - verbose: false, - process_label: "simavr", - }, - ) - .await - .unwrap(); - - match result.outcome { - MonitorOutcome::Error(msg) => { - assert!(msg.contains("halt-on-error pattern matched")); - } - other => panic!("expected error, got {:?}", other), - } - } - - #[tokio::test] - async fn simavr_runner_timeout() { - let (exe, args) = test_process_command(&["simavr: init", "running..."]); - let result = run_qemu_process( - &exe, - &args, - RunQemuOptions { - elf_path: None, - addr2line_path: None, - timeout_secs: Some(0.1), - halt_on_error: None, - halt_on_success: Some("PASS:"), - expect: None, - show_timestamp: false, - verbose: false, - process_label: "simavr", - }, - ) - .await - .unwrap(); - - // Process exits quickly so the outcome depends on whether halt pattern matched - // before the process ended — either timeout or a normal exit without the pattern. - match result.outcome { - MonitorOutcome::Success(_) => { - // Process exited cleanly before timeout, expect not set so counts as success - } - MonitorOutcome::Timeout { .. } => { - // Timed out waiting for halt-on-success pattern — also valid - } - MonitorOutcome::Error(_) => { - // Process exited before halt pattern found — also acceptable - } - } - } - - // ----------------------------------------------------------------------- - // select_runner tests for simavr - // ----------------------------------------------------------------------- - - #[test] - fn select_runner_explicit_simavr_for_uno() { - let result = select_runner( - Path::new("/tmp/test"), - "uno", - fbuild_core::Platform::AtmelAvr, - "uno", - &HashMap::new(), - Some("simavr"), - ); - assert!(result.is_ok(), "select_runner should accept simavr for uno"); - assert_eq!(result.unwrap().name(), "simavr"); - } - - #[test] - fn select_runner_explicit_simavr_for_mega() { - let result = select_runner( - Path::new("/tmp/test"), - "megaatmega2560", - fbuild_core::Platform::AtmelAvr, - "megaatmega2560", - &HashMap::new(), - Some("simavr"), - ); - assert!( - result.is_ok(), - "select_runner should accept simavr for mega" - ); - assert_eq!(result.unwrap().name(), "simavr"); - } - - #[test] - fn select_runner_explicit_simavr_for_leonardo() { - let result = select_runner( - Path::new("/tmp/test"), - "leonardo", - fbuild_core::Platform::AtmelAvr, - "leonardo", - &HashMap::new(), - Some("simavr"), - ); - assert!( - result.is_ok(), - "select_runner should accept simavr for leonardo" - ); - assert_eq!(result.unwrap().name(), "simavr"); - } - - #[test] - fn select_runner_explicit_simavr_rejects_esp32() { - let result = select_runner( - Path::new("/tmp/test"), - "esp32dev", - fbuild_core::Platform::Espressif32, - "esp32dev", - &HashMap::new(), - Some("simavr"), - ); - assert!(result.is_err(), "simavr should reject ESP32 boards"); - } - - #[test] - fn select_runner_auto_detects_simavr_for_mega() { - // ATmega2560 should auto-detect simavr since it's not ATmega328P - let result = select_runner( - Path::new("/tmp/test"), - "megaatmega2560", - fbuild_core::Platform::AtmelAvr, - "megaatmega2560", - &HashMap::new(), - None, - ); - assert!( - result.is_ok(), - "auto-detect should find simavr for mega: {:?}", - result.err() - ); - assert_eq!(result.unwrap().name(), "simavr"); - } - - #[test] - fn select_runner_auto_detects_simavr_for_leonardo() { - let result = select_runner( - Path::new("/tmp/test"), - "leonardo", - fbuild_core::Platform::AtmelAvr, - "leonardo", - &HashMap::new(), - None, - ); - assert!( - result.is_ok(), - "auto-detect should find simavr for leonardo: {:?}", - result.err() - ); - assert_eq!(result.unwrap().name(), "simavr"); - } - - #[test] - fn select_runner_auto_detects_avr8js_for_uno() { - // ATmega328P should still default to avr8js, not simavr - let result = select_runner( - Path::new("/tmp/test"), - "uno", - fbuild_core::Platform::AtmelAvr, - "uno", - &HashMap::new(), - None, - ); - assert!(result.is_ok()); - assert_eq!( - result.unwrap().name(), - "avr8js ATmega328P", - "ATmega328P should default to avr8js" - ); - } - - #[test] - fn select_runner_simavr_name_matches() { - let board = - fbuild_config::BoardConfig::from_board_id("megaatmega2560", &HashMap::new()).unwrap(); - let runner = SimavrRunner::new(board); - assert_eq!(runner.name(), "simavr"); - } - - // ----------------------------------------------------------------------- - // select_runner tests for ESP32 QEMU (Issue #25) - // ----------------------------------------------------------------------- - - #[test] - fn select_runner_explicit_qemu_for_esp32dev() { - let result = select_runner( - Path::new("/tmp/test"), - "esp32dev", - fbuild_core::Platform::Espressif32, - "esp32dev", - &HashMap::new(), - Some("qemu"), - ); - assert!( - result.is_ok(), - "select_runner should accept qemu for esp32dev: {:?}", - result.err() - ); - assert_eq!(result.unwrap().name(), "QEMU ESP32"); - } - - #[test] - fn select_runner_explicit_qemu_for_esp32s3() { - let result = select_runner( - Path::new("/tmp/test"), - "esp32-s3-devkitc-1", - fbuild_core::Platform::Espressif32, - "esp32-s3-devkitc-1", - &HashMap::new(), - Some("qemu"), - ); - assert!( - result.is_ok(), - "select_runner should accept qemu for esp32-s3: {:?}", - result.err() - ); - assert_eq!(result.unwrap().name(), "QEMU ESP32S3"); - } - - #[test] - fn select_runner_explicit_qemu_accepts_esp32c3() { - let result = select_runner( - Path::new("/tmp/test"), - "esp32-c3-devkitm-1", - fbuild_core::Platform::Espressif32, - "esp32-c3-devkitm-1", - &HashMap::new(), - Some("qemu"), - ); - assert!( - result.is_ok(), - "QEMU should accept ESP32-C3 via qemu-system-riscv32: {:?}", - result.err() - ); - } - - #[test] - fn select_runner_explicit_qemu_accepts_esp32c6() { - let result = select_runner( - Path::new("/tmp/test"), - "esp32-c6-devkitc-1", - fbuild_core::Platform::Espressif32, - "esp32-c6-devkitc-1", - &HashMap::new(), - Some("qemu"), - ); - assert!( - result.is_ok(), - "QEMU should accept ESP32-C6 via qemu-system-riscv32: {:?}", - result.err() - ); - } - - #[test] - fn select_runner_explicit_qemu_accepts_esp32h2() { - let result = select_runner( - Path::new("/tmp/test"), - "esp32-h2-devkitm-1", - fbuild_core::Platform::Espressif32, - "esp32-h2-devkitm-1", - &HashMap::new(), - Some("qemu"), - ); - assert!( - result.is_ok(), - "QEMU should accept ESP32-H2 via qemu-system-riscv32: {:?}", - result.err() - ); - } - - #[test] - fn select_runner_explicit_qemu_rejects_esp32s2() { - let result = select_runner( - Path::new("/tmp/test"), - "esp32-s2-saola-1", - fbuild_core::Platform::Espressif32, - "esp32-s2-saola-1", - &HashMap::new(), - Some("qemu"), - ); - assert!( - result.is_err(), - "QEMU should reject ESP32-S2 (not emulated upstream)" - ); - } - - #[test] - fn select_runner_auto_detects_qemu_for_esp32dev() { - let result = select_runner( - Path::new("/tmp/test"), - "esp32dev", - fbuild_core::Platform::Espressif32, - "esp32dev", - &HashMap::new(), - None, - ); - assert!( - result.is_ok(), - "auto-detect should find qemu for esp32dev: {:?}", - result.err() - ); - assert_eq!(result.unwrap().name(), "QEMU ESP32"); - } - - #[test] - fn select_runner_auto_detects_qemu_for_esp32s3() { - let result = select_runner( - Path::new("/tmp/test"), - "esp32-s3-devkitc-1", - fbuild_core::Platform::Espressif32, - "esp32-s3-devkitc-1", - &HashMap::new(), - None, - ); - assert!( - result.is_ok(), - "auto-detect should find qemu for esp32-s3: {:?}", - result.err() - ); - assert_eq!(result.unwrap().name(), "QEMU ESP32S3"); - } - - #[test] - fn is_qemu_supported_esp32_mcu_accepts_xtensa() { - assert!(is_qemu_supported_esp32_mcu("esp32")); - assert!(is_qemu_supported_esp32_mcu("ESP32")); - assert!(is_qemu_supported_esp32_mcu("esp32s3")); - assert!(is_qemu_supported_esp32_mcu("ESP32S3")); - } - - #[test] - fn is_qemu_supported_esp32_mcu_accepts_riscv() { - assert!(is_qemu_supported_esp32_mcu("esp32c3")); - assert!(is_qemu_supported_esp32_mcu("ESP32C3")); - assert!(is_qemu_supported_esp32_mcu("esp32c6")); - assert!(is_qemu_supported_esp32_mcu("esp32h2")); - } - - #[test] - fn is_qemu_supported_esp32_mcu_rejects_unsupported() { - assert!(!is_qemu_supported_esp32_mcu("esp32s2")); - assert!(!is_qemu_supported_esp32_mcu("atmega328p")); - assert!(!is_qemu_supported_esp32_mcu("esp32p4")); - } - - // ----------------------------------------------------------------------- - // avr8js npm cache integrity tests (issue #86) - // ----------------------------------------------------------------------- - - /// Serialises tests that mutate process-wide env vars (PATH). Without - /// this, parallel cargo-test workers would clobber each other's PATH. - fn env_lock() -> std::sync::MutexGuard<'static, ()> { - use std::sync::{Mutex, OnceLock}; - static LOCK: OnceLock> = OnceLock::new(); - LOCK.get_or_init(|| Mutex::new(())) - .lock() - .unwrap_or_else(|p| p.into_inner()) - } - - #[test] - fn avr8js_cache_is_intact_detects_missing_dir() { - let tmp = tempfile::TempDir::new().unwrap(); - // Empty cache dir: no node_modules at all. - assert!(!avr8js_cache_is_intact(tmp.path())); - } - - #[test] - fn avr8js_cache_is_intact_detects_missing_package_json() { - let tmp = tempfile::TempDir::new().unwrap(); - // Simulate a partial/corrupt install: dir exists, but no package.json. - std::fs::create_dir_all(tmp.path().join("node_modules").join("avr8js")).unwrap(); - assert!( - !avr8js_cache_is_intact(tmp.path()), - "corrupt install (no package.json) must not count as intact" - ); - } - - #[test] - fn avr8js_cache_is_intact_accepts_marker_present() { - let tmp = tempfile::TempDir::new().unwrap(); - let module_dir = tmp.path().join("node_modules").join("avr8js"); - std::fs::create_dir_all(&module_dir).unwrap(); - std::fs::write(module_dir.join("package.json"), b"{\"name\":\"avr8js\"}").unwrap(); - assert!(avr8js_cache_is_intact(tmp.path())); - } - - #[test] - fn prepare_avr8js_cache_skips_when_intact() { - let tmp = tempfile::TempDir::new().unwrap(); - let module_dir = tmp.path().join("node_modules").join("avr8js"); - std::fs::create_dir_all(&module_dir).unwrap(); - std::fs::write(module_dir.join("package.json"), b"{}").unwrap(); - assert_eq!( - prepare_avr8js_cache_for_install(tmp.path(), false), - Avr8jsCachePrep::AlreadyIntact - ); - // Tree must survive untouched. - assert!(module_dir.join("package.json").exists()); - } - - #[test] - fn prepare_avr8js_cache_wipes_node_modules_when_corrupt() { - let tmp = tempfile::TempDir::new().unwrap(); - let node_modules = tmp.path().join("node_modules"); - let module_dir = node_modules.join("avr8js"); - std::fs::create_dir_all(&module_dir).unwrap(); - // Stray partial file inside an otherwise package.json-less install. - std::fs::write(module_dir.join("index.mjs"), b"partial").unwrap(); - assert!(!avr8js_cache_is_intact(tmp.path())); - - let prep = prepare_avr8js_cache_for_install(tmp.path(), false); - assert_eq!(prep, Avr8jsCachePrep::WipedNodeModules); - assert!( - !node_modules.exists(), - "corrupt node_modules tree must be wiped before reinstall" - ); - } - - #[test] - fn prepare_avr8js_cache_force_refresh_wipes_entire_dir() { - let tmp_parent = tempfile::TempDir::new().unwrap(); - let cache = tmp_parent.path().join("avr8js-node"); - let module_dir = cache.join("node_modules").join("avr8js"); - std::fs::create_dir_all(&module_dir).unwrap(); - std::fs::write(module_dir.join("package.json"), b"{}").unwrap(); - // Even a fully-intact install must be wiped under force_refresh=true. - let prep = prepare_avr8js_cache_for_install(&cache, true); - assert_eq!(prep, Avr8jsCachePrep::ForceWiped); - assert!(!cache.exists(), "force-refresh must remove the cache dir"); - } - - #[test] - fn prepare_avr8js_cache_nothing_to_clean_on_fresh_path() { - let tmp_parent = tempfile::TempDir::new().unwrap(); - let cache = tmp_parent.path().join("avr8js-node"); // doesn't exist yet - assert_eq!( - prepare_avr8js_cache_for_install(&cache, false), - Avr8jsCachePrep::NothingToClean - ); - } - - #[test] - fn refresh_emu_cache_requested_recognises_truthy_values() { - let _guard = env_lock(); - for (val, expected) in [ - ("1", true), - ("true", true), - ("TRUE", true), - ("yes", true), - ("YES", true), - ("0", false), - ("", false), - ("no", false), - ] { - std::env::set_var(REFRESH_EMU_CACHE_ENV, val); - assert_eq!( - refresh_emu_cache_requested(), - expected, - "{}={:?} should parse as {}", - REFRESH_EMU_CACHE_ENV, - val, - expected - ); - } - std::env::remove_var(REFRESH_EMU_CACHE_ENV); - assert!(!refresh_emu_cache_requested()); - } - - /// When npm isn't on PATH, `ensure_avr8js_npm_in` must return an - /// `FbuildError::DeployFailed` that names both `npm` and the cache dir. - /// This is the fix for issue #86's silent `ERR_MODULE_NOT_FOUND`. - #[test] - fn ensure_avr8js_npm_in_reports_clear_error_without_npm() { - let _guard = env_lock(); - let saved_path = std::env::var_os("PATH"); - // PATHEXT matters on Windows for command resolution of .cmd files. - let saved_pathext = std::env::var_os("PATHEXT"); - - std::env::set_var("PATH", ""); - #[cfg(windows)] - { - // Ensure .cmd isn't resolved via some fallback. - std::env::set_var("PATHEXT", ""); - } - - let tmp = tempfile::TempDir::new().unwrap(); - let cache = tmp.path().join("avr8js-node"); - let result = ensure_avr8js_npm_in(&cache, false); - - // Restore BEFORE asserting so a panic doesn't leak PATH="" to sibling tests. - if let Some(p) = saved_path { - std::env::set_var("PATH", p); - } else { - std::env::remove_var("PATH"); - } - if let Some(p) = saved_pathext { - std::env::set_var("PATHEXT", p); - } else { - std::env::remove_var("PATHEXT"); - } - - let err = result.expect_err("npm must be unresolvable with PATH=\"\""); - let msg = err.to_string(); - assert!( - msg.contains("npm"), - "error message must mention 'npm'; got: {}", - msg - ); - assert!( - msg.contains(&cache.display().to_string()), - "error message must include cache dir path; got: {}", - msg - ); - assert!( - msg.contains(REFRESH_EMU_CACHE_ENV), - "error message must reference {} for recovery; got: {}", - REFRESH_EMU_CACHE_ENV, - msg - ); - } - - /// When the cache dir contains a corrupt partial install, the reinstall - /// path must fire (detected here by asserting the partial tree is wiped - /// even when the downstream npm call subsequently fails). - #[test] - fn ensure_avr8js_npm_in_wipes_corrupt_before_reinstall() { - let _guard = env_lock(); - let saved_path = std::env::var_os("PATH"); - - // Force the npm spawn to fail so we isolate the wipe behaviour. - std::env::set_var("PATH", ""); - - let tmp = tempfile::TempDir::new().unwrap(); - let cache = tmp.path().join("avr8js-node"); - let node_modules = cache.join("node_modules"); - let module_dir = node_modules.join("avr8js"); - std::fs::create_dir_all(&module_dir).unwrap(); - // Deliberately omit package.json → corrupt. - std::fs::write(module_dir.join("garbage"), b"partial").unwrap(); - assert!(!avr8js_cache_is_intact(&cache)); - - let result = ensure_avr8js_npm_in(&cache, false); - - if let Some(p) = saved_path { - std::env::set_var("PATH", p); - } else { - std::env::remove_var("PATH"); - } - - // npm spawn must fail (no PATH), but the corrupt tree must be gone. - assert!(result.is_err(), "empty PATH should prevent npm install"); - assert!( - !node_modules.exists(), - "corrupt node_modules must be wiped before reinstall attempt" - ); - } -} diff --git a/crates/fbuild-daemon/src/handlers/emulator/README.md b/crates/fbuild-daemon/src/handlers/emulator/README.md new file mode 100644 index 00000000..58dbcaf7 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/README.md @@ -0,0 +1,17 @@ +# Emulator Handlers + +`POST /api/deploy` emulator targets, `POST /api/test-emu`, and the `EmulatorRunner` abstraction. Split into focused submodules to stay under the 900-LOC per-file gate. + +- **`mod.rs`** -- Submodule declarations and public re-exports preserving `handlers::emulator::*` paths. +- **`shared.rs`** -- Streaming subprocess runner (`run_qemu_process`), `ProcessEvent`/`RunQemuOptions`/`QemuRunResult`, `EmulatorRunConfig`, `monitor_outcome_to_emulator`, `qemu_session_dir`, ESP32 toolchain GCC resolution. +- **`avr8js_web.rs`** -- Browser-side HTML/JS/session.json/firmware.hex handlers and the `Avr8jsSessionManifest` type. +- **`avr8js_npm.rs`** -- `find_node`, `ensure_avr8js_npm[_in]`, `Avr8jsCachePrep`, and the `FBUILD_REFRESH_EMU_CACHE` env-var helpers. +- **`avr8js_headless.rs`** -- `run_avr8js_headless` (Node.js subprocess streaming) plus `AVR8JS_HEADLESS_MJS`. +- **`avr8js_deploy.rs`** -- `DeployAvr8jsRequest` and the `deploy_avr8js` handler (stages firmware, optionally runs headless). +- **`qemu_deploy.rs`** -- `DeployQemuRequest`, the `deploy_qemu` handler, `resolve_esp_qemu_for_mcu`, `check_qemu_flash_mode`, `is_qemu_supported_esp32_mcu`. +- **`runners.rs`** -- `EmulatorRunner` trait and concrete `QemuRunner` / `Avr8jsRunner` / `SimavrRunner` impls (plus `find_simavr`). +- **`select.rs`** -- `select_runner` and the `test_emu` build-then-emulate handler. +- **`tests_outcome.rs`** -- `monitor_outcome_to_emulator` mapping tests. +- **`tests_process.rs`** -- Subprocess runner tests, an ignored ESP32-S3 fixture integration test, and simavr/avr8js streaming tests. +- **`tests_select_runner.rs`** -- `select_runner` and `is_qemu_supported_esp32_mcu` coverage. +- **`tests_npm_cache.rs`** -- avr8js npm cache integrity and `ensure_avr8js_npm_in` error-message tests. diff --git a/crates/fbuild-daemon/src/handlers/emulator/avr8js_deploy.rs b/crates/fbuild-daemon/src/handlers/emulator/avr8js_deploy.rs new file mode 100644 index 00000000..b5bd43c5 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/avr8js_deploy.rs @@ -0,0 +1,321 @@ +//! `POST /api/deploy` (avr8js variant) — stages firmware for an in-browser +//! AVR8js session, optionally executes a headless run, and returns either a +//! launch URL or captured stdout/stderr. + +use super::avr8js_headless::{run_avr8js_headless, RunAvr8jsHeadlessOptions, AVR8JS_HEADLESS_MJS}; +use super::avr8js_npm::{ensure_avr8js_npm, find_node}; +use super::avr8js_web::{now_unix, Avr8jsSessionManifest}; +use crate::context::DaemonContext; +use crate::handlers::operations::MonitorOutcome; +use crate::models::OperationResponse; +use axum::http::StatusCode; +use axum::Json; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; + +pub struct DeployAvr8jsRequest { + pub request_id: String, + pub project_dir: PathBuf, + pub env_name: String, + pub board_id: String, + pub platform: fbuild_core::Platform, + pub firmware_path: PathBuf, + pub elf_path: Option, + pub monitor_after: bool, + pub output_file: String, + pub output_dir: Option, + pub monitor_timeout: Option, + pub halt_on_error: Option, + pub halt_on_success: Option, + pub expect: Option, + pub show_timestamp: bool, + pub verbose: bool, +} + +pub async fn deploy_avr8js( + ctx: Arc, + req: DeployAvr8jsRequest, +) -> (StatusCode, Json) { + let DeployAvr8jsRequest { + request_id, + project_dir, + env_name, + board_id, + platform, + firmware_path, + elf_path, + monitor_after, + output_file, + output_dir, + monitor_timeout, + halt_on_error, + halt_on_success, + expect, + show_timestamp, + verbose, + } = req; + + if !matches!( + platform, + fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr + ) { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + "avr8js deploy target is only supported for AVR boards".to_string(), + )), + ); + } + if board_id != "uno" { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!( + "avr8js deploy target currently supports only board 'uno' (got '{}')", + board_id + ), + )), + ); + } + if firmware_path.extension().and_then(|ext| ext.to_str()) != Some("hex") { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!( + "avr8js deploy target requires firmware.hex, got '{}'", + firmware_path.display() + ), + )), + ); + } + + let board = match fbuild_config::BoardConfig::from_board_id(&board_id, &HashMap::new()) { + Ok(board) => board, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("failed to load AVR8js board config: {}", e), + )), + ); + } + }; + if !board.mcu.eq_ignore_ascii_case("atmega328p") { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!( + "avr8js deploy target currently supports only ATmega328P, got '{}'", + board.mcu + ), + )), + ); + } + + let session_id = uuid::Uuid::new_v4().to_string(); + let session_dir = fbuild_paths::get_project_fbuild_dir(&project_dir) + .join("emulators") + .join("avr8js") + .join(&env_name) + .join(&session_id); + if let Err(e) = std::fs::create_dir_all(&session_dir) { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("failed to create AVR8js session dir: {}", e), + )), + ); + } + + let staged_hex = session_dir.join("firmware.hex"); + if let Err(e) = std::fs::copy(&firmware_path, &staged_hex) { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("failed to stage AVR8js firmware.hex: {}", e), + )), + ); + } + + let staged_elf = if let Some(ref elf) = elf_path { + let dest = session_dir.join("firmware.elf"); + match std::fs::copy(elf, &dest) { + Ok(_) => Some(dest), + Err(_) => None, + } + } else { + None + }; + + let manifest = Avr8jsSessionManifest { + session_id: session_id.clone(), + project_dir: project_dir.to_string_lossy().to_string(), + env_name: env_name.clone(), + board_id, + platform: format!("{:?}", platform), + mcu: board.mcu.clone(), + f_cpu_hz: board + .f_cpu + .trim_end_matches('L') + .parse::() + .unwrap_or(16_000_000), + firmware_hex: staged_hex.to_string_lossy().to_string(), + firmware_elf: staged_elf.map(|p| p.to_string_lossy().to_string()), + created_at_unix: now_unix(), + }; + let manifest_path = session_dir.join("session.json"); + if let Err(e) = std::fs::write( + &manifest_path, + serde_json::to_vec_pretty(&manifest).unwrap_or_default(), + ) { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("failed to write AVR8js session manifest: {}", e), + )), + ); + } + ctx.avr8js_sessions + .insert(session_id.clone(), manifest_path); + + if monitor_after { + // Headless path: run avr8js in Node.js subprocess, capture UART on stdout + let node_path = match find_node() { + Ok(p) => p, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail(request_id, e.to_string())), + ); + } + }; + let avr8js_cache = match ensure_avr8js_npm() { + Ok(p) => p, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail(request_id, e.to_string())), + ); + } + }; + + let script_path = session_dir.join("headless.mjs"); + if let Err(e) = std::fs::write(&script_path, AVR8JS_HEADLESS_MJS) { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("failed to write headless.mjs: {}", e), + )), + ); + } + + let avr8js_result = match run_avr8js_headless( + &node_path, + &script_path, + &staged_hex, + manifest.f_cpu_hz, + &avr8js_cache, + RunAvr8jsHeadlessOptions { + timeout_secs: monitor_timeout, + halt_on_error: halt_on_error.as_deref(), + halt_on_success: halt_on_success.as_deref(), + expect: expect.as_deref(), + show_timestamp, + verbose, + }, + ) + .await + { + Ok(r) => r, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail(request_id, e.to_string())), + ); + } + }; + + match avr8js_result.outcome { + MonitorOutcome::Success(message) => ( + StatusCode::OK, + Json(OperationResponse { + success: true, + request_id, + message: format!("avr8js run succeeded: {}", message), + exit_code: 0, + output_file: Some(output_file), + output_dir, + launch_url: None, + stdout: Some(avr8js_result.stdout), + stderr: Some(avr8js_result.stderr), + }), + ), + MonitorOutcome::Error(message) => ( + StatusCode::OK, + Json(OperationResponse { + success: false, + request_id, + message: format!("avr8js run failed: {}", message), + exit_code: 1, + output_file: Some(output_file), + output_dir, + launch_url: None, + stdout: Some(avr8js_result.stdout), + stderr: Some(avr8js_result.stderr), + }), + ), + MonitorOutcome::Timeout { expect_found } => { + let success = expect.is_none() || expect_found; + let exit_code = if success { 0 } else { 1 }; + ( + StatusCode::OK, + Json(OperationResponse { + success, + request_id, + message: if success { + "avr8js run completed (timeout)".to_string() + } else { + "avr8js run timed out (expected pattern not found)".to_string() + }, + exit_code, + output_file: Some(output_file), + output_dir, + launch_url: None, + stdout: Some(avr8js_result.stdout), + stderr: Some(avr8js_result.stderr), + }), + ) + } + } + } else { + // Browser path: return URL for the avr8js web UI + let launch_url = Some(format!( + "http://127.0.0.1:{}/emulator/avr8js/{}", + ctx.port, session_id + )); + ( + StatusCode::OK, + Json(OperationResponse { + success: true, + request_id, + message: "deploy complete".to_string(), + exit_code: 0, + output_file: Some(output_file), + output_dir, + launch_url, + stdout: None, + stderr: None, + }), + ) + } +} diff --git a/crates/fbuild-daemon/src/handlers/emulator/avr8js_headless.rs b/crates/fbuild-daemon/src/handlers/emulator/avr8js_headless.rs new file mode 100644 index 00000000..c74d49f8 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/avr8js_headless.rs @@ -0,0 +1,209 @@ +//! Headless AVR8js runner: spawns Node.js with the bundled `headless.mjs` +//! shim and streams the simulated UART output through the same monitor +//! state machine the real serial pipeline uses. + +use super::avr8js_npm::{avr8js_cache_is_intact, REFRESH_EMU_CACHE_ENV}; +use super::shared::{spawn_line_reader, ProcessEvent}; +use crate::handlers::operations::{MonitorOutcome, MonitorState}; +use std::path::Path; +use std::process::Stdio; + +pub(crate) const AVR8JS_HEADLESS_MJS: &str = include_str!("../../../web/avr8js/headless.mjs"); + +pub(crate) struct Avr8jsRunResult { + pub outcome: MonitorOutcome, + pub stdout: String, + pub stderr: String, + pub exit_code: Option, +} + +pub(crate) struct RunAvr8jsHeadlessOptions<'a> { + pub timeout_secs: Option, + pub halt_on_error: Option<&'a str>, + pub halt_on_success: Option<&'a str>, + pub expect: Option<&'a str>, + pub show_timestamp: bool, + pub verbose: bool, +} + +pub(crate) async fn run_avr8js_headless( + node_path: &Path, + script_path: &Path, + hex_path: &Path, + f_cpu_hz: u32, + avr8js_cache_dir: &Path, + options: RunAvr8jsHeadlessOptions<'_>, +) -> fbuild_core::Result { + // Fail-fast: re-verify the cache is intact before we spawn Node. This + // guards against race conditions (cache wiped between `ensure_avr8js_npm` + // and this call) and makes the error actionable instead of a cryptic + // Node `ERR_MODULE_NOT_FOUND` stack trace (issue #86). + if !avr8js_cache_is_intact(avr8js_cache_dir) { + tracing::error!( + "avr8js cache not populated at {}; aborting (expected \ + node_modules/avr8js/package.json). Set {}=1 to force reinstall.", + avr8js_cache_dir.display(), + REFRESH_EMU_CACHE_ENV + ); + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "avr8js cache not populated at {} (missing \ + node_modules/avr8js/package.json). \ + Set {}=1 to force reinstall.", + avr8js_cache_dir.display(), + REFRESH_EMU_CACHE_ENV + ))); + } + + // allow-direct-spawn: tokio streaming emulator; blocking NativeProcess unsuitable. + let mut cmd = tokio::process::Command::new(node_path); + cmd.arg(script_path) + .arg("--hex") + .arg(hex_path) + .arg("--f-cpu") + .arg(f_cpu_hz.to_string()) + .env("NODE_PATH", avr8js_cache_dir.join("node_modules")) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + const CREATE_NO_WINDOW: u32 = 0x08000000; + cmd.creation_flags(CREATE_NO_WINDOW); + } + + if options.verbose { + tracing::info!( + "avr8js headless: {} {} --hex {} --f-cpu {}", + node_path.display(), + script_path.display(), + hex_path.display(), + f_cpu_hz + ); + } + + // Route through containment (#32) so a daemon crash mid-emulation + // takes node.exe and any helper processes it spawned with it. + let mut child = + fbuild_core::containment::tokio_spawn::spawn_contained(&mut cmd).map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!( + "failed to launch Node.js for avr8js: {}", + e + )) + })?; + + let stdout = child.stdout.take().ok_or_else(|| { + fbuild_core::FbuildError::DeployFailed("failed to capture avr8js stdout".to_string()) + })?; + let stderr = child.stderr.take().ok_or_else(|| { + fbuild_core::FbuildError::DeployFailed("failed to capture avr8js stderr".to_string()) + })?; + + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); + let stdout_task = tokio::spawn(spawn_line_reader(stdout, false, tx.clone())); + let stderr_task = tokio::spawn(spawn_line_reader(stderr, true, tx)); + + let mut monitor = MonitorState::new( + options.timeout_secs, + options.halt_on_error, + options.halt_on_success, + options.expect, + options.show_timestamp, + ); + let mut stdout_buf = String::new(); + let mut stderr_buf = String::new(); + let mut streams_open = 2usize; + let mut child_exit: Option = None; + let mut final_outcome: Option = None; + + loop { + if monitor.timed_out() { + final_outcome = Some(monitor.timeout_outcome()); + let _ = child.kill().await; + break; + } + + let recv_timeout = monitor + .remaining() + .unwrap_or(std::time::Duration::from_secs(1)); + + tokio::select! { + status = child.wait(), if child_exit.is_none() => { + child_exit = Some(status.map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!("avr8js wait failed: {}", e)) + })?); + if streams_open == 0 { + break; + } + } + maybe_event = tokio::time::timeout(recv_timeout, rx.recv()) => { + match maybe_event { + Ok(Some(ProcessEvent::Line(line))) => { + let target = if line.is_stderr { &mut stderr_buf } else { &mut stdout_buf }; + target.push_str(&line.line); + target.push('\n'); + + if let Some(outcome) = monitor.process_line(&line.line) { + final_outcome = Some(outcome); + let _ = child.kill().await; + break; + } + } + Ok(Some(ProcessEvent::StreamClosed)) => { + streams_open = streams_open.saturating_sub(1); + if streams_open == 0 && child_exit.is_some() { + break; + } + } + Ok(None) => { + if child_exit.is_some() { + break; + } + } + Err(_) => { + final_outcome = Some(monitor.timeout_outcome()); + let _ = child.kill().await; + break; + } + } + } + } + } + + if child_exit.is_none() { + child_exit = Some(child.wait().await.map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!("avr8js wait failed: {}", e)) + })?); + } + + let _ = stdout_task.await; + let _ = stderr_task.await; + + let outcome = if let Some(outcome) = final_outcome { + outcome + } else if let Some(status) = child_exit { + if status.success() { + if options.expect.is_some() && !monitor.expect_found() { + MonitorOutcome::Error( + "avr8js exited before the expected pattern was found".to_string(), + ) + } else { + MonitorOutcome::Success("avr8js exited normally".to_string()) + } + } else { + MonitorOutcome::Error(format!( + "avr8js exited with code {}", + status.code().unwrap_or(-1) + )) + } + } else { + MonitorOutcome::Error("avr8js exited unexpectedly".to_string()) + }; + + Ok(Avr8jsRunResult { + outcome, + stdout: stdout_buf, + stderr: stderr_buf, + exit_code: child_exit.and_then(|s| s.code()), + }) +} diff --git a/crates/fbuild-daemon/src/handlers/emulator/avr8js_npm.rs b/crates/fbuild-daemon/src/handlers/emulator/avr8js_npm.rs new file mode 100644 index 00000000..0cf2a0b4 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/avr8js_npm.rs @@ -0,0 +1,192 @@ +//! Manage the cached `node_modules/avr8js` install used by the headless AVR8js +//! runner. Detects corrupt/partial installs, optionally force-refreshes (env +//! `FBUILD_REFRESH_EMU_CACHE=1`), and surfaces actionable errors when `node` / +//! `npm` aren't on PATH. + +use std::path::{Path, PathBuf}; + +pub(crate) fn find_node() -> fbuild_core::Result { + let node = if cfg!(windows) { "node.exe" } else { "node" }; + // Route through fbuild-core's `run_command` so the probe spawn is + // captured by the daemon's containment group (issue #32). The probe + // is short-lived (`node --version`) but a missing binary should + // still bubble up the same way. + match fbuild_core::subprocess::run_command(&[node, "--version"], None, None, None) { + Ok(output) if output.success() => Ok(PathBuf::from(node)), + _ => Err(fbuild_core::FbuildError::DeployFailed( + "Node.js is required for headless avr8js emulation but 'node' was not found on PATH. \ + Install Node.js 18+ from https://nodejs.org/" + .to_string(), + )), + } +} + +/// Env-var that forces `ensure_avr8js_npm` to wipe and reinstall the cache +/// regardless of the integrity marker. Set `FBUILD_REFRESH_EMU_CACHE=1` before +/// invoking `fbuild test-emu` (or restart the daemon with it in the env) to +/// recover from a corrupt or partial avr8js install. +pub(crate) const REFRESH_EMU_CACHE_ENV: &str = "FBUILD_REFRESH_EMU_CACHE"; + +pub(crate) fn refresh_emu_cache_requested() -> bool { + matches!( + std::env::var(REFRESH_EMU_CACHE_ENV) + .ok() + .as_deref() + .map(str::trim), + Some("1") | Some("true") | Some("TRUE") | Some("yes") | Some("YES") + ) +} + +pub(crate) fn avr8js_cache_is_intact(cache_dir: &Path) -> bool { + // Cheapest proof of a non-corrupt install: both the `avr8js` module dir + // *and* its `package.json` must exist. A stale/partial `npm install` + // sometimes leaves the directory without the manifest and causes + // Node to fail with `ERR_MODULE_NOT_FOUND` at runtime. + let module_dir = cache_dir.join("node_modules").join("avr8js"); + let marker = module_dir.join("package.json"); + module_dir.is_dir() && marker.is_file() +} + +/// Describes what `prepare_avr8js_cache_for_install` did to the cache dir +/// before a reinstall attempt. Exposed for unit testing; production code +/// only needs the side-effect on disk. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Avr8jsCachePrep { + /// Cache was already intact — no-op, reinstall not required. + AlreadyIntact, + /// No preexisting cache tree; reinstall will populate a fresh dir. + NothingToClean, + /// `node_modules/` was present but corrupt (missing marker); wiped. + WipedNodeModules, + /// Force-refresh was requested; entire `cache_dir` was wiped. + ForceWiped, +} + +/// Inspect `cache_dir`, wipe corrupt/partial installs, and return what was +/// done. Does NOT run `npm install`. +pub(crate) fn prepare_avr8js_cache_for_install( + cache_dir: &Path, + force_refresh: bool, +) -> Avr8jsCachePrep { + if !force_refresh && avr8js_cache_is_intact(cache_dir) { + return Avr8jsCachePrep::AlreadyIntact; + } + + if force_refresh && cache_dir.exists() { + tracing::info!( + "{}=1 set; wiping avr8js cache at {}", + REFRESH_EMU_CACHE_ENV, + cache_dir.display() + ); + if let Err(e) = std::fs::remove_dir_all(cache_dir) { + tracing::warn!( + "failed to wipe avr8js cache at {}: {} (continuing with reinstall)", + cache_dir.display(), + e + ); + } + return Avr8jsCachePrep::ForceWiped; + } + + if cache_dir.exists() { + let node_modules = cache_dir.join("node_modules"); + if node_modules.exists() { + tracing::warn!( + "avr8js cache at {} is corrupt (missing node_modules/avr8js/package.json); reinstalling", + cache_dir.display() + ); + if let Err(e) = std::fs::remove_dir_all(&node_modules) { + tracing::warn!( + "failed to wipe avr8js node_modules at {}: {} (continuing with reinstall)", + node_modules.display(), + e + ); + } + return Avr8jsCachePrep::WipedNodeModules; + } + } + + Avr8jsCachePrep::NothingToClean +} + +pub(crate) fn ensure_avr8js_npm() -> fbuild_core::Result { + let cache_dir = fbuild_paths::get_cache_root().join("avr8js-node"); + ensure_avr8js_npm_in(&cache_dir, refresh_emu_cache_requested())?; + Ok(cache_dir) +} + +/// Populate `cache_dir` with a fresh `node_modules/avr8js` install, wiping +/// a corrupt or partial install as needed. Split out from `ensure_avr8js_npm` +/// so unit tests can inject a temporary cache dir without touching env vars. +pub(crate) fn ensure_avr8js_npm_in( + cache_dir: &Path, + force_refresh: bool, +) -> fbuild_core::Result<()> { + if prepare_avr8js_cache_for_install(cache_dir, force_refresh) == Avr8jsCachePrep::AlreadyIntact + { + return Ok(()); + } + + std::fs::create_dir_all(cache_dir).map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!( + "failed to create avr8js cache dir at {}: {}", + cache_dir.display(), + e + )) + })?; + + let npm = if cfg!(windows) { "npm.cmd" } else { "npm" }; + // Route through `run_command` (which spawns via the daemon's + // containment group) so an `npm install` killed mid-flight doesn't + // leak node processes after the daemon dies. See FastLED/fbuild#32. + let cache_dir_str = cache_dir.to_string_lossy().to_string(); + let output = fbuild_core::subprocess::run_command( + &[ + npm, + "install", + "--save", + "avr8js@0.21.0", + "--prefix", + &cache_dir_str, + ], + None, + None, + None, + ) + .map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!( + "failed to launch 'npm' (for `npm install avr8js@0.21.0 --prefix {}`): {}. \ + Ensure `npm` is installed alongside Node.js and on PATH \ + (https://nodejs.org/). If npm is installed, set \ + {}=1 to force a clean reinstall.", + cache_dir.display(), + e, + REFRESH_EMU_CACHE_ENV + )) + })?; + if !output.success() { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "`npm install avr8js@0.21.0 --prefix {}` exited with status {}.\n\ + --- stdout ---\n{}\n--- stderr ---\n{}", + cache_dir.display(), + output.exit_code, + output.stdout.trim_end(), + output.stderr.trim_end() + ))); + } + + // Post-install integrity check: guard against npm exiting 0 without + // actually extracting the package (rare, but has been seen on Windows + // under antivirus interference). + if !avr8js_cache_is_intact(cache_dir) { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "avr8js npm install at {} reported success but the cache is still \ + incomplete (missing node_modules/avr8js/package.json). \ + Try rerunning with {}=1.", + cache_dir.display(), + REFRESH_EMU_CACHE_ENV + ))); + } + + Ok(()) +} diff --git a/crates/fbuild-daemon/src/handlers/emulator/avr8js_web.rs b/crates/fbuild-daemon/src/handlers/emulator/avr8js_web.rs new file mode 100644 index 00000000..c20d45c9 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/avr8js_web.rs @@ -0,0 +1,238 @@ +//! AVR8js browser-side web UI handlers and session manifest types. +//! +//! Serves the static HTML shell, the `app.js` ES module, the per-session +//! `session.json`, and the raw firmware hex bytes for the in-browser AVR8js +//! emulator. The session manifest is registered into [`DaemonContext`] from +//! [`super::avr8js_deploy::deploy_avr8js`]. + +use crate::context::DaemonContext; +use axum::extract::{Path as AxumPath, State}; +use axum::http::{header, HeaderValue, StatusCode}; +use axum::response::{Html, IntoResponse}; +use axum::Json; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +const AVR8JS_APP_JS: &str = include_str!("../../../web/avr8js/app.js"); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct Avr8jsSessionManifest { + pub session_id: String, + pub project_dir: String, + pub env_name: String, + pub board_id: String, + pub platform: String, + pub mcu: String, + pub f_cpu_hz: u32, + pub firmware_hex: String, + pub firmware_elf: Option, + pub created_at_unix: f64, +} + +#[derive(Debug, Serialize)] +struct Avr8jsSessionResponse { + session_id: String, + env_name: String, + board_id: String, + platform: String, + mcu: String, + f_cpu_hz: u32, + firmware_hex_url: String, +} + +pub(crate) fn now_unix() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} + +pub(crate) fn render_page(session_id: &str) -> String { + format!( + r#" + + + + + AVR8js Monitor + + + +
+
+
AVR8js Emulator
+
Loading session...
+
+ +
+
Session {}
+
Browser-side ATmega328P serial monitor
+
+
+ + + + +"#, + session_id, + serde_json::to_string(session_id).unwrap_or_else(|_| "\"\"".to_string()), + ) +} + +pub(crate) fn load_session_manifest( + ctx: &DaemonContext, + session_id: &str, +) -> fbuild_core::Result { + let manifest_path = ctx + .avr8js_sessions + .get(session_id) + .map(|entry| entry.value().clone()) + .ok_or_else(|| { + fbuild_core::FbuildError::Other(format!("unknown AVR8js session '{}'", session_id)) + })?; + let raw = std::fs::read_to_string(&manifest_path)?; + serde_json::from_str(&raw).map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to parse AVR8js session manifest: {}", e)) + }) +} + +pub async fn avr8js_page( + AxumPath(session_id): AxumPath, + State(ctx): State>, +) -> impl IntoResponse { + match load_session_manifest(&ctx, &session_id) { + Ok(_) => Html(render_page(&session_id)).into_response(), + Err(e) => (StatusCode::NOT_FOUND, e.to_string()).into_response(), + } +} + +pub async fn avr8js_app_js() -> impl IntoResponse { + ( + [( + header::CONTENT_TYPE, + HeaderValue::from_static("application/javascript; charset=utf-8"), + )], + AVR8JS_APP_JS, + ) +} + +pub async fn avr8js_session_json( + AxumPath(session_id): AxumPath, + State(ctx): State>, +) -> impl IntoResponse { + match load_session_manifest(&ctx, &session_id) { + Ok(manifest) => ( + StatusCode::OK, + Json(Avr8jsSessionResponse { + session_id: manifest.session_id.clone(), + env_name: manifest.env_name, + board_id: manifest.board_id, + platform: manifest.platform, + mcu: manifest.mcu, + f_cpu_hz: manifest.f_cpu_hz, + firmware_hex_url: format!("/api/emulator/avr8js/{}/firmware.hex", session_id), + }), + ) + .into_response(), + Err(e) => (StatusCode::NOT_FOUND, e.to_string()).into_response(), + } +} + +pub async fn avr8js_firmware_hex( + AxumPath(session_id): AxumPath, + State(ctx): State>, +) -> impl IntoResponse { + match load_session_manifest(&ctx, &session_id) { + Ok(manifest) => match std::fs::read_to_string(&manifest.firmware_hex) { + Ok(hex) => ( + [( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + )], + hex, + ) + .into_response(), + Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), + }, + Err(e) => (StatusCode::NOT_FOUND, e.to_string()).into_response(), + } +} diff --git a/crates/fbuild-daemon/src/handlers/emulator/mod.rs b/crates/fbuild-daemon/src/handlers/emulator/mod.rs new file mode 100644 index 00000000..88f02faa --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/mod.rs @@ -0,0 +1,35 @@ +//! Emulator deploy handlers, runner abstractions, and the `POST /api/test-emu` +//! build-then-emulate flow. +//! +//! This module is split into focused submodules to keep individual files small. +//! The public API (`deploy_avr8js`, `deploy_qemu`, `test_emu`, the avr8js web +//! handlers, the `EmulatorRunner` trait, `select_runner`, and the request +//! structs) is re-exported here so external callers continue to access it via +//! `handlers::emulator::*`. + +mod avr8js_deploy; +mod avr8js_headless; +mod avr8js_npm; +mod avr8js_web; +mod qemu_deploy; +mod runners; +mod select; +mod shared; + +#[cfg(test)] +mod tests_npm_cache; +#[cfg(test)] +mod tests_outcome; +#[cfg(test)] +mod tests_process; +#[cfg(test)] +mod tests_select_runner; + +// --- Public API re-exports (preserve `handlers::emulator::*` paths) --- + +pub use avr8js_deploy::{deploy_avr8js, DeployAvr8jsRequest}; +pub use avr8js_web::{avr8js_app_js, avr8js_firmware_hex, avr8js_page, avr8js_session_json}; +pub use qemu_deploy::{deploy_qemu, DeployQemuRequest}; +pub use runners::{Avr8jsRunner, EmulatorRunner, QemuRunner, SimavrRunner}; +pub use select::{select_runner, test_emu}; +pub use shared::EmulatorRunConfig; diff --git a/crates/fbuild-daemon/src/handlers/emulator/qemu_deploy.rs b/crates/fbuild-daemon/src/handlers/emulator/qemu_deploy.rs new file mode 100644 index 00000000..68baa7b2 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/qemu_deploy.rs @@ -0,0 +1,339 @@ +//! `POST /api/deploy` (QEMU variant) — builds the flash image, resolves the +//! Espressif QEMU binary, and streams the run through the shared process +//! runner. Also exposes helpers used by the runner-trait implementations +//! (`resolve_esp_qemu_for_mcu`, `check_qemu_flash_mode`, +//! `is_qemu_supported_esp32_mcu`). + +use super::shared::{ + build_linux_macos_qemu_hint, qemu_session_dir, resolve_esp32_toolchain_gcc_path, + run_qemu_process, RunQemuOptions, +}; +use crate::context::DaemonContext; +use crate::handlers::operations::MonitorOutcome; +use crate::models::OperationResponse; +use axum::http::StatusCode; +use axum::Json; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +pub struct DeployQemuRequest { + pub request_id: String, + pub project_dir: PathBuf, + pub env_name: String, + pub board_id: String, + pub platform: fbuild_core::Platform, + pub firmware_path: PathBuf, + pub elf_path: Option, + pub output_file: String, + pub output_dir: Option, + pub monitor_timeout: Option, + pub qemu_timeout_secs: u32, + pub halt_on_error: Option, + pub halt_on_success: Option, + pub expect: Option, + pub show_timestamp: bool, + pub verbose: bool, + pub board_overrides: HashMap, +} + +/// Check whether a given MCU is supported by the QEMU runner. +/// +/// Supported MCUs: +/// - Xtensa (`qemu-system-xtensa`): `esp32`, `esp32s3` +/// - RISC-V (`qemu-system-riscv32`): `esp32c3`, `esp32c6`, `esp32h2` +pub(crate) fn is_qemu_supported_esp32_mcu(mcu: &str) -> bool { + fbuild_packages::toolchain::EspQemuArch::for_mcu(mcu).is_some() +} + +/// Resolve the Espressif QEMU executable appropriate for the given MCU. +/// +/// Picks `qemu-system-xtensa` for ESP32/ESP32-S3 and `qemu-system-riscv32` +/// for ESP32-C3/C6/H2. Returns the resolved binary path (downloading into +/// the managed fbuild cache if required). +pub(crate) fn resolve_esp_qemu_for_mcu( + project_dir: &Path, + mcu: &str, +) -> fbuild_core::Result { + let arch = fbuild_packages::toolchain::EspQemuArch::for_mcu(mcu).ok_or_else(|| { + fbuild_core::FbuildError::DeployFailed(format!( + "no QEMU backend available for MCU '{}'", + mcu + )) + })?; + let pkg = fbuild_packages::toolchain::EspQemu::new(project_dir, arch)?; + pkg.resolve_executable() +} + +/// Fail fast if the board's flash mode is incompatible with QEMU (DIO only). +pub(crate) fn check_qemu_flash_mode(board: &fbuild_config::BoardConfig) -> fbuild_core::Result<()> { + let mcu_config = fbuild_build::esp32::mcu_config::get_mcu_config(&board.mcu)?; + let effective_flash_mode = board + .flash_mode + .as_deref() + .unwrap_or(mcu_config.default_flash_mode()); + if !effective_flash_mode.eq_ignore_ascii_case("dio") { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "QEMU requires DIO flash mode; board '{}' uses '{}'", + board.name, effective_flash_mode + ))); + } + Ok(()) +} + +pub async fn deploy_qemu( + _ctx: Arc, + req: DeployQemuRequest, +) -> (StatusCode, Json) { + let DeployQemuRequest { + request_id, + project_dir, + env_name, + board_id, + platform, + firmware_path, + elf_path, + output_file, + output_dir, + monitor_timeout, + qemu_timeout_secs, + halt_on_error, + halt_on_success, + expect, + show_timestamp, + verbose, + board_overrides, + } = req; + + if platform != fbuild_core::Platform::Espressif32 { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + "QEMU deploy target is currently supported only for ESP32-family boards" + .to_string(), + )), + ); + } + if firmware_path.extension().and_then(|ext| ext.to_str()) != Some("bin") { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!( + "QEMU deploy target requires firmware.bin, got '{}'", + firmware_path.display() + ), + )), + ); + } + + let board = match fbuild_config::BoardConfig::from_board_id(&board_id, &board_overrides) { + Ok(board) => board, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("failed to load board config for QEMU: {}", e), + )), + ); + } + }; + if !is_qemu_supported_esp32_mcu(&board.mcu) { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!( + "native QEMU deploy currently supports ESP32, ESP32-S3 (Xtensa) and \ + ESP32-C3, ESP32-C6, ESP32-H2 (RISC-V) boards, got '{}'", + board.mcu + ), + )), + ); + } + + let mcu_config = match fbuild_build::esp32::mcu_config::get_mcu_config(&board.mcu) { + Ok(cfg) => cfg, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("failed to load MCU config for QEMU: {}", e), + )), + ); + } + }; + + let effective_flash_mode = board + .flash_mode + .as_deref() + .unwrap_or(mcu_config.default_flash_mode()); + if !effective_flash_mode.eq_ignore_ascii_case("dio") { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!( + "QEMU requires a DIO-compatible flash image; effective flash mode is '{}'", + effective_flash_mode + ), + )), + ); + } + + let flash_size_bytes = match fbuild_deploy::esp32::resolve_qemu_flash_size_bytes( + &board, + mcu_config.default_flash_size(), + ) { + Ok(size) => size, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail(request_id, e.to_string())), + ); + } + }; + + let qemu = match resolve_esp_qemu_for_mcu(&project_dir, &board.mcu) { + Ok(path) => path, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + build_linux_macos_qemu_hint(&e.to_string()), + )), + ); + } + }; + + let session_dir = qemu_session_dir(&project_dir, &env_name); + if let Err(e) = std::fs::create_dir_all(&session_dir) { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("failed to create QEMU session dir: {}", e), + )), + ); + } + let flash_image = session_dir.join("qemu_flash.bin"); + // Only apply the ESP32-S3 ADC calibration patch for S3 variants. + let elf_for_adc_patch = if board.mcu.eq_ignore_ascii_case("esp32s3") { + elf_path.as_deref() + } else { + None + }; + if let Err(e) = fbuild_deploy::esp32::create_qemu_flash_image( + &firmware_path, + &flash_image, + flash_size_bytes, + mcu_config.bootloader_offset(), + mcu_config.partitions_offset(), + mcu_config.firmware_offset(), + elf_for_adc_patch, + ) { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("failed to create QEMU flash image: {}", e), + )), + ); + } + + let args = fbuild_deploy::esp32::build_qemu_args( + &board.mcu, + &flash_image, + board.qemu_esp32_psram_config(), + ); + let addr2line_path = elf_path.as_ref().and_then(|_| { + resolve_esp32_toolchain_gcc_path(&project_dir, &mcu_config) + .ok() + .and_then(|gcc| fbuild_serial::crash_decoder::derive_addr2line_path(&gcc)) + }); + + let timeout_secs = monitor_timeout.or(Some(qemu_timeout_secs as f64)); + let qemu_result = match run_qemu_process( + &qemu, + &args, + RunQemuOptions { + elf_path, + addr2line_path, + timeout_secs, + halt_on_error: halt_on_error.as_deref(), + halt_on_success: halt_on_success.as_deref(), + expect: expect.as_deref(), + show_timestamp, + verbose, + process_label: "QEMU", + }, + ) + .await + { + Ok(result) => result, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail(request_id, e.to_string())), + ); + } + }; + + let real_exit_code = qemu_result.exit_code; + match qemu_result.outcome { + MonitorOutcome::Success(message) => ( + StatusCode::OK, + Json(OperationResponse { + success: true, + request_id, + message: format!("QEMU run succeeded: {}", message), + exit_code: real_exit_code.unwrap_or(0), + output_file: Some(output_file), + output_dir, + launch_url: None, + stdout: Some(qemu_result.stdout), + stderr: Some(qemu_result.stderr), + }), + ), + MonitorOutcome::Error(message) => ( + StatusCode::OK, + Json(OperationResponse { + success: false, + request_id, + message: format!("QEMU run failed: {}", message), + exit_code: real_exit_code.unwrap_or(1), + output_file: Some(output_file), + output_dir, + launch_url: None, + stdout: Some(qemu_result.stdout), + stderr: Some(qemu_result.stderr), + }), + ), + MonitorOutcome::Timeout { expect_found } => { + let success = expect.is_none() || expect_found; + let exit_code = real_exit_code.unwrap_or(if success { 0 } else { 1 }); + ( + StatusCode::OK, + Json(OperationResponse { + success, + request_id, + message: if success { + "QEMU run completed (timeout)".to_string() + } else { + "QEMU run timed out (expected pattern not found)".to_string() + }, + exit_code, + output_file: Some(output_file), + output_dir, + launch_url: None, + stdout: Some(qemu_result.stdout), + stderr: Some(qemu_result.stderr), + }), + ) + } + } +} diff --git a/crates/fbuild-daemon/src/handlers/emulator/runners.rs b/crates/fbuild-daemon/src/handlers/emulator/runners.rs new file mode 100644 index 00000000..4448e974 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/runners.rs @@ -0,0 +1,345 @@ +//! `EmulatorRunner` trait and three concrete runners (`QemuRunner`, +//! `Avr8jsRunner`, `SimavrRunner`) used by `select_runner` / `test_emu`. + +use super::avr8js_headless::{run_avr8js_headless, RunAvr8jsHeadlessOptions, AVR8JS_HEADLESS_MJS}; +use super::avr8js_npm::{ensure_avr8js_npm, find_node}; +use super::qemu_deploy::resolve_esp_qemu_for_mcu; +use super::shared::{ + monitor_outcome_to_emulator, qemu_session_dir, resolve_esp32_toolchain_gcc_path, + run_qemu_process, EmulatorRunConfig, RunQemuOptions, +}; +use fbuild_core::emulator::{EmulatorOutcome, EmulatorRunResult}; +use std::path::PathBuf; + +/// Abstraction over emulator backends. Each implementation knows how to set up +/// and execute a specific emulator (QEMU, avr8js, etc.). +#[async_trait::async_trait] +pub trait EmulatorRunner: Send + Sync { + /// Human-readable name of this runner (e.g. "QEMU ESP32-S3", "avr8js ATmega328P"). + fn name(&self) -> &str; + + /// Run the emulator with the given configuration. + async fn run(&self, config: &EmulatorRunConfig) -> fbuild_core::Result; +} + +/// QEMU-based emulator runner for ESP32-family boards. +pub struct QemuRunner { + project_dir: PathBuf, + env_name: String, + board: fbuild_config::BoardConfig, + display_name: String, +} + +impl QemuRunner { + pub fn new(project_dir: PathBuf, env_name: String, board: fbuild_config::BoardConfig) -> Self { + let display_name = format!("QEMU {}", board.mcu.to_uppercase()); + Self { + project_dir, + env_name, + board, + display_name, + } + } +} + +#[async_trait::async_trait] +impl EmulatorRunner for QemuRunner { + fn name(&self) -> &str { + &self.display_name + } + + async fn run(&self, config: &EmulatorRunConfig) -> fbuild_core::Result { + if let Err(msg) = config + .artifact_bundle + .validate_for(fbuild_core::emulator::RunnerKind::QemuEsp32) + { + return Err(fbuild_core::FbuildError::DeployFailed(msg)); + } + let mcu_config = fbuild_build::esp32::mcu_config::get_mcu_config(&self.board.mcu)?; + + let effective_flash_mode = self + .board + .flash_mode + .as_deref() + .unwrap_or(mcu_config.default_flash_mode()); + if !effective_flash_mode.eq_ignore_ascii_case("dio") { + return Ok(EmulatorRunResult { + outcome: EmulatorOutcome::Unsupported(format!( + "QEMU requires DIO flash mode; effective mode is '{}'", + effective_flash_mode + )), + stdout: String::new(), + stderr: String::new(), + command_line: String::new(), + exit_code: None, + }); + } + + let flash_size_bytes = fbuild_deploy::esp32::resolve_qemu_flash_size_bytes( + &self.board, + mcu_config.default_flash_size(), + )?; + + let qemu = resolve_esp_qemu_for_mcu(&self.project_dir, &self.board.mcu)?; + + let session_dir = qemu_session_dir(&self.project_dir, &self.env_name); + std::fs::create_dir_all(&session_dir)?; + + let flash_image = session_dir.join("qemu_flash.bin"); + + // Only apply the ESP32-S3 ADC calibration patch for S3 variants. + let elf_for_adc_patch = if self.board.mcu.eq_ignore_ascii_case("esp32s3") { + config.elf_path.as_deref() + } else { + None + }; + fbuild_deploy::esp32::create_qemu_flash_image( + &config.firmware_path, + &flash_image, + flash_size_bytes, + mcu_config.bootloader_offset(), + mcu_config.partitions_offset(), + mcu_config.firmware_offset(), + elf_for_adc_patch, + )?; + + let args = fbuild_deploy::esp32::build_qemu_args( + &self.board.mcu, + &flash_image, + self.board.qemu_esp32_psram_config(), + ); + let addr2line_path = config.elf_path.as_ref().and_then(|_| { + resolve_esp32_toolchain_gcc_path(&self.project_dir, &mcu_config) + .ok() + .and_then(|gcc| fbuild_serial::crash_decoder::derive_addr2line_path(&gcc)) + }); + + let command_line = format!("{} {}", qemu.display(), args.join(" ")); + + let qemu_result = run_qemu_process( + &qemu, + &args, + RunQemuOptions { + elf_path: config.elf_path.clone(), + addr2line_path, + timeout_secs: config.timeout, + halt_on_error: config.halt_on_error.as_deref(), + halt_on_success: config.halt_on_success.as_deref(), + expect: config.expect.as_deref(), + show_timestamp: config.show_timestamp, + verbose: config.verbose, + process_label: "QEMU", + }, + ) + .await?; + + let exit_code = qemu_result.exit_code; + let outcome = monitor_outcome_to_emulator(qemu_result.outcome, exit_code); + + Ok(EmulatorRunResult { + outcome, + stdout: qemu_result.stdout, + stderr: qemu_result.stderr, + command_line, + exit_code, + }) + } +} + +/// AVR8js-based emulator runner for ATmega328P (headless Node.js). +pub struct Avr8jsRunner { + board: fbuild_config::BoardConfig, +} + +impl Avr8jsRunner { + pub fn new(board: fbuild_config::BoardConfig) -> Self { + Self { board } + } +} + +#[async_trait::async_trait] +impl EmulatorRunner for Avr8jsRunner { + fn name(&self) -> &str { + "avr8js ATmega328P" + } + + async fn run(&self, config: &EmulatorRunConfig) -> fbuild_core::Result { + if let Err(msg) = config + .artifact_bundle + .validate_for(fbuild_core::emulator::RunnerKind::Avr8js) + { + return Err(fbuild_core::FbuildError::DeployFailed(msg)); + } + let node_path = find_node()?; + let avr8js_cache = ensure_avr8js_npm()?; + + let session_dir = tempfile::TempDir::new()?; + let script_path = session_dir.path().join("headless.mjs"); + std::fs::write(&script_path, AVR8JS_HEADLESS_MJS)?; + + let f_cpu_hz: u32 = self + .board + .f_cpu + .trim_end_matches('L') + .parse() + .unwrap_or(16_000_000); + + let command_line = format!( + "{} {} --hex {} --f-cpu {}", + node_path.display(), + script_path.display(), + config.firmware_path.display(), + f_cpu_hz + ); + + let avr8js_result = run_avr8js_headless( + &node_path, + &script_path, + &config.firmware_path, + f_cpu_hz, + &avr8js_cache, + RunAvr8jsHeadlessOptions { + timeout_secs: config.timeout, + halt_on_error: config.halt_on_error.as_deref(), + halt_on_success: config.halt_on_success.as_deref(), + expect: config.expect.as_deref(), + show_timestamp: config.show_timestamp, + verbose: config.verbose, + }, + ) + .await?; + + let exit_code = avr8js_result.exit_code; + let outcome = monitor_outcome_to_emulator(avr8js_result.outcome, exit_code); + + Ok(EmulatorRunResult { + outcome, + stdout: avr8js_result.stdout, + stderr: avr8js_result.stderr, + command_line, + exit_code, + }) + } +} + +/// Find the `simavr` binary on PATH. +/// +/// SimAVR is a native AVR simulator. Install via: +/// - Linux: `apt install simavr` or build from source +/// - macOS: `brew install simavr` +/// - Windows: build from source (MSYS2/MinGW) — limited support +fn find_simavr() -> fbuild_core::Result { + let simavr = if cfg!(windows) { + "simavr.exe" + } else { + "simavr" + }; + // Try running simavr to verify it exists; route through containment + // (issue #32). This is a short-lived probe so the containment + // difference is purely consistency. + match fbuild_core::subprocess::run_command(&[simavr, "--help"], None, None, None) { + Ok(_) => Ok(PathBuf::from(simavr)), + Err(_) => { + let install_hint = if cfg!(target_os = "linux") { + "Install via: apt install simavr (Debian/Ubuntu) or your distro's package manager" + } else if cfg!(target_os = "macos") { + "Install via: brew install simavr" + } else { + "SimAVR has limited Windows support. Build from source via MSYS2/MinGW, \ + or use --emulator avr8js for ATmega328P boards instead" + }; + Err(fbuild_core::FbuildError::DeployFailed(format!( + "simavr is required but '{}' was not found on PATH. {}", + simavr, install_hint + ))) + } + } +} + +/// SimAVR-based native emulator runner for AVR boards. +/// +/// Supports any AVR board that advertises `simavr` in its debug_tools. +/// Primary targets: ATmega328P (Uno), ATmega32U4 (Leonardo), ATmega2560 (Mega). +/// Consumes `firmware.elf` directly. +pub struct SimavrRunner { + board: fbuild_config::BoardConfig, +} + +impl SimavrRunner { + pub fn new(board: fbuild_config::BoardConfig) -> Self { + Self { board } + } +} + +#[async_trait::async_trait] +impl EmulatorRunner for SimavrRunner { + fn name(&self) -> &str { + "simavr" + } + + async fn run(&self, config: &EmulatorRunConfig) -> fbuild_core::Result { + if let Err(msg) = config + .artifact_bundle + .validate_for(fbuild_core::emulator::RunnerKind::Simavr) + { + return Err(fbuild_core::FbuildError::DeployFailed(msg)); + } + let simavr_path = find_simavr()?; + + // simavr requires an ELF file + let elf_path = config.elf_path.as_ref().ok_or_else(|| { + fbuild_core::FbuildError::DeployFailed( + "simavr requires firmware.elf but no ELF path was produced by the build" + .to_string(), + ) + })?; + + let f_cpu_hz: u32 = self + .board + .f_cpu + .trim_end_matches('L') + .parse() + .unwrap_or(16_000_000); + + let mcu = self.board.mcu.to_lowercase(); + + let args: Vec = vec![ + "-m".to_string(), + mcu.clone(), + "-f".to_string(), + f_cpu_hz.to_string(), + elf_path.display().to_string(), + ]; + + let command_line = format!("{} {}", simavr_path.display(), args.join(" ")); + + // Reuse the generic process runner (run_qemu_process) with no crash decoder + let result = run_qemu_process( + &simavr_path, + &args, + RunQemuOptions { + elf_path: None, // no ESP32 crash decoder needed + addr2line_path: None, // no addr2line for AVR in this path + timeout_secs: config.timeout, + halt_on_error: config.halt_on_error.as_deref(), + halt_on_success: config.halt_on_success.as_deref(), + expect: config.expect.as_deref(), + show_timestamp: config.show_timestamp, + verbose: config.verbose, + process_label: "simavr", + }, + ) + .await?; + + let exit_code = result.exit_code; + let outcome = monitor_outcome_to_emulator(result.outcome, exit_code); + + Ok(EmulatorRunResult { + outcome, + stdout: result.stdout, + stderr: result.stderr, + command_line, + exit_code, + }) + } +} diff --git a/crates/fbuild-daemon/src/handlers/emulator/select.rs b/crates/fbuild-daemon/src/handlers/emulator/select.rs new file mode 100644 index 00000000..a755ed60 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/select.rs @@ -0,0 +1,379 @@ +//! Runner selection (`select_runner`) and the build-then-emulate flow exposed +//! as `POST /api/test-emu` (`test_emu`). + +use super::qemu_deploy::{check_qemu_flash_mode, is_qemu_supported_esp32_mcu}; +use super::runners::{Avr8jsRunner, EmulatorRunner, QemuRunner, SimavrRunner}; +use super::shared::EmulatorRunConfig; +use crate::context::DaemonContext; +use crate::models::OperationResponse; +use axum::extract::State; +use axum::http::StatusCode; +use axum::Json; +use fbuild_core::emulator::EmulatorArtifactBundle; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +/// Select the appropriate emulator runner based on platform, MCU, and optional +/// explicit emulator choice. +/// +/// Returns `Err` with `EmulatorOutcome::Unsupported` information if no runner +/// matches. +pub fn select_runner( + project_dir: &Path, + env_name: &str, + platform: fbuild_core::Platform, + board_id: &str, + board_overrides: &HashMap, + emulator: Option<&str>, +) -> fbuild_core::Result> { + let board = fbuild_config::BoardConfig::from_board_id(board_id, board_overrides)?; + + if let Some(explicit) = emulator { + return match explicit { + "qemu" => { + if platform != fbuild_core::Platform::Espressif32 { + return Err(fbuild_core::FbuildError::DeployFailed( + "QEMU runner is only supported for ESP32-family boards".to_string(), + )); + } + if !is_qemu_supported_esp32_mcu(&board.mcu) { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "QEMU runner currently supports ESP32, ESP32-S3 (Xtensa) and \ + ESP32-C3, ESP32-C6, ESP32-H2 (RISC-V), got '{}'", + board.mcu + ))); + } + check_qemu_flash_mode(&board)?; + Ok(Box::new(QemuRunner::new( + project_dir.to_path_buf(), + env_name.to_string(), + board, + ))) + } + "avr8js" => { + if !matches!( + platform, + fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr + ) { + return Err(fbuild_core::FbuildError::DeployFailed( + "avr8js runner is only supported for AVR boards".to_string(), + )); + } + if !board.mcu.eq_ignore_ascii_case("atmega328p") { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "avr8js runner currently supports only ATmega328P, got '{}'", + board.mcu + ))); + } + Ok(Box::new(Avr8jsRunner::new(board))) + } + "simavr" => { + if !matches!( + platform, + fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr + ) { + return Err(fbuild_core::FbuildError::DeployFailed( + "simavr runner is only supported for AVR boards".to_string(), + )); + } + if !board.has_emulator("simavr") { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "board '{}' does not advertise simavr support in its debug_tools", + board_id + ))); + } + Ok(Box::new(SimavrRunner::new(board))) + } + other => Err(fbuild_core::FbuildError::DeployFailed(format!( + "unsupported emulator '{}'; available: qemu, avr8js, simavr", + other + ))), + }; + } + + // Auto-detect based on platform and MCU + match platform { + fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr => { + if board.mcu.eq_ignore_ascii_case("atmega328p") { + // Default to avr8js for ATmega328P (no external binary needed) + Ok(Box::new(Avr8jsRunner::new(board))) + } else if board.has_emulator("simavr") { + // Use simavr for other AVR MCUs that advertise it + Ok(Box::new(SimavrRunner::new(board))) + } else { + Err(fbuild_core::FbuildError::DeployFailed(format!( + "no emulator runner available for AVR MCU '{}'; \ + ATmega328P is supported via avr8js, other AVR boards require simavr in debug_tools", + board.mcu + ))) + } + } + fbuild_core::Platform::Espressif32 => { + if is_qemu_supported_esp32_mcu(&board.mcu) { + check_qemu_flash_mode(&board)?; + Ok(Box::new(QemuRunner::new( + project_dir.to_path_buf(), + env_name.to_string(), + board, + ))) + } else { + Err(fbuild_core::FbuildError::DeployFailed(format!( + "no emulator runner available for ESP32 MCU '{}'; \ + ESP32, ESP32-S3 (Xtensa) and ESP32-C3, ESP32-C6, ESP32-H2 (RISC-V) \ + are supported via QEMU", + board.mcu + ))) + } + } + _ => Err(fbuild_core::FbuildError::DeployFailed(format!( + "no emulator runner available for platform {:?}", + platform + ))), + } +} + +/// POST /api/test-emu handler — build firmware then run it in an emulator. +pub async fn test_emu( + State(ctx): State>, + Json(req): Json, +) -> (StatusCode, Json) { + let request_id = req + .request_id + .clone() + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let project_dir = PathBuf::from(&req.project_dir); + + // Mark the daemon as busy for the full build + emulate lifecycle. + // Without this guard the 30 s self-eviction loop sees an "empty" + // daemon during long (>30 s) ESP32/QEMU builds and triggers graceful + // shutdown, which closes the in-flight HTTP connection and surfaces + // as `error sending request for url (.../api/test-emu)` on the CLI + // side. See issue #130. + let _op_guard = crate::handlers::operations::OperationGuard::new( + &ctx, + fbuild_core::DaemonState::Building, + Some(format!("test-emu {}", req.project_dir)), + ); + + if !project_dir.exists() { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("project directory does not exist: {}", req.project_dir), + )), + ); + } + + // Parse config + let config = + match fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")) { + Ok(c) => c, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("failed to parse platformio.ini: {}", e), + )), + ); + } + }; + + let env_name = req + .environment + .clone() + .or_else(|| config.get_default_environment().map(|s| s.to_string())) + .unwrap_or_else(|| "default".to_string()); + + let env_config = match config.get_env_config(&env_name) { + Ok(c) => c, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("invalid environment '{}': {}", env_name, e), + )), + ); + } + }; + + let platform_str = env_config.get("platform").cloned().unwrap_or_default(); + let platform = match fbuild_core::Platform::from_platform_str(&platform_str) { + Some(p) => p, + None => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("unsupported platform: {}", platform_str), + )), + ); + } + }; + + let board_id = env_config.get("board").cloned().unwrap_or_else(|| { + fbuild_build::get_platform_support(platform) + .map(|s| s.default_board_id().to_string()) + .unwrap_or_else(|_| "unknown".to_string()) + }); + let board_overrides = config.get_board_overrides(&env_name).unwrap_or_default(); + + // Select the emulator runner before building (fail fast on unsupported boards) + let runner = match select_runner( + &project_dir, + &env_name, + platform, + &board_id, + &board_overrides, + req.emulator.as_deref(), + ) { + Ok(r) => r, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail(request_id, e.to_string())), + ); + } + }; + + // Build firmware (hold project lock only during build, not the emulator phase) + let build_result = { + let lock = ctx.project_lock(&project_dir); + let _guard = lock.lock().await; + + let needs_qemu_flags = platform == fbuild_core::Platform::Espressif32 + && req.emulator.as_deref() != Some("avr8js"); + let board_for_flags = if needs_qemu_flags { + fbuild_config::BoardConfig::from_board_id(&board_id, &board_overrides).ok() + } else { + None + }; + + let build_dir = fbuild_paths::get_project_build_root(&project_dir); + let params = fbuild_build::BuildParams { + project_dir: project_dir.clone(), + env_name: env_name.clone(), + clean: false, + profile: fbuild_core::BuildProfile::Release, + build_dir, + verbose: req.verbose, + jobs: None, + generate_compiledb: false, + compiledb_only: false, + log_sender: None, + symbol_analysis: false, + symbol_analysis_path: None, + no_timestamp: false, + src_dir: None, + pio_env: req.pio_env.clone(), + extra_build_flags: if needs_qemu_flags { + board_for_flags + .as_ref() + .map(|b| crate::handlers::operations::qemu_extra_build_flags(platform, &b.mcu)) + .unwrap_or_default() + } else { + Vec::new() + }, + watch_set_cache: Some(std::sync::Arc::clone(&ctx.watch_set_cache) as std::sync::Arc<_>), + }; + + let p = platform; + tokio::task::spawn_blocking(move || { + let orchestrator = fbuild_build::get_orchestrator(p)?; + orchestrator.build(¶ms) + }) + .await + }; + + let (firmware_path, elf_path) = match build_result { + Ok(Ok(r)) if r.success => { + let fw = r.firmware_path.clone().unwrap_or_else(|| { + r.elf_path + .clone() + .unwrap_or_else(|| PathBuf::from("firmware.bin")) + }); + (fw, r.elf_path) + } + Ok(Ok(r)) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("build failed: {}", r.message), + )), + ); + } + Ok(Err(e)) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("build error: {}", e), + )), + ); + } + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("build task panicked: {}", e), + )), + ); + } + }; + + // Run the emulator + let artifact_bundle = EmulatorArtifactBundle::from_paths(&firmware_path, elf_path.as_deref()); + let run_config = EmulatorRunConfig { + firmware_path, + elf_path, + artifact_bundle, + timeout: req.timeout, + halt_on_error: req.halt_on_error.clone(), + halt_on_success: req.halt_on_success.clone(), + expect: req.expect.clone(), + show_timestamp: req.show_timestamp, + verbose: req.verbose, + }; + + let emu_result = match runner.run(&run_config).await { + Ok(r) => r, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("emulator error: {}", e), + )), + ); + } + }; + + let success = emu_result.is_success(); + let exit_code = emu_result.exit_code.unwrap_or(if success { 0 } else { 1 }); + let message = format!( + "{} test-emu {}: {}", + runner.name(), + if success { "passed" } else { "failed" }, + emu_result.outcome + ); + + ( + StatusCode::OK, + Json(OperationResponse { + success, + request_id, + message, + exit_code, + output_file: None, + output_dir: None, + launch_url: None, + stdout: Some(emu_result.stdout), + stderr: Some(emu_result.stderr), + }), + ) +} diff --git a/crates/fbuild-daemon/src/handlers/emulator/shared.rs b/crates/fbuild-daemon/src/handlers/emulator/shared.rs new file mode 100644 index 00000000..302ea9a0 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/shared.rs @@ -0,0 +1,340 @@ +//! Process runner primitives shared by every emulator backend (QEMU, simavr, +//! avr8js headless). +//! +//! Owns the streaming subprocess loop (`run_qemu_process`), line-reader task, +//! shared option/result structs, and a few small helpers used across runners +//! (`qemu_session_dir`, ESP32 toolchain GCC resolution, Windows process +//! flags, `monitor_outcome_to_emulator`). + +use crate::handlers::operations::{MonitorOutcome, MonitorState}; +use fbuild_core::emulator::EmulatorOutcome; +use fbuild_packages::{Package, Toolchain}; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use tokio::io::{AsyncBufReadExt, BufReader}; + +pub(crate) struct ProcessLine { + pub is_stderr: bool, + pub line: String, +} + +pub(crate) enum ProcessEvent { + Line(ProcessLine), + StreamClosed, +} + +pub(crate) struct QemuRunResult { + pub outcome: MonitorOutcome, + pub stdout: String, + pub stderr: String, + pub exit_code: Option, +} + +pub(crate) struct RunQemuOptions<'a> { + pub elf_path: Option, + pub addr2line_path: Option, + pub timeout_secs: Option, + pub halt_on_error: Option<&'a str>, + pub halt_on_success: Option<&'a str>, + pub expect: Option<&'a str>, + pub show_timestamp: bool, + pub verbose: bool, + /// Label used in user-visible messages (e.g. "QEMU", "simavr"). + pub process_label: &'a str, +} + +/// Configuration for an emulator test run (user-facing options). +pub struct EmulatorRunConfig { + pub firmware_path: PathBuf, + pub elf_path: Option, + /// Structured artifact bundle for runner validation. + pub artifact_bundle: fbuild_core::emulator::EmulatorArtifactBundle, + pub timeout: Option, + pub halt_on_error: Option, + pub halt_on_success: Option, + pub expect: Option, + pub show_timestamp: bool, + pub verbose: bool, +} + +/// Convert a `MonitorOutcome` into an `EmulatorOutcome`. +pub(crate) fn monitor_outcome_to_emulator( + outcome: MonitorOutcome, + exit_code: Option, +) -> EmulatorOutcome { + match outcome { + MonitorOutcome::Success(msg) => EmulatorOutcome::Passed(msg), + MonitorOutcome::Error(msg) => { + // Heuristic: if the process crashed (non-zero exit with crash signature) + if let Some(code) = exit_code { + if code != 0 && (msg.contains("abort()") || msg.contains("Guru Meditation")) { + return EmulatorOutcome::Crashed(msg); + } + } + EmulatorOutcome::Failed(msg) + } + MonitorOutcome::Timeout { expect_found } => EmulatorOutcome::TimedOut { expect_found }, + } +} + +pub(crate) fn qemu_session_dir(project_dir: &Path, env_name: &str) -> PathBuf { + fbuild_paths::get_project_fbuild_dir(project_dir) + .join("emulators") + .join("qemu") + .join(env_name) + .join(uuid::Uuid::new_v4().to_string()) +} + +pub(crate) fn build_linux_macos_qemu_hint(err: &str) -> String { + if cfg!(any(target_os = "linux", target_os = "macos")) { + format!( + "{}. On Linux/macOS, ensure QEMU runtime deps are installed: libgcrypt, glib2, pixman, SDL2, and libslirp.", + err + ) + } else { + err.to_string() + } +} + +pub(crate) fn resolve_esp32_toolchain_gcc_path( + project_dir: &Path, + mcu_config: &fbuild_build::esp32::mcu_config::Esp32McuConfig, +) -> fbuild_core::Result { + let platform = fbuild_packages::library::Esp32Platform::new(project_dir); + Package::ensure_installed(&platform)?; + + let is_riscv = mcu_config.is_riscv(); + let prefix = mcu_config.toolchain_prefix(); + let toolchain_name = if is_riscv { + "toolchain-riscv32-esp" + } else { + "toolchain-xtensa-esp-elf" + }; + + let toolchain = match platform.get_toolchain_metadata_url(is_riscv) { + Ok(metadata_url) => { + let cache = fbuild_packages::Cache::new(project_dir); + let cache_dir = cache.toolchains_dir().join(toolchain_name); + match fbuild_packages::toolchain::esp32_metadata::resolve_toolchain_url_sync( + &metadata_url, + toolchain_name, + &cache_dir, + ) { + Ok(resolved) => fbuild_packages::toolchain::Esp32Toolchain::from_resolved( + project_dir, + &resolved.url, + resolved.sha256.as_deref(), + is_riscv, + &prefix, + ), + Err(_) => { + fbuild_packages::toolchain::Esp32Toolchain::new(project_dir, is_riscv, &prefix) + } + } + } + Err(_) => fbuild_packages::toolchain::Esp32Toolchain::new(project_dir, is_riscv, &prefix), + }; + + let _ = Package::ensure_installed(&toolchain)?; + Ok(toolchain.get_gcc_path()) +} + +#[cfg(windows)] +fn apply_windows_process_flags(cmd: &mut tokio::process::Command, exe_path: &Path) { + const CREATE_NO_WINDOW: u32 = 0x08000000; + cmd.creation_flags(CREATE_NO_WINDOW); + + let current_path = std::env::var("PATH").unwrap_or_default(); + if let Ok(path_env) = + fbuild_packages::toolchain::build_windows_qemu_path_env(exe_path, ¤t_path) + { + cmd.env("PATH", path_env); + } +} + +#[cfg(not(windows))] +fn apply_windows_process_flags(_cmd: &mut tokio::process::Command, _exe_path: &Path) {} + +pub(crate) async fn spawn_line_reader( + stream: impl tokio::io::AsyncRead + Unpin + Send + 'static, + is_stderr: bool, + tx: tokio::sync::mpsc::UnboundedSender, +) { + let mut lines = BufReader::new(stream).lines(); + while let Ok(Some(line)) = lines.next_line().await { + let _ = tx.send(ProcessEvent::Line(ProcessLine { is_stderr, line })); + } + let _ = tx.send(ProcessEvent::StreamClosed); +} + +pub(crate) async fn run_qemu_process( + qemu_path: &Path, + args: &[String], + options: RunQemuOptions<'_>, +) -> fbuild_core::Result { + // allow-direct-spawn: tokio streaming QEMU emulator; blocking NativeProcess unsuitable. + let mut cmd = tokio::process::Command::new(qemu_path); + cmd.args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + apply_windows_process_flags(&mut cmd, qemu_path); + + let label = options.process_label; + if options.verbose { + tracing::info!("{}: {} {}", label, qemu_path.display(), args.join(" ")); + } + + // Route through containment (#32). QEMU is a long-running process + // — a daemon hard-kill mid-emulation must not leave `qemu-system-*` + // or its `conhost.exe` wrapper behind. + let mut child = + fbuild_core::containment::tokio_spawn::spawn_contained(&mut cmd).map_err(|e| { + fbuild_core::FbuildError::DeployFailed(build_linux_macos_qemu_hint(&format!( + "failed to launch {} at {}: {}", + label, + qemu_path.display(), + e + ))) + })?; + + let stdout = child.stdout.take().ok_or_else(|| { + fbuild_core::FbuildError::DeployFailed(format!("failed to capture {} stdout", label)) + })?; + let stderr = child.stderr.take().ok_or_else(|| { + fbuild_core::FbuildError::DeployFailed(format!("failed to capture {} stderr", label)) + })?; + + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); + let stdout_task = tokio::spawn(spawn_line_reader(stdout, false, tx.clone())); + let stderr_task = tokio::spawn(spawn_line_reader(stderr, true, tx)); + + let mut monitor = MonitorState::new( + options.timeout_secs, + options.halt_on_error, + options.halt_on_success, + options.expect, + options.show_timestamp, + ); + let mut crash_decoder = + fbuild_serial::crash_decoder::CrashDecoder::new(options.elf_path, options.addr2line_path); + let mut stdout_buf = String::new(); + let mut stderr_buf = String::new(); + let mut synthetic_buf = String::new(); + let mut streams_open = 2usize; + let mut child_exit: Option = None; + let mut final_outcome: Option = None; + + loop { + if monitor.timed_out() { + final_outcome = Some(monitor.timeout_outcome()); + let _ = child.kill().await; + break; + } + + let recv_timeout = monitor + .remaining() + .unwrap_or(std::time::Duration::from_secs(1)); + + tokio::select! { + status = child.wait(), if child_exit.is_none() => { + child_exit = Some(status.map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!("{} wait failed: {}", label, e)) + })?); + if streams_open == 0 { + break; + } + } + maybe_event = tokio::time::timeout(recv_timeout, rx.recv()) => { + match maybe_event { + Ok(Some(ProcessEvent::Line(line))) => { + let target = if line.is_stderr { &mut stderr_buf } else { &mut stdout_buf }; + target.push_str(&line.line); + target.push('\n'); + + if let Some(outcome) = monitor.process_line(&line.line) { + final_outcome = Some(outcome); + let _ = child.kill().await; + break; + } + + if let Some(decoded_lines) = crash_decoder.process_line(&line.line) { + for decoded in decoded_lines { + synthetic_buf.push_str(&decoded); + synthetic_buf.push('\n'); + if let Some(outcome) = monitor.process_line(&decoded) { + final_outcome = Some(outcome); + let _ = child.kill().await; + break; + } + } + if final_outcome.is_some() { + break; + } + } + } + Ok(Some(ProcessEvent::StreamClosed)) => { + streams_open = streams_open.saturating_sub(1); + if streams_open == 0 && child_exit.is_some() { + break; + } + } + Ok(None) => { + if child_exit.is_some() { + break; + } + } + Err(_) => { + final_outcome = Some(monitor.timeout_outcome()); + let _ = child.kill().await; + break; + } + } + } + } + } + + if child_exit.is_none() { + child_exit = Some(child.wait().await.map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!("{} wait failed: {}", label, e)) + })?); + } + + let _ = stdout_task.await; + let _ = stderr_task.await; + + if !synthetic_buf.is_empty() { + stdout_buf.push_str(&synthetic_buf); + } + + let outcome = if let Some(outcome) = final_outcome { + outcome + } else if let Some(status) = child_exit { + if status.success() { + if options.expect.is_some() && !monitor.expect_found() { + MonitorOutcome::Error(format!( + "{} exited before the expected pattern was found", + label + )) + } else { + MonitorOutcome::Success(format!("{} exited normally", label)) + } + } else { + MonitorOutcome::Error(format!( + "{} exited with code {}", + label, + status.code().unwrap_or(-1) + )) + } + } else { + MonitorOutcome::Error(format!("{} exited unexpectedly", label)) + }; + + Ok(QemuRunResult { + outcome, + stdout: stdout_buf, + stderr: stderr_buf, + exit_code: child_exit.and_then(|s| s.code()), + }) +} + diff --git a/crates/fbuild-daemon/src/handlers/emulator/tests_npm_cache.rs b/crates/fbuild-daemon/src/handlers/emulator/tests_npm_cache.rs new file mode 100644 index 00000000..2953d7ad --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/tests_npm_cache.rs @@ -0,0 +1,219 @@ +//! avr8js npm cache integrity tests (issue #86). +//! +//! These tests probe `avr8js_cache_is_intact`, `prepare_avr8js_cache_for_install`, +//! `refresh_emu_cache_requested`, and `ensure_avr8js_npm_in` — covering the +//! detection of partial/corrupt npm installs, the force-refresh env var, and +//! actionable errors when `npm` is unreachable. + +use super::avr8js_npm::{ + avr8js_cache_is_intact, ensure_avr8js_npm_in, prepare_avr8js_cache_for_install, + refresh_emu_cache_requested, Avr8jsCachePrep, REFRESH_EMU_CACHE_ENV, +}; + +/// Serialises tests that mutate process-wide env vars (PATH). Without +/// this, parallel cargo-test workers would clobber each other's PATH. +fn env_lock() -> std::sync::MutexGuard<'static, ()> { + use std::sync::{Mutex, OnceLock}; + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|p| p.into_inner()) +} + +#[test] +fn avr8js_cache_is_intact_detects_missing_dir() { + let tmp = tempfile::TempDir::new().unwrap(); + // Empty cache dir: no node_modules at all. + assert!(!avr8js_cache_is_intact(tmp.path())); +} + +#[test] +fn avr8js_cache_is_intact_detects_missing_package_json() { + let tmp = tempfile::TempDir::new().unwrap(); + // Simulate a partial/corrupt install: dir exists, but no package.json. + std::fs::create_dir_all(tmp.path().join("node_modules").join("avr8js")).unwrap(); + assert!( + !avr8js_cache_is_intact(tmp.path()), + "corrupt install (no package.json) must not count as intact" + ); +} + +#[test] +fn avr8js_cache_is_intact_accepts_marker_present() { + let tmp = tempfile::TempDir::new().unwrap(); + let module_dir = tmp.path().join("node_modules").join("avr8js"); + std::fs::create_dir_all(&module_dir).unwrap(); + std::fs::write(module_dir.join("package.json"), b"{\"name\":\"avr8js\"}").unwrap(); + assert!(avr8js_cache_is_intact(tmp.path())); +} + +#[test] +fn prepare_avr8js_cache_skips_when_intact() { + let tmp = tempfile::TempDir::new().unwrap(); + let module_dir = tmp.path().join("node_modules").join("avr8js"); + std::fs::create_dir_all(&module_dir).unwrap(); + std::fs::write(module_dir.join("package.json"), b"{}").unwrap(); + assert_eq!( + prepare_avr8js_cache_for_install(tmp.path(), false), + Avr8jsCachePrep::AlreadyIntact + ); + // Tree must survive untouched. + assert!(module_dir.join("package.json").exists()); +} + +#[test] +fn prepare_avr8js_cache_wipes_node_modules_when_corrupt() { + let tmp = tempfile::TempDir::new().unwrap(); + let node_modules = tmp.path().join("node_modules"); + let module_dir = node_modules.join("avr8js"); + std::fs::create_dir_all(&module_dir).unwrap(); + // Stray partial file inside an otherwise package.json-less install. + std::fs::write(module_dir.join("index.mjs"), b"partial").unwrap(); + assert!(!avr8js_cache_is_intact(tmp.path())); + + let prep = prepare_avr8js_cache_for_install(tmp.path(), false); + assert_eq!(prep, Avr8jsCachePrep::WipedNodeModules); + assert!( + !node_modules.exists(), + "corrupt node_modules tree must be wiped before reinstall" + ); +} + +#[test] +fn prepare_avr8js_cache_force_refresh_wipes_entire_dir() { + let tmp_parent = tempfile::TempDir::new().unwrap(); + let cache = tmp_parent.path().join("avr8js-node"); + let module_dir = cache.join("node_modules").join("avr8js"); + std::fs::create_dir_all(&module_dir).unwrap(); + std::fs::write(module_dir.join("package.json"), b"{}").unwrap(); + // Even a fully-intact install must be wiped under force_refresh=true. + let prep = prepare_avr8js_cache_for_install(&cache, true); + assert_eq!(prep, Avr8jsCachePrep::ForceWiped); + assert!(!cache.exists(), "force-refresh must remove the cache dir"); +} + +#[test] +fn prepare_avr8js_cache_nothing_to_clean_on_fresh_path() { + let tmp_parent = tempfile::TempDir::new().unwrap(); + let cache = tmp_parent.path().join("avr8js-node"); // doesn't exist yet + assert_eq!( + prepare_avr8js_cache_for_install(&cache, false), + Avr8jsCachePrep::NothingToClean + ); +} + +#[test] +fn refresh_emu_cache_requested_recognises_truthy_values() { + let _guard = env_lock(); + for (val, expected) in [ + ("1", true), + ("true", true), + ("TRUE", true), + ("yes", true), + ("YES", true), + ("0", false), + ("", false), + ("no", false), + ] { + std::env::set_var(REFRESH_EMU_CACHE_ENV, val); + assert_eq!( + refresh_emu_cache_requested(), + expected, + "{}={:?} should parse as {}", + REFRESH_EMU_CACHE_ENV, + val, + expected + ); + } + std::env::remove_var(REFRESH_EMU_CACHE_ENV); + assert!(!refresh_emu_cache_requested()); +} + +/// When npm isn't on PATH, `ensure_avr8js_npm_in` must return an +/// `FbuildError::DeployFailed` that names both `npm` and the cache dir. +/// This is the fix for issue #86's silent `ERR_MODULE_NOT_FOUND`. +#[test] +fn ensure_avr8js_npm_in_reports_clear_error_without_npm() { + let _guard = env_lock(); + let saved_path = std::env::var_os("PATH"); + // PATHEXT matters on Windows for command resolution of .cmd files. + let saved_pathext = std::env::var_os("PATHEXT"); + + std::env::set_var("PATH", ""); + #[cfg(windows)] + { + // Ensure .cmd isn't resolved via some fallback. + std::env::set_var("PATHEXT", ""); + } + + let tmp = tempfile::TempDir::new().unwrap(); + let cache = tmp.path().join("avr8js-node"); + let result = ensure_avr8js_npm_in(&cache, false); + + // Restore BEFORE asserting so a panic doesn't leak PATH="" to sibling tests. + if let Some(p) = saved_path { + std::env::set_var("PATH", p); + } else { + std::env::remove_var("PATH"); + } + if let Some(p) = saved_pathext { + std::env::set_var("PATHEXT", p); + } else { + std::env::remove_var("PATHEXT"); + } + + let err = result.expect_err("npm must be unresolvable with PATH=\"\""); + let msg = err.to_string(); + assert!( + msg.contains("npm"), + "error message must mention 'npm'; got: {}", + msg + ); + assert!( + msg.contains(&cache.display().to_string()), + "error message must include cache dir path; got: {}", + msg + ); + assert!( + msg.contains(REFRESH_EMU_CACHE_ENV), + "error message must reference {} for recovery; got: {}", + REFRESH_EMU_CACHE_ENV, + msg + ); +} + +/// When the cache dir contains a corrupt partial install, the reinstall +/// path must fire (detected here by asserting the partial tree is wiped +/// even when the downstream npm call subsequently fails). +#[test] +fn ensure_avr8js_npm_in_wipes_corrupt_before_reinstall() { + let _guard = env_lock(); + let saved_path = std::env::var_os("PATH"); + + // Force the npm spawn to fail so we isolate the wipe behaviour. + std::env::set_var("PATH", ""); + + let tmp = tempfile::TempDir::new().unwrap(); + let cache = tmp.path().join("avr8js-node"); + let node_modules = cache.join("node_modules"); + let module_dir = node_modules.join("avr8js"); + std::fs::create_dir_all(&module_dir).unwrap(); + // Deliberately omit package.json → corrupt. + std::fs::write(module_dir.join("garbage"), b"partial").unwrap(); + assert!(!avr8js_cache_is_intact(&cache)); + + let result = ensure_avr8js_npm_in(&cache, false); + + if let Some(p) = saved_path { + std::env::set_var("PATH", p); + } else { + std::env::remove_var("PATH"); + } + + // npm spawn must fail (no PATH), but the corrupt tree must be gone. + assert!(result.is_err(), "empty PATH should prevent npm install"); + assert!( + !node_modules.exists(), + "corrupt node_modules must be wiped before reinstall attempt" + ); +} diff --git a/crates/fbuild-daemon/src/handlers/emulator/tests_outcome.rs b/crates/fbuild-daemon/src/handlers/emulator/tests_outcome.rs new file mode 100644 index 00000000..e402e664 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/tests_outcome.rs @@ -0,0 +1,36 @@ +//! Unit tests for the `MonitorOutcome` -> `EmulatorOutcome` mapping. + +use super::shared::monitor_outcome_to_emulator; +use crate::handlers::operations::MonitorOutcome; +use fbuild_core::emulator::EmulatorOutcome; + +#[test] +fn monitor_outcome_to_emulator_maps_success() { + let outcome = monitor_outcome_to_emulator(MonitorOutcome::Success("ok".into()), Some(0)); + assert_eq!(outcome, EmulatorOutcome::Passed("ok".into())); +} + +#[test] +fn monitor_outcome_to_emulator_maps_error() { + let outcome = monitor_outcome_to_emulator(MonitorOutcome::Error("bad".into()), Some(1)); + assert_eq!(outcome, EmulatorOutcome::Failed("bad".into())); +} + +#[test] +fn monitor_outcome_to_emulator_maps_crash() { + let outcome = monitor_outcome_to_emulator( + MonitorOutcome::Error("abort() was called at PC 0x4200".into()), + Some(134), + ); + assert_eq!( + outcome, + EmulatorOutcome::Crashed("abort() was called at PC 0x4200".into()) + ); +} + +#[test] +fn monitor_outcome_to_emulator_maps_timeout() { + let outcome = + monitor_outcome_to_emulator(MonitorOutcome::Timeout { expect_found: true }, None); + assert_eq!(outcome, EmulatorOutcome::TimedOut { expect_found: true }); +} diff --git a/crates/fbuild-daemon/src/handlers/emulator/tests_process.rs b/crates/fbuild-daemon/src/handlers/emulator/tests_process.rs new file mode 100644 index 00000000..eb3ff581 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/tests_process.rs @@ -0,0 +1,435 @@ +//! Tests for the shared subprocess runner (`run_qemu_process`) and an ignored +//! integration test that exercises a real ESP32-S3 fixture under QEMU. + +use super::shared::{resolve_esp32_toolchain_gcc_path, run_qemu_process, RunQemuOptions}; +use crate::handlers::operations::MonitorOutcome; +use std::path::PathBuf; + +pub(super) fn test_process_command(lines: &[&str]) -> (PathBuf, Vec) { + #[cfg(windows)] + { + let system_root = + std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string()); + let exe = PathBuf::from(system_root).join(r"System32\cmd.exe"); + let script = lines + .iter() + .map(|line| format!("echo {}", line)) + .collect::>() + .join(" & "); + (exe, vec!["/C".to_string(), script]) + } + + #[cfg(not(windows))] + { + let script = lines + .iter() + .map(|line| format!("printf '%s\\n' '{}'", line.replace('\'', "'\"'\"'"))) + .collect::>() + .join("; "); + (PathBuf::from("sh"), vec!["-c".to_string(), script]) + } +} + +#[tokio::test] +async fn run_qemu_process_reports_expected_success_output() { + let (exe, args) = test_process_command(&["Hello from ESP32-S3!"]); + let result = run_qemu_process( + &exe, + &args, + RunQemuOptions { + elf_path: None, + addr2line_path: None, + timeout_secs: Some(2.0), + halt_on_error: None, + halt_on_success: None, + expect: Some("Hello from ESP32-S3"), + show_timestamp: false, + verbose: false, + process_label: "QEMU", + }, + ) + .await + .unwrap(); + + assert!(result.stdout.contains("Hello from ESP32-S3!")); + match result.outcome { + MonitorOutcome::Success(message) => { + assert!(message.contains("QEMU exited normally")); + } + other => panic!("expected success outcome, got {:?}", other), + } +} + +#[tokio::test] +async fn run_qemu_process_surfaces_crash_decoder_output() { + let (exe, args) = + test_process_command(&["abort() was called at PC 0x42002a3c", "Rebooting..."]); + let result = run_qemu_process( + &exe, + &args, + RunQemuOptions { + elf_path: None, + addr2line_path: None, + timeout_secs: Some(2.0), + halt_on_error: Some("no firmware\\.elf found"), + halt_on_success: None, + expect: None, + show_timestamp: false, + verbose: false, + process_label: "QEMU", + }, + ) + .await + .unwrap(); + + assert!(result + .stdout + .contains("abort() was called at PC 0x42002a3c")); + assert!(result.stdout.contains("no firmware.elf found")); + match result.outcome { + MonitorOutcome::Error(message) => { + assert!(message.contains("halt-on-error pattern matched")); + } + other => panic!("expected error outcome, got {:?}", other), + } +} + +#[test] +#[ignore] +fn run_real_esp32s3_fixture_in_qemu() { + use fbuild_build::{BuildOrchestrator, BuildParams}; + use fbuild_core::BuildProfile; + + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + let project_dir = repo_root.join("tests/platform/esp32s3"); + if !project_dir.exists() { + eprintln!("SKIP: {} does not exist", project_dir.display()); + return; + } + + let build_dir = project_dir.join(".fbuild/build-qemu"); + let params = BuildParams { + project_dir: project_dir.clone(), + env_name: "esp32s3".to_string(), + clean: true, + profile: BuildProfile::Release, + build_dir, + verbose: true, + jobs: None, + generate_compiledb: false, + compiledb_only: false, + log_sender: None, + symbol_analysis: false, + symbol_analysis_path: None, + no_timestamp: false, + src_dir: None, + pio_env: Default::default(), + extra_build_flags: vec![ + "-DARDUINO_USB_MODE=0".to_string(), + "-DARDUINO_USB_CDC_ON_BOOT=0".to_string(), + ], + watch_set_cache: None, + }; + + let orchestrator = fbuild_build::esp32::orchestrator::Esp32Orchestrator; + let build_result = orchestrator + .build(¶ms) + .expect("ESP32-S3 fixture build should succeed"); + assert!(build_result.success); + + let firmware_path = build_result + .firmware_path + .clone() + .expect("should produce firmware.bin"); + let elf_path = build_result.elf_path.clone(); + + let board = + fbuild_config::BoardConfig::from_board_id("esp32-s3-devkitc-1", &Default::default()) + .unwrap(); + let mcu_config = fbuild_build::esp32::mcu_config::get_mcu_config("esp32s3").unwrap(); + let flash_size_bytes = fbuild_deploy::esp32::resolve_qemu_flash_size_bytes( + &board, + mcu_config.default_flash_size(), + ) + .unwrap(); + + let session_dir = tempfile::TempDir::new().unwrap(); + let flash_image = session_dir.path().join("flash.bin"); + fbuild_deploy::esp32::create_qemu_flash_image( + &firmware_path, + &flash_image, + flash_size_bytes, + mcu_config.bootloader_offset(), + mcu_config.partitions_offset(), + mcu_config.firmware_offset(), + elf_path.as_deref(), + ) + .unwrap(); + + let qemu = fbuild_packages::toolchain::EspQemuXtensa::new(&project_dir) + .and_then(|pkg| pkg.resolve_executable()) + .expect("native QEMU should resolve for ignored integration test"); + let args = fbuild_deploy::esp32::build_qemu_args( + &board.mcu, + &flash_image, + board.qemu_esp32_psram_config(), + ); + let addr2line_path = resolve_esp32_toolchain_gcc_path(&project_dir, &mcu_config) + .ok() + .and_then(|gcc| fbuild_serial::crash_decoder::derive_addr2line_path(&gcc)); + + let rt = tokio::runtime::Runtime::new().unwrap(); + let result = rt + .block_on(run_qemu_process( + &qemu, + &args, + RunQemuOptions { + elf_path, + addr2line_path, + timeout_secs: Some(15.0), + halt_on_error: None, + halt_on_success: Some("Hello from ESP32-S3!"), + expect: Some("Hello from ESP32-S3!"), + show_timestamp: false, + verbose: true, + process_label: "QEMU", + }, + )) + .unwrap(); + + assert!(result.stdout.contains("Hello from ESP32-S3!")); + match result.outcome { + MonitorOutcome::Success(_) => {} + other => panic!("expected success outcome, got {:?}", other), + } +} + +// ----------------------------------------------------------------------- +// avr8js headless tests (use fake process, no real Node.js needed) +// ----------------------------------------------------------------------- + +#[tokio::test] +async fn run_avr8js_headless_captures_stdout() { + let (exe, args) = test_process_command(&["Hello from AVR!"]); + // run_avr8js_headless expects (node, script, hex, f_cpu, cache_dir, options). + // We bypass that by calling the lower-level run with the fake exe directly. + // Since run_avr8js_headless builds its own command, we test the same + // subprocess loop via run_qemu_process which shares identical logic. + let result = run_qemu_process( + &exe, + &args, + RunQemuOptions { + elf_path: None, + addr2line_path: None, + timeout_secs: Some(2.0), + halt_on_error: None, + halt_on_success: None, + expect: Some("Hello from AVR"), + show_timestamp: false, + verbose: false, + process_label: "QEMU", + }, + ) + .await + .unwrap(); + + assert!(result.stdout.contains("Hello from AVR!")); + match result.outcome { + MonitorOutcome::Success(msg) => { + assert!(msg.contains("QEMU exited normally")); + } + other => panic!("expected success, got {:?}", other), + } +} + +#[tokio::test] +async fn run_avr8js_headless_halt_on_success() { + let (exe, args) = + test_process_command(&["booting...", "PASS: all tests passed", "more output"]); + let result = run_qemu_process( + &exe, + &args, + RunQemuOptions { + elf_path: None, + addr2line_path: None, + timeout_secs: Some(2.0), + halt_on_error: None, + halt_on_success: Some("PASS:"), + expect: None, + show_timestamp: false, + verbose: false, + process_label: "QEMU", + }, + ) + .await + .unwrap(); + + match result.outcome { + MonitorOutcome::Success(msg) => { + assert!(msg.contains("halt-on-success pattern matched")); + } + other => panic!("expected success, got {:?}", other), + } +} + +#[tokio::test] +async fn run_avr8js_headless_halt_on_error() { + let (exe, args) = test_process_command(&["booting...", "FAIL: assertion failed"]); + let result = run_qemu_process( + &exe, + &args, + RunQemuOptions { + elf_path: None, + addr2line_path: None, + timeout_secs: Some(2.0), + halt_on_error: Some("FAIL:"), + halt_on_success: None, + expect: None, + show_timestamp: false, + verbose: false, + process_label: "QEMU", + }, + ) + .await + .unwrap(); + + match result.outcome { + MonitorOutcome::Error(msg) => { + assert!(msg.contains("halt-on-error pattern matched")); + } + other => panic!("expected error, got {:?}", other), + } +} + +// ----------------------------------------------------------------------- +// SimAVR runner tests (use fake process, no real simavr needed) +// ----------------------------------------------------------------------- + +#[tokio::test] +async fn simavr_runner_captures_stdout_via_process_runner() { + // SimavrRunner delegates to run_qemu_process, so we verify the same + // subprocess monitoring path works for simavr-style output. + let (exe, args) = test_process_command(&["Hello from ATmega2560!"]); + let result = run_qemu_process( + &exe, + &args, + RunQemuOptions { + elf_path: None, + addr2line_path: None, + timeout_secs: Some(2.0), + halt_on_error: None, + halt_on_success: None, + expect: Some("Hello from ATmega2560"), + show_timestamp: false, + verbose: false, + process_label: "simavr", + }, + ) + .await + .unwrap(); + + assert!(result.stdout.contains("Hello from ATmega2560!")); + match result.outcome { + MonitorOutcome::Success(_) => {} + other => panic!("expected success, got {:?}", other), + } +} + +#[tokio::test] +async fn simavr_runner_halt_on_success() { + let (exe, args) = test_process_command(&["simavr: init", "PASS: all tests passed", "done"]); + let result = run_qemu_process( + &exe, + &args, + RunQemuOptions { + elf_path: None, + addr2line_path: None, + timeout_secs: Some(2.0), + halt_on_error: None, + halt_on_success: Some("PASS:"), + expect: None, + show_timestamp: false, + verbose: false, + process_label: "simavr", + }, + ) + .await + .unwrap(); + + match result.outcome { + MonitorOutcome::Success(msg) => { + assert!(msg.contains("halt-on-success pattern matched")); + } + other => panic!("expected success, got {:?}", other), + } +} + +#[tokio::test] +async fn simavr_runner_halt_on_error() { + let (exe, args) = test_process_command(&["simavr: init", "FAIL: assertion failed"]); + let result = run_qemu_process( + &exe, + &args, + RunQemuOptions { + elf_path: None, + addr2line_path: None, + timeout_secs: Some(2.0), + halt_on_error: Some("FAIL:"), + halt_on_success: None, + expect: None, + show_timestamp: false, + verbose: false, + process_label: "simavr", + }, + ) + .await + .unwrap(); + + match result.outcome { + MonitorOutcome::Error(msg) => { + assert!(msg.contains("halt-on-error pattern matched")); + } + other => panic!("expected error, got {:?}", other), + } +} + +#[tokio::test] +async fn simavr_runner_timeout() { + let (exe, args) = test_process_command(&["simavr: init", "running..."]); + let result = run_qemu_process( + &exe, + &args, + RunQemuOptions { + elf_path: None, + addr2line_path: None, + timeout_secs: Some(0.1), + halt_on_error: None, + halt_on_success: Some("PASS:"), + expect: None, + show_timestamp: false, + verbose: false, + process_label: "simavr", + }, + ) + .await + .unwrap(); + + // Process exits quickly so the outcome depends on whether halt pattern matched + // before the process ended — either timeout or a normal exit without the pattern. + match result.outcome { + MonitorOutcome::Success(_) => { + // Process exited cleanly before timeout, expect not set so counts as success + } + MonitorOutcome::Timeout { .. } => { + // Timed out waiting for halt-on-success pattern — also valid + } + MonitorOutcome::Error(_) => { + // Process exited before halt pattern found — also acceptable + } + } +} diff --git a/crates/fbuild-daemon/src/handlers/emulator/tests_select_runner.rs b/crates/fbuild-daemon/src/handlers/emulator/tests_select_runner.rs new file mode 100644 index 00000000..08434ecb --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/emulator/tests_select_runner.rs @@ -0,0 +1,303 @@ +//! Tests for `select_runner` and the `is_qemu_supported_esp32_mcu` helper. + +use super::qemu_deploy::is_qemu_supported_esp32_mcu; +use super::runners::SimavrRunner; +use super::select::select_runner; +use std::collections::HashMap; +use std::path::Path; + +// ----------------------------------------------------------------------- +// select_runner tests for simavr +// ----------------------------------------------------------------------- + +#[test] +fn select_runner_explicit_simavr_for_uno() { + let result = select_runner( + Path::new("/tmp/test"), + "uno", + fbuild_core::Platform::AtmelAvr, + "uno", + &HashMap::new(), + Some("simavr"), + ); + assert!(result.is_ok(), "select_runner should accept simavr for uno"); + assert_eq!(result.unwrap().name(), "simavr"); +} + +#[test] +fn select_runner_explicit_simavr_for_mega() { + let result = select_runner( + Path::new("/tmp/test"), + "megaatmega2560", + fbuild_core::Platform::AtmelAvr, + "megaatmega2560", + &HashMap::new(), + Some("simavr"), + ); + assert!( + result.is_ok(), + "select_runner should accept simavr for mega" + ); + assert_eq!(result.unwrap().name(), "simavr"); +} + +#[test] +fn select_runner_explicit_simavr_for_leonardo() { + let result = select_runner( + Path::new("/tmp/test"), + "leonardo", + fbuild_core::Platform::AtmelAvr, + "leonardo", + &HashMap::new(), + Some("simavr"), + ); + assert!( + result.is_ok(), + "select_runner should accept simavr for leonardo" + ); + assert_eq!(result.unwrap().name(), "simavr"); +} + +#[test] +fn select_runner_explicit_simavr_rejects_esp32() { + let result = select_runner( + Path::new("/tmp/test"), + "esp32dev", + fbuild_core::Platform::Espressif32, + "esp32dev", + &HashMap::new(), + Some("simavr"), + ); + assert!(result.is_err(), "simavr should reject ESP32 boards"); +} + +#[test] +fn select_runner_auto_detects_simavr_for_mega() { + // ATmega2560 should auto-detect simavr since it's not ATmega328P + let result = select_runner( + Path::new("/tmp/test"), + "megaatmega2560", + fbuild_core::Platform::AtmelAvr, + "megaatmega2560", + &HashMap::new(), + None, + ); + assert!( + result.is_ok(), + "auto-detect should find simavr for mega: {:?}", + result.err() + ); + assert_eq!(result.unwrap().name(), "simavr"); +} + +#[test] +fn select_runner_auto_detects_simavr_for_leonardo() { + let result = select_runner( + Path::new("/tmp/test"), + "leonardo", + fbuild_core::Platform::AtmelAvr, + "leonardo", + &HashMap::new(), + None, + ); + assert!( + result.is_ok(), + "auto-detect should find simavr for leonardo: {:?}", + result.err() + ); + assert_eq!(result.unwrap().name(), "simavr"); +} + +#[test] +fn select_runner_auto_detects_avr8js_for_uno() { + // ATmega328P should still default to avr8js, not simavr + let result = select_runner( + Path::new("/tmp/test"), + "uno", + fbuild_core::Platform::AtmelAvr, + "uno", + &HashMap::new(), + None, + ); + assert!(result.is_ok()); + assert_eq!( + result.unwrap().name(), + "avr8js ATmega328P", + "ATmega328P should default to avr8js" + ); +} + +#[test] +fn select_runner_simavr_name_matches() { + use super::runners::EmulatorRunner; + let board = + fbuild_config::BoardConfig::from_board_id("megaatmega2560", &HashMap::new()).unwrap(); + let runner = SimavrRunner::new(board); + assert_eq!(runner.name(), "simavr"); +} + +// ----------------------------------------------------------------------- +// select_runner tests for ESP32 QEMU (Issue #25) +// ----------------------------------------------------------------------- + +#[test] +fn select_runner_explicit_qemu_for_esp32dev() { + let result = select_runner( + Path::new("/tmp/test"), + "esp32dev", + fbuild_core::Platform::Espressif32, + "esp32dev", + &HashMap::new(), + Some("qemu"), + ); + assert!( + result.is_ok(), + "select_runner should accept qemu for esp32dev: {:?}", + result.err() + ); + assert_eq!(result.unwrap().name(), "QEMU ESP32"); +} + +#[test] +fn select_runner_explicit_qemu_for_esp32s3() { + let result = select_runner( + Path::new("/tmp/test"), + "esp32-s3-devkitc-1", + fbuild_core::Platform::Espressif32, + "esp32-s3-devkitc-1", + &HashMap::new(), + Some("qemu"), + ); + assert!( + result.is_ok(), + "select_runner should accept qemu for esp32-s3: {:?}", + result.err() + ); + assert_eq!(result.unwrap().name(), "QEMU ESP32S3"); +} + +#[test] +fn select_runner_explicit_qemu_accepts_esp32c3() { + let result = select_runner( + Path::new("/tmp/test"), + "esp32-c3-devkitm-1", + fbuild_core::Platform::Espressif32, + "esp32-c3-devkitm-1", + &HashMap::new(), + Some("qemu"), + ); + assert!( + result.is_ok(), + "QEMU should accept ESP32-C3 via qemu-system-riscv32: {:?}", + result.err() + ); +} + +#[test] +fn select_runner_explicit_qemu_accepts_esp32c6() { + let result = select_runner( + Path::new("/tmp/test"), + "esp32-c6-devkitc-1", + fbuild_core::Platform::Espressif32, + "esp32-c6-devkitc-1", + &HashMap::new(), + Some("qemu"), + ); + assert!( + result.is_ok(), + "QEMU should accept ESP32-C6 via qemu-system-riscv32: {:?}", + result.err() + ); +} + +#[test] +fn select_runner_explicit_qemu_accepts_esp32h2() { + let result = select_runner( + Path::new("/tmp/test"), + "esp32-h2-devkitm-1", + fbuild_core::Platform::Espressif32, + "esp32-h2-devkitm-1", + &HashMap::new(), + Some("qemu"), + ); + assert!( + result.is_ok(), + "QEMU should accept ESP32-H2 via qemu-system-riscv32: {:?}", + result.err() + ); +} + +#[test] +fn select_runner_explicit_qemu_rejects_esp32s2() { + let result = select_runner( + Path::new("/tmp/test"), + "esp32-s2-saola-1", + fbuild_core::Platform::Espressif32, + "esp32-s2-saola-1", + &HashMap::new(), + Some("qemu"), + ); + assert!( + result.is_err(), + "QEMU should reject ESP32-S2 (not emulated upstream)" + ); +} + +#[test] +fn select_runner_auto_detects_qemu_for_esp32dev() { + let result = select_runner( + Path::new("/tmp/test"), + "esp32dev", + fbuild_core::Platform::Espressif32, + "esp32dev", + &HashMap::new(), + None, + ); + assert!( + result.is_ok(), + "auto-detect should find qemu for esp32dev: {:?}", + result.err() + ); + assert_eq!(result.unwrap().name(), "QEMU ESP32"); +} + +#[test] +fn select_runner_auto_detects_qemu_for_esp32s3() { + let result = select_runner( + Path::new("/tmp/test"), + "esp32-s3-devkitc-1", + fbuild_core::Platform::Espressif32, + "esp32-s3-devkitc-1", + &HashMap::new(), + None, + ); + assert!( + result.is_ok(), + "auto-detect should find qemu for esp32-s3: {:?}", + result.err() + ); + assert_eq!(result.unwrap().name(), "QEMU ESP32S3"); +} + +#[test] +fn is_qemu_supported_esp32_mcu_accepts_xtensa() { + assert!(is_qemu_supported_esp32_mcu("esp32")); + assert!(is_qemu_supported_esp32_mcu("ESP32")); + assert!(is_qemu_supported_esp32_mcu("esp32s3")); + assert!(is_qemu_supported_esp32_mcu("ESP32S3")); +} + +#[test] +fn is_qemu_supported_esp32_mcu_accepts_riscv() { + assert!(is_qemu_supported_esp32_mcu("esp32c3")); + assert!(is_qemu_supported_esp32_mcu("ESP32C3")); + assert!(is_qemu_supported_esp32_mcu("esp32c6")); + assert!(is_qemu_supported_esp32_mcu("esp32h2")); +} + +#[test] +fn is_qemu_supported_esp32_mcu_rejects_unsupported() { + assert!(!is_qemu_supported_esp32_mcu("esp32s2")); + assert!(!is_qemu_supported_esp32_mcu("atmega328p")); + assert!(!is_qemu_supported_esp32_mcu("esp32p4")); +} diff --git a/crates/fbuild-daemon/src/handlers/operations.rs b/crates/fbuild-daemon/src/handlers/operations.rs deleted file mode 100644 index d7f45ecf..00000000 --- a/crates/fbuild-daemon/src/handlers/operations.rs +++ /dev/null @@ -1,2366 +0,0 @@ -//! Build, deploy, and monitor operation handlers. - -use crate::context::DaemonContext; -use crate::models::{ - BuildRequest, DeployRequest, InstallDepsRequest, MonitorRequest, OperationResponse, - ResetRequest, -}; -use axum::extract::State; -use axum::http::StatusCode; -use axum::Json; -use serde::Serialize; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::atomic::Ordering; -use std::sync::Arc; - -/// Returns `true` when the daemon should route ESP32 `verify-flash` -/// pre-checks through the native [`espflash`] crate (issue #66) instead -/// of the Python `esptool` subprocess. -/// -/// Controlled by the `FBUILD_USE_ESPFLASH_VERIFY` environment variable. -/// Native verify is enabled by default when compiled in. -/// Set this variable to `0`, `false`, `no`, or `off` (case-insensitive) -/// to force esptool. -#[cfg(feature = "espflash-native")] -pub(crate) fn native_verify_enabled() -> bool { - env_default_enabled("FBUILD_USE_ESPFLASH_VERIFY") -} - -/// Returns `true` when the daemon should trust an in-memory firmware -/// hash (keyed by port) and skip the `verify-flash` MD5 round-trip on -/// a warm redeploy. -/// -/// Safety model: the hash is only honoured if the port has been -/// continuously enumerated since the hash was recorded (enforced by -/// [`DeviceManager::trusted_firmware_hash`]). If the user unplugged -/// the board and something else flashed it before it came back, the -/// disconnect edge invalidates the cached hash and the deploy falls -/// through to the normal verify-flash path. -/// -/// Controlled by the `FBUILD_TRUST_DEVICE_HASH` environment variable -/// (set to `1`, `true`, `yes`, or `on` — case-insensitive). Default -/// off while the path accumulates bench time, mirroring the opt-in -/// convention used by `FBUILD_USE_ESPFLASH_{VERIFY,WRITE}`. -pub(crate) fn trust_device_hash_enabled() -> bool { - match std::env::var("FBUILD_TRUST_DEVICE_HASH") { - Ok(v) => matches!( - v.trim().to_ascii_lowercase().as_str(), - "1" | "true" | "yes" | "on" - ), - Err(_) => false, - } -} - -/// Compute a stable SHA-256 over the three ESP32 flash regions the -/// daemon would otherwise MD5 with `verify-flash`. Hashes the tuple -/// `(offset_le, len_le, bytes)` for each region in fixed order -/// (bootloader → partitions → firmware) so the digest uniquely -/// identifies the image that would be written; two builds of the -/// same source with identical output hash to the same value. -/// -/// Memoized on [`crate::context::DaemonContext::image_hash_memo`] -/// keyed by firmware path: if all three files' `mtime` matches the -/// previously-stored tuple, the cached hash is reused (skipping the -/// 2–4 MB disk read + SHA-256) — the dominant non-serial cost on the -/// trust-skip path. Cache entries self-invalidate when any `mtime` -/// advances. -/// -/// Returns `None` if any of the three files is missing on disk, so -/// the caller treats it as "can't trust-skip, fall through to -/// verify-flash." -pub(crate) fn compute_esp32_image_hash( - ctx: &crate::context::DaemonContext, - firmware_path: &std::path::Path, - bootloader_offset: u32, - partitions_offset: u32, - firmware_offset: u32, -) -> Option<[u8; 32]> { - use sha2::{Digest, Sha256}; - let build_dir = firmware_path.parent()?; - let bootloader_path = build_dir.join("bootloader.bin"); - let partitions_path = build_dir.join("partitions.bin"); - let firmware = firmware_path.to_path_buf(); - - let mt = |p: &std::path::Path| -> Option { - std::fs::metadata(p).ok()?.modified().ok() - }; - let mtimes = (mt(&bootloader_path)?, mt(&partitions_path)?, mt(&firmware)?); - - // Fast path: the three files have the same `mtime` as last time - // we hashed them, so the output bytes are unchanged. Reuse the - // stored digest instead of re-reading + re-hashing (~5-15 ms). - if let Some(memo) = ctx.image_hash_memo.get(&firmware) { - if memo.bootloader_mtime == mtimes.0 - && memo.partitions_mtime == mtimes.1 - && memo.firmware_mtime == mtimes.2 - { - return Some(memo.hash); - } - } - - // Miss: rebuild the digest over the current file contents and - // record it alongside the captured `mtime`s. - let regions: [(u32, &std::path::Path); 3] = [ - (bootloader_offset, bootloader_path.as_path()), - (partitions_offset, partitions_path.as_path()), - (firmware_offset, firmware.as_path()), - ]; - let mut hasher = Sha256::new(); - for (offset, path) in ®ions { - let bytes = std::fs::read(path).ok()?; - hasher.update(offset.to_le_bytes()); - hasher.update((bytes.len() as u64).to_le_bytes()); - hasher.update(&bytes); - } - let hash: [u8; 32] = hasher.finalize().into(); - ctx.image_hash_memo.insert( - firmware.clone(), - crate::context::ImageHashMemo { - bootloader_mtime: mtimes.0, - partitions_mtime: mtimes.1, - firmware_mtime: mtimes.2, - hash, - }, - ); - Some(hash) -} - -/// Returns `true` when the daemon should route ESP32 `write-flash` -/// through the native [`espflash`] crate (issue #66) instead of the -/// Python `esptool` subprocess. -/// -/// Controlled by the `FBUILD_USE_ESPFLASH_WRITE` environment variable. -/// Native write is enabled by default when compiled in. Independent of -/// `FBUILD_USE_ESPFLASH_VERIFY`. Set it to `0`, `false`, `no`, or `off` -/// (case-insensitive) to force esptool. -#[cfg(feature = "espflash-native")] -pub(crate) fn native_write_enabled() -> bool { - env_default_enabled("FBUILD_USE_ESPFLASH_WRITE") -} - -#[cfg(feature = "espflash-native")] -fn env_default_enabled(name: &str) -> bool { - match std::env::var(name) { - Ok(v) => !matches!( - v.trim().to_ascii_lowercase().as_str(), - "0" | "false" | "no" | "off" - ), - Err(_) => true, - } -} - -pub(crate) fn qemu_extra_build_flags(platform: fbuild_core::Platform, mcu: &str) -> Vec { - if platform == fbuild_core::Platform::Espressif32 && mcu.eq_ignore_ascii_case("esp32s3") { - vec![ - "-DARDUINO_USB_MODE=0".to_string(), - "-DARDUINO_USB_CDC_ON_BOOT=0".to_string(), - ] - } else { - Vec::new() - } -} - -/// RAII guard that sets `operation_in_progress` to true on creation -/// and false on drop. Also tracks daemon state and current operation description. -pub(crate) struct OperationGuard { - flag: Arc, - state: Arc>, - operation: Arc>>, -} - -impl OperationGuard { - pub(crate) fn new( - ctx: &DaemonContext, - daemon_state: fbuild_core::DaemonState, - description: Option, - ) -> Self { - ctx.touch_activity(); - ctx.operation_in_progress.store(true, Ordering::Relaxed); - if let Ok(mut s) = ctx.daemon_state.write() { - *s = daemon_state; - } - if let Ok(mut op) = ctx.current_operation.write() { - *op = description; - } - Self { - flag: Arc::clone(&ctx.operation_in_progress), - state: Arc::clone(&ctx.daemon_state), - operation: Arc::clone(&ctx.current_operation), - } - } -} - -impl Drop for OperationGuard { - fn drop(&mut self) { - self.flag.store(false, Ordering::Relaxed); - if let Ok(mut s) = self.state.write() { - *s = fbuild_core::DaemonState::Idle; - } - if let Ok(mut op) = self.operation.write() { - *op = None; - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum EmulatorKind { - Qemu, - Avr8js, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum DeployRoute { - Device, - Emulator(EmulatorKind), -} - -fn parse_emulator_kind(raw: &str) -> fbuild_core::Result { - match raw { - "qemu" => Ok(EmulatorKind::Qemu), - "avr8js" => Ok(EmulatorKind::Avr8js), - other => Err(fbuild_core::FbuildError::DeployFailed(format!( - "unsupported emulator '{}'", - other - ))), - } -} - -fn infer_default_emulator_kind(platform: fbuild_core::Platform, mcu: &str) -> Option { - match platform { - fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr => { - Some(EmulatorKind::Avr8js) - } - fbuild_core::Platform::Espressif32 if mcu.eq_ignore_ascii_case("esp32s3") => { - Some(EmulatorKind::Qemu) - } - _ => None, - } -} - -fn parse_deploy_route( - req: &DeployRequest, - default_emulator: Option, -) -> fbuild_core::Result { - if let Some(target) = req.target.as_deref() { - return match target { - "device" => Ok(DeployRoute::Device), - "qemu" => Ok(DeployRoute::Emulator(EmulatorKind::Qemu)), - "avr8js" => Ok(DeployRoute::Emulator(EmulatorKind::Avr8js)), - other => Err(fbuild_core::FbuildError::DeployFailed(format!( - "unsupported deploy target '{}'", - other - ))), - }; - } - - let destination = req.to.as_deref().unwrap_or("device"); - match destination { - "device" => { - if req.qemu { - return Err(fbuild_core::FbuildError::DeployFailed( - "--qemu cannot be combined with --to device".to_string(), - )); - } - if let Some(emulator) = req.emulator.as_deref() { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "--emulator {} requires --to emu", - emulator - ))); - } - Ok(DeployRoute::Device) - } - "emu" | "emulator" => { - let emulator = if req.qemu { - if let Some(explicit) = req.emulator.as_deref() { - if explicit != "qemu" { - return Err(fbuild_core::FbuildError::DeployFailed( - "--qemu cannot be combined with a different --emulator".to_string(), - )); - } - } - "qemu" - } else { - match req.emulator.as_deref() { - Some(explicit) => explicit, - None => match default_emulator { - Some(EmulatorKind::Qemu) => "qemu", - Some(EmulatorKind::Avr8js) => "avr8js", - None => { - return Err(fbuild_core::FbuildError::DeployFailed( - "--to emu requires an explicit --emulator for this board" - .to_string(), - )) - } - }, - } - }; - Ok(DeployRoute::Emulator(parse_emulator_kind(emulator)?)) - } - other => Err(fbuild_core::FbuildError::DeployFailed(format!( - "unsupported deploy destination '{}'", - other - ))), - } -} - -fn resolve_client_path(raw: &str, caller_cwd: Option<&str>, project_dir: &Path) -> PathBuf { - let path = PathBuf::from(raw); - if path.is_absolute() { - path - } else if let Some(cwd) = caller_cwd { - PathBuf::from(cwd).join(path) - } else { - project_dir.join(path) - } -} - -#[derive(Debug, Serialize)] -struct ArtifactFileEntry { - name: String, - role: String, -} - -#[derive(Debug, Serialize)] -struct ArtifactManifest { - platform: String, - environment: String, - primary_firmware: Option, - elf: Option, - files: Vec, -} - -struct ArtifactExportResult { - output_dir: PathBuf, - primary_output: Option, -} - -fn artifact_role(name: &str, primary_firmware: Option<&Path>, elf_path: Option<&Path>) -> String { - if primary_firmware - .and_then(|p| p.file_name()) - .is_some_and(|n| n == name) - { - "firmware".to_string() - } else if elf_path - .and_then(|p| p.file_name()) - .is_some_and(|n| n == name) - { - "elf".to_string() - } else { - match name { - "bootloader.bin" => "bootloader".to_string(), - "partitions.bin" => "partitions".to_string(), - "compile_commands.json" => "compile_database".to_string(), - "symbol_analysis.txt" => "symbol_analysis".to_string(), - _ => "artifact".to_string(), - } - } -} - -fn export_artifacts_bundle( - output_dir: &Path, - platform: fbuild_core::Platform, - env_name: &str, - primary_firmware: Option<&Path>, - elf_path: Option<&Path>, -) -> fbuild_core::Result { - std::fs::create_dir_all(output_dir)?; - - let source_dir = primary_firmware - .and_then(|p| p.parent()) - .or_else(|| elf_path.and_then(|p| p.parent())) - .ok_or_else(|| { - fbuild_core::FbuildError::Other( - "could not determine source artifact directory for export".to_string(), - ) - })?; - - let mut copied_names = Vec::new(); - for entry in std::fs::read_dir(source_dir)? { - let entry = entry?; - let path = entry.path(); - if !path.is_file() { - continue; - } - let file_name = match path.file_name() { - Some(name) => name, - None => continue, - }; - let dest = output_dir.join(file_name); - if path != dest { - std::fs::copy(&path, &dest)?; - } - copied_names.push(file_name.to_string_lossy().to_string()); - } - copied_names.sort(); - copied_names.dedup(); - - let manifest = ArtifactManifest { - platform: format!("{:?}", platform), - environment: env_name.to_string(), - primary_firmware: primary_firmware - .and_then(|p| p.file_name()) - .map(|p| p.to_string_lossy().to_string()), - elf: elf_path - .and_then(|p| p.file_name()) - .map(|p| p.to_string_lossy().to_string()), - files: copied_names - .iter() - .map(|name| ArtifactFileEntry { - name: name.clone(), - role: artifact_role(name, primary_firmware, elf_path), - }) - .collect(), - }; - - std::fs::write( - output_dir.join("artifacts.json"), - serde_json::to_vec_pretty(&manifest).map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to serialize artifact manifest: {}", e)) - })?, - )?; - - let primary_output = primary_firmware - .and_then(|p| p.file_name()) - .map(|name| output_dir.join(name)); - - Ok(ArtifactExportResult { - output_dir: output_dir.to_path_buf(), - primary_output, - }) -} - -/// POST /api/build -pub async fn build( - State(ctx): State>, - Json(req): Json, -) -> axum::response::Response { - use axum::response::IntoResponse; - - let request_id = req - .request_id - .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); - let project_dir = PathBuf::from(&req.project_dir); - let stream = req.stream; - - if !project_dir.exists() { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("project directory does not exist: {}", req.project_dir), - )), - ) - .into_response(); - } - - // Validation: parse config, resolve platform - let config = - match fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")) { - Ok(c) => c, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("failed to parse platformio.ini: {}", e), - )), - ) - .into_response(); - } - }; - - let env_name = req - .environment - .clone() - .or_else(|| config.get_default_environment().map(|s| s.to_string())) - .unwrap_or_else(|| "default".to_string()); - - let env_config = match config.get_env_config(&env_name) { - Ok(c) => c, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("invalid environment '{}': {}", env_name, e), - )), - ) - .into_response(); - } - }; - - let platform_str = env_config.get("platform").cloned().unwrap_or_default(); - let platform = match fbuild_core::Platform::from_platform_str(&platform_str) { - Some(p) => p, - None => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("unsupported platform: {}", platform_str), - )), - ) - .into_response(); - } - }; - - let profile = match req.profile.as_deref() { - Some("quick") => fbuild_core::BuildProfile::Quick, - _ => fbuild_core::BuildProfile::Release, - }; - - let build_dir = fbuild_paths::get_project_build_root(&project_dir); - let compiledb_env = std::env::var("FBUILD_COMPILEDB") - .map(|v| v != "0") - .unwrap_or(true); - let generate_compiledb = req.generate_compiledb || compiledb_env; - let resolved_symbol_analysis_path = req - .symbol_analysis_path - .as_deref() - .map(|p| resolve_client_path(p, req.caller_cwd.as_deref(), &project_dir)); - let resolved_output_dir = req - .output_dir - .as_deref() - .map(|p| resolve_client_path(p, req.caller_cwd.as_deref(), &project_dir)); - - if stream { - // --- STREAMING PATH --- - // Build runs in a background task; log lines stream to client as NDJSON. - let (sync_tx, sync_rx) = std::sync::mpsc::channel::(); - let (async_tx, async_rx) = tokio::sync::mpsc::unbounded_channel::(); - - let params = fbuild_build::BuildParams { - project_dir: project_dir.clone(), - env_name: env_name.clone(), - clean: req.clean_build, - profile, - build_dir, - verbose: req.verbose, - jobs: req.jobs, - generate_compiledb, - compiledb_only: req.compiledb_only, - log_sender: Some(sync_tx), - symbol_analysis: req.symbol_analysis, - symbol_analysis_path: resolved_symbol_analysis_path.clone(), - no_timestamp: req.no_timestamp, - src_dir: req.src_dir.clone(), - pio_env: req.pio_env.clone(), - extra_build_flags: Vec::new(), - watch_set_cache: Some(Arc::clone(&ctx.watch_set_cache) as Arc<_>), - }; - - let project_dir_desc = req.project_dir.clone(); - tokio::spawn(async move { - // FBUILD_PERF_LOG=1 enables daemon-side coarse phase timing - // (lock-wait + build). Zero overhead when unset. - let perf_enabled = std::env::var("FBUILD_PERF_LOG") - .map(|v| !v.is_empty() && v != "0") - .unwrap_or(false); - let daemon_start = std::time::Instant::now(); - - let _op_guard = OperationGuard::new( - &ctx, - fbuild_core::DaemonState::Building, - Some(format!("Building {}", project_dir_desc)), - ); - let lock_wait_start = std::time::Instant::now(); - let lock = ctx.project_lock(&project_dir); - let _lock_guard = lock.lock().await; - let lock_wait = lock_wait_start.elapsed(); - - // Bridge: sync log lines → async NDJSON chunks - let bridge_tx = async_tx.clone(); - let bridge = tokio::task::spawn_blocking(move || { - for line in sync_rx { - let event = serde_json::json!({"type": "log", "message": line}); - let mut chunk = event.to_string(); - chunk.push('\n'); - if bridge_tx.send(bytes::Bytes::from(chunk)).is_err() { - break; - } - } - }); - - // Run build - let build_wallclock_start = std::time::Instant::now(); - let build_result = tokio::task::spawn_blocking(move || { - let orchestrator = fbuild_build::get_orchestrator(platform)?; - orchestrator.build(¶ms) - }) - .await; - let build_wallclock = build_wallclock_start.elapsed(); - if perf_enabled { - let summary = format!( - "[perf-log daemon-handler] lock-wait={} ms, build-wallclock={} ms, total={} ms", - lock_wait.as_millis(), - build_wallclock.as_millis(), - daemon_start.elapsed().as_millis(), - ); - tracing::info!(target: "fbuild_daemon::perf_log", "{}", summary); - eprintln!("{}", summary); - } - - // Extract result (drops BuildLog sender so bridge can finish) - let (success, rid, msg, code, output_file, output_dir) = match build_result { - Ok(Ok(br)) => { - let exported = if br.success { - if let Some(ref out_dir) = resolved_output_dir { - Some(export_artifacts_bundle( - out_dir, - platform, - &env_name, - br.firmware_path.as_deref(), - br.elf_path.as_deref(), - )) - } else { - None - } - } else { - None - }; - let _lines = br.build_log.into_lines(); // drop sender - let summary = if br.success { - let size_str = br - .size_info - .as_ref() - .map(|s| { - format!( - " (flash: {} bytes, ram: {} bytes)", - s.total_flash, s.total_ram - ) - }) - .unwrap_or_default(); - let export_suffix = match exported.as_ref() { - Some(Ok(result)) => { - format!("; artifacts exported to {}", result.output_dir.display()) - } - Some(Err(e)) => { - format!("; artifact export failed: {}", e) - } - None => String::new(), - }; - format!( - "build succeeded in {:.1}s{}{}", - br.build_time_secs, size_str, export_suffix - ) - } else { - br.message.clone() - }; - let output = match exported.as_ref() { - Some(Ok(result)) => result - .primary_output - .clone() - .or(br.firmware_path.clone()) - .or(br.elf_path.clone()), - _ => br.firmware_path.clone().or(br.elf_path.clone()), - } - .map(|p| p.to_string_lossy().to_string()); - let output_dir = match exported.as_ref() { - Some(Ok(result)) => Some(result.output_dir.to_string_lossy().to_string()), - _ => None, - }; - let c = if br.success { 0 } else { 1 }; - ( - br.success, - request_id.clone(), - summary, - c, - output, - output_dir, - ) - } - Ok(Err(e)) => ( - false, - request_id.clone(), - format!("build error: {}", e), - 1, - None, - None, - ), - Err(e) => ( - false, - request_id.clone(), - format!("build task panicked: {}", e), - 1, - None, - None, - ), - }; - - let _ = bridge.await; - - let result_event = serde_json::json!({ - "type": "result", - "success": success, - "request_id": rid, - "message": msg, - "exit_code": code, - "output_file": output_file, - "output_dir": output_dir, - }); - let mut chunk = result_event.to_string(); - chunk.push('\n'); - let _ = async_tx.send(bytes::Bytes::from(chunk)); - }); - - // Return streaming response immediately - let stream = futures::stream::unfold(async_rx, |mut rx| async move { - rx.recv() - .await - .map(|data| (Ok::<_, std::convert::Infallible>(data), rx)) - }); - let body = axum::body::Body::from_stream(stream); - axum::response::Response::builder() - .header("content-type", "application/x-ndjson") - .body(body) - .unwrap() - .into_response() - } else { - // --- NON-STREAMING PATH (existing behavior) --- - let _op_guard = OperationGuard::new( - &ctx, - fbuild_core::DaemonState::Building, - Some(format!("Building {}", req.project_dir)), - ); - let lock = ctx.project_lock(&project_dir); - let _guard = lock.lock().await; - - let params = fbuild_build::BuildParams { - project_dir: project_dir.clone(), - env_name: env_name.clone(), - clean: req.clean_build, - profile, - build_dir, - verbose: req.verbose, - jobs: req.jobs, - generate_compiledb, - compiledb_only: req.compiledb_only, - log_sender: None, - symbol_analysis: req.symbol_analysis, - symbol_analysis_path: resolved_symbol_analysis_path, - no_timestamp: req.no_timestamp, - src_dir: req.src_dir, - pio_env: req.pio_env, - extra_build_flags: Vec::new(), - watch_set_cache: Some(Arc::clone(&ctx.watch_set_cache) as Arc<_>), - }; - - let result = tokio::task::spawn_blocking(move || { - let orchestrator = fbuild_build::get_orchestrator(platform)?; - orchestrator.build(¶ms) - }) - .await; - - match result { - Ok(Ok(build_result)) => { - let exported = if build_result.success { - if let Some(ref out_dir) = resolved_output_dir { - Some(export_artifacts_bundle( - out_dir, - platform, - &env_name, - build_result.firmware_path.as_deref(), - build_result.elf_path.as_deref(), - )) - } else { - None - } - } else { - None - }; - let summary = if build_result.success { - let size_str = build_result - .size_info - .as_ref() - .map(|s| { - format!( - " (flash: {} bytes, ram: {} bytes)", - s.total_flash, s.total_ram - ) - }) - .unwrap_or_default(); - let export_suffix = match exported.as_ref() { - Some(Ok(result)) => { - format!("; artifacts exported to {}", result.output_dir.display()) - } - Some(Err(e)) => format!("; artifact export failed: {}", e), - None => String::new(), - }; - format!( - "build succeeded in {:.1}s{}{}", - build_result.build_time_secs, size_str, export_suffix - ) - } else { - build_result.message.clone() - }; - let msg = if build_result.build_log.is_empty() { - summary - } else { - let mut lines = build_result.build_log.into_lines(); - lines.push(summary); - lines.join("\n") - }; - let output_file = match exported.as_ref() { - Some(Ok(result)) => result - .primary_output - .clone() - .or(build_result.firmware_path.clone()) - .or(build_result.elf_path.clone()), - _ => build_result - .firmware_path - .clone() - .or(build_result.elf_path.clone()), - } - .map(|p| p.to_string_lossy().to_string()); - let output_dir = match exported.as_ref() { - Some(Ok(result)) => Some(result.output_dir.to_string_lossy().to_string()), - _ => None, - }; - let code = if build_result.success { 0 } else { 1 }; - ( - StatusCode::OK, - Json(OperationResponse { - success: build_result.success, - request_id, - message: msg, - exit_code: code, - output_file, - output_dir, - launch_url: None, - stdout: None, - stderr: None, - }), - ) - .into_response() - } - Ok(Err(e)) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("build error: {}", e), - )), - ) - .into_response(), - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("build task panicked: {}", e), - )), - ) - .into_response(), - } - } -} - -/// POST /api/deploy -pub async fn deploy( - State(ctx): State>, - Json(req): Json, -) -> (StatusCode, Json) { - let request_id = req - .request_id - .clone() - .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); - let project_dir = PathBuf::from(&req.project_dir); - - if !project_dir.exists() { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("project directory does not exist: {}", req.project_dir), - )), - ); - } - - let _op_guard = OperationGuard::new( - &ctx, - fbuild_core::DaemonState::Deploying, - Some(format!("Deploying {}", req.project_dir)), - ); - - // Parse config - let config = - match fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")) { - Ok(c) => c, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("failed to parse platformio.ini: {}", e), - )), - ); - } - }; - - let env_name = req - .environment - .clone() - .or_else(|| config.get_default_environment().map(|s| s.to_string())) - .unwrap_or_else(|| "default".to_string()); - - let env_config = match config.get_env_config(&env_name) { - Ok(c) => c, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("invalid environment '{}': {}", env_name, e), - )), - ); - } - }; - let resolved_output_dir = req - .output_dir - .as_deref() - .map(|p| resolve_client_path(p, req.caller_cwd.as_deref(), &project_dir)); - - let platform_str = env_config.get("platform").cloned().unwrap_or_default(); - let platform = match fbuild_core::Platform::from_platform_str(&platform_str) { - Some(p) => p, - None => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("unsupported platform: {}", platform_str), - )), - ); - } - }; - let board_id = env_config.get("board").cloned().unwrap_or_else(|| { - fbuild_build::get_platform_support(platform) - .map(|s| s.default_board_id().to_string()) - .unwrap_or_else(|_| "unknown".to_string()) - }); - let board_overrides = config.get_board_overrides(&env_name).unwrap_or_default(); - let board = fbuild_config::BoardConfig::from_board_id(&board_id, &board_overrides) - .or_else(|_| fbuild_config::BoardConfig::from_board_id(&board_id, &HashMap::new())) - .ok(); - let deploy_route = match parse_deploy_route( - &req, - board - .as_ref() - .and_then(|board| infer_default_emulator_kind(platform, &board.mcu)), - ) { - Ok(route) => route, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail(request_id, e.to_string())), - ); - } - }; - - // Build first unless skip_build - let (firmware_path, elf_path) = if req.skip_build { - // Look for existing firmware using the standard search order - // (profiles: release/quick, base env dir, legacy .pio/build) - match fbuild_paths::find_firmware(&project_dir, &env_name, None) { - Some(path) => { - let elf = path.parent().map(|dir| dir.join("firmware.elf")); - let elf = elf.filter(|p| p.exists()); - (path, elf) - } - None => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - "no firmware found; run build first or remove skip_build".to_string(), - )), - ); - } - } - } else { - // Run build first - let lock = ctx.project_lock(&project_dir); - let _guard = lock.lock().await; - - let build_dir = fbuild_paths::get_project_build_root(&project_dir); - let params = fbuild_build::BuildParams { - project_dir: project_dir.clone(), - env_name: env_name.clone(), - clean: req.clean_build, - profile: fbuild_core::BuildProfile::Release, - build_dir, - verbose: req.verbose, - jobs: None, - generate_compiledb: false, - compiledb_only: false, - log_sender: None, - symbol_analysis: false, - symbol_analysis_path: None, - no_timestamp: false, - src_dir: req.src_dir, - pio_env: req.pio_env, - extra_build_flags: if deploy_route == DeployRoute::Emulator(EmulatorKind::Qemu) { - board - .as_ref() - .map(|board| qemu_extra_build_flags(platform, &board.mcu)) - .unwrap_or_default() - } else { - Vec::new() - }, - watch_set_cache: Some(Arc::clone(&ctx.watch_set_cache) as Arc<_>), - }; - - let build_result = { - let p = platform; - tokio::task::spawn_blocking(move || { - let orchestrator = fbuild_build::get_orchestrator(p)?; - orchestrator.build(¶ms) - }) - .await - }; - - match build_result { - Ok(Ok(r)) if r.success => { - let fw = r.firmware_path.clone().unwrap_or_else(|| { - r.elf_path - .clone() - .unwrap_or_else(|| PathBuf::from("firmware.bin")) - }); - (fw, r.elf_path) - } - Ok(Ok(r)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("build failed: {}", r.message), - )), - ); - } - Ok(Err(e)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("build error: {}", e), - )), - ); - } - Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("build task panicked: {}", e), - )), - ); - } - } - }; - - let artifact_export = match resolved_output_dir.as_ref() { - Some(out_dir) => match export_artifacts_bundle( - out_dir, - platform, - &env_name, - Some(&firmware_path), - elf_path.as_deref(), - ) { - Ok(result) => Some(result), - Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("failed to export artifacts: {}", e), - )), - ); - } - }, - None => None, - }; - - let reported_output_file = artifact_export - .as_ref() - .and_then(|r| r.primary_output.clone()) - .unwrap_or_else(|| firmware_path.clone()) - .to_string_lossy() - .to_string(); - let reported_output_dir = artifact_export - .as_ref() - .map(|r| r.output_dir.to_string_lossy().to_string()); - - if deploy_route == DeployRoute::Emulator(EmulatorKind::Avr8js) { - return crate::handlers::emulator::deploy_avr8js( - ctx, - crate::handlers::emulator::DeployAvr8jsRequest { - request_id, - project_dir, - env_name, - board_id, - platform, - firmware_path, - elf_path, - monitor_after: req.monitor_after, - output_file: reported_output_file, - output_dir: reported_output_dir, - monitor_timeout: req.monitor_timeout, - halt_on_error: req.monitor_halt_on_error.clone(), - halt_on_success: req.monitor_halt_on_success.clone(), - expect: req.monitor_expect.clone(), - show_timestamp: req.monitor_show_timestamp, - verbose: req.verbose, - }, - ) - .await; - } - - if deploy_route == DeployRoute::Emulator(EmulatorKind::Qemu) { - return crate::handlers::emulator::deploy_qemu( - ctx, - crate::handlers::emulator::DeployQemuRequest { - request_id, - project_dir, - env_name, - board_id, - platform, - firmware_path, - elf_path, - output_file: reported_output_file, - output_dir: reported_output_dir, - monitor_timeout: req.monitor_timeout, - qemu_timeout_secs: req.qemu_timeout, - halt_on_error: req.monitor_halt_on_error.clone(), - halt_on_success: req.monitor_halt_on_success.clone(), - expect: req.monitor_expect.clone(), - show_timestamp: req.monitor_show_timestamp, - verbose: req.verbose, - board_overrides, - }, - ) - .await; - } - - // Preempt serial if port specified - let deploy_port_str = req.port.clone(); - if let Some(ref p) = deploy_port_str { - let _ = ctx - .serial_manager - .preempt_for_deploy(p, "deploy".to_string(), request_id.clone()) - .await; - } - - // Extract board ID before spawn_blocking (env_config borrows config) - let board_id = env_config.get("board").cloned().unwrap_or_else(|| { - fbuild_build::get_platform_support(platform) - .map(|s| s.default_board_id().to_string()) - .unwrap_or_else(|_| "unknown".to_string()) - }); - - // Extract env-section board_build.* / board_upload.* overrides BEFORE - // spawn_blocking (env_config borrows config). Without these, fbuild - // would silently ignore `board_build.flash_mode = dio` (and friends) - // from the user's [env:X] section and fall back to whatever the board - // JSON says — producing a firmware/bootloader that doesn't match the - // hardware. The build phase already passes overrides; the deploy phase - // must do the same so esptool flashes with the right --flash-mode. - let board_overrides = config.get_board_overrides(&env_name).unwrap_or_default(); - - // Deploy - let deploy_env = env_name.clone(); - let deploy_project = project_dir.clone(); - let deploy_port = deploy_port_str.clone(); - let deploy_fw = firmware_path.clone(); - let baud_override = req.baud_rate; - let deploy_board_overrides = board_overrides.clone(); - // Snapshot the ctx pointer so the spawn_blocking closure can - // consult / update the daemon's in-memory trusted-hash cache - // without needing a cross-thread lock handshake. - let ctx_for_deploy = Arc::clone(&ctx); - let trusted_hash_enabled = trust_device_hash_enabled(); - // Refresh the enumeration cache so the trust-hash invalidation - // path (`last_disconnect_at`) sees any unplug/replug that - // happened between the previous deploy and now. Without this, - // a user who swapped boards at the same COM port without hitting - // a device-list endpoint could trip the trust check into a - // false match. - // - // Back-to-back warm deploys (the 4 s / 1 s budget target) would - // otherwise re-pay ~20–30 ms per deploy on Windows; cap the cost - // at one enumeration per 2 s. The window is short enough that a - // physically-sneaky board swap between two in-flight deploys - // still needs to happen inside that window to trip trust, and - // the trust-check still requires `is_connected == true` on the - // cached DeviceState, which the most-recent refresh supplied. - if trusted_hash_enabled { - ctx.device_manager - .refresh_devices_if_stale(std::time::Duration::from_secs(2)); - } - let deploy_result = tokio::task::spawn_blocking(move || { - // Populated by the Espressif32 arm with (image_hash, port). - // The tail of the closure consults it after `deployer.deploy` - // returns to record or invalidate the daemon's trusted-hash - // cache. Other platforms leave it `None`. - let mut trusted_hash_update: Option<([u8; 32], String)> = None; - let deployer: Box = match platform { - fbuild_core::Platform::Espressif32 => { - let board_config = - fbuild_config::BoardConfig::from_board_id(&board_id, &deploy_board_overrides) - .unwrap_or_else(|_| { - fbuild_config::BoardConfig::from_board_id( - "esp32dev", - &deploy_board_overrides, - ) - .unwrap() - }); - // Load MCU config to get flash offsets and esptool defaults. - let mcu_config = fbuild_build::esp32::mcu_config::get_mcu_config(&board_config.mcu) - .unwrap_or_else(|_| { - fbuild_build::esp32::mcu_config::get_mcu_config("esp32").unwrap() - }); - // Flash mode: `board_config.flash_mode` is `None` for ESP32 - // chips unless the user explicitly set `board_build.flash_mode` - // in their `[env:X]` section (see `BoardConfig::from_board_id` - // — the JSON-shipped value is intentionally dropped for ESP32 - // because ESP32-S3's QIE-bit init is unreliable). The unwrap - // therefore falls back to the per-MCU default "dio". - let esptool_params = fbuild_deploy::esp32::EsptoolParams { - flash_mode: board_config - .flash_mode - .as_deref() - .unwrap_or(mcu_config.default_flash_mode()) - .to_string(), - flash_freq: { - let f_for_image = board_config - .f_image - .as_deref() - .or(board_config.f_flash.as_deref()); - fbuild_build::esp32::esp32_linker::f_flash_to_esptool_freq( - f_for_image, - mcu_config.default_flash_freq(), - ) - }, - default_baud: mcu_config.default_baud().to_string(), - before_reset: mcu_config.before_reset().to_string(), - after_reset: mcu_config.after_reset().to_string(), - }; - let deployer = fbuild_deploy::esp32::Esp32Deployer::from_board_config( - &board_config, - mcu_config.bootloader_offset(), - mcu_config.partitions_offset(), - mcu_config.firmware_offset(), - &esptool_params, - false, - ); - let deployer = if let Some(baud) = baud_override { - deployer.with_baud_rate(&baud.to_string()) - } else { - deployer - }; - // Issue #66: native `verify-flash` + `write-flash` via - // the `espflash` crate. This is compiled in by default; - // `FBUILD_USE_ESPFLASH_*` are opt-out switches, and the - // deployer falls back to esptool automatically when a - // native operation fails. - #[cfg(feature = "espflash-native")] - let deployer = deployer - .with_native_verify(native_verify_enabled()) - .with_native_write(native_write_enabled()); - - // Compute a deterministic SHA-256 over the three - // regions we'd otherwise verify-flash. Used twice - // below: once to consult the daemon's trusted-hash - // cache (opt-in via `FBUILD_TRUST_DEVICE_HASH=1`), - // and once to *record* the hash after a successful - // deploy so the next warm redeploy can skip the MD5 - // round-trip entirely. A `None` return means one of - // the three files isn't on disk yet — treat it as - // "can't trust-skip" rather than erroring, so the - // fallback path is free to rebuild missing artefacts. - let image_hash = compute_esp32_image_hash( - &ctx_for_deploy, - &deploy_fw, - u32::from_str_radix( - mcu_config.bootloader_offset().trim_start_matches("0x"), - 16, - ) - .unwrap_or(0), - u32::from_str_radix( - mcu_config.partitions_offset().trim_start_matches("0x"), - 16, - ) - .unwrap_or(0), - u32::from_str_radix( - mcu_config.firmware_offset().trim_start_matches("0x"), - 16, - ) - .unwrap_or(0), - ); - - // Session-trusted verify-skip: if the daemon last - // flashed *this exact image* onto *this port* and the - // port has been continuously enumerated since then, - // no external agent could have re-flashed the chip - // without breaking our enumeration (`last_disconnect_at` - // is how we detect it). Skip the entire serial open + - // espflash connect + MD5 round-trip. - if let (Some(port), Some(hash)) = (deploy_port.as_deref(), image_hash) { - if trusted_hash_enabled { - if let Some(trusted) = - ctx_for_deploy.device_manager.trusted_firmware_hash(port) - { - if trusted == hash { - tracing::info!( - port, - "trusted-hash: session-trusted match; skipping verify-flash entirely" - ); - return Ok(fbuild_deploy::DeploymentResult { - success: true, - message: format!( - "firmware already current on {} (skipped via session trust)", - port - ), - port: Some(port.to_string()), - stdout: String::new(), - stderr: String::new(), - outcome: fbuild_deploy::DeployOutcome::VerifySkip, - }); - } - } - } - } - - // Fast deploy: ask the device whether it already holds - // the exact firmware/bootloader/partitions we'd be about - // to write. Uses esptool's `verify-flash` which dispatches - // to the stub flasher's `FLASH_MD5SUM` command — no full - // read-back, just one MD5 round-trip per region. - // - // Measured on a 2.4 MB FastLED esp32s3 image: - // * fresh write-flash: ~25 s - // * verify-flash skip: ~6 s (-19 s, ~76% faster) - // - // Falls through to the normal flash path silently on - // mismatch or transport error so we never break a deploy - // that the verify call didn't understand. - let mut selective_regions: Option> = None; - if let Some(port) = deploy_port.as_deref() { - match deployer.try_verify_deployment(&deploy_fw, port) { - Ok(fbuild_deploy::esp32::VerifyOutcome::Match { stdout, stderr }) => { - tracing::info!( - port, - "verify-flash: device already running this exact image; skipping write" - ); - return Ok(fbuild_deploy::DeploymentResult { - success: true, - message: format!( - "firmware already current on {} (skipped via verify-flash)", - port - ), - port: Some(port.to_string()), - stdout, - stderr, - outcome: fbuild_deploy::DeployOutcome::VerifySkip, - }); - } - Ok(fbuild_deploy::esp32::VerifyOutcome::Mismatch { regions, .. }) => { - // Pick only the regions that actually differ - // so we avoid the ~1s bootloader/partitions - // rewrite when only firmware changed. Empty - // `regions` means parsing failed — fall back - // to full flash. - let to_write: Vec<_> = regions - .iter() - .filter(|r| !r.matched) - .map(|r| r.region) - .collect(); - if !regions.is_empty() && !to_write.is_empty() && to_write.len() < 3 { - tracing::info!( - port, - "verify-flash: only {} region(s) differ; flashing selectively", - to_write.len() - ); - selective_regions = Some(to_write); - } else { - tracing::info!( - port, - "verify-flash: device image differs; proceeding with full flash" - ); - } - } - Err(e) => { - tracing::warn!( - port, - "verify-flash pre-check failed ({}); proceeding with full flash", - e - ); - } - } - } - - if let (Some(regions), Some(port)) = (selective_regions, deploy_port.as_deref()) { - let result = deployer.deploy_regions(&deploy_fw, port, ®ions); - // Record/invalidate the trusted hash based on - // the selective-flash outcome so the next warm - // redeploy can short-circuit via trust-skip. - if let Some(hash) = image_hash { - match &result { - Ok(r) if r.success => { - ctx_for_deploy - .device_manager - .set_trusted_firmware_hash(port, hash); - } - _ => { - ctx_for_deploy.device_manager.clear_trusted_firmware_hash(port); - } - } - } - return result; - } - - // Capture the hash into the boxed deployer closure - // below — the full-flash path (`deployer.deploy(...)`) - // at the end of this `spawn_blocking` applies the - // same record/invalidate rule. We stash the hash + - // port via a small impl-only wrapper because the - // `Deployer` trait doesn't expose the cache hook. - // Stored here so the common tail after the `match` - // can reach it without re-threading every arm. - // (AVR / other arms leave it `None` → no-op.) - trusted_hash_update = image_hash.zip(deploy_port.as_deref().map(str::to_string)); - - Box::new(deployer) - } - fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr => { - let board_config = - fbuild_config::BoardConfig::from_board_id(&board_id, &deploy_board_overrides) - .unwrap_or_else(|_| { - fbuild_config::BoardConfig::from_board_id( - "uno", - &deploy_board_overrides, - ) - .unwrap() - }); - let avr_config = fbuild_build::avr::mcu_config::get_avr_config().unwrap(); - let avrdude_params = fbuild_deploy::avr::AvrdudeParams { - default_programmer: avr_config.avrdude.default_programmer.clone(), - default_baud: avr_config.avrdude.default_baud.to_string(), - timeout_secs: avr_config.avrdude.timeout_secs, - }; - let deployer = fbuild_deploy::avr::AvrDeployer::from_board_config( - &board_config, - &avrdude_params, - false, - ); - let deployer = if let Some(baud) = baud_override { - deployer.with_baud_rate(&baud.to_string()) - } else { - deployer - }; - Box::new(deployer) - } - _ => { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "deployer for {:?} not yet implemented", - platform - ))); - } - }; - let result = deployer.deploy( - &deploy_project, - &deploy_env, - &deploy_fw, - deploy_port.as_deref(), - ); - // Session-trusted verify-skip: record (or invalidate) the - // image hash the daemon associates with this port. Scoped - // to the Espressif32 arm above — other platforms leave - // `trusted_hash_update` as `None` and this block no-ops. - // Failed or partial deploys clear the cache so the next - // warm run falls back to the verify-flash path. - if let Some((hash, port)) = trusted_hash_update { - match &result { - Ok(r) if r.success => { - ctx_for_deploy - .device_manager - .set_trusted_firmware_hash(&port, hash); - } - _ => { - ctx_for_deploy - .device_manager - .clear_trusted_firmware_hash(&port); - } - } - } - result - }) - .await; - - // Clear preemption then wait for USB re-enumeration. - // Fast-poll the serial port instead of a hard 2s sleep — most ESP32-S3 - // boards with native USB re-enumerate in <500ms. - // - // Skip the poll entirely when the deploy didn't actually touch - // flash (VerifySkip): no reset happened, no USB re-enumeration - // is coming, and the poll's `open()` probe would conflict with - // any already-attached monitor and burn the full 3 s timeout. - // This is load-bearing for the < 4 s warm-trust-skip budget. - let deploy_skipped_bus_work = matches!( - &deploy_result, - Ok(Ok(r)) if r.success && matches!( - r.outcome, - fbuild_deploy::DeployOutcome::VerifySkip - ) - ); - if let Some(ref p) = deploy_port_str { - ctx.serial_manager.clear_preemption(p).await; - if !deploy_skipped_bus_work { - let port_name = p.clone(); - let _ = tokio::task::spawn_blocking(move || { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); - while std::time::Instant::now() < deadline { - if serialport::new(&port_name, 115200) - .timeout(std::time::Duration::from_millis(50)) - .open() - .is_ok() - { - return; - } - std::thread::sleep(std::time::Duration::from_millis(100)); - } - tracing::warn!( - "USB re-enumeration: port {} not available after 3s", - port_name - ); - }) - .await; - } - } - - let (deploy_success, deploy_stdout, deploy_stderr, deploy_outcome) = match deploy_result { - Ok(Ok(r)) if r.success => (true, Some(r.stdout), Some(r.stderr), r.outcome), - Ok(Ok(r)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse { - success: false, - request_id, - message: r.message, - exit_code: 1, - output_file: Some(reported_output_file.clone()), - output_dir: reported_output_dir.clone(), - launch_url: None, - stdout: Some(r.stdout), - stderr: Some(r.stderr), - }), - ); - } - Ok(Err(e)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("deploy error: {}", e), - )), - ); - } - Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("deploy task panicked: {}", e), - )), - ); - } - }; - // Build the "deploy succeeded (...)" prefix used by every - // monitor-attached and non-monitor-attached response below. Stable - // wording — see GitHub issue #76 and the DeployOutcome::describe - // test in fbuild-deploy. - let deploy_prefix = format!("deploy succeeded ({})", deploy_outcome.describe()); - - // Post-deploy monitoring: if monitor_after is set, open the serial port - // and stream lines checking halt conditions (matching Python behavior). - if deploy_success && req.monitor_after { - let monitor_port = deploy_port_str.unwrap_or_else(|| "/dev/ttyUSB0".to_string()); - let baud_rate = 115200u32; - - // Open the port for monitoring - if let Err(e) = ctx - .serial_manager - .open_port(&monitor_port, baud_rate, &request_id) - .await - { - return ( - StatusCode::OK, - Json(OperationResponse { - success: true, - request_id, - message: format!("{} but monitor failed to open port: {}", deploy_prefix, e), - exit_code: 0, - output_file: Some(reported_output_file.clone()), - output_dir: reported_output_dir.clone(), - launch_url: None, - stdout: deploy_stdout, - stderr: deploy_stderr, - }), - ); - } - - // Subscribe to broadcast channel - let mut rx = match ctx.serial_manager.attach_reader(&monitor_port, &request_id) { - Some(rx) => rx, - None => { - return ( - StatusCode::OK, - Json(OperationResponse { - success: true, - request_id, - message: format!("{} but monitor could not attach reader", deploy_prefix), - exit_code: 0, - output_file: Some(reported_output_file.clone()), - output_dir: reported_output_dir.clone(), - launch_url: None, - stdout: deploy_stdout, - stderr: deploy_stderr, - }), - ); - } - }; - - let monitor_result = run_monitor_loop( - &mut rx, - req.monitor_timeout, - req.monitor_halt_on_error.as_deref(), - req.monitor_halt_on_success.as_deref(), - req.monitor_expect.as_deref(), - req.monitor_show_timestamp, - ) - .await; - - ctx.serial_manager.detach_reader(&monitor_port, &request_id); - - return match monitor_result { - MonitorOutcome::Success(msg) => ( - StatusCode::OK, - Json(OperationResponse { - success: true, - request_id, - message: format!("{}; monitor: {}", deploy_prefix, msg), - exit_code: 0, - output_file: Some(reported_output_file.clone()), - output_dir: reported_output_dir.clone(), - launch_url: None, - stdout: deploy_stdout, - stderr: deploy_stderr, - }), - ), - MonitorOutcome::Error(msg) => ( - StatusCode::OK, - Json(OperationResponse { - success: false, - request_id, - message: format!("{}; monitor error: {}", deploy_prefix, msg), - exit_code: 1, - output_file: Some(reported_output_file.clone()), - output_dir: reported_output_dir.clone(), - launch_url: None, - stdout: deploy_stdout, - stderr: deploy_stderr, - }), - ), - MonitorOutcome::Timeout { expect_found } => { - let (success, code) = if expect_found { - (true, 0) - } else { - // If expect was set and not found, that's an error - ( - req.monitor_expect.is_none(), - if req.monitor_expect.is_none() { 0 } else { 1 }, - ) - }; - ( - StatusCode::OK, - Json(OperationResponse { - success, - request_id, - message: format!( - "{}; monitor timed out{}", - deploy_prefix, - if !expect_found && req.monitor_expect.is_some() { - " (expected pattern not found)" - } else { - "" - } - ), - exit_code: code, - output_file: Some(reported_output_file.clone()), - output_dir: reported_output_dir.clone(), - launch_url: None, - stdout: deploy_stdout, - stderr: deploy_stderr, - }), - ) - } - }; - } - - ( - StatusCode::OK, - Json(OperationResponse { - success: true, - request_id, - message: deploy_prefix, - exit_code: 0, - output_file: Some(reported_output_file), - output_dir: reported_output_dir, - launch_url: None, - stdout: deploy_stdout, - stderr: deploy_stderr, - }), - ) -} - -/// Outcome of a post-deploy monitor session. -#[derive(Debug)] -pub(crate) enum MonitorOutcome { - /// halt-on-success pattern matched - Success(String), - /// halt-on-error pattern matched - Error(String), - /// Timeout reached - Timeout { expect_found: bool }, -} - -pub(crate) struct MonitorState { - halt_error_re: Option, - halt_success_re: Option, - expect_re: Option, - start: std::time::Instant, - timeout_dur: Option, - expect_found: bool, - show_timestamp: bool, -} - -impl MonitorState { - pub(crate) fn new( - timeout_secs: Option, - halt_on_error: Option<&str>, - halt_on_success: Option<&str>, - expect: Option<&str>, - show_timestamp: bool, - ) -> Self { - let halt_error_re = halt_on_error.and_then(|p| { - regex::RegexBuilder::new(p) - .case_insensitive(true) - .build() - .ok() - }); - let halt_success_re = halt_on_success.and_then(|p| { - regex::RegexBuilder::new(p) - .case_insensitive(true) - .build() - .ok() - }); - let expect_re = expect.and_then(|p| { - regex::RegexBuilder::new(p) - .case_insensitive(true) - .build() - .ok() - }); - Self { - halt_error_re, - halt_success_re, - expect_re, - start: std::time::Instant::now(), - timeout_dur: timeout_secs.map(std::time::Duration::from_secs_f64), - expect_found: false, - show_timestamp, - } - } - - pub(crate) fn timed_out(&self) -> bool { - self.timeout_dur - .is_some_and(|dur| self.start.elapsed() >= dur) - } - - pub(crate) fn remaining(&self) -> Option { - self.timeout_dur - .map(|dur| dur.saturating_sub(self.start.elapsed())) - } - - pub(crate) fn timeout_outcome(&self) -> MonitorOutcome { - MonitorOutcome::Timeout { - expect_found: self.expect_found, - } - } - - pub(crate) fn expect_found(&self) -> bool { - self.expect_found - } - - pub(crate) fn process_line(&mut self, line: &str) -> Option { - if self.show_timestamp { - let total_secs = self.start.elapsed().as_secs_f64(); - let minutes = (total_secs / 60.0) as u64; - let seconds = total_secs % 60.0; - tracing::info!("{:02}:{:05.2} {}", minutes, seconds, line); - } else { - tracing::info!("{}", line); - } - - if let Some(ref re) = self.expect_re { - if re.is_match(line) { - self.expect_found = true; - } - } - - if let Some(ref re) = self.halt_error_re { - if re.is_match(line) { - return Some(MonitorOutcome::Error(format!( - "halt-on-error pattern matched: {}", - line - ))); - } - } - - if let Some(ref re) = self.halt_success_re { - if re.is_match(line) { - return Some(MonitorOutcome::Success(format!( - "halt-on-success pattern matched: {}", - line - ))); - } - } - - None - } -} - -/// Run a monitor loop reading lines from broadcast, checking halt conditions -/// using case-insensitive regex (matching Python's re.search behavior). -async fn run_monitor_loop( - rx: &mut tokio::sync::broadcast::Receiver, - timeout_secs: Option, - halt_on_error: Option<&str>, - halt_on_success: Option<&str>, - expect: Option<&str>, - show_timestamp: bool, -) -> MonitorOutcome { - let mut state = MonitorState::new( - timeout_secs, - halt_on_error, - halt_on_success, - expect, - show_timestamp, - ); - loop { - if state.timed_out() { - return state.timeout_outcome(); - } - - let recv_timeout = state - .remaining() - .unwrap_or(std::time::Duration::from_secs(1)); - - match tokio::time::timeout(recv_timeout, rx.recv()).await { - Ok(Ok(line)) => { - if let Some(outcome) = state.process_line(&line) { - return outcome; - } - } - Ok(Err(tokio::sync::broadcast::error::RecvError::Lagged(n))) => { - tracing::warn!("monitor lagged, skipped {} messages", n); - } - Ok(Err(tokio::sync::broadcast::error::RecvError::Closed)) => { - return state.timeout_outcome(); - } - Err(_) => { - // Timeout on recv — check if overall timeout expired - if state.timed_out() { - return state.timeout_outcome(); - } - // No overall timeout: just keep waiting - } - } - } -} - -/// POST /api/monitor -pub async fn monitor( - State(ctx): State>, - Json(req): Json, -) -> (StatusCode, Json) { - let request_id = req - .request_id - .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); - let port = req.port.unwrap_or_else(|| "/dev/ttyUSB0".to_string()); - let baud_rate = req.baud_rate.unwrap_or(115200); - - if let Err(e) = ctx - .serial_manager - .open_port(&port, baud_rate, &request_id) - .await - { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("failed to open port: {}", e), - )), - ); - } - - // If halt conditions or timeout are set, run a monitor loop - let has_conditions = req.halt_on_error.is_some() - || req.halt_on_success.is_some() - || req.expect.is_some() - || req.timeout.is_some(); - - if has_conditions { - let mut rx = match ctx.serial_manager.attach_reader(&port, &request_id) { - Some(rx) => rx, - None => { - return ( - StatusCode::OK, - Json(OperationResponse::ok( - request_id, - format!( - "monitoring {} at {} baud (no broadcast channel)", - port, baud_rate - ), - )), - ); - } - }; - - let result = run_monitor_loop( - &mut rx, - req.timeout, - req.halt_on_error.as_deref(), - req.halt_on_success.as_deref(), - req.expect.as_deref(), - req.show_timestamp, - ) - .await; - - ctx.serial_manager.detach_reader(&port, &request_id); - - return match result { - MonitorOutcome::Success(msg) => { - (StatusCode::OK, Json(OperationResponse::ok(request_id, msg))) - } - MonitorOutcome::Error(msg) => ( - StatusCode::OK, - Json(OperationResponse::fail(request_id, msg)), - ), - MonitorOutcome::Timeout { expect_found } => { - if req.expect.is_some() && !expect_found { - ( - StatusCode::OK, - Json(OperationResponse::fail( - request_id, - "monitor timed out (expected pattern not found)".to_string(), - )), - ) - } else { - ( - StatusCode::OK, - Json(OperationResponse::ok( - request_id, - "monitor completed (timeout)".to_string(), - )), - ) - } - } - }; - } - - ( - StatusCode::OK, - Json(OperationResponse::ok( - request_id, - format!("monitoring {} at {} baud", port, baud_rate), - )), - ) -} - -/// POST /api/reset -/// -/// Reset a device via serial port DTR/RTS toggling. -/// Uses platform-specific reset sequence based on board identifier. -pub async fn reset( - State(ctx): State>, - Json(req): Json, -) -> (StatusCode, Json) { - let request_id = req - .request_id - .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); - let port = req.port.clone(); - let verbose = req.verbose; - - let platform = req - .board - .as_deref() - .map(fbuild_deploy::reset::detect_platform_for_reset) - .unwrap_or("generic"); - - let _op_guard = OperationGuard::new( - &ctx, - fbuild_core::DaemonState::Deploying, - Some(format!("Resetting device on {}", port)), - ); - - // Preempt serial if someone is monitoring this port - let _ = ctx - .serial_manager - .preempt_for_deploy(&port, "reset".to_string(), request_id.clone()) - .await; - - let platform_str = platform.to_string(); - let result = tokio::task::spawn_blocking(move || { - fbuild_deploy::reset::reset_device(&platform_str, &port, verbose) - }) - .await; - - // Clear preemption - ctx.serial_manager.clear_preemption(&req.port).await; - - match result { - Ok(Ok(true)) => ( - StatusCode::OK, - Json(OperationResponse::ok( - request_id, - format!("device reset successful on {}", req.port), - )), - ), - Ok(Ok(false)) => ( - StatusCode::OK, - Json(OperationResponse::fail( - request_id, - format!("device reset reported failure on {}", req.port), - )), - ), - Ok(Err(e)) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("reset error: {}", e), - )), - ), - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("reset task panicked: {}", e), - )), - ), - } -} - -/// POST /api/install-deps -/// -/// Install toolchain, framework, and library dependencies without building. -/// Matches the Python daemon's `/api/install-deps` endpoint contract. -pub async fn install_deps( - State(ctx): State>, - Json(req): Json, -) -> (StatusCode, Json) { - let request_id = req - .request_id - .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); - let project_dir = PathBuf::from(&req.project_dir); - - if !project_dir.exists() { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("project directory does not exist: {}", req.project_dir), - )), - ); - } - - let _op_guard = OperationGuard::new( - &ctx, - fbuild_core::DaemonState::Building, - Some(format!("Installing deps for {}", req.project_dir)), - ); - - // Acquire per-project lock - let lock = ctx.project_lock(&project_dir); - let _guard = lock.lock().await; - - // Parse platformio.ini to determine platform and resolve packages - let config = - match fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")) { - Ok(c) => c, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("failed to parse platformio.ini: {}", e), - )), - ); - } - }; - - let env_name = req - .environment - .clone() - .or_else(|| config.get_default_environment().map(|s| s.to_string())) - .unwrap_or_else(|| "default".to_string()); - - let env_config = match config.get_env_config(&env_name) { - Ok(c) => c, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("invalid environment '{}': {}", env_name, e), - )), - ); - } - }; - - let platform_str = env_config.get("platform").cloned().unwrap_or_default(); - let platform = match fbuild_core::Platform::from_platform_str(&platform_str) { - Some(p) => p, - None => { - return ( - StatusCode::BAD_REQUEST, - Json(OperationResponse::fail( - request_id, - format!("unsupported platform: {}", platform_str), - )), - ); - } - }; - - // Install dependencies via the package manager - let env_label = env_name.clone(); - let result = tokio::task::spawn_blocking(move || { - fbuild_build::install_platform_deps(platform, &project_dir) - }) - .await; - - match result { - Ok(Ok(())) => ( - StatusCode::OK, - Json(OperationResponse::ok( - request_id, - format!("Dependencies installed for environment '{}'", env_label), - )), - ), - Ok(Err(e)) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("install-deps error: {}", e), - )), - ), - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("install-deps task panicked: {}", e), - )), - ), - } -} - -#[cfg(test)] -mod deploy_message_tests { - //! Verifies the `/api/deploy` response message exposes the real - //! deploy outcome (full / verify-skip / selective) instead of the - //! generic `"deploy succeeded"`. See GitHub issue #76. - //! - //! These tests cover only the pure string-formatting contract; the - //! underlying outcome computation is tested in `fbuild-deploy`. - use fbuild_deploy::{esp32::FlashRegion, DeployOutcome}; - - fn prefix_for(outcome: &DeployOutcome) -> String { - format!("deploy succeeded ({})", outcome.describe()) - } - - #[test] - fn full_flash_prefix() { - assert_eq!( - prefix_for(&DeployOutcome::FullFlash), - "deploy succeeded (full flash)" - ); - } - - #[test] - fn verify_skip_prefix() { - assert_eq!( - prefix_for(&DeployOutcome::VerifySkip), - "deploy succeeded (verify skipped, device already matched)" - ); - } - - #[test] - fn selective_flash_firmware_prefix() { - let outcome = DeployOutcome::SelectiveFlash { - regions: vec![FlashRegion::Firmware], - }; - assert_eq!( - prefix_for(&outcome), - "deploy succeeded (selective flash: firmware)" - ); - } - - #[test] - fn monitor_suffix_preserved_on_selective_flash() { - let outcome = DeployOutcome::SelectiveFlash { - regions: vec![FlashRegion::Firmware], - }; - let prefix = prefix_for(&outcome); - let combined = format!("{}; monitor: ok", prefix); - assert_eq!( - combined, - "deploy succeeded (selective flash: firmware); monitor: ok" - ); - } - - #[test] - fn monitor_error_suffix_preserved_on_verify_skip() { - let prefix = prefix_for(&DeployOutcome::VerifySkip); - let combined = format!("{}; monitor error: pattern matched", prefix); - assert_eq!( - combined, - "deploy succeeded (verify skipped, device already matched); monitor error: pattern matched" - ); - } -} - -#[cfg(all(test, feature = "espflash-native"))] -mod espflash_env_tests { - use super::{native_verify_enabled, native_write_enabled}; - - // This lock only serializes these unit tests while they mutate the - // process environment. Production callers and any other tests reading - // the same env vars remain unsynchronized. - static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - - #[test] - fn native_verify_defaults_on_and_allows_opt_out() { - let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - std::env::remove_var("FBUILD_USE_ESPFLASH_VERIFY"); - assert!(native_verify_enabled()); - - std::env::set_var("FBUILD_USE_ESPFLASH_VERIFY", "0"); - assert!(!native_verify_enabled()); - - std::env::set_var("FBUILD_USE_ESPFLASH_VERIFY", "false"); - assert!(!native_verify_enabled()); - - std::env::set_var("FBUILD_USE_ESPFLASH_VERIFY", "1"); - assert!(native_verify_enabled()); - std::env::remove_var("FBUILD_USE_ESPFLASH_VERIFY"); - } - - #[test] - fn native_write_defaults_on_and_allows_opt_out() { - let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - std::env::remove_var("FBUILD_USE_ESPFLASH_WRITE"); - assert!(native_write_enabled()); - - std::env::set_var("FBUILD_USE_ESPFLASH_WRITE", "off"); - assert!(!native_write_enabled()); - - std::env::set_var("FBUILD_USE_ESPFLASH_WRITE", "yes"); - assert!(native_write_enabled()); - std::env::remove_var("FBUILD_USE_ESPFLASH_WRITE"); - } -} - -#[cfg(test)] -mod image_hash_memo_tests { - //! Memo-cache correctness for [`compute_esp32_image_hash`]: the - //! memo must *reuse* the stored hash when none of the three - //! region files have changed, and *re-hash* when any of them - //! changes on disk. - use super::compute_esp32_image_hash; - use crate::context::DaemonContext; - use std::io::Write; - use std::path::Path; - - fn write(path: &Path, bytes: &[u8]) { - let mut f = std::fs::File::create(path).unwrap(); - f.write_all(bytes).unwrap(); - } - - fn fresh_ctx() -> std::sync::Arc { - let (tx, _rx) = tokio::sync::watch::channel(false); - std::sync::Arc::new(DaemonContext::new(8765, tx, "unknown".to_string())) - } - - fn seed_image(dir: &Path) { - write(&dir.join("bootloader.bin"), b"BOOT0"); - write(&dir.join("partitions.bin"), b"PART00"); - write(&dir.join("firmware.bin"), b"FW__FIRST_BUILD"); - } - - /// Second call with unchanged files must hit the memo (same - /// result, no work). We verify the memo side by directly - /// inspecting `ctx.image_hash_memo`. - #[test] - fn memo_hit_reuses_hash() { - let tmp = tempfile::tempdir().unwrap(); - seed_image(tmp.path()); - let ctx = fresh_ctx(); - let fw = tmp.path().join("firmware.bin"); - - let h1 = compute_esp32_image_hash(&ctx, &fw, 0x0, 0x8000, 0x10000).unwrap(); - assert_eq!(ctx.image_hash_memo.len(), 1); - let h2 = compute_esp32_image_hash(&ctx, &fw, 0x0, 0x8000, 0x10000).unwrap(); - assert_eq!(h1, h2); - assert_eq!(ctx.image_hash_memo.len(), 1, "memo must not grow on a hit"); - } - - /// When any of the three files changes on disk, the memo - /// invalidates via `mtime` change and the hash recomputes. - /// We assert the hash *differs* because the file contents - /// changed — so this catches both the bytes going through the - /// hasher AND the invalidation path. - #[test] - fn memo_miss_on_firmware_mtime_change() { - let tmp = tempfile::tempdir().unwrap(); - seed_image(tmp.path()); - let ctx = fresh_ctx(); - let fw = tmp.path().join("firmware.bin"); - - let h1 = compute_esp32_image_hash(&ctx, &fw, 0x0, 0x8000, 0x10000).unwrap(); - // Rewrite firmware.bin with new content; `std::fs::File::create` - // bumps the `mtime` on Windows with enough resolution (100 ns) - // even for sub-millisecond follow-ups. - std::thread::sleep(std::time::Duration::from_millis(20)); - write(&fw, b"FW__SECOND_BUILD_DIFFERENT"); - - let h2 = compute_esp32_image_hash(&ctx, &fw, 0x0, 0x8000, 0x10000).unwrap(); - assert_ne!(h1, h2, "content-changed image must hash to a new value"); - } - - /// Missing files on disk short-circuit to `None` — the caller - /// falls through to the regular verify-flash path instead of - /// trust-skipping with a stale hash. The memo must NOT store an - /// entry for an input that couldn't be hashed. - #[test] - fn memo_skipped_when_inputs_missing() { - let tmp = tempfile::tempdir().unwrap(); - // Only create firmware.bin — bootloader/partitions absent. - write(&tmp.path().join("firmware.bin"), b"FW"); - let ctx = fresh_ctx(); - let fw = tmp.path().join("firmware.bin"); - - assert!(compute_esp32_image_hash(&ctx, &fw, 0x0, 0x8000, 0x10000).is_none()); - assert_eq!( - ctx.image_hash_memo.len(), - 0, - "memo must not record entries for inputs that fail to hash" - ); - } -} diff --git a/crates/fbuild-daemon/src/handlers/operations/README.md b/crates/fbuild-daemon/src/handlers/operations/README.md new file mode 100644 index 00000000..a2dfe34d --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/operations/README.md @@ -0,0 +1,24 @@ +# operations + +Submodules for the build, deploy, monitor, reset, and install-deps HTTP +handlers. Originally `handlers/operations.rs`; split here to keep every +`.rs` under the 1000-LOC CI gate. + +- **`mod.rs`** -- Re-exports the public handlers and the `pub(crate)` items + shared with `handlers::emulator`. External paths + (`crate::handlers::operations::build`, etc.) are unchanged. +- **`common.rs`** -- `OperationGuard`, env-var feature switches + (`trust_device_hash_enabled`, `native_verify_enabled`, + `native_write_enabled`), `compute_esp32_image_hash`, + `qemu_extra_build_flags`, deploy-route parsing, client-path resolution, + and the artifact bundle exporter. +- **`build.rs`** -- `POST /api/build` handler (streaming + buffered paths). +- **`deploy.rs`** -- `POST /api/deploy` handler with the ESP32 trust-hash / + verify-flash fast paths and AVR fallback. +- **`monitor.rs`** -- `POST /api/monitor` handler, `MonitorState`, + `MonitorOutcome`, and the shared `run_monitor_loop`. +- **`reset.rs`** -- `POST /api/reset` handler (DTR/RTS toggling). +- **`install_deps.rs`** -- `POST /api/install-deps` handler. +- **`tests.rs`** -- Unit tests previously inlined at the bottom of + `operations.rs` (deploy message formatting, espflash env switches, + image-hash memo cache). diff --git a/crates/fbuild-daemon/src/handlers/operations/build.rs b/crates/fbuild-daemon/src/handlers/operations/build.rs new file mode 100644 index 00000000..7c69b3bc --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/operations/build.rs @@ -0,0 +1,434 @@ +//! `POST /api/build` — kick off a build (streaming or buffered). + +use super::common::{export_artifacts_bundle, resolve_client_path, OperationGuard}; +use crate::context::DaemonContext; +use crate::models::{BuildRequest, OperationResponse}; +use axum::extract::State; +use axum::http::StatusCode; +use axum::Json; +use std::path::PathBuf; +use std::sync::Arc; + +/// POST /api/build +pub async fn build( + State(ctx): State>, + Json(req): Json, +) -> axum::response::Response { + use axum::response::IntoResponse; + + let request_id = req + .request_id + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let project_dir = PathBuf::from(&req.project_dir); + let stream = req.stream; + + if !project_dir.exists() { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("project directory does not exist: {}", req.project_dir), + )), + ) + .into_response(); + } + + // Validation: parse config, resolve platform + let config = + match fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")) { + Ok(c) => c, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("failed to parse platformio.ini: {}", e), + )), + ) + .into_response(); + } + }; + + let env_name = req + .environment + .clone() + .or_else(|| config.get_default_environment().map(|s| s.to_string())) + .unwrap_or_else(|| "default".to_string()); + + let env_config = match config.get_env_config(&env_name) { + Ok(c) => c, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("invalid environment '{}': {}", env_name, e), + )), + ) + .into_response(); + } + }; + + let platform_str = env_config.get("platform").cloned().unwrap_or_default(); + let platform = match fbuild_core::Platform::from_platform_str(&platform_str) { + Some(p) => p, + None => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("unsupported platform: {}", platform_str), + )), + ) + .into_response(); + } + }; + + let profile = match req.profile.as_deref() { + Some("quick") => fbuild_core::BuildProfile::Quick, + _ => fbuild_core::BuildProfile::Release, + }; + + let build_dir = fbuild_paths::get_project_build_root(&project_dir); + let compiledb_env = std::env::var("FBUILD_COMPILEDB") + .map(|v| v != "0") + .unwrap_or(true); + let generate_compiledb = req.generate_compiledb || compiledb_env; + let resolved_symbol_analysis_path = req + .symbol_analysis_path + .as_deref() + .map(|p| resolve_client_path(p, req.caller_cwd.as_deref(), &project_dir)); + let resolved_output_dir = req + .output_dir + .as_deref() + .map(|p| resolve_client_path(p, req.caller_cwd.as_deref(), &project_dir)); + + if stream { + // --- STREAMING PATH --- + // Build runs in a background task; log lines stream to client as NDJSON. + let (sync_tx, sync_rx) = std::sync::mpsc::channel::(); + let (async_tx, async_rx) = tokio::sync::mpsc::unbounded_channel::(); + + let params = fbuild_build::BuildParams { + project_dir: project_dir.clone(), + env_name: env_name.clone(), + clean: req.clean_build, + profile, + build_dir, + verbose: req.verbose, + jobs: req.jobs, + generate_compiledb, + compiledb_only: req.compiledb_only, + log_sender: Some(sync_tx), + symbol_analysis: req.symbol_analysis, + symbol_analysis_path: resolved_symbol_analysis_path.clone(), + no_timestamp: req.no_timestamp, + src_dir: req.src_dir.clone(), + pio_env: req.pio_env.clone(), + extra_build_flags: Vec::new(), + watch_set_cache: Some(Arc::clone(&ctx.watch_set_cache) as Arc<_>), + }; + + let project_dir_desc = req.project_dir.clone(); + tokio::spawn(async move { + // FBUILD_PERF_LOG=1 enables daemon-side coarse phase timing + // (lock-wait + build). Zero overhead when unset. + let perf_enabled = std::env::var("FBUILD_PERF_LOG") + .map(|v| !v.is_empty() && v != "0") + .unwrap_or(false); + let daemon_start = std::time::Instant::now(); + + let _op_guard = OperationGuard::new( + &ctx, + fbuild_core::DaemonState::Building, + Some(format!("Building {}", project_dir_desc)), + ); + let lock_wait_start = std::time::Instant::now(); + let lock = ctx.project_lock(&project_dir); + let _lock_guard = lock.lock().await; + let lock_wait = lock_wait_start.elapsed(); + + // Bridge: sync log lines → async NDJSON chunks + let bridge_tx = async_tx.clone(); + let bridge = tokio::task::spawn_blocking(move || { + for line in sync_rx { + let event = serde_json::json!({"type": "log", "message": line}); + let mut chunk = event.to_string(); + chunk.push('\n'); + if bridge_tx.send(bytes::Bytes::from(chunk)).is_err() { + break; + } + } + }); + + // Run build + let build_wallclock_start = std::time::Instant::now(); + let build_result = tokio::task::spawn_blocking(move || { + let orchestrator = fbuild_build::get_orchestrator(platform)?; + orchestrator.build(¶ms) + }) + .await; + let build_wallclock = build_wallclock_start.elapsed(); + if perf_enabled { + let summary = format!( + "[perf-log daemon-handler] lock-wait={} ms, build-wallclock={} ms, total={} ms", + lock_wait.as_millis(), + build_wallclock.as_millis(), + daemon_start.elapsed().as_millis(), + ); + tracing::info!(target: "fbuild_daemon::perf_log", "{}", summary); + eprintln!("{}", summary); + } + + // Extract result (drops BuildLog sender so bridge can finish) + let (success, rid, msg, code, output_file, output_dir) = match build_result { + Ok(Ok(br)) => { + let exported = if br.success { + if let Some(ref out_dir) = resolved_output_dir { + Some(export_artifacts_bundle( + out_dir, + platform, + &env_name, + br.firmware_path.as_deref(), + br.elf_path.as_deref(), + )) + } else { + None + } + } else { + None + }; + let _lines = br.build_log.into_lines(); // drop sender + let summary = if br.success { + let size_str = br + .size_info + .as_ref() + .map(|s| { + format!( + " (flash: {} bytes, ram: {} bytes)", + s.total_flash, s.total_ram + ) + }) + .unwrap_or_default(); + let export_suffix = match exported.as_ref() { + Some(Ok(result)) => { + format!("; artifacts exported to {}", result.output_dir.display()) + } + Some(Err(e)) => { + format!("; artifact export failed: {}", e) + } + None => String::new(), + }; + format!( + "build succeeded in {:.1}s{}{}", + br.build_time_secs, size_str, export_suffix + ) + } else { + br.message.clone() + }; + let output = match exported.as_ref() { + Some(Ok(result)) => result + .primary_output + .clone() + .or(br.firmware_path.clone()) + .or(br.elf_path.clone()), + _ => br.firmware_path.clone().or(br.elf_path.clone()), + } + .map(|p| p.to_string_lossy().to_string()); + let output_dir = match exported.as_ref() { + Some(Ok(result)) => Some(result.output_dir.to_string_lossy().to_string()), + _ => None, + }; + let c = if br.success { 0 } else { 1 }; + ( + br.success, + request_id.clone(), + summary, + c, + output, + output_dir, + ) + } + Ok(Err(e)) => ( + false, + request_id.clone(), + format!("build error: {}", e), + 1, + None, + None, + ), + Err(e) => ( + false, + request_id.clone(), + format!("build task panicked: {}", e), + 1, + None, + None, + ), + }; + + let _ = bridge.await; + + let result_event = serde_json::json!({ + "type": "result", + "success": success, + "request_id": rid, + "message": msg, + "exit_code": code, + "output_file": output_file, + "output_dir": output_dir, + }); + let mut chunk = result_event.to_string(); + chunk.push('\n'); + let _ = async_tx.send(bytes::Bytes::from(chunk)); + }); + + // Return streaming response immediately + let stream = futures::stream::unfold(async_rx, |mut rx| async move { + rx.recv() + .await + .map(|data| (Ok::<_, std::convert::Infallible>(data), rx)) + }); + let body = axum::body::Body::from_stream(stream); + axum::response::Response::builder() + .header("content-type", "application/x-ndjson") + .body(body) + .unwrap() + .into_response() + } else { + // --- NON-STREAMING PATH (existing behavior) --- + let _op_guard = OperationGuard::new( + &ctx, + fbuild_core::DaemonState::Building, + Some(format!("Building {}", req.project_dir)), + ); + let lock = ctx.project_lock(&project_dir); + let _guard = lock.lock().await; + + let params = fbuild_build::BuildParams { + project_dir: project_dir.clone(), + env_name: env_name.clone(), + clean: req.clean_build, + profile, + build_dir, + verbose: req.verbose, + jobs: req.jobs, + generate_compiledb, + compiledb_only: req.compiledb_only, + log_sender: None, + symbol_analysis: req.symbol_analysis, + symbol_analysis_path: resolved_symbol_analysis_path, + no_timestamp: req.no_timestamp, + src_dir: req.src_dir, + pio_env: req.pio_env, + extra_build_flags: Vec::new(), + watch_set_cache: Some(Arc::clone(&ctx.watch_set_cache) as Arc<_>), + }; + + let result = tokio::task::spawn_blocking(move || { + let orchestrator = fbuild_build::get_orchestrator(platform)?; + orchestrator.build(¶ms) + }) + .await; + + match result { + Ok(Ok(build_result)) => { + let exported = if build_result.success { + if let Some(ref out_dir) = resolved_output_dir { + Some(export_artifacts_bundle( + out_dir, + platform, + &env_name, + build_result.firmware_path.as_deref(), + build_result.elf_path.as_deref(), + )) + } else { + None + } + } else { + None + }; + let summary = if build_result.success { + let size_str = build_result + .size_info + .as_ref() + .map(|s| { + format!( + " (flash: {} bytes, ram: {} bytes)", + s.total_flash, s.total_ram + ) + }) + .unwrap_or_default(); + let export_suffix = match exported.as_ref() { + Some(Ok(result)) => { + format!("; artifacts exported to {}", result.output_dir.display()) + } + Some(Err(e)) => format!("; artifact export failed: {}", e), + None => String::new(), + }; + format!( + "build succeeded in {:.1}s{}{}", + build_result.build_time_secs, size_str, export_suffix + ) + } else { + build_result.message.clone() + }; + let msg = if build_result.build_log.is_empty() { + summary + } else { + let mut lines = build_result.build_log.into_lines(); + lines.push(summary); + lines.join("\n") + }; + let output_file = match exported.as_ref() { + Some(Ok(result)) => result + .primary_output + .clone() + .or(build_result.firmware_path.clone()) + .or(build_result.elf_path.clone()), + _ => build_result + .firmware_path + .clone() + .or(build_result.elf_path.clone()), + } + .map(|p| p.to_string_lossy().to_string()); + let output_dir = match exported.as_ref() { + Some(Ok(result)) => Some(result.output_dir.to_string_lossy().to_string()), + _ => None, + }; + let code = if build_result.success { 0 } else { 1 }; + ( + StatusCode::OK, + Json(OperationResponse { + success: build_result.success, + request_id, + message: msg, + exit_code: code, + output_file, + output_dir, + launch_url: None, + stdout: None, + stderr: None, + }), + ) + .into_response() + } + Ok(Err(e)) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("build error: {}", e), + )), + ) + .into_response(), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("build task panicked: {}", e), + )), + ) + .into_response(), + } + } +} diff --git a/crates/fbuild-daemon/src/handlers/operations/common.rs b/crates/fbuild-daemon/src/handlers/operations/common.rs new file mode 100644 index 00000000..c86821d5 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/operations/common.rs @@ -0,0 +1,433 @@ +//! Shared helpers used by every operation handler: +//! env-var feature switches, the RAII [`OperationGuard`], deploy-route +//! parsing, client-path resolution, and the artifact-bundle exporter. + +use crate::context::DaemonContext; +use crate::models::DeployRequest; +use serde::Serialize; +use std::path::{Path, PathBuf}; +use std::sync::atomic::Ordering; +use std::sync::Arc; + +/// Returns `true` when the daemon should route ESP32 `verify-flash` +/// pre-checks through the native [`espflash`] crate (issue #66) instead +/// of the Python `esptool` subprocess. +/// +/// Controlled by the `FBUILD_USE_ESPFLASH_VERIFY` environment variable. +/// Native verify is enabled by default when compiled in. +/// Set this variable to `0`, `false`, `no`, or `off` (case-insensitive) +/// to force esptool. +#[cfg(feature = "espflash-native")] +pub(crate) fn native_verify_enabled() -> bool { + env_default_enabled("FBUILD_USE_ESPFLASH_VERIFY") +} + +/// Returns `true` when the daemon should trust an in-memory firmware +/// hash (keyed by port) and skip the `verify-flash` MD5 round-trip on +/// a warm redeploy. +/// +/// Safety model: the hash is only honoured if the port has been +/// continuously enumerated since the hash was recorded (enforced by +/// [`DeviceManager::trusted_firmware_hash`]). If the user unplugged +/// the board and something else flashed it before it came back, the +/// disconnect edge invalidates the cached hash and the deploy falls +/// through to the normal verify-flash path. +/// +/// Controlled by the `FBUILD_TRUST_DEVICE_HASH` environment variable +/// (set to `1`, `true`, `yes`, or `on` — case-insensitive). Default +/// off while the path accumulates bench time, mirroring the opt-in +/// convention used by `FBUILD_USE_ESPFLASH_{VERIFY,WRITE}`. +pub(crate) fn trust_device_hash_enabled() -> bool { + match std::env::var("FBUILD_TRUST_DEVICE_HASH") { + Ok(v) => matches!( + v.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ), + Err(_) => false, + } +} + +/// Compute a stable SHA-256 over the three ESP32 flash regions the +/// daemon would otherwise MD5 with `verify-flash`. Hashes the tuple +/// `(offset_le, len_le, bytes)` for each region in fixed order +/// (bootloader → partitions → firmware) so the digest uniquely +/// identifies the image that would be written; two builds of the +/// same source with identical output hash to the same value. +/// +/// Memoized on [`crate::context::DaemonContext::image_hash_memo`] +/// keyed by firmware path: if all three files' `mtime` matches the +/// previously-stored tuple, the cached hash is reused (skipping the +/// 2–4 MB disk read + SHA-256) — the dominant non-serial cost on the +/// trust-skip path. Cache entries self-invalidate when any `mtime` +/// advances. +/// +/// Returns `None` if any of the three files is missing on disk, so +/// the caller treats it as "can't trust-skip, fall through to +/// verify-flash." +pub(crate) fn compute_esp32_image_hash( + ctx: &crate::context::DaemonContext, + firmware_path: &std::path::Path, + bootloader_offset: u32, + partitions_offset: u32, + firmware_offset: u32, +) -> Option<[u8; 32]> { + use sha2::{Digest, Sha256}; + let build_dir = firmware_path.parent()?; + let bootloader_path = build_dir.join("bootloader.bin"); + let partitions_path = build_dir.join("partitions.bin"); + let firmware = firmware_path.to_path_buf(); + + let mt = |p: &std::path::Path| -> Option { + std::fs::metadata(p).ok()?.modified().ok() + }; + let mtimes = (mt(&bootloader_path)?, mt(&partitions_path)?, mt(&firmware)?); + + // Fast path: the three files have the same `mtime` as last time + // we hashed them, so the output bytes are unchanged. Reuse the + // stored digest instead of re-reading + re-hashing (~5-15 ms). + if let Some(memo) = ctx.image_hash_memo.get(&firmware) { + if memo.bootloader_mtime == mtimes.0 + && memo.partitions_mtime == mtimes.1 + && memo.firmware_mtime == mtimes.2 + { + return Some(memo.hash); + } + } + + // Miss: rebuild the digest over the current file contents and + // record it alongside the captured `mtime`s. + let regions: [(u32, &std::path::Path); 3] = [ + (bootloader_offset, bootloader_path.as_path()), + (partitions_offset, partitions_path.as_path()), + (firmware_offset, firmware.as_path()), + ]; + let mut hasher = Sha256::new(); + for (offset, path) in ®ions { + let bytes = std::fs::read(path).ok()?; + hasher.update(offset.to_le_bytes()); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(&bytes); + } + let hash: [u8; 32] = hasher.finalize().into(); + ctx.image_hash_memo.insert( + firmware.clone(), + crate::context::ImageHashMemo { + bootloader_mtime: mtimes.0, + partitions_mtime: mtimes.1, + firmware_mtime: mtimes.2, + hash, + }, + ); + Some(hash) +} + +/// Returns `true` when the daemon should route ESP32 `write-flash` +/// through the native [`espflash`] crate (issue #66) instead of the +/// Python `esptool` subprocess. +/// +/// Controlled by the `FBUILD_USE_ESPFLASH_WRITE` environment variable. +/// Native write is enabled by default when compiled in. Independent of +/// `FBUILD_USE_ESPFLASH_VERIFY`. Set it to `0`, `false`, `no`, or `off` +/// (case-insensitive) to force esptool. +#[cfg(feature = "espflash-native")] +pub(crate) fn native_write_enabled() -> bool { + env_default_enabled("FBUILD_USE_ESPFLASH_WRITE") +} + +#[cfg(feature = "espflash-native")] +fn env_default_enabled(name: &str) -> bool { + match std::env::var(name) { + Ok(v) => !matches!( + v.trim().to_ascii_lowercase().as_str(), + "0" | "false" | "no" | "off" + ), + Err(_) => true, + } +} + +pub(crate) fn qemu_extra_build_flags(platform: fbuild_core::Platform, mcu: &str) -> Vec { + if platform == fbuild_core::Platform::Espressif32 && mcu.eq_ignore_ascii_case("esp32s3") { + vec![ + "-DARDUINO_USB_MODE=0".to_string(), + "-DARDUINO_USB_CDC_ON_BOOT=0".to_string(), + ] + } else { + Vec::new() + } +} + +/// RAII guard that sets `operation_in_progress` to true on creation +/// and false on drop. Also tracks daemon state and current operation description. +pub(crate) struct OperationGuard { + flag: Arc, + state: Arc>, + operation: Arc>>, +} + +impl OperationGuard { + pub(crate) fn new( + ctx: &DaemonContext, + daemon_state: fbuild_core::DaemonState, + description: Option, + ) -> Self { + ctx.touch_activity(); + ctx.operation_in_progress.store(true, Ordering::Relaxed); + if let Ok(mut s) = ctx.daemon_state.write() { + *s = daemon_state; + } + if let Ok(mut op) = ctx.current_operation.write() { + *op = description; + } + Self { + flag: Arc::clone(&ctx.operation_in_progress), + state: Arc::clone(&ctx.daemon_state), + operation: Arc::clone(&ctx.current_operation), + } + } +} + +impl Drop for OperationGuard { + fn drop(&mut self) { + self.flag.store(false, Ordering::Relaxed); + if let Ok(mut s) = self.state.write() { + *s = fbuild_core::DaemonState::Idle; + } + if let Ok(mut op) = self.operation.write() { + *op = None; + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EmulatorKind { + Qemu, + Avr8js, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DeployRoute { + Device, + Emulator(EmulatorKind), +} + +fn parse_emulator_kind(raw: &str) -> fbuild_core::Result { + match raw { + "qemu" => Ok(EmulatorKind::Qemu), + "avr8js" => Ok(EmulatorKind::Avr8js), + other => Err(fbuild_core::FbuildError::DeployFailed(format!( + "unsupported emulator '{}'", + other + ))), + } +} + +pub(crate) fn infer_default_emulator_kind( + platform: fbuild_core::Platform, + mcu: &str, +) -> Option { + match platform { + fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr => { + Some(EmulatorKind::Avr8js) + } + fbuild_core::Platform::Espressif32 if mcu.eq_ignore_ascii_case("esp32s3") => { + Some(EmulatorKind::Qemu) + } + _ => None, + } +} + +pub(crate) fn parse_deploy_route( + req: &DeployRequest, + default_emulator: Option, +) -> fbuild_core::Result { + if let Some(target) = req.target.as_deref() { + return match target { + "device" => Ok(DeployRoute::Device), + "qemu" => Ok(DeployRoute::Emulator(EmulatorKind::Qemu)), + "avr8js" => Ok(DeployRoute::Emulator(EmulatorKind::Avr8js)), + other => Err(fbuild_core::FbuildError::DeployFailed(format!( + "unsupported deploy target '{}'", + other + ))), + }; + } + + let destination = req.to.as_deref().unwrap_or("device"); + match destination { + "device" => { + if req.qemu { + return Err(fbuild_core::FbuildError::DeployFailed( + "--qemu cannot be combined with --to device".to_string(), + )); + } + if let Some(emulator) = req.emulator.as_deref() { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "--emulator {} requires --to emu", + emulator + ))); + } + Ok(DeployRoute::Device) + } + "emu" | "emulator" => { + let emulator = if req.qemu { + if let Some(explicit) = req.emulator.as_deref() { + if explicit != "qemu" { + return Err(fbuild_core::FbuildError::DeployFailed( + "--qemu cannot be combined with a different --emulator".to_string(), + )); + } + } + "qemu" + } else { + match req.emulator.as_deref() { + Some(explicit) => explicit, + None => match default_emulator { + Some(EmulatorKind::Qemu) => "qemu", + Some(EmulatorKind::Avr8js) => "avr8js", + None => { + return Err(fbuild_core::FbuildError::DeployFailed( + "--to emu requires an explicit --emulator for this board" + .to_string(), + )) + } + }, + } + }; + Ok(DeployRoute::Emulator(parse_emulator_kind(emulator)?)) + } + other => Err(fbuild_core::FbuildError::DeployFailed(format!( + "unsupported deploy destination '{}'", + other + ))), + } +} + +pub(crate) fn resolve_client_path( + raw: &str, + caller_cwd: Option<&str>, + project_dir: &Path, +) -> PathBuf { + let path = PathBuf::from(raw); + if path.is_absolute() { + path + } else if let Some(cwd) = caller_cwd { + PathBuf::from(cwd).join(path) + } else { + project_dir.join(path) + } +} + +#[derive(Debug, Serialize)] +struct ArtifactFileEntry { + name: String, + role: String, +} + +#[derive(Debug, Serialize)] +struct ArtifactManifest { + platform: String, + environment: String, + primary_firmware: Option, + elf: Option, + files: Vec, +} + +pub(crate) struct ArtifactExportResult { + pub(crate) output_dir: PathBuf, + pub(crate) primary_output: Option, +} + +fn artifact_role(name: &str, primary_firmware: Option<&Path>, elf_path: Option<&Path>) -> String { + if primary_firmware + .and_then(|p| p.file_name()) + .is_some_and(|n| n == name) + { + "firmware".to_string() + } else if elf_path + .and_then(|p| p.file_name()) + .is_some_and(|n| n == name) + { + "elf".to_string() + } else { + match name { + "bootloader.bin" => "bootloader".to_string(), + "partitions.bin" => "partitions".to_string(), + "compile_commands.json" => "compile_database".to_string(), + "symbol_analysis.txt" => "symbol_analysis".to_string(), + _ => "artifact".to_string(), + } + } +} + +pub(crate) fn export_artifacts_bundle( + output_dir: &Path, + platform: fbuild_core::Platform, + env_name: &str, + primary_firmware: Option<&Path>, + elf_path: Option<&Path>, +) -> fbuild_core::Result { + std::fs::create_dir_all(output_dir)?; + + let source_dir = primary_firmware + .and_then(|p| p.parent()) + .or_else(|| elf_path.and_then(|p| p.parent())) + .ok_or_else(|| { + fbuild_core::FbuildError::Other( + "could not determine source artifact directory for export".to_string(), + ) + })?; + + let mut copied_names = Vec::new(); + for entry in std::fs::read_dir(source_dir)? { + let entry = entry?; + let path = entry.path(); + if !path.is_file() { + continue; + } + let file_name = match path.file_name() { + Some(name) => name, + None => continue, + }; + let dest = output_dir.join(file_name); + if path != dest { + std::fs::copy(&path, &dest)?; + } + copied_names.push(file_name.to_string_lossy().to_string()); + } + copied_names.sort(); + copied_names.dedup(); + + let manifest = ArtifactManifest { + platform: format!("{:?}", platform), + environment: env_name.to_string(), + primary_firmware: primary_firmware + .and_then(|p| p.file_name()) + .map(|p| p.to_string_lossy().to_string()), + elf: elf_path + .and_then(|p| p.file_name()) + .map(|p| p.to_string_lossy().to_string()), + files: copied_names + .iter() + .map(|name| ArtifactFileEntry { + name: name.clone(), + role: artifact_role(name, primary_firmware, elf_path), + }) + .collect(), + }; + + std::fs::write( + output_dir.join("artifacts.json"), + serde_json::to_vec_pretty(&manifest).map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to serialize artifact manifest: {}", e)) + })?, + )?; + + let primary_output = primary_firmware + .and_then(|p| p.file_name()) + .map(|name| output_dir.join(name)); + + Ok(ArtifactExportResult { + output_dir: output_dir.to_path_buf(), + primary_output, + }) +} diff --git a/crates/fbuild-daemon/src/handlers/operations/deploy.rs b/crates/fbuild-daemon/src/handlers/operations/deploy.rs new file mode 100644 index 00000000..9c15b39d --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/operations/deploy.rs @@ -0,0 +1,885 @@ +//! `POST /api/deploy` — build (or reuse) firmware, flash, optionally monitor. + +use super::common::{ + compute_esp32_image_hash, export_artifacts_bundle, infer_default_emulator_kind, + parse_deploy_route, qemu_extra_build_flags, resolve_client_path, trust_device_hash_enabled, + DeployRoute, EmulatorKind, OperationGuard, +}; +use super::monitor::{run_monitor_loop, MonitorOutcome}; +use crate::context::DaemonContext; +use crate::models::{DeployRequest, OperationResponse}; +use axum::extract::State; +use axum::http::StatusCode; +use axum::Json; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; + +#[cfg(feature = "espflash-native")] +use super::common::{native_verify_enabled, native_write_enabled}; + +/// POST /api/deploy +pub async fn deploy( + State(ctx): State>, + Json(req): Json, +) -> (StatusCode, Json) { + let request_id = req + .request_id + .clone() + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let project_dir = PathBuf::from(&req.project_dir); + + if !project_dir.exists() { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("project directory does not exist: {}", req.project_dir), + )), + ); + } + + let _op_guard = OperationGuard::new( + &ctx, + fbuild_core::DaemonState::Deploying, + Some(format!("Deploying {}", req.project_dir)), + ); + + // Parse config + let config = + match fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")) { + Ok(c) => c, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("failed to parse platformio.ini: {}", e), + )), + ); + } + }; + + let env_name = req + .environment + .clone() + .or_else(|| config.get_default_environment().map(|s| s.to_string())) + .unwrap_or_else(|| "default".to_string()); + + let env_config = match config.get_env_config(&env_name) { + Ok(c) => c, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("invalid environment '{}': {}", env_name, e), + )), + ); + } + }; + let resolved_output_dir = req + .output_dir + .as_deref() + .map(|p| resolve_client_path(p, req.caller_cwd.as_deref(), &project_dir)); + + let platform_str = env_config.get("platform").cloned().unwrap_or_default(); + let platform = match fbuild_core::Platform::from_platform_str(&platform_str) { + Some(p) => p, + None => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("unsupported platform: {}", platform_str), + )), + ); + } + }; + let board_id = env_config.get("board").cloned().unwrap_or_else(|| { + fbuild_build::get_platform_support(platform) + .map(|s| s.default_board_id().to_string()) + .unwrap_or_else(|_| "unknown".to_string()) + }); + let board_overrides = config.get_board_overrides(&env_name).unwrap_or_default(); + let board = fbuild_config::BoardConfig::from_board_id(&board_id, &board_overrides) + .or_else(|_| fbuild_config::BoardConfig::from_board_id(&board_id, &HashMap::new())) + .ok(); + let deploy_route = match parse_deploy_route( + &req, + board + .as_ref() + .and_then(|board| infer_default_emulator_kind(platform, &board.mcu)), + ) { + Ok(route) => route, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail(request_id, e.to_string())), + ); + } + }; + + // Build first unless skip_build + let (firmware_path, elf_path) = if req.skip_build { + // Look for existing firmware using the standard search order + // (profiles: release/quick, base env dir, legacy .pio/build) + match fbuild_paths::find_firmware(&project_dir, &env_name, None) { + Some(path) => { + let elf = path.parent().map(|dir| dir.join("firmware.elf")); + let elf = elf.filter(|p| p.exists()); + (path, elf) + } + None => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + "no firmware found; run build first or remove skip_build".to_string(), + )), + ); + } + } + } else { + // Run build first + let lock = ctx.project_lock(&project_dir); + let _guard = lock.lock().await; + + let build_dir = fbuild_paths::get_project_build_root(&project_dir); + let params = fbuild_build::BuildParams { + project_dir: project_dir.clone(), + env_name: env_name.clone(), + clean: req.clean_build, + profile: fbuild_core::BuildProfile::Release, + build_dir, + verbose: req.verbose, + jobs: None, + generate_compiledb: false, + compiledb_only: false, + log_sender: None, + symbol_analysis: false, + symbol_analysis_path: None, + no_timestamp: false, + src_dir: req.src_dir, + pio_env: req.pio_env, + extra_build_flags: if deploy_route == DeployRoute::Emulator(EmulatorKind::Qemu) { + board + .as_ref() + .map(|board| qemu_extra_build_flags(platform, &board.mcu)) + .unwrap_or_default() + } else { + Vec::new() + }, + watch_set_cache: Some(Arc::clone(&ctx.watch_set_cache) as Arc<_>), + }; + + let build_result = { + let p = platform; + tokio::task::spawn_blocking(move || { + let orchestrator = fbuild_build::get_orchestrator(p)?; + orchestrator.build(¶ms) + }) + .await + }; + + match build_result { + Ok(Ok(r)) if r.success => { + let fw = r.firmware_path.clone().unwrap_or_else(|| { + r.elf_path + .clone() + .unwrap_or_else(|| PathBuf::from("firmware.bin")) + }); + (fw, r.elf_path) + } + Ok(Ok(r)) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("build failed: {}", r.message), + )), + ); + } + Ok(Err(e)) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("build error: {}", e), + )), + ); + } + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("build task panicked: {}", e), + )), + ); + } + } + }; + + let artifact_export = match resolved_output_dir.as_ref() { + Some(out_dir) => match export_artifacts_bundle( + out_dir, + platform, + &env_name, + Some(&firmware_path), + elf_path.as_deref(), + ) { + Ok(result) => Some(result), + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("failed to export artifacts: {}", e), + )), + ); + } + }, + None => None, + }; + + let reported_output_file = artifact_export + .as_ref() + .and_then(|r| r.primary_output.clone()) + .unwrap_or_else(|| firmware_path.clone()) + .to_string_lossy() + .to_string(); + let reported_output_dir = artifact_export + .as_ref() + .map(|r| r.output_dir.to_string_lossy().to_string()); + + if deploy_route == DeployRoute::Emulator(EmulatorKind::Avr8js) { + return crate::handlers::emulator::deploy_avr8js( + ctx, + crate::handlers::emulator::DeployAvr8jsRequest { + request_id, + project_dir, + env_name, + board_id, + platform, + firmware_path, + elf_path, + monitor_after: req.monitor_after, + output_file: reported_output_file, + output_dir: reported_output_dir, + monitor_timeout: req.monitor_timeout, + halt_on_error: req.monitor_halt_on_error.clone(), + halt_on_success: req.monitor_halt_on_success.clone(), + expect: req.monitor_expect.clone(), + show_timestamp: req.monitor_show_timestamp, + verbose: req.verbose, + }, + ) + .await; + } + + if deploy_route == DeployRoute::Emulator(EmulatorKind::Qemu) { + return crate::handlers::emulator::deploy_qemu( + ctx, + crate::handlers::emulator::DeployQemuRequest { + request_id, + project_dir, + env_name, + board_id, + platform, + firmware_path, + elf_path, + output_file: reported_output_file, + output_dir: reported_output_dir, + monitor_timeout: req.monitor_timeout, + qemu_timeout_secs: req.qemu_timeout, + halt_on_error: req.monitor_halt_on_error.clone(), + halt_on_success: req.monitor_halt_on_success.clone(), + expect: req.monitor_expect.clone(), + show_timestamp: req.monitor_show_timestamp, + verbose: req.verbose, + board_overrides, + }, + ) + .await; + } + + // Preempt serial if port specified + let deploy_port_str = req.port.clone(); + if let Some(ref p) = deploy_port_str { + let _ = ctx + .serial_manager + .preempt_for_deploy(p, "deploy".to_string(), request_id.clone()) + .await; + } + + // Extract board ID before spawn_blocking (env_config borrows config) + let board_id = env_config.get("board").cloned().unwrap_or_else(|| { + fbuild_build::get_platform_support(platform) + .map(|s| s.default_board_id().to_string()) + .unwrap_or_else(|_| "unknown".to_string()) + }); + + // Extract env-section board_build.* / board_upload.* overrides BEFORE + // spawn_blocking (env_config borrows config). Without these, fbuild + // would silently ignore `board_build.flash_mode = dio` (and friends) + // from the user's [env:X] section and fall back to whatever the board + // JSON says — producing a firmware/bootloader that doesn't match the + // hardware. The build phase already passes overrides; the deploy phase + // must do the same so esptool flashes with the right --flash-mode. + let board_overrides = config.get_board_overrides(&env_name).unwrap_or_default(); + + // Deploy + let deploy_env = env_name.clone(); + let deploy_project = project_dir.clone(); + let deploy_port = deploy_port_str.clone(); + let deploy_fw = firmware_path.clone(); + let baud_override = req.baud_rate; + let deploy_board_overrides = board_overrides.clone(); + // Snapshot the ctx pointer so the spawn_blocking closure can + // consult / update the daemon's in-memory trusted-hash cache + // without needing a cross-thread lock handshake. + let ctx_for_deploy = Arc::clone(&ctx); + let trusted_hash_enabled = trust_device_hash_enabled(); + // Refresh the enumeration cache so the trust-hash invalidation + // path (`last_disconnect_at`) sees any unplug/replug that + // happened between the previous deploy and now. Without this, + // a user who swapped boards at the same COM port without hitting + // a device-list endpoint could trip the trust check into a + // false match. + // + // Back-to-back warm deploys (the 4 s / 1 s budget target) would + // otherwise re-pay ~20–30 ms per deploy on Windows; cap the cost + // at one enumeration per 2 s. The window is short enough that a + // physically-sneaky board swap between two in-flight deploys + // still needs to happen inside that window to trip trust, and + // the trust-check still requires `is_connected == true` on the + // cached DeviceState, which the most-recent refresh supplied. + if trusted_hash_enabled { + ctx.device_manager + .refresh_devices_if_stale(std::time::Duration::from_secs(2)); + } + let deploy_result = tokio::task::spawn_blocking(move || { + // Populated by the Espressif32 arm with (image_hash, port). + // The tail of the closure consults it after `deployer.deploy` + // returns to record or invalidate the daemon's trusted-hash + // cache. Other platforms leave it `None`. + let mut trusted_hash_update: Option<([u8; 32], String)> = None; + let deployer: Box = match platform { + fbuild_core::Platform::Espressif32 => { + let board_config = + fbuild_config::BoardConfig::from_board_id(&board_id, &deploy_board_overrides) + .unwrap_or_else(|_| { + fbuild_config::BoardConfig::from_board_id( + "esp32dev", + &deploy_board_overrides, + ) + .unwrap() + }); + // Load MCU config to get flash offsets and esptool defaults. + let mcu_config = fbuild_build::esp32::mcu_config::get_mcu_config(&board_config.mcu) + .unwrap_or_else(|_| { + fbuild_build::esp32::mcu_config::get_mcu_config("esp32").unwrap() + }); + // Flash mode: `board_config.flash_mode` is `None` for ESP32 + // chips unless the user explicitly set `board_build.flash_mode` + // in their `[env:X]` section (see `BoardConfig::from_board_id` + // — the JSON-shipped value is intentionally dropped for ESP32 + // because ESP32-S3's QIE-bit init is unreliable). The unwrap + // therefore falls back to the per-MCU default "dio". + let esptool_params = fbuild_deploy::esp32::EsptoolParams { + flash_mode: board_config + .flash_mode + .as_deref() + .unwrap_or(mcu_config.default_flash_mode()) + .to_string(), + flash_freq: { + let f_for_image = board_config + .f_image + .as_deref() + .or(board_config.f_flash.as_deref()); + fbuild_build::esp32::esp32_linker::f_flash_to_esptool_freq( + f_for_image, + mcu_config.default_flash_freq(), + ) + }, + default_baud: mcu_config.default_baud().to_string(), + before_reset: mcu_config.before_reset().to_string(), + after_reset: mcu_config.after_reset().to_string(), + }; + let deployer = fbuild_deploy::esp32::Esp32Deployer::from_board_config( + &board_config, + mcu_config.bootloader_offset(), + mcu_config.partitions_offset(), + mcu_config.firmware_offset(), + &esptool_params, + false, + ); + let deployer = if let Some(baud) = baud_override { + deployer.with_baud_rate(&baud.to_string()) + } else { + deployer + }; + // Issue #66: native `verify-flash` + `write-flash` via + // the `espflash` crate. This is compiled in by default; + // `FBUILD_USE_ESPFLASH_*` are opt-out switches, and the + // deployer falls back to esptool automatically when a + // native operation fails. + #[cfg(feature = "espflash-native")] + let deployer = deployer + .with_native_verify(native_verify_enabled()) + .with_native_write(native_write_enabled()); + + // Compute a deterministic SHA-256 over the three + // regions we'd otherwise verify-flash. Used twice + // below: once to consult the daemon's trusted-hash + // cache (opt-in via `FBUILD_TRUST_DEVICE_HASH=1`), + // and once to *record* the hash after a successful + // deploy so the next warm redeploy can skip the MD5 + // round-trip entirely. A `None` return means one of + // the three files isn't on disk yet — treat it as + // "can't trust-skip" rather than erroring, so the + // fallback path is free to rebuild missing artefacts. + let image_hash = compute_esp32_image_hash( + &ctx_for_deploy, + &deploy_fw, + u32::from_str_radix( + mcu_config.bootloader_offset().trim_start_matches("0x"), + 16, + ) + .unwrap_or(0), + u32::from_str_radix( + mcu_config.partitions_offset().trim_start_matches("0x"), + 16, + ) + .unwrap_or(0), + u32::from_str_radix( + mcu_config.firmware_offset().trim_start_matches("0x"), + 16, + ) + .unwrap_or(0), + ); + + // Session-trusted verify-skip: if the daemon last + // flashed *this exact image* onto *this port* and the + // port has been continuously enumerated since then, + // no external agent could have re-flashed the chip + // without breaking our enumeration (`last_disconnect_at` + // is how we detect it). Skip the entire serial open + + // espflash connect + MD5 round-trip. + if let (Some(port), Some(hash)) = (deploy_port.as_deref(), image_hash) { + if trusted_hash_enabled { + if let Some(trusted) = + ctx_for_deploy.device_manager.trusted_firmware_hash(port) + { + if trusted == hash { + tracing::info!( + port, + "trusted-hash: session-trusted match; skipping verify-flash entirely" + ); + return Ok(fbuild_deploy::DeploymentResult { + success: true, + message: format!( + "firmware already current on {} (skipped via session trust)", + port + ), + port: Some(port.to_string()), + stdout: String::new(), + stderr: String::new(), + outcome: fbuild_deploy::DeployOutcome::VerifySkip, + }); + } + } + } + } + + // Fast deploy: ask the device whether it already holds + // the exact firmware/bootloader/partitions we'd be about + // to write. Uses esptool's `verify-flash` which dispatches + // to the stub flasher's `FLASH_MD5SUM` command — no full + // read-back, just one MD5 round-trip per region. + // + // Measured on a 2.4 MB FastLED esp32s3 image: + // * fresh write-flash: ~25 s + // * verify-flash skip: ~6 s (-19 s, ~76% faster) + // + // Falls through to the normal flash path silently on + // mismatch or transport error so we never break a deploy + // that the verify call didn't understand. + let mut selective_regions: Option> = None; + if let Some(port) = deploy_port.as_deref() { + match deployer.try_verify_deployment(&deploy_fw, port) { + Ok(fbuild_deploy::esp32::VerifyOutcome::Match { stdout, stderr }) => { + tracing::info!( + port, + "verify-flash: device already running this exact image; skipping write" + ); + return Ok(fbuild_deploy::DeploymentResult { + success: true, + message: format!( + "firmware already current on {} (skipped via verify-flash)", + port + ), + port: Some(port.to_string()), + stdout, + stderr, + outcome: fbuild_deploy::DeployOutcome::VerifySkip, + }); + } + Ok(fbuild_deploy::esp32::VerifyOutcome::Mismatch { regions, .. }) => { + // Pick only the regions that actually differ + // so we avoid the ~1s bootloader/partitions + // rewrite when only firmware changed. Empty + // `regions` means parsing failed — fall back + // to full flash. + let to_write: Vec<_> = regions + .iter() + .filter(|r| !r.matched) + .map(|r| r.region) + .collect(); + if !regions.is_empty() && !to_write.is_empty() && to_write.len() < 3 { + tracing::info!( + port, + "verify-flash: only {} region(s) differ; flashing selectively", + to_write.len() + ); + selective_regions = Some(to_write); + } else { + tracing::info!( + port, + "verify-flash: device image differs; proceeding with full flash" + ); + } + } + Err(e) => { + tracing::warn!( + port, + "verify-flash pre-check failed ({}); proceeding with full flash", + e + ); + } + } + } + if let (Some(regions), Some(port)) = (selective_regions, deploy_port.as_deref()) { + let result = deployer.deploy_regions(&deploy_fw, port, ®ions); + // Record/invalidate the trusted hash based on + // the selective-flash outcome so the next warm + // redeploy can short-circuit via trust-skip. + if let Some(hash) = image_hash { + match &result { + Ok(r) if r.success => { + ctx_for_deploy + .device_manager + .set_trusted_firmware_hash(port, hash); + } + _ => { + ctx_for_deploy.device_manager.clear_trusted_firmware_hash(port); + } + } + } + return result; + } + + // Capture the hash into the boxed deployer closure + // below — the full-flash path (`deployer.deploy(...)`) + // at the end of this `spawn_blocking` applies the + // same record/invalidate rule. We stash the hash + + // port via a small impl-only wrapper because the + // `Deployer` trait doesn't expose the cache hook. + // Stored here so the common tail after the `match` + // can reach it without re-threading every arm. + // (AVR / other arms leave it `None` → no-op.) + trusted_hash_update = image_hash.zip(deploy_port.as_deref().map(str::to_string)); + + Box::new(deployer) + } + fbuild_core::Platform::AtmelAvr | fbuild_core::Platform::AtmelMegaAvr => { + let board_config = + fbuild_config::BoardConfig::from_board_id(&board_id, &deploy_board_overrides) + .unwrap_or_else(|_| { + fbuild_config::BoardConfig::from_board_id( + "uno", + &deploy_board_overrides, + ) + .unwrap() + }); + let avr_config = fbuild_build::avr::mcu_config::get_avr_config().unwrap(); + let avrdude_params = fbuild_deploy::avr::AvrdudeParams { + default_programmer: avr_config.avrdude.default_programmer.clone(), + default_baud: avr_config.avrdude.default_baud.to_string(), + timeout_secs: avr_config.avrdude.timeout_secs, + }; + let deployer = fbuild_deploy::avr::AvrDeployer::from_board_config( + &board_config, + &avrdude_params, + false, + ); + let deployer = if let Some(baud) = baud_override { + deployer.with_baud_rate(&baud.to_string()) + } else { + deployer + }; + Box::new(deployer) + } + _ => { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "deployer for {:?} not yet implemented", + platform + ))); + } + }; + let result = deployer.deploy( + &deploy_project, + &deploy_env, + &deploy_fw, + deploy_port.as_deref(), + ); + // Session-trusted verify-skip: record (or invalidate) the + // image hash the daemon associates with this port. Scoped + // to the Espressif32 arm above — other platforms leave + // `trusted_hash_update` as `None` and this block no-ops. + // Failed or partial deploys clear the cache so the next + // warm run falls back to the verify-flash path. + if let Some((hash, port)) = trusted_hash_update { + match &result { + Ok(r) if r.success => { + ctx_for_deploy + .device_manager + .set_trusted_firmware_hash(&port, hash); + } + _ => { + ctx_for_deploy + .device_manager + .clear_trusted_firmware_hash(&port); + } + } + } + result + }) + .await; + + // Clear preemption then wait for USB re-enumeration. + // Fast-poll the serial port instead of a hard 2s sleep — most ESP32-S3 + // boards with native USB re-enumerate in <500ms. + // + // Skip the poll entirely when the deploy didn't actually touch + // flash (VerifySkip): no reset happened, no USB re-enumeration + // is coming, and the poll's `open()` probe would conflict with + // any already-attached monitor and burn the full 3 s timeout. + // This is load-bearing for the < 4 s warm-trust-skip budget. + let deploy_skipped_bus_work = matches!( + &deploy_result, + Ok(Ok(r)) if r.success && matches!( + r.outcome, + fbuild_deploy::DeployOutcome::VerifySkip + ) + ); + if let Some(ref p) = deploy_port_str { + ctx.serial_manager.clear_preemption(p).await; + if !deploy_skipped_bus_work { + let port_name = p.clone(); + let _ = tokio::task::spawn_blocking(move || { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); + while std::time::Instant::now() < deadline { + if serialport::new(&port_name, 115200) + .timeout(std::time::Duration::from_millis(50)) + .open() + .is_ok() + { + return; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + tracing::warn!( + "USB re-enumeration: port {} not available after 3s", + port_name + ); + }) + .await; + } + } + + let (deploy_success, deploy_stdout, deploy_stderr, deploy_outcome) = match deploy_result { + Ok(Ok(r)) if r.success => (true, Some(r.stdout), Some(r.stderr), r.outcome), + Ok(Ok(r)) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse { + success: false, + request_id, + message: r.message, + exit_code: 1, + output_file: Some(reported_output_file.clone()), + output_dir: reported_output_dir.clone(), + launch_url: None, + stdout: Some(r.stdout), + stderr: Some(r.stderr), + }), + ); + } + Ok(Err(e)) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("deploy error: {}", e), + )), + ); + } + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("deploy task panicked: {}", e), + )), + ); + } + }; + // Build the "deploy succeeded (...)" prefix used by every + // monitor-attached and non-monitor-attached response below. Stable + // wording — see GitHub issue #76 and the DeployOutcome::describe + // test in fbuild-deploy. + let deploy_prefix = format!("deploy succeeded ({})", deploy_outcome.describe()); + + // Post-deploy monitoring: if monitor_after is set, open the serial port + // and stream lines checking halt conditions (matching Python behavior). + if deploy_success && req.monitor_after { + let monitor_port = deploy_port_str.unwrap_or_else(|| "/dev/ttyUSB0".to_string()); + let baud_rate = 115200u32; + + // Open the port for monitoring + if let Err(e) = ctx + .serial_manager + .open_port(&monitor_port, baud_rate, &request_id) + .await + { + return ( + StatusCode::OK, + Json(OperationResponse { + success: true, + request_id, + message: format!("{} but monitor failed to open port: {}", deploy_prefix, e), + exit_code: 0, + output_file: Some(reported_output_file.clone()), + output_dir: reported_output_dir.clone(), + launch_url: None, + stdout: deploy_stdout, + stderr: deploy_stderr, + }), + ); + } + + // Subscribe to broadcast channel + let mut rx = match ctx.serial_manager.attach_reader(&monitor_port, &request_id) { + Some(rx) => rx, + None => { + return ( + StatusCode::OK, + Json(OperationResponse { + success: true, + request_id, + message: format!("{} but monitor could not attach reader", deploy_prefix), + exit_code: 0, + output_file: Some(reported_output_file.clone()), + output_dir: reported_output_dir.clone(), + launch_url: None, + stdout: deploy_stdout, + stderr: deploy_stderr, + }), + ); + } + }; + + let monitor_result = run_monitor_loop( + &mut rx, + req.monitor_timeout, + req.monitor_halt_on_error.as_deref(), + req.monitor_halt_on_success.as_deref(), + req.monitor_expect.as_deref(), + req.monitor_show_timestamp, + ) + .await; + + ctx.serial_manager.detach_reader(&monitor_port, &request_id); + + return match monitor_result { + MonitorOutcome::Success(msg) => ( + StatusCode::OK, + Json(OperationResponse { + success: true, + request_id, + message: format!("{}; monitor: {}", deploy_prefix, msg), + exit_code: 0, + output_file: Some(reported_output_file.clone()), + output_dir: reported_output_dir.clone(), + launch_url: None, + stdout: deploy_stdout, + stderr: deploy_stderr, + }), + ), + MonitorOutcome::Error(msg) => ( + StatusCode::OK, + Json(OperationResponse { + success: false, + request_id, + message: format!("{}; monitor error: {}", deploy_prefix, msg), + exit_code: 1, + output_file: Some(reported_output_file.clone()), + output_dir: reported_output_dir.clone(), + launch_url: None, + stdout: deploy_stdout, + stderr: deploy_stderr, + }), + ), + MonitorOutcome::Timeout { expect_found } => { + let (success, code) = if expect_found { + (true, 0) + } else { + // If expect was set and not found, that's an error + ( + req.monitor_expect.is_none(), + if req.monitor_expect.is_none() { 0 } else { 1 }, + ) + }; + ( + StatusCode::OK, + Json(OperationResponse { + success, + request_id, + message: format!( + "{}; monitor timed out{}", + deploy_prefix, + if !expect_found && req.monitor_expect.is_some() { + " (expected pattern not found)" + } else { + "" + } + ), + exit_code: code, + output_file: Some(reported_output_file.clone()), + output_dir: reported_output_dir.clone(), + launch_url: None, + stdout: deploy_stdout, + stderr: deploy_stderr, + }), + ) + } + }; + } + + ( + StatusCode::OK, + Json(OperationResponse { + success: true, + request_id, + message: deploy_prefix, + exit_code: 0, + output_file: Some(reported_output_file), + output_dir: reported_output_dir, + launch_url: None, + stdout: deploy_stdout, + stderr: deploy_stderr, + }), + ) +} diff --git a/crates/fbuild-daemon/src/handlers/operations/install_deps.rs b/crates/fbuild-daemon/src/handlers/operations/install_deps.rs new file mode 100644 index 00000000..0a99f46a --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/operations/install_deps.rs @@ -0,0 +1,124 @@ +//! `POST /api/install-deps` — fetch toolchains, frameworks, and libraries +//! without building. + +use super::common::OperationGuard; +use crate::context::DaemonContext; +use crate::models::{InstallDepsRequest, OperationResponse}; +use axum::extract::State; +use axum::http::StatusCode; +use axum::Json; +use std::path::PathBuf; +use std::sync::Arc; + +/// POST /api/install-deps +/// +/// Install toolchain, framework, and library dependencies without building. +/// Matches the Python daemon's `/api/install-deps` endpoint contract. +pub async fn install_deps( + State(ctx): State>, + Json(req): Json, +) -> (StatusCode, Json) { + let request_id = req + .request_id + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let project_dir = PathBuf::from(&req.project_dir); + + if !project_dir.exists() { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("project directory does not exist: {}", req.project_dir), + )), + ); + } + + let _op_guard = OperationGuard::new( + &ctx, + fbuild_core::DaemonState::Building, + Some(format!("Installing deps for {}", req.project_dir)), + ); + + // Acquire per-project lock + let lock = ctx.project_lock(&project_dir); + let _guard = lock.lock().await; + + // Parse platformio.ini to determine platform and resolve packages + let config = + match fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")) { + Ok(c) => c, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("failed to parse platformio.ini: {}", e), + )), + ); + } + }; + + let env_name = req + .environment + .clone() + .or_else(|| config.get_default_environment().map(|s| s.to_string())) + .unwrap_or_else(|| "default".to_string()); + + let env_config = match config.get_env_config(&env_name) { + Ok(c) => c, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("invalid environment '{}': {}", env_name, e), + )), + ); + } + }; + + let platform_str = env_config.get("platform").cloned().unwrap_or_default(); + let platform = match fbuild_core::Platform::from_platform_str(&platform_str) { + Some(p) => p, + None => { + return ( + StatusCode::BAD_REQUEST, + Json(OperationResponse::fail( + request_id, + format!("unsupported platform: {}", platform_str), + )), + ); + } + }; + + // Install dependencies via the package manager + let env_label = env_name.clone(); + let result = tokio::task::spawn_blocking(move || { + fbuild_build::install_platform_deps(platform, &project_dir) + }) + .await; + + match result { + Ok(Ok(())) => ( + StatusCode::OK, + Json(OperationResponse::ok( + request_id, + format!("Dependencies installed for environment '{}'", env_label), + )), + ), + Ok(Err(e)) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("install-deps error: {}", e), + )), + ), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("install-deps task panicked: {}", e), + )), + ), + } +} diff --git a/crates/fbuild-daemon/src/handlers/operations/mod.rs b/crates/fbuild-daemon/src/handlers/operations/mod.rs new file mode 100644 index 00000000..05d4f74f --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/operations/mod.rs @@ -0,0 +1,29 @@ +//! Build, deploy, and monitor operation handlers. +//! +//! This module is split into per-RPC submodules so each `.rs` file +//! stays under the 1000-LOC CI gate. The public API is unchanged: +//! callers still reach `build`, `deploy`, `monitor`, `reset`, and +//! `install_deps` through `crate::handlers::operations::*`. + +mod build; +mod common; +mod deploy; +mod install_deps; +mod monitor; +mod reset; + +#[cfg(test)] +mod tests; + +// Public HTTP handlers — these are wired up by `main.rs` and must +// keep their original paths (`crate::handlers::operations::`). +pub use build::build; +pub use deploy::deploy; +pub use install_deps::install_deps; +pub use monitor::monitor; +pub use reset::reset; + +// `pub(crate)` re-exports for sibling handler modules +// (`handlers::emulator` consumes these). +pub(crate) use common::{qemu_extra_build_flags, OperationGuard}; +pub(crate) use monitor::{MonitorOutcome, MonitorState}; diff --git a/crates/fbuild-daemon/src/handlers/operations/monitor.rs b/crates/fbuild-daemon/src/handlers/operations/monitor.rs new file mode 100644 index 00000000..958b126c --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/operations/monitor.rs @@ -0,0 +1,274 @@ +//! `POST /api/monitor` and the shared monitor-loop state machine used +//! by both the standalone monitor handler and the post-deploy monitor +//! attached to `/api/deploy`. + +use crate::context::DaemonContext; +use crate::models::{MonitorRequest, OperationResponse}; +use axum::extract::State; +use axum::http::StatusCode; +use axum::Json; +use std::sync::Arc; + +/// Outcome of a post-deploy monitor session. +#[derive(Debug)] +pub(crate) enum MonitorOutcome { + /// halt-on-success pattern matched + Success(String), + /// halt-on-error pattern matched + Error(String), + /// Timeout reached + Timeout { expect_found: bool }, +} + +pub(crate) struct MonitorState { + halt_error_re: Option, + halt_success_re: Option, + expect_re: Option, + start: std::time::Instant, + timeout_dur: Option, + expect_found: bool, + show_timestamp: bool, +} + +impl MonitorState { + pub(crate) fn new( + timeout_secs: Option, + halt_on_error: Option<&str>, + halt_on_success: Option<&str>, + expect: Option<&str>, + show_timestamp: bool, + ) -> Self { + let halt_error_re = halt_on_error.and_then(|p| { + regex::RegexBuilder::new(p) + .case_insensitive(true) + .build() + .ok() + }); + let halt_success_re = halt_on_success.and_then(|p| { + regex::RegexBuilder::new(p) + .case_insensitive(true) + .build() + .ok() + }); + let expect_re = expect.and_then(|p| { + regex::RegexBuilder::new(p) + .case_insensitive(true) + .build() + .ok() + }); + Self { + halt_error_re, + halt_success_re, + expect_re, + start: std::time::Instant::now(), + timeout_dur: timeout_secs.map(std::time::Duration::from_secs_f64), + expect_found: false, + show_timestamp, + } + } + + pub(crate) fn timed_out(&self) -> bool { + self.timeout_dur + .is_some_and(|dur| self.start.elapsed() >= dur) + } + + pub(crate) fn remaining(&self) -> Option { + self.timeout_dur + .map(|dur| dur.saturating_sub(self.start.elapsed())) + } + + pub(crate) fn timeout_outcome(&self) -> MonitorOutcome { + MonitorOutcome::Timeout { + expect_found: self.expect_found, + } + } + + pub(crate) fn expect_found(&self) -> bool { + self.expect_found + } + + pub(crate) fn process_line(&mut self, line: &str) -> Option { + if self.show_timestamp { + let total_secs = self.start.elapsed().as_secs_f64(); + let minutes = (total_secs / 60.0) as u64; + let seconds = total_secs % 60.0; + tracing::info!("{:02}:{:05.2} {}", minutes, seconds, line); + } else { + tracing::info!("{}", line); + } + + if let Some(ref re) = self.expect_re { + if re.is_match(line) { + self.expect_found = true; + } + } + + if let Some(ref re) = self.halt_error_re { + if re.is_match(line) { + return Some(MonitorOutcome::Error(format!( + "halt-on-error pattern matched: {}", + line + ))); + } + } + + if let Some(ref re) = self.halt_success_re { + if re.is_match(line) { + return Some(MonitorOutcome::Success(format!( + "halt-on-success pattern matched: {}", + line + ))); + } + } + + None + } +} + +/// Run a monitor loop reading lines from broadcast, checking halt conditions +/// using case-insensitive regex (matching Python's re.search behavior). +pub(crate) async fn run_monitor_loop( + rx: &mut tokio::sync::broadcast::Receiver, + timeout_secs: Option, + halt_on_error: Option<&str>, + halt_on_success: Option<&str>, + expect: Option<&str>, + show_timestamp: bool, +) -> MonitorOutcome { + let mut state = MonitorState::new( + timeout_secs, + halt_on_error, + halt_on_success, + expect, + show_timestamp, + ); + loop { + if state.timed_out() { + return state.timeout_outcome(); + } + + let recv_timeout = state + .remaining() + .unwrap_or(std::time::Duration::from_secs(1)); + + match tokio::time::timeout(recv_timeout, rx.recv()).await { + Ok(Ok(line)) => { + if let Some(outcome) = state.process_line(&line) { + return outcome; + } + } + Ok(Err(tokio::sync::broadcast::error::RecvError::Lagged(n))) => { + tracing::warn!("monitor lagged, skipped {} messages", n); + } + Ok(Err(tokio::sync::broadcast::error::RecvError::Closed)) => { + return state.timeout_outcome(); + } + Err(_) => { + // Timeout on recv — check if overall timeout expired + if state.timed_out() { + return state.timeout_outcome(); + } + // No overall timeout: just keep waiting + } + } + } +} + +/// POST /api/monitor +pub async fn monitor( + State(ctx): State>, + Json(req): Json, +) -> (StatusCode, Json) { + let request_id = req + .request_id + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let port = req.port.unwrap_or_else(|| "/dev/ttyUSB0".to_string()); + let baud_rate = req.baud_rate.unwrap_or(115200); + + if let Err(e) = ctx + .serial_manager + .open_port(&port, baud_rate, &request_id) + .await + { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("failed to open port: {}", e), + )), + ); + } + + // If halt conditions or timeout are set, run a monitor loop + let has_conditions = req.halt_on_error.is_some() + || req.halt_on_success.is_some() + || req.expect.is_some() + || req.timeout.is_some(); + + if has_conditions { + let mut rx = match ctx.serial_manager.attach_reader(&port, &request_id) { + Some(rx) => rx, + None => { + return ( + StatusCode::OK, + Json(OperationResponse::ok( + request_id, + format!( + "monitoring {} at {} baud (no broadcast channel)", + port, baud_rate + ), + )), + ); + } + }; + + let result = run_monitor_loop( + &mut rx, + req.timeout, + req.halt_on_error.as_deref(), + req.halt_on_success.as_deref(), + req.expect.as_deref(), + req.show_timestamp, + ) + .await; + + ctx.serial_manager.detach_reader(&port, &request_id); + + return match result { + MonitorOutcome::Success(msg) => { + (StatusCode::OK, Json(OperationResponse::ok(request_id, msg))) + } + MonitorOutcome::Error(msg) => ( + StatusCode::OK, + Json(OperationResponse::fail(request_id, msg)), + ), + MonitorOutcome::Timeout { expect_found } => { + if req.expect.is_some() && !expect_found { + ( + StatusCode::OK, + Json(OperationResponse::fail( + request_id, + "monitor timed out (expected pattern not found)".to_string(), + )), + ) + } else { + ( + StatusCode::OK, + Json(OperationResponse::ok( + request_id, + "monitor completed (timeout)".to_string(), + )), + ) + } + } + }; + } + + ( + StatusCode::OK, + Json(OperationResponse::ok( + request_id, + format!("monitoring {} at {} baud", port, baud_rate), + )), + ) +} diff --git a/crates/fbuild-daemon/src/handlers/operations/reset.rs b/crates/fbuild-daemon/src/handlers/operations/reset.rs new file mode 100644 index 00000000..76226539 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/operations/reset.rs @@ -0,0 +1,82 @@ +//! `POST /api/reset` — toggle DTR/RTS to reset a board. + +use super::common::OperationGuard; +use crate::context::DaemonContext; +use crate::models::{OperationResponse, ResetRequest}; +use axum::extract::State; +use axum::http::StatusCode; +use axum::Json; +use std::sync::Arc; + +/// POST /api/reset +/// +/// Reset a device via serial port DTR/RTS toggling. +/// Uses platform-specific reset sequence based on board identifier. +pub async fn reset( + State(ctx): State>, + Json(req): Json, +) -> (StatusCode, Json) { + let request_id = req + .request_id + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let port = req.port.clone(); + let verbose = req.verbose; + + let platform = req + .board + .as_deref() + .map(fbuild_deploy::reset::detect_platform_for_reset) + .unwrap_or("generic"); + + let _op_guard = OperationGuard::new( + &ctx, + fbuild_core::DaemonState::Deploying, + Some(format!("Resetting device on {}", port)), + ); + + // Preempt serial if someone is monitoring this port + let _ = ctx + .serial_manager + .preempt_for_deploy(&port, "reset".to_string(), request_id.clone()) + .await; + + let platform_str = platform.to_string(); + let result = tokio::task::spawn_blocking(move || { + fbuild_deploy::reset::reset_device(&platform_str, &port, verbose) + }) + .await; + + // Clear preemption + ctx.serial_manager.clear_preemption(&req.port).await; + + match result { + Ok(Ok(true)) => ( + StatusCode::OK, + Json(OperationResponse::ok( + request_id, + format!("device reset successful on {}", req.port), + )), + ), + Ok(Ok(false)) => ( + StatusCode::OK, + Json(OperationResponse::fail( + request_id, + format!("device reset reported failure on {}", req.port), + )), + ), + Ok(Err(e)) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("reset error: {}", e), + )), + ), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(OperationResponse::fail( + request_id, + format!("reset task panicked: {}", e), + )), + ), + } +} diff --git a/crates/fbuild-daemon/src/handlers/operations/tests.rs b/crates/fbuild-daemon/src/handlers/operations/tests.rs new file mode 100644 index 00000000..9c23c407 --- /dev/null +++ b/crates/fbuild-daemon/src/handlers/operations/tests.rs @@ -0,0 +1,196 @@ +//! Unit tests for the operations handlers split. +//! +//! All tests previously inlined at the bottom of `operations.rs` now +//! live here. They reach into sibling submodules via `super::*`. + +mod deploy_message_tests { + //! Verifies the `/api/deploy` response message exposes the real + //! deploy outcome (full / verify-skip / selective) instead of the + //! generic `"deploy succeeded"`. See GitHub issue #76. + //! + //! These tests cover only the pure string-formatting contract; the + //! underlying outcome computation is tested in `fbuild-deploy`. + use fbuild_deploy::{esp32::FlashRegion, DeployOutcome}; + + fn prefix_for(outcome: &DeployOutcome) -> String { + format!("deploy succeeded ({})", outcome.describe()) + } + + #[test] + fn full_flash_prefix() { + assert_eq!( + prefix_for(&DeployOutcome::FullFlash), + "deploy succeeded (full flash)" + ); + } + + #[test] + fn verify_skip_prefix() { + assert_eq!( + prefix_for(&DeployOutcome::VerifySkip), + "deploy succeeded (verify skipped, device already matched)" + ); + } + + #[test] + fn selective_flash_firmware_prefix() { + let outcome = DeployOutcome::SelectiveFlash { + regions: vec![FlashRegion::Firmware], + }; + assert_eq!( + prefix_for(&outcome), + "deploy succeeded (selective flash: firmware)" + ); + } + + #[test] + fn monitor_suffix_preserved_on_selective_flash() { + let outcome = DeployOutcome::SelectiveFlash { + regions: vec![FlashRegion::Firmware], + }; + let prefix = prefix_for(&outcome); + let combined = format!("{}; monitor: ok", prefix); + assert_eq!( + combined, + "deploy succeeded (selective flash: firmware); monitor: ok" + ); + } + + #[test] + fn monitor_error_suffix_preserved_on_verify_skip() { + let prefix = prefix_for(&DeployOutcome::VerifySkip); + let combined = format!("{}; monitor error: pattern matched", prefix); + assert_eq!( + combined, + "deploy succeeded (verify skipped, device already matched); monitor error: pattern matched" + ); + } +} + +#[cfg(feature = "espflash-native")] +mod espflash_env_tests { + use super::super::common::{native_verify_enabled, native_write_enabled}; + + // This lock only serializes these unit tests while they mutate the + // process environment. Production callers and any other tests reading + // the same env vars remain unsynchronized. + static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + #[test] + fn native_verify_defaults_on_and_allows_opt_out() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + std::env::remove_var("FBUILD_USE_ESPFLASH_VERIFY"); + assert!(native_verify_enabled()); + + std::env::set_var("FBUILD_USE_ESPFLASH_VERIFY", "0"); + assert!(!native_verify_enabled()); + + std::env::set_var("FBUILD_USE_ESPFLASH_VERIFY", "false"); + assert!(!native_verify_enabled()); + + std::env::set_var("FBUILD_USE_ESPFLASH_VERIFY", "1"); + assert!(native_verify_enabled()); + std::env::remove_var("FBUILD_USE_ESPFLASH_VERIFY"); + } + + #[test] + fn native_write_defaults_on_and_allows_opt_out() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + std::env::remove_var("FBUILD_USE_ESPFLASH_WRITE"); + assert!(native_write_enabled()); + + std::env::set_var("FBUILD_USE_ESPFLASH_WRITE", "off"); + assert!(!native_write_enabled()); + + std::env::set_var("FBUILD_USE_ESPFLASH_WRITE", "yes"); + assert!(native_write_enabled()); + std::env::remove_var("FBUILD_USE_ESPFLASH_WRITE"); + } +} + +mod image_hash_memo_tests { + //! Memo-cache correctness for [`compute_esp32_image_hash`]: the + //! memo must *reuse* the stored hash when none of the three + //! region files have changed, and *re-hash* when any of them + //! changes on disk. + use super::super::common::compute_esp32_image_hash; + use crate::context::DaemonContext; + use std::io::Write; + use std::path::Path; + + fn write(path: &Path, bytes: &[u8]) { + let mut f = std::fs::File::create(path).unwrap(); + f.write_all(bytes).unwrap(); + } + + fn fresh_ctx() -> std::sync::Arc { + let (tx, _rx) = tokio::sync::watch::channel(false); + std::sync::Arc::new(DaemonContext::new(8765, tx, "unknown".to_string())) + } + + fn seed_image(dir: &Path) { + write(&dir.join("bootloader.bin"), b"BOOT0"); + write(&dir.join("partitions.bin"), b"PART00"); + write(&dir.join("firmware.bin"), b"FW__FIRST_BUILD"); + } + + /// Second call with unchanged files must hit the memo (same + /// result, no work). We verify the memo side by directly + /// inspecting `ctx.image_hash_memo`. + #[test] + fn memo_hit_reuses_hash() { + let tmp = tempfile::tempdir().unwrap(); + seed_image(tmp.path()); + let ctx = fresh_ctx(); + let fw = tmp.path().join("firmware.bin"); + + let h1 = compute_esp32_image_hash(&ctx, &fw, 0x0, 0x8000, 0x10000).unwrap(); + assert_eq!(ctx.image_hash_memo.len(), 1); + let h2 = compute_esp32_image_hash(&ctx, &fw, 0x0, 0x8000, 0x10000).unwrap(); + assert_eq!(h1, h2); + assert_eq!(ctx.image_hash_memo.len(), 1, "memo must not grow on a hit"); + } + + /// When any of the three files changes on disk, the memo + /// invalidates via `mtime` change and the hash recomputes. + /// We assert the hash *differs* because the file contents + /// changed — so this catches both the bytes going through the + /// hasher AND the invalidation path. + #[test] + fn memo_miss_on_firmware_mtime_change() { + let tmp = tempfile::tempdir().unwrap(); + seed_image(tmp.path()); + let ctx = fresh_ctx(); + let fw = tmp.path().join("firmware.bin"); + + let h1 = compute_esp32_image_hash(&ctx, &fw, 0x0, 0x8000, 0x10000).unwrap(); + // Rewrite firmware.bin with new content; `std::fs::File::create` + // bumps the `mtime` on Windows with enough resolution (100 ns) + // even for sub-millisecond follow-ups. + std::thread::sleep(std::time::Duration::from_millis(20)); + write(&fw, b"FW__SECOND_BUILD_DIFFERENT"); + + let h2 = compute_esp32_image_hash(&ctx, &fw, 0x0, 0x8000, 0x10000).unwrap(); + assert_ne!(h1, h2, "content-changed image must hash to a new value"); + } + + /// Missing files on disk short-circuit to `None` — the caller + /// falls through to the regular verify-flash path instead of + /// trust-skipping with a stale hash. The memo must NOT store an + /// entry for an input that couldn't be hashed. + #[test] + fn memo_skipped_when_inputs_missing() { + let tmp = tempfile::tempdir().unwrap(); + // Only create firmware.bin — bootloader/partitions absent. + write(&tmp.path().join("firmware.bin"), b"FW"); + let ctx = fresh_ctx(); + let fw = tmp.path().join("firmware.bin"); + + assert!(compute_esp32_image_hash(&ctx, &fw, 0x0, 0x8000, 0x10000).is_none()); + assert_eq!( + ctx.image_hash_memo.len(), + 0, + "memo must not record entries for inputs that fail to hash" + ); + } +} diff --git a/crates/fbuild-deploy/src/esp32.rs b/crates/fbuild-deploy/src/esp32.rs deleted file mode 100644 index ba104840..00000000 --- a/crates/fbuild-deploy/src/esp32.rs +++ /dev/null @@ -1,2045 +0,0 @@ -//! ESP32 deployer using esptool.py. -//! -//! Flashes firmware to ESP32 boards via serial port using esptool. -//! Bootloader offset varies by MCU: -//! - `0x1000`: esp32, esp32s2 -//! - `0x0`: esp32c2, esp32c3, esp32c5, esp32c6, esp32h2, esp32s3 -//! - `0x2000`: esp32p4 - -use std::io::{Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; - -use fbuild_core::subprocess::run_command; -use fbuild_core::Result; -use object::{Object, ObjectSymbol}; -use sha2::{Digest, Sha256}; - -use crate::{DeployOutcome, Deployer, DeploymentResult}; - -/// Esptool flash parameters sourced from MCU config JSON. -/// -/// All fields correspond to `esptool` section fields in the MCU config. -pub struct EsptoolParams { - pub flash_mode: String, - pub flash_freq: String, - pub default_baud: String, - pub before_reset: String, - pub after_reset: String, -} - -/// ESP32 deployer using `esptool`. -pub struct Esp32Deployer { - /// MCU chip type for esptool --chip flag (e.g. "esp32c6"). - chip: String, - /// Baud rate for flashing (e.g. "460800"). - baud_rate: String, - /// Flash offsets. - bootloader_offset: String, - partitions_offset: String, - firmware_offset: String, - /// Flash mode for esptool (e.g. "dio", "qio"). - flash_mode: String, - /// Flash frequency for esptool (e.g. "80m", "40m"). - flash_freq: String, - /// Reset mode before flashing. - before_reset: String, - /// Reset mode after flashing. - after_reset: String, - verbose: bool, - /// Route `verify-flash` through the native [`espflash`] crate - /// instead of the Python `esptool` subprocess. - /// - /// The daemon sets this from the `FBUILD_USE_ESPFLASH_VERIFY` env - /// var. When enabled, the deployer still falls back to esptool if - /// the native path fails on a given board or host. - /// - /// Only compiled in when the `espflash-native` cargo feature is - /// enabled. - #[cfg(feature = "espflash-native")] - use_native_verify: bool, - /// Route `write-flash` through the native [`espflash`] crate - /// instead of the Python `esptool` subprocess (issue #66). - /// - /// The daemon sets this from the `FBUILD_USE_ESPFLASH_WRITE` env - /// var, independently of `use_native_verify`. When enabled, the - /// deployer still falls back to esptool if the native path fails. - /// - /// Feature-gated — see `use_native_verify`. - #[cfg(feature = "espflash-native")] - use_native_write: bool, -} - -#[cfg(feature = "espflash-native")] -fn native_write_or_fallback(port: &str, label: &str, native: F) -> Option -where - F: FnOnce() -> Result, -{ - match native() { - Ok(result) if result.success => Some(result), - Ok(result) => { - tracing::warn!( - port, - "native {} failed ({}); falling back to esptool", - label, - result.message - ); - None - } - Err(e) => { - tracing::warn!( - port, - "native {} failed ({}); falling back to esptool", - label, - e - ); - None - } - } -} - -impl Esp32Deployer { - #[allow(clippy::too_many_arguments)] - pub fn new( - chip: &str, - baud_rate: &str, - bootloader_offset: &str, - partitions_offset: &str, - firmware_offset: &str, - esptool_params: &EsptoolParams, - verbose: bool, - ) -> Self { - Self { - chip: chip.to_string(), - baud_rate: baud_rate.to_string(), - bootloader_offset: bootloader_offset.to_string(), - partitions_offset: partitions_offset.to_string(), - firmware_offset: firmware_offset.to_string(), - flash_mode: esptool_params.flash_mode.clone(), - flash_freq: esptool_params.flash_freq.clone(), - before_reset: esptool_params.before_reset.clone(), - after_reset: esptool_params.after_reset.clone(), - verbose, - #[cfg(feature = "espflash-native")] - use_native_verify: false, - #[cfg(feature = "espflash-native")] - use_native_write: false, - } - } - - /// Opt this deployer into the native espflash-based verify path - /// (issue #66). Independent of `with_native_write`. - /// - /// Only present when the `espflash-native` cargo feature is enabled; - /// without it the esptool-subprocess path is the only code path. - #[cfg(feature = "espflash-native")] - #[must_use] - pub fn with_native_verify(mut self, enabled: bool) -> Self { - self.use_native_verify = enabled; - self - } - - /// Opt this deployer into the native espflash-based write-flash - /// path (issue #66). Independent of `with_native_verify`. When - /// enabled, both [`Deployer::deploy`] and - /// [`Esp32Deployer::deploy_regions`] route through the in-process - /// espflash `Flasher`, skipping the ~1.5 s Python/esptool startup - /// per flash unless they need to fall back. - /// - /// Only present when the `espflash-native` cargo feature is enabled. - #[cfg(feature = "espflash-native")] - #[must_use] - pub fn with_native_write(mut self, enabled: bool) -> Self { - self.use_native_write = enabled; - self - } - - /// Create an ESP32 deployer from board config with explicit flash offsets. - pub fn from_board_config( - board: &fbuild_config::BoardConfig, - bootloader_offset: &str, - partitions_offset: &str, - firmware_offset: &str, - esptool_params: &EsptoolParams, - verbose: bool, - ) -> Self { - let baud = board - .upload_speed - .as_deref() - .unwrap_or(&esptool_params.default_baud); - // Board-level flash_mode overrides MCU default. - let flash_mode = board - .flash_mode - .as_deref() - .unwrap_or(&esptool_params.flash_mode); - let params = EsptoolParams { - flash_mode: flash_mode.to_string(), - flash_freq: esptool_params.flash_freq.clone(), - default_baud: esptool_params.default_baud.clone(), - before_reset: esptool_params.before_reset.clone(), - after_reset: esptool_params.after_reset.clone(), - }; - Self::new( - &board.mcu, - baud, - bootloader_offset, - partitions_offset, - firmware_offset, - ¶ms, - verbose, - ) - } - - /// Override the baud rate (e.g. from a CLI `--baud` flag). - pub fn with_baud_rate(mut self, baud: &str) -> Self { - self.baud_rate = baud.to_string(); - self - } - - /// Find the esptool executable. - /// - /// Uses standalone `esptool` command (available when esptool is pip-installed). - fn find_esptool() -> Vec { - vec!["esptool".to_string()] - } - - /// Build the `esptool verify-flash` command line that this deployer - /// would run to verify a candidate firmware image is already on the - /// device. Pure (no I/O) so we can unit-test the argument layout - /// without touching real hardware. - /// - /// `verify-flash` uses the device-side `FLASH_MD5SUM` command (issued - /// by the stub flasher), so it does NOT read the entire flash region - /// back over UART — verification of a 2.4 MB ESP32-S3 image takes - /// ~6 seconds end-to-end vs ~25 seconds for a full re-flash. - /// See ISSUES.md "Fast deploy via verify-then-skip". - pub fn build_verify_flash_args(&self, firmware_path: &Path, port: &str) -> Vec { - let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); - let bootloader_path = build_dir.join("bootloader.bin"); - let partitions_path = build_dir.join("partitions.bin"); - - let mut args = Self::find_esptool(); - args.extend([ - "--chip".to_string(), - self.chip.clone(), - "--port".to_string(), - port.to_string(), - "--baud".to_string(), - self.baud_rate.clone(), - "--before".to_string(), - self.before_reset.clone(), - "--after".to_string(), - self.after_reset.clone(), - "verify-flash".to_string(), - ]); - - // Verify all three regions in a single esptool invocation so we - // pay the stub-flasher upload cost (~3s) once, not three times. - if bootloader_path.exists() { - args.push(self.bootloader_offset.clone()); - args.push(bootloader_path.to_string_lossy().to_string()); - } - if partitions_path.exists() { - args.push(self.partitions_offset.clone()); - args.push(partitions_path.to_string_lossy().to_string()); - } - args.push(self.firmware_offset.clone()); - args.push(firmware_path.to_string_lossy().to_string()); - args - } - - /// Run `esptool verify-flash` on bootloader + partitions + firmware - /// against the live device. Returns `Ok(true)` when every region's - /// FLASH_MD5SUM matches the local file (i.e. flashing would be a - /// no-op), `Ok(false)` when at least one region differs, and `Err` - /// only when esptool itself failed to run (port not found, stub - /// upload failed, etc.). - /// - /// On success the chip is hard-reset by esptool's `--after hard-reset`, - /// matching the post-flash behavior — so callers can treat a `true` - /// return as "device is now running the requested firmware" without - /// any extra reset. - pub fn try_verify_deployment(&self, firmware_path: &Path, port: &str) -> Result { - #[cfg(feature = "espflash-native")] - if self.use_native_verify { - match self.try_verify_deployment_native(firmware_path, port) { - Ok(outcome) => return Ok(outcome), - Err(e) => { - tracing::warn!( - port, - "native verify-flash failed ({}); falling back to esptool", - e - ); - } - } - } - - self.try_verify_deployment_esptool(firmware_path, port) - } - - fn try_verify_deployment_esptool( - &self, - firmware_path: &Path, - port: &str, - ) -> Result { - let args = self.build_verify_flash_args(firmware_path, port); - let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - - if self.verbose { - tracing::info!("verify: {}", args.join(" ")); - } - tracing::info!( - "verifying {} on {} via esptool ({})", - firmware_path.display(), - port, - self.chip - ); - - let result = run_command( - &args_ref, - None, - None, - // Verify is bounded: the slowest case is ~10s for a maximum - // image. 30s gives plenty of headroom for slow USB-CDC stacks - // and stub flasher upload retries. - Some(std::time::Duration::from_secs(30)), - )?; - - if result.success() { - Ok(VerifyOutcome::Match { - stdout: result.stdout, - stderr: result.stderr, - }) - } else { - // esptool exits non-zero on a digest mismatch *and* on real - // failures (port unreachable, stub upload error). Distinguish - // them by looking at stderr — a true digest mismatch always - // contains the literal "Verification failed" string. - let combined = format!("{}\n{}", result.stdout, result.stderr); - if combined.contains("Verification failed") || combined.contains("digest mismatch") { - let regions = parse_verify_regions( - &combined, - &self.bootloader_offset, - &self.partitions_offset, - &self.firmware_offset, - ); - Ok(VerifyOutcome::Mismatch { - stdout: result.stdout, - stderr: result.stderr, - regions, - }) - } else { - Err(fbuild_core::FbuildError::DeployFailed(format!( - "esptool verify-flash failed (exit {}): {}", - result.exit_code, result.stderr - ))) - } - } - } - - /// Native `verify-flash` via the [`espflash`] crate (issue #66). - /// - /// Saves the Python interpreter startup (~1 s) and subprocess spawn - /// (~0.5 s) that the esptool path pays per invocation. Same three - /// regions (bootloader / partitions / firmware) and same - /// [`VerifyOutcome`] semantics as the esptool path, so callers can - /// swap between the two behind the `use_native_verify` flag without - /// any result-handling changes. - #[cfg(feature = "espflash-native")] - fn try_verify_deployment_native( - &self, - firmware_path: &Path, - port: &str, - ) -> Result { - let baud: u32 = self.baud_rate.parse().map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!( - "native verify: invalid baud rate '{}': {}", - self.baud_rate, e - )) - })?; - let boot_off = parse_hex_offset_u32(&self.bootloader_offset)?; - let parts_off = parse_hex_offset_u32(&self.partitions_offset)?; - let fw_off = parse_hex_offset_u32(&self.firmware_offset)?; - - let regions = crate::esp32_native::collect_standard_regions( - firmware_path, - boot_off, - parts_off, - fw_off, - ); - - if self.verbose { - tracing::info!( - "native verify: chip={} port={} baud={} regions={}", - self.chip, - port, - baud, - regions.len() - ); - } - tracing::info!( - "verifying {} on {} via espflash ({})", - firmware_path.display(), - port, - self.chip - ); - - crate::esp32_native::try_verify_deployment_native( - &self.chip, - port, - baud, - &self.before_reset, - &self.after_reset, - ®ions, - boot_off, - parts_off, - fw_off, - ) - } - - /// Native `write-flash` via the [`espflash`] crate (issue #66). - /// - /// Writes the full three-region set (bootloader + partitions + - /// firmware where present). Saves the Python interpreter startup - /// (~1 s) plus subprocess spawn (~0.5 s) vs the esptool path, and - /// surfaces per-region progress via `tracing` (bridged into the - /// daemon's existing log broadcaster). Same [`DeploymentResult`] - /// shape as the esptool path so callers swap behind a single flag. - #[cfg(feature = "espflash-native")] - fn try_deploy_native(&self, firmware_path: &Path, port: &str) -> Result { - let baud = self.parse_native_baud()?; - let (boot_off, parts_off, fw_off) = self.parse_native_offsets()?; - - let regions = crate::esp32_native::collect_standard_write_regions( - firmware_path, - boot_off, - parts_off, - fw_off, - ); - - if self.verbose { - tracing::info!( - "native write: chip={} port={} baud={} regions={}", - self.chip, - port, - baud, - regions.len() - ); - } - tracing::info!( - "flashing {} to {} via espflash ({})", - firmware_path.display(), - port, - self.chip - ); - - crate::esp32_native::try_write_deployment_native( - &self.chip, - port, - baud, - &self.before_reset, - &self.after_reset, - ®ions, - /* selective */ false, - ) - } - - /// Native `write-flash` for a caller-chosen subset of regions - /// (issue #66). Used after a verify-mismatch to rewrite only the - /// regions that actually differ — skipping the ~1s - /// bootloader/partitions rewrite when only firmware changed. - #[cfg(feature = "espflash-native")] - fn try_deploy_regions_native( - &self, - firmware_path: &Path, - port: &str, - regions: &[FlashRegion], - ) -> Result { - let baud = self.parse_native_baud()?; - let (boot_off, parts_off, fw_off) = self.parse_native_offsets()?; - - let write_regions = crate::esp32_native::collect_selected_write_regions( - firmware_path, - boot_off, - parts_off, - fw_off, - regions, - )?; - - if self.verbose { - tracing::info!( - "native write (selective): chip={} port={} baud={} regions={:?}", - self.chip, - port, - baud, - regions - ); - } - tracing::info!( - "flashing regions {:?} of {} to {} via espflash ({})", - regions, - firmware_path.display(), - port, - self.chip - ); - - crate::esp32_native::try_write_deployment_native( - &self.chip, - port, - baud, - &self.before_reset, - &self.after_reset, - &write_regions, - /* selective */ true, - ) - } - - #[cfg(feature = "espflash-native")] - fn parse_native_baud(&self) -> Result { - self.baud_rate.parse().map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!( - "native write: invalid baud rate '{}': {}", - self.baud_rate, e - )) - }) - } - - #[cfg(feature = "espflash-native")] - fn parse_native_offsets(&self) -> Result<(u32, u32, u32)> { - let boot = parse_hex_offset_u32(&self.bootloader_offset)?; - let parts = parse_hex_offset_u32(&self.partitions_offset)?; - let fw = parse_hex_offset_u32(&self.firmware_offset)?; - Ok((boot, parts, fw)) - } -} - -/// Parse a hex flash offset (accepts `0x` prefix) as a `u32`. espflash's -/// `FLASH_MD5SUM` command takes 32-bit offsets, so we narrow the shared -/// [`parse_hex_offset`] result here. Only used by the native verify/write -/// path, so gated with the crate's feature to avoid dead-code lints on -/// default builds. -#[cfg(feature = "espflash-native")] -fn parse_hex_offset_u32(raw: &str) -> Result { - let as_u64 = parse_hex_offset(raw)?; - u32::try_from(as_u64).map_err(|_| { - fbuild_core::FbuildError::DeployFailed(format!( - "native verify: flash offset {} does not fit in u32", - raw - )) - }) -} - -/// Which of the three logical flash regions a verify/write targets. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FlashRegion { - Bootloader, - Partitions, - Firmware, -} - -/// Per-region outcome parsed from `esptool verify-flash` stdout. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct RegionVerifyResult { - pub region: FlashRegion, - pub matched: bool, -} - -/// Result of a `try_verify_deployment` call. -#[derive(Debug, Clone)] -pub enum VerifyOutcome { - /// All flashed regions match the candidate image; flashing would be - /// a no-op. The device has been hard-reset by esptool's - /// `--after hard-reset` so it's already running the requested image. - Match { stdout: String, stderr: String }, - /// At least one region differs from the local files. `regions` carries - /// the parsed per-region verdict when stdout was understood; empty - /// when parsing failed and the caller must flash everything. - Mismatch { - stdout: String, - stderr: String, - regions: Vec, - }, -} - -impl VerifyOutcome { - /// Convenience: returns `true` only for `Match`. - pub fn is_match(&self) -> bool { - matches!(self, VerifyOutcome::Match { .. }) - } -} - -/// Parse `esptool verify-flash` stdout into per-region results. -/// -/// esptool 5.x emits, for each region: -/// -/// ```text -/// Verifying 0x6060 (24672) bytes at 0x00000000 in flash against 'bootloader.bin'... -/// Verification successful (digest matched). -/// ``` -/// -/// or, on mismatch, `Verification failed (digest mismatch).`. We map the -/// `at 0x{addr:#010x}` line to one of the three known offsets, then read -/// the next non-blank line as the verdict. Unknown addresses are skipped. -/// Returns an empty `Vec` when nothing could be parsed — callers must then -/// fall back to flashing all regions. -pub fn parse_verify_regions( - stdout: &str, - bootloader_offset: &str, - partitions_offset: &str, - firmware_offset: &str, -) -> Vec { - let boot_addr = parse_hex_offset(bootloader_offset).ok(); - let parts_addr = parse_hex_offset(partitions_offset).ok(); - let fw_addr = parse_hex_offset(firmware_offset).ok(); - - let mut results: Vec = Vec::new(); - let mut pending: Option = None; - - for raw in stdout.lines() { - let line = raw.trim(); - if let Some(addr) = extract_verifying_at_address(line) { - pending = if Some(addr) == boot_addr { - Some(FlashRegion::Bootloader) - } else if Some(addr) == parts_addr { - Some(FlashRegion::Partitions) - } else if Some(addr) == fw_addr { - Some(FlashRegion::Firmware) - } else { - None - }; - continue; - } - if let Some(region) = pending { - if line.starts_with("Verification successful") { - if !results.iter().any(|r| r.region == region) { - results.push(RegionVerifyResult { - region, - matched: true, - }); - } - pending = None; - } else if line.starts_with("Verification failed") { - if !results.iter().any(|r| r.region == region) { - results.push(RegionVerifyResult { - region, - matched: false, - }); - } - pending = None; - } - } - } - results -} - -/// Extract the `0x...` address from an esptool `Verifying ... at 0xNNNNNNNN in flash ...` line. -fn extract_verifying_at_address(line: &str) -> Option { - if !line.starts_with("Verifying ") { - return None; - } - let marker = " at 0x"; - let start = line.find(marker)? + marker.len(); - let tail = &line[start..]; - let end = tail - .find(|c: char| !c.is_ascii_hexdigit()) - .unwrap_or(tail.len()); - u64::from_str_radix(&tail[..end], 16).ok() -} - -/// Resolve the flash-image size to use for QEMU. -/// -/// ESP32 QEMU supports 2MB, 4MB, 8MB, and 16MB flash images. We derive -/// the size from the board's `maximum_size` when present, otherwise fall back -/// to the MCU config's default flash size label. -pub fn resolve_qemu_flash_size_bytes( - board: &fbuild_config::BoardConfig, - default_flash_size: &str, -) -> Result { - let size_bytes = match board.max_flash { - Some(bytes) => bytes, - None => parse_flash_size_bytes(default_flash_size)?, - }; - if matches!(size_bytes, 2_097_152 | 4_194_304 | 8_388_608 | 16_777_216) { - Ok(size_bytes) - } else { - Err(fbuild_core::FbuildError::DeployFailed(format!( - "ESP32 QEMU supports only 2MB, 4MB, 8MB, or 16MB flash images; got {} bytes", - size_bytes - ))) - } -} - -/// Create a merged raw flash image for ESP32 QEMU from bootloader, -/// partitions, and application firmware. -/// -/// When `elf_path` is `Some`, the ESP32-S3 ADC calibration patch is applied. -/// Pass `None` for non-S3 variants to skip the patch. -pub fn create_qemu_flash_image( - firmware_path: &Path, - output_path: &Path, - flash_size_bytes: u64, - bootloader_offset: &str, - partitions_offset: &str, - firmware_offset: &str, - elf_path: Option<&Path>, -) -> Result { - let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); - let bootloader_path = build_dir.join("bootloader.bin"); - let boot_app0_path = build_dir.join("boot_app0.bin"); - let partitions_path = build_dir.join("partitions.bin"); - let firmware_offset = parse_hex_offset(firmware_offset)?; - - for required in [&bootloader_path, &partitions_path, firmware_path] { - if !required.is_file() { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "required QEMU artifact not found: {}", - required.display() - ))); - } - } - - if let Some(parent) = output_path.parent() { - std::fs::create_dir_all(parent)?; - } - - let mut output = std::fs::File::create(output_path)?; - fill_with_ff(&mut output, flash_size_bytes)?; - - write_binary_at_offset( - &mut output, - &bootloader_path, - parse_hex_offset(bootloader_offset)?, - flash_size_bytes, - )?; - write_binary_at_offset( - &mut output, - &partitions_path, - parse_hex_offset(partitions_offset)?, - flash_size_bytes, - )?; - if boot_app0_path.is_file() { - write_binary_at_offset(&mut output, &boot_app0_path, 0xE000, flash_size_bytes)?; - } - write_binary_at_offset( - &mut output, - firmware_path, - firmware_offset, - flash_size_bytes, - )?; - if let Some(elf_path) = elf_path { - patch_qemu_esp32s3_adc_calibration(output_path, firmware_path, elf_path, firmware_offset)?; - } - - Ok(output_path.to_path_buf()) -} - -/// Build the QEMU argv for ESP32-family emulation. -/// -/// The `mcu` parameter selects the QEMU machine type and watchdog timer -/// driver name. Supported values: `esp32`, `esp32s3` (via -/// `qemu-system-xtensa`) and `esp32c3`, `esp32c6`, `esp32h2` (via -/// `qemu-system-riscv32`). Callers are responsible for launching the -/// matching QEMU binary; this function only emits the argv. -pub fn build_qemu_args( - mcu: &str, - flash_image: &Path, - psram: Option, -) -> Vec { - let machine = mcu.to_lowercase(); - let mut args = vec![ - "-nographic".to_string(), - "-machine".to_string(), - machine.clone(), - ]; - if let Some(psram) = psram { - args.push("-m".to_string()); - args.push(format!("{}M", psram.size_mib)); - } - args.extend([ - "-drive".to_string(), - format!("file={},if=mtd,format=raw", flash_image.display()), - "-serial".to_string(), - "mon:stdio".to_string(), - "-monitor".to_string(), - "none".to_string(), - "-global".to_string(), - format!( - "driver=timer.{}.timg,property=wdt_disable,value=true", - machine - ), - ]); - if let Some(psram) = psram { - if psram.is_octal { - args.push("-global".to_string()); - args.push("driver=ssi_psram,property=is_octal,value=true".to_string()); - } - } - args -} - -/// Build the QEMU argv for ESP32-S3 emulation (convenience wrapper). -pub fn build_qemu_esp32s3_args( - flash_image: &Path, - psram: Option, -) -> Vec { - build_qemu_args("esp32s3", flash_image, psram) -} - -fn parse_hex_offset(raw: &str) -> Result { - let trimmed = raw.trim_start_matches("0x").trim_start_matches("0X"); - u64::from_str_radix(trimmed, 16).map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!("invalid flash offset '{}': {}", raw, e)) - }) -} - -fn parse_flash_size_bytes(raw: &str) -> Result { - let upper = raw.trim().to_ascii_uppercase(); - if let Some(num) = upper.strip_suffix("MB") { - return num - .trim() - .parse::() - .map(|n| n * 1024 * 1024) - .map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!( - "invalid flash size '{}': {}", - raw, e - )) - }); - } - if let Some(num) = upper.strip_suffix("KB") { - return num.trim().parse::().map(|n| n * 1024).map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!("invalid flash size '{}': {}", raw, e)) - }); - } - Err(fbuild_core::FbuildError::DeployFailed(format!( - "unsupported flash size label '{}'", - raw - ))) -} - -const ESP_IMAGE_HEADER_LEN: usize = 24; -const ESP_IMAGE_SEGMENT_HEADER_LEN: usize = 8; -const ESP_IMAGE_HEADER_MAGIC: u8 = 0xE9; -const ESP_ROM_CHECKSUM_INITIAL: u32 = 0xEF; -const ESP_IMAGE_APPENDED_HASH_LEN: usize = 32; -const QEMU_ADC_CALIBRATION_SYMBOL: &str = "adc_hw_calibration"; -const QEMU_ADC_CALIBRATION_PATCH_OFFSET: u32 = 3; -const QEMU_ADC_CALIBRATION_EXPECTED_BYTES: [u8; 2] = [0x0C, 0x0A]; -const QEMU_ADC_CALIBRATION_PATCH_BYTES: [u8; 2] = [0x1D, 0xF0]; - -fn patch_qemu_esp32s3_adc_calibration( - flash_image_path: &Path, - firmware_path: &Path, - elf_path: &Path, - firmware_offset: u64, -) -> Result<()> { - let symbol_addr = resolve_local_elf_symbol_address(elf_path, QEMU_ADC_CALIBRATION_SYMBOL)?; - let patch_addr = symbol_addr - .checked_add(QEMU_ADC_CALIBRATION_PATCH_OFFSET) - .ok_or_else(|| { - fbuild_core::FbuildError::DeployFailed(format!( - "QEMU workaround address overflow for symbol {}", - QEMU_ADC_CALIBRATION_SYMBOL - )) - })?; - let mut firmware_bytes = std::fs::read(firmware_path)?; - let firmware_file_offset = resolve_esp_image_file_offset(&firmware_bytes, patch_addr)?; - patch_bytes( - &mut firmware_bytes, - firmware_file_offset, - &QEMU_ADC_CALIBRATION_EXPECTED_BYTES, - &QEMU_ADC_CALIBRATION_PATCH_BYTES, - )?; - repair_esp_image_checksum_and_hash(&mut firmware_bytes)?; - - let mut flash_image = std::fs::OpenOptions::new() - .write(true) - .open(flash_image_path)?; - flash_image.seek(SeekFrom::Start(firmware_offset))?; - flash_image.write_all(&firmware_bytes)?; - tracing::info!("patched ESP32-S3 QEMU image to skip adc_hw_calibration at 0x{patch_addr:08x}"); - Ok(()) -} - -fn resolve_local_elf_symbol_address(elf_path: &Path, symbol_name: &str) -> Result { - let bytes = std::fs::read(elf_path)?; - let object = object::File::parse(bytes.as_slice()).map_err(|e| { - fbuild_core::FbuildError::DeployFailed(format!( - "failed to parse ELF {}: {}", - elf_path.display(), - e - )) - })?; - - let symbol = object - .symbols() - .find(|symbol| symbol.name().ok() == Some(symbol_name)) - .ok_or_else(|| { - fbuild_core::FbuildError::DeployFailed(format!( - "required ELF symbol '{}' not found in {}", - symbol_name, - elf_path.display() - )) - })?; - - u32::try_from(symbol.address()).map_err(|_| { - fbuild_core::FbuildError::DeployFailed(format!( - "ELF symbol '{}' address 0x{:x} does not fit in u32", - symbol_name, - symbol.address() - )) - }) -} - -fn resolve_esp_image_file_offset(firmware_bin: &[u8], load_addr: u32) -> Result { - if firmware_bin.len() < ESP_IMAGE_HEADER_LEN { - return Err(fbuild_core::FbuildError::DeployFailed( - "firmware.bin is too small to contain an ESP image header".to_string(), - )); - } - if firmware_bin[0] != ESP_IMAGE_HEADER_MAGIC { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "firmware.bin does not start with ESP image magic 0x{:02x}", - ESP_IMAGE_HEADER_MAGIC - ))); - } - - let segment_count = firmware_bin[1] as usize; - let mut cursor = ESP_IMAGE_HEADER_LEN; - for _ in 0..segment_count { - if cursor + ESP_IMAGE_SEGMENT_HEADER_LEN > firmware_bin.len() { - return Err(fbuild_core::FbuildError::DeployFailed( - "firmware.bin ended before segment header".to_string(), - )); - } - let seg_load_addr = - u32::from_le_bytes(firmware_bin[cursor..cursor + 4].try_into().unwrap()); - let seg_len = - u32::from_le_bytes(firmware_bin[cursor + 4..cursor + 8].try_into().unwrap()) as usize; - let data_start = cursor + ESP_IMAGE_SEGMENT_HEADER_LEN; - let data_end = data_start + seg_len; - if data_end > firmware_bin.len() { - return Err(fbuild_core::FbuildError::DeployFailed( - "firmware.bin ended before segment payload".to_string(), - )); - } - let seg_end_addr = seg_load_addr.checked_add(seg_len as u32).ok_or_else(|| { - fbuild_core::FbuildError::DeployFailed("ESP image segment address overflow".to_string()) - })?; - if (seg_load_addr..seg_end_addr).contains(&load_addr) { - return Ok(data_start + (load_addr - seg_load_addr) as usize); - } - cursor = data_end; - } - - Err(fbuild_core::FbuildError::DeployFailed(format!( - "firmware.bin does not contain a segment covering 0x{load_addr:08x}" - ))) -} - -fn patch_bytes(bytes: &mut [u8], offset: usize, expected: &[u8], replacement: &[u8]) -> Result<()> { - if expected.len() != replacement.len() { - return Err(fbuild_core::FbuildError::DeployFailed( - "patch replacement length mismatch".to_string(), - )); - } - - let end = offset.checked_add(expected.len()).ok_or_else(|| { - fbuild_core::FbuildError::DeployFailed("patch offset overflow".to_string()) - })?; - if end > bytes.len() { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "patch range 0x{:x}..0x{:x} exceeds image size {}", - offset, - end, - bytes.len() - ))); - } - let actual = &bytes[offset..end]; - if actual != expected { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "QEMU workaround expected bytes {:02x?} at 0x{:x}, found {:02x?}", - expected, offset, actual - ))); - } - bytes[offset..end].copy_from_slice(replacement); - Ok(()) -} - -fn repair_esp_image_checksum_and_hash(image: &mut [u8]) -> Result<()> { - if image.len() < ESP_IMAGE_HEADER_LEN { - return Err(fbuild_core::FbuildError::DeployFailed( - "firmware.bin is too small to repair".to_string(), - )); - } - if image[0] != ESP_IMAGE_HEADER_MAGIC { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "firmware.bin does not start with ESP image magic 0x{:02x}", - ESP_IMAGE_HEADER_MAGIC - ))); - } - - let segment_count = image[1] as usize; - let hash_appended = image[23] != 0; - let mut checksum_word = ESP_ROM_CHECKSUM_INITIAL; - let mut cursor = ESP_IMAGE_HEADER_LEN; - for _ in 0..segment_count { - if cursor + ESP_IMAGE_SEGMENT_HEADER_LEN > image.len() { - return Err(fbuild_core::FbuildError::DeployFailed( - "firmware.bin ended before segment header".to_string(), - )); - } - let seg_len = - u32::from_le_bytes(image[cursor + 4..cursor + 8].try_into().unwrap()) as usize; - let data_start = cursor + ESP_IMAGE_SEGMENT_HEADER_LEN; - let data_end = data_start + seg_len; - if data_end > image.len() { - return Err(fbuild_core::FbuildError::DeployFailed( - "firmware.bin ended before segment payload".to_string(), - )); - } - for chunk in image[data_start..data_end].chunks(4) { - let mut word = [0u8; 4]; - word[..chunk.len()].copy_from_slice(chunk); - checksum_word ^= u32::from_le_bytes(word); - } - cursor = data_end; - } - - let checksum_block_len = ((cursor + 1 + 15) & !15) - cursor; - let checksum_offset = cursor + checksum_block_len - 1; - if checksum_offset >= image.len() { - return Err(fbuild_core::FbuildError::DeployFailed( - "firmware.bin ended before checksum byte".to_string(), - )); - } - image[checksum_offset] = - ((checksum_word >> 24) ^ (checksum_word >> 16) ^ (checksum_word >> 8) ^ checksum_word) - as u8; - - if hash_appended { - let hash_start = checksum_offset + 1; - let hash_end = hash_start + ESP_IMAGE_APPENDED_HASH_LEN; - if hash_end > image.len() { - return Err(fbuild_core::FbuildError::DeployFailed( - "firmware.bin ended before appended hash".to_string(), - )); - } - let digest = Sha256::digest(&image[..hash_start]); - image[hash_start..hash_end].copy_from_slice(&digest); - } - Ok(()) -} - -fn fill_with_ff(file: &mut std::fs::File, total_size: u64) -> Result<()> { - file.seek(SeekFrom::Start(0))?; - let chunk = vec![0xFFu8; 64 * 1024]; - let mut remaining = total_size; - while remaining > 0 { - let to_write = std::cmp::min(remaining, chunk.len() as u64) as usize; - file.write_all(&chunk[..to_write])?; - remaining -= to_write as u64; - } - Ok(()) -} - -fn write_binary_at_offset( - output: &mut std::fs::File, - input_path: &Path, - offset: u64, - flash_size_bytes: u64, -) -> Result<()> { - let metadata = std::fs::metadata(input_path)?; - let end = offset.saturating_add(metadata.len()); - if end > flash_size_bytes { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "artifact {} at offset 0x{:x} exceeds flash image size {}", - input_path.display(), - offset, - flash_size_bytes - ))); - } - - output.seek(SeekFrom::Start(offset))?; - let mut input = std::fs::File::open(input_path)?; - let mut buffer = [0u8; 64 * 1024]; - loop { - let read = input.read(&mut buffer)?; - if read == 0 { - break; - } - output.write_all(&buffer[..read])?; - } - Ok(()) -} - -impl Esp32Deployer { - /// Build `esptool write-flash` argv for a caller-chosen subset of - /// regions. Pass `None` for `regions` to flash all three (bootloader + - /// partitions + firmware) when present on disk — the default deploy - /// path. Pass `Some(&[...])` to restrict the write to specific regions - /// (see `deploy_regions`). - pub fn build_write_flash_args( - &self, - firmware_path: &Path, - port: &str, - regions: Option<&[FlashRegion]>, - ) -> Vec { - let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); - let bootloader_path = build_dir.join("bootloader.bin"); - let partitions_path = build_dir.join("partitions.bin"); - - let mut args = Self::find_esptool(); - args.extend([ - "--chip".to_string(), - self.chip.clone(), - "--port".to_string(), - port.to_string(), - "--baud".to_string(), - self.baud_rate.clone(), - "--before".to_string(), - self.before_reset.clone(), - "--after".to_string(), - self.after_reset.clone(), - "write-flash".to_string(), - "-z".to_string(), - "--flash-mode".to_string(), - self.flash_mode.clone(), - "--flash-freq".to_string(), - self.flash_freq.clone(), - "--flash-size".to_string(), - "detect".to_string(), - ]); - - let include = |r: FlashRegion| regions.map_or(true, |rs| rs.contains(&r)); - - if include(FlashRegion::Bootloader) && bootloader_path.exists() { - args.push(self.bootloader_offset.clone()); - args.push(bootloader_path.to_string_lossy().to_string()); - } - if include(FlashRegion::Partitions) && partitions_path.exists() { - args.push(self.partitions_offset.clone()); - args.push(partitions_path.to_string_lossy().to_string()); - } - if include(FlashRegion::Firmware) { - args.push(self.firmware_offset.clone()); - args.push(firmware_path.to_string_lossy().to_string()); - } - args - } - - /// Flash only the specified regions. Use after `try_verify_deployment` - /// returns a `Mismatch` with `regions` populated — we skip the ~1s - /// bootloader/partitions rewrite when only firmware differs (the - /// overwhelmingly common case for iterative development). - /// - /// Returns an error when `regions` is empty; esptool rejects a - /// write-flash call with no offset/file pair and the message would be - /// opaque. - pub fn deploy_regions( - &self, - firmware_path: &Path, - port: &str, - regions: &[FlashRegion], - ) -> Result { - if regions.is_empty() { - return Err(fbuild_core::FbuildError::DeployFailed( - "deploy_regions called with no regions; pass at least one".to_string(), - )); - } - - // Fail loudly if the caller asked for a region whose file isn't - // on disk. Without this check `build_write_flash_args` would - // silently drop the request and esptool would complain with an - // opaque usage error. - let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); - for region in regions { - let (name, path) = match region { - FlashRegion::Bootloader => ("bootloader.bin", build_dir.join("bootloader.bin")), - FlashRegion::Partitions => ("partitions.bin", build_dir.join("partitions.bin")), - FlashRegion::Firmware => continue, - }; - if !path.exists() { - return Err(fbuild_core::FbuildError::DeployFailed(format!( - "deploy_regions requested {:?} but {} is missing from {}", - region, - name, - build_dir.display() - ))); - } - } - - #[cfg(feature = "espflash-native")] - if self.use_native_write { - if let Some(result) = native_write_or_fallback(port, "selective write-flash", || { - self.try_deploy_regions_native(firmware_path, port, regions) - }) { - return Ok(result); - } - } - - let args = self.build_write_flash_args(firmware_path, port, Some(regions)); - let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - - if self.verbose { - tracing::info!("deploy (selective): {}", args.join(" ")); - } - tracing::info!( - "flashing regions {:?} of {} to {} via esptool ({})", - regions, - firmware_path.display(), - port, - self.chip - ); - - let result = run_command( - &args_ref, - None, - None, - Some(std::time::Duration::from_secs(120)), - )?; - - if result.success() { - Ok(DeploymentResult { - success: true, - message: format!( - "{} region(s) flashed to {} ({})", - regions.len(), - port, - self.chip - ), - port: Some(port.to_string()), - stdout: result.stdout, - stderr: result.stderr, - outcome: DeployOutcome::SelectiveFlash { - regions: regions.to_vec(), - }, - }) - } else { - Ok(DeploymentResult { - success: false, - message: format!("esptool failed (exit code {})", result.exit_code), - port: Some(port.to_string()), - stdout: result.stdout, - stderr: result.stderr, - outcome: DeployOutcome::SelectiveFlash { - regions: regions.to_vec(), - }, - }) - } - } -} - -impl Deployer for Esp32Deployer { - fn deploy( - &self, - _project_dir: &Path, - _env_name: &str, - firmware_path: &Path, - port: Option<&str>, - ) -> Result { - let port = port.ok_or_else(|| { - fbuild_core::FbuildError::DeployFailed( - "serial port required for ESP32 deploy (use --port)".to_string(), - ) - })?; - - #[cfg(feature = "espflash-native")] - if self.use_native_write { - if let Some(result) = native_write_or_fallback(port, "write-flash", || { - self.try_deploy_native(firmware_path, port) - }) { - return Ok(result); - } - } - - let args = self.build_write_flash_args(firmware_path, port, None); - let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - - if self.verbose { - tracing::info!("deploy: {}", args.join(" ")); - } - - tracing::info!( - "flashing {} to {} via esptool ({})", - firmware_path.display(), - port, - self.chip - ); - - let result = run_command( - &args_ref, - None, - None, - Some(std::time::Duration::from_secs(120)), - )?; - - if result.success() { - Ok(DeploymentResult { - success: true, - message: format!("firmware flashed to {} ({})", port, self.chip), - port: Some(port.to_string()), - stdout: result.stdout, - stderr: result.stderr, - outcome: DeployOutcome::FullFlash, - }) - } else { - Ok(DeploymentResult { - success: false, - message: format!("esptool failed (exit code {})", result.exit_code), - port: Some(port.to_string()), - stdout: result.stdout, - stderr: result.stderr, - outcome: DeployOutcome::FullFlash, - }) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Test params matching ESP32-C6 JSON config values. - fn test_esptool_params() -> EsptoolParams { - EsptoolParams { - flash_mode: "dio".to_string(), - flash_freq: "80m".to_string(), - default_baud: "460800".to_string(), - before_reset: "default-reset".to_string(), - after_reset: "hard-reset".to_string(), - } - } - - #[test] - fn test_esp32_deployer_creation() { - let params = test_esptool_params(); - let deployer = Esp32Deployer::new( - "esp32c6", "460800", "0x0", "0x8000", "0x10000", ¶ms, false, - ); - assert_eq!(deployer.chip, "esp32c6"); - assert_eq!(deployer.baud_rate, "460800"); - assert_eq!(deployer.bootloader_offset, "0x0"); - assert_eq!(deployer.firmware_offset, "0x10000"); - assert_eq!(deployer.flash_mode, "dio"); - assert_eq!(deployer.before_reset, "default-reset"); - } - - #[test] - fn qemu_flash_size_resolution_accepts_supported_sizes() { - let mut board = - fbuild_config::BoardConfig::from_board_id("esp32-s3-devkitc-1", &Default::default()) - .unwrap(); - board.max_flash = Some(8 * 1024 * 1024); - assert_eq!( - resolve_qemu_flash_size_bytes(&board, "4MB").unwrap(), - 8 * 1024 * 1024 - ); - } - - #[test] - fn qemu_flash_size_resolution_rejects_unsupported_size() { - let mut board = - fbuild_config::BoardConfig::from_board_id("esp32-s3-devkitc-1", &Default::default()) - .unwrap(); - board.max_flash = Some(32 * 1024 * 1024); - let err = resolve_qemu_flash_size_bytes(&board, "4MB").unwrap_err(); - assert!(err - .to_string() - .contains("supports only 2MB, 4MB, 8MB, or 16MB")); - } - - #[test] - fn create_qemu_flash_image_writes_regions_at_offsets() { - let tmp = tempfile::TempDir::new().unwrap(); - let build_dir = tmp.path().join("build"); - std::fs::create_dir_all(&build_dir).unwrap(); - - let boot = build_dir.join("bootloader.bin"); - let parts = build_dir.join("partitions.bin"); - let fw = build_dir.join("firmware.bin"); - std::fs::write(&boot, b"BOOT").unwrap(); - std::fs::write(&parts, b"PART").unwrap(); - std::fs::write(&fw, b"FIRM").unwrap(); - - let flash = tmp.path().join("qemu_flash.bin"); - create_qemu_flash_image( - &fw, - &flash, - 2 * 1024 * 1024, - "0x0", - "0x8000", - "0x10000", - None, - ) - .unwrap(); - - let bytes = std::fs::read(&flash).unwrap(); - assert_eq!(&bytes[0..4], b"BOOT"); - assert_eq!(&bytes[0x8000..0x8004], b"PART"); - assert_eq!(&bytes[0x10000..0x10004], b"FIRM"); - assert_eq!(bytes.len(), 2 * 1024 * 1024); - assert_eq!(bytes[0x200], 0xFF); - } - - #[test] - fn create_qemu_flash_image_includes_boot_app0_when_present() { - let tmp = tempfile::TempDir::new().unwrap(); - let build_dir = tmp.path().join("build"); - std::fs::create_dir_all(&build_dir).unwrap(); - - let boot = build_dir.join("bootloader.bin"); - let boot_app0 = build_dir.join("boot_app0.bin"); - let parts = build_dir.join("partitions.bin"); - let fw = build_dir.join("firmware.bin"); - std::fs::write(&boot, b"BOOT").unwrap(); - std::fs::write(&boot_app0, b"APP0").unwrap(); - std::fs::write(&parts, b"PART").unwrap(); - std::fs::write(&fw, b"FIRM").unwrap(); - - let flash = tmp.path().join("qemu_flash.bin"); - create_qemu_flash_image( - &fw, - &flash, - 2 * 1024 * 1024, - "0x0", - "0x8000", - "0x10000", - None, - ) - .unwrap(); - - let bytes = std::fs::read(&flash).unwrap(); - assert_eq!(&bytes[0xE000..0xE004], b"APP0"); - } - - #[test] - fn resolve_esp_image_file_offset_maps_address_into_segment_data() { - let mut image = vec![0u8; ESP_IMAGE_HEADER_LEN]; - image[0] = ESP_IMAGE_HEADER_MAGIC; - image[1] = 1; - image.extend_from_slice(&0x4200_0000u32.to_le_bytes()); - image.extend_from_slice(&6u32.to_le_bytes()); - image.extend_from_slice(&[0x11, 0x22, 0x33, 0x44, 0x55, 0x66]); - - let offset = resolve_esp_image_file_offset(&image, 0x4200_0003).unwrap(); - assert_eq!( - offset, - ESP_IMAGE_HEADER_LEN + ESP_IMAGE_SEGMENT_HEADER_LEN + 3 - ); - } - - #[test] - fn patch_bytes_rewrites_expected_bytes_only() { - let mut flash = [0xFFu8; 16]; - flash[6..8].copy_from_slice(&QEMU_ADC_CALIBRATION_EXPECTED_BYTES); - - patch_bytes( - &mut flash, - 6, - &QEMU_ADC_CALIBRATION_EXPECTED_BYTES, - &QEMU_ADC_CALIBRATION_PATCH_BYTES, - ) - .unwrap(); - - assert_eq!(&flash[6..8], &QEMU_ADC_CALIBRATION_PATCH_BYTES); - } - - #[test] - fn repair_esp_image_checksum_and_hash_updates_trailers_after_patch() { - let mut image = vec![0u8; ESP_IMAGE_HEADER_LEN]; - image[0] = ESP_IMAGE_HEADER_MAGIC; - image[1] = 1; - image[23] = 1; - image.extend_from_slice(&0x4200_0000u32.to_le_bytes()); - image.extend_from_slice(&8u32.to_le_bytes()); - image.extend_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); - image.extend_from_slice(&[0u8; 16]); - image.extend_from_slice(&[0u8; ESP_IMAGE_APPENDED_HASH_LEN]); - - patch_bytes( - &mut image, - ESP_IMAGE_HEADER_LEN + ESP_IMAGE_SEGMENT_HEADER_LEN + 3, - &[4], - &[9], - ) - .unwrap(); - repair_esp_image_checksum_and_hash(&mut image).unwrap(); - - let checksum_offset = - ((ESP_IMAGE_HEADER_LEN + ESP_IMAGE_SEGMENT_HEADER_LEN + 8 + 1 + 15) & !15) - 1; - let expected_checksum = { - let mut checksum_word = ESP_ROM_CHECKSUM_INITIAL; - for chunk in image[ESP_IMAGE_HEADER_LEN + ESP_IMAGE_SEGMENT_HEADER_LEN - ..ESP_IMAGE_HEADER_LEN + ESP_IMAGE_SEGMENT_HEADER_LEN + 8] - .chunks(4) - { - let mut word = [0u8; 4]; - word[..chunk.len()].copy_from_slice(chunk); - checksum_word ^= u32::from_le_bytes(word); - } - ((checksum_word >> 24) ^ (checksum_word >> 16) ^ (checksum_word >> 8) ^ checksum_word) - as u8 - }; - let expected_hash = Sha256::digest(&image[..checksum_offset + 1]); - assert_eq!(image[checksum_offset], expected_checksum); - assert_eq!( - &image[checksum_offset + 1..checksum_offset + 1 + ESP_IMAGE_APPENDED_HASH_LEN], - expected_hash.as_slice() - ); - } - - #[test] - fn qemu_command_builder_uses_expected_machine_and_watchdog_override() { - let args = build_qemu_esp32s3_args(Path::new("flash.bin"), None); - assert!(args.contains(&"esp32s3".to_string())); - assert!(args - .iter() - .any(|arg| arg == "driver=timer.esp32s3.timg,property=wdt_disable,value=true")); - assert!(args - .iter() - .any(|arg| arg.contains("file=flash.bin,if=mtd,format=raw"))); - } - - #[test] - fn qemu_command_builder_uses_esp32_machine_for_base_variant() { - let args = build_qemu_args("esp32", Path::new("flash.bin"), None); - assert!(args.contains(&"esp32".to_string())); - assert!(args - .iter() - .any(|arg| arg == "driver=timer.esp32.timg,property=wdt_disable,value=true")); - assert!(args - .iter() - .any(|arg| arg.contains("file=flash.bin,if=mtd,format=raw"))); - } - - #[test] - fn qemu_command_builder_adds_psram_args_when_requested() { - let args = build_qemu_esp32s3_args( - Path::new("flash.bin"), - Some(fbuild_config::Esp32QemuPsramConfig { - size_mib: 8, - is_octal: true, - }), - ); - assert!(args.windows(2).any(|pair| pair == ["-m", "8M"])); - assert!(args - .iter() - .any(|arg| arg == "driver=ssi_psram,property=is_octal,value=true")); - } - - #[test] - fn test_esp32_deployer_from_board_config() { - let board = - fbuild_config::BoardConfig::from_board_id("esp32c6", &std::collections::HashMap::new()) - .unwrap(); - let params = test_esptool_params(); - let deployer = - Esp32Deployer::from_board_config(&board, "0x0", "0x8000", "0x10000", ¶ms, false); - assert_eq!(deployer.chip, "esp32c6"); - assert_eq!(deployer.bootloader_offset, "0x0"); - } - - #[test] - fn test_deploy_requires_port() { - let params = test_esptool_params(); - let deployer = Esp32Deployer::new( - "esp32c6", "460800", "0x0", "0x8000", "0x10000", ¶ms, false, - ); - let tmp = tempfile::TempDir::new().unwrap(); - let result = deployer.deploy(tmp.path(), "esp32c6", Path::new("firmware.bin"), None); - assert!(result.is_err()); - assert!(result - .unwrap_err() - .to_string() - .contains("serial port required")); - } - - /// Fast deploy: the verify-flash command line must include the - /// `verify-flash` subcommand and pair every flash region with its - /// matching offset, in the order bootloader, partitions, firmware. - /// Verifying all three in a single esptool call amortises the - /// ~3-second stub flasher upload. - #[test] - fn build_verify_flash_args_includes_all_three_regions_when_present() { - let params = test_esptool_params(); - let deployer = Esp32Deployer::new( - "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, - ); - let tmp = tempfile::TempDir::new().unwrap(); - std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); - std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firm").unwrap(); - - let args = deployer.build_verify_flash_args(&fw, "COM13"); - - // Subcommand - assert!( - args.contains(&"verify-flash".to_string()), - "missing verify-flash subcommand: {:?}", - args - ); - // Chip + port - assert!(args.contains(&"--chip".to_string())); - assert!(args.contains(&"esp32s3".to_string())); - assert!(args.contains(&"--port".to_string())); - assert!(args.contains(&"COM13".to_string())); - - // All three (offset, file) pairs in the right order. - let pos_verify = args.iter().position(|a| a == "verify-flash").unwrap(); - let pos_boot = args.iter().position(|a| a == "0x0").unwrap(); - let pos_parts = args.iter().position(|a| a == "0x8000").unwrap(); - let pos_fw = args.iter().position(|a| a == "0x10000").unwrap(); - assert!( - pos_verify < pos_boot && pos_boot < pos_parts && pos_parts < pos_fw, - "regions must appear after verify-flash in bootloader→partitions→firmware order: {:?}", - args - ); - - // verify-flash MUST NOT carry --flash-mode/freq/size flags; - // those are write-flash options and esptool 5.x rejects them - // here. We were burned by this when we copied the deploy() - // argument layout wholesale. - assert!( - !args.contains(&"--flash-mode".to_string()), - "verify-flash must not include --flash-mode (write-flash only): {:?}", - args - ); - assert!( - !args.contains(&"--flash-freq".to_string()), - "verify-flash must not include --flash-freq (write-flash only): {:?}", - args - ); - } - - #[test] - fn build_verify_flash_args_skips_missing_bootloader_and_partitions() { - // When bootloader.bin / partitions.bin haven't been built (e.g. - // an upload-only test fixture), verify must still cover firmware - // alone. Otherwise we'd skip the only thing we have. - let params = test_esptool_params(); - let deployer = Esp32Deployer::new( - "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, - ); - let tmp = tempfile::TempDir::new().unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firm").unwrap(); - - let args = deployer.build_verify_flash_args(&fw, "COM13"); - - // No bootloader or partitions paths in the args. - assert!(!args.iter().any(|a| a.ends_with("bootloader.bin"))); - assert!(!args.iter().any(|a| a.ends_with("partitions.bin"))); - // Firmware offset is still present. - assert!(args.contains(&"0x10000".to_string())); - assert!(args.iter().any(|a| a.ends_with("firmware.bin"))); - } - - /// esptool 5.x emits one `Verifying ... at 0x{addr:#010x} ...` line - /// followed by `Verification successful/failed` for each region. - /// Parser must pair them and classify the region by offset. - #[test] - fn parse_verify_regions_classifies_each_region_by_offset() { - let stdout = "\ -Verifying 0x6060 (24672) bytes at 0x00000000 in flash against 'bootloader.bin'...\n\ -Verification successful (digest matched).\n\ -Verifying 0xc00 (3072) bytes at 0x00008000 in flash against 'partitions.bin'...\n\ -Verification successful (digest matched).\n\ -Verifying 0x260a60 (2493536) bytes at 0x00010000 in flash against 'firmware.bin'...\n\ -Verification failed (digest mismatch).\n"; - let regions = parse_verify_regions(stdout, "0x0", "0x8000", "0x10000"); - assert_eq!( - regions, - vec![ - RegionVerifyResult { - region: FlashRegion::Bootloader, - matched: true - }, - RegionVerifyResult { - region: FlashRegion::Partitions, - matched: true - }, - RegionVerifyResult { - region: FlashRegion::Firmware, - matched: false - }, - ] - ); - } - - /// When esptool output doesn't match the expected pattern (older - /// version, localized output, truncated log), parse returns an empty - /// vec so the daemon falls back to flashing all regions. - #[test] - fn parse_verify_regions_returns_empty_on_unknown_format() { - let stdout = "some unrelated failure output\nexit 1\n"; - let regions = parse_verify_regions(stdout, "0x0", "0x8000", "0x10000"); - assert!(regions.is_empty()); - } - - /// Regions whose offset doesn't match any of the three knowns are - /// silently skipped — we stay conservative and return only what we - /// understand. - #[test] - fn parse_verify_regions_skips_unknown_offsets() { - let stdout = "\ -Verifying 0x1000 (4096) bytes at 0x00001000 in flash against 'bootloader.bin'...\n\ -Verification failed (digest mismatch).\n\ -Verifying 0x1000 (4096) bytes at 0x00010000 in flash against 'firmware.bin'...\n\ -Verification successful (digest matched).\n"; - let regions = parse_verify_regions(stdout, "0x1000", "0x8000", "0x10000"); - assert_eq!( - regions, - vec![ - RegionVerifyResult { - region: FlashRegion::Bootloader, - matched: false - }, - RegionVerifyResult { - region: FlashRegion::Firmware, - matched: true - }, - ] - ); - } - - /// The selective write-flash argv must include the write-flash - /// subcommand and only the requested region's offset/file pair. - /// Skipping bootloader + partitions is the ~1s save targeted by #67. - #[test] - fn build_write_flash_args_firmware_only_skips_bootloader_and_partitions() { - let params = test_esptool_params(); - let deployer = Esp32Deployer::new( - "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, - ); - let tmp = tempfile::TempDir::new().unwrap(); - std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); - std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firm").unwrap(); - - let args = deployer.build_write_flash_args(&fw, "COM13", Some(&[FlashRegion::Firmware])); - - assert!(args.contains(&"write-flash".to_string())); - assert!(!args.iter().any(|a| a.ends_with("bootloader.bin"))); - assert!(!args.iter().any(|a| a.ends_with("partitions.bin"))); - assert!(args.contains(&"0x10000".to_string())); - assert!(args.iter().any(|a| a.ends_with("firmware.bin"))); - assert!(!args.contains(&"0x8000".to_string())); - } - - /// `None` regions (default deploy) must still include all three - /// present files — we can't regress the baseline path. - #[test] - fn build_write_flash_args_default_includes_all_regions() { - let params = test_esptool_params(); - let deployer = Esp32Deployer::new( - "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, - ); - let tmp = tempfile::TempDir::new().unwrap(); - std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); - std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firm").unwrap(); - - let args = deployer.build_write_flash_args(&fw, "COM13", None); - assert!(args.contains(&"0x0".to_string())); - assert!(args.contains(&"0x8000".to_string())); - assert!(args.contains(&"0x10000".to_string())); - } - - /// If a caller requests a region whose file is missing on disk, fail - /// with a clear error rather than silently emitting a write-flash - /// call with no offset/file pair (which would produce an opaque - /// esptool usage error). Addresses CodeRabbit review on PR #71. - #[test] - fn deploy_regions_errors_when_requested_region_file_missing() { - let params = test_esptool_params(); - let deployer = Esp32Deployer::new( - "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, - ); - let tmp = tempfile::TempDir::new().unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firm").unwrap(); - // Note: no bootloader.bin written. - let err = deployer - .deploy_regions(&fw, "COM13", &[FlashRegion::Bootloader]) - .unwrap_err(); - assert!( - err.to_string().contains("bootloader.bin"), - "error must name the missing file: {}", - err - ); - } - - /// Empty region slice -> usage error; we surface it rather than let - /// esptool barf. - #[test] - fn deploy_regions_rejects_empty_slice() { - let params = test_esptool_params(); - let deployer = Esp32Deployer::new( - "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, - ); - let err = deployer - .deploy_regions(Path::new("firmware.bin"), "COM13", &[]) - .unwrap_err(); - assert!(err.to_string().contains("no regions")); - } - - #[test] - fn verify_outcome_is_match_helper() { - let m = VerifyOutcome::Match { - stdout: "ok".into(), - stderr: String::new(), - }; - let mm = VerifyOutcome::Mismatch { - stdout: String::new(), - stderr: "Verification failed".into(), - regions: Vec::new(), - }; - assert!(m.is_match()); - assert!(!mm.is_match()); - } - - // --------------------------------------------------------------- - // Hardware-gated verify-deployment tests for each ESP32 family MCU. - // - // These tests are `#[ignore]` so they never run in CI. To exercise - // them on a local bench, set the env vars described below and run: - // - // uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real -- --ignored --nocapture - // - // Each test reads **two** environment variables: - // - // _PORT – serial port the board is attached to (e.g. COM13, /dev/ttyUSB0) - // _FIRMWARE – absolute path to a pre-flashed firmware.bin - // - // where is one of ESP32, ESP32S2, ESP32S3, ESP32C2, ESP32C3, - // ESP32C6, ESP32H2, ESP32P4. - // - // The firmware directory must also contain `bootloader.bin` and - // `partitions.bin` so that verify-flash can check all three regions - // in a single esptool invocation. - // - // Bootloader offsets per chip (from esp32.rs header comment): - // 0x1000 – esp32, esp32s2 - // 0x0 – esp32c2, esp32c3, esp32c5, esp32c6, esp32h2, esp32s3 - // 0x2000 – esp32p4 - // --------------------------------------------------------------- - - /// Shared implementation for all per-chip hardware-gated verify tests. - /// - /// 1. Reads `{port_env}` and `{firmware_env}` from the environment. - /// 2. Asserts that verify against the pre-flashed image returns `Match` - /// in under 15 seconds. - /// 3. Asserts that a tampered image (1 byte flipped) returns `Mismatch`. - fn run_verify_deployment_test( - chip: &str, - bootloader_offset: &str, - port_env: &str, - firmware_env: &str, - ) { - let port = std::env::var(port_env).unwrap_or_else(|_| { - panic!( - "set {} to the serial port your {} board is attached to (e.g. COM13)", - port_env, chip - ) - }); - let firmware_path = std::env::var(firmware_env).unwrap_or_else(|_| { - panic!( - "set {} to the absolute path of the pre-flashed firmware.bin for {}", - firmware_env, chip - ) - }); - let reference = std::path::PathBuf::from(&firmware_path); - assert!( - reference.is_file(), - "reference firmware not found at {}; build and flash it first", - reference.display() - ); - let ref_dir = reference - .parent() - .unwrap_or_else(|| std::path::Path::new(".")); - for name in ["bootloader.bin", "partitions.bin"] { - let artifact = ref_dir.join(name); - assert!( - artifact.is_file(), - "[{}] missing {} next to {}; otherwise this only verifies firmware.bin", - chip, - name, - reference.display() - ); - } - - let params = EsptoolParams { - flash_mode: "dio".to_string(), - flash_freq: "80m".to_string(), - default_baud: "921600".to_string(), - before_reset: "default-reset".to_string(), - after_reset: "hard-reset".to_string(), - }; - let deployer = Esp32Deployer::new( - chip, - "921600", - bootloader_offset, - "0x8000", - "0x10000", - ¶ms, - true, - ); - - // Phase 1: matching image -> Match - let start = std::time::Instant::now(); - let outcome = deployer - .try_verify_deployment(&reference, &port) - .unwrap_or_else(|e| panic!("verify must not fail against attached {}: {}", chip, e)); - let elapsed = start.elapsed(); - assert!( - outcome.is_match(), - "[{}] expected Match against pre-flashed firmware; got {:?}", - chip, - outcome - ); - assert!( - elapsed < std::time::Duration::from_secs(15), - "[{}] verify took {:?} -- should complete in <15s", - chip, - elapsed - ); - eprintln!("[{}] verify (Match) elapsed: {:?}", chip, elapsed); - - // Phase 2: tampered image -> Mismatch - let tmp = tempfile::TempDir::new().unwrap(); - // Copy bootloader and partitions next to the tampered firmware so - // build_verify_flash_args picks them up alongside firmware.bin. - for name in ["bootloader.bin", "partitions.bin"] { - std::fs::copy(ref_dir.join(name), tmp.path().join(name)).unwrap(); - } - let tampered = tmp.path().join("firmware.bin"); - let mut bytes = std::fs::read(&reference).unwrap(); - // Flip a byte well past the image header to avoid invalidating - // the ESP-IDF magic and triggering an esptool parse error rather - // than a clean digest mismatch. - let target = bytes.len() / 2; - bytes[target] ^= 0x55; - std::fs::write(&tampered, &bytes).unwrap(); - - let outcome = deployer - .try_verify_deployment(&tampered, &port) - .unwrap_or_else(|e| { - panic!( - "[{}] verify must not fail with tampered firmware: {}", - chip, e - ) - }); - assert!( - !outcome.is_match(), - "[{}] expected Mismatch for tampered firmware; got {:?}", - chip, - outcome - ); - eprintln!("[{}] verify (Mismatch) detected correctly", chip); - } - - /// ESP32 (Xtensa, bootloader at 0x1000). - /// - /// ```text - /// ESP32_PORT=COM5 ESP32_FIRMWARE=C:\path\to\firmware.bin \ - /// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32 -- --ignored --nocapture - /// ``` - #[test] - #[ignore = "requires real ESP32 board — set ESP32_PORT and ESP32_FIRMWARE"] - fn try_verify_deployment_real_esp32() { - run_verify_deployment_test("esp32", "0x1000", "ESP32_PORT", "ESP32_FIRMWARE"); - } - - /// ESP32-S2 (Xtensa single-core, bootloader at 0x1000). - /// - /// ```text - /// ESP32S2_PORT=COM6 ESP32S2_FIRMWARE=C:\path\to\firmware.bin \ - /// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32s2 -- --ignored --nocapture - /// ``` - #[test] - #[ignore = "requires real ESP32-S2 board — set ESP32S2_PORT and ESP32S2_FIRMWARE"] - fn try_verify_deployment_real_esp32s2() { - run_verify_deployment_test("esp32s2", "0x1000", "ESP32S2_PORT", "ESP32S2_FIRMWARE"); - } - - /// ESP32-S3 (Xtensa dual-core, bootloader at 0x0). - /// - /// This is the original baseline test, now using env-var configuration - /// consistent with the rest of the family. - /// - /// ```text - /// ESP32S3_PORT=COM13 ESP32S3_FIRMWARE=C:\Users\niteris\dev\fastled\.pio\build\esp32s3\firmware.bin \ - /// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32s3 -- --ignored --nocapture - /// ``` - #[test] - #[ignore = "requires real ESP32-S3 board — set ESP32S3_PORT and ESP32S3_FIRMWARE"] - fn try_verify_deployment_real_esp32s3() { - run_verify_deployment_test("esp32s3", "0x0", "ESP32S3_PORT", "ESP32S3_FIRMWARE"); - } - - /// ESP32-C2 (RISC-V single-core, bootloader at 0x0). - /// - /// ```text - /// ESP32C2_PORT=COM7 ESP32C2_FIRMWARE=C:\path\to\firmware.bin \ - /// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32c2 -- --ignored --nocapture - /// ``` - #[test] - #[ignore = "requires real ESP32-C2 board — set ESP32C2_PORT and ESP32C2_FIRMWARE"] - fn try_verify_deployment_real_esp32c2() { - run_verify_deployment_test("esp32c2", "0x0", "ESP32C2_PORT", "ESP32C2_FIRMWARE"); - } - - /// ESP32-C3 (RISC-V single-core, bootloader at 0x0). - /// - /// ```text - /// ESP32C3_PORT=COM8 ESP32C3_FIRMWARE=C:\path\to\firmware.bin \ - /// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32c3 -- --ignored --nocapture - /// ``` - #[test] - #[ignore = "requires real ESP32-C3 board — set ESP32C3_PORT and ESP32C3_FIRMWARE"] - fn try_verify_deployment_real_esp32c3() { - run_verify_deployment_test("esp32c3", "0x0", "ESP32C3_PORT", "ESP32C3_FIRMWARE"); - } - - /// ESP32-C6 (RISC-V single-core, bootloader at 0x0). - /// - /// ```text - /// ESP32C6_PORT=COM9 ESP32C6_FIRMWARE=C:\path\to\firmware.bin \ - /// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32c6 -- --ignored --nocapture - /// ``` - #[test] - #[ignore = "requires real ESP32-C6 board — set ESP32C6_PORT and ESP32C6_FIRMWARE"] - fn try_verify_deployment_real_esp32c6() { - run_verify_deployment_test("esp32c6", "0x0", "ESP32C6_PORT", "ESP32C6_FIRMWARE"); - } - - /// ESP32-H2 (RISC-V single-core, bootloader at 0x0). - /// - /// ```text - /// ESP32H2_PORT=COM10 ESP32H2_FIRMWARE=C:\path\to\firmware.bin \ - /// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32h2 -- --ignored --nocapture - /// ``` - #[test] - #[ignore = "requires real ESP32-H2 board — set ESP32H2_PORT and ESP32H2_FIRMWARE"] - fn try_verify_deployment_real_esp32h2() { - run_verify_deployment_test("esp32h2", "0x0", "ESP32H2_PORT", "ESP32H2_FIRMWARE"); - } - - /// ESP32-P4 (RISC-V dual-core, OPI flash, bootloader at 0x2000). - /// - /// Note: ESP32-P4 uses OPI flash and has a different bootloader offset - /// (0x2000) compared to other ESP32 chips. - /// - /// ```text - /// ESP32P4_PORT=COM11 ESP32P4_FIRMWARE=C:\path\to\firmware.bin \ - /// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32p4 -- --ignored --nocapture - /// ``` - #[test] - #[ignore = "requires real ESP32-P4 board — set ESP32P4_PORT and ESP32P4_FIRMWARE"] - fn try_verify_deployment_real_esp32p4() { - run_verify_deployment_test("esp32p4", "0x2000", "ESP32P4_PORT", "ESP32P4_FIRMWARE"); - } -} diff --git a/crates/fbuild-deploy/src/esp32/deployer.rs b/crates/fbuild-deploy/src/esp32/deployer.rs new file mode 100644 index 00000000..92ee9160 --- /dev/null +++ b/crates/fbuild-deploy/src/esp32/deployer.rs @@ -0,0 +1,740 @@ +//! `Esp32Deployer` core: construction, args, verify and write paths, and +//! the [`Deployer`] trait implementation. + +use std::path::Path; + +use fbuild_core::subprocess::run_command; +use fbuild_core::Result; + +#[cfg(feature = "espflash-native")] +use super::parse::parse_hex_offset_u32; +use super::verify::{parse_verify_regions, FlashRegion, VerifyOutcome}; +use crate::{DeployOutcome, Deployer, DeploymentResult}; + +/// Esptool flash parameters sourced from MCU config JSON. +/// +/// All fields correspond to `esptool` section fields in the MCU config. +pub struct EsptoolParams { + pub flash_mode: String, + pub flash_freq: String, + pub default_baud: String, + pub before_reset: String, + pub after_reset: String, +} + +/// ESP32 deployer using `esptool`. +pub struct Esp32Deployer { + /// MCU chip type for esptool --chip flag (e.g. "esp32c6"). + pub(super) chip: String, + /// Baud rate for flashing (e.g. "460800"). + pub(super) baud_rate: String, + /// Flash offsets. + pub(super) bootloader_offset: String, + pub(super) partitions_offset: String, + pub(super) firmware_offset: String, + /// Flash mode for esptool (e.g. "dio", "qio"). + pub(super) flash_mode: String, + /// Flash frequency for esptool (e.g. "80m", "40m"). + pub(super) flash_freq: String, + /// Reset mode before flashing. + pub(super) before_reset: String, + /// Reset mode after flashing. + pub(super) after_reset: String, + pub(super) verbose: bool, + /// Route `verify-flash` through the native [`espflash`] crate + /// instead of the Python `esptool` subprocess. + /// + /// The daemon sets this from the `FBUILD_USE_ESPFLASH_VERIFY` env + /// var. When enabled, the deployer still falls back to esptool if + /// the native path fails on a given board or host. + /// + /// Only compiled in when the `espflash-native` cargo feature is + /// enabled. + #[cfg(feature = "espflash-native")] + pub(super) use_native_verify: bool, + /// Route `write-flash` through the native [`espflash`] crate + /// instead of the Python `esptool` subprocess (issue #66). + /// + /// The daemon sets this from the `FBUILD_USE_ESPFLASH_WRITE` env + /// var, independently of `use_native_verify`. When enabled, the + /// deployer still falls back to esptool if the native path fails. + /// + /// Feature-gated — see `use_native_verify`. + #[cfg(feature = "espflash-native")] + pub(super) use_native_write: bool, +} + +#[cfg(feature = "espflash-native")] +pub(super) fn native_write_or_fallback( + port: &str, + label: &str, + native: F, +) -> Option +where + F: FnOnce() -> Result, +{ + match native() { + Ok(result) if result.success => Some(result), + Ok(result) => { + tracing::warn!( + port, + "native {} failed ({}); falling back to esptool", + label, + result.message + ); + None + } + Err(e) => { + tracing::warn!( + port, + "native {} failed ({}); falling back to esptool", + label, + e + ); + None + } + } +} + +impl Esp32Deployer { + #[allow(clippy::too_many_arguments)] + pub fn new( + chip: &str, + baud_rate: &str, + bootloader_offset: &str, + partitions_offset: &str, + firmware_offset: &str, + esptool_params: &EsptoolParams, + verbose: bool, + ) -> Self { + Self { + chip: chip.to_string(), + baud_rate: baud_rate.to_string(), + bootloader_offset: bootloader_offset.to_string(), + partitions_offset: partitions_offset.to_string(), + firmware_offset: firmware_offset.to_string(), + flash_mode: esptool_params.flash_mode.clone(), + flash_freq: esptool_params.flash_freq.clone(), + before_reset: esptool_params.before_reset.clone(), + after_reset: esptool_params.after_reset.clone(), + verbose, + #[cfg(feature = "espflash-native")] + use_native_verify: false, + #[cfg(feature = "espflash-native")] + use_native_write: false, + } + } + + /// Opt this deployer into the native espflash-based verify path + /// (issue #66). Independent of `with_native_write`. + /// + /// Only present when the `espflash-native` cargo feature is enabled; + /// without it the esptool-subprocess path is the only code path. + #[cfg(feature = "espflash-native")] + #[must_use] + pub fn with_native_verify(mut self, enabled: bool) -> Self { + self.use_native_verify = enabled; + self + } + + /// Opt this deployer into the native espflash-based write-flash + /// path (issue #66). Independent of `with_native_verify`. When + /// enabled, both [`Deployer::deploy`] and + /// [`Esp32Deployer::deploy_regions`] route through the in-process + /// espflash `Flasher`, skipping the ~1.5 s Python/esptool startup + /// per flash unless they need to fall back. + /// + /// Only present when the `espflash-native` cargo feature is enabled. + #[cfg(feature = "espflash-native")] + #[must_use] + pub fn with_native_write(mut self, enabled: bool) -> Self { + self.use_native_write = enabled; + self + } + + /// Create an ESP32 deployer from board config with explicit flash offsets. + pub fn from_board_config( + board: &fbuild_config::BoardConfig, + bootloader_offset: &str, + partitions_offset: &str, + firmware_offset: &str, + esptool_params: &EsptoolParams, + verbose: bool, + ) -> Self { + let baud = board + .upload_speed + .as_deref() + .unwrap_or(&esptool_params.default_baud); + // Board-level flash_mode overrides MCU default. + let flash_mode = board + .flash_mode + .as_deref() + .unwrap_or(&esptool_params.flash_mode); + let params = EsptoolParams { + flash_mode: flash_mode.to_string(), + flash_freq: esptool_params.flash_freq.clone(), + default_baud: esptool_params.default_baud.clone(), + before_reset: esptool_params.before_reset.clone(), + after_reset: esptool_params.after_reset.clone(), + }; + Self::new( + &board.mcu, + baud, + bootloader_offset, + partitions_offset, + firmware_offset, + ¶ms, + verbose, + ) + } + + /// Override the baud rate (e.g. from a CLI `--baud` flag). + pub fn with_baud_rate(mut self, baud: &str) -> Self { + self.baud_rate = baud.to_string(); + self + } + + /// Find the esptool executable. + /// + /// Uses standalone `esptool` command (available when esptool is pip-installed). + pub(super) fn find_esptool() -> Vec { + vec!["esptool".to_string()] + } + + /// Build the `esptool verify-flash` command line that this deployer + /// would run to verify a candidate firmware image is already on the + /// device. Pure (no I/O) so we can unit-test the argument layout + /// without touching real hardware. + /// + /// `verify-flash` uses the device-side `FLASH_MD5SUM` command (issued + /// by the stub flasher), so it does NOT read the entire flash region + /// back over UART — verification of a 2.4 MB ESP32-S3 image takes + /// ~6 seconds end-to-end vs ~25 seconds for a full re-flash. + /// See ISSUES.md "Fast deploy via verify-then-skip". + pub fn build_verify_flash_args(&self, firmware_path: &Path, port: &str) -> Vec { + let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); + let bootloader_path = build_dir.join("bootloader.bin"); + let partitions_path = build_dir.join("partitions.bin"); + + let mut args = Self::find_esptool(); + args.extend([ + "--chip".to_string(), + self.chip.clone(), + "--port".to_string(), + port.to_string(), + "--baud".to_string(), + self.baud_rate.clone(), + "--before".to_string(), + self.before_reset.clone(), + "--after".to_string(), + self.after_reset.clone(), + "verify-flash".to_string(), + ]); + + // Verify all three regions in a single esptool invocation so we + // pay the stub-flasher upload cost (~3s) once, not three times. + if bootloader_path.exists() { + args.push(self.bootloader_offset.clone()); + args.push(bootloader_path.to_string_lossy().to_string()); + } + if partitions_path.exists() { + args.push(self.partitions_offset.clone()); + args.push(partitions_path.to_string_lossy().to_string()); + } + args.push(self.firmware_offset.clone()); + args.push(firmware_path.to_string_lossy().to_string()); + args + } + + /// Run `esptool verify-flash` on bootloader + partitions + firmware + /// against the live device. Returns `Ok(true)` when every region's + /// FLASH_MD5SUM matches the local file (i.e. flashing would be a + /// no-op), `Ok(false)` when at least one region differs, and `Err` + /// only when esptool itself failed to run (port not found, stub + /// upload failed, etc.). + /// + /// On success the chip is hard-reset by esptool's `--after hard-reset`, + /// matching the post-flash behavior — so callers can treat a `true` + /// return as "device is now running the requested firmware" without + /// any extra reset. + pub fn try_verify_deployment(&self, firmware_path: &Path, port: &str) -> Result { + #[cfg(feature = "espflash-native")] + if self.use_native_verify { + match self.try_verify_deployment_native(firmware_path, port) { + Ok(outcome) => return Ok(outcome), + Err(e) => { + tracing::warn!( + port, + "native verify-flash failed ({}); falling back to esptool", + e + ); + } + } + } + + self.try_verify_deployment_esptool(firmware_path, port) + } + + fn try_verify_deployment_esptool( + &self, + firmware_path: &Path, + port: &str, + ) -> Result { + let args = self.build_verify_flash_args(firmware_path, port); + let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + + if self.verbose { + tracing::info!("verify: {}", args.join(" ")); + } + tracing::info!( + "verifying {} on {} via esptool ({})", + firmware_path.display(), + port, + self.chip + ); + + let result = run_command( + &args_ref, + None, + None, + // Verify is bounded: the slowest case is ~10s for a maximum + // image. 30s gives plenty of headroom for slow USB-CDC stacks + // and stub flasher upload retries. + Some(std::time::Duration::from_secs(30)), + )?; + + if result.success() { + Ok(VerifyOutcome::Match { + stdout: result.stdout, + stderr: result.stderr, + }) + } else { + // esptool exits non-zero on a digest mismatch *and* on real + // failures (port unreachable, stub upload error). Distinguish + // them by looking at stderr — a true digest mismatch always + // contains the literal "Verification failed" string. + let combined = format!("{}\n{}", result.stdout, result.stderr); + if combined.contains("Verification failed") || combined.contains("digest mismatch") { + let regions = parse_verify_regions( + &combined, + &self.bootloader_offset, + &self.partitions_offset, + &self.firmware_offset, + ); + Ok(VerifyOutcome::Mismatch { + stdout: result.stdout, + stderr: result.stderr, + regions, + }) + } else { + Err(fbuild_core::FbuildError::DeployFailed(format!( + "esptool verify-flash failed (exit {}): {}", + result.exit_code, result.stderr + ))) + } + } + } + + /// Native `verify-flash` via the [`espflash`] crate (issue #66). + /// + /// Saves the Python interpreter startup (~1 s) and subprocess spawn + /// (~0.5 s) that the esptool path pays per invocation. Same three + /// regions (bootloader / partitions / firmware) and same + /// [`VerifyOutcome`] semantics as the esptool path, so callers can + /// swap between the two behind the `use_native_verify` flag without + /// any result-handling changes. + #[cfg(feature = "espflash-native")] + fn try_verify_deployment_native( + &self, + firmware_path: &Path, + port: &str, + ) -> Result { + let baud: u32 = self.baud_rate.parse().map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!( + "native verify: invalid baud rate '{}': {}", + self.baud_rate, e + )) + })?; + let boot_off = parse_hex_offset_u32(&self.bootloader_offset)?; + let parts_off = parse_hex_offset_u32(&self.partitions_offset)?; + let fw_off = parse_hex_offset_u32(&self.firmware_offset)?; + + let regions = crate::esp32_native::collect_standard_regions( + firmware_path, + boot_off, + parts_off, + fw_off, + ); + + if self.verbose { + tracing::info!( + "native verify: chip={} port={} baud={} regions={}", + self.chip, + port, + baud, + regions.len() + ); + } + tracing::info!( + "verifying {} on {} via espflash ({})", + firmware_path.display(), + port, + self.chip + ); + + crate::esp32_native::try_verify_deployment_native( + &self.chip, + port, + baud, + &self.before_reset, + &self.after_reset, + ®ions, + boot_off, + parts_off, + fw_off, + ) + } + + /// Native `write-flash` via the [`espflash`] crate (issue #66). + /// + /// Writes the full three-region set (bootloader + partitions + + /// firmware where present). Saves the Python interpreter startup + /// (~1 s) plus subprocess spawn (~0.5 s) vs the esptool path, and + /// surfaces per-region progress via `tracing` (bridged into the + /// daemon's existing log broadcaster). Same [`DeploymentResult`] + /// shape as the esptool path so callers swap behind a single flag. + #[cfg(feature = "espflash-native")] + pub(super) fn try_deploy_native( + &self, + firmware_path: &Path, + port: &str, + ) -> Result { + let baud = self.parse_native_baud()?; + let (boot_off, parts_off, fw_off) = self.parse_native_offsets()?; + + let regions = crate::esp32_native::collect_standard_write_regions( + firmware_path, + boot_off, + parts_off, + fw_off, + ); + + if self.verbose { + tracing::info!( + "native write: chip={} port={} baud={} regions={}", + self.chip, + port, + baud, + regions.len() + ); + } + tracing::info!( + "flashing {} to {} via espflash ({})", + firmware_path.display(), + port, + self.chip + ); + + crate::esp32_native::try_write_deployment_native( + &self.chip, + port, + baud, + &self.before_reset, + &self.after_reset, + ®ions, + /* selective */ false, + ) + } + + /// Native `write-flash` for a caller-chosen subset of regions + /// (issue #66). Used after a verify-mismatch to rewrite only the + /// regions that actually differ — skipping the ~1s + /// bootloader/partitions rewrite when only firmware changed. + #[cfg(feature = "espflash-native")] + pub(super) fn try_deploy_regions_native( + &self, + firmware_path: &Path, + port: &str, + regions: &[FlashRegion], + ) -> Result { + let baud = self.parse_native_baud()?; + let (boot_off, parts_off, fw_off) = self.parse_native_offsets()?; + + let write_regions = crate::esp32_native::collect_selected_write_regions( + firmware_path, + boot_off, + parts_off, + fw_off, + regions, + )?; + + if self.verbose { + tracing::info!( + "native write (selective): chip={} port={} baud={} regions={:?}", + self.chip, + port, + baud, + regions + ); + } + tracing::info!( + "flashing regions {:?} of {} to {} via espflash ({})", + regions, + firmware_path.display(), + port, + self.chip + ); + + crate::esp32_native::try_write_deployment_native( + &self.chip, + port, + baud, + &self.before_reset, + &self.after_reset, + &write_regions, + /* selective */ true, + ) + } + + #[cfg(feature = "espflash-native")] + fn parse_native_baud(&self) -> Result { + self.baud_rate.parse().map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!( + "native write: invalid baud rate '{}': {}", + self.baud_rate, e + )) + }) + } + + #[cfg(feature = "espflash-native")] + fn parse_native_offsets(&self) -> Result<(u32, u32, u32)> { + let boot = parse_hex_offset_u32(&self.bootloader_offset)?; + let parts = parse_hex_offset_u32(&self.partitions_offset)?; + let fw = parse_hex_offset_u32(&self.firmware_offset)?; + Ok((boot, parts, fw)) + } +} + +impl Esp32Deployer { + /// Build `esptool write-flash` argv for a caller-chosen subset of + /// regions. Pass `None` for `regions` to flash all three (bootloader + + /// partitions + firmware) when present on disk — the default deploy + /// path. Pass `Some(&[...])` to restrict the write to specific regions + /// (see `deploy_regions`). + pub fn build_write_flash_args( + &self, + firmware_path: &Path, + port: &str, + regions: Option<&[FlashRegion]>, + ) -> Vec { + let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); + let bootloader_path = build_dir.join("bootloader.bin"); + let partitions_path = build_dir.join("partitions.bin"); + + let mut args = Self::find_esptool(); + args.extend([ + "--chip".to_string(), + self.chip.clone(), + "--port".to_string(), + port.to_string(), + "--baud".to_string(), + self.baud_rate.clone(), + "--before".to_string(), + self.before_reset.clone(), + "--after".to_string(), + self.after_reset.clone(), + "write-flash".to_string(), + "-z".to_string(), + "--flash-mode".to_string(), + self.flash_mode.clone(), + "--flash-freq".to_string(), + self.flash_freq.clone(), + "--flash-size".to_string(), + "detect".to_string(), + ]); + + let include = |r: FlashRegion| regions.map_or(true, |rs| rs.contains(&r)); + + if include(FlashRegion::Bootloader) && bootloader_path.exists() { + args.push(self.bootloader_offset.clone()); + args.push(bootloader_path.to_string_lossy().to_string()); + } + if include(FlashRegion::Partitions) && partitions_path.exists() { + args.push(self.partitions_offset.clone()); + args.push(partitions_path.to_string_lossy().to_string()); + } + if include(FlashRegion::Firmware) { + args.push(self.firmware_offset.clone()); + args.push(firmware_path.to_string_lossy().to_string()); + } + args + } + + /// Flash only the specified regions. Use after `try_verify_deployment` + /// returns a `Mismatch` with `regions` populated — we skip the ~1s + /// bootloader/partitions rewrite when only firmware differs (the + /// overwhelmingly common case for iterative development). + /// + /// Returns an error when `regions` is empty; esptool rejects a + /// write-flash call with no offset/file pair and the message would be + /// opaque. + pub fn deploy_regions( + &self, + firmware_path: &Path, + port: &str, + regions: &[FlashRegion], + ) -> Result { + if regions.is_empty() { + return Err(fbuild_core::FbuildError::DeployFailed( + "deploy_regions called with no regions; pass at least one".to_string(), + )); + } + + // Fail loudly if the caller asked for a region whose file isn't + // on disk. Without this check `build_write_flash_args` would + // silently drop the request and esptool would complain with an + // opaque usage error. + let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); + for region in regions { + let (name, path) = match region { + FlashRegion::Bootloader => ("bootloader.bin", build_dir.join("bootloader.bin")), + FlashRegion::Partitions => ("partitions.bin", build_dir.join("partitions.bin")), + FlashRegion::Firmware => continue, + }; + if !path.exists() { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "deploy_regions requested {:?} but {} is missing from {}", + region, + name, + build_dir.display() + ))); + } + } + + #[cfg(feature = "espflash-native")] + if self.use_native_write { + if let Some(result) = native_write_or_fallback(port, "selective write-flash", || { + self.try_deploy_regions_native(firmware_path, port, regions) + }) { + return Ok(result); + } + } + + let args = self.build_write_flash_args(firmware_path, port, Some(regions)); + let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + + if self.verbose { + tracing::info!("deploy (selective): {}", args.join(" ")); + } + tracing::info!( + "flashing regions {:?} of {} to {} via esptool ({})", + regions, + firmware_path.display(), + port, + self.chip + ); + + let result = run_command( + &args_ref, + None, + None, + Some(std::time::Duration::from_secs(120)), + )?; + + if result.success() { + Ok(DeploymentResult { + success: true, + message: format!( + "{} region(s) flashed to {} ({})", + regions.len(), + port, + self.chip + ), + port: Some(port.to_string()), + stdout: result.stdout, + stderr: result.stderr, + outcome: DeployOutcome::SelectiveFlash { + regions: regions.to_vec(), + }, + }) + } else { + Ok(DeploymentResult { + success: false, + message: format!("esptool failed (exit code {})", result.exit_code), + port: Some(port.to_string()), + stdout: result.stdout, + stderr: result.stderr, + outcome: DeployOutcome::SelectiveFlash { + regions: regions.to_vec(), + }, + }) + } + } +} + +impl Deployer for Esp32Deployer { + fn deploy( + &self, + _project_dir: &Path, + _env_name: &str, + firmware_path: &Path, + port: Option<&str>, + ) -> Result { + let port = port.ok_or_else(|| { + fbuild_core::FbuildError::DeployFailed( + "serial port required for ESP32 deploy (use --port)".to_string(), + ) + })?; + + #[cfg(feature = "espflash-native")] + if self.use_native_write { + if let Some(result) = native_write_or_fallback(port, "write-flash", || { + self.try_deploy_native(firmware_path, port) + }) { + return Ok(result); + } + } + + let args = self.build_write_flash_args(firmware_path, port, None); + let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + + if self.verbose { + tracing::info!("deploy: {}", args.join(" ")); + } + + tracing::info!( + "flashing {} to {} via esptool ({})", + firmware_path.display(), + port, + self.chip + ); + + let result = run_command( + &args_ref, + None, + None, + Some(std::time::Duration::from_secs(120)), + )?; + + if result.success() { + Ok(DeploymentResult { + success: true, + message: format!("firmware flashed to {} ({})", port, self.chip), + port: Some(port.to_string()), + stdout: result.stdout, + stderr: result.stderr, + outcome: DeployOutcome::FullFlash, + }) + } else { + Ok(DeploymentResult { + success: false, + message: format!("esptool failed (exit code {})", result.exit_code), + port: Some(port.to_string()), + stdout: result.stdout, + stderr: result.stderr, + outcome: DeployOutcome::FullFlash, + }) + } + } +} + diff --git a/crates/fbuild-deploy/src/esp32/image.rs b/crates/fbuild-deploy/src/esp32/image.rs new file mode 100644 index 00000000..b95d7fd4 --- /dev/null +++ b/crates/fbuild-deploy/src/esp32/image.rs @@ -0,0 +1,270 @@ +//! ESP image header constants, patching, checksum repair, and raw binary +//! I/O helpers used when assembling a QEMU flash image. + +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::Path; + +use fbuild_core::Result; +use object::{Object, ObjectSymbol}; +use sha2::{Digest, Sha256}; + +pub(super) const ESP_IMAGE_HEADER_LEN: usize = 24; +pub(super) const ESP_IMAGE_SEGMENT_HEADER_LEN: usize = 8; +pub(super) const ESP_IMAGE_HEADER_MAGIC: u8 = 0xE9; +pub(super) const ESP_ROM_CHECKSUM_INITIAL: u32 = 0xEF; +pub(super) const ESP_IMAGE_APPENDED_HASH_LEN: usize = 32; +pub(super) const QEMU_ADC_CALIBRATION_SYMBOL: &str = "adc_hw_calibration"; +pub(super) const QEMU_ADC_CALIBRATION_PATCH_OFFSET: u32 = 3; +pub(super) const QEMU_ADC_CALIBRATION_EXPECTED_BYTES: [u8; 2] = [0x0C, 0x0A]; +pub(super) const QEMU_ADC_CALIBRATION_PATCH_BYTES: [u8; 2] = [0x1D, 0xF0]; + +pub(super) fn patch_qemu_esp32s3_adc_calibration( + flash_image_path: &Path, + firmware_path: &Path, + elf_path: &Path, + firmware_offset: u64, +) -> Result<()> { + let symbol_addr = resolve_local_elf_symbol_address(elf_path, QEMU_ADC_CALIBRATION_SYMBOL)?; + let patch_addr = symbol_addr + .checked_add(QEMU_ADC_CALIBRATION_PATCH_OFFSET) + .ok_or_else(|| { + fbuild_core::FbuildError::DeployFailed(format!( + "QEMU workaround address overflow for symbol {}", + QEMU_ADC_CALIBRATION_SYMBOL + )) + })?; + let mut firmware_bytes = std::fs::read(firmware_path)?; + let firmware_file_offset = resolve_esp_image_file_offset(&firmware_bytes, patch_addr)?; + patch_bytes( + &mut firmware_bytes, + firmware_file_offset, + &QEMU_ADC_CALIBRATION_EXPECTED_BYTES, + &QEMU_ADC_CALIBRATION_PATCH_BYTES, + )?; + repair_esp_image_checksum_and_hash(&mut firmware_bytes)?; + + let mut flash_image = std::fs::OpenOptions::new() + .write(true) + .open(flash_image_path)?; + flash_image.seek(SeekFrom::Start(firmware_offset))?; + flash_image.write_all(&firmware_bytes)?; + tracing::info!("patched ESP32-S3 QEMU image to skip adc_hw_calibration at 0x{patch_addr:08x}"); + Ok(()) +} + +fn resolve_local_elf_symbol_address(elf_path: &Path, symbol_name: &str) -> Result { + let bytes = std::fs::read(elf_path)?; + let object = object::File::parse(bytes.as_slice()).map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!( + "failed to parse ELF {}: {}", + elf_path.display(), + e + )) + })?; + + let symbol = object + .symbols() + .find(|symbol| symbol.name().ok() == Some(symbol_name)) + .ok_or_else(|| { + fbuild_core::FbuildError::DeployFailed(format!( + "required ELF symbol '{}' not found in {}", + symbol_name, + elf_path.display() + )) + })?; + + u32::try_from(symbol.address()).map_err(|_| { + fbuild_core::FbuildError::DeployFailed(format!( + "ELF symbol '{}' address 0x{:x} does not fit in u32", + symbol_name, + symbol.address() + )) + }) +} + +pub(super) fn resolve_esp_image_file_offset(firmware_bin: &[u8], load_addr: u32) -> Result { + if firmware_bin.len() < ESP_IMAGE_HEADER_LEN { + return Err(fbuild_core::FbuildError::DeployFailed( + "firmware.bin is too small to contain an ESP image header".to_string(), + )); + } + if firmware_bin[0] != ESP_IMAGE_HEADER_MAGIC { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "firmware.bin does not start with ESP image magic 0x{:02x}", + ESP_IMAGE_HEADER_MAGIC + ))); + } + + let segment_count = firmware_bin[1] as usize; + let mut cursor = ESP_IMAGE_HEADER_LEN; + for _ in 0..segment_count { + if cursor + ESP_IMAGE_SEGMENT_HEADER_LEN > firmware_bin.len() { + return Err(fbuild_core::FbuildError::DeployFailed( + "firmware.bin ended before segment header".to_string(), + )); + } + let seg_load_addr = + u32::from_le_bytes(firmware_bin[cursor..cursor + 4].try_into().unwrap()); + let seg_len = + u32::from_le_bytes(firmware_bin[cursor + 4..cursor + 8].try_into().unwrap()) as usize; + let data_start = cursor + ESP_IMAGE_SEGMENT_HEADER_LEN; + let data_end = data_start + seg_len; + if data_end > firmware_bin.len() { + return Err(fbuild_core::FbuildError::DeployFailed( + "firmware.bin ended before segment payload".to_string(), + )); + } + let seg_end_addr = seg_load_addr.checked_add(seg_len as u32).ok_or_else(|| { + fbuild_core::FbuildError::DeployFailed("ESP image segment address overflow".to_string()) + })?; + if (seg_load_addr..seg_end_addr).contains(&load_addr) { + return Ok(data_start + (load_addr - seg_load_addr) as usize); + } + cursor = data_end; + } + + Err(fbuild_core::FbuildError::DeployFailed(format!( + "firmware.bin does not contain a segment covering 0x{load_addr:08x}" + ))) +} + +pub(super) fn patch_bytes( + bytes: &mut [u8], + offset: usize, + expected: &[u8], + replacement: &[u8], +) -> Result<()> { + if expected.len() != replacement.len() { + return Err(fbuild_core::FbuildError::DeployFailed( + "patch replacement length mismatch".to_string(), + )); + } + + let end = offset.checked_add(expected.len()).ok_or_else(|| { + fbuild_core::FbuildError::DeployFailed("patch offset overflow".to_string()) + })?; + if end > bytes.len() { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "patch range 0x{:x}..0x{:x} exceeds image size {}", + offset, + end, + bytes.len() + ))); + } + let actual = &bytes[offset..end]; + if actual != expected { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "QEMU workaround expected bytes {:02x?} at 0x{:x}, found {:02x?}", + expected, offset, actual + ))); + } + bytes[offset..end].copy_from_slice(replacement); + Ok(()) +} + +pub(super) fn repair_esp_image_checksum_and_hash(image: &mut [u8]) -> Result<()> { + if image.len() < ESP_IMAGE_HEADER_LEN { + return Err(fbuild_core::FbuildError::DeployFailed( + "firmware.bin is too small to repair".to_string(), + )); + } + if image[0] != ESP_IMAGE_HEADER_MAGIC { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "firmware.bin does not start with ESP image magic 0x{:02x}", + ESP_IMAGE_HEADER_MAGIC + ))); + } + + let segment_count = image[1] as usize; + let hash_appended = image[23] != 0; + let mut checksum_word = ESP_ROM_CHECKSUM_INITIAL; + let mut cursor = ESP_IMAGE_HEADER_LEN; + for _ in 0..segment_count { + if cursor + ESP_IMAGE_SEGMENT_HEADER_LEN > image.len() { + return Err(fbuild_core::FbuildError::DeployFailed( + "firmware.bin ended before segment header".to_string(), + )); + } + let seg_len = + u32::from_le_bytes(image[cursor + 4..cursor + 8].try_into().unwrap()) as usize; + let data_start = cursor + ESP_IMAGE_SEGMENT_HEADER_LEN; + let data_end = data_start + seg_len; + if data_end > image.len() { + return Err(fbuild_core::FbuildError::DeployFailed( + "firmware.bin ended before segment payload".to_string(), + )); + } + for chunk in image[data_start..data_end].chunks(4) { + let mut word = [0u8; 4]; + word[..chunk.len()].copy_from_slice(chunk); + checksum_word ^= u32::from_le_bytes(word); + } + cursor = data_end; + } + + let checksum_block_len = ((cursor + 1 + 15) & !15) - cursor; + let checksum_offset = cursor + checksum_block_len - 1; + if checksum_offset >= image.len() { + return Err(fbuild_core::FbuildError::DeployFailed( + "firmware.bin ended before checksum byte".to_string(), + )); + } + image[checksum_offset] = + ((checksum_word >> 24) ^ (checksum_word >> 16) ^ (checksum_word >> 8) ^ checksum_word) + as u8; + + if hash_appended { + let hash_start = checksum_offset + 1; + let hash_end = hash_start + ESP_IMAGE_APPENDED_HASH_LEN; + if hash_end > image.len() { + return Err(fbuild_core::FbuildError::DeployFailed( + "firmware.bin ended before appended hash".to_string(), + )); + } + let digest = Sha256::digest(&image[..hash_start]); + image[hash_start..hash_end].copy_from_slice(&digest); + } + Ok(()) +} + +pub(super) fn fill_with_ff(file: &mut std::fs::File, total_size: u64) -> Result<()> { + file.seek(SeekFrom::Start(0))?; + let chunk = vec![0xFFu8; 64 * 1024]; + let mut remaining = total_size; + while remaining > 0 { + let to_write = std::cmp::min(remaining, chunk.len() as u64) as usize; + file.write_all(&chunk[..to_write])?; + remaining -= to_write as u64; + } + Ok(()) +} + +pub(super) fn write_binary_at_offset( + output: &mut std::fs::File, + input_path: &Path, + offset: u64, + flash_size_bytes: u64, +) -> Result<()> { + let metadata = std::fs::metadata(input_path)?; + let end = offset.saturating_add(metadata.len()); + if end > flash_size_bytes { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "artifact {} at offset 0x{:x} exceeds flash image size {}", + input_path.display(), + offset, + flash_size_bytes + ))); + } + + output.seek(SeekFrom::Start(offset))?; + let mut input = std::fs::File::open(input_path)?; + let mut buffer = [0u8; 64 * 1024]; + loop { + let read = input.read(&mut buffer)?; + if read == 0 { + break; + } + output.write_all(&buffer[..read])?; + } + Ok(()) +} diff --git a/crates/fbuild-deploy/src/esp32/mod.rs b/crates/fbuild-deploy/src/esp32/mod.rs new file mode 100644 index 00000000..1d136b81 --- /dev/null +++ b/crates/fbuild-deploy/src/esp32/mod.rs @@ -0,0 +1,34 @@ +//! ESP32 deployer using esptool.py. +//! +//! Flashes firmware to ESP32 boards via serial port using esptool. +//! Bootloader offset varies by MCU: +//! - `0x1000`: esp32, esp32s2 +//! - `0x0`: esp32c2, esp32c3, esp32c5, esp32c6, esp32h2, esp32s3 +//! - `0x2000`: esp32p4 +//! +//! This file is the module entrypoint; the implementation is split +//! across sibling files to keep each one under the LOC gate: +//! +//! * [`deployer`] — [`Esp32Deployer`], [`EsptoolParams`], esptool argv, +//! verify/write paths, and the [`crate::Deployer`] impl. +//! * [`verify`] — [`FlashRegion`], [`RegionVerifyResult`], +//! [`VerifyOutcome`], and the [`parse_verify_regions`] helper. +//! * [`qemu`] — QEMU flash-image assembly and argv builders. +//! * [`image`] — ESP image header constants, byte patching, checksum +//! and SHA-256 trailer repair, and raw binary I/O helpers. +//! * [`parse`] — Hex offset / flash-size string parsers shared across +//! the submodules. + +mod deployer; +mod image; +mod parse; +mod qemu; +#[cfg(test)] +mod tests; +mod verify; + +pub use deployer::{Esp32Deployer, EsptoolParams}; +pub use qemu::{ + build_qemu_args, build_qemu_esp32s3_args, create_qemu_flash_image, resolve_qemu_flash_size_bytes, +}; +pub use verify::{parse_verify_regions, FlashRegion, RegionVerifyResult, VerifyOutcome}; diff --git a/crates/fbuild-deploy/src/esp32/parse.rs b/crates/fbuild-deploy/src/esp32/parse.rs new file mode 100644 index 00000000..f4a92653 --- /dev/null +++ b/crates/fbuild-deploy/src/esp32/parse.rs @@ -0,0 +1,51 @@ +//! Hex offset and flash-size parsers shared across the esp32 module. + +use fbuild_core::Result; + +pub(super) fn parse_hex_offset(raw: &str) -> Result { + let trimmed = raw.trim_start_matches("0x").trim_start_matches("0X"); + u64::from_str_radix(trimmed, 16).map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!("invalid flash offset '{}': {}", raw, e)) + }) +} + +/// Parse a hex flash offset (accepts `0x` prefix) as a `u32`. espflash's +/// `FLASH_MD5SUM` command takes 32-bit offsets, so we narrow the shared +/// [`parse_hex_offset`] result here. Only used by the native verify/write +/// path, so gated with the crate's feature to avoid dead-code lints on +/// default builds. +#[cfg(feature = "espflash-native")] +pub(super) fn parse_hex_offset_u32(raw: &str) -> Result { + let as_u64 = parse_hex_offset(raw)?; + u32::try_from(as_u64).map_err(|_| { + fbuild_core::FbuildError::DeployFailed(format!( + "native verify: flash offset {} does not fit in u32", + raw + )) + }) +} + +pub(super) fn parse_flash_size_bytes(raw: &str) -> Result { + let upper = raw.trim().to_ascii_uppercase(); + if let Some(num) = upper.strip_suffix("MB") { + return num + .trim() + .parse::() + .map(|n| n * 1024 * 1024) + .map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!( + "invalid flash size '{}': {}", + raw, e + )) + }); + } + if let Some(num) = upper.strip_suffix("KB") { + return num.trim().parse::().map(|n| n * 1024).map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!("invalid flash size '{}': {}", raw, e)) + }); + } + Err(fbuild_core::FbuildError::DeployFailed(format!( + "unsupported flash size label '{}'", + raw + ))) +} diff --git a/crates/fbuild-deploy/src/esp32/qemu.rs b/crates/fbuild-deploy/src/esp32/qemu.rs new file mode 100644 index 00000000..885078cc --- /dev/null +++ b/crates/fbuild-deploy/src/esp32/qemu.rs @@ -0,0 +1,149 @@ +//! QEMU flash image assembly and argv builders for ESP32-family emulation. + +use std::path::{Path, PathBuf}; + +use fbuild_core::Result; + +use super::image::{ + fill_with_ff, patch_qemu_esp32s3_adc_calibration, write_binary_at_offset, +}; +use super::parse::{parse_flash_size_bytes, parse_hex_offset}; + +/// Resolve the flash-image size to use for QEMU. +/// +/// ESP32 QEMU supports 2MB, 4MB, 8MB, and 16MB flash images. We derive +/// the size from the board's `maximum_size` when present, otherwise fall back +/// to the MCU config's default flash size label. +pub fn resolve_qemu_flash_size_bytes( + board: &fbuild_config::BoardConfig, + default_flash_size: &str, +) -> Result { + let size_bytes = match board.max_flash { + Some(bytes) => bytes, + None => parse_flash_size_bytes(default_flash_size)?, + }; + if matches!(size_bytes, 2_097_152 | 4_194_304 | 8_388_608 | 16_777_216) { + Ok(size_bytes) + } else { + Err(fbuild_core::FbuildError::DeployFailed(format!( + "ESP32 QEMU supports only 2MB, 4MB, 8MB, or 16MB flash images; got {} bytes", + size_bytes + ))) + } +} + +/// Create a merged raw flash image for ESP32 QEMU from bootloader, +/// partitions, and application firmware. +/// +/// When `elf_path` is `Some`, the ESP32-S3 ADC calibration patch is applied. +/// Pass `None` for non-S3 variants to skip the patch. +pub fn create_qemu_flash_image( + firmware_path: &Path, + output_path: &Path, + flash_size_bytes: u64, + bootloader_offset: &str, + partitions_offset: &str, + firmware_offset: &str, + elf_path: Option<&Path>, +) -> Result { + let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); + let bootloader_path = build_dir.join("bootloader.bin"); + let boot_app0_path = build_dir.join("boot_app0.bin"); + let partitions_path = build_dir.join("partitions.bin"); + let firmware_offset = parse_hex_offset(firmware_offset)?; + + for required in [&bootloader_path, &partitions_path, firmware_path] { + if !required.is_file() { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "required QEMU artifact not found: {}", + required.display() + ))); + } + } + + if let Some(parent) = output_path.parent() { + std::fs::create_dir_all(parent)?; + } + + let mut output = std::fs::File::create(output_path)?; + fill_with_ff(&mut output, flash_size_bytes)?; + + write_binary_at_offset( + &mut output, + &bootloader_path, + parse_hex_offset(bootloader_offset)?, + flash_size_bytes, + )?; + write_binary_at_offset( + &mut output, + &partitions_path, + parse_hex_offset(partitions_offset)?, + flash_size_bytes, + )?; + if boot_app0_path.is_file() { + write_binary_at_offset(&mut output, &boot_app0_path, 0xE000, flash_size_bytes)?; + } + write_binary_at_offset( + &mut output, + firmware_path, + firmware_offset, + flash_size_bytes, + )?; + if let Some(elf_path) = elf_path { + patch_qemu_esp32s3_adc_calibration(output_path, firmware_path, elf_path, firmware_offset)?; + } + + Ok(output_path.to_path_buf()) +} + +/// Build the QEMU argv for ESP32-family emulation. +/// +/// The `mcu` parameter selects the QEMU machine type and watchdog timer +/// driver name. Supported values: `esp32`, `esp32s3` (via +/// `qemu-system-xtensa`) and `esp32c3`, `esp32c6`, `esp32h2` (via +/// `qemu-system-riscv32`). Callers are responsible for launching the +/// matching QEMU binary; this function only emits the argv. +pub fn build_qemu_args( + mcu: &str, + flash_image: &Path, + psram: Option, +) -> Vec { + let machine = mcu.to_lowercase(); + let mut args = vec![ + "-nographic".to_string(), + "-machine".to_string(), + machine.clone(), + ]; + if let Some(psram) = psram { + args.push("-m".to_string()); + args.push(format!("{}M", psram.size_mib)); + } + args.extend([ + "-drive".to_string(), + format!("file={},if=mtd,format=raw", flash_image.display()), + "-serial".to_string(), + "mon:stdio".to_string(), + "-monitor".to_string(), + "none".to_string(), + "-global".to_string(), + format!( + "driver=timer.{}.timg,property=wdt_disable,value=true", + machine + ), + ]); + if let Some(psram) = psram { + if psram.is_octal { + args.push("-global".to_string()); + args.push("driver=ssi_psram,property=is_octal,value=true".to_string()); + } + } + args +} + +/// Build the QEMU argv for ESP32-S3 emulation (convenience wrapper). +pub fn build_qemu_esp32s3_args( + flash_image: &Path, + psram: Option, +) -> Vec { + build_qemu_args("esp32s3", flash_image, psram) +} diff --git a/crates/fbuild-deploy/src/esp32/tests.rs b/crates/fbuild-deploy/src/esp32/tests.rs new file mode 100644 index 00000000..9d9852ff --- /dev/null +++ b/crates/fbuild-deploy/src/esp32/tests.rs @@ -0,0 +1,766 @@ +//! Unit and hardware-gated integration tests for the esp32 module. + +#![cfg(test)] + +use std::path::Path; + +use sha2::{Digest, Sha256}; + +use super::deployer::{Esp32Deployer, EsptoolParams}; +use super::image::{ + repair_esp_image_checksum_and_hash, resolve_esp_image_file_offset, ESP_IMAGE_APPENDED_HASH_LEN, + ESP_IMAGE_HEADER_LEN, ESP_IMAGE_HEADER_MAGIC, ESP_IMAGE_SEGMENT_HEADER_LEN, + ESP_ROM_CHECKSUM_INITIAL, QEMU_ADC_CALIBRATION_EXPECTED_BYTES, QEMU_ADC_CALIBRATION_PATCH_BYTES, +}; +use super::image::patch_bytes; +use super::qemu::{ + build_qemu_args, build_qemu_esp32s3_args, create_qemu_flash_image, resolve_qemu_flash_size_bytes, +}; +use super::verify::{ + parse_verify_regions, FlashRegion, RegionVerifyResult, VerifyOutcome, +}; +use crate::Deployer; + +/// Test params matching ESP32-C6 JSON config values. +fn test_esptool_params() -> EsptoolParams { + EsptoolParams { + flash_mode: "dio".to_string(), + flash_freq: "80m".to_string(), + default_baud: "460800".to_string(), + before_reset: "default-reset".to_string(), + after_reset: "hard-reset".to_string(), + } +} + +#[test] +fn test_esp32_deployer_creation() { + let params = test_esptool_params(); + let deployer = Esp32Deployer::new( + "esp32c6", "460800", "0x0", "0x8000", "0x10000", ¶ms, false, + ); + assert_eq!(deployer.chip, "esp32c6"); + assert_eq!(deployer.baud_rate, "460800"); + assert_eq!(deployer.bootloader_offset, "0x0"); + assert_eq!(deployer.firmware_offset, "0x10000"); + assert_eq!(deployer.flash_mode, "dio"); + assert_eq!(deployer.before_reset, "default-reset"); +} + +#[test] +fn qemu_flash_size_resolution_accepts_supported_sizes() { + let mut board = + fbuild_config::BoardConfig::from_board_id("esp32-s3-devkitc-1", &Default::default()) + .unwrap(); + board.max_flash = Some(8 * 1024 * 1024); + assert_eq!( + resolve_qemu_flash_size_bytes(&board, "4MB").unwrap(), + 8 * 1024 * 1024 + ); +} + +#[test] +fn qemu_flash_size_resolution_rejects_unsupported_size() { + let mut board = + fbuild_config::BoardConfig::from_board_id("esp32-s3-devkitc-1", &Default::default()) + .unwrap(); + board.max_flash = Some(32 * 1024 * 1024); + let err = resolve_qemu_flash_size_bytes(&board, "4MB").unwrap_err(); + assert!(err + .to_string() + .contains("supports only 2MB, 4MB, 8MB, or 16MB")); +} + +#[test] +fn create_qemu_flash_image_writes_regions_at_offsets() { + let tmp = tempfile::TempDir::new().unwrap(); + let build_dir = tmp.path().join("build"); + std::fs::create_dir_all(&build_dir).unwrap(); + + let boot = build_dir.join("bootloader.bin"); + let parts = build_dir.join("partitions.bin"); + let fw = build_dir.join("firmware.bin"); + std::fs::write(&boot, b"BOOT").unwrap(); + std::fs::write(&parts, b"PART").unwrap(); + std::fs::write(&fw, b"FIRM").unwrap(); + + let flash = tmp.path().join("qemu_flash.bin"); + create_qemu_flash_image( + &fw, + &flash, + 2 * 1024 * 1024, + "0x0", + "0x8000", + "0x10000", + None, + ) + .unwrap(); + + let bytes = std::fs::read(&flash).unwrap(); + assert_eq!(&bytes[0..4], b"BOOT"); + assert_eq!(&bytes[0x8000..0x8004], b"PART"); + assert_eq!(&bytes[0x10000..0x10004], b"FIRM"); + assert_eq!(bytes.len(), 2 * 1024 * 1024); + assert_eq!(bytes[0x200], 0xFF); +} + +#[test] +fn create_qemu_flash_image_includes_boot_app0_when_present() { + let tmp = tempfile::TempDir::new().unwrap(); + let build_dir = tmp.path().join("build"); + std::fs::create_dir_all(&build_dir).unwrap(); + + let boot = build_dir.join("bootloader.bin"); + let boot_app0 = build_dir.join("boot_app0.bin"); + let parts = build_dir.join("partitions.bin"); + let fw = build_dir.join("firmware.bin"); + std::fs::write(&boot, b"BOOT").unwrap(); + std::fs::write(&boot_app0, b"APP0").unwrap(); + std::fs::write(&parts, b"PART").unwrap(); + std::fs::write(&fw, b"FIRM").unwrap(); + + let flash = tmp.path().join("qemu_flash.bin"); + create_qemu_flash_image( + &fw, + &flash, + 2 * 1024 * 1024, + "0x0", + "0x8000", + "0x10000", + None, + ) + .unwrap(); + + let bytes = std::fs::read(&flash).unwrap(); + assert_eq!(&bytes[0xE000..0xE004], b"APP0"); +} + +#[test] +fn resolve_esp_image_file_offset_maps_address_into_segment_data() { + let mut image = vec![0u8; ESP_IMAGE_HEADER_LEN]; + image[0] = ESP_IMAGE_HEADER_MAGIC; + image[1] = 1; + image.extend_from_slice(&0x4200_0000u32.to_le_bytes()); + image.extend_from_slice(&6u32.to_le_bytes()); + image.extend_from_slice(&[0x11, 0x22, 0x33, 0x44, 0x55, 0x66]); + + let offset = resolve_esp_image_file_offset(&image, 0x4200_0003).unwrap(); + assert_eq!( + offset, + ESP_IMAGE_HEADER_LEN + ESP_IMAGE_SEGMENT_HEADER_LEN + 3 + ); +} + +#[test] +fn patch_bytes_rewrites_expected_bytes_only() { + let mut flash = [0xFFu8; 16]; + flash[6..8].copy_from_slice(&QEMU_ADC_CALIBRATION_EXPECTED_BYTES); + + patch_bytes( + &mut flash, + 6, + &QEMU_ADC_CALIBRATION_EXPECTED_BYTES, + &QEMU_ADC_CALIBRATION_PATCH_BYTES, + ) + .unwrap(); + + assert_eq!(&flash[6..8], &QEMU_ADC_CALIBRATION_PATCH_BYTES); +} + +#[test] +fn repair_esp_image_checksum_and_hash_updates_trailers_after_patch() { + let mut image = vec![0u8; ESP_IMAGE_HEADER_LEN]; + image[0] = ESP_IMAGE_HEADER_MAGIC; + image[1] = 1; + image[23] = 1; + image.extend_from_slice(&0x4200_0000u32.to_le_bytes()); + image.extend_from_slice(&8u32.to_le_bytes()); + image.extend_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); + image.extend_from_slice(&[0u8; 16]); + image.extend_from_slice(&[0u8; ESP_IMAGE_APPENDED_HASH_LEN]); + + patch_bytes( + &mut image, + ESP_IMAGE_HEADER_LEN + ESP_IMAGE_SEGMENT_HEADER_LEN + 3, + &[4], + &[9], + ) + .unwrap(); + repair_esp_image_checksum_and_hash(&mut image).unwrap(); + + let checksum_offset = + ((ESP_IMAGE_HEADER_LEN + ESP_IMAGE_SEGMENT_HEADER_LEN + 8 + 1 + 15) & !15) - 1; + let expected_checksum = { + let mut checksum_word = ESP_ROM_CHECKSUM_INITIAL; + for chunk in image[ESP_IMAGE_HEADER_LEN + ESP_IMAGE_SEGMENT_HEADER_LEN + ..ESP_IMAGE_HEADER_LEN + ESP_IMAGE_SEGMENT_HEADER_LEN + 8] + .chunks(4) + { + let mut word = [0u8; 4]; + word[..chunk.len()].copy_from_slice(chunk); + checksum_word ^= u32::from_le_bytes(word); + } + ((checksum_word >> 24) ^ (checksum_word >> 16) ^ (checksum_word >> 8) ^ checksum_word) + as u8 + }; + let expected_hash = Sha256::digest(&image[..checksum_offset + 1]); + assert_eq!(image[checksum_offset], expected_checksum); + assert_eq!( + &image[checksum_offset + 1..checksum_offset + 1 + ESP_IMAGE_APPENDED_HASH_LEN], + expected_hash.as_slice() + ); +} + +#[test] +fn qemu_command_builder_uses_expected_machine_and_watchdog_override() { + let args = build_qemu_esp32s3_args(Path::new("flash.bin"), None); + assert!(args.contains(&"esp32s3".to_string())); + assert!(args + .iter() + .any(|arg| arg == "driver=timer.esp32s3.timg,property=wdt_disable,value=true")); + assert!(args + .iter() + .any(|arg| arg.contains("file=flash.bin,if=mtd,format=raw"))); +} + +#[test] +fn qemu_command_builder_uses_esp32_machine_for_base_variant() { + let args = build_qemu_args("esp32", Path::new("flash.bin"), None); + assert!(args.contains(&"esp32".to_string())); + assert!(args + .iter() + .any(|arg| arg == "driver=timer.esp32.timg,property=wdt_disable,value=true")); + assert!(args + .iter() + .any(|arg| arg.contains("file=flash.bin,if=mtd,format=raw"))); +} + +#[test] +fn qemu_command_builder_adds_psram_args_when_requested() { + let args = build_qemu_esp32s3_args( + Path::new("flash.bin"), + Some(fbuild_config::Esp32QemuPsramConfig { + size_mib: 8, + is_octal: true, + }), + ); + assert!(args.windows(2).any(|pair| pair == ["-m", "8M"])); + assert!(args + .iter() + .any(|arg| arg == "driver=ssi_psram,property=is_octal,value=true")); +} + +#[test] +fn test_esp32_deployer_from_board_config() { + let board = + fbuild_config::BoardConfig::from_board_id("esp32c6", &std::collections::HashMap::new()) + .unwrap(); + let params = test_esptool_params(); + let deployer = + Esp32Deployer::from_board_config(&board, "0x0", "0x8000", "0x10000", ¶ms, false); + assert_eq!(deployer.chip, "esp32c6"); + assert_eq!(deployer.bootloader_offset, "0x0"); +} + +#[test] +fn test_deploy_requires_port() { + let params = test_esptool_params(); + let deployer = Esp32Deployer::new( + "esp32c6", "460800", "0x0", "0x8000", "0x10000", ¶ms, false, + ); + let tmp = tempfile::TempDir::new().unwrap(); + let result = deployer.deploy(tmp.path(), "esp32c6", Path::new("firmware.bin"), None); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("serial port required")); +} + +/// Fast deploy: the verify-flash command line must include the +/// `verify-flash` subcommand and pair every flash region with its +/// matching offset, in the order bootloader, partitions, firmware. +/// Verifying all three in a single esptool call amortises the +/// ~3-second stub flasher upload. +#[test] +fn build_verify_flash_args_includes_all_three_regions_when_present() { + let params = test_esptool_params(); + let deployer = Esp32Deployer::new( + "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, + ); + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); + std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firm").unwrap(); + + let args = deployer.build_verify_flash_args(&fw, "COM13"); + + // Subcommand + assert!( + args.contains(&"verify-flash".to_string()), + "missing verify-flash subcommand: {:?}", + args + ); + // Chip + port + assert!(args.contains(&"--chip".to_string())); + assert!(args.contains(&"esp32s3".to_string())); + assert!(args.contains(&"--port".to_string())); + assert!(args.contains(&"COM13".to_string())); + + // All three (offset, file) pairs in the right order. + let pos_verify = args.iter().position(|a| a == "verify-flash").unwrap(); + let pos_boot = args.iter().position(|a| a == "0x0").unwrap(); + let pos_parts = args.iter().position(|a| a == "0x8000").unwrap(); + let pos_fw = args.iter().position(|a| a == "0x10000").unwrap(); + assert!( + pos_verify < pos_boot && pos_boot < pos_parts && pos_parts < pos_fw, + "regions must appear after verify-flash in bootloader→partitions→firmware order: {:?}", + args + ); + + // verify-flash MUST NOT carry --flash-mode/freq/size flags; + // those are write-flash options and esptool 5.x rejects them + // here. We were burned by this when we copied the deploy() + // argument layout wholesale. + assert!( + !args.contains(&"--flash-mode".to_string()), + "verify-flash must not include --flash-mode (write-flash only): {:?}", + args + ); + assert!( + !args.contains(&"--flash-freq".to_string()), + "verify-flash must not include --flash-freq (write-flash only): {:?}", + args + ); +} + +#[test] +fn build_verify_flash_args_skips_missing_bootloader_and_partitions() { + // When bootloader.bin / partitions.bin haven't been built (e.g. + // an upload-only test fixture), verify must still cover firmware + // alone. Otherwise we'd skip the only thing we have. + let params = test_esptool_params(); + let deployer = Esp32Deployer::new( + "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, + ); + let tmp = tempfile::TempDir::new().unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firm").unwrap(); + + let args = deployer.build_verify_flash_args(&fw, "COM13"); + + // No bootloader or partitions paths in the args. + assert!(!args.iter().any(|a| a.ends_with("bootloader.bin"))); + assert!(!args.iter().any(|a| a.ends_with("partitions.bin"))); + // Firmware offset is still present. + assert!(args.contains(&"0x10000".to_string())); + assert!(args.iter().any(|a| a.ends_with("firmware.bin"))); +} + +/// esptool 5.x emits one `Verifying ... at 0x{addr:#010x} ...` line +/// followed by `Verification successful/failed` for each region. +/// Parser must pair them and classify the region by offset. +#[test] +fn parse_verify_regions_classifies_each_region_by_offset() { + let stdout = "\ +Verifying 0x6060 (24672) bytes at 0x00000000 in flash against 'bootloader.bin'...\n\ +Verification successful (digest matched).\n\ +Verifying 0xc00 (3072) bytes at 0x00008000 in flash against 'partitions.bin'...\n\ +Verification successful (digest matched).\n\ +Verifying 0x260a60 (2493536) bytes at 0x00010000 in flash against 'firmware.bin'...\n\ +Verification failed (digest mismatch).\n"; + let regions = parse_verify_regions(stdout, "0x0", "0x8000", "0x10000"); + assert_eq!( + regions, + vec![ + RegionVerifyResult { + region: FlashRegion::Bootloader, + matched: true + }, + RegionVerifyResult { + region: FlashRegion::Partitions, + matched: true + }, + RegionVerifyResult { + region: FlashRegion::Firmware, + matched: false + }, + ] + ); +} + +/// When esptool output doesn't match the expected pattern (older +/// version, localized output, truncated log), parse returns an empty +/// vec so the daemon falls back to flashing all regions. +#[test] +fn parse_verify_regions_returns_empty_on_unknown_format() { + let stdout = "some unrelated failure output\nexit 1\n"; + let regions = parse_verify_regions(stdout, "0x0", "0x8000", "0x10000"); + assert!(regions.is_empty()); +} + +/// Regions whose offset doesn't match any of the three knowns are +/// silently skipped — we stay conservative and return only what we +/// understand. +#[test] +fn parse_verify_regions_skips_unknown_offsets() { + let stdout = "\ +Verifying 0x1000 (4096) bytes at 0x00001000 in flash against 'bootloader.bin'...\n\ +Verification failed (digest mismatch).\n\ +Verifying 0x1000 (4096) bytes at 0x00010000 in flash against 'firmware.bin'...\n\ +Verification successful (digest matched).\n"; + let regions = parse_verify_regions(stdout, "0x1000", "0x8000", "0x10000"); + assert_eq!( + regions, + vec![ + RegionVerifyResult { + region: FlashRegion::Bootloader, + matched: false + }, + RegionVerifyResult { + region: FlashRegion::Firmware, + matched: true + }, + ] + ); +} + +/// The selective write-flash argv must include the write-flash +/// subcommand and only the requested region's offset/file pair. +/// Skipping bootloader + partitions is the ~1s save targeted by #67. +#[test] +fn build_write_flash_args_firmware_only_skips_bootloader_and_partitions() { + let params = test_esptool_params(); + let deployer = Esp32Deployer::new( + "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, + ); + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); + std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firm").unwrap(); + + let args = deployer.build_write_flash_args(&fw, "COM13", Some(&[FlashRegion::Firmware])); + + assert!(args.contains(&"write-flash".to_string())); + assert!(!args.iter().any(|a| a.ends_with("bootloader.bin"))); + assert!(!args.iter().any(|a| a.ends_with("partitions.bin"))); + assert!(args.contains(&"0x10000".to_string())); + assert!(args.iter().any(|a| a.ends_with("firmware.bin"))); + assert!(!args.contains(&"0x8000".to_string())); +} + +/// `None` regions (default deploy) must still include all three +/// present files — we can't regress the baseline path. +#[test] +fn build_write_flash_args_default_includes_all_regions() { + let params = test_esptool_params(); + let deployer = Esp32Deployer::new( + "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, + ); + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); + std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firm").unwrap(); + + let args = deployer.build_write_flash_args(&fw, "COM13", None); + assert!(args.contains(&"0x0".to_string())); + assert!(args.contains(&"0x8000".to_string())); + assert!(args.contains(&"0x10000".to_string())); +} + +/// If a caller requests a region whose file is missing on disk, fail +/// with a clear error rather than silently emitting a write-flash +/// call with no offset/file pair (which would produce an opaque +/// esptool usage error). Addresses CodeRabbit review on PR #71. +#[test] +fn deploy_regions_errors_when_requested_region_file_missing() { + let params = test_esptool_params(); + let deployer = Esp32Deployer::new( + "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, + ); + let tmp = tempfile::TempDir::new().unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firm").unwrap(); + // Note: no bootloader.bin written. + let err = deployer + .deploy_regions(&fw, "COM13", &[FlashRegion::Bootloader]) + .unwrap_err(); + assert!( + err.to_string().contains("bootloader.bin"), + "error must name the missing file: {}", + err + ); +} + +/// Empty region slice -> usage error; we surface it rather than let +/// esptool barf. +#[test] +fn deploy_regions_rejects_empty_slice() { + let params = test_esptool_params(); + let deployer = Esp32Deployer::new( + "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, + ); + let err = deployer + .deploy_regions(Path::new("firmware.bin"), "COM13", &[]) + .unwrap_err(); + assert!(err.to_string().contains("no regions")); +} + +#[test] +fn verify_outcome_is_match_helper() { + let m = VerifyOutcome::Match { + stdout: "ok".into(), + stderr: String::new(), + }; + let mm = VerifyOutcome::Mismatch { + stdout: String::new(), + stderr: "Verification failed".into(), + regions: Vec::new(), + }; + assert!(m.is_match()); + assert!(!mm.is_match()); +} + +// --------------------------------------------------------------- +// Hardware-gated verify-deployment tests for each ESP32 family MCU. +// +// These tests are `#[ignore]` so they never run in CI. To exercise +// them on a local bench, set the env vars described below and run: +// +// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real -- --ignored --nocapture +// +// Each test reads **two** environment variables: +// +// _PORT – serial port the board is attached to (e.g. COM13, /dev/ttyUSB0) +// _FIRMWARE – absolute path to a pre-flashed firmware.bin +// +// where is one of ESP32, ESP32S2, ESP32S3, ESP32C2, ESP32C3, +// ESP32C6, ESP32H2, ESP32P4. +// +// The firmware directory must also contain `bootloader.bin` and +// `partitions.bin` so that verify-flash can check all three regions +// in a single esptool invocation. +// +// Bootloader offsets per chip (from esp32.rs header comment): +// 0x1000 – esp32, esp32s2 +// 0x0 – esp32c2, esp32c3, esp32c5, esp32c6, esp32h2, esp32s3 +// 0x2000 – esp32p4 +// --------------------------------------------------------------- + +/// Shared implementation for all per-chip hardware-gated verify tests. +/// +/// 1. Reads `{port_env}` and `{firmware_env}` from the environment. +/// 2. Asserts that verify against the pre-flashed image returns `Match` +/// in under 15 seconds. +/// 3. Asserts that a tampered image (1 byte flipped) returns `Mismatch`. +fn run_verify_deployment_test( + chip: &str, + bootloader_offset: &str, + port_env: &str, + firmware_env: &str, +) { + let port = std::env::var(port_env).unwrap_or_else(|_| { + panic!( + "set {} to the serial port your {} board is attached to (e.g. COM13)", + port_env, chip + ) + }); + let firmware_path = std::env::var(firmware_env).unwrap_or_else(|_| { + panic!( + "set {} to the absolute path of the pre-flashed firmware.bin for {}", + firmware_env, chip + ) + }); + let reference = std::path::PathBuf::from(&firmware_path); + assert!( + reference.is_file(), + "reference firmware not found at {}; build and flash it first", + reference.display() + ); + let ref_dir = reference + .parent() + .unwrap_or_else(|| std::path::Path::new(".")); + for name in ["bootloader.bin", "partitions.bin"] { + let artifact = ref_dir.join(name); + assert!( + artifact.is_file(), + "[{}] missing {} next to {}; otherwise this only verifies firmware.bin", + chip, + name, + reference.display() + ); + } + + let params = EsptoolParams { + flash_mode: "dio".to_string(), + flash_freq: "80m".to_string(), + default_baud: "921600".to_string(), + before_reset: "default-reset".to_string(), + after_reset: "hard-reset".to_string(), + }; + let deployer = Esp32Deployer::new( + chip, + "921600", + bootloader_offset, + "0x8000", + "0x10000", + ¶ms, + true, + ); + + // Phase 1: matching image -> Match + let start = std::time::Instant::now(); + let outcome = deployer + .try_verify_deployment(&reference, &port) + .unwrap_or_else(|e| panic!("verify must not fail against attached {}: {}", chip, e)); + let elapsed = start.elapsed(); + assert!( + outcome.is_match(), + "[{}] expected Match against pre-flashed firmware; got {:?}", + chip, + outcome + ); + assert!( + elapsed < std::time::Duration::from_secs(15), + "[{}] verify took {:?} -- should complete in <15s", + chip, + elapsed + ); + eprintln!("[{}] verify (Match) elapsed: {:?}", chip, elapsed); + + // Phase 2: tampered image -> Mismatch + let tmp = tempfile::TempDir::new().unwrap(); + // Copy bootloader and partitions next to the tampered firmware so + // build_verify_flash_args picks them up alongside firmware.bin. + for name in ["bootloader.bin", "partitions.bin"] { + std::fs::copy(ref_dir.join(name), tmp.path().join(name)).unwrap(); + } + let tampered = tmp.path().join("firmware.bin"); + let mut bytes = std::fs::read(&reference).unwrap(); + // Flip a byte well past the image header to avoid invalidating + // the ESP-IDF magic and triggering an esptool parse error rather + // than a clean digest mismatch. + let target = bytes.len() / 2; + bytes[target] ^= 0x55; + std::fs::write(&tampered, &bytes).unwrap(); + + let outcome = deployer + .try_verify_deployment(&tampered, &port) + .unwrap_or_else(|e| { + panic!( + "[{}] verify must not fail with tampered firmware: {}", + chip, e + ) + }); + assert!( + !outcome.is_match(), + "[{}] expected Mismatch for tampered firmware; got {:?}", + chip, + outcome + ); + eprintln!("[{}] verify (Mismatch) detected correctly", chip); +} + +/// ESP32 (Xtensa, bootloader at 0x1000). +/// +/// ```text +/// ESP32_PORT=COM5 ESP32_FIRMWARE=C:\path\to\firmware.bin \ +/// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32 -- --ignored --nocapture +/// ``` +#[test] +#[ignore = "requires real ESP32 board — set ESP32_PORT and ESP32_FIRMWARE"] +fn try_verify_deployment_real_esp32() { + run_verify_deployment_test("esp32", "0x1000", "ESP32_PORT", "ESP32_FIRMWARE"); +} + +/// ESP32-S2 (Xtensa single-core, bootloader at 0x1000). +/// +/// ```text +/// ESP32S2_PORT=COM6 ESP32S2_FIRMWARE=C:\path\to\firmware.bin \ +/// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32s2 -- --ignored --nocapture +/// ``` +#[test] +#[ignore = "requires real ESP32-S2 board — set ESP32S2_PORT and ESP32S2_FIRMWARE"] +fn try_verify_deployment_real_esp32s2() { + run_verify_deployment_test("esp32s2", "0x1000", "ESP32S2_PORT", "ESP32S2_FIRMWARE"); +} + +/// ESP32-S3 (Xtensa dual-core, bootloader at 0x0). +/// +/// This is the original baseline test, now using env-var configuration +/// consistent with the rest of the family. +/// +/// ```text +/// ESP32S3_PORT=COM13 ESP32S3_FIRMWARE=C:\Users\niteris\dev\fastled\.pio\build\esp32s3\firmware.bin \ +/// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32s3 -- --ignored --nocapture +/// ``` +#[test] +#[ignore = "requires real ESP32-S3 board — set ESP32S3_PORT and ESP32S3_FIRMWARE"] +fn try_verify_deployment_real_esp32s3() { + run_verify_deployment_test("esp32s3", "0x0", "ESP32S3_PORT", "ESP32S3_FIRMWARE"); +} + +/// ESP32-C2 (RISC-V single-core, bootloader at 0x0). +/// +/// ```text +/// ESP32C2_PORT=COM7 ESP32C2_FIRMWARE=C:\path\to\firmware.bin \ +/// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32c2 -- --ignored --nocapture +/// ``` +#[test] +#[ignore = "requires real ESP32-C2 board — set ESP32C2_PORT and ESP32C2_FIRMWARE"] +fn try_verify_deployment_real_esp32c2() { + run_verify_deployment_test("esp32c2", "0x0", "ESP32C2_PORT", "ESP32C2_FIRMWARE"); +} + +/// ESP32-C3 (RISC-V single-core, bootloader at 0x0). +/// +/// ```text +/// ESP32C3_PORT=COM8 ESP32C3_FIRMWARE=C:\path\to\firmware.bin \ +/// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32c3 -- --ignored --nocapture +/// ``` +#[test] +#[ignore = "requires real ESP32-C3 board — set ESP32C3_PORT and ESP32C3_FIRMWARE"] +fn try_verify_deployment_real_esp32c3() { + run_verify_deployment_test("esp32c3", "0x0", "ESP32C3_PORT", "ESP32C3_FIRMWARE"); +} + +/// ESP32-C6 (RISC-V single-core, bootloader at 0x0). +/// +/// ```text +/// ESP32C6_PORT=COM9 ESP32C6_FIRMWARE=C:\path\to\firmware.bin \ +/// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32c6 -- --ignored --nocapture +/// ``` +#[test] +#[ignore = "requires real ESP32-C6 board — set ESP32C6_PORT and ESP32C6_FIRMWARE"] +fn try_verify_deployment_real_esp32c6() { + run_verify_deployment_test("esp32c6", "0x0", "ESP32C6_PORT", "ESP32C6_FIRMWARE"); +} + +/// ESP32-H2 (RISC-V single-core, bootloader at 0x0). +/// +/// ```text +/// ESP32H2_PORT=COM10 ESP32H2_FIRMWARE=C:\path\to\firmware.bin \ +/// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32h2 -- --ignored --nocapture +/// ``` +#[test] +#[ignore = "requires real ESP32-H2 board — set ESP32H2_PORT and ESP32H2_FIRMWARE"] +fn try_verify_deployment_real_esp32h2() { + run_verify_deployment_test("esp32h2", "0x0", "ESP32H2_PORT", "ESP32H2_FIRMWARE"); +} + +/// ESP32-P4 (RISC-V dual-core, OPI flash, bootloader at 0x2000). +/// +/// Note: ESP32-P4 uses OPI flash and has a different bootloader offset +/// (0x2000) compared to other ESP32 chips. +/// +/// ```text +/// ESP32P4_PORT=COM11 ESP32P4_FIRMWARE=C:\path\to\firmware.bin \ +/// uv run soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32p4 -- --ignored --nocapture +/// ``` +#[test] +#[ignore = "requires real ESP32-P4 board — set ESP32P4_PORT and ESP32P4_FIRMWARE"] +fn try_verify_deployment_real_esp32p4() { + run_verify_deployment_test("esp32p4", "0x2000", "ESP32P4_PORT", "ESP32P4_FIRMWARE"); +} diff --git a/crates/fbuild-deploy/src/esp32/verify.rs b/crates/fbuild-deploy/src/esp32/verify.rs new file mode 100644 index 00000000..02b7417d --- /dev/null +++ b/crates/fbuild-deploy/src/esp32/verify.rs @@ -0,0 +1,120 @@ +//! Verify-flash region types and stdout parser. + +use super::parse::parse_hex_offset; + +/// Which of the three logical flash regions a verify/write targets. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FlashRegion { + Bootloader, + Partitions, + Firmware, +} + +/// Per-region outcome parsed from `esptool verify-flash` stdout. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RegionVerifyResult { + pub region: FlashRegion, + pub matched: bool, +} + +/// Result of a `try_verify_deployment` call. +#[derive(Debug, Clone)] +pub enum VerifyOutcome { + /// All flashed regions match the candidate image; flashing would be + /// a no-op. The device has been hard-reset by esptool's + /// `--after hard-reset` so it's already running the requested image. + Match { stdout: String, stderr: String }, + /// At least one region differs from the local files. `regions` carries + /// the parsed per-region verdict when stdout was understood; empty + /// when parsing failed and the caller must flash everything. + Mismatch { + stdout: String, + stderr: String, + regions: Vec, + }, +} + +impl VerifyOutcome { + /// Convenience: returns `true` only for `Match`. + pub fn is_match(&self) -> bool { + matches!(self, VerifyOutcome::Match { .. }) + } +} + +/// Parse `esptool verify-flash` stdout into per-region results. +/// +/// esptool 5.x emits, for each region: +/// +/// ```text +/// Verifying 0x6060 (24672) bytes at 0x00000000 in flash against 'bootloader.bin'... +/// Verification successful (digest matched). +/// ``` +/// +/// or, on mismatch, `Verification failed (digest mismatch).`. We map the +/// `at 0x{addr:#010x}` line to one of the three known offsets, then read +/// the next non-blank line as the verdict. Unknown addresses are skipped. +/// Returns an empty `Vec` when nothing could be parsed — callers must then +/// fall back to flashing all regions. +pub fn parse_verify_regions( + stdout: &str, + bootloader_offset: &str, + partitions_offset: &str, + firmware_offset: &str, +) -> Vec { + let boot_addr = parse_hex_offset(bootloader_offset).ok(); + let parts_addr = parse_hex_offset(partitions_offset).ok(); + let fw_addr = parse_hex_offset(firmware_offset).ok(); + + let mut results: Vec = Vec::new(); + let mut pending: Option = None; + + for raw in stdout.lines() { + let line = raw.trim(); + if let Some(addr) = extract_verifying_at_address(line) { + pending = if Some(addr) == boot_addr { + Some(FlashRegion::Bootloader) + } else if Some(addr) == parts_addr { + Some(FlashRegion::Partitions) + } else if Some(addr) == fw_addr { + Some(FlashRegion::Firmware) + } else { + None + }; + continue; + } + if let Some(region) = pending { + if line.starts_with("Verification successful") { + if !results.iter().any(|r| r.region == region) { + results.push(RegionVerifyResult { + region, + matched: true, + }); + } + pending = None; + } else if line.starts_with("Verification failed") { + if !results.iter().any(|r| r.region == region) { + results.push(RegionVerifyResult { + region, + matched: false, + }); + } + pending = None; + } + } + } + results +} + +/// Extract the `0x...` address from an esptool `Verifying ... at 0xNNNNNNNN in flash ...` line. +fn extract_verifying_at_address(line: &str) -> Option { + if !line.starts_with("Verifying ") { + return None; + } + let marker = " at 0x"; + let start = line.find(marker)? + marker.len(); + let tail = &line[start..]; + let end = tail + .find(|c: char| !c.is_ascii_hexdigit()) + .unwrap_or(tail.len()); + u64::from_str_radix(&tail[..end], 16).ok() +} diff --git a/crates/fbuild-deploy/src/esp32_native.rs b/crates/fbuild-deploy/src/esp32_native.rs deleted file mode 100644 index b9fed9d0..00000000 --- a/crates/fbuild-deploy/src/esp32_native.rs +++ /dev/null @@ -1,1042 +0,0 @@ -//! Native ESP32 `verify-flash` **and** `write-flash` implementations -//! backed by the [`espflash`] crate. Alternatives to the default -//! [`crate::esp32::Esp32Deployer`] path, which shells out to Python -//! `esptool`. -//! -//! # Why (issue #66) -//! -//! `esptool.py` spends ~1 s on Python interpreter startup plus another -//! ~0.5 s on subprocess/stub-flasher handshake before it even issues the -//! first real command. Calling `espflash` in-process skips both — we -//! saved ~4 s on a cold 2.4 MB ESP32-S3 verify in the first half of this -//! work, and a full write gets the same baseline savings plus a -//! progress stream that the daemon can surface without scraping -//! subprocess stdout. -//! -//! # Scope -//! -//! * `verify-flash` — three regions (bootloader / partitions / -//! firmware), same [`crate::esp32::VerifyOutcome`] semantics as the -//! esptool path. -//! * `write-flash` — same three regions, same -//! [`DeploymentResult`]/[`DeployOutcome`] shape as the esptool path. -//! Progress callbacks from espflash are bridged into `tracing` so the -//! daemon's existing log plumbing picks them up without any new API -//! surface. Full WebSocket progress frames are a follow-up — see -//! `log_only_progress_bridge` below. -//! -//! # Serial-port lease -//! -//! The daemon pre-empts monitor sessions via -//! [`fbuild_serial::SharedSerialManager::preempt_for_deploy`] before -//! calling into this module. `preempt_for_deploy` explicitly closes the -//! OS-level port handle, so we can open our own here — exactly the same -//! way the existing esptool-subprocess path does. No second lease is -//! held concurrently. -//! -//! # Opt-in -//! -//! `verify-flash` is guarded by -//! [`crate::esp32::Esp32Deployer::with_native_verify`] (daemon env: -//! `FBUILD_USE_ESPFLASH_VERIFY`), and `write-flash` by -//! [`crate::esp32::Esp32Deployer::with_native_write`] (daemon env: -//! `FBUILD_USE_ESPFLASH_WRITE`). The two flags are independent — -//! users can flip one without the other while the native write path -//! accumulates bench time on every ESP32 family member. - -use std::path::Path; -use std::str::FromStr; -use std::time::Duration; - -use espflash::connection::{Connection, ResetAfterOperation, ResetBeforeOperation}; -use espflash::flasher::Flasher; -use espflash::target::{Chip, ProgressCallbacks}; -use md5::{Digest, Md5}; -use serialport::{FlowControl, SerialPortType, UsbPortInfo}; - -use fbuild_core::{FbuildError, Result}; - -use crate::esp32::{FlashRegion, RegionVerifyResult, VerifyOutcome}; -use crate::{DeployOutcome, DeploymentResult}; - -/// A single region (flash offset + local firmware file) to verify. -#[derive(Debug, Clone)] -pub struct NativeVerifyRegion { - pub region: FlashRegion, - pub offset: u32, - pub path: std::path::PathBuf, -} - -/// Native `verify-flash` — reads each file from disk, asks the stub -/// flasher to compute `FLASH_MD5SUM` over the same region on-chip, and -/// reports per-region match/mismatch. -/// -/// On `Match` the chip is hard-reset (matching esptool's -/// `--after hard-reset`) so callers can treat a `Match` as "device is -/// now running the requested firmware" just like the subprocess path. -/// -/// Port ownership: caller must ensure the OS-level serial port is free -/// before calling (the daemon does this via `preempt_for_deploy`). The -/// opened handle is closed on function return. -/// -/// `before_reset` / `after_reset` accept the same strings the esptool -/// path uses (`default-reset`, `no-reset`, `no-reset-no-sync`, -/// `usb-reset` for before; `hard-reset`, `no-reset`, `no-reset-no-stub`, -/// `watchdog-reset` for after) so the daemon wiring doesn't need a -/// separate config surface. -#[allow(clippy::too_many_arguments)] -pub fn try_verify_deployment_native( - chip_name: &str, - port: &str, - baud: u32, - before_reset: &str, - after_reset: &str, - regions: &[NativeVerifyRegion], - bootloader_offset: u32, - partitions_offset: u32, - firmware_offset: u32, -) -> Result { - // Map config strings → espflash enums. We intentionally keep the - // surface narrow: anything outside the supported set is a hard - // error so a typo in a board JSON doesn't silently degrade to - // default-reset. - let chip = parse_chip(chip_name)?; - let before = parse_before_reset(before_reset)?; - let after = parse_after_reset(after_reset)?; - - // Open the port at 115_200 (espflash renegotiates to `baud` after - // stub upload). `open_native` returns the platform-specific `Port` - // (COMPort / TTYPort) expected by `Connection::new`. - let serial_port = serialport::new(port, 115_200) - .flow_control(FlowControl::None) - .timeout(Duration::from_secs(3)) - .open_native() - .map_err(|e| { - FbuildError::DeployFailed(format!( - "native verify: failed to open serial port {}: {}", - port, e - )) - })?; - - // Look up the USB VID/PID for the port so reset strategy selection - // in espflash (USB-JTAG vs classic) picks the right sequence. - // Missing entries default to zero, matching espflash's own CLI. - let port_info = discover_usb_port_info(port); - - let connection = Connection::new(serial_port, port_info, after, before, baud); - - // `Flasher::connect` handles reset, sync, chip detect, stub upload, - // and baud renegotiation. Errors here are fatal — on real hardware - // we treat them as "verify couldn't run, fall back to full flash" - // at the caller, so we surface them via `FbuildError` rather than - // embedding them in a `Mismatch`. - let use_stub = true; - let mut flasher = Flasher::connect( - connection, - use_stub, - /* verify */ false, // we do our own per-region verify below - /* skip */ false, - Some(chip), - Some(baud), - ) - .map_err(|e| { - FbuildError::DeployFailed(format!( - "native verify: espflash connect failed on {}: {}", - port, e - )) - })?; - - // Compute per-region verdicts. - let mut results: Vec = Vec::with_capacity(regions.len()); - for r in regions { - let bytes = std::fs::read(&r.path).map_err(|e| { - FbuildError::DeployFailed(format!( - "native verify: failed to read {}: {}", - r.path.display(), - e - )) - })?; - let local = local_md5(&bytes); - let remote = flasher - .checksum_md5(r.offset, bytes.len() as u32) - .map_err(|e| { - FbuildError::DeployFailed(format!( - "native verify: FLASH_MD5SUM failed at 0x{:x}: {}", - r.offset, e - )) - })?; - let matched = remote == local; - tracing::debug!( - port, - region = ?r.region, - offset = format!("0x{:x}", r.offset), - size = bytes.len(), - matched, - "native verify region result" - ); - results.push(RegionVerifyResult { - region: r.region, - matched, - }); - } - - // Keep the three offsets threaded through even though we no longer - // parse them from stdout — callers of `VerifyOutcome::Mismatch` use - // `regions` directly and ignore stdout/stderr when they came from - // the native path. - let _ = (bootloader_offset, partitions_offset, firmware_offset); - - let all_match = !results.is_empty() && results.iter().all(|r| r.matched); - - // Hard-reset the chip back into the app on success so behavior - // matches the esptool `--after hard-reset` contract. On mismatch we - // leave the reset policy to the subsequent flash call. - if all_match { - let mut connection = flasher.into_connection(); - if let Err(e) = connection.reset_after(use_stub, chip) { - tracing::warn!(port, "native verify: reset_after failed: {}", e); - } - } - - if all_match { - Ok(VerifyOutcome::Match { - stdout: render_native_stdout(&results), - stderr: String::new(), - }) - } else { - Ok(VerifyOutcome::Mismatch { - stdout: render_native_stdout(&results), - stderr: String::new(), - regions: results, - }) - } -} - -/// A single region (flash offset + local firmware file) to write. -/// -/// Same shape as [`NativeVerifyRegion`] but kept as a distinct type so -/// a future change (e.g. encrypted-write flags per-region) doesn't -/// silently leak back into the verify path. -#[derive(Debug, Clone)] -pub struct NativeWriteRegion { - pub region: FlashRegion, - pub offset: u32, - pub path: std::path::PathBuf, -} - -/// Native `write-flash` — reads each file from disk and streams it to -/// the chip via espflash's stub flasher. Same three regions -/// (bootloader / partitions / firmware) as the esptool path, same -/// [`DeploymentResult`] / [`DeployOutcome`] semantics so callers can -/// route between the two behind a single flag. -/// -/// Progress from espflash is bridged into `tracing` so the daemon's -/// existing log broadcaster surfaces it without new API surface (see -/// the private `LoggingProgressBridge` below). Structured WebSocket -/// progress frames are a follow-up: the bridge is a drop-in -/// replacement point for a richer callback without touching any of -/// the call sites. -/// -/// On success the chip is hard-reset (matching esptool's -/// `--after hard-reset`) so callers can treat the `Ok` return as -/// "device is now running the requested firmware" without an extra -/// reset. -/// -/// Port ownership: caller must ensure the OS-level serial port is free -/// before calling (the daemon does this via `preempt_for_deploy`). The -/// opened handle is closed on function return. -/// -/// Error recovery: on any region failure we short-circuit, log which -/// region failed with its flash offset, return a failed -/// [`DeploymentResult`], and let the caller decide whether to retry via -/// esptool. Partial writes are never silently swallowed. -#[allow(clippy::too_many_arguments)] -pub fn try_write_deployment_native( - chip_name: &str, - port: &str, - baud: u32, - before_reset: &str, - after_reset: &str, - regions: &[NativeWriteRegion], - selective: bool, -) -> Result { - let chip = parse_chip(chip_name)?; - let before = parse_before_reset(before_reset)?; - let after = parse_after_reset(after_reset)?; - - if regions.is_empty() { - return Err(FbuildError::DeployFailed( - "native write: called with no regions; at least firmware is required".to_string(), - )); - } - - let serial_port = serialport::new(port, 115_200) - .flow_control(FlowControl::None) - // Writes are much longer than verifies; the stub flasher can - // take tens of seconds between UART responses while erasing - // the partition table sector on a cold boot. 10s matches - // espflash's own CLI default. - .timeout(Duration::from_secs(10)) - .open_native() - .map_err(|e| { - FbuildError::DeployFailed(format!( - "native write: failed to open serial port {}: {}", - port, e - )) - })?; - - let port_info = discover_usb_port_info(port); - let connection = Connection::new(serial_port, port_info, after, before, baud); - - let use_stub = true; - let mut flasher = Flasher::connect( - connection, - use_stub, - /* verify */ - false, // espflash's own post-write verify; we rely on the separate verify path - /* skip */ false, - Some(chip), - Some(baud), - ) - .map_err(|e| { - FbuildError::DeployFailed(format!( - "native write: espflash connect failed on {}: {}", - port, e - )) - })?; - - let mut bridge = LoggingProgressBridge::new(port); - let mut rendered = String::new(); - let mut bytes_written: u64 = 0; - - for r in regions { - let bytes = std::fs::read(&r.path).map_err(|e| { - FbuildError::DeployFailed(format!( - "native write: failed to read {}: {}", - r.path.display(), - e - )) - })?; - let size = bytes.len(); - bridge.enter_region(r.region); - tracing::info!( - port, - region = ?r.region, - offset = format!("0x{:x}", r.offset), - size, - "native write: writing region" - ); - if let Err(e) = flasher.write_bin_to_flash(r.offset, &bytes, &mut bridge) { - let msg = format!( - "native write: region {:?} at 0x{:x} failed after {}/{} bytes: {}", - r.region, r.offset, bridge.last_current, size, e - ); - tracing::error!(port, "{}", msg); - rendered.push_str(&msg); - rendered.push('\n'); - return Ok(DeploymentResult { - success: false, - message: format!("espflash native write failed on {} ({})", port, chip_name), - port: Some(port.to_string()), - stdout: rendered, - stderr: e.to_string(), - outcome: outcome_for(selective, regions), - }); - } - bytes_written += size as u64; - rendered.push_str(&format!( - "native write: {} at 0x{:x} ({} bytes) OK\n", - region_name(r.region), - r.offset, - size - )); - } - - // Hard-reset (or whatever after_reset selects) the chip so the app - // boots once the stub releases the bus. Mirrors esptool's - // `--after hard-reset` contract. - let mut connection = flasher.into_connection(); - if let Err(e) = connection.reset_after(use_stub, chip) { - // A failed final reset is annoying but doesn't invalidate the - // flash: the image is on-device, a manual power cycle will - // boot it. Log and report success. - tracing::warn!(port, "native write: reset_after failed: {}", e); - } - - tracing::info!( - port, - bytes_written, - regions = regions.len(), - "native write: completed successfully" - ); - - Ok(DeploymentResult { - success: true, - message: format!( - "firmware flashed to {} ({}) via espflash ({} bytes, {} region(s))", - port, - chip_name, - bytes_written, - regions.len() - ), - port: Some(port.to_string()), - stdout: rendered, - stderr: String::new(), - outcome: outcome_for(selective, regions), - }) -} - -/// Pick the correct [`DeployOutcome`] for a native write — preserves -/// the existing invariant that the full three-region path reports -/// `FullFlash` and selective rewrites carry the region list. -fn outcome_for(selective: bool, regions: &[NativeWriteRegion]) -> DeployOutcome { - if selective { - DeployOutcome::SelectiveFlash { - regions: regions.iter().map(|r| r.region).collect(), - } - } else { - DeployOutcome::FullFlash - } -} - -fn region_name(r: FlashRegion) -> &'static str { - match r { - FlashRegion::Bootloader => "bootloader", - FlashRegion::Partitions => "partitions", - FlashRegion::Firmware => "firmware", - } -} - -/// Collect the full three-region write set (bootloader + partitions + -/// firmware where present) from a firmware path. Mirrors -/// [`collect_standard_regions`] but returns [`NativeWriteRegion`]. -pub fn collect_standard_write_regions( - firmware_path: &Path, - bootloader_offset: u32, - partitions_offset: u32, - firmware_offset: u32, -) -> Vec { - let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); - let bootloader_path = build_dir.join("bootloader.bin"); - let partitions_path = build_dir.join("partitions.bin"); - - let mut out = Vec::with_capacity(3); - if bootloader_path.exists() { - out.push(NativeWriteRegion { - region: FlashRegion::Bootloader, - offset: bootloader_offset, - path: bootloader_path, - }); - } - if partitions_path.exists() { - out.push(NativeWriteRegion { - region: FlashRegion::Partitions, - offset: partitions_offset, - path: partitions_path, - }); - } - out.push(NativeWriteRegion { - region: FlashRegion::Firmware, - offset: firmware_offset, - path: firmware_path.to_path_buf(), - }); - out -} - -/// Collect a caller-chosen subset of write regions. Used by the daemon's -/// selective-flash path (post-verify-mismatch) so we don't rewrite -/// bootloader/partitions when only firmware differs. `requested` must -/// not be empty. -pub fn collect_selected_write_regions( - firmware_path: &Path, - bootloader_offset: u32, - partitions_offset: u32, - firmware_offset: u32, - requested: &[FlashRegion], -) -> Result> { - if requested.is_empty() { - return Err(FbuildError::DeployFailed( - "native write: collect_selected_write_regions called with empty request".to_string(), - )); - } - let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); - let mut out = Vec::with_capacity(requested.len()); - for r in requested { - let (path, offset) = match r { - FlashRegion::Bootloader => (build_dir.join("bootloader.bin"), bootloader_offset), - FlashRegion::Partitions => (build_dir.join("partitions.bin"), partitions_offset), - FlashRegion::Firmware => (firmware_path.to_path_buf(), firmware_offset), - }; - if !path.exists() { - return Err(FbuildError::DeployFailed(format!( - "native write: requested {:?} but {} is missing", - r, - path.display() - ))); - } - out.push(NativeWriteRegion { - region: *r, - offset, - path, - }); - } - Ok(out) -} - -/// Bridges espflash's [`ProgressCallbacks`] into `tracing` so the -/// daemon's existing log broadcaster picks up write progress without -/// any new API surface. -/// -/// Logging-only by design: a richer WebSocket bridge (structured -/// progress frames on the deploy WS channel) is a follow-up — this -/// type is the single seam where that upgrade lands. -struct LoggingProgressBridge { - port: String, - region: Option, - total: usize, - last_current: usize, - /// Last percentage we logged at. Throttles the per-block `update` - /// spam to one line per 10% of a region so the daemon log stream - /// stays readable. - last_pct_logged: u8, -} - -impl LoggingProgressBridge { - fn new(port: &str) -> Self { - Self { - port: port.to_string(), - region: None, - total: 0, - last_current: 0, - last_pct_logged: 0, - } - } - - fn enter_region(&mut self, region: FlashRegion) { - self.region = Some(region); - self.total = 0; - self.last_current = 0; - self.last_pct_logged = 0; - } - - fn region_label(&self) -> &'static str { - match self.region { - Some(r) => region_name(r), - None => "unknown", - } - } -} - -impl ProgressCallbacks for LoggingProgressBridge { - fn init(&mut self, addr: u32, total: usize) { - self.total = total; - self.last_current = 0; - self.last_pct_logged = 0; - tracing::info!( - port = %self.port, - region = self.region_label(), - addr = format!("0x{:x}", addr), - total, - "native write: begin region" - ); - } - - fn update(&mut self, current: usize) { - self.last_current = current; - if self.total == 0 { - return; - } - let pct = ((current as u64 * 100) / self.total as u64).min(100) as u8; - // Emit a log line every 10% boundary so a 1 MB write produces - // ~10 lines rather than hundreds. - if pct >= self.last_pct_logged + 10 { - self.last_pct_logged = pct - (pct % 10); - tracing::info!( - port = %self.port, - region = self.region_label(), - pct, - current, - total = self.total, - "native write: progress" - ); - } - } - - fn verifying(&mut self) { - tracing::debug!( - port = %self.port, - region = self.region_label(), - "native write: verifying region (espflash internal)" - ); - } - - fn finish(&mut self, skipped: bool) { - tracing::info!( - port = %self.port, - region = self.region_label(), - skipped, - "native write: region complete" - ); - } -} - -/// Parse a chip name string (`"esp32s3"`, `"esp32c6"`, ...) into -/// espflash's [`Chip`] enum. -/// -/// espflash derives [`strum::EnumString`] with `serialize_all = -/// "lowercase"` on `Chip`, so this is a thin wrapper around the existing -/// `FromStr` impl. Kept as a named helper so error messages point at -/// this module instead of at espflash internals. -fn parse_chip(name: &str) -> Result { - Chip::from_str(&name.to_ascii_lowercase()).map_err(|_| { - FbuildError::DeployFailed(format!( - "native verify: unknown chip name '{}' (espflash does not recognize it)", - name - )) - }) -} - -fn parse_before_reset(s: &str) -> Result { - // Match the esptool CLI spellings we already accept in board JSON. - match s { - "default-reset" | "default_reset" => Ok(ResetBeforeOperation::DefaultReset), - "no-reset" | "no_reset" => Ok(ResetBeforeOperation::NoReset), - "no-reset-no-sync" | "no_reset_no_sync" => Ok(ResetBeforeOperation::NoResetNoSync), - "usb-reset" | "usb_reset" => Ok(ResetBeforeOperation::UsbReset), - other => Err(FbuildError::DeployFailed(format!( - "native verify: unsupported before-reset mode '{}'", - other - ))), - } -} - -fn parse_after_reset(s: &str) -> Result { - match s { - "hard-reset" | "hard_reset" => Ok(ResetAfterOperation::HardReset), - "no-reset" | "no_reset" => Ok(ResetAfterOperation::NoReset), - "no-reset-no-stub" | "no_reset_no_stub" => Ok(ResetAfterOperation::NoResetNoStub), - "watchdog-reset" | "watchdog_reset" => Ok(ResetAfterOperation::WatchdogReset), - other => Err(FbuildError::DeployFailed(format!( - "native verify: unsupported after-reset mode '{}'", - other - ))), - } -} - -/// Compute MD5 of a local buffer and pack it into a `u128` in the same -/// byte order espflash's `checksum_md5` returns. espflash parses the -/// 16-byte on-chip digest as little-endian `u128`, so we do the same -/// here; equality over the packed ints is equivalent to equality over -/// the 16 digest bytes. -fn local_md5(bytes: &[u8]) -> u128 { - let mut hasher = Md5::new(); - hasher.update(bytes); - let digest = hasher.finalize(); - let arr: [u8; 16] = digest.into(); - u128::from_le_bytes(arr) -} - -/// Best-effort USB VID/PID lookup for the opened port, mirroring -/// espflash's own CLI fallback. Failure → zeros, which just means -/// reset-strategy selection uses generic defaults. -fn discover_usb_port_info(port: &str) -> UsbPortInfo { - match serialport::available_ports() { - Ok(list) => { - for p in list { - if p.port_name == port { - if let SerialPortType::UsbPort(info) = p.port_type { - return info; - } - } - } - } - Err(e) => { - tracing::debug!("native verify: available_ports failed: {}", e); - } - } - UsbPortInfo { - vid: 0, - pid: 0, - serial_number: None, - manufacturer: None, - product: None, - } -} - -/// Render a compact text description of the per-region results that -/// callers can log or return in `VerifyOutcome::Match::stdout`. Keeps -/// the outcome surface identical between the esptool and native paths -/// for anything that reads `stdout` for display. -fn render_native_stdout(results: &[RegionVerifyResult]) -> String { - let mut out = String::new(); - for r in results { - let name = match r.region { - FlashRegion::Bootloader => "bootloader", - FlashRegion::Partitions => "partitions", - FlashRegion::Firmware => "firmware", - }; - let verdict = if r.matched { - "Verification successful (digest matched)." - } else { - "Verification failed (digest mismatch)." - }; - out.push_str(&format!("native verify: {}: {}\n", name, verdict)); - } - out -} - -/// Collect the standard three-region set from a firmware path. The -/// caller supplies the offsets (parsed once from board config by -/// [`super::esp32::Esp32Deployer`]). -/// -/// Mirrors [`super::esp32::Esp32Deployer::build_verify_flash_args`]: -/// bootloader and partitions are optional (absent files are skipped); -/// firmware is mandatory. -pub fn collect_standard_regions( - firmware_path: &Path, - bootloader_offset: u32, - partitions_offset: u32, - firmware_offset: u32, -) -> Vec { - let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); - let bootloader_path = build_dir.join("bootloader.bin"); - let partitions_path = build_dir.join("partitions.bin"); - - let mut out = Vec::with_capacity(3); - if bootloader_path.exists() { - out.push(NativeVerifyRegion { - region: FlashRegion::Bootloader, - offset: bootloader_offset, - path: bootloader_path, - }); - } - if partitions_path.exists() { - out.push(NativeVerifyRegion { - region: FlashRegion::Partitions, - offset: partitions_offset, - path: partitions_path, - }); - } - out.push(NativeVerifyRegion { - region: FlashRegion::Firmware, - offset: firmware_offset, - path: firmware_path.to_path_buf(), - }); - out -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_chip_accepts_lowercase_family_members() { - // All ten currently-supported chips must round-trip through - // our wrapper. Guards against a future espflash bump that - // renames a variant. - for name in [ - "esp32", "esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32c61", "esp32h2", "esp32p4", - "esp32s2", "esp32s3", - ] { - parse_chip(name) - .unwrap_or_else(|e| panic!("chip '{}' must parse in espflash: {:?}", name, e)); - } - } - - #[test] - fn parse_chip_accepts_uppercase_because_we_lowercase_first() { - parse_chip("ESP32S3").unwrap(); - parse_chip("Esp32C6").unwrap(); - } - - #[test] - fn parse_chip_rejects_unknown() { - assert!(parse_chip("esp99").is_err()); - } - - #[test] - fn parse_before_reset_covers_esptool_spellings() { - assert!(matches!( - parse_before_reset("default-reset").unwrap(), - ResetBeforeOperation::DefaultReset - )); - assert!(matches!( - parse_before_reset("no-reset").unwrap(), - ResetBeforeOperation::NoReset - )); - assert!(matches!( - parse_before_reset("usb-reset").unwrap(), - ResetBeforeOperation::UsbReset - )); - assert!(parse_before_reset("bogus").is_err()); - } - - #[test] - fn parse_after_reset_covers_esptool_spellings() { - assert!(matches!( - parse_after_reset("hard-reset").unwrap(), - ResetAfterOperation::HardReset - )); - assert!(matches!( - parse_after_reset("no-reset").unwrap(), - ResetAfterOperation::NoReset - )); - assert!(parse_after_reset("bogus").is_err()); - } - - #[test] - fn local_md5_matches_known_vector() { - // RFC 1321 test vector: MD5("") = d41d8cd98f00b204e9800998ecf8427e - let empty = local_md5(b""); - let expected_bytes: [u8; 16] = [ - 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, - 0x42, 0x7e, - ]; - assert_eq!(empty, u128::from_le_bytes(expected_bytes)); - - // MD5("abc") = 900150983cd24fb0d6963f7d28e17f72 - let abc = local_md5(b"abc"); - let expected_bytes: [u8; 16] = [ - 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0, 0xd6, 0x96, 0x3f, 0x7d, 0x28, 0xe1, - 0x7f, 0x72, - ]; - assert_eq!(abc, u128::from_le_bytes(expected_bytes)); - } - - #[test] - fn collect_standard_regions_skips_missing_optional_files() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firmware").unwrap(); - - let regions = collect_standard_regions(&fw, 0x0, 0x8000, 0x10000); - assert_eq!(regions.len(), 1); - assert_eq!(regions[0].region, FlashRegion::Firmware); - assert_eq!(regions[0].offset, 0x10000); - } - - #[test] - fn collect_standard_regions_includes_optional_files_when_present() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firmware").unwrap(); - std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); - std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); - - let regions = collect_standard_regions(&fw, 0x0, 0x8000, 0x10000); - assert_eq!(regions.len(), 3); - assert_eq!(regions[0].region, FlashRegion::Bootloader); - assert_eq!(regions[0].offset, 0x0); - assert_eq!(regions[1].region, FlashRegion::Partitions); - assert_eq!(regions[1].offset, 0x8000); - assert_eq!(regions[2].region, FlashRegion::Firmware); - assert_eq!(regions[2].offset, 0x10000); - } - - #[test] - fn render_native_stdout_mentions_all_regions() { - let results = vec![ - RegionVerifyResult { - region: FlashRegion::Bootloader, - matched: true, - }, - RegionVerifyResult { - region: FlashRegion::Firmware, - matched: false, - }, - ]; - let out = render_native_stdout(&results); - assert!(out.contains("bootloader")); - assert!(out.contains("firmware")); - assert!(out.contains("digest matched")); - assert!(out.contains("digest mismatch")); - } - - // --- native write-flash tests (issue #66 PR #89 follow-up) --- - // - // Most of the write flow is live hardware code. What we can test - // without a board attached is the region assembly, the - // DeployOutcome mapping, and the progress-bridge throttling — the - // three pure pieces where a regression would silently corrupt the - // daemon's deploy response. - - #[test] - fn collect_standard_write_regions_skips_missing_optional_files() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firmware").unwrap(); - - let regions = collect_standard_write_regions(&fw, 0x0, 0x8000, 0x10000); - assert_eq!(regions.len(), 1); - assert_eq!(regions[0].region, FlashRegion::Firmware); - assert_eq!(regions[0].offset, 0x10000); - } - - #[test] - fn collect_standard_write_regions_includes_optional_files_when_present() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firmware").unwrap(); - std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); - std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); - - let regions = collect_standard_write_regions(&fw, 0x0, 0x8000, 0x10000); - assert_eq!(regions.len(), 3); - assert_eq!(regions[0].region, FlashRegion::Bootloader); - assert_eq!(regions[0].offset, 0x0); - assert_eq!(regions[1].region, FlashRegion::Partitions); - assert_eq!(regions[1].offset, 0x8000); - assert_eq!(regions[2].region, FlashRegion::Firmware); - assert_eq!(regions[2].offset, 0x10000); - } - - #[test] - fn collect_selected_write_regions_errors_on_empty_request() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firmware").unwrap(); - let err = collect_selected_write_regions(&fw, 0x0, 0x8000, 0x10000, &[]).unwrap_err(); - assert!(err.to_string().contains("empty request")); - } - - #[test] - fn collect_selected_write_regions_errors_when_file_missing() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firmware").unwrap(); - // No bootloader.bin on disk. - let err = - collect_selected_write_regions(&fw, 0x0, 0x8000, 0x10000, &[FlashRegion::Bootloader]) - .unwrap_err(); - assert!(err.to_string().contains("missing")); - } - - #[test] - fn collect_selected_write_regions_returns_requested_subset() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = tmp.path().join("firmware.bin"); - std::fs::write(&fw, b"firmware").unwrap(); - std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); - std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); - - let out = collect_selected_write_regions( - &fw, - 0x0, - 0x8000, - 0x10000, - &[FlashRegion::Firmware, FlashRegion::Bootloader], - ) - .unwrap(); - assert_eq!(out.len(), 2); - assert_eq!(out[0].region, FlashRegion::Firmware); - assert_eq!(out[0].offset, 0x10000); - assert_eq!(out[1].region, FlashRegion::Bootloader); - assert_eq!(out[1].offset, 0x0); - } - - #[test] - fn outcome_for_full_write_reports_full_flash() { - let regions = vec![NativeWriteRegion { - region: FlashRegion::Firmware, - offset: 0x10000, - path: std::path::PathBuf::from("firmware.bin"), - }]; - assert!(matches!( - outcome_for(false, ®ions), - DeployOutcome::FullFlash - )); - } - - #[test] - fn outcome_for_selective_write_carries_region_list() { - let regions = vec![ - NativeWriteRegion { - region: FlashRegion::Bootloader, - offset: 0x0, - path: std::path::PathBuf::from("bootloader.bin"), - }, - NativeWriteRegion { - region: FlashRegion::Firmware, - offset: 0x10000, - path: std::path::PathBuf::from("firmware.bin"), - }, - ]; - match outcome_for(true, ®ions) { - DeployOutcome::SelectiveFlash { regions } => { - assert_eq!( - regions, - vec![FlashRegion::Bootloader, FlashRegion::Firmware] - ); - } - other => panic!("expected SelectiveFlash, got {:?}", other), - } - } - - #[test] - fn try_write_deployment_native_rejects_empty_regions() { - let err = try_write_deployment_native( - "esp32s3", - "COM99", - 460_800, - "default-reset", - "hard-reset", - &[], - false, - ) - .unwrap_err(); - assert!(err.to_string().contains("no regions")); - } - - #[test] - fn progress_bridge_throttles_updates_to_ten_percent_boundaries() { - // Assert the 10%-boundary throttle actually fires — without it - // a 1 MB write spams hundreds of log lines per region. - let mut bridge = LoggingProgressBridge::new("COM13"); - bridge.enter_region(FlashRegion::Firmware); - bridge.init(0x10000, 1000); - - // Simulate espflash calling update every 10 bytes. After 1000 - // calls we should have logged at roughly 0, 10, 20, ..., 100% - // — i.e. last_pct_logged should have landed on a 10-multiple. - for current in (10..=1000).step_by(10) { - bridge.update(current); - } - assert_eq!(bridge.last_current, 1000); - assert_eq!(bridge.last_pct_logged % 10, 0); - // Finishing resets the per-region state for the next region. - bridge.finish(false); - } - - #[test] - fn progress_bridge_handles_zero_total_without_panic() { - // A zero-byte region is defensive: espflash shouldn't ever - // emit one, but our arithmetic must not divide by zero. - let mut bridge = LoggingProgressBridge::new("COM13"); - bridge.enter_region(FlashRegion::Firmware); - bridge.init(0x10000, 0); - bridge.update(0); - bridge.finish(true); - } - - #[test] - fn progress_bridge_reports_correct_region_label() { - let mut bridge = LoggingProgressBridge::new("COM13"); - assert_eq!(bridge.region_label(), "unknown"); - bridge.enter_region(FlashRegion::Bootloader); - assert_eq!(bridge.region_label(), "bootloader"); - bridge.enter_region(FlashRegion::Partitions); - assert_eq!(bridge.region_label(), "partitions"); - bridge.enter_region(FlashRegion::Firmware); - assert_eq!(bridge.region_label(), "firmware"); - } - - #[test] - fn region_name_is_stable() { - // Daemon log messages and integration tests depend on these - // literal names. Pin them here so a refactor that renames - // them trips a unit test first. - assert_eq!(region_name(FlashRegion::Bootloader), "bootloader"); - assert_eq!(region_name(FlashRegion::Partitions), "partitions"); - assert_eq!(region_name(FlashRegion::Firmware), "firmware"); - } -} diff --git a/crates/fbuild-deploy/src/esp32_native/README.md b/crates/fbuild-deploy/src/esp32_native/README.md new file mode 100644 index 00000000..4b5f21b1 --- /dev/null +++ b/crates/fbuild-deploy/src/esp32_native/README.md @@ -0,0 +1,25 @@ +# esp32_native + +Native ESP32 `verify-flash` and `write-flash` implementations backed by +the `espflash` crate, compiled in only when the `espflash-native` cargo +feature is enabled. Alternative to the default +`crate::esp32::Esp32Deployer` path that shells out to Python `esptool`. + +See the module docstring in [`mod.rs`](mod.rs) for the rationale (issue +#66), scope, serial-port lease behavior, and opt-in env vars. + +## Layout + +| File | Responsibility | +| -------------- | ------------------------------------------------------- | +| `mod.rs` | Module docstring; re-exports the public API | +| `types.rs` | `NativeVerifyRegion`, `NativeWriteRegion` | +| `verify.rs` | `try_verify_deployment_native`, `collect_standard_regions` | +| `write.rs` | `try_write_deployment_native`, write-region collectors | +| `transport.rs` | Chip / reset-string parsing, MD5, port discovery, stdout renderer | +| `progress.rs` | `LoggingProgressBridge` — espflash → `tracing` adapter | +| `tests.rs` | Unit tests for all pure helpers | + +Split from a single 1042-LOC `esp32_native.rs` to satisfy the workspace +LOC gate (1000 LOC max per `.rs` file). Public surface is preserved +exactly — `crate::esp32_native::*` paths still resolve. diff --git a/crates/fbuild-deploy/src/esp32_native/mod.rs b/crates/fbuild-deploy/src/esp32_native/mod.rs new file mode 100644 index 00000000..9e9de91d --- /dev/null +++ b/crates/fbuild-deploy/src/esp32_native/mod.rs @@ -0,0 +1,70 @@ +//! Native ESP32 `verify-flash` **and** `write-flash` implementations +//! backed by the [`espflash`] crate. Alternatives to the default +//! [`crate::esp32::Esp32Deployer`] path, which shells out to Python +//! `esptool`. +//! +//! # Why (issue #66) +//! +//! `esptool.py` spends ~1 s on Python interpreter startup plus another +//! ~0.5 s on subprocess/stub-flasher handshake before it even issues the +//! first real command. Calling `espflash` in-process skips both — we +//! saved ~4 s on a cold 2.4 MB ESP32-S3 verify in the first half of this +//! work, and a full write gets the same baseline savings plus a +//! progress stream that the daemon can surface without scraping +//! subprocess stdout. +//! +//! # Scope +//! +//! * `verify-flash` — three regions (bootloader / partitions / +//! firmware), same [`crate::esp32::VerifyOutcome`] semantics as the +//! esptool path. +//! * `write-flash` — same three regions, same +//! [`crate::DeploymentResult`]/[`crate::DeployOutcome`] shape as the +//! esptool path. Progress callbacks from espflash are bridged into +//! `tracing` so the daemon's existing log plumbing picks them up +//! without any new API surface. Full WebSocket progress frames are a +//! follow-up — see the `progress` submodule. +//! +//! # Serial-port lease +//! +//! The daemon pre-empts monitor sessions via +//! [`fbuild_serial::SharedSerialManager::preempt_for_deploy`] before +//! calling into this module. `preempt_for_deploy` explicitly closes the +//! OS-level port handle, so we can open our own here — exactly the same +//! way the existing esptool-subprocess path does. No second lease is +//! held concurrently. +//! +//! # Opt-in +//! +//! `verify-flash` is guarded by +//! [`crate::esp32::Esp32Deployer::with_native_verify`] (daemon env: +//! `FBUILD_USE_ESPFLASH_VERIFY`), and `write-flash` by +//! [`crate::esp32::Esp32Deployer::with_native_write`] (daemon env: +//! `FBUILD_USE_ESPFLASH_WRITE`). The two flags are independent — +//! users can flip one without the other while the native write path +//! accumulates bench time on every ESP32 family member. +//! +//! # Module layout +//! +//! Split into submodules so no single file exceeds the workspace LOC +//! gate. Public surface (`try_verify_deployment_native`, +//! `try_write_deployment_native`, `NativeVerifyRegion`, +//! `NativeWriteRegion`, `collect_standard_regions`, +//! `collect_standard_write_regions`, +//! `collect_selected_write_regions`) is re-exported from this `mod.rs` +//! so external callers see the same paths as before. + +mod progress; +mod transport; +mod types; +mod verify; +mod write; + +pub use types::{NativeVerifyRegion, NativeWriteRegion}; +pub use verify::{collect_standard_regions, try_verify_deployment_native}; +pub use write::{ + collect_selected_write_regions, collect_standard_write_regions, try_write_deployment_native, +}; + +#[cfg(test)] +mod tests; diff --git a/crates/fbuild-deploy/src/esp32_native/progress.rs b/crates/fbuild-deploy/src/esp32_native/progress.rs new file mode 100644 index 00000000..ef2d302b --- /dev/null +++ b/crates/fbuild-deploy/src/esp32_native/progress.rs @@ -0,0 +1,104 @@ +//! Espflash progress -> `tracing` bridge for the native write path. + +use espflash::target::ProgressCallbacks; + +use crate::esp32::FlashRegion; + +use super::transport::region_name; + +/// Bridges espflash's [`ProgressCallbacks`] into `tracing` so the +/// daemon's existing log broadcaster picks up write progress without +/// any new API surface. +/// +/// Logging-only by design: a richer WebSocket bridge (structured +/// progress frames on the deploy WS channel) is a follow-up — this +/// type is the single seam where that upgrade lands. +pub(super) struct LoggingProgressBridge { + pub(super) port: String, + pub(super) region: Option, + pub(super) total: usize, + pub(super) last_current: usize, + /// Last percentage we logged at. Throttles the per-block `update` + /// spam to one line per 10% of a region so the daemon log stream + /// stays readable. + pub(super) last_pct_logged: u8, +} + +impl LoggingProgressBridge { + pub(super) fn new(port: &str) -> Self { + Self { + port: port.to_string(), + region: None, + total: 0, + last_current: 0, + last_pct_logged: 0, + } + } + + pub(super) fn enter_region(&mut self, region: FlashRegion) { + self.region = Some(region); + self.total = 0; + self.last_current = 0; + self.last_pct_logged = 0; + } + + pub(super) fn region_label(&self) -> &'static str { + match self.region { + Some(r) => region_name(r), + None => "unknown", + } + } +} + +impl ProgressCallbacks for LoggingProgressBridge { + fn init(&mut self, addr: u32, total: usize) { + self.total = total; + self.last_current = 0; + self.last_pct_logged = 0; + tracing::info!( + port = %self.port, + region = self.region_label(), + addr = format!("0x{:x}", addr), + total, + "native write: begin region" + ); + } + + fn update(&mut self, current: usize) { + self.last_current = current; + if self.total == 0 { + return; + } + let pct = ((current as u64 * 100) / self.total as u64).min(100) as u8; + // Emit a log line every 10% boundary so a 1 MB write produces + // ~10 lines rather than hundreds. + if pct >= self.last_pct_logged + 10 { + self.last_pct_logged = pct - (pct % 10); + tracing::info!( + port = %self.port, + region = self.region_label(), + pct, + current, + total = self.total, + "native write: progress" + ); + } + } + + fn verifying(&mut self) { + tracing::debug!( + port = %self.port, + region = self.region_label(), + "native write: verifying region (espflash internal)" + ); + } + + fn finish(&mut self, skipped: bool) { + tracing::info!( + port = %self.port, + region = self.region_label(), + skipped, + "native write: region complete" + ); + } +} diff --git a/crates/fbuild-deploy/src/esp32_native/tests.rs b/crates/fbuild-deploy/src/esp32_native/tests.rs new file mode 100644 index 00000000..292bc7d8 --- /dev/null +++ b/crates/fbuild-deploy/src/esp32_native/tests.rs @@ -0,0 +1,332 @@ +//! Unit tests for the `esp32_native` module tree. Kept in a single +//! file so a refactor that splits the module doesn't fan tests out +//! into multiple test binaries. + +use espflash::connection::{ResetAfterOperation, ResetBeforeOperation}; + +use crate::esp32::{FlashRegion, RegionVerifyResult}; +use crate::DeployOutcome; + +use super::progress::LoggingProgressBridge; +use super::transport::{ + local_md5, parse_after_reset, parse_before_reset, parse_chip, region_name, + render_native_stdout, +}; +use super::types::NativeWriteRegion; +use super::verify::collect_standard_regions; +use super::write::{ + collect_selected_write_regions, collect_standard_write_regions, outcome_for, + try_write_deployment_native, +}; + +#[test] +fn parse_chip_accepts_lowercase_family_members() { + // All ten currently-supported chips must round-trip through + // our wrapper. Guards against a future espflash bump that + // renames a variant. + for name in [ + "esp32", "esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32c61", "esp32h2", "esp32p4", + "esp32s2", "esp32s3", + ] { + parse_chip(name) + .unwrap_or_else(|e| panic!("chip '{}' must parse in espflash: {:?}", name, e)); + } +} + +#[test] +fn parse_chip_accepts_uppercase_because_we_lowercase_first() { + parse_chip("ESP32S3").unwrap(); + parse_chip("Esp32C6").unwrap(); +} + +#[test] +fn parse_chip_rejects_unknown() { + assert!(parse_chip("esp99").is_err()); +} + +#[test] +fn parse_before_reset_covers_esptool_spellings() { + assert!(matches!( + parse_before_reset("default-reset").unwrap(), + ResetBeforeOperation::DefaultReset + )); + assert!(matches!( + parse_before_reset("no-reset").unwrap(), + ResetBeforeOperation::NoReset + )); + assert!(matches!( + parse_before_reset("usb-reset").unwrap(), + ResetBeforeOperation::UsbReset + )); + assert!(parse_before_reset("bogus").is_err()); +} + +#[test] +fn parse_after_reset_covers_esptool_spellings() { + assert!(matches!( + parse_after_reset("hard-reset").unwrap(), + ResetAfterOperation::HardReset + )); + assert!(matches!( + parse_after_reset("no-reset").unwrap(), + ResetAfterOperation::NoReset + )); + assert!(parse_after_reset("bogus").is_err()); +} + +#[test] +fn local_md5_matches_known_vector() { + // RFC 1321 test vector: MD5("") = d41d8cd98f00b204e9800998ecf8427e + let empty = local_md5(b""); + let expected_bytes: [u8; 16] = [ + 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, + 0x7e, + ]; + assert_eq!(empty, u128::from_le_bytes(expected_bytes)); + + // MD5("abc") = 900150983cd24fb0d6963f7d28e17f72 + let abc = local_md5(b"abc"); + let expected_bytes: [u8; 16] = [ + 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0, 0xd6, 0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, + 0x72, + ]; + assert_eq!(abc, u128::from_le_bytes(expected_bytes)); +} + +#[test] +fn collect_standard_regions_skips_missing_optional_files() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firmware").unwrap(); + + let regions = collect_standard_regions(&fw, 0x0, 0x8000, 0x10000); + assert_eq!(regions.len(), 1); + assert_eq!(regions[0].region, FlashRegion::Firmware); + assert_eq!(regions[0].offset, 0x10000); +} + +#[test] +fn collect_standard_regions_includes_optional_files_when_present() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firmware").unwrap(); + std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); + std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); + + let regions = collect_standard_regions(&fw, 0x0, 0x8000, 0x10000); + assert_eq!(regions.len(), 3); + assert_eq!(regions[0].region, FlashRegion::Bootloader); + assert_eq!(regions[0].offset, 0x0); + assert_eq!(regions[1].region, FlashRegion::Partitions); + assert_eq!(regions[1].offset, 0x8000); + assert_eq!(regions[2].region, FlashRegion::Firmware); + assert_eq!(regions[2].offset, 0x10000); +} + +#[test] +fn render_native_stdout_mentions_all_regions() { + let results = vec![ + RegionVerifyResult { + region: FlashRegion::Bootloader, + matched: true, + }, + RegionVerifyResult { + region: FlashRegion::Firmware, + matched: false, + }, + ]; + let out = render_native_stdout(&results); + assert!(out.contains("bootloader")); + assert!(out.contains("firmware")); + assert!(out.contains("digest matched")); + assert!(out.contains("digest mismatch")); +} + +// --- native write-flash tests (issue #66 PR #89 follow-up) --- +// +// Most of the write flow is live hardware code. What we can test +// without a board attached is the region assembly, the +// DeployOutcome mapping, and the progress-bridge throttling — the +// three pure pieces where a regression would silently corrupt the +// daemon's deploy response. + +#[test] +fn collect_standard_write_regions_skips_missing_optional_files() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firmware").unwrap(); + + let regions = collect_standard_write_regions(&fw, 0x0, 0x8000, 0x10000); + assert_eq!(regions.len(), 1); + assert_eq!(regions[0].region, FlashRegion::Firmware); + assert_eq!(regions[0].offset, 0x10000); +} + +#[test] +fn collect_standard_write_regions_includes_optional_files_when_present() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firmware").unwrap(); + std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); + std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); + + let regions = collect_standard_write_regions(&fw, 0x0, 0x8000, 0x10000); + assert_eq!(regions.len(), 3); + assert_eq!(regions[0].region, FlashRegion::Bootloader); + assert_eq!(regions[0].offset, 0x0); + assert_eq!(regions[1].region, FlashRegion::Partitions); + assert_eq!(regions[1].offset, 0x8000); + assert_eq!(regions[2].region, FlashRegion::Firmware); + assert_eq!(regions[2].offset, 0x10000); +} + +#[test] +fn collect_selected_write_regions_errors_on_empty_request() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firmware").unwrap(); + let err = collect_selected_write_regions(&fw, 0x0, 0x8000, 0x10000, &[]).unwrap_err(); + assert!(err.to_string().contains("empty request")); +} + +#[test] +fn collect_selected_write_regions_errors_when_file_missing() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firmware").unwrap(); + // No bootloader.bin on disk. + let err = collect_selected_write_regions(&fw, 0x0, 0x8000, 0x10000, &[FlashRegion::Bootloader]) + .unwrap_err(); + assert!(err.to_string().contains("missing")); +} + +#[test] +fn collect_selected_write_regions_returns_requested_subset() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = tmp.path().join("firmware.bin"); + std::fs::write(&fw, b"firmware").unwrap(); + std::fs::write(tmp.path().join("bootloader.bin"), b"boot").unwrap(); + std::fs::write(tmp.path().join("partitions.bin"), b"part").unwrap(); + + let out = collect_selected_write_regions( + &fw, + 0x0, + 0x8000, + 0x10000, + &[FlashRegion::Firmware, FlashRegion::Bootloader], + ) + .unwrap(); + assert_eq!(out.len(), 2); + assert_eq!(out[0].region, FlashRegion::Firmware); + assert_eq!(out[0].offset, 0x10000); + assert_eq!(out[1].region, FlashRegion::Bootloader); + assert_eq!(out[1].offset, 0x0); +} + +#[test] +fn outcome_for_full_write_reports_full_flash() { + let regions = vec![NativeWriteRegion { + region: FlashRegion::Firmware, + offset: 0x10000, + path: std::path::PathBuf::from("firmware.bin"), + }]; + assert!(matches!( + outcome_for(false, ®ions), + DeployOutcome::FullFlash + )); +} + +#[test] +fn outcome_for_selective_write_carries_region_list() { + let regions = vec![ + NativeWriteRegion { + region: FlashRegion::Bootloader, + offset: 0x0, + path: std::path::PathBuf::from("bootloader.bin"), + }, + NativeWriteRegion { + region: FlashRegion::Firmware, + offset: 0x10000, + path: std::path::PathBuf::from("firmware.bin"), + }, + ]; + match outcome_for(true, ®ions) { + DeployOutcome::SelectiveFlash { regions } => { + assert_eq!( + regions, + vec![FlashRegion::Bootloader, FlashRegion::Firmware] + ); + } + other => panic!("expected SelectiveFlash, got {:?}", other), + } +} + +#[test] +fn try_write_deployment_native_rejects_empty_regions() { + let err = try_write_deployment_native( + "esp32s3", + "COM99", + 460_800, + "default-reset", + "hard-reset", + &[], + false, + ) + .unwrap_err(); + assert!(err.to_string().contains("no regions")); +} + +#[test] +fn progress_bridge_throttles_updates_to_ten_percent_boundaries() { + use espflash::target::ProgressCallbacks; + // Assert the 10%-boundary throttle actually fires — without it + // a 1 MB write spams hundreds of log lines per region. + let mut bridge = LoggingProgressBridge::new("COM13"); + bridge.enter_region(FlashRegion::Firmware); + bridge.init(0x10000, 1000); + + // Simulate espflash calling update every 10 bytes. After 1000 + // calls we should have logged at roughly 0, 10, 20, ..., 100% + // — i.e. last_pct_logged should have landed on a 10-multiple. + for current in (10..=1000).step_by(10) { + bridge.update(current); + } + assert_eq!(bridge.last_current, 1000); + assert_eq!(bridge.last_pct_logged % 10, 0); + // Finishing resets the per-region state for the next region. + bridge.finish(false); +} + +#[test] +fn progress_bridge_handles_zero_total_without_panic() { + use espflash::target::ProgressCallbacks; + // A zero-byte region is defensive: espflash shouldn't ever + // emit one, but our arithmetic must not divide by zero. + let mut bridge = LoggingProgressBridge::new("COM13"); + bridge.enter_region(FlashRegion::Firmware); + bridge.init(0x10000, 0); + bridge.update(0); + bridge.finish(true); +} + +#[test] +fn progress_bridge_reports_correct_region_label() { + let mut bridge = LoggingProgressBridge::new("COM13"); + assert_eq!(bridge.region_label(), "unknown"); + bridge.enter_region(FlashRegion::Bootloader); + assert_eq!(bridge.region_label(), "bootloader"); + bridge.enter_region(FlashRegion::Partitions); + assert_eq!(bridge.region_label(), "partitions"); + bridge.enter_region(FlashRegion::Firmware); + assert_eq!(bridge.region_label(), "firmware"); +} + +#[test] +fn region_name_is_stable() { + // Daemon log messages and integration tests depend on these + // literal names. Pin them here so a refactor that renames + // them trips a unit test first. + assert_eq!(region_name(FlashRegion::Bootloader), "bootloader"); + assert_eq!(region_name(FlashRegion::Partitions), "partitions"); + assert_eq!(region_name(FlashRegion::Firmware), "firmware"); +} diff --git a/crates/fbuild-deploy/src/esp32_native/transport.rs b/crates/fbuild-deploy/src/esp32_native/transport.rs new file mode 100644 index 00000000..f2a6f163 --- /dev/null +++ b/crates/fbuild-deploy/src/esp32_native/transport.rs @@ -0,0 +1,127 @@ +//! Transport-layer helpers shared by the native verify and write paths: +//! chip / reset-string parsing, local MD5, USB port discovery, and the +//! per-region stdout renderer. + +use std::str::FromStr; + +use espflash::connection::{ResetAfterOperation, ResetBeforeOperation}; +use espflash::target::Chip; +use md5::{Digest, Md5}; +use serialport::{SerialPortType, UsbPortInfo}; + +use fbuild_core::{FbuildError, Result}; + +use crate::esp32::{FlashRegion, RegionVerifyResult}; + +/// Parse a chip name string (`"esp32s3"`, `"esp32c6"`, ...) into +/// espflash's [`Chip`] enum. +/// +/// espflash derives [`strum::EnumString`] with `serialize_all = +/// "lowercase"` on `Chip`, so this is a thin wrapper around the existing +/// `FromStr` impl. Kept as a named helper so error messages point at +/// this module instead of at espflash internals. +pub(super) fn parse_chip(name: &str) -> Result { + Chip::from_str(&name.to_ascii_lowercase()).map_err(|_| { + FbuildError::DeployFailed(format!( + "native verify: unknown chip name '{}' (espflash does not recognize it)", + name + )) + }) +} + +pub(super) fn parse_before_reset(s: &str) -> Result { + // Match the esptool CLI spellings we already accept in board JSON. + match s { + "default-reset" | "default_reset" => Ok(ResetBeforeOperation::DefaultReset), + "no-reset" | "no_reset" => Ok(ResetBeforeOperation::NoReset), + "no-reset-no-sync" | "no_reset_no_sync" => Ok(ResetBeforeOperation::NoResetNoSync), + "usb-reset" | "usb_reset" => Ok(ResetBeforeOperation::UsbReset), + other => Err(FbuildError::DeployFailed(format!( + "native verify: unsupported before-reset mode '{}'", + other + ))), + } +} + +pub(super) fn parse_after_reset(s: &str) -> Result { + match s { + "hard-reset" | "hard_reset" => Ok(ResetAfterOperation::HardReset), + "no-reset" | "no_reset" => Ok(ResetAfterOperation::NoReset), + "no-reset-no-stub" | "no_reset_no_stub" => Ok(ResetAfterOperation::NoResetNoStub), + "watchdog-reset" | "watchdog_reset" => Ok(ResetAfterOperation::WatchdogReset), + other => Err(FbuildError::DeployFailed(format!( + "native verify: unsupported after-reset mode '{}'", + other + ))), + } +} + +/// Compute MD5 of a local buffer and pack it into a `u128` in the same +/// byte order espflash's `checksum_md5` returns. espflash parses the +/// 16-byte on-chip digest as little-endian `u128`, so we do the same +/// here; equality over the packed ints is equivalent to equality over +/// the 16 digest bytes. +pub(super) fn local_md5(bytes: &[u8]) -> u128 { + let mut hasher = Md5::new(); + hasher.update(bytes); + let digest = hasher.finalize(); + let arr: [u8; 16] = digest.into(); + u128::from_le_bytes(arr) +} + +/// Best-effort USB VID/PID lookup for the opened port, mirroring +/// espflash's own CLI fallback. Failure → zeros, which just means +/// reset-strategy selection uses generic defaults. +pub(super) fn discover_usb_port_info(port: &str) -> UsbPortInfo { + match serialport::available_ports() { + Ok(list) => { + for p in list { + if p.port_name == port { + if let SerialPortType::UsbPort(info) = p.port_type { + return info; + } + } + } + } + Err(e) => { + tracing::debug!("native verify: available_ports failed: {}", e); + } + } + UsbPortInfo { + vid: 0, + pid: 0, + serial_number: None, + manufacturer: None, + product: None, + } +} + +/// Render a compact text description of the per-region results that +/// callers can log or return in `VerifyOutcome::Match::stdout`. Keeps +/// the outcome surface identical between the esptool and native paths +/// for anything that reads `stdout` for display. +pub(super) fn render_native_stdout(results: &[RegionVerifyResult]) -> String { + let mut out = String::new(); + for r in results { + let name = match r.region { + FlashRegion::Bootloader => "bootloader", + FlashRegion::Partitions => "partitions", + FlashRegion::Firmware => "firmware", + }; + let verdict = if r.matched { + "Verification successful (digest matched)." + } else { + "Verification failed (digest mismatch)." + }; + out.push_str(&format!("native verify: {}: {}\n", name, verdict)); + } + out +} + +pub(super) fn region_name(r: FlashRegion) -> &'static str { + match r { + FlashRegion::Bootloader => "bootloader", + FlashRegion::Partitions => "partitions", + FlashRegion::Firmware => "firmware", + } +} diff --git a/crates/fbuild-deploy/src/esp32_native/types.rs b/crates/fbuild-deploy/src/esp32_native/types.rs new file mode 100644 index 00000000..0f500c32 --- /dev/null +++ b/crates/fbuild-deploy/src/esp32_native/types.rs @@ -0,0 +1,26 @@ +//! Shared region types used by the native verify and write paths. +//! +//! Kept as a distinct module so a future change (e.g. encrypted-write +//! flags per-region) doesn't silently leak between paths. + +use crate::esp32::FlashRegion; + +/// A single region (flash offset + local firmware file) to verify. +#[derive(Debug, Clone)] +pub struct NativeVerifyRegion { + pub region: FlashRegion, + pub offset: u32, + pub path: std::path::PathBuf, +} + +/// A single region (flash offset + local firmware file) to write. +/// +/// Same shape as [`NativeVerifyRegion`] but kept as a distinct type so +/// a future change (e.g. encrypted-write flags per-region) doesn't +/// silently leak back into the verify path. +#[derive(Debug, Clone)] +pub struct NativeWriteRegion { + pub region: FlashRegion, + pub offset: u32, + pub path: std::path::PathBuf, +} diff --git a/crates/fbuild-deploy/src/esp32_native/verify.rs b/crates/fbuild-deploy/src/esp32_native/verify.rs new file mode 100644 index 00000000..684f9f84 --- /dev/null +++ b/crates/fbuild-deploy/src/esp32_native/verify.rs @@ -0,0 +1,203 @@ +//! Native `verify-flash` implementation backed by espflash. + +use std::path::Path; +use std::time::Duration; + +use espflash::connection::Connection; +use espflash::flasher::Flasher; +use serialport::FlowControl; + +use fbuild_core::{FbuildError, Result}; + +use crate::esp32::{FlashRegion, RegionVerifyResult, VerifyOutcome}; + +use super::transport::{ + discover_usb_port_info, local_md5, parse_after_reset, parse_before_reset, parse_chip, + render_native_stdout, +}; +use super::types::NativeVerifyRegion; + +/// Native `verify-flash` — reads each file from disk, asks the stub +/// flasher to compute `FLASH_MD5SUM` over the same region on-chip, and +/// reports per-region match/mismatch. +/// +/// On `Match` the chip is hard-reset (matching esptool's +/// `--after hard-reset`) so callers can treat a `Match` as "device is +/// now running the requested firmware" just like the subprocess path. +/// +/// Port ownership: caller must ensure the OS-level serial port is free +/// before calling (the daemon does this via `preempt_for_deploy`). The +/// opened handle is closed on function return. +/// +/// `before_reset` / `after_reset` accept the same strings the esptool +/// path uses (`default-reset`, `no-reset`, `no-reset-no-sync`, +/// `usb-reset` for before; `hard-reset`, `no-reset`, `no-reset-no-stub`, +/// `watchdog-reset` for after) so the daemon wiring doesn't need a +/// separate config surface. +#[allow(clippy::too_many_arguments)] +pub fn try_verify_deployment_native( + chip_name: &str, + port: &str, + baud: u32, + before_reset: &str, + after_reset: &str, + regions: &[NativeVerifyRegion], + bootloader_offset: u32, + partitions_offset: u32, + firmware_offset: u32, +) -> Result { + // Map config strings → espflash enums. We intentionally keep the + // surface narrow: anything outside the supported set is a hard + // error so a typo in a board JSON doesn't silently degrade to + // default-reset. + let chip = parse_chip(chip_name)?; + let before = parse_before_reset(before_reset)?; + let after = parse_after_reset(after_reset)?; + + // Open the port at 115_200 (espflash renegotiates to `baud` after + // stub upload). `open_native` returns the platform-specific `Port` + // (COMPort / TTYPort) expected by `Connection::new`. + let serial_port = serialport::new(port, 115_200) + .flow_control(FlowControl::None) + .timeout(Duration::from_secs(3)) + .open_native() + .map_err(|e| { + FbuildError::DeployFailed(format!( + "native verify: failed to open serial port {}: {}", + port, e + )) + })?; + + // Look up the USB VID/PID for the port so reset strategy selection + // in espflash (USB-JTAG vs classic) picks the right sequence. + // Missing entries default to zero, matching espflash's own CLI. + let port_info = discover_usb_port_info(port); + + let connection = Connection::new(serial_port, port_info, after, before, baud); + + // `Flasher::connect` handles reset, sync, chip detect, stub upload, + // and baud renegotiation. Errors here are fatal — on real hardware + // we treat them as "verify couldn't run, fall back to full flash" + // at the caller, so we surface them via `FbuildError` rather than + // embedding them in a `Mismatch`. + let use_stub = true; + let mut flasher = Flasher::connect( + connection, + use_stub, + /* verify */ false, // we do our own per-region verify below + /* skip */ false, + Some(chip), + Some(baud), + ) + .map_err(|e| { + FbuildError::DeployFailed(format!( + "native verify: espflash connect failed on {}: {}", + port, e + )) + })?; + + // Compute per-region verdicts. + let mut results: Vec = Vec::with_capacity(regions.len()); + for r in regions { + let bytes = std::fs::read(&r.path).map_err(|e| { + FbuildError::DeployFailed(format!( + "native verify: failed to read {}: {}", + r.path.display(), + e + )) + })?; + let local = local_md5(&bytes); + let remote = flasher + .checksum_md5(r.offset, bytes.len() as u32) + .map_err(|e| { + FbuildError::DeployFailed(format!( + "native verify: FLASH_MD5SUM failed at 0x{:x}: {}", + r.offset, e + )) + })?; + let matched = remote == local; + tracing::debug!( + port, + region = ?r.region, + offset = format!("0x{:x}", r.offset), + size = bytes.len(), + matched, + "native verify region result" + ); + results.push(RegionVerifyResult { + region: r.region, + matched, + }); + } + + // Keep the three offsets threaded through even though we no longer + // parse them from stdout — callers of `VerifyOutcome::Mismatch` use + // `regions` directly and ignore stdout/stderr when they came from + // the native path. + let _ = (bootloader_offset, partitions_offset, firmware_offset); + + let all_match = !results.is_empty() && results.iter().all(|r| r.matched); + + // Hard-reset the chip back into the app on success so behavior + // matches the esptool `--after hard-reset` contract. On mismatch we + // leave the reset policy to the subsequent flash call. + if all_match { + let mut connection = flasher.into_connection(); + if let Err(e) = connection.reset_after(use_stub, chip) { + tracing::warn!(port, "native verify: reset_after failed: {}", e); + } + } + + if all_match { + Ok(VerifyOutcome::Match { + stdout: render_native_stdout(&results), + stderr: String::new(), + }) + } else { + Ok(VerifyOutcome::Mismatch { + stdout: render_native_stdout(&results), + stderr: String::new(), + regions: results, + }) + } +} + +/// Collect the standard three-region set from a firmware path. The +/// caller supplies the offsets (parsed once from board config by +/// [`super::super::esp32::Esp32Deployer`]). +/// +/// Mirrors [`super::super::esp32::Esp32Deployer::build_verify_flash_args`]: +/// bootloader and partitions are optional (absent files are skipped); +/// firmware is mandatory. +pub fn collect_standard_regions( + firmware_path: &Path, + bootloader_offset: u32, + partitions_offset: u32, + firmware_offset: u32, +) -> Vec { + let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); + let bootloader_path = build_dir.join("bootloader.bin"); + let partitions_path = build_dir.join("partitions.bin"); + + let mut out = Vec::with_capacity(3); + if bootloader_path.exists() { + out.push(NativeVerifyRegion { + region: FlashRegion::Bootloader, + offset: bootloader_offset, + path: bootloader_path, + }); + } + if partitions_path.exists() { + out.push(NativeVerifyRegion { + region: FlashRegion::Partitions, + offset: partitions_offset, + path: partitions_path, + }); + } + out.push(NativeVerifyRegion { + region: FlashRegion::Firmware, + offset: firmware_offset, + path: firmware_path.to_path_buf(), + }); + out +} diff --git a/crates/fbuild-deploy/src/esp32_native/write.rs b/crates/fbuild-deploy/src/esp32_native/write.rs new file mode 100644 index 00000000..e1145377 --- /dev/null +++ b/crates/fbuild-deploy/src/esp32_native/write.rs @@ -0,0 +1,273 @@ +//! Native `write-flash` implementation backed by espflash, plus the +//! per-region collection helpers used by the daemon's selective flash +//! path. + +use std::path::Path; +use std::time::Duration; + +use espflash::connection::Connection; +use espflash::flasher::Flasher; +use serialport::FlowControl; + +use fbuild_core::{FbuildError, Result}; + +use crate::esp32::FlashRegion; +use crate::{DeployOutcome, DeploymentResult}; + +use super::progress::LoggingProgressBridge; +use super::transport::{ + discover_usb_port_info, parse_after_reset, parse_before_reset, parse_chip, region_name, +}; +use super::types::NativeWriteRegion; + +/// Native `write-flash` — reads each file from disk and streams it to +/// the chip via espflash's stub flasher. Same three regions +/// (bootloader / partitions / firmware) as the esptool path, same +/// [`DeploymentResult`] / [`DeployOutcome`] semantics so callers can +/// route between the two behind a single flag. +/// +/// Progress from espflash is bridged into `tracing` so the daemon's +/// existing log broadcaster surfaces it without new API surface (see +/// the private `LoggingProgressBridge` in [`super::progress`]). +/// Structured WebSocket progress frames are a follow-up: the bridge is +/// a drop-in replacement point for a richer callback without touching +/// any of the call sites. +/// +/// On success the chip is hard-reset (matching esptool's +/// `--after hard-reset`) so callers can treat the `Ok` return as +/// "device is now running the requested firmware" without an extra +/// reset. +/// +/// Port ownership: caller must ensure the OS-level serial port is free +/// before calling (the daemon does this via `preempt_for_deploy`). The +/// opened handle is closed on function return. +/// +/// Error recovery: on any region failure we short-circuit, log which +/// region failed with its flash offset, return a failed +/// [`DeploymentResult`], and let the caller decide whether to retry via +/// esptool. Partial writes are never silently swallowed. +#[allow(clippy::too_many_arguments)] +pub fn try_write_deployment_native( + chip_name: &str, + port: &str, + baud: u32, + before_reset: &str, + after_reset: &str, + regions: &[NativeWriteRegion], + selective: bool, +) -> Result { + let chip = parse_chip(chip_name)?; + let before = parse_before_reset(before_reset)?; + let after = parse_after_reset(after_reset)?; + + if regions.is_empty() { + return Err(FbuildError::DeployFailed( + "native write: called with no regions; at least firmware is required".to_string(), + )); + } + + let serial_port = serialport::new(port, 115_200) + .flow_control(FlowControl::None) + // Writes are much longer than verifies; the stub flasher can + // take tens of seconds between UART responses while erasing + // the partition table sector on a cold boot. 10s matches + // espflash's own CLI default. + .timeout(Duration::from_secs(10)) + .open_native() + .map_err(|e| { + FbuildError::DeployFailed(format!( + "native write: failed to open serial port {}: {}", + port, e + )) + })?; + + let port_info = discover_usb_port_info(port); + let connection = Connection::new(serial_port, port_info, after, before, baud); + + let use_stub = true; + let mut flasher = Flasher::connect( + connection, + use_stub, + /* verify */ + false, // espflash's own post-write verify; we rely on the separate verify path + /* skip */ false, + Some(chip), + Some(baud), + ) + .map_err(|e| { + FbuildError::DeployFailed(format!( + "native write: espflash connect failed on {}: {}", + port, e + )) + })?; + + let mut bridge = LoggingProgressBridge::new(port); + let mut rendered = String::new(); + let mut bytes_written: u64 = 0; + + for r in regions { + let bytes = std::fs::read(&r.path).map_err(|e| { + FbuildError::DeployFailed(format!( + "native write: failed to read {}: {}", + r.path.display(), + e + )) + })?; + let size = bytes.len(); + bridge.enter_region(r.region); + tracing::info!( + port, + region = ?r.region, + offset = format!("0x{:x}", r.offset), + size, + "native write: writing region" + ); + if let Err(e) = flasher.write_bin_to_flash(r.offset, &bytes, &mut bridge) { + let msg = format!( + "native write: region {:?} at 0x{:x} failed after {}/{} bytes: {}", + r.region, r.offset, bridge.last_current, size, e + ); + tracing::error!(port, "{}", msg); + rendered.push_str(&msg); + rendered.push('\n'); + return Ok(DeploymentResult { + success: false, + message: format!("espflash native write failed on {} ({})", port, chip_name), + port: Some(port.to_string()), + stdout: rendered, + stderr: e.to_string(), + outcome: outcome_for(selective, regions), + }); + } + bytes_written += size as u64; + rendered.push_str(&format!( + "native write: {} at 0x{:x} ({} bytes) OK\n", + region_name(r.region), + r.offset, + size + )); + } + + // Hard-reset (or whatever after_reset selects) the chip so the app + // boots once the stub releases the bus. Mirrors esptool's + // `--after hard-reset` contract. + let mut connection = flasher.into_connection(); + if let Err(e) = connection.reset_after(use_stub, chip) { + // A failed final reset is annoying but doesn't invalidate the + // flash: the image is on-device, a manual power cycle will + // boot it. Log and report success. + tracing::warn!(port, "native write: reset_after failed: {}", e); + } + + tracing::info!( + port, + bytes_written, + regions = regions.len(), + "native write: completed successfully" + ); + + Ok(DeploymentResult { + success: true, + message: format!( + "firmware flashed to {} ({}) via espflash ({} bytes, {} region(s))", + port, + chip_name, + bytes_written, + regions.len() + ), + port: Some(port.to_string()), + stdout: rendered, + stderr: String::new(), + outcome: outcome_for(selective, regions), + }) +} + +/// Pick the correct [`DeployOutcome`] for a native write — preserves +/// the existing invariant that the full three-region path reports +/// `FullFlash` and selective rewrites carry the region list. +pub(super) fn outcome_for(selective: bool, regions: &[NativeWriteRegion]) -> DeployOutcome { + if selective { + DeployOutcome::SelectiveFlash { + regions: regions.iter().map(|r| r.region).collect(), + } + } else { + DeployOutcome::FullFlash + } +} + +/// Collect the full three-region write set (bootloader + partitions + +/// firmware where present) from a firmware path. Mirrors +/// [`super::verify::collect_standard_regions`] but returns +/// [`NativeWriteRegion`]. +pub fn collect_standard_write_regions( + firmware_path: &Path, + bootloader_offset: u32, + partitions_offset: u32, + firmware_offset: u32, +) -> Vec { + let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); + let bootloader_path = build_dir.join("bootloader.bin"); + let partitions_path = build_dir.join("partitions.bin"); + + let mut out = Vec::with_capacity(3); + if bootloader_path.exists() { + out.push(NativeWriteRegion { + region: FlashRegion::Bootloader, + offset: bootloader_offset, + path: bootloader_path, + }); + } + if partitions_path.exists() { + out.push(NativeWriteRegion { + region: FlashRegion::Partitions, + offset: partitions_offset, + path: partitions_path, + }); + } + out.push(NativeWriteRegion { + region: FlashRegion::Firmware, + offset: firmware_offset, + path: firmware_path.to_path_buf(), + }); + out +} + +/// Collect a caller-chosen subset of write regions. Used by the daemon's +/// selective-flash path (post-verify-mismatch) so we don't rewrite +/// bootloader/partitions when only firmware differs. `requested` must +/// not be empty. +pub fn collect_selected_write_regions( + firmware_path: &Path, + bootloader_offset: u32, + partitions_offset: u32, + firmware_offset: u32, + requested: &[FlashRegion], +) -> Result> { + if requested.is_empty() { + return Err(FbuildError::DeployFailed( + "native write: collect_selected_write_regions called with empty request".to_string(), + )); + } + let build_dir = firmware_path.parent().unwrap_or_else(|| Path::new(".")); + let mut out = Vec::with_capacity(requested.len()); + for r in requested { + let (path, offset) = match r { + FlashRegion::Bootloader => (build_dir.join("bootloader.bin"), bootloader_offset), + FlashRegion::Partitions => (build_dir.join("partitions.bin"), partitions_offset), + FlashRegion::Firmware => (firmware_path.to_path_buf(), firmware_offset), + }; + if !path.exists() { + return Err(FbuildError::DeployFailed(format!( + "native write: requested {:?} but {} is missing", + r, + path.display() + ))); + } + out.push(NativeWriteRegion { + region: *r, + offset, + path, + }); + } + Ok(out) +} diff --git a/crates/fbuild-packages/src/disk_cache/index.rs b/crates/fbuild-packages/src/disk_cache/index.rs deleted file mode 100644 index c2b6a3c2..00000000 --- a/crates/fbuild-packages/src/disk_cache/index.rs +++ /dev/null @@ -1,1053 +0,0 @@ -//! SQLite-based crash-safe index for the two-phase disk cache. -//! -//! Opened in WAL mode with `synchronous=NORMAL` for multi-reader/single-writer -//! safety and crash recovery. The WAL is replayed on next open after a crash. -//! -//! # Schema migrations -//! -//! Migrations are tracked in the `schema_migrations` table keyed by a -//! stable migration id (`m001_initial_schema`, `m002_add_leases_refcount`, -//! ...). On open, every registered migration is applied in order, inside -//! its own transaction, unless it has already been recorded. This lets -//! older production databases (created before a column existed) pick up -//! schema deltas idempotently — adding a new migration is append-only. -//! -//! The legacy `cache_meta.schema_version` key is still written for -//! backwards compatibility with any tooling that inspects it, but the -//! authoritative source of truth is `schema_migrations`. - -use super::paths::{self, Kind}; -use rusqlite::{params, Connection, OptionalExtension}; -use std::path::{Path, PathBuf}; -use std::sync::Mutex; -use std::time::{SystemTime, UNIX_EPOCH}; - -/// Legacy version pin written to `cache_meta` for backwards compatibility. -/// -/// The real source of truth is the `schema_migrations` table. This value -/// is mirrored so older tools that read `cache_meta.schema_version` keep -/// working. -const LEGACY_SCHEMA_VERSION: i64 = 1; - -/// A single ordered schema migration. Applied idempotently on open. -/// -/// `id` must be stable and unique. Append new migrations to [`MIGRATIONS`] -/// — never reorder or rename existing ids. -struct Migration { - id: &'static str, - up: fn(&Connection) -> rusqlite::Result<()>, -} - -/// The ordered list of migrations. Append-only. -/// -/// - `m001_initial_schema` — the original tables + indexes. Uses -/// `CREATE TABLE IF NOT EXISTS` so it is safe on databases where these -/// objects already exist (e.g. older production caches). -/// - `m002_add_leases_refcount` — adds `leases.refcount` when absent. -/// Older databases (pre-#119) had a `leases` table without this column, -/// which broke `pin()`/`unpin()`. New databases already get it from -/// `m001`, so this migration only mutates old caches. -const MIGRATIONS: &[Migration] = &[ - Migration { - id: "m001_initial_schema", - up: migrate_m001_initial_schema, - }, - Migration { - id: "m002_add_leases_refcount", - up: migrate_m002_add_leases_refcount, - }, -]; - -/// A row in the `entries` table. -#[derive(Debug, Clone)] -pub struct CacheEntry { - pub id: i64, - pub kind: Kind, - pub url: String, - pub stem: String, - pub hash: String, - pub version: String, - pub archive_path: Option, - pub archive_bytes: Option, - pub archive_sha256: Option, - pub installed_path: Option, - pub installed_bytes: Option, - pub installed_at: Option, - pub archived_at: Option, - pub last_used_at: i64, - pub use_count: i64, - pub pinned: i64, -} - -/// The crash-safe SQLite index. -/// -/// Wraps the connection in a `Mutex` so the index is `Send + Sync` -/// and can be shared via `Arc` across threads. -pub struct CacheIndex { - conn: Mutex, - cache_root: PathBuf, -} - -impl CacheIndex { - /// Open (or create) the index at the standard location under `cache_root`. - /// Runs migrations if needed. - pub fn open(cache_root: &Path) -> rusqlite::Result { - let db_path = paths::index_path(cache_root); - std::fs::create_dir_all(cache_root).map_err(|e| { - rusqlite::Error::InvalidPath(PathBuf::from(format!( - "failed to create cache root: {}", - e - ))) - })?; - let conn = Connection::open(&db_path)?; - - // WAL mode for crash safety and concurrent access - conn.pragma_update(None, "journal_mode", "WAL")?; - conn.pragma_update(None, "synchronous", "NORMAL")?; - // Enable foreign keys - conn.pragma_update(None, "foreign_keys", "ON")?; - - let idx = Self { - conn: Mutex::new(conn), - cache_root: cache_root.to_path_buf(), - }; - idx.migrate()?; - Ok(idx) - } - - /// Open with an in-memory database (for testing). - #[cfg(test)] - pub fn open_in_memory() -> rusqlite::Result { - let conn = Connection::open_in_memory()?; - conn.pragma_update(None, "foreign_keys", "ON")?; - let idx = Self { - conn: Mutex::new(conn), - cache_root: PathBuf::from("/tmp/test_cache"), - }; - idx.migrate()?; - Ok(idx) - } - - fn migrate(&self) -> rusqlite::Result<()> { - let conn = self.conn.lock().unwrap(); - Self::run_migrations(&conn) - } - - /// Bootstrap the migrations-tracking table and run each registered - /// migration exactly once. Safe to call against fresh *or* pre-existing - /// databases; each migration is wrapped in its own transaction. - fn run_migrations(conn: &Connection) -> rusqlite::Result<()> { - // Bootstrap the migrations-tracking table. We can't use the - // migration framework itself for this because it's what records - // whether a migration ran. - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS schema_migrations ( - id TEXT PRIMARY KEY, - applied_at INTEGER NOT NULL - );", - )?; - - for m in MIGRATIONS { - let already_applied: bool = conn - .query_row( - "SELECT 1 FROM schema_migrations WHERE id = ?1", - params![m.id], - |_| Ok(true), - ) - .optional()? - .unwrap_or(false); - if already_applied { - continue; - } - - let tx = conn.unchecked_transaction()?; - (m.up)(&tx)?; - tx.execute( - "INSERT INTO schema_migrations (id, applied_at) VALUES (?1, ?2)", - params![m.id, Self::now_epoch()], - )?; - tx.commit()?; - } - - // Mirror the legacy cache_meta.schema_version key so any tooling - // that inspects it sees the expected value. Best-effort — the - // authoritative source is now `schema_migrations`. - let _ = conn.execute( - "INSERT OR REPLACE INTO cache_meta (key, value) VALUES ('schema_version', ?1)", - params![LEGACY_SCHEMA_VERSION.to_string()], - ); - - Ok(()) - } - - /// Get the current schema version. - pub fn schema_version(&self) -> rusqlite::Result { - let conn = self.conn.lock().unwrap(); - conn.query_row( - "SELECT value FROM cache_meta WHERE key = 'schema_version'", - [], - |row| { - let v: String = row.get(0)?; - Ok(v.parse::().unwrap_or(0)) - }, - ) - .optional() - .map(|opt| opt.unwrap_or(0)) - } - - fn now_epoch() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() as i64 - } - - /// Look up an entry by kind, url, and version. - pub fn lookup( - &self, - kind: Kind, - url: &str, - version: &str, - ) -> rusqlite::Result> { - let (_, hash) = paths::stem_and_hash(url); - let conn = self.conn.lock().unwrap(); - conn.query_row( - "SELECT id, kind, url, stem, hash, version, - archive_path, archive_bytes, archive_sha256, - installed_path, installed_bytes, installed_at, - archived_at, last_used_at, use_count, pinned - FROM entries - WHERE kind = ?1 AND hash = ?2 AND version = ?3", - params![kind.as_str(), hash, version], - Self::row_to_entry, - ) - .optional() - } - - /// Record that an archive was downloaded. - pub fn record_archive( - &self, - kind: Kind, - url: &str, - version: &str, - archive_path: &str, - archive_bytes: i64, - archive_sha256: &str, - ) -> rusqlite::Result { - let (stem, hash) = paths::stem_and_hash(url); - let now = Self::now_epoch(); - let conn = self.conn.lock().unwrap(); - conn.execute( - "INSERT INTO entries (kind, url, stem, hash, version, - archive_path, archive_bytes, archive_sha256, - archived_at, last_used_at, use_count, pinned) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 0, 0) - ON CONFLICT(kind, hash, version) DO UPDATE SET - archive_path = excluded.archive_path, - archive_bytes = excluded.archive_bytes, - archive_sha256 = excluded.archive_sha256, - archived_at = excluded.archived_at, - last_used_at = excluded.last_used_at", - params![ - kind.as_str(), - url, - stem, - hash, - version, - archive_path, - archive_bytes, - archive_sha256, - now, - now, - ], - )?; - drop(conn); - - self.lookup(kind, url, version) - .map(|opt| opt.expect("entry must exist after insert")) - } - - /// Record that an archive was extracted/installed. - pub fn record_install( - &self, - kind: Kind, - url: &str, - version: &str, - installed_path: &str, - installed_bytes: i64, - ) -> rusqlite::Result { - let (stem, hash) = paths::stem_and_hash(url); - let now = Self::now_epoch(); - let conn = self.conn.lock().unwrap(); - conn.execute( - "INSERT INTO entries (kind, url, stem, hash, version, - installed_path, installed_bytes, installed_at, - last_used_at, use_count, pinned) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 0, 0) - ON CONFLICT(kind, hash, version) DO UPDATE SET - installed_path = excluded.installed_path, - installed_bytes = excluded.installed_bytes, - installed_at = excluded.installed_at, - last_used_at = excluded.last_used_at", - params![ - kind.as_str(), - url, - stem, - hash, - version, - installed_path, - installed_bytes, - now, - now, - ], - )?; - drop(conn); - - self.lookup(kind, url, version) - .map(|opt| opt.expect("entry must exist after insert")) - } - - /// Bump LRU timestamp and use count for an entry. - pub fn touch(&self, entry_id: i64) -> rusqlite::Result<()> { - let now = Self::now_epoch(); - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE entries SET last_used_at = ?1, use_count = use_count + 1 WHERE id = ?2", - params![now, entry_id], - )?; - Ok(()) - } - - /// Increment the pinned count for an entry (lease acquired). - /// Uses a per-(PID, nonce) refcount so multiple `Lease` guards in the same - /// process correctly track independent pins. - pub fn pin(&self, entry_id: i64, holder_pid: u32, holder_nonce: u64) -> rusqlite::Result<()> { - let now = Self::now_epoch(); - let conn = self.conn.lock().unwrap(); - let updated = conn.execute( - "UPDATE leases SET refcount = refcount + 1 - WHERE entry_id = ?1 AND holder_pid = ?2 AND holder_nonce = ?3", - params![entry_id, holder_pid as i64, holder_nonce as i64], - )?; - if updated == 0 { - conn.execute( - "INSERT INTO leases (entry_id, holder_pid, holder_nonce, refcount, acquired_at) - VALUES (?1, ?2, ?3, 1, ?4)", - params![entry_id, holder_pid as i64, holder_nonce as i64, now], - )?; - } - conn.execute( - "UPDATE entries SET pinned = (SELECT COALESCE(SUM(refcount), 0) FROM leases WHERE entry_id = ?1) WHERE id = ?1", - params![entry_id], - )?; - Ok(()) - } - - /// Decrement the pinned count for an entry (lease released). - /// Decrements refcount; removes the row when it reaches zero. - pub fn unpin(&self, entry_id: i64, holder_pid: u32, holder_nonce: u64) -> rusqlite::Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE leases SET refcount = refcount - 1 - WHERE entry_id = ?1 AND holder_pid = ?2 AND holder_nonce = ?3", - params![entry_id, holder_pid as i64, holder_nonce as i64], - )?; - conn.execute( - "DELETE FROM leases - WHERE entry_id = ?1 AND holder_pid = ?2 AND holder_nonce = ?3 AND refcount <= 0", - params![entry_id, holder_pid as i64, holder_nonce as i64], - )?; - conn.execute( - "UPDATE entries SET pinned = (SELECT COALESCE(SUM(refcount), 0) FROM leases WHERE entry_id = ?1) WHERE id = ?1", - params![entry_id], - )?; - Ok(()) - } - - /// Get total bytes for all archives. - pub fn total_archive_bytes(&self) -> rusqlite::Result { - let conn = self.conn.lock().unwrap(); - conn.query_row( - "SELECT COALESCE(SUM(archive_bytes), 0) FROM entries WHERE archive_path IS NOT NULL", - [], - |row| row.get::<_, i64>(0), - ) - } - - /// Get total bytes for all installed entries. - pub fn total_installed_bytes(&self) -> rusqlite::Result { - let conn = self.conn.lock().unwrap(); - conn.query_row( - "SELECT COALESCE(SUM(installed_bytes), 0) FROM entries WHERE installed_path IS NOT NULL", - [], - |row| row.get::<_, i64>(0), - ) - } - - /// Get LRU installed entries (oldest first), skipping pinned. - pub fn lru_installed_entries(&self, limit: usize) -> rusqlite::Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( - "SELECT id, kind, url, stem, hash, version, - archive_path, archive_bytes, archive_sha256, - installed_path, installed_bytes, installed_at, - archived_at, last_used_at, use_count, pinned - FROM entries - WHERE installed_path IS NOT NULL AND pinned = 0 - ORDER BY last_used_at ASC - LIMIT ?1", - )?; - - let entries = stmt - .query_map(params![limit as i64], Self::row_to_entry)? - .collect::, _>>()?; - Ok(entries) - } - - /// Get LRU archive entries (oldest first), skipping pinned. - pub fn lru_archive_entries(&self, limit: usize) -> rusqlite::Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( - "SELECT id, kind, url, stem, hash, version, - archive_path, archive_bytes, archive_sha256, - installed_path, installed_bytes, installed_at, - archived_at, last_used_at, use_count, pinned - FROM entries - WHERE archive_path IS NOT NULL AND pinned = 0 - ORDER BY last_used_at ASC - LIMIT ?1", - )?; - - let entries = stmt - .query_map(params![limit as i64], Self::row_to_entry)? - .collect::, _>>()?; - Ok(entries) - } - - /// Null out the installed_path for an entry (after evicting its directory). - pub fn clear_installed(&self, entry_id: i64) -> rusqlite::Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE entries SET installed_path = NULL, installed_bytes = NULL, installed_at = NULL WHERE id = ?1", - params![entry_id], - )?; - Ok(()) - } - - /// Null out the archive_path for an entry (after evicting its archive). - pub fn clear_archive(&self, entry_id: i64) -> rusqlite::Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE entries SET archive_path = NULL, archive_bytes = NULL, archive_sha256 = NULL, archived_at = NULL WHERE id = ?1", - params![entry_id], - )?; - Ok(()) - } - - /// Delete an entry entirely (when both archive and installed are gone). - pub fn delete_entry(&self, entry_id: i64) -> rusqlite::Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute("DELETE FROM entries WHERE id = ?1", params![entry_id])?; - Ok(()) - } - - /// Reap leases for dead PIDs. - pub fn reap_dead_leases(&self) -> rusqlite::Result { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare("SELECT DISTINCT holder_pid FROM leases")?; - let pids: Vec = stmt - .query_map([], |row| row.get::<_, i64>(0))? - .collect::, _>>()?; - drop(stmt); - - let mut reaped = 0; - for pid in pids { - if !is_pid_alive(pid as u32) { - conn.execute("DELETE FROM leases WHERE holder_pid = ?1", params![pid])?; - reaped += 1; - } - } - - // Refresh pinned counts for all affected entries — use SUM(refcount) - // to match pin()/unpin() which also use SUM(refcount), not COUNT(*). - if reaped > 0 { - conn.execute_batch( - "UPDATE entries SET pinned = (SELECT COALESCE(SUM(refcount), 0) FROM leases WHERE leases.entry_id = entries.id)", - )?; - } - - Ok(reaped) - } - - /// Get all entries (for reconciliation). - pub fn all_entries(&self) -> rusqlite::Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( - "SELECT id, kind, url, stem, hash, version, - archive_path, archive_bytes, archive_sha256, - installed_path, installed_bytes, installed_at, - archived_at, last_used_at, use_count, pinned - FROM entries", - )?; - - let entries = stmt - .query_map([], Self::row_to_entry)? - .collect::, _>>()?; - Ok(entries) - } - - /// Look up the most recently used entry for a kind+url pair (any version). - /// Returns the entry with the highest `last_used_at` that has an installed path. - pub fn lookup_latest(&self, kind: Kind, url: &str) -> rusqlite::Result> { - let (_, hash) = paths::stem_and_hash(url); - let conn = self.conn.lock().unwrap(); - conn.query_row( - "SELECT id, kind, url, stem, hash, version, - archive_path, archive_bytes, archive_sha256, - installed_path, installed_bytes, installed_at, - archived_at, last_used_at, use_count, pinned - FROM entries - WHERE kind = ?1 AND hash = ?2 AND installed_path IS NOT NULL - ORDER BY last_used_at DESC - LIMIT 1", - params![kind.as_str(), hash], - Self::row_to_entry, - ) - .optional() - } - - /// Get total entry count. - pub fn entry_count(&self) -> rusqlite::Result { - let conn = self.conn.lock().unwrap(); - conn.query_row("SELECT COUNT(*) FROM entries", [], |row| { - row.get::<_, i64>(0) - }) - } - - /// Get a reference to the cache root path. - pub fn cache_root(&self) -> &Path { - &self.cache_root - } - - fn row_to_entry(row: &rusqlite::Row<'_>) -> rusqlite::Result { - let kind_str: String = row.get(1)?; - let kind: Kind = kind_str.parse().map_err(|e: String| { - rusqlite::Error::FromSqlConversionFailure( - 1, - rusqlite::types::Type::Text, - Box::new(std::io::Error::new(std::io::ErrorKind::InvalidData, e)), - ) - })?; - Ok(CacheEntry { - id: row.get(0)?, - kind, - url: row.get(2)?, - stem: row.get(3)?, - hash: row.get(4)?, - version: row.get(5)?, - archive_path: row.get(6)?, - archive_bytes: row.get(7)?, - archive_sha256: row.get(8)?, - installed_path: row.get(9)?, - installed_bytes: row.get(10)?, - installed_at: row.get(11)?, - archived_at: row.get(12)?, - last_used_at: row.get(13)?, - use_count: row.get(14)?, - pinned: row.get(15)?, - }) - } -} - -/// m001 — original schema. Creates `cache_meta`, `entries`, the LRU -/// indexes, and `leases` (with `refcount`). Uses `IF NOT EXISTS` so it -/// is safe against caches that already have these objects from the -/// pre-migration-framework era. -fn migrate_m001_initial_schema(conn: &Connection) -> rusqlite::Result<()> { - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS cache_meta ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS entries ( - id INTEGER PRIMARY KEY, - kind TEXT NOT NULL, - url TEXT NOT NULL, - stem TEXT NOT NULL, - hash TEXT NOT NULL, - version TEXT NOT NULL, - archive_path TEXT, - archive_bytes INTEGER, - archive_sha256 TEXT, - installed_path TEXT, - installed_bytes INTEGER, - installed_at INTEGER, - archived_at INTEGER, - last_used_at INTEGER NOT NULL, - use_count INTEGER NOT NULL DEFAULT 0, - pinned INTEGER NOT NULL DEFAULT 0, - UNIQUE(kind, hash, version) - ); - - CREATE INDEX IF NOT EXISTS idx_lru_installed - ON entries(last_used_at) WHERE installed_path IS NOT NULL; - CREATE INDEX IF NOT EXISTS idx_lru_archive - ON entries(last_used_at) WHERE archive_path IS NOT NULL; - - CREATE TABLE IF NOT EXISTS leases ( - entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE, - holder_pid INTEGER NOT NULL, - holder_nonce INTEGER NOT NULL DEFAULT 0, - refcount INTEGER NOT NULL DEFAULT 1, - acquired_at INTEGER NOT NULL, - PRIMARY KEY(entry_id, holder_pid, holder_nonce) - );", - )?; - Ok(()) -} - -/// m002 — add `leases.refcount` on pre-existing caches. -/// -/// Older fbuild versions created `leases` without the `refcount` column. -/// New caches already get it from `m001_initial_schema`, so this -/// migration is a no-op for them. For old ones, we `ALTER TABLE` to add -/// the column with `DEFAULT 1` (matching the semantic that existing rows -/// represent a single held lease) and then rebaseline to `1` in case the -/// column was added but left at its implicit NULL by older sqlite. -fn migrate_m002_add_leases_refcount(conn: &Connection) -> rusqlite::Result<()> { - if leases_has_refcount(conn)? { - return Ok(()); - } - conn.execute_batch( - "ALTER TABLE leases ADD COLUMN refcount INTEGER NOT NULL DEFAULT 1; - UPDATE leases SET refcount = 1 WHERE refcount IS NULL OR refcount <= 0;", - )?; - Ok(()) -} - -/// Returns true iff the `leases` table has a column named `refcount`. -/// Used by migrations to decide whether an `ALTER TABLE` is needed. -fn leases_has_refcount(conn: &Connection) -> rusqlite::Result { - let mut stmt = conn.prepare("PRAGMA table_info(leases)")?; - let rows = stmt.query_map([], |row| row.get::<_, String>(1))?; - for col in rows { - if col? == "refcount" { - return Ok(true); - } - } - Ok(false) -} - -/// Check if a PID is alive. Platform-specific. -fn is_pid_alive(pid: u32) -> bool { - #[cfg(unix)] - { - // kill(pid, 0) checks if process exists without sending a signal. - // Use raw FFI to avoid a libc crate dependency (matters for musl builds). - extern "C" { - fn kill(pid: i32, sig: i32) -> i32; - } - unsafe { kill(pid as i32, 0) == 0 } - } - #[cfg(windows)] - { - // Use OpenProcess to check if PID is alive (fast, no subprocess). - // PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 - extern "system" { - fn OpenProcess(access: u32, inherit: i32, pid: u32) -> *mut std::ffi::c_void; - fn CloseHandle(handle: *mut std::ffi::c_void) -> i32; - } - const PROCESS_QUERY_LIMITED_INFORMATION: u32 = 0x1000; - let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; - if handle.is_null() { - false - } else { - unsafe { CloseHandle(handle) }; - true - } - } - #[cfg(not(any(unix, windows)))] - { - let _ = pid; - false - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_index_open_creates_schema() { - let tmp = tempfile::TempDir::new().unwrap(); - let idx = CacheIndex::open(tmp.path()).unwrap(); - assert_eq!(idx.schema_version().unwrap(), 1); - assert_eq!(idx.entry_count().unwrap(), 0); - } - - #[test] - fn test_index_reopen_preserves_data() { - let tmp = tempfile::TempDir::new().unwrap(); - { - let idx = CacheIndex::open(tmp.path()).unwrap(); - idx.record_archive( - Kind::Packages, - "https://example.com/pkg.tar.gz", - "1.0.0", - "archives/packages/example-pkg/abc123/1.0.0/pkg.tar.gz", - 1024, - "sha256abc", - ) - .unwrap(); - } - // Reopen - let idx = CacheIndex::open(tmp.path()).unwrap(); - let entry = idx - .lookup(Kind::Packages, "https://example.com/pkg.tar.gz", "1.0.0") - .unwrap(); - assert!(entry.is_some()); - let entry = entry.unwrap(); - assert_eq!(entry.archive_bytes, Some(1024)); - } - - #[test] - fn test_record_archive_then_install_roundtrip() { - let idx = CacheIndex::open_in_memory().unwrap(); - let url = "https://example.com/tool.tar.gz"; - - let entry = idx - .record_archive( - Kind::Toolchains, - url, - "7.3.0", - "archives/tool.tar.gz", - 5000, - "deadbeef", - ) - .unwrap(); - assert_eq!(entry.kind, Kind::Toolchains); - assert_eq!(entry.version, "7.3.0"); - assert_eq!(entry.archive_bytes, Some(5000)); - assert!(entry.installed_path.is_none()); - - let entry = idx - .record_install( - Kind::Toolchains, - url, - "7.3.0", - "installed/toolchains/tool/abc/7.3.0", - 20000, - ) - .unwrap(); - assert_eq!(entry.archive_bytes, Some(5000)); // archive still there - assert_eq!(entry.installed_bytes, Some(20000)); - assert!(entry.installed_path.is_some()); - } - - #[test] - fn test_lookup_returns_none_for_missing() { - let idx = CacheIndex::open_in_memory().unwrap(); - let result = idx - .lookup(Kind::Packages, "https://example.com/nope", "1.0.0") - .unwrap(); - assert!(result.is_none()); - } - - #[test] - fn test_touch_bumps_use_count() { - let idx = CacheIndex::open_in_memory().unwrap(); - let entry = idx - .record_archive( - Kind::Packages, - "https://example.com/a", - "1.0", - "path", - 100, - "sha", - ) - .unwrap(); - assert_eq!(entry.use_count, 0); - - idx.touch(entry.id).unwrap(); - let entry = idx - .lookup(Kind::Packages, "https://example.com/a", "1.0") - .unwrap() - .unwrap(); - assert_eq!(entry.use_count, 1); - - idx.touch(entry.id).unwrap(); - let entry = idx - .lookup(Kind::Packages, "https://example.com/a", "1.0") - .unwrap() - .unwrap(); - assert_eq!(entry.use_count, 2); - } - - #[test] - fn test_pin_and_unpin() { - let idx = CacheIndex::open_in_memory().unwrap(); - let entry = idx - .record_archive( - Kind::Packages, - "https://example.com/a", - "1.0", - "path", - 100, - "sha", - ) - .unwrap(); - assert_eq!(entry.pinned, 0); - - idx.pin(entry.id, 12345, 1).unwrap(); - let entry = idx - .lookup(Kind::Packages, "https://example.com/a", "1.0") - .unwrap() - .unwrap(); - assert_eq!(entry.pinned, 1); - - // Pin with another PID - idx.pin(entry.id, 67890, 2).unwrap(); - let entry = idx - .lookup(Kind::Packages, "https://example.com/a", "1.0") - .unwrap() - .unwrap(); - assert_eq!(entry.pinned, 2); - - // Unpin one - idx.unpin(entry.id, 12345, 1).unwrap(); - let entry = idx - .lookup(Kind::Packages, "https://example.com/a", "1.0") - .unwrap() - .unwrap(); - assert_eq!(entry.pinned, 1); - } - - #[test] - fn test_lru_installed_entries_skip_pinned() { - let idx = CacheIndex::open_in_memory().unwrap(); - - // Create two installed entries - let e1 = idx - .record_install( - Kind::Packages, - "https://example.com/a", - "1.0", - "path_a", - 1000, - ) - .unwrap(); - let _e2 = idx - .record_install( - Kind::Packages, - "https://example.com/b", - "1.0", - "path_b", - 2000, - ) - .unwrap(); - - // Pin e1 - idx.pin(e1.id, 99999, 1).unwrap(); - - // LRU should only return e2 (unpinned) - let lru = idx.lru_installed_entries(10).unwrap(); - assert_eq!(lru.len(), 1); - assert_eq!(lru[0].url, "https://example.com/b"); - } - - #[test] - fn test_clear_installed_nulls_fields() { - let idx = CacheIndex::open_in_memory().unwrap(); - let url = "https://example.com/pkg"; - idx.record_archive(Kind::Packages, url, "1.0", "archive_path", 100, "sha") - .unwrap(); - let entry = idx - .record_install(Kind::Packages, url, "1.0", "install_path", 500) - .unwrap(); - - idx.clear_installed(entry.id).unwrap(); - let entry = idx.lookup(Kind::Packages, url, "1.0").unwrap().unwrap(); - assert!(entry.installed_path.is_none()); - assert!(entry.installed_bytes.is_none()); - // Archive still present - assert!(entry.archive_path.is_some()); - } - - #[test] - fn test_clear_archive_nulls_fields() { - let idx = CacheIndex::open_in_memory().unwrap(); - let url = "https://example.com/pkg"; - let entry = idx - .record_archive(Kind::Packages, url, "1.0", "archive_path", 100, "sha") - .unwrap(); - - idx.clear_archive(entry.id).unwrap(); - let entry = idx.lookup(Kind::Packages, url, "1.0").unwrap().unwrap(); - assert!(entry.archive_path.is_none()); - assert!(entry.archive_bytes.is_none()); - assert!(entry.archive_sha256.is_none()); - } - - #[test] - fn test_delete_entry() { - let idx = CacheIndex::open_in_memory().unwrap(); - let entry = idx - .record_archive( - Kind::Packages, - "https://example.com/a", - "1.0", - "path", - 100, - "sha", - ) - .unwrap(); - assert_eq!(idx.entry_count().unwrap(), 1); - - idx.delete_entry(entry.id).unwrap(); - assert_eq!(idx.entry_count().unwrap(), 0); - } - - #[test] - fn test_total_bytes_accounting() { - let idx = CacheIndex::open_in_memory().unwrap(); - assert_eq!(idx.total_archive_bytes().unwrap(), 0); - assert_eq!(idx.total_installed_bytes().unwrap(), 0); - - idx.record_archive(Kind::Packages, "https://a.com/a", "1.0", "p1", 1000, "s1") - .unwrap(); - idx.record_archive(Kind::Packages, "https://b.com/b", "1.0", "p2", 2000, "s2") - .unwrap(); - assert_eq!(idx.total_archive_bytes().unwrap(), 3000); - - idx.record_install(Kind::Toolchains, "https://c.com/c", "1.0", "p3", 5000) - .unwrap(); - assert_eq!(idx.total_installed_bytes().unwrap(), 5000); - } - - #[test] - fn test_reconcile_orphan_row_nulled() { - let tmp = tempfile::TempDir::new().unwrap(); - let idx = CacheIndex::open(tmp.path()).unwrap(); - - let entry = idx - .record_archive( - Kind::Packages, - "https://example.com/orphan", - "1.0", - "archives/packages/orphan/hash/1.0/pkg.tar.gz", - 500, - "sha", - ) - .unwrap(); - - assert!(entry.archive_path.is_some()); - idx.clear_archive(entry.id).unwrap(); - let entry = idx - .lookup(Kind::Packages, "https://example.com/orphan", "1.0") - .unwrap() - .unwrap(); - assert!(entry.archive_path.is_none()); - assert_eq!(idx.entry_count().unwrap(), 1); - } - - /// A freshly-migrated database must report every registered migration - /// as applied — regression guard against adding a migration to - /// `MIGRATIONS` but forgetting to record it. - #[test] - fn test_all_migrations_recorded_after_open() { - let tmp = tempfile::TempDir::new().unwrap(); - let idx = CacheIndex::open(tmp.path()).unwrap(); - let conn = idx.conn.lock().unwrap(); - for m in MIGRATIONS { - let applied: Option = conn - .query_row( - "SELECT 1 FROM schema_migrations WHERE id = ?1", - params![m.id], - |row| row.get(0), - ) - .optional() - .unwrap(); - assert!(applied.is_some(), "migration {} not recorded", m.id); - } - } - - /// Pre-migration schema: `leases` without `refcount`. Opening via - /// `CacheIndex::open` must transparently add the column, and - /// subsequent `pin()` calls must succeed. - #[test] - fn test_legacy_schema_missing_refcount_is_migrated() { - let tmp = tempfile::TempDir::new().unwrap(); - let db_path = paths::index_path(tmp.path()); - std::fs::create_dir_all(db_path.parent().unwrap()).unwrap(); - - // Hand-craft the legacy schema: leases has no refcount column. - { - let raw = Connection::open(&db_path).unwrap(); - raw.execute_batch( - "CREATE TABLE cache_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); - CREATE TABLE entries ( - id INTEGER PRIMARY KEY, - kind TEXT NOT NULL, - url TEXT NOT NULL, - stem TEXT NOT NULL, - hash TEXT NOT NULL, - version TEXT NOT NULL, - archive_path TEXT, - archive_bytes INTEGER, - archive_sha256 TEXT, - installed_path TEXT, - installed_bytes INTEGER, - installed_at INTEGER, - archived_at INTEGER, - last_used_at INTEGER NOT NULL, - use_count INTEGER NOT NULL DEFAULT 0, - pinned INTEGER NOT NULL DEFAULT 0, - UNIQUE(kind, hash, version) - ); - CREATE TABLE leases ( - entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE, - holder_pid INTEGER NOT NULL, - holder_nonce INTEGER NOT NULL DEFAULT 0, - acquired_at INTEGER NOT NULL, - PRIMARY KEY(entry_id, holder_pid, holder_nonce) - ); - INSERT INTO cache_meta(key, value) VALUES ('schema_version', '1');", - ) - .unwrap(); - // raw goes out of scope → connection closes and file is flushed. - } - - // Re-open through the normal path. Migrations should run. - let idx = CacheIndex::open(tmp.path()).unwrap(); - assert!( - leases_has_refcount(&idx.conn.lock().unwrap()).unwrap(), - "m002 should have added leases.refcount" - ); - - // pin()/unpin() must now succeed end-to-end. - let entry = idx - .record_archive(Kind::Packages, "https://example.com/x", "1.0", "p", 1, "s") - .unwrap(); - idx.pin(entry.id, 4242, 7).unwrap(); - let entry = idx - .lookup(Kind::Packages, "https://example.com/x", "1.0") - .unwrap() - .unwrap(); - assert_eq!(entry.pinned, 1); - idx.unpin(entry.id, 4242, 7).unwrap(); - } - - /// Migrations must be idempotent: opening twice must not re-apply - /// them and must not double-error on the `ALTER TABLE`. - #[test] - fn test_migrations_idempotent_across_reopens() { - let tmp = tempfile::TempDir::new().unwrap(); - { - let _idx = CacheIndex::open(tmp.path()).unwrap(); - } - // Second open must succeed — no "duplicate column name" error. - let idx = CacheIndex::open(tmp.path()).unwrap(); - assert!(leases_has_refcount(&idx.conn.lock().unwrap()).unwrap()); - } -} diff --git a/crates/fbuild-packages/src/disk_cache/index/README.md b/crates/fbuild-packages/src/disk_cache/index/README.md new file mode 100644 index 00000000..48e7b92f --- /dev/null +++ b/crates/fbuild-packages/src/disk_cache/index/README.md @@ -0,0 +1,13 @@ +# disk_cache/index + +SQLite-backed crash-safe index for the two-phase disk cache. + +Split across multiple files so each stays under the workspace LOC gate: + +- `mod.rs` — core types (`CacheEntry`, `CacheIndex`), open / migrate / lifecycle. +- `queries.rs` — lookup, mutation, lease bookkeeping, LRU, reconciliation queries. +- `migrations.rs` — append-only ordered migrations and helpers. +- `pid.rs` — platform-specific PID liveness check used for lease reaping. +- `tests.rs` — index test suite (gated by `#[cfg(test)]`). + +External callers still import as `super::index::{CacheIndex, CacheEntry}`. diff --git a/crates/fbuild-packages/src/disk_cache/index/migrations.rs b/crates/fbuild-packages/src/disk_cache/index/migrations.rs new file mode 100644 index 00000000..ee5deb79 --- /dev/null +++ b/crates/fbuild-packages/src/disk_cache/index/migrations.rs @@ -0,0 +1,116 @@ +//! Append-only schema migrations for the SQLite cache index. +//! +//! Each migration has a stable id (e.g. `m001_initial_schema`) recorded in +//! the `schema_migrations` table. Append new migrations to [`MIGRATIONS`] — +//! never reorder or rename existing ids. + +use rusqlite::Connection; + +/// A single ordered schema migration. Applied idempotently on open. +/// +/// `id` must be stable and unique. Append new migrations to [`MIGRATIONS`] +/// — never reorder or rename existing ids. +pub(super) struct Migration { + pub(super) id: &'static str, + pub(super) up: fn(&Connection) -> rusqlite::Result<()>, +} + +/// The ordered list of migrations. Append-only. +/// +/// - `m001_initial_schema` — the original tables + indexes. Uses +/// `CREATE TABLE IF NOT EXISTS` so it is safe on databases where these +/// objects already exist (e.g. older production caches). +/// - `m002_add_leases_refcount` — adds `leases.refcount` when absent. +/// Older databases (pre-#119) had a `leases` table without this column, +/// which broke `pin()`/`unpin()`. New databases already get it from +/// `m001`, so this migration only mutates old caches. +pub(super) const MIGRATIONS: &[Migration] = &[ + Migration { + id: "m001_initial_schema", + up: migrate_m001_initial_schema, + }, + Migration { + id: "m002_add_leases_refcount", + up: migrate_m002_add_leases_refcount, + }, +]; + +/// m001 — original schema. Creates `cache_meta`, `entries`, the LRU +/// indexes, and `leases` (with `refcount`). Uses `IF NOT EXISTS` so it +/// is safe against caches that already have these objects from the +/// pre-migration-framework era. +fn migrate_m001_initial_schema(conn: &Connection) -> rusqlite::Result<()> { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS cache_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS entries ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + url TEXT NOT NULL, + stem TEXT NOT NULL, + hash TEXT NOT NULL, + version TEXT NOT NULL, + archive_path TEXT, + archive_bytes INTEGER, + archive_sha256 TEXT, + installed_path TEXT, + installed_bytes INTEGER, + installed_at INTEGER, + archived_at INTEGER, + last_used_at INTEGER NOT NULL, + use_count INTEGER NOT NULL DEFAULT 0, + pinned INTEGER NOT NULL DEFAULT 0, + UNIQUE(kind, hash, version) + ); + + CREATE INDEX IF NOT EXISTS idx_lru_installed + ON entries(last_used_at) WHERE installed_path IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_lru_archive + ON entries(last_used_at) WHERE archive_path IS NOT NULL; + + CREATE TABLE IF NOT EXISTS leases ( + entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE, + holder_pid INTEGER NOT NULL, + holder_nonce INTEGER NOT NULL DEFAULT 0, + refcount INTEGER NOT NULL DEFAULT 1, + acquired_at INTEGER NOT NULL, + PRIMARY KEY(entry_id, holder_pid, holder_nonce) + );", + )?; + Ok(()) +} + +/// m002 — add `leases.refcount` on pre-existing caches. +/// +/// Older fbuild versions created `leases` without the `refcount` column. +/// New caches already get it from `m001_initial_schema`, so this +/// migration is a no-op for them. For old ones, we `ALTER TABLE` to add +/// the column with `DEFAULT 1` (matching the semantic that existing rows +/// represent a single held lease) and then rebaseline to `1` in case the +/// column was added but left at its implicit NULL by older sqlite. +fn migrate_m002_add_leases_refcount(conn: &Connection) -> rusqlite::Result<()> { + if leases_has_refcount(conn)? { + return Ok(()); + } + conn.execute_batch( + "ALTER TABLE leases ADD COLUMN refcount INTEGER NOT NULL DEFAULT 1; + UPDATE leases SET refcount = 1 WHERE refcount IS NULL OR refcount <= 0;", + )?; + Ok(()) +} + +/// Returns true iff the `leases` table has a column named `refcount`. +/// Used by migrations to decide whether an `ALTER TABLE` is needed. +pub(super) fn leases_has_refcount(conn: &Connection) -> rusqlite::Result { + let mut stmt = conn.prepare("PRAGMA table_info(leases)")?; + let rows = stmt.query_map([], |row| row.get::<_, String>(1))?; + for col in rows { + if col? == "refcount" { + return Ok(true); + } + } + Ok(false) +} diff --git a/crates/fbuild-packages/src/disk_cache/index/mod.rs b/crates/fbuild-packages/src/disk_cache/index/mod.rs new file mode 100644 index 00000000..86432f30 --- /dev/null +++ b/crates/fbuild-packages/src/disk_cache/index/mod.rs @@ -0,0 +1,205 @@ +//! SQLite-based crash-safe index for the two-phase disk cache. +//! +//! Opened in WAL mode with `synchronous=NORMAL` for multi-reader/single-writer +//! safety and crash recovery. The WAL is replayed on next open after a crash. +//! +//! # Schema migrations +//! +//! Migrations are tracked in the `schema_migrations` table keyed by a +//! stable migration id (`m001_initial_schema`, `m002_add_leases_refcount`, +//! ...). On open, every registered migration is applied in order, inside +//! its own transaction, unless it has already been recorded. This lets +//! older production databases (created before a column existed) pick up +//! schema deltas idempotently — adding a new migration is append-only. +//! +//! The legacy `cache_meta.schema_version` key is still written for +//! backwards compatibility with any tooling that inspects it, but the +//! authoritative source of truth is `schema_migrations`. +//! +//! # Module layout +//! +//! This module is split across several files to keep individual files +//! under the workspace LOC gate: +//! +//! - `mod.rs` — core types ([`CacheEntry`], [`CacheIndex`]) plus +//! `open`/`migrate`/`schema_version` lifecycle. +//! - `queries.rs` — `impl CacheIndex` block with all lookup / mutation / +//! lease / LRU / reconciliation methods. +//! - `migrations.rs` — append-only migration list and helpers. +//! - `pid.rs` — platform-specific PID liveness check. +//! - `tests.rs` — the index test suite (gated by `#[cfg(test)]`). +//! +//! The public API is unchanged: external callers continue to refer to +//! `super::index::{CacheIndex, CacheEntry}`. + +use super::paths::{self, Kind}; +use rusqlite::{params, Connection, OptionalExtension}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; + +mod migrations; +mod pid; +mod queries; + +#[cfg(test)] +mod tests; + +use migrations::MIGRATIONS; + +/// Legacy version pin written to `cache_meta` for backwards compatibility. +/// +/// The real source of truth is the `schema_migrations` table. This value +/// is mirrored so older tools that read `cache_meta.schema_version` keep +/// working. +const LEGACY_SCHEMA_VERSION: i64 = 1; + +/// A row in the `entries` table. +#[derive(Debug, Clone)] +pub struct CacheEntry { + pub id: i64, + pub kind: Kind, + pub url: String, + pub stem: String, + pub hash: String, + pub version: String, + pub archive_path: Option, + pub archive_bytes: Option, + pub archive_sha256: Option, + pub installed_path: Option, + pub installed_bytes: Option, + pub installed_at: Option, + pub archived_at: Option, + pub last_used_at: i64, + pub use_count: i64, + pub pinned: i64, +} + +/// The crash-safe SQLite index. +/// +/// Wraps the connection in a `Mutex` so the index is `Send + Sync` +/// and can be shared via `Arc` across threads. +pub struct CacheIndex { + pub(super) conn: Mutex, + cache_root: PathBuf, +} + +impl CacheIndex { + /// Open (or create) the index at the standard location under `cache_root`. + /// Runs migrations if needed. + pub fn open(cache_root: &Path) -> rusqlite::Result { + let db_path = paths::index_path(cache_root); + std::fs::create_dir_all(cache_root).map_err(|e| { + rusqlite::Error::InvalidPath(PathBuf::from(format!( + "failed to create cache root: {}", + e + ))) + })?; + let conn = Connection::open(&db_path)?; + + // WAL mode for crash safety and concurrent access + conn.pragma_update(None, "journal_mode", "WAL")?; + conn.pragma_update(None, "synchronous", "NORMAL")?; + // Enable foreign keys + conn.pragma_update(None, "foreign_keys", "ON")?; + + let idx = Self { + conn: Mutex::new(conn), + cache_root: cache_root.to_path_buf(), + }; + idx.migrate()?; + Ok(idx) + } + + /// Open with an in-memory database (for testing). + #[cfg(test)] + pub fn open_in_memory() -> rusqlite::Result { + let conn = Connection::open_in_memory()?; + conn.pragma_update(None, "foreign_keys", "ON")?; + let idx = Self { + conn: Mutex::new(conn), + cache_root: PathBuf::from("/tmp/test_cache"), + }; + idx.migrate()?; + Ok(idx) + } + + fn migrate(&self) -> rusqlite::Result<()> { + let conn = self.conn.lock().unwrap(); + Self::run_migrations(&conn) + } + + /// Bootstrap the migrations-tracking table and run each registered + /// migration exactly once. Safe to call against fresh *or* pre-existing + /// databases; each migration is wrapped in its own transaction. + fn run_migrations(conn: &Connection) -> rusqlite::Result<()> { + // Bootstrap the migrations-tracking table. We can't use the + // migration framework itself for this because it's what records + // whether a migration ran. + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS schema_migrations ( + id TEXT PRIMARY KEY, + applied_at INTEGER NOT NULL + );", + )?; + + for m in MIGRATIONS { + let already_applied: bool = conn + .query_row( + "SELECT 1 FROM schema_migrations WHERE id = ?1", + params![m.id], + |_| Ok(true), + ) + .optional()? + .unwrap_or(false); + if already_applied { + continue; + } + + let tx = conn.unchecked_transaction()?; + (m.up)(&tx)?; + tx.execute( + "INSERT INTO schema_migrations (id, applied_at) VALUES (?1, ?2)", + params![m.id, Self::now_epoch()], + )?; + tx.commit()?; + } + + // Mirror the legacy cache_meta.schema_version key so any tooling + // that inspects it sees the expected value. Best-effort — the + // authoritative source is now `schema_migrations`. + let _ = conn.execute( + "INSERT OR REPLACE INTO cache_meta (key, value) VALUES ('schema_version', ?1)", + params![LEGACY_SCHEMA_VERSION.to_string()], + ); + + Ok(()) + } + + /// Get the current schema version. + pub fn schema_version(&self) -> rusqlite::Result { + let conn = self.conn.lock().unwrap(); + conn.query_row( + "SELECT value FROM cache_meta WHERE key = 'schema_version'", + [], + |row| { + let v: String = row.get(0)?; + Ok(v.parse::().unwrap_or(0)) + }, + ) + .optional() + .map(|opt| opt.unwrap_or(0)) + } + + pub(super) fn now_epoch() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 + } + + /// Get a reference to the cache root path. + pub fn cache_root(&self) -> &Path { + &self.cache_root + } +} diff --git a/crates/fbuild-packages/src/disk_cache/index/pid.rs b/crates/fbuild-packages/src/disk_cache/index/pid.rs new file mode 100644 index 00000000..9fffc30c --- /dev/null +++ b/crates/fbuild-packages/src/disk_cache/index/pid.rs @@ -0,0 +1,36 @@ +//! Platform-specific PID liveness check used by lease reaping. + +/// Check if a PID is alive. Platform-specific. +pub(super) fn is_pid_alive(pid: u32) -> bool { + #[cfg(unix)] + { + // kill(pid, 0) checks if process exists without sending a signal. + // Use raw FFI to avoid a libc crate dependency (matters for musl builds). + extern "C" { + fn kill(pid: i32, sig: i32) -> i32; + } + unsafe { kill(pid as i32, 0) == 0 } + } + #[cfg(windows)] + { + // Use OpenProcess to check if PID is alive (fast, no subprocess). + // PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 + extern "system" { + fn OpenProcess(access: u32, inherit: i32, pid: u32) -> *mut std::ffi::c_void; + fn CloseHandle(handle: *mut std::ffi::c_void) -> i32; + } + const PROCESS_QUERY_LIMITED_INFORMATION: u32 = 0x1000; + let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; + if handle.is_null() { + false + } else { + unsafe { CloseHandle(handle) }; + true + } + } + #[cfg(not(any(unix, windows)))] + { + let _ = pid; + false + } +} diff --git a/crates/fbuild-packages/src/disk_cache/index/queries.rs b/crates/fbuild-packages/src/disk_cache/index/queries.rs new file mode 100644 index 00000000..aabd582c --- /dev/null +++ b/crates/fbuild-packages/src/disk_cache/index/queries.rs @@ -0,0 +1,366 @@ +//! Query, mutation, and lease-bookkeeping methods on [`CacheIndex`]. +//! +//! Split out from `mod.rs` to keep individual files under the LOC gate. +//! All methods continue to live on the same `CacheIndex` type via +//! additional `impl` blocks — public API is unchanged. + +use rusqlite::params; + +use super::super::paths::{self, Kind}; +use super::pid::is_pid_alive; +use super::{CacheEntry, CacheIndex}; + +impl CacheIndex { + /// Look up an entry by kind, url, and version. + pub fn lookup( + &self, + kind: Kind, + url: &str, + version: &str, + ) -> rusqlite::Result> { + use rusqlite::OptionalExtension; + let (_, hash) = paths::stem_and_hash(url); + let conn = self.conn.lock().unwrap(); + conn.query_row( + "SELECT id, kind, url, stem, hash, version, + archive_path, archive_bytes, archive_sha256, + installed_path, installed_bytes, installed_at, + archived_at, last_used_at, use_count, pinned + FROM entries + WHERE kind = ?1 AND hash = ?2 AND version = ?3", + params![kind.as_str(), hash, version], + Self::row_to_entry, + ) + .optional() + } + + /// Record that an archive was downloaded. + pub fn record_archive( + &self, + kind: Kind, + url: &str, + version: &str, + archive_path: &str, + archive_bytes: i64, + archive_sha256: &str, + ) -> rusqlite::Result { + let (stem, hash) = paths::stem_and_hash(url); + let now = Self::now_epoch(); + let conn = self.conn.lock().unwrap(); + conn.execute( + "INSERT INTO entries (kind, url, stem, hash, version, + archive_path, archive_bytes, archive_sha256, + archived_at, last_used_at, use_count, pinned) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 0, 0) + ON CONFLICT(kind, hash, version) DO UPDATE SET + archive_path = excluded.archive_path, + archive_bytes = excluded.archive_bytes, + archive_sha256 = excluded.archive_sha256, + archived_at = excluded.archived_at, + last_used_at = excluded.last_used_at", + params![ + kind.as_str(), + url, + stem, + hash, + version, + archive_path, + archive_bytes, + archive_sha256, + now, + now, + ], + )?; + drop(conn); + + self.lookup(kind, url, version) + .map(|opt| opt.expect("entry must exist after insert")) + } + + /// Record that an archive was extracted/installed. + pub fn record_install( + &self, + kind: Kind, + url: &str, + version: &str, + installed_path: &str, + installed_bytes: i64, + ) -> rusqlite::Result { + let (stem, hash) = paths::stem_and_hash(url); + let now = Self::now_epoch(); + let conn = self.conn.lock().unwrap(); + conn.execute( + "INSERT INTO entries (kind, url, stem, hash, version, + installed_path, installed_bytes, installed_at, + last_used_at, use_count, pinned) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 0, 0) + ON CONFLICT(kind, hash, version) DO UPDATE SET + installed_path = excluded.installed_path, + installed_bytes = excluded.installed_bytes, + installed_at = excluded.installed_at, + last_used_at = excluded.last_used_at", + params![ + kind.as_str(), + url, + stem, + hash, + version, + installed_path, + installed_bytes, + now, + now, + ], + )?; + drop(conn); + + self.lookup(kind, url, version) + .map(|opt| opt.expect("entry must exist after insert")) + } + + /// Bump LRU timestamp and use count for an entry. + pub fn touch(&self, entry_id: i64) -> rusqlite::Result<()> { + let now = Self::now_epoch(); + let conn = self.conn.lock().unwrap(); + conn.execute( + "UPDATE entries SET last_used_at = ?1, use_count = use_count + 1 WHERE id = ?2", + params![now, entry_id], + )?; + Ok(()) + } + + /// Increment the pinned count for an entry (lease acquired). + /// Uses a per-(PID, nonce) refcount so multiple `Lease` guards in the same + /// process correctly track independent pins. + pub fn pin(&self, entry_id: i64, holder_pid: u32, holder_nonce: u64) -> rusqlite::Result<()> { + let now = Self::now_epoch(); + let conn = self.conn.lock().unwrap(); + let updated = conn.execute( + "UPDATE leases SET refcount = refcount + 1 + WHERE entry_id = ?1 AND holder_pid = ?2 AND holder_nonce = ?3", + params![entry_id, holder_pid as i64, holder_nonce as i64], + )?; + if updated == 0 { + conn.execute( + "INSERT INTO leases (entry_id, holder_pid, holder_nonce, refcount, acquired_at) + VALUES (?1, ?2, ?3, 1, ?4)", + params![entry_id, holder_pid as i64, holder_nonce as i64, now], + )?; + } + conn.execute( + "UPDATE entries SET pinned = (SELECT COALESCE(SUM(refcount), 0) FROM leases WHERE entry_id = ?1) WHERE id = ?1", + params![entry_id], + )?; + Ok(()) + } + + /// Decrement the pinned count for an entry (lease released). + /// Decrements refcount; removes the row when it reaches zero. + pub fn unpin(&self, entry_id: i64, holder_pid: u32, holder_nonce: u64) -> rusqlite::Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "UPDATE leases SET refcount = refcount - 1 + WHERE entry_id = ?1 AND holder_pid = ?2 AND holder_nonce = ?3", + params![entry_id, holder_pid as i64, holder_nonce as i64], + )?; + conn.execute( + "DELETE FROM leases + WHERE entry_id = ?1 AND holder_pid = ?2 AND holder_nonce = ?3 AND refcount <= 0", + params![entry_id, holder_pid as i64, holder_nonce as i64], + )?; + conn.execute( + "UPDATE entries SET pinned = (SELECT COALESCE(SUM(refcount), 0) FROM leases WHERE entry_id = ?1) WHERE id = ?1", + params![entry_id], + )?; + Ok(()) + } + + /// Get total bytes for all archives. + pub fn total_archive_bytes(&self) -> rusqlite::Result { + let conn = self.conn.lock().unwrap(); + conn.query_row( + "SELECT COALESCE(SUM(archive_bytes), 0) FROM entries WHERE archive_path IS NOT NULL", + [], + |row| row.get::<_, i64>(0), + ) + } + + /// Get total bytes for all installed entries. + pub fn total_installed_bytes(&self) -> rusqlite::Result { + let conn = self.conn.lock().unwrap(); + conn.query_row( + "SELECT COALESCE(SUM(installed_bytes), 0) FROM entries WHERE installed_path IS NOT NULL", + [], + |row| row.get::<_, i64>(0), + ) + } + + /// Get LRU installed entries (oldest first), skipping pinned. + pub fn lru_installed_entries(&self, limit: usize) -> rusqlite::Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare( + "SELECT id, kind, url, stem, hash, version, + archive_path, archive_bytes, archive_sha256, + installed_path, installed_bytes, installed_at, + archived_at, last_used_at, use_count, pinned + FROM entries + WHERE installed_path IS NOT NULL AND pinned = 0 + ORDER BY last_used_at ASC + LIMIT ?1", + )?; + + let entries = stmt + .query_map(params![limit as i64], Self::row_to_entry)? + .collect::, _>>()?; + Ok(entries) + } + + /// Get LRU archive entries (oldest first), skipping pinned. + pub fn lru_archive_entries(&self, limit: usize) -> rusqlite::Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare( + "SELECT id, kind, url, stem, hash, version, + archive_path, archive_bytes, archive_sha256, + installed_path, installed_bytes, installed_at, + archived_at, last_used_at, use_count, pinned + FROM entries + WHERE archive_path IS NOT NULL AND pinned = 0 + ORDER BY last_used_at ASC + LIMIT ?1", + )?; + + let entries = stmt + .query_map(params![limit as i64], Self::row_to_entry)? + .collect::, _>>()?; + Ok(entries) + } + + /// Null out the installed_path for an entry (after evicting its directory). + pub fn clear_installed(&self, entry_id: i64) -> rusqlite::Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "UPDATE entries SET installed_path = NULL, installed_bytes = NULL, installed_at = NULL WHERE id = ?1", + params![entry_id], + )?; + Ok(()) + } + + /// Null out the archive_path for an entry (after evicting its archive). + pub fn clear_archive(&self, entry_id: i64) -> rusqlite::Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "UPDATE entries SET archive_path = NULL, archive_bytes = NULL, archive_sha256 = NULL, archived_at = NULL WHERE id = ?1", + params![entry_id], + )?; + Ok(()) + } + + /// Delete an entry entirely (when both archive and installed are gone). + pub fn delete_entry(&self, entry_id: i64) -> rusqlite::Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute("DELETE FROM entries WHERE id = ?1", params![entry_id])?; + Ok(()) + } + + /// Reap leases for dead PIDs. + pub fn reap_dead_leases(&self) -> rusqlite::Result { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare("SELECT DISTINCT holder_pid FROM leases")?; + let pids: Vec = stmt + .query_map([], |row| row.get::<_, i64>(0))? + .collect::, _>>()?; + drop(stmt); + + let mut reaped = 0; + for pid in pids { + if !is_pid_alive(pid as u32) { + conn.execute("DELETE FROM leases WHERE holder_pid = ?1", params![pid])?; + reaped += 1; + } + } + + // Refresh pinned counts for all affected entries — use SUM(refcount) + // to match pin()/unpin() which also use SUM(refcount), not COUNT(*). + if reaped > 0 { + conn.execute_batch( + "UPDATE entries SET pinned = (SELECT COALESCE(SUM(refcount), 0) FROM leases WHERE leases.entry_id = entries.id)", + )?; + } + + Ok(reaped) + } + + /// Get all entries (for reconciliation). + pub fn all_entries(&self) -> rusqlite::Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare( + "SELECT id, kind, url, stem, hash, version, + archive_path, archive_bytes, archive_sha256, + installed_path, installed_bytes, installed_at, + archived_at, last_used_at, use_count, pinned + FROM entries", + )?; + + let entries = stmt + .query_map([], Self::row_to_entry)? + .collect::, _>>()?; + Ok(entries) + } + + /// Look up the most recently used entry for a kind+url pair (any version). + /// Returns the entry with the highest `last_used_at` that has an installed path. + pub fn lookup_latest(&self, kind: Kind, url: &str) -> rusqlite::Result> { + use rusqlite::OptionalExtension; + let (_, hash) = paths::stem_and_hash(url); + let conn = self.conn.lock().unwrap(); + conn.query_row( + "SELECT id, kind, url, stem, hash, version, + archive_path, archive_bytes, archive_sha256, + installed_path, installed_bytes, installed_at, + archived_at, last_used_at, use_count, pinned + FROM entries + WHERE kind = ?1 AND hash = ?2 AND installed_path IS NOT NULL + ORDER BY last_used_at DESC + LIMIT 1", + params![kind.as_str(), hash], + Self::row_to_entry, + ) + .optional() + } + + /// Get total entry count. + pub fn entry_count(&self) -> rusqlite::Result { + let conn = self.conn.lock().unwrap(); + conn.query_row("SELECT COUNT(*) FROM entries", [], |row| { + row.get::<_, i64>(0) + }) + } + + pub(super) fn row_to_entry(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let kind_str: String = row.get(1)?; + let kind: Kind = kind_str.parse().map_err(|e: String| { + rusqlite::Error::FromSqlConversionFailure( + 1, + rusqlite::types::Type::Text, + Box::new(std::io::Error::new(std::io::ErrorKind::InvalidData, e)), + ) + })?; + Ok(CacheEntry { + id: row.get(0)?, + kind, + url: row.get(2)?, + stem: row.get(3)?, + hash: row.get(4)?, + version: row.get(5)?, + archive_path: row.get(6)?, + archive_bytes: row.get(7)?, + archive_sha256: row.get(8)?, + installed_path: row.get(9)?, + installed_bytes: row.get(10)?, + installed_at: row.get(11)?, + archived_at: row.get(12)?, + last_used_at: row.get(13)?, + use_count: row.get(14)?, + pinned: row.get(15)?, + }) + } +} diff --git a/crates/fbuild-packages/src/disk_cache/index/tests.rs b/crates/fbuild-packages/src/disk_cache/index/tests.rs new file mode 100644 index 00000000..1acb630f --- /dev/null +++ b/crates/fbuild-packages/src/disk_cache/index/tests.rs @@ -0,0 +1,380 @@ +//! Tests for the SQLite cache index. + +use rusqlite::{params, Connection, OptionalExtension}; + +use super::super::paths::{self, Kind}; +use super::migrations::{leases_has_refcount, MIGRATIONS}; +use super::CacheIndex; + +#[test] +fn test_index_open_creates_schema() { + let tmp = tempfile::TempDir::new().unwrap(); + let idx = CacheIndex::open(tmp.path()).unwrap(); + assert_eq!(idx.schema_version().unwrap(), 1); + assert_eq!(idx.entry_count().unwrap(), 0); +} + +#[test] +fn test_index_reopen_preserves_data() { + let tmp = tempfile::TempDir::new().unwrap(); + { + let idx = CacheIndex::open(tmp.path()).unwrap(); + idx.record_archive( + Kind::Packages, + "https://example.com/pkg.tar.gz", + "1.0.0", + "archives/packages/example-pkg/abc123/1.0.0/pkg.tar.gz", + 1024, + "sha256abc", + ) + .unwrap(); + } + // Reopen + let idx = CacheIndex::open(tmp.path()).unwrap(); + let entry = idx + .lookup(Kind::Packages, "https://example.com/pkg.tar.gz", "1.0.0") + .unwrap(); + assert!(entry.is_some()); + let entry = entry.unwrap(); + assert_eq!(entry.archive_bytes, Some(1024)); +} + +#[test] +fn test_record_archive_then_install_roundtrip() { + let idx = CacheIndex::open_in_memory().unwrap(); + let url = "https://example.com/tool.tar.gz"; + + let entry = idx + .record_archive( + Kind::Toolchains, + url, + "7.3.0", + "archives/tool.tar.gz", + 5000, + "deadbeef", + ) + .unwrap(); + assert_eq!(entry.kind, Kind::Toolchains); + assert_eq!(entry.version, "7.3.0"); + assert_eq!(entry.archive_bytes, Some(5000)); + assert!(entry.installed_path.is_none()); + + let entry = idx + .record_install( + Kind::Toolchains, + url, + "7.3.0", + "installed/toolchains/tool/abc/7.3.0", + 20000, + ) + .unwrap(); + assert_eq!(entry.archive_bytes, Some(5000)); // archive still there + assert_eq!(entry.installed_bytes, Some(20000)); + assert!(entry.installed_path.is_some()); +} + +#[test] +fn test_lookup_returns_none_for_missing() { + let idx = CacheIndex::open_in_memory().unwrap(); + let result = idx + .lookup(Kind::Packages, "https://example.com/nope", "1.0.0") + .unwrap(); + assert!(result.is_none()); +} + +#[test] +fn test_touch_bumps_use_count() { + let idx = CacheIndex::open_in_memory().unwrap(); + let entry = idx + .record_archive( + Kind::Packages, + "https://example.com/a", + "1.0", + "path", + 100, + "sha", + ) + .unwrap(); + assert_eq!(entry.use_count, 0); + + idx.touch(entry.id).unwrap(); + let entry = idx + .lookup(Kind::Packages, "https://example.com/a", "1.0") + .unwrap() + .unwrap(); + assert_eq!(entry.use_count, 1); + + idx.touch(entry.id).unwrap(); + let entry = idx + .lookup(Kind::Packages, "https://example.com/a", "1.0") + .unwrap() + .unwrap(); + assert_eq!(entry.use_count, 2); +} + +#[test] +fn test_pin_and_unpin() { + let idx = CacheIndex::open_in_memory().unwrap(); + let entry = idx + .record_archive( + Kind::Packages, + "https://example.com/a", + "1.0", + "path", + 100, + "sha", + ) + .unwrap(); + assert_eq!(entry.pinned, 0); + + idx.pin(entry.id, 12345, 1).unwrap(); + let entry = idx + .lookup(Kind::Packages, "https://example.com/a", "1.0") + .unwrap() + .unwrap(); + assert_eq!(entry.pinned, 1); + + // Pin with another PID + idx.pin(entry.id, 67890, 2).unwrap(); + let entry = idx + .lookup(Kind::Packages, "https://example.com/a", "1.0") + .unwrap() + .unwrap(); + assert_eq!(entry.pinned, 2); + + // Unpin one + idx.unpin(entry.id, 12345, 1).unwrap(); + let entry = idx + .lookup(Kind::Packages, "https://example.com/a", "1.0") + .unwrap() + .unwrap(); + assert_eq!(entry.pinned, 1); +} + +#[test] +fn test_lru_installed_entries_skip_pinned() { + let idx = CacheIndex::open_in_memory().unwrap(); + + // Create two installed entries + let e1 = idx + .record_install( + Kind::Packages, + "https://example.com/a", + "1.0", + "path_a", + 1000, + ) + .unwrap(); + let _e2 = idx + .record_install( + Kind::Packages, + "https://example.com/b", + "1.0", + "path_b", + 2000, + ) + .unwrap(); + + // Pin e1 + idx.pin(e1.id, 99999, 1).unwrap(); + + // LRU should only return e2 (unpinned) + let lru = idx.lru_installed_entries(10).unwrap(); + assert_eq!(lru.len(), 1); + assert_eq!(lru[0].url, "https://example.com/b"); +} + +#[test] +fn test_clear_installed_nulls_fields() { + let idx = CacheIndex::open_in_memory().unwrap(); + let url = "https://example.com/pkg"; + idx.record_archive(Kind::Packages, url, "1.0", "archive_path", 100, "sha") + .unwrap(); + let entry = idx + .record_install(Kind::Packages, url, "1.0", "install_path", 500) + .unwrap(); + + idx.clear_installed(entry.id).unwrap(); + let entry = idx.lookup(Kind::Packages, url, "1.0").unwrap().unwrap(); + assert!(entry.installed_path.is_none()); + assert!(entry.installed_bytes.is_none()); + // Archive still present + assert!(entry.archive_path.is_some()); +} + +#[test] +fn test_clear_archive_nulls_fields() { + let idx = CacheIndex::open_in_memory().unwrap(); + let url = "https://example.com/pkg"; + let entry = idx + .record_archive(Kind::Packages, url, "1.0", "archive_path", 100, "sha") + .unwrap(); + + idx.clear_archive(entry.id).unwrap(); + let entry = idx.lookup(Kind::Packages, url, "1.0").unwrap().unwrap(); + assert!(entry.archive_path.is_none()); + assert!(entry.archive_bytes.is_none()); + assert!(entry.archive_sha256.is_none()); +} + +#[test] +fn test_delete_entry() { + let idx = CacheIndex::open_in_memory().unwrap(); + let entry = idx + .record_archive( + Kind::Packages, + "https://example.com/a", + "1.0", + "path", + 100, + "sha", + ) + .unwrap(); + assert_eq!(idx.entry_count().unwrap(), 1); + + idx.delete_entry(entry.id).unwrap(); + assert_eq!(idx.entry_count().unwrap(), 0); +} + +#[test] +fn test_total_bytes_accounting() { + let idx = CacheIndex::open_in_memory().unwrap(); + assert_eq!(idx.total_archive_bytes().unwrap(), 0); + assert_eq!(idx.total_installed_bytes().unwrap(), 0); + + idx.record_archive(Kind::Packages, "https://a.com/a", "1.0", "p1", 1000, "s1") + .unwrap(); + idx.record_archive(Kind::Packages, "https://b.com/b", "1.0", "p2", 2000, "s2") + .unwrap(); + assert_eq!(idx.total_archive_bytes().unwrap(), 3000); + + idx.record_install(Kind::Toolchains, "https://c.com/c", "1.0", "p3", 5000) + .unwrap(); + assert_eq!(idx.total_installed_bytes().unwrap(), 5000); +} + +#[test] +fn test_reconcile_orphan_row_nulled() { + let tmp = tempfile::TempDir::new().unwrap(); + let idx = CacheIndex::open(tmp.path()).unwrap(); + + let entry = idx + .record_archive( + Kind::Packages, + "https://example.com/orphan", + "1.0", + "archives/packages/orphan/hash/1.0/pkg.tar.gz", + 500, + "sha", + ) + .unwrap(); + + assert!(entry.archive_path.is_some()); + idx.clear_archive(entry.id).unwrap(); + let entry = idx + .lookup(Kind::Packages, "https://example.com/orphan", "1.0") + .unwrap() + .unwrap(); + assert!(entry.archive_path.is_none()); + assert_eq!(idx.entry_count().unwrap(), 1); +} + +/// A freshly-migrated database must report every registered migration +/// as applied — regression guard against adding a migration to +/// `MIGRATIONS` but forgetting to record it. +#[test] +fn test_all_migrations_recorded_after_open() { + let tmp = tempfile::TempDir::new().unwrap(); + let idx = CacheIndex::open(tmp.path()).unwrap(); + let conn = idx.conn.lock().unwrap(); + for m in MIGRATIONS { + let applied: Option = conn + .query_row( + "SELECT 1 FROM schema_migrations WHERE id = ?1", + params![m.id], + |row| row.get(0), + ) + .optional() + .unwrap(); + assert!(applied.is_some(), "migration {} not recorded", m.id); + } +} + +/// Pre-migration schema: `leases` without `refcount`. Opening via +/// `CacheIndex::open` must transparently add the column, and +/// subsequent `pin()` calls must succeed. +#[test] +fn test_legacy_schema_missing_refcount_is_migrated() { + let tmp = tempfile::TempDir::new().unwrap(); + let db_path = paths::index_path(tmp.path()); + std::fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + + // Hand-craft the legacy schema: leases has no refcount column. + { + let raw = Connection::open(&db_path).unwrap(); + raw.execute_batch( + "CREATE TABLE cache_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE entries ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + url TEXT NOT NULL, + stem TEXT NOT NULL, + hash TEXT NOT NULL, + version TEXT NOT NULL, + archive_path TEXT, + archive_bytes INTEGER, + archive_sha256 TEXT, + installed_path TEXT, + installed_bytes INTEGER, + installed_at INTEGER, + archived_at INTEGER, + last_used_at INTEGER NOT NULL, + use_count INTEGER NOT NULL DEFAULT 0, + pinned INTEGER NOT NULL DEFAULT 0, + UNIQUE(kind, hash, version) + ); + CREATE TABLE leases ( + entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE, + holder_pid INTEGER NOT NULL, + holder_nonce INTEGER NOT NULL DEFAULT 0, + acquired_at INTEGER NOT NULL, + PRIMARY KEY(entry_id, holder_pid, holder_nonce) + ); + INSERT INTO cache_meta(key, value) VALUES ('schema_version', '1');", + ) + .unwrap(); + // raw goes out of scope → connection closes and file is flushed. + } + + // Re-open through the normal path. Migrations should run. + let idx = CacheIndex::open(tmp.path()).unwrap(); + assert!( + leases_has_refcount(&idx.conn.lock().unwrap()).unwrap(), + "m002 should have added leases.refcount" + ); + + // pin()/unpin() must now succeed end-to-end. + let entry = idx + .record_archive(Kind::Packages, "https://example.com/x", "1.0", "p", 1, "s") + .unwrap(); + idx.pin(entry.id, 4242, 7).unwrap(); + let entry = idx + .lookup(Kind::Packages, "https://example.com/x", "1.0") + .unwrap() + .unwrap(); + assert_eq!(entry.pinned, 1); + idx.unpin(entry.id, 4242, 7).unwrap(); +} + +/// Migrations must be idempotent: opening twice must not re-apply +/// them and must not double-error on the `ALTER TABLE`. +#[test] +fn test_migrations_idempotent_across_reopens() { + let tmp = tempfile::TempDir::new().unwrap(); + { + let _idx = CacheIndex::open(tmp.path()).unwrap(); + } + // Second open must succeed — no "duplicate column name" error. + let idx = CacheIndex::open(tmp.path()).unwrap(); + assert!(leases_has_refcount(&idx.conn.lock().unwrap()).unwrap()); +} diff --git a/crates/fbuild-packages/src/library/esp32_framework.rs b/crates/fbuild-packages/src/library/esp32_framework.rs deleted file mode 100644 index 9630ed09..00000000 --- a/crates/fbuild-packages/src/library/esp32_framework.rs +++ /dev/null @@ -1,1077 +0,0 @@ -//! ESP32 Arduino framework package. -//! -//! Downloads and manages the Arduino-ESP32 core + ESP-IDF precompiled libraries. -//! This combines what PlatformIO splits into two packages: -//! - `framework-arduinoespressif32`: Arduino core, variants, libraries -//! - `framework-arduinoespressif32-libs`: ESP-IDF SDK includes + precompiled `.a` libs -//! -//! Key methods provide paths to: -//! - Core sources: `cores/esp32/` -//! - Board variants: `variants/{mcu}/` -//! - SDK include dirs: `tools/sdk/{mcu}/include/` (305+ paths) -//! - SDK precompiled libs: `tools/sdk/{mcu}/lib/` (100+ .a files) -//! - Linker scripts: `tools/sdk/{mcu}/ld/` -//! - Bootloader/partitions: `tools/sdk/{mcu}/bin/` - -use std::path::{Path, PathBuf}; - -use crate::{CacheSubdir, Framework, PackageBase, PackageInfo}; - -const ESP32_FRAMEWORK_VERSION: &str = "3.1.1"; -const ESP32_FRAMEWORK_URL: &str = - "https://github.com/pioarduino/arduino-esp32/releases/download/3.1.1/framework-arduinoespressif32-3.1.1.tar.gz"; - -/// ESP32 Arduino framework manager. -pub struct Esp32Framework { - base: PackageBase, - install_dir: Option, -} - -impl Esp32Framework { - /// Create with hardcoded URL (legacy, for tests). - pub fn new(project_dir: &Path, _mcu: &str) -> Self { - Self { - base: PackageBase::new( - "esp32-arduino", - ESP32_FRAMEWORK_VERSION, - ESP32_FRAMEWORK_URL, - ESP32_FRAMEWORK_URL, - None, - CacheSubdir::Platforms, - project_dir, - ), - install_dir: None, - } - } - - #[cfg(test)] - fn with_cache_root(project_dir: &Path, cache_root: &Path, _mcu: &str) -> Self { - Self { - base: PackageBase::with_cache_root( - "esp32-arduino", - ESP32_FRAMEWORK_VERSION, - ESP32_FRAMEWORK_URL, - ESP32_FRAMEWORK_URL, - None, - CacheSubdir::Platforms, - project_dir, - cache_root, - ), - install_dir: None, - } - } - - /// Create from a resolved URL (from platform.json). - /// - /// The orchestrator reads `platform.json` → `packages.framework-arduinoespressif32.version` - /// to get the correct download URL (e.g. espressif/arduino-esp32 release). - pub fn from_url(project_dir: &Path, url: &str) -> Self { - // Extract version from URL (e.g., "3.3.7" from ".../3.3.7/esp32-core-3.3.7.tar.xz") - let version = extract_framework_version(url); - - Self { - base: PackageBase::new( - "esp32-arduino", - &version, - url, - "framework-arduinoespressif32", - None, - CacheSubdir::Platforms, - project_dir, - ), - install_dir: None, - } - } - - /// Ensure the SDK libs are downloaded and extracted into the framework's `tools/` dir. - pub fn ensure_libs(&self, libs_url: &str) -> fbuild_core::Result<()> { - let root = self.resolved_dir(); - let tools_dir = root.join("tools"); - - // Already have SDK libs? Check both old (sdk/) and new (esp32-arduino-libs/) layouts - for dir_name in &["esp32-arduino-libs", "sdk"] { - let sdk_dir = tools_dir.join(dir_name); - if sdk_dir.exists() && sdk_dir.is_dir() { - if let Ok(mut entries) = std::fs::read_dir(&sdk_dir) { - if entries.next().is_some() { - return Ok(()); - } - } - } - } - - std::fs::create_dir_all(&tools_dir)?; - - // Check for already-downloaded archive (skip re-download) - let archive_filename = libs_url.rsplit('/').next().unwrap_or("libs.tar.xz"); - let archive_path = tools_dir.join(archive_filename); - - if !archive_path.exists() { - tracing::info!("downloading ESP32 SDK libs"); - let rt = tokio::runtime::Handle::try_current().ok(); - if let Some(handle) = rt { - handle.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; - } - } - - // Extract to a short temp path to avoid Windows MAX_PATH (260 char) limit. - // Then rename (atomic on same filesystem) to final location. - let temp_dir = std::env::temp_dir().join(format!("fbuild_sdk_{}", std::process::id())); - if temp_dir.exists() { - let _ = std::fs::remove_dir_all(&temp_dir); - } - std::fs::create_dir_all(&temp_dir)?; - - tracing::info!( - "extracting ESP32 SDK libs ({} MB)", - archive_path - .metadata() - .map(|m| m.len() / 1_000_000) - .unwrap_or(0) - ); - crate::extractor::extract(&archive_path, &temp_dir)?; - let _ = std::fs::remove_file(&archive_path); - - // Move extracted content to final tools/ dir (same filesystem = fast rename) - if let Ok(entries) = std::fs::read_dir(&temp_dir) { - for entry in entries.flatten() { - let src = entry.path(); - let dest = tools_dir.join(entry.file_name()); - if dest.exists() { - let _ = std::fs::remove_dir_all(&dest); - } - if std::fs::rename(&src, &dest).is_err() { - copy_dir_recursive(&src, &dest)?; - } - } - } - let _ = std::fs::remove_dir_all(&temp_dir); - - tracing::info!("ESP32 SDK libs installed"); - Ok(()) - } - - /// Ensure MCU-specific skeleton libs are downloaded and merged into the framework's `tools/` dir. - /// - /// Some MCUs (e.g. ESP32-C2, ESP32-C61) ship their SDK libs in a separate - /// skeleton package rather than the main `framework-arduinoespressif32-libs`. - /// This merges the skeleton into the existing `tools/` directory without - /// clobbering other MCU subdirs. - pub fn ensure_mcu_libs(&self, libs_url: &str, mcu: &str) -> fbuild_core::Result<()> { - let root = self.resolved_dir(); - let tools_dir = root.join("tools"); - - // Already have this MCU's SDK? Check both layouts. - for dir_name in &["esp32-arduino-libs", "sdk"] { - let mcu_dir = tools_dir.join(dir_name).join(mcu); - if mcu_dir.exists() && mcu_dir.is_dir() { - return Ok(()); - } - } - - std::fs::create_dir_all(&tools_dir)?; - - let archive_filename = libs_url.rsplit('/').next().unwrap_or("skeleton.zip"); - let archive_path = tools_dir.join(archive_filename); - - if !archive_path.exists() { - tracing::info!("downloading {} skeleton libs", mcu); - let rt = tokio::runtime::Handle::try_current().ok(); - if let Some(handle) = rt { - handle.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; - } - } - - let temp_dir = std::env::temp_dir().join(format!("fbuild_skel_{}", std::process::id())); - if temp_dir.exists() { - let _ = std::fs::remove_dir_all(&temp_dir); - } - std::fs::create_dir_all(&temp_dir)?; - - tracing::info!("extracting {} skeleton libs", mcu); - crate::extractor::extract(&archive_path, &temp_dir)?; - let _ = std::fs::remove_file(&archive_path); - - // Merge into tools/ — use copy_dir_recursive so existing MCU dirs are preserved. - if let Ok(entries) = std::fs::read_dir(&temp_dir) { - for entry in entries.flatten() { - let src = entry.path(); - let dest = tools_dir.join(entry.file_name()); - if src.is_dir() { - copy_dir_recursive(&src, &dest)?; - } else if std::fs::rename(&src, &dest).is_err() { - std::fs::copy(&src, &dest)?; - } - } - } - let _ = std::fs::remove_dir_all(&temp_dir); - - tracing::info!("{} skeleton libs installed", mcu); - Ok(()) - } - - /// Get the resolved root directory of the framework. - fn resolved_dir(&self) -> PathBuf { - self.install_dir - .clone() - .unwrap_or_else(|| find_framework_root(&self.base.install_path())) - } - - /// Validate the extracted framework has required structure. - fn validate(install_dir: &Path) -> fbuild_core::Result<()> { - let root = find_framework_root(install_dir); - - let cores_dir = root.join("cores").join("esp32"); - if !cores_dir.exists() { - return Err(fbuild_core::FbuildError::PackageError(format!( - "ESP32 framework missing cores/esp32/ directory (in {})", - root.display() - ))); - } - - let arduino_h = cores_dir.join("Arduino.h"); - if !arduino_h.exists() { - return Err(fbuild_core::FbuildError::PackageError( - "ESP32 framework missing cores/esp32/Arduino.h".to_string(), - )); - } - - Ok(()) - } - - /// Get the core source directory (e.g. `cores/esp32`). - pub fn get_core_dir(&self, core_name: &str) -> PathBuf { - self.resolved_dir().join("cores").join(core_name) - } - - /// Get the variant directory for a board (e.g. `variants/esp32c6`). - pub fn get_variant_dir(&self, variant_name: &str) -> PathBuf { - self.resolved_dir().join("variants").join(variant_name) - } - - /// Get the SDK directory for a given MCU. - /// - /// Tries new layout (`tools/esp32-arduino-libs/{mcu}`) first, falls back to - /// old layout (`tools/sdk/{mcu}`). - fn sdk_mcu_dir(&self, mcu: &str) -> PathBuf { - let root = self.resolved_dir(); - let new_path = root.join("tools").join("esp32-arduino-libs").join(mcu); - if new_path.exists() { - return new_path; - } - root.join("tools").join("sdk").join(mcu) - } - - fn sdk_memory_variant_dir(sdk_dir: &Path, requested: Option<&str>) -> Option { - if let Some(requested) = requested { - let requested_dir = sdk_dir.join(requested); - if requested_dir.exists() { - return Some(requested_dir); - } - } - - for variant in &["qio_opi", "dio_opi", "opi_opi", "qio_qspi", "dio_qspi"] { - let candidate = sdk_dir.join(variant); - if candidate.exists() { - return Some(candidate); - } - } - - None - } - - /// Get SDK include directories for a given MCU. - /// - /// Reads the `flags/includes` file from the SDK directory, which lists - /// all 305+ include paths. Falls back to scanning `include/` subdirectories. - pub fn get_sdk_include_dirs(&self, mcu: &str, memory_type: Option<&str>) -> Vec { - let root = self.resolved_dir(); - let sdk_dir = self.sdk_mcu_dir(mcu); - - // Try reading the includes list file (supports both -I and -iwithprefixbefore formats) - let includes_file = sdk_dir.join("flags").join("includes"); - if includes_file.exists() { - if let Ok(content) = std::fs::read_to_string(&includes_file) { - let include_base = sdk_dir.join("include"); - let mut dirs = parse_include_flags(&content, &include_base, &root); - - // Add flash/PSRAM variant include dir (contains sdkconfig.h). - // Try common variants in preference order; the correct one depends - // on board_build.flash_mode and board_build.arduino.memory_type. - if let Some(variant_dir) = Self::sdk_memory_variant_dir(&sdk_dir, memory_type) { - let v_include = variant_dir.join("include"); - if v_include.exists() { - dirs.push(v_include); - } - } - - return dirs; - } - } - - // Fallback: recursively scan include/ subdirectories. - // The 2.x framework (PlatformIO-compat) has deeply nested includes - // under tools/sdk/{mcu}/include/ (e.g., freertos/include/freertos, - // freertos/port/xtensa/include). - let include_dir = sdk_dir.join("include"); - if !include_dir.exists() { - return Vec::new(); - } - - let mut dirs = Vec::new(); - - // Prepend newlib/platform_include (provides assert.h, errno.h, time.h) - // which must come before SDK headers. PlatformIO also puts this first. - let newlib_platform = include_dir.join("newlib").join("platform_include"); - if newlib_platform.exists() { - dirs.push(newlib_platform); - } - - // Scan 4 levels deep — matches PlatformIO's actual include depth. - // ESP-IDF components have nested includes up to 4 levels deep - // (e.g., freertos/include/esp_additions/freertos/). - scan_include_dirs_recursive(&include_dir, &mut dirs, 0, 4); - - // Add well-known ESP-IDF Xtensa/RISC-V port include paths that the - // scanner misses because headers are nested too deeply for detection. - for sub_mcu in &["esp32", "esp32s2", "esp32s3"] { - let xtensa_inc = include_dir.join("xtensa").join(sub_mcu).join("include"); - if xtensa_inc.exists() && !dirs.contains(&xtensa_inc) { - dirs.push(xtensa_inc); - } - } - - // Also add flash/PSRAM variant include dir (contains sdkconfig.h). - if let Some(variant_dir) = Self::sdk_memory_variant_dir(&sdk_dir, memory_type) { - let v_include = variant_dir.join("include"); - if v_include.exists() { - dirs.push(v_include); - } - } - - dirs.sort(); - dirs - } - - /// Get all precompiled `.a` library files from the ESP-IDF SDK. - pub fn get_sdk_libs(&self, mcu: &str) -> Vec { - let lib_dir = self.sdk_mcu_dir(mcu).join("lib"); - collect_archive_files(&lib_dir) - } - - /// Get the ordered SDK linker library flags from `flags/ld_libs`. - /// - /// Returns the pre-ordered `-l` flags (with duplicates for circular deps) - /// as specified by the SDK. Falls back to scanning `lib/` for `.a` files - /// if the flags file doesn't exist. - pub fn get_sdk_lib_flags(&self, mcu: &str, memory_type: Option<&str>) -> Vec { - let sdk_dir = self.sdk_mcu_dir(mcu); - let ld_libs_file = sdk_dir.join("flags").join("ld_libs"); - - if let Ok(content) = std::fs::read_to_string(&ld_libs_file) { - let mut flags = vec![format!("-L{}", sdk_dir.join("lib").display())]; - // Add ld/ directory as a library search path - let ld_dir = sdk_dir.join("ld"); - if ld_dir.exists() { - flags.push(format!("-L{}", ld_dir.display())); - } - // Add flash-mode-specific directory (contains libspi_flash.a and others). - // Default to dio_qspi (most common for ESP32dev boards). - if let Some(variant_dir) = Self::sdk_memory_variant_dir(&sdk_dir, memory_type) { - flags.push(format!("-L{}", variant_dir.display())); - } - flags.extend(fbuild_core::shell_split::split(&content)); - return flags; - } - - // Fallback: scan lib/ directory for .a files - let lib_dir = sdk_dir.join("lib"); - let mut flags = Vec::new(); - if lib_dir.exists() { - flags.push(format!("-L{}", lib_dir.display())); - } - for lib in collect_archive_files(&lib_dir) { - if let Some(stem) = lib.file_stem() { - let name = stem.to_string_lossy(); - if let Some(stripped) = name.strip_prefix("lib") { - flags.push(format!("-l{}", stripped)); - } - } - } - flags - } - - /// Get the SDK compiler defines from `flags/defines`. - /// - /// Returns `-D` flags that must be passed to the compiler for SDK headers - /// to work correctly (e.g., `MBEDTLS_CONFIG_FILE`, `IDF_VER`). - /// Returns empty if the flags file doesn't exist. - /// - /// Uses `split_defines` instead of `shell_split` because define values - /// like `-DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\"` contain escaped - /// quotes that must be preserved for GCC. - pub fn get_sdk_defines(&self, mcu: &str) -> Vec { - let defines_file = self.sdk_mcu_dir(mcu).join("flags").join("defines"); - if let Ok(content) = std::fs::read_to_string(&defines_file) { - return split_defines(&content); - } - Vec::new() - } - - /// Get the ordered SDK linker flags from `flags/ld_flags`. - /// - /// Returns the linker flags (undefined symbols, wrap directives, etc.) - /// as specified by the SDK. Returns empty if the flags file doesn't exist. - pub fn get_sdk_ld_flags(&self, mcu: &str) -> Vec { - let ld_flags_file = self.sdk_mcu_dir(mcu).join("flags").join("ld_flags"); - if let Ok(content) = std::fs::read_to_string(&ld_flags_file) { - return fbuild_core::shell_split::split(&content); - } - Vec::new() - } - - /// Get the SDK linker script flags from `flags/ld_scripts`. - /// - /// Returns the `-T` flags in the correct order, with the ld directory - /// as the search path. Falls back to the ld/ directory if no flags file. - pub fn get_sdk_ld_scripts(&self, mcu: &str) -> Vec { - let sdk_dir = self.sdk_mcu_dir(mcu); - let ld_scripts_file = sdk_dir.join("flags").join("ld_scripts"); - - let mut flags = vec![format!("-L{}", sdk_dir.join("ld").display())]; - - if let Ok(content) = std::fs::read_to_string(&ld_scripts_file) { - flags.extend(fbuild_core::shell_split::split(&content)); - return flags; - } - - // Fallback: no scripts - flags - } - - /// Get the linker scripts directory for a given MCU. - pub fn get_linker_scripts_dir(&self, mcu: &str) -> PathBuf { - self.sdk_mcu_dir(mcu).join("ld") - } - - /// Get the path to the bootloader binary. - /// - /// First checks for a pre-built `bootloader.bin`. If not found, looks for - /// `bootloader_{flash_mode}_{flash_freq}.elf` which needs elf2image conversion. - pub fn get_bootloader_bin(&self, mcu: &str) -> PathBuf { - self.sdk_mcu_dir(mcu).join("bin").join("bootloader.bin") - } - - /// Get the path to the bootloader ELF for a given flash mode and frequency. - /// - /// ESP32 Arduino core provides pre-built bootloader ELFs named - /// `bootloader_{mode}_{freq}.elf`. The ROM bootloader on ESP32-S3 and - /// similar chips can only load the second-stage bootloader in DIO mode, - /// so `flash_mode` should typically be "dio" regardless of application mode. - pub fn get_bootloader_elf(&self, mcu: &str, flash_mode: &str, flash_freq: &str) -> PathBuf { - let filename = format!("bootloader_{}_{}.elf", flash_mode, flash_freq); - self.sdk_mcu_dir(mcu).join("bin").join(filename) - } - - /// Get the path to the partitions binary. - pub fn get_partitions_bin(&self, mcu: &str) -> PathBuf { - self.sdk_mcu_dir(mcu).join("bin").join("partitions.bin") - } - - /// Get the path to the default ESP-IDF boot_app0 helper binary. - pub fn get_boot_app0_bin(&self) -> PathBuf { - self.resolved_dir() - .join("tools") - .join("partitions") - .join("boot_app0.bin") - } - - /// Get the path to the partitions CSV file. - pub fn get_partitions_csv(&self, partitions_name: &str) -> PathBuf { - self.resolved_dir() - .join("tools") - .join("partitions") - .join(partitions_name) - } - - /// Get the path to gen_esp32part.py for generating partition tables. - pub fn get_gen_esp32part(&self) -> PathBuf { - self.resolved_dir().join("tools").join("gen_esp32part.py") - } - - /// List all source files in a core directory. - pub fn get_core_sources(&self, core_name: &str) -> Vec { - collect_sources(&self.get_core_dir(core_name)) - } -} - -impl crate::Package for Esp32Framework { - fn ensure_installed(&self) -> fbuild_core::Result { - if self.is_installed() { - return Ok(self.resolved_dir()); - } - - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - - Ok(find_framework_root(&install_path)) - } - - fn is_installed(&self) -> bool { - if !self.base.is_cached() { - return false; - } - let root = find_framework_root(&self.base.install_path()); - root.join("cores").join("esp32").join("Arduino.h").exists() - } - - fn get_info(&self) -> PackageInfo { - self.base.get_info() - } -} - -impl Framework for Esp32Framework { - fn get_cores_dir(&self) -> PathBuf { - self.resolved_dir().join("cores") - } - - fn get_variants_dir(&self) -> PathBuf { - self.resolved_dir().join("variants") - } - - fn get_libraries_dir(&self) -> PathBuf { - self.resolved_dir().join("libraries") - } -} - -/// Parse include flags from the `flags/includes` file. -/// -/// Uses `fbuild_core::shell_split::split` to tokenize (handles quoted paths, -/// safe on Windows). Iterates with an index, consuming flag+path pairs. -/// Split a defines string into individual `-D` flags, preserving escaped quotes. -/// -/// The `flags/defines` file contains flags like: -/// ```text -/// -DFOO=1 -DBAR=\"baz.h\" -DQUX -/// ``` -/// The `\"` sequences must be preserved because GCC needs the quotes in -/// define values (e.g., `MBEDTLS_CONFIG_FILE` expands to `"mbedtls/esp_config.h"`). -/// -/// Unlike `shell_split`, this splits on whitespace boundaries that precede `-D` -/// and keeps the raw content of each flag intact. -fn split_defines(content: &str) -> Vec { - let mut tokens = Vec::new(); - // Split on " -D" boundaries (preserving the -D prefix) - let trimmed = content.trim(); - if trimmed.is_empty() { - return tokens; - } - // Find each -D token boundary - let mut start = 0; - let bytes = trimmed.as_bytes(); - let len = bytes.len(); - let mut i = 1; // skip first char - while i < len { - // A new -D token starts when we see whitespace followed by -D - if bytes[i] == b'-' - && i + 1 < len - && bytes[i + 1] == b'D' - && i > 0 - && bytes[i - 1].is_ascii_whitespace() - { - let token = trimmed[start..i].trim(); - if !token.is_empty() { - tokens.push(token.to_string()); - } - start = i; - } - i += 1; - } - // Last token - let token = trimmed[start..].trim(); - if !token.is_empty() { - tokens.push(token.to_string()); - } - tokens -} - -/// Handles two flag formats: -/// - `-iwithprefixbefore relative/path` (new 3.3.7+, resolved against include_base) -/// - `-I/absolute/path` or `-Irelative/path` (legacy 3.1.x) -fn parse_include_flags(content: &str, include_base: &Path, root: &Path) -> Vec { - let mut dirs = Vec::new(); - let parts = fbuild_core::shell_split::split(content); - let mut i = 0; - while i < parts.len() { - if parts[i] == "-iwithprefixbefore" { - if i + 1 < parts.len() { - let resolved = include_base.join(&parts[i + 1]); - if resolved.exists() { - dirs.push(resolved); - } - i += 2; - } else { - i += 1; - } - } else if let Some(path_str) = parts[i].strip_prefix("-I") { - if !path_str.is_empty() { - let p = if Path::new(path_str).is_absolute() { - PathBuf::from(path_str) - } else { - root.join(path_str) - }; - if p.exists() { - dirs.push(p); - } - } - i += 1; - } else { - i += 1; - } - } - dirs -} - -/// Extract a version string from a framework URL. -/// -/// E.g., `".../download/3.3.7/esp32-core-3.3.7.tar.xz"` → `"3.3.7"` -fn extract_framework_version(url: &str) -> String { - // Look for a path segment that is purely a version number (digits + dots) - for segment in url.rsplit('/') { - let s = segment - .trim_end_matches(".tar.xz") - .trim_end_matches(".tar.gz") - .trim_end_matches(".zip"); - if s.chars().all(|c| c.is_ascii_digit() || c == '.') && s.contains('.') && !s.is_empty() { - return s.to_string(); - } - } - // Fallback: hash - crate::cache::hash_url(url) -} - -/// Find the actual framework root inside an extracted archive. -/// Recursively copy a directory tree. -fn copy_dir_recursive(src: &Path, dest: &Path) -> fbuild_core::Result<()> { - std::fs::create_dir_all(dest)?; - for entry in std::fs::read_dir(src)?.flatten() { - let src_path = entry.path(); - let dest_path = dest.join(entry.file_name()); - if src_path.is_dir() { - copy_dir_recursive(&src_path, &dest_path)?; - } else { - std::fs::copy(&src_path, &dest_path)?; - } - } - Ok(()) -} - -fn find_framework_root(install_dir: &Path) -> PathBuf { - if install_dir.join("cores").exists() { - return install_dir.to_path_buf(); - } - - // Check one level deep - if let Ok(entries) = std::fs::read_dir(install_dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() && path.join("cores").exists() { - return path; - } - } - } - - install_dir.to_path_buf() -} - -/// Recursively scan for include paths that are useful for compilation. -/// -/// A directory is added if it contains headers directly OR has an immediate -/// child directory with headers (supporting `#include "subdir/header.h"`). -/// This matches PlatformIO's include strategy without adding every directory. -/// -/// Used for the 2.x framework layout where `flags/includes` doesn't exist. -fn scan_include_dirs_recursive( - dir: &Path, - dirs: &mut Vec, - depth: usize, - max_depth: usize, -) { - if depth > max_depth { - return; - } - if let Ok(entries) = std::fs::read_dir(dir) { - let mut has_headers = false; - let mut child_has_headers = false; - let mut subdirs = Vec::new(); - - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - // Check if this child dir has headers (one level peek) - if !child_has_headers && dir_contains_headers(&path) { - child_has_headers = true; - } - subdirs.push(path); - } else if !has_headers { - if let Some(ext) = path.extension() { - if ext == "h" || ext == "hpp" { - has_headers = true; - } - } - } - } - - // Add if dir has headers or a child dir has headers - if has_headers || child_has_headers { - dirs.push(dir.to_path_buf()); - } - - for subdir in subdirs { - scan_include_dirs_recursive(&subdir, dirs, depth + 1, max_depth); - } - } -} - -/// Check if a directory directly contains any header files. -fn dir_contains_headers(dir: &Path) -> bool { - if let Ok(entries) = std::fs::read_dir(dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_file() { - if let Some(ext) = path.extension() { - if ext == "h" || ext == "hpp" { - return true; - } - } - } - } - } - false -} - -/// Collect all `.a` archive files from a directory (non-recursive). -fn collect_archive_files(dir: &Path) -> Vec { - let mut libs = Vec::new(); - if let Ok(entries) = std::fs::read_dir(dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_file() && path.extension().is_some_and(|e| e == "a") { - libs.push(path); - } - } - } - libs.sort(); - libs -} - -/// Collect source files from a directory (non-recursive). -fn collect_sources(dir: &Path) -> Vec { - let mut sources = Vec::new(); - if let Ok(entries) = std::fs::read_dir(dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_file() { - let ext = path - .extension() - .unwrap_or_default() - .to_string_lossy() - .to_lowercase(); - if matches!(ext.as_str(), "c" | "cpp" | "cc" | "s") { - sources.push(path); - } - } - } - } - sources.sort(); - sources -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Package; - - #[test] - fn test_esp32_framework_not_installed() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = Esp32Framework::with_cache_root(tmp.path(), &tmp.path().join("cache"), "esp32c6"); - assert!(!fw.is_installed()); - } - - #[test] - fn test_find_framework_root_direct() { - let tmp = tempfile::TempDir::new().unwrap(); - std::fs::create_dir_all(tmp.path().join("cores")).unwrap(); - assert_eq!(find_framework_root(tmp.path()), tmp.path().to_path_buf()); - } - - #[test] - fn test_find_framework_root_nested() { - let tmp = tempfile::TempDir::new().unwrap(); - let nested = tmp.path().join("framework-arduinoespressif32"); - std::fs::create_dir_all(nested.join("cores")).unwrap(); - assert_eq!(find_framework_root(tmp.path()), nested); - } - - #[test] - fn test_get_core_dir() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = Esp32Framework::new(tmp.path(), "esp32c6"); - let core_dir = fw.get_core_dir("esp32"); - assert!(core_dir.to_string_lossy().contains("cores")); - assert!(core_dir.to_string_lossy().contains("esp32")); - } - - #[test] - fn test_get_variant_dir() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = Esp32Framework::new(tmp.path(), "esp32c6"); - let variant_dir = fw.get_variant_dir("esp32c6"); - assert!(variant_dir.to_string_lossy().contains("variants")); - assert!(variant_dir.to_string_lossy().contains("esp32c6")); - } - - #[test] - fn test_sdk_paths() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = Esp32Framework::new(tmp.path(), "esp32c6"); - let ld_dir = fw.get_linker_scripts_dir("esp32c6"); - assert!(ld_dir.to_string_lossy().contains("sdk")); - assert!(ld_dir.to_string_lossy().contains("esp32c6")); - assert!(ld_dir.to_string_lossy().contains("ld")); - } - - #[test] - fn test_collect_archive_files() { - let tmp = tempfile::TempDir::new().unwrap(); - std::fs::write(tmp.path().join("libfreertos.a"), "").unwrap(); - std::fs::write(tmp.path().join("libesp_system.a"), "").unwrap(); - std::fs::write(tmp.path().join("readme.txt"), "").unwrap(); - let libs = collect_archive_files(tmp.path()); - assert_eq!(libs.len(), 2); - assert!(libs.iter().all(|p| p.extension().unwrap() == "a")); - } - - #[test] - fn test_get_sdk_libs_empty() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = Esp32Framework::new(tmp.path(), "esp32c6"); - let libs = fw.get_sdk_libs("esp32c6"); - assert!(libs.is_empty()); // No SDK installed - } - - #[test] - fn test_validate_missing_cores() { - let tmp = tempfile::TempDir::new().unwrap(); - let result = Esp32Framework::validate(tmp.path()); - assert!(result.is_err()); - } - - #[test] - fn test_validate_missing_arduino_h() { - let tmp = tempfile::TempDir::new().unwrap(); - std::fs::create_dir_all(tmp.path().join("cores").join("esp32")).unwrap(); - let result = Esp32Framework::validate(tmp.path()); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("Arduino.h")); - } - - #[test] - fn test_bootloader_bin_path() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = Esp32Framework::new(tmp.path(), "esp32c6"); - let boot = fw.get_bootloader_bin("esp32c6"); - assert!(boot.to_string_lossy().contains("bootloader.bin")); - } - - #[test] - fn test_partitions_bin_path() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = Esp32Framework::new(tmp.path(), "esp32c6"); - let parts = fw.get_partitions_bin("esp32c6"); - assert!(parts.to_string_lossy().contains("partitions.bin")); - } - - #[test] - fn test_boot_app0_bin_path() { - let tmp = tempfile::TempDir::new().unwrap(); - let fw = Esp32Framework::new(tmp.path(), "esp32c6"); - let boot_app0 = fw.get_boot_app0_bin(); - assert!(boot_app0.to_string_lossy().contains("boot_app0.bin")); - } - - #[test] - fn test_parse_iwithprefixbefore_format() { - let tmp = tempfile::TempDir::new().unwrap(); - let include_base = tmp.path().join("include"); - - // Create dirs that match the relative paths - let freertos = include_base.join("freertos/include/freertos"); - let esp_sys = include_base.join("esp_system/include"); - std::fs::create_dir_all(&freertos).unwrap(); - std::fs::create_dir_all(&esp_sys).unwrap(); - - // This is the actual format from flags/includes files - let content = - "-iwithprefixbefore freertos/include/freertos -iwithprefixbefore esp_system/include"; - let dirs = parse_include_flags(content, &include_base, tmp.path()); - - assert_eq!(dirs.len(), 2); - assert_eq!(dirs[0], freertos); - assert_eq!(dirs[1], esp_sys); - } - - #[test] - fn test_sdk_include_dirs_with_mock() { - let tmp = tempfile::TempDir::new().unwrap(); - // Create mock SDK structure with includes file - let sdk_dir = tmp.path().join("tools").join("sdk").join("esp32c6"); - let flags_dir = sdk_dir.join("flags"); - std::fs::create_dir_all(&flags_dir).unwrap(); - - // Create some include dirs - let inc1 = sdk_dir.join("include").join("freertos"); - let inc2 = sdk_dir.join("include").join("esp_system"); - std::fs::create_dir_all(&inc1).unwrap(); - std::fs::create_dir_all(&inc2).unwrap(); - - // Write includes file with absolute paths - let includes_content = format!("-I{}\n-I{}\n", inc1.display(), inc2.display()); - std::fs::write(flags_dir.join("includes"), &includes_content).unwrap(); - - let fw = Esp32Framework { - base: PackageBase::new( - "test", - "1.0", - "http://example.com", - "http://example.com", - None, - CacheSubdir::Platforms, - tmp.path(), - ), - install_dir: Some(tmp.path().to_path_buf()), - }; - - let dirs = fw.get_sdk_include_dirs("esp32c6", None); - assert_eq!(dirs.len(), 2); - } - - #[test] - fn test_sdk_include_dirs_prefers_requested_memory_variant() { - let tmp = tempfile::TempDir::new().unwrap(); - let sdk_dir = tmp.path().join("tools").join("sdk").join("esp32s3"); - let flags_dir = sdk_dir.join("flags"); - std::fs::create_dir_all(&flags_dir).unwrap(); - std::fs::create_dir_all(sdk_dir.join("include")).unwrap(); - std::fs::write(flags_dir.join("includes"), "").unwrap(); - std::fs::create_dir_all(sdk_dir.join("qio_opi").join("include")).unwrap(); - std::fs::create_dir_all(sdk_dir.join("dio_qspi").join("include")).unwrap(); - - let fw = Esp32Framework { - base: PackageBase::new( - "test", - "1.0", - "http://example.com", - "http://example.com", - None, - CacheSubdir::Platforms, - tmp.path(), - ), - install_dir: Some(tmp.path().to_path_buf()), - }; - - let dirs = fw.get_sdk_include_dirs("esp32s3", Some("dio_qspi")); - assert!(dirs - .iter() - .any(|d| d.ends_with(Path::new("dio_qspi").join("include")))); - assert!(!dirs - .iter() - .any(|d| d.ends_with(Path::new("qio_opi").join("include")))); - } - - #[test] - fn test_sdk_lib_flags_prefers_requested_memory_variant() { - let tmp = tempfile::TempDir::new().unwrap(); - let sdk_dir = tmp.path().join("tools").join("sdk").join("esp32s3"); - let flags_dir = sdk_dir.join("flags"); - std::fs::create_dir_all(&flags_dir).unwrap(); - std::fs::write(flags_dir.join("ld_libs"), "-lfoo").unwrap(); - std::fs::create_dir_all(sdk_dir.join("lib")).unwrap(); - std::fs::create_dir_all(sdk_dir.join("dio_qspi")).unwrap(); - std::fs::create_dir_all(sdk_dir.join("qio_opi")).unwrap(); - - let fw = Esp32Framework { - base: PackageBase::new( - "test", - "1.0", - "http://example.com", - "http://example.com", - None, - CacheSubdir::Platforms, - tmp.path(), - ), - install_dir: Some(tmp.path().to_path_buf()), - }; - - let flags = fw.get_sdk_lib_flags("esp32s3", Some("dio_qspi")); - assert!(flags - .iter() - .any(|f| f.ends_with("\\esp32s3\\dio_qspi") || f.ends_with("/esp32s3/dio_qspi"))); - assert!(!flags - .iter() - .any(|f| f.ends_with("\\esp32s3\\qio_opi") || f.ends_with("/esp32s3/qio_opi"))); - } - - #[test] - fn test_split_defines_preserves_escaped_quotes() { - let content = - r#"-DFOO=1 -DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\" -DBAR -DIDF_VER=\"v5.5.2\""#; - let tokens = split_defines(content); - assert_eq!(tokens.len(), 4); - assert_eq!(tokens[0], "-DFOO=1"); - assert_eq!( - tokens[1], - r#"-DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\""# - ); - assert_eq!(tokens[2], "-DBAR"); - assert_eq!(tokens[3], r#"-DIDF_VER=\"v5.5.2\""#); - } - - #[test] - fn test_split_defines_empty() { - assert!(split_defines("").is_empty()); - assert!(split_defines(" ").is_empty()); - } - - #[test] - fn test_split_defines_single() { - assert_eq!(split_defines("-DFOO=1"), vec!["-DFOO=1"]); - } -} diff --git a/crates/fbuild-packages/src/library/esp32_framework/README.md b/crates/fbuild-packages/src/library/esp32_framework/README.md new file mode 100644 index 00000000..e114e78a --- /dev/null +++ b/crates/fbuild-packages/src/library/esp32_framework/README.md @@ -0,0 +1,25 @@ +# esp32_framework + +ESP32 Arduino framework package manager. Split into submodules to keep each +file below the 1000 LOC gate. + +## Layout + +- `mod.rs` — `Esp32Framework` struct, constructors, `Package`/`Framework` trait + impls, and the legacy `validate` helper. +- `libs.rs` — `ensure_libs` / `ensure_mcu_libs` (download + extract the ESP-IDF + SDK and per-MCU skeleton libs into `tools/`). +- `paths.rs` — Simple framework path accessors (cores, variants, bootloader, + partitions, gen_esp32part.py, core sources). +- `sdk_paths.rs` — SDK include directories, library flags, defines, linker + flags, and linker-script flags. Includes the `sdk_mcu_dir` and memory-variant + resolver. +- `parsing.rs` — Tokenizers for the SDK `flags/*` files (`-D` defines and + `-I` / `-iwithprefixbefore` include flags) and URL version extraction. +- `fs_utils.rs` — Generic filesystem helpers: recursive copy, framework-root + detection, recursive include scanner, archive/source collectors. +- `tests.rs` — Unit tests for the module (cfg(test) only). + +External API surface (`Esp32Framework` and its inherent / trait methods) is +unchanged. The only re-export from the parent module remains +`crate::library::Esp32Framework`. diff --git a/crates/fbuild-packages/src/library/esp32_framework/fs_utils.rs b/crates/fbuild-packages/src/library/esp32_framework/fs_utils.rs new file mode 100644 index 00000000..fe9d4a3a --- /dev/null +++ b/crates/fbuild-packages/src/library/esp32_framework/fs_utils.rs @@ -0,0 +1,140 @@ +//! Filesystem helpers for the ESP32 framework module. + +use std::path::{Path, PathBuf}; + +/// Find the actual framework root inside an extracted archive. +/// Recursively copy a directory tree. +pub(crate) fn copy_dir_recursive(src: &Path, dest: &Path) -> fbuild_core::Result<()> { + std::fs::create_dir_all(dest)?; + for entry in std::fs::read_dir(src)?.flatten() { + let src_path = entry.path(); + let dest_path = dest.join(entry.file_name()); + if src_path.is_dir() { + copy_dir_recursive(&src_path, &dest_path)?; + } else { + std::fs::copy(&src_path, &dest_path)?; + } + } + Ok(()) +} + +pub(crate) fn find_framework_root(install_dir: &Path) -> PathBuf { + if install_dir.join("cores").exists() { + return install_dir.to_path_buf(); + } + + // Check one level deep + if let Ok(entries) = std::fs::read_dir(install_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() && path.join("cores").exists() { + return path; + } + } + } + + install_dir.to_path_buf() +} + +/// Recursively scan for include paths that are useful for compilation. +/// +/// A directory is added if it contains headers directly OR has an immediate +/// child directory with headers (supporting `#include "subdir/header.h"`). +/// This matches PlatformIO's include strategy without adding every directory. +/// +/// Used for the 2.x framework layout where `flags/includes` doesn't exist. +pub(crate) fn scan_include_dirs_recursive( + dir: &Path, + dirs: &mut Vec, + depth: usize, + max_depth: usize, +) { + if depth > max_depth { + return; + } + if let Ok(entries) = std::fs::read_dir(dir) { + let mut has_headers = false; + let mut child_has_headers = false; + let mut subdirs = Vec::new(); + + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + // Check if this child dir has headers (one level peek) + if !child_has_headers && dir_contains_headers(&path) { + child_has_headers = true; + } + subdirs.push(path); + } else if !has_headers { + if let Some(ext) = path.extension() { + if ext == "h" || ext == "hpp" { + has_headers = true; + } + } + } + } + + // Add if dir has headers or a child dir has headers + if has_headers || child_has_headers { + dirs.push(dir.to_path_buf()); + } + + for subdir in subdirs { + scan_include_dirs_recursive(&subdir, dirs, depth + 1, max_depth); + } + } +} + +/// Check if a directory directly contains any header files. +fn dir_contains_headers(dir: &Path) -> bool { + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_file() { + if let Some(ext) = path.extension() { + if ext == "h" || ext == "hpp" { + return true; + } + } + } + } + } + false +} + +/// Collect all `.a` archive files from a directory (non-recursive). +pub(crate) fn collect_archive_files(dir: &Path) -> Vec { + let mut libs = Vec::new(); + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_file() && path.extension().is_some_and(|e| e == "a") { + libs.push(path); + } + } + } + libs.sort(); + libs +} + +/// Collect source files from a directory (non-recursive). +pub(crate) fn collect_sources(dir: &Path) -> Vec { + let mut sources = Vec::new(); + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_file() { + let ext = path + .extension() + .unwrap_or_default() + .to_string_lossy() + .to_lowercase(); + if matches!(ext.as_str(), "c" | "cpp" | "cc" | "s") { + sources.push(path); + } + } + } + } + sources.sort(); + sources +} diff --git a/crates/fbuild-packages/src/library/esp32_framework/libs.rs b/crates/fbuild-packages/src/library/esp32_framework/libs.rs new file mode 100644 index 00000000..456b0588 --- /dev/null +++ b/crates/fbuild-packages/src/library/esp32_framework/libs.rs @@ -0,0 +1,149 @@ +//! Logic for downloading and installing the ESP-IDF SDK libs and MCU skeleton libs. + +use super::fs_utils::copy_dir_recursive; +use super::Esp32Framework; + +impl Esp32Framework { + /// Ensure the SDK libs are downloaded and extracted into the framework's `tools/` dir. + pub fn ensure_libs(&self, libs_url: &str) -> fbuild_core::Result<()> { + let root = self.resolved_dir(); + let tools_dir = root.join("tools"); + + // Already have SDK libs? Check both old (sdk/) and new (esp32-arduino-libs/) layouts + for dir_name in &["esp32-arduino-libs", "sdk"] { + let sdk_dir = tools_dir.join(dir_name); + if sdk_dir.exists() && sdk_dir.is_dir() { + if let Ok(mut entries) = std::fs::read_dir(&sdk_dir) { + if entries.next().is_some() { + return Ok(()); + } + } + } + } + + std::fs::create_dir_all(&tools_dir)?; + + // Check for already-downloaded archive (skip re-download) + let archive_filename = libs_url.rsplit('/').next().unwrap_or("libs.tar.xz"); + let archive_path = tools_dir.join(archive_filename); + + if !archive_path.exists() { + tracing::info!("downloading ESP32 SDK libs"); + let rt = tokio::runtime::Handle::try_current().ok(); + if let Some(handle) = rt { + handle.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; + } else { + let rt = tokio::runtime::Runtime::new().map_err(|e| { + fbuild_core::FbuildError::PackageError(format!( + "failed to create tokio runtime: {}", + e + )) + })?; + rt.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; + } + } + + // Extract to a short temp path to avoid Windows MAX_PATH (260 char) limit. + // Then rename (atomic on same filesystem) to final location. + let temp_dir = std::env::temp_dir().join(format!("fbuild_sdk_{}", std::process::id())); + if temp_dir.exists() { + let _ = std::fs::remove_dir_all(&temp_dir); + } + std::fs::create_dir_all(&temp_dir)?; + + tracing::info!( + "extracting ESP32 SDK libs ({} MB)", + archive_path + .metadata() + .map(|m| m.len() / 1_000_000) + .unwrap_or(0) + ); + crate::extractor::extract(&archive_path, &temp_dir)?; + let _ = std::fs::remove_file(&archive_path); + + // Move extracted content to final tools/ dir (same filesystem = fast rename) + if let Ok(entries) = std::fs::read_dir(&temp_dir) { + for entry in entries.flatten() { + let src = entry.path(); + let dest = tools_dir.join(entry.file_name()); + if dest.exists() { + let _ = std::fs::remove_dir_all(&dest); + } + if std::fs::rename(&src, &dest).is_err() { + copy_dir_recursive(&src, &dest)?; + } + } + } + let _ = std::fs::remove_dir_all(&temp_dir); + + tracing::info!("ESP32 SDK libs installed"); + Ok(()) + } + + /// Ensure MCU-specific skeleton libs are downloaded and merged into the framework's `tools/` dir. + /// + /// Some MCUs (e.g. ESP32-C2, ESP32-C61) ship their SDK libs in a separate + /// skeleton package rather than the main `framework-arduinoespressif32-libs`. + /// This merges the skeleton into the existing `tools/` directory without + /// clobbering other MCU subdirs. + pub fn ensure_mcu_libs(&self, libs_url: &str, mcu: &str) -> fbuild_core::Result<()> { + let root = self.resolved_dir(); + let tools_dir = root.join("tools"); + + // Already have this MCU's SDK? Check both layouts. + for dir_name in &["esp32-arduino-libs", "sdk"] { + let mcu_dir = tools_dir.join(dir_name).join(mcu); + if mcu_dir.exists() && mcu_dir.is_dir() { + return Ok(()); + } + } + + std::fs::create_dir_all(&tools_dir)?; + + let archive_filename = libs_url.rsplit('/').next().unwrap_or("skeleton.zip"); + let archive_path = tools_dir.join(archive_filename); + + if !archive_path.exists() { + tracing::info!("downloading {} skeleton libs", mcu); + let rt = tokio::runtime::Handle::try_current().ok(); + if let Some(handle) = rt { + handle.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; + } else { + let rt = tokio::runtime::Runtime::new().map_err(|e| { + fbuild_core::FbuildError::PackageError(format!( + "failed to create tokio runtime: {}", + e + )) + })?; + rt.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; + } + } + + let temp_dir = std::env::temp_dir().join(format!("fbuild_skel_{}", std::process::id())); + if temp_dir.exists() { + let _ = std::fs::remove_dir_all(&temp_dir); + } + std::fs::create_dir_all(&temp_dir)?; + + tracing::info!("extracting {} skeleton libs", mcu); + crate::extractor::extract(&archive_path, &temp_dir)?; + let _ = std::fs::remove_file(&archive_path); + + // Merge into tools/ — use copy_dir_recursive so existing MCU dirs are preserved. + if let Ok(entries) = std::fs::read_dir(&temp_dir) { + for entry in entries.flatten() { + let src = entry.path(); + let dest = tools_dir.join(entry.file_name()); + if src.is_dir() { + copy_dir_recursive(&src, &dest)?; + } else if std::fs::rename(&src, &dest).is_err() { + std::fs::copy(&src, &dest)?; + } + } + } + let _ = std::fs::remove_dir_all(&temp_dir); + + tracing::info!("{} skeleton libs installed", mcu); + Ok(()) + } +} diff --git a/crates/fbuild-packages/src/library/esp32_framework/mod.rs b/crates/fbuild-packages/src/library/esp32_framework/mod.rs new file mode 100644 index 00000000..5a8cc508 --- /dev/null +++ b/crates/fbuild-packages/src/library/esp32_framework/mod.rs @@ -0,0 +1,175 @@ +//! ESP32 Arduino framework package. +//! +//! Downloads and manages the Arduino-ESP32 core + ESP-IDF precompiled libraries. +//! This combines what PlatformIO splits into two packages: +//! - `framework-arduinoespressif32`: Arduino core, variants, libraries +//! - `framework-arduinoespressif32-libs`: ESP-IDF SDK includes + precompiled `.a` libs +//! +//! Key methods provide paths to: +//! - Core sources: `cores/esp32/` +//! - Board variants: `variants/{mcu}/` +//! - SDK include dirs: `tools/sdk/{mcu}/include/` (305+ paths) +//! - SDK precompiled libs: `tools/sdk/{mcu}/lib/` (100+ .a files) +//! - Linker scripts: `tools/sdk/{mcu}/ld/` +//! - Bootloader/partitions: `tools/sdk/{mcu}/bin/` + +use std::path::{Path, PathBuf}; + +use crate::{CacheSubdir, Framework, PackageBase, PackageInfo}; + +mod fs_utils; +mod libs; +mod parsing; +mod paths; +mod sdk_paths; + +#[cfg(test)] +mod tests; + +use fs_utils::find_framework_root; +use parsing::extract_framework_version; + +const ESP32_FRAMEWORK_VERSION: &str = "3.1.1"; +const ESP32_FRAMEWORK_URL: &str = + "https://github.com/pioarduino/arduino-esp32/releases/download/3.1.1/framework-arduinoespressif32-3.1.1.tar.gz"; + +/// ESP32 Arduino framework manager. +pub struct Esp32Framework { + pub(crate) base: PackageBase, + pub(crate) install_dir: Option, +} + +impl Esp32Framework { + /// Create with hardcoded URL (legacy, for tests). + pub fn new(project_dir: &Path, _mcu: &str) -> Self { + Self { + base: PackageBase::new( + "esp32-arduino", + ESP32_FRAMEWORK_VERSION, + ESP32_FRAMEWORK_URL, + ESP32_FRAMEWORK_URL, + None, + CacheSubdir::Platforms, + project_dir, + ), + install_dir: None, + } + } + + #[cfg(test)] + fn with_cache_root(project_dir: &Path, cache_root: &Path, _mcu: &str) -> Self { + Self { + base: PackageBase::with_cache_root( + "esp32-arduino", + ESP32_FRAMEWORK_VERSION, + ESP32_FRAMEWORK_URL, + ESP32_FRAMEWORK_URL, + None, + CacheSubdir::Platforms, + project_dir, + cache_root, + ), + install_dir: None, + } + } + + /// Create from a resolved URL (from platform.json). + /// + /// The orchestrator reads `platform.json` → `packages.framework-arduinoespressif32.version` + /// to get the correct download URL (e.g. espressif/arduino-esp32 release). + pub fn from_url(project_dir: &Path, url: &str) -> Self { + // Extract version from URL (e.g., "3.3.7" from ".../3.3.7/esp32-core-3.3.7.tar.xz") + let version = extract_framework_version(url); + + Self { + base: PackageBase::new( + "esp32-arduino", + &version, + url, + "framework-arduinoespressif32", + None, + CacheSubdir::Platforms, + project_dir, + ), + install_dir: None, + } + } + + /// Get the resolved root directory of the framework. + pub(crate) fn resolved_dir(&self) -> PathBuf { + self.install_dir + .clone() + .unwrap_or_else(|| find_framework_root(&self.base.install_path())) + } + + /// Validate the extracted framework has required structure. + fn validate(install_dir: &Path) -> fbuild_core::Result<()> { + let root = find_framework_root(install_dir); + + let cores_dir = root.join("cores").join("esp32"); + if !cores_dir.exists() { + return Err(fbuild_core::FbuildError::PackageError(format!( + "ESP32 framework missing cores/esp32/ directory (in {})", + root.display() + ))); + } + + let arduino_h = cores_dir.join("Arduino.h"); + if !arduino_h.exists() { + return Err(fbuild_core::FbuildError::PackageError( + "ESP32 framework missing cores/esp32/Arduino.h".to_string(), + )); + } + + Ok(()) + } +} + +impl crate::Package for Esp32Framework { + fn ensure_installed(&self) -> fbuild_core::Result { + if self.is_installed() { + return Ok(self.resolved_dir()); + } + + let rt = tokio::runtime::Handle::try_current().ok(); + let install_path = if let Some(handle) = rt { + handle.block_on(self.base.staged_install(Self::validate))? + } else { + let rt = tokio::runtime::Runtime::new().map_err(|e| { + fbuild_core::FbuildError::PackageError(format!( + "failed to create tokio runtime: {}", + e + )) + })?; + rt.block_on(self.base.staged_install(Self::validate))? + }; + + Ok(find_framework_root(&install_path)) + } + + fn is_installed(&self) -> bool { + if !self.base.is_cached() { + return false; + } + let root = find_framework_root(&self.base.install_path()); + root.join("cores").join("esp32").join("Arduino.h").exists() + } + + fn get_info(&self) -> PackageInfo { + self.base.get_info() + } +} + +impl Framework for Esp32Framework { + fn get_cores_dir(&self) -> PathBuf { + self.resolved_dir().join("cores") + } + + fn get_variants_dir(&self) -> PathBuf { + self.resolved_dir().join("variants") + } + + fn get_libraries_dir(&self) -> PathBuf { + self.resolved_dir().join("libraries") + } +} diff --git a/crates/fbuild-packages/src/library/esp32_framework/parsing.rs b/crates/fbuild-packages/src/library/esp32_framework/parsing.rs new file mode 100644 index 00000000..81c02515 --- /dev/null +++ b/crates/fbuild-packages/src/library/esp32_framework/parsing.rs @@ -0,0 +1,109 @@ +//! Parsers for the flag files shipped with the ESP-IDF SDK package. + +use std::path::{Path, PathBuf}; + +/// Parse include flags from the `flags/includes` file. +/// +/// Uses `fbuild_core::shell_split::split` to tokenize (handles quoted paths, +/// safe on Windows). Iterates with an index, consuming flag+path pairs. +/// Split a defines string into individual `-D` flags, preserving escaped quotes. +/// +/// The `flags/defines` file contains flags like: +/// ```text +/// -DFOO=1 -DBAR=\"baz.h\" -DQUX +/// ``` +/// The `\"` sequences must be preserved because GCC needs the quotes in +/// define values (e.g., `MBEDTLS_CONFIG_FILE` expands to `"mbedtls/esp_config.h"`). +/// +/// Unlike `shell_split`, this splits on whitespace boundaries that precede `-D` +/// and keeps the raw content of each flag intact. +pub(crate) fn split_defines(content: &str) -> Vec { + let mut tokens = Vec::new(); + // Split on " -D" boundaries (preserving the -D prefix) + let trimmed = content.trim(); + if trimmed.is_empty() { + return tokens; + } + // Find each -D token boundary + let mut start = 0; + let bytes = trimmed.as_bytes(); + let len = bytes.len(); + let mut i = 1; // skip first char + while i < len { + // A new -D token starts when we see whitespace followed by -D + if bytes[i] == b'-' + && i + 1 < len + && bytes[i + 1] == b'D' + && i > 0 + && bytes[i - 1].is_ascii_whitespace() + { + let token = trimmed[start..i].trim(); + if !token.is_empty() { + tokens.push(token.to_string()); + } + start = i; + } + i += 1; + } + // Last token + let token = trimmed[start..].trim(); + if !token.is_empty() { + tokens.push(token.to_string()); + } + tokens +} + +/// Handles two flag formats: +/// - `-iwithprefixbefore relative/path` (new 3.3.7+, resolved against include_base) +/// - `-I/absolute/path` or `-Irelative/path` (legacy 3.1.x) +pub(crate) fn parse_include_flags(content: &str, include_base: &Path, root: &Path) -> Vec { + let mut dirs = Vec::new(); + let parts = fbuild_core::shell_split::split(content); + let mut i = 0; + while i < parts.len() { + if parts[i] == "-iwithprefixbefore" { + if i + 1 < parts.len() { + let resolved = include_base.join(&parts[i + 1]); + if resolved.exists() { + dirs.push(resolved); + } + i += 2; + } else { + i += 1; + } + } else if let Some(path_str) = parts[i].strip_prefix("-I") { + if !path_str.is_empty() { + let p = if Path::new(path_str).is_absolute() { + PathBuf::from(path_str) + } else { + root.join(path_str) + }; + if p.exists() { + dirs.push(p); + } + } + i += 1; + } else { + i += 1; + } + } + dirs +} + +/// Extract a version string from a framework URL. +/// +/// E.g., `".../download/3.3.7/esp32-core-3.3.7.tar.xz"` → `"3.3.7"` +pub(crate) fn extract_framework_version(url: &str) -> String { + // Look for a path segment that is purely a version number (digits + dots) + for segment in url.rsplit('/') { + let s = segment + .trim_end_matches(".tar.xz") + .trim_end_matches(".tar.gz") + .trim_end_matches(".zip"); + if s.chars().all(|c| c.is_ascii_digit() || c == '.') && s.contains('.') && !s.is_empty() { + return s.to_string(); + } + } + // Fallback: hash + crate::cache::hash_url(url) +} diff --git a/crates/fbuild-packages/src/library/esp32_framework/paths.rs b/crates/fbuild-packages/src/library/esp32_framework/paths.rs new file mode 100644 index 00000000..916936f8 --- /dev/null +++ b/crates/fbuild-packages/src/library/esp32_framework/paths.rs @@ -0,0 +1,74 @@ +//! Simple path accessors for ESP32 framework directories. + +use std::path::PathBuf; + +use super::fs_utils::collect_sources; +use super::sdk_paths::sdk_mcu_dir; +use super::Esp32Framework; + +impl Esp32Framework { + /// Get the core source directory (e.g. `cores/esp32`). + pub fn get_core_dir(&self, core_name: &str) -> PathBuf { + self.resolved_dir().join("cores").join(core_name) + } + + /// Get the variant directory for a board (e.g. `variants/esp32c6`). + pub fn get_variant_dir(&self, variant_name: &str) -> PathBuf { + self.resolved_dir().join("variants").join(variant_name) + } + + /// Get the linker scripts directory for a given MCU. + pub fn get_linker_scripts_dir(&self, mcu: &str) -> PathBuf { + sdk_mcu_dir(self, mcu).join("ld") + } + + /// Get the path to the bootloader binary. + /// + /// First checks for a pre-built `bootloader.bin`. If not found, looks for + /// `bootloader_{flash_mode}_{flash_freq}.elf` which needs elf2image conversion. + pub fn get_bootloader_bin(&self, mcu: &str) -> PathBuf { + sdk_mcu_dir(self, mcu).join("bin").join("bootloader.bin") + } + + /// Get the path to the bootloader ELF for a given flash mode and frequency. + /// + /// ESP32 Arduino core provides pre-built bootloader ELFs named + /// `bootloader_{mode}_{freq}.elf`. The ROM bootloader on ESP32-S3 and + /// similar chips can only load the second-stage bootloader in DIO mode, + /// so `flash_mode` should typically be "dio" regardless of application mode. + pub fn get_bootloader_elf(&self, mcu: &str, flash_mode: &str, flash_freq: &str) -> PathBuf { + let filename = format!("bootloader_{}_{}.elf", flash_mode, flash_freq); + sdk_mcu_dir(self, mcu).join("bin").join(filename) + } + + /// Get the path to the partitions binary. + pub fn get_partitions_bin(&self, mcu: &str) -> PathBuf { + sdk_mcu_dir(self, mcu).join("bin").join("partitions.bin") + } + + /// Get the path to the default ESP-IDF boot_app0 helper binary. + pub fn get_boot_app0_bin(&self) -> PathBuf { + self.resolved_dir() + .join("tools") + .join("partitions") + .join("boot_app0.bin") + } + + /// Get the path to the partitions CSV file. + pub fn get_partitions_csv(&self, partitions_name: &str) -> PathBuf { + self.resolved_dir() + .join("tools") + .join("partitions") + .join(partitions_name) + } + + /// Get the path to gen_esp32part.py for generating partition tables. + pub fn get_gen_esp32part(&self) -> PathBuf { + self.resolved_dir().join("tools").join("gen_esp32part.py") + } + + /// List all source files in a core directory. + pub fn get_core_sources(&self, core_name: &str) -> Vec { + collect_sources(&self.get_core_dir(core_name)) + } +} diff --git a/crates/fbuild-packages/src/library/esp32_framework/sdk_paths.rs b/crates/fbuild-packages/src/library/esp32_framework/sdk_paths.rs new file mode 100644 index 00000000..e0c9f61e --- /dev/null +++ b/crates/fbuild-packages/src/library/esp32_framework/sdk_paths.rs @@ -0,0 +1,218 @@ +//! SDK include, library, define, and linker flag accessors for the ESP-IDF SDK +//! shipped with the ESP32 Arduino framework. + +use std::path::{Path, PathBuf}; + +use super::fs_utils::{collect_archive_files, scan_include_dirs_recursive}; +use super::parsing::{parse_include_flags, split_defines}; +use super::Esp32Framework; + +/// Get the SDK directory for a given MCU. +/// +/// Tries new layout (`tools/esp32-arduino-libs/{mcu}`) first, falls back to +/// old layout (`tools/sdk/{mcu}`). +pub(super) fn sdk_mcu_dir(fw: &Esp32Framework, mcu: &str) -> PathBuf { + let root = fw.resolved_dir(); + let new_path = root.join("tools").join("esp32-arduino-libs").join(mcu); + if new_path.exists() { + return new_path; + } + root.join("tools").join("sdk").join(mcu) +} + +fn sdk_memory_variant_dir(sdk_dir: &Path, requested: Option<&str>) -> Option { + if let Some(requested) = requested { + let requested_dir = sdk_dir.join(requested); + if requested_dir.exists() { + return Some(requested_dir); + } + } + + for variant in &["qio_opi", "dio_opi", "opi_opi", "qio_qspi", "dio_qspi"] { + let candidate = sdk_dir.join(variant); + if candidate.exists() { + return Some(candidate); + } + } + + None +} + +impl Esp32Framework { + /// Get the SDK directory for a given MCU. + /// + /// Tries new layout (`tools/esp32-arduino-libs/{mcu}`) first, falls back to + /// old layout (`tools/sdk/{mcu}`). + fn sdk_mcu_dir(&self, mcu: &str) -> PathBuf { + sdk_mcu_dir(self, mcu) + } + + /// Get SDK include directories for a given MCU. + /// + /// Reads the `flags/includes` file from the SDK directory, which lists + /// all 305+ include paths. Falls back to scanning `include/` subdirectories. + pub fn get_sdk_include_dirs(&self, mcu: &str, memory_type: Option<&str>) -> Vec { + let root = self.resolved_dir(); + let sdk_dir = self.sdk_mcu_dir(mcu); + + // Try reading the includes list file (supports both -I and -iwithprefixbefore formats) + let includes_file = sdk_dir.join("flags").join("includes"); + if includes_file.exists() { + if let Ok(content) = std::fs::read_to_string(&includes_file) { + let include_base = sdk_dir.join("include"); + let mut dirs = parse_include_flags(&content, &include_base, &root); + + // Add flash/PSRAM variant include dir (contains sdkconfig.h). + // Try common variants in preference order; the correct one depends + // on board_build.flash_mode and board_build.arduino.memory_type. + if let Some(variant_dir) = sdk_memory_variant_dir(&sdk_dir, memory_type) { + let v_include = variant_dir.join("include"); + if v_include.exists() { + dirs.push(v_include); + } + } + + return dirs; + } + } + + // Fallback: recursively scan include/ subdirectories. + // The 2.x framework (PlatformIO-compat) has deeply nested includes + // under tools/sdk/{mcu}/include/ (e.g., freertos/include/freertos, + // freertos/port/xtensa/include). + let include_dir = sdk_dir.join("include"); + if !include_dir.exists() { + return Vec::new(); + } + + let mut dirs = Vec::new(); + + // Prepend newlib/platform_include (provides assert.h, errno.h, time.h) + // which must come before SDK headers. PlatformIO also puts this first. + let newlib_platform = include_dir.join("newlib").join("platform_include"); + if newlib_platform.exists() { + dirs.push(newlib_platform); + } + + // Scan 4 levels deep — matches PlatformIO's actual include depth. + // ESP-IDF components have nested includes up to 4 levels deep + // (e.g., freertos/include/esp_additions/freertos/). + scan_include_dirs_recursive(&include_dir, &mut dirs, 0, 4); + + // Add well-known ESP-IDF Xtensa/RISC-V port include paths that the + // scanner misses because headers are nested too deeply for detection. + for sub_mcu in &["esp32", "esp32s2", "esp32s3"] { + let xtensa_inc = include_dir.join("xtensa").join(sub_mcu).join("include"); + if xtensa_inc.exists() && !dirs.contains(&xtensa_inc) { + dirs.push(xtensa_inc); + } + } + + // Also add flash/PSRAM variant include dir (contains sdkconfig.h). + if let Some(variant_dir) = sdk_memory_variant_dir(&sdk_dir, memory_type) { + let v_include = variant_dir.join("include"); + if v_include.exists() { + dirs.push(v_include); + } + } + + dirs.sort(); + dirs + } + + /// Get all precompiled `.a` library files from the ESP-IDF SDK. + pub fn get_sdk_libs(&self, mcu: &str) -> Vec { + let lib_dir = self.sdk_mcu_dir(mcu).join("lib"); + collect_archive_files(&lib_dir) + } + + /// Get the ordered SDK linker library flags from `flags/ld_libs`. + /// + /// Returns the pre-ordered `-l` flags (with duplicates for circular deps) + /// as specified by the SDK. Falls back to scanning `lib/` for `.a` files + /// if the flags file doesn't exist. + pub fn get_sdk_lib_flags(&self, mcu: &str, memory_type: Option<&str>) -> Vec { + let sdk_dir = self.sdk_mcu_dir(mcu); + let ld_libs_file = sdk_dir.join("flags").join("ld_libs"); + + if let Ok(content) = std::fs::read_to_string(&ld_libs_file) { + let mut flags = vec![format!("-L{}", sdk_dir.join("lib").display())]; + // Add ld/ directory as a library search path + let ld_dir = sdk_dir.join("ld"); + if ld_dir.exists() { + flags.push(format!("-L{}", ld_dir.display())); + } + // Add flash-mode-specific directory (contains libspi_flash.a and others). + // Default to dio_qspi (most common for ESP32dev boards). + if let Some(variant_dir) = sdk_memory_variant_dir(&sdk_dir, memory_type) { + flags.push(format!("-L{}", variant_dir.display())); + } + flags.extend(fbuild_core::shell_split::split(&content)); + return flags; + } + + // Fallback: scan lib/ directory for .a files + let lib_dir = sdk_dir.join("lib"); + let mut flags = Vec::new(); + if lib_dir.exists() { + flags.push(format!("-L{}", lib_dir.display())); + } + for lib in collect_archive_files(&lib_dir) { + if let Some(stem) = lib.file_stem() { + let name = stem.to_string_lossy(); + if let Some(stripped) = name.strip_prefix("lib") { + flags.push(format!("-l{}", stripped)); + } + } + } + flags + } + + /// Get the SDK compiler defines from `flags/defines`. + /// + /// Returns `-D` flags that must be passed to the compiler for SDK headers + /// to work correctly (e.g., `MBEDTLS_CONFIG_FILE`, `IDF_VER`). + /// Returns empty if the flags file doesn't exist. + /// + /// Uses `split_defines` instead of `shell_split` because define values + /// like `-DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\"` contain escaped + /// quotes that must be preserved for GCC. + pub fn get_sdk_defines(&self, mcu: &str) -> Vec { + let defines_file = self.sdk_mcu_dir(mcu).join("flags").join("defines"); + if let Ok(content) = std::fs::read_to_string(&defines_file) { + return split_defines(&content); + } + Vec::new() + } + + /// Get the ordered SDK linker flags from `flags/ld_flags`. + /// + /// Returns the linker flags (undefined symbols, wrap directives, etc.) + /// as specified by the SDK. Returns empty if the flags file doesn't exist. + pub fn get_sdk_ld_flags(&self, mcu: &str) -> Vec { + let ld_flags_file = self.sdk_mcu_dir(mcu).join("flags").join("ld_flags"); + if let Ok(content) = std::fs::read_to_string(&ld_flags_file) { + return fbuild_core::shell_split::split(&content); + } + Vec::new() + } + + /// Get the SDK linker script flags from `flags/ld_scripts`. + /// + /// Returns the `-T` flags in the correct order, with the ld directory + /// as the search path. Falls back to the ld/ directory if no flags file. + pub fn get_sdk_ld_scripts(&self, mcu: &str) -> Vec { + let sdk_dir = self.sdk_mcu_dir(mcu); + let ld_scripts_file = sdk_dir.join("flags").join("ld_scripts"); + + let mut flags = vec![format!("-L{}", sdk_dir.join("ld").display())]; + + if let Ok(content) = std::fs::read_to_string(&ld_scripts_file) { + flags.extend(fbuild_core::shell_split::split(&content)); + return flags; + } + + // Fallback: no scripts + flags + } +} diff --git a/crates/fbuild-packages/src/library/esp32_framework/tests.rs b/crates/fbuild-packages/src/library/esp32_framework/tests.rs new file mode 100644 index 00000000..baa1cb4b --- /dev/null +++ b/crates/fbuild-packages/src/library/esp32_framework/tests.rs @@ -0,0 +1,263 @@ +use std::path::Path; + +use super::fs_utils::{collect_archive_files, find_framework_root}; +use super::parsing::{parse_include_flags, split_defines}; +use super::Esp32Framework; +use crate::{CacheSubdir, Package, PackageBase}; + +#[test] +fn test_esp32_framework_not_installed() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = Esp32Framework::with_cache_root(tmp.path(), &tmp.path().join("cache"), "esp32c6"); + assert!(!fw.is_installed()); +} + +#[test] +fn test_find_framework_root_direct() { + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::create_dir_all(tmp.path().join("cores")).unwrap(); + assert_eq!(find_framework_root(tmp.path()), tmp.path().to_path_buf()); +} + +#[test] +fn test_find_framework_root_nested() { + let tmp = tempfile::TempDir::new().unwrap(); + let nested = tmp.path().join("framework-arduinoespressif32"); + std::fs::create_dir_all(nested.join("cores")).unwrap(); + assert_eq!(find_framework_root(tmp.path()), nested); +} + +#[test] +fn test_get_core_dir() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = Esp32Framework::new(tmp.path(), "esp32c6"); + let core_dir = fw.get_core_dir("esp32"); + assert!(core_dir.to_string_lossy().contains("cores")); + assert!(core_dir.to_string_lossy().contains("esp32")); +} + +#[test] +fn test_get_variant_dir() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = Esp32Framework::new(tmp.path(), "esp32c6"); + let variant_dir = fw.get_variant_dir("esp32c6"); + assert!(variant_dir.to_string_lossy().contains("variants")); + assert!(variant_dir.to_string_lossy().contains("esp32c6")); +} + +#[test] +fn test_sdk_paths() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = Esp32Framework::new(tmp.path(), "esp32c6"); + let ld_dir = fw.get_linker_scripts_dir("esp32c6"); + assert!(ld_dir.to_string_lossy().contains("sdk")); + assert!(ld_dir.to_string_lossy().contains("esp32c6")); + assert!(ld_dir.to_string_lossy().contains("ld")); +} + +#[test] +fn test_collect_archive_files() { + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::write(tmp.path().join("libfreertos.a"), "").unwrap(); + std::fs::write(tmp.path().join("libesp_system.a"), "").unwrap(); + std::fs::write(tmp.path().join("readme.txt"), "").unwrap(); + let libs = collect_archive_files(tmp.path()); + assert_eq!(libs.len(), 2); + assert!(libs.iter().all(|p| p.extension().unwrap() == "a")); +} + +#[test] +fn test_get_sdk_libs_empty() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = Esp32Framework::new(tmp.path(), "esp32c6"); + let libs = fw.get_sdk_libs("esp32c6"); + assert!(libs.is_empty()); // No SDK installed +} + +#[test] +fn test_validate_missing_cores() { + let tmp = tempfile::TempDir::new().unwrap(); + let result = Esp32Framework::validate(tmp.path()); + assert!(result.is_err()); +} + +#[test] +fn test_validate_missing_arduino_h() { + let tmp = tempfile::TempDir::new().unwrap(); + std::fs::create_dir_all(tmp.path().join("cores").join("esp32")).unwrap(); + let result = Esp32Framework::validate(tmp.path()); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("Arduino.h")); +} + +#[test] +fn test_bootloader_bin_path() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = Esp32Framework::new(tmp.path(), "esp32c6"); + let boot = fw.get_bootloader_bin("esp32c6"); + assert!(boot.to_string_lossy().contains("bootloader.bin")); +} + +#[test] +fn test_partitions_bin_path() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = Esp32Framework::new(tmp.path(), "esp32c6"); + let parts = fw.get_partitions_bin("esp32c6"); + assert!(parts.to_string_lossy().contains("partitions.bin")); +} + +#[test] +fn test_boot_app0_bin_path() { + let tmp = tempfile::TempDir::new().unwrap(); + let fw = Esp32Framework::new(tmp.path(), "esp32c6"); + let boot_app0 = fw.get_boot_app0_bin(); + assert!(boot_app0.to_string_lossy().contains("boot_app0.bin")); +} + +#[test] +fn test_parse_iwithprefixbefore_format() { + let tmp = tempfile::TempDir::new().unwrap(); + let include_base = tmp.path().join("include"); + + // Create dirs that match the relative paths + let freertos = include_base.join("freertos/include/freertos"); + let esp_sys = include_base.join("esp_system/include"); + std::fs::create_dir_all(&freertos).unwrap(); + std::fs::create_dir_all(&esp_sys).unwrap(); + + // This is the actual format from flags/includes files + let content = + "-iwithprefixbefore freertos/include/freertos -iwithprefixbefore esp_system/include"; + let dirs = parse_include_flags(content, &include_base, tmp.path()); + + assert_eq!(dirs.len(), 2); + assert_eq!(dirs[0], freertos); + assert_eq!(dirs[1], esp_sys); +} + +#[test] +fn test_sdk_include_dirs_with_mock() { + let tmp = tempfile::TempDir::new().unwrap(); + // Create mock SDK structure with includes file + let sdk_dir = tmp.path().join("tools").join("sdk").join("esp32c6"); + let flags_dir = sdk_dir.join("flags"); + std::fs::create_dir_all(&flags_dir).unwrap(); + + // Create some include dirs + let inc1 = sdk_dir.join("include").join("freertos"); + let inc2 = sdk_dir.join("include").join("esp_system"); + std::fs::create_dir_all(&inc1).unwrap(); + std::fs::create_dir_all(&inc2).unwrap(); + + // Write includes file with absolute paths + let includes_content = format!("-I{}\n-I{}\n", inc1.display(), inc2.display()); + std::fs::write(flags_dir.join("includes"), &includes_content).unwrap(); + + let fw = Esp32Framework { + base: PackageBase::new( + "test", + "1.0", + "http://example.com", + "http://example.com", + None, + CacheSubdir::Platforms, + tmp.path(), + ), + install_dir: Some(tmp.path().to_path_buf()), + }; + + let dirs = fw.get_sdk_include_dirs("esp32c6", None); + assert_eq!(dirs.len(), 2); +} + +#[test] +fn test_sdk_include_dirs_prefers_requested_memory_variant() { + let tmp = tempfile::TempDir::new().unwrap(); + let sdk_dir = tmp.path().join("tools").join("sdk").join("esp32s3"); + let flags_dir = sdk_dir.join("flags"); + std::fs::create_dir_all(&flags_dir).unwrap(); + std::fs::create_dir_all(sdk_dir.join("include")).unwrap(); + std::fs::write(flags_dir.join("includes"), "").unwrap(); + std::fs::create_dir_all(sdk_dir.join("qio_opi").join("include")).unwrap(); + std::fs::create_dir_all(sdk_dir.join("dio_qspi").join("include")).unwrap(); + + let fw = Esp32Framework { + base: PackageBase::new( + "test", + "1.0", + "http://example.com", + "http://example.com", + None, + CacheSubdir::Platforms, + tmp.path(), + ), + install_dir: Some(tmp.path().to_path_buf()), + }; + + let dirs = fw.get_sdk_include_dirs("esp32s3", Some("dio_qspi")); + assert!(dirs + .iter() + .any(|d| d.ends_with(Path::new("dio_qspi").join("include")))); + assert!(!dirs + .iter() + .any(|d| d.ends_with(Path::new("qio_opi").join("include")))); +} + +#[test] +fn test_sdk_lib_flags_prefers_requested_memory_variant() { + let tmp = tempfile::TempDir::new().unwrap(); + let sdk_dir = tmp.path().join("tools").join("sdk").join("esp32s3"); + let flags_dir = sdk_dir.join("flags"); + std::fs::create_dir_all(&flags_dir).unwrap(); + std::fs::write(flags_dir.join("ld_libs"), "-lfoo").unwrap(); + std::fs::create_dir_all(sdk_dir.join("lib")).unwrap(); + std::fs::create_dir_all(sdk_dir.join("dio_qspi")).unwrap(); + std::fs::create_dir_all(sdk_dir.join("qio_opi")).unwrap(); + + let fw = Esp32Framework { + base: PackageBase::new( + "test", + "1.0", + "http://example.com", + "http://example.com", + None, + CacheSubdir::Platforms, + tmp.path(), + ), + install_dir: Some(tmp.path().to_path_buf()), + }; + + let flags = fw.get_sdk_lib_flags("esp32s3", Some("dio_qspi")); + assert!(flags + .iter() + .any(|f| f.ends_with("\\esp32s3\\dio_qspi") || f.ends_with("/esp32s3/dio_qspi"))); + assert!(!flags + .iter() + .any(|f| f.ends_with("\\esp32s3\\qio_opi") || f.ends_with("/esp32s3/qio_opi"))); +} + +#[test] +fn test_split_defines_preserves_escaped_quotes() { + let content = + r#"-DFOO=1 -DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\" -DBAR -DIDF_VER=\"v5.5.2\""#; + let tokens = split_defines(content); + assert_eq!(tokens.len(), 4); + assert_eq!(tokens[0], "-DFOO=1"); + assert_eq!( + tokens[1], + r#"-DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\""# + ); + assert_eq!(tokens[2], "-DBAR"); + assert_eq!(tokens[3], r#"-DIDF_VER=\"v5.5.2\""#); +} + +#[test] +fn test_split_defines_empty() { + assert!(split_defines("").is_empty()); + assert!(split_defines(" ").is_empty()); +} + +#[test] +fn test_split_defines_single() { + assert_eq!(split_defines("-DFOO=1"), vec!["-DFOO=1"]); +} diff --git a/crates/fbuild-python/src/async_daemon_connection.rs b/crates/fbuild-python/src/async_daemon_connection.rs new file mode 100644 index 00000000..0ac962ea --- /dev/null +++ b/crates/fbuild-python/src/async_daemon_connection.rs @@ -0,0 +1,230 @@ +//! Asynchronous `AsyncDaemonConnection` PyO3 binding — native async +//! counterpart to `DaemonConnection` (FastLED/fbuild#65). + +use pyo3::prelude::*; + +use crate::outcome::{ + build_url, deploy_url, monitor_url, outcome_to_pydict, send_op_async, OpRequest, +}; + +/// Python-visible AsyncDaemonConnection class. +/// +/// Native async counterpart to `DaemonConnection`. Exposes `build`, +/// `deploy`, and `monitor` (and their `_result` variants) as async methods +/// that call the daemon over `reqwest::Client` (non-blocking) instead of +/// the blocking client used by the sync sibling. This is the method set +/// FastLED/fbuild#65 explicitly targets under "Daemon/DaemonConnection: +/// send_op and any other HTTP call". +/// +/// ```python +/// import asyncio +/// from fbuild._native import AsyncDaemonConnection +/// +/// async def main(): +/// conn = AsyncDaemonConnection(project_dir="tests/platform/uno", environment="uno") +/// ok = await conn.build() +/// result = await conn.build_result() +/// +/// asyncio.run(main()) +/// ``` +#[pyclass] +pub(crate) struct AsyncDaemonConnection { + project_dir: String, + environment: String, +} + +#[pymethods] +impl AsyncDaemonConnection { + #[new] + pub(crate) fn new(project_dir: String, environment: String) -> Self { + Self { + project_dir, + environment, + } + } + + /// Async context manager entry. Returns self so callers can + /// `async with AsyncDaemonConnection(...) as conn:`. + fn __aenter__<'py>(slf: PyRef<'py, Self>, py: Python<'py>) -> PyResult> { + let project_dir = slf.project_dir.clone(); + let environment = slf.environment.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + Python::with_gil(|py| { + let obj = Py::new( + py, + AsyncDaemonConnection { + project_dir, + environment, + }, + )?; + Ok(obj.to_object(py)) + }) + }) + } + + #[pyo3(signature = (_exc_type=None, _exc_val=None, _exc_tb=None))] + fn __aexit__<'py>( + &self, + py: Python<'py>, + _exc_type: Option, + _exc_val: Option, + _exc_tb: Option, + ) -> PyResult> { + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(false) }) + } + + /// Async counterpart to `DaemonConnection::build`. Awaits the daemon's + /// `POST /api/build` response and returns the `success` bool. + #[pyo3(signature = (clean=false, verbose=false, timeout=1800.0))] + fn build<'py>( + &self, + py: Python<'py>, + clean: bool, + verbose: bool, + timeout: f64, + ) -> PyResult> { + let url = build_url(); + let req = self.build_request(clean, verbose); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + Ok(send_op_async(url, req, timeout).await.success) + }) + } + + /// Async counterpart to `DaemonConnection::deploy`. + #[pyo3(signature = (port=None, clean=false, skip_build=false, monitor_after=false, timeout=1800.0))] + fn deploy<'py>( + &self, + py: Python<'py>, + port: Option, + clean: bool, + skip_build: bool, + monitor_after: bool, + timeout: f64, + ) -> PyResult> { + let url = deploy_url(); + let req = self.deploy_request(port, clean, skip_build, monitor_after); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + Ok(send_op_async(url, req, timeout).await.success) + }) + } + + /// Async counterpart to `DaemonConnection::monitor`. + #[pyo3(signature = (port=None, baud_rate=None, timeout=None))] + fn monitor<'py>( + &self, + py: Python<'py>, + port: Option, + baud_rate: Option, + timeout: Option, + ) -> PyResult> { + let url = monitor_url(); + let req = self.monitor_request(port, baud_rate); + let t = timeout.unwrap_or(1800.0); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + Ok(send_op_async(url, req, t).await.success) + }) + } + + /// Async counterpart to `DaemonConnection::build_result`. Returns the + /// full structured outcome dict (`success`, `message`, `exit_code`, + /// `stdout`, `stderr`) — matches the sync surface exactly. + #[pyo3(signature = (clean=false, verbose=false, timeout=1800.0))] + fn build_result<'py>( + &self, + py: Python<'py>, + clean: bool, + verbose: bool, + timeout: f64, + ) -> PyResult> { + let url = build_url(); + let req = self.build_request(clean, verbose); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let outcome = send_op_async(url, req, timeout).await; + Python::with_gil(|py| Ok(outcome_to_pydict(py, &outcome)?.unbind())) + }) + } + + /// Async counterpart to `DaemonConnection::deploy_result`. + #[pyo3(signature = (port=None, clean=false, skip_build=false, monitor_after=false, timeout=1800.0))] + fn deploy_result<'py>( + &self, + py: Python<'py>, + port: Option, + clean: bool, + skip_build: bool, + monitor_after: bool, + timeout: f64, + ) -> PyResult> { + let url = deploy_url(); + let req = self.deploy_request(port, clean, skip_build, monitor_after); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let outcome = send_op_async(url, req, timeout).await; + Python::with_gil(|py| Ok(outcome_to_pydict(py, &outcome)?.unbind())) + }) + } + + /// Async counterpart to `DaemonConnection::monitor_result`. + #[pyo3(signature = (port=None, baud_rate=None, timeout=None))] + fn monitor_result<'py>( + &self, + py: Python<'py>, + port: Option, + baud_rate: Option, + timeout: Option, + ) -> PyResult> { + let url = monitor_url(); + let req = self.monitor_request(port, baud_rate); + let t = timeout.unwrap_or(1800.0); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let outcome = send_op_async(url, req, t).await; + Python::with_gil(|py| Ok(outcome_to_pydict(py, &outcome)?.unbind())) + }) + } +} + +impl AsyncDaemonConnection { + fn build_request(&self, clean: bool, verbose: bool) -> OpRequest { + OpRequest { + project_dir: self.project_dir.clone(), + environment: Some(self.environment.clone()), + clean_build: clean, + verbose, + port: None, + monitor_after: false, + skip_build: false, + baud_rate: None, + } + } + + fn deploy_request( + &self, + port: Option, + clean: bool, + skip_build: bool, + monitor_after: bool, + ) -> OpRequest { + OpRequest { + project_dir: self.project_dir.clone(), + environment: Some(self.environment.clone()), + clean_build: clean, + verbose: false, + port, + monitor_after, + skip_build, + baud_rate: None, + } + } + + fn monitor_request(&self, port: Option, baud_rate: Option) -> OpRequest { + OpRequest { + project_dir: self.project_dir.clone(), + environment: Some(self.environment.clone()), + clean_build: false, + verbose: false, + port, + monitor_after: false, + skip_build: false, + baud_rate, + } + } +} diff --git a/crates/fbuild-python/src/async_serial_monitor.rs b/crates/fbuild-python/src/async_serial_monitor.rs new file mode 100644 index 00000000..f461f152 --- /dev/null +++ b/crates/fbuild-python/src/async_serial_monitor.rs @@ -0,0 +1,323 @@ +//! Asynchronous `AsyncSerialMonitor` PyO3 binding — the native async +//! counterpart to `SerialMonitor` (FastLED/fbuild#65). + +use futures::{SinkExt, StreamExt}; +use pyo3::prelude::*; +use serde::Serialize; +use std::sync::Arc; +use tokio_tungstenite::tungstenite; + +use crate::json_rpc::{ + encode_payload, read_lines_async, wait_for_remote_json_rpc_response_async, write_async, +}; +use crate::messages::{ClientMessage, ServerMessage, WsSink, WsSource}; + +/// Python-visible AsyncSerialMonitor class. +/// +/// Native async equivalent of `SerialMonitor`. Exposes every long-running +/// serial operation (`read_lines`, `write`, `write_json_rpc`) plus the +/// context-manager pair (`__aenter__` / `__aexit__`) as `async def` so +/// callers can `await` them directly from an asyncio event loop without +/// FastLED's thread-pool shim (`_run_in_thread`). See FastLED/fbuild#65. +/// +/// ## Send + Sync refactor +/// +/// The sync `SerialMonitor` wraps its `WsSink`/`WsSource` halves in +/// `std::sync::Mutex`, which cannot be held across `.await`. For the async +/// surface we switch to `Arc>>`: +/// +/// * `tokio::sync::Mutex` — safe to hold across `.await` points, which +/// we do when calling `sink.send(...).await` / `source.next().await`. +/// * `Arc<...>` — lets us `clone` the handle into `async move { ... }` +/// blocks handed to `future_into_py`, so each future owns its own +/// reference into the shared connection state. +/// * `Option<_>` — the WS halves are absent before `__aenter__` and +/// after `__aexit__`, and the outer `Arc>>` stays +/// alive across both transitions. +/// * Read/write halves live in **separate** mutexes so a pending +/// `read_lines` does not serialize an unrelated `write` (each direction +/// of the WebSocket has independent framing state). +/// +/// The sync `SerialMonitor` is untouched — this is a purely additive +/// surface. +/// +/// ```python +/// import asyncio +/// from fbuild._native import AsyncSerialMonitor +/// +/// async def main(): +/// async with AsyncSerialMonitor(port="COM13", baud_rate=115200) as mon: +/// lines = await mon.read_lines(timeout_secs=5.0) +/// await mon.write("hello\n") +/// ok = await mon.reset_device(board="esp32s3") +/// +/// asyncio.run(main()) +/// ``` +#[pyclass] +pub(crate) struct AsyncSerialMonitor { + port: String, + baud_rate: u32, + auto_reconnect: bool, + verbose: bool, + client_id: String, + ws_write: Arc>>, + ws_read: Arc>>, +} + +#[pymethods] +impl AsyncSerialMonitor { + #[new] + #[pyo3(signature = (port, baud_rate=115200, auto_reconnect=true, verbose=false))] + fn new(port: String, baud_rate: u32, auto_reconnect: bool, verbose: bool) -> Self { + Self { + port, + baud_rate, + auto_reconnect, + verbose, + client_id: uuid::Uuid::new_v4().to_string(), + ws_write: Arc::new(tokio::sync::Mutex::new(None)), + ws_read: Arc::new(tokio::sync::Mutex::new(None)), + } + } + + /// Async context-manager entry. Connects to the daemon's + /// `/ws/serial-monitor` endpoint, sends the `attach` handshake, and + /// stores the split sink/source halves for subsequent `read_lines` / + /// `write` calls. Mirrors the sync `SerialMonitor::__enter__` contract. + fn __aenter__<'py>(slf: PyRef<'py, Self>, py: Python<'py>) -> PyResult> { + let port = slf.port.clone(); + let baud_rate = slf.baud_rate; + let client_id = slf.client_id.clone(); + let verbose = slf.verbose; + let ws_write_slot = slf.ws_write.clone(); + let ws_read_slot = slf.ws_read.clone(); + let slf_obj: PyObject = slf.into_py(py); + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let daemon_port = fbuild_paths::get_daemon_port(); + let ws_url = format!("ws://127.0.0.1:{}/ws/serial-monitor", daemon_port); + + let (ws_stream, _) = tokio_tungstenite::connect_async(&ws_url) + .await + .map_err(|e| { + pyo3::exceptions::PyConnectionError::new_err(format!( + "failed to connect to daemon WebSocket at {}: {}", + ws_url, e + )) + })?; + + let (mut write, mut read) = ws_stream.split(); + + let attach = ClientMessage::Attach { + client_id: client_id.clone(), + port: port.clone(), + baud_rate, + open_if_needed: true, + pre_acquire_writer: true, + }; + let attach_json = serde_json::to_string(&attach).unwrap(); + + write + .send(tungstenite::Message::Text(attach_json)) + .await + .map_err(|e| { + pyo3::exceptions::PyConnectionError::new_err(format!( + "failed to send attach: {}", + e + )) + })?; + + let msg = read + .next() + .await + .ok_or_else(|| { + pyo3::exceptions::PyConnectionError::new_err("WebSocket closed before attach") + })? + .map_err(|e| { + pyo3::exceptions::PyConnectionError::new_err(format!("WebSocket error: {}", e)) + })?; + + if let tungstenite::Message::Text(text) = msg { + match serde_json::from_str::(&text) { + Ok(ServerMessage::Attached { success, .. }) if success => { + if verbose { + eprintln!("attached to {} at {} baud", port, baud_rate); + } + } + Ok(ServerMessage::Error { message }) => { + return Err(pyo3::exceptions::PyRuntimeError::new_err(format!( + "attach failed: {}", + message + ))); + } + _ => { + return Err(pyo3::exceptions::PyRuntimeError::new_err( + "unexpected response to attach", + )); + } + } + } + + *ws_write_slot.lock().await = Some(write); + *ws_read_slot.lock().await = Some(read); + + Ok(slf_obj) + }) + } + + /// Async context-manager exit. Sends a `detach` + `Close` frame and + /// clears the stored sink/source halves. Mirrors sync `__exit__`. + #[pyo3(signature = (_exc_type=None, _exc_val=None, _exc_tb=None))] + fn __aexit__<'py>( + &self, + py: Python<'py>, + _exc_type: Option, + _exc_val: Option, + _exc_tb: Option, + ) -> PyResult> { + let ws_write_slot = self.ws_write.clone(); + let ws_read_slot = self.ws_read.clone(); + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + if let Some(mut write) = ws_write_slot.lock().await.take() { + let detach = serde_json::to_string(&ClientMessage::Detach).unwrap(); + let _ = write.send(tungstenite::Message::Text(detach)).await; + let _ = write.send(tungstenite::Message::Close(None)).await; + } + let _ = ws_read_slot.lock().await.take(); + Ok(false) + }) + } + + /// Async counterpart to `SerialMonitor::read_lines`. Pulls batches of + /// lines from the daemon until at least one batch arrives or the + /// timeout elapses. Handles `Preempted` / `Reconnected` transparently + /// when `auto_reconnect=true` just like the sync path. + #[pyo3(signature = (timeout_secs=30.0))] + fn read_lines<'py>(&self, py: Python<'py>, timeout_secs: f64) -> PyResult> { + let ws_read_slot = self.ws_read.clone(); + let auto_reconnect = self.auto_reconnect; + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + Ok(read_lines_async(ws_read_slot, auto_reconnect, timeout_secs).await) + }) + } + + /// Async counterpart to `SerialMonitor::write`. Returns `true` on + /// successful delivery (daemon acknowledged with `write_ack`), + /// `false` otherwise. Mirrors the sync contract except the return + /// type is a bool rather than `bytes_written` to match the async + /// signature spec in FastLED/fbuild#65. + fn write<'py>(&self, py: Python<'py>, data: &str) -> PyResult> { + let ws_write_slot = self.ws_write.clone(); + let ws_read_slot = self.ws_read.clone(); + let encoded = encode_payload(data); + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + Ok(write_async(ws_write_slot, ws_read_slot, encoded).await) + }) + } + + /// Async counterpart to `SerialMonitor::write_json_rpc`. Serializes + /// `request` to JSON, sends it with a trailing newline, then polls + /// `read_lines` until a `REMOTE:` response arrives or the full + /// `timeout_secs` elapses. Honors the PR #57 guarantee that an empty + /// batch does not short-circuit the overall deadline. + /// + /// Returns the JSON-decoded response on success, raises + /// `TimeoutError` on timeout. + #[pyo3(signature = (request, timeout_secs=5.0))] + fn write_json_rpc<'py>( + &self, + py: Python<'py>, + request: &Bound<'_, PyAny>, + timeout_secs: f64, + ) -> PyResult> { + // Serialize the request on the calling thread (needs the GIL) + // before we enter the async block, mirroring the send_op_async + // pattern of moving only owned primitives across the .await + // boundary. + let json_str: String = py + .import_bound("json")? + .call_method1("dumps", (request,))? + .extract()?; + let data = format!("{}\n", json_str); + let encoded = encode_payload(&data); + + let ws_write_slot = self.ws_write.clone(); + let ws_read_slot = self.ws_read.clone(); + let auto_reconnect = self.auto_reconnect; + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + // Fire-and-acknowledge the write first; we don't abort on + // a write failure because the sync surface also proceeds + // to poll for a REMOTE: response (which is how the Python + // tests exercise this path). + let _ = write_async(ws_write_slot, ws_read_slot.clone(), encoded).await; + + let json_part = + wait_for_remote_json_rpc_response_async(timeout_secs, ws_read_slot, auto_reconnect) + .await; + + match json_part { + Some(payload) => Python::with_gil(|py| { + let json_module = py.import_bound("json")?; + let parsed = json_module.call_method1("loads", (payload.trim(),))?; + Ok(parsed.unbind()) + }), + None => Err(pyo3::exceptions::PyTimeoutError::new_err(format!( + "no REMOTE: response within {} seconds", + timeout_secs + ))), + } + }) + } + + /// Asynchronously reset the device via the daemon's `POST /api/reset` + /// endpoint. Returns `True` if the daemon reported success. + #[pyo3(signature = (board=None))] + fn reset_device<'py>( + &self, + py: Python<'py>, + board: Option, + ) -> PyResult> { + let url = format!("{}/api/reset", fbuild_paths::get_daemon_url()); + let port = self.port.clone(); + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + #[derive(Serialize)] + struct ResetPayload { + port: String, + #[serde(skip_serializing_if = "Option::is_none")] + board: Option, + } + + let payload = ResetPayload { port, board }; + + let resp = reqwest::Client::new() + .post(&url) + .json(&payload) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await + .map_err(|e| { + pyo3::exceptions::PyConnectionError::new_err(format!( + "failed to send reset request to daemon: {}", + e + )) + })?; + + let body: serde_json::Value = resp.json().await.map_err(|e| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "failed to parse reset response: {}", + e + )) + })?; + + Ok(body + .get("success") + .and_then(|v| v.as_bool()) + .unwrap_or(false)) + }) + } +} diff --git a/crates/fbuild-python/src/daemon.rs b/crates/fbuild-python/src/daemon.rs new file mode 100644 index 00000000..ced05a80 --- /dev/null +++ b/crates/fbuild-python/src/daemon.rs @@ -0,0 +1,214 @@ +//! Synchronous `Daemon` and asynchronous `AsyncDaemon` PyO3 bindings. + +use pyo3::prelude::*; + +/// Python-visible Daemon class (high-level API). +#[pyclass] +pub(crate) struct Daemon; + +#[pymethods] +impl Daemon { + #[staticmethod] + fn ensure_running() -> bool { + let url = format!("{}/health", fbuild_paths::get_daemon_url()); + if let Ok(resp) = reqwest::blocking::get(&url) { + if resp.status().is_success() { + return true; + } + } + + // INTENTIONALLY DETACHED (FastLED/fbuild#32): the Python host + // spawns the daemon and then the Python interpreter may exit — + // the daemon must survive. This PyO3 binding runs inside the + // Python interpreter process, which has no global containment + // group, so `spawn()` is already uncontained; see the matching + // comment in fbuild-cli/src/daemon_client.rs. + // allow-direct-spawn: daemon must outlive the Python interpreter. + let mut cmd = std::process::Command::new("fbuild-daemon"); + if fbuild_paths::is_dev_mode() { + cmd.arg("--dev"); + } + cmd.stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + + if cmd.spawn().is_err() { + return false; + } + + for _ in 0..100 { + std::thread::sleep(std::time::Duration::from_millis(100)); + if let Ok(resp) = reqwest::blocking::get(&url) { + if resp.status().is_success() { + return true; + } + } + } + false + } + + #[staticmethod] + fn stop() -> bool { + let url = format!("{}/api/daemon/shutdown", fbuild_paths::get_daemon_url()); + reqwest::blocking::Client::new() + .post(&url) + .send() + .map(|r| r.status().is_success()) + .unwrap_or(false) + } + + #[staticmethod] + fn status(py: Python<'_>) -> PyResult { + let url = format!("{}/api/daemon/info", fbuild_paths::get_daemon_url()); + let resp = reqwest::blocking::get(&url).map_err(|e| { + pyo3::exceptions::PyConnectionError::new_err(format!( + "failed to connect to daemon: {}", + e + )) + })?; + let text = resp.text().map_err(|e| { + pyo3::exceptions::PyRuntimeError::new_err(format!("failed to read response: {}", e)) + })?; + let json_module = py.import_bound("json")?; + let result = json_module.call_method1("loads", (text,))?; + Ok(result.to_object(py)) + } +} + +/// Python-visible AsyncDaemon class. +/// +/// Native async counterpart to `Daemon`. Follows the same additive +/// pattern as `AsyncSerialMonitor` (Issue #65): the sync `Daemon` class +/// stays unchanged, and this one exposes async methods so callers under +/// an asyncio event loop can `await` them directly. +/// +/// ```python +/// import asyncio +/// from fbuild._native import AsyncDaemon +/// +/// async def main(): +/// info = await AsyncDaemon.status() +/// +/// asyncio.run(main()) +/// ``` +#[pyclass] +pub(crate) struct AsyncDaemon; + +#[pymethods] +impl AsyncDaemon { + /// Asynchronously fetch `/api/daemon/info` from the daemon. Returns + /// a JSON-deserialized Python object on success, or raises a + /// ConnectionError/RuntimeError. + #[staticmethod] + fn status(py: Python<'_>) -> PyResult> { + let url = format!("{}/api/daemon/info", fbuild_paths::get_daemon_url()); + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let resp = reqwest::Client::new() + .get(&url) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await + .map_err(|e| { + pyo3::exceptions::PyConnectionError::new_err(format!( + "failed to connect to daemon: {}", + e + )) + })?; + + let text = resp.text().await.map_err(|e| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "failed to read daemon response: {}", + e + )) + })?; + + Python::with_gil(|py| { + let json_module = py.import_bound("json")?; + let parsed = json_module.call_method1("loads", (text,))?; + Ok(parsed.unbind()) + }) + }) + } + + /// Asynchronously ensure the daemon is running. Mirrors the sync + /// `Daemon.ensure_running` contract: returns `True` if the daemon + /// responds to `/health`, spawning a new `fbuild-daemon` process if + /// needed and polling until the health endpoint succeeds. + /// + /// The spawn itself is synchronous (`std::process::Command::spawn`) + /// because `tokio::process::Command` adds no value for a detached + /// child — the child does not need an async stdio pipe. The key win + /// for async callers is that the health poll loop uses + /// `tokio::time::sleep` and async reqwest instead of blocking the + /// event loop thread. + #[staticmethod] + fn ensure_running(py: Python<'_>) -> PyResult> { + let url = format!("{}/health", fbuild_paths::get_daemon_url()); + let dev_mode = fbuild_paths::is_dev_mode(); + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let client = reqwest::Client::new(); + + // Fast path: daemon is already up. + if let Ok(resp) = client + .get(&url) + .timeout(std::time::Duration::from_secs(5)) + .send() + .await + { + if resp.status().is_success() { + return Ok(true); + } + } + + // INTENTIONALLY DETACHED (FastLED/fbuild#32): see the + // matching comment in `Daemon::ensure_running` above. + // allow-direct-spawn: daemon must outlive the Python interpreter. + let mut cmd = std::process::Command::new("fbuild-daemon"); + if dev_mode { + cmd.arg("--dev"); + } + cmd.stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + + if cmd.spawn().is_err() { + return Ok(false); + } + + for _ in 0..100 { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + if let Ok(resp) = client + .get(&url) + .timeout(std::time::Duration::from_secs(5)) + .send() + .await + { + if resp.status().is_success() { + return Ok(true); + } + } + } + Ok(false) + }) + } + + /// Asynchronously shut down the daemon via `POST /api/daemon/shutdown`. + /// Returns `True` if the daemon acknowledged with a 2xx response. + #[staticmethod] + fn stop(py: Python<'_>) -> PyResult> { + let url = format!("{}/api/daemon/shutdown", fbuild_paths::get_daemon_url()); + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let ok = reqwest::Client::new() + .post(&url) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await + .map(|r| r.status().is_success()) + .unwrap_or(false); + Ok(ok) + }) + } +} diff --git a/crates/fbuild-python/src/daemon_connection.rs b/crates/fbuild-python/src/daemon_connection.rs new file mode 100644 index 00000000..f80411ee --- /dev/null +++ b/crates/fbuild-python/src/daemon_connection.rs @@ -0,0 +1,170 @@ +//! Synchronous `DaemonConnection` PyO3 binding. + +use pyo3::prelude::*; + +use crate::outcome::{ + build_url, deploy_url, monitor_url, outcome_to_pydict, send_op, OpRequest, +}; + +/// Python-visible DaemonConnection (context manager). +#[pyclass] +pub(crate) struct DaemonConnection { + project_dir: String, + environment: String, +} + +#[pymethods] +impl DaemonConnection { + #[new] + pub(crate) fn new(project_dir: String, environment: String) -> Self { + Self { + project_dir, + environment, + } + } + + fn __enter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } + + #[pyo3(signature = (_exc_type=None, _exc_val=None, _exc_tb=None))] + fn __exit__( + &self, + _exc_type: Option<&Bound<'_, PyAny>>, + _exc_val: Option<&Bound<'_, PyAny>>, + _exc_tb: Option<&Bound<'_, PyAny>>, + ) -> bool { + false + } + + #[pyo3(signature = (clean=false, verbose=false, timeout=1800.0))] + fn build(&self, clean: bool, verbose: bool, timeout: f64) -> bool { + send_op(&build_url(), &self.build_request(clean, verbose), timeout).success + } + + #[pyo3(signature = (port=None, clean=false, skip_build=false, monitor_after=false, timeout=1800.0))] + fn deploy( + &self, + port: Option, + clean: bool, + skip_build: bool, + monitor_after: bool, + timeout: f64, + ) -> bool { + send_op( + &deploy_url(), + &self.deploy_request(port, clean, skip_build, monitor_after), + timeout, + ) + .success + } + + #[pyo3(signature = (port=None, baud_rate=None, timeout=None))] + fn monitor(&self, port: Option, baud_rate: Option, timeout: Option) -> bool { + send_op( + &monitor_url(), + &self.monitor_request(port, baud_rate), + timeout.unwrap_or(1800.0), + ) + .success + } + + /// Same as `build()` but returns a dict with structured result fields: + /// `success`, `message`, `exit_code`, `stdout`, `stderr`. Callers that + /// need to branch on failure mode can inspect the dict instead of + /// swallowing a bare bool. See FastLED/fbuild#18. + #[pyo3(signature = (clean=false, verbose=false, timeout=1800.0))] + fn build_result<'py>( + &self, + py: Python<'py>, + clean: bool, + verbose: bool, + timeout: f64, + ) -> PyResult> { + let outcome = send_op(&build_url(), &self.build_request(clean, verbose), timeout); + outcome_to_pydict(py, &outcome) + } + + /// Structured-result counterpart to `deploy()`. See `build_result()`. + #[pyo3(signature = (port=None, clean=false, skip_build=false, monitor_after=false, timeout=1800.0))] + fn deploy_result<'py>( + &self, + py: Python<'py>, + port: Option, + clean: bool, + skip_build: bool, + monitor_after: bool, + timeout: f64, + ) -> PyResult> { + let outcome = send_op( + &deploy_url(), + &self.deploy_request(port, clean, skip_build, monitor_after), + timeout, + ); + outcome_to_pydict(py, &outcome) + } + + /// Structured-result counterpart to `monitor()`. See `build_result()`. + #[pyo3(signature = (port=None, baud_rate=None, timeout=None))] + fn monitor_result<'py>( + &self, + py: Python<'py>, + port: Option, + baud_rate: Option, + timeout: Option, + ) -> PyResult> { + let outcome = send_op( + &monitor_url(), + &self.monitor_request(port, baud_rate), + timeout.unwrap_or(1800.0), + ); + outcome_to_pydict(py, &outcome) + } +} + +impl DaemonConnection { + fn build_request(&self, clean: bool, verbose: bool) -> OpRequest { + OpRequest { + project_dir: self.project_dir.clone(), + environment: Some(self.environment.clone()), + clean_build: clean, + verbose, + port: None, + monitor_after: false, + skip_build: false, + baud_rate: None, + } + } + + fn deploy_request( + &self, + port: Option, + clean: bool, + skip_build: bool, + monitor_after: bool, + ) -> OpRequest { + OpRequest { + project_dir: self.project_dir.clone(), + environment: Some(self.environment.clone()), + clean_build: clean, + verbose: false, + port, + monitor_after, + skip_build, + baud_rate: None, + } + } + + fn monitor_request(&self, port: Option, baud_rate: Option) -> OpRequest { + OpRequest { + project_dir: self.project_dir.clone(), + environment: Some(self.environment.clone()), + clean_build: false, + verbose: false, + port, + monitor_after: false, + skip_build: false, + baud_rate, + } + } +} diff --git a/crates/fbuild-python/src/json_rpc.rs b/crates/fbuild-python/src/json_rpc.rs new file mode 100644 index 00000000..298b53b9 --- /dev/null +++ b/crates/fbuild-python/src/json_rpc.rs @@ -0,0 +1,161 @@ +//! REMOTE: JSON-RPC response helpers and shared async read/write loops +//! used by both `SerialMonitor` and `AsyncSerialMonitor`. + +use base64::Engine; +use futures::{SinkExt, StreamExt}; +use std::sync::Arc; +use tokio_tungstenite::tungstenite; + +use crate::messages::{ClientMessage, ServerMessage, WsSink, WsSource}; + +pub(crate) fn extract_remote_json_rpc_response(lines: &[String]) -> Option { + lines.iter().find_map(|line| { + line.strip_prefix("REMOTE:") + .map(|json_part| json_part.to_string()) + }) +} + +pub(crate) fn wait_for_remote_json_rpc_response(timeout: f64, mut poll: F) -> Option +where + F: FnMut(f64) -> Vec, +{ + let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout); + + while std::time::Instant::now() < deadline { + let remaining = (deadline - std::time::Instant::now()).as_secs_f64(); + let lines = poll(remaining); + if let Some(json_part) = extract_remote_json_rpc_response(&lines) { + return Some(json_part); + } + } + + None +} + +/// Shared async read-batch loop used by `AsyncSerialMonitor::read_lines` +/// and `write_json_rpc`. Acquires the source mutex only for the duration +/// of each `.next()` call so that concurrent `write` / `__aexit__` +/// futures can still progress between iterations. +pub(crate) async fn read_lines_async( + ws_read_slot: Arc>>, + auto_reconnect: bool, + timeout_secs: f64, +) -> Vec { + let mut lines = Vec::new(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout_secs); + + loop { + let now = std::time::Instant::now(); + if now >= deadline { + break; + } + let remaining = deadline - now; + + let mut guard = ws_read_slot.lock().await; + let Some(source) = guard.as_mut() else { + // Session not entered (or already exited). Nothing to read. + break; + }; + + let result = tokio::time::timeout(remaining, source.next()).await; + // Drop the guard before continuing the loop so another task + // can take the mutex (e.g. __aexit__). + drop(guard); + + match result { + Ok(Some(Ok(tungstenite::Message::Text(text)))) => { + match serde_json::from_str::(&text) { + Ok(ServerMessage::Data { + lines: data_lines, .. + }) => { + lines.extend(data_lines); + if !lines.is_empty() { + break; + } + } + Ok(ServerMessage::Preempted { .. }) => { + if auto_reconnect { + continue; + } + break; + } + Ok(ServerMessage::Reconnected { .. }) => continue, + _ => continue, + } + } + Ok(Some(Ok(tungstenite::Message::Close(_)))) | Ok(None) => break, + Err(_) => break, // timeout + _ => continue, + } + } + + lines +} + +/// Shared async write path used by `AsyncSerialMonitor::write` and +/// `write_json_rpc`. Sends the already-base64-encoded payload, then +/// awaits a `write_ack` (with a 5-second bound matching the sync path). +/// Returns `true` if the ack reported any bytes written. +pub(crate) async fn write_async( + ws_write_slot: Arc>>, + ws_read_slot: Arc>>, + encoded: String, +) -> bool { + let msg = serde_json::to_string(&ClientMessage::Write { data: encoded }).unwrap(); + + { + let mut guard = ws_write_slot.lock().await; + let Some(sink) = guard.as_mut() else { + return false; + }; + if sink.send(tungstenite::Message::Text(msg)).await.is_err() { + return false; + } + } + + // Wait for write_ack. Hold the read half's mutex only across this + // single `.next()` so concurrent `read_lines` futures can resume. + let mut guard = ws_read_slot.lock().await; + let Some(source) = guard.as_mut() else { + return false; + }; + let ack_timeout = std::time::Duration::from_secs(5); + match tokio::time::timeout(ack_timeout, source.next()).await { + Ok(Some(Ok(tungstenite::Message::Text(text)))) => { + matches!( + serde_json::from_str::(&text), + Ok(ServerMessage::WriteAck { bytes_written, .. }) if bytes_written > 0 + ) + } + _ => false, + } +} + +/// Async counterpart to `wait_for_remote_json_rpc_response`. Keeps +/// polling `read_lines_async` until the deadline expires, even if an +/// individual batch comes back empty — preserving the PR #57 fix. +pub(crate) async fn wait_for_remote_json_rpc_response_async( + timeout_secs: f64, + ws_read_slot: Arc>>, + auto_reconnect: bool, +) -> Option { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout_secs); + + while std::time::Instant::now() < deadline { + let remaining = (deadline - std::time::Instant::now()).as_secs_f64(); + if remaining <= 0.0 { + break; + } + let lines = read_lines_async(ws_read_slot.clone(), auto_reconnect, remaining).await; + if let Some(json_part) = extract_remote_json_rpc_response(&lines) { + return Some(json_part); + } + } + + None +} + +/// Helper: produce a base64-encoded write payload from raw bytes. +pub(crate) fn encode_payload(data: &str) -> String { + base64::engine::general_purpose::STANDARD.encode(data.as_bytes()) +} diff --git a/crates/fbuild-python/src/lib.rs b/crates/fbuild-python/src/lib.rs index fbf5027e..8dfb3327 100644 --- a/crates/fbuild-python/src/lib.rs +++ b/crates/fbuild-python/src/lib.rs @@ -18,1010 +18,31 @@ //! Python classes are thin wrappers around Rust types. The SerialMonitor //! maintains a tokio runtime internally for async serial operations, //! exposed as sync methods via `block_on()`. +//! +//! Implementation is split across topic-focused submodules; this file is +//! intentionally slim and contains only the `#[pymodule]` entry point, +//! the version constant, the small standalone `#[pyfunction]`s, and the +//! integration tests. All Python-visible classes and helper types live in +//! their respective sibling modules — see `mod` declarations below. #![allow(clippy::useless_conversion)] -use base64::Engine; -use futures::{SinkExt, StreamExt}; use pyo3::prelude::*; -use serde::{Deserialize, Serialize}; -use std::sync::{Arc, Mutex}; -use tokio::runtime::Runtime; -use tokio_tungstenite::tungstenite; - -type WsStream = - tokio_tungstenite::WebSocketStream>; -type WsSink = futures::stream::SplitSink; -type WsSource = futures::stream::SplitStream; - -/// Python-visible SerialMonitor class. -/// -/// This is the critical binding that FastLED depends on. -/// It wraps the Rust SharedSerialManager via WebSocket, -/// matching the original Python API: -/// -/// ```python -/// with SerialMonitor(port="COM13", baud_rate=115200) as mon: -/// for line in mon.read_lines(timeout=30.0): -/// print(line) -/// mon.write("hello\n") -/// ``` -#[pyclass] -struct SerialMonitor { - port: String, - baud_rate: u32, - auto_reconnect: bool, - verbose: bool, - hooks: Vec, - runtime: Option, - ws_write: Option>, - ws_read: Option>, - client_id: String, - last_line: String, - #[allow(dead_code)] - preempted: bool, -} - -/// Messages we receive from the daemon (subset we care about). -#[derive(Debug, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum ServerMessage { - Attached { - success: bool, - #[allow(dead_code)] - message: String, - #[allow(dead_code)] - writer_pre_acquired: bool, - }, - Data { - lines: Vec, - #[allow(dead_code)] - current_index: u64, - }, - WriteAck { - #[allow(dead_code)] - success: bool, - bytes_written: usize, - #[allow(dead_code)] - message: Option, - }, - Preempted { - #[allow(dead_code)] - reason: String, - #[allow(dead_code)] - preempted_by: String, - }, - Reconnected { - #[allow(dead_code)] - message: String, - }, - Error { - message: String, - }, - #[serde(other)] - Other, -} - -/// Client message to send to the daemon. -#[derive(Debug, Serialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum ClientMessage { - Attach { - client_id: String, - port: String, - baud_rate: u32, - open_if_needed: bool, - pre_acquire_writer: bool, - }, - Write { - data: String, - }, - Detach, -} - -#[pymethods] -impl SerialMonitor { - #[new] - #[pyo3(signature = (port, baud_rate=115200, hooks=None, auto_reconnect=true, verbose=false))] - fn new( - port: String, - baud_rate: u32, - hooks: Option>, - auto_reconnect: bool, - verbose: bool, - ) -> Self { - Self { - port, - baud_rate, - auto_reconnect, - verbose, - hooks: hooks.unwrap_or_default(), - runtime: None, - ws_write: None, - ws_read: None, - client_id: uuid::Uuid::new_v4().to_string(), - last_line: String::new(), - preempted: false, - } - } - - fn __enter__(mut slf: PyRefMut<'_, Self>) -> PyResult> { - let rt = Runtime::new().map_err(|e| { - pyo3::exceptions::PyRuntimeError::new_err(format!("failed to create runtime: {}", e)) - })?; - - // Connect to daemon WebSocket - let daemon_port = fbuild_paths::get_daemon_port(); - let ws_url = format!("ws://127.0.0.1:{}/ws/serial-monitor", daemon_port); - - let (ws_stream, _) = rt - .block_on(tokio_tungstenite::connect_async(&ws_url)) - .map_err(|e| { - pyo3::exceptions::PyConnectionError::new_err(format!( - "failed to connect to daemon WebSocket at {}: {}", - ws_url, e - )) - })?; - - let (mut write, mut read) = ws_stream.split(); - - // Send attach message - let attach = ClientMessage::Attach { - client_id: slf.client_id.clone(), - port: slf.port.clone(), - baud_rate: slf.baud_rate, - open_if_needed: true, - pre_acquire_writer: true, - }; - let attach_json = serde_json::to_string(&attach).unwrap(); - - rt.block_on(write.send(tungstenite::Message::Text(attach_json))) - .map_err(|e| { - pyo3::exceptions::PyConnectionError::new_err(format!( - "failed to send attach: {}", - e - )) - })?; - - // Wait for attached response - let msg: tungstenite::Message = rt - .block_on(read.next()) - .ok_or_else(|| { - pyo3::exceptions::PyConnectionError::new_err("WebSocket closed before attach") - })? - .map_err(|e| { - pyo3::exceptions::PyConnectionError::new_err(format!("WebSocket error: {}", e)) - })?; - - if let tungstenite::Message::Text(text) = msg { - match serde_json::from_str::(&text) { - Ok(ServerMessage::Attached { success, .. }) if success => { - if slf.verbose { - eprintln!("attached to {} at {} baud", slf.port, slf.baud_rate); - } - } - Ok(ServerMessage::Error { message }) => { - return Err(pyo3::exceptions::PyRuntimeError::new_err(format!( - "attach failed: {}", - message - ))); - } - _ => { - return Err(pyo3::exceptions::PyRuntimeError::new_err( - "unexpected response to attach", - )); - } - } - } - - slf.ws_write = Some(Mutex::new(write)); - slf.ws_read = Some(Mutex::new(read)); - slf.runtime = Some(rt); - Ok(slf) - } - - #[pyo3(signature = (_exc_type=None, _exc_val=None, _exc_tb=None))] - fn __exit__( - &mut self, - _exc_type: Option<&Bound<'_, PyAny>>, - _exc_val: Option<&Bound<'_, PyAny>>, - _exc_tb: Option<&Bound<'_, PyAny>>, - ) -> bool { - if let (Some(ref rt), Some(ref ws_write)) = (&self.runtime, &self.ws_write) { - let detach = serde_json::to_string(&ClientMessage::Detach).unwrap(); - if let Ok(mut write) = ws_write.lock() { - let _ = rt.block_on(write.send(tungstenite::Message::Text(detach))); - let _ = rt.block_on(write.send(tungstenite::Message::Close(None))); - } - } - self.ws_write = None; - self.ws_read = None; - self.runtime = None; - false - } - - /// The last line received from the serial port. - #[getter] - fn last_line(&self) -> &str { - &self.last_line - } - - /// Iterate over serial output lines. - /// - /// Returns a list of lines received within the timeout period. - #[pyo3(signature = (timeout=30.0))] - fn read_lines(&mut self, py: Python<'_>, timeout: f64) -> Vec { - let (Some(ref rt), Some(ref ws_read)) = (&self.runtime, &self.ws_read) else { - return vec![]; - }; - - let mut lines = Vec::new(); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout); - let auto_reconnect = self.auto_reconnect; - - py.allow_threads(|| { - while std::time::Instant::now() < deadline { - let remaining = deadline - std::time::Instant::now(); - let result = { - let mut read = ws_read.lock().unwrap(); - // tokio::time::timeout MUST be constructed inside the - // runtime context, otherwise it panics with "there is - // no reactor running" because the Sleep future needs - // Handle::current() to register with the timer driver. - rt.block_on(async { tokio::time::timeout(remaining, read.next()).await }) - }; - - match result { - Ok(Some(Ok(tungstenite::Message::Text(text)))) => { - match serde_json::from_str::(&text) { - Ok(ServerMessage::Data { - lines: data_lines, .. - }) => { - lines.extend(data_lines); - if !lines.is_empty() { - break; - } - } - Ok(ServerMessage::Preempted { .. }) => { - // Pause — deploy is happening - if auto_reconnect { - continue; - } - break; - } - Ok(ServerMessage::Reconnected { .. }) => { - // Resume after deploy - continue; - } - _ => continue, - } - } - Ok(Some(Ok(tungstenite::Message::Close(_)))) | Ok(None) => break, - Err(_) => break, // timeout - _ => continue, - } - } - }); - - // Update last_line and dispatch hooks - if let Some(last) = lines.last() { - self.last_line = last.clone(); - } - - // Dispatch hooks for each line - if !self.hooks.is_empty() && !lines.is_empty() { - Python::with_gil(|py| { - for line in &lines { - for hook in &self.hooks { - let _ = hook.call1(py, (line,)); - } - } - }); - } - - lines - } - - /// Write data to the serial port. - fn write(&self, data: &str) -> usize { - let (Some(ref rt), Some(ref ws_write), Some(ref ws_read)) = - (&self.runtime, &self.ws_write, &self.ws_read) - else { - return 0; - }; - - let encoded = base64::engine::general_purpose::STANDARD.encode(data.as_bytes()); - let msg = serde_json::to_string(&ClientMessage::Write { data: encoded }).unwrap(); - - { - let mut write = ws_write.lock().unwrap(); - if rt - .block_on(write.send(tungstenite::Message::Text(msg))) - .is_err() - { - return 0; - } - } - - // Wait for write_ack - let mut read = ws_read.lock().unwrap(); - let timeout = std::time::Duration::from_secs(5); - // tokio::time::timeout must be created inside the runtime context. - match rt.block_on(async { tokio::time::timeout(timeout, read.next()).await }) { - Ok(Some(Ok(tungstenite::Message::Text(text)))) => { - if let Ok(ServerMessage::WriteAck { bytes_written, .. }) = - serde_json::from_str(&text) - { - return bytes_written; - } - 0 - } - _ => 0, - } - } - - /// Run monitor until condition returns True or timeout expires. - /// - /// Calls `condition(line)` for each received line. Returns True if - /// the condition was met, False on timeout. - #[pyo3(signature = (condition, timeout=30.0))] - fn run_until(&mut self, py: Python<'_>, condition: PyObject, timeout: f64) -> PyResult { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout); - - while std::time::Instant::now() < deadline { - let remaining = (deadline - std::time::Instant::now()).as_secs_f64(); - if remaining <= 0.0 { - break; - } - let lines = self.read_lines(py, remaining.min(1.0)); - for line in &lines { - let result: bool = condition.call1(py, (line,))?.extract(py)?; - if result { - return Ok(true); - } - } - if lines.is_empty() { - // Timeout on read_lines, check overall deadline - continue; - } - } - - Ok(false) - } - - /// Send a JSON-RPC request and wait for matching response. - #[pyo3(signature = (request, timeout=5.0))] - fn write_json_rpc( - &self, - py: Python<'_>, - request: &Bound<'_, PyAny>, - timeout: f64, - ) -> PyResult { - let json_str: String = py - .import_bound("json")? - .call_method1("dumps", (request,))? - .extract()?; - - let data = format!("{}\n", json_str); - self.write(&data); - - if let Some(json_part) = wait_for_remote_json_rpc_response(timeout, |remaining| { - // read_lines takes &mut self but we only have &self here — - // use the raw WS read directly. - self.read_lines_inner(remaining.min(1.0)) - }) { - let json_module = py.import_bound("json")?; - let result = json_module.call_method1("loads", (json_part.trim(),))?; - return Ok(result.to_object(py)); - } - - Err(pyo3::exceptions::PyTimeoutError::new_err(format!( - "no REMOTE: response within {} seconds", - timeout - ))) - } - - /// Reset the device via the daemon's DTR/RTS reset endpoint. - /// - /// Sends POST /api/reset to the daemon, which preempts any active - /// serial monitor session, toggles DTR/RTS to reset the device, - /// then clears preemption so monitors can reconnect. - /// - /// Works whether or not `__enter__` has been called — the reset goes - /// through the daemon's HTTP API, not the WebSocket session. - /// - /// Args: - /// board: Board identifier (e.g. "esp32s3", "teensy40"). - /// Determines the platform-specific reset sequence. - /// If None, a generic DTR toggle is used. - /// wait_for_output: If True, block until serial output is detected - /// after the reset (device has rebooted and is producing data). - /// If False (default), return immediately after reset. - /// timeout: Maximum seconds to wait for output (only used when - /// wait_for_output is True). Default: 5.0. - /// - /// Returns: - /// True if reset succeeded (and output detected, if wait_for_output). - /// False on failure or timeout. - #[pyo3(signature = (board=None, wait_for_output=false, timeout=5.0))] - fn reset_device( - &self, - board: Option, - wait_for_output: bool, - timeout: f64, - ) -> PyResult { - let url = format!("{}/api/reset", fbuild_paths::get_daemon_url()); - - #[derive(Serialize)] - struct ResetPayload { - port: String, - #[serde(skip_serializing_if = "Option::is_none")] - board: Option, - } - - let payload = ResetPayload { - port: self.port.clone(), - board, - }; - - let resp = reqwest::blocking::Client::new() - .post(&url) - .json(&payload) - .timeout(std::time::Duration::from_secs(10)) - .send() - .map_err(|e| { - pyo3::exceptions::PyConnectionError::new_err(format!( - "failed to send reset request to daemon: {}", - e - )) - })?; - - let body: serde_json::Value = resp.json().map_err(|e| { - pyo3::exceptions::PyRuntimeError::new_err(format!( - "failed to parse reset response: {}", - e - )) - })?; - - let success = body - .get("success") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - - if !success || !wait_for_output { - return Ok(success); - } - - // Wait for the device to produce serial output after reset. - let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout); - - // Brief pause for USB re-enumeration after DTR toggle - std::thread::sleep(std::time::Duration::from_millis(300)); - - // If WebSocket is connected (__enter__ was called), poll via read_lines. - // Note: the daemon preempts our session during reset and sends a - // "Reconnected" message after. With auto_reconnect=true the WebSocket - // transparently re-attaches, so read_lines_inner will see new output. - if self.runtime.is_some() && self.ws_read.is_some() { - while std::time::Instant::now() < deadline { - let remaining = (deadline - std::time::Instant::now()) - .as_secs_f64() - .min(0.2); - let lines = self.read_lines_inner(remaining); - if !lines.is_empty() { - return Ok(true); - } - } - return Ok(false); - } - - // No WebSocket — we can't observe output directly. - // Wait a conservative 1 second (ESP32-S3 USB-CDC typically boots - // in <500ms). The caller can pass a shorter timeout if needed. - let wait = timeout.min(1.0); - std::thread::sleep(std::time::Duration::from_secs_f64(wait)); - Ok(true) - } -} - -impl SerialMonitor { - /// Internal read_lines without hook dispatch (for write_json_rpc which has &self). - fn read_lines_inner(&self, timeout: f64) -> Vec { - let (Some(ref rt), Some(ref ws_read)) = (&self.runtime, &self.ws_read) else { - return vec![]; - }; - let mut lines = Vec::new(); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout); - let auto_reconnect = self.auto_reconnect; - - while std::time::Instant::now() < deadline { - let remaining = deadline - std::time::Instant::now(); - let result = { - let mut read = ws_read.lock().unwrap(); - // tokio::time::timeout must be created inside the runtime - // context (otherwise: "there is no reactor running" panic). - rt.block_on(async { tokio::time::timeout(remaining, read.next()).await }) - }; - - match result { - Ok(Some(Ok(tungstenite::Message::Text(text)))) => { - match serde_json::from_str::(&text) { - Ok(ServerMessage::Data { - lines: data_lines, .. - }) => { - lines.extend(data_lines); - if !lines.is_empty() { - break; - } - } - Ok(ServerMessage::Preempted { .. }) => { - if auto_reconnect { - continue; - } - break; - } - Ok(ServerMessage::Reconnected { .. }) => { - continue; - } - _ => continue, - } - } - Ok(Some(Ok(tungstenite::Message::Close(_)))) | Ok(None) => break, - Err(_) => break, - _ => continue, - } - } - - lines - } -} - -fn extract_remote_json_rpc_response(lines: &[String]) -> Option { - lines.iter().find_map(|line| { - line.strip_prefix("REMOTE:") - .map(|json_part| json_part.to_string()) - }) -} - -fn wait_for_remote_json_rpc_response(timeout: f64, mut poll: F) -> Option -where - F: FnMut(f64) -> Vec, -{ - let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout); - - while std::time::Instant::now() < deadline { - let remaining = (deadline - std::time::Instant::now()).as_secs_f64(); - let lines = poll(remaining); - if let Some(json_part) = extract_remote_json_rpc_response(&lines) { - return Some(json_part); - } - } - - None -} - -/// Python-visible Daemon class (high-level API). -#[pyclass] -struct Daemon; - -#[pymethods] -impl Daemon { - #[staticmethod] - fn ensure_running() -> bool { - let url = format!("{}/health", fbuild_paths::get_daemon_url()); - if let Ok(resp) = reqwest::blocking::get(&url) { - if resp.status().is_success() { - return true; - } - } - - // INTENTIONALLY DETACHED (FastLED/fbuild#32): the Python host - // spawns the daemon and then the Python interpreter may exit — - // the daemon must survive. This PyO3 binding runs inside the - // Python interpreter process, which has no global containment - // group, so `spawn()` is already uncontained; see the matching - // comment in fbuild-cli/src/daemon_client.rs. - // allow-direct-spawn: daemon must outlive the Python interpreter. - let mut cmd = std::process::Command::new("fbuild-daemon"); - if fbuild_paths::is_dev_mode() { - cmd.arg("--dev"); - } - cmd.stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()); - - if cmd.spawn().is_err() { - return false; - } - - for _ in 0..100 { - std::thread::sleep(std::time::Duration::from_millis(100)); - if let Ok(resp) = reqwest::blocking::get(&url) { - if resp.status().is_success() { - return true; - } - } - } - false - } - - #[staticmethod] - fn stop() -> bool { - let url = format!("{}/api/daemon/shutdown", fbuild_paths::get_daemon_url()); - reqwest::blocking::Client::new() - .post(&url) - .send() - .map(|r| r.status().is_success()) - .unwrap_or(false) - } - - #[staticmethod] - fn status(py: Python<'_>) -> PyResult { - let url = format!("{}/api/daemon/info", fbuild_paths::get_daemon_url()); - let resp = reqwest::blocking::get(&url).map_err(|e| { - pyo3::exceptions::PyConnectionError::new_err(format!( - "failed to connect to daemon: {}", - e - )) - })?; - let text = resp.text().map_err(|e| { - pyo3::exceptions::PyRuntimeError::new_err(format!("failed to read response: {}", e)) - })?; - let json_module = py.import_bound("json")?; - let result = json_module.call_method1("loads", (text,))?; - Ok(result.to_object(py)) - } -} - -/// Python-visible DaemonConnection (context manager). -#[pyclass] -struct DaemonConnection { - project_dir: String, - environment: String, -} - -#[derive(Clone, Serialize)] -struct OpRequest { - project_dir: String, - #[serde(skip_serializing_if = "Option::is_none")] - environment: Option, - #[serde(skip_serializing_if = "std::ops::Not::not")] - clean_build: bool, - #[serde(skip_serializing_if = "std::ops::Not::not")] - verbose: bool, - #[serde(skip_serializing_if = "Option::is_none")] - port: Option, - #[serde(skip_serializing_if = "std::ops::Not::not")] - monitor_after: bool, - #[serde(skip_serializing_if = "std::ops::Not::not")] - skip_build: bool, - #[serde(skip_serializing_if = "Option::is_none")] - baud_rate: Option, -} - -#[pymethods] -impl DaemonConnection { - #[new] - fn new(project_dir: String, environment: String) -> Self { - Self { - project_dir, - environment, - } - } - - fn __enter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { - slf - } - - #[pyo3(signature = (_exc_type=None, _exc_val=None, _exc_tb=None))] - fn __exit__( - &self, - _exc_type: Option<&Bound<'_, PyAny>>, - _exc_val: Option<&Bound<'_, PyAny>>, - _exc_tb: Option<&Bound<'_, PyAny>>, - ) -> bool { - false - } - - #[pyo3(signature = (clean=false, verbose=false, timeout=1800.0))] - fn build(&self, clean: bool, verbose: bool, timeout: f64) -> bool { - send_op(&build_url(), &self.build_request(clean, verbose), timeout).success - } - - #[pyo3(signature = (port=None, clean=false, skip_build=false, monitor_after=false, timeout=1800.0))] - fn deploy( - &self, - port: Option, - clean: bool, - skip_build: bool, - monitor_after: bool, - timeout: f64, - ) -> bool { - send_op( - &deploy_url(), - &self.deploy_request(port, clean, skip_build, monitor_after), - timeout, - ) - .success - } - - #[pyo3(signature = (port=None, baud_rate=None, timeout=None))] - fn monitor(&self, port: Option, baud_rate: Option, timeout: Option) -> bool { - send_op( - &monitor_url(), - &self.monitor_request(port, baud_rate), - timeout.unwrap_or(1800.0), - ) - .success - } - - /// Same as `build()` but returns a dict with structured result fields: - /// `success`, `message`, `exit_code`, `stdout`, `stderr`. Callers that - /// need to branch on failure mode can inspect the dict instead of - /// swallowing a bare bool. See FastLED/fbuild#18. - #[pyo3(signature = (clean=false, verbose=false, timeout=1800.0))] - fn build_result<'py>( - &self, - py: Python<'py>, - clean: bool, - verbose: bool, - timeout: f64, - ) -> PyResult> { - let outcome = send_op(&build_url(), &self.build_request(clean, verbose), timeout); - outcome_to_pydict(py, &outcome) - } - - /// Structured-result counterpart to `deploy()`. See `build_result()`. - #[pyo3(signature = (port=None, clean=false, skip_build=false, monitor_after=false, timeout=1800.0))] - fn deploy_result<'py>( - &self, - py: Python<'py>, - port: Option, - clean: bool, - skip_build: bool, - monitor_after: bool, - timeout: f64, - ) -> PyResult> { - let outcome = send_op( - &deploy_url(), - &self.deploy_request(port, clean, skip_build, monitor_after), - timeout, - ); - outcome_to_pydict(py, &outcome) - } - - /// Structured-result counterpart to `monitor()`. See `build_result()`. - #[pyo3(signature = (port=None, baud_rate=None, timeout=None))] - fn monitor_result<'py>( - &self, - py: Python<'py>, - port: Option, - baud_rate: Option, - timeout: Option, - ) -> PyResult> { - let outcome = send_op( - &monitor_url(), - &self.monitor_request(port, baud_rate), - timeout.unwrap_or(1800.0), - ); - outcome_to_pydict(py, &outcome) - } -} - -impl DaemonConnection { - fn build_request(&self, clean: bool, verbose: bool) -> OpRequest { - OpRequest { - project_dir: self.project_dir.clone(), - environment: Some(self.environment.clone()), - clean_build: clean, - verbose, - port: None, - monitor_after: false, - skip_build: false, - baud_rate: None, - } - } - - fn deploy_request( - &self, - port: Option, - clean: bool, - skip_build: bool, - monitor_after: bool, - ) -> OpRequest { - OpRequest { - project_dir: self.project_dir.clone(), - environment: Some(self.environment.clone()), - clean_build: clean, - verbose: false, - port, - monitor_after, - skip_build, - baud_rate: None, - } - } - - fn monitor_request(&self, port: Option, baud_rate: Option) -> OpRequest { - OpRequest { - project_dir: self.project_dir.clone(), - environment: Some(self.environment.clone()), - clean_build: false, - verbose: false, - port, - monitor_after: false, - skip_build: false, - baud_rate, - } - } -} - -fn build_url() -> String { - format!("{}/api/build", fbuild_paths::get_daemon_url()) -} - -fn deploy_url() -> String { - format!("{}/api/deploy", fbuild_paths::get_daemon_url()) -} - -fn monitor_url() -> String { - format!("{}/api/monitor", fbuild_paths::get_daemon_url()) -} - -/// Structured result of a daemon operation (build/deploy/monitor). -/// -/// Used internally by `send_op` and exposed to Python callers via -/// `DaemonConnection::{build,deploy,monitor}_result`. Lets callers branch -/// on specific failure modes (transport error vs. build error vs. no -/// response) instead of inspecting a bare bool. See FastLED/fbuild#18. -#[derive(Debug, Clone, Default)] -struct OperationOutcome { - success: bool, - message: Option, - exit_code: Option, - stdout: Option, - stderr: Option, -} - -fn outcome_to_pydict<'py>( - py: Python<'py>, - outcome: &OperationOutcome, -) -> PyResult> { - let dict = pyo3::types::PyDict::new_bound(py); - dict.set_item("success", outcome.success)?; - dict.set_item("message", outcome.message.clone())?; - dict.set_item("exit_code", outcome.exit_code)?; - dict.set_item("stdout", outcome.stdout.clone())?; - dict.set_item("stderr", outcome.stderr.clone())?; - Ok(dict) -} - -fn parse_outcome(body: &serde_json::Value) -> OperationOutcome { - OperationOutcome { - success: body - .get("success") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - message: body - .get("message") - .and_then(|v| v.as_str()) - .map(str::to_string), - exit_code: body - .get("exit_code") - .and_then(|v| v.as_i64()) - .and_then(|n| { - if n >= i32::MIN as i64 && n <= i32::MAX as i64 { - Some(n as i32) - } else { - None - } - }), - stdout: body - .get("stdout") - .and_then(|v| v.as_str()) - .map(str::to_string), - stderr: body - .get("stderr") - .and_then(|v| v.as_str()) - .map(str::to_string), - } -} - -fn send_op(url: &str, req: &OpRequest, timeout: f64) -> OperationOutcome { - let client = reqwest::blocking::Client::new(); - match client - .post(url) - .json(req) - .timeout(std::time::Duration::from_secs_f64(timeout)) - .send() - { - Ok(resp) => match resp.json::() { - Ok(body) => { - let outcome = parse_outcome(&body); - if !outcome.success { - if let Some(ref msg) = outcome.message { - eprintln!("[fbuild] operation failed: {}", msg); - } - if let Some(ref stderr) = outcome.stderr { - if !stderr.is_empty() { - eprintln!("[fbuild] stderr:\n{}", stderr); - } - } - } - outcome - } - Err(e) => { - let msg = format!("failed to parse daemon response: {}", e); - eprintln!("[fbuild] {}", msg); - OperationOutcome { - success: false, - message: Some(msg), - ..Default::default() - } - } - }, - Err(e) => { - let msg = format!("request failed: {}", e); - eprintln!("[fbuild] {}", msg); - OperationOutcome { - success: false, - message: Some(msg), - ..Default::default() - } - } - } -} - -/// Native-async counterpart to `send_op`. Issues the same HTTP POST against -/// the daemon but yields on I/O instead of blocking a thread, so callers on -/// an asyncio event loop don't need FastLED's `_run_in_thread` shim. -/// -/// Returns the same `OperationOutcome` so the sync and async surfaces share -/// `parse_outcome` and `outcome_to_pydict`. See FastLED/fbuild#65. -async fn send_op_async(url: String, req: OpRequest, timeout: f64) -> OperationOutcome { - let client = reqwest::Client::new(); - let request = client - .post(&url) - .json(&req) - .timeout(std::time::Duration::from_secs_f64(timeout)); - - match request.send().await { - Ok(resp) => match resp.json::().await { - Ok(body) => { - let outcome = parse_outcome(&body); - if !outcome.success { - if let Some(ref msg) = outcome.message { - eprintln!("[fbuild] operation failed: {}", msg); - } - if let Some(ref stderr) = outcome.stderr { - if !stderr.is_empty() { - eprintln!("[fbuild] stderr:\n{}", stderr); - } - } - } - outcome - } - Err(e) => { - let msg = format!("failed to parse daemon response: {}", e); - eprintln!("[fbuild] {}", msg); - OperationOutcome { - success: false, - message: Some(msg), - ..Default::default() - } - } - }, - Err(e) => { - let msg = format!("request failed: {}", e); - eprintln!("[fbuild] {}", msg); - OperationOutcome { - success: false, - message: Some(msg), - ..Default::default() - } - } - } -} +mod async_daemon_connection; +mod async_serial_monitor; +mod daemon; +mod daemon_connection; +mod json_rpc; +mod messages; +mod outcome; +mod serial_monitor; + +use async_daemon_connection::AsyncDaemonConnection; +use async_serial_monitor::AsyncSerialMonitor; +use daemon::{AsyncDaemon, Daemon}; +use daemon_connection::DaemonConnection; +use serial_monitor::SerialMonitor; /// Factory function matching `from fbuild import connect_daemon`. #[pyfunction] @@ -1044,799 +65,6 @@ fn connect_daemon_async(project_dir: String, environment: String) -> AsyncDaemon /// against the native binary unreliable. const PYTHON_MODULE_VERSION: &str = env!("CARGO_PKG_VERSION"); -/// Python-visible AsyncSerialMonitor class. -/// -/// Native async equivalent of `SerialMonitor`. Exposes every long-running -/// serial operation (`read_lines`, `write`, `write_json_rpc`) plus the -/// context-manager pair (`__aenter__` / `__aexit__`) as `async def` so -/// callers can `await` them directly from an asyncio event loop without -/// FastLED's thread-pool shim (`_run_in_thread`). See FastLED/fbuild#65. -/// -/// ## Send + Sync refactor -/// -/// The sync `SerialMonitor` wraps its `WsSink`/`WsSource` halves in -/// `std::sync::Mutex`, which cannot be held across `.await`. For the async -/// surface we switch to `Arc>>`: -/// -/// * `tokio::sync::Mutex` — safe to hold across `.await` points, which -/// we do when calling `sink.send(...).await` / `source.next().await`. -/// * `Arc<...>` — lets us `clone` the handle into `async move { ... }` -/// blocks handed to `future_into_py`, so each future owns its own -/// reference into the shared connection state. -/// * `Option<_>` — the WS halves are absent before `__aenter__` and -/// after `__aexit__`, and the outer `Arc>>` stays -/// alive across both transitions. -/// * Read/write halves live in **separate** mutexes so a pending -/// `read_lines` does not serialize an unrelated `write` (each direction -/// of the WebSocket has independent framing state). -/// -/// The sync `SerialMonitor` is untouched — this is a purely additive -/// surface. -/// -/// ```python -/// import asyncio -/// from fbuild._native import AsyncSerialMonitor -/// -/// async def main(): -/// async with AsyncSerialMonitor(port="COM13", baud_rate=115200) as mon: -/// lines = await mon.read_lines(timeout_secs=5.0) -/// await mon.write("hello\n") -/// ok = await mon.reset_device(board="esp32s3") -/// -/// asyncio.run(main()) -/// ``` -#[pyclass] -struct AsyncSerialMonitor { - port: String, - baud_rate: u32, - auto_reconnect: bool, - verbose: bool, - client_id: String, - ws_write: Arc>>, - ws_read: Arc>>, -} - -#[pymethods] -impl AsyncSerialMonitor { - #[new] - #[pyo3(signature = (port, baud_rate=115200, auto_reconnect=true, verbose=false))] - fn new(port: String, baud_rate: u32, auto_reconnect: bool, verbose: bool) -> Self { - Self { - port, - baud_rate, - auto_reconnect, - verbose, - client_id: uuid::Uuid::new_v4().to_string(), - ws_write: Arc::new(tokio::sync::Mutex::new(None)), - ws_read: Arc::new(tokio::sync::Mutex::new(None)), - } - } - - /// Async context-manager entry. Connects to the daemon's - /// `/ws/serial-monitor` endpoint, sends the `attach` handshake, and - /// stores the split sink/source halves for subsequent `read_lines` / - /// `write` calls. Mirrors the sync `SerialMonitor::__enter__` contract. - fn __aenter__<'py>(slf: PyRef<'py, Self>, py: Python<'py>) -> PyResult> { - let port = slf.port.clone(); - let baud_rate = slf.baud_rate; - let client_id = slf.client_id.clone(); - let verbose = slf.verbose; - let ws_write_slot = slf.ws_write.clone(); - let ws_read_slot = slf.ws_read.clone(); - let slf_obj: PyObject = slf.into_py(py); - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let daemon_port = fbuild_paths::get_daemon_port(); - let ws_url = format!("ws://127.0.0.1:{}/ws/serial-monitor", daemon_port); - - let (ws_stream, _) = tokio_tungstenite::connect_async(&ws_url) - .await - .map_err(|e| { - pyo3::exceptions::PyConnectionError::new_err(format!( - "failed to connect to daemon WebSocket at {}: {}", - ws_url, e - )) - })?; - - let (mut write, mut read) = ws_stream.split(); - - let attach = ClientMessage::Attach { - client_id: client_id.clone(), - port: port.clone(), - baud_rate, - open_if_needed: true, - pre_acquire_writer: true, - }; - let attach_json = serde_json::to_string(&attach).unwrap(); - - write - .send(tungstenite::Message::Text(attach_json)) - .await - .map_err(|e| { - pyo3::exceptions::PyConnectionError::new_err(format!( - "failed to send attach: {}", - e - )) - })?; - - let msg = read - .next() - .await - .ok_or_else(|| { - pyo3::exceptions::PyConnectionError::new_err("WebSocket closed before attach") - })? - .map_err(|e| { - pyo3::exceptions::PyConnectionError::new_err(format!("WebSocket error: {}", e)) - })?; - - if let tungstenite::Message::Text(text) = msg { - match serde_json::from_str::(&text) { - Ok(ServerMessage::Attached { success, .. }) if success => { - if verbose { - eprintln!("attached to {} at {} baud", port, baud_rate); - } - } - Ok(ServerMessage::Error { message }) => { - return Err(pyo3::exceptions::PyRuntimeError::new_err(format!( - "attach failed: {}", - message - ))); - } - _ => { - return Err(pyo3::exceptions::PyRuntimeError::new_err( - "unexpected response to attach", - )); - } - } - } - - *ws_write_slot.lock().await = Some(write); - *ws_read_slot.lock().await = Some(read); - - Ok(slf_obj) - }) - } - - /// Async context-manager exit. Sends a `detach` + `Close` frame and - /// clears the stored sink/source halves. Mirrors sync `__exit__`. - #[pyo3(signature = (_exc_type=None, _exc_val=None, _exc_tb=None))] - fn __aexit__<'py>( - &self, - py: Python<'py>, - _exc_type: Option, - _exc_val: Option, - _exc_tb: Option, - ) -> PyResult> { - let ws_write_slot = self.ws_write.clone(); - let ws_read_slot = self.ws_read.clone(); - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - if let Some(mut write) = ws_write_slot.lock().await.take() { - let detach = serde_json::to_string(&ClientMessage::Detach).unwrap(); - let _ = write.send(tungstenite::Message::Text(detach)).await; - let _ = write.send(tungstenite::Message::Close(None)).await; - } - let _ = ws_read_slot.lock().await.take(); - Ok(false) - }) - } - - /// Async counterpart to `SerialMonitor::read_lines`. Pulls batches of - /// lines from the daemon until at least one batch arrives or the - /// timeout elapses. Handles `Preempted` / `Reconnected` transparently - /// when `auto_reconnect=true` just like the sync path. - #[pyo3(signature = (timeout_secs=30.0))] - fn read_lines<'py>(&self, py: Python<'py>, timeout_secs: f64) -> PyResult> { - let ws_read_slot = self.ws_read.clone(); - let auto_reconnect = self.auto_reconnect; - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - Ok(read_lines_async(ws_read_slot, auto_reconnect, timeout_secs).await) - }) - } - - /// Async counterpart to `SerialMonitor::write`. Returns `true` on - /// successful delivery (daemon acknowledged with `write_ack`), - /// `false` otherwise. Mirrors the sync contract except the return - /// type is a bool rather than `bytes_written` to match the async - /// signature spec in FastLED/fbuild#65. - fn write<'py>(&self, py: Python<'py>, data: &str) -> PyResult> { - let ws_write_slot = self.ws_write.clone(); - let ws_read_slot = self.ws_read.clone(); - let encoded = base64::engine::general_purpose::STANDARD.encode(data.as_bytes()); - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - Ok(write_async(ws_write_slot, ws_read_slot, encoded).await) - }) - } - - /// Async counterpart to `SerialMonitor::write_json_rpc`. Serializes - /// `request` to JSON, sends it with a trailing newline, then polls - /// `read_lines` until a `REMOTE:` response arrives or the full - /// `timeout_secs` elapses. Honors the PR #57 guarantee that an empty - /// batch does not short-circuit the overall deadline. - /// - /// Returns the JSON-decoded response on success, raises - /// `TimeoutError` on timeout. - #[pyo3(signature = (request, timeout_secs=5.0))] - fn write_json_rpc<'py>( - &self, - py: Python<'py>, - request: &Bound<'_, PyAny>, - timeout_secs: f64, - ) -> PyResult> { - // Serialize the request on the calling thread (needs the GIL) - // before we enter the async block, mirroring the send_op_async - // pattern of moving only owned primitives across the .await - // boundary. - let json_str: String = py - .import_bound("json")? - .call_method1("dumps", (request,))? - .extract()?; - let data = format!("{}\n", json_str); - let encoded = base64::engine::general_purpose::STANDARD.encode(data.as_bytes()); - - let ws_write_slot = self.ws_write.clone(); - let ws_read_slot = self.ws_read.clone(); - let auto_reconnect = self.auto_reconnect; - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - // Fire-and-acknowledge the write first; we don't abort on - // a write failure because the sync surface also proceeds - // to poll for a REMOTE: response (which is how the Python - // tests exercise this path). - let _ = write_async(ws_write_slot, ws_read_slot.clone(), encoded).await; - - let json_part = - wait_for_remote_json_rpc_response_async(timeout_secs, ws_read_slot, auto_reconnect) - .await; - - match json_part { - Some(payload) => Python::with_gil(|py| { - let json_module = py.import_bound("json")?; - let parsed = json_module.call_method1("loads", (payload.trim(),))?; - Ok(parsed.unbind()) - }), - None => Err(pyo3::exceptions::PyTimeoutError::new_err(format!( - "no REMOTE: response within {} seconds", - timeout_secs - ))), - } - }) - } - - /// Asynchronously reset the device via the daemon's `POST /api/reset` - /// endpoint. Returns `True` if the daemon reported success. - #[pyo3(signature = (board=None))] - fn reset_device<'py>( - &self, - py: Python<'py>, - board: Option, - ) -> PyResult> { - let url = format!("{}/api/reset", fbuild_paths::get_daemon_url()); - let port = self.port.clone(); - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - #[derive(Serialize)] - struct ResetPayload { - port: String, - #[serde(skip_serializing_if = "Option::is_none")] - board: Option, - } - - let payload = ResetPayload { port, board }; - - let resp = reqwest::Client::new() - .post(&url) - .json(&payload) - .timeout(std::time::Duration::from_secs(10)) - .send() - .await - .map_err(|e| { - pyo3::exceptions::PyConnectionError::new_err(format!( - "failed to send reset request to daemon: {}", - e - )) - })?; - - let body: serde_json::Value = resp.json().await.map_err(|e| { - pyo3::exceptions::PyRuntimeError::new_err(format!( - "failed to parse reset response: {}", - e - )) - })?; - - Ok(body - .get("success") - .and_then(|v| v.as_bool()) - .unwrap_or(false)) - }) - } -} - -/// Shared async read-batch loop used by `AsyncSerialMonitor::read_lines` -/// and `write_json_rpc`. Acquires the source mutex only for the duration -/// of each `.next()` call so that concurrent `write` / `__aexit__` -/// futures can still progress between iterations. -async fn read_lines_async( - ws_read_slot: Arc>>, - auto_reconnect: bool, - timeout_secs: f64, -) -> Vec { - let mut lines = Vec::new(); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout_secs); - - loop { - let now = std::time::Instant::now(); - if now >= deadline { - break; - } - let remaining = deadline - now; - - let mut guard = ws_read_slot.lock().await; - let Some(source) = guard.as_mut() else { - // Session not entered (or already exited). Nothing to read. - break; - }; - - let result = tokio::time::timeout(remaining, source.next()).await; - // Drop the guard before continuing the loop so another task - // can take the mutex (e.g. __aexit__). - drop(guard); - - match result { - Ok(Some(Ok(tungstenite::Message::Text(text)))) => { - match serde_json::from_str::(&text) { - Ok(ServerMessage::Data { - lines: data_lines, .. - }) => { - lines.extend(data_lines); - if !lines.is_empty() { - break; - } - } - Ok(ServerMessage::Preempted { .. }) => { - if auto_reconnect { - continue; - } - break; - } - Ok(ServerMessage::Reconnected { .. }) => continue, - _ => continue, - } - } - Ok(Some(Ok(tungstenite::Message::Close(_)))) | Ok(None) => break, - Err(_) => break, // timeout - _ => continue, - } - } - - lines -} - -/// Shared async write path used by `AsyncSerialMonitor::write` and -/// `write_json_rpc`. Sends the already-base64-encoded payload, then -/// awaits a `write_ack` (with a 5-second bound matching the sync path). -/// Returns `true` if the ack reported any bytes written. -async fn write_async( - ws_write_slot: Arc>>, - ws_read_slot: Arc>>, - encoded: String, -) -> bool { - let msg = serde_json::to_string(&ClientMessage::Write { data: encoded }).unwrap(); - - { - let mut guard = ws_write_slot.lock().await; - let Some(sink) = guard.as_mut() else { - return false; - }; - if sink.send(tungstenite::Message::Text(msg)).await.is_err() { - return false; - } - } - - // Wait for write_ack. Hold the read half's mutex only across this - // single `.next()` so concurrent `read_lines` futures can resume. - let mut guard = ws_read_slot.lock().await; - let Some(source) = guard.as_mut() else { - return false; - }; - let ack_timeout = std::time::Duration::from_secs(5); - match tokio::time::timeout(ack_timeout, source.next()).await { - Ok(Some(Ok(tungstenite::Message::Text(text)))) => { - matches!( - serde_json::from_str::(&text), - Ok(ServerMessage::WriteAck { bytes_written, .. }) if bytes_written > 0 - ) - } - _ => false, - } -} - -/// Async counterpart to `wait_for_remote_json_rpc_response`. Keeps -/// polling `read_lines_async` until the deadline expires, even if an -/// individual batch comes back empty — preserving the PR #57 fix. -async fn wait_for_remote_json_rpc_response_async( - timeout_secs: f64, - ws_read_slot: Arc>>, - auto_reconnect: bool, -) -> Option { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout_secs); - - while std::time::Instant::now() < deadline { - let remaining = (deadline - std::time::Instant::now()).as_secs_f64(); - if remaining <= 0.0 { - break; - } - let lines = read_lines_async(ws_read_slot.clone(), auto_reconnect, remaining).await; - if let Some(json_part) = extract_remote_json_rpc_response(&lines) { - return Some(json_part); - } - } - - None -} - -/// Python-visible AsyncDaemon class. -/// -/// Native async counterpart to `Daemon`. Follows the same additive -/// pattern as `AsyncSerialMonitor` (Issue #65): the sync `Daemon` class -/// stays unchanged, and this one exposes async methods so callers under -/// an asyncio event loop can `await` them directly. -/// -/// ```python -/// import asyncio -/// from fbuild._native import AsyncDaemon -/// -/// async def main(): -/// info = await AsyncDaemon.status() -/// -/// asyncio.run(main()) -/// ``` -#[pyclass] -struct AsyncDaemon; - -#[pymethods] -impl AsyncDaemon { - /// Asynchronously fetch `/api/daemon/info` from the daemon. Returns - /// a JSON-deserialized Python object on success, or raises a - /// ConnectionError/RuntimeError. - #[staticmethod] - fn status(py: Python<'_>) -> PyResult> { - let url = format!("{}/api/daemon/info", fbuild_paths::get_daemon_url()); - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let resp = reqwest::Client::new() - .get(&url) - .timeout(std::time::Duration::from_secs(10)) - .send() - .await - .map_err(|e| { - pyo3::exceptions::PyConnectionError::new_err(format!( - "failed to connect to daemon: {}", - e - )) - })?; - - let text = resp.text().await.map_err(|e| { - pyo3::exceptions::PyRuntimeError::new_err(format!( - "failed to read daemon response: {}", - e - )) - })?; - - Python::with_gil(|py| { - let json_module = py.import_bound("json")?; - let parsed = json_module.call_method1("loads", (text,))?; - Ok(parsed.unbind()) - }) - }) - } - - /// Asynchronously ensure the daemon is running. Mirrors the sync - /// `Daemon.ensure_running` contract: returns `True` if the daemon - /// responds to `/health`, spawning a new `fbuild-daemon` process if - /// needed and polling until the health endpoint succeeds. - /// - /// The spawn itself is synchronous (`std::process::Command::spawn`) - /// because `tokio::process::Command` adds no value for a detached - /// child — the child does not need an async stdio pipe. The key win - /// for async callers is that the health poll loop uses - /// `tokio::time::sleep` and async reqwest instead of blocking the - /// event loop thread. - #[staticmethod] - fn ensure_running(py: Python<'_>) -> PyResult> { - let url = format!("{}/health", fbuild_paths::get_daemon_url()); - let dev_mode = fbuild_paths::is_dev_mode(); - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let client = reqwest::Client::new(); - - // Fast path: daemon is already up. - if let Ok(resp) = client - .get(&url) - .timeout(std::time::Duration::from_secs(5)) - .send() - .await - { - if resp.status().is_success() { - return Ok(true); - } - } - - // INTENTIONALLY DETACHED (FastLED/fbuild#32): see the - // matching comment in `Daemon::ensure_running` above. - // allow-direct-spawn: daemon must outlive the Python interpreter. - let mut cmd = std::process::Command::new("fbuild-daemon"); - if dev_mode { - cmd.arg("--dev"); - } - cmd.stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()); - - if cmd.spawn().is_err() { - return Ok(false); - } - - for _ in 0..100 { - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - if let Ok(resp) = client - .get(&url) - .timeout(std::time::Duration::from_secs(5)) - .send() - .await - { - if resp.status().is_success() { - return Ok(true); - } - } - } - Ok(false) - }) - } - - /// Asynchronously shut down the daemon via `POST /api/daemon/shutdown`. - /// Returns `True` if the daemon acknowledged with a 2xx response. - #[staticmethod] - fn stop(py: Python<'_>) -> PyResult> { - let url = format!("{}/api/daemon/shutdown", fbuild_paths::get_daemon_url()); - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let ok = reqwest::Client::new() - .post(&url) - .timeout(std::time::Duration::from_secs(10)) - .send() - .await - .map(|r| r.status().is_success()) - .unwrap_or(false); - Ok(ok) - }) - } -} - -/// Python-visible AsyncDaemonConnection class. -/// -/// Native async counterpart to `DaemonConnection`. Exposes `build`, -/// `deploy`, and `monitor` (and their `_result` variants) as async methods -/// that call the daemon over `reqwest::Client` (non-blocking) instead of -/// the blocking client used by the sync sibling. This is the method set -/// FastLED/fbuild#65 explicitly targets under "Daemon/DaemonConnection: -/// send_op and any other HTTP call". -/// -/// ```python -/// import asyncio -/// from fbuild._native import AsyncDaemonConnection -/// -/// async def main(): -/// conn = AsyncDaemonConnection(project_dir="tests/platform/uno", environment="uno") -/// ok = await conn.build() -/// result = await conn.build_result() -/// -/// asyncio.run(main()) -/// ``` -#[pyclass] -struct AsyncDaemonConnection { - project_dir: String, - environment: String, -} - -#[pymethods] -impl AsyncDaemonConnection { - #[new] - fn new(project_dir: String, environment: String) -> Self { - Self { - project_dir, - environment, - } - } - - /// Async context manager entry. Returns self so callers can - /// `async with AsyncDaemonConnection(...) as conn:`. - fn __aenter__<'py>(slf: PyRef<'py, Self>, py: Python<'py>) -> PyResult> { - let project_dir = slf.project_dir.clone(); - let environment = slf.environment.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - Python::with_gil(|py| { - let obj = Py::new( - py, - AsyncDaemonConnection { - project_dir, - environment, - }, - )?; - Ok(obj.to_object(py)) - }) - }) - } - - #[pyo3(signature = (_exc_type=None, _exc_val=None, _exc_tb=None))] - fn __aexit__<'py>( - &self, - py: Python<'py>, - _exc_type: Option, - _exc_val: Option, - _exc_tb: Option, - ) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(false) }) - } - - /// Async counterpart to `DaemonConnection::build`. Awaits the daemon's - /// `POST /api/build` response and returns the `success` bool. - #[pyo3(signature = (clean=false, verbose=false, timeout=1800.0))] - fn build<'py>( - &self, - py: Python<'py>, - clean: bool, - verbose: bool, - timeout: f64, - ) -> PyResult> { - let url = build_url(); - let req = self.build_request(clean, verbose); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - Ok(send_op_async(url, req, timeout).await.success) - }) - } - - /// Async counterpart to `DaemonConnection::deploy`. - #[pyo3(signature = (port=None, clean=false, skip_build=false, monitor_after=false, timeout=1800.0))] - fn deploy<'py>( - &self, - py: Python<'py>, - port: Option, - clean: bool, - skip_build: bool, - monitor_after: bool, - timeout: f64, - ) -> PyResult> { - let url = deploy_url(); - let req = self.deploy_request(port, clean, skip_build, monitor_after); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - Ok(send_op_async(url, req, timeout).await.success) - }) - } - - /// Async counterpart to `DaemonConnection::monitor`. - #[pyo3(signature = (port=None, baud_rate=None, timeout=None))] - fn monitor<'py>( - &self, - py: Python<'py>, - port: Option, - baud_rate: Option, - timeout: Option, - ) -> PyResult> { - let url = monitor_url(); - let req = self.monitor_request(port, baud_rate); - let t = timeout.unwrap_or(1800.0); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - Ok(send_op_async(url, req, t).await.success) - }) - } - - /// Async counterpart to `DaemonConnection::build_result`. Returns the - /// full structured outcome dict (`success`, `message`, `exit_code`, - /// `stdout`, `stderr`) — matches the sync surface exactly. - #[pyo3(signature = (clean=false, verbose=false, timeout=1800.0))] - fn build_result<'py>( - &self, - py: Python<'py>, - clean: bool, - verbose: bool, - timeout: f64, - ) -> PyResult> { - let url = build_url(); - let req = self.build_request(clean, verbose); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let outcome = send_op_async(url, req, timeout).await; - Python::with_gil(|py| Ok(outcome_to_pydict(py, &outcome)?.unbind())) - }) - } - - /// Async counterpart to `DaemonConnection::deploy_result`. - #[pyo3(signature = (port=None, clean=false, skip_build=false, monitor_after=false, timeout=1800.0))] - fn deploy_result<'py>( - &self, - py: Python<'py>, - port: Option, - clean: bool, - skip_build: bool, - monitor_after: bool, - timeout: f64, - ) -> PyResult> { - let url = deploy_url(); - let req = self.deploy_request(port, clean, skip_build, monitor_after); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let outcome = send_op_async(url, req, timeout).await; - Python::with_gil(|py| Ok(outcome_to_pydict(py, &outcome)?.unbind())) - }) - } - - /// Async counterpart to `DaemonConnection::monitor_result`. - #[pyo3(signature = (port=None, baud_rate=None, timeout=None))] - fn monitor_result<'py>( - &self, - py: Python<'py>, - port: Option, - baud_rate: Option, - timeout: Option, - ) -> PyResult> { - let url = monitor_url(); - let req = self.monitor_request(port, baud_rate); - let t = timeout.unwrap_or(1800.0); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let outcome = send_op_async(url, req, t).await; - Python::with_gil(|py| Ok(outcome_to_pydict(py, &outcome)?.unbind())) - }) - } -} - -impl AsyncDaemonConnection { - fn build_request(&self, clean: bool, verbose: bool) -> OpRequest { - OpRequest { - project_dir: self.project_dir.clone(), - environment: Some(self.environment.clone()), - clean_build: clean, - verbose, - port: None, - monitor_after: false, - skip_build: false, - baud_rate: None, - } - } - - fn deploy_request( - &self, - port: Option, - clean: bool, - skip_build: bool, - monitor_after: bool, - ) -> OpRequest { - OpRequest { - project_dir: self.project_dir.clone(), - environment: Some(self.environment.clone()), - clean_build: clean, - verbose: false, - port, - monitor_after, - skip_build, - baud_rate: None, - } - } - - fn monitor_request(&self, port: Option, baud_rate: Option) -> OpRequest { - OpRequest { - project_dir: self.project_dir.clone(), - environment: Some(self.environment.clone()), - clean_build: false, - verbose: false, - port, - monitor_after: false, - skip_build: false, - baud_rate, - } - } -} - /// The fbuild Python module (imported as fbuild._native). #[pymodule] fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { @@ -1854,10 +82,9 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { #[cfg(test)] mod tests { - use super::{ - extract_remote_json_rpc_response, parse_outcome, send_op_async, - wait_for_remote_json_rpc_response, OpRequest, PYTHON_MODULE_VERSION, - }; + use crate::json_rpc::{extract_remote_json_rpc_response, wait_for_remote_json_rpc_response}; + use crate::outcome::{parse_outcome, send_op_async, OpRequest}; + use crate::PYTHON_MODULE_VERSION; use tokio::io::{AsyncReadExt, AsyncWriteExt}; /// `parse_outcome` must faithfully extract every field the daemon's diff --git a/crates/fbuild-python/src/messages.rs b/crates/fbuild-python/src/messages.rs new file mode 100644 index 00000000..a404b4a2 --- /dev/null +++ b/crates/fbuild-python/src/messages.rs @@ -0,0 +1,67 @@ +//! Shared WebSocket message types and type aliases used by the +//! synchronous and asynchronous SerialMonitor implementations. + +use serde::{Deserialize, Serialize}; +use tokio_tungstenite::tungstenite; + +pub(crate) type WsStream = + tokio_tungstenite::WebSocketStream>; +pub(crate) type WsSink = futures::stream::SplitSink; +pub(crate) type WsSource = futures::stream::SplitStream; + +/// Messages we receive from the daemon (subset we care about). +#[derive(Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub(crate) enum ServerMessage { + Attached { + success: bool, + #[allow(dead_code)] + message: String, + #[allow(dead_code)] + writer_pre_acquired: bool, + }, + Data { + lines: Vec, + #[allow(dead_code)] + current_index: u64, + }, + WriteAck { + #[allow(dead_code)] + success: bool, + bytes_written: usize, + #[allow(dead_code)] + message: Option, + }, + Preempted { + #[allow(dead_code)] + reason: String, + #[allow(dead_code)] + preempted_by: String, + }, + Reconnected { + #[allow(dead_code)] + message: String, + }, + Error { + message: String, + }, + #[serde(other)] + Other, +} + +/// Client message to send to the daemon. +#[derive(Debug, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub(crate) enum ClientMessage { + Attach { + client_id: String, + port: String, + baud_rate: u32, + open_if_needed: bool, + pre_acquire_writer: bool, + }, + Write { + data: String, + }, + Detach, +} diff --git a/crates/fbuild-python/src/outcome.rs b/crates/fbuild-python/src/outcome.rs new file mode 100644 index 00000000..071ad188 --- /dev/null +++ b/crates/fbuild-python/src/outcome.rs @@ -0,0 +1,191 @@ +//! Shared `OperationOutcome` / `OpRequest` types plus the sync and async +//! HTTP transports used by `DaemonConnection` and `AsyncDaemonConnection`. + +use pyo3::prelude::*; +use serde::Serialize; + +#[derive(Clone, Serialize)] +pub(crate) struct OpRequest { + pub(crate) project_dir: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) environment: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub(crate) clean_build: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub(crate) verbose: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) port: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub(crate) monitor_after: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub(crate) skip_build: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) baud_rate: Option, +} + +pub(crate) fn build_url() -> String { + format!("{}/api/build", fbuild_paths::get_daemon_url()) +} + +pub(crate) fn deploy_url() -> String { + format!("{}/api/deploy", fbuild_paths::get_daemon_url()) +} + +pub(crate) fn monitor_url() -> String { + format!("{}/api/monitor", fbuild_paths::get_daemon_url()) +} + +/// Structured result of a daemon operation (build/deploy/monitor). +/// +/// Used internally by `send_op` and exposed to Python callers via +/// `DaemonConnection::{build,deploy,monitor}_result`. Lets callers branch +/// on specific failure modes (transport error vs. build error vs. no +/// response) instead of inspecting a bare bool. See FastLED/fbuild#18. +#[derive(Debug, Clone, Default)] +pub(crate) struct OperationOutcome { + pub(crate) success: bool, + pub(crate) message: Option, + pub(crate) exit_code: Option, + pub(crate) stdout: Option, + pub(crate) stderr: Option, +} + +pub(crate) fn outcome_to_pydict<'py>( + py: Python<'py>, + outcome: &OperationOutcome, +) -> PyResult> { + let dict = pyo3::types::PyDict::new_bound(py); + dict.set_item("success", outcome.success)?; + dict.set_item("message", outcome.message.clone())?; + dict.set_item("exit_code", outcome.exit_code)?; + dict.set_item("stdout", outcome.stdout.clone())?; + dict.set_item("stderr", outcome.stderr.clone())?; + Ok(dict) +} + +pub(crate) fn parse_outcome(body: &serde_json::Value) -> OperationOutcome { + OperationOutcome { + success: body + .get("success") + .and_then(|v| v.as_bool()) + .unwrap_or(false), + message: body + .get("message") + .and_then(|v| v.as_str()) + .map(str::to_string), + exit_code: body + .get("exit_code") + .and_then(|v| v.as_i64()) + .and_then(|n| { + if n >= i32::MIN as i64 && n <= i32::MAX as i64 { + Some(n as i32) + } else { + None + } + }), + stdout: body + .get("stdout") + .and_then(|v| v.as_str()) + .map(str::to_string), + stderr: body + .get("stderr") + .and_then(|v| v.as_str()) + .map(str::to_string), + } +} + +pub(crate) fn send_op(url: &str, req: &OpRequest, timeout: f64) -> OperationOutcome { + let client = reqwest::blocking::Client::new(); + match client + .post(url) + .json(req) + .timeout(std::time::Duration::from_secs_f64(timeout)) + .send() + { + Ok(resp) => match resp.json::() { + Ok(body) => { + let outcome = parse_outcome(&body); + if !outcome.success { + if let Some(ref msg) = outcome.message { + eprintln!("[fbuild] operation failed: {}", msg); + } + if let Some(ref stderr) = outcome.stderr { + if !stderr.is_empty() { + eprintln!("[fbuild] stderr:\n{}", stderr); + } + } + } + outcome + } + Err(e) => { + let msg = format!("failed to parse daemon response: {}", e); + eprintln!("[fbuild] {}", msg); + OperationOutcome { + success: false, + message: Some(msg), + ..Default::default() + } + } + }, + Err(e) => { + let msg = format!("request failed: {}", e); + eprintln!("[fbuild] {}", msg); + OperationOutcome { + success: false, + message: Some(msg), + ..Default::default() + } + } + } +} + +/// Native-async counterpart to `send_op`. Issues the same HTTP POST against +/// the daemon but yields on I/O instead of blocking a thread, so callers on +/// an asyncio event loop don't need FastLED's `_run_in_thread` shim. +/// +/// Returns the same `OperationOutcome` so the sync and async surfaces share +/// `parse_outcome` and `outcome_to_pydict`. See FastLED/fbuild#65. +pub(crate) async fn send_op_async(url: String, req: OpRequest, timeout: f64) -> OperationOutcome { + let client = reqwest::Client::new(); + let request = client + .post(&url) + .json(&req) + .timeout(std::time::Duration::from_secs_f64(timeout)); + + match request.send().await { + Ok(resp) => match resp.json::().await { + Ok(body) => { + let outcome = parse_outcome(&body); + if !outcome.success { + if let Some(ref msg) = outcome.message { + eprintln!("[fbuild] operation failed: {}", msg); + } + if let Some(ref stderr) = outcome.stderr { + if !stderr.is_empty() { + eprintln!("[fbuild] stderr:\n{}", stderr); + } + } + } + outcome + } + Err(e) => { + let msg = format!("failed to parse daemon response: {}", e); + eprintln!("[fbuild] {}", msg); + OperationOutcome { + success: false, + message: Some(msg), + ..Default::default() + } + } + }, + Err(e) => { + let msg = format!("request failed: {}", e); + eprintln!("[fbuild] {}", msg); + OperationOutcome { + success: false, + message: Some(msg), + ..Default::default() + } + } + } +} diff --git a/crates/fbuild-python/src/serial_monitor.rs b/crates/fbuild-python/src/serial_monitor.rs new file mode 100644 index 00000000..fcd09b12 --- /dev/null +++ b/crates/fbuild-python/src/serial_monitor.rs @@ -0,0 +1,498 @@ +//! Synchronous `SerialMonitor` PyO3 binding — the API FastLED depends on. + +use base64::Engine; +use futures::{SinkExt, StreamExt}; +use pyo3::prelude::*; +use serde::Serialize; +use std::sync::Mutex; +use tokio::runtime::Runtime; +use tokio_tungstenite::tungstenite; + +use crate::json_rpc::wait_for_remote_json_rpc_response; +use crate::messages::{ClientMessage, ServerMessage, WsSink, WsSource}; + +/// Python-visible SerialMonitor class. +/// +/// This is the critical binding that FastLED depends on. +/// It wraps the Rust SharedSerialManager via WebSocket, +/// matching the original Python API: +/// +/// ```python +/// with SerialMonitor(port="COM13", baud_rate=115200) as mon: +/// for line in mon.read_lines(timeout=30.0): +/// print(line) +/// mon.write("hello\n") +/// ``` +#[pyclass] +pub(crate) struct SerialMonitor { + port: String, + baud_rate: u32, + auto_reconnect: bool, + verbose: bool, + hooks: Vec, + runtime: Option, + ws_write: Option>, + ws_read: Option>, + client_id: String, + last_line: String, + #[allow(dead_code)] + preempted: bool, +} + +#[pymethods] +impl SerialMonitor { + #[new] + #[pyo3(signature = (port, baud_rate=115200, hooks=None, auto_reconnect=true, verbose=false))] + fn new( + port: String, + baud_rate: u32, + hooks: Option>, + auto_reconnect: bool, + verbose: bool, + ) -> Self { + Self { + port, + baud_rate, + auto_reconnect, + verbose, + hooks: hooks.unwrap_or_default(), + runtime: None, + ws_write: None, + ws_read: None, + client_id: uuid::Uuid::new_v4().to_string(), + last_line: String::new(), + preempted: false, + } + } + + fn __enter__(mut slf: PyRefMut<'_, Self>) -> PyResult> { + let rt = Runtime::new().map_err(|e| { + pyo3::exceptions::PyRuntimeError::new_err(format!("failed to create runtime: {}", e)) + })?; + + // Connect to daemon WebSocket + let daemon_port = fbuild_paths::get_daemon_port(); + let ws_url = format!("ws://127.0.0.1:{}/ws/serial-monitor", daemon_port); + + let (ws_stream, _) = rt + .block_on(tokio_tungstenite::connect_async(&ws_url)) + .map_err(|e| { + pyo3::exceptions::PyConnectionError::new_err(format!( + "failed to connect to daemon WebSocket at {}: {}", + ws_url, e + )) + })?; + + let (mut write, mut read) = ws_stream.split(); + + // Send attach message + let attach = ClientMessage::Attach { + client_id: slf.client_id.clone(), + port: slf.port.clone(), + baud_rate: slf.baud_rate, + open_if_needed: true, + pre_acquire_writer: true, + }; + let attach_json = serde_json::to_string(&attach).unwrap(); + + rt.block_on(write.send(tungstenite::Message::Text(attach_json))) + .map_err(|e| { + pyo3::exceptions::PyConnectionError::new_err(format!( + "failed to send attach: {}", + e + )) + })?; + + // Wait for attached response + let msg: tungstenite::Message = rt + .block_on(read.next()) + .ok_or_else(|| { + pyo3::exceptions::PyConnectionError::new_err("WebSocket closed before attach") + })? + .map_err(|e| { + pyo3::exceptions::PyConnectionError::new_err(format!("WebSocket error: {}", e)) + })?; + + if let tungstenite::Message::Text(text) = msg { + match serde_json::from_str::(&text) { + Ok(ServerMessage::Attached { success, .. }) if success => { + if slf.verbose { + eprintln!("attached to {} at {} baud", slf.port, slf.baud_rate); + } + } + Ok(ServerMessage::Error { message }) => { + return Err(pyo3::exceptions::PyRuntimeError::new_err(format!( + "attach failed: {}", + message + ))); + } + _ => { + return Err(pyo3::exceptions::PyRuntimeError::new_err( + "unexpected response to attach", + )); + } + } + } + + slf.ws_write = Some(Mutex::new(write)); + slf.ws_read = Some(Mutex::new(read)); + slf.runtime = Some(rt); + Ok(slf) + } + + #[pyo3(signature = (_exc_type=None, _exc_val=None, _exc_tb=None))] + fn __exit__( + &mut self, + _exc_type: Option<&Bound<'_, PyAny>>, + _exc_val: Option<&Bound<'_, PyAny>>, + _exc_tb: Option<&Bound<'_, PyAny>>, + ) -> bool { + if let (Some(ref rt), Some(ref ws_write)) = (&self.runtime, &self.ws_write) { + let detach = serde_json::to_string(&ClientMessage::Detach).unwrap(); + if let Ok(mut write) = ws_write.lock() { + let _ = rt.block_on(write.send(tungstenite::Message::Text(detach))); + let _ = rt.block_on(write.send(tungstenite::Message::Close(None))); + } + } + self.ws_write = None; + self.ws_read = None; + self.runtime = None; + false + } + + /// The last line received from the serial port. + #[getter] + fn last_line(&self) -> &str { + &self.last_line + } + + /// Iterate over serial output lines. + /// + /// Returns a list of lines received within the timeout period. + #[pyo3(signature = (timeout=30.0))] + fn read_lines(&mut self, py: Python<'_>, timeout: f64) -> Vec { + let (Some(ref rt), Some(ref ws_read)) = (&self.runtime, &self.ws_read) else { + return vec![]; + }; + + let mut lines = Vec::new(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout); + let auto_reconnect = self.auto_reconnect; + + py.allow_threads(|| { + while std::time::Instant::now() < deadline { + let remaining = deadline - std::time::Instant::now(); + let result = { + let mut read = ws_read.lock().unwrap(); + // tokio::time::timeout MUST be constructed inside the + // runtime context, otherwise it panics with "there is + // no reactor running" because the Sleep future needs + // Handle::current() to register with the timer driver. + rt.block_on(async { tokio::time::timeout(remaining, read.next()).await }) + }; + + match result { + Ok(Some(Ok(tungstenite::Message::Text(text)))) => { + match serde_json::from_str::(&text) { + Ok(ServerMessage::Data { + lines: data_lines, .. + }) => { + lines.extend(data_lines); + if !lines.is_empty() { + break; + } + } + Ok(ServerMessage::Preempted { .. }) => { + // Pause — deploy is happening + if auto_reconnect { + continue; + } + break; + } + Ok(ServerMessage::Reconnected { .. }) => { + // Resume after deploy + continue; + } + _ => continue, + } + } + Ok(Some(Ok(tungstenite::Message::Close(_)))) | Ok(None) => break, + Err(_) => break, // timeout + _ => continue, + } + } + }); + + // Update last_line and dispatch hooks + if let Some(last) = lines.last() { + self.last_line = last.clone(); + } + + // Dispatch hooks for each line + if !self.hooks.is_empty() && !lines.is_empty() { + Python::with_gil(|py| { + for line in &lines { + for hook in &self.hooks { + let _ = hook.call1(py, (line,)); + } + } + }); + } + + lines + } + + /// Write data to the serial port. + fn write(&self, data: &str) -> usize { + let (Some(ref rt), Some(ref ws_write), Some(ref ws_read)) = + (&self.runtime, &self.ws_write, &self.ws_read) + else { + return 0; + }; + + let encoded = base64::engine::general_purpose::STANDARD.encode(data.as_bytes()); + let msg = serde_json::to_string(&ClientMessage::Write { data: encoded }).unwrap(); + + { + let mut write = ws_write.lock().unwrap(); + if rt + .block_on(write.send(tungstenite::Message::Text(msg))) + .is_err() + { + return 0; + } + } + + // Wait for write_ack + let mut read = ws_read.lock().unwrap(); + let timeout = std::time::Duration::from_secs(5); + // tokio::time::timeout must be created inside the runtime context. + match rt.block_on(async { tokio::time::timeout(timeout, read.next()).await }) { + Ok(Some(Ok(tungstenite::Message::Text(text)))) => { + if let Ok(ServerMessage::WriteAck { bytes_written, .. }) = + serde_json::from_str(&text) + { + return bytes_written; + } + 0 + } + _ => 0, + } + } + + /// Run monitor until condition returns True or timeout expires. + /// + /// Calls `condition(line)` for each received line. Returns True if + /// the condition was met, False on timeout. + #[pyo3(signature = (condition, timeout=30.0))] + fn run_until(&mut self, py: Python<'_>, condition: PyObject, timeout: f64) -> PyResult { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout); + + while std::time::Instant::now() < deadline { + let remaining = (deadline - std::time::Instant::now()).as_secs_f64(); + if remaining <= 0.0 { + break; + } + let lines = self.read_lines(py, remaining.min(1.0)); + for line in &lines { + let result: bool = condition.call1(py, (line,))?.extract(py)?; + if result { + return Ok(true); + } + } + if lines.is_empty() { + // Timeout on read_lines, check overall deadline + continue; + } + } + + Ok(false) + } + + /// Send a JSON-RPC request and wait for matching response. + #[pyo3(signature = (request, timeout=5.0))] + fn write_json_rpc( + &self, + py: Python<'_>, + request: &Bound<'_, PyAny>, + timeout: f64, + ) -> PyResult { + let json_str: String = py + .import_bound("json")? + .call_method1("dumps", (request,))? + .extract()?; + + let data = format!("{}\n", json_str); + self.write(&data); + + if let Some(json_part) = wait_for_remote_json_rpc_response(timeout, |remaining| { + // read_lines takes &mut self but we only have &self here — + // use the raw WS read directly. + self.read_lines_inner(remaining.min(1.0)) + }) { + let json_module = py.import_bound("json")?; + let result = json_module.call_method1("loads", (json_part.trim(),))?; + return Ok(result.to_object(py)); + } + + Err(pyo3::exceptions::PyTimeoutError::new_err(format!( + "no REMOTE: response within {} seconds", + timeout + ))) + } + + /// Reset the device via the daemon's DTR/RTS reset endpoint. + /// + /// Sends POST /api/reset to the daemon, which preempts any active + /// serial monitor session, toggles DTR/RTS to reset the device, + /// then clears preemption so monitors can reconnect. + /// + /// Works whether or not `__enter__` has been called — the reset goes + /// through the daemon's HTTP API, not the WebSocket session. + /// + /// Args: + /// board: Board identifier (e.g. "esp32s3", "teensy40"). + /// Determines the platform-specific reset sequence. + /// If None, a generic DTR toggle is used. + /// wait_for_output: If True, block until serial output is detected + /// after the reset (device has rebooted and is producing data). + /// If False (default), return immediately after reset. + /// timeout: Maximum seconds to wait for output (only used when + /// wait_for_output is True). Default: 5.0. + /// + /// Returns: + /// True if reset succeeded (and output detected, if wait_for_output). + /// False on failure or timeout. + #[pyo3(signature = (board=None, wait_for_output=false, timeout=5.0))] + fn reset_device( + &self, + board: Option, + wait_for_output: bool, + timeout: f64, + ) -> PyResult { + let url = format!("{}/api/reset", fbuild_paths::get_daemon_url()); + + #[derive(Serialize)] + struct ResetPayload { + port: String, + #[serde(skip_serializing_if = "Option::is_none")] + board: Option, + } + + let payload = ResetPayload { + port: self.port.clone(), + board, + }; + + let resp = reqwest::blocking::Client::new() + .post(&url) + .json(&payload) + .timeout(std::time::Duration::from_secs(10)) + .send() + .map_err(|e| { + pyo3::exceptions::PyConnectionError::new_err(format!( + "failed to send reset request to daemon: {}", + e + )) + })?; + + let body: serde_json::Value = resp.json().map_err(|e| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "failed to parse reset response: {}", + e + )) + })?; + + let success = body + .get("success") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + if !success || !wait_for_output { + return Ok(success); + } + + // Wait for the device to produce serial output after reset. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout); + + // Brief pause for USB re-enumeration after DTR toggle + std::thread::sleep(std::time::Duration::from_millis(300)); + + // If WebSocket is connected (__enter__ was called), poll via read_lines. + // Note: the daemon preempts our session during reset and sends a + // "Reconnected" message after. With auto_reconnect=true the WebSocket + // transparently re-attaches, so read_lines_inner will see new output. + if self.runtime.is_some() && self.ws_read.is_some() { + while std::time::Instant::now() < deadline { + let remaining = (deadline - std::time::Instant::now()) + .as_secs_f64() + .min(0.2); + let lines = self.read_lines_inner(remaining); + if !lines.is_empty() { + return Ok(true); + } + } + return Ok(false); + } + + // No WebSocket — we can't observe output directly. + // Wait a conservative 1 second (ESP32-S3 USB-CDC typically boots + // in <500ms). The caller can pass a shorter timeout if needed. + let wait = timeout.min(1.0); + std::thread::sleep(std::time::Duration::from_secs_f64(wait)); + Ok(true) + } +} + +impl SerialMonitor { + /// Internal read_lines without hook dispatch (for write_json_rpc which has &self). + fn read_lines_inner(&self, timeout: f64) -> Vec { + let (Some(ref rt), Some(ref ws_read)) = (&self.runtime, &self.ws_read) else { + return vec![]; + }; + + let mut lines = Vec::new(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs_f64(timeout); + let auto_reconnect = self.auto_reconnect; + + while std::time::Instant::now() < deadline { + let remaining = deadline - std::time::Instant::now(); + let result = { + let mut read = ws_read.lock().unwrap(); + // tokio::time::timeout must be created inside the runtime + // context (otherwise: "there is no reactor running" panic). + rt.block_on(async { tokio::time::timeout(remaining, read.next()).await }) + }; + + match result { + Ok(Some(Ok(tungstenite::Message::Text(text)))) => { + match serde_json::from_str::(&text) { + Ok(ServerMessage::Data { + lines: data_lines, .. + }) => { + lines.extend(data_lines); + if !lines.is_empty() { + break; + } + } + Ok(ServerMessage::Preempted { .. }) => { + if auto_reconnect { + continue; + } + break; + } + Ok(ServerMessage::Reconnected { .. }) => { + continue; + } + _ => continue, + } + } + Ok(Some(Ok(tungstenite::Message::Close(_)))) | Ok(None) => break, + Err(_) => break, + _ => continue, + } + } + + lines + } +}