Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions .github/workflows/dylint.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
name: Dylint

# Runs the custom `ban_raw_subprocess` dylint over the workspace.
# Lives in its own job (and uses its own nightly toolchain pin) so the
# stable workspace check on `check-ubuntu.yml` isn't blocked by dylint
# infrastructure issues. See #264.
# Runs the custom dylints over the workspace. This is enforced as its own
# required GitHub status check rather than being folded into `./lint`, because
# Dylint builds the lint crates and rustc-dev driver and would force a heavy
# recompile on every local lint call. See #264 and #994.

on:
workflow_dispatch: {}
Expand Down Expand Up @@ -69,8 +69,7 @@ jobs:
# `<name>@<toolchain>.<ext>` but cargo emits them as bare
# `<name>.<ext>` on first build. After each cargo-dylint
# invocation, alias any just-built libraries that lack the
# `@<toolchain>` suffix and retry. Same loop zccache uses in
# `ci/lint.py::lint_dylint_only`.
# `@<toolchain>` suffix and retry.
run: |
export PATH="${CARGO_HOME}/bin:${PATH}"
set +e
Expand Down
38 changes: 38 additions & 0 deletions crates/fbuild-build/src/ch32v/ch32v_compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,44 @@ impl Compiler for Ch32vCompiler {
self.build_unflags(),
)
}

fn artifact_cache_signature(
&self,
project_dir: &Path,
source: &Path,
extra_flags: &[String],
) -> String {
let ext = source
.extension()
.unwrap_or_default()
.to_string_lossy()
.to_lowercase();
let (compiler_path, flags) = match ext.as_str() {
"c" | "s" => (self.gcc_path(), self.c_flags()),
_ => (self.gxx_path(), self.cpp_flags()),
};
let extra_owned: Vec<String> = if self.is_framework_source(source) {
extra_flags
.iter()
.cloned()
.chain(
framework_suppression_flags()
.iter()
.map(|s| (*s).to_string()),
)
.collect()
} else {
extra_flags.to_vec()
};
crate::compiler::build_rebuild_signature_for_project(
project_dir,
compiler_path,
&flags,
&[],
&extra_owned,
self.build_unflags(),
)
}
}

#[cfg(test)]
Expand Down
111 changes: 105 additions & 6 deletions crates/fbuild-build/src/compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,39 @@ pub trait Compiler: Send + Sync {
self.build_unflags(),
)
}

/// Project-independent fingerprint for reusable artifact caches.
///
/// Unlike [`Self::rebuild_signature`], this normalizes absolute
/// project-local include/source paths relative to `project_dir`. The normal
/// rebuild signature remains project-local because it is written into a
/// project's own `.cmdhash`; global artifact caches need the same logical
/// project shape to hash identically even when copied to a different
/// absolute checkout path or basename.
fn artifact_cache_signature(
&self,
project_dir: &Path,
source: &Path,
extra_flags: &[String],
) -> String {
let ext = source
.extension()
.unwrap_or_default()
.to_string_lossy()
.to_lowercase();
let (compiler_path, flags) = match ext.as_str() {
"c" | "s" => (self.gcc_path(), self.c_flags()),
_ => (self.gxx_path(), self.cpp_flags()),
};
build_rebuild_signature_for_project(
project_dir,
compiler_path,
&flags,
&[],
extra_flags,
self.build_unflags(),
)
}
}

/// Shared compiler utilities used by all platform-specific compilers.
Expand Down Expand Up @@ -392,6 +425,48 @@ pub fn build_rebuild_signature(
pre_flags: &[String],
extra_flags: &[String],
unflags: &[String],
) -> String {
build_rebuild_signature_with_normalizer(
compiler_path,
flags,
pre_flags,
extra_flags,
unflags,
&normalize_signature_value,
)
}

/// Variant of [`build_rebuild_signature`] for global artifact cache keys.
///
/// Any absolute path under `project_dir` is reduced to `.project/<relative>`
/// before hashing, so two fresh checkouts with the same project layout produce
/// the same cache key even when their absolute roots or basenames differ.
pub fn build_rebuild_signature_for_project(
project_dir: &Path,
compiler_path: &Path,
flags: &[String],
pre_flags: &[String],
extra_flags: &[String],
unflags: &[String],
) -> String {
let normalize = |value: &str| normalize_signature_value_for_project(value, project_dir);
build_rebuild_signature_with_normalizer(
compiler_path,
flags,
pre_flags,
extra_flags,
unflags,
&normalize,
)
}

fn build_rebuild_signature_with_normalizer(
compiler_path: &Path,
flags: &[String],
pre_flags: &[String],
extra_flags: &[String],
unflags: &[String],
normalize_value: &dyn Fn(&str) -> String,
) -> String {
let strip = |group: &[String]| -> Vec<String> {
if unflags.is_empty() {
Expand All @@ -408,21 +483,25 @@ pub fn build_rebuild_signature(
hasher.update(compiler_identity(compiler_path).as_bytes());
hasher.update([0]);
for group in [flags.as_slice(), pre_flags, extra_flags.as_slice()] {
hash_signature_group(&mut hasher, group);
hash_signature_group(&mut hasher, group, normalize_value);
hasher.update([0xff]);
}
format!("{:x}", hasher.finalize())
}

fn hash_signature_group(hasher: &mut Sha256, group: &[String]) {
fn hash_signature_group(
hasher: &mut Sha256,
group: &[String],
normalize_value: &dyn Fn(&str) -> String,
) {
let mut expects_path_value = false;
for flag in group {
let normalized = if expects_path_value {
expects_path_value = false;
normalize_signature_value(flag)
normalize_value(flag)
} else {
expects_path_value = is_split_path_flag(flag);
normalize_signature_flag(flag)
normalize_signature_flag(flag, normalize_value)
};
hasher.update(normalized.as_bytes());
hasher.update([0]);
Expand All @@ -436,10 +515,10 @@ fn is_split_path_flag(flag: &str) -> bool {
)
}

fn normalize_signature_flag(flag: &str) -> String {
fn normalize_signature_flag(flag: &str, normalize_value: &dyn Fn(&str) -> String) -> String {
for prefix in ["-I", "-isystem=", "-iquote=", "-include=", "--sysroot="] {
if let Some(value) = flag.strip_prefix(prefix) {
return format!("{prefix}{}", normalize_signature_value(value));
return format!("{prefix}{}", normalize_value(value));
}
}
flag.to_string()
Expand All @@ -456,6 +535,26 @@ fn normalize_signature_value(value: &str) -> String {
normalize_signature_path(path)
}

fn normalize_signature_value_for_project(value: &str, project_dir: &Path) -> String {
if value.is_empty() {
return String::new();
}
let path = Path::new(value);
if !looks_like_absolute_path(path, value) {
return value.to_string();
}
let arg = fbuild_core::path::path_arg_for_compile_cwd(path, project_dir);
if !looks_like_absolute_path(Path::new(&arg), &arg) {
if arg == "." {
".project".to_string()
} else {
format!(".project/{arg}")
}
} else {
normalize_signature_path(path)
}
}

fn normalize_signature_path(path: &Path) -> String {
let normalized = normalize_signature_components(path);
if let Some(index) = normalized
Expand Down
35 changes: 35 additions & 0 deletions crates/fbuild-build/src/compiler_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,41 @@ fn test_build_rebuild_signature_ignores_attached_include_root() {
assert_eq!(sig_a, sig_b);
}

#[test]
fn test_artifact_cache_signature_ignores_project_directory_name() {
let tmp = tempfile::tempdir().unwrap();
let project_a = tmp.path().join("nds");
let project_b = tmp.path().join("nds-copy");
let include_a = project_a.join("src");
let include_b = project_b.join("src");
std::fs::create_dir_all(&include_a).unwrap();
std::fs::create_dir_all(&include_b).unwrap();
let flags_a = vec![format!("-I{}", include_a.display())];
let flags_b = vec![format!("-I{}", include_b.display())];

let sig_a = build_rebuild_signature_for_project(
&project_a,
Path::new("/tmp/tool/bin/xtensa-esp32-elf-g++"),
&flags_a,
&[],
&[],
&[],
);
let sig_b = build_rebuild_signature_for_project(
&project_b,
Path::new("/tmp/tool/bin/xtensa-esp32-elf-g++"),
&flags_b,
&[],
&[],
&[],
);

assert_eq!(
sig_a, sig_b,
"artifact cache keys must depend on project layout, not checkout basename"
);
}

#[test]
fn test_build_rebuild_signature_ignores_split_path_flag_values() {
let flags_a = vec![
Expand Down
30 changes: 30 additions & 0 deletions crates/fbuild-build/src/esp32/esp32_compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,36 @@ impl Compiler for Esp32Compiler {
Compiler::build_unflags(self),
)
}

fn artifact_cache_signature(
&self,
project_dir: &Path,
source: &Path,
extra_flags: &[String],
) -> String {
let ext = source
.extension()
.unwrap_or_default()
.to_string_lossy()
.to_lowercase();
let base_flags = match ext.as_str() {
"c" | "s" => self.c_flags(),
_ => self.cpp_flags(),
};
let include_flags = self.base.build_include_flags();
let compiler_path = match ext.as_str() {
"c" | "s" => self.gcc_path(),
_ => self.gxx_path(),
};
crate::compiler::build_rebuild_signature_for_project(
project_dir,
compiler_path,
&base_flags,
&include_flags,
extra_flags,
Compiler::build_unflags(self),
)
}
}

// Response file utilities (write_response_file, replace_path_backslashes)
Expand Down
2 changes: 1 addition & 1 deletion crates/fbuild-build/src/esp32/orchestrator/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,7 @@ impl BuildOrchestrator for Esp32Orchestrator {
};
if let Some(cache) = core_cache.as_ref() {
let _g = perf.phase("core-cache-hydrate");
match cache.hydrate(core_build_dir) {
match cache.hydrate(core_build_dir, &compiler, &all_core_sources, &user_overlay) {
Ok(stats) if stats.copied > 0 || stats.skipped > 0 => tracing::info!(
"framework core cache hydrate key={} copied={} skipped={} from {}",
cache.key(),
Expand Down
Loading
Loading