diff --git a/Cargo.toml b/Cargo.toml index 2e8d5e1a2a98..eb9057a1d46e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -276,6 +276,13 @@ inherits = "release" opt-level = 1 lto = "off" +[profile.profiling] +inherits = "dev" +opt-level = 0 +debug = true +split-debuginfo = "unpacked" +panic = "abort" + [profile.release] # https://doc.rust-lang.org/cargo/reference/profiles.html#strip strip = true @@ -295,9 +302,10 @@ doctest-private = [] # see benchmark-private = [] # see lib.rs::benchmark_private interop-tests-private = [] # see lib.rs::interop_tests_private -# Allocator +# Allocator. Use at most one of these. rustalloc = [] jemalloc = ["dep:tikv-jemallocator"] +system-alloc = [] # Use the platform allocator (for memory profiling). tokio-console = ["dep:console-subscriber"] tracing-loki = ["dep:tracing-loki"] diff --git a/Makefile b/Makefile index f34d7a080b2d..abdf1497d1c0 100644 --- a/Makefile +++ b/Makefile @@ -117,4 +117,38 @@ license: docs: cargo doc --no-deps +## +## Memory Profiling +## + +# Read up on memory profiling in Forest: https://rumcajs.dev/posts/memory-analysis-in-rust/ + +# Memory profiling is done with the `profiling` profile. There's no silver bullet for memory profiling, so we provide a few options here. + +### Gperftools +# https://github.com/gperftools/gperftools + +# Profile with gperftools (Memory/Heap profiler) +# There is a workaround there, as outlined in https://github.com/gperftools/gperftools/issues/1603 +gperfheapprofile = FOREST_PROFILING_GPERFTOOLS_BUILD=1 cargo build --no-default-features --features system-alloc --profile=profiling --bin $(1); \ + ulimit -n 8192; \ + HEAPPROFILE_USE_PID=t HEAPPROFILE=/tmp/gperfheap.$(1).prof target/profiling/$(1) $(2) + +gperfheapprofile.forest: + $(call gperfheapprofile,forest, --chain calibnet --encrypt-keystore=false) + +# To visualize the heap profile, run: +# pprof -http=localhost:8080 bool { std::env::var("DOCS_RS").is_ok() } -fn is_sidecar_ffi_enabled() -> bool { - // Opt-out building the F3 sidecar staticlib - match std::env::var("FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT") { - Ok(value) => !matches!(value.to_lowercase().as_str(), "1" | "true"), - _ => true, - } +fn is_env_truthy(env: &str) -> bool { + std::env::var(env) + .ok() + .map(|var| matches!(var.to_lowercase().as_str(), "1" | "true" | "yes" | "_yes_")) + .unwrap_or_default() } fn rpc_regression_tests_gen() { diff --git a/documentation/src/SUMMARY.md b/documentation/src/SUMMARY.md index 87127022c593..5d53eb260902 100644 --- a/documentation/src/SUMMARY.md +++ b/documentation/src/SUMMARY.md @@ -11,7 +11,6 @@ - [Developer documentation](./developer_documentation/introduction.md) - [Database migrations](./developer_documentation/database_migrations.md) - [Local GH Actions](./developer_documentation/local_actions.md) - - [Memory analysis](./developer_documentation/memory-analysis.md) - [Release checklist](./developer_documentation/release_checklist.md) - [State migration guide](./developer_documentation/state_migration_guide.md) - [Devnet Notes](./developer_documentation/devnet_notes.md) diff --git a/documentation/src/developer_documentation/heaptrack/bottom_up.png b/documentation/src/developer_documentation/heaptrack/bottom_up.png deleted file mode 100644 index e506d976bb60..000000000000 Binary files a/documentation/src/developer_documentation/heaptrack/bottom_up.png and /dev/null differ diff --git a/documentation/src/developer_documentation/heaptrack/caller_callee.png b/documentation/src/developer_documentation/heaptrack/caller_callee.png deleted file mode 100644 index bc2267ef7d10..000000000000 Binary files a/documentation/src/developer_documentation/heaptrack/caller_callee.png and /dev/null differ diff --git a/documentation/src/developer_documentation/heaptrack/consumed.png b/documentation/src/developer_documentation/heaptrack/consumed.png deleted file mode 100644 index 9a6ffe05e95a..000000000000 Binary files a/documentation/src/developer_documentation/heaptrack/consumed.png and /dev/null differ diff --git a/documentation/src/developer_documentation/heaptrack/flamegraph.png b/documentation/src/developer_documentation/heaptrack/flamegraph.png deleted file mode 100644 index 2887dfa44a9d..000000000000 Binary files a/documentation/src/developer_documentation/heaptrack/flamegraph.png and /dev/null differ diff --git a/documentation/src/developer_documentation/heaptrack/sizes.png b/documentation/src/developer_documentation/heaptrack/sizes.png deleted file mode 100644 index 7d52f7ad1865..000000000000 Binary files a/documentation/src/developer_documentation/heaptrack/sizes.png and /dev/null differ diff --git a/documentation/src/developer_documentation/heaptrack/summary.png b/documentation/src/developer_documentation/heaptrack/summary.png deleted file mode 100644 index 94283c25fef9..000000000000 Binary files a/documentation/src/developer_documentation/heaptrack/summary.png and /dev/null differ diff --git a/documentation/src/developer_documentation/memory-analysis.md b/documentation/src/developer_documentation/memory-analysis.md deleted file mode 100644 index eec44b12a9d8..000000000000 --- a/documentation/src/developer_documentation/memory-analysis.md +++ /dev/null @@ -1,135 +0,0 @@ -In case of memory leaks, either coming from _unsafe_ libraries or just Forest -pushing shamelessly into some collection, it is useful to not guess where the -leak happened but to use proper tooling. - -# [HeapTrack](https://github.com/KDE/heaptrack) - -## Installation - -Either build it with the instructions provided in the repository or download a -ready AppImage, e.g. from -[here](https://invent.kde.org/sdk/heaptrack/-/releases). You may not want to use -the `heaptrack` available in your OS packages as it may be a bit outdated. - -## Preparation - -To get the most out of the tool, you may want to add debug information to the -binary, regardless if you are running it in release or debug mode. - -```toml -[profile.dev] -debug = 2 - -[profile.release] -debug = 2 -``` - -## Usage - -You can grab the trace on your host machine or in a VPS (e.g. Digital Ocean -Droplet). - -Start tracing with `heaptrack `, e.g. - -``` -heaptrack target/release/forest --encrypt-keystore=false --target-peer-count 50 --chain calibnet --import-snapshot forest_snapshot.car -``` - -This will push traces to a file, e.g. `heaptrack.forest.12345.gz`. The longer -your process will be running, the bigger it will get, so double check your free -space before leaving it overnight. - -Now analyze the trace. You can do it after Forest has e.g. crashed due to OOM or -even during its execution. If you were capturing traces in a Droplet, copy the -file to your host, e.g. -`scp chainsafe@123.45.66.77:/home/chainsafe/heaptrack.forest.12345.gz .`. - -Depending on the size of the trace, it may take a while (but there is a nice -progress bar so you will know if you can grab a coffee in the meantime). - -``` -heaptrack --analyze heaptrack.forest.12345.gz -``` - -### Summary - -Here we can see memory usage overview. Keep in mind that _leaks_ here are not -necessarily leaks - it's just memory that hasn't been yet freed. A global cache -would always show as a leak. - -While most of the potential _culprits_ are not necessarily interesting (e.g. -`alloc::*`) because even a `String` constructor calls them, we immediately see -that among specific ones, it's the `rocksdb` that gets into the spotlight. - -![summary](heaptrack/summary.png) - -### Bottom-up - -View in which you see _low-level_ methods first. In such view, the first methods -would almost always be allocator methods, finally unwinding into `main`. - -![bottom-up](heaptrack/bottom_up.png) - -### Caller/callee - -All the methods called along with their allocations, where one can easily -navigate between their callers and callees, also showing you the location in -code (you can configure `heaptrack` to take you to that code with -`Settings/Code Navigation`). Most useful tab when you delve into the details. - -![caller-callee](heaptrack/caller_callee.png) - -### Top-down - -Basically an inverse of _Bottom-up_ view. High-level methods first, then you can -drill down. - -### Flamegraph - -A graphical form of Bottom-up and Top-Down (you can switch). Helps with -visualizing the heavy allocators. - -![flamegraph-up](heaptrack/flamegraph.png) - -### Consumed - -Shows the heap memory consumption over time. Here we can notice some patterns, -e.g. what happens with memory during snapshot import, then downloading headers -and syncing. - -![consumed](heaptrack/consumed.png) - -### Allocations - -Shows total number of allocations over time. - -### Temporary allocations - -Shows the number of temporary allocations over time. Temporary allocation is an -allocation followed by its deallocation, i.e. there are no other allocations -in-between. - -### Sizes - -This tab will show you the allocation sizes during runtime and their frequency. -If you hover over a bar you will see that e.g. `LZ4_createStream` (most likely -used by `rocksdb`) made 5,624,180 allocations, total 92.3G, on average 14.4kB -per allocation. - -![sizes](heaptrack/sizes.png) - -### Miscellaneous - -- Keep in mind that running Forest _with_ heaptrack gives a non-negligible - memory and CPU overhead. You may not be able to run mainnet node on a 16G - machine even if normally it would be fine. -- Optimizations may play tricks on the developer, e.g. inlining functions so - they won't even appear in your trace. If you think a particular method should - have been called but for mysterious reasons it does not appear in the - analysis, you may want to put `#[inline(never)]` on top of it. Analyzing a - debug build may also be useful, but depending on where the leak happens, it - may be too slow. -- There is a lot of noise coming from dependencies and standard library. It's - useful to mentally filter them out a bit and focus on the biggest culprits in - Forest methods. Flamegraph and caller/callee view are the most useful for - this. diff --git a/src/lib.rs b/src/lib.rs index e1780a1964c1..f91dc0b05932 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,6 +30,10 @@ cfg_if::cfg_if! { use crate::cli_shared::tikv_jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; + } else if #[cfg(feature = "system-alloc")] { + use std::alloc::System; + #[global_allocator] + static GLOBAL: System = System; } }