From e095203a546caadc5ac26755383d0b2880957cf7 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Fri, 17 Jul 2026 15:37:28 +0200 Subject: [PATCH] =?UTF-8?q?feat(nativemem):=20categorized=20native-memory?= =?UTF-8?q?=20accounting=20=E2=80=94=20first=20cut?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a NativeMem facility that tracks the profiler's own native memory usage per category, with a moving-window average and a running peak. - NativeMemCategory enum whose per-category live gauges partition the total (each backing allocation belongs to exactly one category, so there is no double counting). - Live accounting is always-on and independent of the COUNTERS build flag: record() is a single relaxed atomic add, async-signal-safe and usable from signal handlers. - sample() folds the live gauges into a moving-window average and a high-water max; it is ticked once per JFR chunk finish. - Totals mirror into the existing NATIVE_MEM_{LIVE,AVG,MAX}_BYTES counters (JFR + JNI counter path); per-category values are emitted as native_mem_{live,avg,max}_bytes. counter events, reusing the existing counter event format (no new event type). Instrumented sites (tagged CALLTRACE, their sole use today): the LinearAllocator chunk alloc/free (the call-trace arena) and the per-shard calltrace buffers. Accounting lives with the semantic owner rather than OS::safeAlloc, which stays category-agnostic. First-cut limits, documented in code: only CALLTRACE sites are instrumented so far (other categories read 0 until tagged); the max is sampled rather than spike-accurate; transient negative live is clamped to 0. The existing reserved/used/waste counters (CALLTRACE_STORAGE_BYTES, DICTIONARY_ARENA_WASTE_BYTES) remain an independent nested dimension and are intentionally not summed into the per-category total. Co-Authored-By: Claude Opus 4.8 (1M context) --- ddprof-lib/src/main/cpp/counters.h | 3 + ddprof-lib/src/main/cpp/flightRecorder.cpp | 47 +++++++++ ddprof-lib/src/main/cpp/flightRecorder.h | 3 + ddprof-lib/src/main/cpp/linearAllocator.cpp | 6 ++ ddprof-lib/src/main/cpp/nativeMem.cpp | 103 ++++++++++++++++++++ ddprof-lib/src/main/cpp/nativeMem.h | 99 +++++++++++++++++++ ddprof-lib/src/main/cpp/profiler.cpp | 6 ++ ddprof-lib/src/test/cpp/nativeMem_ut.cpp | 89 +++++++++++++++++ 8 files changed, 356 insertions(+) create mode 100644 ddprof-lib/src/main/cpp/nativeMem.cpp create mode 100644 ddprof-lib/src/main/cpp/nativeMem.h create mode 100644 ddprof-lib/src/test/cpp/nativeMem_ut.cpp diff --git a/ddprof-lib/src/main/cpp/counters.h b/ddprof-lib/src/main/cpp/counters.h index 6550136646..868dbe9c7e 100644 --- a/ddprof-lib/src/main/cpp/counters.h +++ b/ddprof-lib/src/main/cpp/counters.h @@ -52,6 +52,9 @@ X(CALLTRACE_STORAGE_TRACES, "calltrace_storage_traces") \ X(LINEAR_ALLOCATOR_BYTES, "linear_allocator_bytes") \ X(LINEAR_ALLOCATOR_CHUNKS, "linear_allocator_chunks") \ + X(NATIVE_MEM_LIVE_BYTES, "native_mem_live_bytes") \ + X(NATIVE_MEM_MAX_BYTES, "native_mem_max_bytes") \ + X(NATIVE_MEM_AVG_BYTES, "native_mem_avg_bytes") \ X(THREAD_IDS_COUNT, "thread_ids_count") \ X(THREAD_NAMES_COUNT, "thread_names_count") \ X(THREAD_FILTER_PAGES, "thread_filter_pages") \ diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index 1bd4ec1bcd..ddd634d47a 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -12,6 +12,7 @@ #include "context.h" #include "context_api.h" #include "counters.h" +#include "nativeMem.h" #include "dictionary.h" #include "flightRecorder.inline.h" #include "incbin.h" @@ -754,7 +755,9 @@ off_t Recording::finishChunk(bool end_recording, bool do_cleanup) { // dictionary) will reflect the previous serialization. That is, some level of // familiarity with the code base will be required to use this diagnostic // information for now. + updateNativeMemStats(); writeCounters(_buf); + writeNativeMem(_buf); // Keep a simple stats for where we failed to unwind // For the sakes of simplicity we are not keeping the count of failed unwinds which would also be @@ -1720,6 +1723,50 @@ void Recording::writeLogLevels(Buffer *buf) { } } +void Recording::updateNativeMemStats() { + // Fold the per-category live gauges into their moving-window averages and + // running peaks. This is a sampled max: peaks that rise and fall entirely + // between two chunk finishes are not observed. A spike-accurate max would + // track the high-water mark at allocation time. + NativeMem::sample(); + + // Mirror the totals into the flat counter table so they flow out through the + // existing counter path (JFR T_DATADOG_COUNTER events and the JNI debug + // counters). Per-category values are emitted separately by writeNativeMem(). + Counters::set(NATIVE_MEM_LIVE_BYTES, NativeMem::liveTotal()); + Counters::set(NATIVE_MEM_AVG_BYTES, NativeMem::avgTotal()); + Counters::set(NATIVE_MEM_MAX_BYTES, NativeMem::maxTotal()); +} + +void Recording::writeNativeMem(Buffer *buf) { + // Emit live/avg/max per category as counter events, reusing the counter event + // format with a "." name so they land alongside the totals + // without needing a dedicated event type or a slot in the counter table. + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + NativeMemCategory cat = (NativeMemCategory)c; + const char *name = NativeMem::categoryName(cat); + const struct { + const char *prefix; + long long value; + } metrics[] = { + {"native_mem_live_bytes.", NativeMem::live(cat)}, + {"native_mem_avg_bytes.", NativeMem::avg(cat)}, + {"native_mem_max_bytes.", NativeMem::max(cat)}, + }; + for (const auto &m : metrics) { + char label[64]; + snprintf(label, sizeof(label), "%s%s", m.prefix, name); + int start = buf->skip(1); + buf->putVar64(T_DATADOG_COUNTER); + buf->putVar64(_start_ticks); + buf->putUtf8(label); + buf->putVar64(m.value); + writeEventSizePrefix(buf, start); + flushIfNeeded(buf); + } + } +} + void Recording::writeCounters(Buffer *buf) { long long *counters = Counters::getCounters(); if (counters) { diff --git a/ddprof-lib/src/main/cpp/flightRecorder.h b/ddprof-lib/src/main/cpp/flightRecorder.h index fd5bffda59..ee6929b4f1 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.h +++ b/ddprof-lib/src/main/cpp/flightRecorder.h @@ -304,6 +304,9 @@ class Recording { void writeCounters(Buffer *buf); + void updateNativeMemStats(); + void writeNativeMem(Buffer *buf); + void writeUnwindFailures(Buffer *buf); void writeContextSnapshot(Buffer *buf, Context &context); diff --git a/ddprof-lib/src/main/cpp/linearAllocator.cpp b/ddprof-lib/src/main/cpp/linearAllocator.cpp index 4aea288fee..cc12da403c 100644 --- a/ddprof-lib/src/main/cpp/linearAllocator.cpp +++ b/ddprof-lib/src/main/cpp/linearAllocator.cpp @@ -17,6 +17,7 @@ #include "linearAllocator.h" #include "counters.h" +#include "nativeMem.h" #include "os.h" #include "common.h" #include @@ -167,6 +168,9 @@ void LinearAllocator::freeChunks(ChunkList& chunks) { OS::safeFree(current, chunks.chunk_size); Counters::decrement(LINEAR_ALLOCATOR_BYTES, chunks.chunk_size); Counters::decrement(LINEAR_ALLOCATOR_CHUNKS); + // The LinearAllocator's only user is call-trace storage, so all of its + // chunk memory is attributed to the CALLTRACE category. + NativeMem::record(NM_CALLTRACE, -(long long)chunks.chunk_size); current = prev; } @@ -260,6 +264,7 @@ Chunk *LinearAllocator::allocateChunk(Chunk *current) { Counters::increment(LINEAR_ALLOCATOR_BYTES, _chunk_size); Counters::increment(LINEAR_ALLOCATOR_CHUNKS); + NativeMem::record(NM_CALLTRACE, (long long)_chunk_size); } return chunk; } @@ -275,6 +280,7 @@ void LinearAllocator::freeChunk(Chunk *current) { OS::safeFree(current, _chunk_size); Counters::decrement(LINEAR_ALLOCATOR_BYTES, _chunk_size); Counters::decrement(LINEAR_ALLOCATOR_CHUNKS); + NativeMem::record(NM_CALLTRACE, -(long long)_chunk_size); } void LinearAllocator::reserveChunk(Chunk *current) { diff --git a/ddprof-lib/src/main/cpp/nativeMem.cpp b/ddprof-lib/src/main/cpp/nativeMem.cpp new file mode 100644 index 0000000000..4373e1f778 --- /dev/null +++ b/ddprof-lib/src/main/cpp/nativeMem.cpp @@ -0,0 +1,103 @@ +/* + * Copyright 2026 Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "nativeMem.h" + +volatile long long NativeMem::_live[NM_NUM_CATEGORIES] = {}; +long long NativeMem::_window[NM_NUM_CATEGORIES][NativeMem::WINDOW] = {}; +long long NativeMem::_total_window[NativeMem::WINDOW] = {}; +int NativeMem::_window_pos = 0; +int NativeMem::_window_count = 0; +long long NativeMem::_avg[NM_NUM_CATEGORIES] = {}; +long long NativeMem::_max[NM_NUM_CATEGORIES] = {}; +long long NativeMem::_total_avg = 0; +long long NativeMem::_total_max = 0; + +long long NativeMem::liveTotal() { + long long total = 0; + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + total += load(_live[c]); + } + return total; +} + +void NativeMem::sample() { + long long total = 0; + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + long long v = load(_live[c]); + // Clamp transient negatives from not-yet-instrumented free sites so they do + // not skew the window; accounting is currently best-effort per category. + if (v < 0) { + v = 0; + } + _window[c][_window_pos] = v; + if (v > _max[c]) { + _max[c] = v; + } + total += v; + } + + _total_window[_window_pos] = total; + if (total > _total_max) { + _total_max = total; + } + + _window_pos = (_window_pos + 1) % WINDOW; + if (_window_count < WINDOW) { + _window_count++; + } + + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + long long sum = 0; + for (int i = 0; i < _window_count; i++) { + sum += _window[c][i]; + } + _avg[c] = sum / _window_count; + } + + long long total_sum = 0; + for (int i = 0; i < _window_count; i++) { + total_sum += _total_window[i]; + } + _total_avg = total_sum / _window_count; +} + +void NativeMem::reset() { + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + store(_live[c], (long long)0); + _avg[c] = 0; + _max[c] = 0; + for (int i = 0; i < WINDOW; i++) { + _window[c][i] = 0; + } + } + for (int i = 0; i < WINDOW; i++) { + _total_window[i] = 0; + } + _window_pos = 0; + _window_count = 0; + _total_avg = 0; + _total_max = 0; +} + +const char *NativeMem::categoryName(NativeMemCategory category) { +#define X_NM_NAME(a, b) b, + static const char *const names[] = {DD_NATIVE_MEM_CATEGORY_TABLE(X_NM_NAME)}; +#undef X_NM_NAME + if (category < 0 || category >= NM_NUM_CATEGORIES) { + return "unknown"; + } + return names[category]; +} diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h new file mode 100644 index 0000000000..16807b4d68 --- /dev/null +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -0,0 +1,99 @@ +/* + * Copyright 2026 Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _NATIVEMEM_H +#define _NATIVEMEM_H + +#include "arch.h" + +// Physical native-memory categories used by the profiler's own allocations. +// Every tracked backing allocation belongs to exactly one category, so the +// per-category live byte gauges partition the total: sum(category) == total, +// with no double counting. +// +// The "reserved vs used vs wasted" breakdowns exposed by the existing counters +// (CALLTRACE_STORAGE_BYTES is the used slice of the CALLTRACE arena; +// DICTIONARY_ARENA_WASTE_BYTES is the wasted slice of the DICTIONARY arena) are +// a separate, nested dimension. They are intentionally NOT summed in here. +#define DD_NATIVE_MEM_CATEGORY_TABLE(X) \ + X(CALLTRACE, "calltrace") \ + X(DICTIONARY, "dictionary") \ + X(CONTEXT, "context") \ + X(THREAD_FILTER, "thread_filter") \ + X(CODECACHE, "codecache") \ + X(LINE_TABLES, "line_tables") \ + X(PERF, "perf") \ + X(THREAD_LOCAL, "thread_local") \ + X(JFR_BUFFERS, "jfr_buffers") \ + X(MISC, "misc") + +#define X_NM_ENUM(a, b) NM_##a, +typedef enum NativeMemCategory : int { + DD_NATIVE_MEM_CATEGORY_TABLE(X_NM_ENUM) NM_NUM_CATEGORIES +} NativeMemCategory; +#undef X_NM_ENUM + +// Tracks the profiler's live native memory per category, plus a moving-window +// average and running peak. Live accounting is always-on and independent of the +// COUNTERS build flag: record() is a single relaxed atomic add, which is +// async-signal-safe and therefore usable from signal handlers. +// +// The moving average and peak are refreshed by sample(), which is expected to +// be called from a single thread (the JFR chunk-finish path). +class NativeMem { +private: + // Number of sample() ticks retained in the moving-average window. + static const int WINDOW = 64; + + static volatile long long _live[NM_NUM_CATEGORIES]; + + // sample()-owned state; not touched from allocation sites. + static long long _window[NM_NUM_CATEGORIES][WINDOW]; + static long long _total_window[WINDOW]; + static int _window_pos; + static int _window_count; + static long long _avg[NM_NUM_CATEGORIES]; + static long long _max[NM_NUM_CATEGORIES]; + static long long _total_avg; + static long long _total_max; + +public: + // Account for a backing allocation (delta > 0) or free (delta < 0). + static void record(NativeMemCategory category, long long delta) { + atomicIncRelaxed(_live[category], delta); + } + + static long long live(NativeMemCategory category) { + return load(_live[category]); + } + static long long liveTotal(); + + // Fold the current live gauges into the moving-average window and the running + // peak. Call once per sampling tick from a single thread. + static void sample(); + + static long long avg(NativeMemCategory category) { return _avg[category]; } + static long long max(NativeMemCategory category) { return _max[category]; } + static long long avgTotal() { return _total_avg; } + static long long maxTotal() { return _total_max; } + + // Clear all live gauges and window/peak state. Not thread-safe against + // concurrent record()/sample(); intended for process init and tests. + static void reset(); + + static const char *categoryName(NativeMemCategory category); +}; + +#endif // _NATIVEMEM_H diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 561009e221..a49bf7a801 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -13,6 +13,7 @@ #include "guards.h" #include "common.h" #include "counters.h" +#include "nativeMem.h" #include "ctimer.h" #include "signalInflight.h" #include "dwarf.h" @@ -1321,6 +1322,7 @@ Error Profiler::start(Arguments &args, bool reset) { // (Re-)allocate calltrace buffers if (_max_stack_depth != args._jstackdepth) { + size_t prev_nelem = _max_stack_depth + RESERVED_FRAMES; _max_stack_depth = args._jstackdepth; size_t nelem = _max_stack_depth + RESERVED_FRAMES; @@ -1334,6 +1336,7 @@ Error Profiler::start(Arguments &args, bool reset) { return Error("Not enough memory to allocate stack trace buffers (try " "smaller jstackdepth)"); } + NativeMem::record(NM_CALLTRACE, (long long)(nelem * sizeof(CallTraceBuffer))); // Swap under the per-shard lock: all readers (recordJVMTISample, // recordExternalSample) acquire this lock via tryLock before reading // _calltrace_buffer, so no reader can observe a freed pointer mid-replacement. @@ -1341,6 +1344,9 @@ Error Profiler::start(Arguments &args, bool reset) { CallTraceBuffer *prev = _calltrace_buffer[i]; _calltrace_buffer[i] = fresh; _locks[i].unlock(); + if (prev != NULL) { + NativeMem::record(NM_CALLTRACE, -(long long)(prev_nelem * sizeof(CallTraceBuffer))); + } free(prev); } } diff --git a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp new file mode 100644 index 0000000000..d17e8cb0c9 --- /dev/null +++ b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp @@ -0,0 +1,89 @@ +/* + * Copyright 2026 Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "nativeMem.h" + +class NativeMemTest : public ::testing::Test { +protected: + void SetUp() override { NativeMem::reset(); } + void TearDown() override { NativeMem::reset(); } +}; + +// record() adds to and subtracts from the per-category live gauge, and the +// total is the sum across categories. +TEST_F(NativeMemTest, RecordTracksLivePerCategoryAndTotal) { + NativeMem::record(NM_CALLTRACE, 1000); + NativeMem::record(NM_DICTIONARY, 250); + EXPECT_EQ(1000, NativeMem::live(NM_CALLTRACE)); + EXPECT_EQ(250, NativeMem::live(NM_DICTIONARY)); + EXPECT_EQ(0, NativeMem::live(NM_CONTEXT)); + EXPECT_EQ(1250, NativeMem::liveTotal()); + + NativeMem::record(NM_CALLTRACE, -400); + EXPECT_EQ(600, NativeMem::live(NM_CALLTRACE)); + EXPECT_EQ(850, NativeMem::liveTotal()); +} + +// sample() folds the current live gauge into the moving average; a single +// sample yields avg == live and max == live. +TEST_F(NativeMemTest, SingleSampleAverageEqualsLive) { + NativeMem::record(NM_CALLTRACE, 800); + NativeMem::sample(); + EXPECT_EQ(800, NativeMem::avg(NM_CALLTRACE)); + EXPECT_EQ(800, NativeMem::max(NM_CALLTRACE)); + EXPECT_EQ(800, NativeMem::avgTotal()); + EXPECT_EQ(800, NativeMem::maxTotal()); +} + +// The moving average is the mean over the sampled ticks. +TEST_F(NativeMemTest, MovingAverageOverSamples) { + NativeMem::record(NM_CALLTRACE, 100); + NativeMem::sample(); // window: [100] + NativeMem::record(NM_CALLTRACE, 200); // live now 300 + NativeMem::sample(); // window: [100, 300] -> avg 200 + EXPECT_EQ(200, NativeMem::avg(NM_CALLTRACE)); +} + +// The peak is a high-water mark: it retains the largest sampled value even +// after live memory has been freed back down. +TEST_F(NativeMemTest, MaxIsHighWaterMark) { + NativeMem::record(NM_CALLTRACE, 5000); + NativeMem::sample(); + NativeMem::record(NM_CALLTRACE, -4900); // live drops to 100 + NativeMem::sample(); + EXPECT_EQ(100, NativeMem::live(NM_CALLTRACE)); + EXPECT_EQ(5000, NativeMem::max(NM_CALLTRACE)); +} + +// Transient negative live (an over-counted free at a not-yet-paired site) is +// clamped to zero when sampled so it does not skew the average or total. +TEST_F(NativeMemTest, NegativeLiveClampedInSample) { + NativeMem::record(NM_MISC, -1234); + NativeMem::sample(); + EXPECT_EQ(0, NativeMem::avg(NM_MISC)); + EXPECT_EQ(0, NativeMem::maxTotal()); +} + +// Every category exposes a distinct, non-empty name. +TEST_F(NativeMemTest, CategoryNamesPresent) { + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + const char *name = NativeMem::categoryName((NativeMemCategory)c); + ASSERT_NE(nullptr, name); + EXPECT_GT((int)strlen(name), 0); + } +}