From 1872942a3f29c8883b524f44723029aa8986f98d Mon Sep 17 00:00:00 2001 From: Douglas Whittingham Date: Sat, 22 Aug 2026 17:33:55 -1000 Subject: [PATCH] Add --state-in-memory: keep the guest register file out of SSA The LLVM backend loads every state slot a region touches into an alloca at entry and lets mem2reg promote it. That is the textbook move, and on this workload it loses to the C backend. Promoting a whole guest register file gives the allocator far more simultaneously live values than the machine has registers, so it spills them straight back. Measured on GM4E01, one boot chunk carries 631 state phi nodes across 228 basic blocks, 20+ per block in hot loops. x86-64 has 16 GPRs; the live set is many times that. The result is phi construction, a much larger function and worse allocation, arriving back at values in memory with more instructions around them. --state-in-memory points state_[slot] straight into CPUState instead. Every load and store site works unchanged; what disappears is the entry prologue and, with it, the materialization barriers -- those exist only to flush values that were hoisted, and nothing is hoisted. LLVM still forwards stores to loads and keeps values in registers where that pays; it is simply no longer forced to keep the entire register file live across a region. Measured against the C backend, same DOL, same module compiler (clang 20.1.8, -O2 -flto=thin), interleaved paired runs: x86-64 Windows, GM4E01 race.sav 0.8533 -> 1.2464 12/12 pairs AArch64 Pi 4, GLME01 cold boot ~1.21 -> 1.3035 6/6 pairs Both are ratios against the C backend, measured directly rather than chained. On x86-64 that is 1.46x the current LLVM backend. Module size on GM4E01 falls from 389.1 MB to 161.0 MB, and on AArch64 from 126.0 MB to 78.7 MB. The gain is larger on x86-64 than on AArch64, which is what the register pressure explanation predicts: AArch64's 31 GPRs already absorb much of the promoted live set, which is also why the LLVM backend was already ahead of C there and behind it on x86-64. Three slot families keep their allocas. CR0-CR7 are 4-bit nibbles of CPUState.cr, XER_CA-XER_SO are individual bits of xer, and XER itself writes only the low 29 bits while preserving the flags. storeContext and loadContext pack and unpack them, so they have no standalone storage to point at. slotIsPacked() keeps them on the promoted path. The option participates in the object cache key. It changes emitted code, so a cache entry built with it must not collide with one built without it; without that, toggling the flag silently returns the other configuration's objects. Off by default. Note for anyone reproducing: skipping the PromoteMemToReg call alone does nothing. optimizeModule() runs the standard -O2 pipeline before emission and SROA/mem2reg promote the allocas anyway -- verified byte-identical IR either way. The allocas have to not exist. --- src/app/cli.c | 10 ++++++++ src/app/cli.h | 1 + src/app/pipeline.c | 8 ++++++ src/backend/llvm/branch_targets.cpp | 2 ++ src/backend/llvm/emitter.cpp | 1 + src/backend/llvm/emitter.h | 10 +++++++- src/backend/llvm/exits.cpp | 13 ++++++++++ src/backend/llvm/llvm_backend.h | 4 +++ src/backend/llvm/register_state.cpp | 38 ++++++++++++++++++++++++++--- src/backend/llvm/runtime.cpp | 4 +++ 10 files changed, 87 insertions(+), 4 deletions(-) diff --git a/src/app/cli.c b/src/app/cli.c index 648dfd8..896526e 100644 --- a/src/app/cli.c +++ b/src/app/cli.c @@ -16,6 +16,8 @@ void print_usage(const char* argv0) { fprintf(stderr, " -jN Use N worker jobs for split C output (e.g. -j14)\n"); fprintf(stderr, " --cpu gekko|broadway|espresso Select CPU profile (default: broadway)\n"); fprintf(stderr, " --backend c|llvm Select generated-code backend (default: c)\n"); + fprintf(stderr, " --state-in-memory Keep guest state in CPUState instead of\n"); + fprintf(stderr, " hoisting it into promoted allocas\n"); fprintf(stderr, " --targets host, x86-64-v2, x86-64-v3, aarch64, aarch64-a57\n"); fprintf(stderr, " --semantics exact|fast PowerPC floating-point semantics (default: exact)\n"); fprintf(stderr, " --instrumentation none|lockstep Compile release or state-journal objects\n"); @@ -191,6 +193,14 @@ int parse_cli(int argc, char** argv, CliOptions* opts) { opts->gamecube_mode = 1; continue; } + if (strcmp(arg, "--state-in-memory") == 0) { + opts->state_in_memory = 1; + continue; + } + if (strcmp(arg, "--no-state-in-memory") == 0) { + opts->state_in_memory = 0; + continue; + } if (strcmp(arg, "--backend") == 0) { if (i + 1 >= argc) { diff --git a/src/app/cli.h b/src/app/cli.h index 77f867a..a088f1f 100644 --- a/src/app/cli.h +++ b/src/app/cli.h @@ -30,6 +30,7 @@ typedef struct { int setup_mode; int show_help; int fast_semantics; + int state_in_memory; int lockstep_instrumentation; } CliOptions; diff --git a/src/app/pipeline.c b/src/app/pipeline.c index 3306611..98e8f11 100644 --- a/src/app/pipeline.c +++ b/src/app/pipeline.c @@ -73,6 +73,7 @@ typedef struct { const char* profile_generate_path; const char* profile_use_path; u64 partition_seed; + int state_in_memory; u32 ram_size; u32 mem2_size; char symbol_suffix[32]; @@ -260,6 +261,11 @@ static u64 llvm_job_hash(const LLVMChunkJob* job) { hash = hash_bytes(hash, &job->semantics, sizeof(job->semantics)); hash = hash_bytes(hash, &job->instrumentation, sizeof(job->instrumentation)); + /* Changes the emitted code, so a plan built with it must not collide with + one built without it. Omitting this is how a toggled codegen option + silently returns another configuration's objects. */ + hash = hash_bytes(hash, &job->state_in_memory, + sizeof(job->state_in_memory)); hash = hash_bytes(hash, &job->partition_seed, sizeof(job->partition_seed)); hash = hash_bytes(hash, &job->ram_size, sizeof(job->ram_size)); @@ -465,6 +471,7 @@ static int emit_llvm_chunk_job(const void* data, void* user) { options.profile_generate_path = job->profile_generate_path; options.profile_use_path = job->profile_use_path; options.partition_seed = job->partition_seed; + options.state_in_memory = job->state_in_memory; options.emit_thinlto = 1; options.thinlto_path = job->thinlto_path; options.fixed_memory_layout = 1; @@ -880,6 +887,7 @@ static int emit_code_sections_llvm(const LoadedCodeSection* sections, options->profile_generate_path; target_job->profile_use_path = options->profile_use_path; target_job->partition_seed = options->partition_seed; + target_job->state_in_memory = options->state_in_memory; target_job->ram_size = GC_MAIN_RAM_SIZE; target_job->mem2_size = cpu == DOLRECOMP_CPU_GEKKO ? 0u diff --git a/src/backend/llvm/branch_targets.cpp b/src/backend/llvm/branch_targets.cpp index 43ed05c..5f953f8 100644 --- a/src/backend/llvm/branch_targets.cpp +++ b/src/backend/llvm/branch_targets.cpp @@ -93,6 +93,8 @@ BasicBlock *FunctionEmitter::externalDestination(const DolIRTerminator &term, if (!used_[state]) continue; auto stateSlot = static_cast(state); + if (slotInMemory(stateSlot)) + continue; builder_.CreateStore(loadContext(stateSlot), state_[state]); } builder_.CreateBr(blocks_[continuationBlock]); diff --git a/src/backend/llvm/emitter.cpp b/src/backend/llvm/emitter.cpp index e26aa07..59b3aa8 100644 --- a/src/backend/llvm/emitter.cpp +++ b/src/backend/llvm/emitter.cpp @@ -29,6 +29,7 @@ FunctionEmitter::FunctionEmitter(LLVMContext &context, Module &module, semantics_(options.semantics), symbol_suffix_(options.symbol_suffix ? options.symbol_suffix : ""), fixed_memory_layout_(options.fixed_memory_layout != 0), + state_in_memory_(options.state_in_memory != 0), expected_ram_size_(options.ram_size), expected_mem2_size_(options.mem2_size) {} diff --git a/src/backend/llvm/emitter.h b/src/backend/llvm/emitter.h index dbce5d9..8e08132 100644 --- a/src/backend/llvm/emitter.h +++ b/src/backend/llvm/emitter.h @@ -65,6 +65,10 @@ class FunctionEmitter final { void scanLoopHeaders(); void scanRegionLeaders(); void finalizeStateSSA(); + bool stateInMemory() const; + bool state_in_memory_ = false; + static bool slotIsPacked(DolIRStateSlot slot); + bool slotInMemory(DolIRStateSlot slot) const; void emitEntry(); bool emitWrapper(llvm::raw_ostream &diagnostics); @@ -190,7 +194,11 @@ class FunctionEmitter final { llvm::Value *mem2_size_ = nullptr; llvm::BasicBlock *fallback_block_ = nullptr; llvm::PHINode *fallback_pc_ = nullptr; - std::array state_{}; + // Where each guest state slot lives in this function. By default an alloca + // that mem2reg promotes; with --state-in-memory a pointer straight into + // CPUState, so every load and store site works unchanged either way. Packed + // slots always keep an alloca -- they have no standalone storage to point at. + std::array state_{}; std::array pair_f32_{}; std::array pair_f64_{}; std::array fp_rep_{}; diff --git a/src/backend/llvm/exits.cpp b/src/backend/llvm/exits.cpp index 9773699..b273c57 100644 --- a/src/backend/llvm/exits.cpp +++ b/src/backend/llvm/exits.cpp @@ -27,6 +27,12 @@ void FunctionEmitter::emitEntry() { if (!used_[slot]) continue; auto stateSlot = static_cast(slot); + if (slotInMemory(stateSlot)) { + // No alloca and no entry copy: the slot is read and written where it + // already lives, inside CPUState. + state_[slot] = bytePtr(stateOffset(stateSlot)); + continue; + } state_[slot] = builder_.CreateAlloca(type(dolir_state_type(stateSlot)), nullptr, "state"); } @@ -97,6 +103,9 @@ void FunctionEmitter::emitEntry() { if (!used_[slot]) continue; auto stateSlot = static_cast(slot); + // A slot that already points into CPUState needs no prologue copy. + if (slotInMemory(stateSlot)) + continue; builder_.CreateStore(loadContext(stateSlot), state_[slot]); } initializeEntryControls(); @@ -132,6 +141,10 @@ void FunctionEmitter::syncDirtyState() { if (!dirty_[slot] || slot == DOLIR_STATE_FPSCR) continue; auto stateSlot = static_cast(slot); + // Nothing was hoisted for this slot, so nothing has gone stale; the load + // would read a CPUState field and store it straight back to itself. + if (slotInMemory(stateSlot)) + continue; storeContext( stateSlot, builder_.CreateLoad(type(dolir_state_type(stateSlot)), state_[slot])); diff --git a/src/backend/llvm/llvm_backend.h b/src/backend/llvm/llvm_backend.h index 71790a3..2cc37b1 100644 --- a/src/backend/llvm/llvm_backend.h +++ b/src/backend/llvm/llvm_backend.h @@ -48,6 +48,10 @@ typedef struct { u32 ram_size; u32 mem2_size; u64 partition_seed; + /* Keep guest state in CPUState instead of hoisting it into allocas that + mem2reg promotes. Changes emitted code, so it participates in the object + cache key. */ + int state_in_memory; const DolLLVMFunctionRange* function_ranges; u32 function_range_count; const u32* entry_points; diff --git a/src/backend/llvm/register_state.cpp b/src/backend/llvm/register_state.cpp index 8de0335..b0dd77b 100644 --- a/src/backend/llvm/register_state.cpp +++ b/src/backend/llvm/register_state.cpp @@ -10,11 +10,43 @@ namespace dolllvm { using namespace llvm; +// Guest state can live in CPUState instead of being hoisted into allocas at +// region entry. +// +// Promoting the guest register file gives the register allocator far more +// simultaneously live values than x86-64 has registers, so it spills them +// straight back. Measured on GM4E01: 631 state phi nodes across 228 basic +// blocks in a single chunk, 20+ per block in hot loops. AArch64's 31 GPRs +// absorb much of that; x86-64's 16 do not. +// +// Note that skipping PromoteMemToReg alone achieves nothing: optimizeModule() +// runs the standard -O2 pipeline before emission and SROA/mem2reg promote the +// allocas anyway. The allocas have to not exist. +bool FunctionEmitter::stateInMemory() const { return state_in_memory_; } + +// These slots are bitfields inside a wider CPUState word rather than +// addressable storage: CR0-CR7 are nibbles of cr, XER_CA-XER_SO are bits of +// xer, and XER itself preserves those flag bits while writing only the low 29. +// storeContext and loadContext pack and unpack them, so they cannot be pointed +// at directly and always keep an alloca. +bool FunctionEmitter::slotIsPacked(DolIRStateSlot slot) { + return (slot >= DOLIR_STATE_CR0 && slot <= DOLIR_STATE_CR7) || + (slot >= DOLIR_STATE_XER_CA && slot <= DOLIR_STATE_XER_SO) || + slot == DOLIR_STATE_XER; +} + +bool FunctionEmitter::slotInMemory(DolIRStateSlot slot) const { + return stateInMemory() && !slotIsPacked(slot); +} + void FunctionEmitter::finalizeStateSSA() { + // Nothing was hoisted, so there is nothing to promote. + if (stateInMemory()) + return; SmallVector registers; - for (AllocaInst *slot : state_) - if (slot) - registers.push_back(slot); + for (Value *slot : state_) + if (auto *alloca = dyn_cast_or_null(slot)) + registers.push_back(alloca); for (AllocaInst *pair : pair_f32_) if (pair) registers.push_back(pair); diff --git a/src/backend/llvm/runtime.cpp b/src/backend/llvm/runtime.cpp index 218f2eb..f37e268 100644 --- a/src/backend/llvm/runtime.cpp +++ b/src/backend/llvm/runtime.cpp @@ -32,6 +32,10 @@ void FunctionEmitter::reloadState(DolIRStateSlot slot) { known_state_[slot] = nullptr; if (slot == DOLIR_STATE_FPSCR) pending_fprf_ = nullptr; + // The helper wrote CPUState directly and the slot points there, so the value + // is already current; only the cached constant needed clearing. + if (slotInMemory(slot)) + return; builder_.CreateStore(loadContext(slot), state_[slot]); }