diff --git a/src/app/cli.c b/src/app/cli.c index 648dfd8..896526e 100644 --- a/src/app/cli.c +++ b/src/app/cli.c @@ -16,6 +16,8 @@ void print_usage(const char* argv0) { fprintf(stderr, " -jN Use N worker jobs for split C output (e.g. -j14)\n"); fprintf(stderr, " --cpu gekko|broadway|espresso Select CPU profile (default: broadway)\n"); fprintf(stderr, " --backend c|llvm Select generated-code backend (default: c)\n"); + fprintf(stderr, " --state-in-memory Keep guest state in CPUState instead of\n"); + fprintf(stderr, " hoisting it into promoted allocas\n"); fprintf(stderr, " --targets host, x86-64-v2, x86-64-v3, aarch64, aarch64-a57\n"); fprintf(stderr, " --semantics exact|fast PowerPC floating-point semantics (default: exact)\n"); fprintf(stderr, " --instrumentation none|lockstep Compile release or state-journal objects\n"); @@ -191,6 +193,14 @@ int parse_cli(int argc, char** argv, CliOptions* opts) { opts->gamecube_mode = 1; continue; } + if (strcmp(arg, "--state-in-memory") == 0) { + opts->state_in_memory = 1; + continue; + } + if (strcmp(arg, "--no-state-in-memory") == 0) { + opts->state_in_memory = 0; + continue; + } if (strcmp(arg, "--backend") == 0) { if (i + 1 >= argc) { diff --git a/src/app/cli.h b/src/app/cli.h index 77f867a..a088f1f 100644 --- a/src/app/cli.h +++ b/src/app/cli.h @@ -30,6 +30,7 @@ typedef struct { int setup_mode; int show_help; int fast_semantics; + int state_in_memory; int lockstep_instrumentation; } CliOptions; diff --git a/src/app/pipeline.c b/src/app/pipeline.c index 3306611..98e8f11 100644 --- a/src/app/pipeline.c +++ b/src/app/pipeline.c @@ -73,6 +73,7 @@ typedef struct { const char* profile_generate_path; const char* profile_use_path; u64 partition_seed; + int state_in_memory; u32 ram_size; u32 mem2_size; char symbol_suffix[32]; @@ -260,6 +261,11 @@ static u64 llvm_job_hash(const LLVMChunkJob* job) { hash = hash_bytes(hash, &job->semantics, sizeof(job->semantics)); hash = hash_bytes(hash, &job->instrumentation, sizeof(job->instrumentation)); + /* Changes the emitted code, so a plan built with it must not collide with + one built without it. Omitting this is how a toggled codegen option + silently returns another configuration's objects. */ + hash = hash_bytes(hash, &job->state_in_memory, + sizeof(job->state_in_memory)); hash = hash_bytes(hash, &job->partition_seed, sizeof(job->partition_seed)); hash = hash_bytes(hash, &job->ram_size, sizeof(job->ram_size)); @@ -465,6 +471,7 @@ static int emit_llvm_chunk_job(const void* data, void* user) { options.profile_generate_path = job->profile_generate_path; options.profile_use_path = job->profile_use_path; options.partition_seed = job->partition_seed; + options.state_in_memory = job->state_in_memory; options.emit_thinlto = 1; options.thinlto_path = job->thinlto_path; options.fixed_memory_layout = 1; @@ -880,6 +887,7 @@ static int emit_code_sections_llvm(const LoadedCodeSection* sections, options->profile_generate_path; target_job->profile_use_path = options->profile_use_path; target_job->partition_seed = options->partition_seed; + target_job->state_in_memory = options->state_in_memory; target_job->ram_size = GC_MAIN_RAM_SIZE; target_job->mem2_size = cpu == DOLRECOMP_CPU_GEKKO ? 0u diff --git a/src/backend/llvm/branch_targets.cpp b/src/backend/llvm/branch_targets.cpp index 43ed05c..5f953f8 100644 --- a/src/backend/llvm/branch_targets.cpp +++ b/src/backend/llvm/branch_targets.cpp @@ -93,6 +93,8 @@ BasicBlock *FunctionEmitter::externalDestination(const DolIRTerminator &term, if (!used_[state]) continue; auto stateSlot = static_cast(state); + if (slotInMemory(stateSlot)) + continue; builder_.CreateStore(loadContext(stateSlot), state_[state]); } builder_.CreateBr(blocks_[continuationBlock]); diff --git a/src/backend/llvm/emitter.cpp b/src/backend/llvm/emitter.cpp index e26aa07..59b3aa8 100644 --- a/src/backend/llvm/emitter.cpp +++ b/src/backend/llvm/emitter.cpp @@ -29,6 +29,7 @@ FunctionEmitter::FunctionEmitter(LLVMContext &context, Module &module, semantics_(options.semantics), symbol_suffix_(options.symbol_suffix ? options.symbol_suffix : ""), fixed_memory_layout_(options.fixed_memory_layout != 0), + state_in_memory_(options.state_in_memory != 0), expected_ram_size_(options.ram_size), expected_mem2_size_(options.mem2_size) {} diff --git a/src/backend/llvm/emitter.h b/src/backend/llvm/emitter.h index dbce5d9..8e08132 100644 --- a/src/backend/llvm/emitter.h +++ b/src/backend/llvm/emitter.h @@ -65,6 +65,10 @@ class FunctionEmitter final { void scanLoopHeaders(); void scanRegionLeaders(); void finalizeStateSSA(); + bool stateInMemory() const; + bool state_in_memory_ = false; + static bool slotIsPacked(DolIRStateSlot slot); + bool slotInMemory(DolIRStateSlot slot) const; void emitEntry(); bool emitWrapper(llvm::raw_ostream &diagnostics); @@ -190,7 +194,11 @@ class FunctionEmitter final { llvm::Value *mem2_size_ = nullptr; llvm::BasicBlock *fallback_block_ = nullptr; llvm::PHINode *fallback_pc_ = nullptr; - std::array state_{}; + // Where each guest state slot lives in this function. By default an alloca + // that mem2reg promotes; with --state-in-memory a pointer straight into + // CPUState, so every load and store site works unchanged either way. Packed + // slots always keep an alloca -- they have no standalone storage to point at. + std::array state_{}; std::array pair_f32_{}; std::array pair_f64_{}; std::array fp_rep_{}; diff --git a/src/backend/llvm/exits.cpp b/src/backend/llvm/exits.cpp index 9773699..b273c57 100644 --- a/src/backend/llvm/exits.cpp +++ b/src/backend/llvm/exits.cpp @@ -27,6 +27,12 @@ void FunctionEmitter::emitEntry() { if (!used_[slot]) continue; auto stateSlot = static_cast(slot); + if (slotInMemory(stateSlot)) { + // No alloca and no entry copy: the slot is read and written where it + // already lives, inside CPUState. + state_[slot] = bytePtr(stateOffset(stateSlot)); + continue; + } state_[slot] = builder_.CreateAlloca(type(dolir_state_type(stateSlot)), nullptr, "state"); } @@ -97,6 +103,9 @@ void FunctionEmitter::emitEntry() { if (!used_[slot]) continue; auto stateSlot = static_cast(slot); + // A slot that already points into CPUState needs no prologue copy. + if (slotInMemory(stateSlot)) + continue; builder_.CreateStore(loadContext(stateSlot), state_[slot]); } initializeEntryControls(); @@ -132,6 +141,10 @@ void FunctionEmitter::syncDirtyState() { if (!dirty_[slot] || slot == DOLIR_STATE_FPSCR) continue; auto stateSlot = static_cast(slot); + // Nothing was hoisted for this slot, so nothing has gone stale; the load + // would read a CPUState field and store it straight back to itself. + if (slotInMemory(stateSlot)) + continue; storeContext( stateSlot, builder_.CreateLoad(type(dolir_state_type(stateSlot)), state_[slot])); diff --git a/src/backend/llvm/llvm_backend.h b/src/backend/llvm/llvm_backend.h index 71790a3..2cc37b1 100644 --- a/src/backend/llvm/llvm_backend.h +++ b/src/backend/llvm/llvm_backend.h @@ -48,6 +48,10 @@ typedef struct { u32 ram_size; u32 mem2_size; u64 partition_seed; + /* Keep guest state in CPUState instead of hoisting it into allocas that + mem2reg promotes. Changes emitted code, so it participates in the object + cache key. */ + int state_in_memory; const DolLLVMFunctionRange* function_ranges; u32 function_range_count; const u32* entry_points; diff --git a/src/backend/llvm/register_state.cpp b/src/backend/llvm/register_state.cpp index 8de0335..b0dd77b 100644 --- a/src/backend/llvm/register_state.cpp +++ b/src/backend/llvm/register_state.cpp @@ -10,11 +10,43 @@ namespace dolllvm { using namespace llvm; +// Guest state can live in CPUState instead of being hoisted into allocas at +// region entry. +// +// Promoting the guest register file gives the register allocator far more +// simultaneously live values than x86-64 has registers, so it spills them +// straight back. Measured on GM4E01: 631 state phi nodes across 228 basic +// blocks in a single chunk, 20+ per block in hot loops. AArch64's 31 GPRs +// absorb much of that; x86-64's 16 do not. +// +// Note that skipping PromoteMemToReg alone achieves nothing: optimizeModule() +// runs the standard -O2 pipeline before emission and SROA/mem2reg promote the +// allocas anyway. The allocas have to not exist. +bool FunctionEmitter::stateInMemory() const { return state_in_memory_; } + +// These slots are bitfields inside a wider CPUState word rather than +// addressable storage: CR0-CR7 are nibbles of cr, XER_CA-XER_SO are bits of +// xer, and XER itself preserves those flag bits while writing only the low 29. +// storeContext and loadContext pack and unpack them, so they cannot be pointed +// at directly and always keep an alloca. +bool FunctionEmitter::slotIsPacked(DolIRStateSlot slot) { + return (slot >= DOLIR_STATE_CR0 && slot <= DOLIR_STATE_CR7) || + (slot >= DOLIR_STATE_XER_CA && slot <= DOLIR_STATE_XER_SO) || + slot == DOLIR_STATE_XER; +} + +bool FunctionEmitter::slotInMemory(DolIRStateSlot slot) const { + return stateInMemory() && !slotIsPacked(slot); +} + void FunctionEmitter::finalizeStateSSA() { + // Nothing was hoisted, so there is nothing to promote. + if (stateInMemory()) + return; SmallVector registers; - for (AllocaInst *slot : state_) - if (slot) - registers.push_back(slot); + for (Value *slot : state_) + if (auto *alloca = dyn_cast_or_null(slot)) + registers.push_back(alloca); for (AllocaInst *pair : pair_f32_) if (pair) registers.push_back(pair); diff --git a/src/backend/llvm/runtime.cpp b/src/backend/llvm/runtime.cpp index 218f2eb..f37e268 100644 --- a/src/backend/llvm/runtime.cpp +++ b/src/backend/llvm/runtime.cpp @@ -32,6 +32,10 @@ void FunctionEmitter::reloadState(DolIRStateSlot slot) { known_state_[slot] = nullptr; if (slot == DOLIR_STATE_FPSCR) pending_fprf_ = nullptr; + // The helper wrote CPUState directly and the slot points there, so the value + // is already current; only the cached constant needed clearing. + if (slotInMemory(slot)) + return; builder_.CreateStore(loadContext(slot), state_[slot]); }