Add --state-in-memory: keep the guest register file out of SSA

src/app/cli.c

@@ -16,6 +16,8 @@ void print_usage(const char* argv0) {
     fprintf(stderr, "  -jN                            Use N worker jobs for split C output (e.g. -j14)\n");
     fprintf(stderr, "  --cpu gekko|broadway|espresso  Select CPU profile (default: broadway)\n");
     fprintf(stderr, "  --backend c|llvm               Select generated-code backend (default: c)\n");
+    fprintf(stderr, "  --state-in-memory              Keep guest state in CPUState instead of\n");
+    fprintf(stderr, "                                 hoisting it into promoted allocas\n");
     fprintf(stderr, "  --targets <set>                host, x86-64-v2, x86-64-v3, aarch64, aarch64-a57\n");
     fprintf(stderr, "  --semantics exact|fast         PowerPC floating-point semantics (default: exact)\n");
     fprintf(stderr, "  --instrumentation none|lockstep  Compile release or state-journal objects\n");
@@ -191,6 +193,14 @@ int parse_cli(int argc, char** argv, CliOptions* opts) {
             opts->gamecube_mode = 1;
             continue;
         }
+        if (strcmp(arg, "--state-in-memory") == 0) {
+            opts->state_in_memory = 1;
+            continue;
+        }
+        if (strcmp(arg, "--no-state-in-memory") == 0) {
+            opts->state_in_memory = 0;
+            continue;
+        }
 
         if (strcmp(arg, "--backend") == 0) {
             if (i + 1 >= argc) {

src/app/cli.h

@@ -30,6 +30,7 @@ typedef struct {
     int setup_mode;
     int show_help;
     int fast_semantics;
+    int state_in_memory;
     int lockstep_instrumentation;
 } CliOptions;
 

src/app/pipeline.c

@@ -73,6 +73,7 @@ typedef struct {
     const char* profile_generate_path;
     const char* profile_use_path;
     u64 partition_seed;
+    int state_in_memory;
     u32 ram_size;
     u32 mem2_size;
     char symbol_suffix[32];
@@ -260,6 +261,11 @@ static u64 llvm_job_hash(const LLVMChunkJob* job) {
     hash = hash_bytes(hash, &job->semantics, sizeof(job->semantics));
     hash = hash_bytes(hash, &job->instrumentation,
                       sizeof(job->instrumentation));
+    /* Changes the emitted code, so a plan built with it must not collide with
+       one built without it. Omitting this is how a toggled codegen option
+       silently returns another configuration's objects. */
+    hash = hash_bytes(hash, &job->state_in_memory,
+                      sizeof(job->state_in_memory));
     hash = hash_bytes(hash, &job->partition_seed,
                       sizeof(job->partition_seed));
     hash = hash_bytes(hash, &job->ram_size, sizeof(job->ram_size));
@@ -465,6 +471,7 @@ static int emit_llvm_chunk_job(const void* data, void* user) {
     options.profile_generate_path = job->profile_generate_path;
     options.profile_use_path = job->profile_use_path;
     options.partition_seed = job->partition_seed;
+    options.state_in_memory = job->state_in_memory;
     options.emit_thinlto = 1;
     options.thinlto_path = job->thinlto_path;
     options.fixed_memory_layout = 1;
@@ -880,6 +887,7 @@ static int emit_code_sections_llvm(const LoadedCodeSection* sections,
                     options->profile_generate_path;
                 target_job->profile_use_path = options->profile_use_path;
                 target_job->partition_seed = options->partition_seed;
+                target_job->state_in_memory = options->state_in_memory;
                 target_job->ram_size = GC_MAIN_RAM_SIZE;
                 target_job->mem2_size = cpu == DOLRECOMP_CPU_GEKKO
                                              ? 0u

src/backend/llvm/branch_targets.cpp

@@ -93,6 +93,8 @@ BasicBlock *FunctionEmitter::externalDestination(const DolIRTerminator &term,
       if (!used_[state])
         continue;
       auto stateSlot = static_cast<DolIRStateSlot>(state);
+      if (slotInMemory(stateSlot))
+        continue;
       builder_.CreateStore(loadContext(stateSlot), state_[state]);
     }
     builder_.CreateBr(blocks_[continuationBlock]);

src/backend/llvm/emitter.cpp

@@ -29,6 +29,7 @@ FunctionEmitter::FunctionEmitter(LLVMContext &context, Module &module,
       semantics_(options.semantics),
       symbol_suffix_(options.symbol_suffix ? options.symbol_suffix : ""),
       fixed_memory_layout_(options.fixed_memory_layout != 0),
+      state_in_memory_(options.state_in_memory != 0),
       expected_ram_size_(options.ram_size),
       expected_mem2_size_(options.mem2_size) {}
 

src/backend/llvm/emitter.h

@@ -65,6 +65,10 @@ class FunctionEmitter final {
   void scanLoopHeaders();
   void scanRegionLeaders();
   void finalizeStateSSA();
+  bool stateInMemory() const;
+  bool state_in_memory_ = false;
+  static bool slotIsPacked(DolIRStateSlot slot);
+  bool slotInMemory(DolIRStateSlot slot) const;
 
   void emitEntry();
   bool emitWrapper(llvm::raw_ostream &diagnostics);
@@ -190,7 +194,11 @@ class FunctionEmitter final {
   llvm::Value *mem2_size_ = nullptr;
   llvm::BasicBlock *fallback_block_ = nullptr;
   llvm::PHINode *fallback_pc_ = nullptr;
-  std::array<llvm::AllocaInst *, DOLIR_STATE_COUNT> state_{};
+  // Where each guest state slot lives in this function. By default an alloca
+  // that mem2reg promotes; with --state-in-memory a pointer straight into
+  // CPUState, so every load and store site works unchanged either way. Packed
+  // slots always keep an alloca -- they have no standalone storage to point at.
+  std::array<llvm::Value *, DOLIR_STATE_COUNT> state_{};
   std::array<llvm::AllocaInst *, 32> pair_f32_{};
   std::array<llvm::AllocaInst *, 32> pair_f64_{};
   std::array<FPRepresentation, 32> fp_rep_{};

src/backend/llvm/exits.cpp

@@ -27,6 +27,12 @@ void FunctionEmitter::emitEntry() {
     if (!used_[slot])
       continue;
     auto stateSlot = static_cast<DolIRStateSlot>(slot);
+    if (slotInMemory(stateSlot)) {
+      // No alloca and no entry copy: the slot is read and written where it
+      // already lives, inside CPUState.
+      state_[slot] = bytePtr(stateOffset(stateSlot));
+      continue;
+    }
     state_[slot] = builder_.CreateAlloca(type(dolir_state_type(stateSlot)),
                                          nullptr, "state");
   }
@@ -97,6 +103,9 @@ void FunctionEmitter::emitEntry() {
     if (!used_[slot])
       continue;
     auto stateSlot = static_cast<DolIRStateSlot>(slot);
+    // A slot that already points into CPUState needs no prologue copy.
+    if (slotInMemory(stateSlot))
+      continue;
     builder_.CreateStore(loadContext(stateSlot), state_[slot]);
   }
   initializeEntryControls();
@@ -132,6 +141,10 @@ void FunctionEmitter::syncDirtyState() {
     if (!dirty_[slot] || slot == DOLIR_STATE_FPSCR)
       continue;
     auto stateSlot = static_cast<DolIRStateSlot>(slot);
+    // Nothing was hoisted for this slot, so nothing has gone stale; the load
+    // would read a CPUState field and store it straight back to itself.
+    if (slotInMemory(stateSlot))
+      continue;
     storeContext(
         stateSlot,
         builder_.CreateLoad(type(dolir_state_type(stateSlot)), state_[slot]));

src/backend/llvm/llvm_backend.h

@@ -48,6 +48,10 @@ typedef struct {
     u32 ram_size;
     u32 mem2_size;
     u64 partition_seed;
+    /* Keep guest state in CPUState instead of hoisting it into allocas that
+       mem2reg promotes. Changes emitted code, so it participates in the object
+       cache key. */
+    int state_in_memory;
     const DolLLVMFunctionRange* function_ranges;
     u32 function_range_count;
     const u32* entry_points;

src/backend/llvm/register_state.cpp

@@ -10,11 +10,43 @@ namespace dolllvm {
 
 using namespace llvm;
 
+// Guest state can live in CPUState instead of being hoisted into allocas at
+// region entry.
+//
+// Promoting the guest register file gives the register allocator far more
+// simultaneously live values than x86-64 has registers, so it spills them
+// straight back. Measured on GM4E01: 631 state phi nodes across 228 basic
+// blocks in a single chunk, 20+ per block in hot loops. AArch64's 31 GPRs
+// absorb much of that; x86-64's 16 do not.
+//
+// Note that skipping PromoteMemToReg alone achieves nothing: optimizeModule()
+// runs the standard -O2 pipeline before emission and SROA/mem2reg promote the
+// allocas anyway. The allocas have to not exist.
+bool FunctionEmitter::stateInMemory() const { return state_in_memory_; }
+
+// These slots are bitfields inside a wider CPUState word rather than
+// addressable storage: CR0-CR7 are nibbles of cr, XER_CA-XER_SO are bits of
+// xer, and XER itself preserves those flag bits while writing only the low 29.
+// storeContext and loadContext pack and unpack them, so they cannot be pointed
+// at directly and always keep an alloca.
+bool FunctionEmitter::slotIsPacked(DolIRStateSlot slot) {
+  return (slot >= DOLIR_STATE_CR0 && slot <= DOLIR_STATE_CR7) ||
+         (slot >= DOLIR_STATE_XER_CA && slot <= DOLIR_STATE_XER_SO) ||
+         slot == DOLIR_STATE_XER;
+}
+
+bool FunctionEmitter::slotInMemory(DolIRStateSlot slot) const {
+  return stateInMemory() && !slotIsPacked(slot);
+}
+
 void FunctionEmitter::finalizeStateSSA() {
+  // Nothing was hoisted, so there is nothing to promote.
+  if (stateInMemory())
+    return;
   SmallVector<AllocaInst *, DOLIR_STATE_COUNT + 64> registers;
-  for (AllocaInst *slot : state_)
-    if (slot)
-      registers.push_back(slot);
+  for (Value *slot : state_)
+    if (auto *alloca = dyn_cast_or_null<AllocaInst>(slot))
+      registers.push_back(alloca);
   for (AllocaInst *pair : pair_f32_)
     if (pair)
       registers.push_back(pair);

src/backend/llvm/runtime.cpp

@@ -32,6 +32,10 @@ void FunctionEmitter::reloadState(DolIRStateSlot slot) {
   known_state_[slot] = nullptr;
   if (slot == DOLIR_STATE_FPSCR)
     pending_fprf_ = nullptr;
+  // The helper wrote CPUState directly and the slot points there, so the value
+  // is already current; only the cached constant needed clearing.
+  if (slotInMemory(slot))
+    return;
   builder_.CreateStore(loadContext(slot), state_[slot]);
 }