diff --git a/CMakeLists.txt b/CMakeLists.txt index dbba0e8..68c999f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,17 +135,27 @@ if(DOLRECOMP_ENABLE_LLVM) MC native nativecodegen - # Specific target backends required by target.cpp - X86CodeGen - X86AsmParser - AArch64CodeGen - AArch64AsmParser - ARMCodeGen - ARMAsmParser - RISCVCodeGen - RISCVAsmParser + # Target backends are appended below, gated on what this LLVM ships. ) + # target.cpp registers only the targets it can emit for, and only those + # this LLVM was built with. Naming them unconditionally breaks two ways: + # against an LLVM without one of them the call does not even compile, and + # registering every target via InitializeAll* drags unused backends into + # the link. LLVM_TARGETS_TO_BUILD is what LLVMConfig.cmake reports. + if("X86" IN_LIST LLVM_TARGETS_TO_BUILD) + list(APPEND DOLRECOMP_LLVM_COMPONENTS X86CodeGen X86AsmParser) + target_compile_definitions(dr_llvm PRIVATE DOLLLVM_HAVE_X86_TARGET=1) + endif() + if("AArch64" IN_LIST LLVM_TARGETS_TO_BUILD) + list(APPEND DOLRECOMP_LLVM_COMPONENTS AArch64CodeGen AArch64AsmParser) + target_compile_definitions(dr_llvm PRIVATE DOLLLVM_HAVE_AARCH64_TARGET=1) + endif() + if(NOT "X86" IN_LIST LLVM_TARGETS_TO_BUILD AND NOT "AArch64" IN_LIST LLVM_TARGETS_TO_BUILD) + message(FATAL_ERROR + "DolRecomp's LLVM backend needs the X86 or AArch64 target; this LLVM has: ${LLVM_TARGETS_TO_BUILD}") + endif() + if(TARGET LLVM) target_link_libraries(dr_llvm PRIVATE LLVM) elseif(MINGW) diff --git a/benchmarks/llvm_backend_bench.c b/benchmarks/llvm_backend_bench.c index e56bde9..3a2450f 100644 --- a/benchmarks/llvm_backend_bench.c +++ b/benchmarks/llvm_backend_bench.c @@ -5,6 +5,10 @@ #include #include +#if defined(_WIN32) +#include +#endif + void func_80003100(CPUState *cpu); void func_80003500(CPUState *cpu); void func_80003750(CPUState *cpu); @@ -33,9 +37,17 @@ static bool region_query(CPUState *cpu, u32 address) { } static double seconds(void) { +#if defined(_WIN32) + LARGE_INTEGER frequency; + LARGE_INTEGER now; + QueryPerformanceFrequency(&frequency); + QueryPerformanceCounter(&now); + return (double)now.QuadPart / (double)frequency.QuadPart; +#else struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); return (double)now.tv_sec + (double)now.tv_nsec * 1e-9; +#endif } static void prepare(CPUState *cpu, u32 pc) { diff --git a/src/app/cli.c b/src/app/cli.c index 648dfd8..896526e 100644 --- a/src/app/cli.c +++ b/src/app/cli.c @@ -16,6 +16,8 @@ void print_usage(const char* argv0) { fprintf(stderr, " -jN Use N worker jobs for split C output (e.g. -j14)\n"); fprintf(stderr, " --cpu gekko|broadway|espresso Select CPU profile (default: broadway)\n"); fprintf(stderr, " --backend c|llvm Select generated-code backend (default: c)\n"); + fprintf(stderr, " --state-in-memory Keep guest state in CPUState instead of\n"); + fprintf(stderr, " hoisting it into promoted allocas\n"); fprintf(stderr, " --targets host, x86-64-v2, x86-64-v3, aarch64, aarch64-a57\n"); fprintf(stderr, " --semantics exact|fast PowerPC floating-point semantics (default: exact)\n"); fprintf(stderr, " --instrumentation none|lockstep Compile release or state-journal objects\n"); @@ -191,6 +193,14 @@ int parse_cli(int argc, char** argv, CliOptions* opts) { opts->gamecube_mode = 1; continue; } + if (strcmp(arg, "--state-in-memory") == 0) { + opts->state_in_memory = 1; + continue; + } + if (strcmp(arg, "--no-state-in-memory") == 0) { + opts->state_in_memory = 0; + continue; + } if (strcmp(arg, "--backend") == 0) { if (i + 1 >= argc) { diff --git a/src/app/cli.h b/src/app/cli.h index 77f867a..a088f1f 100644 --- a/src/app/cli.h +++ b/src/app/cli.h @@ -30,6 +30,7 @@ typedef struct { int setup_mode; int show_help; int fast_semantics; + int state_in_memory; int lockstep_instrumentation; } CliOptions; diff --git a/src/app/pipeline.c b/src/app/pipeline.c index 3306611..98e8f11 100644 --- a/src/app/pipeline.c +++ b/src/app/pipeline.c @@ -73,6 +73,7 @@ typedef struct { const char* profile_generate_path; const char* profile_use_path; u64 partition_seed; + int state_in_memory; u32 ram_size; u32 mem2_size; char symbol_suffix[32]; @@ -260,6 +261,11 @@ static u64 llvm_job_hash(const LLVMChunkJob* job) { hash = hash_bytes(hash, &job->semantics, sizeof(job->semantics)); hash = hash_bytes(hash, &job->instrumentation, sizeof(job->instrumentation)); + /* Changes the emitted code, so a plan built with it must not collide with + one built without it. Omitting this is how a toggled codegen option + silently returns another configuration's objects. */ + hash = hash_bytes(hash, &job->state_in_memory, + sizeof(job->state_in_memory)); hash = hash_bytes(hash, &job->partition_seed, sizeof(job->partition_seed)); hash = hash_bytes(hash, &job->ram_size, sizeof(job->ram_size)); @@ -465,6 +471,7 @@ static int emit_llvm_chunk_job(const void* data, void* user) { options.profile_generate_path = job->profile_generate_path; options.profile_use_path = job->profile_use_path; options.partition_seed = job->partition_seed; + options.state_in_memory = job->state_in_memory; options.emit_thinlto = 1; options.thinlto_path = job->thinlto_path; options.fixed_memory_layout = 1; @@ -880,6 +887,7 @@ static int emit_code_sections_llvm(const LoadedCodeSection* sections, options->profile_generate_path; target_job->profile_use_path = options->profile_use_path; target_job->partition_seed = options->partition_seed; + target_job->state_in_memory = options->state_in_memory; target_job->ram_size = GC_MAIN_RAM_SIZE; target_job->mem2_size = cpu == DOLRECOMP_CPU_GEKKO ? 0u diff --git a/src/backend/llvm/branch_targets.cpp b/src/backend/llvm/branch_targets.cpp index 37e8421..b0bb431 100644 --- a/src/backend/llvm/branch_targets.cpp +++ b/src/backend/llvm/branch_targets.cpp @@ -93,6 +93,8 @@ BasicBlock *FunctionEmitter::externalDestination(const DolIRTerminator &term, if (!used_[state]) continue; auto stateSlot = static_cast(state); + if (slotInMemory(stateSlot)) + continue; builder_.CreateStore(loadContext(stateSlot), state_[state]); } builder_.CreateBr(blocks_[continuationBlock]); diff --git a/src/backend/llvm/emitter.cpp b/src/backend/llvm/emitter.cpp index b28bea4..b763592 100644 --- a/src/backend/llvm/emitter.cpp +++ b/src/backend/llvm/emitter.cpp @@ -29,6 +29,7 @@ FunctionEmitter::FunctionEmitter(LLVMContext &context, Module &module, semantics_(options.semantics), symbol_suffix_(options.symbol_suffix ? options.symbol_suffix : ""), fixed_memory_layout_(options.fixed_memory_layout != 0), + state_in_memory_(options.state_in_memory != 0), expected_ram_size_(options.ram_size), expected_mem2_size_(options.mem2_size) {} diff --git a/src/backend/llvm/emitter.h b/src/backend/llvm/emitter.h index dbce5d9..8e08132 100644 --- a/src/backend/llvm/emitter.h +++ b/src/backend/llvm/emitter.h @@ -65,6 +65,10 @@ class FunctionEmitter final { void scanLoopHeaders(); void scanRegionLeaders(); void finalizeStateSSA(); + bool stateInMemory() const; + bool state_in_memory_ = false; + static bool slotIsPacked(DolIRStateSlot slot); + bool slotInMemory(DolIRStateSlot slot) const; void emitEntry(); bool emitWrapper(llvm::raw_ostream &diagnostics); @@ -190,7 +194,11 @@ class FunctionEmitter final { llvm::Value *mem2_size_ = nullptr; llvm::BasicBlock *fallback_block_ = nullptr; llvm::PHINode *fallback_pc_ = nullptr; - std::array state_{}; + // Where each guest state slot lives in this function. By default an alloca + // that mem2reg promotes; with --state-in-memory a pointer straight into + // CPUState, so every load and store site works unchanged either way. Packed + // slots always keep an alloca -- they have no standalone storage to point at. + std::array state_{}; std::array pair_f32_{}; std::array pair_f64_{}; std::array fp_rep_{}; diff --git a/src/backend/llvm/exits.cpp b/src/backend/llvm/exits.cpp index 9773699..b273c57 100644 --- a/src/backend/llvm/exits.cpp +++ b/src/backend/llvm/exits.cpp @@ -27,6 +27,12 @@ void FunctionEmitter::emitEntry() { if (!used_[slot]) continue; auto stateSlot = static_cast(slot); + if (slotInMemory(stateSlot)) { + // No alloca and no entry copy: the slot is read and written where it + // already lives, inside CPUState. + state_[slot] = bytePtr(stateOffset(stateSlot)); + continue; + } state_[slot] = builder_.CreateAlloca(type(dolir_state_type(stateSlot)), nullptr, "state"); } @@ -97,6 +103,9 @@ void FunctionEmitter::emitEntry() { if (!used_[slot]) continue; auto stateSlot = static_cast(slot); + // A slot that already points into CPUState needs no prologue copy. + if (slotInMemory(stateSlot)) + continue; builder_.CreateStore(loadContext(stateSlot), state_[slot]); } initializeEntryControls(); @@ -132,6 +141,10 @@ void FunctionEmitter::syncDirtyState() { if (!dirty_[slot] || slot == DOLIR_STATE_FPSCR) continue; auto stateSlot = static_cast(slot); + // Nothing was hoisted for this slot, so nothing has gone stale; the load + // would read a CPUState field and store it straight back to itself. + if (slotInMemory(stateSlot)) + continue; storeContext( stateSlot, builder_.CreateLoad(type(dolir_state_type(stateSlot)), state_[slot])); diff --git a/src/backend/llvm/instructions.cpp b/src/backend/llvm/instructions.cpp index 1e7f36a..e74e81f 100644 --- a/src/backend/llvm/instructions.cpp +++ b/src/backend/llvm/instructions.cpp @@ -101,6 +101,38 @@ bool FunctionEmitter::emitInstruction(const DolIRInstruction &inst, result = stateValue(static_cast(inst.aux)); break; case DOLIR_OP_STATE_WRITE: + if (static_cast(inst.aux) == DOLIR_STATE_MSR) { + // MSR[EE] going 0->1 is an interrupt delivery point. The interpreter + // and the C backend reach one at every block boundary, but generated + // code here runs whole critical sections between dispatcher visits, + // and the budget guards that end a burst sit at call sites -- which + // in OS code are almost all inside interrupt-disabled windows. A + // guest that waits by yield-spinning (enable, check, disable, + // reschedule) then never presents an enabled window at a dispatch + // boundary, and pending external interrupts starve: Colosseum + // renders one frame per 3-second timeout that way. Hand control + // back to the dispatcher whenever mtmsr enables EE, exactly like + // the block-ending JITs do. + Value *oldMSR = builder_.CreateLoad( + type(dolir_state_type(DOLIR_STATE_MSR)), state_[inst.aux]); + builder_.CreateStore(operand(inst, 0), state_[inst.aux]); + noteStateWrite(DOLIR_STATE_MSR, operand(inst, 0)); + materializeFPRF(); + Value *enabling = builder_.CreateAnd( + builder_.CreateAnd(builder_.CreateNot(oldMSR), operand(inst, 0)), + builder_.getInt32(0x8000)); + BasicBlock *eeExit = + BasicBlock::Create(context_, "msr_ee_exit", function_); + BasicBlock *eeCont = + BasicBlock::Create(context_, "msr_ee_cont", function_); + builder_.CreateCondBr( + builder_.CreateICmpNE(enabling, builder_.getInt32(0)), eeExit, + eeCont); + builder_.SetInsertPoint(eeExit); + sideExit(inst.guest_pc + 4u); + builder_.SetInsertPoint(eeCont); + break; + } builder_.CreateStore(operand(inst, 0), state_[inst.aux]); noteStateWrite(static_cast(inst.aux), operand(inst, 0)); break; diff --git a/src/backend/llvm/llvm_backend.h b/src/backend/llvm/llvm_backend.h index 71790a3..2cc37b1 100644 --- a/src/backend/llvm/llvm_backend.h +++ b/src/backend/llvm/llvm_backend.h @@ -48,6 +48,10 @@ typedef struct { u32 ram_size; u32 mem2_size; u64 partition_seed; + /* Keep guest state in CPUState instead of hoisting it into allocas that + mem2reg promotes. Changes emitted code, so it participates in the object + cache key. */ + int state_in_memory; const DolLLVMFunctionRange* function_ranges; u32 function_range_count; const u32* entry_points; diff --git a/src/backend/llvm/register_state.cpp b/src/backend/llvm/register_state.cpp index 8de0335..b0dd77b 100644 --- a/src/backend/llvm/register_state.cpp +++ b/src/backend/llvm/register_state.cpp @@ -10,11 +10,43 @@ namespace dolllvm { using namespace llvm; +// Guest state can live in CPUState instead of being hoisted into allocas at +// region entry. +// +// Promoting the guest register file gives the register allocator far more +// simultaneously live values than x86-64 has registers, so it spills them +// straight back. Measured on GM4E01: 631 state phi nodes across 228 basic +// blocks in a single chunk, 20+ per block in hot loops. AArch64's 31 GPRs +// absorb much of that; x86-64's 16 do not. +// +// Note that skipping PromoteMemToReg alone achieves nothing: optimizeModule() +// runs the standard -O2 pipeline before emission and SROA/mem2reg promote the +// allocas anyway. The allocas have to not exist. +bool FunctionEmitter::stateInMemory() const { return state_in_memory_; } + +// These slots are bitfields inside a wider CPUState word rather than +// addressable storage: CR0-CR7 are nibbles of cr, XER_CA-XER_SO are bits of +// xer, and XER itself preserves those flag bits while writing only the low 29. +// storeContext and loadContext pack and unpack them, so they cannot be pointed +// at directly and always keep an alloca. +bool FunctionEmitter::slotIsPacked(DolIRStateSlot slot) { + return (slot >= DOLIR_STATE_CR0 && slot <= DOLIR_STATE_CR7) || + (slot >= DOLIR_STATE_XER_CA && slot <= DOLIR_STATE_XER_SO) || + slot == DOLIR_STATE_XER; +} + +bool FunctionEmitter::slotInMemory(DolIRStateSlot slot) const { + return stateInMemory() && !slotIsPacked(slot); +} + void FunctionEmitter::finalizeStateSSA() { + // Nothing was hoisted, so there is nothing to promote. + if (stateInMemory()) + return; SmallVector registers; - for (AllocaInst *slot : state_) - if (slot) - registers.push_back(slot); + for (Value *slot : state_) + if (auto *alloca = dyn_cast_or_null(slot)) + registers.push_back(alloca); for (AllocaInst *pair : pair_f32_) if (pair) registers.push_back(pair); diff --git a/src/backend/llvm/runtime.cpp b/src/backend/llvm/runtime.cpp index 218f2eb..f37e268 100644 --- a/src/backend/llvm/runtime.cpp +++ b/src/backend/llvm/runtime.cpp @@ -32,6 +32,10 @@ void FunctionEmitter::reloadState(DolIRStateSlot slot) { known_state_[slot] = nullptr; if (slot == DOLIR_STATE_FPSCR) pending_fprf_ = nullptr; + // The helper wrote CPUState directly and the slot points there, so the value + // is already current; only the cached constant needed clearing. + if (slotInMemory(slot)) + return; builder_.CreateStore(loadContext(slot), state_[slot]); } diff --git a/src/backend/llvm/target.cpp b/src/backend/llvm/target.cpp index 8e177c0..2bdaa32 100644 --- a/src/backend/llvm/target.cpp +++ b/src/backend/llvm/target.cpp @@ -45,11 +45,26 @@ static const ProfileDefinition *definition(DolLLVMTargetProfile id) { void initializeTargets() { static const bool once = [] { - InitializeAllTargetInfos(); - InitializeAllTargets(); - InitializeAllTargetMCs(); - InitializeAllAsmPrinters(); - InitializeAllAsmParsers(); + // Only the targets DolRecomp can emit, and only those the host LLVM + // actually ships. InitializeAll* expands to every target this LLVM was + // configured with, which forces the link to carry backends the emitter + // never uses; naming X86 and AArch64 unconditionally instead would fail to + // compile against an LLVM built without one of them. CMake derives these + // two macros from LLVM_TARGETS_TO_BUILD, so this tracks the host build. +#if defined(DOLLLVM_HAVE_X86_TARGET) + LLVMInitializeX86TargetInfo(); + LLVMInitializeX86Target(); + LLVMInitializeX86TargetMC(); + LLVMInitializeX86AsmPrinter(); + LLVMInitializeX86AsmParser(); +#endif +#if defined(DOLLLVM_HAVE_AARCH64_TARGET) + LLVMInitializeAArch64TargetInfo(); + LLVMInitializeAArch64Target(); + LLVMInitializeAArch64TargetMC(); + LLVMInitializeAArch64AsmPrinter(); + LLVMInitializeAArch64AsmParser(); +#endif return true; }(); (void)once;