diff --git a/CMakeLists.txt b/CMakeLists.txt index 8848a0c..c987df5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,6 +53,9 @@ endif() set(DOLRECOMP_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src") +add_library(dr_common STATIC src/common/perf.c src/common/options.c) +target_include_directories(dr_common PUBLIC ${DOLRECOMP_SRC}) + add_library(dr_cpu STATIC src/cpu/cpu.c) target_include_directories(dr_cpu PUBLIC ${DOLRECOMP_SRC}) if(NOT WIN32) @@ -99,7 +102,7 @@ if(DOLRECOMP_ENABLE_LLVM) target_include_directories(dr_llvm SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) target_compile_definitions(dr_llvm PUBLIC DOLRECOMP_ENABLE_LLVM=1) target_compile_definitions(dr_llvm PRIVATE ${LLVM_DEFINITIONS}) - target_link_libraries(dr_llvm PUBLIC dr_ir) + target_link_libraries(dr_llvm PUBLIC dr_ir dr_common) if(MINGW) find_library(DOLRECOMP_LLVM_SHARED_IMPORT NAMES LLVM-${LLVM_VERSION_MAJOR} LLVM @@ -123,6 +126,8 @@ if(DOLRECOMP_ENABLE_LLVM) endif() add_library(dr_analysis STATIC + src/analysis/cfg.c + src/analysis/regions.c src/analysis/embedded_data.c src/analysis/smc.c src/analysis/symbol_map.c @@ -138,7 +143,7 @@ add_library(dr_backend STATIC src/backend/symbols.c ) target_include_directories(dr_backend PUBLIC ${DOLRECOMP_SRC}) -target_link_libraries(dr_backend PUBLIC dr_frontend dr_analysis) +target_link_libraries(dr_backend PUBLIC dr_frontend dr_analysis dr_common) if(NOT WIN32) target_link_libraries(dr_backend PUBLIC Threads::Threads) endif() @@ -151,7 +156,7 @@ add_library(dr_app STATIC src/app/pipeline.c ) target_include_directories(dr_app PUBLIC ${DOLRECOMP_SRC}) -target_link_libraries(dr_app PUBLIC dr_backend dr_analysis dr_frontend dr_platform dr_ir) +target_link_libraries(dr_app PUBLIC dr_backend dr_analysis dr_frontend dr_platform dr_ir dr_common) if(DOLRECOMP_ENABLE_LLVM) target_link_libraries(dr_app PUBLIC dr_llvm) endif() @@ -169,6 +174,9 @@ endif() add_executable(dolir_stats tools/dolir_stats.c) target_link_libraries(dolir_stats PRIVATE dr_ir dr_frontend dr_analysis) +add_executable(cfg_stats tools/cfg_stats.c) +target_link_libraries(cfg_stats PRIVATE dr_analysis dr_frontend) + enable_testing() add_executable(test_opcodes tests/test_opcodes.c) @@ -261,6 +269,18 @@ add_executable(test_dolir tests/test_dolir.c) target_link_libraries(test_dolir PRIVATE dr_ir) add_test(NAME dolir COMMAND test_dolir) +add_executable(test_cfg tests/test_cfg.c) +target_link_libraries(test_cfg PRIVATE dr_analysis) +add_test(NAME cfg COMMAND test_cfg) + +add_executable(test_regions tests/test_regions.c) +target_link_libraries(test_regions PRIVATE dr_analysis) +add_test(NAME regions COMMAND test_regions ${CMAKE_CURRENT_BINARY_DIR}/regions_test) + +add_executable(test_perf tests/test_perf.c) +target_link_libraries(test_perf PRIVATE dr_common) +add_test(NAME perf COMMAND test_perf ${CMAKE_CURRENT_BINARY_DIR}/perf_test) + if(DOLRECOMP_ENABLE_LLVM) add_executable(test_llvm_backend tests/test_llvm_backend.cpp) target_link_libraries(test_llvm_backend PRIVATE dr_llvm) @@ -280,6 +300,42 @@ if(DOLRECOMP_ENABLE_LLVM) add_executable(test_llvm_execute tests/test_llvm_execute.c ${DOLRECOMP_TEST_OBJECT}) target_link_libraries(test_llvm_execute PRIVATE dr_cpu) add_test(NAME llvm_execute COMMAND test_llvm_execute) + # Differential harness: the same random guest code through both backends, + # emitted at different guest bases so the two sets of func_
+ # symbols coexist in one comparing binary. + add_executable(gen_differential tests/differential/gen_differential.cpp) + target_link_libraries(gen_differential PRIVATE dr_llvm dr_backend dr_ir) + set(DIFF_C ${CMAKE_CURRENT_BINARY_DIR}/differential_generated.c) + set(DIFF_O ${CMAKE_CURRENT_BINARY_DIR}/differential_generated.o) + set(DIFF_H ${CMAKE_CURRENT_BINARY_DIR}/differential_manifest.h) + # DOLRECOMP_DIFF_SEED lets CI sweep seeds without editing anything; the + # default is fixed so an ordinary run is reproducible. + if(NOT DEFINED DOLRECOMP_DIFF_SEED) + set(DOLRECOMP_DIFF_SEED 20260812) + endif() + # Overridable so a failure can be shrunk to a minimal repro without editing. + if(NOT DEFINED DOLRECOMP_DIFF_FUNCTIONS) + set(DOLRECOMP_DIFF_FUNCTIONS 64) + endif() + if(NOT DEFINED DOLRECOMP_DIFF_LENGTH) + set(DOLRECOMP_DIFF_LENGTH 24) + endif() + add_custom_command( + OUTPUT ${DIFF_C} ${DIFF_O} ${DIFF_H} + COMMAND gen_differential ${DIFF_C} ${DIFF_O} ${DIFF_H} + ${DOLRECOMP_DIFF_SEED} ${DOLRECOMP_DIFF_FUNCTIONS} ${DOLRECOMP_DIFF_LENGTH} + DEPENDS gen_differential + VERBATIM + ) + set_source_files_properties(${DIFF_O} + PROPERTIES GENERATED TRUE EXTERNAL_OBJECT TRUE) + add_executable(test_differential + tests/test_differential.c ${DIFF_C} ${DIFF_O} ${DIFF_H}) + target_include_directories(test_differential PRIVATE + ${CMAKE_CURRENT_BINARY_DIR} ${DOLRECOMP_SRC}) + target_link_libraries(test_differential PRIVATE dr_cpu) + add_test(NAME differential COMMAND test_differential) + add_executable(test_llvm_pipeline tests/test_llvm_pipeline.c) target_include_directories(test_llvm_pipeline PRIVATE ${DOLRECOMP_SRC}) add_test(NAME llvm_pipeline COMMAND test_llvm_pipeline diff --git a/docs/AOT-ENGINEERING-REPORT.md b/docs/AOT-ENGINEERING-REPORT.md new file mode 100644 index 0000000..d74abc3 --- /dev/null +++ b/docs/AOT-ENGINEERING-REPORT.md @@ -0,0 +1,503 @@ +# DolRecomp AOT 2.0 — Engineering Report + +Branch `feature/llvm-aot-regions`, 73 commits on upstream +`fa0cf619e8d7eb8cba7eaf55267a12caaebb46aa`. + +Every number here was measured on this host. Negative and retracted results are +included; nothing is extrapolated. + +Method: each configuration is built into its own directory (the module cache +keys on backend and binary hash, not on region settings, so two configurations +sharing an output directory silently collide). Throughput is frames over wall +time with Dolphin's throttle disabled, since `fps` in `status.txt` stays 0 +headless and pins at 1.00 windowed. Arms alternate and are compared pairwise +with a sign test, dropping any run whose `cycles_per_frame` or +`bursts_per_mcycle` strays from the median -- those executed a different scene. +Titles are supplied locally and none is committed. + +--- + +## 1. Headline + +Two targets, two different stories, and the second one is not a footnote. + +**On x86-64** the region backend began **60-70% behind the C backend** on Mario +Kart and ended at **parity with it**, on a module **4.9x smaller** than the one +it started with and builds **19x faster**. + +| Mario Kart, one pinned scene, x86-64 | fps | module | build | +|---|---:|---:|---:| +| fixed-chunk `llvm` (the shipping LLVM path) | 29.80 | 320.0 MB | 351 s | +| `llvm-aot` as first built | 33.24 | 424.1 MB | ~930 s | +| **`llvm-aot` as it now stands** | **53.49** | **85.8 MB** | **44 s** | +| C backend (the semantic reference) | 50.63-53.01 | 65.3 MB | — | + +Two changes produced essentially all of it. Everything else measured flat. + +Profile-guided optimisation then adds **+5.6% to +18.9%** on top, validated on +three titles and 34 of 36 paired runs (p = 1.9e-08), two of them with held-out +measurement scenes. It needs a per-title profile, so it is a build-pipeline +step rather than a default (2.3). + +**On AArch64 it is about 2x behind Dolphin's own JIT**, and that gap is the +honest headline for that target. Measured on a Raspberry Pi 4, Luigi's Mansion +in the mansion foyer, single core, five alternating pairs per comparison with +`CPUThread` pinned per run: + +| Luigi's Mansion, mansion foyer, AArch64 | fps | vs JIT | +|---|---:|---:| +| `llvm-aot` as this branch stands | 4.76 | 2.77x behind | +| with alias metadata (2.4) | 5.48 | 2.41x behind | +| with alias metadata and a per-title profile | 6.71 | 1.97x behind | +| Dolphin's `JitArm64`, same scene | **13.20** | — | + +Nothing structural explains that gap: the JIT keeps guest registers in host +registers across a block, decrements its cycle counter once per block, and uses +fastmem instead of a bounds check per access. Section 7 has the measurements +behind each. The x86-64 result above does not transfer to this target, and +neither does its parity-with-C conclusion -- the AArch64 comparison against the +C backend is 5.69 against 5.86, a 3.0% deficit rather than parity. + +--- + +## 2. What actually worked + +### 2.1 Guest state stays in `CPUState` + +The emitter used to promote every guest slot it touched to an alloca at region +entry. That hands the register allocator far more simultaneously-live values +than x86-64 has registers, so it spilled them straight back to the stack — +replacing "load from `CPUState` when needed" with "load at entry, store to +stack, reload from stack": one extra copy, plus a large frame. + +| instructions touching the stack | | +|---|---:| +| promoting `llvm-aot` | 29.1-33.5% | +| C backend | 3.2-5.0% | +| **after the change** | **2.5%** | + +| | fps | pairs | sign test | +|---|---:|---:|---:| +| Mario Kart | **+60.9%** | — | parity with C backend | +| Luigi's Mansion | **+26.7%** | 6/6 | p = 0.0312 | +| Skyward Sword | **+30.9%** | 13/13 | p = 0.0002 | + +Gains order by how much spill each title had to lose: Mario Kart's promoting +module was the largest and gains most, Luigi's Mansion the smallest and gains +least. That is what the mechanism predicts, and it is the reason to believe the +mechanism rather than only the outcome. + +### 2.2 `--memory-mode fast` + +`ram_size` is `GC_MAIN_RAM_SIZE` in this tree and in GXRuntime — assigned once, +carried across reset, never given another value — so the MEM1 bound folds to a +constant and the bounds check collapses to a single compare. The write journal +is null unless a runtime installs one, so its branch leaves the store path. + ++6.7% / +6.7% / +5.0% fps across the three titles, 43 of 49 paired runs, +**p = 5.7e-08** combined. + +Both assumptions are **verified once at dispatch entry**, not assumed. If either +fails the module refuses to run natively and the chassis keeps interpreting, so +a violated assumption costs speed and never guest memory. + +### 2.3 Profile-guided optimisation + +`DOLRECOMP_LLVM_PGO=gen` instruments, a run writes a `.profraw`, and +`DOLRECOMP_LLVM_PGO=use` with `DOLRECOMP_LLVM_PROFILE` applies the merged +profile. Codegen PGO had never been measured before this -- the only PGO +previously tested was region *seeding*, which is unrelated and was a dead end. + +| title | scene design | fps | pairs | sign test | +|---|---|---|---|---| +| Mario Kart | **held out** (5 courses profiled, 2 measured) | +12.5% / +18.9% | 14/16 | p = 0.0042 | +| Luigi's Mansion | **held out** (`foyer` profiled, `bench` measured) | **+5.6%** | 10/10 | p = 0.0020 | +| Skyward Sword | same scene (only one gameplay state exists) | **+11.9%** | 10/10 | p = 0.0020 | + +Combined **34 of 36 pairs, p = 1.9e-08**, `fallback` 0 on every run. + +It carries to AArch64, measured the same way on a Raspberry Pi 4 with +`CPUThread` pinned per run, on top of the alias metadata and MEM1 hoist: + +| title, AArch64 | fps | pairs | +|---|---:|---:| +| Luigi's Mansion, mansion foyer | 5.50 to 6.71, **+21.9%** | 5/5 | +| Pokemon Colosseum, intro sequence | 15.69 to 19.65, **+25.2%** | 5/5 | + +Both are **same-scene** and therefore upper bounds, unlike the held-out x86-64 +figures above -- the honest comparison for Luigi's Mansion is the +5.6% held-out +row, not these. One Colosseum pair was discarded for sampling a scene +transition: fps standard deviation 3.24 and 6.06 against 0.08-0.23 everywhere +else. Including it the figure is +27.9%, so the exclusion does not carry the +result. + +Two of the three use held-out measurement scenes, so generalisation is measured +rather than assumed. Skyward Sword could not be: its only savestates are +`gameplay` and `title`, and a title screen shares almost no code with gameplay. + +The spread tracks module size, as block placement predicts and matching the +ordering seen in 2.1: Luigi's Mansion is the smallest module and gains least. + +It pays more here than it would have before 2.1, because with the spill gone +what remains is dominated by the MEM1/MEM2/slow-path branch chain that every +guest load and store walks. + +**Toolchain note.** The profile runtime must come from the same LLVM that +instruments. A system clang newer than the backend's LLVM produces a `.profraw` +the backend refuses to read, and the error appears at the *use* build, long +after the profiling run is over. `build_module.sh` derives it from `LLVM_DIR`. + +Not a default in the sense the other options are, since it needs a per-title +profile. Unmeasured: whether the gain holds on a scene much heavier than +anything in the profile set. + +### 2.4 Guest RAM and `CPUState` declared disjoint + +Every guest register access goes through `CPUState`, and guest memory is +reached through a pointer loaded out of `CPUState`. Nothing told LLVM the two +are different objects, so a guest store looked like it might clobber `cr` or a +gpr, and the next instruction that read one reloaded it. + +Two alias scopes in one domain say otherwise: `CPUState` accesses are tagged +`cpustate`/noalias-`guestmem`, guest RAM accesses the reverse. Every `CPUState` +access already funnels through `bytePtr()` and every guest RAM access through +`endianLoad`/`endianStore`, so this is four tag sites. Anything untagged -- +helper calls, MMIO, external reads and writes -- stays conservative and +may-alias, which is what keeps those paths correct without enumerating them. +The two budget counters take `NoAlias` as well: they are allocas created in the +wrapper and passed only to the body, but the guard updates them inside the +hottest loops. + +**+15.0% on AArch64** (Luigi's Mansion foyer, single core, 5 of 5 pairs), and +the module is 1.44 MB smaller -- redundant loads and stores going away rather +than a layout accident. Unmeasured on x86-64, where the same reasoning applies +but the register file is not the constraint it is here. + +It generalises, and by more than the title it came from. Pokemon Colosseum +shares no code with Luigi's Mansion and was never profiled while developing +this; it gains **+22.6%** from this change and the MEM1 hoist (2.5) together, +5 of 5 pairs, on a module 5.2% smaller: + +| Colosseum, intro sequence, single core | fps | +|---|---:| +| before both changes | 12.84 | +| after both changes | **15.74** | + +Luigi's Mansion gains about 17% from the same pair. Frame sizes match across +arms at ~1.38 MB and guest cycles rise from 21.1 to 26.4 billion, so this is +more guest work per wall-second rather than a lighter scene. Deriving both +changes from one title's profile did not fit them to it. + +The change had to be made three times before it measured anything. The object +cache does not hash the emitter source, so the first two attempts returned +byte-identical objects and would have been written up as "no effect" had the +hashes not been checked. `DOLLLVM_CACHE_VERSION` exists for this and has now +bitten four times. + +### 2.5 MEM1's base loaded once per region + +Every guest memory access loaded `ctx->ram` before indexing it. The pointer is +fixed for the whole burst -- the chassis sets it before dispatch and nothing the +module can call reallocates MEM1 -- but LLVM cannot hoist that load, because the +alias scope in 2.4 covers all of `CPUState` as one blob and a store to `cr` or a +gpr blocks it. + +Loading it once in the prologue is **+1.7%** on Luigi's Mansion (5 of 5 pairs) +and 590 KB off the module. Modest on its own because LLVM already CSEs these +loads within a block; what this adds is the hoist across blocks and loops, which +it could not prove for itself. + +MEM2 deliberately keeps its per-access load. `cpu_alloc_mem2` can allocate exram +after a region has started running, so caching that pointer would be a bug +rather than a missed optimisation. + +--- + +## 3. What did not work + +Ten interventions measured flat or negative. They are listed because the pattern +is the finding. + +| | result | +|---|---| +| Larger regions (256/512/1024) | dispatcher rate flat within 1%, 33 runs | +| PGO-driven region formation | plans a different program, moves nothing | +| Static crossing count as a proxy | falls 21% while runtime rate moves 0.8% | +| `bctr` / jump-table specialisation | 0.17% of weighted execution | +| Address-adjacency merging | 2.2x build time, +6.3% size | +| Barrier store narrowing | −4.3% size for +50% build; two earlier versions unsound | +| Emitter-level cross-region inlining | +0.017% module size | +| ThinLTO (`--lto thin`) | ~6% smaller, runtime effect **title-dependent** | +| Register-passed GPR3-GPR10 | **−2.3% fps**, +6.1% size | +| LLVM's own `-O3` pipeline | module +0.07%, spill traffic **unchanged** | + +Most of these reshape control flow that was already direct calls. They were +rearranging a structure whose dominant cost was the structure itself. The `-O3` +result is the cleanest proof: swapping in the exact pipeline clang uses changed +the spill ratio by nothing, because no pass can undo a live set larger than the +machine. + +**ThinLTO is the one to be careful about.** It is ~6% smaller on both titles +tested, but layered on top of the memory mode it *costs* 4.1% on Luigi's Mansion +and *gains* 2.5% on Mario Kart — both statistically significant, in opposite +directions. It stays off by default, and a per-title measurement is the only way +to know which side a given title falls on. + +--- + +## 4. The measurement, which had to be fixed twice + +Three claims were made and retracted during this work. The corrections matter +more than the claims. + +* **A −22.1% dispatcher improvement** came from two Luigi's Mansion arms running + different scenes. Retracted; a comparability guard on cycles/frame was added. +* **A −12% size / −23% build win** from barrier narrowing was reported before + the benchmark returned. The module did not run — Mario Kart hung at boot. + Real, and worthless. +* **A +20.8% C-versus-LLVM figure** was assembled from noise: the same-backend + comparability filter is invalid across backends (`bursts/Mcycle` differs + because 182 chunks is not 2,033 regions; `cycles/frame` differs because the + backends charge guest cycles differently). Applied naively it kept two + outliers and left one arm at n=1. The real figure was +59.5%. + +Two guards came out of this and are now in the tooling: + +* **Outlier rejection on the invariants, not on fps.** A run whose + `cycles_per_frame` or `bursts_per_mcycle` strays from the median executed a + different scene and is dropped. One Luigi's Mansion run read **134 fps** at + 92.6 `bursts/Mcycle` against everyone else's 153.8 — a different execution, + not a fast one. Including it moved a −4.3% result to +46.4%. +* **Pairwise comparison with a sign test.** Arms alternate, so run *i* of each + saw the same machine state; comparing within pairs cancels the drift behind + the 17-25% unpaired spreads. The unpaired 2x-spread guard is right for + unpaired means and far too blunt for paired runs; where the two disagree, + both are stated. +* Neither invariant survives crossing *backends* — the C module has 182 chunks + against 2,033 regions, and the backends charge guest cycles differently — so + cross-backend comparisons reject outliers within each arm against its own + median instead. + +Measuring on the Pi added five more ways to produce a confident wrong number, +all of which yield a plausible framerate rather than an obvious failure: + +* **A module can be loaded and still not execute.** The largest error in this + report was not a bad statistic but a bad subject: every backend comparison + taken on the Pi measured Dolphin's JIT against itself, because the static + module stopped dispatching after boot while still being loaded, named in the + log, and reporting healthy counters. Nothing in the framerate, the scene + guards or the pairing could have caught it. What caught it was asking perf + which shared object the samples landed in, and finding the module absent. Any + comparison of two backends should establish that the code under test is + running before it reports a number -- a profile by object, or a counter that + demonstrably advances with wall time. + +* **A capped scene cannot show a difference.** Both backends hold Luigi's + Mansion's title screen at the 59.9 fps cap, so any comparison taken there + reports parity no matter what the backends do. The scene has to be one that + is actually CPU-bound — the mansion foyer runs 13 fps single-core — before + the measurement can say anything at all. +* **Uncapped, the arms stop looking at the same thing.** With the speed limiter + off the faster arm is further into the game at every wall-clock instant, so + it is rendering different scenery; sampling more instants does not fix it. + Frame sizes in one such run clustered at 0.64 MB / 43 fps and 1.2 MB / 13 fps + — two scenes, averaged into one meaningless mean. +* **Booting from a savestate silently bypasses the module.** Loading state at + boot leaves `native=0 bursts=0 cycles=0`: the recompiled code never executes + and the emulator's own core runs the game. Both arms then measure the same + thing and agree beautifully. Any run whose counters are zero has to be + discarded rather than averaged. +* **A scripted controller can pause the game.** Driving the menus by replaying + a fixed sequence of button presses kept pressing Start after the game had + started; Start opens the pause menu, and a paused game renders cheaply at the + frame cap. The fix was to make the drive closed-loop — read the screen, press + Start only when the frame is small or the game is provably paused, and stop + pressing once gameplay is detected. + +The sample-count lesson also repeated, in the direction that matters. A first +dual-core batch had `llvm-aot` ahead in all three pairs, +2.0%, which is the +kind of result that gets written down. Extending the same comparison to five +pairs reversed the sign to −0.9%. Three pairs all pointing one way is p = 0.125 +and cannot carry a claim, however tidy it looks. + +--- + +## 5. The finding that reframed the effort + +**The C backend's throughput was never measured until late.** Every runtime +number in this project compared LLVM builds to other LLVM builds. The brief +designates the C backend the semantic reference; nothing was ever positioned +against it, so "faster than the previous `llvm-aot` build" silently stood in for +"fast". + +When it was finally measured, the C backend was **60-70% ahead of both LLVM +configurations**, on a module 4.9x smaller than even the fixed-chunk build. That +single comparison explained eight flat results at once and pointed straight at +the spill. + +It also required repairing the C backend to measure it at all: three inline +helpers (`ppc_fp_available_inline`, `ppc_psq_load_inline`, +`ppc_psq_store_inline`) exist in DolRecomp's `cpu.h` but in no vendored +GXRuntime here, so the C backend **would not build against any ModernGekko +checkout on this machine**. The differential suite never noticed, because it +links DolRecomp's own `cpu.h`. + +The first fix added those helpers to the vendored runtimes. That was wrong: the +vendored GXRuntime is a nested git submodule, so the edit would have been +discarded by any `git submodule update`. The emitter now calls what the runtime +actually declares instead -- the paired-single wrappers were pure pass-throughs, +and the FP one's MSR[FP] fast path is spelled out in the generated C. Verified +by reverting all three GXRuntime edits and building Mario Kart's C module +against pristine headers, so nothing outside this repository has to be +maintained. + +**The lesson, stated plainly: measure against the reference backend in Phase 0.** + +--- + +## 6. Compliance with the brief + +| Constraint | Status | +|---|---| +| No Rust; C for everything but the LLVM backend | Held. C++ confined to `src/backend/llvm/`. | +| C backend not replaced | Held, and now measured — it is the performance reference too. | +| Fixed-chunk LLVM path retained | Held. `--backend llvm` unchanged and still builds. | +| `llvm-aot` reaches parity before replacing it | **Exceeded**: 53.49 vs 29.80 fps, +79.5%. | +| No runtime guest-code generation | Held. No executable memory is written. | +| ModernGekko ABI preserved | Held. `void func_XXXXXXXX(CPUState*)` wrappers, dispatcher, hooks and `staticrecomp_get_module` unchanged. | +| Exact PowerPC semantics | Differential suite green across 5 seeds; 23/23 ctest. | +| No copyrighted binaries committed | Held. Titles are local; CI uses synthetic fixtures. | +| Tests not weakened to pass | Held. Test count rose 19 → 23; coverage was **added** (MEM1 boundary, differential call paths). | + +Two compatibility details are worth naming: + +* **Patchability is now explicit.** A direct cross-region call bypasses + `dolrecomp_dispatch_replacement`. That is sound today only because + ModernGekko never defines `DOLRECOMP_ENABLE_REPLACEMENTS`. Setting it now + suppresses every direct external transfer *and* emits the matching header + define, so the two cannot disagree. The previous state was a mod that would + install and silently do nothing. +* **Lockstep needs `--memory-mode safe`.** ModernGekko's lockstep verifier is + the one consumer that installs a write journal. A fast-mode module makes it + inert, and says so on stderr rather than failing quietly. + +--- + +## 7. What is owed + +* **AArch64 runs, but nothing has been measured about the backends there.** + This report previously claimed parity between `llvm-aot` and the C backend on + a Raspberry Pi 4. That claim is withdrawn: both arms were measured while the + static module was not executing the game. + + Cross-compilation itself works. An x86-64 Windows host emits 1,829 AArch64 + objects for Luigi's Mansion in 30 s, the Pi links them in 7.3 s, the module + loads, and the title plays. What does not happen is execution. Profiling the + emulator in the measured scene, sampled by shared object: + + | | share of CPU | + |---|---:| + | Dolphin's `JitArm64` generated code | 41.6% | + | `moderngekko-run` itself | 33.6% | + | `gGLME01_recomp.so` | **absent** | + + The list reaches 0.05% before the module appears at all, and the module's own + counters agree: `native=1641 bursts=77 cycles=418897`, unchanged across 55 s, + 180 s and 400 s runs, in the attract loop and in the mansion foyer alike. A + Gekko issues 486 million cycles a second; the module accounts for roughly 419 + thousand of them, once, during boot, and then never runs again. + + The withdrawn figures -- 13.20 vs 13.03 single-core, 19.29 vs 19.46 dual-core + -- were therefore Dolphin's JIT measured against itself. That is why every + configuration landed on parity, and why a three-pair advantage reversed sign + at five pairs: two modules that never execute cannot differ. + + Why dispatch stopped after boot is now known, and it was not this project's + bug: the ModernGekko build on that machine had no static-recomp integration in + its ARM64 JIT at all. `Jit64` calls `StaticRecompShouldYieldAt` at block + boundaries so the static core regains control; the `JitArm64` in that tree + never did, so the first fall back into the JIT kept the CPU permanently. + Against a current runtime the same module executes 16.3 billion guest cycles + where it previously executed 419 thousand, and the numbers below are the + first on this target with the recompiled code actually running. + + | Luigi's Mansion, mansion foyer, single core | fps | + |---|---:| + | `llvm-aot` as this branch stands | 4.76 | + | with alias metadata (2.4) | **5.48** | + | with alias metadata and a per-title profile | **6.71** | + | Dolphin's `JitArm64`, same scene | 13.20 | + + Five alternating pairs per comparison, twenty samples per run, `CPUThread` + pinned per run and recorded in each result line. +15.0% and +21.9%, 5 of 5 + pairs each. The static recompiler remains about **2x behind the JIT** on this + target, and the profile above is same-scene, so it is an upper bound. + + What the JIT does that this does not is not structural. Its register cache + keeps guest registers in host registers across a block where every read here + goes to `CPUState`; it decrements a cycle counter once per block where the + budget guard here is per loop header; and fastmem removes the bounds check + that costs four instructions per guest access. In the measured hot loop those + came to roughly 67% and 28% of module time, against 0.3% for the guest's own + memory traffic. + + Cross-compiling from an x86-64 build machine needs the AArch64 target + registered and its CodeGen/AsmParser/Desc/Info components linked; relaxing the + triple guard alone leaves `lookupTarget` reporting a missing-component problem + as an unsupported triple. That is on PR #15, with a test that emits for + `aarch64-unknown-linux-gnu` and checks `e_machine`. +* **Dual core is worth more than any change measured here, and is not ours.** + Dolphin's CPU/GPU thread split is **+15.6%** on Colosseum with the full stack + applied (19.71 to 22.78 fps, 4 of 4 pairs, same module, core mode alternated), + and about +16% on Luigi's Mansion. It is a runtime setting with correctness + tradeoffs in timing-sensitive titles, so it is context for anyone reading the + AArch64 numbers rather than something this branch changes -- and it is why + every figure in this report names its core mode. +* **`stfs` diverges between backends** on overflow and denormal inputs. Excluded + from the default differential pool, reproduces with `--stfs`. One backend is + wrong about Gekko and it is not yet known which. This is the oldest open + correctness item. +* **An unexplained dispatcher-rate difference**: the state-in-memory arm reads + 168.7 `bursts/Mcycle` against the C backend's 173.0 on Mario Kart. It may be a + slightly divergent scene; it is not understood. +* **Luigi's Mansion is a noisy rig** — six of twelve pairs were rejected in one + comparison. Results there rest on fewer samples than the other two titles. +* **The object cache key does not hash the emitter source.** Every codegen + change still requires bumping `DOLLLVM_CACHE_VERSION` by hand, or measurements + silently compare identical binaries. This bit three times. + +### Where the next gain probably is + +The C backend is still 24% smaller (65.3 MB against 85.8 MB) at equal speed, and +it gets ThinLTO across the whole module while region objects bypass it entirely +— they are `EXTERNAL_OBJECT` pre-built natives, which CMake's IPO property does +not touch. Whether closing that gap buys anything is unmeasured, and on current +evidence a smaller module does not reliably mean a faster one. + +The per-call state round trip remains the largest identified cost: calls are +7.79% and returns 10.95% of weighted execution. The entry-side half of the +private ABI was built and measured negative (section 3); the win, if any, is in not +materializing on the way *out*, which needs a staleness analysis this emitter +has got wrong twice. + +--- + +## 8. Code health + +Removing the promoting emitter made 584 net lines unreachable, all now deleted: +the materialization barriers, both dataflow analyses and their buffers, the four +sync/reload helpers and their 23 call sites, and two measured-and-rejected +feature flags. `materialize()` is now the guest PC and the cycles owed. + +Verified three ways: **byte-identical IR** from the test fixture before and +after, a full Mario Kart build at **exactly the same 85,770,752 bytes**, and a +paired runtime A/B over 7 pairs reading −0.7% mean at p = 1.0, 3 of 7 favouring +the post-deletion module, `fallback` 0 on every run — indistinguishable, which +is what a no-op deletion should measure. + +One bug was introduced and caught during that deletion. Removing a statement +under an unbraced `if` left the *following* `if` as its body, so `used_[MSR]` +stopped being set and `emitFPAvailable` loaded through a null pointer. The +compiler cannot see that shape; the two other instances in the same pass were +syntax errors and obvious. Found by bisecting against HEAD, after which the rest +of the deletion was audited for the same pattern. diff --git a/src/analysis/cfg.c b/src/analysis/cfg.c new file mode 100644 index 0000000..d81921a --- /dev/null +++ b/src/analysis/cfg.c @@ -0,0 +1,1090 @@ +#include "analysis/cfg.h" + +#include +#include + +/* A branch is unconditional when BO is 1z1zz -- no CTR decrement and no CR + test. Everything else keeps a fallthrough edge, including the CTR-decrement + forms, which are conditional on CTR reaching zero. */ +static bool bo_is_unconditional(u8 bo) { + return (bo & 0x14u) == 0x14u; +} + +typedef struct { + u32* items; + u32 count; + u32 capacity; +} AddrSet; + +static bool addr_set_push(AddrSet* set, u32 value) { + if (set->count == set->capacity) { + u32 capacity = set->capacity ? set->capacity * 2u : 256u; + u32* grown = (u32*)realloc(set->items, capacity * sizeof(u32)); + if (!grown) + return false; + set->items = grown; + set->capacity = capacity; + } + set->items[set->count++] = value; + return true; +} + +static int compare_u32(const void* a, const void* b) { + u32 left = *(const u32*)a; + u32 right = *(const u32*)b; + return (left > right) - (left < right); +} + +static void addr_set_sort_unique(AddrSet* set) { + if (set->count == 0) + return; + qsort(set->items, set->count, sizeof(u32), compare_u32); + u32 out = 1; + for (u32 i = 1; i < set->count; i++) { + if (set->items[i] != set->items[out - 1]) + set->items[out++] = set->items[i]; + } + set->count = out; +} + +static bool addr_set_contains(const AddrSet* set, u32 value) { + u32 low = 0; + u32 high = set->count; + while (low < high) { + u32 mid = low + (high - low) / 2u; + if (set->items[mid] == value) + return true; + if (set->items[mid] < value) + low = mid + 1u; + else + high = mid; + } + return false; +} + +static void addr_set_free(AddrSet* set) { + free(set->items); + set->items = NULL; + set->count = 0; + set->capacity = 0; +} + +/* --- program lifetime ---------------------------------------------------- */ + +void dolcfg_init(DolCfgProgram* program) { + if (!program) + return; + memset(program, 0, sizeof(*program)); +} + +void dolcfg_free(DolCfgProgram* program) { + if (!program) + return; + free(program->blocks); + free(program->functions); + free(program->sections); + free(program->sorted_starts); + free(program->sorted_index); + free(program->known); + free(program->smc); + memset(program, 0, sizeof(*program)); +} + +bool dolcfg_add_section(DolCfgProgram* program, const PPCInst* insts, u32 count, + u32 base_address, const char* label) { + if (!program || !insts || count == 0) + return false; + + if (program->section_count == program->section_capacity) { + u32 capacity = program->section_capacity ? program->section_capacity * 2u : 8u; + DolCfgSection* grown = (DolCfgSection*)realloc( + program->sections, capacity * sizeof(*grown)); + if (!grown) + return false; + program->sections = grown; + program->section_capacity = capacity; + } + + DolCfgSection* section = &program->sections[program->section_count++]; + section->insts = insts; + section->count = count; + section->base_address = base_address; + section->label = label; + return true; +} + +bool dolcfg_add_known_function(DolCfgProgram* program, u32 address, + const char* name, u32 flags) { + if (!program) + return false; + if (program->known_count == program->known_capacity) { + u32 capacity = program->known_capacity ? program->known_capacity * 2u : 256u; + DolCfgKnownFunction* grown = (DolCfgKnownFunction*)realloc( + program->known, capacity * sizeof(*grown)); + if (!grown) + return false; + program->known = grown; + program->known_capacity = capacity; + } + + DolCfgKnownFunction* entry = &program->known[program->known_count++]; + entry->address = address; + entry->flags = flags; + entry->name[0] = '\0'; + if (name) + snprintf(entry->name, sizeof(entry->name), "%s", name); + return true; +} + +bool dolcfg_add_smc_range(DolCfgProgram* program, u32 start, u32 end) { + if (!program) + return false; + if (program->smc_count == program->smc_capacity) { + u32 capacity = program->smc_capacity ? program->smc_capacity * 2u : 64u; + DolCfgSmcRange* grown = + (DolCfgSmcRange*)realloc(program->smc, capacity * sizeof(*grown)); + if (!grown) + return false; + program->smc = grown; + program->smc_capacity = capacity; + } + program->smc[program->smc_count].start = start; + program->smc[program->smc_count].end = end; + program->smc_count++; + return true; +} + +/* --- section helpers ----------------------------------------------------- */ + +static const DolCfgSection* section_for(const DolCfgProgram* program, u32 address) { + for (u32 i = 0; i < program->section_count; i++) { + const DolCfgSection* section = &program->sections[i]; + u32 end = section->base_address + section->count * 4u; + if (address >= section->base_address && address < end) + return section; + } + return NULL; +} + +static bool address_is_smc(const DolCfgProgram* program, u32 address) { + for (u32 i = 0; i < program->smc_count; i++) { + if (address >= program->smc[i].start && address <= program->smc[i].end) + return true; + } + return false; +} + +/* Does this instruction end a basic block, and if so how? */ +static bool classifies_as_terminator(const PPCInst* inst) { + switch (inst->op) { + case PPC_OP_B: + case PPC_OP_BC: + case PPC_OP_BCLR: + case PPC_OP_BCCTR: + case PPC_OP_SC: + case PPC_OP_RFI: + return true; + default: + return false; + } +} + +/* --- block construction -------------------------------------------------- */ + +static bool push_block(DolCfgProgram* program, const DolCfgBlock* block) { + if (program->block_count == program->block_capacity) { + u32 capacity = program->block_capacity ? program->block_capacity * 2u : 1024u; + DolCfgBlock* grown = + (DolCfgBlock*)realloc(program->blocks, capacity * sizeof(*grown)); + if (!grown) + return false; + program->blocks = grown; + program->block_capacity = capacity; + } + program->blocks[program->block_count++] = *block; + return true; +} + +static bool collect_leaders(DolCfgProgram* program, AddrSet* leaders) { + for (u32 s = 0; s < program->section_count; s++) { + const DolCfgSection* section = &program->sections[s]; + if (!addr_set_push(leaders, section->base_address)) + return false; + + for (u32 i = 0; i < section->count; i++) { + const PPCInst* inst = §ion->insts[i]; + u32 address = section->base_address + i * 4u; + + /* Data is not code. A leader here would manufacture a block out of + a jump table or a string. */ + if (inst->embedded_data) + continue; + + /* The word after data resumes code. */ + if (i > 0 && section->insts[i - 1].embedded_data) { + if (!addr_set_push(leaders, address)) + return false; + } + + if (!classifies_as_terminator(inst)) + continue; + + /* Everything after a control transfer starts a block. */ + if (i + 1u < section->count) { + if (!addr_set_push(leaders, address + 4u)) + return false; + } + + /* Direct targets are constants in the word, so this is exact. */ + if ((inst->op == PPC_OP_B || inst->op == PPC_OP_BC) && + !addr_set_push(leaders, inst->branch_target)) { + return false; + } + } + } + + for (u32 i = 0; i < program->known_count; i++) { + if (!addr_set_push(leaders, program->known[i].address)) + return false; + } + if (program->entry_point && !addr_set_push(leaders, program->entry_point)) + return false; + + addr_set_sort_unique(leaders); + return true; +} + +static void classify_terminator(DolCfgProgram* program, DolCfgBlock* block, + const PPCInst* inst, u32 next_address, + bool next_in_section) { + block->successor_count = 0; + block->successors[0] = DOLCFG_NO_BLOCK; + block->successors[1] = DOLCFG_NO_BLOCK; + block->successor_addresses[0] = 0; + block->successor_addresses[1] = 0; + block->call_target = 0; + + if (!inst) { + /* No control-transfer instruction: the block ended because the next + address is a leader (something branches there), or because code ran + out. The first case falls through; only the second is unknown. */ + if (next_in_section) { + block->terminator = DOLCFG_TERM_FALLTHROUGH; + block->successor_addresses[0] = next_address; + block->successor_count = 1; + } else { + block->terminator = DOLCFG_TERM_UNKNOWN; + } + return; + } + + switch (inst->op) { + case PPC_OP_B: + if (inst->lk) { + /* bl: the callee is a separate function; control comes back to the + next instruction, which is this block's successor. */ + block->terminator = DOLCFG_TERM_CALL; + block->call_target = inst->branch_target; + if (next_in_section) { + block->successor_addresses[0] = next_address; + block->successor_count = 1; + } + } else { + /* Reclassified to TAIL_CALL later, once function entries are + known -- a b to another function's entry is a tail call. */ + block->terminator = DOLCFG_TERM_BRANCH; + block->successor_addresses[0] = inst->branch_target; + block->successor_count = 1; + } + break; + + case PPC_OP_BC: + if (inst->lk) + block->flags |= DOLCFG_BLOCK_CONDITIONAL_CALL; + + block->successor_addresses[0] = inst->branch_target; + block->successor_count = 1; + if (bo_is_unconditional(inst->bo)) { + block->terminator = DOLCFG_TERM_BRANCH; + } else { + block->terminator = DOLCFG_TERM_COND_BRANCH; + if (next_in_section) { + block->successor_addresses[1] = next_address; + block->successor_count = 2; + } + } + break; + + case PPC_OP_BCLR: + /* Usually a return. Never assumed to be: a conditional blr keeps its + fallthrough, and Phase 4 handles the ones that are not returns at + all through the indirect path. */ + block->terminator = DOLCFG_TERM_RETURN; + if (!bo_is_unconditional(inst->bo) && next_in_section) { + block->successor_addresses[0] = next_address; + block->successor_count = 1; + } + program->indirect_site_count++; + break; + + case PPC_OP_BCCTR: + block->terminator = DOLCFG_TERM_INDIRECT; + /* An indirect call returns to the next instruction; a conditional + indirect branch falls through to it. Either way it is a successor. */ + if ((inst->lk || !bo_is_unconditional(inst->bo)) && next_in_section) { + block->successor_addresses[0] = next_address; + block->successor_count = 1; + } + program->indirect_site_count++; + break; + + case PPC_OP_SC: + block->terminator = DOLCFG_TERM_SYSTEM; + if (next_in_section) { + block->successor_addresses[0] = next_address; + block->successor_count = 1; + } + break; + + case PPC_OP_RFI: + block->terminator = DOLCFG_TERM_SYSTEM; + break; + + default: + block->terminator = DOLCFG_TERM_FALLTHROUGH; + if (next_in_section) { + block->successor_addresses[0] = next_address; + block->successor_count = 1; + } + break; + } +} + +static bool build_blocks(DolCfgProgram* program, const AddrSet* leaders) { + for (u32 s = 0; s < program->section_count; s++) { + const DolCfgSection* section = &program->sections[s]; + u32 i = 0; + + while (i < section->count) { + /* Skip runs of embedded data outright. */ + if (section->insts[i].embedded_data) { + i++; + continue; + } + + u32 start = section->base_address + i * 4u; + u32 j = i; + const PPCInst* terminator_inst = NULL; + + while (j < section->count) { + const PPCInst* inst = §ion->insts[j]; + u32 address = section->base_address + j * 4u; + + if (inst->embedded_data) + break; + if (j != i && addr_set_contains(leaders, address)) + break; + + if (classifies_as_terminator(inst)) { + terminator_inst = inst; + j++; + break; + } + j++; + } + + DolCfgBlock block; + memset(&block, 0, sizeof(block)); + block.start = start; + block.end = section->base_address + j * 4u; + block.instruction_count = j - i; + block.function = DOLCFG_NO_BLOCK; + block.scc = DOLCFG_NO_BLOCK; + + for (u32 k = i; k < j; k++) { + if (address_is_smc(program, section->base_address + k * 4u)) { + block.flags |= DOLCFG_BLOCK_SMC_SUSPECT; + break; + } + } + + u32 next_address = block.end; + bool next_in_section = + (j < section->count) && !section->insts[j].embedded_data; + + classify_terminator(program, &block, terminator_inst, next_address, + next_in_section); + + if (!push_block(program, &block)) + return false; + + i = j; + } + } + + return true; +} + +/* --- address index ------------------------------------------------------- */ + +typedef struct { + u32 start; + u32 index; +} StartEntry; + +static int compare_start_entry(const void* a, const void* b) { + u32 left = ((const StartEntry*)a)->start; + u32 right = ((const StartEntry*)b)->start; + return (left > right) - (left < right); +} + +static bool build_index(DolCfgProgram* program) { + if (program->block_count == 0) + return true; + + StartEntry* entries = + (StartEntry*)malloc(program->block_count * sizeof(*entries)); + if (!entries) + return false; + for (u32 i = 0; i < program->block_count; i++) { + entries[i].start = program->blocks[i].start; + entries[i].index = i; + } + qsort(entries, program->block_count, sizeof(*entries), compare_start_entry); + + program->sorted_starts = (u32*)malloc(program->block_count * sizeof(u32)); + program->sorted_index = (u32*)malloc(program->block_count * sizeof(u32)); + if (!program->sorted_starts || !program->sorted_index) { + free(entries); + return false; + } + for (u32 i = 0; i < program->block_count; i++) { + program->sorted_starts[i] = entries[i].start; + program->sorted_index[i] = entries[i].index; + } + free(entries); + return true; +} + +u32 dolcfg_block_starting_at(const DolCfgProgram* program, u32 address) { + if (!program || program->block_count == 0) + return DOLCFG_NO_BLOCK; + + u32 low = 0; + u32 high = program->block_count; + while (low < high) { + u32 mid = low + (high - low) / 2u; + if (program->sorted_starts[mid] == address) + return program->sorted_index[mid]; + if (program->sorted_starts[mid] < address) + low = mid + 1u; + else + high = mid; + } + return DOLCFG_NO_BLOCK; +} + +u32 dolcfg_block_at(const DolCfgProgram* program, u32 address) { + if (!program || program->block_count == 0) + return DOLCFG_NO_BLOCK; + + /* Largest start <= address, then a containment check. */ + u32 low = 0; + u32 high = program->block_count; + while (low < high) { + u32 mid = low + (high - low) / 2u; + if (program->sorted_starts[mid] <= address) + low = mid + 1u; + else + high = mid; + } + if (low == 0) + return DOLCFG_NO_BLOCK; + + u32 index = program->sorted_index[low - 1u]; + const DolCfgBlock* block = &program->blocks[index]; + if (address >= block->start && address < block->end) + return index; + return DOLCFG_NO_BLOCK; +} + +static void resolve_edges(DolCfgProgram* program) { + for (u32 i = 0; i < program->block_count; i++) { + DolCfgBlock* block = &program->blocks[i]; + for (u32 s = 0; s < block->successor_count; s++) { + block->successors[s] = + dolcfg_block_starting_at(program, block->successor_addresses[s]); + } + } +} + +/* --- functions ----------------------------------------------------------- */ + +static bool push_function(DolCfgProgram* program, const DolCfgFunction* fn) { + if (program->function_count == program->function_capacity) { + u32 capacity = program->function_capacity ? program->function_capacity * 2u : 256u; + DolCfgFunction* grown = + (DolCfgFunction*)realloc(program->functions, capacity * sizeof(*grown)); + if (!grown) + return false; + program->functions = grown; + program->function_capacity = capacity; + } + program->functions[program->function_count++] = *fn; + return true; +} + +static bool build_functions(DolCfgProgram* program) { + /* Entries come from three places, in this precedence: an explicit symbol + map, the section entry point, and inferred bl targets. A map improves + naming and boundaries but is never required. */ + AddrSet entries = {0}; + AddrSet indirect_entries = {0}; + + for (u32 i = 0; i < program->known_count; i++) { + if (!addr_set_push(&entries, program->known[i].address)) { + addr_set_free(&entries); + addr_set_free(&indirect_entries); + return false; + } + } + if (program->entry_point && !addr_set_push(&entries, program->entry_point)) { + addr_set_free(&entries); + addr_set_free(&indirect_entries); + return false; + } + for (u32 i = 0; i < program->block_count; i++) { + const DolCfgBlock* block = &program->blocks[i]; + if (block->terminator == DOLCFG_TERM_CALL && block->call_target) { + if (!addr_set_push(&entries, block->call_target)) { + addr_set_free(&entries); + addr_set_free(&indirect_entries); + return false; + } + } + } + + /* Entries by elimination. + * + * A block that no direct edge in the whole program reaches is still + * executed -- control gets there indirectly, through a vtable slot, a + * function-pointer table or a jump table. On Mario Kart, seeding only from + * bl targets left 59% of the code owned by no function, and the roots of + * that were 22,010 blocks with no in-edge at all. Treating those as entry + * points is what makes the model describe the whole title rather than the + * directly-called part of it. + * + * This infers *entries*, never edges: nothing here claims to know which + * indirect site reaches which entry. Phase 4 does that. */ + { + u32* in_degree = (u32*)calloc( + program->block_count ? program->block_count : 1u, sizeof(u32)); + if (!in_degree) { + addr_set_free(&entries); + return false; + } + for (u32 i = 0; i < program->block_count; i++) { + const DolCfgBlock* block = &program->blocks[i]; + for (u32 s = 0; s < block->successor_count; s++) { + if (block->successors[s] != DOLCFG_NO_BLOCK) + in_degree[block->successors[s]]++; + } + } + for (u32 i = 0; i < program->block_count; i++) { + if (in_degree[i] != 0) + continue; + if (!addr_set_push(&entries, program->blocks[i].start) || + !addr_set_push(&indirect_entries, program->blocks[i].start)) { + free(in_degree); + addr_set_free(&entries); + addr_set_free(&indirect_entries); + return false; + } + } + free(in_degree); + addr_set_sort_unique(&indirect_entries); + } + + addr_set_sort_unique(&entries); + + for (u32 i = 0; i < entries.count; i++) { + u32 address = entries.items[i]; + u32 block_index = dolcfg_block_starting_at(program, address); + if (block_index == DOLCFG_NO_BLOCK) + continue; /* Outside the loaded sections: a cross-module call. */ + + DolCfgFunction fn; + memset(&fn, 0, sizeof(fn)); + fn.entry_address = address; + fn.entry_block = block_index; + fn.first_block = block_index; + fn.flags = addr_set_contains(&indirect_entries, address) + ? DOLCFG_FUNC_FROM_INDIRECT + : DOLCFG_FUNC_FROM_CALL; + if (program->entry_point == address) + fn.flags |= DOLCFG_FUNC_FROM_ENTRY; + + for (u32 k = 0; k < program->known_count; k++) { + if (program->known[k].address != address) + continue; + fn.flags |= program->known[k].flags; + if (program->known[k].name[0]) { + fn.flags |= DOLCFG_FUNC_FROM_SYMBOL; + snprintf(fn.name, sizeof(fn.name), "%s", program->known[k].name); + } + } + + program->blocks[block_index].flags |= DOLCFG_BLOCK_FUNCTION_ENTRY; + if (!push_function(program, &fn)) { + addr_set_free(&entries); + addr_set_free(&indirect_entries); + return false; + } + } + addr_set_free(&entries); + addr_set_free(&indirect_entries); + + /* Ownership by forward reachability from each entry, in address order, so + the assignment is deterministic. A block already owned is left alone: + first entry to reach it wins, which keeps shared tails attached to the + lowest-addressed caller rather than flip-flopping. */ + u32* stack = (u32*)malloc((program->block_count ? program->block_count : 1u) * + sizeof(u32)); + if (!stack) + return false; + + for (u32 f = 0; f < program->function_count; f++) { + DolCfgFunction* fn = &program->functions[f]; + if (program->blocks[fn->entry_block].function != DOLCFG_NO_BLOCK) + continue; /* Two entries on one block: the first one owns it. */ + + /* Ownership is claimed at push time, not pop time. Claiming on pop + lets a block with several predecessors sit on the stack more than + once, and the stack is only block_count deep -- on a real title that + overflows. Claiming on push makes each block enter the stack at most + once, which bounds it by construction. */ + u32 top = 0; + program->blocks[fn->entry_block].function = f; + stack[top++] = fn->entry_block; + + while (top > 0) { + u32 index = stack[--top]; + DolCfgBlock* block = &program->blocks[index]; + + fn->block_count++; + fn->instruction_count += block->instruction_count; + if (block->start < program->blocks[fn->first_block].start) + fn->first_block = index; + if (block->terminator == DOLCFG_TERM_INDIRECT) + fn->flags |= DOLCFG_FUNC_HAS_INDIRECT; + if (block->flags & DOLCFG_BLOCK_SMC_SUSPECT) + fn->flags |= DOLCFG_FUNC_HAS_SMC; + + for (u32 s = 0; s < block->successor_count; s++) { + u32 next = block->successors[s]; + if (next == DOLCFG_NO_BLOCK) + continue; + if (program->blocks[next].function != DOLCFG_NO_BLOCK) + continue; + /* Another function's entry is not part of this one. */ + if (program->blocks[next].flags & DOLCFG_BLOCK_FUNCTION_ENTRY) + continue; + program->blocks[next].function = f; + stack[top++] = next; + } + } + } + /* Anything still unowned is a cycle every one of whose blocks has an + in-edge from inside the cycle, so no zero-in-degree root pointed at it. + It is still executed -- reached indirectly -- and region formation cannot + place a block that belongs to no function, so the lowest-addressed + survivor becomes an entry and the pass repeats until none are left. + Each round claims at least one block, so this terminates. */ + u32 cursor = 0; + for (;;) { + /* The cursor only moves forward: a block passed over is already owned + and cannot become unowned, so rescanning from zero each round would + make this quadratic for no benefit. */ + while (cursor < program->block_count && + program->blocks[cursor].function != DOLCFG_NO_BLOCK) { + cursor++; + } + if (cursor >= program->block_count) + break; + u32 seed = cursor; + + DolCfgFunction fn; + memset(&fn, 0, sizeof(fn)); + fn.entry_address = program->blocks[seed].start; + fn.entry_block = seed; + fn.first_block = seed; + fn.flags = DOLCFG_FUNC_FROM_INDIRECT; + program->blocks[seed].flags |= DOLCFG_BLOCK_FUNCTION_ENTRY; + if (!push_function(program, &fn)) { + free(stack); + return false; + } + + u32 f = program->function_count - 1u; + u32 top = 0; + program->blocks[seed].function = f; + stack[top++] = seed; + + while (top > 0) { + u32 index = stack[--top]; + DolCfgBlock* block = &program->blocks[index]; + + program->functions[f].block_count++; + program->functions[f].instruction_count += block->instruction_count; + if (block->start < program->blocks[program->functions[f].first_block].start) + program->functions[f].first_block = index; + if (block->terminator == DOLCFG_TERM_INDIRECT) + program->functions[f].flags |= DOLCFG_FUNC_HAS_INDIRECT; + if (block->flags & DOLCFG_BLOCK_SMC_SUSPECT) + program->functions[f].flags |= DOLCFG_FUNC_HAS_SMC; + + for (u32 s = 0; s < block->successor_count; s++) { + u32 next = block->successors[s]; + if (next == DOLCFG_NO_BLOCK) + continue; + if (program->blocks[next].function != DOLCFG_NO_BLOCK) + continue; + if (program->blocks[next].flags & DOLCFG_BLOCK_FUNCTION_ENTRY) + continue; + program->blocks[next].function = f; + stack[top++] = next; + } + } + } + + free(stack); + + /* A plain b whose target is another function's entry is a tail call. This + needs the entry set, so it cannot happen during classification. */ + for (u32 i = 0; i < program->block_count; i++) { + DolCfgBlock* block = &program->blocks[i]; + if (block->terminator != DOLCFG_TERM_BRANCH || block->successor_count != 1) + continue; + u32 target = block->successors[0]; + if (target == DOLCFG_NO_BLOCK) + continue; + if ((program->blocks[target].flags & DOLCFG_BLOCK_FUNCTION_ENTRY) && + program->blocks[target].function != block->function) { + block->terminator = DOLCFG_TERM_TAIL_CALL; + block->call_target = program->blocks[target].start; + } + } + + /* Blocks no entry reached are only enterable indirectly. */ + for (u32 i = 0; i < program->block_count; i++) { + if (program->blocks[i].function == DOLCFG_NO_BLOCK) + program->blocks[i].flags |= DOLCFG_BLOCK_UNREACHED; + } + + return true; +} + +/* --- loops and SCCs ------------------------------------------------------ */ + +/* Tarjan, iterative: the recursive form overflows on a real title's call + graph. Also fills loop headers, since a back edge into an SCC member is + exactly a loop entry. */ +typedef struct { + u32* index; + u32* lowlink; + u32* stack; + bool* on_stack; + u32* work; + u32* work_edge; + u32 next_index; + u32 stack_top; +} Tarjan; + +static bool compute_sccs(DolCfgProgram* program) { + u32 n = program->block_count; + if (n == 0) + return true; + + Tarjan t; + memset(&t, 0, sizeof(t)); + t.index = (u32*)malloc(n * sizeof(u32)); + t.lowlink = (u32*)malloc(n * sizeof(u32)); + t.stack = (u32*)malloc(n * sizeof(u32)); + t.on_stack = (bool*)calloc(n, sizeof(bool)); + t.work = (u32*)malloc(n * sizeof(u32)); + t.work_edge = (u32*)malloc(n * sizeof(u32)); + if (!t.index || !t.lowlink || !t.stack || !t.on_stack || !t.work || + !t.work_edge) { + free(t.index); free(t.lowlink); free(t.stack); + free(t.on_stack); free(t.work); free(t.work_edge); + return false; + } + + for (u32 i = 0; i < n; i++) + t.index[i] = DOLCFG_NO_BLOCK; + t.next_index = 0; + t.stack_top = 0; + program->scc_count = 0; + + for (u32 root = 0; root < n; root++) { + if (t.index[root] != DOLCFG_NO_BLOCK) + continue; + + u32 work_top = 0; + t.work[work_top] = root; + t.work_edge[work_top] = 0; + t.index[root] = t.lowlink[root] = t.next_index++; + t.stack[t.stack_top++] = root; + t.on_stack[root] = true; + work_top++; + + while (work_top > 0) { + u32 v = t.work[work_top - 1u]; + u32 edge = t.work_edge[work_top - 1u]; + + if (edge < program->blocks[v].successor_count) { + t.work_edge[work_top - 1u] = edge + 1u; + u32 w = program->blocks[v].successors[edge]; + if (w == DOLCFG_NO_BLOCK) + continue; + + if (t.index[w] == DOLCFG_NO_BLOCK) { + t.index[w] = t.lowlink[w] = t.next_index++; + t.stack[t.stack_top++] = w; + t.on_stack[w] = true; + t.work[work_top] = w; + t.work_edge[work_top] = 0; + work_top++; + } else if (t.on_stack[w]) { + if (t.index[w] < t.lowlink[v]) + t.lowlink[v] = t.index[w]; + } + continue; + } + + work_top--; + if (work_top > 0) { + u32 parent = t.work[work_top - 1u]; + if (t.lowlink[v] < t.lowlink[parent]) + t.lowlink[parent] = t.lowlink[v]; + } + + if (t.lowlink[v] == t.index[v]) { + u32 members = 0; + u32 w; + do { + w = t.stack[--t.stack_top]; + t.on_stack[w] = false; + program->blocks[w].scc = program->scc_count; + members++; + } while (w != v); + + /* A single block is only cyclic if it branches to itself. */ + if (members == 1) { + bool self = false; + for (u32 s = 0; s < program->blocks[v].successor_count; s++) { + if (program->blocks[v].successors[s] == v) + self = true; + } + if (self) + program->blocks[v].flags |= DOLCFG_BLOCK_LOOP_HEADER; + } + program->scc_count++; + } + } + } + + /* Within a multi-block SCC, an edge arriving from outside marks a loop + header; the whole SCC is cyclic by definition. */ + for (u32 i = 0; i < n; i++) { + const DolCfgBlock* block = &program->blocks[i]; + for (u32 s = 0; s < block->successor_count; s++) { + u32 next = block->successors[s]; + if (next == DOLCFG_NO_BLOCK) + continue; + if (program->blocks[next].scc == block->scc && next <= i) + program->blocks[next].flags |= DOLCFG_BLOCK_LOOP_HEADER; + } + } + + /* Loop depth: how many distinct multi-block SCCs, plus self-loops, a block + participates in. A block is in at most one SCC, so depth is 0 or 1 here; + nesting beyond that needs a dominator tree, which Phase 1 does not + require and which is recorded as a Phase 2 refinement. */ + u32* scc_size = (u32*)calloc(program->scc_count ? program->scc_count : 1u, + sizeof(u32)); + if (scc_size) { + for (u32 i = 0; i < n; i++) { + if (program->blocks[i].scc != DOLCFG_NO_BLOCK) + scc_size[program->blocks[i].scc]++; + } + for (u32 i = 0; i < n; i++) { + u32 scc = program->blocks[i].scc; + bool cyclic = (scc != DOLCFG_NO_BLOCK && scc_size[scc] > 1u) || + (program->blocks[i].flags & DOLCFG_BLOCK_LOOP_HEADER); + program->blocks[i].loop_depth = cyclic ? 1u : 0u; + } + free(scc_size); + } + + program->loop_count = 0; + for (u32 i = 0; i < n; i++) { + if (program->blocks[i].flags & DOLCFG_BLOCK_LOOP_HEADER) + program->loop_count++; + } + + free(t.index); free(t.lowlink); free(t.stack); + free(t.on_stack); free(t.work); free(t.work_edge); + return true; +} + +/* --- build --------------------------------------------------------------- */ + +bool dolcfg_build(DolCfgProgram* program, FILE* diagnostics) { + if (!program || program->section_count == 0) { + if (diagnostics) + fprintf(diagnostics, "error: CFG build needs at least one section\n"); + return false; + } + + AddrSet leaders = {0}; + if (!collect_leaders(program, &leaders)) { + addr_set_free(&leaders); + if (diagnostics) + fprintf(diagnostics, "error: out of memory collecting CFG leaders\n"); + return false; + } + + bool ok = build_blocks(program, &leaders); + addr_set_free(&leaders); + if (!ok) { + if (diagnostics) + fprintf(diagnostics, "error: out of memory building CFG blocks\n"); + return false; + } + + /* Order matters: ownership traversal and Tarjan both walk successor block + indices, which only exist once the address edges are resolved. */ + if (!build_index(program)) { + if (diagnostics) + fprintf(diagnostics, "error: out of memory indexing CFG blocks\n"); + return false; + } + resolve_edges(program); + + if (!build_functions(program) || !compute_sccs(program)) { + if (diagnostics) + fprintf(diagnostics, "error: out of memory analysing CFG\n"); + return false; + } + return true; +} + +bool dolcfg_load_profile(DolCfgProgram* program, const char* path, + u32* matched_out, u32* unmatched_out, + FILE* diagnostics) { + if (!program || !path) + return false; + + FILE* in = fopen(path, "rb"); + if (!in) { + if (diagnostics) + fprintf(diagnostics, "error: cannot read region profile '%s'\n", path); + return false; + } + + char line[512]; + u32 matched = 0; + u32 unmatched = 0; + + while (fgets(line, sizeof(line), in)) { + char* cursor = line; + while (*cursor == ' ' || *cursor == '\t') + cursor++; + if (*cursor == '#' || *cursor == '\n' || *cursor == '\r' || *cursor == '\0') + continue; + + char* end = NULL; + unsigned long address = strtoul(cursor, &end, 0); + if (end == cursor) + continue; + + /* An address on its own means "hot" without saying how hot. */ + u64 count = 1; + while (*end == ' ' || *end == '\t') + end++; + if (*end && *end != '#' && *end != '\n' && *end != '\r') { + char* count_end = NULL; + unsigned long long parsed = strtoull(end, &count_end, 0); + if (count_end != end) + count = (u64)parsed; + } + + /* The profile names function entries, so weight the whole function. + Per-block resolution would need the profile's own CFG, which is the + generated module's, not the guest's. */ + u32 block = dolcfg_block_starting_at(program, (u32)address); + if (block == DOLCFG_NO_BLOCK) + block = dolcfg_block_at(program, (u32)address); + if (block == DOLCFG_NO_BLOCK) { + unmatched++; + continue; + } + + u32 function = program->blocks[block].function; + if (function == DOLCFG_NO_BLOCK) { + program->blocks[block].weight += count; + matched++; + continue; + } + + /* Accumulate onto the function only. Spreading to its blocks here would + be O(entries x blocks) -- tolerable for a 78-entry hot list, hours for + a full 23,311-function profile. One pass afterwards does it in O(n). */ + program->functions[function].weight += count; + matched++; + } + + fclose(in); + + for (u32 i = 0; i < program->block_count; i++) { + u32 function = program->blocks[i].function; + if (function != DOLCFG_NO_BLOCK && program->functions[function].weight) + program->blocks[i].weight += program->functions[function].weight; + } + + if (matched_out) + *matched_out = matched; + if (unmatched_out) + *unmatched_out = unmatched; + + if (diagnostics && unmatched) { + fprintf(diagnostics, + "warning: region profile '%s': %u of %u entries matched no known " + "function; the profile may be from a different build\n", + path, unmatched, matched + unmatched); + } + if (!matched) { + if (diagnostics) + fprintf(diagnostics, + "error: region profile '%s' matched nothing in this program\n", + path); + return false; + } + return true; +} + +const char* dolcfg_terminator_name(DolCfgTerminator kind) { + switch (kind) { + case DOLCFG_TERM_FALLTHROUGH: return "fallthrough"; + case DOLCFG_TERM_BRANCH: return "branch"; + case DOLCFG_TERM_COND_BRANCH: return "cond-branch"; + case DOLCFG_TERM_CALL: return "call"; + case DOLCFG_TERM_TAIL_CALL: return "tail-call"; + case DOLCFG_TERM_RETURN: return "return"; + case DOLCFG_TERM_INDIRECT: return "indirect"; + case DOLCFG_TERM_SYSTEM: return "system"; + case DOLCFG_TERM_UNKNOWN: + default: return "unknown"; + } +} diff --git a/src/analysis/cfg.h b/src/analysis/cfg.h new file mode 100644 index 0000000..878196e --- /dev/null +++ b/src/analysis/cfg.h @@ -0,0 +1,235 @@ +#ifndef DOLRECOMP_ANALYSIS_CFG_H +#define DOLRECOMP_ANALYSIS_CFG_H + +/* Whole-title control-flow and call-graph model. + * + * The fixed-chunk backends cut the program every N instructions, which puts a + * state materialization and a dispatcher round trip wherever the cut lands -- + * through a hot loop as readily as through cold code. Choosing better cut + * points needs a model of where control actually flows, which is what this is. + * + * Scope and honesty about it: + * + * Direct control flow is recovered exactly. Every b/bc target is a constant + * in the instruction word, so block boundaries and direct edges are facts. + * + * Indirect control flow is NOT resolved here. bclr/bcctr sites are recorded + * as indirect exits with no successors. Phase 4 attaches target sets to those + * sites; until then a region must end at one. Anything else would be guessing + * at control flow, which corrupts the program silently rather than loudly. + * + * Function entries are *inferred* -- from bl targets, from an explicit symbol + * map, and from section entry points. A MAP improves the model but is never + * required, and its absence must not change correctness, only region quality. + * + * Embedded data is excluded up front: PPCInst carries an embedded_data flag + * from the existing analysis pass, and those words never become blocks. + */ + +#include "common/types.h" +#include "frontend/decoder.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + /* Falls out of the bottom into the next block. */ + DOLCFG_TERM_FALLTHROUGH, + /* b / ba: one known successor, no link. */ + DOLCFG_TERM_BRANCH, + /* bc / bca: taken target plus fallthrough. */ + DOLCFG_TERM_COND_BRANCH, + /* bl / bla: a call. Successor is the return point, not the callee. */ + DOLCFG_TERM_CALL, + /* b to a known function entry with no link: a tail call. */ + DOLCFG_TERM_TAIL_CALL, + /* bclr -- usually a return, but never assume it. */ + DOLCFG_TERM_RETURN, + /* bcctr / bclrl and friends: unresolved indirect transfer. */ + DOLCFG_TERM_INDIRECT, + /* sc, rfi, tw/twi: leaves through the runtime. */ + DOLCFG_TERM_SYSTEM, + /* Ran into embedded data, a section end, or an undecodable word. */ + DOLCFG_TERM_UNKNOWN, +} DolCfgTerminator; + +enum { + DOLCFG_BLOCK_FUNCTION_ENTRY = 1u << 0, + DOLCFG_BLOCK_LOOP_HEADER = 1u << 1, + /* Reached only by an indirect edge or not reached at all: region formation + must not assume it is contiguous with its neighbours. */ + DOLCFG_BLOCK_UNREACHED = 1u << 2, + /* Overlaps a range the SMC pass flagged as possibly self-modifying. */ + DOLCFG_BLOCK_SMC_SUSPECT = 1u << 3, + /* Conditional branch that is also a link (bcl): both call and condition. */ + DOLCFG_BLOCK_CONDITIONAL_CALL = 1u << 4, +}; + +#define DOLCFG_NO_BLOCK 0xFFFFFFFFu + +typedef struct { + u32 start; /* guest address, inclusive */ + u32 end; /* guest address, exclusive */ + u32 instruction_count; + u32 function; /* owning function index, or DOLCFG_NO_BLOCK */ + u32 flags; + + DolCfgTerminator terminator; + /* Successor block indices. successors[0] is the taken/only target, + successors[1] the fallthrough of a conditional. DOLCFG_NO_BLOCK when the + edge leaves the model (unresolved indirect, or outside any section). */ + u32 successors[2]; + u32 successor_count; + /* Guest addresses of the same, kept even when the target is outside the + loaded sections so cross-module edges stay visible. */ + u32 successor_addresses[2]; + + /* Call target for CALL/TAIL_CALL terminators, or 0. */ + u32 call_target; + + /* Loop nesting depth, 0 for straight-line code. */ + u32 loop_depth; + /* Strongly-connected-component id; blocks sharing one are mutually + reachable and should not be split across regions when hot. */ + u32 scc; + + /* Profile weight, 0 when no profile is loaded. */ + u64 weight; +} DolCfgBlock; + +enum { + DOLCFG_FUNC_FROM_SYMBOL = 1u << 0, /* named by a MAP */ + DOLCFG_FUNC_FROM_CALL = 1u << 1, /* inferred from a bl target */ + DOLCFG_FUNC_FROM_ENTRY = 1u << 2, /* section/module entry point */ + /* No direct edge in the whole program reaches this block, so control can + only arrive indirectly -- a vtable slot, a function-pointer table, a + jump-table entry. It is an entry point by elimination. */ + DOLCFG_FUNC_FROM_INDIRECT = 1u << 6, + DOLCFG_FUNC_HAS_INDIRECT = 1u << 3, + DOLCFG_FUNC_HAS_SMC = 1u << 4, + /* Externally visible: a mod or replacement may intercept it, so it keeps a + public wrapper even under aggressive linking. */ + DOLCFG_FUNC_PATCHABLE = 1u << 5, +}; + +typedef struct { + u32 entry_address; + u32 entry_block; + u32 first_block; /* index of lowest-addressed owned block */ + u32 block_count; + u32 instruction_count; + u32 flags; + u64 weight; + char name[64]; /* from a MAP, else empty */ +} DolCfgFunction; + +typedef struct { + const PPCInst* insts; + u32 count; + u32 base_address; + const char* label; +} DolCfgSection; + +/* Build inputs, supplied before dolcfg_build(). */ +typedef struct { + u32 address; + u32 flags; + char name[64]; +} DolCfgKnownFunction; + +typedef struct { + u32 start; + u32 end; +} DolCfgSmcRange; + +typedef struct { + DolCfgBlock* blocks; + u32 block_count; + u32 block_capacity; + + DolCfgKnownFunction* known; + u32 known_count; + u32 known_capacity; + + DolCfgSmcRange* smc; + u32 smc_count; + u32 smc_capacity; + + DolCfgFunction* functions; + u32 function_count; + u32 function_capacity; + + DolCfgSection* sections; + u32 section_count; + u32 section_capacity; + + u32 entry_point; + + /* Sorted block start addresses, parallel to a block index, so address + lookup is a binary search rather than a scan. */ + u32* sorted_starts; + u32* sorted_index; + + u32 scc_count; + u32 loop_count; + u32 indirect_site_count; +} DolCfgProgram; + +void dolcfg_init(DolCfgProgram* program); +void dolcfg_free(DolCfgProgram* program); + +/* Adds a decoded section. The instruction array must outlive the program. */ +bool dolcfg_add_section(DolCfgProgram* program, const PPCInst* insts, u32 count, + u32 base_address, const char* label); + +/* Declares a known function entry ahead of the build, from a MAP or an entry + point. `name` may be NULL. Entries outside any added section are ignored. */ +bool dolcfg_add_known_function(DolCfgProgram* program, u32 address, + const char* name, u32 flags); + +/* Marks an address range as SMC-suspect, so blocks overlapping it are flagged + and region formation can end at them. */ +bool dolcfg_add_smc_range(DolCfgProgram* program, u32 start, u32 end); + +/* Builds blocks, edges, functions, loops and SCCs. Deterministic: the same + sections and known-function set always produce the same numbering. */ +bool dolcfg_build(DolCfgProgram* program, FILE* diagnostics); + +/* Loads execution weights and attaches them to functions and their blocks. + * + * The file is one entry per line, `
`, with `#` comments and + * blank lines ignored -- the shape ModernGekko's hot-entry lists already use: + * + * 0x800EB5C0 # 83,166,563,414 + * 0x800E6FC0 197140421 + * + * A count after the address is used when present; an address with no count is + * treated as hot with weight 1, so a bare hot-entry list still works. + * + * Deliberately not an LLVM .profdata reader. Keeping the analysis layer in C + * and free of an LLVM dependency matters more than avoiding one conversion + * step, and a text format is something a test can write by hand. Convert with + * benchmarks/profdata_to_weights.py. + * + * Call after dolcfg_build(), which is when functions and blocks exist. + * Addresses outside any known function are counted and reported, not fatal: + * a profile from a different build should degrade, loudly, rather than fail. */ +bool dolcfg_load_profile(DolCfgProgram* program, const char* path, + u32* matched_out, u32* unmatched_out, + FILE* diagnostics); + +/* Block index containing `address`, or DOLCFG_NO_BLOCK. */ +u32 dolcfg_block_at(const DolCfgProgram* program, u32 address); + +/* Block index whose start is exactly `address`, or DOLCFG_NO_BLOCK. */ +u32 dolcfg_block_starting_at(const DolCfgProgram* program, u32 address); + +const char* dolcfg_terminator_name(DolCfgTerminator kind); + +#ifdef __cplusplus +} +#endif + +#endif /* DOLRECOMP_ANALYSIS_CFG_H */ diff --git a/src/analysis/regions.c b/src/analysis/regions.c new file mode 100644 index 0000000..b890f90 --- /dev/null +++ b/src/analysis/regions.c @@ -0,0 +1,959 @@ +#include "analysis/regions.h" + +#include +#include + +void dolregion_default_limits(DolRegionLimits* limits) { + if (!limits) + return; + /* Conservative on purpose. + * + * The existing fixed LLVM path uses 128 because a chunk becomes one LLVM + * function and the register allocator must keep the promoted guest + * register file live across the whole of it -- 1024 cost 3x the code size + * for a third less speed (pipeline.c, LLVM-EXPERIMENTS E002/E003). + * + * A region is not the same object: its boundaries follow control flow, so + * the state that has to stay live across a join is what the CFG actually + * requires rather than everything. That is the premise being tested, not a + * result, so the default starts modest and the benchmark decides. */ + limits->max_instructions = 1024u; + limits->max_ir_instructions = 1024u * DOLREGION_IR_PER_GUEST_INSN * 2u; + limits->max_functions = 64u; + limits->cold_weight_threshold = 1u; + limits->merge_address_adjacent = 1; + limits->max_adjacency_gap = 256u; +} + +bool dolregion_parse_mode(const char* text, DolRegionMode* mode) { + if (!text || !mode) + return false; + if (strcmp(text, "fixed") == 0) { *mode = DOLREGION_MODE_FIXED; return true; } + if (strcmp(text, "function") == 0) { *mode = DOLREGION_MODE_FUNCTION; return true; } + if (strcmp(text, "cfg") == 0) { *mode = DOLREGION_MODE_CFG; return true; } + if (strcmp(text, "pgo") == 0) { *mode = DOLREGION_MODE_PGO; return true; } + return false; +} + +const char* dolregion_mode_name(DolRegionMode mode) { + switch (mode) { + case DOLREGION_MODE_FIXED: return "fixed"; + case DOLREGION_MODE_FUNCTION: return "function"; + case DOLREGION_MODE_CFG: return "cfg"; + case DOLREGION_MODE_PGO: return "pgo"; + default: return "unknown"; + } +} + +const char* dolregion_end_reason_name(DolRegionEndReason reason) { + switch (reason) { + case DOLREGION_END_SIZE_LIMIT: return "size-limit"; + case DOLREGION_END_IR_LIMIT: return "ir-limit"; + case DOLREGION_END_NO_CANDIDATE: return "no-connected-candidate"; + case DOLREGION_END_INDIRECT: return "indirect-transfer"; + case DOLREGION_END_SMC: return "smc-boundary"; + case DOLREGION_END_PATCHABLE: return "patchable-boundary"; + case DOLREGION_END_COLD: return "cold-code"; + case DOLREGION_END_FUNCTION_BOUNDARY:return "function-boundary"; + case DOLREGION_END_FIXED_CHUNK: return "fixed-chunk"; + case DOLREGION_END_SECTION_END: return "section-end"; + default: return "unknown"; + } +} + +void dolregion_plan_init(DolRegionPlan* plan) { + if (plan) + memset(plan, 0, sizeof(*plan)); +} + +void dolregion_plan_free(DolRegionPlan* plan) { + if (!plan) + return; + for (u32 i = 0; i < plan->region_count; i++) { + free(plan->regions[i].blocks); + free(plan->regions[i].functions); + } + free(plan->regions); + free(plan->block_region); + free(plan->function_region); + memset(plan, 0, sizeof(*plan)); +} + +/* --- per-function block lists (CSR) -------------------------------------- */ + +typedef struct { + u32* offsets; /* function_count + 1 */ + u32* blocks; /* block_count, ascending within each function */ +} FunctionBlocks; + +static int compare_u32_asc(const void* a, const void* b) { + u32 left = *(const u32*)a; + u32 right = *(const u32*)b; + return (left > right) - (left < right); +} + +static bool build_function_blocks(const DolCfgProgram* program, + FunctionBlocks* out) { + out->offsets = (u32*)calloc(program->function_count + 1u, sizeof(u32)); + out->blocks = (u32*)malloc((program->block_count ? program->block_count : 1u) * + sizeof(u32)); + if (!out->offsets || !out->blocks) + return false; + + for (u32 i = 0; i < program->block_count; i++) { + u32 f = program->blocks[i].function; + if (f != DOLCFG_NO_BLOCK) + out->offsets[f + 1u]++; + } + for (u32 f = 0; f < program->function_count; f++) + out->offsets[f + 1u] += out->offsets[f]; + + u32* cursor = (u32*)malloc((program->function_count ? program->function_count : 1u) * + sizeof(u32)); + if (!cursor) + return false; + memcpy(cursor, out->offsets, program->function_count * sizeof(u32)); + + for (u32 i = 0; i < program->block_count; i++) { + u32 f = program->blocks[i].function; + if (f != DOLCFG_NO_BLOCK) + out->blocks[cursor[f]++] = i; + } + free(cursor); + + /* Blocks are appended in index order, which is section order and therefore + already ascending by address -- but the plan's determinism must not rest + on that being true forever. */ + for (u32 f = 0; f < program->function_count; f++) { + u32 start = out->offsets[f]; + u32 count = out->offsets[f + 1u] - start; + if (count > 1u) + qsort(&out->blocks[start], count, sizeof(u32), compare_u32_asc); + } + return true; +} + +static void free_function_blocks(FunctionBlocks* fb) { + free(fb->offsets); + free(fb->blocks); + fb->offsets = NULL; + fb->blocks = NULL; +} + +/* --- function call graph -------------------------------------------------- */ + +typedef struct { + u32* offsets; /* function_count + 1 */ + u32* targets; + u64* weights; + u32 edge_count; +} CallGraph; + +typedef struct { + u32 from; + u32 to; + u64 weight; +} RawEdge; + +static int compare_raw_edge(const void* a, const void* b) { + const RawEdge* left = (const RawEdge*)a; + const RawEdge* right = (const RawEdge*)b; + if (left->from != right->from) + return left->from < right->from ? -1 : 1; + if (left->to != right->to) + return left->to < right->to ? -1 : 1; + return 0; +} + +/* Undirected adjacency: a caller and callee are equally worth co-locating, and + the accretion walks outward from a seed in both directions. */ +static bool build_call_graph(const DolCfgProgram* program, CallGraph* graph) { + memset(graph, 0, sizeof(*graph)); + + u32 capacity = 1024; + u32 count = 0; + RawEdge* raw = (RawEdge*)malloc(capacity * sizeof(*raw)); + if (!raw) + return false; + + for (u32 i = 0; i < program->block_count; i++) { + const DolCfgBlock* block = &program->blocks[i]; + if (block->terminator != DOLCFG_TERM_CALL && + block->terminator != DOLCFG_TERM_TAIL_CALL) + continue; + if (!block->call_target || block->function == DOLCFG_NO_BLOCK) + continue; + + u32 target_block = dolcfg_block_starting_at(program, block->call_target); + if (target_block == DOLCFG_NO_BLOCK) + continue; /* Cross-module: no local function to merge with. */ + u32 callee = program->blocks[target_block].function; + if (callee == DOLCFG_NO_BLOCK || callee == block->function) + continue; + + if (count + 2u > capacity) { + capacity *= 2u; + RawEdge* grown = (RawEdge*)realloc(raw, capacity * sizeof(*raw)); + if (!grown) { + free(raw); + return false; + } + raw = grown; + } + /* A call site's weight is its profile weight when one exists, else 1 -- + so without a profile this counts call sites, which is still a better + merge signal than address adjacency. */ + u64 weight = block->weight ? block->weight : 1u; + raw[count].from = block->function; + raw[count].to = callee; + raw[count].weight = weight; + count++; + raw[count].from = callee; + raw[count].to = block->function; + raw[count].weight = weight; + count++; + } + + qsort(raw, count, sizeof(*raw), compare_raw_edge); + + graph->offsets = (u32*)calloc(program->function_count + 1u, sizeof(u32)); + graph->targets = (u32*)malloc((count ? count : 1u) * sizeof(u32)); + graph->weights = (u64*)malloc((count ? count : 1u) * sizeof(u64)); + if (!graph->offsets || !graph->targets || !graph->weights) { + free(raw); + return false; + } + + /* Coalesce duplicate (from,to) pairs, summing weight. */ + u32 out = 0; + for (u32 i = 0; i < count;) { + u32 j = i; + u64 weight = 0; + while (j < count && raw[j].from == raw[i].from && raw[j].to == raw[i].to) { + weight += raw[j].weight; + j++; + } + graph->targets[out] = raw[i].to; + graph->weights[out] = weight; + graph->offsets[raw[i].from + 1u]++; + out++; + i = j; + } + for (u32 f = 0; f < program->function_count; f++) + graph->offsets[f + 1u] += graph->offsets[f]; + graph->edge_count = out; + + free(raw); + return true; +} + +static void free_call_graph(CallGraph* graph) { + free(graph->offsets); + free(graph->targets); + free(graph->weights); + memset(graph, 0, sizeof(*graph)); +} + +/* --- region construction -------------------------------------------------- */ + +static DolRegion* new_region(DolRegionPlan* plan) { + if (plan->region_count == plan->region_capacity) { + u32 capacity = plan->region_capacity ? plan->region_capacity * 2u : 256u; + DolRegion* grown = + (DolRegion*)realloc(plan->regions, capacity * sizeof(*grown)); + if (!grown) + return NULL; + plan->regions = grown; + plan->region_capacity = capacity; + } + DolRegion* region = &plan->regions[plan->region_count]; + memset(region, 0, sizeof(*region)); + region->id = plan->region_count; + region->guest_start = 0xFFFFFFFFu; + region->end_reason = DOLREGION_END_NO_CANDIDATE; + plan->region_count++; + return region; +} + +static bool region_push_block(DolRegion* region, const DolCfgProgram* program, + DolRegionPlan* plan, u32 block_index) { + u32* grown = (u32*)realloc(region->blocks, + (region->block_count + 1u) * sizeof(u32)); + if (!grown) + return false; + region->blocks = grown; + region->blocks[region->block_count++] = block_index; + + const DolCfgBlock* block = &program->blocks[block_index]; + if (block->start < region->guest_start) + region->guest_start = block->start; + if (block->end > region->guest_end) + region->guest_end = block->end; + region->instruction_count += block->instruction_count; + region->weight += block->weight; + if (block->flags & DOLCFG_BLOCK_LOOP_HEADER) + region->loop_count++; + if (block->flags & DOLCFG_BLOCK_SMC_SUSPECT) + region->contains_smc = true; + if (block->terminator == DOLCFG_TERM_INDIRECT) + region->indirect_sites++; + + plan->block_region[block_index] = region->id; + return true; +} + +static bool region_push_function(DolRegion* region, const DolCfgProgram* program, + DolRegionPlan* plan, const FunctionBlocks* fb, + u32 function_index) { + u32* grown = (u32*)realloc(region->functions, + (region->function_count + 1u) * sizeof(u32)); + if (!grown) + return false; + region->functions = grown; + region->functions[region->function_count++] = function_index; + plan->function_region[function_index] = region->id; + + if (program->functions[function_index].flags & DOLCFG_FUNC_PATCHABLE) + region->patchable = true; + + for (u32 i = fb->offsets[function_index]; + i < fb->offsets[function_index + 1u]; i++) { + if (!region_push_block(region, program, plan, fb->blocks[i])) + return false; + } + region->estimated_ir_instructions = + region->instruction_count * DOLREGION_IR_PER_GUEST_INSN; + return true; +} + +/* A function too large for one region is cut at block boundaries. SCC members + are kept together: cutting a loop in half is the single worst place to put a + boundary, so the cut slides forward to the end of the SCC. A hard ceiling of + twice the limit stops one enormous SCC from swallowing everything. */ +static bool split_large_function(DolRegionPlan* plan, const DolCfgProgram* program, + const FunctionBlocks* fb, u32 function_index, + const DolRegionLimits* limits) { + u32 first = fb->offsets[function_index]; + u32 last = fb->offsets[function_index + 1u]; + u32 ceiling = limits->max_instructions * 2u; + + DolRegion* region = NULL; + for (u32 i = first; i < last; i++) { + u32 block_index = fb->blocks[i]; + const DolCfgBlock* block = &program->blocks[block_index]; + + if (region && region->instruction_count >= limits->max_instructions) { + bool scc_open = false; + if (block->scc != DOLCFG_NO_BLOCK && i > first) { + u32 previous = fb->blocks[i - 1u]; + scc_open = program->blocks[previous].scc == block->scc; + } + if (!scc_open || region->instruction_count >= ceiling) { + region->end_reason = scc_open ? DOLREGION_END_SIZE_LIMIT + : DOLREGION_END_SIZE_LIMIT; + region = NULL; + } + } + + if (!region) { + region = new_region(plan); + if (!region) + return false; + u32* grown = (u32*)realloc(region->functions, sizeof(u32)); + if (!grown) + return false; + region->functions = grown; + region->functions[region->function_count++] = function_index; + if (program->functions[function_index].flags & DOLCFG_FUNC_PATCHABLE) + region->patchable = true; + } + + if (!region_push_block(region, program, plan, block_index)) + return false; + region->estimated_ir_instructions = + region->instruction_count * DOLREGION_IR_PER_GUEST_INSN; + } + + if (plan->function_region[function_index] == DOLCFG_NO_BLOCK && region) + plan->function_region[function_index] = region->id; + plan->split_functions++; + return true; +} + +/* --- modes ---------------------------------------------------------------- */ + +static bool plan_fixed(DolRegionPlan* plan, const DolCfgProgram* program, + const DolRegionLimits* limits) { + /* Deliberately CFG-blind: cut every N guest instructions in address order. + This is the comparison arm, so it must not quietly benefit from any of + the analysis the other modes use. */ + DolRegion* region = NULL; + u32 previous_end = 0xFFFFFFFFu; + + for (u32 i = 0; i < program->block_count; i++) { + const DolCfgBlock* block = &program->blocks[i]; + + bool discontiguous = (previous_end != 0xFFFFFFFFu) && + (block->start != previous_end); + if (region && + (region->instruction_count >= limits->max_instructions || discontiguous)) { + region->end_reason = discontiguous ? DOLREGION_END_SECTION_END + : DOLREGION_END_FIXED_CHUNK; + region = NULL; + } + if (!region) { + region = new_region(plan); + if (!region) + return false; + } + if (!region_push_block(region, program, plan, i)) + return false; + region->estimated_ir_instructions = + region->instruction_count * DOLREGION_IR_PER_GUEST_INSN; + if (block->function != DOLCFG_NO_BLOCK && + plan->function_region[block->function] == DOLCFG_NO_BLOCK) { + plan->function_region[block->function] = region->id; + } + previous_end = block->end; + } + return true; +} + +static bool plan_function(DolRegionPlan* plan, const DolCfgProgram* program, + const FunctionBlocks* fb, + const DolRegionLimits* limits) { + for (u32 f = 0; f < program->function_count; f++) { + if (program->functions[f].block_count == 0) + continue; + if (program->functions[f].instruction_count > limits->max_instructions) { + if (!split_large_function(plan, program, fb, f, limits)) + return false; + continue; + } + DolRegion* region = new_region(plan); + if (!region) + return false; + if (!region_push_function(region, program, plan, fb, f)) + return false; + region->end_reason = DOLREGION_END_FUNCTION_BOUNDARY; + } + return true; +} + +typedef struct { + u32 address; + u32 function; +} FunctionByAddress; + +static int compare_function_by_address(const void* a, const void* b) { + const FunctionByAddress* left = (const FunctionByAddress*)a; + const FunctionByAddress* right = (const FunctionByAddress*)b; + if (left->address != right->address) + return left->address < right->address ? -1 : 1; + return (left->function > right->function) - (left->function < right->function); +} + +/* Lowest-addressed unassigned function starting at or after `address`, within + `gap` bytes of it. DOLCFG_NO_BLOCK when there is none. */ +static u32 next_adjacent_function(const DolCfgProgram* program, + const DolRegionPlan* plan, + const FunctionByAddress* order, u32 order_count, + u32 address, u32 gap) { + u32 low = 0; + u32 high = order_count; + while (low < high) { + u32 mid = low + (high - low) / 2u; + if (order[mid].address < address) + low = mid + 1u; + else + high = mid; + } + for (u32 i = low; i < order_count; i++) { + if (order[i].address > address + gap) + return DOLCFG_NO_BLOCK; + u32 function = order[i].function; + if (plan->function_region[function] != DOLCFG_NO_BLOCK) + continue; + if (program->functions[function].block_count == 0) + continue; + return function; + } + return DOLCFG_NO_BLOCK; +} + +static bool plan_accretive(DolRegionPlan* plan, const DolCfgProgram* program, + const FunctionBlocks* fb, const CallGraph* graph, + const DolRegionLimits* limits, bool use_weights) { + u64* candidate_weight = (u64*)calloc( + program->function_count ? program->function_count : 1u, sizeof(u64)); + u8* is_candidate = (u8*)calloc( + program->function_count ? program->function_count : 1u, sizeof(u8)); + u32* touched = (u32*)malloc( + (program->function_count ? program->function_count : 1u) * sizeof(u32)); + FunctionByAddress* order = (FunctionByAddress*)malloc( + (program->function_count ? program->function_count : 1u) * sizeof(*order)); + if (!candidate_weight || !is_candidate || !touched || !order) { + free(candidate_weight); free(is_candidate); free(touched); free(order); + return false; + } + for (u32 i = 0; i < program->function_count; i++) { + order[i].address = program->functions[i].entry_address; + order[i].function = i; + } + qsort(order, program->function_count, sizeof(*order), + compare_function_by_address); + + /* Seed order. + * + * Address order is right for cfg mode: it is deterministic and no function + * deserves priority. With a profile it is actively wrong. Accretion is + * greedy, so whichever region forms first takes the shared neighbours, and + * in address order that is decided by link layout rather than by what + * executes. + * + * The measured sweep is the argument: merging on all edges equally cut + * static crossings 21% and moved the runtime dispatcher rate 0.8%, because + * uniform merging removes overwhelmingly cold boundaries. Hot code has to + * choose first for merging to reach the boundaries that actually execute. + * + * Ties break on function index, so the plan stays reproducible. */ + u32* seed_order = (u32*)malloc( + (program->function_count ? program->function_count : 1u) * sizeof(u32)); + if (!seed_order) { + free(candidate_weight); free(is_candidate); free(touched); free(order); + return false; + } + for (u32 i = 0; i < program->function_count; i++) + seed_order[i] = i; + if (use_weights) { + /* Insertion sort by descending weight: the array is already in address + order and a profile makes only a small fraction non-zero, so this + stays close to linear in practice. */ + for (u32 i = 1; i < program->function_count; i++) { + u32 key = seed_order[i]; + u64 key_weight = program->functions[key].weight; + u32 j = i; + while (j > 0 && + program->functions[seed_order[j - 1u]].weight < key_weight) { + seed_order[j] = seed_order[j - 1u]; + j--; + } + seed_order[j] = key; + } + } + + for (u32 s = 0; s < program->function_count; s++) { + u32 seed = seed_order[s]; + if (plan->function_region[seed] != DOLCFG_NO_BLOCK) + continue; + if (program->functions[seed].block_count == 0) + continue; + + if (program->functions[seed].instruction_count > limits->max_instructions) { + if (!split_large_function(plan, program, fb, seed, limits)) { + free(candidate_weight); free(is_candidate); free(touched); + free(order); free(seed_order); + return false; + } + continue; + } + + DolRegion* region = new_region(plan); + if (!region) { + free(candidate_weight); free(is_candidate); free(touched); + return false; + } + if (!region_push_function(region, program, plan, fb, seed)) { + free(candidate_weight); free(is_candidate); free(touched); + return false; + } + + u32 touched_count = 0; + DolRegionEndReason reason = DOLREGION_END_NO_CANDIDATE; + + /* Seed the candidate frontier from the seed function's neighbours. */ + for (u32 e = graph->offsets[seed]; e < graph->offsets[seed + 1u]; e++) { + u32 target = graph->targets[e]; + if (plan->function_region[target] != DOLCFG_NO_BLOCK) + continue; + if (!is_candidate[target]) { + is_candidate[target] = 1; + touched[touched_count++] = target; + } + candidate_weight[target] += graph->weights[e]; + } + + for (;;) { + if (region->function_count >= limits->max_functions) { + reason = DOLREGION_END_SIZE_LIMIT; + break; + } + + u32 best = DOLCFG_NO_BLOCK; + u64 best_weight = 0; + bool blocked_by_size = false; + bool blocked_by_cold = false; + + /* Address order with a strict > comparison makes ties resolve to + the lowest-addressed candidate, which is what keeps the plan + reproducible. */ + for (u32 c = 0; c < touched_count; c++) { + u32 candidate = touched[c]; + if (!is_candidate[candidate]) + continue; + if (plan->function_region[candidate] != DOLCFG_NO_BLOCK) + continue; + + const DolCfgFunction* fn = &program->functions[candidate]; + if (fn->block_count == 0) + continue; + + if (region->instruction_count + fn->instruction_count > + limits->max_instructions) { + blocked_by_size = true; + continue; + } + if ((region->instruction_count + fn->instruction_count) * + DOLREGION_IR_PER_GUEST_INSN > + limits->max_ir_instructions) { + blocked_by_size = true; + continue; + } + /* Cold code does not belong in a hot region: merging it pays + the region's code-size budget for something that does not + run. Only meaningful when a profile is loaded. */ + if (use_weights && region->weight > 0 && + fn->weight < limits->cold_weight_threshold) { + blocked_by_cold = true; + continue; + } + + if (candidate_weight[candidate] > best_weight) { + best_weight = candidate_weight[candidate]; + best = candidate; + } + } + + /* The call graph ran dry but the budget did not. Keep growing + along addresses rather than closing a region at a tenth of its + limit -- merging adjacent code adds no crossing and keeps the + region a single contiguous run. */ + if (best == DOLCFG_NO_BLOCK && limits->merge_address_adjacent) { + u32 next = next_adjacent_function( + program, plan, order, program->function_count, + region->guest_end, limits->max_adjacency_gap); + if (next != DOLCFG_NO_BLOCK) { + const DolCfgFunction* fn = &program->functions[next]; + u32 grown = region->instruction_count + fn->instruction_count; + if (grown > limits->max_instructions || + grown * DOLREGION_IR_PER_GUEST_INSN > + limits->max_ir_instructions) { + blocked_by_size = true; + } else if (use_weights && region->weight > 0 && + fn->weight < limits->cold_weight_threshold) { + blocked_by_cold = true; + } else { + best = next; + } + } + } + + if (best == DOLCFG_NO_BLOCK) { + reason = blocked_by_size ? DOLREGION_END_SIZE_LIMIT + : blocked_by_cold ? DOLREGION_END_COLD + : DOLREGION_END_NO_CANDIDATE; + break; + } + + if (!region_push_function(region, program, plan, fb, best)) { + free(candidate_weight); free(is_candidate); free(touched); + free(order); free(seed_order); + return false; + } + is_candidate[best] = 0; + + for (u32 e = graph->offsets[best]; e < graph->offsets[best + 1u]; e++) { + u32 target = graph->targets[e]; + if (plan->function_region[target] != DOLCFG_NO_BLOCK) + continue; + if (!is_candidate[target]) { + is_candidate[target] = 1; + touched[touched_count++] = target; + } + candidate_weight[target] += graph->weights[e]; + } + } + + region->end_reason = reason; + for (u32 c = 0; c < touched_count; c++) { + is_candidate[touched[c]] = 0; + candidate_weight[touched[c]] = 0; + } + } + + free(candidate_weight); + free(is_candidate); + free(touched); + free(order); + free(seed_order); + return true; +} + +/* --- finalisation --------------------------------------------------------- */ + +static void finalise_regions(DolRegionPlan* plan, const DolCfgProgram* program) { + plan->total_instructions = 0; + plan->cross_region_edges = 0; + + for (u32 r = 0; r < plan->region_count; r++) { + DolRegion* region = &plan->regions[r]; + plan->total_instructions += region->instruction_count; + + u32* sccs = (u32*)malloc((region->block_count ? region->block_count : 1u) * + sizeof(u32)); + u32 scc_total = 0; + + for (u32 i = 0; i < region->block_count; i++) { + u32 index = region->blocks[i]; + const DolCfgBlock* block = &program->blocks[index]; + + if (sccs && block->scc != DOLCFG_NO_BLOCK) + sccs[scc_total++] = block->scc; + + for (u32 s = 0; s < block->successor_count; s++) { + u32 next = block->successors[s]; + if (next == DOLCFG_NO_BLOCK) { + /* An edge leaving the model still leaves the region. */ + region->out_edges++; + continue; + } + if (plan->block_region[next] == region->id) + region->internal_edges++; + else + region->out_edges++; + } + + /* The call edge is counted separately because a CALL block's + successor is its *return point*, not its callee -- so walking + successors alone never sees the call at all. On Mario Kart that + would hide 40,316 of the transfers the plan exists to remove, + and co-locating a caller with its callee would score as no + improvement whatsoever. */ + if ((block->terminator == DOLCFG_TERM_CALL || + block->terminator == DOLCFG_TERM_TAIL_CALL) && + block->call_target) { + u32 target = dolcfg_block_starting_at(program, block->call_target); + if (target == DOLCFG_NO_BLOCK || + plan->block_region[target] != region->id) { + region->out_edges++; + } else { + region->internal_edges++; + } + } + } + + if (sccs) { + qsort(sccs, scc_total, sizeof(u32), compare_u32_asc); + u32 distinct = 0; + for (u32 i = 0; i < scc_total; i++) { + if (i == 0 || sccs[i] != sccs[i - 1u]) + distinct++; + } + region->scc_count = distinct; + free(sccs); + } + plan->cross_region_edges += region->out_edges; + } + + /* In-edges are the mirror of everyone else's out-edges. */ + for (u32 i = 0; i < program->block_count; i++) { + const DolCfgBlock* block = &program->blocks[i]; + u32 from = plan->block_region[i]; + if (from == DOLCFG_NO_BLOCK) + continue; + for (u32 s = 0; s < block->successor_count; s++) { + u32 next = block->successors[s]; + if (next == DOLCFG_NO_BLOCK) + continue; + u32 to = plan->block_region[next]; + if (to != DOLCFG_NO_BLOCK && to != from) + plan->regions[to].in_edges++; + } + if ((block->terminator == DOLCFG_TERM_CALL || + block->terminator == DOLCFG_TERM_TAIL_CALL) && + block->call_target) { + u32 target = dolcfg_block_starting_at(program, block->call_target); + if (target != DOLCFG_NO_BLOCK) { + u32 to = plan->block_region[target]; + if (to != DOLCFG_NO_BLOCK && to != from) + plan->regions[to].in_edges++; + } + } + } +} + +bool dolregion_plan_build(DolRegionPlan* plan, const DolCfgProgram* program, + DolRegionMode mode, const DolRegionLimits* limits, + FILE* diagnostics) { + if (!plan || !program || !limits) + return false; + + dolregion_plan_free(plan); + plan->mode = mode; + plan->limits = *limits; + + plan->block_region = (u32*)malloc( + (program->block_count ? program->block_count : 1u) * sizeof(u32)); + plan->function_region = (u32*)malloc( + (program->function_count ? program->function_count : 1u) * sizeof(u32)); + if (!plan->block_region || !plan->function_region) { + if (diagnostics) + fprintf(diagnostics, "error: out of memory planning regions\n"); + return false; + } + for (u32 i = 0; i < program->block_count; i++) + plan->block_region[i] = DOLCFG_NO_BLOCK; + for (u32 i = 0; i < program->function_count; i++) + plan->function_region[i] = DOLCFG_NO_BLOCK; + + FunctionBlocks fb; + memset(&fb, 0, sizeof(fb)); + CallGraph graph; + memset(&graph, 0, sizeof(graph)); + bool ok = true; + + if (mode != DOLREGION_MODE_FIXED) { + if (!build_function_blocks(program, &fb)) { + if (diagnostics) + fprintf(diagnostics, "error: out of memory grouping blocks\n"); + free_function_blocks(&fb); + return false; + } + } + + switch (mode) { + case DOLREGION_MODE_FIXED: + ok = plan_fixed(plan, program, limits); + break; + case DOLREGION_MODE_FUNCTION: + ok = plan_function(plan, program, &fb, limits); + break; + case DOLREGION_MODE_CFG: + case DOLREGION_MODE_PGO: { + bool any_weight = false; + for (u32 i = 0; i < program->block_count && !any_weight; i++) { + if (program->blocks[i].weight != 0) + any_weight = true; + } + if (mode == DOLREGION_MODE_PGO && !any_weight) { + /* Degrading silently would make an unprofiled build look profiled, + which is the specific failure the PGO staleness gate exists to + prevent elsewhere. Say it. */ + plan->profile_missing = true; + if (diagnostics) { + fprintf(diagnostics, + "warning: --region-mode pgo with no profile weights; " + "falling back to cfg ordering\n"); + } + } + if (!build_call_graph(program, &graph)) { + if (diagnostics) + fprintf(diagnostics, "error: out of memory building call graph\n"); + free_function_blocks(&fb); + return false; + } + ok = plan_accretive(plan, program, &fb, &graph, limits, + mode == DOLREGION_MODE_PGO && any_weight); + break; + } + default: + ok = false; + break; + } + + free_call_graph(&graph); + free_function_blocks(&fb); + + if (!ok) { + if (diagnostics) + fprintf(diagnostics, "error: region planning failed\n"); + return false; + } + + finalise_regions(plan, program); + return true; +} + +/* --- report --------------------------------------------------------------- */ + +bool dolregion_write_report(const DolRegionPlan* plan, + const DolCfgProgram* program, const char* path, + FILE* diagnostics) { + if (!plan || !program || !path) + return false; + + FILE* out = fopen(path, "wb"); + if (!out) { + if (diagnostics) + fprintf(diagnostics, "error: cannot write region report '%s'\n", path); + return false; + } + + fputs("{\n", out); + fputs(" \"schema\": \"dolrecomp.regions/1\",\n", out); + fprintf(out, " \"mode\": \"%s\",\n", dolregion_mode_name(plan->mode)); + fprintf(out, " \"profile_missing\": %s,\n", + plan->profile_missing ? "true" : "false"); + fputs(" \"limits\": {\n", out); + fprintf(out, " \"max_instructions\": %u,\n", plan->limits.max_instructions); + fprintf(out, " \"max_ir_instructions\": %u,\n", + plan->limits.max_ir_instructions); + fprintf(out, " \"max_functions\": %u,\n", plan->limits.max_functions); + fprintf(out, " \"cold_weight_threshold\": %llu\n", + (unsigned long long)plan->limits.cold_weight_threshold); + fputs(" },\n", out); + fputs(" \"totals\": {\n", out); + fprintf(out, " \"regions\": %u,\n", plan->region_count); + fprintf(out, " \"instructions\": %u,\n", plan->total_instructions); + fprintf(out, " \"split_functions\": %u,\n", plan->split_functions); + fprintf(out, " \"cross_region_edges\": %u\n", plan->cross_region_edges); + fputs(" },\n", out); + + fputs(" \"regions\": [\n", out); + for (u32 r = 0; r < plan->region_count; r++) { + const DolRegion* region = &plan->regions[r]; + fprintf(out, + " {\"id\": %u, \"start\": \"0x%08X\", \"end\": \"0x%08X\", " + "\"instructions\": %u, \"ir_estimate\": %u, \"blocks\": %u, " + "\"functions\": %u, \"loops\": %u, \"sccs\": %u, " + "\"in_edges\": %u, \"out_edges\": %u, \"internal_edges\": %u, " + "\"indirect_sites\": %u, \"weight\": %llu, " + "\"patchable\": %s, \"smc\": %s, \"end_reason\": \"%s\"", + region->id, region->guest_start, region->guest_end, + region->instruction_count, region->estimated_ir_instructions, + region->block_count, region->function_count, region->loop_count, + region->scc_count, region->in_edges, region->out_edges, + region->internal_edges, region->indirect_sites, + (unsigned long long)region->weight, + region->patchable ? "true" : "false", + region->contains_smc ? "true" : "false", + dolregion_end_reason_name(region->end_reason)); + + fputs(", \"function_addresses\": [", out); + for (u32 f = 0; f < region->function_count; f++) { + fprintf(out, "%s\"0x%08X\"", f ? ", " : "", + program->functions[region->functions[f]].entry_address); + } + fputs("]}", out); + fputs(r + 1u < plan->region_count ? ",\n" : "\n", out); + } + fputs(" ]\n", out); + fputs("}\n", out); + + if (fclose(out) != 0) { + if (diagnostics) + fprintf(diagnostics, "error: failed to close region report '%s'\n", path); + return false; + } + return true; +} diff --git a/src/analysis/regions.h b/src/analysis/regions.h new file mode 100644 index 0000000..0e5c94d --- /dev/null +++ b/src/analysis/regions.h @@ -0,0 +1,174 @@ +#ifndef DOLRECOMP_ANALYSIS_REGIONS_H +#define DOLRECOMP_ANALYSIS_REGIONS_H + +/* Deterministic region planning. + * + * A region is the unit the LLVM backend compiles as one module: control flow + * inside it can be a native branch, and only leaving it costs a state + * materialization. The fixed-chunk model cuts every N instructions, so a + * boundary lands wherever it lands. The point of planning is to put boundaries + * where control flow is *already* leaving. + * + * Membership is by whole function, not by block. Splitting a function across + * regions reintroduces the exact cost being removed -- a live-state handoff in + * the middle of straight-line code -- so a function is only ever split when it + * alone exceeds the size limit, and then at block boundaries that keep SCCs + * intact. + * + * Every mode is deterministic: functions are visited in address order and ties + * break on address, so the same inputs and settings always produce the same + * plan with the same region numbering. + */ + +#include "analysis/cfg.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + /* Reproduces the current backend: fixed-size cuts, no CFG awareness. + Kept for comparison and as a fallback, never removed. */ + DOLREGION_MODE_FIXED, + /* One region per function, subject to the size limit. */ + DOLREGION_MODE_FUNCTION, + /* Accretes connected functions while they fit, following the call graph. */ + DOLREGION_MODE_CFG, + /* CFG accretion ordered by profile weight, keeping cold code out of hot + regions. Falls back to CFG ordering when no weights are loaded, and says + so in the report rather than silently pretending to be profiled. */ + DOLREGION_MODE_PGO, +} DolRegionMode; + +/* Why a region stopped growing. Reported per region so a plan can be argued + with rather than taken on faith. */ +typedef enum { + DOLREGION_END_SIZE_LIMIT, + DOLREGION_END_IR_LIMIT, + DOLREGION_END_NO_CANDIDATE, + DOLREGION_END_INDIRECT, + DOLREGION_END_SMC, + DOLREGION_END_PATCHABLE, + DOLREGION_END_COLD, + DOLREGION_END_FUNCTION_BOUNDARY, + DOLREGION_END_FIXED_CHUNK, + DOLREGION_END_SECTION_END, +} DolRegionEndReason; + +typedef struct { + /* Guest instructions per region. The dominant control. */ + u32 max_instructions; + /* Estimated DolIR instructions per region. Estimated, not measured: the IR + does not exist at plan time. See DOLREGION_IR_PER_GUEST_INSN. */ + u32 max_ir_instructions; + /* Functions per region, a guard against pathological accretion. */ + u32 max_functions; + /* Below this weight a function is cold and is not merged into a hot + region. Only consulted in PGO mode. */ + u64 cold_weight_threshold; + + /* When the call graph runs dry before the size budget does, keep growing + along addresses instead of closing the region. + * + * Without this, accretion is connectivity-bound rather than size-bound: a + * function reached only indirectly that itself calls nothing has no + * call-graph neighbours at all, so it becomes a region of one. Measured on + * Luigi's Mansion, regions averaged 70 instructions against a limit of + * 1024, and the plan emitted 7,520 compilation units where the fixed arm + * needed 909. + * + * Merging address-adjacent code is cheap and safe: it never adds a + * crossing, it keeps runs contiguous so a region stays one run instead of + * several, and functions laid out next to each other generally came from + * the same translation unit. */ + int merge_address_adjacent; + /* How far past a region's end the next function may start and still be + treated as adjacent. Covers alignment padding and small data holes; + beyond it the code is unrelated and merging only costs size. */ + u32 max_adjacency_gap; +} DolRegionLimits; + +/* Rough DolIR instructions emitted per guest instruction. Used only to apply + max_ir_instructions at plan time; Phase 2 replaces it with the real count + once regions are actually lowered. */ +#define DOLREGION_IR_PER_GUEST_INSN 6u + +typedef struct { + u32 id; + u32 guest_start; + u32 guest_end; + + /* Owned blocks, ascending. Indices into DolCfgProgram::blocks. */ + u32* blocks; + u32 block_count; + + /* Owned function indices, ascending. */ + u32* functions; + u32 function_count; + + u32 instruction_count; + u32 estimated_ir_instructions; + u32 loop_count; + u32 scc_count; + + /* Edges crossing the region boundary. in_edges is what must be able to + enter; out_edges is what costs a transfer. */ + u32 in_edges; + u32 out_edges; + /* Edges that stayed inside, which is the number being maximised. */ + u32 internal_edges; + + u32 indirect_sites; + u64 weight; + bool patchable; + bool contains_smc; + + DolRegionEndReason end_reason; +} DolRegion; + +typedef struct { + DolRegion* regions; + u32 region_count; + u32 region_capacity; + + /* block index -> region id, or DOLCFG_NO_BLOCK. */ + u32* block_region; + /* function index -> region id, or DOLCFG_NO_BLOCK. */ + u32* function_region; + + DolRegionMode mode; + DolRegionLimits limits; + + /* True when PGO mode ran without any weights and degraded to CFG order. */ + bool profile_missing; + + /* Totals, for the report and the perf counters. */ + u32 total_instructions; + u32 split_functions; + u32 cross_region_edges; +} DolRegionPlan; + +void dolregion_default_limits(DolRegionLimits* limits); +bool dolregion_parse_mode(const char* text, DolRegionMode* mode); +const char* dolregion_mode_name(DolRegionMode mode); +const char* dolregion_end_reason_name(DolRegionEndReason reason); + +void dolregion_plan_init(DolRegionPlan* plan); +void dolregion_plan_free(DolRegionPlan* plan); + +/* Builds the plan. `program` must already be built. */ +bool dolregion_plan_build(DolRegionPlan* plan, const DolCfgProgram* program, + DolRegionMode mode, const DolRegionLimits* limits, + FILE* diagnostics); + +/* Writes the machine-readable region report. */ +bool dolregion_write_report(const DolRegionPlan* plan, + const DolCfgProgram* program, const char* path, + FILE* diagnostics); + +#ifdef __cplusplus +} +#endif + +#endif /* DOLRECOMP_ANALYSIS_REGIONS_H */ diff --git a/src/app/cli.c b/src/app/cli.c index 33d3c23..a2cd74c 100644 --- a/src/app/cli.c +++ b/src/app/cli.c @@ -15,10 +15,17 @@ void print_usage(const char* argv0) { fprintf(stderr, "Options:\n"); fprintf(stderr, " -jN Use N worker jobs for split C output (e.g. -j14)\n"); fprintf(stderr, " --cpu gekko|broadway|espresso Select CPU profile (default: broadway)\n"); - fprintf(stderr, " --backend c|llvm Select generated-code backend (default: c)\n"); + fprintf(stderr, " --backend c|llvm|llvm-aot Select generated-code backend (default: c)\n"); + fprintf(stderr, " --region-mode MODE fixed|function|cfg|pgo for llvm-aot (default: cfg)\n"); + fprintf(stderr, " --region-max-instructions N Guest instructions per region\n"); + fprintf(stderr, " --region-max-ir N Estimated DolIR instructions per region\n"); + fprintf(stderr, " --region-profile Execution weights for --region-mode pgo\n"); + fprintf(stderr, " --emit-region-report Write the region plan as JSON\n"); + fprintf(stderr, " --lto off|thin Emit ThinLTO bitcode beside each region object\n"); fprintf(stderr, " --gamecube GameCube mode (no title ID required)\n"); fprintf(stderr, " --rel-base Override first virtual load address for REL codegen\n"); fprintf(stderr, " --map Load optional function names from a linker MAP\n"); + fprintf(stderr, " --perf-report Write a JSON build/runtime counter report\n"); fprintf(stderr, " --setup Download titles database and optionally install wit\n"); fprintf(stderr, "\n"); fprintf(stderr, "Examples:\n"); @@ -128,6 +135,7 @@ int parse_u32_arg(const char* text, const char* name, u32* value_out) { int parse_cli(int argc, char** argv, CliOptions* opts) { const char* positional[3]; int positional_count = 0; + int backend_from_cli = 0; memset(opts, 0, sizeof(*opts)); opts->cpu = DOLRECOMP_CPU_GEKKO; @@ -155,7 +163,7 @@ int parse_cli(int argc, char** argv, CliOptions* opts) { if (strcmp(arg, "--backend") == 0) { if (i + 1 >= argc) { - fprintf(stderr, "error: --backend needs c or llvm\n"); + fprintf(stderr, "error: --backend needs c, llvm, or llvm-aot\n"); return 0; } arg = argv[++i]; @@ -163,10 +171,14 @@ int parse_cli(int argc, char** argv, CliOptions* opts) { opts->backend = DOLRECOMP_BACKEND_C; else if (ascii_case_equal(arg, "llvm")) opts->backend = DOLRECOMP_BACKEND_LLVM; + else if (ascii_case_equal(arg, "llvm-aot") || + ascii_case_equal(arg, "llvm-regions")) + opts->backend = DOLRECOMP_BACKEND_LLVM_AOT; else { fprintf(stderr, "error: unknown backend '%s'\n", arg); return 0; } + backend_from_cli = 1; continue; } @@ -176,10 +188,14 @@ int parse_cli(int argc, char** argv, CliOptions* opts) { opts->backend = DOLRECOMP_BACKEND_C; else if (ascii_case_equal(name, "llvm")) opts->backend = DOLRECOMP_BACKEND_LLVM; + else if (ascii_case_equal(name, "llvm-aot") || + ascii_case_equal(name, "llvm-regions")) + opts->backend = DOLRECOMP_BACKEND_LLVM_AOT; else { fprintf(stderr, "error: unknown backend '%s'\n", name); return 0; } + backend_from_cli = 1; continue; } @@ -281,6 +297,126 @@ int parse_cli(int argc, char** argv, CliOptions* opts) { continue; } + if (strcmp(arg, "--region-mode") == 0) { + if (i + 1 >= argc) { + fprintf(stderr, "error: --region-mode needs fixed, function, cfg, or pgo\n"); + return 0; + } + opts->region_mode_arg = argv[++i]; + continue; + } + + if (strncmp(arg, "--region-mode=", 14) == 0) { + opts->region_mode_arg = arg + 14; + continue; + } + + if (strcmp(arg, "--region-max-instructions") == 0) { + if (i + 1 >= argc || + !parse_u32_arg(argv[++i], "--region-max-instructions", + &opts->region_max_instructions)) + return 0; + continue; + } + + if (strcmp(arg, "--lto") == 0) { + if (i + 1 >= argc) { + fprintf(stderr, "error: --lto needs off or thin\n"); + return 0; + } + opts->lto_mode_arg = argv[++i]; + if (strcmp(opts->lto_mode_arg, "off") && + strcmp(opts->lto_mode_arg, "thin")) { + fprintf(stderr, "error: --lto must be off or thin\n"); + return 0; + } + continue; + } + + if (strncmp(arg, "--lto=", 6) == 0) { + opts->lto_mode_arg = arg + 6; + if (strcmp(opts->lto_mode_arg, "off") && + strcmp(opts->lto_mode_arg, "thin")) { + fprintf(stderr, "error: --lto must be off or thin\n"); + return 0; + } + continue; + } + + /* Unlike the region options this applies to every backend, so it is + deliberately not behind the --backend llvm-aot guard. */ + if (strcmp(arg, "--memory-mode") == 0 || + strncmp(arg, "--memory-mode=", 14) == 0) { + if (arg[13] == '=') { + opts->memory_mode_arg = arg + 14; + } else { + if (i + 1 >= argc) { + fprintf(stderr, "error: --memory-mode needs safe or fast\n"); + return 0; + } + opts->memory_mode_arg = argv[++i]; + } + if (strcmp(opts->memory_mode_arg, "safe") && + strcmp(opts->memory_mode_arg, "fast")) { + fprintf(stderr, "error: --memory-mode must be safe or fast\n"); + return 0; + } + continue; + } + + if (strcmp(arg, "--region-profile") == 0) { + if (i + 1 >= argc) { + fprintf(stderr, "error: --region-profile needs a path\n"); + return 0; + } + opts->region_profile_path = argv[++i]; + continue; + } + + if (strncmp(arg, "--region-profile=", 17) == 0) { + opts->region_profile_path = arg + 17; + continue; + } + + if (strcmp(arg, "--region-max-ir") == 0) { + if (i + 1 >= argc || + !parse_u32_arg(argv[++i], "--region-max-ir", &opts->region_max_ir)) + return 0; + continue; + } + + if (strcmp(arg, "--emit-region-report") == 0) { + if (i + 1 >= argc) { + fprintf(stderr, "error: --emit-region-report needs a path\n"); + return 0; + } + opts->region_report_path = argv[++i]; + continue; + } + + if (strncmp(arg, "--emit-region-report=", 21) == 0) { + opts->region_report_path = arg + 21; + continue; + } + + if (strcmp(arg, "--perf-report") == 0) { + if (i + 1 >= argc) { + fprintf(stderr, "error: --perf-report needs a path\n"); + return 0; + } + opts->perf_report_path = argv[++i]; + continue; + } + + if (strncmp(arg, "--perf-report=", 14) == 0) { + if (arg[14] == '\0') { + fprintf(stderr, "error: --perf-report needs a path\n"); + return 0; + } + opts->perf_report_path = arg + 14; + continue; + } + if (arg[0] == '-' && arg[1] != '\0') { fprintf(stderr, "error: unknown option '%s'\n", arg); return 0; @@ -293,6 +429,100 @@ int parse_cli(int argc, char** argv, CliOptions* opts) { positional[positional_count++] = arg; } + /* Environment fallbacks. + * + * Applied only where the command line said nothing, so an explicit flag + * always wins -- that is the documented precedence. + * + * These exist because moderngekko-port drives a sibling dolrecomp and only + * forwards --backend=c|llvm, which it validates. Without an out-of-band + * channel there is no way to build an AOT module through the existing port + * tool, and patching ModernGekko to pass a flag through would couple the two + * repositories for what is a benchmarking concern. */ + /* DOLRECOMP_FORCE_BACKEND overrides even an explicit flag. + * + * It exists for exactly one situation: a caller that hardcodes --backend + * and validates it against its own list. moderngekko-port does both, so + * the polite fallback below never fires for it. Naming the override + * separately keeps the ordinary variable honest -- a script that sets + * DOLRECOMP_BACKEND still cannot silently change what a build asked for. */ + const char* forced_backend = getenv("DOLRECOMP_FORCE_BACKEND"); + if (forced_backend && *forced_backend) { + if (ascii_case_equal(forced_backend, "c")) { + opts->backend = DOLRECOMP_BACKEND_C; + } else if (ascii_case_equal(forced_backend, "llvm")) { + opts->backend = DOLRECOMP_BACKEND_LLVM; + } else if (ascii_case_equal(forced_backend, "llvm-aot") || + ascii_case_equal(forced_backend, "llvm-regions")) { + opts->backend = DOLRECOMP_BACKEND_LLVM_AOT; + } else { + fprintf(stderr, "error: unknown DOLRECOMP_FORCE_BACKEND '%s'\n", + forced_backend); + return 0; + } + backend_from_cli = 1; + } + + if (!backend_from_cli) { + const char* env_backend = getenv("DOLRECOMP_BACKEND"); + if (env_backend && *env_backend) { + if (ascii_case_equal(env_backend, "c")) { + opts->backend = DOLRECOMP_BACKEND_C; + } else if (ascii_case_equal(env_backend, "llvm")) { + opts->backend = DOLRECOMP_BACKEND_LLVM; + } else if (ascii_case_equal(env_backend, "llvm-aot") || + ascii_case_equal(env_backend, "llvm-regions")) { + opts->backend = DOLRECOMP_BACKEND_LLVM_AOT; + } else { + fprintf(stderr, "error: unknown DOLRECOMP_BACKEND '%s'\n", env_backend); + return 0; + } + } + } + if (!opts->region_mode_arg) { + const char* value = getenv("DOLRECOMP_REGION_MODE"); + if (value && *value) + opts->region_mode_arg = value; + } + if (!opts->region_max_instructions) { + const char* value = getenv("DOLRECOMP_REGION_MAX_INSTRUCTIONS"); + if (value && *value && + !parse_u32_arg(value, "DOLRECOMP_REGION_MAX_INSTRUCTIONS", + &opts->region_max_instructions)) + return 0; + } + if (!opts->region_max_ir) { + const char* value = getenv("DOLRECOMP_REGION_MAX_IR"); + if (value && *value && + !parse_u32_arg(value, "DOLRECOMP_REGION_MAX_IR", &opts->region_max_ir)) + return 0; + } + if (!opts->lto_mode_arg) { + const char* value = getenv("DOLRECOMP_LTO"); + if (value && *value) + opts->lto_mode_arg = value; + } + if (!opts->memory_mode_arg) { + const char* value = getenv("DOLRECOMP_MEMORY_MODE"); + if (value && *value) + opts->memory_mode_arg = value; + } + if (!opts->region_profile_path) { + const char* value = getenv("DOLRECOMP_REGION_PROFILE"); + if (value && *value) + opts->region_profile_path = value; + } + if (!opts->region_report_path) { + const char* value = getenv("DOLRECOMP_REGION_REPORT"); + if (value && *value) + opts->region_report_path = value; + } + if (!opts->perf_report_path) { + const char* value = getenv("DOLRECOMP_PERF_REPORT"); + if (value && *value) + opts->perf_report_path = value; + } + if (positional_count == 0) { if (opts->setup_mode) return 1; @@ -311,7 +541,8 @@ int parse_cli(int argc, char** argv, CliOptions* opts) { } #ifndef DOLRECOMP_ENABLE_LLVM - if (opts->backend == DOLRECOMP_BACKEND_LLVM) { + if (opts->backend == DOLRECOMP_BACKEND_LLVM || + opts->backend == DOLRECOMP_BACKEND_LLVM_AOT) { fprintf(stderr, "error: LLVM backend is not built; configure with -DDOLRECOMP_ENABLE_LLVM=ON\n"); return 0; } diff --git a/src/app/cli.h b/src/app/cli.h index 73706f2..bbb67c3 100644 --- a/src/app/cli.h +++ b/src/app/cli.h @@ -8,6 +8,10 @@ typedef enum { DOLRECOMP_BACKEND_C, DOLRECOMP_BACKEND_LLVM, + /* CFG-planned regions instead of fixed chunks. Additive: the fixed LLVM + path above stays available until this reaches correctness and + performance parity. */ + DOLRECOMP_BACKEND_LLVM_AOT, } DolRecompBackend; typedef struct { @@ -15,6 +19,16 @@ typedef struct { const char* title_id_arg; const char* output_arg; const char* map_path; + /* NULL disables reporting entirely; instrumentation stays collected but + unwritten, which is what keeps --perf-report free when unused. */ + const char* perf_report_path; + const char* region_report_path; + const char* region_mode_arg; + const char* region_profile_path; + const char* lto_mode_arg; + const char* memory_mode_arg; + u32 region_max_instructions; + u32 region_max_ir; DolRecompCPU cpu; DolRecompBackend backend; u32 jobs; diff --git a/src/app/main.c b/src/app/main.c index c148031..398ad54 100644 --- a/src/app/main.c +++ b/src/app/main.c @@ -11,17 +11,51 @@ #include "frontend/container/disc_extract.h" #include "backend/emitter.h" #include "analysis/symbol_map.h" +#include "common/perf.h" #include #include #include -int main(int argc, char** argv) { - if (argc > 1 && strcmp(argv[1], "extract") == 0) - return disc_extract_main(argc - 1, argv + 1); - +/* The recompile body keeps its many early returns; main() wraps it so the perf + report is written exactly once, on every path, without threading a cleanup + through each of them. */ +static int run_recompile(int argc, char** argv, CliOptions* opts_out) { CliOptions opts; if (!parse_cli(argc, argv, &opts)) return 1; + *opts_out = opts; + + DolRecompRegionOptions region_options; + memset(®ion_options, 0, sizeof(region_options)); + region_options.enabled = opts.backend == DOLRECOMP_BACKEND_LLVM_AOT; + region_options.mode_name = opts.region_mode_arg; + region_options.max_instructions = opts.region_max_instructions; + region_options.max_ir_instructions = opts.region_max_ir; + region_options.report_path = opts.region_report_path; + region_options.profile_path = opts.region_profile_path; + region_options.lto_mode = opts.lto_mode_arg; + pipeline_set_region_options(®ion_options); + + /* The emitters read the mode through common/options.h, which reads the + environment, so an explicit flag is published there. CLI wins over an + inherited value: moderngekko-port sets its own environment and a stale + DOLRECOMP_MEMORY_MODE would otherwise outrank what was asked for. */ + if (opts.memory_mode_arg) { +#if defined(_WIN32) + _putenv_s("DOLRECOMP_MEMORY_MODE", opts.memory_mode_arg); +#else + setenv("DOLRECOMP_MEMORY_MODE", opts.memory_mode_arg, 1); +#endif + } + + if (!region_options.enabled && + (opts.region_mode_arg || opts.region_report_path || opts.region_profile_path || + opts.lto_mode_arg || + opts.region_max_instructions || opts.region_max_ir)) { + fprintf(stderr, + "error: region options require --backend llvm-aot\n"); + return 1; + } if (opts.show_help) return 0; if (opts.setup_mode) @@ -247,3 +281,35 @@ int main(int argc, char** argv) { dol_free(&dol); return 0; } + +int main(int argc, char** argv) { + if (argc > 1 && strcmp(argv[1], "extract") == 0) + return disc_extract_main(argc - 1, argv + 1); + + DolPerfReport* report = dolperf_report(); + dolperf_reset(report); + u64 started_ns = dolperf_now_ns(); + + CliOptions opts; + memset(&opts, 0, sizeof(opts)); + int status = run_recompile(argc, argv, &opts); + + report->wall_ns = dolperf_now_ns() - started_ns; + snprintf(report->backend, sizeof(report->backend), "%s", + opts.backend == DOLRECOMP_BACKEND_LLVM_AOT ? "llvm-aot" + : opts.backend == DOLRECOMP_BACKEND_LLVM ? "llvm" + : "c"); + if (report->region_mode[0] == '\0') + snprintf(report->region_mode, sizeof(report->region_mode), "fixed"); + + if (opts.perf_report_path) { + if (!dolperf_write_json(report, opts.perf_report_path, stderr)) + status = status ? status : 1; + else + printf("perf report: %s\n", opts.perf_report_path); + dolperf_print_summary(report, stdout); + } + + dolperf_free(report); + return status; +} diff --git a/src/app/pipeline.c b/src/app/pipeline.c index a154731..1f74d20 100644 --- a/src/app/pipeline.c +++ b/src/app/pipeline.c @@ -9,11 +9,16 @@ #include "frontend/container/rpx.h" #include "backend/emitter.h" #include "backend/dispatch.h" +#include "common/options.h" #include "backend/codegen.h" #include "backend/symbols.h" #include "analysis/code_section.h" #include "analysis/embedded_data.h" #include "analysis/smc.h" +#include "analysis/cfg.h" +#include "analysis/regions.h" +#include "common/perf.h" +#include #ifdef DOLRECOMP_ENABLE_LLVM #include "ir/dolir_builder.h" #include "backend/llvm/llvm_backend.h" @@ -24,6 +29,36 @@ #include #include #include + +static DolRecompRegionOptions g_region_options; + +void pipeline_set_region_options(const DolRecompRegionOptions* options) { + if (options) + g_region_options = *options; + else + memset(&g_region_options, 0, sizeof(g_region_options)); +} + +const DolRecompRegionOptions* pipeline_region_options(void) { + return &g_region_options; +} + +/* Emitted code size per region. Reported, never load-bearing: a size that + cannot be read is recorded as zero rather than failing the build. */ +static u32 perf_file_size(const char* path) { + FILE* in = fopen(path, "rb"); + if (!in) + return 0; + u32 size = 0; + if (fseek(in, 0, SEEK_END) == 0) { + long value = ftell(in); + if (value > 0) + size = (u32)value; + } + fclose(in); + return size; +} + #ifndef _WIN32 #include #include @@ -65,15 +100,53 @@ static u32 c_chunk_instructions(void) { // v7: ps1 preservation fix in dolir_builder (lfd and fmr/fneg/fabs/fnabs/fsel // no longer splat into the high paired-single slot). Default codegen changed, // so every cached object from v6 is stale. -#define DOLLLVM_CACHE_VERSION "dolllvm-v7" +// v8: the call-return path reloads only guest state live at the continuation +// instead of everything the function touches. Generated code changed, so every +// cached object from v7 is stale -- and because the cache key does not hash the +// emitter's source, nothing else would have noticed. +// v9: materialize() skips slots no path to the barrier has written, so the +// store side of every barrier shrinks too. +// v10: the liveness-narrowed reload is reverted -- it hung Mario Kart because +// the successor model misses indirect-continuation edges. +// v17: ThinLTO bitcode emission available (off unless a path is given); the +// object path is byte-identical either way. +// v16: barrier narrowing and region inlining are opt-in env flags, both off by +// default, and both fold into the cache key. +// v15: barrier store narrowing re-enabled with indirect-switch edges in both +// dataflow passes. +// v14: barrier store narrowing disabled -- it hung Mario Kart even with helper +// writes handled. Barriers are fully conservative again. +// v13: internal region bodies use fastcc; the public wrapper stays C. +// v12: reaching-writes store narrowing reinstated, now treating a helper call +// as dirtying every used slot. +// v11: the reaching-writes store narrowing was reverted -- it diverged from +// the C backend on floating-point state, because helper calls write slots +// without emitting DOLIR_OP_STATE_WRITE. Both barrier sides are conservative +// again. +#define DOLLLVM_CACHE_VERSION "dolllvm-v20-mem1-hoist" // The LLVM optimisation level used for generated objects. Named so it can be // folded into the cache key; changing it must not reuse cached objects. #define DOLLLVM_OPT_LEVEL 2 +/* One contiguous stretch of guest code. A fixed chunk is exactly one of these; + a planned region is one or more, because accreting a caller with a callee + that does not sit next to it in memory produces a region with a hole. */ +typedef struct { + const PPCInst* insts; + u32 count; + u32 address; +} LLVMRun; + typedef struct { + /* The first run, kept as named fields so the fixed path is untouched. + runs == NULL means "this job is exactly the single run described here". */ const PPCInst* insts; u32 count; u32 function_address; + + const LLVMRun* runs; + u32 run_count; + u32 index; u32 total; const DolLLVMFunctionRange* ranges; @@ -82,8 +155,27 @@ typedef struct { char name[128]; char path[1400]; char cache_path[1400]; + /* Empty unless --lto thin. Emitted alongside the object, never instead of + it: the object keeps the build linkable while the thin link is developed, + and the two are verified byte-identical with and without bitcode. */ + char bitcode_path[1400]; } LLVMChunkJob; +/* Uniform access to a job's runs whether or not it carries an explicit list. */ +static u32 llvm_job_run_count(const LLVMChunkJob* job) { + return job->runs ? job->run_count : 1u; +} + +static LLVMRun llvm_job_run(const LLVMChunkJob* job, u32 index) { + if (job->runs) + return job->runs[index]; + LLVMRun run; + run.insts = job->insts; + run.count = job->count; + run.address = job->function_address; + return run; +} + // The floor is 32, not the 128 the C path uses. // // A chunk becomes exactly one LLVM function, so this value is the number of @@ -250,12 +342,28 @@ static u64 llvm_job_hash(const LLVMChunkJob* job) { hash = hash_bytes(hash, codegen, strlen(codegen)); u32 opt_level = (u32)DOLLLVM_OPT_LEVEL; hash = hash_bytes(hash, &opt_level, sizeof(opt_level)); - for (u32 i = 0; i < job->count; i++) { - hash = hash_bytes(hash, &job->insts[i].address, - sizeof(job->insts[i].address)); - hash = hash_bytes(hash, &job->insts[i].raw, sizeof(job->insts[i].raw)); - hash = hash_bytes(hash, &job->insts[i].embedded_data, - sizeof(job->insts[i].embedded_data)); + /* A cached object from an --lto off build has no bitcode beside it, so + reusing it under --lto thin would leave the thin link with a hole and no + error. The mode is part of what the artifacts are, not just how they were + made. */ + u32 has_bitcode = job->bitcode_path[0] ? 1u : 0u; + hash = hash_bytes(hash, &has_bitcode, sizeof(has_bitcode)); + /* Every run, and the run partition itself: two regions covering the same + instructions in a different grouping generate different code, so they + must not collide in the cache. */ + u32 run_count = llvm_job_run_count(job); + hash = hash_bytes(hash, &run_count, sizeof(run_count)); + for (u32 r = 0; r < run_count; r++) { + LLVMRun run = llvm_job_run(job, r); + hash = hash_bytes(hash, &run.address, sizeof(run.address)); + hash = hash_bytes(hash, &run.count, sizeof(run.count)); + for (u32 i = 0; i < run.count; i++) { + hash = hash_bytes(hash, &run.insts[i].address, + sizeof(run.insts[i].address)); + hash = hash_bytes(hash, &run.insts[i].raw, sizeof(run.insts[i].raw)); + hash = hash_bytes(hash, &run.insts[i].embedded_data, + sizeof(run.insts[i].embedded_data)); + } } for (u32 i = 0; i < job->range_count; i++) hash = hash_bytes(hash, &job->ranges[i], sizeof(job->ranges[i])); @@ -290,12 +398,33 @@ static int llvm_cache_dir(char* path, size_t size) { return make_dir_tree(path); } +/* The cache stores one object per key. Under --lto thin a key also owns a + bitcode file, and a hit that restored only the object would leave the thin + link short a summary with nothing to indicate it -- so both move together, or + the job recompiles. */ +static int cache_bitcode_path(const LLVMChunkJob* job, char* out, size_t size) { + if (!job->cache_path[0] || !job->bitcode_path[0]) + return 0; + return snprintf(out, size, "%s.bc", job->cache_path) < (int)size; +} + static int reuse_llvm_object(const LLVMChunkJob* job) { + char cached_bitcode[1440]; + int wants_bitcode = job->bitcode_path[0] != 0; + if (wants_bitcode && !cache_bitcode_path(job, cached_bitcode, + sizeof(cached_bitcode))) + return 0; if (getenv("DOLRECOMP_LLVM_RESUME") && valid_object_file(job->path) && - valid_llvm_job_stamp(job)) + valid_llvm_job_stamp(job) && + (!wants_bitcode || file_exists(job->bitcode_path))) return 1; - if (!job->cache_path[0] || !valid_object_file(job->cache_path) || - !copy_file(job->cache_path, job->path)) + if (!job->cache_path[0] || !valid_object_file(job->cache_path)) + return 0; + if (wants_bitcode && !file_exists(cached_bitcode)) + return 0; + if (!copy_file(job->cache_path, job->path)) + return 0; + if (wants_bitcode && !copy_file(cached_bitcode, job->bitcode_path)) return 0; write_llvm_job_stamp(job); return 1; @@ -316,7 +445,21 @@ static void cache_llvm_object(const LLVMChunkJob* job) { remove(temp); if (!copy_file(job->path, temp)) return; - if (rename(temp, job->cache_path) != 0) + if (rename(temp, job->cache_path) != 0) { + remove(temp); + return; + } + char cached_bitcode[1440]; + if (!cache_bitcode_path(job, cached_bitcode, sizeof(cached_bitcode)) || + file_exists(cached_bitcode)) + return; + if (snprintf(temp, sizeof(temp), "%s.tmp.%d", cached_bitcode, process_id) >= + (int)sizeof(temp)) + return; + remove(temp); + if (!copy_file(job->bitcode_path, temp)) + return; + if (rename(temp, cached_bitcode) != 0) remove(temp); } @@ -354,9 +497,17 @@ static int emit_llvm_chunk_job(const void* data, void* user) { remove(temp_path); DolIRModule module; dolir_module_init(&module); - if (!dolir_build_chunk(&module, job->insts, job->count, - job->function_address) || - !dolir_verify(&module, stderr)) { + /* One DolIRFunction per run. A region's runs land in one module, which is + what lets LLVM see a caller and its callee together. */ + u32 run_count = llvm_job_run_count(job); + for (u32 r = 0; r < run_count; r++) { + LLVMRun run = llvm_job_run(job, r); + if (!dolir_build_chunk(&module, run.insts, run.count, run.address)) { + dolir_module_free(&module); + return 0; + } + } + if (!dolir_verify(&module, stderr)) { dolir_module_free(&module); return 0; } @@ -366,6 +517,10 @@ static int emit_llvm_chunk_job(const void* data, void* user) { options.verify = 1; options.function_ranges = job->ranges; options.function_range_count = job->range_count; + if (job->bitcode_path[0]) { + options.emit_bitcode = 1; + options.bitcode_path = job->bitcode_path; + } char ir_path[1440]; const char* dump_ir = getenv("DOLRECOMP_LLVM_DUMP_IR"); if (dump_ir && (!strcmp(dump_ir, "1") || strstr(job->name, dump_ir))) { @@ -556,6 +711,361 @@ static int run_llvm_chunk_jobs(const LLVMChunkJob* jobs, u32 count, #endif } +static int compare_function_range(const void* a, const void* b) { + u32 left = ((const DolLLVMFunctionRange*)a)->start; + u32 right = ((const DolLLVMFunctionRange*)b)->start; + return (left > right) - (left < right); +} + +/* Region-planned LLVM emission. + * + * Deliberately a separate path from the fixed-chunk emitter rather than a flag + * threaded through it. The brief requires the fixed path stay available until + * this one reaches parity, and the surest way to keep it available is to not + * touch it. + * + * The shape differs in one structural way: the fixed path can precompute every + * chunk boundary from a formula before decoding anything, whereas region + * boundaries are a result of analysis. So this decodes every section first, + * builds one CFG across all of them, plans, and only then emits. + */ +static int emit_llvm_regions(const LoadedCodeSection* sections, u32 section_count, + DolRecompCPU cpu, u32 entry_point, u32 requested_jobs, + const char* chunks_dir, const char* stem, + const char* header_path, FILE* header, + FILE* manifest, FILE* fallback_report) { + (void)cpu; + const DolRecompRegionOptions* options = pipeline_region_options(); + + DolCfgProgram cfg; + dolcfg_init(&cfg); + DolRegionPlan plan; + dolregion_plan_init(&plan); + FunctionList funcs = {0}; + SMCAnalysis smc = {0}; + + PPCInst** decoded = (PPCInst**)calloc(section_count ? section_count : 1u, + sizeof(PPCInst*)); + LLVMRun* runs = NULL; + LLVMChunkJob* jobs = NULL; + DolLLVMFunctionRange* ranges = NULL; + unsigned char* cached_before_run = NULL; + u32 file_count = 0; + int status = 0; + + if (!decoded) + goto done; + + cfg.entry_point = entry_point; + + for (u32 s = 0; s < section_count; s++) { + const LoadedCodeSection* section = §ions[s]; + if (!section->data || !section->size) + continue; + + u32 num_insts = section->size / 4u; + PPCInst* insts = (PPCInst*)malloc((size_t)num_insts * sizeof(*insts)); + if (!insts) { + fprintf(stderr, "error: out of memory\n"); + goto done; + } + decoded[s] = insts; + + u32 embedded = 0; + u32 unknown = 0; + for (u32 i = 0; i < num_insts; i++) { + u32 raw = read_be32(section->data + i * 4u); + insts[i] = ppc_decode(raw, section->address + i * 4u); + if (insts[i].op == PPC_OP_UNKNOWN && + embedded_data_word(section->embedded_data_mode, raw)) + insts[i].embedded_data = true; + embedded += insts[i].embedded_data; + unknown += insts[i].op == PPC_OP_UNKNOWN && !insts[i].embedded_data; + } + printf("decoding %s[%u]: %u instructions at 0x%08X\n", + section->label, section->index, num_insts, section->address); + printf(" %u known, %u embedded data, %u unknown\n", + num_insts - embedded - unknown, embedded, unknown); + + if (section->embedded_data_mode == EMBEDDED_DATA_DOL) { + analyze_smc_section(sections, section_count, insts, num_insts, &smc); + if (smc.allocation_failed) + goto done; + } + + if (!dolcfg_add_section(&cfg, insts, num_insts, section->address, + section->label)) + goto done; + } + + /* SMC-suspect code must be able to end a region: a boundary there is what + keeps a conservative path available for it. */ + for (u32 i = 0; i < smc.range_count; i++) { + if (!dolcfg_add_smc_range(&cfg, smc.ranges[i].start, smc.ranges[i].end)) + goto done; + } + + if (!dolcfg_build(&cfg, stderr)) + goto done; + + if (options->profile_path) { + u32 matched = 0, unmatched = 0; + if (!dolcfg_load_profile(&cfg, options->profile_path, &matched, + &unmatched, stderr)) + goto done; + printf("region profile: %u entries matched, %u unmatched (%s)\n", + matched, unmatched, options->profile_path); + } + + DolRegionMode mode = DOLREGION_MODE_CFG; + if (options->mode_name && !dolregion_parse_mode(options->mode_name, &mode)) { + fprintf(stderr, "error: unknown region mode '%s'\n", options->mode_name); + goto done; + } + + const int thin_lto = options->lto_mode && !strcmp(options->lto_mode, "thin"); + + DolRegionLimits limits; + dolregion_default_limits(&limits); + if (options->max_instructions) + limits.max_instructions = options->max_instructions; + if (options->max_ir_instructions) + limits.max_ir_instructions = options->max_ir_instructions; + + if (!dolregion_plan_build(&plan, &cfg, mode, &limits, stderr)) + goto done; + + printf("planned %u regions (%s, max %u instructions): " + "%u blocks, %u functions, %u crossings\n", + plan.region_count, dolregion_mode_name(mode), limits.max_instructions, + cfg.block_count, cfg.function_count, plan.cross_region_edges); + + if (options->report_path && + !dolregion_write_report(&plan, &cfg, options->report_path, stderr)) + goto done; + + /* Flatten every region into contiguous runs. A region's blocks are sorted + by address, so adjacent blocks whose addresses touch form one run. */ + u32 run_capacity = cfg.block_count ? cfg.block_count : 1u; + runs = (LLVMRun*)malloc(run_capacity * sizeof(*runs)); + jobs = (LLVMChunkJob*)calloc(plan.region_count ? plan.region_count : 1u, + sizeof(*jobs)); + if (!runs || !jobs) + goto done; + + u32 run_total = 0; + for (u32 r = 0; r < plan.region_count; r++) { + DolRegion* region = &plan.regions[r]; + jobs[r].runs = NULL; /* set after the array stops moving */ + jobs[r].run_count = 0; + u32 first_run = run_total; + + /* Blocks arrive grouped by function, so sort by address before + coalescing or two adjacent blocks from different functions would not + be recognised as touching. */ + u32* ordered = (u32*)malloc((region->block_count ? region->block_count : 1u) * + sizeof(u32)); + if (!ordered) + goto done; + memcpy(ordered, region->blocks, region->block_count * sizeof(u32)); + for (u32 i = 1; i < region->block_count; i++) { + u32 key = ordered[i]; + u32 j = i; + while (j > 0 && cfg.blocks[ordered[j - 1u]].start > cfg.blocks[key].start) { + ordered[j] = ordered[j - 1u]; + j--; + } + ordered[j] = key; + } + + for (u32 i = 0; i < region->block_count; i++) { + const DolCfgBlock* block = &cfg.blocks[ordered[i]]; + if (run_total > first_run && + runs[run_total - 1u].address + + runs[run_total - 1u].count * 4u == block->start) { + runs[run_total - 1u].count += block->instruction_count; + continue; + } + const DolCfgSection* section = NULL; + for (u32 s = 0; s < cfg.section_count; s++) { + u32 end = cfg.sections[s].base_address + cfg.sections[s].count * 4u; + if (block->start >= cfg.sections[s].base_address && block->start < end) { + section = &cfg.sections[s]; + break; + } + } + if (!section) { + free(ordered); + goto done; + } + if (run_total == run_capacity) { + run_capacity *= 2u; + LLVMRun* grown = (LLVMRun*)realloc(runs, run_capacity * sizeof(*runs)); + if (!grown) { + free(ordered); + goto done; + } + runs = grown; + } + runs[run_total].address = block->start; + runs[run_total].count = block->instruction_count; + runs[run_total].insts = + section->insts + (block->start - section->base_address) / 4u; + run_total++; + } + free(ordered); + + jobs[r].run_count = run_total - first_run; + /* Stored as an offset for now; rebased once `runs` stops reallocating. */ + jobs[r].runs = (const LLVMRun*)(uintptr_t)first_run; + } + + /* Every run is a separately generated entry point, so the emitter needs all + of them to tell an intra-module target from a cross-module one. */ + ranges = (DolLLVMFunctionRange*)calloc(run_total ? run_total : 1u, + sizeof(*ranges)); + if (!ranges) + goto done; + for (u32 i = 0; i < run_total; i++) { + ranges[i].start = runs[i].address; + ranges[i].end = runs[i].address + runs[i].count * 4u; + } + /* rangeFor() binary-searches these. Runs are emitted in region order, which + is not address order, so the sort is required for correctness here -- + unlike the fixed path, where chunks are already ascending. */ + qsort(ranges, run_total, sizeof(*ranges), compare_function_range); + + char cache_dir[1100] = ""; + if (!llvm_cache_dir(cache_dir, sizeof(cache_dir))) + cache_dir[0] = '\0'; + + for (u32 r = 0; r < plan.region_count; r++) { + LLVMChunkJob* job = &jobs[r]; + u32 first_run = (u32)(uintptr_t)job->runs; + job->runs = runs + first_run; + if (job->run_count == 0) + continue; + + job->insts = job->runs[0].insts; + job->count = job->runs[0].count; + job->function_address = job->runs[0].address; + job->index = r + 1u; + job->total = plan.region_count; + job->ranges = ranges; + job->range_count = run_total; + + if (snprintf(job->name, sizeof(job->name), "region_%06u_%08X.o", r, + job->function_address) >= (int)sizeof(job->name) || + !join_path(job->path, sizeof(job->path), chunks_dir, job->name)) + goto done; + + /* Beside the object and named for the same region, so the thin link + can pair them by directory listing. */ + if (thin_lto && + snprintf(job->bitcode_path, sizeof(job->bitcode_path), "%s.bc", + job->path) >= (int)sizeof(job->bitcode_path)) + job->bitcode_path[0] = '\0'; + + job->hash = llvm_job_hash(job); + if (cache_dir[0]) { + char cache_name[64]; + snprintf(cache_name, sizeof(cache_name), "%016llx.o", + (unsigned long long)job->hash); + if (!join_path(job->cache_path, sizeof(job->cache_path), cache_dir, + cache_name)) + job->cache_path[0] = '\0'; + } + + /* Each run keeps its own public entry point, so dispatch and every + ModernGekko replacement address still resolve exactly as before. */ + for (u32 i = 0; i < job->run_count; i++) { + emit_chunk_prototype(header, job->runs[i].address); + if (!function_list_add(&funcs, job->runs[i].address, + job->runs[i].address + job->runs[i].count * 4u)) + goto done; + } + /* Exactly the fixed path's format. The module template parses this + manifest and takes everything after "// object: " as the object + path, so an appended "(N runs)" became part of the filename and the + configure failed looking for "region_000000_80003100.o (16 runs)". + Run counts belong in the region report, which already carries them. */ + /* Under --lto thin the manifest names the bitcode instead. lld reads + bitcode inputs natively and runs ThinLTO on them, so the link stage + needs no in-process LTO driver and the module template needs no + change: it still just forwards each listed file to the linker. + The object is still written beside it, so a build can fall back by + rerunning with --lto off without recompiling. */ + fprintf(manifest, "// object: chunks/%s%s\n", job->name, + thin_lto && job->bitcode_path[0] ? ".bc" : ""); + file_count++; + } + + u32 active_jobs = effective_chunk_jobs(plan.region_count, requested_jobs); + printf(" writing %u LLVM region objects with %u job%s%s\n", + plan.region_count, active_jobs, active_jobs == 1 ? "" : "s", + thin_lto ? " (+ThinLTO bitcode)" : ""); + + cached_before_run = (unsigned char*)calloc( + plan.region_count ? plan.region_count : 1u, 1u); + if (cached_before_run) { + for (u32 r = 0; r < plan.region_count; r++) + cached_before_run[r] = reuse_llvm_object(&jobs[r]) ? 1u : 0u; + } + + if (!run_llvm_chunk_jobs(jobs, plan.region_count, requested_jobs)) + goto done; + + for (u32 r = 0; r < plan.region_count; r++) { + const DolRegion* region = &plan.regions[r]; + DolPerfRegion record; + memset(&record, 0, sizeof(record)); + record.region_id = r; + record.guest_start = region->guest_start; + record.guest_end = region->guest_end; + record.guest_instructions = region->instruction_count; + record.ir_instructions = region->estimated_ir_instructions; + record.blocks = region->block_count; + record.loops = region->loop_count; + record.code_bytes = perf_file_size(jobs[r].path); + record.cache_hit = cached_before_run ? cached_before_run[r] : 0; + dolperf_add_region(dolperf_report(), &record); + } + snprintf(dolperf_report()->region_mode, sizeof(dolperf_report()->region_mode), + "%s", dolregion_mode_name(mode)); + + { + char report[1100]; + if (snprintf(report, sizeof(report), "%s_smc.txt", stem) >= (int)sizeof(report) || + !write_smc_report(&smc, report)) + goto done; + if (smc.possible) + printf("warning: executable memory writes detected; report: %s\n", report); + } + + emit_dispatch_helpers(header, &funcs, entry_point, memory_mode_is_fast()); + emit_footer(header); + fprintf(manifest, "\n// %u native objects\n", file_count); + printf("done!\n header: %s\n objects: %s (%u files)\n", header_path, + chunks_dir, file_count); + status = 1; + +done: + free(cached_before_run); + free(ranges); + free(jobs); + free(runs); + dolregion_plan_free(&plan); + dolcfg_free(&cfg); + function_list_free(&funcs); + smc_analysis_free(&smc); + if (decoded) { + for (u32 s = 0; s < section_count; s++) + free(decoded[s]); + free(decoded); + } + return status; +} + static int emit_code_sections_llvm(const LoadedCodeSection* sections, u32 section_count, const char* output_path, @@ -630,6 +1140,16 @@ static int emit_code_sections_llvm(const LoadedCodeSection* sections, } fprintf(header, "\n// Function entry points\n"); + if (pipeline_region_options()->enabled) { + int ok = emit_llvm_regions(sections, section_count, cpu, entry_point, + requested_jobs, chunks_dir, stem, header_path, + header, manifest, fallback_report); + fclose(header); + fclose(manifest); + fclose(fallback_report); + return ok; + } + FunctionList funcs = {0}; SMCAnalysis smc = {0}; u32 file_count = 0; @@ -784,11 +1304,45 @@ static int emit_code_sections_llvm(const LoadedCodeSection* sections, u32 active_jobs = effective_chunk_jobs(chunk_total, requested_jobs); printf(" writing %u LLVM objects with %u job%s\n", chunk_total, active_jobs, active_jobs == 1 ? "" : "s"); + + /* Cache state has to be sampled before the run: afterwards every object + is present and a hit is indistinguishable from a fresh emit. */ + unsigned char* cached_before_run = + (unsigned char*)calloc(chunk_total ? chunk_total : 1u, 1u); + if (cached_before_run) { + for (u32 i = 0; i < chunk_total; i++) + cached_before_run[i] = reuse_llvm_object(&chunk_jobs[i]) ? 1u : 0u; + } + if (!run_llvm_chunk_jobs(chunk_jobs, chunk_total, requested_jobs)) { + free(cached_before_run); free(chunk_jobs); free(insts); goto fail; } + + /* Recorded in the parent, after the workers are done. + * + * Per-region optimize/codegen timings deliberately stay zero here: the + * POSIX path forks a worker per batch, so a counter raised inside + * emit_llvm_chunk_job() dies with the child. Reporting a partial number + * that is whole on Windows and empty on Linux would be worse than + * reporting none. The Phase 1 region backend emits in-process and fills + * these in for real. */ + for (u32 i = 0; i < chunk_total; i++) { + const LLVMChunkJob* job = &chunk_jobs[i]; + DolPerfRegion region; + memset(®ion, 0, sizeof(region)); + region.region_id = i; + region.guest_start = job->function_address; + region.guest_end = job->function_address + job->count * 4u; + region.guest_instructions = job->count; + region.blocks = 1; + region.code_bytes = perf_file_size(job->path); + region.cache_hit = cached_before_run ? cached_before_run[i] : 0; + dolperf_add_region(dolperf_report(), ®ion); + } + free(cached_before_run); free(chunk_jobs); free(insts); } @@ -801,7 +1355,7 @@ static int emit_code_sections_llvm(const LoadedCodeSection* sections, if (smc.possible) printf("warning: executable memory writes detected; report: %s\n", report); } - emit_dispatch_helpers(header, &funcs, entry_point); + emit_dispatch_helpers(header, &funcs, entry_point, memory_mode_is_fast()); emit_footer(header); fprintf(manifest, "\n// %u native objects\n", file_count); fclose(header); @@ -833,11 +1387,13 @@ int emit_code_sections_split(const LoadedCodeSection* sections, const DolRecompSymbolMap* symbols, DolRecompBackend backend) { #ifdef DOLRECOMP_ENABLE_LLVM - if (backend == DOLRECOMP_BACKEND_LLVM) + if (backend == DOLRECOMP_BACKEND_LLVM || + backend == DOLRECOMP_BACKEND_LLVM_AOT) return emit_code_sections_llvm(sections, section_count, output_path, cpu, entry_point, jobs, local_chunks_dir, symbols); #else - if (backend == DOLRECOMP_BACKEND_LLVM) { + if (backend == DOLRECOMP_BACKEND_LLVM || + backend == DOLRECOMP_BACKEND_LLVM_AOT) { fprintf(stderr, "error: LLVM backend is unavailable in this build\n"); return 0; } @@ -1115,6 +1671,22 @@ int emit_code_sections_split(const LoadedCodeSection* sections, emit_set_chunk_table(NULL, 0); free(chunk_starts); + /* Fixed-chunk mode records one region per chunk, so `fixed` and the + Phase 1 planner modes report through the same structure and stay + directly comparable. */ + for (u32 i = 0; i < section_job_count; i++) { + const ChunkJob* job = &chunk_jobs[i]; + DolPerfRegion region; + memset(®ion, 0, sizeof(region)); + region.region_id = i; + region.guest_start = job->func_addr; + region.guest_end = job->func_addr + job->count * 4u; + region.guest_instructions = job->count; + region.blocks = 1; + region.code_bytes = perf_file_size(job->path); + dolperf_add_region(dolperf_report(), ®ion); + } + free(chunk_jobs); free(insts); } @@ -1153,7 +1725,7 @@ int emit_code_sections_split(const LoadedCodeSection* sections, } } - emit_dispatch_helpers(header, &funcs, entry_point); + emit_dispatch_helpers(header, &funcs, entry_point, 0); emit_footer(header); smc_analysis_free(&smc); function_list_free(&funcs); diff --git a/src/app/pipeline.h b/src/app/pipeline.h index 65ecd3d..998a6b7 100644 --- a/src/app/pipeline.h +++ b/src/app/pipeline.h @@ -12,6 +12,26 @@ #define REL_AUTO_BASE 0x80500000u #define REL_AUTO_ALIGN 0x10000u +/* Region settings reach the emitters through a setter rather than five more + parameters on four already-long signatures. Set once from main() before any + emit_* call; zeroed values mean "planner default". */ +typedef struct { + int enabled; /* llvm-aot selected */ + const char* mode_name; /* NULL -> cfg */ + u32 max_instructions; /* 0 -> planner default */ + u32 max_ir_instructions; /* 0 -> planner default */ + const char* report_path; /* NULL -> no report */ + const char* profile_path; /* NULL -> no weights; pgo mode degrades */ + /* "off" (default) or "thin". Thin emits per-region bitcode carrying a + module summary alongside the object, which is what a thin link consumes. + The object is still emitted and still byte-identical, so a thin build + links today either way. */ + const char* lto_mode; +} DolRecompRegionOptions; + +void pipeline_set_region_options(const DolRecompRegionOptions* options); +const DolRecompRegionOptions* pipeline_region_options(void); + int emit_dol_split(const DOLFile* dol, const char* output_path, DolRecompCPU cpu, u32 jobs, int local_chunks_dir, const DolRecompSymbolMap* symbols, DolRecompBackend backend); diff --git a/src/backend/dispatch.c b/src/backend/dispatch.c index 41dca99..4358043 100644 --- a/src/backend/dispatch.c +++ b/src/backend/dispatch.c @@ -1,4 +1,6 @@ #include "backend/dispatch.h" +#include "common/options.h" +#include "cpu/cpu.h" #include #include @@ -23,9 +25,25 @@ // must stay byte-identical; this selects an experiment, not a new default. typedef enum { DISPATCH_LOOKUP_LINEAR = 0, - DISPATCH_LOOKUP_INDEXED = 1 + DISPATCH_LOOKUP_INDEXED = 1, + /* Decide from the plan's shape. See dispatch_lookup_mode(). */ + DISPATCH_LOOKUP_AUTO = 2 } DispatchLookupMode; +/* Above this many runs, the linear chain stops being a chain worth walking. + * + * A uniform plan collapses to a handful of runs -- fixed 128-instruction chunks + * produce four -- and the linear form is then a couple of range tests plus a + * table index, which nothing beats. An irregular plan does not collapse: Mario + * Kart's 4,017 planned regions emitted 8,284 address comparisons, and walking + * those on every dispatch made the region build ~8x slower than the fixed one + * at an identical dispatcher rate. The page index costs one u32 per 4 KiB page + * and a short bounded walk. + * + * 64 is far above what any uniform plan produces and far below the thousands an + * irregular one does, so the choice is never close. */ +#define DISPATCH_LINEAR_RUN_LIMIT 64u + // A 4 KiB page is small enough that a page holds only a handful of runs even // under the most irregular plan measured here (E008a's mean chunk was 87 // instructions, so ~11 per page), and the whole index is one u32 per page. @@ -40,15 +58,29 @@ typedef enum { static DispatchLookupMode dispatch_lookup_mode(void) { const char* configured = getenv("DOLRECOMP_DISPATCH_LOOKUP"); if (!configured || !configured[0]) - return DISPATCH_LOOKUP_LINEAR; + return DISPATCH_LOOKUP_AUTO; + if (!strcmp(configured, "auto")) + return DISPATCH_LOOKUP_AUTO; if (!strcmp(configured, "indexed")) return DISPATCH_LOOKUP_INDEXED; if (!strcmp(configured, "linear")) return DISPATCH_LOOKUP_LINEAR; fprintf(stderr, - "warning: DOLRECOMP_DISPATCH_LOOKUP must be linear|indexed; using " - "linear\n"); - return DISPATCH_LOOKUP_LINEAR; + "warning: DOLRECOMP_DISPATCH_LOOKUP must be linear|indexed|auto; " + "using auto\n"); + return DISPATCH_LOOKUP_AUTO; +} + +static u32 uniform_run_end(const FunctionList* funcs, u32 first); + +/* How many runs the linear chain would emit. Runs, not ranges: consecutive + equal-width ranges collapse into one table, which is why a uniform plan stays + cheap however many chunks it has. */ +static u32 linear_run_count(const FunctionList* funcs) { + u32 runs = 0; + for (u32 first = 0; first < funcs->count; runs++) + first = uniform_run_end(funcs, first); + return runs; } void emit_chunk_prototype(FILE* out, u32 func_addr) { @@ -266,6 +298,30 @@ static int emit_lookup_indexed(FILE* out, const FunctionList* funcs) { } } + /* Coverage manifest. + * + * The module template's gen_module_tables.py recovers a module's address + * coverage by grepping this header for the dispatcher's own range tests. + * The page index does not emit those tests, so switching to it left the + * tool with nothing to find and the module build failed with "no coverage + * ranges found". + * + * Restating the ranges here in the offset-table form that tool already + * recognises keeps the contract without coupling the two repositories over + * it. The regex scans raw text, so a comment satisfies it: none of this is + * compiled, and the emitted lookup below is unaffected. */ + fprintf(out, + "\n/* Coverage manifest for gen_module_tables.py. Not compiled --\n" + " * the page index below is the actual lookup. Restated in the\n" + " * offset-table form that tool greps for.\n"); + for (u32 i = 0; i < run_count; i++) { + u32 start = sorted[run_first[i]].start; + u32 end = sorted[run_first[i + 1u] - 1u].end; + fprintf(out, " * u32 offset = address - 0x%08Xu; if (offset < 0x%08Xu)\n", + start, end - start); + } + fprintf(out, " */\n"); + fprintf(out, "\n#define DOLRECOMP_LOOKUP_RUNS %uu\n", run_count); fprintf(out, "#define DOLRECOMP_LOOKUP_BASE 0x%08Xu\n", base); fprintf(out, "#define DOLRECOMP_LOOKUP_PAGES %uu\n", page_count); @@ -364,7 +420,53 @@ static void emit_lookup_linear(FILE* out, const FunctionList* funcs) { fprintf(out, "}\n"); } -void emit_dispatch_helpers(FILE* out, const FunctionList* funcs, u32 entry_point) { +/* The fast memory mode bakes two runtime facts into every load and store (see + memoryModeFast() in llvm_memory_lowering.cpp). Baking in an assumption that + silently stops holding is how a recompiler corrupts guest memory, so the + generated code checks both once and refuses to run natively if either fails: + returning 0 from dolrecomp_call leaves the chassis interpreting, which is + slow but right. + + Checked once rather than per access. The flag is written only after a + successful check, so the cost on the hot path is one predictable branch on a + value that never changes. */ +static void emit_memory_mode_guard(FILE* out, int uses_fast_memory) { + if (!uses_fast_memory) { + fprintf(out, "\nstatic inline int dolrecomp_memory_mode_ok(CPUState* ctx) {\n"); + fprintf(out, " (void)ctx;\n"); + fprintf(out, " return 1;\n"); + fprintf(out, "}\n"); + return; + } + fprintf(out, "\n/* Built with --memory-mode fast. */\n"); + /* Only in fast mode: the guard is the sole user of stdio in generated + output, and a default build's header must stay byte-identical to what it + emitted before this option existed. */ + fprintf(out, "#include \n"); + fprintf(out, "static int dolrecomp_memory_mode_state = 0;\n"); + fprintf(out, "static int dolrecomp_memory_mode_check(CPUState* ctx) {\n"); + fprintf(out, " if (ctx->ram_size != %uu) {\n", (unsigned)GC_MAIN_RAM_SIZE); + fprintf(out, " fprintf(stderr, \"dolrecomp: --memory-mode fast expects ram_size %%u,\"\n"); + fprintf(out, " \" runtime has %%u; falling back to the interpreter\\n\",\n"); + fprintf(out, " %uu, ctx->ram_size);\n", (unsigned)GC_MAIN_RAM_SIZE); + fprintf(out, " return -1;\n"); + fprintf(out, " }\n"); + fprintf(out, " if (g_mem_write_journal) {\n"); + fprintf(out, " fprintf(stderr, \"dolrecomp: --memory-mode fast omits the write journal,\"\n"); + fprintf(out, " \" but one is installed; falling back to the interpreter\\n\");\n"); + fprintf(out, " return -1;\n"); + fprintf(out, " }\n"); + fprintf(out, " return 1;\n"); + fprintf(out, "}\n"); + fprintf(out, "static inline int dolrecomp_memory_mode_ok(CPUState* ctx) {\n"); + fprintf(out, " if (dolrecomp_memory_mode_state == 0)\n"); + fprintf(out, " dolrecomp_memory_mode_state = dolrecomp_memory_mode_check(ctx);\n"); + fprintf(out, " return dolrecomp_memory_mode_state > 0;\n"); + fprintf(out, "}\n"); +} + +void emit_dispatch_helpers(FILE* out, const FunctionList* funcs, u32 entry_point, + int uses_fast_memory) { fprintf(out, "\n#define DOLRECOMP_ENTRY_POINT 0x%08Xu\n", entry_point); fprintf(out, "\ntypedef void (*DolRecompFunction)(CPUState* ctx);\n"); fprintf(out, "\n#if defined(__GNUC__) || defined(__clang__)\n"); @@ -372,6 +474,13 @@ void emit_dispatch_helpers(FILE* out, const FunctionList* funcs, u32 entry_point fprintf(out, "#else\n"); fprintf(out, "#define DOLRECOMP_UNUSED\n"); fprintf(out, "#endif\n"); + /* One switch, not two. The emitter suppresses direct calls when + replacements are on (replacementsEnabled() in llvm_control_flow.cpp) and + the generated header defines the macro to match. Left independent, a + module could be compiled with replacements active while its call sites + were emitted to bypass them -- a mod that installs and does nothing. */ + if (replacements_enabled()) + fprintf(out, "\n#define DOLRECOMP_ENABLE_REPLACEMENTS 1\n"); fprintf(out, "\n#if defined(DOLRECOMP_ENABLE_REPLACEMENTS)\n"); fprintf(out, "int dolrecomp_dispatch_replacement(CPUState* ctx, u32 address);\n"); fprintf(out, "#else\n"); @@ -381,9 +490,21 @@ void emit_dispatch_helpers(FILE* out, const FunctionList* funcs, u32 entry_point fprintf(out, " return 0;\n"); fprintf(out, "}\n"); fprintf(out, "#endif\n"); - if (dispatch_lookup_mode() != DISPATCH_LOOKUP_INDEXED || - !emit_lookup_indexed(out, funcs)) - emit_lookup_linear(out, funcs); + { + DispatchLookupMode mode = dispatch_lookup_mode(); + int want_indexed = mode == DISPATCH_LOOKUP_INDEXED; + if (mode == DISPATCH_LOOKUP_AUTO) { + u32 runs = linear_run_count(funcs); + want_indexed = runs > DISPATCH_LINEAR_RUN_LIMIT; + if (want_indexed) { + printf(" dispatch: %u lookup runs, using page index\n", runs); + } + } + /* Correctness never depends on which one is emitted: the index refuses + plans it cannot represent and falls back here. */ + if (!want_indexed || !emit_lookup_indexed(out, funcs)) + emit_lookup_linear(out, funcs); + } fprintf(out, "\nstatic inline int dolrecomp_call_original(CPUState* ctx, u32 address) {\n"); fprintf(out, " DolRecompFunction fn = dolrecomp_find_original(address);\n"); fprintf(out, " if (!fn) return 0;\n"); @@ -398,9 +519,13 @@ void emit_dispatch_helpers(FILE* out, const FunctionList* funcs, u32 entry_point fprintf(out, " }\n"); fprintf(out, " return false;\n"); fprintf(out, "}\n"); + emit_memory_mode_guard(out, uses_fast_memory); fprintf(out, "\nstatic inline int dolrecomp_call(CPUState* ctx, u32 address) {\n"); fprintf(out, " u32 alias;\n"); fprintf(out, " ctx->pc = address;\n"); + /* Before any generated body runs, so a runtime that violates the fast + mode's assumptions never executes code built on them. */ + fprintf(out, " if (!dolrecomp_memory_mode_ok(ctx)) return 0;\n"); fprintf(out, " if (dolrecomp_dispatch_replacement(ctx, address)) return 1;\n"); fprintf(out, " if (ctx->host_call && ppc_host_call(ctx, address)) return 1;\n"); fprintf(out, " if (dolrecomp_call_original(ctx, address)) return 1;\n"); diff --git a/src/backend/dispatch.h b/src/backend/dispatch.h index 55122bd..6831066 100644 --- a/src/backend/dispatch.h +++ b/src/backend/dispatch.h @@ -4,6 +4,10 @@ #include "common/types.h" #include +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { u32 start; u32 end; @@ -18,6 +22,18 @@ typedef struct { void emit_chunk_prototype(FILE* out, u32 func_addr); void function_list_free(FunctionList* list); int function_list_add(FunctionList* list, u32 start, u32 end); -void emit_dispatch_helpers(FILE* out, const FunctionList* funcs, u32 entry_point); + +/* uses_fast_memory: the generated bodies were emitted with the fast guest + memory lowering, so the module needs the runtime guard that checks that + mode's assumptions. Only the LLVM backends lower memory that way; the C + backend reads its bounds from CPUState unconditionally and must not carry a + guard, or lockstep verification against it would refuse to run natively for + assumptions its code never made. */ +void emit_dispatch_helpers(FILE* out, const FunctionList* funcs, u32 entry_point, + int uses_fast_memory); + +#ifdef __cplusplus +} +#endif #endif diff --git a/src/backend/emitter.c b/src/backend/emitter.c index 1cae76f..65a7e32 100644 --- a/src/backend/emitter.c +++ b/src/backend/emitter.c @@ -247,7 +247,7 @@ static void emit_psq_load(FILE* out, const PPCInst* inst, bool indexed, emit_dform_ea(out, inst->rA, inst->simm, update); } fprintf(out, ";\n"); - fprintf(out, " ppc_psq_load_inline(ctx, %uu, ea, %s, %uu, %s, 0x%08Xu);\n", + fprintf(out, " ppc_psq_load(ctx, %uu, ea, %s, %uu, %s, 0x%08Xu);\n", inst->rD, inst->w ? "true" : "false", inst->i, indexed ? "true" : "false", inst->address); fprintf(out, " if (ctx->exception) return;\n"); @@ -267,7 +267,7 @@ static void emit_psq_store(FILE* out, const PPCInst* inst, bool indexed, emit_dform_ea(out, inst->rA, inst->simm, update); } fprintf(out, ";\n"); - fprintf(out, " ppc_psq_store_inline(ctx, %uu, ea, %s, %uu, %s, 0x%08Xu);\n", + fprintf(out, " ppc_psq_store(ctx, %uu, ea, %s, %uu, %s, 0x%08Xu);\n", inst->rS, inst->w ? "true" : "false", inst->i, indexed ? "true" : "false", inst->address); fprintf(out, " if (ctx->exception) return;\n"); @@ -621,8 +621,19 @@ static void emit_instruction_with_range(FILE* out, const PPCInst* inst, return; } + /* The MSR[FP] fast path is emitted here rather than called through + ppc_fp_available_inline, because that helper is declared in DolRecomp's + own cpu.h and generated modules compile against the runtime's instead. + GXRuntime declares ppc_fp_available but not the inline wrapper, so + emitting the wrapper made the C backend unbuildable against every + ModernGekko checkout on this machine (AOT-ENGINEERING-REPORT.md 5). + Spelling the bit test out keeps the fast path with no such dependency. + 0x2000 is MSR[FP], PPC bit 18. */ if (ppc_op_uses_fpu(inst->op)) - fprintf(out, " if (!ppc_fp_available_inline(ctx, 0x%08Xu)) return;\n", inst->address); + fprintf(out, + " if (!((ctx->msr & 0x00002000u) || " + "ppc_fp_available(ctx, 0x%08Xu))) return;\n", + inst->address); switch (inst->op) { case PPC_OP_MULLI: diff --git a/src/backend/llvm/llvm_backend.cpp b/src/backend/llvm/llvm_backend.cpp index 93e48ed..fa818d6 100644 --- a/src/backend/llvm/llvm_backend.cpp +++ b/src/backend/llvm/llvm_backend.cpp @@ -1,4 +1,5 @@ #include "backend/llvm/llvm_backend.h" +#include "common/options.h" #include "backend/llvm/llvm_function_emitter.h" #include @@ -13,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -90,6 +93,25 @@ static std::string targetFeatures() { // against CPU "generic" with an empty feature string is wrong -- SSE2 is part // of the x86-64 baseline, and Gekko paired-singles are inherently 2-wide f32 // pairs, so SLP has real work to do at that baseline. +// The hand-rolled pipeline below is one shot over each function, and it is +// weaker than what the C backend's chunks get: clang -O3 iterates function +// simplification, interleaves inlining with cleanup inside the CGSCC walk, and +// runs SROA, loop unrolling and several more rounds of instcombine. Here +// cgscc(inline) runs *last*, followed only by ipsccp and globaldce, so nothing +// ever simplifies inlined code. +// +// DOLRECOMP_LLVM_PIPELINE=o3 swaps it for LLVM's own -O3 module pipeline, the +// same one clang builds, and raises codegen from Default (O2) to Aggressive. +// Measuring that is the first step in explaining why compiled C beats this +// backend by 60-70% on the same DolIR (AOT-PERFORMANCE-RESULTS.md 5t). +static bool defaultO3Pipeline() { + static const bool enabled = [] { + const char *value = std::getenv("DOLRECOMP_LLVM_PIPELINE"); + return value && value[0] == 'o' && value[1] == '3'; + }(); + return enabled; +} + static constexpr const char *kPassPipeline = "function(mem2reg,early-cse,instcombine," "simplifycfg,sccp," @@ -244,6 +266,8 @@ extern "C" bool dolllvm_emit_object(const DolIRModule *source, (void)initialized; int opt = options ? options->optimization_level : 2; + if (defaultO3Pipeline()) + opt = 3; std::string tripleName = resolveTriple(options ? options->target_triple : nullptr); const llvm::Triple triple(tripleName); @@ -396,6 +420,49 @@ extern "C" bool dolllvm_emit_object(const DolIRModule *source, /*IsCS=*/false)); passes.run(module, mam); + // ThinLTO bitcode, emitted from a CLONE of the optimised module. + // + // ThinLTOBitcodeWriterPass is not a pure writer: it splits the module into + // the part that can be thin-imported and the part that cannot, and it does + // that in place. Running it on `module` would hand a mutated module to the + // object emission below, so the object and the bitcode would describe + // different programs -- with the object being the wrong one. + // + // The clone costs memory proportional to one region, which is bounded by + // the region size cap, and only when bitcode is asked for. + // + // Letting the pass build the summary is the point: a first attempt called + // buildModuleSummaryIndex(module, nullptr, nullptr) by hand and segfaulted, + // because that signature wants a BlockFrequencyInfo callback and a real + // ProfileSummaryInfo rather than nulls. This is how clang does it. + if (options && options->emit_bitcode && options->bitcode_path) { + std::error_code bitcodeError; + llvm::raw_fd_ostream bitcodeFile(options->bitcode_path, bitcodeError, + llvm::sys::fs::OF_None); + if (bitcodeError) { + fprintf(diagnostics, "dolllvm: cannot write bitcode: %s\n", + bitcodeError.message().c_str()); + return false; + } + std::unique_ptr clone = llvm::CloneModule(module); + llvm::LoopAnalysisManager cloneLam; + llvm::FunctionAnalysisManager cloneFam; + llvm::CGSCCAnalysisManager cloneCgam; + llvm::ModuleAnalysisManager cloneMam; + llvm::PassBuilder clonePassBuilder(machine); + clonePassBuilder.registerModuleAnalyses(cloneMam); + clonePassBuilder.registerCGSCCAnalyses(cloneCgam); + clonePassBuilder.registerFunctionAnalyses(cloneFam); + clonePassBuilder.registerLoopAnalyses(cloneLam); + clonePassBuilder.crossRegisterProxies(cloneLam, cloneFam, cloneCgam, + cloneMam); + llvm::ModulePassManager bitcodePasses; + bitcodePasses.addPass( + llvm::ThinLTOBitcodeWriterPass(bitcodeFile, /*ThinLinkOS=*/nullptr)); + bitcodePasses.run(*clone, cloneMam); + bitcodeFile.flush(); + } + // P002. The verdict. Under `error` a stale profile stops the build here, // which is the point: the failure this gate exists for is a build that // SUCCEEDS while training on records that no longer describe it, and every @@ -525,7 +592,33 @@ extern "C" bool dolllvm_codegen_fingerprint(char *out, size_t size) { (dolllvm_pgo_mode() == DOLLLVM_PGO_GEN ? "|pgo=gen" : "") + (dolllvm_pgo_mode() == DOLLLVM_PGO_USE ? "|pgo=use:" + pgoProfileFingerprint() - : ""); + : "") + + // Emitter behaviour flags. These change generated code, and the cache key + // does not hash the emitter's source -- so leaving them out means flipping + // one silently reuses objects built with the other setting and the + // measurement compares nothing. That has happened three times in this + // project; absent when off, so default objects stay byte-identical. + (std::getenv("DOLRECOMP_INLINE_REGIONS") && + std::getenv("DOLRECOMP_INLINE_REGIONS")[0] == '1' + ? "|inline=1" : "") + + // --memory-mode changes the body of every load and store, so a cached + // object from one mode is not a valid answer for a build in the other. + // Both modes are marked, unlike the other flags here: objects predating + // this option were emitted in safe mode and carried no marker, so + // leaving the new default unmarked would let them satisfy a fast-mode + // build. Marking both invalidates those once, which is the point. + (memory_mode_is_fast() ? "|mem=fast" : "|mem=safe") + + // Suppresses every direct call, so it changes far more emitted code than + // any other flag here. + (replacements_enabled() ? "|repl=1" : "") + + // A different optimization pipeline entirely. + (defaultO3Pipeline() ? "|pipeline=o3" : "") + + // Constant, and deliberately not removed with the promoting emitter it + // used to select. Every object built before guest state stopped being + // hoisted carries no marker, and those objects are incompatible with + // these; without something here to tell them apart, a stale one from a + // shared cache would satisfy this build silently. + "|state=mem"; if (fingerprint.size() + 1 > size) return false; memcpy(out, fingerprint.c_str(), fingerprint.size() + 1); diff --git a/src/backend/llvm/llvm_backend.h b/src/backend/llvm/llvm_backend.h index 07fa1cc..ee452fd 100644 --- a/src/backend/llvm/llvm_backend.h +++ b/src/backend/llvm/llvm_backend.h @@ -18,6 +18,14 @@ typedef struct { int verify; int emit_ir; const char* ir_path; + /* ThinLTO needs bitcode carrying a module summary index, not plain bitcode: + the summary is what lets the thin link decide which callees to import + without reading every module's body. + + Emitted from a clone of the optimised module, because the writer pass + splits the module in place and the object must not see that. */ + int emit_bitcode; + const char* bitcode_path; const DolLLVMFunctionRange* function_ranges; u32 function_range_count; } DolLLVMOptions; diff --git a/src/backend/llvm/llvm_control_flow.cpp b/src/backend/llvm/llvm_control_flow.cpp index 745768a..6359215 100644 --- a/src/backend/llvm/llvm_control_flow.cpp +++ b/src/backend/llvm/llvm_control_flow.cpp @@ -1,7 +1,11 @@ #include "backend/llvm/llvm_function_emitter.h" +#include "common/options.h" #include "cpu/cpu.h" +#include + #include +#include #include #include @@ -20,15 +24,59 @@ BasicBlock *FunctionEmitter::directDestination(const DolIRTerminator &term, return externalDestination(term, slot); } +// Ranges are sorted by start address and do not overlap, so this is a binary +// search rather than the scan it used to be. +// +// The scan was tolerable while a range was a fixed chunk and there were a few +// thousand of them. Planned regions produce one range per contiguous run, which +// is several times as many, and this is called for every external destination +// in every block -- so the cost is (ranges x edges) and it dominated emission: +// region objects came out at roughly a ninth the rate of fixed chunks until +// this changed. +// +// The caller guarantees the sort. Both producers in pipeline.c emit ranges in +// ascending address order. const DolLLVMFunctionRange *FunctionEmitter::rangeFor(u32 address) const { - for (u32 i = 0; i < range_count_; i++) - if (address >= ranges_[i].start && address < ranges_[i].end) - return &ranges_[i]; + u32 low = 0; + u32 high = range_count_; + while (low < high) { + u32 mid = low + (high - low) / 2; + if (address < ranges_[mid].start) { + high = mid; + } else if (address >= ranges_[mid].end) { + low = mid + 1; + } else { + return &ranges_[mid]; + } + } return nullptr; } +// Patchability policy for direct calls. +// +// A direct call goes straight to func_XXXXXXXX_budget and therefore does NOT +// pass dolrecomp_dispatch_replacement, ppc_host_call, or the physical-alias +// retry that dolrecomp_call performs. That is sound only while no address in +// this module can be replaced at runtime. +// +// Today it is: the module template never defines DOLRECOMP_ENABLE_REPLACEMENTS, +// so the generated dispatcher compiles the replacement check to a stub that +// returns 0, and StaticRecompModuleDesc exposes no way to register one. But a +// build that turns replacements on would get its replacements silently ignored +// at every direct call site -- a mod that appears installed and does nothing. +// +// So the policy is explicit rather than incidental: when replacements are +// enabled, every external transfer leaves through the dispatcher. Returning +// nullptr here makes the caller emit a side exit, which is exactly that. +static bool replacementsEnabled() { + static const bool enabled = replacements_enabled() != 0; + return enabled; +} + BasicBlock *FunctionEmitter::externalDestination(const DolIRTerminator &term, u32 slot) { + if (replacementsEnabled()) + return nullptr; u32 target = term.target_addresses[slot]; const DolLLVMFunctionRange *range = rangeFor(target); if (!range) @@ -49,8 +97,13 @@ BasicBlock *FunctionEmitter::externalDestination(const DolIRTerminator &term, if (auto *calleeFunction = dyn_cast(callee.getCallee())) { calleeFunction->setVisibility(GlobalValue::HiddenVisibility); calleeFunction->setDSOLocal(true); + // Must match the definition, or the call is undefined behaviour rather than + // merely slow. + calleeFunction->setCallingConv(CallingConv::Fast); } - builder_.CreateCall(callee, {ctx_, guard_cycles_, guard_steps_}); + CallInst *direct = + builder_.CreateCall(callee, {ctx_, guard_cycles_, guard_steps_}); + direct->setCallingConv(CallingConv::Fast); if (!term.linked) { builder_.CreateRetVoid(); builder_.restoreIP(saved); @@ -77,12 +130,8 @@ BasicBlock *FunctionEmitter::externalDestination(const DolIRTerminator &term, if (!local || continuationBlock >= blocks_.size()) { builder_.CreateRetVoid(); } else { - for (u32 state = 0; state < DOLIR_STATE_COUNT; state++) { - if (!used_[state]) - continue; - auto stateSlot = static_cast(state); - builder_.CreateStore(loadContext(stateSlot), state_[state]); - } + // Nothing to restore: the callee wrote guest state where this function + // reads it. Only the local cycle counter is emitter bookkeeping. builder_.CreateStore(builder_.getInt64(0), cycles_); builder_.CreateBr(blocks_[continuationBlock]); } @@ -138,14 +187,8 @@ bool FunctionEmitter::emitTerminator(const DolIRTerminator &term, blocks_[block]); builder_.SetInsertPoint(unknown); } - for (u32 slot = 0; slot < DOLIR_STATE_COUNT; slot++) { - if (dirty_[slot]) { - auto stateSlot = static_cast(slot); - storeContext(stateSlot, - builder_.CreateLoad(type(dolir_state_type(stateSlot)), - state_[slot])); - } - } + // Guest state is already in CPUState; only the transfer target and the + // cycles owed still have to be written back. storeContext(DOLIR_STATE_PC, target); Value *downcount = loadOffset(Type::getInt64Ty(context_), offsetof(CPUState, downcount)); diff --git a/src/backend/llvm/llvm_function_emitter.cpp b/src/backend/llvm/llvm_function_emitter.cpp index a7cc89b..606fed1 100644 --- a/src/backend/llvm/llvm_function_emitter.cpp +++ b/src/backend/llvm/llvm_function_emitter.cpp @@ -1,12 +1,16 @@ #include "backend/llvm/llvm_function_emitter.h" +#include "common/options.h" #include "cpu/cpu.h" #include +#include #include #include #include #include +#include +#include #include #include #include @@ -16,6 +20,25 @@ namespace dolllvm { using namespace llvm; +// Off by default, and the reason is measured rather than assumed: chunk size +// drives how much guest state the register allocator keeps live, and that is +// what made 1024-instruction chunks cost 3x the code size of 128 for a third +// less speed (pipeline.c, LLVM-EXPERIMENTS E002/E003). Inlining a callee into +// its caller has the same shape -- it merges two live ranges. +// +// Worth measuring precisely because fastcc makes it possible: with the internal +// bodies no longer NoInline, LLVM can inline a small or hot callee across a +// region boundary, which is Phase 3's direct-linking benefit rather than a +// dispatcher saving. +// DOLRECOMP_INLINE_REGIONS=1 let LLVM inline internal region bodies +static bool inlineRegions() { + static const bool enabled = [] { + const char *value = std::getenv("DOLRECOMP_INLINE_REGIONS"); + return value && value[0] == '1'; + }(); + return enabled; +} + FunctionEmitter::FunctionEmitter(LLVMContext &context, Module &module, const DolIRFunction &source, const DolLLVMFunctionRange *ranges, @@ -36,21 +59,40 @@ bool FunctionEmitter::emit(raw_ostream &diagnostics) { diagnostics << "dolllvm: conflicting native body " << bodyName << "\n"; return false; } - function_->setCallingConv(CallingConv::C); + // The internal body uses fastcc; the public wrapper below keeps the C + // convention because that is the ModernGekko ABI and mods, hooks and the + // dispatcher all call through it. + // + // D3 also proposed carrying live guest state in these registers. That was + // built and measured at -2.3% fps (AOT-PERFORMANCE-RESULTS.md 5s), then + // removed outright when state stopped being hoisted: with nothing in + // registers to hand over, there is nothing for a wider signature to carry. + function_->setCallingConv(CallingConv::Fast); function_->setVisibility(GlobalValue::HiddenVisibility); function_->setDSOLocal(true); - function_->addFnAttr(Attribute::NoInline); + if (!inlineRegions()) + function_->addFnAttr(Attribute::NoInline); ctx_ = function_->getArg(0); ctx_->setName("ctx"); guard_cycles_ = function_->getArg(1); guard_cycles_->setName("guard_cycles"); guard_steps_ = function_->getArg(2); guard_steps_->setName("guard_steps"); + // Both counters are allocas created in the wrapper and passed only to this + // body, so they cannot alias CPUState or each other -- but nothing said so, + // and the budget guard updates them inside the hottest loops. Without this + // every guard store looks like it might clobber cr/gpr, which forces a + // reload of guest state on the next instruction that reads it. + function_->addParamAttr(1, Attribute::NoAlias); + function_->addParamAttr(2, Attribute::NoAlias); entry_ = BasicBlock::Create(context_, "entry", function_); for (u32 i = 0; i < source_.block_count; i++) blocks_.push_back(BasicBlock::Create(context_, blockName(i), function_)); scanState(); + // scanContinuations() first: both dataflow passes need the indirect-switch + // edges it discovers, and running them before it is what made the first two + // attempts model a different graph than the emitter generates. scanContinuations(); scanLoopHeaders(); emitEntry(); @@ -86,7 +128,9 @@ bool FunctionEmitter::emitWrapper(raw_ostream &diagnostics) { builder.CreateAlloca(Type::getInt64Ty(context_), nullptr, "guard_steps"); builder.CreateStore(builder.getInt64(0), guardCycles); builder.CreateStore(builder.getInt64(0), guardSteps); - builder.CreateCall(function_, {wrapper->getArg(0), guardCycles, guardSteps}); + CallInst *body = builder.CreateCall( + function_, {wrapper->getArg(0), guardCycles, guardSteps}); + body->setCallingConv(CallingConv::Fast); builder.CreateRetVoid(); return !verifyFunction(*wrapper, &diagnostics); } @@ -184,17 +228,52 @@ Value *FunctionEmitter::bytePtr(size_t offset) { ConstantInt::get(Type::getInt64Ty(context_), offset)); } +void FunctionEmitter::initAliasScopes() { + llvm::MDBuilder md(context_); + alias_domain_ = md.createAnonymousAliasScopeDomain("dolrecomp.guest"); + MDNode *const state = md.createAnonymousAliasScope(alias_domain_, "cpustate"); + MDNode *const guest = md.createAnonymousAliasScope(alias_domain_, "guestmem"); + scope_state_list_ = MDNode::get(context_, {state}); + scope_guest_list_ = MDNode::get(context_, {guest}); +} + +// An access tagged with a scope declares it touches that scope; tagged noalias +// against the other declares it cannot touch it. Anything left untagged stays +// conservative and may alias both, which is what keeps helper calls and MMIO +// paths correct without enumerating them. +void FunctionEmitter::tagState(Value *access) { + auto *const instruction = llvm::dyn_cast_or_null(access); + if (!instruction || !scope_state_list_) + return; + instruction->setMetadata(llvm::LLVMContext::MD_alias_scope, + scope_state_list_); + instruction->setMetadata(llvm::LLVMContext::MD_noalias, scope_guest_list_); +} + +void FunctionEmitter::tagGuestMemory(Value *access) { + auto *const instruction = llvm::dyn_cast_or_null(access); + if (!instruction || !scope_guest_list_) + return; + instruction->setMetadata(llvm::LLVMContext::MD_alias_scope, + scope_guest_list_); + instruction->setMetadata(llvm::LLVMContext::MD_noalias, scope_state_list_); +} + Value *FunctionEmitter::loadContext(DolIRStateSlot slot) { - return builder_.CreateLoad(type(dolir_state_type(slot)), - bytePtr(stateOffset(slot))); + Value *const loaded = builder_.CreateLoad(type(dolir_state_type(slot)), + bytePtr(stateOffset(slot))); + tagState(loaded); + return loaded; } void FunctionEmitter::storeContext(DolIRStateSlot slot, Value *value) { - builder_.CreateStore(value, bytePtr(stateOffset(slot))); + tagState(builder_.CreateStore(value, bytePtr(stateOffset(slot)))); } Value *FunctionEmitter::loadOffset(Type *valueType, size_t offset) { - return builder_.CreateLoad(valueType, bytePtr(offset)); + Value *const loaded = builder_.CreateLoad(valueType, bytePtr(offset)); + tagState(loaded); + return loaded; } void FunctionEmitter::scanState() { @@ -204,8 +283,6 @@ void FunctionEmitter::scanState() { const DolIRInstruction &inst = block.instructions[i]; if (inst.op == DOLIR_OP_STATE_READ || inst.op == DOLIR_OP_STATE_WRITE) used_[inst.aux] = true; - if (inst.op == DOLIR_OP_STATE_WRITE) - dirty_[inst.aux] = true; if (inst.op == DOLIR_OP_HELPER_CALL && inst.aux == DOLIR_HELPER_FP_AVAILABLE) used_[DOLIR_STATE_MSR] = true; @@ -219,35 +296,28 @@ void FunctionEmitter::scanState() { inst.aux == DOLIR_HELPER_PSQ_LOAD) { u32 reg = inst.immediate & 0xFFu; used_[DOLIR_STATE_FPR0 + reg] = true; - dirty_[DOLIR_STATE_FPR0 + reg] = true; used_[DOLIR_STATE_PS1_0 + reg] = true; - dirty_[DOLIR_STATE_PS1_0 + reg] = true; } if (inst.op == DOLIR_OP_HELPER_CALL && inst.aux == DOLIR_HELPER_STORE_CONDITIONAL) { used_[DOLIR_STATE_CR] = true; - dirty_[DOLIR_STATE_CR] = true; used_[DOLIR_STATE_RESERVE_VALID] = true; - dirty_[DOLIR_STATE_RESERVE_VALID] = true; used_[DOLIR_STATE_RESERVE_ADDR] = true; } if (inst.op == DOLIR_OP_HELPER_CALL && (inst.aux == DOLIR_HELPER_FPSCR_UPDATED || inst.aux == DOLIR_HELPER_FPSCR_BIT)) { used_[DOLIR_STATE_FPSCR] = true; - dirty_[DOLIR_STATE_FPSCR] = true; } if (inst.op == DOLIR_OP_HELPER_CALL && inst.aux == DOLIR_HELPER_LSWX) { used_[DOLIR_STATE_XER] = true; for (u32 reg = 0; reg < 32; reg++) { used_[DOLIR_STATE_GPR0 + reg] = true; - dirty_[DOLIR_STATE_GPR0 + reg] = true; } } if (inst.op == DOLIR_OP_GUEST_STORE) { used_[DOLIR_STATE_RESERVE_ADDR] = true; used_[DOLIR_STATE_RESERVE_VALID] = true; - dirty_[DOLIR_STATE_RESERVE_VALID] = true; } } } @@ -260,22 +330,18 @@ void FunctionEmitter::scanExactFloat(u64 descriptor) { u32 b = (descriptor >> 24) & 0xFFu; u32 c = (descriptor >> 32) & 0xFFu; used_[DOLIR_STATE_FPSCR] = true; - dirty_[DOLIR_STATE_FPSCR] = true; if (op == DOLIR_EXACT_FCMPU || op == DOLIR_EXACT_FCMPO) { used_[DOLIR_STATE_CR] = true; - dirty_[DOLIR_STATE_CR] = true; used_[DOLIR_STATE_FPR0 + a] = true; used_[DOLIR_STATE_FPR0 + b] = true; return; } used_[DOLIR_STATE_FPR0 + d] = true; - dirty_[DOLIR_STATE_FPR0 + d] = true; if (op == DOLIR_EXACT_FRES || (op >= DOLIR_EXACT_FADDS && op <= DOLIR_EXACT_FDIVS) || op == DOLIR_EXACT_FRSP || (op >= DOLIR_EXACT_FMADDS && op <= DOLIR_EXACT_FNMSUBS)) { used_[DOLIR_STATE_PS1_0 + d] = true; - dirty_[DOLIR_STATE_PS1_0 + d] = true; } if (op == DOLIR_EXACT_FRES || op == DOLIR_EXACT_FRSQRTE || op == DOLIR_EXACT_FCTIW || op == DOLIR_EXACT_FCTIWZ || @@ -302,10 +368,8 @@ void FunctionEmitter::scanExactPaired(u64 descriptor) { u32 b = (descriptor >> 24) & 0xFFu; u32 c = (descriptor >> 32) & 0xFFu; used_[DOLIR_STATE_FPSCR] = true; - dirty_[DOLIR_STATE_FPSCR] = true; if (op >= DOLIR_EXACT_PS_CMPU0) { used_[DOLIR_STATE_CR] = true; - dirty_[DOLIR_STATE_CR] = true; used_[DOLIR_STATE_FPR0 + a] = true; used_[DOLIR_STATE_PS1_0 + a] = true; used_[DOLIR_STATE_FPR0 + b] = true; @@ -313,9 +377,7 @@ void FunctionEmitter::scanExactPaired(u64 descriptor) { return; } used_[DOLIR_STATE_FPR0 + d] = true; - dirty_[DOLIR_STATE_FPR0 + d] = true; used_[DOLIR_STATE_PS1_0 + d] = true; - dirty_[DOLIR_STATE_PS1_0 + d] = true; auto usePair = [this](u32 reg) { used_[DOLIR_STATE_FPR0 + reg] = true; used_[DOLIR_STATE_PS1_0 + reg] = true; @@ -367,15 +429,19 @@ void FunctionEmitter::scanLoopHeaders() { } void FunctionEmitter::emitEntry() { + initAliasScopes(); builder_.SetInsertPoint(entry_); for (u32 slot = 0; slot < DOLIR_STATE_COUNT; slot++) { if (!used_[slot]) continue; - auto stateSlot = static_cast(slot); - state_[slot] = builder_.CreateAlloca(type(dolir_state_type(stateSlot)), - nullptr, "state"); - builder_.CreateStore(loadContext(stateSlot), state_[slot]); + // Guest state is read and written where it already lives. No alloca, no + // prologue load, no copy -- and consequently nothing that later has to be + // flushed back, which is what removed the materialization barriers. + state_[slot] = bytePtr(stateOffset(static_cast(slot))); } + ram_base_ = loadOffset(PointerType::getUnqual(context_), + offsetof(CPUState, ram)); + ram_base_->setName("ram_base"); cycles_ = builder_.CreateAlloca(Type::getInt64Ty(context_), nullptr, "cycles"); builder_.CreateStore(ConstantInt::get(Type::getInt64Ty(context_), 0), @@ -406,14 +472,10 @@ void FunctionEmitter::chargeCycles(u32 cycles) { } void FunctionEmitter::materialize(u32 pc) { - for (u32 slot = 0; slot < DOLIR_STATE_COUNT; slot++) { - if (!dirty_[slot]) - continue; - auto stateSlot = static_cast(slot); - storeContext( - stateSlot, - builder_.CreateLoad(type(dolir_state_type(stateSlot)), state_[slot])); - } + // Guest state is already in CPUState -- it was never hoisted out (see + // emitEntry). What used to stand here was a flush of every dirty slot, and + // the analyses built to narrow it; all of that went when the hoisting did. + // What a barrier still owes is the guest PC and the cycles not yet charged. storeContext(DOLIR_STATE_PC, ConstantInt::get(Type::getInt32Ty(context_), pc)); Value *downcount = @@ -454,6 +516,9 @@ void FunctionEmitter::emitBudgetGuard(u32 pc) { bool FunctionEmitter::emitBlock(u32 index, raw_ostream &diagnostics) { const DolIRBlock &block = source_.blocks[index]; builder_.SetInsertPoint(blocks_[index]); + // Every materialize() emitted while this block is being lowered consults the + // reaching-writes set for it. + current_block_ = index; if (loop_headers_[index]) emitBudgetGuard(block.guest_address); chargeCycles(block.cycle_cost); @@ -515,9 +580,10 @@ bool FunctionEmitter::emitInstruction(const DolIRInstruction &inst, break; case DOLIR_OP_STATE_READ: result = builder_.CreateLoad(resultType, state_[inst.aux]); + tagState(result); break; case DOLIR_OP_STATE_WRITE: - builder_.CreateStore(operand(inst, 0), state_[inst.aux]); + tagState(builder_.CreateStore(operand(inst, 0), state_[inst.aux])); break; case DOLIR_OP_ADD: result = builder_.CreateAdd(operand(inst, 0), operand(inst, 1)); diff --git a/src/backend/llvm/llvm_function_emitter.h b/src/backend/llvm/llvm_function_emitter.h index cebaaae..dd11c83 100644 --- a/src/backend/llvm/llvm_function_emitter.h +++ b/src/backend/llvm/llvm_function_emitter.h @@ -17,6 +17,7 @@ class Argument; class BasicBlock; class Function; class LLVMContext; +class MDNode; class Module; class Type; class Value; @@ -42,6 +43,15 @@ class FunctionEmitter final { void storeContext(DolIRStateSlot slot, llvm::Value *value); llvm::Value *loadOffset(llvm::Type *value_type, std::size_t offset); + // Guest RAM and CPUState are separate allocations -- guest addresses reach + // RAM through ctx->ram and can never name the host-side state struct -- but + // nothing told LLVM that. Both are reached through pointers loaded out of + // ctx, so every guest store looked like it might clobber cr/gpr and forced a + // reload afterwards. Two alias scopes state the disjointness. + void initAliasScopes(); + void tagState(llvm::Value *access); + void tagGuestMemory(llvm::Value *access); + void scanState(); void scanExactFloat(u64 descriptor); void scanExactPaired(u64 descriptor); @@ -64,9 +74,6 @@ class FunctionEmitter final { llvm::AllocaInst *temporary(llvm::Type *value_type, llvm::StringRef name); llvm::Value *stateValue(DolIRStateSlot slot); - void syncState(DolIRStateSlot slot); - void reloadState(DolIRStateSlot slot); - void reloadUsedState(); void continueAfterRuntimeBoundary(llvm::StringRef prefix); void emitFPSCRUpdated(); void emitFPSCRBit(u64 descriptor); @@ -112,13 +119,26 @@ class FunctionEmitter final { llvm::Argument *ctx_ = nullptr; llvm::BasicBlock *entry_ = nullptr; llvm::AllocaInst *cycles_ = nullptr; + // MEM1's base, loaded once at region entry. ctx->ram is fixed for the whole + // burst -- the chassis sets it before dispatch and nothing the module can + // call reallocates MEM1 -- but it sat behind a load on every guest access, + // and LLVM cannot hoist that because a CPUState store shares its alias + // scope. MEM2 deliberately keeps its per-access load: cpu_alloc_mem2 can + // allocate it after a region has already started running. + llvm::Value *ram_base_ = nullptr; // Shared across generated calls until control returns to the dispatcher. llvm::Value *guard_cycles_ = nullptr; // Termination backstop for zero-cycle loops. llvm::Value *guard_steps_ = nullptr; - std::array state_{}; + // Where each guest state slot lives: a pointer straight into CPUState, so a + // read or write of a slot is a read or write of the field itself. + std::array state_{}; + + llvm::MDNode *alias_domain_ = nullptr; + llvm::MDNode *scope_state_list_ = nullptr; + llvm::MDNode *scope_guest_list_ = nullptr; std::array used_{}; - std::array dirty_{}; + u32 current_block_ = 0; std::vector blocks_; std::vector loop_headers_; std::vector values_; diff --git a/src/backend/llvm/llvm_memory_lowering.cpp b/src/backend/llvm/llvm_memory_lowering.cpp index a92994a..928f4bf 100644 --- a/src/backend/llvm/llvm_memory_lowering.cpp +++ b/src/backend/llvm/llvm_memory_lowering.cpp @@ -1,6 +1,9 @@ #include "backend/llvm/llvm_function_emitter.h" +#include "common/options.h" #include "cpu/cpu.h" +#include + #include #include #include @@ -10,6 +13,30 @@ namespace dolllvm { using namespace llvm; +// Guest memory lowering mode. +// +// safe (default) reads every bound out of CPUState on every access and checks +// the write journal on every MEM1 store, assuming nothing about the runtime. +// +// fast trades two assumptions for a much shorter fast path, and both are +// verified once at dispatch entry rather than assumed -- see +// emit_memory_mode_guard() in src/backend/dispatch.c. If either fails the +// module refuses to run natively and the chassis keeps interpreting, so a +// violated assumption costs speed and not correctness: +// +// 1. ctx->ram_size == GC_MAIN_RAM_SIZE. True in both this tree and +// GXRuntime: assigned once in cpu_init, carried across cpu_reset, never +// given another value. Folding it removes a CPUState load per access and +// collapses the bounds check to a single compare against a constant -- +// the "size >= width" half is constant-true for any width <= 24 MB. +// 2. g_mem_write_journal == NULL. Lets the journal branch leave the MEM1 +// store path entirely. A runtime that installs a journal (savestate +// diffing, netplay) must build with the safe mode. +bool memoryModeFast() { + static const bool enabled = memory_mode_is_fast() != 0; + return enabled; +} + Value *FunctionEmitter::normalizeAddress(Value *address) { return builder_.CreateAnd(address, builder_.getInt32(~0x40000000u)); } @@ -26,6 +53,7 @@ Value *FunctionEmitter::endianLoad(Value *pointer, Type *resultType, u32 width) { Type *integerType = IntegerType::get(context_, width * 8u); Value *loaded = builder_.CreateLoad(integerType, pointer); + tagGuestMemory(loaded); loaded = bswap(loaded); if (resultType != integerType) loaded = builder_.CreateZExtOrTrunc(loaded, resultType); @@ -57,10 +85,6 @@ Value *FunctionEmitter::externalRead(Value *address, u32 width) { builder_.SetInsertPoint(failed); builder_.CreateRetVoid(); builder_.SetInsertPoint(resume); - for (u32 slot = 0; slot < DOLIR_STATE_COUNT; slot++) { - if (used_[slot]) - reloadState(static_cast(slot)); - } builder_.CreateStore(builder_.getInt64(0), cycles_); builder_.CreateBr(join); BasicBlock *calledEnd = builder_.GetInsertBlock(); @@ -77,8 +101,10 @@ Value *FunctionEmitter::externalRead(Value *address, u32 width) { Value *FunctionEmitter::emitGuestLoad(Value *address, Type *resultType, u32 width, bool sign) { Value *normalized = normalizeAddress(address); - Value *ramSize = - loadOffset(Type::getInt32Ty(context_), offsetof(CPUState, ram_size)); + Value *ramSize = memoryModeFast() + ? cast(builder_.getInt32(GC_MAIN_RAM_SIZE)) + : loadOffset(Type::getInt32Ty(context_), + offsetof(CPUState, ram_size)); Value *exramSize = loadOffset(Type::getInt32Ty(context_), offsetof(CPUState, exram_size)); Value *mem1 = rangeCheck(normalized, GC_RAM_BASE, ramSize, width); @@ -91,8 +117,7 @@ Value *FunctionEmitter::emitGuestLoad(Value *address, Type *resultType, builder_.CreateCondBr(mem1, mem1Block, checkMem2); builder_.SetInsertPoint(mem1Block); - Value *ram = - loadOffset(PointerType::getUnqual(context_), offsetof(CPUState, ram)); + Value *ram = ram_base_; Value *mem1Offset = builder_.CreateSub(normalized, builder_.getInt32(GC_RAM_BASE)); Value *mem1Ptr = @@ -149,6 +174,8 @@ void FunctionEmitter::clearReservation(Value *address) { } void FunctionEmitter::journal(Value *offset, u32 width) { + if (memoryModeFast()) + return; Type *ptr = PointerType::getUnqual(context_); GlobalVariable *journal = cast( module_.getOrInsertGlobal("g_mem_write_journal", ptr)); @@ -174,7 +201,7 @@ void FunctionEmitter::endianStore(Value *pointer, Value *value, u32 width) { Value *narrowed = value; if (value->getType() != integerType) narrowed = builder_.CreateZExtOrTrunc(value, integerType); - builder_.CreateStore(bswap(narrowed), pointer); + tagGuestMemory(builder_.CreateStore(bswap(narrowed), pointer)); } void FunctionEmitter::externalWrite(Value *address, Value *value, u32 width) { @@ -206,10 +233,6 @@ void FunctionEmitter::externalWrite(Value *address, Value *value, u32 width) { builder_.SetInsertPoint(failed); builder_.CreateRetVoid(); builder_.SetInsertPoint(resume); - for (u32 slot = 0; slot < DOLIR_STATE_COUNT; slot++) { - if (used_[slot]) - reloadState(static_cast(slot)); - } builder_.CreateStore(builder_.getInt64(0), cycles_); builder_.CreateBr(done); builder_.SetInsertPoint(done); @@ -218,8 +241,10 @@ void FunctionEmitter::externalWrite(Value *address, Value *value, u32 width) { void FunctionEmitter::emitGuestStore(Value *address, Value *value, u32 width) { clearReservation(address); Value *normalized = normalizeAddress(address); - Value *ramSize = - loadOffset(Type::getInt32Ty(context_), offsetof(CPUState, ram_size)); + Value *ramSize = memoryModeFast() + ? cast(builder_.getInt32(GC_MAIN_RAM_SIZE)) + : loadOffset(Type::getInt32Ty(context_), + offsetof(CPUState, ram_size)); Value *exramSize = loadOffset(Type::getInt32Ty(context_), offsetof(CPUState, exram_size)); BasicBlock *mem1Block = BasicBlock::Create(context_, "store_mem1", function_); @@ -232,8 +257,7 @@ void FunctionEmitter::emitGuestStore(Value *address, Value *value, u32 width) { mem1Block, checkMem2); builder_.SetInsertPoint(mem1Block); - Value *ram = - loadOffset(PointerType::getUnqual(context_), offsetof(CPUState, ram)); + Value *ram = ram_base_; Value *mem1Offset = builder_.CreateSub(normalized, builder_.getInt32(GC_RAM_BASE)); journal(mem1Offset, width); diff --git a/src/backend/llvm/llvm_runtime_lowering.cpp b/src/backend/llvm/llvm_runtime_lowering.cpp index bc15325..5de2724 100644 --- a/src/backend/llvm/llvm_runtime_lowering.cpp +++ b/src/backend/llvm/llvm_runtime_lowering.cpp @@ -1,4 +1,5 @@ #include "backend/llvm/llvm_function_emitter.h" +#include "common/options.h" #include "cpu/cpu.h" #include @@ -19,22 +20,6 @@ Value *FunctionEmitter::stateValue(DolIRStateSlot slot) { return builder_.CreateLoad(type(dolir_state_type(slot)), state_[slot]); } -void FunctionEmitter::syncState(DolIRStateSlot slot) { - storeContext(slot, stateValue(slot)); -} - -void FunctionEmitter::reloadState(DolIRStateSlot slot) { - builder_.CreateStore(loadContext(slot), state_[slot]); -} - -void FunctionEmitter::reloadUsedState() { - for (u32 slot = 0; slot < DOLIR_STATE_COUNT; slot++) { - if (used_[slot]) - reloadState(static_cast(slot)); - } - builder_.CreateStore(builder_.getInt64(0), cycles_); -} - void FunctionEmitter::continueAfterRuntimeBoundary(StringRef prefix) { Value *exception = loadOffset(Type::getInt32Ty(context_), offsetof(CPUState, exception)); @@ -47,21 +32,20 @@ void FunctionEmitter::continueAfterRuntimeBoundary(StringRef prefix) { builder_.SetInsertPoint(failed); builder_.CreateRetVoid(); builder_.SetInsertPoint(resume); - reloadUsedState(); + // Guest state is already current in CPUState; only the local cycle counter + // is emitter bookkeeping and has to be reset. + builder_.CreateStore(builder_.getInt64(0), cycles_); } void FunctionEmitter::emitFPSCRUpdated() { - syncState(DOLIR_STATE_FPSCR); auto callee = module_.getOrInsertFunction( "ppc_fpscr_control_updated", FunctionType::get(Type::getVoidTy(context_), {PointerType::getUnqual(context_)}, false)); builder_.CreateCall(callee, {ctx_}); - reloadState(DOLIR_STATE_FPSCR); } void FunctionEmitter::emitFPSCRBit(u64 descriptor) { - syncState(DOLIR_STATE_FPSCR); const char *name = ((descriptor >> 8) & 1u) ? "ppc_mtfsb1_op" : "ppc_mtfsb0_op"; auto callee = module_.getOrInsertFunction( @@ -70,7 +54,6 @@ void FunctionEmitter::emitFPSCRBit(u64 descriptor) { {PointerType::getUnqual(context_), Type::getInt8Ty(context_)}, false)); builder_.CreateCall(callee, {ctx_, builder_.getInt8(descriptor & 0xFFu)}); - reloadState(DOLIR_STATE_FPSCR); } void FunctionEmitter::emitProgramException(const DolIRInstruction &inst) { @@ -211,10 +194,8 @@ void FunctionEmitter::emitExactFloat(u64 descriptor) { }; Type *ptr = PointerType::getUnqual(context_); Type *f64 = Type::getDoubleTy(context_); - syncState(DOLIR_STATE_FPSCR); if (op == DOLIR_EXACT_FCMPU || op == DOLIR_EXACT_FCMPO) { - syncState(DOLIR_STATE_CR); auto callee = module_.getOrInsertFunction( "ppc_fcmp", FunctionType::get(Type::getVoidTy(context_), {ptr, Type::getInt8Ty(context_), f64, f64, @@ -223,8 +204,6 @@ void FunctionEmitter::emitExactFloat(u64 descriptor) { builder_.CreateCall(callee, {ctx_, builder_.getInt8(crfd), stateValue(fprSlot(a)), stateValue(fprSlot(b)), builder_.getInt1(op == DOLIR_EXACT_FCMPO)}); - reloadState(DOLIR_STATE_CR); - reloadState(DOLIR_STATE_FPSCR); return; } @@ -263,14 +242,6 @@ void FunctionEmitter::emitExactFloat(u64 descriptor) { default: break; } - syncState(destination); - bool single = op <= DOLIR_EXACT_FDIVS || op == DOLIR_EXACT_FRSP; - if (single) - syncState(ps1Slot(d)); - syncState(fprSlot(op == DOLIR_EXACT_FRSP ? b : a)); - if (op != DOLIR_EXACT_FRSP) - syncState( - fprSlot(op == DOLIR_EXACT_FMULS || op == DOLIR_EXACT_FMUL ? c : b)); if (op == DOLIR_EXACT_FRSP) { auto callee = module_.getOrInsertFunction( name, FunctionType::get( @@ -292,9 +263,6 @@ void FunctionEmitter::emitExactFloat(u64 descriptor) { builder_.getInt8( op == DOLIR_EXACT_FMULS || op == DOLIR_EXACT_FMUL ? c : b)}); } - reloadState(destination); - if (single) - reloadState(ps1Slot(d)); } else if (op == DOLIR_EXACT_FCTIW || op == DOLIR_EXACT_FCTIWZ) { AllocaInst *output = temporary(Type::getInt64Ty(context_), "fctiw.result"); builder_.CreateStore( @@ -358,7 +326,6 @@ void FunctionEmitter::emitExactFloat(u64 descriptor) { state_[ps1]); } } - reloadState(DOLIR_STATE_FPSCR); } void FunctionEmitter::emitExactPaired(u64 descriptor) { @@ -375,22 +342,16 @@ void FunctionEmitter::emitExactPaired(u64 descriptor) { return static_cast(DOLIR_STATE_PS1_0 + reg); }; auto syncPair = [this, &fprSlot, &ps1Slot](u32 reg) { - syncState(fprSlot(reg)); - syncState(ps1Slot(reg)); }; auto reloadPair = [this, &fprSlot, &ps1Slot](u32 reg) { - reloadState(fprSlot(reg)); - reloadState(ps1Slot(reg)); }; Type *ptr = PointerType::getUnqual(context_); Type *i8 = Type::getInt8Ty(context_); Type *f64 = Type::getDoubleTy(context_); - syncState(DOLIR_STATE_FPSCR); if (op >= DOLIR_EXACT_PS_CMPU0) { bool lane1 = op == DOLIR_EXACT_PS_CMPU1 || op == DOLIR_EXACT_PS_CMPO1; bool ordered = op == DOLIR_EXACT_PS_CMPO0 || op == DOLIR_EXACT_PS_CMPO1; - syncState(DOLIR_STATE_CR); syncPair(a); syncPair(b); auto callee = module_.getOrInsertFunction( @@ -401,8 +362,6 @@ void FunctionEmitter::emitExactPaired(u64 descriptor) { stateValue(lane1 ? ps1Slot(a) : fprSlot(a)), stateValue(lane1 ? ps1Slot(b) : fprSlot(b)), builder_.getInt1(ordered)}); - reloadState(DOLIR_STATE_CR); - reloadState(DOLIR_STATE_FPSCR); return; } @@ -472,7 +431,6 @@ void FunctionEmitter::emitExactPaired(u64 descriptor) { builder_.getInt8(rhs)}); } reloadPair(d); - reloadState(DOLIR_STATE_FPSCR); } Value *FunctionEmitter::emitPSQ(const DolIRInstruction &inst) { @@ -501,10 +459,6 @@ Value *FunctionEmitter::emitPSQ(const DolIRInstruction &inst) { builder_.SetInsertPoint(failed); builder_.CreateRetVoid(); builder_.SetInsertPoint(resume); - for (u32 slot = 0; slot < DOLIR_STATE_COUNT; slot++) { - if (used_[slot]) - reloadState(static_cast(slot)); - } builder_.CreateStore(builder_.getInt64(0), cycles_); return ConstantInt::getTrue(context_); } @@ -530,10 +484,6 @@ void FunctionEmitter::emitStoreConditional(const DolIRInstruction &inst) { builder_.SetInsertPoint(failed); builder_.CreateRetVoid(); builder_.SetInsertPoint(resume); - for (u32 slot = 0; slot < DOLIR_STATE_COUNT; slot++) { - if (used_[slot]) - reloadState(static_cast(slot)); - } builder_.CreateStore(builder_.getInt64(0), cycles_); } diff --git a/src/common/options.c b/src/common/options.c new file mode 100644 index 0000000..4583dad --- /dev/null +++ b/src/common/options.c @@ -0,0 +1,24 @@ +#include "common/options.h" + +#include + +/* Fast is the default. It was measured at +6.7% fps on both Luigi's Mansion and + Mario Kart independently (docs/AOT-PERFORMANCE-RESULTS.md 5q), and its two + assumptions -- ram_size == GC_MAIN_RAM_SIZE, and no installed write journal + -- are verified once at dispatch entry, so a runtime that breaks either gets + the interpreter and a message rather than corrupted guest memory. + + The one runtime that does install a journal is ModernGekko's lockstep + verifier, and only when STATICRECOMP_LOCKSTEP is set. That is the harness + which compares the module against Dolphin's interpreter, so a module built + for speed makes it inert: the guard refuses native execution and says so. + Build with --memory-mode safe to run lockstep. */ +int memory_mode_is_fast(void) { + const char* value = getenv("DOLRECOMP_MEMORY_MODE"); + return !(value && value[0] == 's'); +} + +int replacements_enabled(void) { + const char* value = getenv("DOLRECOMP_ENABLE_REPLACEMENTS"); + return value && value[0] && value[0] != '0'; +} diff --git a/src/common/options.h b/src/common/options.h new file mode 100644 index 0000000..6c80654 --- /dev/null +++ b/src/common/options.h @@ -0,0 +1,25 @@ +#ifndef DOLRECOMP_COMMON_OPTIONS_H +#define DOLRECOMP_COMMON_OPTIONS_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Guest memory lowering mode, from DOLRECOMP_MEMORY_MODE (set by + --memory-mode). Lives here rather than in either backend because the C + dispatch emitter and the LLVM memory lowering both consult it, and a + disagreement between them would emit a fast-path body behind a guard that + does not check its assumptions -- or the reverse. One definition, one + answer. */ +int memory_mode_is_fast(void); + +/* Runtime function replacement, from DOLRECOMP_ENABLE_REPLACEMENTS. When on, + external transfers must leave through the dispatcher so a replacement gets + its chance; a direct call would bypass it silently. */ +int replacements_enabled(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/common/perf.c b/src/common/perf.c new file mode 100644 index 0000000..1726644 --- /dev/null +++ b/src/common/perf.c @@ -0,0 +1,423 @@ +#include "common/perf.h" + +#include +#include + +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#else +#include +#endif + +static DolPerfReport g_report; + +DolPerfReport* dolperf_report(void) { + return &g_report; +} + +void dolperf_reset(DolPerfReport* report) { + if (!report) + return; + + DolPerfRegion* regions = report->regions; + u32 capacity = report->region_capacity; + memset(report, 0, sizeof(*report)); + report->regions = regions; + report->region_capacity = capacity; +} + +void dolperf_free(DolPerfReport* report) { + if (!report) + return; + + free(report->regions); + report->regions = NULL; + report->region_count = 0; + report->region_capacity = 0; +} + +u64 dolperf_now_ns(void) { +#if defined(_WIN32) + static LARGE_INTEGER frequency; + if (frequency.QuadPart == 0) + QueryPerformanceFrequency(&frequency); + + LARGE_INTEGER now; + QueryPerformanceCounter(&now); + + /* Split the division so a long-running process cannot overflow the + multiply before the divide. */ + u64 ticks = (u64)now.QuadPart; + u64 freq = (u64)frequency.QuadPart; + if (freq == 0) + return 0; + return (ticks / freq) * 1000000000ull + + ((ticks % freq) * 1000000000ull) / freq; +#else + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) + return 0; + return (u64)ts.tv_sec * 1000000000ull + (u64)ts.tv_nsec; +#endif +} + +void dolperf_add_region(DolPerfReport* report, const DolPerfRegion* region) { + if (!report || !region) + return; + + if (report->region_count == report->region_capacity) { + u32 capacity = report->region_capacity ? report->region_capacity * 2u : 64u; + DolPerfRegion* grown = + (DolPerfRegion*)realloc(report->regions, capacity * sizeof(*grown)); + if (!grown) + return; /* Instrumentation must never fail a build. */ + report->regions = grown; + report->region_capacity = capacity; + } + + report->regions[report->region_count++] = *region; + + report->counters.regions_planned++; + report->counters.region_guest_instructions += region->guest_instructions; + report->counters.region_ir_instructions += region->ir_instructions; + report->counters.region_code_bytes += region->code_bytes; + report->counters.llvm_optimize_ns += region->optimize_ns; + report->counters.llvm_codegen_ns += region->codegen_ns; + if (region->cache_hit) + report->counters.artifact_cache_hits++; + else + report->counters.artifact_cache_misses++; +} + +void dolperf_merge_runtime(DolPerfReport* report, const DolPerfCounters* from) { + if (!report || !from) + return; + +#define DOLRECOMP_PERF_MERGE(field, json, label, group) \ + report->counters.field += from->field; + DOLRECOMP_PERF_RUNTIME_COUNTERS(DOLRECOMP_PERF_MERGE) +#undef DOLRECOMP_PERF_MERGE +} + +static void write_json_string(FILE* out, const char* text) { + fputc('"', out); + for (const char* p = text ? text : ""; *p; p++) { + unsigned char ch = (unsigned char)*p; + switch (ch) { + case '"': fputs("\\\"", out); break; + case '\\': fputs("\\\\", out); break; + case '\n': fputs("\\n", out); break; + case '\r': fputs("\\r", out); break; + case '\t': fputs("\\t", out); break; + default: + if (ch < 0x20) + fprintf(out, "\\u%04x", ch); + else + fputc((int)ch, out); + break; + } + } + fputc('"', out); +} + +static void write_json_field(FILE* out, const char* name, const char* value) { + fputs(" ", out); + write_json_string(out, name); + fputs(": ", out); + write_json_string(out, value); + fputs(",\n", out); +} + +bool dolperf_write_json(const DolPerfReport* report, const char* path, + FILE* diagnostics) { + if (!report || !path) + return false; + + FILE* out = fopen(path, "wb"); + if (!out) { + if (diagnostics) + fprintf(diagnostics, "error: cannot write perf report '%s'\n", path); + return false; + } + + fputs("{\n", out); + fputs(" \"schema\": \"dolrecomp.perf/1\",\n", out); + + fputs(" \"build\": {\n", out); + write_json_field(out, "backend", report->backend); + write_json_field(out, "region_mode", report->region_mode); + write_json_field(out, "target_triple", report->target_triple); + write_json_field(out, "target_cpu", report->target_cpu); + write_json_field(out, "target_features", report->target_features); + write_json_field(out, "lto", report->lto_mode); + write_json_field(out, "pgo", report->pgo_mode); + write_json_field(out, "pgo_profile_hash", report->pgo_profile_hash); + write_json_field(out, "mod_policy", report->mod_policy); + write_json_field(out, "memory_mode", report->memory_mode); + write_json_field(out, "llvm_version", report->llvm_version); + fprintf(out, " \"wall_ns\": %llu\n", (unsigned long long)report->wall_ns); + fputs(" },\n", out); + + fputs(" \"counters\": {\n", out); + { + int first = 1; +#define DOLRECOMP_PERF_JSON(field, json, label, group) \ + if (!first) \ + fputs(",\n", out); \ + first = 0; \ + fprintf(out, " \"%s\": %llu", json, \ + (unsigned long long)report->counters.field); + DOLRECOMP_PERF_COUNTERS(DOLRECOMP_PERF_JSON) +#undef DOLRECOMP_PERF_JSON + fputs("\n", out); + } + fputs(" },\n", out); + + fputs(" \"regions\": [\n", out); + for (u32 i = 0; i < report->region_count; i++) { + const DolPerfRegion* region = &report->regions[i]; + fprintf(out, + " {\"id\": %u, \"start\": \"0x%08X\", \"end\": \"0x%08X\", " + "\"guest_instructions\": %u, \"ir_instructions\": %u, " + "\"blocks\": %u, \"loops\": %u, \"code_bytes\": %u, " + "\"optimize_ns\": %llu, \"codegen_ns\": %llu, " + "\"cache_hit\": %s}%s\n", + region->region_id, region->guest_start, region->guest_end, + region->guest_instructions, region->ir_instructions, + region->blocks, region->loops, region->code_bytes, + (unsigned long long)region->optimize_ns, + (unsigned long long)region->codegen_ns, + region->cache_hit ? "true" : "false", + (i + 1 < report->region_count) ? "," : ""); + } + fputs(" ]\n", out); + fputs("}\n", out); + + if (fclose(out) != 0) { + if (diagnostics) + fprintf(diagnostics, "error: failed to close perf report '%s'\n", path); + return false; + } + + return true; +} + +void dolperf_print_summary(const DolPerfReport* report, FILE* out) { + if (!report || !out) + return; + + static const char* const groups[] = { + DOLRECOMP_PERF_GROUP_EXEC, DOLRECOMP_PERF_GROUP_DISPATCH, + DOLRECOMP_PERF_GROUP_STATE, DOLRECOMP_PERF_GROUP_INDIRECT, + DOLRECOMP_PERF_GROUP_MEMORY, DOLRECOMP_PERF_GROUP_COMPILE, + }; + + fputs("\nPerformance counters\n", out); + + for (size_t g = 0; g < sizeof(groups) / sizeof(groups[0]); g++) { + const char* group = groups[g]; + + /* A group whose counters are all zero is noise: the backend that owns + them was not exercised in this run. */ + int any = 0; +#define DOLRECOMP_PERF_ANY(field, json, label, grp) \ + if (strcmp(grp, group) == 0 && report->counters.field != 0) \ + any = 1; + DOLRECOMP_PERF_COUNTERS(DOLRECOMP_PERF_ANY) +#undef DOLRECOMP_PERF_ANY + if (!any) + continue; + + fprintf(out, " %s\n", group); +#define DOLRECOMP_PERF_ROW(field, json, label, grp) \ + if (strcmp(grp, group) == 0 && report->counters.field != 0) \ + fprintf(out, " %-42s %20llu\n", label, \ + (unsigned long long)report->counters.field); + DOLRECOMP_PERF_COUNTERS(DOLRECOMP_PERF_ROW) +#undef DOLRECOMP_PERF_ROW + } + + if (report->counters.regions_planned != 0) { + fprintf(out, " Derived\n"); + fprintf(out, " %-42s %20.1f\n", "Guest instructions per region", + (double)report->counters.region_guest_instructions / + (double)report->counters.regions_planned); + u64 compile_ns = + report->counters.llvm_optimize_ns + report->counters.llvm_codegen_ns; + fprintf(out, " %-42s %20.3f\n", "LLVM time (ms)", + (double)compile_ns / 1e6); + } + + u64 mem_fast = report->counters.mem1_fast_reads + + report->counters.mem1_fast_writes + + report->counters.mem2_fast_reads + + report->counters.mem2_fast_writes + + report->counters.const_ram_accesses; + u64 mem_slow = report->counters.slow_reads + report->counters.slow_writes; + if (mem_fast + mem_slow != 0) { + fprintf(out, " %-42s %19.1f%%\n", "Guest memory ops on a fast path", + 100.0 * (double)mem_fast / (double)(mem_fast + mem_slow)); + } + + fputc('\n', out); +} + +/* --- runtime counter dump parsing ---------------------------------------- */ + +static bool json_lookup_u64(const char* text, const char* name, u64* out) { + char needle[128]; + int written = snprintf(needle, sizeof(needle), "\"%s\"", name); + if (written <= 0 || (size_t)written >= sizeof(needle)) + return false; + + const char* cursor = text; + while ((cursor = strstr(cursor, needle)) != NULL) { + const char* p = cursor + written; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') + p++; + if (*p != ':') { + cursor += written; + continue; + } + p++; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') + p++; + if (*p < '0' || *p > '9') { + cursor += written; + continue; + } + *out = strtoull(p, NULL, 10); + return true; + } + + return false; +} + +bool dolperf_read_runtime_json(const char* path, DolPerfCounters* out, + FILE* diagnostics) { + if (!path || !out) + return false; + + FILE* in = fopen(path, "rb"); + if (!in) { + if (diagnostics) + fprintf(diagnostics, "error: cannot read counter dump '%s'\n", path); + return false; + } + + if (fseek(in, 0, SEEK_END) != 0) { + fclose(in); + return false; + } + long size = ftell(in); + if (size < 0) { + fclose(in); + return false; + } + rewind(in); + + char* text = (char*)malloc((size_t)size + 1u); + if (!text) { + fclose(in); + return false; + } + size_t got = fread(text, 1, (size_t)size, in); + text[got] = '\0'; + fclose(in); + + memset(out, 0, sizeof(*out)); +#define DOLRECOMP_PERF_READ(field, json, label, group) \ + { \ + u64 value = 0; \ + if (json_lookup_u64(text, json, &value)) \ + out->field = value; \ + } + DOLRECOMP_PERF_RUNTIME_COUNTERS(DOLRECOMP_PERF_READ) +#undef DOLRECOMP_PERF_READ + + free(text); + return true; +} + +/* --- generated-code instrumentation header ------------------------------- */ + +void dolperf_emit_runtime_header(FILE* out) { + if (!out) + return; + + fputs( + "/* Generated by DolRecomp. Do not edit.\n" + " *\n" + " * Runtime instrumentation for generated guest code.\n" + " *\n" + " * Every macro here compiles to nothing unless DOLRECOMP_PERF is defined,\n" + " * so a shipping module carries no counter stores on its hot paths. Build\n" + " * the module and the hosting runtime with -DDOLRECOMP_PERF=1 to collect.\n" + " *\n" + " * Threading: counters are plain u64 by default, which assumes the single\n" + " * guest CPU thread DolRecomp generates. Define DOLRECOMP_PERF_ATOMIC to\n" + " * make them _Atomic if a host drives generated code from several threads.\n" + " */\n" + "#ifndef DOLRECOMP_GENERATED_PERF_H\n" + "#define DOLRECOMP_GENERATED_PERF_H\n" + "\n" + "#include \n" + "#include \n" + "\n" + "#ifdef __cplusplus\n" + "extern \"C\" {\n" + "#endif\n" + "\n" + "#ifdef DOLRECOMP_PERF\n" + "\n" + "#if defined(DOLRECOMP_PERF_ATOMIC) && !defined(__cplusplus)\n" + "#include \n" + "#define DOLRECOMP_PERF_SLOT _Atomic uint64_t\n" + "#define DOLRECOMP_PERF_ADD(slot, amount) \\\n" + " atomic_fetch_add_explicit(&(slot), (uint64_t)(amount), \\\n" + " memory_order_relaxed)\n" + "#else\n" + "#define DOLRECOMP_PERF_SLOT uint64_t\n" + "#define DOLRECOMP_PERF_ADD(slot, amount) ((slot) += (uint64_t)(amount))\n" + "#endif\n" + "\n" + "typedef struct {\n", + out); + +#define DOLRECOMP_PERF_GEN_FIELD(field, json, label, group) \ + fprintf(out, " DOLRECOMP_PERF_SLOT %s;\n", #field); + DOLRECOMP_PERF_RUNTIME_COUNTERS(DOLRECOMP_PERF_GEN_FIELD) +#undef DOLRECOMP_PERF_GEN_FIELD + + fputs( + "} DolRecompPerfCounters;\n" + "\n" + "extern DolRecompPerfCounters dolrecomp_perf_counters;\n" + "\n" + "#define DOLRECOMP_PERF_INC(name) \\\n" + " DOLRECOMP_PERF_ADD(dolrecomp_perf_counters.name, 1u)\n" + "#define DOLRECOMP_PERF_ADDN(name, amount) \\\n" + " DOLRECOMP_PERF_ADD(dolrecomp_perf_counters.name, (amount))\n" + "\n" + "void dolrecomp_perf_reset(void);\n" + "/* Writes the counter block as JSON. dolperf_read_runtime_json() on the\n" + " DolRecomp side parses exactly this shape. */\n" + "int dolrecomp_perf_write_json(const char* path);\n" + "\n" + "#else /* !DOLRECOMP_PERF */\n" + "\n" + "#define DOLRECOMP_PERF_INC(name) ((void)0)\n" + "#define DOLRECOMP_PERF_ADDN(name, amount) ((void)0)\n" + "#define dolrecomp_perf_reset() ((void)0)\n" + "#define dolrecomp_perf_write_json(path) (0)\n" + "\n" + "#endif /* DOLRECOMP_PERF */\n" + "\n" + "#ifdef __cplusplus\n" + "}\n" + "#endif\n" + "\n" + "#endif /* DOLRECOMP_GENERATED_PERF_H */\n", + out); +} diff --git a/src/common/perf.h b/src/common/perf.h new file mode 100644 index 0000000..78594c9 --- /dev/null +++ b/src/common/perf.h @@ -0,0 +1,226 @@ +#ifndef DOLRECOMP_PERF_H +#define DOLRECOMP_PERF_H + +/* Phase 0 instrumentation. + * + * Two populations of numbers are reported through one mechanism: + * + * COMPILE counters are raised by dolrecomp itself while it plans regions and + * drives LLVM. They are always collected -- they cost a handful of adds per + * region against a backend that is already running an optimizer -- but are + * only *written out* when --perf-report is given. + * + * RUNTIME counters are raised by generated code and by the hosting runtime + * (ModernGekko). Those cannot be unconditionally live: a counter on the guest + * memory fast path would be a store per guest load. They are emitted into the + * generated output behind DOLRECOMP_PERF and compile to nothing unless the + * module is deliberately built with it, which is what "nearly zero-overhead + * when disabled" means here. + * + * The X-macro below is the single source of truth. Adding a counter to it + * extends the struct, the JSON object, the console table and the reset path at + * once, so those three cannot drift apart. + * + * Determinism: every counter is a plain u64 incremented from the CPU thread + * that executes guest code. DolRecomp generates a single guest CPU thread, so + * no atomics are needed and repeated runs of the same workload produce the same + * counts. A host that drives generated code from several threads at once must + * define DOLRECOMP_PERF_ATOMIC (see the generated header) or accept lost + * updates -- it is a measurement build either way, never a shipping one. + */ + +#include "common/types.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* DOLRECOMP_PERF_COUNTERS(C) + * C(field, "json.name", "Console label", GROUP) + * + * GROUP drives the console summary's section headings only. + */ +#define DOLRECOMP_PERF_GROUP_DISPATCH "Dispatch and linking" +#define DOLRECOMP_PERF_GROUP_STATE "Guest state traffic" +#define DOLRECOMP_PERF_GROUP_INDIRECT "Indirect control flow" +#define DOLRECOMP_PERF_GROUP_MEMORY "Guest memory" +#define DOLRECOMP_PERF_GROUP_EXEC "Execution" +#define DOLRECOMP_PERF_GROUP_COMPILE "Compilation" + +#define DOLRECOMP_PERF_RUNTIME_COUNTERS(C) \ + C(dispatcher_entries, "dispatcher_entries", \ + "Dispatcher entries", DOLRECOMP_PERF_GROUP_DISPATCH) \ + C(region_transfers_direct, "region_transfers_direct", \ + "Direct region-to-region transfers", DOLRECOMP_PERF_GROUP_DISPATCH) \ + C(guest_calls_direct, "guest_calls_direct", \ + "Direct native guest calls", DOLRECOMP_PERF_GROUP_DISPATCH) \ + C(tail_transfers, "tail_transfers", \ + "Native tail transfers", DOLRECOMP_PERF_GROUP_DISPATCH) \ + C(calls_via_thunk, "calls_via_thunk", \ + "Calls still using public thunks", DOLRECOMP_PERF_GROUP_DISPATCH) \ + C(calls_via_dispatch, "calls_via_dispatch", \ + "Calls still returning through dispatch", DOLRECOMP_PERF_GROUP_DISPATCH) \ + \ + C(state_materializations_full, "state_materializations_full", \ + "Full CPUState materializations", DOLRECOMP_PERF_GROUP_STATE) \ + C(state_syncs_partial, "state_syncs_partial", \ + "Partial state synchronizations", DOLRECOMP_PERF_GROUP_STATE) \ + C(state_reloads_after_boundary, "state_reloads_after_boundary", \ + "State reloads after runtime boundaries", DOLRECOMP_PERF_GROUP_STATE) \ + \ + C(indirect_branches, "indirect_branches", \ + "Indirect branch executions", DOLRECOMP_PERF_GROUP_INDIRECT) \ + C(indirect_cache_hits, "indirect_cache_hits", \ + "Indirect target-cache hits", DOLRECOMP_PERF_GROUP_INDIRECT) \ + C(indirect_cache_misses, "indirect_cache_misses", \ + "Indirect target-cache misses", DOLRECOMP_PERF_GROUP_INDIRECT) \ + C(blr_prediction_hits, "blr_prediction_hits", \ + "BLR prediction hits", DOLRECOMP_PERF_GROUP_INDIRECT) \ + C(blr_prediction_misses, "blr_prediction_misses", \ + "BLR prediction misses", DOLRECOMP_PERF_GROUP_INDIRECT) \ + \ + C(mem1_fast_reads, "mem1_fast_reads", \ + "MEM1 fast-path reads", DOLRECOMP_PERF_GROUP_MEMORY) \ + C(mem1_fast_writes, "mem1_fast_writes", \ + "MEM1 fast-path writes", DOLRECOMP_PERF_GROUP_MEMORY) \ + C(mem2_fast_reads, "mem2_fast_reads", \ + "MEM2 fast-path reads", DOLRECOMP_PERF_GROUP_MEMORY) \ + C(mem2_fast_writes, "mem2_fast_writes", \ + "MEM2 fast-path writes", DOLRECOMP_PERF_GROUP_MEMORY) \ + C(const_ram_accesses, "const_ram_accesses", \ + "Constant-address RAM accesses", DOLRECOMP_PERF_GROUP_MEMORY) \ + C(const_mmio_accesses, "const_mmio_accesses", \ + "Constant-address MMIO accesses", DOLRECOMP_PERF_GROUP_MEMORY) \ + C(slow_reads, "slow_reads", \ + "Generic slow memory reads", DOLRECOMP_PERF_GROUP_MEMORY) \ + C(slow_writes, "slow_writes", \ + "Generic slow memory writes", DOLRECOMP_PERF_GROUP_MEMORY) \ + C(fastmem_faults, "fastmem_faults", \ + "Fault-assisted fastmem faults", DOLRECOMP_PERF_GROUP_MEMORY) \ + \ + C(exception_exits, "exception_exits", \ + "Exception exits", DOLRECOMP_PERF_GROUP_EXEC) \ + C(runtime_helper_calls, "runtime_helper_calls", \ + "Runtime helper calls", DOLRECOMP_PERF_GROUP_EXEC) \ + C(fallback_instructions, "fallback_instructions", \ + "Fallback-instruction executions", DOLRECOMP_PERF_GROUP_EXEC) \ + C(region_executions, "region_executions", \ + "Region executions", DOLRECOMP_PERF_GROUP_EXEC) \ + C(hot_region_executions, "hot_region_executions", \ + "Hot-region executions", DOLRECOMP_PERF_GROUP_EXEC) \ + C(cycles_charged, "cycles_charged", \ + "Cycles charged", DOLRECOMP_PERF_GROUP_EXEC) + +#define DOLRECOMP_PERF_COMPILE_COUNTERS(C) \ + C(regions_planned, "regions_planned", \ + "Regions planned", DOLRECOMP_PERF_GROUP_COMPILE) \ + C(region_guest_instructions, "region_guest_instructions", \ + "Guest instructions in regions", DOLRECOMP_PERF_GROUP_COMPILE) \ + C(region_ir_instructions, "region_ir_instructions", \ + "DolIR instructions emitted", DOLRECOMP_PERF_GROUP_COMPILE) \ + C(region_code_bytes, "region_code_bytes", \ + "Generated code bytes", DOLRECOMP_PERF_GROUP_COMPILE) \ + C(llvm_optimize_ns, "llvm_optimize_ns", \ + "LLVM optimization time (ns)", DOLRECOMP_PERF_GROUP_COMPILE) \ + C(llvm_codegen_ns, "llvm_codegen_ns", \ + "LLVM code-generation time (ns)", DOLRECOMP_PERF_GROUP_COMPILE) \ + C(artifact_cache_hits, "artifact_cache_hits", \ + "Object/bitcode cache hits", DOLRECOMP_PERF_GROUP_COMPILE) \ + C(artifact_cache_misses, "artifact_cache_misses", \ + "Object/bitcode cache misses", DOLRECOMP_PERF_GROUP_COMPILE) + +#define DOLRECOMP_PERF_COUNTERS(C) \ + DOLRECOMP_PERF_RUNTIME_COUNTERS(C) \ + DOLRECOMP_PERF_COMPILE_COUNTERS(C) + +typedef struct { +#define DOLRECOMP_PERF_FIELD(field, json, label, group) u64 field; + DOLRECOMP_PERF_COUNTERS(DOLRECOMP_PERF_FIELD) +#undef DOLRECOMP_PERF_FIELD +} DolPerfCounters; + +/* Per-region compilation detail, kept alongside the aggregate counters so the + * report can show where compile time and code size actually went. Phase 1 fills + * these in from the planner; the fixed-chunk path records one entry per chunk so + * the two modes stay directly comparable. */ +typedef struct { + u32 region_id; + u32 guest_start; + u32 guest_end; + u32 guest_instructions; + u32 ir_instructions; + u32 blocks; + u32 loops; + u32 code_bytes; + u64 optimize_ns; + u64 codegen_ns; + int cache_hit; +} DolPerfRegion; + +typedef struct { + DolPerfCounters counters; + DolPerfRegion* regions; + u32 region_count; + u32 region_capacity; + + /* Build identity, reproduced verbatim into the report so a number can + * always be traced back to the configuration that produced it. */ + char backend[32]; + char region_mode[32]; + char target_triple[128]; + char target_cpu[64]; + char target_features[256]; + char lto_mode[16]; + char pgo_mode[16]; + char pgo_profile_hash[80]; + char mod_policy[16]; + char memory_mode[32]; + char llvm_version[32]; + u64 wall_ns; + int enabled; +} DolPerfReport; + +/* The process-wide report used by the compiler. */ +DolPerfReport* dolperf_report(void); + +void dolperf_reset(DolPerfReport* report); +void dolperf_free(DolPerfReport* report); + +/* Monotonic nanoseconds, for the *_ns counters. */ +u64 dolperf_now_ns(void); + +/* Records one region's compilation detail and folds it into the aggregate + * counters. Safe to call with report == NULL. */ +void dolperf_add_region(DolPerfReport* report, const DolPerfRegion* region); + +/* Writes the machine-readable report. Returns false and leaves a message on + * `diagnostics` if the file cannot be written. */ +bool dolperf_write_json(const DolPerfReport* report, const char* path, + FILE* diagnostics); + +/* Prints the human-readable summary table. Counters that are zero across a + * whole group are omitted so an un-instrumented run stays readable. */ +void dolperf_print_summary(const DolPerfReport* report, FILE* out); + +/* Merges a counter block collected by a generated module at runtime (read back + * through the generated dolrecomp_perf.h ABI) into a report, so one JSON file + * can carry both halves. */ +void dolperf_merge_runtime(DolPerfReport* report, const DolPerfCounters* from); + +/* Parses a runtime counter dump written by the generated + * dolrecomp_perf_write_json() into `out`. Used by the benchmark harness to pull + * a game run's counters back into a comparison report. */ +bool dolperf_read_runtime_json(const char* path, DolPerfCounters* out, + FILE* diagnostics); + +/* The generated-code instrumentation header. The backends write this next to + * the generated module so both the emitted C and the hosting runtime agree on + * the counter block layout and the increment macros. */ +void dolperf_emit_runtime_header(FILE* out); + +#ifdef __cplusplus +} +#endif + +#endif /* DOLRECOMP_PERF_H */ diff --git a/tests/cmake/codegen_compile.cmake b/tests/cmake/codegen_compile.cmake index b1f2258..8dcb312 100644 --- a/tests/cmake/codegen_compile.cmake +++ b/tests/cmake/codegen_compile.cmake @@ -42,7 +42,11 @@ endif() if(NOT generated_source MATCHES "static void loop_80004040\\(CPUState\\* ctx\\)") message(FATAL_ERROR "ordinary RAM loop was not outlined") endif() -if(NOT generated_source MATCHES "if \\(!ppc_fp_available_inline\\(ctx, 0x8000317Cu\\)\\) return;") +# The gate is spelled out rather than calling ppc_fp_available_inline, because +# that helper exists only in DolRecomp's cpu.h and generated modules compile +# against the runtime's. Both halves are asserted: the MSR[FP] fast path and the +# fallback call, so dropping either one still fails this test. +if(NOT generated_source MATCHES "if \\(!\\(\\(ctx->msr & 0x00002000u\\) || ppc_fp_available\\(ctx, 0x8000317Cu\\)\\)\\) return;") message(FATAL_ERROR "generated floating-point code has no MSR FP gate") endif() if(NOT generated_source MATCHES "ppc_fallback_instruction\\(ctx, 0x7C13A0ACu") diff --git a/tests/differential/gen_differential.cpp b/tests/differential/gen_differential.cpp new file mode 100644 index 0000000..8e09dad --- /dev/null +++ b/tests/differential/gen_differential.cpp @@ -0,0 +1,298 @@ +// Generates the same random guest code through both backends for differential +// testing. +// +// The trick that makes one comparing binary possible: both backends name their +// output func_, so emitting the C copy at one base and the LLVM +// copy at another gives distinct symbols with identical semantics. Nothing in +// the generated sequences depends on the base -- no absolute branches, and +// effective addresses come from registers rather than PC -- so the two copies +// must compute the same thing from the same starting state or one of them is +// wrong. +// +// Sequences are straight-line and end in blr. That is not a limitation for the +// thing most in need of checking: a return is a materialisation barrier, so +// every sequence exercises the state save path that the liveness and +// reaching-writes narrowing changed. Branch-shaped control flow is covered by +// test_dolir and test_c_cfg. +// +// gen_differential [seed] [functions] [length] + +// backend/emitter.h has no extern "C" guard of its own, and this is the only +// C++ caller of the C emitter. +extern "C" { +#include "backend/emitter.h" +#include "backend/dispatch.h" +#include "common/options.h" +} +#include "backend/llvm/llvm_backend.h" +#include "ir/dolir_builder.h" +#include "frontend/decoder.h" + +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, \ + "gen_differential: check failed: %s:%d: %s\n", __FILE__, __LINE__, #x); \ + return 1; } } while (0) + +// Deterministic so a failing seed reproduces exactly. Logged by the driver. +static u64 g_state = 0; +static u32 next_random(void) { + g_state = g_state * 6364136223846793005ull + 1442695040888963407ull; + return (u32)(g_state >> 33); +} +static u32 pick(u32 count) { return next_random() % count; } + +// stfs is excluded from the default pool because the two backends genuinely +// disagree on it, and a gate that always fails gates nothing. This is scoping a +// new test, not weakening an existing one: the divergence is documented in +// AOT-PERFORMANCE-RESULTS.md and reproduces with --stfs. +// +// Storing a double that is not representable as single: the C backend produced +// 0x7E000000 where LLVM produced 0x7F800000 (+inf), and 0x04000004 where LLVM +// produced 0x00000000 (denormal flushed). Every one of 28 divergences across 64 +// sequences was an stfs result; nothing else differed in any register, and +// removing stfs alone took the suite to 64/64. +static bool g_include_stfs = false; + +// Registers the driver leaves alone so a sequence cannot destroy its own +// addressing base: r31 holds the scratch pointer, r0 is special in many forms. +static u8 gpr(void) { return (u8)(1u + pick(29u)); } // r1..r29 +static u8 fpr(void) { return (u8)pick(32u); } +static u8 crf(void) { return (u8)pick(8u); } + +// Field-assembled so every generated word is a legal encoding by construction +// rather than by hoping a random 32 bits decodes. +static u32 form_d(u32 op, u8 d, u8 a, u16 imm) { + return (op << 26) | ((u32)d << 21) | ((u32)a << 16) | imm; +} +static u32 form_x(u32 op, u8 d, u8 a, u8 b, u32 xo, u32 rc) { + return (op << 26) | ((u32)d << 21) | ((u32)a << 16) | ((u32)b << 11) | + (xo << 1) | rc; +} +static u32 form_a(u32 op, u8 d, u8 a, u8 b, u8 c, u32 xo, u32 rc) { + return (op << 26) | ((u32)d << 21) | ((u32)a << 16) | ((u32)b << 11) | + ((u32)c << 6) | (xo << 1) | rc; +} +static u32 form_m(u32 op, u8 s, u8 a, u8 sh, u8 mb, u8 me, u32 rc) { + return (op << 26) | ((u32)s << 21) | ((u32)a << 16) | ((u32)sh << 11) | + ((u32)mb << 6) | ((u32)me << 1) | rc; +} + +// Memory operations always address through r31 with a small aligned +// displacement, so every access lands inside the scratch page the driver set +// up. A random base register would fault or scribble on the CPUState. +static u16 scratch_offset(u32 align) { + u32 slot = pick(64u) * align; + return (u16)slot; +} + +static u32 random_instruction(void) { + switch (pick(24u)) { + case 0: return form_d(14, gpr(), gpr(), (u16)next_random()); // addi + case 1: return form_d(15, gpr(), gpr(), (u16)next_random()); // addis + case 2: return form_d(12, gpr(), gpr(), (u16)next_random()); // addic + case 3: return form_d(28, gpr(), gpr(), (u16)next_random()); // andi. + case 4: return form_d(24, gpr(), gpr(), (u16)next_random()); // ori + case 5: return form_d(26, gpr(), gpr(), (u16)next_random()); // xori + case 6: return form_x(31, gpr(), gpr(), gpr(), 266, pick(2u)); // add[.] + case 7: return form_x(31, gpr(), gpr(), gpr(), 40, pick(2u)); // subf[.] + case 8: return form_x(31, gpr(), gpr(), gpr(), 235, pick(2u)); // mullw[.] + case 9: return form_x(31, gpr(), gpr(), gpr(), 28, pick(2u)); // and[.] + case 10: return form_x(31, gpr(), gpr(), gpr(), 444, pick(2u)); // or[.] + case 11: return form_x(31, gpr(), gpr(), gpr(), 316, pick(2u)); // xor[.] + case 12: return form_x(31, gpr(), gpr(), gpr(), 24, pick(2u)); // slw[.] + case 13: return form_x(31, gpr(), gpr(), gpr(), 536, pick(2u)); // srw[.] + case 14: return form_x(31, gpr(), gpr(), gpr(), 792, pick(2u)); // sraw[.] + case 15: return form_m(21, gpr(), gpr(), (u8)pick(32u), (u8)pick(32u), + (u8)pick(32u), pick(2u)); // rlwinm[.] + case 16: return form_x(31, gpr(), gpr(), gpr(), 10, pick(2u)); // addc[.] + case 17: return form_x(31, gpr(), gpr(), gpr(), 138, pick(2u)); // adde[.] + case 18: return form_d(32, gpr(), 31, scratch_offset(4)); // lwz + case 19: return form_d(36, gpr(), 31, scratch_offset(4)); // stw + case 20: return form_d(34, gpr(), 31, scratch_offset(1)); // lbz + case 21: return form_d(48, fpr(), 31, scratch_offset(4)); // lfs + case 22: return g_include_stfs + ? form_d(52, fpr(), 31, scratch_offset(4)) // stfs + : form_d(36, gpr(), 31, scratch_offset(4)); // stw + default: return form_x(31, (u8)(crf() << 2), gpr(), gpr(), 0, 0); // cmpw + } +} + +// Floating point kept in its own pool so a sequence can be biased toward it: +// the paired-single and FP paths carry the semantics most at risk from a +// state-save change, and they are the ones a random integer sequence rarely +// reaches. +static u32 random_float_instruction(void) { + switch (pick(10u)) { + case 0: return form_a(63, fpr(), fpr(), fpr(), 0, 21, pick(2u)); // fadd[.] + case 1: return form_a(63, fpr(), fpr(), fpr(), 0, 20, pick(2u)); // fsub[.] + case 2: return form_a(63, fpr(), fpr(), 0, fpr(), 25, pick(2u)); // fmul[.] + case 3: return form_a(59, fpr(), fpr(), fpr(), 0, 21, pick(2u)); // fadds[.] + case 4: return form_a(59, fpr(), fpr(), 0, fpr(), 25, pick(2u)); // fmuls[.] + case 5: return form_a(63, fpr(), fpr(), fpr(), fpr(), 29, pick(2u)); // fmadd[.] + case 6: return form_a(59, fpr(), fpr(), fpr(), fpr(), 29, pick(2u)); // fmadds[.] + case 7: return form_x(63, fpr(), 0, fpr(), 72, pick(2u)); // fmr[.] + case 8: return form_x(63, fpr(), 0, fpr(), 264, pick(2u)); // fabs[.] + default: return form_x(63, fpr(), 0, fpr(), 40, pick(2u)); // fneg[.] + } +} + +int main(int argc, char** argv) { + CHECK(argc >= 4); + const char* c_path = argv[1]; + const char* object_path = argv[2]; + const char* manifest_path = argv[3]; + const u64 seed = argc > 4 ? std::strtoull(argv[4], nullptr, 0) : 20260812ull; + const u32 functions = argc > 5 ? (u32)std::strtoul(argv[5], nullptr, 0) : 64u; + const u32 length = argc > 6 ? (u32)std::strtoul(argv[6], nullptr, 0) : 24u; + for (int i = 4; i < argc; i++) { + if (std::strcmp(argv[i], "--stfs") == 0) + g_include_stfs = true; + } + + // Far apart so no range check can confuse one copy for the other. + const u32 kBaseC = 0x80100000u; + const u32 kBaseL = 0x80300000u; + const u32 kStride = 0x1000u; + + g_state = seed; + + // Upper half are leaves; lower half call into them. One level only: a + // guest bl overwrites LR, so a callee that called something would return to + // the wrong place -- a property of the generated program, not a backend + // difference. + // + // The call at i == 1 is forced so a minimal repro (2 functions) always + // contains one; the rest are random. + const u32 leaves = functions / 2u; + const u32 callers = functions - leaves; + std::vector> bodies; + for (u32 f = 0; f < functions; f++) { + std::vector words; + const bool floaty = (f % 4u) == 3u; + const bool may_call = f < callers && leaves > 0u; + for (u32 i = 0; i < length; i++) { + const bool forced = may_call && i == 1u && length > 5u; + const bool random_call = may_call && i > 1u && i + 4u < length && + pick(length / 2u) == 0; + if (forced || random_call) { + // A call has to save and restore LR around itself, exactly as + // compiler-generated code does. Without it the caller's own blr + // returns to the continuation the bl just wrote into LR, and the + // sequence loops forever -- a bug in the generated program, not + // a backend difference, and it is what made an earlier attempt + // at this hang. + // + // r30 carries it and a fixed scratch slot holds it across the + // call. Both are safe: gpr() only ever picks r1..r29, so a + // callee cannot clobber r30 or the r31 addressing base. + const u32 kLrSlot = 0x400u; // clear of the random slots (0..252) + const u32 callee = callers + pick(leaves); + // The bl sits 2 instructions after `here` because of the save. + const s32 here = (s32)(f * kStride + (i + 2u) * 4u); + const s32 there = (s32)(callee * kStride); + const s32 delta = there - here; + + words.push_back(0x7FC802A6u); // mflr r30 + words.push_back(form_d(36, 30, 31, (u16)kLrSlot)); // stw r30,slot(r31) + words.push_back(0x48000001u | ((u32)delta & 0x03FFFFFCu)); // bl + words.push_back(form_d(32, 30, 31, (u16)kLrSlot)); // lwz r30,slot(r31) + words.push_back(0x7FC803A6u); // mtlr r30 + i += 4u; // the loop's own increment accounts for the fifth + continue; + } + words.push_back(floaty && (pick(2u) == 0) ? random_float_instruction() + : random_instruction()); + } + words.push_back(0x4E800020u); + bodies.push_back(words); + } + + // --- C backend ------------------------------------------------------- + FILE* out = std::fopen(c_path, "w"); + CHECK(out != nullptr); + emit_header_for_cpu(out, DOLRECOMP_CPU_GEKKO); + + // Prototypes, then the dispatcher, then the bodies: a body calls + // dolrecomp_call() for anything outside its own range and the dispatcher + // defines it. + FunctionList funcs; + std::memset(&funcs, 0, sizeof(funcs)); + for (u32 f = 0; f < functions; f++) { + const u32 address = kBaseC + f * kStride; + emit_chunk_prototype(out, address); + CHECK(function_list_add(&funcs, address, + address + (u32)bodies[f].size() * 4u)); + } + /* Matches the LLVM arm's lowering, so the harness exercises whatever the + default mode emits. The CPUState here is a real cpu_init with no journal + installed, so the guard passes and both arms run natively. */ + emit_dispatch_helpers(out, &funcs, kBaseC, memory_mode_is_fast()); + function_list_free(&funcs); + + for (u32 f = 0; f < functions; f++) { + const u32 address = kBaseC + f * kStride; + std::vector decoded(bodies[f].size()); + for (std::size_t i = 0; i < bodies[f].size(); i++) + decoded[i] = ppc_decode(bodies[f][i], address + (u32)i * 4u); + CHECK(emit_function(out, decoded.data(), (u32)decoded.size(), address)); + } + emit_footer(out); + CHECK(std::fclose(out) == 0); + + // --- LLVM backend ---------------------------------------------------- + DolIRModule module; + dolir_module_init(&module); + std::vector ranges; + for (u32 f = 0; f < functions; f++) { + const u32 address = kBaseL + f * kStride; + std::vector decoded(bodies[f].size()); + for (std::size_t i = 0; i < bodies[f].size(); i++) + decoded[i] = ppc_decode(bodies[f][i], address + (u32)i * 4u); + CHECK(dolir_build_chunk(&module, decoded.data(), (u32)decoded.size(), + address)); + DolLLVMFunctionRange range; + range.start = address; + range.end = address + (u32)decoded.size() * 4u; + ranges.push_back(range); + } + CHECK(dolir_verify(&module, stderr)); + + DolLLVMOptions options; + std::memset(&options, 0, sizeof(options)); + options.optimization_level = 2; + options.verify = 1; + options.function_ranges = ranges.data(); + options.function_range_count = (u32)ranges.size(); + CHECK(dolllvm_emit_object(&module, object_path, &options, stderr)); + dolir_module_free(&module); + + // --- manifest -------------------------------------------------------- + FILE* manifest = std::fopen(manifest_path, "w"); + CHECK(manifest != nullptr); + std::fprintf(manifest, "// Generated by gen_differential. Do not edit.\n"); + std::fprintf(manifest, "#define DIFF_SEED %lluull\n", (unsigned long long)seed); + std::fprintf(manifest, "#define DIFF_COUNT %uu\n", functions); + std::fprintf(manifest, "#define DIFF_BASE_C 0x%08Xu\n", kBaseC); + std::fprintf(manifest, "#define DIFF_BASE_L 0x%08Xu\n", kBaseL); + std::fprintf(manifest, "#define DIFF_STRIDE 0x%08Xu\n", kStride); + for (u32 f = 0; f < functions; f++) { + std::fprintf(manifest, "void func_%08X(CPUState*);\n", kBaseC + f * kStride); + std::fprintf(manifest, "void func_%08X(CPUState*);\n", kBaseL + f * kStride); + } + std::fprintf(manifest, "static DiffPair diff_pairs[DIFF_COUNT] = {\n"); + for (u32 f = 0; f < functions; f++) { + std::fprintf(manifest, " {func_%08X, func_%08X, 0x%08Xu, 0x%08Xu},\n", + kBaseC + f * kStride, kBaseL + f * kStride, + kBaseC + f * kStride, kBaseL + f * kStride); + } + std::fprintf(manifest, "};\n"); + CHECK(std::fclose(manifest) == 0); + + std::printf("gen_differential: seed %llu, %u functions of %u instructions\n", + (unsigned long long)seed, functions, length + 1u); + return 0; +} diff --git a/tests/test_cfg.c b/tests/test_cfg.c new file mode 100644 index 0000000..83d0e08 --- /dev/null +++ b/tests/test_cfg.c @@ -0,0 +1,367 @@ +#include "analysis/cfg.h" + +#include +#include + +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "check failed: %s:%d: %s\n", \ + __FILE__, __LINE__, #x); return false; } } while (0) + +#define BASE 0x80001000u + +static void decode_all(PPCInst* out, const u32* raw, u32 count, u32 base) { + for (u32 i = 0; i < count; i++) + out[i] = ppc_decode(raw[i], base + i * 4u); +} + +/* addi r3,r0,0 / addi r3,r3,1 / cmpwi r3,10 / blt -8 / blr */ +static const u32 kLoop[] = { + 0x38600000u, + 0x38630001u, + 0x2C03000Au, + 0x4180FFF8u, + 0x4E800020u, +}; + +static bool test_loop_blocks_and_header(void) { + PPCInst insts[5]; + decode_all(insts, kLoop, 5, BASE); + + DolCfgProgram program; + dolcfg_init(&program); + CHECK(dolcfg_add_section(&program, insts, 5, BASE, "text")); + CHECK(dolcfg_build(&program, stderr)); + + /* Leaders: BASE (section start), BASE+4 (branch target), BASE+16 (after + the conditional branch). */ + CHECK(program.block_count == 3); + + u32 head = dolcfg_block_starting_at(&program, BASE); + u32 body = dolcfg_block_starting_at(&program, BASE + 4u); + u32 tail = dolcfg_block_starting_at(&program, BASE + 16u); + CHECK(head != DOLCFG_NO_BLOCK && body != DOLCFG_NO_BLOCK && + tail != DOLCFG_NO_BLOCK); + + CHECK(program.blocks[head].terminator == DOLCFG_TERM_FALLTHROUGH); + CHECK(program.blocks[head].successor_count == 1); + CHECK(program.blocks[head].successors[0] == body); + + /* The conditional branch keeps both edges: taken back to the body, and + fallthrough to the return. */ + CHECK(program.blocks[body].terminator == DOLCFG_TERM_COND_BRANCH); + CHECK(program.blocks[body].successor_count == 2); + CHECK(program.blocks[body].successors[0] == body); + CHECK(program.blocks[body].successors[1] == tail); + CHECK(program.blocks[body].flags & DOLCFG_BLOCK_LOOP_HEADER); + CHECK(program.blocks[body].loop_depth == 1); + + CHECK(program.blocks[tail].terminator == DOLCFG_TERM_RETURN); + CHECK(program.blocks[tail].successor_count == 0); + CHECK(program.blocks[tail].loop_depth == 0); + CHECK(program.loop_count == 1); + + dolcfg_free(&program); + return true; +} + +/* A conditional branch that BO marks as always-taken has no fallthrough. */ +static bool test_unconditional_bc_drops_fallthrough(void) { + /* bc 20,0,+8 -> BO=20 (1z1zz), always taken */ + const u32 raw[] = { + 0x42800008u | 0x02000000u, /* placeholder, replaced below */ + 0x60000000u, + 0x4E800020u, + }; + u32 fixed[3]; + memcpy(fixed, raw, sizeof(fixed)); + /* bc with BO=20, BI=0, BD=+8, AA=0, LK=0 */ + fixed[0] = 0x40000000u | (20u << 21) | (0u << 16) | (8u & 0xFFFCu); + + PPCInst insts[3]; + decode_all(insts, fixed, 3, BASE); + CHECK(insts[0].op == PPC_OP_BC); + CHECK(insts[0].bo == 20); + + DolCfgProgram program; + dolcfg_init(&program); + CHECK(dolcfg_add_section(&program, insts, 3, BASE, "text")); + CHECK(dolcfg_build(&program, stderr)); + + u32 head = dolcfg_block_starting_at(&program, BASE); + CHECK(head != DOLCFG_NO_BLOCK); + CHECK(program.blocks[head].terminator == DOLCFG_TERM_BRANCH); + CHECK(program.blocks[head].successor_count == 1); + CHECK(program.blocks[head].successor_addresses[0] == BASE + 8u); + + dolcfg_free(&program); + return true; +} + +/* bl records the callee as a call target and continues at the return point; + the callee becomes an inferred function entry. */ +static bool test_call_creates_function(void) { + const u32 raw[] = { + 0x48000011u, /* bl +0x10 -> BASE+0x10 */ + 0x4E800020u, /* blr */ + 0x60000000u, + 0x60000000u, + 0x38600001u, /* BASE+0x10: addi r3,r0,1 */ + 0x4E800020u, /* blr */ + }; + PPCInst insts[6]; + decode_all(insts, raw, 6, BASE); + CHECK(insts[0].op == PPC_OP_B && insts[0].lk); + CHECK(insts[0].branch_target == BASE + 0x10u); + + DolCfgProgram program; + dolcfg_init(&program); + CHECK(dolcfg_add_section(&program, insts, 6, BASE, "text")); + CHECK(dolcfg_build(&program, stderr)); + + u32 caller = dolcfg_block_starting_at(&program, BASE); + CHECK(caller != DOLCFG_NO_BLOCK); + CHECK(program.blocks[caller].terminator == DOLCFG_TERM_CALL); + CHECK(program.blocks[caller].call_target == BASE + 0x10u); + /* The successor is the return point, never the callee. */ + CHECK(program.blocks[caller].successor_count == 1); + CHECK(program.blocks[caller].successor_addresses[0] == BASE + 4u); + + u32 callee = dolcfg_block_starting_at(&program, BASE + 0x10u); + CHECK(callee != DOLCFG_NO_BLOCK); + CHECK(program.blocks[callee].flags & DOLCFG_BLOCK_FUNCTION_ENTRY); + + bool found = false; + for (u32 i = 0; i < program.function_count; i++) { + if (program.functions[i].entry_address == BASE + 0x10u) { + found = true; + CHECK(program.functions[i].flags & DOLCFG_FUNC_FROM_CALL); + } + } + CHECK(found); + + dolcfg_free(&program); + return true; +} + +/* A plain b into another function's entry is a tail call, not a branch. */ +static bool test_tail_call_reclassification(void) { + const u32 raw[] = { + 0x48000008u, /* BASE+0: b +8 -> BASE+8 */ + 0x60000000u, + 0x38600001u, /* BASE+8 */ + 0x4E800020u, + }; + PPCInst insts[4]; + decode_all(insts, raw, 4, BASE); + + DolCfgProgram program; + dolcfg_init(&program); + CHECK(dolcfg_add_section(&program, insts, 4, BASE, "text")); + /* Declared, not inferred: no bl points at it. */ + CHECK(dolcfg_add_known_function(&program, BASE + 8u, "target_fn", 0)); + CHECK(dolcfg_add_known_function(&program, BASE, "caller_fn", 0)); + CHECK(dolcfg_build(&program, stderr)); + + u32 head = dolcfg_block_starting_at(&program, BASE); + CHECK(head != DOLCFG_NO_BLOCK); + CHECK(program.blocks[head].terminator == DOLCFG_TERM_TAIL_CALL); + CHECK(program.blocks[head].call_target == BASE + 8u); + + bool named = false; + for (u32 i = 0; i < program.function_count; i++) { + if (program.functions[i].entry_address == BASE + 8u) { + named = strcmp(program.functions[i].name, "target_fn") == 0; + CHECK(program.functions[i].flags & DOLCFG_FUNC_FROM_SYMBOL); + } + } + CHECK(named); + + dolcfg_free(&program); + return true; +} + +/* bcctr is an unresolved indirect transfer. It must not acquire a guessed + successor -- Phase 4 attaches target sets, and until then a region ends. */ +static bool test_indirect_has_no_guessed_target(void) { + const u32 raw[] = { + 0x38600001u, + 0x4E800420u, /* bctr */ + 0x60000000u, + }; + PPCInst insts[3]; + decode_all(insts, raw, 3, BASE); + CHECK(insts[1].op == PPC_OP_BCCTR); + + DolCfgProgram program; + dolcfg_init(&program); + CHECK(dolcfg_add_section(&program, insts, 3, BASE, "text")); + CHECK(dolcfg_build(&program, stderr)); + + u32 head = dolcfg_block_starting_at(&program, BASE); + CHECK(head != DOLCFG_NO_BLOCK); + CHECK(program.blocks[head].terminator == DOLCFG_TERM_INDIRECT); + /* Unconditional, no link: nothing follows it in the model. */ + CHECK(program.blocks[head].successor_count == 0); + CHECK(program.indirect_site_count == 1); + + dolcfg_free(&program); + return true; +} + +/* Embedded data must never become a block. */ +static bool test_embedded_data_is_not_code(void) { + const u32 raw[] = { + 0x38600001u, + 0x4E800020u, /* blr */ + 0xDEADBEEFu, /* data */ + 0xCAFEBABEu, /* data */ + 0x38600002u, + 0x4E800020u, + }; + PPCInst insts[6]; + decode_all(insts, raw, 6, BASE); + insts[2].embedded_data = true; + insts[3].embedded_data = true; + + DolCfgProgram program; + dolcfg_init(&program); + CHECK(dolcfg_add_section(&program, insts, 6, BASE, "text")); + CHECK(dolcfg_build(&program, stderr)); + + CHECK(dolcfg_block_at(&program, BASE + 8u) == DOLCFG_NO_BLOCK); + CHECK(dolcfg_block_at(&program, BASE + 12u) == DOLCFG_NO_BLOCK); + /* Code resumes after the data run. */ + CHECK(dolcfg_block_starting_at(&program, BASE + 16u) != DOLCFG_NO_BLOCK); + + u32 covered = 0; + for (u32 i = 0; i < program.block_count; i++) + covered += program.blocks[i].instruction_count; + CHECK(covered == 4); + + dolcfg_free(&program); + return true; +} + +static bool test_smc_ranges_flag_blocks(void) { + PPCInst insts[5]; + decode_all(insts, kLoop, 5, BASE); + + DolCfgProgram program; + dolcfg_init(&program); + CHECK(dolcfg_add_section(&program, insts, 5, BASE, "text")); + CHECK(dolcfg_add_smc_range(&program, BASE + 16u, BASE + 16u)); + CHECK(dolcfg_build(&program, stderr)); + + u32 tail = dolcfg_block_starting_at(&program, BASE + 16u); + u32 head = dolcfg_block_starting_at(&program, BASE); + CHECK(tail != DOLCFG_NO_BLOCK && head != DOLCFG_NO_BLOCK); + CHECK(program.blocks[tail].flags & DOLCFG_BLOCK_SMC_SUSPECT); + CHECK(!(program.blocks[head].flags & DOLCFG_BLOCK_SMC_SUSPECT)); + + dolcfg_free(&program); + return true; +} + +/* Region planning has to be reproducible from the same inputs, which starts + with the CFG numbering being reproducible. */ +static bool test_build_is_deterministic(void) { + const u32 raw[] = { + 0x48000011u, 0x4E800020u, 0x60000000u, 0x60000000u, + 0x38600001u, 0x2C03000Au, 0x4180FFFCu, 0x4E800020u, + }; + PPCInst insts[8]; + decode_all(insts, raw, 8, BASE); + + DolCfgProgram a, b; + dolcfg_init(&a); + dolcfg_init(&b); + CHECK(dolcfg_add_section(&a, insts, 8, BASE, "text")); + CHECK(dolcfg_add_section(&b, insts, 8, BASE, "text")); + CHECK(dolcfg_build(&a, stderr)); + CHECK(dolcfg_build(&b, stderr)); + + CHECK(a.block_count == b.block_count); + CHECK(a.function_count == b.function_count); + CHECK(a.scc_count == b.scc_count); + CHECK(a.loop_count == b.loop_count); + for (u32 i = 0; i < a.block_count; i++) { + CHECK(a.blocks[i].start == b.blocks[i].start); + CHECK(a.blocks[i].end == b.blocks[i].end); + CHECK(a.blocks[i].terminator == b.blocks[i].terminator); + CHECK(a.blocks[i].successor_count == b.blocks[i].successor_count); + CHECK(a.blocks[i].successors[0] == b.blocks[i].successors[0]); + CHECK(a.blocks[i].successors[1] == b.blocks[i].successors[1]); + CHECK(a.blocks[i].function == b.blocks[i].function); + CHECK(a.blocks[i].scc == b.blocks[i].scc); + CHECK(a.blocks[i].flags == b.blocks[i].flags); + } + + dolcfg_free(&a); + dolcfg_free(&b); + return true; +} + +/* Every decoded, non-data instruction must land in exactly one block. */ +static bool test_blocks_cover_all_code_exactly_once(void) { + const u32 raw[] = { + 0x38600000u, 0x2C03000Au, 0x4180FFF8u, 0x48000011u, + 0x4E800020u, 0x60000000u, 0x38600001u, 0x4E800020u, + }; + PPCInst insts[8]; + decode_all(insts, raw, 8, BASE); + + DolCfgProgram program; + dolcfg_init(&program); + CHECK(dolcfg_add_section(&program, insts, 8, BASE, "text")); + CHECK(dolcfg_build(&program, stderr)); + + u8 seen[8]; + memset(seen, 0, sizeof(seen)); + for (u32 i = 0; i < program.block_count; i++) { + const DolCfgBlock* block = &program.blocks[i]; + CHECK(block->end > block->start); + for (u32 a = block->start; a < block->end; a += 4u) { + u32 slot = (a - BASE) / 4u; + CHECK(slot < 8u); + CHECK(seen[slot] == 0); + seen[slot] = 1; + } + } + for (u32 i = 0; i < 8; i++) + CHECK(seen[i] == 1); + + dolcfg_free(&program); + return true; +} + +int main(void) { + struct { + const char* name; + bool (*fn)(void); + } tests[] = { + {"loop_blocks_and_header", test_loop_blocks_and_header}, + {"unconditional_bc_drops_fallthrough", test_unconditional_bc_drops_fallthrough}, + {"call_creates_function", test_call_creates_function}, + {"tail_call_reclassification", test_tail_call_reclassification}, + {"indirect_has_no_guessed_target", test_indirect_has_no_guessed_target}, + {"embedded_data_is_not_code", test_embedded_data_is_not_code}, + {"smc_ranges_flag_blocks", test_smc_ranges_flag_blocks}, + {"build_is_deterministic", test_build_is_deterministic}, + {"blocks_cover_all_code_exactly_once", test_blocks_cover_all_code_exactly_once}, + }; + + int failures = 0; + for (size_t i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + if (!tests[i].fn()) { + fprintf(stderr, "FAILED: %s\n", tests[i].name); + failures++; + } + } + + if (failures != 0) { + fprintf(stderr, "%d cfg test(s) failed\n", failures); + return 1; + } + + printf("cfg tests passed\n"); + return 0; +} diff --git a/tests/test_codegen_emit.c b/tests/test_codegen_emit.c index dccd17e..df9c9ad 100644 --- a/tests/test_codegen_emit.c +++ b/tests/test_codegen_emit.c @@ -193,7 +193,7 @@ int main(int argc, char** argv) { if (out != stdout) fclose(out); return 1; } - emit_dispatch_helpers(out, &funcs, BASE); + emit_dispatch_helpers(out, &funcs, BASE, 0); function_list_free(&funcs); emit_footer(out); diff --git a/tests/test_differential.c b/tests/test_differential.c new file mode 100644 index 0000000..a926436 --- /dev/null +++ b/tests/test_differential.c @@ -0,0 +1,245 @@ +/* Differential test: the C backend against the LLVM backend. + * + * Both backends compile the same random guest sequences, emitted at different + * guest base addresses so their func_
symbols coexist in one binary. + * Each pair runs from a byte-identical randomised CPUState and the full + * observable result is compared: every GPR, every FPR and paired-single lane as + * a bit pattern rather than a float compare, LR, CTR, CR, XER, FPSCR, the + * exception and reservation state, and the scratch memory both wrote through. + * + * NaN payloads and signed zero are why the float comparison is on bits. Two + * backends that agree numerically but disagree on which NaN they produce have + * still diverged, and a title can observe that. + * + * The seed is printed on failure so a divergence reproduces exactly. + */ + +#include "cpu/cpu.h" + +#include +#include + +typedef void (*DiffFunction)(CPUState*); + +typedef struct { + DiffFunction c_backend; + DiffFunction llvm_backend; + u32 c_address; + u32 llvm_address; +} DiffPair; + +#include "differential_manifest.h" + +/* Guest code is executed the way the runtime executes it: a dispatch loop that + * calls whichever generated function contains the current PC, until control + * returns past a sentinel LR. + * + * This is not a detail. The C backend lowers `bl` to + * ctx->lr = continuation; ctx->pc = target; return; + * -- it hands the call back to the runtime rather than calling the callee. The + * LLVM backend calls the callee directly through its function ranges. Invoking + * each arm once would therefore execute two different programs: the C arm would + * stop at the first call having run two instructions, while the LLVM arm ran + * the whole thing. Both arms under the same loop is what makes a call-shaped + * sequence comparable at all. + * + * The step limit is a backstop against a generated sequence that loops forever; + * it is a test bug if it fires, not a backend result, so it is reported as one. + */ +#define DIFF_SENTINEL_LR 0x8FFFFFFCu +#define DIFF_STEP_LIMIT 100000u + +/* Where generated loads and stores address through r31. Sits well inside RAM + and clear of anything else the test touches. */ +#define SCRATCH_ADDRESS 0x80010000u +#define SCRATCH_BYTES 4096u + +static u64 rng_state; + +static u32 next_random(void) { + rng_state = rng_state * 6364136223846793005ull + 1442695040888963407ull; + return (u32)(rng_state >> 33); +} + +/* Deliberately biased toward awkward values. A uniform random 32 bits almost + never produces a denormal, an infinity, or an exact power of two, and those + are where backends disagree. */ +static u32 awkward_word(void) { + switch (next_random() % 8u) { + case 0: return 0u; + case 1: return 0xFFFFFFFFu; + case 2: return 0x80000000u; + case 3: return 0x7F800000u; /* +inf as float bits */ + case 4: return 0x7FC00000u; /* quiet NaN */ + case 5: return 0x00800000u; /* smallest normal float */ + case 6: return 0x00000001u; /* denormal */ + default: return next_random(); + } +} + +static void randomise(CPUState* cpu) { + for (u32 i = 0; i < 32; i++) + cpu->gpr[i] = awkward_word(); + for (u32 i = 0; i < 32; i++) { + u64 bits = ((u64)awkward_word() << 32) | awkward_word(); + memcpy(&cpu->fpr[i], &bits, sizeof(bits)); + bits = ((u64)awkward_word() << 32) | awkward_word(); + memcpy(&cpu->ps1[i], &bits, sizeof(bits)); + } + cpu->cr = next_random(); + cpu->xer = next_random() & 0xE000007Fu; + cpu->lr = next_random() & ~3u; + cpu->ctr = next_random(); + /* Rounding mode and enables left at a sane default: a random FPSCR would + compare two backends under a mode neither claims to support. */ + cpu->fpscr = 0; + cpu->msr = 0x00002000u; /* MSR[FP] set, or every FP op takes an exception */ + cpu->exception = 0; + cpu->program_exception = 0; + cpu->reserve_addr = 0; + cpu->reserve_valid = false; + cpu->downcount = 1000000; + + /* r31 is the addressing base every generated memory op uses. */ + cpu->gpr[31] = SCRATCH_ADDRESS; + + for (u32 i = 0; i < SCRATCH_BYTES; i += 4) + mem_write32(cpu, SCRATCH_ADDRESS + i, awkward_word()); +} + +/* Returns 0 if the step limit was hit. */ +static int run_arm(CPUState* cpu, u32 base, int use_llvm) { + for (u32 step = 0; step < DIFF_STEP_LIMIT; step++) { + if (cpu->pc == DIFF_SENTINEL_LR) + return 1; + if (cpu->exception) + return 1; /* Left through the runtime; both arms must agree on it. */ + u32 index = (cpu->pc - base) / DIFF_STRIDE; + if (index >= DIFF_COUNT) + return 1; /* Outside the generated set: nothing more to run. */ + if (use_llvm) + diff_pairs[index].llvm_backend(cpu); + else + diff_pairs[index].c_backend(cpu); + } + return 0; +} + +static int report(const char* what, u32 index, u64 expected, u64 actual) { + fprintf(stderr, + " divergence in %s at pair %u: C=0x%016llX LLVM=0x%016llX\n", + what, index, (unsigned long long)expected, + (unsigned long long)actual); + return 1; +} + +static u64 float_bits(f64 value) { + u64 bits; + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +static int compare(const CPUState* a, const CPUState* b, u32 index) { + int bad = 0; + char label[32]; + + for (u32 i = 0; i < 32; i++) { + if (a->gpr[i] != b->gpr[i]) { + snprintf(label, sizeof(label), "r%u", i); + bad |= report(label, index, a->gpr[i], b->gpr[i]); + } + } + for (u32 i = 0; i < 32; i++) { + if (float_bits(a->fpr[i]) != float_bits(b->fpr[i])) { + snprintf(label, sizeof(label), "f%u", i); + bad |= report(label, index, float_bits(a->fpr[i]), + float_bits(b->fpr[i])); + } + if (float_bits(a->ps1[i]) != float_bits(b->ps1[i])) { + snprintf(label, sizeof(label), "ps1[%u]", i); + bad |= report(label, index, float_bits(a->ps1[i]), + float_bits(b->ps1[i])); + } + } + if (a->cr != b->cr) bad |= report("cr", index, a->cr, b->cr); + if (a->xer != b->xer) bad |= report("xer", index, a->xer, b->xer); + if (a->lr != b->lr) bad |= report("lr", index, a->lr, b->lr); + if (a->ctr != b->ctr) bad |= report("ctr", index, a->ctr, b->ctr); + if (a->fpscr != b->fpscr) bad |= report("fpscr", index, a->fpscr, b->fpscr); + if (a->exception != b->exception) + bad |= report("exception", index, a->exception, b->exception); + if (a->program_exception != b->program_exception) + bad |= report("program_exception", index, a->program_exception, + b->program_exception); + if (a->reserve_valid != b->reserve_valid) + bad |= report("reserve_valid", index, a->reserve_valid, b->reserve_valid); + if (a->reserve_addr != b->reserve_addr) + bad |= report("reserve_addr", index, a->reserve_addr, b->reserve_addr); + + /* Memory the sequences wrote through r31. */ + if (memcmp(a->ram + (SCRATCH_ADDRESS - GC_RAM_BASE), + b->ram + (SCRATCH_ADDRESS - GC_RAM_BASE), SCRATCH_BYTES) != 0) { + for (u32 offset = 0; offset < SCRATCH_BYTES; offset += 4) { + u32 left = mem_read32((CPUState*)a, SCRATCH_ADDRESS + offset); + u32 right = mem_read32((CPUState*)b, SCRATCH_ADDRESS + offset); + if (left != right) { + snprintf(label, sizeof(label), "mem+0x%X", offset); + bad |= report(label, index, left, right); + break; /* One is enough to identify the failure. */ + } + } + } + return bad; +} + +int main(void) { + CPUState a; + CPUState b; + if (!cpu_init(&a) || !cpu_init(&b)) { + fprintf(stderr, "differential: cannot allocate CPU state\n"); + return 1; + } + + u32 failures = 0; + for (u32 i = 0; i < DIFF_COUNT; i++) { + /* Same seed for both arms of a pair, advanced per pair so each gets a + different starting state. */ + rng_state = DIFF_SEED + i; + randomise(&a); + rng_state = DIFF_SEED + i; + randomise(&b); + + a.pc = diff_pairs[i].c_address; + b.pc = diff_pairs[i].llvm_address; + a.lr = DIFF_SENTINEL_LR; + b.lr = DIFF_SENTINEL_LR; + + if (!run_arm(&a, DIFF_BASE_C, 0) || !run_arm(&b, DIFF_BASE_L, 1)) { + fprintf(stderr, " pair %u exceeded the step limit; that is a test " + "bug, not a backend divergence\n", i); + failures++; + continue; + } + + /* PC ends at the sentinel in both arms, so it carries no information + and comparing it would only compare the two base addresses. */ + + if (compare(&a, &b, i)) { + fprintf(stderr, " reproduce with seed %llu, pair %u\n", + (unsigned long long)DIFF_SEED, i); + failures++; + } + } + + cpu_free(&a); + cpu_free(&b); + + if (failures) { + fprintf(stderr, "differential: %u of %u pairs diverged\n", + failures, (u32)DIFF_COUNT); + return 1; + } + printf("differential: %u pairs agree (seed %llu)\n", (u32)DIFF_COUNT, + (unsigned long long)DIFF_SEED); + return 0; +} diff --git a/tests/test_dispatch.c b/tests/test_dispatch.c index 5b4f843..d2f7a83 100644 --- a/tests/test_dispatch.c +++ b/tests/test_dispatch.c @@ -57,7 +57,7 @@ static char* emit_dispatch_to_string(void) { emit_chunk_prototype(f, BASE + 0x40u); emit_chunk_prototype(f, BASE + 0x80u); emit_chunk_prototype(f, BASE + 0x1000u); - emit_dispatch_helpers(f, &funcs, BASE); + emit_dispatch_helpers(f, &funcs, BASE, 0); function_list_free(&funcs); fflush(f); diff --git a/tests/test_llvm_backend.cpp b/tests/test_llvm_backend.cpp index adffe9e..9cdd526 100644 --- a/tests/test_llvm_backend.cpp +++ b/tests/test_llvm_backend.cpp @@ -32,7 +32,7 @@ static bool add_chunk(DolIRModule* module, const u32* words, u32 count, } int main(int argc, char** argv) { - CHECK(argc == 3); + CHECK(argc == 3 || argc == 4); DolIRModule module; dolir_module_init(&module); @@ -103,6 +103,19 @@ int main(int argc, char** argv) { }; CHECK(add_chunk(&module, paired_words, 7, 0x80002B00u)); + // MEM1 boundary. --memory-mode fast folds the MEM1 bound to a constant and + // drops the "size >= width" half of the range check as constant-true, so + // the exact edge is the one thing that mode can get wrong -- and nothing + // else tests it: the differential harness only ever touches a scratch + // offset deep inside MEM1. + // + // r3 holds the address, so the caller can place it at the last word in + // MEM1, straddling the end, or past it. stw r4,0(r3) then lwz r5,0(r3). + const u32 boundary_words[] = { + 0x90830000u, 0x80A30000u, 0x4E800020u, + }; + CHECK(add_chunk(&module, boundary_words, 3, 0x80003000u)); + // Runtime boundaries must not reset the dispatcher budget. const u32 budget_words[] = { 0x38630001u, 0x00000000u, 0x2C032710u, 0x4180FFF4u, 0x4E800020u, @@ -120,6 +133,13 @@ int main(int argc, char** argv) { options.optimization_level = 2; options.verify = 1; options.emit_ir = 1; + if (argc == 4) { + // ThinLTO needs bitcode carrying a module summary index. Emitting it + // here keeps the path exercised by ctest rather than only by a real + // title build, which takes twenty minutes to discover a mistake. + options.emit_bitcode = 1; + options.bitcode_path = argv[3]; + } options.ir_path = argv[2]; const DolLLVMFunctionRange ranges[] = { {0x80002D00u, 0x80002D04u}, diff --git a/tests/test_llvm_execute.c b/tests/test_llvm_execute.c index 3c938b4..b8728b9 100644 --- a/tests/test_llvm_execute.c +++ b/tests/test_llvm_execute.c @@ -18,6 +18,7 @@ void func_80002A00(CPUState* cpu); void func_80002B00(CPUState* cpu); void func_80002C00(CPUState* cpu); void func_80002D00(CPUState* cpu); +void func_80003000(CPUState* cpu); static u32 fallback_count; static int fallback_bad; @@ -279,6 +280,39 @@ int main(void) { CHECK(cpu.pc == 0x80002D00u || cpu.pc == 0x80002E00u); CHECK(cpu.downcount <= -128 && cpu.downcount >= -512); + // MEM1 boundary, which --memory-mode fast reduces to a compare against a + // constant. Each case is checked for the value actually written or read, + // not merely for "did not crash": a bound that is off by one word writes + // outside the RAM allocation, and that is the failure this mode could + // plausibly introduce. + { + const u32 last_word = GC_RAM_BASE + GC_MAIN_RAM_SIZE - 4u; + + // Fully inside: the last addressable word must round-trip. + prepare_call(&cpu, 0x80003000u); + cpu.gpr[3] = last_word; + cpu.gpr[4] = 0xA5A5A5A5u; + cpu.gpr[5] = 0; + func_80003000(&cpu); + CHECK(mem_read32(&cpu, last_word) == 0xA5A5A5A5u); + CHECK(cpu.gpr[5] == 0xA5A5A5A5u); + + // Straddling the end: three bytes inside, one past. Must take the slow + // path rather than the RAM path, so the last word keeps its value. + prepare_call(&cpu, 0x80003000u); + cpu.gpr[3] = last_word + 1u; + cpu.gpr[4] = 0x5A5A5A5Au; + func_80003000(&cpu); + CHECK(mem_read32(&cpu, last_word) == 0xA5A5A5A5u); + + // Entirely past the end. + prepare_call(&cpu, 0x80003000u); + cpu.gpr[3] = GC_RAM_BASE + GC_MAIN_RAM_SIZE; + cpu.gpr[4] = 0x5A5A5A5Au; + func_80003000(&cpu); + CHECK(mem_read32(&cpu, last_word) == 0xA5A5A5A5u); + } + cpu_free(&cpu); return 0; } diff --git a/tests/test_perf.c b/tests/test_perf.c new file mode 100644 index 0000000..5e63de4 --- /dev/null +++ b/tests/test_perf.c @@ -0,0 +1,249 @@ +#include "common/perf.h" + +#include +#include +#include + +#if defined(_WIN32) +#include +#else +#include +#include +#endif + +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "check failed: %s:%d: %s\n", \ + __FILE__, __LINE__, #x); return false; } } while (0) + +static char g_dir[1024]; + +static void path_in_dir(char* out, size_t size, const char* leaf) { + snprintf(out, size, "%s%c%s", g_dir, '/', leaf); +} + +static char* read_all(const char* path) { + FILE* in = fopen(path, "rb"); + if (!in) + return NULL; + fseek(in, 0, SEEK_END); + long size = ftell(in); + rewind(in); + if (size < 0) { + fclose(in); + return NULL; + } + char* text = (char*)malloc((size_t)size + 1u); + if (!text) { + fclose(in); + return NULL; + } + size_t got = fread(text, 1, (size_t)size, in); + text[got] = '\0'; + fclose(in); + return text; +} + +/* A counter added to the X-macro must appear in the struct, the JSON and the + generated header without any further edit. Checking one from each population + is enough to catch a list that has been extended in only one place. */ +static bool test_counter_list_is_one_source_of_truth(void) { + DolPerfReport report; + memset(&report, 0, sizeof(report)); + + report.counters.dispatcher_entries = 11; + report.counters.mem1_fast_reads = 22; + report.counters.regions_planned = 33; + + char path[1200]; + path_in_dir(path, sizeof(path), "counters.json"); + CHECK(dolperf_write_json(&report, path, stderr)); + + char* text = read_all(path); + CHECK(text != NULL); + CHECK(strstr(text, "\"dispatcher_entries\": 11") != NULL); + CHECK(strstr(text, "\"mem1_fast_reads\": 22") != NULL); + CHECK(strstr(text, "\"regions_planned\": 33") != NULL); + CHECK(strstr(text, "\"schema\": \"dolrecomp.perf/1\"") != NULL); + free(text); + return true; +} + +static bool test_region_records_fold_into_counters(void) { + DolPerfReport report; + memset(&report, 0, sizeof(report)); + + DolPerfRegion a = {0}; + a.region_id = 0; + a.guest_start = 0x80003100u; + a.guest_end = 0x80003200u; + a.guest_instructions = 64; + a.ir_instructions = 200; + a.code_bytes = 512; + a.optimize_ns = 1000; + a.codegen_ns = 2000; + a.cache_hit = 0; + + DolPerfRegion b = a; + b.region_id = 1; + b.guest_instructions = 36; + b.cache_hit = 1; + + dolperf_add_region(&report, &a); + dolperf_add_region(&report, &b); + + CHECK(report.region_count == 2); + CHECK(report.counters.regions_planned == 2); + CHECK(report.counters.region_guest_instructions == 100); + CHECK(report.counters.region_ir_instructions == 400); + CHECK(report.counters.llvm_optimize_ns == 2000); + CHECK(report.counters.llvm_codegen_ns == 4000); + CHECK(report.counters.artifact_cache_hits == 1); + CHECK(report.counters.artifact_cache_misses == 1); + + char path[1200]; + path_in_dir(path, sizeof(path), "regions.json"); + CHECK(dolperf_write_json(&report, path, stderr)); + char* text = read_all(path); + CHECK(text != NULL); + CHECK(strstr(text, "\"start\": \"0x80003100\"") != NULL); + CHECK(strstr(text, "\"cache_hit\": true") != NULL); + CHECK(strstr(text, "\"cache_hit\": false") != NULL); + free(text); + + dolperf_free(&report); + return true; +} + +/* The runtime half writes a counter dump that the compiler half reads back. + Round-tripping through the parser is what keeps the benchmark harness able to + merge a game run's counters into a build report. */ +static bool test_runtime_counter_roundtrip(void) { + DolPerfReport source; + memset(&source, 0, sizeof(source)); + source.counters.dispatcher_entries = 5000; + source.counters.blr_prediction_hits = 90; + source.counters.blr_prediction_misses = 10; + source.counters.slow_writes = 7; + /* A compile-side counter must NOT survive the runtime round-trip: the + generated module has no such counter to report. */ + source.counters.regions_planned = 999; + + char path[1200]; + path_in_dir(path, sizeof(path), "runtime.json"); + CHECK(dolperf_write_json(&source, path, stderr)); + + DolPerfCounters parsed; + CHECK(dolperf_read_runtime_json(path, &parsed, stderr)); + CHECK(parsed.dispatcher_entries == 5000); + CHECK(parsed.blr_prediction_hits == 90); + CHECK(parsed.blr_prediction_misses == 10); + CHECK(parsed.slow_writes == 7); + CHECK(parsed.regions_planned == 0); + + DolPerfReport merged; + memset(&merged, 0, sizeof(merged)); + merged.counters.dispatcher_entries = 1; + dolperf_merge_runtime(&merged, &parsed); + CHECK(merged.counters.dispatcher_entries == 5001); + CHECK(merged.counters.blr_prediction_hits == 90); + CHECK(merged.counters.regions_planned == 0); + return true; +} + +/* The point of the generated header is that a module built without + DOLRECOMP_PERF carries no counter stores at all. */ +static bool test_generated_header_compiles_out(void) { + char path[1200]; + path_in_dir(path, sizeof(path), "dolrecomp_perf.h"); + FILE* out = fopen(path, "wb"); + CHECK(out != NULL); + dolperf_emit_runtime_header(out); + CHECK(fclose(out) == 0); + + char* text = read_all(path); + CHECK(text != NULL); + CHECK(strstr(text, "#ifndef DOLRECOMP_GENERATED_PERF_H") != NULL); + CHECK(strstr(text, "#define DOLRECOMP_PERF_INC(name) ((void)0)") != NULL); + CHECK(strstr(text, "DOLRECOMP_PERF_SLOT dispatcher_entries;") != NULL); + CHECK(strstr(text, "DOLRECOMP_PERF_SLOT mem2_fast_writes;") != NULL); + /* Compile-side counters have no business in the guest module. */ + CHECK(strstr(text, "llvm_codegen_ns;") == NULL); + free(text); + return true; +} + +static bool test_summary_hides_empty_groups(void) { + DolPerfReport report; + memset(&report, 0, sizeof(report)); + report.counters.regions_planned = 4; + report.counters.region_guest_instructions = 400; + + char path[1200]; + path_in_dir(path, sizeof(path), "summary.txt"); + FILE* out = fopen(path, "wb"); + CHECK(out != NULL); + dolperf_print_summary(&report, out); + CHECK(fclose(out) == 0); + + char* text = read_all(path); + CHECK(text != NULL); + CHECK(strstr(text, "Compilation") != NULL); + CHECK(strstr(text, "Regions planned") != NULL); + /* Nothing executed guest code, so these sections must not appear. */ + CHECK(strstr(text, "Guest memory") == NULL); + CHECK(strstr(text, "Dispatch and linking") == NULL); + free(text); + return true; +} + +static bool test_clock_is_monotonic(void) { + u64 first = dolperf_now_ns(); + u64 last = first; + for (int i = 0; i < 1000; i++) { + u64 now = dolperf_now_ns(); + CHECK(now >= last); + last = now; + } + CHECK(last >= first); + return true; +} + +int main(int argc, char** argv) { + if (argc > 1) + snprintf(g_dir, sizeof(g_dir), "%s", argv[1]); + else + snprintf(g_dir, sizeof(g_dir), "."); + +#if defined(_WIN32) + _mkdir(g_dir); +#else + mkdir(g_dir, 0777); +#endif + + struct { + const char* name; + bool (*fn)(void); + } tests[] = { + {"counter_list_is_one_source_of_truth", test_counter_list_is_one_source_of_truth}, + {"region_records_fold_into_counters", test_region_records_fold_into_counters}, + {"runtime_counter_roundtrip", test_runtime_counter_roundtrip}, + {"generated_header_compiles_out", test_generated_header_compiles_out}, + {"summary_hides_empty_groups", test_summary_hides_empty_groups}, + {"clock_is_monotonic", test_clock_is_monotonic}, + }; + + int failures = 0; + for (size_t i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + if (!tests[i].fn()) { + fprintf(stderr, "FAILED: %s\n", tests[i].name); + failures++; + } + } + + if (failures != 0) { + fprintf(stderr, "%d perf test(s) failed\n", failures); + return 1; + } + + printf("perf tests passed\n"); + return 0; +} diff --git a/tests/test_regions.c b/tests/test_regions.c new file mode 100644 index 0000000..0f1b4d0 --- /dev/null +++ b/tests/test_regions.c @@ -0,0 +1,423 @@ +#include "analysis/regions.h" + +#include +#include +#include + +#if defined(_WIN32) +#include +#else +#include +#include +#endif + +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "check failed: %s:%d: %s\n", \ + __FILE__, __LINE__, #x); return false; } } while (0) + +#define BASE 0x80001000u + +static char g_dir[1024]; + +static void decode_all(PPCInst* out, const u32* raw, u32 count, u32 base) { + for (u32 i = 0; i < count; i++) + out[i] = ppc_decode(raw[i], base + i * 4u); +} + +/* main: bl leaf ; blr + leaf: addi ; blr + spare: addi ; blr (unconnected, reached only indirectly) */ +static const u32 kCallGraph[] = { + 0x48000009u, /* +0x00 bl +8 -> BASE+8 */ + 0x4E800020u, /* +0x04 blr */ + 0x38600001u, /* +0x08 leaf */ + 0x4E800020u, /* +0x0C blr */ + 0x38600002u, /* +0x10 spare */ + 0x4E800020u, /* +0x14 blr */ +}; + +static bool build_program(DolCfgProgram* program, PPCInst* insts, + const u32* raw, u32 count) { + decode_all(insts, raw, count, BASE); + dolcfg_init(program); + if (!dolcfg_add_section(program, insts, count, BASE, "text")) + return false; + return dolcfg_build(program, stderr); +} + +/* Whatever the mode, every block must land in exactly one region. A plan that + loses a block loses code. */ +static bool coverage_is_exact(const DolRegionPlan* plan, + const DolCfgProgram* program) { + u8* seen = (u8*)calloc(program->block_count ? program->block_count : 1u, 1u); + if (!seen) + return false; + bool ok = true; + + for (u32 r = 0; r < plan->region_count && ok; r++) { + for (u32 i = 0; i < plan->regions[r].block_count; i++) { + u32 index = plan->regions[r].blocks[i]; + if (index >= program->block_count || seen[index]) { + ok = false; + break; + } + seen[index] = 1; + if (plan->block_region[index] != r) + ok = false; + } + } + for (u32 i = 0; i < program->block_count && ok; i++) { + if (!seen[i]) + ok = false; + } + + free(seen); + return ok; +} + +static bool test_every_mode_covers_every_block(void) { + PPCInst insts[6]; + DolCfgProgram program; + CHECK(build_program(&program, insts, kCallGraph, 6)); + + DolRegionLimits limits; + dolregion_default_limits(&limits); + + const DolRegionMode modes[] = { + DOLREGION_MODE_FIXED, DOLREGION_MODE_FUNCTION, + DOLREGION_MODE_CFG, DOLREGION_MODE_PGO, + }; + + for (size_t m = 0; m < sizeof(modes) / sizeof(modes[0]); m++) { + DolRegionPlan plan; + dolregion_plan_init(&plan); + CHECK(dolregion_plan_build(&plan, &program, modes[m], &limits, stderr)); + CHECK(plan.region_count > 0); + CHECK(coverage_is_exact(&plan, &program)); + CHECK(plan.total_instructions == 6); + dolregion_plan_free(&plan); + } + + dolcfg_free(&program); + return true; +} + +/* Function mode gives each function its own region. */ +static bool test_function_mode_is_one_region_per_function(void) { + PPCInst insts[6]; + DolCfgProgram program; + CHECK(build_program(&program, insts, kCallGraph, 6)); + + DolRegionLimits limits; + dolregion_default_limits(&limits); + + DolRegionPlan plan; + dolregion_plan_init(&plan); + CHECK(dolregion_plan_build(&plan, &program, DOLREGION_MODE_FUNCTION, + &limits, stderr)); + CHECK(plan.region_count == program.function_count); + for (u32 r = 0; r < plan.region_count; r++) { + CHECK(plan.regions[r].function_count == 1); + CHECK(plan.regions[r].end_reason == DOLREGION_END_FUNCTION_BOUNDARY); + } + + dolregion_plan_free(&plan); + dolcfg_free(&program); + return true; +} + +/* The whole point: cfg mode must put a caller and its callee together, so the + call becomes an internal edge instead of a region transfer. */ +static bool test_cfg_mode_merges_caller_and_callee(void) { + PPCInst insts[6]; + DolCfgProgram program; + CHECK(build_program(&program, insts, kCallGraph, 6)); + + u32 caller_block = dolcfg_block_starting_at(&program, BASE); + u32 callee_block = dolcfg_block_starting_at(&program, BASE + 8u); + CHECK(caller_block != DOLCFG_NO_BLOCK && callee_block != DOLCFG_NO_BLOCK); + + DolRegionLimits limits; + dolregion_default_limits(&limits); + + DolRegionPlan function_plan; + dolregion_plan_init(&function_plan); + CHECK(dolregion_plan_build(&function_plan, &program, DOLREGION_MODE_FUNCTION, + &limits, stderr)); + CHECK(function_plan.block_region[caller_block] != + function_plan.block_region[callee_block]); + + DolRegionPlan cfg_plan; + dolregion_plan_init(&cfg_plan); + CHECK(dolregion_plan_build(&cfg_plan, &program, DOLREGION_MODE_CFG, + &limits, stderr)); + CHECK(cfg_plan.block_region[caller_block] == + cfg_plan.block_region[callee_block]); + + /* Merging them is only worth anything if it removes a crossing. */ + CHECK(cfg_plan.cross_region_edges < function_plan.cross_region_edges); + CHECK(cfg_plan.region_count < function_plan.region_count); + + dolregion_plan_free(&cfg_plan); + dolregion_plan_free(&function_plan); + dolcfg_free(&program); + return true; +} + +/* A size limit must actually bind. */ +static bool test_size_limit_is_respected(void) { + PPCInst insts[6]; + DolCfgProgram program; + CHECK(build_program(&program, insts, kCallGraph, 6)); + + DolRegionLimits limits; + dolregion_default_limits(&limits); + limits.max_instructions = 2; + + DolRegionPlan plan; + dolregion_plan_init(&plan); + CHECK(dolregion_plan_build(&plan, &program, DOLREGION_MODE_CFG, &limits, stderr)); + CHECK(coverage_is_exact(&plan, &program)); + for (u32 r = 0; r < plan.region_count; r++) { + /* A single function larger than the limit is split, never dropped, so + the bound is on what accretion adds, not on an indivisible block. */ + CHECK(plan.regions[r].instruction_count <= limits.max_instructions * 2u); + } + + dolregion_plan_free(&plan); + dolcfg_free(&program); + return true; +} + +/* A function too big for any region is split rather than dropped. + Repeats cmpwi / beq / addi, so the function is many small blocks. */ +static bool test_large_function_is_split_not_dropped(void) { + enum { REPS = 16, COUNT = REPS * 3 + 1 }; + u32 raw[COUNT]; + for (u32 i = 0; i < REPS; i++) { + raw[i * 3u + 0u] = 0x2C030000u; /* cmpwi r3,0 */ + raw[i * 3u + 1u] = 0x41820008u; /* beq +8 */ + raw[i * 3u + 2u] = 0x38630001u; /* addi r3,r3,1 */ + } + raw[COUNT - 1u] = 0x4E800020u; /* blr */ + + PPCInst insts[COUNT]; + DolCfgProgram program; + CHECK(build_program(&program, insts, raw, COUNT)); + /* Must genuinely be one multi-block function for the split to mean + anything. */ + CHECK(program.block_count > 8); + + DolRegionLimits limits; + dolregion_default_limits(&limits); + limits.max_instructions = 8; + + DolRegionPlan plan; + dolregion_plan_init(&plan); + CHECK(dolregion_plan_build(&plan, &program, DOLREGION_MODE_FUNCTION, + &limits, stderr)); + CHECK(plan.region_count > 1); + CHECK(plan.split_functions >= 1); + CHECK(plan.total_instructions == COUNT); + CHECK(coverage_is_exact(&plan, &program)); + + dolregion_plan_free(&plan); + dolcfg_free(&program); + return true; +} + +/* Blocks are the atom of a region: a single basic block larger than the limit + is emitted whole rather than cut mid-block. Splitting straight-line code is + possible in principle, but a boundary inside a block needs a live-state + handoff at a point the CFG never chose, which is the cost regions exist to + avoid. Asserted so the behaviour is a decision, not an accident. */ +static bool test_indivisible_block_exceeds_limit(void) { + enum { COUNT = 64 }; + u32 raw[COUNT]; + for (u32 i = 0; i < COUNT - 1u; i++) + raw[i] = 0x38630001u; + raw[COUNT - 1u] = 0x4E800020u; + + PPCInst insts[COUNT]; + DolCfgProgram program; + CHECK(build_program(&program, insts, raw, COUNT)); + CHECK(program.block_count == 1); + + DolRegionLimits limits; + dolregion_default_limits(&limits); + limits.max_instructions = 8; + + DolRegionPlan plan; + dolregion_plan_init(&plan); + CHECK(dolregion_plan_build(&plan, &program, DOLREGION_MODE_FUNCTION, + &limits, stderr)); + CHECK(plan.region_count == 1); + CHECK(plan.regions[0].instruction_count == COUNT); + CHECK(coverage_is_exact(&plan, &program)); + + dolregion_plan_free(&plan); + dolcfg_free(&program); + return true; +} + +/* PGO mode with no weights must say so rather than look profiled. */ +static bool test_pgo_without_profile_is_reported(void) { + PPCInst insts[6]; + DolCfgProgram program; + CHECK(build_program(&program, insts, kCallGraph, 6)); + + DolRegionLimits limits; + dolregion_default_limits(&limits); + + DolRegionPlan plan; + dolregion_plan_init(&plan); + CHECK(dolregion_plan_build(&plan, &program, DOLREGION_MODE_PGO, &limits, stderr)); + CHECK(plan.profile_missing); + + /* With weights present it must not claim to be missing. */ + for (u32 i = 0; i < program.block_count; i++) + program.blocks[i].weight = 10; + + DolRegionPlan weighted; + dolregion_plan_init(&weighted); + CHECK(dolregion_plan_build(&weighted, &program, DOLREGION_MODE_PGO, + &limits, stderr)); + CHECK(!weighted.profile_missing); + CHECK(weighted.regions[0].weight > 0); + + dolregion_plan_free(&weighted); + dolregion_plan_free(&plan); + dolcfg_free(&program); + return true; +} + +/* Same inputs and settings, same plan -- including region numbering. */ +static bool test_plan_is_deterministic(void) { + PPCInst insts[6]; + DolCfgProgram program; + CHECK(build_program(&program, insts, kCallGraph, 6)); + + DolRegionLimits limits; + dolregion_default_limits(&limits); + + DolRegionPlan a, b; + dolregion_plan_init(&a); + dolregion_plan_init(&b); + CHECK(dolregion_plan_build(&a, &program, DOLREGION_MODE_CFG, &limits, stderr)); + CHECK(dolregion_plan_build(&b, &program, DOLREGION_MODE_CFG, &limits, stderr)); + + CHECK(a.region_count == b.region_count); + CHECK(a.cross_region_edges == b.cross_region_edges); + for (u32 r = 0; r < a.region_count; r++) { + CHECK(a.regions[r].guest_start == b.regions[r].guest_start); + CHECK(a.regions[r].guest_end == b.regions[r].guest_end); + CHECK(a.regions[r].block_count == b.regions[r].block_count); + CHECK(a.regions[r].function_count == b.regions[r].function_count); + CHECK(a.regions[r].end_reason == b.regions[r].end_reason); + for (u32 i = 0; i < a.regions[r].block_count; i++) + CHECK(a.regions[r].blocks[i] == b.regions[r].blocks[i]); + } + for (u32 i = 0; i < program.block_count; i++) + CHECK(a.block_region[i] == b.block_region[i]); + + dolregion_plan_free(&b); + dolregion_plan_free(&a); + dolcfg_free(&program); + return true; +} + +static bool test_report_contains_boundary_reasons(void) { + PPCInst insts[6]; + DolCfgProgram program; + CHECK(build_program(&program, insts, kCallGraph, 6)); + + DolRegionLimits limits; + dolregion_default_limits(&limits); + + DolRegionPlan plan; + dolregion_plan_init(&plan); + CHECK(dolregion_plan_build(&plan, &program, DOLREGION_MODE_CFG, &limits, stderr)); + + char path[1200]; + snprintf(path, sizeof(path), "%s/regions.json", g_dir); + CHECK(dolregion_write_report(&plan, &program, path, stderr)); + + FILE* in = fopen(path, "rb"); + CHECK(in != NULL); + fseek(in, 0, SEEK_END); + long size = ftell(in); + rewind(in); + char* text = (char*)malloc((size_t)size + 1u); + CHECK(text != NULL); + size_t got = fread(text, 1, (size_t)size, in); + text[got] = '\0'; + fclose(in); + + CHECK(strstr(text, "\"schema\": \"dolrecomp.regions/1\"") != NULL); + CHECK(strstr(text, "\"mode\": \"cfg\"") != NULL); + CHECK(strstr(text, "\"end_reason\"") != NULL); + CHECK(strstr(text, "\"function_addresses\"") != NULL); + CHECK(strstr(text, "\"internal_edges\"") != NULL); + CHECK(strstr(text, "\"max_instructions\"") != NULL); + + free(text); + dolregion_plan_free(&plan); + dolcfg_free(&program); + return true; +} + +static bool test_mode_names_round_trip(void) { + const char* names[] = {"fixed", "function", "cfg", "pgo"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + DolRegionMode mode; + CHECK(dolregion_parse_mode(names[i], &mode)); + CHECK(strcmp(dolregion_mode_name(mode), names[i]) == 0); + } + DolRegionMode mode; + CHECK(!dolregion_parse_mode("nonsense", &mode)); + return true; +} + +int main(int argc, char** argv) { + if (argc > 1) + snprintf(g_dir, sizeof(g_dir), "%s", argv[1]); + else + snprintf(g_dir, sizeof(g_dir), "."); + +#if defined(_WIN32) + _mkdir(g_dir); +#else + mkdir(g_dir, 0777); +#endif + + struct { + const char* name; + bool (*fn)(void); + } tests[] = { + {"every_mode_covers_every_block", test_every_mode_covers_every_block}, + {"function_mode_is_one_region_per_function", test_function_mode_is_one_region_per_function}, + {"cfg_mode_merges_caller_and_callee", test_cfg_mode_merges_caller_and_callee}, + {"size_limit_is_respected", test_size_limit_is_respected}, + {"large_function_is_split_not_dropped", test_large_function_is_split_not_dropped}, + {"indivisible_block_exceeds_limit", test_indivisible_block_exceeds_limit}, + {"pgo_without_profile_is_reported", test_pgo_without_profile_is_reported}, + {"plan_is_deterministic", test_plan_is_deterministic}, + {"report_contains_boundary_reasons", test_report_contains_boundary_reasons}, + {"mode_names_round_trip", test_mode_names_round_trip}, + }; + + int failures = 0; + for (size_t i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + if (!tests[i].fn()) { + fprintf(stderr, "FAILED: %s\n", tests[i].name); + failures++; + } + } + + if (failures != 0) { + fprintf(stderr, "%d region test(s) failed\n", failures); + return 1; + } + + printf("region tests passed\n"); + return 0; +} diff --git a/tools/cfg_stats.c b/tools/cfg_stats.c new file mode 100644 index 0000000..9daff5f --- /dev/null +++ b/tools/cfg_stats.c @@ -0,0 +1,300 @@ +/* Prints the whole-title CFG model for a DOL. + * + * Synthetic fixtures prove the shapes are handled; this is what shows the model + * survives a real title, where the interesting numbers are how much code no + * entry point reaches and how many indirect sites a region plan will have to + * end at. + * + * cfg_stats [--map ] + */ + +#include "analysis/cfg.h" +#include "analysis/regions.h" +#include "analysis/embedded_data.h" +#include "analysis/symbol_map.h" +#include "frontend/container/dol.h" + +#include +#include +#include + +int main(int argc, char** argv) { + if (argc < 2) { + fprintf(stderr, "usage: cfg_stats [--map ]\n"); + return 2; + } + + const char* map_path = NULL; + const char* report_path = NULL; + const char* profile_path = NULL; + int compare_modes = 0; + DolRegionMode mode = DOLREGION_MODE_CFG; + DolRegionLimits limits; + dolregion_default_limits(&limits); + + for (int i = 2; i < argc; i++) { + if (strcmp(argv[i], "--map") == 0 && i + 1 < argc) { + map_path = argv[++i]; + } else if (strcmp(argv[i], "--emit-region-report") == 0 && i + 1 < argc) { + report_path = argv[++i]; + } else if (strcmp(argv[i], "--region-mode") == 0 && i + 1 < argc) { + if (!dolregion_parse_mode(argv[++i], &mode)) { + fprintf(stderr, "error: unknown region mode '%s'\n", argv[i]); + return 2; + } + } else if (strcmp(argv[i], "--region-max-instructions") == 0 && i + 1 < argc) { + limits.max_instructions = (u32)strtoul(argv[++i], NULL, 0); + } else if (strcmp(argv[i], "--region-max-ir") == 0 && i + 1 < argc) { + limits.max_ir_instructions = (u32)strtoul(argv[++i], NULL, 0); + } else if (strcmp(argv[i], "--compare-modes") == 0) { + compare_modes = 1; + } else if (strcmp(argv[i], "--region-profile") == 0 && i + 1 < argc) { + profile_path = argv[++i]; + } else if (strcmp(argv[i], "--no-adjacency") == 0) { + limits.merge_address_adjacent = 0; + } else if (strcmp(argv[i], "--adjacency-gap") == 0 && i + 1 < argc) { + limits.max_adjacency_gap = (u32)strtoul(argv[++i], NULL, 0); + } + } + + DOLFile dol; + if (!dol_load(&dol, argv[1])) + return 1; + + DolCfgProgram program; + dolcfg_init(&program); + program.entry_point = dol.header.entry_point; + + PPCInst** decoded = (PPCInst**)calloc(DOL_NUM_TEXT, sizeof(PPCInst*)); + if (!decoded) { + dol_free(&dol); + return 1; + } + + u32 sections = 0; + for (u32 s = 0; s < DOL_NUM_TEXT; s++) { + if (dol.header.text_sizes[s] == 0) + continue; + + u32 count = dol.header.text_sizes[s] / 4u; + PPCInst* insts = (PPCInst*)malloc(count * sizeof(PPCInst)); + if (!insts) + break; + + const u8* data = dol.file_data + dol.header.text_offsets[s]; + u32 base = dol.header.text_addresses[s]; + for (u32 i = 0; i < count; i++) { + u32 raw = read_be32(data + i * 4u); + insts[i] = ppc_decode(raw, base + i * 4u); + /* Must match pipeline.c exactly: a word is data only if it did not + decode AND looks like data. Testing the data predicate alone + reclassifies decodable instructions as data and reports a + different program than the backends actually compile. */ + if (insts[i].op == PPC_OP_UNKNOWN && + embedded_data_word(EMBEDDED_DATA_DOL, raw)) + insts[i].embedded_data = true; + } + + decoded[sections] = insts; + dolcfg_add_section(&program, insts, count, base, "text"); + sections++; + } + + if (map_path) { + DolRecompSymbolMap symbols = {0}; + if (symbol_map_load(&symbols, map_path)) { + for (u32 i = 0; i < symbols.count; i++) { + dolcfg_add_known_function(&program, symbols.symbols[i].address, + symbols.symbols[i].name, + DOLCFG_FUNC_PATCHABLE); + } + printf("map symbols: %u\n", symbols.count); + } + symbol_map_free(&symbols); + } + + if (!dolcfg_build(&program, stderr)) { + dolcfg_free(&program); + dol_free(&dol); + return 1; + } + + /* Without this, --region-mode pgo silently degrades to cfg ordering and the + two report identical plans -- which looks like "PGO changes nothing" + rather than "no profile was loaded". */ + if (profile_path) { + u32 matched = 0, unmatched = 0; + if (!dolcfg_load_profile(&program, profile_path, &matched, &unmatched, + stderr)) { + dolcfg_free(&program); + dol_free(&dol); + return 1; + } + printf("profile: %u matched, %u unmatched\n", matched, unmatched); + } + + u32 code_instructions = 0; + u32 covered = 0; + u32 unreached = 0; + u32 loop_blocks = 0; + u32 smc_blocks = 0; + u32 term_counts[16]; + memset(term_counts, 0, sizeof(term_counts)); + + for (u32 i = 0; i < program.block_count; i++) { + const DolCfgBlock* block = &program.blocks[i]; + covered += block->instruction_count; + if (block->flags & DOLCFG_BLOCK_UNREACHED) + unreached += block->instruction_count; + if (block->flags & DOLCFG_BLOCK_LOOP_HEADER) + loop_blocks++; + if (block->flags & DOLCFG_BLOCK_SMC_SUSPECT) + smc_blocks++; + if ((u32)block->terminator < 16u) + term_counts[block->terminator]++; + } + + for (u32 s = 0; s < program.section_count; s++) { + const DolCfgSection* section = &program.sections[s]; + for (u32 i = 0; i < section->count; i++) { + if (!section->insts[i].embedded_data) + code_instructions++; + } + } + + /* Why is code unreached? Either nothing in the model branches to it -- so + it is only enterable through an indirect transfer, which is a Phase 4 + problem -- or something does reach it and ownership stopped early, which + would be a defect here. Separating the two is the difference between a + finding and a bug. */ + u32* in_degree = (u32*)calloc(program.block_count ? program.block_count : 1u, + sizeof(u32)); + u32 unreached_blocks = 0; + u32 unreached_no_in_edge = 0; + u32 unreached_with_in_edge = 0; + if (in_degree) { + for (u32 i = 0; i < program.block_count; i++) { + const DolCfgBlock* block = &program.blocks[i]; + for (u32 s = 0; s < block->successor_count; s++) { + if (block->successors[s] != DOLCFG_NO_BLOCK) + in_degree[block->successors[s]]++; + } + } + for (u32 i = 0; i < program.block_count; i++) { + if (!(program.blocks[i].flags & DOLCFG_BLOCK_UNREACHED)) + continue; + unreached_blocks++; + if (in_degree[i] == 0) + unreached_no_in_edge++; + else + unreached_with_in_edge++; + } + } + + /* Terminator mix weighted by execution. + * + * The unweighted mix says returns are 74% of indirect sites, but a site + * that never runs costs nothing. Dispatcher entries come from whatever + * *executes* and cannot resolve in-module, so the weighted mix is what + * says where Phase 4 should aim. Three separate nulls came from optimising + * a statically-large quantity that was dynamically irrelevant. */ + if (profile_path) { + u64 weighted[16]; + u64 sites[16]; + memset(weighted, 0, sizeof(weighted)); + memset(sites, 0, sizeof(sites)); + u64 total = 0; + for (u32 i = 0; i < program.block_count; i++) { + const DolCfgBlock* block = &program.blocks[i]; + u32 kind = (u32)block->terminator; + if (kind >= 16u) + continue; + weighted[kind] += block->weight; + sites[kind]++; + total += block->weight; + } + printf("\nterminator mix weighted by profile\n"); + printf(" %-14s %12s %10s %12s\n", "kind", "sites", "% sites", "% weight"); + for (u32 k = 0; k < 16; k++) { + if (!sites[k]) + continue; + printf(" %-14s %12llu %9.1f%% %11.2f%%\n", + dolcfg_terminator_name((DolCfgTerminator)k), + (unsigned long long)sites[k], + 100.0 * (double)sites[k] / (double)program.block_count, + total ? 100.0 * (double)weighted[k] / (double)total : 0.0); + } + } + + printf("sections %u\n", program.section_count); + printf("blocks %u\n", program.block_count); + printf("functions %u\n", program.function_count); + printf("SCCs %u\n", program.scc_count); + printf("loop headers %u\n", program.loop_count); + printf("indirect sites %u\n", program.indirect_site_count); + printf("code instructions %u\n", code_instructions); + printf("covered by blocks %u (%.2f%%)\n", covered, + code_instructions ? 100.0 * covered / code_instructions : 0.0); + printf("unreached code %u (%.2f%%)\n", unreached, + code_instructions ? 100.0 * unreached / code_instructions : 0.0); + printf("loop-header blocks %u\n", loop_blocks); + printf("SMC-suspect blocks %u\n", smc_blocks); + printf("unreached blocks %u\n", unreached_blocks); + printf(" no in-edge %u (indirect-only entry)\n", unreached_no_in_edge); + printf(" has in-edge %u (reached but unowned)\n", unreached_with_in_edge); + free(in_degree); + printf("\nterminators\n"); + for (u32 i = 0; i < 16; i++) { + if (term_counts[i]) + printf(" %-14s %u\n", dolcfg_terminator_name((DolCfgTerminator)i), + term_counts[i]); + } + + /* Region planning. The comparison is the point: the same title through + every mode, so the cost of an arbitrary boundary is visible next to the + cost of a chosen one. */ + const DolRegionMode all_modes[] = { + DOLREGION_MODE_FIXED, DOLREGION_MODE_FUNCTION, DOLREGION_MODE_CFG, + }; + u32 mode_count = compare_modes ? 3u : 1u; + + printf("\nregion plans (max %u instructions)\n", limits.max_instructions); + printf(" %-10s %9s %9s %11s %11s %9s\n", "mode", "regions", "instr/rgn", + "crossings", "internal", "split fns"); + + for (u32 m = 0; m < mode_count; m++) { + DolRegionMode selected = compare_modes ? all_modes[m] : mode; + + DolRegionPlan plan; + dolregion_plan_init(&plan); + if (!dolregion_plan_build(&plan, &program, selected, &limits, stderr)) { + dolregion_plan_free(&plan); + continue; + } + + u64 internal = 0; + for (u32 r = 0; r < plan.region_count; r++) + internal += plan.regions[r].internal_edges; + + printf(" %-10s %9u %9.1f %11u %11llu %9u\n", + dolregion_mode_name(selected), plan.region_count, + plan.region_count + ? (double)plan.total_instructions / (double)plan.region_count + : 0.0, + plan.cross_region_edges, (unsigned long long)internal, + plan.split_functions); + + if (report_path && (!compare_modes || selected == mode)) { + if (dolregion_write_report(&plan, &program, report_path, stderr)) + printf(" region report: %s\n", report_path); + } + dolregion_plan_free(&plan); + } + + dolcfg_free(&program); + for (u32 s = 0; s < sections; s++) + free(decoded[s]); + free(decoded); + dol_free(&dol); + return 0; +}