diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8593a4ef0..63fd4bedc 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -27,6 +27,8 @@ jobs: - name: Run Clippy # Make sure CI fails on all warnings, including Clippy lints run: nix develop --ignore-environment -c bash -c "cd tool/microkit && cargo-clippy --all-targets --all-features -- -D warnings -Wclippy::get_unwrap" + - name: Run Clippy (Loader) + run: nix develop --ignore-environment -c bash -c 'make -C loader clippy CLIPPYFLAGS="-D warnings -Wclippy::get_unwrap" BUILD_DIR=$(mktemp -d) ARCH=dummy BOARD=dummy SEL4_SDK=dummy TARGET_TRIPLE=dummy LLVM=False LINK_ADDRESS=0' rustfmt_check: runs-on: [self-hosted, macos, ARM64] diff --git a/build_sdk.py b/build_sdk.py index 92cc5bfcb..dd5b4f9cf 100644 --- a/build_sdk.py +++ b/build_sdk.py @@ -796,6 +796,26 @@ def build_sel4( json_dst.chmod(0o744) +def test_loader(build_dir: Path) -> None: + build_dir = build_dir / "loader" + build_dir.mkdir(exist_ok=True, parents=True) + + make_args = f"BUILD_DIR={build_dir.absolute()} ARCH=dummy BOARD=dummy SEL4_SDK=dummy TARGET_TRIPLE=dummy LLVM=False LINK_ADDRESS=0" + + r = system( + f"make -C loader tests {make_args}" + ) + if r != 0: + raise Exception(f"Tests failed: loader") + + # We don't pass CLIPPYARGS, so this is warning-only + r = system( + f"make -C loader clippy {make_args}" + ) + if r != 0: + raise Exception(f"Clippy failed: loader") + + def build_elf_component( component_name: str, sdk_dir: Path, @@ -1094,6 +1114,8 @@ def main() -> None: if not args.skip_run_time: build_dir = Path("build") + test_loader(build_dir) + for (board, configs) in build_goals: for config in configs: if not args.skip_sel4: diff --git a/loader/Makefile b/loader/Makefile index c08c18bd9..d106e3261 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -35,27 +35,34 @@ else LD = $(TARGET_TRIPLE)-ld endif +RUSTC := rustc +CLIPPY := clippy-driver + ifeq ($(ARCH),aarch64) CFLAGS_AARCH64 := -mcpu=$(GCC_CPU) -mgeneral-regs-only -mstrict-align -mno-outline-atomics CFLAGS_ARCH := $(CFLAGS_AARCH64) -DARCH_aarch64 ASM_FLAGS_ARCH := -mcpu=$(GCC_CPU) ARCH_DIR := aarch64 + RUST_TARGET_TRIPLE := aarch64-unknown-none else ifeq ($(ARCH),riscv64) CFLAGS_RISCV64 := -mcmodel=medany -march=rv64imac_zicsr_zifencei -mabi=lp64 CFLAGS_ARCH := $(CFLAGS_RISCV64) -DARCH_riscv64 ASM_FLAGS_ARCH := -march=rv64imac_zicsr_zifencei -mabi=lp64 + RUST_TARGET_TRIPLE := riscv64gc-unknown-none-elf ARCH_DIR := riscv endif CFLAGS := -std=gnu11 -g -O3 -nostdlib -ffreestanding \ -MP -MD $(CFLAGS_ARCH) -DBOARD_$(BOARD) -I$(SEL4_SDK)/include \ -Wall -Werror -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations \ - -Wundef -Wno-nonnull -Wnested-externs + -Wundef -Wno-nonnull -Wnested-externs -ffunction-sections -fdata-sections ASM_FLAGS := $(ASM_FLAGS_ARCH) -g -MP -MD -I$(SEL4_SDK)/include +RUSTFLAGS := --edition 2024 -g -C opt-level=2 + PROGS := loader.elf -OBJECTS := loader.o crt0.o uart.o cutil.o +OBJECTS := loader.o crt0.o uart.o cutil.o libpage_tables.a ifeq ($(ARCH),aarch64) OBJECTS += util64.o el.o exceptions.o init.o mmu.o cpus.o @@ -80,6 +87,18 @@ $(BUILD_DIR)/%.o : src/$(ARCH_DIR)/%.c $(BUILD_DIR)/%.o : src/%.c $(CC) -c $(CFLAGS) $< -o $@ +# Note: having multiple libs with staticlib will give duplicate linker symbol +# issues. Use "--crate-type rlib" instead, but then we need to link a single +# copy of the rust corelibs. For now this is fine. +$(BUILD_DIR)/lib%.a : src/%.rs + $(RUSTC) $(RUSTFLAGS) \ + --emit dep-info,metadata,link \ + --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ + --target $(RUST_TARGET_TRIPLE) \ + --crate-type staticlib \ + --crate-name $(patsubst lib%.a,%,$(notdir $@)) \ + $< + -include $(BUILD_DIR)/*.d OBJPROG = $(addprefix $(BUILD_DIR)/, $(PROGS)) @@ -89,5 +108,34 @@ all: $(OBJPROG) $(LINKSCRIPT): $(LINKSCRIPT_INPUT) $(CPP) -DLINK_ADDRESS=$(LINK_ADDRESS) $< | grep -v "^#" > $@ +LDFLAGS := -T$(LINKSCRIPT) --gc-sections + $(OBJPROG): $(addprefix $(BUILD_DIR)/, $(OBJECTS)) $(LINKSCRIPT) - $(LD) -T$(LINKSCRIPT) $(addprefix $(BUILD_DIR)/, $(OBJECTS)) -o $@ + $(LD) $(LDFLAGS) --start-group $(addprefix $(BUILD_DIR)/, $(OBJECTS)) --end-group -o $@ + +rusttest_%: src/%.rs + $(RUSTC) $(RUSTFLAGS) \ + --emit dep-info,metadata,link \ + --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ + -Awarnings \ + --test \ + --crate-name "$@" \ + $< + +tests: rusttest_page_tables + $(BUILD_DIR)/rusttest_page_tables + + +CLIPPYFLAGS ?= + +rustclippy_%: src/%.rs + $(CLIPPY) $(RUSTFLAGS) \ + --emit dep-info,metadata,link \ + --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ + $(CLIPPYFLAGS) \ + -Cpanic=abort \ + --crate-type staticlib \ + --crate-name "$@" \ + $< + +clippy: rustclippy_page_tables diff --git a/loader/aarch64.ld b/loader/aarch64.ld index 977ccc574..2454398a3 100644 --- a/loader/aarch64.ld +++ b/loader/aarch64.ld @@ -8,6 +8,12 @@ PHDRS all PT_LOAD AT (LINK_ADDRESS); } + +// text PT_LOAD FLAGS(5); /* RX */ +// rodata PT_LOAD FLAGS(4); /* RO */ +// data PT_LOAD FLAGS(6); /* RW */ +// bss PT_LOAD FLAGS(6); /* RW */ + SECTIONS { . = LINK_ADDRESS; @@ -17,16 +23,26 @@ SECTIONS .text : { _text = .; - *(.text.start) - *(.text*) - *(.rodata) + + KEEP(*(.text.start)) + *(.text .text.*) + _text_end = .; } :all + .rodata : + { + *(.rodata .rodata.* .rodata..Lanon.*) + } :all + .data : { _data = .; - *(.data) + *(.data .data.*) + *(.data.*) + + KEEP(*(.data.uart_addr)) + _data_end = .; } :all @@ -34,10 +50,13 @@ SECTIONS { _bss = .; *(.bss) + *(.bss.*) *(COMMON) . = ALIGN(4); _bss_end = .; } :all + + _loader_end = .; } diff --git a/loader/riscv64.ld b/loader/riscv64.ld index f7ae1240e..fe98d7450 100644 --- a/loader/riscv64.ld +++ b/loader/riscv64.ld @@ -15,9 +15,11 @@ SECTIONS .text : { _text = .; - *(.text.start) + KEEP(*(.text.start)) *(.text*) + *(.text.*) *(.rodata) + *(.rodata.*) _text_end = .; } :all @@ -25,17 +27,20 @@ SECTIONS { _data = .; *(.data) + *(.data.*) __global_pointer$ = . + 0x800; - *(.srodata) - *(.sdata) + *(.srodata) + *(.sdata) + KEEP(*(.data.uart_addr)) _data_end = .; } :all .bss : { _bss = .; - *(.sbss) + *(.sbss) *(.bss) + *(.bss.*) *(COMMON) . = ALIGN(4); _bss_end = .; diff --git a/loader/src/aarch64/mmu.c b/loader/src/aarch64/mmu.c index 8ef6427ce..75ceba86b 100644 --- a/loader/src/aarch64/mmu.c +++ b/loader/src/aarch64/mmu.c @@ -6,27 +6,56 @@ */ #include +#include #include "el.h" #include "../arch.h" #include "../cutil.h" #include "../uart.h" -void el1_mmu_enable(void); -void el2_mmu_enable(void); +void el1_mmu_enable(uint64_t ttbr0_el1, uint64_t ttbr1_el1); +void el2_mmu_enable(uint64_t ttbr0_el2); -/* Paging structures for kernel mapping */ -uint64_t boot_lvl0_upper[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl1_upper[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl2_upper[1 << 9] ALIGN(1 << 12); +struct AArch64ReturnValue { + uintptr_t ttbr0_el2; + uintptr_t ttbr0_el1; + uintptr_t ttbr1_el1; +}; -/* Paging structures for identity mapping */ -uint64_t boot_lvl0_lower[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl1_lower[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl2_lower[1 << 9] ALIGN(1 << 12); +union RegionArchAttrs { + bool is_ram; + uint64_t raw; +}; + +struct Region { + uint64_t start; + uint64_t end; + union RegionArchAttrs arch_attrs; +}; + +struct Region regions[] = { + { .start = 0x60000000, .end = 0xc0000000, .arch_attrs.is_ram = true }, + { .start = 0x9000000, .end = 0x9000000 + 4096, .arch_attrs.is_ram = false }, +}; + +#define PAGE_TABLE_SIZE 4096 +#define MAX_NUM_PAGE_TABLES 64 + +uint8_t page_table_bytes[PAGE_TABLE_SIZE][MAX_NUM_PAGE_TABLES] ALIGN(4096); + +extern struct AArch64ReturnValue aarch64_setup_pagetables( + uint64_t kernel_first_vaddr, uint64_t kernel_first_paddr, + void *regions_ptr, uintptr_t regions_len, + uint8_t page_table_bytes[4096][64]); int arch_mmu_enable(int logical_cpu) { + struct AArch64ReturnValue pt = aarch64_setup_pagetables( + 0x8060000000, 0x60000000, + ®ions, ARRAY_SIZE(regions), + page_table_bytes + ); + int r; enum el el; r = ensure_correct_el(logical_cpu); @@ -37,9 +66,9 @@ int arch_mmu_enable(int logical_cpu) LDR_PRINT("INFO", logical_cpu, "enabling MMU\n"); el = current_el(); if (el == EL1) { - el1_mmu_enable(); + el1_mmu_enable(pt.ttbr0_el1, pt.ttbr1_el1); } else if (el == EL2) { - el2_mmu_enable(); + el2_mmu_enable(pt.ttbr0_el2); } else { LDR_PRINT("ERROR", logical_cpu, "unknown EL for MMU enable\n"); } diff --git a/loader/src/aarch64/util64.S b/loader/src/aarch64/util64.S index ccf1889b0..fd94ebe36 100644 --- a/loader/src/aarch64/util64.S +++ b/loader/src/aarch64/util64.S @@ -308,7 +308,6 @@ END_FUNC(el1_mmu_disable) BEGIN_FUNC(el2_mmu_disable) stp x29, x30, [sp, #-16]! - stp x27, x28, [sp, #-16]! mov x29, sp /* Disable caches */ @@ -323,14 +322,18 @@ BEGIN_FUNC(el2_mmu_disable) */ bl invalidate_icache - ldp x27, x28, [sp], #16 ldp x29, x30, [sp], #16 ret END_FUNC(el2_mmu_disable) +/* + * Enables the MMU for EL2. + * Takes two arguments the physical address for TTBR0_EL1 (x0) and TTBR1_EL1 (x1). + */ BEGIN_FUNC(el1_mmu_enable) stp x29, x30, [sp, #-16]! stp x27, x28, [sp, #-16]! + /* move caller-saved to callee-saved registers */ mov x29, sp mov x27, x0 mov x28, x1 @@ -358,10 +361,8 @@ BEGIN_FUNC(el1_mmu_enable) msr tcr_el1, x10 /* Setup page tables */ - adrp x8, boot_lvl0_lower - msr ttbr0_el1, x8 - adrp x8, boot_lvl0_upper - msr ttbr1_el1, x8 + msr ttbr0_el1, x27 /* argument 0 */ + msr ttbr1_el1, x28 /* argument 1 */ isb /* invalidate all TLB entries for EL1 */ @@ -374,12 +375,18 @@ BEGIN_FUNC(el1_mmu_enable) ldp x27, x28, [sp], #16 ldp x29, x30, [sp], #16 ret - END_FUNC(el1_mmu_enable) +/* + * Enables the MMU for EL2. + * Takes one argument, the physical address for TTBR0_EL2 (x0). + */ BEGIN_FUNC(el2_mmu_enable) stp x29, x30, [sp, #-16]! + stp x27, x28, [sp, #-16]! + /* move caller-saved to callee-saved registers */ mov x29, sp + mov x28, x0 /* Disable the MMU */ bl el2_mmu_disable @@ -403,8 +410,7 @@ BEGIN_FUNC(el2_mmu_enable) isb /* Setup page tables */ - adrp x8, boot_lvl0_lower - msr ttbr0_el2, x8 + msr ttbr0_el2, x28 /* argument 0 */ isb /* invalidate all TLB entries for EL2 */ @@ -423,9 +429,9 @@ BEGIN_FUNC(el2_mmu_enable) dsb ish isb + ldp x27, x28, [sp], #16 ldp x29, x30, [sp], #16 ret - END_FUNC(el2_mmu_enable) .extern arm_secondary_cpu_c_entry diff --git a/loader/src/c_interop.rs b/loader/src/c_interop.rs new file mode 100644 index 000000000..526ec80f0 --- /dev/null +++ b/loader/src/c_interop.rs @@ -0,0 +1,73 @@ +#[cfg(not(test))] +mod real_hardware { + use core::ffi::c_char; + use core::ffi::CStr; + use core::fmt; + use core::fmt::Write; + use core::panic::PanicInfo; + + unsafe extern "C" { + safe fn fail() -> !; + // safe fn putc(c: c_char); + unsafe fn puts(s: *const c_char); + } + + /// Exposed only for print macro. + #[doc(hidden)] + pub(crate) struct Writer; + + impl fmt::Write for Writer { + fn write_str(&mut self, s: &str) -> Result<(), fmt::Error> { + for c in s.bytes() { + unsafe { puts(CStr::from_bytes_with_nul_unchecked(&[c, 0]).as_ptr()) }; + } + Ok(()) + } + } + + #[macro_export] + macro_rules! __print { + ($($arg:tt)*) => {{ + use core::fmt::Write; + $crate::c_interop::Writer{}.write_fmt(format_args!($($arg)*)).unwrap() + }} + } + + #[macro_export] + macro_rules! __println { + () => {{ + $crate::__print!("\n"); + }}; + + ($($arg:tt)*) => {{ + $crate::__print!("{}\n", format_args!($($arg)*)); + }} + } + + pub(crate) use __println as println; + + #[panic_handler] + fn panic(info: &PanicInfo) -> ! { + println!("panicked"); + + if writeln!(Writer, "{}", info).is_err() { + // If writeln!() fails (which it should never as our fmt::Write) never + // fails, then just don't print the extra information. + println!("panicked (information unknown)"); + } + + fail(); + } +} + +#[cfg(test)] +mod for_tests { + extern crate std; + pub(crate) use std::println; +} + +#[cfg(test)] +pub(crate) use for_tests::*; + +#[cfg(not(test))] +pub(crate) use real_hardware::*; diff --git a/loader/src/loader.c b/loader/src/loader.c index 67c09b149..45a21de04 100644 --- a/loader/src/loader.c +++ b/loader/src/loader.c @@ -100,7 +100,6 @@ static int print_lock = 0; void start_kernel(int logical_cpu) { - LDR_PRINT("INFO", logical_cpu, "enabling MMU\n"); int r = arch_mmu_enable(logical_cpu); if (r != 0) { LDR_PRINT("ERROR", logical_cpu, "failed to enable MMU: "); diff --git a/loader/src/loader.h b/loader/src/loader.h index c144381aa..d1a0e79d3 100644 --- a/loader/src/loader.h +++ b/loader/src/loader.h @@ -7,7 +7,7 @@ #pragma once -#define STACK_SIZE 4096 +#define STACK_SIZE 40960 #define REGION_TYPE_DATA 1 #define REGION_TYPE_ZERO 2 diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs new file mode 100644 index 000000000..6322e8323 --- /dev/null +++ b/loader/src/page_tables.rs @@ -0,0 +1,1136 @@ +// +// Copyright 2026, UNSW +// +// SPDX-License-Identifier: BSD-2-Clause +// + +#![no_std] + +// We prefer indices as it matches the semantics of PT indices +#![allow(clippy::needless_range_loop)] + +mod c_interop; + +use core::cmp::min; +use core::fmt; +use core::mem; +use core::mem::MaybeUninit; +use core::slice; + +use c_interop::println; + +const PAGE_TABLE_SIZE: usize = 4096; + +const fn divmod(x: u64, y: u64) -> (u64, u64) { + (x / y, x % y) +} + +const fn mask(n: u64) -> u64 { + (1 << n) - 1 +} + +const fn round_down(n: u64, x: u64) -> u64 { + let (_, m) = divmod(n, x); + if m == 0 { + n + } else { + n - m + } +} + +const fn align_down(n: u64, bits: u64) -> u64 { + round_down(n, 1 << bits) +} + +unsafe extern "C" { + static mut _text: u8; +} + +pub mod aarch64 { + //! For AArch64, our page tables use the Stage 1 descriptor formats + //! for both EL2 (TTBR0_EL2) and EL1 (TTBR0_EL1/TTBR1_EL1). + //! Stage 2 descriptors are only used when in the EL1&0 regime; which is not + //! the case when in EL2. + + use super::*; + + pub const LVL0_BITS: u64 = 9; + pub const LVL1_BITS: u64 = 9; + pub const LVL2_BITS: u64 = 9; + pub const LVL3_BITS: u64 = 9; + + pub fn lvl0_index(addr: u64) -> usize { + let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS + LVL1_BITS)) & mask(LVL0_BITS); + idx as usize + } + + pub fn lvl1_index(addr: u64) -> usize { + let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS)) & mask(LVL1_BITS); + idx as usize + } + + pub fn lvl2_index(addr: u64) -> usize { + let idx = (addr >> (BLOCK_BITS_2MB)) & mask(LVL2_BITS); + idx as usize + } + + pub fn lvl3_index(addr: u64) -> usize { + let idx = (addr >> PAGE_BITS_4KB) & mask(LVL3_BITS); + idx as usize + } + + /// Stage 1 translation table page/block descriptors have bits[4:2] containing + /// AttrIndex[2:0]. The AttrIndex values depends on our configuration of + /// the `MAIR_EL1` or `MAIR_EL2` registers done in util64.S; + /// This also needs to match the values that seL4 uses. + #[allow(non_upper_case_globals, reason = "matching ARM naming convention")] + pub mod s1_mair_attr_index { + pub const MT_DEVICE_nGnRnE: u64 = 0b000; + pub const MT_DEVICE_nGnRE: u64 = 0b001; + pub const MT_DEVICE_GRE: u64 = 0b010; + pub const MT_NORMAL_NC: u64 = 0b011; + pub const MT_NORMAL: u64 = 0b100; + } + + pub mod descriptor_type { + //! The translation table descriptor formats, as per §D8.3 "Translation + //! table descriptor formats" of ARM DDI 0487 L.b. Specifically, + //! as per "Table D8-48 Determination of descriptor type" + + /// Descriptor type: Table. Condition is lookup level != 3. + pub const TABLE: u64 = 0b11; + /// Descriptor type: Page. Condition is lookup level == 3. + pub const PAGE: u64 = 0b11; + /// Descriptor type: Block. Condition is lookup level != 3. + pub const BLOCK: u64 = 0b01; + /// Descriptor type: Invalid. Strictly speaking bit[1] does not matter. + pub const INVALID: u64 = 0b00; + } + + pub mod shareability_attributes { + //! Per §D8.6.2 "Stage 1 Shareability attributes", these contain the + //! shareability attributes of the descriptor OA for normal-cacheable + //! memory. + + /// Non-shareable + pub const NON_SHAREABLE: u64 = 0b00; + /// Outer-shareable + pub const OUTER_SHAREABLE: u64 = 0b10; + /// Inner-shareable + pub const INNER_SHAREABLE: u64 = 0b11; + } + + /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, + /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address + /// is bits [47:n], and: + /// + /// > For the 4KB granule size, the level 1 descriptor n is 30, + /// > and the level 2 descriptor n is 21. + pub const BLOCK_BITS_1GB: u64 = 30; + + /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, + /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address + /// is bits [47:n], and: + /// + /// > For the 4KB granule size, the level 1 descriptor n is 30, + /// > and the level 2 descriptor n is 21. + pub const BLOCK_BITS_2MB: u64 = 21; + + // TODO: + + pub const BLOCK_BITS_512GB: u64 = 39; + pub const PAGE_BITS_4KB: u64 = 12; + + /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and + /// "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b; + /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" + pub fn block_descriptor(level: usize, addr: u64, attr_index: u64) -> u64 { + // Per Table D8-48, Condition for descriptor_type::BLOCK is level != 3. + assert!(level != 3); + + let upper_attributes: u64 = 0; + + let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { + // Match what the seL4 kernel uses for its page tables, which + // is especially necessary for SMP booting which relies on it + // for coherency. See the comment in seL4 `release_secondary_cpus()`. + shareability_attributes::INNER_SHAREABLE + } else { + // Per $R_{PYFVQ}$: + // > If a region is mapped as Device memory or Normal Non-cacheable + // > memory after all enabled translation stages, then the region + // > has an effective Shareability attribute of Outer Shareable. + // + // We override the value we place in here to OUTER_SHAREABLE to match + // how the hardware behaves. This is not necessary but for clarity. + shareability_attributes::OUTER_SHAREABLE + }; + + // AP[2:1], which we set as 0b00 for read/write access: + // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1 + // stage 2: 0b00 is RW for EL2 and no perms for EL1. + const AP_KERNEL_RW: u64 = 0b00; + + // bit[11] is the not global (nG) field, we leave as 0 (global). + // bit[10] is the access flag; depending on FEAT_HAFDBS, when software + // manages the AF memory accesses to the page/block when AF=0 + // raise an Access Fault; when hardware manages the AF it will + // become 1. + // bit[9:8] is SH[1:0] containing stage 1 shareability attributes + // bit[7:6] contains AP[2:1] + // bit[5] is RES0 + // bit[4:2] contains AttrIndex + let lower_attributes: u64 = + (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); + + // bits[47:n] + let output_address: u64 = addr + & !mask(match level { + 1 => BLOCK_BITS_1GB, + 2 => BLOCK_BITS_2MB, + _ => panic!("unsupported level {level} for block descriptor"), + }); + + // address must not have bits above 47 set. + assert!(addr & mask(48) == addr); + + // bits[63:50] describing the "Upper attributes" are left at 0. + // bits[49:48] are RES0 + // bits[47:n] contain the Output address + // bits[n-1:12] are RES0 + // bits[11:2] contain the "Lower attributes" + // bits[1:0] contains the descriptor type + upper_attributes | output_address | lower_attributes | descriptor_type::BLOCK + } + + /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and + /// "Figure D8-15 VMSAv8-64 Page descriptor formats" of ARM DDI0487L.b; + /// specifically subfigure "4KB granule 48-bit OA". + pub fn page_descriptor(addr: u64, attr_index: u64) -> u64 { + // The main difference between a page descriptor and block descriptor + // is in the size of the output address (OA) and in the descriptor type. + + let upper_attributes: u64 = 0; + + let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { + // Match what the seL4 kernel uses for its page tables, which + // is especially necessary for SMP booting which relies on it + // for coherency. + shareability_attributes::INNER_SHAREABLE + } else { + // Per $R_{PYFVQ}$: + // > If a region is mapped as Device memory or Normal Non-cacheable + // > memory after all enabled translation stages, then the region + // > has an effective Shareability attribute of Outer Shareable. + // We override the value we place in here to OUTER_SHAREABLE to match + // how the hardware behaves. + shareability_attributes::OUTER_SHAREABLE + }; + + // AP[2:1], which we set as 0b00 for read/write access: + // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1/El2 (priv) + const AP_KERNEL_RW: u64 = 0b00; + + // bit[11] is the not global (nG) field, we leave as 0 (global). + // bit[10] is the access flag; depending on FEAT_HAFDBS, when software + // manages the AF memory accesses to the page/block when AF=0 + // raise an Access Fault; when hardware manages the AF it will + // become 1. + // bit[9:8] is SH[1:0] containing stage 1 shareability attributes + // bit[7:6] contains AP[2:1] + // bit[5] is RES0 + // bit[4:2] contains AttrIndex + let lower_attributes: u64 = + (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); + + // bits[47:12] + let output_address: u64 = addr & !mask(12); + + // address must not have bits above 47 set. + assert!(addr & mask(48) == addr); + + // bits[63:50] describing the "Upper attributes" are left at 0. + // bits[49:48] are RES0 + // bits[47:12] contain the Output address + // bits[11:2] contain the "Lower attributes" + // bits[1:0] contains the descriptor type + upper_attributes | output_address | lower_attributes | descriptor_type::PAGE + } + + /// Per "Table D8-50 Stage 1 VMSAv8-64 Table descriptor fields" and + /// "Figure D8-12 VMSAv8-64 Table descriptor formats" of ARM DDI0487L.b; + /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" + pub fn table_descriptor(addr: *const u8) -> u64 { + // Per Table D8-48, Condition for descriptor_type::TABLE is level != 3. + + let addr: u64 = addr.addr().try_into().expect("usize in u64"); + + // We don't set any of these attributes, most are hardware-feature conditional + let attributes: u64 = 0; + + // address must not have bits above 47 or below 12 set + assert!(addr & mask(12) == 0x0); + assert!(addr & mask(48) == addr); + + let next_level_table_address = addr; + + // bits[63:59] are "Attributes" + // bits[58:51] are ignored + // bits[50:48] are RES0 + // bits[47:m] is the next-level table address + // note: here m=12 for 4KB granule + // bits[m-1:12] are RES0 + // so this doesn't exist for 4KB granule + // bits[11:2] are ignored + // bits[1:0] contain the descriptor type + attributes | next_level_table_address | descriptor_type::TABLE + } +} + +mod riscv64 { + pub(crate) const BLOCK_BITS_1GB: u64 = 30; + pub(crate) const BLOCK_BITS_2MB: u64 = 21; + pub(crate) const PAGE_BITS_4K: u64 = 12; + + pub(crate) const PAGE_TABLE_INDEX_BITS: u64 = 9; + pub(crate) const PAGE_SHIFT: u64 = 12; + /// This sets the page table entry bits: D,A,X,W,R. + pub(crate) const PTE_TYPE_BITS: u64 = 0b11001110; + // TODO: where does this come from? + pub(crate) const PTE_TYPE_TABLE: u64 = 0; + pub(crate) const PTE_TYPE_VALID: u64 = 1; + + pub(crate) const PTE_PPN0_SHIFT: u64 = 10; + + /// Due to RISC-V having various virtual memory setups, we have this generic function to + /// figure out the page-table index given the total number of page table levels for the + /// platform and which level we are currently looking at. + pub fn pt_index(pt_levels: usize, addr: u64, level: usize) -> usize { + let pt_index_bits = PAGE_TABLE_INDEX_BITS * (pt_levels - level) as u64; + let idx = (addr >> (pt_index_bits + PAGE_SHIFT)) % 512; + + idx as usize + } + + /// Generate physical page number given an address + pub fn pte_ppn(addr: u64) -> u64 { + (addr >> PAGE_SHIFT) << PTE_PPN0_SHIFT + } + + pub fn pte_next(addr: u64) -> u64 { + pte_ppn(addr) | PTE_TYPE_TABLE | PTE_TYPE_VALID + } + + pub fn pte_leaf(addr: u64) -> u64 { + pte_ppn(addr) | PTE_TYPE_BITS | PTE_TYPE_VALID + } +} + +/// RISC-V 64 page tables for our purposes uses the Sv39 translation scheme +/// (3-level page tables). +/// +/// It is split into two halves: the Upper/Kernel part of the page tables, +/// which matches the format seL4 expects. The lower half contains an +/// identity mapped region for the loader. +/// +/// ```txt +/// (512 GiB) +/// 512 +---- Level 1 ---+ 2^39 +/// | | +/// | (empty) | +/// | | +/// k+1 +----------------+ (1 GiB) +/// | Level 2 Kernel | ----------> +---- Level 2 ---+ +-------------+ +/// k +----------------+ | | ----------> | 2 MiB block | +/// | | 511 |----------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | 510 |----------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | |----------------| +------------- +/// | | (...) (...) (...) Kernel Regions +/// | | |----------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | l+1 |----------------| +-------------+ +/// | | | Level 3 Kernel | ----+ +/// | | l |----------------| | +/// | | | | | (2 MiB) +/// | | | | +-----> +-- Level 3 --+ +------------+ +/// | | | | | | ----------> | 4 KiB page | +/// | | | | 511 |-------------| +------------+ +/// | | | (empty) | | | ----------> | 4 KiB page | +/// | (empty) | | | |-------------| +------------+ +/// | | | | | | ----------> | 4 KiB page | +/// | | | | m |-------------| +------------+ p +/// | | | | | (empty) | +/// | | | | +-------------+ +/// | | | | +/// | | 0 +----------------+ +/// | | +/// | | +/// | | +/// | | +/// | | +/// s+1 +----------------+ (1 GiB) +/// | Level 2 Loader | ----------> +-- Level 2 --+ +-------------+ +/// s +----------------+ | | ----------> | 2 MiB block | +/// | | 511 +-------------+ +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | (empty) | 510 +-------------+ +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | |-------------| +-------------+ +/// 0 +----------------+ | | ----------> | 2 MiB block | +/// |-------------| +-------------+ +/// (...) (...) (...) Loader Regions +/// |-------------| +-------------+ +/// | | ----------> | 2 MiB block | +/// |-------------| +-------------+ +/// | | ----------> | 2 MiB block | +/// t +-------------+ +-------------+ +/// | | +/// | (empty) | +/// | | +/// +-------------+ +/// +/// +/// Where: +/// k = align_down(kernel_first_vaddr, 1GiB), +/// l = align_down(kernel_first_vaddr, 2MiB), +/// m = align_down(kernel_first_vaddr, 4KiB), +/// p = align_down(kernel_first_paddr, 4KiB), +/// +/// s = align_down(text_addr, 1GiB), +/// t = align_down(text_addr, 2MiB), +/// ``` +/// +#[unsafe(no_mangle)] +pub extern "C" fn riscv64_setup_pagetables( + kernel_first_vaddr: u64, + kernel_first_paddr: u64, + page_tables_paddr_start: u64, +) -> u64 { + use riscv64::{pt_index, pte_leaf, pte_next, BLOCK_BITS_1GB, BLOCK_BITS_2MB, PAGE_BITS_4K}; + + let text_addr = &raw const _text as u64; + + // We map the loader using 2MB pages, so make sure the base is actually aligned. + assert!(text_addr.is_multiple_of(1 << BLOCK_BITS_2MB)); + + const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); + + let mut serialise_page_table_to_paddr = { + assert!( + page_tables_paddr_start + == page_tables_paddr_start.next_multiple_of(PAGE_TABLE_SIZE as u64) + ); + + // This maintains the current end of the PT array. + let mut next_pt_paddr = page_tables_paddr_start; + + move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> u64 { + let pt_paddr = next_pt_paddr; + // page_table_bytes.extend(page_table.iter().flat_map(|pte| pte.to_le_bytes())); + next_pt_paddr += PAGE_TABLE_SIZE as u64; + page_table.fill(0); + pt_paddr + } + }; + + struct Config { + riscv_pt_levels: usize, + } + let config = Config { riscv_pt_levels: 3 }; + + let num_pt_levels = config.riscv_pt_levels; + assert!(num_pt_levels == 3); + + // Manufacture the constants as per the diagram. + let k = align_down(kernel_first_vaddr, BLOCK_BITS_1GB); + let l = align_down(kernel_first_vaddr, BLOCK_BITS_2MB); + let m = align_down(kernel_first_vaddr, PAGE_BITS_4K); + let p = align_down(kernel_first_paddr, PAGE_BITS_4K); + + let s = align_down(text_addr, BLOCK_BITS_1GB); + let t = align_down(text_addr, BLOCK_BITS_2MB); + + // Manufacture the kernel page tables + let kernel_lvl2_pt_paddr = { + let mut lvl2_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + + let mut paddr = p; + let index_l = pt_index(num_pt_levels, l, 2); + + lvl2_pt_kernel[index_l] = if kernel_first_vaddr.is_multiple_of(1 << BLOCK_BITS_2MB) { + assert!(paddr.is_multiple_of(1 << BLOCK_BITS_2MB)); + let pte = pte_leaf(paddr); + paddr += 1 << BLOCK_BITS_2MB; + pte + } else { + let mut lvl3_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + + let index_m = pt_index(num_pt_levels, m, 3); + + for index in index_m..512 { + lvl3_pt_kernel[index] = pte_leaf(paddr); + paddr += 1 << PAGE_BITS_4K; + } + + let kernel_lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt_kernel); + pte_next(kernel_lvl3_pt_paddr) + }; + + for index in (index_l + 1)..512 { + lvl2_pt_kernel[index] = pte_leaf(paddr); + paddr += 1 << BLOCK_BITS_2MB; + } + + serialise_page_table_to_paddr(&mut lvl2_pt_kernel) + }; + + // Manufacture the loader page tables, which is relatively straightforward + let loader_lvl2_pt_paddr = { + let mut lvl2_pt_loader = [0u64; PAGE_TABLE_ENTRIES]; + + // Identity mapped, so vaddr == paddr. + let mut paddr = t; + + for index in pt_index(num_pt_levels, t, 2)..512 { + lvl2_pt_loader[index] = pte_leaf(paddr); + paddr += 1 << BLOCK_BITS_2MB; + } + + serialise_page_table_to_paddr(&mut lvl2_pt_loader) + }; + + // Manufacture the Level 1 table + let mut boot_lvl1_pt = [0u64; PAGE_TABLE_ENTRIES]; + + let index_s = pt_index(num_pt_levels, s, 1); + let index_k = pt_index(num_pt_levels, k, 1); + boot_lvl1_pt[index_k] = pte_next(kernel_lvl2_pt_paddr); + boot_lvl1_pt[index_s] = pte_next(loader_lvl2_pt_paddr); + + serialise_page_table_to_paddr(&mut boot_lvl1_pt) +} + +/// Note that "0" is a valid return value; instead the invalid value is +/// '-1', or usize::MAX. +#[repr(C)] +#[derive(Debug)] +pub struct AArch64ReturnValue { + pub ttbr0_el2: *const u8, + pub ttbr0_el1: *const u8, + pub ttbr1_el1: *const u8, +} + +impl AArch64ReturnValue { + const INVALID: *const u8 = usize::MAX as *const _; +} + +#[derive(Copy, Clone)] +#[repr(C)] +pub union RegionArchAttrs { + pub is_ram: bool, + pub raw: u64, +} + +impl fmt::Debug for RegionArchAttrs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + f.debug_struct("RegionArchAttrs") + // SAFETY: raw contains all valid bitpatterns + .field("raw", unsafe { &self.raw }) + .finish() + } +} + +#[derive(Debug, Copy, Clone)] +#[repr(C)] +pub struct Region { + pub start: u64, + pub end: u64, + pub arch_attrs: RegionArchAttrs, +} + +pub const MAX_NUM_PAGE_TABLES: usize = 64; + +/// AArch64 loader page tables have two variations: +/// - Loader in EL2, then Stage 1 translations in use, so we have the +/// singular TTBR0_EL2 register containing the Level 0 table; +/// this allows virtual address in the range [0,2^48). +/// - Loader in EL1, then Stage 1 translations are in use, so we have both +/// the TTBR0_EL1 (covering vaddr in range [0,2^48)) and TTBR1_EL2 ( +/// (covering vaddr in the range [2^64-2^48,2^64)), and containing +/// the "Level 0 Lower" page table, and "Level 0 Upper" page table +/// physical addresses respectively. +/// +/// Thus, for EL2 loader, the singular Level 0 page table contains the table +/// descriptors for the "Level 1 Upper" and "Level 1 Lower" page tables. +/// For the EL1 loader, we instead have two Level 0 page tables, and +/// "Level 0 Lower" contains the "Level 1 Lower" descriptor, and "Level 0 +/// Upper" contains the "Level 1 Upper" descriptor. +/// Otherwise, the page tables layout from Level 1 downwards are identical +/// (but not necessarily the layout within the page/table/block descriptors). +/// +/// ```txt +/// (256 TiB) +/// 512 +-- Level 0 --+ 2^48 +/// | | +/// | (empty) | +/// | | +/// k+1 +-------------+ (512 GiB) +/// | Level 1 Upr | ----------> +-- Level 1 --+ +/// k +-------------+ | | +/// | | | (empty) | +/// | | | | +/// | | l+1 +-------------+ (1 GiB) +/// | | | Level 2 Upr | ----------> +-- Level 2 --+ +-------------+ +/// | | l +-------------+ | | ----------> | 2 MiB block | +/// | | | | 511 |-------------| +-------------+ +/// | | | (empty) | | | ----------> | 2 MiB block | +/// | | | | 510 |-------------| +-------------+ +/// | | +-------------+ | | ----------> | 2 MiB block | +/// | | |-------------| +-------------+ +/// | (empty) | Kernel Regions (...) (...) (...) +/// | | |-------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | m |-------------| +-------------+ p +/// | | | | +/// | | | (empty) | +/// | | | | +/// | | 0 +-------------+ +/// | | +/// | | +/// | | +/// 1 +-------------+ (512 GiB) +/// | Level 1 Lwr | ----------> +-- Level 1 --+ +/// 0 +-------------+ TODO: RAM. +/// +/// +/// Where: +/// k = align_down(kernel_first_vaddr, 512GiB), +/// l = align_down(kernel_first_vaddr, 1GiB), +/// m = align_down(kernel_first_vaddr, 2MiB), +/// p = align_down(kernel_first_paddr, 2MiB), +/// u = align_down(uart_base, 1GiB), +/// ``` +/// +/// # Safety +/// - regions_ptr must be valid for as long as this function runs, +/// and regions_len must repsent its length +/// +#[unsafe(no_mangle)] +pub unsafe extern "C" fn aarch64_setup_pagetables( + kernel_first_vaddr: u64, + kernel_first_paddr: u64, + // In-out param; storage and input + regions_ptr: *mut Region, + regions_len: usize, + // Storage used for page tables + page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], +) -> AArch64ReturnValue { + use aarch64::{ + block_descriptor, lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, + s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}, + table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, + }; + + const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); + + let mut serialise_page_table_to_paddr = { + let page_tables_paddr_start: *const u8 = page_table_bytes.as_ptr().cast(); + + assert!((page_tables_paddr_start as usize).is_multiple_of(PAGE_TABLE_SIZE)); + + // This maintains the current end of the PT array. + let mut next_pt_paddr = page_tables_paddr_start; + let mut i = 0; + + move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> *const _ { + let pt_paddr = next_pt_paddr; + for (j, byte) in page_table + .iter() + .flat_map(|pte| pte.to_le_bytes()) + .enumerate() + { + page_table_bytes[i][j].write(byte); + } + + next_pt_paddr = next_pt_paddr.wrapping_add(PAGE_TABLE_SIZE); + i += 1; + page_table.fill(0); + pt_paddr + } + }; + + let identity_mapped_regions: &mut [Region] = { + let regions = unsafe { slice::from_raw_parts_mut(regions_ptr, regions_len) }; + + println!("{:#x?}", regions); + + for region in regions.iter_mut() { + // SAFETY: We expect users to set is_ram appropriately. + region.arch_attrs.raw = if unsafe { region.arch_attrs.is_ram } { + // FIXME: For now, RAM is also mapped as DEVICE memory. + MT_DEVICE_nGnRnE + } else { + MT_DEVICE_nGnRnE + }; + } + + // Need to use 'sort_unstable_by_key' as sort_by_key is not in-place. + regions.sort_unstable_by_key(|region| region.start); + + regions + }; + + // Manufacture the constants as per the diagram. + let k = align_down(kernel_first_vaddr, BLOCK_BITS_512GB); + let l = align_down(kernel_first_vaddr, BLOCK_BITS_1GB); + let m = align_down(kernel_first_vaddr, BLOCK_BITS_2MB); + let p = align_down(kernel_first_paddr, BLOCK_BITS_2MB); + + // Manufacture the kernel page tables, which is relatively straightforward. + let kernel_lvl1_pt_paddr = { + // First, the Level 2 Upr table. + let lvl2_pt_paddr = { + let mut lvl2_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + + let mut vaddr = m; + let mut paddr = p; + while lvl1_index(m) == lvl1_index(vaddr) { + lvl2_pt_kernel[lvl2_index(vaddr)] = block_descriptor(2, paddr, MT_NORMAL); + + vaddr += 1 << BLOCK_BITS_2MB; + paddr += 1 << BLOCK_BITS_2MB; + } + + serialise_page_table_to_paddr(&mut lvl2_pt_kernel) + }; + + // Then, the Level 1 Upr table. + let mut lvl1_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + lvl1_pt_kernel[lvl1_index(l)] = table_descriptor(lvl2_pt_paddr); + + serialise_page_table_to_paddr(&mut lvl1_pt_kernel) + }; + + // Manufacture the RAM page tables, which is a little bit more complicated. + // We assume that normal RAM lies between 0 <= paddr < 512GiB, i.e. + // that lvl0_index(any ram region addr) = 0. + let ram_lvl1_pt_paddr = { + // Validation of assumptions about the identity mapped regions. + let mut previous_end = None; + for region in identity_mapped_regions.iter() { + assert!(lvl0_index(region.start) == 0); + assert!(lvl0_index(region.end - 1) == 0); + // This is probably an unnecessary assumption. + assert!(region.start.is_multiple_of(4096)); + assert!(region.end.is_multiple_of(4096)); + // This is definitely necessary. + assert!(region.start >= previous_end.unwrap_or(0)); + previous_end = Some(region.end); + } + + // We maintain three active page tables, which contain our previous + // known page table data. As we process regions in ascending order, + // once we have exceeded the bounds of the current reservation we + // can simply push to the page_table_bytes storage and insert into + // the parent PT the descriptor. + // When the current vaddr (/paddr, as identity mapped) exceeds the + // top value we rotate to a new PT. + + let mut lvl1_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut lvl2_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut lvl3_pt = [0u64; PAGE_TABLE_ENTRIES]; + // TODO: These should be defines. Note that the top is the size of 1 level of the next level up. + // TODO: LVL1_ENTRY_RANGE? idk + #[allow(unused_mut)] + let mut lvl1_vaddr_top = 1 << BLOCK_BITS_512GB; + let mut lvl2_vaddr_top = 1 << BLOCK_BITS_1GB; + let mut lvl3_vaddr_top = 1 << BLOCK_BITS_2MB; + + // TODO: Tests... + // This is similar to aligned_power_of_two_regions() for the kernel UT, + // but we restrict it such that the output always is either 1GB, 2MB, or 4KB + // pages. + + // Allowed externally for the final iteration + let mut base = 0u64; + for region in identity_mapped_regions.iter() { + // SAFETY: We went through and initialised raw before. + let attr_index = unsafe { region.arch_attrs.raw }; + + println!( + "Identity-Mapped Region: {:#x}..{:#x}", + region.start, region.end + ); + println!( + " - Current Lvl1: {:#x}..{:#x}, entries: {}", + (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), + lvl1_vaddr_top, + lvl1_pt.iter().filter(|&&v| v != 0).count() + ); + println!( + " - Current Lvl2: {:#x}..{:#x}, entries: {}", + (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), + lvl2_vaddr_top, + lvl2_pt.iter().filter(|&&v| v != 0).count() + ); + println!( + " - Current Lvl3: {:#x}..{:#x}, entries: {}", + (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), + lvl3_vaddr_top, + lvl3_pt.iter().filter(|&&v| v != 0).count() + ); + + // Handle the fact that the regions are not contiguous and that + // we might need to skip PT. + + { + if region.start >= lvl3_vaddr_top { + if lvl3_pt != [0; _] { + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + println!( + "[iter] Serialise lvl3 table: {:#x} for to {:#x}..{lvl3_vaddr_top:#x}", + lvl3_pt_paddr as usize, + (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)) + ); + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + } + + // TODO: just compute it. + while region.start >= lvl3_vaddr_top { + lvl3_vaddr_top += 1 << BLOCK_BITS_2MB; + } + } + + if region.start >= lvl2_vaddr_top { + if lvl2_pt != [0; _] { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + println!("[iter] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", lvl2_pt_paddr as usize, (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + } + + // TODO: just compute it. + while region.start >= lvl2_vaddr_top { + lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; + } + } + + if region.start >= lvl1_vaddr_top { + unreachable!( + "impossible as everything should fit here: {:#x}", + lvl1_vaddr_top + ); + } + } + + // After serialising the old base, update the new one. + base = region.start; + + // Inner Loop: + // Invariant: the page tables in lvl1_pt, lvl2_pt, lvl3_pt + // are either (1) for the current address range, + // or (2) are empty and for a lower level than the current level. + // Also, the values in lvlXXX_vaddr_top are always correct (even if empty) + // Also contiguous within the loop. + // Loop entry: (1) holds by work at the start of each region + while base != region.end { + // Condition is !=, but assert that we never skip it. + assert!(base < region.end); + + let size_bits = region.end.wrapping_sub(base).ilog2(); + let align_bits = min( + size_bits, + // FIXME: Once MSRV is > 1.97, use .lowest_one() method. + if base == 0 { + size_bits + } else { + base.trailing_zeros() + }, + ); + + // Match the size and alignment of the current region to + // the valid PT region sizes. + let (level, bits) = match u64::from(align_bits) { + BLOCK_BITS_1GB.. => (1, BLOCK_BITS_1GB), + BLOCK_BITS_2MB.. => (2, BLOCK_BITS_2MB), + PAGE_BITS_4KB.. => (3, PAGE_BITS_4KB), + 0.. => panic!("impossible; regions should be aligned to 4K at least"), + }; + + let pt_region_size = 1u64 << bits; + let top = base + pt_region_size; + + println!( + "- Aligned PT region: {:#x}..{:#x} (size_bits: {}, align_bits: {}, bits: {})", + base, top, size_bits, align_bits, bits + ); + println!( + " - Current Lvl1: {:#x}..{:#x}, entries: {}", + (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), + lvl1_vaddr_top, + lvl1_pt.iter().filter(|&&v| v != 0).count() + ); + println!( + " - Current Lvl2: {:#x}..{:#x}, entries: {}", + (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), + lvl2_vaddr_top, + lvl2_pt.iter().filter(|&&v| v != 0).count() + ); + println!( + " - Current Lvl3: {:#x}..{:#x}, entries: {}", + (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), + lvl3_vaddr_top, + lvl3_pt.iter().filter(|&&v| v != 0).count() + ); + + match level { + 1 => { + // If it belongs in Level 1 PT, then it must go in + // lvl1 pt. By the inavariant, base < lvl1_vaddr_top. + assert!(base < lvl1_vaddr_top); + // top is <= lvl1_vaddr_top (the case where it is the topmost entry) + assert!(top <= lvl1_vaddr_top); + + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = block_descriptor(1, base, attr_index); + + if top == lvl1_vaddr_top { + // Invariant maintenance: if the new top would be now equal + // the end of the page table's region top, we need a new + // page table object and add it to the list. + + // This should be possible to handle - we just need to break out of this loop + todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + } + + // Invariant: Lower levels are empty. + assert!(lvl2_pt == [0; _]); + assert!(lvl3_pt == [0; _]); + // Invariant maintenance: vaddr_top is right range for current PT. + // it's empty so we need to increment the top to be current top (1G aligned) + 2MIB (512 lvl3 entries) + lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); + // it's empty so we need to increment the top to be current top (1G aligned) + 1G (512 lvl2 entries) + lvl2_vaddr_top = top + (1 << BLOCK_BITS_1GB); + } + 2 => { + // If it is a 2MiB block, it must go in the Level 2 PT; + // by our invariants: base < lvl2_vaddr_top and top <= lvl2_vaddr_top + assert!(base < lvl2_vaddr_top); + assert!(top <= lvl2_vaddr_top); + + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = block_descriptor(2, base, attr_index); + + if top == lvl2_vaddr_top { + // Invariant maintenance: keep for current address range. + // As we're the top of the range, we can serialise the table. + + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} up to {lvl2_vaddr_top:#x}"); + lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; + + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + + if top == lvl1_vaddr_top { + todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + } + } + + // Invariant: Lower levels are empty. + assert!(lvl3_pt == [0; _]); + // Invariant maintenance: vaddr_top is right range for current PT. + // it's empty so we need to increment the top to be current top (2MIB aligned) + 2MIB (512 lvl3 entries) + lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); + } + 3 => { + // If it is a 4K page, it must go in the Level 3 PT; + // by our invariants: base < lvl3_vaddr_top and top <= lvl3_vaddr_top + assert!(base < lvl3_vaddr_top); + assert!(top <= lvl3_vaddr_top); + + assert!(lvl3_pt[lvl3_index(base)] == 0); + lvl3_pt[lvl3_index(base)] = page_descriptor(base, attr_index); + + if top == lvl3_vaddr_top { + // Invariant maintenance: keep for current address range. + // As we're the top of the range, we can serialise the table. + + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + println!( + "Serialise lvl3 table: {:#x} for to {:#x}..{lvl3_vaddr_top:#x}", + lvl3_pt_paddr as usize, + (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)) + ); + lvl3_vaddr_top += 1 << BLOCK_BITS_2MB; + + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + + if top == lvl2_vaddr_top { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + println!( + "Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}", + lvl2_pt_paddr as usize, + (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)) + ); + lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; + + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + + if top == lvl1_vaddr_top { + todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + } + } + } + + // Invariant: lower levels empty is vacuuously true + } + _ => unreachable!("level is 1..=3"), + } + + base += pt_region_size; + } + } + + // By the loop invariant, we know that anything before has been serialised. + // However, as we are at the end of the loop now, we might have + // page tables that have been partially filled out, and we need to + // serialise these. + + if lvl3_pt != [0; _] { + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + println!("[end] Serialise lvl3 table: {:#x}", lvl3_pt_paddr as usize); + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + } + + if lvl2_pt != [0; _] { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + println!("[end] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", lvl2_pt_paddr as usize, (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + } + + // the level1 pt should not be empty. lol. + assert!(lvl1_pt != [0; _]); + + // println!("New lvl1 table"); + serialise_page_table_to_paddr(&mut lvl1_pt) + }; + + struct Config { + hypervisor: bool, + } + let config = Config { hypervisor: true }; + + // Depending on whether we are in hypervisor mode, we either need to + // return the TTBR0_EL2 or TTBR[0,1]_EL1 values. We return u64::MAX + // so as to return garbage - an unaligned address outside of physical + // memory. + if config.hypervisor { + // Manufacture the Level 0 table, containing the kernel table + // and the RAM tables. + + let mut ttbr0_el2_pt = [0u64; PAGE_TABLE_ENTRIES]; + + assert!(lvl0_index(k) != lvl0_index(0)); + ttbr0_el2_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); + ttbr0_el2_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); + + let ttbr0_el2 = serialise_page_table_to_paddr(&mut ttbr0_el2_pt); + + AArch64ReturnValue { + ttbr0_el2, + ttbr0_el1: AArch64ReturnValue::INVALID, + ttbr1_el1: AArch64ReturnValue::INVALID, + } + } else { + let mut ttbr0_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut ttbr1_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; + + // Kernel in TTBR1 (Upper) + ttbr1_el1_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); + // Identity-mapped RAM in TTBR0 (Lower) + ttbr0_el1_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); + + let ttbr0_el1 = serialise_page_table_to_paddr(&mut ttbr0_el1_pt); + let ttbr1_el1 = serialise_page_table_to_paddr(&mut ttbr1_el1_pt); + + AArch64ReturnValue { + ttbr0_el2: AArch64ReturnValue::INVALID, + ttbr0_el1, + ttbr1_el1, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } + + #[test] + fn qemu_aarch64() { + #[repr(align(4096))] + struct PtBytes([[MaybeUninit; 4096]; MAX_NUM_PAGE_TABLES]); + + let mut regions = [ + Region { + start: 0x60000000, + end: 0xc0000000, + arch_attrs: RegionArchAttrs { is_ram: true }, + }, + // UART + Region { + start: 0x9000000, + end: 0x9001000, + arch_attrs: RegionArchAttrs { is_ram: false }, + }, + ]; + + // // FIXME: Derive from the kernel build system. + // if let Some(uart_base) = read_symbol_maybe(elf, "uart_addr") { + // let uart_base = align_down(uart_base, PAGE_BITS_4KB); + // regions.push(( + // PlatformConfigRegion { + // start: uart_base, + // end: uart_base + (1 << PAGE_BITS_4KB), + // }, + // MT_DEVICE_nGnRnE, + // )); + // } + // // FIXME: This is currently assuming implementation details of the BCM2711/ + // // Raspberry Pi 4B spin table implementation, as it is the only + // // platform we have that uses spin tables. Specifically, that + // // it is always located at the 0 page. + // if elf.find_symbol("cpus_release_addr").is_ok() { + // regions.push(( + // PlatformConfigRegion { + // start: 0x0, + // end: 1 << PAGE_BITS_4KB, + // }, + // MT_DEVICE_nGnRnE, + // )); + // } + + let mut page_table_bytes = PtBytes([[MaybeUninit::uninit(); _]; _]); + + let pt_bases = unsafe { + aarch64_setup_pagetables( + /* kernel_first_vaddr */ 0x8060000000, + /* kernel_first_paddr */ 0x60000000, + regions.as_mut_ptr(), + regions.len(), + &mut page_table_bytes.0, + ) + }; + } +} diff --git a/loader/src/riscv/mmu.c b/loader/src/riscv/mmu.c index 7751b25e9..b40350ca7 100644 --- a/loader/src/riscv/mmu.c +++ b/loader/src/riscv/mmu.c @@ -8,15 +8,9 @@ #include #include "../arch.h" -#include "../cutil.h" - -/* Paging structures for kernel mapping */ -uint64_t boot_lvl1_pt[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl2_pt[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl3_pt[1 << 9] ALIGN(1 << 12); -/* Paging structures for identity mapping */ -uint64_t boot_lvl2_pt_loader[1 << 9] ALIGN(1 << 12); +/* Pointers to the top-level paging structures */ +uintptr_t riscv64_boot_lvl1_pt; /* * This is the encoding for the MODE field of the satp register when @@ -36,7 +30,7 @@ int arch_mmu_enable(int logical_cpu) asm volatile( "csrw satp, %0\n" : - : "r"(VM_MODE | (uintptr_t)boot_lvl1_pt >> RISCV_PGSHIFT) + : "r"(VM_MODE | riscv64_boot_lvl1_pt >> RISCV_PGSHIFT) : ); asm volatile("fence.i" ::: "memory"); diff --git a/tool/microkit/src/loader.rs b/tool/microkit/src/loader.rs index 53a2597e2..860caca70 100644 --- a/tool/microkit/src/loader.rs +++ b/tool/microkit/src/loader.rs @@ -9,298 +9,24 @@ use crate::uimage::uimage_serialise; use crate::util::{mb, round_up, struct_to_bytes}; use std::fs::File; use std::io::{BufWriter, Write}; +use std::mem; use std::ops::Range; use std::path::Path; -macro_rules! grab_symbol { - ($elf: expr, $symbol_name: literal) => { - $elf.find_symbol($symbol_name) - .expect(concat!("Could not find '", $symbol_name, "' symbol")) - }; -} - -const PAGE_TABLE_SIZE: usize = 4096; - -pub mod aarch64 { - //! For AArch64, our page tables use the Stage 1 descriptor formats - //! for both EL2 (TTBR0_EL2) and EL1 (TTBR0_EL1/TTBR1_EL1). - //! Stage 2 descriptors are only used when in the EL1&0 regime; which is not - //! the case when in EL2. - - use crate::util::mask; - - pub const LVL0_BITS: u64 = 9; - pub const LVL1_BITS: u64 = 9; - pub const LVL2_BITS: u64 = 9; - - pub fn lvl0_index(addr: u64) -> usize { - let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS + LVL1_BITS)) & mask(LVL0_BITS); - idx as usize - } - - pub fn lvl1_index(addr: u64) -> usize { - let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS)) & mask(LVL1_BITS); - idx as usize - } - - pub fn lvl2_index(addr: u64) -> usize { - let idx = (addr >> (BLOCK_BITS_2MB)) & mask(LVL2_BITS); - idx as usize - } - - /// Stage 1 translation table page/block descriptors have bits[4:2] containing - /// AttrIndex[2:0]. The AttrIndex values depends on our configuration of - /// the `MAIR_EL1` or `MAIR_EL2` registers done in util64.S; - /// This also needs to match the values that seL4 uses. - #[allow(non_upper_case_globals, reason = "matching ARM naming convention")] - pub mod s1_mair_attr_index { - pub const MT_DEVICE_nGnRnE: u64 = 0b000; - pub const MT_DEVICE_nGnRE: u64 = 0b001; - pub const MT_DEVICE_GRE: u64 = 0b010; - pub const MT_NORMAL_NC: u64 = 0b011; - pub const MT_NORMAL: u64 = 0b100; - } - - pub mod descriptor_type { - //! The translation table descriptor formats, as per §D8.3 "Translation - //! table descriptor formats" of ARM DDI 0487 L.b. Specifically, - //! as per "Table D8-48 Determination of descriptor type" - - /// Descriptor type: Table. Condition is lookup level != 3. - pub const TABLE: u64 = 0b11; - /// Descriptor type: Page. Condition is lookup level == 3. - pub const PAGE: u64 = 0b11; - /// Descriptor type: Block. Condition is lookup level != 3. - pub const BLOCK: u64 = 0b01; - /// Descriptor type: Invalid. Strictly speaking bit[1] does not matter. - pub const INVALID: u64 = 0b00; - } - - pub mod shareability_attributes { - //! Per §D8.6.2 "Stage 1 Shareability attributes", these contain the - //! shareability attributes of the descriptor OA for normal-cacheable - //! memory. - - /// Non-shareable - pub const NON_SHAREABLE: u64 = 0b00; - /// Outer-shareable - pub const OUTER_SHAREABLE: u64 = 0b10; - /// Inner-shareable - pub const INNER_SHAREABLE: u64 = 0b11; - } - - /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, - /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address - /// is bits [47:n], and: - /// - /// > For the 4KB granule size, the level 1 descriptor n is 30, - /// > and the level 2 descriptor n is 21. - pub const BLOCK_BITS_1GB: u64 = 30; - - /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, - /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address - /// is bits [47:n], and: - /// - /// > For the 4KB granule size, the level 1 descriptor n is 30, - /// > and the level 2 descriptor n is 21. - pub const BLOCK_BITS_2MB: u64 = 21; - - /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and - /// "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b; - /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" - pub fn block_descriptor(level: usize, addr: u64, attr_index: u64) -> u64 { - // Per Table D8-48, Condition for descriptor_type::BLOCK is level != 3. - assert!(level != 3); - - let upper_attributes: u64 = 0; - - let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { - // Match what the seL4 kernel uses for its page tables, which - // is especially necessary for SMP booting which relies on it - // for coherency. - shareability_attributes::INNER_SHAREABLE - } else { - // Per $R_{PYFVQ}$: - // > If a region is mapped as Device memory or Normal Non-cacheable - // > memory after all enabled translation stages, then the region - // > has an effective Shareability attribute of Outer Shareable. - // - // We override the value we place in here to OUTER_SHAREABLE to match - // how the hardware behaves. This is not necessary but for clarity. - shareability_attributes::OUTER_SHAREABLE - }; - - // AP[2:1], which we set as 0b00 for read/write access: - // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1 - // stage 2: 0b00 is RW for EL2 and no perms for EL1. - const AP_KERNEL_RW: u64 = 0b00; - - // bit[11] is the not global (nG) field, we leave as 0 (global). - // bit[10] is the access flag; depending on FEAT_HAFDBS, when software - // manages the AF memory accesses to the page/block when AF=0 - // raise an Access Fault; when hardware manages the AF it will - // become 1. - // bit[9:8] is SH[1:0] containing stage 1 shareability attributes - // bit[7:6] contains AP[2:1] - // bit[5] is RES0 - // bit[4:2] contains AttrIndex - let lower_attributes: u64 = - (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); - - // bits[47:n] - let output_address: u64 = addr - & !mask(match level { - 1 => BLOCK_BITS_1GB, - 2 => BLOCK_BITS_2MB, - _ => panic!("unsupported level {level} for block descriptor"), - }); - - // address must not have bits above 47 set. - assert!(addr & mask(48) == addr); - - // bits[63:50] describing the "Upper attributes" are left at 0. - // bits[49:48] are RES0 - // bits[47:n] contain the Output address - // bits[n-1:12] are RES0 - // bits[11:2] contain the "Lower attributes" - // bits[1:0] contains the descriptor type - upper_attributes | output_address | lower_attributes | descriptor_type::BLOCK - } - - /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and - /// "Figure D8-15 VMSAv8-64 Page descriptor formats" of ARM DDI0487L.b; - /// specifically subfigure "4KB granule 48-bit OA". - pub fn page_descriptor(addr: u64, attr_index: u64) -> u64 { - // The main difference between a page descriptor and block descriptor - // is in the size of the output address (OA) and in the descriptor type. - - let upper_attributes: u64 = 0; - - let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { - // Match what the seL4 kernel uses for its page tables, which - // is especially necessary for SMP booting which relies on it - // for coherency. - shareability_attributes::INNER_SHAREABLE - } else { - // Per $R_{PYFVQ}$: - // > If a region is mapped as Device memory or Normal Non-cacheable - // > memory after all enabled translation stages, then the region - // > has an effective Shareability attribute of Outer Shareable. - // We override the value we place in here to OUTER_SHAREABLE to match - // how the hardware behaves. - shareability_attributes::OUTER_SHAREABLE - }; - - // AP[2:1], which we set as 0b00 for read/write access: - // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1/El2 (priv) - const AP_KERNEL_RW: u64 = 0b00; - - // bit[11] is the not global (nG) field, we leave as 0 (global). - // bit[10] is the access flag; depending on FEAT_HAFDBS, when software - // manages the AF memory accesses to the page/block when AF=0 - // raise an Access Fault; when hardware manages the AF it will - // become 1. - // bit[9:8] is SH[1:0] containing stage 1 shareability attributes - // bit[7:6] contains AP[2:1] - // bit[5] is RES0 - // bit[4:2] contains AttrIndex - let lower_attributes: u64 = - (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); - - // bits[47:12] - let output_address: u64 = addr & !mask(12); - - // address must not have bits above 47 set. - assert!(addr & mask(48) == addr); - - // bits[63:50] describing the "Upper attributes" are left at 0. - // bits[49:48] are RES0 - // bits[47:12] contain the Output address - // bits[11:2] contain the "Lower attributes" - // bits[1:0] contains the descriptor type - upper_attributes | output_address | lower_attributes | descriptor_type::PAGE - } - - /// Per "Table D8-50 Stage 1 VMSAv8-64 Table descriptor fields" and - /// "Figure D8-12 VMSAv8-64 Table descriptor formats" of ARM DDI0487L.b; - /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" - pub fn table_descriptor(addr: u64) -> u64 { - // Per Table D8-48, Condition for descriptor_type::TABLE is level != 3. - - // We don't set any of these attributes, most are hardware-feature conditional - let attributes: u64 = 0; - - // address must not have bits above 47 or below 12 set - assert!(addr & mask(12) == 0x0); - assert!(addr & mask(48) == addr); - - let next_level_table_address = addr; - - // bits[63:59] are "Attributes" - // bits[58:51] are ignored - // bits[50:48] are RES0 - // bits[47:m] is the next-level table address - // note: here m=12 for 4KB granule - // bits[m-1:12] are RES0 - // so this doesn't exist for 4KB granule - // bits[11:2] are ignored - // bits[1:0] contain the descriptor type - attributes | next_level_table_address | descriptor_type::TABLE - } -} - -mod riscv64 { - pub(crate) const BLOCK_BITS_2MB: u64 = 21; - pub(crate) const PAGE_BITS_4K: u64 = 12; - - pub(crate) const PAGE_TABLE_INDEX_BITS: u64 = 9; - pub(crate) const PAGE_SHIFT: u64 = 12; - /// This sets the page table entry bits: D,A,X,W,R. - pub(crate) const PTE_TYPE_BITS: u64 = 0b11001110; - // TODO: where does this come from? - pub(crate) const PTE_TYPE_TABLE: u64 = 0; - pub(crate) const PTE_TYPE_VALID: u64 = 1; - - pub(crate) const PTE_PPN0_SHIFT: u64 = 10; - - /// Due to RISC-V having various virtual memory setups, we have this generic function to - /// figure out the page-table index given the total number of page table levels for the - /// platform and which level we are currently looking at. - pub fn pt_index(pt_levels: usize, addr: u64, level: usize) -> usize { - let pt_index_bits = PAGE_TABLE_INDEX_BITS * (pt_levels - level) as u64; - let idx = (addr >> (pt_index_bits + PAGE_SHIFT)) % 512; - - idx as usize - } - - /// Generate physical page number given an address - pub fn pte_ppn(addr: u64) -> u64 { - (addr >> PAGE_SHIFT) << PTE_PPN0_SHIFT - } - - pub fn pte_next(addr: u64) -> u64 { - pte_ppn(addr) | PTE_TYPE_TABLE | PTE_TYPE_VALID - } - - pub fn pte_leaf(addr: u64) -> u64 { - pte_ppn(addr) | PTE_TYPE_BITS | PTE_TYPE_VALID - } -} - /// Checks that each region in the given list does not overlap with any other region. /// Panics upon finding an overlapping region -fn check_non_overlapping(regions: &Vec<(u64, &[u8])>) { +fn check_non_overlapping(regions: &Vec<(u64, u64)>) { let mut checked: Vec<(u64, u64)> = Vec::new(); - for (base, data) in regions { - let end = base + data.len() as u64; + for &(base, size) in regions.iter() { + let end = base + size; // Check that this does not overlap with any checked regions - for (b, e) in &checked { - if !(end <= *b || *base >= *e) { + for &(b, e) in checked.iter() { + if !(end <= b || base >= e) { panic!("Overlapping regions: [{base:x}..{end:x}) overlaps [{b:x}..{e:x})"); } } - checked.push((*base, end)); + checked.push((base, end)); } } @@ -419,53 +145,27 @@ impl<'a> Loader<'a> { } } - let Some(kernel_first_vaddr) = kernel_first_vaddr else { - panic!("INTERNAL: could not determine kernel_first_vaddr"); - }; - - let Some(kernel_first_paddr) = kernel_first_paddr else { - panic!("INTERNAL: could not determine kernel_first_paddr"); - }; - - let pagetable_vars = match config.arch { - Arch::Aarch64 => Loader::aarch64_setup_pagetables( - config, - &loader_elf, - kernel_first_vaddr, - kernel_first_paddr, - ), - Arch::Riscv64 => Loader::riscv64_setup_pagetables( - config, - &loader_elf, - kernel_first_vaddr, - kernel_first_paddr, - ), - Arch::X86_64 => unreachable!("x86_64 does not support creating a loader image"), - }; - let image_segment = loader_elf .segments - .into_iter() + .iter() .find(|segment| segment.loadable) .expect("Did not find loadable segment"); + + // Called "vaddr" but due to 1:1 mapping vaddr == paddr. let image_vaddr = image_segment.virt_addr; + // We have to clone here as the image executable is part of this function return object, // and the loader ELF is deserialised in this scope, so its lifetime will be shorter than // the return object. - let mut loader_image = image_segment.data().clone(); + let loader_image = image_segment.data().clone(); + + println!("kernel_first_vaddr: {kernel_first_vaddr:?}"); + println!("kernel_first_paddr: {kernel_first_paddr:?}"); if image_vaddr != loader_elf.entry { panic!("The loader entry point must be the first byte in the image"); } - for (var_addr, var_size, var_data) in pagetable_vars { - let offset = var_addr - image_vaddr; - assert!(var_size == var_data.len() as u64); - assert!(offset > 0); - assert!(offset <= loader_image.len() as u64); - loader_image[offset as usize..(offset + var_size) as usize].copy_from_slice(&var_data); - } - let kernel_entry = kernel_elf.entry; // initial task virt + pv_offset == initial task physical, so @@ -477,11 +177,6 @@ impl<'a> Loader<'a> { ui_p_reg_start + (initial_task_vaddr_range.end - initial_task_vaddr_range.start); assert!(ui_p_reg_end > ui_p_reg_start); - // This clone isn't too bad as it is just a Vec<(u64, &[u8])> - let mut all_regions_with_loader = regions.clone(); - all_regions_with_loader.push((image_vaddr, &loader_image)); - check_non_overlapping(&all_regions_with_loader); - let mut region_metadata = Vec::new(); let mut offset: u64 = 0; for (addr, data) in ®ions { @@ -494,10 +189,19 @@ impl<'a> Loader<'a> { offset += data.len() as u64; } - let size = std::mem::size_of::() as u64 - + region_metadata.iter().fold(0_u64, |acc, x| { - acc + x.size + std::mem::size_of::() as u64 - }); + let size = loader_image.len() as u64 + + mem::size_of::() as u64 + + (region_metadata.len() * mem::size_of::()) as u64 + + offset; + + let mut all_regions_with_loader: Vec<_> = regions + .iter() + .map(|&(base, data)| (base, data.len() as u64)) + .collect(); + all_regions_with_loader.push((image_vaddr, size)); + check_non_overlapping(&all_regions_with_loader); + + // TODO: Check contained within real RAM. let header = LoaderHeader64 { magic, @@ -539,6 +243,8 @@ impl<'a> Loader<'a> { bytes.extend_from_slice(data); } + assert!(bytes.len() as u64 == self.header.size); + bytes } @@ -603,312 +309,4 @@ impl<'a> Loader<'a> { Err(e) => panic!("Could not create '{}': {}", path.display(), e), } } - - fn riscv64_setup_pagetables( - config: &Config, - elf: &ElfFile, - first_vaddr: u64, - first_paddr: u64, - ) -> Vec<(u64, u64, [u8; PAGE_TABLE_SIZE])> { - let (text_addr, _) = grab_symbol!(elf, "_text"); - let (boot_lvl1_pt_addr, boot_lvl1_pt_size) = grab_symbol!(elf, "boot_lvl1_pt"); - let (boot_lvl2_pt_addr, boot_lvl2_pt_size) = grab_symbol!(elf, "boot_lvl2_pt"); - let (boot_lvl3_pt_addr, boot_lvl3_pt_size) = grab_symbol!(elf, "boot_lvl3_pt"); - let (boot_lvl2_pt_loader_addr, boot_lvl2_pt_loader_size) = - grab_symbol!(elf, "boot_lvl2_pt_loader"); - - // We map the loader using 2MB pages, so make sure the base is actually aligned. - assert!(text_addr.is_multiple_of(1 << riscv64::BLOCK_BITS_2MB)); - - let num_pt_levels = config.riscv_pt_levels.unwrap().levels(); - - let mut boot_lvl1_pt: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let text_index_lvl1 = riscv64::pt_index(num_pt_levels, text_addr, 1); - let pt_entry = riscv64::pte_next(boot_lvl2_pt_loader_addr); - let start = 8 * text_index_lvl1; - let end = start + 8; - boot_lvl1_pt[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl2_pt_loader: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let text_index_lvl2 = riscv64::pt_index(num_pt_levels, text_addr, 2); - for (page, i) in (text_index_lvl2..512).enumerate() { - let start = 8 * i; - let end = start + 8; - let addr = text_addr + ((page as u64) << riscv64::BLOCK_BITS_2MB); - let pt_entry = riscv64::pte_leaf(addr); - boot_lvl2_pt_loader[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - } - - { - let index = riscv64::pt_index(num_pt_levels, first_vaddr, 1); - let start = 8 * index; - let end = start + 8; - boot_lvl1_pt[start..end] - .copy_from_slice(&riscv64::pte_next(boot_lvl2_pt_addr).to_le_bytes()); - } - - let mut boot_lvl3_pt: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - let mut boot_lvl2_pt: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let mut index_lvl2 = riscv64::pt_index(num_pt_levels, first_vaddr, 2); - if !first_vaddr.is_multiple_of(1 << riscv64::BLOCK_BITS_2MB) { - let index_lvl3 = riscv64::pt_index(num_pt_levels, first_vaddr, 3); - for (page, i) in (index_lvl3..512).enumerate() { - let start = 8 * i; - let end = start + 8; - let addr = first_paddr + ((page as u64) << riscv64::PAGE_BITS_4K); - assert!(addr.is_multiple_of(1 << riscv64::PAGE_BITS_4K)); - let pt_entry = riscv64::pte_leaf(addr); - boot_lvl3_pt[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - let start = 8 * index_lvl2; - let end = start + 8; - let lvl3_pt_entry = riscv64::pte_next(boot_lvl3_pt_addr); - assert!(boot_lvl3_pt_addr.is_multiple_of(1 << riscv64::PAGE_BITS_4K)); - boot_lvl2_pt[start..end].copy_from_slice(&lvl3_pt_entry.to_le_bytes()); - index_lvl2 += 1; - } - let first_paddr_aligned = round_up(first_paddr, 1 << riscv64::BLOCK_BITS_2MB); - for (page, i) in (index_lvl2..512).enumerate() { - let start = 8 * i; - let end = start + 8; - let addr = first_paddr_aligned + ((page as u64) << riscv64::BLOCK_BITS_2MB); - assert!(addr.is_multiple_of(1 << riscv64::BLOCK_BITS_2MB)); - let pt_entry = riscv64::pte_leaf(addr); - boot_lvl2_pt[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - } - - vec![ - (boot_lvl1_pt_addr, boot_lvl1_pt_size, boot_lvl1_pt), - (boot_lvl2_pt_addr, boot_lvl2_pt_size, boot_lvl2_pt), - (boot_lvl3_pt_addr, boot_lvl3_pt_size, boot_lvl3_pt), - ( - boot_lvl2_pt_loader_addr, - boot_lvl2_pt_loader_size, - boot_lvl2_pt_loader, - ), - ] - } - - /// AArch64 loader page tables have two variations: - /// - Loader in EL2, then Stage 1 translations in use, so we have the - /// singular TTBR0_EL2 register containing the Level 0 table; - /// this allows virtual address in the range [0,2^48). - /// - Loader in EL1, then Stage 1 translations are in use, so we have both - /// the TTBR0_EL1 (covering vaddr in range [0,2^48)) and TTBR1_EL2 ( - /// (covering vaddr in the range [2^64-2^48,2^64)), and containing - /// the "Level 0 Lower" page table, and "Level 0 Upper" page table - /// physical addresses respectively. - /// - /// Thus, for EL2 loader, the singular Level 0 page table contains the table - /// descriptors for the "Level 1 Upper" and "Level 1 Lower" page tables. - /// For the EL1 loader, we instead have two Level 0 page tables, and - /// "Level 0 Lower" contains the "Level 1 Lower" descriptor, and "Level 0 - /// Upper" contains the "Level 1 Upper" descriptor. - /// Otherwise, the page tables layout from Level 1 downwards are identical - /// (but not necessarily the layout within the page/table/block descriptors). - /// - /// ```txt - /// (256 TiB) - /// 512 +-- Level 0 --+ 2^48 - /// | | - /// | (empty) | - /// | | - /// k+1 +-------------+ (512 GiB) - /// | Level 1 Upr | ----------> +-- Level 1 --+ - /// k +-------------+ | | - /// | | | (empty) | - /// | | | | - /// | | l+1 +-------------+ (1 GiB) - /// | | | Level 2 Upr | ----------> +-- Level 2 --+ +-------------+ - /// | | l +-------------+ | | ----------> | 2 MiB block | - /// | | | | 511 |-------------| +-------------+ - /// | | | (empty) | | | ----------> | 2 MiB block | - /// | | | | 510 |-------------| +-------------+ - /// | | +-------------+ | | ----------> | 2 MiB block | - /// | | |-------------| +-------------+ - /// | (empty) | Kernel Regions (...) (...) (...) - /// | | |-------------| +-------------+ - /// | | | | ----------> | 2 MiB block | - /// | | m |-------------| +-------------+ p - /// | | | | - /// | | | (empty) | - /// | | | | - /// | | 0 +-------------+ - /// | | - /// | | - /// | | - /// 1 +-------------+ (512 GiB) - /// | Level 1 Lwr | ----------> +-- Level 1 --+ - /// 0 +-------------+ | | - /// | (empty) | - /// | | - /// u+1 +-------------+ +-------------+ - /// | uart_base | ----------> | 1 GiB block | - /// u +-------------+ +-------------+ - /// | | - /// | (empty) | - /// | | - /// i+1 +-------------+ (1 GiB) - /// | Level 2 Lwr | ----------> +-- Level 2 --+ - /// i +-------------+ | | - /// | | | (empty) | - /// | (empty) | | | - /// | | t +-------------+ +-------------+ - /// +-------------+ | | ----------> | 2 MiB block | - /// |-------------| +-------------+ - /// | | ----------> | 2 MiB block | - /// |-------------| +-------------+ - /// Loader Regions (...) (...) (...) - /// |-------------| +-------------+ - /// | | ----------> | 2 MiB block | - /// |-------------| +-------------+ - /// | | ----------> | 2 MiB block | - /// s +-------------+ +-------------+ - /// | | - /// | (empty) | - /// | | - /// +-------------+ - /// - /// Where: - /// k = align_down(kernel_first_vaddr, 512GiB), - /// l = align_down(kernel_first_vaddr, 1GiB), - /// m = align_down(kernel_first_vaddr, 2MiB), - /// p = align_down(kernel_first_paddr, 2MiB), - /// u = align_down(uart_base, 1GiB), - /// i = align_down(loader_start_addr, 1GiB), - /// s = align_down(loader_start_addr, 2MiB), - /// t = align_up(loader_end_addr, 2MiB), - /// ``` - /// - fn aarch64_setup_pagetables( - _config: &Config, - elf: &ElfFile, - first_vaddr: u64, - first_paddr: u64, - ) -> Vec<(u64, u64, [u8; PAGE_TABLE_SIZE])> { - use aarch64::s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}; - - let (boot_lvl1_lower_addr, boot_lvl1_lower_size) = grab_symbol!(elf, "boot_lvl1_lower"); - let (boot_lvl1_upper_addr, boot_lvl1_upper_size) = grab_symbol!(elf, "boot_lvl1_upper"); - let (boot_lvl2_upper_addr, boot_lvl2_upper_size) = grab_symbol!(elf, "boot_lvl2_upper"); - let (boot_lvl0_lower_addr, boot_lvl0_lower_size) = grab_symbol!(elf, "boot_lvl0_lower"); - let (boot_lvl0_upper_addr, boot_lvl0_upper_size) = grab_symbol!(elf, "boot_lvl0_upper"); - let (boot_lvl2_lower_addr, boot_lvl2_lower_size) = grab_symbol!(elf, "boot_lvl2_lower"); - - let (loader_start_addr, _) = grab_symbol!(elf, "_loader_start"); - let (loader_end_addr, _) = grab_symbol!(elf, "_loader_end"); - - if aarch64::lvl1_index(loader_start_addr) != aarch64::lvl1_index(loader_end_addr) { - panic!("We only map 1GiB, but loader paddr range covers multiple GiB"); - } - - let mut boot_lvl0_lower: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let pt_entry = aarch64::table_descriptor(boot_lvl1_lower_addr); - boot_lvl0_lower[..8].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl1_lower: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - - // map optional UART MMIO in l1 1GB page, only available if CONFIG_PRINTING - if let Ok((uart_addr, uart_addr_size)) = elf.find_symbol("uart_addr") { - let data = elf - .get_data(uart_addr, uart_addr_size) - .expect("uart_addr not initialized"); - - let uart_base = u64::from_le_bytes(data[0..8].try_into().unwrap()); - - let lvl1_idx = aarch64::lvl1_index(uart_base); - - let pt_entry = aarch64::block_descriptor(1, uart_base, MT_DEVICE_nGnRnE); - - let start = 8 * lvl1_idx; - let end = 8 * (lvl1_idx + 1); - boot_lvl1_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl2_lower: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - - // 1GB lvl1 Table entry - let pt_entry = aarch64::table_descriptor(boot_lvl2_lower_addr); - let lvl1_idx = aarch64::lvl1_index(loader_start_addr); - let start = 8 * lvl1_idx; - let end = 8 * (lvl1_idx + 1); - boot_lvl1_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - - // map the loader 1:1 access into 2MB lvl2 Block entries for a 4KB granule - let lvl2_idx = aarch64::lvl2_index(loader_start_addr); - for i in lvl2_idx..=aarch64::lvl2_index(loader_end_addr) { - let entry_idx: u64 = - ((i - aarch64::lvl2_index(loader_start_addr)) << aarch64::BLOCK_BITS_2MB) as u64; - - let pt_entry = - aarch64::block_descriptor(2, loader_start_addr + entry_idx, MT_DEVICE_nGnRnE); - - let start = 8 * i; - let end = 8 * (i + 1); - boot_lvl2_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - - // TODO: this is a complete hack specific to BCM2711/Raspberry Pi 4B and - // will be removed with patches that re-do this loader mapping code. - if elf.find_symbol("cpus_release_addr").is_ok() { - let lvl2_idx = aarch64::lvl2_index(0); - // Make sure we don't override the loader mappings done above. - assert!(aarch64::lvl2_index(loader_start_addr) != lvl2_idx); - assert!(aarch64::lvl1_index(loader_start_addr) == aarch64::lvl1_index(0)); - - let pt_entry = aarch64::block_descriptor(2, lvl2_idx as u64, MT_DEVICE_nGnRnE); - - let start = 8 * lvl2_idx; - let end = 8 * (lvl2_idx + 1); - boot_lvl2_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl0_upper: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let pt_entry = aarch64::table_descriptor(boot_lvl1_upper_addr); - let idx = aarch64::lvl0_index(first_vaddr); - // For EL2. - boot_lvl0_lower[8 * idx..8 * (idx + 1)].copy_from_slice(&pt_entry.to_le_bytes()); - // For EL1. - boot_lvl0_upper[8 * idx..8 * (idx + 1)].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl1_upper: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let pt_entry = aarch64::table_descriptor(boot_lvl2_upper_addr); - let idx = aarch64::lvl1_index(first_vaddr); - boot_lvl1_upper[8 * idx..8 * (idx + 1)].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl2_upper: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - - let lvl2_idx = aarch64::lvl2_index(first_vaddr); - for i in lvl2_idx..512 { - let entry_idx: u64 = - ((i - aarch64::lvl2_index(first_vaddr)) << aarch64::BLOCK_BITS_2MB) as u64; - - let pt_entry = aarch64::block_descriptor(2, first_paddr + entry_idx, MT_NORMAL); - - let start = 8 * i; - let end = 8 * (i + 1); - boot_lvl2_upper[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - - vec![ - (boot_lvl0_lower_addr, boot_lvl0_lower_size, boot_lvl0_lower), - (boot_lvl1_lower_addr, boot_lvl1_lower_size, boot_lvl1_lower), - (boot_lvl0_upper_addr, boot_lvl0_upper_size, boot_lvl0_upper), - (boot_lvl1_upper_addr, boot_lvl1_upper_size, boot_lvl1_upper), - (boot_lvl2_upper_addr, boot_lvl2_upper_size, boot_lvl2_upper), - (boot_lvl2_lower_addr, boot_lvl2_lower_size, boot_lvl2_lower), - ] - } } diff --git a/tool/microkit/src/sel4.rs b/tool/microkit/src/sel4.rs index d95084a74..ead67bb04 100644 --- a/tool/microkit/src/sel4.rs +++ b/tool/microkit/src/sel4.rs @@ -249,7 +249,7 @@ pub fn emulate_kernel_boot( } } -#[derive(Deserialize)] +#[derive(Deserialize, Debug, Clone)] pub struct PlatformConfigRegion { pub start: u64, pub end: u64, diff --git a/tool/microkit/src/util.rs b/tool/microkit/src/util.rs index 6f2c0e275..211c794d6 100644 --- a/tool/microkit/src/util.rs +++ b/tool/microkit/src/util.rs @@ -54,6 +54,14 @@ pub const fn round_down(n: u64, x: u64) -> u64 { } } +pub const fn align_up(n: u64, bits: u64) -> u64 { + round_up(n, 1 << bits) +} + +pub const fn align_down(n: u64, bits: u64) -> u64 { + round_down(n, 1 << bits) +} + pub fn is_power_of_two(n: u64) -> bool { assert!(n > 0); n & (n - 1) == 0