diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8593a4ef0..d530f4de8 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -27,6 +27,8 @@ jobs: - name: Run Clippy # Make sure CI fails on all warnings, including Clippy lints run: nix develop --ignore-environment -c bash -c "cd tool/microkit && cargo-clippy --all-targets --all-features -- -D warnings -Wclippy::get_unwrap" + - name: Run Clippy (loader) + run: nix develop --ignore-environment -c bash -c 'make -C loader clippy CLIPPY_FLAGS="-D warnings -Wclippy::get_unwrap" BUILD_DIR=$(mktemp -d) RUST_ONLY=True' rustfmt_check: runs-on: [self-hosted, macos, ARM64] @@ -34,6 +36,8 @@ jobs: - uses: actions/checkout@v4 - name: Run rustfmt run: nix develop --ignore-environment -c bash -c "cd tool/microkit && cargo-fmt --check" + - name: Run rustfmt (loader) + run: nix develop --ignore-environment -c bash -c 'make -C loader rustfmt-check BUILD_DIR=$(mktemp -d) RUST_ONLY=True' code: name: Freeze Code diff --git a/build_sdk.py b/build_sdk.py index 92cc5bfcb..01130c2a6 100644 --- a/build_sdk.py +++ b/build_sdk.py @@ -796,6 +796,26 @@ def build_sel4( json_dst.chmod(0o744) +def test_loader(build_dir: Path) -> None: + build_dir = build_dir / "loader" + build_dir.mkdir(exist_ok=True, parents=True) + + make_args = f"BUILD_DIR={build_dir.absolute()} RUST_ONLY=True" + + r = system( + f"make -C loader tests {make_args}" + ) + if r != 0: + raise Exception(f"Tests failed: loader") + + # We don't pass CLIPPYARGS, so this is warning-only + r = system( + f"make -C loader clippy {make_args}" + ) + if r != 0: + raise Exception(f"Clippy failed: loader") + + def build_elf_component( component_name: str, sdk_dir: Path, @@ -1094,6 +1114,8 @@ def main() -> None: if not args.skip_run_time: build_dir = Path("build") + test_loader(build_dir) + for (board, configs) in build_goals: for config in configs: if not args.skip_sel4: diff --git a/example/hello/Makefile b/example/hello/Makefile index e33ce05eb..b32c24f51 100644 --- a/example/hello/Makefile +++ b/example/hello/Makefile @@ -27,7 +27,7 @@ ifeq ($(ARCH),aarch64) TARGET_TRIPLE := aarch64-none-elf CFLAGS_ARCH := -mstrict-align else ifeq ($(ARCH),riscv64) - TARGET_TRIPLE := riscv64-unknown-elf + TARGET_TRIPLE := riscv64-none-elf CFLAGS_ARCH := -march=rv64imafdc_zicsr_zifencei -mabi=lp64d else ifeq ($(ARCH),x86_64) TARGET_TRIPLE := x86_64-linux-gnu diff --git a/loader/Makefile b/loader/Makefile index c08c18bd9..a8a9ca943 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -3,6 +3,15 @@ # # SPDX-License-Identifier: BSD-2-Clause # + +ifneq ($(strip $(RUST_ONLY)),) + ARCH := dummy + BOARD := dummy + LINK_ADDRESS := dummy + TARGET_TRIPLE := dummy + SEL4_SDK := dummy +endif + ifeq ($(strip $(BUILD_DIR)),) $(error BUILD_DIR must be specified) endif @@ -35,27 +44,37 @@ else LD = $(TARGET_TRIPLE)-ld endif +RUSTC := rustc +CLIPPY := clippy-driver +RUSTFMT := rustfmt + ifeq ($(ARCH),aarch64) CFLAGS_AARCH64 := -mcpu=$(GCC_CPU) -mgeneral-regs-only -mstrict-align -mno-outline-atomics CFLAGS_ARCH := $(CFLAGS_AARCH64) -DARCH_aarch64 ASM_FLAGS_ARCH := -mcpu=$(GCC_CPU) ARCH_DIR := aarch64 + RUST_TARGET_TRIPLE := aarch64-unknown-none else ifeq ($(ARCH),riscv64) - CFLAGS_RISCV64 := -mcmodel=medany -march=rv64imac_zicsr_zifencei -mabi=lp64 + CFLAGS_RISCV64 := -mcmodel=medany -march=rv64gc -mabi=lp64d CFLAGS_ARCH := $(CFLAGS_RISCV64) -DARCH_riscv64 - ASM_FLAGS_ARCH := -march=rv64imac_zicsr_zifencei -mabi=lp64 + ASM_FLAGS_ARCH := -march=rv64gc -mabi=lp64d + RUST_TARGET_TRIPLE := riscv64gc-unknown-none-elf ARCH_DIR := riscv endif CFLAGS := -std=gnu11 -g -O3 -nostdlib -ffreestanding \ -MP -MD $(CFLAGS_ARCH) -DBOARD_$(BOARD) -I$(SEL4_SDK)/include \ -Wall -Werror -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations \ - -Wundef -Wno-nonnull -Wnested-externs + -Wundef -Wno-nonnull -Wnested-externs -ffunction-sections -fdata-sections ASM_FLAGS := $(ASM_FLAGS_ARCH) -g -MP -MD -I$(SEL4_SDK)/include +RUST_EDITION := 2024 +RUSTFLAGS := --edition $(RUST_EDITION) -g -C opt-level=2 +CLIPPY_FLAGS ?= + PROGS := loader.elf -OBJECTS := loader.o crt0.o uart.o cutil.o +OBJECTS := loader.o crt0.o uart.o cutil.o libpage_tables.a ifeq ($(ARCH),aarch64) OBJECTS += util64.o el.o exceptions.o init.o mmu.o cpus.o @@ -63,6 +82,8 @@ else ifeq ($(ARCH),riscv64) OBJECTS += exceptions.o init.o mmu.o cpus.o sbi.o endif +RUST_CRATES := page_tables + LINKSCRIPT_INPUT := $(ARCH).ld LINKSCRIPT := $(BUILD_DIR)/link.ld @@ -80,6 +101,18 @@ $(BUILD_DIR)/%.o : src/$(ARCH_DIR)/%.c $(BUILD_DIR)/%.o : src/%.c $(CC) -c $(CFLAGS) $< -o $@ +# Note: having multiple libs with staticlib will give duplicate linker symbol +# issues. Use "--crate-type rlib" instead, but then we need to link a single +# copy of the rust corelibs. For now this is fine. +$(BUILD_DIR)/lib%.a : src/%.rs + $(RUSTC) $(RUSTFLAGS) \ + --emit dep-info,metadata,link \ + --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ + --target $(RUST_TARGET_TRIPLE) \ + --crate-type staticlib \ + --crate-name $(patsubst lib%.a,%,$(notdir $@)) \ + $< + -include $(BUILD_DIR)/*.d OBJPROG = $(addprefix $(BUILD_DIR)/, $(PROGS)) @@ -89,5 +122,41 @@ all: $(OBJPROG) $(LINKSCRIPT): $(LINKSCRIPT_INPUT) $(CPP) -DLINK_ADDRESS=$(LINK_ADDRESS) $< | grep -v "^#" > $@ +LDFLAGS := -T$(LINKSCRIPT) + $(OBJPROG): $(addprefix $(BUILD_DIR)/, $(OBJECTS)) $(LINKSCRIPT) - $(LD) -T$(LINKSCRIPT) $(addprefix $(BUILD_DIR)/, $(OBJECTS)) -o $@ + $(LD) $(LDFLAGS) --start-group $(addprefix $(BUILD_DIR)/, $(OBJECTS)) --end-group -o $@ + +rusttest_%: src/%.rs + $(RUSTC) $(RUSTFLAGS) \ + --emit dep-info,metadata,link \ + --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ + -Awarnings \ + --test \ + --crate-name "$@" \ + $< + +tests: $(addprefix rusttest_, $(RUST_CRATES)) + $(BUILD_DIR)/rusttest_page_tables + +rustclippy_%: src/%.rs + $(CLIPPY) $(RUSTFLAGS) \ + --emit dep-info,metadata,link \ + --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ + $(CLIPPYFLAGS) \ + -Cpanic=abort \ + --crate-type staticlib \ + --crate-name "$@" \ + $< + +clippy: $(addprefix rustclippy_, $(RUST_CRATES)) + +rustfmt_%: src/%.rs + $(RUSTFMT) $(RUSTFMT_FLAGS) \ + --edition $(RUST_EDITION) \ + $< + +rustfmt: $(addprefix rustfmt_, $(RUST_CRATES)) + +rustfmt-check: RUSTFMT_FLAGS += --check +rustfmt-check: rustfmt diff --git a/loader/aarch64.ld b/loader/aarch64.ld index 977ccc574..b45561a04 100644 --- a/loader/aarch64.ld +++ b/loader/aarch64.ld @@ -8,6 +8,12 @@ PHDRS all PT_LOAD AT (LINK_ADDRESS); } + +// text PT_LOAD FLAGS(5); /* RX */ +// rodata PT_LOAD FLAGS(4); /* RO */ +// data PT_LOAD FLAGS(6); /* RW */ +// bss PT_LOAD FLAGS(6); /* RW */ + SECTIONS { . = LINK_ADDRESS; @@ -17,16 +23,26 @@ SECTIONS .text : { _text = .; - *(.text.start) - *(.text*) - *(.rodata) + + KEEP(*(.text.start)) + *(.text .text.*) + _text_end = .; } :all + .rodata : + { + *(.rodata .rodata.* .rodata..Lanon.*) + } :all + .data : { _data = .; - *(.data) + *(.data .data.*) + *(.data.*) + + KEEP(*(.data.uart_addr)) + _data_end = .; } :all @@ -34,6 +50,7 @@ SECTIONS { _bss = .; *(.bss) + *(.bss.*) *(COMMON) . = ALIGN(4); _bss_end = .; diff --git a/loader/riscv64.ld b/loader/riscv64.ld index f7ae1240e..98c94c490 100644 --- a/loader/riscv64.ld +++ b/loader/riscv64.ld @@ -15,27 +15,37 @@ SECTIONS .text : { _text = .; - *(.text.start) - *(.text*) - *(.rodata) + + KEEP(*(.text.start)) + *(.text .text.*) + _text_end = .; } :all + .rodata : + { + *(.rodata .rodata.* .rodata..Lanon.*) + } :all + .data : { _data = .; - *(.data) + *(.data .data.*) + *(.data.*) __global_pointer$ = . + 0x800; - *(.srodata) - *(.sdata) - _data_end = .; + *(.srodata) + *(.sdata) + + KEEP(*(.data.uart_addr)) + + _data_end = .; } :all .bss : { _bss = .; - *(.sbss) *(.bss) + *(.bss.*) *(COMMON) . = ALIGN(4); _bss_end = .; diff --git a/loader/src/aarch64/mmu.c b/loader/src/aarch64/mmu.c index 8ef6427ce..95ba14124 100644 --- a/loader/src/aarch64/mmu.c +++ b/loader/src/aarch64/mmu.c @@ -6,27 +6,42 @@ */ #include +#include #include "el.h" #include "../arch.h" #include "../cutil.h" #include "../uart.h" -void el1_mmu_enable(void); -void el2_mmu_enable(void); +void el1_mmu_enable(uintptr_t ttbr0_el1, uintptr_t ttbr1_el1); +void el2_mmu_enable(uintptr_t ttbr0_el2); -/* Paging structures for kernel mapping */ -uint64_t boot_lvl0_upper[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl1_upper[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl2_upper[1 << 9] ALIGN(1 << 12); +struct AArch64ReturnValue { + uintptr_t ttbr0_el2; + uintptr_t ttbr0_el1; + uintptr_t ttbr1_el1; +}; -/* Paging structures for identity mapping */ -uint64_t boot_lvl0_lower[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl1_lower[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl2_lower[1 << 9] ALIGN(1 << 12); +struct Region regions[] = { + { .start = 0x60000000, .top = 0xc0000000 - 1, .arch_attrs.is_ram = true }, + { .start = 0x9000000, .top = 0x9000000 + 0xfff, .arch_attrs.is_ram = false }, +}; + +uint8_t page_table_bytes[PAGE_TABLE_SIZE][MAX_NUM_PAGE_TABLES] ALIGN(PAGE_TABLE_SIZE); + +extern struct AArch64ReturnValue aarch64_setup_pagetables( + uint64_t kernel_first_vaddr, uint64_t kernel_first_paddr, + void *regions_ptr, uintptr_t regions_len, + uint8_t page_table_bytes[PAGE_TABLE_SIZE][MAX_NUM_PAGE_TABLES]); int arch_mmu_enable(int logical_cpu) { + struct AArch64ReturnValue pt = aarch64_setup_pagetables( + 0x8060000000, 0x60000000, + ®ions, ARRAY_SIZE(regions), + page_table_bytes + ); + int r; enum el el; r = ensure_correct_el(logical_cpu); @@ -37,9 +52,9 @@ int arch_mmu_enable(int logical_cpu) LDR_PRINT("INFO", logical_cpu, "enabling MMU\n"); el = current_el(); if (el == EL1) { - el1_mmu_enable(); + el1_mmu_enable(pt.ttbr0_el1, pt.ttbr1_el1); } else if (el == EL2) { - el2_mmu_enable(); + el2_mmu_enable(pt.ttbr0_el2); } else { LDR_PRINT("ERROR", logical_cpu, "unknown EL for MMU enable\n"); } diff --git a/loader/src/aarch64/util64.S b/loader/src/aarch64/util64.S index ccf1889b0..fd94ebe36 100644 --- a/loader/src/aarch64/util64.S +++ b/loader/src/aarch64/util64.S @@ -308,7 +308,6 @@ END_FUNC(el1_mmu_disable) BEGIN_FUNC(el2_mmu_disable) stp x29, x30, [sp, #-16]! - stp x27, x28, [sp, #-16]! mov x29, sp /* Disable caches */ @@ -323,14 +322,18 @@ BEGIN_FUNC(el2_mmu_disable) */ bl invalidate_icache - ldp x27, x28, [sp], #16 ldp x29, x30, [sp], #16 ret END_FUNC(el2_mmu_disable) +/* + * Enables the MMU for EL2. + * Takes two arguments the physical address for TTBR0_EL1 (x0) and TTBR1_EL1 (x1). + */ BEGIN_FUNC(el1_mmu_enable) stp x29, x30, [sp, #-16]! stp x27, x28, [sp, #-16]! + /* move caller-saved to callee-saved registers */ mov x29, sp mov x27, x0 mov x28, x1 @@ -358,10 +361,8 @@ BEGIN_FUNC(el1_mmu_enable) msr tcr_el1, x10 /* Setup page tables */ - adrp x8, boot_lvl0_lower - msr ttbr0_el1, x8 - adrp x8, boot_lvl0_upper - msr ttbr1_el1, x8 + msr ttbr0_el1, x27 /* argument 0 */ + msr ttbr1_el1, x28 /* argument 1 */ isb /* invalidate all TLB entries for EL1 */ @@ -374,12 +375,18 @@ BEGIN_FUNC(el1_mmu_enable) ldp x27, x28, [sp], #16 ldp x29, x30, [sp], #16 ret - END_FUNC(el1_mmu_enable) +/* + * Enables the MMU for EL2. + * Takes one argument, the physical address for TTBR0_EL2 (x0). + */ BEGIN_FUNC(el2_mmu_enable) stp x29, x30, [sp, #-16]! + stp x27, x28, [sp, #-16]! + /* move caller-saved to callee-saved registers */ mov x29, sp + mov x28, x0 /* Disable the MMU */ bl el2_mmu_disable @@ -403,8 +410,7 @@ BEGIN_FUNC(el2_mmu_enable) isb /* Setup page tables */ - adrp x8, boot_lvl0_lower - msr ttbr0_el2, x8 + msr ttbr0_el2, x28 /* argument 0 */ isb /* invalidate all TLB entries for EL2 */ @@ -423,9 +429,9 @@ BEGIN_FUNC(el2_mmu_enable) dsb ish isb + ldp x27, x28, [sp], #16 ldp x29, x30, [sp], #16 ret - END_FUNC(el2_mmu_enable) .extern arm_secondary_cpu_c_entry diff --git a/loader/src/aligned_regions.rs b/loader/src/aligned_regions.rs new file mode 100644 index 000000000..4405e85ac --- /dev/null +++ b/loader/src/aligned_regions.rs @@ -0,0 +1,271 @@ +// +// Copyright 2026, UNSW +// +// SPDX-License-Identifier: BSD-2-Clause +// + +//! This function provides an iterator which is useful for generating +//! page tables in one pass, with a constant amount of "extra" space for +//! bookkeeping. It transforms a set of discontiguous regions that cover +//! multiple levels of page table structures into a set of aligned regions, +//! useful for filling out paging structures. An additional piece of information +//! we need to maintain is when we move between levels. + +use core::array; +use core::cmp::min; + +use crate::Region; + +/// Bits from [start, end) +/// Note: Bit indices are 'u32' as this is what rust tends to use for usize::BITS, +/// and for checked_shl, and otherwise. This makes our life simpler. +fn bits_of_range(value: usize, start: u32, end: u32) -> usize { + assert!(start < end); + assert!(start < usize::BITS); + assert!(end <= usize::BITS); + + // Handle the maximum-shift case. + let mask = if let Some(bit) = 1usize.checked_shl(end) { + bit - 1 + } else { + debug_assert!(end == usize::BITS); + usize::MAX + }; + + (value & mask) >> start +} + +fn indices_of_level( + // [size_bits, count_bits) + level_bits: &[(u32, u32); LEVELS], + level: usize, + value: usize, +) -> usize { + let (size_bits, count_bits) = level_bits[level]; + + bits_of_range(value, size_bits, size_bits + count_bits) +} + +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct AlignedRegionsIter<'a, I, const LEVELS: usize> +where + I: Iterator, +{ + /// Array of (size_bits, count_bits), descending, for each level + level_bits: [(u32, u32); LEVELS], + input_regions_iter: I, + current_input_region: Option<&'a Region>, + current_addr: usize, +} + +impl<'a, I, const LEVELS: usize> AlignedRegionsIter<'a, I, LEVELS> +where + I: Iterator, +{ + // TODO: method on iter? + pub fn new(iter: I, level_bits: [(u32, u32); LEVELS]) -> Self { + for ((upper_size, _), (lower_size, lower_count)) in + level_bits.windows(2).map(|s| (s[0], s[1])) + { + assert!(upper_size == lower_size + lower_count); + } + + Self { + level_bits, + input_regions_iter: iter, + current_input_region: None, + current_addr: 0, + } + } +} + +impl<'a, I, const LEVELS: usize> Iterator for AlignedRegionsIter<'a, I, LEVELS> +where + I: Iterator, +{ + type Item = (usize, [usize; LEVELS], usize, u64); + + fn next(&mut self) -> Option { + let region = match self.current_input_region { + Some(r) => r, + None => { + let Some(region) = self.input_regions_iter.next() else { + // We exit our iterator here as we have no more work to do. + return None; + }; + + assert!(region.start < region.top); + + self.current_input_region = Some(region); + // Guarantees the loop invariant. + self.current_addr = region.start; + + region + } + }; + + let current_addr = self.current_addr; + + // Loop invariant. + assert!(current_addr < region.top); + + let size = region.top.checked_sub(current_addr).unwrap() + 1; + let size_bits = size.ilog2(); + // FIXME: Once MSRV is > 1.97, use .lowest_one() method. + let align_bits = if current_addr == 0 { + size_bits + } else { + current_addr.trailing_zeros() + }; + + // The correct pt size bits we can use it the smallest of the size + // and the alignment; we can't use a 21-bit aligned region if + // we have a 15-bit region, since it would overrun. + let align_bits = min(align_bits, size_bits); + + let level = self + .level_bits + .map(|(size, _)| size) + .iter() + .position(|&level_size_bits| align_bits >= level_size_bits) + .expect("bad input; regions should be aligned to at least the lowest level"); + + let level_indices: [usize; LEVELS] = + array::from_fn(|level| indices_of_level(&self.level_bits, level, current_addr)); + + let next_addr = current_addr.wrapping_add(1 << self.level_bits[level].0); + + if next_addr > region.top || next_addr == 0 { + self.current_input_region = None; + } + + self.current_addr = next_addr; + + // SAFETY: raw contains all valid bitpatterns + let raw_arch_attrs = unsafe { region.arch_attrs.raw }; + + Some((level, level_indices, current_addr, raw_arch_attrs)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + extern crate std; + use std::vec; + use std::vec::Vec; + + use crate::RegionArchAttrs; + + #[test] + fn test_bits_range() { + assert_eq!(bits_of_range(0b0110, 1, 3), 0b11); + assert_eq!(bits_of_range(0b1001, 1, 3), 0b00); + assert_eq!(bits_of_range(usize::MAX, 0, usize::BITS), usize::MAX); + assert_eq!(bits_of_range(usize::MAX, 4, usize::BITS), usize::MAX >> 4); + } + + #[test] + #[should_panic] + fn test_bits_range_not_allowed() { + bits_of_range(0b1001, 1, 1); + } + + #[test] + #[should_panic] + fn test_bits_range_not_allowed2() { + bits_of_range(0b1001, 4, 3); + } + + #[test] + fn test_bits_level() { + let levels = [(32, 16), (24, 8), (12, 12), (8, 4)]; + assert_eq!(indices_of_level(&levels, 0, 10 << 32), 10); + assert_eq!(indices_of_level(&levels, 1, 10 << 24), 10); + assert_eq!(indices_of_level(&levels, 2, 10 << 12), 10); + assert_eq!(indices_of_level(&levels, 3, 10 << 8), 10); + } + + #[test] + #[should_panic] + fn test_invalid_range_aligned_regions() { + let iter = [Region::EMPTY; 5].iter(); + // Should be reverse + AlignedRegionsIter::new(iter, [(8, 4), (12, 12), (24, 8), (32, 16)]); + } + + #[test] + fn test_ok_range_aligned_regions() { + let iter = [Region::EMPTY; 5].iter(); + AlignedRegionsIter::new(iter, [(32, 16), (24, 8), (12, 12), (8, 4)]); + } + + #[test] + fn test_iter_regions_simple() { + let aarch64_levels = [(39, 9), (30, 9), (21, 9), (12, 9)]; + let mut iter = AlignedRegionsIter::new( + // This should give us 4 4k regions + [Region { + start: 0, + top: 0x3fff, + arch_attrs: RegionArchAttrs { is_ram: false }, + }] + .iter(), + aarch64_levels, + ); + + let indices: Vec<_> = iter.collect(); + + assert_eq!( + indices, + vec![ + (3, [0, 0, 0, 0], 0x0000, 0x0), + (3, [0, 0, 0, 1], 0x1000, 0x0), + (3, [0, 0, 0, 2], 0x2000, 0x0), + (3, [0, 0, 0, 3], 0x3000, 0x0), + ] + ); + } + + #[test] + fn test_iter_regions_multi_layer() { + let aarch64_levels = [(39, 9), (30, 9), (21, 9), (12, 9)]; + let mut iter = AlignedRegionsIter::new( + [ + // This should give us 4 4k regions + Region { + start: 0, + top: 0x3fff, + arch_attrs: RegionArchAttrs { is_ram: false }, + }, + // Then this will give us 2 2M region and then 4 4k regions + Region { + start: 0x200000, + top: 0x603fff, + arch_attrs: RegionArchAttrs { is_ram: true }, + }, + ] + .iter(), + aarch64_levels, + ); + + let indices: Vec<_> = iter.collect(); + + assert_eq!( + indices, + vec![ + (3, [0, 0, 0, 0], 0x0000, 0x0), + (3, [0, 0, 0, 1], 0x1000, 0x0), + (3, [0, 0, 0, 2], 0x2000, 0x0), + (3, [0, 0, 0, 3], 0x3000, 0x0), + (2, [0, 0, 1, 0], 0x200000, 0x1), + (2, [0, 0, 2, 0], 0x400000, 0x1), + (3, [0, 0, 3, 0], 0x600000, 0x1), + (3, [0, 0, 3, 1], 0x601000, 0x1), + (3, [0, 0, 3, 2], 0x602000, 0x1), + (3, [0, 0, 3, 3], 0x603000, 0x1), + ] + ); + } +} diff --git a/loader/src/arch.h b/loader/src/arch.h index 69b4ba148..bb83b4ade 100644 --- a/loader/src/arch.h +++ b/loader/src/arch.h @@ -6,6 +6,9 @@ #pragma once +#include +#include + /** * The layout and naming scheme of the functions in these files has meaning: * @@ -35,3 +38,17 @@ void arch_init(void); void arch_set_exception_handler(void); int arch_mmu_enable(int logical_cpu); void arch_jump_to_kernel(int logical_cpu); + +union RegionArchAttrs { + bool is_ram; + uint64_t raw; +}; + +struct Region { + uintptr_t start; + uintptr_t top; + union RegionArchAttrs arch_attrs; +}; + +#define PAGE_TABLE_SIZE 4096 +#define MAX_NUM_PAGE_TABLES 64 diff --git a/loader/src/c_interop.rs b/loader/src/c_interop.rs new file mode 100644 index 000000000..526ec80f0 --- /dev/null +++ b/loader/src/c_interop.rs @@ -0,0 +1,73 @@ +#[cfg(not(test))] +mod real_hardware { + use core::ffi::c_char; + use core::ffi::CStr; + use core::fmt; + use core::fmt::Write; + use core::panic::PanicInfo; + + unsafe extern "C" { + safe fn fail() -> !; + // safe fn putc(c: c_char); + unsafe fn puts(s: *const c_char); + } + + /// Exposed only for print macro. + #[doc(hidden)] + pub(crate) struct Writer; + + impl fmt::Write for Writer { + fn write_str(&mut self, s: &str) -> Result<(), fmt::Error> { + for c in s.bytes() { + unsafe { puts(CStr::from_bytes_with_nul_unchecked(&[c, 0]).as_ptr()) }; + } + Ok(()) + } + } + + #[macro_export] + macro_rules! __print { + ($($arg:tt)*) => {{ + use core::fmt::Write; + $crate::c_interop::Writer{}.write_fmt(format_args!($($arg)*)).unwrap() + }} + } + + #[macro_export] + macro_rules! __println { + () => {{ + $crate::__print!("\n"); + }}; + + ($($arg:tt)*) => {{ + $crate::__print!("{}\n", format_args!($($arg)*)); + }} + } + + pub(crate) use __println as println; + + #[panic_handler] + fn panic(info: &PanicInfo) -> ! { + println!("panicked"); + + if writeln!(Writer, "{}", info).is_err() { + // If writeln!() fails (which it should never as our fmt::Write) never + // fails, then just don't print the extra information. + println!("panicked (information unknown)"); + } + + fail(); + } +} + +#[cfg(test)] +mod for_tests { + extern crate std; + pub(crate) use std::println; +} + +#[cfg(test)] +pub(crate) use for_tests::*; + +#[cfg(not(test))] +pub(crate) use real_hardware::*; diff --git a/loader/src/loader.c b/loader/src/loader.c index 67c09b149..45a21de04 100644 --- a/loader/src/loader.c +++ b/loader/src/loader.c @@ -100,7 +100,6 @@ static int print_lock = 0; void start_kernel(int logical_cpu) { - LDR_PRINT("INFO", logical_cpu, "enabling MMU\n"); int r = arch_mmu_enable(logical_cpu); if (r != 0) { LDR_PRINT("ERROR", logical_cpu, "failed to enable MMU: "); diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs new file mode 100644 index 000000000..43433084a --- /dev/null +++ b/loader/src/page_tables.rs @@ -0,0 +1,1146 @@ +// +// Copyright 2026, UNSW +// +// SPDX-License-Identifier: BSD-2-Clause +// + +#![no_std] +// We prefer indices as it matches the semantics of PT indices +#![allow(clippy::needless_range_loop)] + +mod aligned_regions; +mod c_interop; + +use core::fmt; +use core::mem; +use core::mem::MaybeUninit; +use core::slice; + +use aligned_regions::AlignedRegionsIter; + +const PAGE_TABLE_SIZE: usize = 4096; + +const fn divmod(x: u64, y: u64) -> (u64, u64) { + (x / y, x % y) +} + +const fn mask(n: u64) -> u64 { + (1 << n) - 1 +} + +const fn round_down(n: u64, x: u64) -> u64 { + let (_, m) = divmod(n, x); + if m == 0 { + n + } else { + n - m + } +} + +const fn align_down(n: u64, bits: u64) -> u64 { + round_down(n, 1 << bits) +} + +pub mod aarch64 { + //! For AArch64, our page tables use the Stage 1 descriptor formats + //! for both EL2 (TTBR0_EL2) and EL1 (TTBR0_EL1/TTBR1_EL1). + //! Stage 2 descriptors are only used when in the EL1&0 regime; which is not + //! the case when in EL2. + + use super::*; + + pub const LVL0_BITS: u64 = 9; + pub const LVL1_BITS: u64 = 9; + pub const LVL2_BITS: u64 = 9; + pub const LVL3_BITS: u64 = 9; + + pub fn lvl0_index(addr: u64) -> usize { + let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS + LVL1_BITS)) & mask(LVL0_BITS); + idx as usize + } + + pub fn lvl1_index(addr: u64) -> usize { + let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS)) & mask(LVL1_BITS); + idx as usize + } + + pub fn lvl2_index(addr: u64) -> usize { + let idx = (addr >> (BLOCK_BITS_2MB)) & mask(LVL2_BITS); + idx as usize + } + + pub fn lvl3_index(addr: u64) -> usize { + let idx = (addr >> PAGE_BITS_4KB) & mask(LVL3_BITS); + idx as usize + } + + /// Stage 1 translation table page/block descriptors have bits[4:2] containing + /// AttrIndex[2:0]. The AttrIndex values depends on our configuration of + /// the `MAIR_EL1` or `MAIR_EL2` registers done in util64.S; + /// This also needs to match the values that seL4 uses. + #[allow(non_upper_case_globals, reason = "matching ARM naming convention")] + pub mod s1_mair_attr_index { + pub const MT_DEVICE_nGnRnE: u64 = 0b000; + pub const MT_DEVICE_nGnRE: u64 = 0b001; + pub const MT_DEVICE_GRE: u64 = 0b010; + pub const MT_NORMAL_NC: u64 = 0b011; + pub const MT_NORMAL: u64 = 0b100; + } + + pub mod descriptor_type { + //! The translation table descriptor formats, as per §D8.3 "Translation + //! table descriptor formats" of ARM DDI 0487 L.b. Specifically, + //! as per "Table D8-48 Determination of descriptor type" + + /// Descriptor type: Table. Condition is lookup level != 3. + pub const TABLE: u64 = 0b11; + /// Descriptor type: Page. Condition is lookup level == 3. + pub const PAGE: u64 = 0b11; + /// Descriptor type: Block. Condition is lookup level != 3. + pub const BLOCK: u64 = 0b01; + /// Descriptor type: Invalid. Strictly speaking bit[1] does not matter. + pub const INVALID: u64 = 0b00; + } + + pub mod shareability_attributes { + //! Per §D8.6.2 "Stage 1 Shareability attributes", these contain the + //! shareability attributes of the descriptor OA for normal-cacheable + //! memory. + + /// Non-shareable + pub const NON_SHAREABLE: u64 = 0b00; + /// Outer-shareable + pub const OUTER_SHAREABLE: u64 = 0b10; + /// Inner-shareable + pub const INNER_SHAREABLE: u64 = 0b11; + } + + /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, + /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address + /// is bits [47:n], and: + /// + /// > For the 4KB granule size, the level 1 descriptor n is 30, + /// > and the level 2 descriptor n is 21. + pub const BLOCK_BITS_1GB: u64 = 30; + + /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, + /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address + /// is bits [47:n], and: + /// + /// > For the 4KB granule size, the level 1 descriptor n is 30, + /// > and the level 2 descriptor n is 21. + pub const BLOCK_BITS_2MB: u64 = 21; + + // TODO: + + pub const BLOCK_BITS_512GB: u64 = 39; + pub const PAGE_BITS_4KB: u64 = 12; + + /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and + /// "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b; + /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" + pub fn block_descriptor(level: usize, addr: u64, attr_index: u64) -> u64 { + // Per Table D8-48, Condition for descriptor_type::BLOCK is level != 3. + assert!(level != 3); + + let upper_attributes: u64 = 0; + + let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { + // Match what the seL4 kernel uses for its page tables, which + // is especially necessary for SMP booting which relies on it + // for coherency. See the comment in seL4 `release_secondary_cpus()`. + shareability_attributes::INNER_SHAREABLE + } else { + // Per $R_{PYFVQ}$: + // > If a region is mapped as Device memory or Normal Non-cacheable + // > memory after all enabled translation stages, then the region + // > has an effective Shareability attribute of Outer Shareable. + // + // We override the value we place in here to OUTER_SHAREABLE to match + // how the hardware behaves. This is not necessary but for clarity. + shareability_attributes::OUTER_SHAREABLE + }; + + // AP[2:1], which we set as 0b00 for read/write access: + // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1 + // stage 2: 0b00 is RW for EL2 and no perms for EL1. + const AP_KERNEL_RW: u64 = 0b00; + + // bit[11] is the not global (nG) field, we leave as 0 (global). + // bit[10] is the access flag; depending on FEAT_HAFDBS, when software + // manages the AF memory accesses to the page/block when AF=0 + // raise an Access Fault; when hardware manages the AF it will + // become 1. + // bit[9:8] is SH[1:0] containing stage 1 shareability attributes + // bit[7:6] contains AP[2:1] + // bit[5] is RES0 + // bit[4:2] contains AttrIndex + let lower_attributes: u64 = + (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); + + // bits[47:n] + let output_address: u64 = addr + & !mask(match level { + 1 => BLOCK_BITS_1GB, + 2 => BLOCK_BITS_2MB, + _ => panic!("unsupported level {level} for block descriptor"), + }); + + // address must not have bits above 47 set. + assert!(addr & mask(48) == addr); + + // bits[63:50] describing the "Upper attributes" are left at 0. + // bits[49:48] are RES0 + // bits[47:n] contain the Output address + // bits[n-1:12] are RES0 + // bits[11:2] contain the "Lower attributes" + // bits[1:0] contains the descriptor type + upper_attributes | output_address | lower_attributes | descriptor_type::BLOCK + } + + /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and + /// "Figure D8-15 VMSAv8-64 Page descriptor formats" of ARM DDI0487L.b; + /// specifically subfigure "4KB granule 48-bit OA". + pub fn page_descriptor(addr: u64, attr_index: u64) -> u64 { + // The main difference between a page descriptor and block descriptor + // is in the size of the output address (OA) and in the descriptor type. + + let upper_attributes: u64 = 0; + + let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { + // Match what the seL4 kernel uses for its page tables, which + // is especially necessary for SMP booting which relies on it + // for coherency. + shareability_attributes::INNER_SHAREABLE + } else { + // Per $R_{PYFVQ}$: + // > If a region is mapped as Device memory or Normal Non-cacheable + // > memory after all enabled translation stages, then the region + // > has an effective Shareability attribute of Outer Shareable. + // We override the value we place in here to OUTER_SHAREABLE to match + // how the hardware behaves. + shareability_attributes::OUTER_SHAREABLE + }; + + // AP[2:1], which we set as 0b00 for read/write access: + // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1/El2 (priv) + const AP_KERNEL_RW: u64 = 0b00; + + // bit[11] is the not global (nG) field, we leave as 0 (global). + // bit[10] is the access flag; depending on FEAT_HAFDBS, when software + // manages the AF memory accesses to the page/block when AF=0 + // raise an Access Fault; when hardware manages the AF it will + // become 1. + // bit[9:8] is SH[1:0] containing stage 1 shareability attributes + // bit[7:6] contains AP[2:1] + // bit[5] is RES0 + // bit[4:2] contains AttrIndex + let lower_attributes: u64 = + (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); + + // bits[47:12] + let output_address: u64 = addr & !mask(12); + + // address must not have bits above 47 set. + assert!(addr & mask(48) == addr); + + // bits[63:50] describing the "Upper attributes" are left at 0. + // bits[49:48] are RES0 + // bits[47:12] contain the Output address + // bits[11:2] contain the "Lower attributes" + // bits[1:0] contains the descriptor type + upper_attributes | output_address | lower_attributes | descriptor_type::PAGE + } + + /// Per "Table D8-50 Stage 1 VMSAv8-64 Table descriptor fields" and + /// "Figure D8-12 VMSAv8-64 Table descriptor formats" of ARM DDI0487L.b; + /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" + pub fn table_descriptor(addr: *const u8) -> u64 { + // Per Table D8-48, Condition for descriptor_type::TABLE is level != 3. + + let addr: u64 = addr.addr().try_into().expect("usize in u64"); + + // We don't set any of these attributes, most are hardware-feature conditional + let attributes: u64 = 0; + + // address must not have bits above 47 or below 12 set + assert!(addr & mask(12) == 0x0); + assert!(addr & mask(48) == addr); + + let next_level_table_address = addr; + + // bits[63:59] are "Attributes" + // bits[58:51] are ignored + // bits[50:48] are RES0 + // bits[47:m] is the next-level table address + // note: here m=12 for 4KB granule + // bits[m-1:12] are RES0 + // so this doesn't exist for 4KB granule + // bits[11:2] are ignored + // bits[1:0] contain the descriptor type + attributes | next_level_table_address | descriptor_type::TABLE + } +} + +mod riscv64 { + pub(crate) const BLOCK_BITS_1GB: u64 = 30; + pub(crate) const BLOCK_BITS_2MB: u64 = 21; + pub(crate) const PAGE_BITS_4K: u64 = 12; + + pub(crate) const PAGE_TABLE_INDEX_BITS: u64 = 9; + pub(crate) const PAGE_SHIFT: u64 = 12; + /// This sets the page table entry bits: D,A,X,W,R. + pub(crate) const PTE_TYPE_BITS: u64 = 0b11001110; + // TODO: where does this come from? + pub(crate) const PTE_TYPE_TABLE: u64 = 0; + pub(crate) const PTE_TYPE_VALID: u64 = 1; + + pub(crate) const PTE_PPN0_SHIFT: u64 = 10; + + /// Due to RISC-V having various virtual memory setups, we have this generic function to + /// figure out the page-table index given the total number of page table levels for the + /// platform and which level we are currently looking at. + pub fn pt_index(pt_levels: usize, addr: u64, level: usize) -> usize { + let pt_index_bits = PAGE_TABLE_INDEX_BITS * (pt_levels - level) as u64; + let idx = (addr >> (pt_index_bits + PAGE_SHIFT)) % 512; + + idx as usize + } + + /// Generate physical page number given an address + pub fn pte_ppn(addr: u64) -> u64 { + (addr >> PAGE_SHIFT) << PTE_PPN0_SHIFT + } + + pub fn pte_next(addr: *const u8) -> u64 { + pte_ppn(addr as u64) | PTE_TYPE_TABLE | PTE_TYPE_VALID + } + + pub fn pte_leaf(addr: u64) -> u64 { + pte_ppn(addr) | PTE_TYPE_BITS | PTE_TYPE_VALID + } +} + +/// RISC-V 64 page tables for our purposes uses the Sv39 translation scheme +/// (3-level page tables). +/// +/// It is split into two halves: the Upper/Kernel part of the page tables, +/// which matches the format seL4 expects. The lower half contains an +/// identity mapped region for the loader. +/// +/// ```txt +/// (512 GiB) +/// 512 +---- Level 1 ---+ 2^39 +/// | | +/// | (empty) | +/// | | +/// k+1 +----------------+ (1 GiB) +/// | Level 2 Kernel | ----------> +---- Level 2 ---+ +-------------+ +/// k +----------------+ | | ----------> | 2 MiB block | +/// | | 511 |----------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | 510 |----------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | |----------------| +------------- +/// | | (...) (...) (...) Kernel Regions +/// | | |----------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | l+1 |----------------| +-------------+ +/// | | | Level 3 Kernel | ----+ +/// | | l |----------------| | +/// | | | | | (2 MiB) +/// | | | | +-----> +-- Level 3 --+ +------------+ +/// | | | | | | ----------> | 4 KiB page | +/// | | | | 511 |-------------| +------------+ +/// | | | (empty) | | | ----------> | 4 KiB page | +/// | (empty) | | | |-------------| +------------+ +/// | | | | | | ----------> | 4 KiB page | +/// | | | | m |-------------| +------------+ p +/// | | | | | (empty) | +/// | | | | +-------------+ +/// | | | | +/// | | 0 +----------------+ +/// | | +/// | | +/// | | +/// | | +/// | | +/// 1 +----------------+ +/// | Level 2 Loader | ----------> +-- RAM +/// 0 +----------------+ +/// +/// +/// Where: +/// k = align_down(kernel_first_vaddr, 1GiB), +/// l = align_down(kernel_first_vaddr, 2MiB), +/// m = align_down(kernel_first_vaddr, 4KiB), +/// p = align_down(kernel_first_paddr, 4KiB), +/// ``` +/// +/// # Safety +/// - regions_ptr must be valid for as long as this function runs, +/// and regions_len must represent its length +/// - page_table_bytes must be aligned to PAGE_TABLE_SIZE +/// +/// +#[unsafe(no_mangle)] +pub unsafe extern "C" fn riscv64_setup_pagetables( + kernel_first_vaddr: u64, + kernel_first_paddr: u64, + // In-out param; storage and input + regions_ptr: *mut Region, + regions_len: usize, + // Storage used for page tables + page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], +) -> *const u8 { + use riscv64::{pt_index, pte_leaf, pte_next, BLOCK_BITS_1GB, BLOCK_BITS_2MB, PAGE_BITS_4K}; + + const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); + + let (pt_temporaries, mut serialise_page_table_to_paddr) = + make_helper_pt_serialisers::(page_table_bytes); + + struct Config { + riscv_pt_levels: usize, + } + let config = Config { riscv_pt_levels: 3 }; + + let num_pt_levels = config.riscv_pt_levels; + assert!(num_pt_levels == 3); + + // Manufacture the constants as per the diagram. + let k = align_down(kernel_first_vaddr, BLOCK_BITS_1GB); + let l = align_down(kernel_first_vaddr, BLOCK_BITS_2MB); + let m = align_down(kernel_first_vaddr, PAGE_BITS_4K); + let p = align_down(kernel_first_paddr, PAGE_BITS_4K); + + // Manufacture the kernel page tables + let kernel_lvl2_pt_paddr = { + let mut lvl2_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + + let mut paddr = p; + let index_l = pt_index(num_pt_levels, l, 2); + + lvl2_pt_kernel[index_l] = if kernel_first_vaddr.is_multiple_of(1 << BLOCK_BITS_2MB) { + assert!(paddr.is_multiple_of(1 << BLOCK_BITS_2MB)); + let pte = pte_leaf(paddr); + paddr += 1 << BLOCK_BITS_2MB; + pte + } else { + let mut lvl3_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + + let index_m = pt_index(num_pt_levels, m, 3); + + for index in index_m..512 { + lvl3_pt_kernel[index] = pte_leaf(paddr); + paddr += 1 << PAGE_BITS_4K; + } + + let kernel_lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt_kernel); + pte_next(kernel_lvl3_pt_paddr) + }; + + for index in (index_l + 1)..512 { + lvl2_pt_kernel[index] = pte_leaf(paddr); + paddr += 1 << BLOCK_BITS_2MB; + } + + serialise_page_table_to_paddr(&mut lvl2_pt_kernel) + }; + + let ram_lvl2_pt_paddr = { + let identity_mapped_regions: &mut [Region] = { + let regions = unsafe { slice::from_raw_parts_mut(regions_ptr, regions_len) }; + + for region in regions.iter_mut() { + // RISC-V ignores attributes + region.arch_attrs.raw = 0; + } + + // Need to use 'sort_unstable_by_key' as sort_by_key is not in-place. + regions.sort_unstable_by_key(|region| region.start); + + regions + }; + + setup_identity_page_tables::<3, _, Riscv64PtLayout, _>( + identity_mapped_regions, + pt_temporaries, + &mut serialise_page_table_to_paddr, + ) + }; + + // Manufacture the Level 1 table + let mut boot_lvl1_pt = [0u64; PAGE_TABLE_ENTRIES]; + + let index_k = pt_index(num_pt_levels, k, 1); + boot_lvl1_pt[index_k] = pte_next(kernel_lvl2_pt_paddr); + assert!(index_k != 0); + boot_lvl1_pt[0] = pte_next(ram_lvl2_pt_paddr); + + serialise_page_table_to_paddr(&mut boot_lvl1_pt) +} + +/// Note that "0" is a valid return value; instead the invalid value is +/// '-1', or usize::MAX. +#[repr(C)] +#[derive(Debug)] +pub struct AArch64ReturnValue { + pub ttbr0_el2: *const u8, + pub ttbr0_el1: *const u8, + pub ttbr1_el1: *const u8, +} + +impl AArch64ReturnValue { + const INVALID: *const u8 = usize::MAX as *const _; +} + +/// IMPORTANT: Keep in sync with C's `union RegionArchAttrs` +#[derive(Copy, Clone)] +#[repr(C)] +pub union RegionArchAttrs { + pub is_ram: bool, + pub raw: u64, +} + +impl fmt::Debug for RegionArchAttrs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + f.debug_struct("RegionArchAttrs") + // SAFETY: raw contains all valid bitpatterns + .field("raw", unsafe { &self.raw }) + .finish() + } +} + +/// Region is [start, end] *inclusive* as this avoids overflows. +/// IMPORTANT: Keep in sync with C's `struct Region` +#[derive(Debug, Copy, Clone)] +#[repr(C)] +pub struct Region { + pub start: usize, + pub top: usize, + pub arch_attrs: RegionArchAttrs, +} + +impl Region { + pub const EMPTY: Self = Self { + start: 0, + top: 0, + arch_attrs: RegionArchAttrs { raw: 0 }, + }; +} + +const MAX_NUM_PAGE_TABLES: usize = 64; +const NUM_TEMPORARIES: usize = 4; + +pub trait ArchPtLayout { + const LEVELS: usize = LEVELS; + + const MIN_LEVEL: usize; + const LEVEL_BITS: [(u32, u32); LEVELS]; + + fn leaf_entry(level: usize, address: usize, attributes: u64) -> u64; + fn table_entry(level: usize, address: *const u8) -> u64; +} + +struct AArch64PtLayout; + +impl ArchPtLayout<4> for AArch64PtLayout { + const MIN_LEVEL: usize = 1; + const LEVEL_BITS: [(u32, u32); 4] = [(39, 9), (30, 9), (21, 9), (12, 9)]; + + fn leaf_entry(level: usize, address: usize, attributes: u64) -> u64 { + assert!(level < Self::LEVELS); + + let address = address.try_into().unwrap(); + + if level == 3 { + aarch64::page_descriptor(address, attributes) + } else { + aarch64::block_descriptor(level, address, attributes) + } + } + + fn table_entry(_level: usize, address: *const u8) -> u64 { + aarch64::table_descriptor(address) + } +} + +struct Riscv64PtLayout; + +impl ArchPtLayout<3> for Riscv64PtLayout { + const MIN_LEVEL: usize = 1; + const LEVEL_BITS: [(u32, u32); 3] = [(30, 9), (21, 9), (12, 9)]; + + fn leaf_entry(level: usize, address: usize, attributes: u64) -> u64 { + assert!(level < Self::LEVELS); + assert_eq!(attributes, 0); + + let address = address.try_into().unwrap(); + + riscv64::pte_leaf(address) + } + + fn table_entry(_level: usize, address: *const u8) -> u64 { + riscv64::pte_next(address) + } +} + +fn setup_identity_page_tables< + const LEVELS: usize, + const PAGE_TABLE_ENTRIES: usize, + LAYOUT, + SerialiseFn, +>( + identity_mapped_regions: &mut [Region], + pt_temporaries: &mut [[u64; PAGE_TABLE_ENTRIES]; NUM_TEMPORARIES], + mut serialise_page_table_to_paddr: SerialiseFn, +) -> *const u8 +where + SerialiseFn: FnMut(&mut [u64; PAGE_TABLE_ENTRIES]) -> *const u8, + LAYOUT: ArchPtLayout, +{ + // Manufacture the RAM page tables, which is a little bit more complicated. + + // We maintain three active page tables, which contain our previous + // known page table data. As we process regions in ascending order, + // once we have exceeded the bounds of the current reservation we + // can simply push to the page_table_bytes storage and insert into + // the parent PT the descriptor. + + // We never actually use level 0 here, but it is nice to have because + // then the indices are the same as the level. + let pts_by_level = pt_temporaries.get_disjoint_mut([0, 1, 2, 3]).unwrap(); + + let mut iter = + AlignedRegionsIter::new(identity_mapped_regions.iter(), LAYOUT::LEVEL_BITS).peekable(); + + // RAM should never cross Level 0 boundaries, for the moment at least. + const MIN_LEVEL: usize = 1; + + while let Some((level, level_indices, current_addr, attributes)) = iter.next() { + assert!(level >= MIN_LEVEL); + + assert!(pts_by_level[level][level_indices[level]] == 0); + + pts_by_level[level][level_indices[level]] = + LAYOUT::leaf_entry(level, current_addr, attributes); + + // Invariant: the page tables in pts_by_level are either: + // (1) for the current level_indices, or + // (2) are empty/invalid and for a lower level. + // Similar, the level indices in our array are only meaningful + // from [0..=level]. + // + // Hence, when moving around, we only need to care about page tables + // in the range [0, level) inclusive, and can ignore those on + // lower levels. + // We start from the lowest level (parent) checking if the indices + // prefix (i.e. it, or any above it) have changed. Note that + // checking just the index would be invalid, in the case of say a + // [0, 0, 1, 0] -> [0, 0, 2, 0] change where level=3, as the + // level=2 row has changed, so our level=3 page table must be + // written out. + // We start from the parent and not the current level, because + // the change from [0, 0, 1, 0] -> [0, 0, 1, 1] should not write + // out the page table. (similarly, [0, 0, 1, X] -> [0, 0, 1, X] + // for level=2). + // We don't need to care if next_level is higher than the current + // level, as this still means the current page table is valid. + + for level in (MIN_LEVEL..level).rev() { + // Two cases where we need to write out the page tables: + // either we are reaching the end (iter.peek() = None) + // or if the next one has different page tables to us. + let changed = match iter.peek() { + None => true, + Some((_, next_level_indices, _, _)) => { + level_indices[0..=level] != next_level_indices[0..=level] + } + }; + + // Flush the 'level + 1' (the entry in the current level's PT) + // into the 'level' PT (next level up) + // We could have written instead this for loop as + // `for level in (MIN_LEVEL+1..=level)` + // and then used `let parent_level = level - 1`. + if changed { + let pt_paddr = serialise_page_table_to_paddr(pts_by_level[level + 1]); + pts_by_level[level][level_indices[level]] = LAYOUT::table_entry(level, pt_paddr); + } + } + } + + serialise_page_table_to_paddr(&mut pt_temporaries[MIN_LEVEL]) +} + +fn make_helper_pt_serialisers( + page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], +) -> ( + &mut [[u64; PAGE_TABLE_ENTRIES]; NUM_TEMPORARIES], + impl FnMut(&mut [u64; PAGE_TABLE_ENTRIES]) -> *const u8, +) { + // FIXME: Replace once https://github.com/rust-lang/rust/issues/90091 is merged + let (page_table_bytes, pt_temporaries) = page_table_bytes + .split_first_chunk_mut::<{ MAX_NUM_PAGE_TABLES - NUM_TEMPORARIES }>() + .unwrap(); + + let pt_temporaries = { + let pt_temporaries: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; NUM_TEMPORARIES] = + pt_temporaries.try_into().unwrap(); + + for pt in pt_temporaries.iter_mut() { + for elem in pt { + elem.write(0); + } + } + + // SAFETY: we just initialised it. + let pt_temporaries = unsafe { + mem::transmute::< + &mut [[MaybeUninit; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], + &mut [[u8; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], + >(pt_temporaries) + }; + + // SAFETY: + // - all bitpatterns of u8 can be represented in u8. + // - alignment requirements are met by input requirements + unsafe { + assert!((pt_temporaries.as_ptr() as usize).is_multiple_of(PAGE_TABLE_SIZE)); + mem::transmute::< + &mut [[u8; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], + &mut [[u64; PAGE_TABLE_ENTRIES]; NUM_TEMPORARIES], + >(pt_temporaries) + } + }; + + let serialise_page_table_to_paddr = { + let page_tables_paddr_start: *const u8 = page_table_bytes.as_ptr().cast(); + + assert!((page_tables_paddr_start as usize).is_multiple_of(PAGE_TABLE_SIZE)); + + // This maintains the current end of the PT array. + let mut next_pt_paddr = page_tables_paddr_start; + let mut i = 0; + + move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> *const _ { + let pt_paddr = next_pt_paddr; + for (j, byte) in page_table + .iter() + .flat_map(|pte| pte.to_le_bytes()) + .enumerate() + { + page_table_bytes[i][j].write(byte); + } + + next_pt_paddr = next_pt_paddr.wrapping_add(PAGE_TABLE_SIZE); + i += 1; + page_table.fill(0); + + if cfg!(test) { + // HACK! For tests, we want stable page tables, but due to ASLR + // we get random things every time. Instead, let's make the + // paddr we return a relative-to-start-of-page-tables value. + return unsafe { pt_paddr.offset_from(page_tables_paddr_start) } as *const _; + } + + pt_paddr + } + }; + + (pt_temporaries, serialise_page_table_to_paddr) +} + +/// AArch64 loader page tables have two variations: +/// - Loader in EL2, then Stage 1 translations in use, so we have the +/// singular TTBR0_EL2 register containing the Level 0 table; +/// this allows virtual address in the range [0,2^48). +/// - Loader in EL1, then Stage 1 translations are in use, so we have both +/// the TTBR0_EL1 (covering vaddr in range [0,2^48)) and TTBR1_EL2 ( +/// (covering vaddr in the range [2^64-2^48,2^64)), and containing +/// the "Level 0 Lower" page table, and "Level 0 Upper" page table +/// physical addresses respectively. +/// +/// Thus, for EL2 loader, the singular Level 0 page table contains the table +/// descriptors for the "Level 1 Upper" and "Level 1 Lower" page tables. +/// For the EL1 loader, we instead have two Level 0 page tables, and +/// "Level 0 Lower" contains the "Level 1 Lower" descriptor, and "Level 0 +/// Upper" contains the "Level 1 Upper" descriptor. +/// Otherwise, the page tables layout from Level 1 downwards are identical +/// (but not necessarily the layout within the page/table/block descriptors). +/// +/// ```txt +/// (256 TiB) +/// 512 +-- Level 0 --+ 2^48 +/// | | +/// | (empty) | +/// | | +/// k+1 +-------------+ (512 GiB) +/// | Level 1 Upr | ----------> +-- Level 1 --+ +/// k +-------------+ | | +/// | | | (empty) | +/// | | | | +/// | | l+1 +-------------+ (1 GiB) +/// | | | Level 2 Upr | ----------> +-- Level 2 --+ +-------------+ +/// | | l +-------------+ | | ----------> | 2 MiB block | +/// | | | | 511 |-------------| +-------------+ +/// | | | (empty) | | | ----------> | 2 MiB block | +/// | | | | 510 |-------------| +-------------+ +/// | | +-------------+ | | ----------> | 2 MiB block | +/// | | |-------------| +-------------+ +/// | (empty) | Kernel Regions (...) (...) (...) +/// | | |-------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | m |-------------| +-------------+ p +/// | | | | +/// | | | (empty) | +/// | | | | +/// | | 0 +-------------+ +/// | | +/// | | +/// | | +/// 1 +-------------+ (512 GiB) +/// | Level 1 Lwr | ----------> +-- Level 1 --+ +/// 0 +-------------+ TODO: RAM. +/// +/// +/// Where: +/// k = align_down(kernel_first_vaddr, 512GiB), +/// l = align_down(kernel_first_vaddr, 1GiB), +/// m = align_down(kernel_first_vaddr, 2MiB), +/// p = align_down(kernel_first_paddr, 2MiB), +/// u = align_down(uart_base, 1GiB), +/// ``` +/// +/// # Safety +/// - regions_ptr must be valid for as long as this function runs, +/// and regions_len must represent its length +/// - page_table_bytes must be aligned to PAGE_TABLE_SIZE +/// +#[unsafe(no_mangle)] +pub unsafe extern "C" fn aarch64_setup_pagetables( + kernel_first_vaddr: u64, + kernel_first_paddr: u64, + // In-out param; storage and input + regions_ptr: *mut Region, + regions_len: usize, + // Storage used for page tables + page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], +) -> AArch64ReturnValue { + use aarch64::{ + block_descriptor, lvl0_index, lvl1_index, lvl2_index, + s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}, + table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, + }; + + const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); + + let (pt_temporaries, mut serialise_page_table_to_paddr) = + make_helper_pt_serialisers::(page_table_bytes); + + // Manufacture the constants as per the diagram. + let k = align_down(kernel_first_vaddr, BLOCK_BITS_512GB); + let l = align_down(kernel_first_vaddr, BLOCK_BITS_1GB); + let m = align_down(kernel_first_vaddr, BLOCK_BITS_2MB); + let p = align_down(kernel_first_paddr, BLOCK_BITS_2MB); + + // Manufacture the kernel page tables, which is relatively straightforward. + let kernel_lvl1_pt_paddr = { + // First, the Level 2 Upr table. + let lvl2_pt_paddr = { + let lvl2_pt_kernel = &mut pt_temporaries[0]; + + let mut vaddr = m; + let mut paddr = p; + while lvl1_index(m) == lvl1_index(vaddr) { + lvl2_pt_kernel[lvl2_index(vaddr)] = block_descriptor(2, paddr, MT_NORMAL); + + vaddr += 1 << BLOCK_BITS_2MB; + paddr += 1 << BLOCK_BITS_2MB; + } + + serialise_page_table_to_paddr(lvl2_pt_kernel) + }; + + // Then, the Level 1 Upr table. + let lvl1_pt_kernel = &mut pt_temporaries[0]; + lvl1_pt_kernel[lvl1_index(l)] = table_descriptor(lvl2_pt_paddr); + + serialise_page_table_to_paddr(lvl1_pt_kernel) + }; + + let ram_lvl1_pt_paddr = { + let identity_mapped_regions: &mut [Region] = { + let regions = unsafe { slice::from_raw_parts_mut(regions_ptr, regions_len) }; + + for region in regions.iter_mut() { + // SAFETY: We expect users to set is_ram appropriately. + region.arch_attrs.raw = if unsafe { region.arch_attrs.is_ram } { + // FIXME: For now, RAM is also mapped as DEVICE memory. + MT_DEVICE_nGnRnE + } else { + MT_DEVICE_nGnRnE + }; + } + + // Need to use 'sort_unstable_by_key' as sort_by_key is not in-place. + regions.sort_unstable_by_key(|region| region.start); + + regions + }; + + setup_identity_page_tables::<4, _, AArch64PtLayout, _>( + identity_mapped_regions, + pt_temporaries, + &mut serialise_page_table_to_paddr, + ) + }; + + struct Config { + hypervisor: bool, + } + let config = Config { hypervisor: true }; + + // Depending on whether we are in hypervisor mode, we either need to + // return the TTBR0_EL2 or TTBR[0,1]_EL1 values. We return u64::MAX + // so as to return garbage - an unaligned address outside of physical + // memory. + if config.hypervisor { + // Manufacture the Level 0 table, containing the kernel table + // and the RAM tables. + + let ttbr0_el2_pt = &mut pt_temporaries[0]; + + assert!(lvl0_index(k) != lvl0_index(0)); + ttbr0_el2_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); + ttbr0_el2_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); + + let ttbr0_el2 = serialise_page_table_to_paddr(ttbr0_el2_pt); + + AArch64ReturnValue { + ttbr0_el2, + ttbr0_el1: AArch64ReturnValue::INVALID, + ttbr1_el1: AArch64ReturnValue::INVALID, + } + } else { + let [ttbr0_el1_pt, ttbr1_el1_pt] = pt_temporaries.get_disjoint_mut([0, 1]).unwrap(); + + // Kernel in TTBR1 (Upper) + ttbr1_el1_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); + // Identity-mapped RAM in TTBR0 (Lower) + ttbr0_el1_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); + + let ttbr0_el1 = serialise_page_table_to_paddr(ttbr0_el1_pt); + let ttbr1_el1 = serialise_page_table_to_paddr(ttbr1_el1_pt); + + AArch64ReturnValue { + ttbr0_el2: AArch64ReturnValue::INVALID, + ttbr0_el1, + ttbr1_el1, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + extern crate std; + use std::unreachable; + use std::vec; + use std::vec::Vec; + + // Exclusive [start, end) + #[derive(Debug, PartialEq)] + struct WalkRegion { + v_start: u64, + v_end: u64, + p_start: u64, + p_end: u64, + // This includes *all* page table attributes + arch_value: u64, + } + + fn aarch64_walk_pt_level_order( + level: usize, + pte: u64, + vaddr: &mut u64, + pts: &[[u64; 512]], + regions: &mut Vec, + ) { + use aarch64::descriptor_type; + + // Level is [0, 4) + assert!(level < 4); + + let v_start = *vaddr; + let size = 1 + << match level { + 0 => aarch64::BLOCK_BITS_512GB, + 1 => aarch64::BLOCK_BITS_1GB, + 2 => aarch64::BLOCK_BITS_2MB, + 3 => aarch64::PAGE_BITS_4KB, + _ => unreachable!(), + }; + *vaddr += size; + let v_end = *vaddr; + + if pte == 0 { + return; + } + + let pte_type = pte & 0b11; + // bits [47: 12] + let pte_oa = pte & 0xfffffffff000; + let pte_attrs = pte & !0xfffffffff000; + + if level == 3 { + assert!(pte_type == descriptor_type::PAGE); + } + + if level != 3 && pte_type == descriptor_type::TABLE { + let next_level_pt_idx = (pte_oa as usize) / PAGE_TABLE_SIZE; + + let mut vaddr = v_start; + for &child_pte in pts[next_level_pt_idx].iter() { + aarch64_walk_pt_level_order(level + 1, child_pte, &mut vaddr, pts, regions); + } + } else { + regions.push(WalkRegion { + v_start, + v_end, + p_start: pte_oa, + p_end: pte_oa + size, + arch_value: pte_attrs, + }); + } + } + + fn aarch64_walk_pt_gather_regions(pts: &[[u64; 512]], root_idx: usize) -> Vec { + let mut regions = vec![]; + let mut vaddr = 0; + for &pte in pts[root_idx].iter() { + aarch64_walk_pt_level_order(0, pte, &mut vaddr, pts, &mut regions); + } + + let mut i = regions.len() - 1; + while i > 1 { + if regions[i].p_start == regions[i - 1].p_end + && regions[i].v_start == regions[i - 1].v_end + && regions[i].arch_value == regions[i - 1].arch_value + { + regions[i - 1].p_end = regions[i].p_end; + regions[i - 1].v_end = regions[i].v_end; + regions.remove(i); + } + + i -= 1; + } + + regions + } + + #[test] + fn qemu_aarch64() { + #[repr(align(4096))] + struct PtBytes([[MaybeUninit; 4096]; MAX_NUM_PAGE_TABLES]); + + let mut regions = [ + Region { + start: 0x60000000, + top: 0xc0000000 - 1, + arch_attrs: RegionArchAttrs { is_ram: true }, + }, + // UART + Region { + start: 0x9000000, + top: 0x9000fff, + arch_attrs: RegionArchAttrs { is_ram: false }, + }, + ]; + + // // FIXME: Derive from the kernel build system. + // if let Some(uart_base) = read_symbol_maybe(elf, "uart_addr") { + // let uart_base = align_down(uart_base, PAGE_BITS_4KB); + // regions.push(( + // PlatformConfigRegion { + // start: uart_base, + // end: uart_base + (1 << PAGE_BITS_4KB), + // }, + // MT_DEVICE_nGnRnE, + // )); + // } + // // FIXME: This is currently assuming implementation details of the BCM2711/ + // // Raspberry Pi 4B spin table implementation, as it is the only + // // platform we have that uses spin tables. Specifically, that + // // it is always located at the 0 page. + // if elf.find_symbol("cpus_release_addr").is_ok() { + // regions.push(( + // PlatformConfigRegion { + // start: 0x0, + // end: 1 << PAGE_BITS_4KB, + // }, + // MT_DEVICE_nGnRnE, + // )); + // } + + let mut page_table_bytes = PtBytes([[MaybeUninit::zeroed(); _]; _]); + + let pt_bases = unsafe { + aarch64_setup_pagetables( + /* kernel_first_vaddr */ 0x8060000000, + /* kernel_first_paddr */ 0x60000000, + regions.as_mut_ptr(), + regions.len(), + &mut page_table_bytes.0, + ) + }; + + let page_tables = unsafe { + mem::transmute::< + [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], + [[u64; 512]; MAX_NUM_PAGE_TABLES], + >(page_table_bytes.0) + }; + + assert_eq!(pt_bases.ttbr0_el1, AArch64ReturnValue::INVALID); + assert_eq!(pt_bases.ttbr1_el1, AArch64ReturnValue::INVALID); + assert_ne!(pt_bases.ttbr0_el2, AArch64ReturnValue::INVALID); + + let root_addr = pt_bases.ttbr0_el2 as usize; + + let walk_regions = + aarch64_walk_pt_gather_regions(&page_tables, root_addr / PAGE_TABLE_SIZE); + + assert_eq!( + walk_regions, + vec![ + // UART + WalkRegion { + v_start: 0x9000000, + v_end: 0x9001000, + p_start: 0x9000000, + p_end: 0x9001000, + arch_value: 0x603, + }, + // RAM + WalkRegion { + v_start: 0x60000000, + v_end: 0xc0000000, + p_start: 0x60000000, + p_end: 0xc0000000, + arch_value: 0x601, + }, + // seL4 + WalkRegion { + v_start: 0x8060000000, + v_end: 0x8080000000, + p_start: 0x60000000, + p_end: 0x80000000, + arch_value: 0x711, + }, + ] + ); + } +} diff --git a/loader/src/riscv/mmu.c b/loader/src/riscv/mmu.c index 7751b25e9..8e71db4cb 100644 --- a/loader/src/riscv/mmu.c +++ b/loader/src/riscv/mmu.c @@ -6,17 +6,13 @@ */ #include +#include #include "../arch.h" #include "../cutil.h" -/* Paging structures for kernel mapping */ -uint64_t boot_lvl1_pt[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl2_pt[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl3_pt[1 << 9] ALIGN(1 << 12); -/* Paging structures for identity mapping */ -uint64_t boot_lvl2_pt_loader[1 << 9] ALIGN(1 << 12); - +/* Pointers to the top-level paging structures */ +uintptr_t riscv64_boot_lvl1_pt; /* * This is the encoding for the MODE field of the satp register when @@ -26,8 +22,25 @@ uint64_t boot_lvl2_pt_loader[1 << 9] ALIGN(1 << 12); #define RISCV_PGSHIFT 12 +struct Region regions[] = { + { .start = 0x80200000, .top = 0x100000000 - 1, .arch_attrs.is_ram = true }, +}; + +uint8_t page_table_bytes[PAGE_TABLE_SIZE][MAX_NUM_PAGE_TABLES] ALIGN(PAGE_TABLE_SIZE); + +extern uintptr_t riscv64_setup_pagetables( + uint64_t kernel_first_vaddr, uint64_t kernel_first_paddr, + void *regions_ptr, uintptr_t regions_len, + uint8_t page_table_bytes[PAGE_TABLE_SIZE][MAX_NUM_PAGE_TABLES]); + int arch_mmu_enable(int logical_cpu) { + uintptr_t riscv64_boot_lvl1_pt = riscv64_setup_pagetables( + 0xffffffff80200000, 0x80200000, + ®ions, ARRAY_SIZE(regions), + page_table_bytes + ); + // The RISC-V privileged spec (20211203), section 4.1.11 says that the // SFENCE.VMA instruction may need to be executed before or after writing // to satp. I don't understand why we do it before compared to after. @@ -36,7 +49,7 @@ int arch_mmu_enable(int logical_cpu) asm volatile( "csrw satp, %0\n" : - : "r"(VM_MODE | (uintptr_t)boot_lvl1_pt >> RISCV_PGSHIFT) + : "r"(VM_MODE | riscv64_boot_lvl1_pt >> RISCV_PGSHIFT) : ); asm volatile("fence.i" ::: "memory"); diff --git a/tool/microkit/src/loader.rs b/tool/microkit/src/loader.rs index 53a2597e2..860caca70 100644 --- a/tool/microkit/src/loader.rs +++ b/tool/microkit/src/loader.rs @@ -9,298 +9,24 @@ use crate::uimage::uimage_serialise; use crate::util::{mb, round_up, struct_to_bytes}; use std::fs::File; use std::io::{BufWriter, Write}; +use std::mem; use std::ops::Range; use std::path::Path; -macro_rules! grab_symbol { - ($elf: expr, $symbol_name: literal) => { - $elf.find_symbol($symbol_name) - .expect(concat!("Could not find '", $symbol_name, "' symbol")) - }; -} - -const PAGE_TABLE_SIZE: usize = 4096; - -pub mod aarch64 { - //! For AArch64, our page tables use the Stage 1 descriptor formats - //! for both EL2 (TTBR0_EL2) and EL1 (TTBR0_EL1/TTBR1_EL1). - //! Stage 2 descriptors are only used when in the EL1&0 regime; which is not - //! the case when in EL2. - - use crate::util::mask; - - pub const LVL0_BITS: u64 = 9; - pub const LVL1_BITS: u64 = 9; - pub const LVL2_BITS: u64 = 9; - - pub fn lvl0_index(addr: u64) -> usize { - let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS + LVL1_BITS)) & mask(LVL0_BITS); - idx as usize - } - - pub fn lvl1_index(addr: u64) -> usize { - let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS)) & mask(LVL1_BITS); - idx as usize - } - - pub fn lvl2_index(addr: u64) -> usize { - let idx = (addr >> (BLOCK_BITS_2MB)) & mask(LVL2_BITS); - idx as usize - } - - /// Stage 1 translation table page/block descriptors have bits[4:2] containing - /// AttrIndex[2:0]. The AttrIndex values depends on our configuration of - /// the `MAIR_EL1` or `MAIR_EL2` registers done in util64.S; - /// This also needs to match the values that seL4 uses. - #[allow(non_upper_case_globals, reason = "matching ARM naming convention")] - pub mod s1_mair_attr_index { - pub const MT_DEVICE_nGnRnE: u64 = 0b000; - pub const MT_DEVICE_nGnRE: u64 = 0b001; - pub const MT_DEVICE_GRE: u64 = 0b010; - pub const MT_NORMAL_NC: u64 = 0b011; - pub const MT_NORMAL: u64 = 0b100; - } - - pub mod descriptor_type { - //! The translation table descriptor formats, as per §D8.3 "Translation - //! table descriptor formats" of ARM DDI 0487 L.b. Specifically, - //! as per "Table D8-48 Determination of descriptor type" - - /// Descriptor type: Table. Condition is lookup level != 3. - pub const TABLE: u64 = 0b11; - /// Descriptor type: Page. Condition is lookup level == 3. - pub const PAGE: u64 = 0b11; - /// Descriptor type: Block. Condition is lookup level != 3. - pub const BLOCK: u64 = 0b01; - /// Descriptor type: Invalid. Strictly speaking bit[1] does not matter. - pub const INVALID: u64 = 0b00; - } - - pub mod shareability_attributes { - //! Per §D8.6.2 "Stage 1 Shareability attributes", these contain the - //! shareability attributes of the descriptor OA for normal-cacheable - //! memory. - - /// Non-shareable - pub const NON_SHAREABLE: u64 = 0b00; - /// Outer-shareable - pub const OUTER_SHAREABLE: u64 = 0b10; - /// Inner-shareable - pub const INNER_SHAREABLE: u64 = 0b11; - } - - /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, - /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address - /// is bits [47:n], and: - /// - /// > For the 4KB granule size, the level 1 descriptor n is 30, - /// > and the level 2 descriptor n is 21. - pub const BLOCK_BITS_1GB: u64 = 30; - - /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, - /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address - /// is bits [47:n], and: - /// - /// > For the 4KB granule size, the level 1 descriptor n is 30, - /// > and the level 2 descriptor n is 21. - pub const BLOCK_BITS_2MB: u64 = 21; - - /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and - /// "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b; - /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" - pub fn block_descriptor(level: usize, addr: u64, attr_index: u64) -> u64 { - // Per Table D8-48, Condition for descriptor_type::BLOCK is level != 3. - assert!(level != 3); - - let upper_attributes: u64 = 0; - - let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { - // Match what the seL4 kernel uses for its page tables, which - // is especially necessary for SMP booting which relies on it - // for coherency. - shareability_attributes::INNER_SHAREABLE - } else { - // Per $R_{PYFVQ}$: - // > If a region is mapped as Device memory or Normal Non-cacheable - // > memory after all enabled translation stages, then the region - // > has an effective Shareability attribute of Outer Shareable. - // - // We override the value we place in here to OUTER_SHAREABLE to match - // how the hardware behaves. This is not necessary but for clarity. - shareability_attributes::OUTER_SHAREABLE - }; - - // AP[2:1], which we set as 0b00 for read/write access: - // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1 - // stage 2: 0b00 is RW for EL2 and no perms for EL1. - const AP_KERNEL_RW: u64 = 0b00; - - // bit[11] is the not global (nG) field, we leave as 0 (global). - // bit[10] is the access flag; depending on FEAT_HAFDBS, when software - // manages the AF memory accesses to the page/block when AF=0 - // raise an Access Fault; when hardware manages the AF it will - // become 1. - // bit[9:8] is SH[1:0] containing stage 1 shareability attributes - // bit[7:6] contains AP[2:1] - // bit[5] is RES0 - // bit[4:2] contains AttrIndex - let lower_attributes: u64 = - (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); - - // bits[47:n] - let output_address: u64 = addr - & !mask(match level { - 1 => BLOCK_BITS_1GB, - 2 => BLOCK_BITS_2MB, - _ => panic!("unsupported level {level} for block descriptor"), - }); - - // address must not have bits above 47 set. - assert!(addr & mask(48) == addr); - - // bits[63:50] describing the "Upper attributes" are left at 0. - // bits[49:48] are RES0 - // bits[47:n] contain the Output address - // bits[n-1:12] are RES0 - // bits[11:2] contain the "Lower attributes" - // bits[1:0] contains the descriptor type - upper_attributes | output_address | lower_attributes | descriptor_type::BLOCK - } - - /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and - /// "Figure D8-15 VMSAv8-64 Page descriptor formats" of ARM DDI0487L.b; - /// specifically subfigure "4KB granule 48-bit OA". - pub fn page_descriptor(addr: u64, attr_index: u64) -> u64 { - // The main difference between a page descriptor and block descriptor - // is in the size of the output address (OA) and in the descriptor type. - - let upper_attributes: u64 = 0; - - let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { - // Match what the seL4 kernel uses for its page tables, which - // is especially necessary for SMP booting which relies on it - // for coherency. - shareability_attributes::INNER_SHAREABLE - } else { - // Per $R_{PYFVQ}$: - // > If a region is mapped as Device memory or Normal Non-cacheable - // > memory after all enabled translation stages, then the region - // > has an effective Shareability attribute of Outer Shareable. - // We override the value we place in here to OUTER_SHAREABLE to match - // how the hardware behaves. - shareability_attributes::OUTER_SHAREABLE - }; - - // AP[2:1], which we set as 0b00 for read/write access: - // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1/El2 (priv) - const AP_KERNEL_RW: u64 = 0b00; - - // bit[11] is the not global (nG) field, we leave as 0 (global). - // bit[10] is the access flag; depending on FEAT_HAFDBS, when software - // manages the AF memory accesses to the page/block when AF=0 - // raise an Access Fault; when hardware manages the AF it will - // become 1. - // bit[9:8] is SH[1:0] containing stage 1 shareability attributes - // bit[7:6] contains AP[2:1] - // bit[5] is RES0 - // bit[4:2] contains AttrIndex - let lower_attributes: u64 = - (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); - - // bits[47:12] - let output_address: u64 = addr & !mask(12); - - // address must not have bits above 47 set. - assert!(addr & mask(48) == addr); - - // bits[63:50] describing the "Upper attributes" are left at 0. - // bits[49:48] are RES0 - // bits[47:12] contain the Output address - // bits[11:2] contain the "Lower attributes" - // bits[1:0] contains the descriptor type - upper_attributes | output_address | lower_attributes | descriptor_type::PAGE - } - - /// Per "Table D8-50 Stage 1 VMSAv8-64 Table descriptor fields" and - /// "Figure D8-12 VMSAv8-64 Table descriptor formats" of ARM DDI0487L.b; - /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" - pub fn table_descriptor(addr: u64) -> u64 { - // Per Table D8-48, Condition for descriptor_type::TABLE is level != 3. - - // We don't set any of these attributes, most are hardware-feature conditional - let attributes: u64 = 0; - - // address must not have bits above 47 or below 12 set - assert!(addr & mask(12) == 0x0); - assert!(addr & mask(48) == addr); - - let next_level_table_address = addr; - - // bits[63:59] are "Attributes" - // bits[58:51] are ignored - // bits[50:48] are RES0 - // bits[47:m] is the next-level table address - // note: here m=12 for 4KB granule - // bits[m-1:12] are RES0 - // so this doesn't exist for 4KB granule - // bits[11:2] are ignored - // bits[1:0] contain the descriptor type - attributes | next_level_table_address | descriptor_type::TABLE - } -} - -mod riscv64 { - pub(crate) const BLOCK_BITS_2MB: u64 = 21; - pub(crate) const PAGE_BITS_4K: u64 = 12; - - pub(crate) const PAGE_TABLE_INDEX_BITS: u64 = 9; - pub(crate) const PAGE_SHIFT: u64 = 12; - /// This sets the page table entry bits: D,A,X,W,R. - pub(crate) const PTE_TYPE_BITS: u64 = 0b11001110; - // TODO: where does this come from? - pub(crate) const PTE_TYPE_TABLE: u64 = 0; - pub(crate) const PTE_TYPE_VALID: u64 = 1; - - pub(crate) const PTE_PPN0_SHIFT: u64 = 10; - - /// Due to RISC-V having various virtual memory setups, we have this generic function to - /// figure out the page-table index given the total number of page table levels for the - /// platform and which level we are currently looking at. - pub fn pt_index(pt_levels: usize, addr: u64, level: usize) -> usize { - let pt_index_bits = PAGE_TABLE_INDEX_BITS * (pt_levels - level) as u64; - let idx = (addr >> (pt_index_bits + PAGE_SHIFT)) % 512; - - idx as usize - } - - /// Generate physical page number given an address - pub fn pte_ppn(addr: u64) -> u64 { - (addr >> PAGE_SHIFT) << PTE_PPN0_SHIFT - } - - pub fn pte_next(addr: u64) -> u64 { - pte_ppn(addr) | PTE_TYPE_TABLE | PTE_TYPE_VALID - } - - pub fn pte_leaf(addr: u64) -> u64 { - pte_ppn(addr) | PTE_TYPE_BITS | PTE_TYPE_VALID - } -} - /// Checks that each region in the given list does not overlap with any other region. /// Panics upon finding an overlapping region -fn check_non_overlapping(regions: &Vec<(u64, &[u8])>) { +fn check_non_overlapping(regions: &Vec<(u64, u64)>) { let mut checked: Vec<(u64, u64)> = Vec::new(); - for (base, data) in regions { - let end = base + data.len() as u64; + for &(base, size) in regions.iter() { + let end = base + size; // Check that this does not overlap with any checked regions - for (b, e) in &checked { - if !(end <= *b || *base >= *e) { + for &(b, e) in checked.iter() { + if !(end <= b || base >= e) { panic!("Overlapping regions: [{base:x}..{end:x}) overlaps [{b:x}..{e:x})"); } } - checked.push((*base, end)); + checked.push((base, end)); } } @@ -419,53 +145,27 @@ impl<'a> Loader<'a> { } } - let Some(kernel_first_vaddr) = kernel_first_vaddr else { - panic!("INTERNAL: could not determine kernel_first_vaddr"); - }; - - let Some(kernel_first_paddr) = kernel_first_paddr else { - panic!("INTERNAL: could not determine kernel_first_paddr"); - }; - - let pagetable_vars = match config.arch { - Arch::Aarch64 => Loader::aarch64_setup_pagetables( - config, - &loader_elf, - kernel_first_vaddr, - kernel_first_paddr, - ), - Arch::Riscv64 => Loader::riscv64_setup_pagetables( - config, - &loader_elf, - kernel_first_vaddr, - kernel_first_paddr, - ), - Arch::X86_64 => unreachable!("x86_64 does not support creating a loader image"), - }; - let image_segment = loader_elf .segments - .into_iter() + .iter() .find(|segment| segment.loadable) .expect("Did not find loadable segment"); + + // Called "vaddr" but due to 1:1 mapping vaddr == paddr. let image_vaddr = image_segment.virt_addr; + // We have to clone here as the image executable is part of this function return object, // and the loader ELF is deserialised in this scope, so its lifetime will be shorter than // the return object. - let mut loader_image = image_segment.data().clone(); + let loader_image = image_segment.data().clone(); + + println!("kernel_first_vaddr: {kernel_first_vaddr:?}"); + println!("kernel_first_paddr: {kernel_first_paddr:?}"); if image_vaddr != loader_elf.entry { panic!("The loader entry point must be the first byte in the image"); } - for (var_addr, var_size, var_data) in pagetable_vars { - let offset = var_addr - image_vaddr; - assert!(var_size == var_data.len() as u64); - assert!(offset > 0); - assert!(offset <= loader_image.len() as u64); - loader_image[offset as usize..(offset + var_size) as usize].copy_from_slice(&var_data); - } - let kernel_entry = kernel_elf.entry; // initial task virt + pv_offset == initial task physical, so @@ -477,11 +177,6 @@ impl<'a> Loader<'a> { ui_p_reg_start + (initial_task_vaddr_range.end - initial_task_vaddr_range.start); assert!(ui_p_reg_end > ui_p_reg_start); - // This clone isn't too bad as it is just a Vec<(u64, &[u8])> - let mut all_regions_with_loader = regions.clone(); - all_regions_with_loader.push((image_vaddr, &loader_image)); - check_non_overlapping(&all_regions_with_loader); - let mut region_metadata = Vec::new(); let mut offset: u64 = 0; for (addr, data) in ®ions { @@ -494,10 +189,19 @@ impl<'a> Loader<'a> { offset += data.len() as u64; } - let size = std::mem::size_of::() as u64 - + region_metadata.iter().fold(0_u64, |acc, x| { - acc + x.size + std::mem::size_of::() as u64 - }); + let size = loader_image.len() as u64 + + mem::size_of::() as u64 + + (region_metadata.len() * mem::size_of::()) as u64 + + offset; + + let mut all_regions_with_loader: Vec<_> = regions + .iter() + .map(|&(base, data)| (base, data.len() as u64)) + .collect(); + all_regions_with_loader.push((image_vaddr, size)); + check_non_overlapping(&all_regions_with_loader); + + // TODO: Check contained within real RAM. let header = LoaderHeader64 { magic, @@ -539,6 +243,8 @@ impl<'a> Loader<'a> { bytes.extend_from_slice(data); } + assert!(bytes.len() as u64 == self.header.size); + bytes } @@ -603,312 +309,4 @@ impl<'a> Loader<'a> { Err(e) => panic!("Could not create '{}': {}", path.display(), e), } } - - fn riscv64_setup_pagetables( - config: &Config, - elf: &ElfFile, - first_vaddr: u64, - first_paddr: u64, - ) -> Vec<(u64, u64, [u8; PAGE_TABLE_SIZE])> { - let (text_addr, _) = grab_symbol!(elf, "_text"); - let (boot_lvl1_pt_addr, boot_lvl1_pt_size) = grab_symbol!(elf, "boot_lvl1_pt"); - let (boot_lvl2_pt_addr, boot_lvl2_pt_size) = grab_symbol!(elf, "boot_lvl2_pt"); - let (boot_lvl3_pt_addr, boot_lvl3_pt_size) = grab_symbol!(elf, "boot_lvl3_pt"); - let (boot_lvl2_pt_loader_addr, boot_lvl2_pt_loader_size) = - grab_symbol!(elf, "boot_lvl2_pt_loader"); - - // We map the loader using 2MB pages, so make sure the base is actually aligned. - assert!(text_addr.is_multiple_of(1 << riscv64::BLOCK_BITS_2MB)); - - let num_pt_levels = config.riscv_pt_levels.unwrap().levels(); - - let mut boot_lvl1_pt: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let text_index_lvl1 = riscv64::pt_index(num_pt_levels, text_addr, 1); - let pt_entry = riscv64::pte_next(boot_lvl2_pt_loader_addr); - let start = 8 * text_index_lvl1; - let end = start + 8; - boot_lvl1_pt[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl2_pt_loader: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let text_index_lvl2 = riscv64::pt_index(num_pt_levels, text_addr, 2); - for (page, i) in (text_index_lvl2..512).enumerate() { - let start = 8 * i; - let end = start + 8; - let addr = text_addr + ((page as u64) << riscv64::BLOCK_BITS_2MB); - let pt_entry = riscv64::pte_leaf(addr); - boot_lvl2_pt_loader[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - } - - { - let index = riscv64::pt_index(num_pt_levels, first_vaddr, 1); - let start = 8 * index; - let end = start + 8; - boot_lvl1_pt[start..end] - .copy_from_slice(&riscv64::pte_next(boot_lvl2_pt_addr).to_le_bytes()); - } - - let mut boot_lvl3_pt: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - let mut boot_lvl2_pt: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let mut index_lvl2 = riscv64::pt_index(num_pt_levels, first_vaddr, 2); - if !first_vaddr.is_multiple_of(1 << riscv64::BLOCK_BITS_2MB) { - let index_lvl3 = riscv64::pt_index(num_pt_levels, first_vaddr, 3); - for (page, i) in (index_lvl3..512).enumerate() { - let start = 8 * i; - let end = start + 8; - let addr = first_paddr + ((page as u64) << riscv64::PAGE_BITS_4K); - assert!(addr.is_multiple_of(1 << riscv64::PAGE_BITS_4K)); - let pt_entry = riscv64::pte_leaf(addr); - boot_lvl3_pt[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - let start = 8 * index_lvl2; - let end = start + 8; - let lvl3_pt_entry = riscv64::pte_next(boot_lvl3_pt_addr); - assert!(boot_lvl3_pt_addr.is_multiple_of(1 << riscv64::PAGE_BITS_4K)); - boot_lvl2_pt[start..end].copy_from_slice(&lvl3_pt_entry.to_le_bytes()); - index_lvl2 += 1; - } - let first_paddr_aligned = round_up(first_paddr, 1 << riscv64::BLOCK_BITS_2MB); - for (page, i) in (index_lvl2..512).enumerate() { - let start = 8 * i; - let end = start + 8; - let addr = first_paddr_aligned + ((page as u64) << riscv64::BLOCK_BITS_2MB); - assert!(addr.is_multiple_of(1 << riscv64::BLOCK_BITS_2MB)); - let pt_entry = riscv64::pte_leaf(addr); - boot_lvl2_pt[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - } - - vec![ - (boot_lvl1_pt_addr, boot_lvl1_pt_size, boot_lvl1_pt), - (boot_lvl2_pt_addr, boot_lvl2_pt_size, boot_lvl2_pt), - (boot_lvl3_pt_addr, boot_lvl3_pt_size, boot_lvl3_pt), - ( - boot_lvl2_pt_loader_addr, - boot_lvl2_pt_loader_size, - boot_lvl2_pt_loader, - ), - ] - } - - /// AArch64 loader page tables have two variations: - /// - Loader in EL2, then Stage 1 translations in use, so we have the - /// singular TTBR0_EL2 register containing the Level 0 table; - /// this allows virtual address in the range [0,2^48). - /// - Loader in EL1, then Stage 1 translations are in use, so we have both - /// the TTBR0_EL1 (covering vaddr in range [0,2^48)) and TTBR1_EL2 ( - /// (covering vaddr in the range [2^64-2^48,2^64)), and containing - /// the "Level 0 Lower" page table, and "Level 0 Upper" page table - /// physical addresses respectively. - /// - /// Thus, for EL2 loader, the singular Level 0 page table contains the table - /// descriptors for the "Level 1 Upper" and "Level 1 Lower" page tables. - /// For the EL1 loader, we instead have two Level 0 page tables, and - /// "Level 0 Lower" contains the "Level 1 Lower" descriptor, and "Level 0 - /// Upper" contains the "Level 1 Upper" descriptor. - /// Otherwise, the page tables layout from Level 1 downwards are identical - /// (but not necessarily the layout within the page/table/block descriptors). - /// - /// ```txt - /// (256 TiB) - /// 512 +-- Level 0 --+ 2^48 - /// | | - /// | (empty) | - /// | | - /// k+1 +-------------+ (512 GiB) - /// | Level 1 Upr | ----------> +-- Level 1 --+ - /// k +-------------+ | | - /// | | | (empty) | - /// | | | | - /// | | l+1 +-------------+ (1 GiB) - /// | | | Level 2 Upr | ----------> +-- Level 2 --+ +-------------+ - /// | | l +-------------+ | | ----------> | 2 MiB block | - /// | | | | 511 |-------------| +-------------+ - /// | | | (empty) | | | ----------> | 2 MiB block | - /// | | | | 510 |-------------| +-------------+ - /// | | +-------------+ | | ----------> | 2 MiB block | - /// | | |-------------| +-------------+ - /// | (empty) | Kernel Regions (...) (...) (...) - /// | | |-------------| +-------------+ - /// | | | | ----------> | 2 MiB block | - /// | | m |-------------| +-------------+ p - /// | | | | - /// | | | (empty) | - /// | | | | - /// | | 0 +-------------+ - /// | | - /// | | - /// | | - /// 1 +-------------+ (512 GiB) - /// | Level 1 Lwr | ----------> +-- Level 1 --+ - /// 0 +-------------+ | | - /// | (empty) | - /// | | - /// u+1 +-------------+ +-------------+ - /// | uart_base | ----------> | 1 GiB block | - /// u +-------------+ +-------------+ - /// | | - /// | (empty) | - /// | | - /// i+1 +-------------+ (1 GiB) - /// | Level 2 Lwr | ----------> +-- Level 2 --+ - /// i +-------------+ | | - /// | | | (empty) | - /// | (empty) | | | - /// | | t +-------------+ +-------------+ - /// +-------------+ | | ----------> | 2 MiB block | - /// |-------------| +-------------+ - /// | | ----------> | 2 MiB block | - /// |-------------| +-------------+ - /// Loader Regions (...) (...) (...) - /// |-------------| +-------------+ - /// | | ----------> | 2 MiB block | - /// |-------------| +-------------+ - /// | | ----------> | 2 MiB block | - /// s +-------------+ +-------------+ - /// | | - /// | (empty) | - /// | | - /// +-------------+ - /// - /// Where: - /// k = align_down(kernel_first_vaddr, 512GiB), - /// l = align_down(kernel_first_vaddr, 1GiB), - /// m = align_down(kernel_first_vaddr, 2MiB), - /// p = align_down(kernel_first_paddr, 2MiB), - /// u = align_down(uart_base, 1GiB), - /// i = align_down(loader_start_addr, 1GiB), - /// s = align_down(loader_start_addr, 2MiB), - /// t = align_up(loader_end_addr, 2MiB), - /// ``` - /// - fn aarch64_setup_pagetables( - _config: &Config, - elf: &ElfFile, - first_vaddr: u64, - first_paddr: u64, - ) -> Vec<(u64, u64, [u8; PAGE_TABLE_SIZE])> { - use aarch64::s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}; - - let (boot_lvl1_lower_addr, boot_lvl1_lower_size) = grab_symbol!(elf, "boot_lvl1_lower"); - let (boot_lvl1_upper_addr, boot_lvl1_upper_size) = grab_symbol!(elf, "boot_lvl1_upper"); - let (boot_lvl2_upper_addr, boot_lvl2_upper_size) = grab_symbol!(elf, "boot_lvl2_upper"); - let (boot_lvl0_lower_addr, boot_lvl0_lower_size) = grab_symbol!(elf, "boot_lvl0_lower"); - let (boot_lvl0_upper_addr, boot_lvl0_upper_size) = grab_symbol!(elf, "boot_lvl0_upper"); - let (boot_lvl2_lower_addr, boot_lvl2_lower_size) = grab_symbol!(elf, "boot_lvl2_lower"); - - let (loader_start_addr, _) = grab_symbol!(elf, "_loader_start"); - let (loader_end_addr, _) = grab_symbol!(elf, "_loader_end"); - - if aarch64::lvl1_index(loader_start_addr) != aarch64::lvl1_index(loader_end_addr) { - panic!("We only map 1GiB, but loader paddr range covers multiple GiB"); - } - - let mut boot_lvl0_lower: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let pt_entry = aarch64::table_descriptor(boot_lvl1_lower_addr); - boot_lvl0_lower[..8].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl1_lower: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - - // map optional UART MMIO in l1 1GB page, only available if CONFIG_PRINTING - if let Ok((uart_addr, uart_addr_size)) = elf.find_symbol("uart_addr") { - let data = elf - .get_data(uart_addr, uart_addr_size) - .expect("uart_addr not initialized"); - - let uart_base = u64::from_le_bytes(data[0..8].try_into().unwrap()); - - let lvl1_idx = aarch64::lvl1_index(uart_base); - - let pt_entry = aarch64::block_descriptor(1, uart_base, MT_DEVICE_nGnRnE); - - let start = 8 * lvl1_idx; - let end = 8 * (lvl1_idx + 1); - boot_lvl1_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl2_lower: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - - // 1GB lvl1 Table entry - let pt_entry = aarch64::table_descriptor(boot_lvl2_lower_addr); - let lvl1_idx = aarch64::lvl1_index(loader_start_addr); - let start = 8 * lvl1_idx; - let end = 8 * (lvl1_idx + 1); - boot_lvl1_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - - // map the loader 1:1 access into 2MB lvl2 Block entries for a 4KB granule - let lvl2_idx = aarch64::lvl2_index(loader_start_addr); - for i in lvl2_idx..=aarch64::lvl2_index(loader_end_addr) { - let entry_idx: u64 = - ((i - aarch64::lvl2_index(loader_start_addr)) << aarch64::BLOCK_BITS_2MB) as u64; - - let pt_entry = - aarch64::block_descriptor(2, loader_start_addr + entry_idx, MT_DEVICE_nGnRnE); - - let start = 8 * i; - let end = 8 * (i + 1); - boot_lvl2_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - - // TODO: this is a complete hack specific to BCM2711/Raspberry Pi 4B and - // will be removed with patches that re-do this loader mapping code. - if elf.find_symbol("cpus_release_addr").is_ok() { - let lvl2_idx = aarch64::lvl2_index(0); - // Make sure we don't override the loader mappings done above. - assert!(aarch64::lvl2_index(loader_start_addr) != lvl2_idx); - assert!(aarch64::lvl1_index(loader_start_addr) == aarch64::lvl1_index(0)); - - let pt_entry = aarch64::block_descriptor(2, lvl2_idx as u64, MT_DEVICE_nGnRnE); - - let start = 8 * lvl2_idx; - let end = 8 * (lvl2_idx + 1); - boot_lvl2_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl0_upper: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let pt_entry = aarch64::table_descriptor(boot_lvl1_upper_addr); - let idx = aarch64::lvl0_index(first_vaddr); - // For EL2. - boot_lvl0_lower[8 * idx..8 * (idx + 1)].copy_from_slice(&pt_entry.to_le_bytes()); - // For EL1. - boot_lvl0_upper[8 * idx..8 * (idx + 1)].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl1_upper: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let pt_entry = aarch64::table_descriptor(boot_lvl2_upper_addr); - let idx = aarch64::lvl1_index(first_vaddr); - boot_lvl1_upper[8 * idx..8 * (idx + 1)].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl2_upper: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - - let lvl2_idx = aarch64::lvl2_index(first_vaddr); - for i in lvl2_idx..512 { - let entry_idx: u64 = - ((i - aarch64::lvl2_index(first_vaddr)) << aarch64::BLOCK_BITS_2MB) as u64; - - let pt_entry = aarch64::block_descriptor(2, first_paddr + entry_idx, MT_NORMAL); - - let start = 8 * i; - let end = 8 * (i + 1); - boot_lvl2_upper[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } - - vec![ - (boot_lvl0_lower_addr, boot_lvl0_lower_size, boot_lvl0_lower), - (boot_lvl1_lower_addr, boot_lvl1_lower_size, boot_lvl1_lower), - (boot_lvl0_upper_addr, boot_lvl0_upper_size, boot_lvl0_upper), - (boot_lvl1_upper_addr, boot_lvl1_upper_size, boot_lvl1_upper), - (boot_lvl2_upper_addr, boot_lvl2_upper_size, boot_lvl2_upper), - (boot_lvl2_lower_addr, boot_lvl2_lower_size, boot_lvl2_lower), - ] - } } diff --git a/tool/microkit/src/sel4.rs b/tool/microkit/src/sel4.rs index d95084a74..ead67bb04 100644 --- a/tool/microkit/src/sel4.rs +++ b/tool/microkit/src/sel4.rs @@ -249,7 +249,7 @@ pub fn emulate_kernel_boot( } } -#[derive(Deserialize)] +#[derive(Deserialize, Debug, Clone)] pub struct PlatformConfigRegion { pub start: u64, pub end: u64, diff --git a/tool/microkit/src/util.rs b/tool/microkit/src/util.rs index 6f2c0e275..211c794d6 100644 --- a/tool/microkit/src/util.rs +++ b/tool/microkit/src/util.rs @@ -54,6 +54,14 @@ pub const fn round_down(n: u64, x: u64) -> u64 { } } +pub const fn align_up(n: u64, bits: u64) -> u64 { + round_up(n, 1 << bits) +} + +pub const fn align_down(n: u64, bits: u64) -> u64 { + round_down(n, 1 << bits) +} + pub fn is_power_of_two(n: u64) -> bool { assert!(n > 0); n & (n - 1) == 0