From b5917a782aa29ae5859ef88cabd0eb6f120d3edd Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Wed, 6 May 2026 15:14:41 +1000 Subject: [PATCH 01/29] loader(aarch64): pass PTs to enable MMU assembler As opposed to them being accessed via global variables directly. The reason to do this is to enable us to vary the page table placement from the C side without needing to touch or modify the assembly any further, which will enable the future commits for making the number of page page table structures dynamic, which is necessary for making the loader 1:1 all of RAM. Also, remove some redundant stack pops/pushs to util64.S. Signed-off-by: Julia Vassiliki --- loader/src/aarch64/mmu.c | 8 ++++---- loader/src/aarch64/util64.S | 26 ++++++++++++++++---------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/loader/src/aarch64/mmu.c b/loader/src/aarch64/mmu.c index 8ef6427ce..15fe83564 100644 --- a/loader/src/aarch64/mmu.c +++ b/loader/src/aarch64/mmu.c @@ -12,8 +12,8 @@ #include "../cutil.h" #include "../uart.h" -void el1_mmu_enable(void); -void el2_mmu_enable(void); +void el1_mmu_enable(void *a, void *b); +void el2_mmu_enable(void *a); /* Paging structures for kernel mapping */ uint64_t boot_lvl0_upper[1 << 9] ALIGN(1 << 12); @@ -37,9 +37,9 @@ int arch_mmu_enable(int logical_cpu) LDR_PRINT("INFO", logical_cpu, "enabling MMU\n"); el = current_el(); if (el == EL1) { - el1_mmu_enable(); + el1_mmu_enable(&boot_lvl0_lower, &boot_lvl0_upper); } else if (el == EL2) { - el2_mmu_enable(); + el2_mmu_enable(&boot_lvl0_lower); } else { LDR_PRINT("ERROR", logical_cpu, "unknown EL for MMU enable\n"); } diff --git a/loader/src/aarch64/util64.S b/loader/src/aarch64/util64.S index ccf1889b0..fd94ebe36 100644 --- a/loader/src/aarch64/util64.S +++ b/loader/src/aarch64/util64.S @@ -308,7 +308,6 @@ END_FUNC(el1_mmu_disable) BEGIN_FUNC(el2_mmu_disable) stp x29, x30, [sp, #-16]! - stp x27, x28, [sp, #-16]! mov x29, sp /* Disable caches */ @@ -323,14 +322,18 @@ BEGIN_FUNC(el2_mmu_disable) */ bl invalidate_icache - ldp x27, x28, [sp], #16 ldp x29, x30, [sp], #16 ret END_FUNC(el2_mmu_disable) +/* + * Enables the MMU for EL2. + * Takes two arguments the physical address for TTBR0_EL1 (x0) and TTBR1_EL1 (x1). + */ BEGIN_FUNC(el1_mmu_enable) stp x29, x30, [sp, #-16]! stp x27, x28, [sp, #-16]! + /* move caller-saved to callee-saved registers */ mov x29, sp mov x27, x0 mov x28, x1 @@ -358,10 +361,8 @@ BEGIN_FUNC(el1_mmu_enable) msr tcr_el1, x10 /* Setup page tables */ - adrp x8, boot_lvl0_lower - msr ttbr0_el1, x8 - adrp x8, boot_lvl0_upper - msr ttbr1_el1, x8 + msr ttbr0_el1, x27 /* argument 0 */ + msr ttbr1_el1, x28 /* argument 1 */ isb /* invalidate all TLB entries for EL1 */ @@ -374,12 +375,18 @@ BEGIN_FUNC(el1_mmu_enable) ldp x27, x28, [sp], #16 ldp x29, x30, [sp], #16 ret - END_FUNC(el1_mmu_enable) +/* + * Enables the MMU for EL2. + * Takes one argument, the physical address for TTBR0_EL2 (x0). + */ BEGIN_FUNC(el2_mmu_enable) stp x29, x30, [sp, #-16]! + stp x27, x28, [sp, #-16]! + /* move caller-saved to callee-saved registers */ mov x29, sp + mov x28, x0 /* Disable the MMU */ bl el2_mmu_disable @@ -403,8 +410,7 @@ BEGIN_FUNC(el2_mmu_enable) isb /* Setup page tables */ - adrp x8, boot_lvl0_lower - msr ttbr0_el2, x8 + msr ttbr0_el2, x28 /* argument 0 */ isb /* invalidate all TLB entries for EL2 */ @@ -423,9 +429,9 @@ BEGIN_FUNC(el2_mmu_enable) dsb ish isb + ldp x27, x28, [sp], #16 ldp x29, x30, [sp], #16 ret - END_FUNC(el2_mmu_enable) .extern arm_secondary_cpu_c_entry From 75b307c342b15b8d3654b3e1f351cf27c9f29ffd Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Mon, 18 May 2026 17:18:47 +1000 Subject: [PATCH 02/29] loader(aarch64): dynamically allocate PTs This performs cleanups to the loader code, to remove a bunch of boilerplate, and also makes the page table entries placed in the loader binary at the end. This is in preparation for moving towards mapping all of RAM in the loader. Signed-off-by: Julia Vassiliki --- loader/src/aarch64/mmu.c | 21 +- tool/microkit/src/loader.rs | 375 +++++++++++++++++++++++------------- tool/microkit/src/sel4.rs | 2 +- tool/microkit/src/util.rs | 8 + 4 files changed, 256 insertions(+), 150 deletions(-) diff --git a/loader/src/aarch64/mmu.c b/loader/src/aarch64/mmu.c index 15fe83564..39b4676c8 100644 --- a/loader/src/aarch64/mmu.c +++ b/loader/src/aarch64/mmu.c @@ -12,18 +12,13 @@ #include "../cutil.h" #include "../uart.h" -void el1_mmu_enable(void *a, void *b); -void el2_mmu_enable(void *a); +void el1_mmu_enable(uint64_t aarch64_pt_ttbr0_el1, uint64_t aarch64_pt_ttbr1_el1); +void el2_mmu_enable(uint64_t aarch64_pt_ttbr0_el2); -/* Paging structures for kernel mapping */ -uint64_t boot_lvl0_upper[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl1_upper[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl2_upper[1 << 9] ALIGN(1 << 12); - -/* Paging structures for identity mapping */ -uint64_t boot_lvl0_lower[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl1_lower[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl2_lower[1 << 9] ALIGN(1 << 12); +/* Pointers to the top-level paging structures */ +uint64_t aarch64_pt_ttbr0_el1; +uint64_t aarch64_pt_ttbr1_el1; +uint64_t aarch64_pt_ttbr0_el2; int arch_mmu_enable(int logical_cpu) { @@ -37,9 +32,9 @@ int arch_mmu_enable(int logical_cpu) LDR_PRINT("INFO", logical_cpu, "enabling MMU\n"); el = current_el(); if (el == EL1) { - el1_mmu_enable(&boot_lvl0_lower, &boot_lvl0_upper); + el1_mmu_enable(aarch64_pt_ttbr0_el1, aarch64_pt_ttbr1_el1); } else if (el == EL2) { - el2_mmu_enable(&boot_lvl0_lower); + el2_mmu_enable(aarch64_pt_ttbr0_el2); } else { LDR_PRINT("ERROR", logical_cpu, "unknown EL for MMU enable\n"); } diff --git a/tool/microkit/src/loader.rs b/tool/microkit/src/loader.rs index 53a2597e2..fd4519f10 100644 --- a/tool/microkit/src/loader.rs +++ b/tool/microkit/src/loader.rs @@ -6,19 +6,38 @@ use crate::elf::{ElfFile, ElfSegmentData}; use crate::sel4::{Arch, Config}; use crate::uimage::uimage_serialise; -use crate::util::{mb, round_up, struct_to_bytes}; +use crate::util::{align_down, align_up, mask, mb, round_up, struct_to_bytes}; +use std::cmp::min; use std::fs::File; use std::io::{BufWriter, Write}; +use std::mem; use std::ops::Range; use std::path::Path; macro_rules! grab_symbol { - ($elf: expr, $symbol_name: literal) => { + ($elf: expr, $symbol_name: expr) => { $elf.find_symbol($symbol_name) .expect(concat!("Could not find '", $symbol_name, "' symbol")) }; } +macro_rules! write_symbol { + ($loader_image: expr, $image_vaddr: expr, $elf: expr, $symbol: literal, $symbol_var: expr) => { + let (addr, size) = grab_symbol!($elf, $symbol); + let addr = usize::try_from(addr).expect("addr fits in usize"); + let size = usize::try_from(size).expect("size fits in usize"); + let image_vaddr = usize::try_from($image_vaddr).expect("vaddr fits in usize"); + + assert!(addr >= image_vaddr); + assert!(size == ::std::mem::size_of_val(&$symbol_var)); + + let offset: usize = (addr - image_vaddr); + assert!(offset <= $loader_image.len()); + + $loader_image[offset..(offset + size)].copy_from_slice(&$symbol_var.to_le_bytes()); + }; +} + const PAGE_TABLE_SIZE: usize = 4096; pub mod aarch64 { @@ -32,6 +51,7 @@ pub mod aarch64 { pub const LVL0_BITS: u64 = 9; pub const LVL1_BITS: u64 = 9; pub const LVL2_BITS: u64 = 9; + pub const LVL3_BITS: u64 = 9; pub fn lvl0_index(addr: u64) -> usize { let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS + LVL1_BITS)) & mask(LVL0_BITS); @@ -48,6 +68,11 @@ pub mod aarch64 { idx as usize } + pub fn lvl3_index(addr: u64) -> usize { + let idx = (addr >> PAGE_BITS_4KB) & mask(LVL3_BITS); + idx as usize + } + /// Stage 1 translation table page/block descriptors have bits[4:2] containing /// AttrIndex[2:0]. The AttrIndex values depends on our configuration of /// the `MAIR_EL1` or `MAIR_EL2` registers done in util64.S; @@ -105,6 +130,11 @@ pub mod aarch64 { /// > and the level 2 descriptor n is 21. pub const BLOCK_BITS_2MB: u64 = 21; + // TODO: + + pub const BLOCK_BITS_512GB: u64 = 39; + pub const PAGE_BITS_4KB: u64 = 12; + /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and /// "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b; /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" @@ -289,18 +319,18 @@ mod riscv64 { /// Checks that each region in the given list does not overlap with any other region. /// Panics upon finding an overlapping region -fn check_non_overlapping(regions: &Vec<(u64, &[u8])>) { +fn check_non_overlapping(regions: &Vec<(u64, u64)>) { let mut checked: Vec<(u64, u64)> = Vec::new(); - for (base, data) in regions { - let end = base + data.len() as u64; + for &(base, size) in regions.iter() { + let end = base + size; // Check that this does not overlap with any checked regions - for (b, e) in &checked { - if !(end <= *b || *base >= *e) { + for &(b, e) in checked.iter() { + if !(end <= b || base >= e) { panic!("Overlapping regions: [{base:x}..{end:x}) overlaps [{b:x}..{e:x})"); } } - checked.push((*base, end)); + checked.push((base, end)); } } @@ -330,6 +360,7 @@ pub struct Loader<'a> { header: LoaderHeader64, region_metadata: Vec, regions: Vec<(u64, &'a [u8])>, + page_table_bytes: Vec, word_size: usize, elf_machine: u16, entry: u64, @@ -427,28 +458,15 @@ impl<'a> Loader<'a> { panic!("INTERNAL: could not determine kernel_first_paddr"); }; - let pagetable_vars = match config.arch { - Arch::Aarch64 => Loader::aarch64_setup_pagetables( - config, - &loader_elf, - kernel_first_vaddr, - kernel_first_paddr, - ), - Arch::Riscv64 => Loader::riscv64_setup_pagetables( - config, - &loader_elf, - kernel_first_vaddr, - kernel_first_paddr, - ), - Arch::X86_64 => unreachable!("x86_64 does not support creating a loader image"), - }; - let image_segment = loader_elf .segments - .into_iter() + .iter() .find(|segment| segment.loadable) .expect("Did not find loadable segment"); + + // Called "vaddr" but due to 1:1 mapping vaddr == paddr. let image_vaddr = image_segment.virt_addr; + // We have to clone here as the image executable is part of this function return object, // and the loader ELF is deserialised in this scope, so its lifetime will be shorter than // the return object. @@ -458,14 +476,6 @@ impl<'a> Loader<'a> { panic!("The loader entry point must be the first byte in the image"); } - for (var_addr, var_size, var_data) in pagetable_vars { - let offset = var_addr - image_vaddr; - assert!(var_size == var_data.len() as u64); - assert!(offset > 0); - assert!(offset <= loader_image.len() as u64); - loader_image[offset as usize..(offset + var_size) as usize].copy_from_slice(&var_data); - } - let kernel_entry = kernel_elf.entry; // initial task virt + pv_offset == initial task physical, so @@ -477,11 +487,6 @@ impl<'a> Loader<'a> { ui_p_reg_start + (initial_task_vaddr_range.end - initial_task_vaddr_range.start); assert!(ui_p_reg_end > ui_p_reg_start); - // This clone isn't too bad as it is just a Vec<(u64, &[u8])> - let mut all_regions_with_loader = regions.clone(); - all_regions_with_loader.push((image_vaddr, &loader_image)); - check_non_overlapping(&all_regions_with_loader); - let mut region_metadata = Vec::new(); let mut offset: u64 = 0; for (addr, data) in ®ions { @@ -494,10 +499,63 @@ impl<'a> Loader<'a> { offset += data.len() as u64; } - let size = std::mem::size_of::() as u64 - + region_metadata.iter().fold(0_u64, |acc, x| { - acc + x.size + std::mem::size_of::() as u64 - }); + let partial_size = loader_image.len() as u64 + + mem::size_of::() as u64 + + (region_metadata.len() * mem::size_of::()) as u64 + + offset; + + let page_tables_paddr_start = image_vaddr + partial_size; + + let mut page_table_bytes = Vec::::new(); + match config.arch { + Arch::Aarch64 => { + let (ttbr0_el2, ttbr0_el1, ttbr1_el1) = Loader::aarch64_setup_pagetables( + config, + &loader_elf, + kernel_first_vaddr, + kernel_first_paddr, + page_tables_paddr_start, + &mut page_table_bytes, + ); + + write_symbol!( + loader_image, + image_vaddr, + loader_elf, + "aarch64_pt_ttbr0_el2", + ttbr0_el2 + ); + write_symbol!( + loader_image, + image_vaddr, + loader_elf, + "aarch64_pt_ttbr0_el1", + ttbr0_el1 + ); + write_symbol!( + loader_image, + image_vaddr, + loader_elf, + "aarch64_pt_ttbr1_el1", + ttbr1_el1 + ); + } + Arch::Riscv64 => { + todo!(); + } + Arch::X86_64 => unreachable!("x86_64 does not support creating a loader image"), + }; + + let size = partial_size + page_table_bytes.len() as u64; + + let mut all_regions_with_loader: Vec<_> = regions + .iter() + .map(|&(base, data)| (base, data.len() as u64)) + .collect(); + all_regions_with_loader.push((image_vaddr, size)); + check_non_overlapping(&all_regions_with_loader); + + // TODO: Check contained within real RAM. let header = LoaderHeader64 { magic, @@ -516,6 +574,7 @@ impl<'a> Loader<'a> { header, region_metadata, regions, + page_table_bytes, word_size: kernel_elf.word_size, elf_machine: kernel_elf.machine, entry: loader_elf.entry, @@ -539,6 +598,10 @@ impl<'a> Loader<'a> { bytes.extend_from_slice(data); } + bytes.extend_from_slice(&self.page_table_bytes); + + assert!(bytes.len() as u64 == self.header.size); + bytes } @@ -672,7 +735,7 @@ impl<'a> Loader<'a> { boot_lvl2_pt[start..end].copy_from_slice(&lvl3_pt_entry.to_le_bytes()); index_lvl2 += 1; } - let first_paddr_aligned = round_up(first_paddr, 1 << riscv64::BLOCK_BITS_2MB); + let first_paddr_aligned = align_up(first_paddr, riscv64::BLOCK_BITS_2MB); for (page, i) in (index_lvl2..512).enumerate() { let start = 8 * i; let end = start + 8; @@ -787,128 +850,168 @@ impl<'a> Loader<'a> { /// ``` /// fn aarch64_setup_pagetables( - _config: &Config, + config: &Config, elf: &ElfFile, - first_vaddr: u64, - first_paddr: u64, - ) -> Vec<(u64, u64, [u8; PAGE_TABLE_SIZE])> { - use aarch64::s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}; + kernel_first_vaddr: u64, + kernel_first_paddr: u64, + page_tables_paddr_start: u64, + page_table_bytes: &mut Vec, + ) -> (u64, u64, u64) { + use aarch64::{ + block_descriptor, lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, + s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}, + table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, + }; + + const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); + + let mut serialise_page_table_to_paddr = { + let page_tables_paddr_start = { + let aligned_pt_paddr_start = + page_tables_paddr_start.next_multiple_of(PAGE_TABLE_SIZE as u64); + if aligned_pt_paddr_start != page_tables_paddr_start { + let alignment_diff = + (aligned_pt_paddr_start - page_tables_paddr_start) as usize; + page_table_bytes.resize(alignment_diff, 0); + } + + aligned_pt_paddr_start + }; - let (boot_lvl1_lower_addr, boot_lvl1_lower_size) = grab_symbol!(elf, "boot_lvl1_lower"); - let (boot_lvl1_upper_addr, boot_lvl1_upper_size) = grab_symbol!(elf, "boot_lvl1_upper"); - let (boot_lvl2_upper_addr, boot_lvl2_upper_size) = grab_symbol!(elf, "boot_lvl2_upper"); - let (boot_lvl0_lower_addr, boot_lvl0_lower_size) = grab_symbol!(elf, "boot_lvl0_lower"); - let (boot_lvl0_upper_addr, boot_lvl0_upper_size) = grab_symbol!(elf, "boot_lvl0_upper"); - let (boot_lvl2_lower_addr, boot_lvl2_lower_size) = grab_symbol!(elf, "boot_lvl2_lower"); + // This maintains the current end of the PT array. + let mut next_pt_paddr = page_tables_paddr_start; + + move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> u64 { + let pt_paddr = next_pt_paddr; + page_table_bytes.extend(page_table.iter().flat_map(|pte| pte.to_le_bytes())); + next_pt_paddr += PAGE_TABLE_SIZE as u64; + page_table.fill(0); + pt_paddr + } + }; let (loader_start_addr, _) = grab_symbol!(elf, "_loader_start"); let (loader_end_addr, _) = grab_symbol!(elf, "_loader_end"); - - if aarch64::lvl1_index(loader_start_addr) != aarch64::lvl1_index(loader_end_addr) { + if lvl1_index(loader_start_addr) != lvl1_index(loader_end_addr) { panic!("We only map 1GiB, but loader paddr range covers multiple GiB"); } - let mut boot_lvl0_lower: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let pt_entry = aarch64::table_descriptor(boot_lvl1_lower_addr); - boot_lvl0_lower[..8].copy_from_slice(&pt_entry.to_le_bytes()); - } - - let mut boot_lvl1_lower: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - - // map optional UART MMIO in l1 1GB page, only available if CONFIG_PRINTING - if let Ok((uart_addr, uart_addr_size)) = elf.find_symbol("uart_addr") { + let uart_base = if let Ok((uart_addr, uart_addr_size)) = elf.find_symbol("uart_addr") { let data = elf .get_data(uart_addr, uart_addr_size) .expect("uart_addr not initialized"); - let uart_base = u64::from_le_bytes(data[0..8].try_into().unwrap()); + Some(u64::from_le_bytes(data[0..8].try_into().unwrap())) + } else { + None + }; + + // Manufacture the constants as per the diagram. + let k = align_down(kernel_first_vaddr, BLOCK_BITS_512GB); + let l = align_down(kernel_first_vaddr, BLOCK_BITS_1GB); + let m = align_down(kernel_first_vaddr, BLOCK_BITS_2MB); + let p = align_down(kernel_first_paddr, BLOCK_BITS_2MB); + let i = align_down(loader_start_addr, BLOCK_BITS_1GB); + let u = uart_base.map(|addr| align_down(addr, BLOCK_BITS_1GB)); + let s = align_down(loader_start_addr, BLOCK_BITS_2MB); + let t = align_up(loader_end_addr, BLOCK_BITS_2MB); + + // Manufacture the kernel page tables, which is relatively straightforward. + let kernel_lvl1_pt_paddr = { + // First, the Level 2 Upr table. + let lvl2_pt_paddr = { + let mut lvl2_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + + let mut vaddr = m; + let mut paddr = p; + while lvl1_index(m) == lvl1_index(vaddr) { + lvl2_pt_kernel[lvl2_index(vaddr)] = block_descriptor(2, paddr, MT_NORMAL); + + vaddr += 1 << BLOCK_BITS_2MB; + paddr += 1 << BLOCK_BITS_2MB; + } - let lvl1_idx = aarch64::lvl1_index(uart_base); + serialise_page_table_to_paddr(&mut lvl2_pt_kernel) + }; - let pt_entry = aarch64::block_descriptor(1, uart_base, MT_DEVICE_nGnRnE); + // Then, the Level 1 Upr table. + let mut lvl1_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + lvl1_pt_kernel[lvl1_index(l)] = table_descriptor(lvl2_pt_paddr); - let start = 8 * lvl1_idx; - let end = 8 * (lvl1_idx + 1); - boot_lvl1_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } + serialise_page_table_to_paddr(&mut lvl1_pt_kernel) + }; - let mut boot_lvl2_lower: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; + // Manufacture the loader page tables. + let loader_lvl1_pt_paddr = { + // First, the Level 2 Lwr table + let lvl2_pt_paddr = { + let mut lvl2_pt_loader = [0u64; PAGE_TABLE_ENTRIES]; - // 1GB lvl1 Table entry - let pt_entry = aarch64::table_descriptor(boot_lvl2_lower_addr); - let lvl1_idx = aarch64::lvl1_index(loader_start_addr); - let start = 8 * lvl1_idx; - let end = 8 * (lvl1_idx + 1); - boot_lvl1_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); + // Identity mapped: vaddr == paddr. + let mut addr = s; + while addr < t { + lvl2_pt_loader[lvl2_index(addr)] = block_descriptor(2, addr, MT_DEVICE_nGnRnE); - // map the loader 1:1 access into 2MB lvl2 Block entries for a 4KB granule - let lvl2_idx = aarch64::lvl2_index(loader_start_addr); - for i in lvl2_idx..=aarch64::lvl2_index(loader_end_addr) { - let entry_idx: u64 = - ((i - aarch64::lvl2_index(loader_start_addr)) << aarch64::BLOCK_BITS_2MB) as u64; + addr += 1 << BLOCK_BITS_2MB; + } - let pt_entry = - aarch64::block_descriptor(2, loader_start_addr + entry_idx, MT_DEVICE_nGnRnE); + // TODO: this is a complete hack specific to BCM2711/Raspberry Pi 4B and + // will be reworked with patches that re-do this loader mapping code. + if elf.find_symbol("cpus_release_addr").is_ok() { + // Make sure we don't override the loader mappings done above; + // and that this is located at 0x0. + assert!(s != 0); + assert!(i == 0); - let start = 8 * i; - let end = 8 * (i + 1); - boot_lvl2_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } + lvl2_pt_loader[lvl2_index(0)] = block_descriptor(2, 0, MT_DEVICE_nGnRnE); + } - // TODO: this is a complete hack specific to BCM2711/Raspberry Pi 4B and - // will be removed with patches that re-do this loader mapping code. - if elf.find_symbol("cpus_release_addr").is_ok() { - let lvl2_idx = aarch64::lvl2_index(0); - // Make sure we don't override the loader mappings done above. - assert!(aarch64::lvl2_index(loader_start_addr) != lvl2_idx); - assert!(aarch64::lvl1_index(loader_start_addr) == aarch64::lvl1_index(0)); + serialise_page_table_to_paddr(&mut lvl2_pt_loader) + }; - let pt_entry = aarch64::block_descriptor(2, lvl2_idx as u64, MT_DEVICE_nGnRnE); + // Then, the Level 1 Lwr table. + let mut lvl1_pt_loader = [0u64; PAGE_TABLE_ENTRIES]; + lvl1_pt_loader[lvl1_index(i)] = table_descriptor(lvl2_pt_paddr); - let start = 8 * lvl2_idx; - let end = 8 * (lvl2_idx + 1); - boot_lvl2_lower[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } + // map optional UART MMIO in l1 1GB page, only available if CONFIG_PRINTING + if let Some(u) = u { + // UART no overlap with Loader. + assert!(lvl1_index(i) != lvl1_index(u)); + lvl1_pt_loader[lvl1_index(u)] = block_descriptor(1, u, MT_DEVICE_nGnRnE); + } - let mut boot_lvl0_upper: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let pt_entry = aarch64::table_descriptor(boot_lvl1_upper_addr); - let idx = aarch64::lvl0_index(first_vaddr); - // For EL2. - boot_lvl0_lower[8 * idx..8 * (idx + 1)].copy_from_slice(&pt_entry.to_le_bytes()); - // For EL1. - boot_lvl0_upper[8 * idx..8 * (idx + 1)].copy_from_slice(&pt_entry.to_le_bytes()); - } + serialise_page_table_to_paddr(&mut lvl1_pt_loader) + }; - let mut boot_lvl1_upper: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let pt_entry = aarch64::table_descriptor(boot_lvl2_upper_addr); - let idx = aarch64::lvl1_index(first_vaddr); - boot_lvl1_upper[8 * idx..8 * (idx + 1)].copy_from_slice(&pt_entry.to_le_bytes()); - } + // Depending on whether we are in hypervisor mode, we either need to + // return the TTBR0_EL2 or TTBR[0,1]_EL1 values. We return u64::MAX + // so as to return garbage - an unaligned address outside of physical + // memory. + if config.hypervisor { + // Manufacture the Level 0 table, containing the kernel table + // and the RAM tables. - let mut boot_lvl2_upper: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; + let mut ttbr0_el2_pt = [0u64; PAGE_TABLE_ENTRIES]; - let lvl2_idx = aarch64::lvl2_index(first_vaddr); - for i in lvl2_idx..512 { - let entry_idx: u64 = - ((i - aarch64::lvl2_index(first_vaddr)) << aarch64::BLOCK_BITS_2MB) as u64; + ttbr0_el2_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); + ttbr0_el2_pt[lvl0_index(0)] = table_descriptor(loader_lvl1_pt_paddr); - let pt_entry = aarch64::block_descriptor(2, first_paddr + entry_idx, MT_NORMAL); + let ttbr0_el2 = serialise_page_table_to_paddr(&mut ttbr0_el2_pt); - let start = 8 * i; - let end = 8 * (i + 1); - boot_lvl2_upper[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } + (ttbr0_el2, u64::MAX, u64::MAX) + } else { + let mut ttbr0_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut ttbr1_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; - vec![ - (boot_lvl0_lower_addr, boot_lvl0_lower_size, boot_lvl0_lower), - (boot_lvl1_lower_addr, boot_lvl1_lower_size, boot_lvl1_lower), - (boot_lvl0_upper_addr, boot_lvl0_upper_size, boot_lvl0_upper), - (boot_lvl1_upper_addr, boot_lvl1_upper_size, boot_lvl1_upper), - (boot_lvl2_upper_addr, boot_lvl2_upper_size, boot_lvl2_upper), - (boot_lvl2_lower_addr, boot_lvl2_lower_size, boot_lvl2_lower), - ] + // Kernel in TTBR1 (Upper) + ttbr1_el1_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); + // Loader in TTBR0 (Lower) + ttbr0_el1_pt[lvl0_index(k)] = table_descriptor(loader_lvl1_pt_paddr); + + let ttbr0_el1 = serialise_page_table_to_paddr(&mut ttbr0_el1_pt); + let ttbr1_el1 = serialise_page_table_to_paddr(&mut ttbr1_el1_pt); + + (u64::MAX, ttbr0_el1, ttbr1_el1) + } } } diff --git a/tool/microkit/src/sel4.rs b/tool/microkit/src/sel4.rs index d95084a74..2c30309bd 100644 --- a/tool/microkit/src/sel4.rs +++ b/tool/microkit/src/sel4.rs @@ -249,7 +249,7 @@ pub fn emulate_kernel_boot( } } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] pub struct PlatformConfigRegion { pub start: u64, pub end: u64, diff --git a/tool/microkit/src/util.rs b/tool/microkit/src/util.rs index 6f2c0e275..211c794d6 100644 --- a/tool/microkit/src/util.rs +++ b/tool/microkit/src/util.rs @@ -54,6 +54,14 @@ pub const fn round_down(n: u64, x: u64) -> u64 { } } +pub const fn align_up(n: u64, bits: u64) -> u64 { + round_up(n, 1 << bits) +} + +pub const fn align_down(n: u64, bits: u64) -> u64 { + round_down(n, 1 << bits) +} + pub fn is_power_of_two(n: u64) -> bool { assert!(n > 0); n & (n - 1) == 0 From e8ee048fc88786c768998b9c8d5e6f09866e2518 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Mon, 18 May 2026 18:28:16 +1000 Subject: [PATCH 03/29] loader(riscv64): apply the aarch64 treatment Signed-off-by: Julia Vassiliki --- loader/src/riscv/mmu.c | 12 +- tool/microkit/src/loader.rs | 262 ++++++++++++++++++++++++++---------- 2 files changed, 191 insertions(+), 83 deletions(-) diff --git a/loader/src/riscv/mmu.c b/loader/src/riscv/mmu.c index 7751b25e9..b40350ca7 100644 --- a/loader/src/riscv/mmu.c +++ b/loader/src/riscv/mmu.c @@ -8,15 +8,9 @@ #include #include "../arch.h" -#include "../cutil.h" - -/* Paging structures for kernel mapping */ -uint64_t boot_lvl1_pt[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl2_pt[1 << 9] ALIGN(1 << 12); -uint64_t boot_lvl3_pt[1 << 9] ALIGN(1 << 12); -/* Paging structures for identity mapping */ -uint64_t boot_lvl2_pt_loader[1 << 9] ALIGN(1 << 12); +/* Pointers to the top-level paging structures */ +uintptr_t riscv64_boot_lvl1_pt; /* * This is the encoding for the MODE field of the satp register when @@ -36,7 +30,7 @@ int arch_mmu_enable(int logical_cpu) asm volatile( "csrw satp, %0\n" : - : "r"(VM_MODE | (uintptr_t)boot_lvl1_pt >> RISCV_PGSHIFT) + : "r"(VM_MODE | riscv64_boot_lvl1_pt >> RISCV_PGSHIFT) : ); asm volatile("fence.i" ::: "memory"); diff --git a/tool/microkit/src/loader.rs b/tool/microkit/src/loader.rs index fd4519f10..8205a50e6 100644 --- a/tool/microkit/src/loader.rs +++ b/tool/microkit/src/loader.rs @@ -280,6 +280,7 @@ pub mod aarch64 { } mod riscv64 { + pub(crate) const BLOCK_BITS_1GB: u64 = 30; pub(crate) const BLOCK_BITS_2MB: u64 = 21; pub(crate) const PAGE_BITS_4K: u64 = 12; @@ -541,7 +542,21 @@ impl<'a> Loader<'a> { ); } Arch::Riscv64 => { - todo!(); + let boot_lvl1_pt = Loader::riscv64_setup_pagetables( + config, + &loader_elf, + kernel_first_vaddr, + kernel_first_paddr, + page_tables_paddr_start, + &mut page_table_bytes, + ); + write_symbol!( + loader_image, + image_vaddr, + loader_elf, + "riscv64_boot_lvl1_pt", + boot_lvl1_pt + ); } Arch::X86_64 => unreachable!("x86_64 does not support creating a loader image"), }; @@ -667,95 +682,194 @@ impl<'a> Loader<'a> { } } + /// RISC-V 64 page tables for our purposes uses the Sv39 translation scheme + /// (3-level page tables). + /// + /// It is split into two halves: the Upper/Kernel part of the page tables, + /// which matches the format seL4 expects. The lower half contains an + /// identity mapped region for the loader. + /// + /// ```txt + /// (512 GiB) + /// 512 +---- Level 1 ---+ 2^39 + /// | | + /// | (empty) | + /// | | + /// k+1 +----------------+ (1 GiB) + /// | Level 2 Kernel | ----------> +---- Level 2 ---+ +-------------+ + /// k +----------------+ | | ----------> | 2 MiB block | + /// | | 511 |----------------| +-------------+ + /// | | | | ----------> | 2 MiB block | + /// | | 510 |----------------| +-------------+ + /// | | | | ----------> | 2 MiB block | + /// | | |----------------| +------------- + /// | | (...) (...) (...) Kernel Regions + /// | | |----------------| +-------------+ + /// | | | | ----------> | 2 MiB block | + /// | | l+1 |----------------| +-------------+ + /// | | | Level 3 Kernel | ----+ + /// | | l |----------------| | + /// | | | | | (2 MiB) + /// | | | | +-----> +-- Level 3 --+ +------------+ + /// | | | | | | ----------> | 4 KiB page | + /// | | | | 511 |-------------| +------------+ + /// | | | (empty) | | | ----------> | 4 KiB page | + /// | (empty) | | | |-------------| +------------+ + /// | | | | | | ----------> | 4 KiB page | + /// | | | | m |-------------| +------------+ p + /// | | | | | (empty) | + /// | | | | +-------------+ + /// | | | | + /// | | 0 +----------------+ + /// | | + /// | | + /// | | + /// | | + /// | | + /// s+1 +----------------+ (1 GiB) + /// | Level 2 Loader | ----------> +-- Level 2 --+ +-------------+ + /// s +----------------+ | | ----------> | 2 MiB block | + /// | | 511 +-------------+ +-------------+ + /// | | | | ----------> | 2 MiB block | + /// | (empty) | 510 +-------------+ +-------------+ + /// | | | | ----------> | 2 MiB block | + /// | | |-------------| +-------------+ + /// 0 +----------------+ | | ----------> | 2 MiB block | + /// |-------------| +-------------+ + /// (...) (...) (...) Loader Regions + /// |-------------| +-------------+ + /// | | ----------> | 2 MiB block | + /// |-------------| +-------------+ + /// | | ----------> | 2 MiB block | + /// t +-------------+ +-------------+ + /// | | + /// | (empty) | + /// | | + /// +-------------+ + /// + /// + /// Where: + /// k = align_down(kernel_first_vaddr, 1GiB), + /// l = align_down(kernel_first_vaddr, 2MiB), + /// m = align_down(kernel_first_vaddr, 4KiB), + /// p = align_down(kernel_first_paddr, 4KiB), + /// + /// s = align_down(text_addr, 1GiB), + /// t = align_down(text_addr, 2MiB), + /// ``` + /// fn riscv64_setup_pagetables( config: &Config, elf: &ElfFile, - first_vaddr: u64, - first_paddr: u64, - ) -> Vec<(u64, u64, [u8; PAGE_TABLE_SIZE])> { + kernel_first_vaddr: u64, + kernel_first_paddr: u64, + page_tables_paddr_start: u64, + page_table_bytes: &mut Vec, + ) -> u64 { + use riscv64::{pt_index, pte_leaf, pte_next, BLOCK_BITS_1GB, BLOCK_BITS_2MB, PAGE_BITS_4K}; + let (text_addr, _) = grab_symbol!(elf, "_text"); - let (boot_lvl1_pt_addr, boot_lvl1_pt_size) = grab_symbol!(elf, "boot_lvl1_pt"); - let (boot_lvl2_pt_addr, boot_lvl2_pt_size) = grab_symbol!(elf, "boot_lvl2_pt"); - let (boot_lvl3_pt_addr, boot_lvl3_pt_size) = grab_symbol!(elf, "boot_lvl3_pt"); - let (boot_lvl2_pt_loader_addr, boot_lvl2_pt_loader_size) = - grab_symbol!(elf, "boot_lvl2_pt_loader"); // We map the loader using 2MB pages, so make sure the base is actually aligned. - assert!(text_addr.is_multiple_of(1 << riscv64::BLOCK_BITS_2MB)); + assert!(text_addr.is_multiple_of(1 << BLOCK_BITS_2MB)); - let num_pt_levels = config.riscv_pt_levels.unwrap().levels(); + const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); - let mut boot_lvl1_pt: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let text_index_lvl1 = riscv64::pt_index(num_pt_levels, text_addr, 1); - let pt_entry = riscv64::pte_next(boot_lvl2_pt_loader_addr); - let start = 8 * text_index_lvl1; - let end = start + 8; - boot_lvl1_pt[start..end].copy_from_slice(&pt_entry.to_le_bytes()); - } + let mut serialise_page_table_to_paddr = { + let page_tables_paddr_start = { + let aligned_pt_paddr_start = + page_tables_paddr_start.next_multiple_of(PAGE_TABLE_SIZE as u64); + if aligned_pt_paddr_start != page_tables_paddr_start { + let alignment_diff = + (aligned_pt_paddr_start - page_tables_paddr_start) as usize; + page_table_bytes.resize(alignment_diff, 0); + } + + aligned_pt_paddr_start + }; + + // This maintains the current end of the PT array. + let mut next_pt_paddr = page_tables_paddr_start; - let mut boot_lvl2_pt_loader: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let text_index_lvl2 = riscv64::pt_index(num_pt_levels, text_addr, 2); - for (page, i) in (text_index_lvl2..512).enumerate() { - let start = 8 * i; - let end = start + 8; - let addr = text_addr + ((page as u64) << riscv64::BLOCK_BITS_2MB); - let pt_entry = riscv64::pte_leaf(addr); - boot_lvl2_pt_loader[start..end].copy_from_slice(&pt_entry.to_le_bytes()); + move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> u64 { + let pt_paddr = next_pt_paddr; + page_table_bytes.extend(page_table.iter().flat_map(|pte| pte.to_le_bytes())); + next_pt_paddr += PAGE_TABLE_SIZE as u64; + page_table.fill(0); + pt_paddr } - } + }; - { - let index = riscv64::pt_index(num_pt_levels, first_vaddr, 1); - let start = 8 * index; - let end = start + 8; - boot_lvl1_pt[start..end] - .copy_from_slice(&riscv64::pte_next(boot_lvl2_pt_addr).to_le_bytes()); - } + let num_pt_levels = config.riscv_pt_levels.unwrap().levels(); + assert!(num_pt_levels == 3); - let mut boot_lvl3_pt: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - let mut boot_lvl2_pt: [u8; PAGE_TABLE_SIZE] = [0; PAGE_TABLE_SIZE]; - { - let mut index_lvl2 = riscv64::pt_index(num_pt_levels, first_vaddr, 2); - if !first_vaddr.is_multiple_of(1 << riscv64::BLOCK_BITS_2MB) { - let index_lvl3 = riscv64::pt_index(num_pt_levels, first_vaddr, 3); - for (page, i) in (index_lvl3..512).enumerate() { - let start = 8 * i; - let end = start + 8; - let addr = first_paddr + ((page as u64) << riscv64::PAGE_BITS_4K); - assert!(addr.is_multiple_of(1 << riscv64::PAGE_BITS_4K)); - let pt_entry = riscv64::pte_leaf(addr); - boot_lvl3_pt[start..end].copy_from_slice(&pt_entry.to_le_bytes()); + // Manufacture the constants as per the diagram. + let k = align_down(kernel_first_vaddr, BLOCK_BITS_1GB); + let l = align_down(kernel_first_vaddr, BLOCK_BITS_2MB); + let m = align_down(kernel_first_vaddr, PAGE_BITS_4K); + let p = align_down(kernel_first_paddr, PAGE_BITS_4K); + + let s = align_down(text_addr, BLOCK_BITS_1GB); + let t = align_down(text_addr, BLOCK_BITS_2MB); + + // Manufacture the kernel page tables + let kernel_lvl2_pt_paddr = { + let mut lvl2_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + + let mut paddr = p; + let index_l = pt_index(num_pt_levels, l, 2); + + lvl2_pt_kernel[index_l] = if kernel_first_vaddr.is_multiple_of(1 << BLOCK_BITS_2MB) { + assert!(paddr.is_multiple_of(1 << BLOCK_BITS_2MB)); + let pte = pte_leaf(paddr); + paddr += 1 << BLOCK_BITS_2MB; + pte + } else { + let mut lvl3_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + + let index_m = pt_index(num_pt_levels, m, 3); + + for index in index_m..512 { + lvl3_pt_kernel[index] = pte_leaf(paddr); + paddr += 1 << PAGE_BITS_4K; } - let start = 8 * index_lvl2; - let end = start + 8; - let lvl3_pt_entry = riscv64::pte_next(boot_lvl3_pt_addr); - assert!(boot_lvl3_pt_addr.is_multiple_of(1 << riscv64::PAGE_BITS_4K)); - boot_lvl2_pt[start..end].copy_from_slice(&lvl3_pt_entry.to_le_bytes()); - index_lvl2 += 1; + + let kernel_lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt_kernel); + pte_next(kernel_lvl3_pt_paddr) + }; + + for index in (index_l + 1)..512 { + lvl2_pt_kernel[index] = pte_leaf(paddr); + paddr += 1 << BLOCK_BITS_2MB; } - let first_paddr_aligned = align_up(first_paddr, riscv64::BLOCK_BITS_2MB); - for (page, i) in (index_lvl2..512).enumerate() { - let start = 8 * i; - let end = start + 8; - let addr = first_paddr_aligned + ((page as u64) << riscv64::BLOCK_BITS_2MB); - assert!(addr.is_multiple_of(1 << riscv64::BLOCK_BITS_2MB)); - let pt_entry = riscv64::pte_leaf(addr); - boot_lvl2_pt[start..end].copy_from_slice(&pt_entry.to_le_bytes()); + + serialise_page_table_to_paddr(&mut lvl2_pt_kernel) + }; + + // Manufacture the loader page tables, which is relatively straightforward + let loader_lvl2_pt_paddr = { + let mut lvl2_pt_loader = [0u64; PAGE_TABLE_ENTRIES]; + + // Identity mapped, so vaddr == paddr. + let mut paddr = t; + + for index in pt_index(num_pt_levels, t, 2)..512 { + lvl2_pt_loader[index] = pte_leaf(paddr); + paddr += 1 << BLOCK_BITS_2MB; } - } - vec![ - (boot_lvl1_pt_addr, boot_lvl1_pt_size, boot_lvl1_pt), - (boot_lvl2_pt_addr, boot_lvl2_pt_size, boot_lvl2_pt), - (boot_lvl3_pt_addr, boot_lvl3_pt_size, boot_lvl3_pt), - ( - boot_lvl2_pt_loader_addr, - boot_lvl2_pt_loader_size, - boot_lvl2_pt_loader, - ), - ] + serialise_page_table_to_paddr(&mut lvl2_pt_loader) + }; + + // Manufacture the Level 1 table + let mut boot_lvl1_pt = [0u64; PAGE_TABLE_ENTRIES]; + + let index_s = pt_index(num_pt_levels, s, 1); + let index_k = pt_index(num_pt_levels, k, 1); + boot_lvl1_pt[index_k] = pte_next(kernel_lvl2_pt_paddr); + boot_lvl1_pt[index_s] = pte_next(loader_lvl2_pt_paddr); + + serialise_page_table_to_paddr(&mut boot_lvl1_pt) } /// AArch64 loader page tables have two variations: From 3aba5722f3f8c393abe79e4386687c5683edbe12 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Fri, 22 May 2026 10:48:45 +1000 Subject: [PATCH 04/29] [WIP] ram --- tool/microkit/src/loader.rs | 419 +++++++++++++++++++++++++++++------- tool/microkit/src/sel4.rs | 2 +- 2 files changed, 337 insertions(+), 84 deletions(-) diff --git a/tool/microkit/src/loader.rs b/tool/microkit/src/loader.rs index 8205a50e6..d41225f6a 100644 --- a/tool/microkit/src/loader.rs +++ b/tool/microkit/src/loader.rs @@ -4,9 +4,9 @@ // SPDX-License-Identifier: BSD-2-Clause // use crate::elf::{ElfFile, ElfSegmentData}; -use crate::sel4::{Arch, Config}; +use crate::sel4::{Arch, Config, PlatformConfigRegion}; use crate::uimage::uimage_serialise; -use crate::util::{align_down, align_up, mask, mb, round_up, struct_to_bytes}; +use crate::util::{align_down, mb, round_up, struct_to_bytes}; use std::cmp::min; use std::fs::File; use std::io::{BufWriter, Write}; @@ -21,6 +21,18 @@ macro_rules! grab_symbol { }; } +// XX: This could be generic on arbitrary if we could specify T:: implements from_le_bytes, +// but we can't. +fn read_symbol_maybe(elf: &ElfFile, symbol_name: &str) -> Option { + let (addr, size) = elf.find_symbol(symbol_name).ok()?; + + let symbol_bytes = elf.get_data(addr, size)?; + + assert!(mem::size_of::() == symbol_bytes.len()); + + Some(u64::from_le_bytes(symbol_bytes.try_into().ok()?)) +} + macro_rules! write_symbol { ($loader_image: expr, $image_vaddr: expr, $elf: expr, $symbol: literal, $symbol_var: expr) => { let (addr, size) = grab_symbol!($elf, $symbol); @@ -147,7 +159,7 @@ pub mod aarch64 { let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { // Match what the seL4 kernel uses for its page tables, which // is especially necessary for SMP booting which relies on it - // for coherency. + // for coherency. See the comment in seL4 `release_secondary_cpus()`. shareability_attributes::INNER_SHAREABLE } else { // Per $R_{PYFVQ}$: @@ -922,35 +934,8 @@ impl<'a> Loader<'a> { /// | | /// 1 +-------------+ (512 GiB) /// | Level 1 Lwr | ----------> +-- Level 1 --+ - /// 0 +-------------+ | | - /// | (empty) | - /// | | - /// u+1 +-------------+ +-------------+ - /// | uart_base | ----------> | 1 GiB block | - /// u +-------------+ +-------------+ - /// | | - /// | (empty) | - /// | | - /// i+1 +-------------+ (1 GiB) - /// | Level 2 Lwr | ----------> +-- Level 2 --+ - /// i +-------------+ | | - /// | | | (empty) | - /// | (empty) | | | - /// | | t +-------------+ +-------------+ - /// +-------------+ | | ----------> | 2 MiB block | - /// |-------------| +-------------+ - /// | | ----------> | 2 MiB block | - /// |-------------| +-------------+ - /// Loader Regions (...) (...) (...) - /// |-------------| +-------------+ - /// | | ----------> | 2 MiB block | - /// |-------------| +-------------+ - /// | | ----------> | 2 MiB block | - /// s +-------------+ +-------------+ - /// | | - /// | (empty) | - /// | | - /// +-------------+ + /// 0 +-------------+ TODO: RAM. + /// /// /// Where: /// k = align_down(kernel_first_vaddr, 512GiB), @@ -958,9 +943,6 @@ impl<'a> Loader<'a> { /// m = align_down(kernel_first_vaddr, 2MiB), /// p = align_down(kernel_first_paddr, 2MiB), /// u = align_down(uart_base, 1GiB), - /// i = align_down(loader_start_addr, 1GiB), - /// s = align_down(loader_start_addr, 2MiB), - /// t = align_up(loader_end_addr, 2MiB), /// ``` /// fn aarch64_setup_pagetables( @@ -1004,20 +986,49 @@ impl<'a> Loader<'a> { } }; - let (loader_start_addr, _) = grab_symbol!(elf, "_loader_start"); - let (loader_end_addr, _) = grab_symbol!(elf, "_loader_end"); - if lvl1_index(loader_start_addr) != lvl1_index(loader_end_addr) { - panic!("We only map 1GiB, but loader paddr range covers multiple GiB"); - } + let identity_mapped_regions = { + let ram_regions = config + .normal_regions + .as_ref() + .expect("AArch64 should have normal_regions"); + + // println!("{:#x?}", ram_regions); + + let mut regions: Vec<_> = ram_regions + .iter() + .cloned() + .map(|region| (region, MT_DEVICE_nGnRnE)) + .collect(); + + // FIXME: Derive from the kernel build system. + if let Some(uart_base) = read_symbol_maybe(elf, "uart_addr") { + let uart_base = align_down(uart_base, PAGE_BITS_4KB); + regions.push(( + PlatformConfigRegion { + start: uart_base, + end: uart_base + (1 << PAGE_BITS_4KB), + }, + MT_DEVICE_nGnRnE, + )); + } + + // FIXME: This is currently assuming implementation details of the BCM2711/ + // Raspberry Pi 4B spin table implementation, as it is the only + // platform we have that uses spin tables. Specifically, that + // it is always located at the 0 page. + if elf.find_symbol("cpus_release_addr").is_ok() { + regions.push(( + PlatformConfigRegion { + start: 0x0, + end: 1 << PAGE_BITS_4KB, + }, + MT_DEVICE_nGnRnE, + )); + } - let uart_base = if let Ok((uart_addr, uart_addr_size)) = elf.find_symbol("uart_addr") { - let data = elf - .get_data(uart_addr, uart_addr_size) - .expect("uart_addr not initialized"); + regions.sort_by_key(|(region, _)| region.start); - Some(u64::from_le_bytes(data[0..8].try_into().unwrap())) - } else { - None + regions }; // Manufacture the constants as per the diagram. @@ -1025,10 +1036,6 @@ impl<'a> Loader<'a> { let l = align_down(kernel_first_vaddr, BLOCK_BITS_1GB); let m = align_down(kernel_first_vaddr, BLOCK_BITS_2MB); let p = align_down(kernel_first_paddr, BLOCK_BITS_2MB); - let i = align_down(loader_start_addr, BLOCK_BITS_1GB); - let u = uart_base.map(|addr| align_down(addr, BLOCK_BITS_1GB)); - let s = align_down(loader_start_addr, BLOCK_BITS_2MB); - let t = align_up(loader_end_addr, BLOCK_BITS_2MB); // Manufacture the kernel page tables, which is relatively straightforward. let kernel_lvl1_pt_paddr = { @@ -1055,46 +1062,291 @@ impl<'a> Loader<'a> { serialise_page_table_to_paddr(&mut lvl1_pt_kernel) }; - // Manufacture the loader page tables. - let loader_lvl1_pt_paddr = { - // First, the Level 2 Lwr table - let lvl2_pt_paddr = { - let mut lvl2_pt_loader = [0u64; PAGE_TABLE_ENTRIES]; + // Manufacture the RAM page tables, which is a little bit more complicated. + // We assume that normal RAM lies between 0 <= paddr < 512GiB, i.e. + // that lvl0_index(any ram region addr) = 0. + let ram_lvl1_pt_paddr = { + // Validation of assumptions about the identity mapped regions. + let mut previous_end = None; + for (region, _) in identity_mapped_regions.iter() { + assert!(lvl0_index(region.start) == 0); + assert!(lvl0_index(region.end - 1) == 0); + // This is probably an unnecessary assumption. + assert!(region.start.is_multiple_of(4096)); + assert!(region.end.is_multiple_of(4096)); + // This is definitely necessary. + assert!(region.start >= previous_end.unwrap_or(0)); + previous_end = Some(region.end); + } + + // We maintain three active page tables, which contain our previous + // known page table data. As we process regions in ascending order, + // once we have exceeded the bounds of the current reservation we + // can simply push to the page_table_bytes storage and insert into + // the parent PT the descriptor. + // When the current vaddr (/paddr, as identity mapped) exceeds the + // top value we rotate to a new PT. + + let mut lvl1_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut lvl2_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut lvl3_pt = [0u64; PAGE_TABLE_ENTRIES]; + // TODO: These should be defines. Note that the top is the size of 1 level of the next level up. + // TODO: LVL1_ENTRY_RANGE? idk + #[allow(unused_mut)] + let mut lvl1_vaddr_top = 1 << BLOCK_BITS_512GB; + let mut lvl2_vaddr_top = 1 << BLOCK_BITS_1GB; + let mut lvl3_vaddr_top = 1 << BLOCK_BITS_2MB; + + // TODO: Tests... + // This is similar to aligned_power_of_two_regions() for the kernel UT, + // but we restrict it such that the output always is either 1GB, 2MB, or 4KB + // pages. + + // Allowed externally for the final iteration + let mut base = 0u64; + for &(ref region, attr_index) in identity_mapped_regions.iter() { + // println!("RAM Region: {:#x}..{:#x}", base, region.end); + // println!( + // " - Current Lvl1: {:#x}..{:#x}, entries: {}", + // (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), + // lvl1_vaddr_top, + // lvl1_pt.iter().filter(|&&v| v != 0).count() + // ); + // println!( + // " - Current Lvl2: {:#x}..{:#x}, entries: {}", + // (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), + // lvl2_vaddr_top, + // lvl2_pt.iter().filter(|&&v| v != 0).count() + // ); + // println!( + // " - Current Lvl3: {:#x}..{:#x}, entries: {}", + // (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), + // lvl3_vaddr_top, + // lvl3_pt.iter().filter(|&&v| v != 0).count() + // ); + + // Handle the fact that the regions are not contiguous and that + // we might need to skip PT. + + { + if region.start >= lvl3_vaddr_top { + if lvl3_pt != [0; _] { + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + // println!("[iter] Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{lvl3_vaddr_top:#x}", (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB))); + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + } + + // TODO: just compute it. + while region.start >= lvl3_vaddr_top { + lvl3_vaddr_top += 1 << BLOCK_BITS_2MB; + } + } - // Identity mapped: vaddr == paddr. - let mut addr = s; - while addr < t { - lvl2_pt_loader[lvl2_index(addr)] = block_descriptor(2, addr, MT_DEVICE_nGnRnE); + if region.start >= lvl2_vaddr_top { + if lvl2_pt != [0; _] { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + // println!("[iter] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + } + + // TODO: just compute it. + while region.start >= lvl2_vaddr_top { + lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; + } + } - addr += 1 << BLOCK_BITS_2MB; + if region.start >= lvl1_vaddr_top { + unreachable!( + "impossible as everything should fit here: {lvl1_vaddr_top:#x}" + ); + } } - // TODO: this is a complete hack specific to BCM2711/Raspberry Pi 4B and - // will be reworked with patches that re-do this loader mapping code. - if elf.find_symbol("cpus_release_addr").is_ok() { - // Make sure we don't override the loader mappings done above; - // and that this is located at 0x0. - assert!(s != 0); - assert!(i == 0); + // After serialising the old base, update the new one. + base = region.start; + + // Inner Loop: + // Invariant: the page tables in lvl1_pt, lvl2_pt, lvl3_pt + // are either (1) for the current address range, + // or (2) are empty and for a lower level than the current level. + // Also, the values in lvlXXX_vaddr_top are always correct (even if empty) + // Also contiguous within the loop. + // Loop entry: (1) holds by work at the start of each region + while base != region.end { + // Condition is !=, but assert that we never skip it. + assert!(base < region.end); + + let size_bits = region.end.wrapping_sub(base).ilog2(); + let align_bits = min( + size_bits, + // FIXME: Once MSRV is > 1.97, use .lowest_one() method. + if base == 0 { + size_bits + } else { + base.trailing_zeros() + }, + ); + + // Match the size and alignment of the current region to + // the valid PT region sizes. + let (level, bits) = match u64::from(align_bits) { + BLOCK_BITS_1GB.. => (1, BLOCK_BITS_1GB), + BLOCK_BITS_2MB.. => (2, BLOCK_BITS_2MB), + PAGE_BITS_4KB.. => (3, PAGE_BITS_4KB), + 0.. => panic!("impossible; regions should be aligned to 4K at least"), + }; + + let pt_region_size = 1u64 << bits; + let top = base + pt_region_size; + + // println!("- Aligned PT region: {:#x}..{:#x} (size_bits: {}, align_bits: {}, bits: {})", base, top, size_bits, align_bits, bits); + // println!( + // " - Current Lvl1: {:#x}..{:#x}, entries: {}", + // (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), + // lvl1_vaddr_top, + // lvl1_pt.iter().filter(|&&v| v != 0).count() + // ); + // println!( + // " - Current Lvl2: {:#x}..{:#x}, entries: {}", + // (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), + // lvl2_vaddr_top, + // lvl2_pt.iter().filter(|&&v| v != 0).count() + // ); + // println!( + // " - Current Lvl3: {:#x}..{:#x}, entries: {}", + // (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), + // lvl3_vaddr_top, + // lvl3_pt.iter().filter(|&&v| v != 0).count() + // ); + + match level { + 1 => { + // If it belongs in Level 1 PT, then it must go in + // lvl1 pt. By the inavariant, base < lvl1_vaddr_top. + assert!(base < lvl1_vaddr_top); + // top is <= lvl1_vaddr_top (the case where it is the topmost entry) + assert!(top <= lvl1_vaddr_top); + + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = block_descriptor(1, base, attr_index); + + if top == lvl1_vaddr_top { + // Invariant maintenance: if the new top would be now equal + // the end of the page table's region top, we need a new + // page table object and add it to the list. + + // This should be possible to handle - we just need to break out of this loop + todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + } + + // Invariant: Lower levels are empty. + assert!(lvl2_pt == [0; _]); + assert!(lvl3_pt == [0; _]); + // Invariant maintenance: vaddr_top is right range for current PT. + // it's empty so we need to increment the top to be current top (1G aligned) + 2MIB (512 lvl3 entries) + lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); + // it's empty so we need to increment the top to be current top (1G aligned) + 1G (512 lvl2 entries) + lvl2_vaddr_top = top + (1 << BLOCK_BITS_1GB); + } + 2 => { + // If it is a 2MiB block, it must go in the Level 2 PT; + // by our invariants: base < lvl2_vaddr_top and top <= lvl2_vaddr_top + assert!(base < lvl2_vaddr_top); + assert!(top <= lvl2_vaddr_top); + + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = block_descriptor(2, base, attr_index); + + if top == lvl2_vaddr_top { + // Invariant maintenance: keep for current address range. + // As we're the top of the range, we can serialise the table. + + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} up to {lvl2_vaddr_top:#x}"); + lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; + + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + + if top == lvl1_vaddr_top { + todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + } + } + + // Invariant: Lower levels are empty. + assert!(lvl3_pt == [0; _]); + // Invariant maintenance: vaddr_top is right range for current PT. + // it's empty so we need to increment the top to be current top (2MIB aligned) + 2MIB (512 lvl3 entries) + lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); + } + 3 => { + // If it is a 4K page, it must go in the Level 3 PT; + // by our invariants: base < lvl3_vaddr_top and top <= lvl3_vaddr_top + assert!(base < lvl3_vaddr_top); + assert!(top <= lvl3_vaddr_top); + + assert!(lvl3_pt[lvl3_index(base)] == 0); + lvl3_pt[lvl3_index(base)] = page_descriptor(base, attr_index); + + if top == lvl3_vaddr_top { + // Invariant maintenance: keep for current address range. + // As we're the top of the range, we can serialise the table. + + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + // println!("Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{lvl3_vaddr_top:#x}", (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB))); + lvl3_vaddr_top += 1 << BLOCK_BITS_2MB; + + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + + if top == lvl2_vaddr_top { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB))); + lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; + + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + + if top == lvl1_vaddr_top { + todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + } + } + } + + // Invariant: lower levels empty is vacuuously true + } + _ => unreachable!("level is 1..=3"), + } - lvl2_pt_loader[lvl2_index(0)] = block_descriptor(2, 0, MT_DEVICE_nGnRnE); + base = base + pt_region_size; } + } - serialise_page_table_to_paddr(&mut lvl2_pt_loader) - }; + // By the loop invariant, we know that anything before has been serialised. + // However, as we are at the end of the loop now, we might have + // page tables that have been partially filled out, and we need to + // serialise these. - // Then, the Level 1 Lwr table. - let mut lvl1_pt_loader = [0u64; PAGE_TABLE_ENTRIES]; - lvl1_pt_loader[lvl1_index(i)] = table_descriptor(lvl2_pt_paddr); + if lvl3_pt != [0; _] { + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + // println!("[end] Serialise lvl3 table: {lvl3_pt_paddr:#x}"); + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + } - // map optional UART MMIO in l1 1GB page, only available if CONFIG_PRINTING - if let Some(u) = u { - // UART no overlap with Loader. - assert!(lvl1_index(i) != lvl1_index(u)); - lvl1_pt_loader[lvl1_index(u)] = block_descriptor(1, u, MT_DEVICE_nGnRnE); + if lvl2_pt != [0; _] { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + // println!("[end] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); } - serialise_page_table_to_paddr(&mut lvl1_pt_loader) + // the level1 pt should not be empty. lol. + assert!(lvl1_pt != [0; _]); + + // println!("New lvl1 table"); + serialise_page_table_to_paddr(&mut lvl1_pt) }; // Depending on whether we are in hypervisor mode, we either need to @@ -1107,8 +1359,9 @@ impl<'a> Loader<'a> { let mut ttbr0_el2_pt = [0u64; PAGE_TABLE_ENTRIES]; + assert!(lvl0_index(k) != lvl0_index(0)); ttbr0_el2_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); - ttbr0_el2_pt[lvl0_index(0)] = table_descriptor(loader_lvl1_pt_paddr); + ttbr0_el2_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); let ttbr0_el2 = serialise_page_table_to_paddr(&mut ttbr0_el2_pt); @@ -1119,8 +1372,8 @@ impl<'a> Loader<'a> { // Kernel in TTBR1 (Upper) ttbr1_el1_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); - // Loader in TTBR0 (Lower) - ttbr0_el1_pt[lvl0_index(k)] = table_descriptor(loader_lvl1_pt_paddr); + // Identity-mapped RAM in TTBR0 (Lower) + ttbr0_el1_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); let ttbr0_el1 = serialise_page_table_to_paddr(&mut ttbr0_el1_pt); let ttbr1_el1 = serialise_page_table_to_paddr(&mut ttbr1_el1_pt); diff --git a/tool/microkit/src/sel4.rs b/tool/microkit/src/sel4.rs index 2c30309bd..ead67bb04 100644 --- a/tool/microkit/src/sel4.rs +++ b/tool/microkit/src/sel4.rs @@ -249,7 +249,7 @@ pub fn emulate_kernel_boot( } } -#[derive(Deserialize, Debug)] +#[derive(Deserialize, Debug, Clone)] pub struct PlatformConfigRegion { pub start: u64, pub end: u64, From 5417be07f78a8e4a2ad5de6d5be533957b1e3945 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 28 Jul 2026 11:00:01 +1000 Subject: [PATCH 05/29] loader: gc-sections Useful for rust. Signed-off-by: Julia Vassiliki --- loader/Makefile | 6 ++++-- loader/aarch64.ld | 7 ++++++- loader/riscv64.ld | 13 +++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/loader/Makefile b/loader/Makefile index c08c18bd9..2811fd8f9 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -50,7 +50,7 @@ endif CFLAGS := -std=gnu11 -g -O3 -nostdlib -ffreestanding \ -MP -MD $(CFLAGS_ARCH) -DBOARD_$(BOARD) -I$(SEL4_SDK)/include \ -Wall -Werror -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations \ - -Wundef -Wno-nonnull -Wnested-externs + -Wundef -Wno-nonnull -Wnested-externs -ffunction-sections -fdata-sections ASM_FLAGS := $(ASM_FLAGS_ARCH) -g -MP -MD -I$(SEL4_SDK)/include @@ -89,5 +89,7 @@ all: $(OBJPROG) $(LINKSCRIPT): $(LINKSCRIPT_INPUT) $(CPP) -DLINK_ADDRESS=$(LINK_ADDRESS) $< | grep -v "^#" > $@ +LDFLAGS := -T$(LINKSCRIPT) --gc-sections + $(OBJPROG): $(addprefix $(BUILD_DIR)/, $(OBJECTS)) $(LINKSCRIPT) - $(LD) -T$(LINKSCRIPT) $(addprefix $(BUILD_DIR)/, $(OBJECTS)) -o $@ + $(LD) $(LDFLAGS) $(addprefix $(BUILD_DIR)/, $(OBJECTS)) -o $@ diff --git a/loader/aarch64.ld b/loader/aarch64.ld index 977ccc574..34636e9e4 100644 --- a/loader/aarch64.ld +++ b/loader/aarch64.ld @@ -17,9 +17,11 @@ SECTIONS .text : { _text = .; - *(.text.start) + KEEP(*(.text.start)) *(.text*) + *(.text.*) *(.rodata) + *(.rodata.*) _text_end = .; } :all @@ -27,6 +29,8 @@ SECTIONS { _data = .; *(.data) + *(.data.*) + KEEP(*(.data.uart_addr)) _data_end = .; } :all @@ -34,6 +38,7 @@ SECTIONS { _bss = .; *(.bss) + *(.bss.*) *(COMMON) . = ALIGN(4); _bss_end = .; diff --git a/loader/riscv64.ld b/loader/riscv64.ld index f7ae1240e..fe98d7450 100644 --- a/loader/riscv64.ld +++ b/loader/riscv64.ld @@ -15,9 +15,11 @@ SECTIONS .text : { _text = .; - *(.text.start) + KEEP(*(.text.start)) *(.text*) + *(.text.*) *(.rodata) + *(.rodata.*) _text_end = .; } :all @@ -25,17 +27,20 @@ SECTIONS { _data = .; *(.data) + *(.data.*) __global_pointer$ = . + 0x800; - *(.srodata) - *(.sdata) + *(.srodata) + *(.sdata) + KEEP(*(.data.uart_addr)) _data_end = .; } :all .bss : { _bss = .; - *(.sbss) + *(.sbss) *(.bss) + *(.bss.*) *(COMMON) . = ALIGN(4); _bss_end = .; From 0557f16ab54b9a60e9202fb105d94cd75946b5e1 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Wed, 29 Jul 2026 15:26:32 +1000 Subject: [PATCH 06/29] move the stuff to runtime (but it breaks it!! need fix!!) Signed-off-by: Julia Vassiliki --- loader/Makefile | 23 +- loader/aarch64.ld | 24 +- loader/src/aarch64/mmu.c | 15 + loader/src/loader.c | 1 - loader/src/loader.h | 2 +- loader/src/page_tables.rs | 1090 +++++++++++++++++++++++++++++++++++ tool/microkit/src/loader.rs | 1083 +--------------------------------- 7 files changed, 1150 insertions(+), 1088 deletions(-) create mode 100644 loader/src/page_tables.rs diff --git a/loader/Makefile b/loader/Makefile index 2811fd8f9..dd78ce5d9 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -35,15 +35,19 @@ else LD = $(TARGET_TRIPLE)-ld endif +RUSTC := rustc + ifeq ($(ARCH),aarch64) CFLAGS_AARCH64 := -mcpu=$(GCC_CPU) -mgeneral-regs-only -mstrict-align -mno-outline-atomics CFLAGS_ARCH := $(CFLAGS_AARCH64) -DARCH_aarch64 ASM_FLAGS_ARCH := -mcpu=$(GCC_CPU) ARCH_DIR := aarch64 + RUST_TARGET_TRIPLE := aarch64-unknown-none else ifeq ($(ARCH),riscv64) CFLAGS_RISCV64 := -mcmodel=medany -march=rv64imac_zicsr_zifencei -mabi=lp64 CFLAGS_ARCH := $(CFLAGS_RISCV64) -DARCH_riscv64 ASM_FLAGS_ARCH := -march=rv64imac_zicsr_zifencei -mabi=lp64 + RUST_TARGET_TRIPLE := riscv64gc-unknown-none-elf ARCH_DIR := riscv endif @@ -54,8 +58,10 @@ CFLAGS := -std=gnu11 -g -O3 -nostdlib -ffreestanding \ ASM_FLAGS := $(ASM_FLAGS_ARCH) -g -MP -MD -I$(SEL4_SDK)/include +RUSTFLAGS := --target $(RUST_TARGET_TRIPLE) --edition 2024 -g -C opt-level=2 + PROGS := loader.elf -OBJECTS := loader.o crt0.o uart.o cutil.o +OBJECTS := loader.o crt0.o uart.o cutil.o libpage_tables.a ifeq ($(ARCH),aarch64) OBJECTS += util64.o el.o exceptions.o init.o mmu.o cpus.o @@ -80,7 +86,19 @@ $(BUILD_DIR)/%.o : src/$(ARCH_DIR)/%.c $(BUILD_DIR)/%.o : src/%.c $(CC) -c $(CFLAGS) $< -o $@ +# Note: having multiple rlib with staticlib will give duplicate linker symbol +# issues. Use "--crate-type rlib" instead, but then we need to link a single +# copy of the rust corelibs. +$(BUILD_DIR)/lib%.a : src/%.rs + $(RUSTC) $(RUSTFLAGS) \ + --emit dep-info,metadata,link \ + --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ + --crate-type staticlib \ + --crate-name $(patsubst lib%.a,%,$(notdir $@)) \ + $< + -include $(BUILD_DIR)/*.d +-include $(BUILD_DIR)/mmu.d OBJPROG = $(addprefix $(BUILD_DIR)/, $(PROGS)) @@ -92,4 +110,5 @@ $(LINKSCRIPT): $(LINKSCRIPT_INPUT) LDFLAGS := -T$(LINKSCRIPT) --gc-sections $(OBJPROG): $(addprefix $(BUILD_DIR)/, $(OBJECTS)) $(LINKSCRIPT) - $(LD) $(LDFLAGS) $(addprefix $(BUILD_DIR)/, $(OBJECTS)) -o $@ + $(LD) $(LDFLAGS) --start-group $(addprefix $(BUILD_DIR)/, $(OBJECTS)) --end-group -o $@ + diff --git a/loader/aarch64.ld b/loader/aarch64.ld index 34636e9e4..2454398a3 100644 --- a/loader/aarch64.ld +++ b/loader/aarch64.ld @@ -8,6 +8,12 @@ PHDRS all PT_LOAD AT (LINK_ADDRESS); } + +// text PT_LOAD FLAGS(5); /* RX */ +// rodata PT_LOAD FLAGS(4); /* RO */ +// data PT_LOAD FLAGS(6); /* RW */ +// bss PT_LOAD FLAGS(6); /* RW */ + SECTIONS { . = LINK_ADDRESS; @@ -17,20 +23,26 @@ SECTIONS .text : { _text = .; + KEEP(*(.text.start)) - *(.text*) - *(.text.*) - *(.rodata) - *(.rodata.*) + *(.text .text.*) + _text_end = .; } :all + .rodata : + { + *(.rodata .rodata.* .rodata..Lanon.*) + } :all + .data : { _data = .; - *(.data) + *(.data .data.*) *(.data.*) + KEEP(*(.data.uart_addr)) + _data_end = .; } :all @@ -44,5 +56,7 @@ SECTIONS _bss_end = .; } :all + + _loader_end = .; } diff --git a/loader/src/aarch64/mmu.c b/loader/src/aarch64/mmu.c index 39b4676c8..2fbdcc150 100644 --- a/loader/src/aarch64/mmu.c +++ b/loader/src/aarch64/mmu.c @@ -20,8 +20,23 @@ uint64_t aarch64_pt_ttbr0_el1; uint64_t aarch64_pt_ttbr1_el1; uint64_t aarch64_pt_ttbr0_el2; +struct ret { + uint64_t a; + uint64_t b; + uint64_t c; +}; + +extern struct ret aarch64_setup_pagetables(uint64_t kernel_first_vaddr, uint64_t kernel_first_paddr, uint64_t page_tables_paddr_start); + int arch_mmu_enable(int logical_cpu) { + puts("setup1\n"); + struct ret x = aarch64_setup_pagetables(0, 0, 0); + aarch64_pt_ttbr0_el1 = x.a; + aarch64_pt_ttbr1_el1 = x.b; + aarch64_pt_ttbr0_el2 = x.c; + puts("setup\n"); + int r; enum el el; r = ensure_correct_el(logical_cpu); diff --git a/loader/src/loader.c b/loader/src/loader.c index 67c09b149..45a21de04 100644 --- a/loader/src/loader.c +++ b/loader/src/loader.c @@ -100,7 +100,6 @@ static int print_lock = 0; void start_kernel(int logical_cpu) { - LDR_PRINT("INFO", logical_cpu, "enabling MMU\n"); int r = arch_mmu_enable(logical_cpu); if (r != 0) { LDR_PRINT("ERROR", logical_cpu, "failed to enable MMU: "); diff --git a/loader/src/loader.h b/loader/src/loader.h index c144381aa..d1a0e79d3 100644 --- a/loader/src/loader.h +++ b/loader/src/loader.h @@ -7,7 +7,7 @@ #pragma once -#define STACK_SIZE 4096 +#define STACK_SIZE 40960 #define REGION_TYPE_DATA 1 #define REGION_TYPE_ZERO 2 diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs new file mode 100644 index 000000000..31bcca1c9 --- /dev/null +++ b/loader/src/page_tables.rs @@ -0,0 +1,1090 @@ +// +// Copyright 2026, UNSW +// +// SPDX-License-Identifier: BSD-2-Clause +// + +#![no_std] + +use core::cmp::min; +use core::ffi::c_char; +use core::fmt; +use core::fmt::Write; +use core::mem; +use core::panic::PanicInfo; + +unsafe extern "C" { + safe fn fail() -> !; + // safe fn putc(c: c_char); + unsafe fn puts(s: *const c_char); +} + +#[panic_handler] +fn panic(info: &PanicInfo) -> ! { + unsafe { puts(c"panicked\n".as_ptr()) }; + + struct DebugWriter; + impl fmt::Write for DebugWriter { + fn write_str(&mut self, s: &str) -> fmt::Result { + for c in s.bytes() { + unsafe { + puts(core::ffi::CStr::from_bytes_with_nul_unchecked(&[c.into(), 0]).as_ptr()) + }; + } + + Ok(()) + } + } + + if let Err(_) = writeln!(DebugWriter, "{}", info) { + // If writeln!() fails (which it should never as our fmt::Write) never + // fails, then just don't print the extra information. + unsafe { puts(c"panicked (information unknown)\n".as_ptr()) }; + } + + fail(); +} + +const PAGE_TABLE_SIZE: usize = 4096; + +const fn divmod(x: u64, y: u64) -> (u64, u64) { + (x / y, x % y) +} + +const fn mask(n: u64) -> u64 { + (1 << n) - 1 +} + +const fn round_up(n: u64, x: u64) -> u64 { + let (_, m) = divmod(n, x); + if m == 0 { + n + } else { + n + x - m + } +} + +const fn round_down(n: u64, x: u64) -> u64 { + let (_, m) = divmod(n, x); + if m == 0 { + n + } else { + n - m + } +} + +const fn align_up(n: u64, bits: u64) -> u64 { + round_up(n, 1 << bits) +} + +const fn align_down(n: u64, bits: u64) -> u64 { + round_down(n, 1 << bits) +} + +unsafe extern "C" { + static mut _text: u8; +} + +pub mod aarch64 { + //! For AArch64, our page tables use the Stage 1 descriptor formats + //! for both EL2 (TTBR0_EL2) and EL1 (TTBR0_EL1/TTBR1_EL1). + //! Stage 2 descriptors are only used when in the EL1&0 regime; which is not + //! the case when in EL2. + + use crate::mask; + + pub const LVL0_BITS: u64 = 9; + pub const LVL1_BITS: u64 = 9; + pub const LVL2_BITS: u64 = 9; + pub const LVL3_BITS: u64 = 9; + + pub fn lvl0_index(addr: u64) -> usize { + let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS + LVL1_BITS)) & mask(LVL0_BITS); + idx as usize + } + + pub fn lvl1_index(addr: u64) -> usize { + let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS)) & mask(LVL1_BITS); + idx as usize + } + + pub fn lvl2_index(addr: u64) -> usize { + let idx = (addr >> (BLOCK_BITS_2MB)) & mask(LVL2_BITS); + idx as usize + } + + pub fn lvl3_index(addr: u64) -> usize { + let idx = (addr >> PAGE_BITS_4KB) & mask(LVL3_BITS); + idx as usize + } + + /// Stage 1 translation table page/block descriptors have bits[4:2] containing + /// AttrIndex[2:0]. The AttrIndex values depends on our configuration of + /// the `MAIR_EL1` or `MAIR_EL2` registers done in util64.S; + /// This also needs to match the values that seL4 uses. + #[allow(non_upper_case_globals, reason = "matching ARM naming convention")] + pub mod s1_mair_attr_index { + pub const MT_DEVICE_nGnRnE: u64 = 0b000; + pub const MT_DEVICE_nGnRE: u64 = 0b001; + pub const MT_DEVICE_GRE: u64 = 0b010; + pub const MT_NORMAL_NC: u64 = 0b011; + pub const MT_NORMAL: u64 = 0b100; + } + + pub mod descriptor_type { + //! The translation table descriptor formats, as per §D8.3 "Translation + //! table descriptor formats" of ARM DDI 0487 L.b. Specifically, + //! as per "Table D8-48 Determination of descriptor type" + + /// Descriptor type: Table. Condition is lookup level != 3. + pub const TABLE: u64 = 0b11; + /// Descriptor type: Page. Condition is lookup level == 3. + pub const PAGE: u64 = 0b11; + /// Descriptor type: Block. Condition is lookup level != 3. + pub const BLOCK: u64 = 0b01; + /// Descriptor type: Invalid. Strictly speaking bit[1] does not matter. + pub const INVALID: u64 = 0b00; + } + + pub mod shareability_attributes { + //! Per §D8.6.2 "Stage 1 Shareability attributes", these contain the + //! shareability attributes of the descriptor OA for normal-cacheable + //! memory. + + /// Non-shareable + pub const NON_SHAREABLE: u64 = 0b00; + /// Outer-shareable + pub const OUTER_SHAREABLE: u64 = 0b10; + /// Inner-shareable + pub const INNER_SHAREABLE: u64 = 0b11; + } + + /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, + /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address + /// is bits [47:n], and: + /// + /// > For the 4KB granule size, the level 1 descriptor n is 30, + /// > and the level 2 descriptor n is 21. + pub const BLOCK_BITS_1GB: u64 = 30; + + /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, + /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address + /// is bits [47:n], and: + /// + /// > For the 4KB granule size, the level 1 descriptor n is 30, + /// > and the level 2 descriptor n is 21. + pub const BLOCK_BITS_2MB: u64 = 21; + + // TODO: + + pub const BLOCK_BITS_512GB: u64 = 39; + pub const PAGE_BITS_4KB: u64 = 12; + + /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and + /// "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b; + /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" + pub fn block_descriptor(level: usize, addr: u64, attr_index: u64) -> u64 { + // Per Table D8-48, Condition for descriptor_type::BLOCK is level != 3. + assert!(level != 3); + + let upper_attributes: u64 = 0; + + let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { + // Match what the seL4 kernel uses for its page tables, which + // is especially necessary for SMP booting which relies on it + // for coherency. See the comment in seL4 `release_secondary_cpus()`. + shareability_attributes::INNER_SHAREABLE + } else { + // Per $R_{PYFVQ}$: + // > If a region is mapped as Device memory or Normal Non-cacheable + // > memory after all enabled translation stages, then the region + // > has an effective Shareability attribute of Outer Shareable. + // + // We override the value we place in here to OUTER_SHAREABLE to match + // how the hardware behaves. This is not necessary but for clarity. + shareability_attributes::OUTER_SHAREABLE + }; + + // AP[2:1], which we set as 0b00 for read/write access: + // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1 + // stage 2: 0b00 is RW for EL2 and no perms for EL1. + const AP_KERNEL_RW: u64 = 0b00; + + // bit[11] is the not global (nG) field, we leave as 0 (global). + // bit[10] is the access flag; depending on FEAT_HAFDBS, when software + // manages the AF memory accesses to the page/block when AF=0 + // raise an Access Fault; when hardware manages the AF it will + // become 1. + // bit[9:8] is SH[1:0] containing stage 1 shareability attributes + // bit[7:6] contains AP[2:1] + // bit[5] is RES0 + // bit[4:2] contains AttrIndex + let lower_attributes: u64 = + (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); + + // bits[47:n] + let output_address: u64 = addr + & !mask(match level { + 1 => BLOCK_BITS_1GB, + 2 => BLOCK_BITS_2MB, + _ => panic!("unsupported level {level} for block descriptor"), + }); + + // address must not have bits above 47 set. + assert!(addr & mask(48) == addr); + + // bits[63:50] describing the "Upper attributes" are left at 0. + // bits[49:48] are RES0 + // bits[47:n] contain the Output address + // bits[n-1:12] are RES0 + // bits[11:2] contain the "Lower attributes" + // bits[1:0] contains the descriptor type + upper_attributes | output_address | lower_attributes | descriptor_type::BLOCK + } + + /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and + /// "Figure D8-15 VMSAv8-64 Page descriptor formats" of ARM DDI0487L.b; + /// specifically subfigure "4KB granule 48-bit OA". + pub fn page_descriptor(addr: u64, attr_index: u64) -> u64 { + // The main difference between a page descriptor and block descriptor + // is in the size of the output address (OA) and in the descriptor type. + + let upper_attributes: u64 = 0; + + let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { + // Match what the seL4 kernel uses for its page tables, which + // is especially necessary for SMP booting which relies on it + // for coherency. + shareability_attributes::INNER_SHAREABLE + } else { + // Per $R_{PYFVQ}$: + // > If a region is mapped as Device memory or Normal Non-cacheable + // > memory after all enabled translation stages, then the region + // > has an effective Shareability attribute of Outer Shareable. + // We override the value we place in here to OUTER_SHAREABLE to match + // how the hardware behaves. + shareability_attributes::OUTER_SHAREABLE + }; + + // AP[2:1], which we set as 0b00 for read/write access: + // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1/El2 (priv) + const AP_KERNEL_RW: u64 = 0b00; + + // bit[11] is the not global (nG) field, we leave as 0 (global). + // bit[10] is the access flag; depending on FEAT_HAFDBS, when software + // manages the AF memory accesses to the page/block when AF=0 + // raise an Access Fault; when hardware manages the AF it will + // become 1. + // bit[9:8] is SH[1:0] containing stage 1 shareability attributes + // bit[7:6] contains AP[2:1] + // bit[5] is RES0 + // bit[4:2] contains AttrIndex + let lower_attributes: u64 = + (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); + + // bits[47:12] + let output_address: u64 = addr & !mask(12); + + // address must not have bits above 47 set. + assert!(addr & mask(48) == addr); + + // bits[63:50] describing the "Upper attributes" are left at 0. + // bits[49:48] are RES0 + // bits[47:12] contain the Output address + // bits[11:2] contain the "Lower attributes" + // bits[1:0] contains the descriptor type + upper_attributes | output_address | lower_attributes | descriptor_type::PAGE + } + + /// Per "Table D8-50 Stage 1 VMSAv8-64 Table descriptor fields" and + /// "Figure D8-12 VMSAv8-64 Table descriptor formats" of ARM DDI0487L.b; + /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" + pub fn table_descriptor(addr: u64) -> u64 { + // Per Table D8-48, Condition for descriptor_type::TABLE is level != 3. + + // We don't set any of these attributes, most are hardware-feature conditional + let attributes: u64 = 0; + + // address must not have bits above 47 or below 12 set + assert!(addr & mask(12) == 0x0); + assert!(addr & mask(48) == addr); + + let next_level_table_address = addr; + + // bits[63:59] are "Attributes" + // bits[58:51] are ignored + // bits[50:48] are RES0 + // bits[47:m] is the next-level table address + // note: here m=12 for 4KB granule + // bits[m-1:12] are RES0 + // so this doesn't exist for 4KB granule + // bits[11:2] are ignored + // bits[1:0] contain the descriptor type + attributes | next_level_table_address | descriptor_type::TABLE + } +} + +mod riscv64 { + pub(crate) const BLOCK_BITS_1GB: u64 = 30; + pub(crate) const BLOCK_BITS_2MB: u64 = 21; + pub(crate) const PAGE_BITS_4K: u64 = 12; + + pub(crate) const PAGE_TABLE_INDEX_BITS: u64 = 9; + pub(crate) const PAGE_SHIFT: u64 = 12; + /// This sets the page table entry bits: D,A,X,W,R. + pub(crate) const PTE_TYPE_BITS: u64 = 0b11001110; + // TODO: where does this come from? + pub(crate) const PTE_TYPE_TABLE: u64 = 0; + pub(crate) const PTE_TYPE_VALID: u64 = 1; + + pub(crate) const PTE_PPN0_SHIFT: u64 = 10; + + /// Due to RISC-V having various virtual memory setups, we have this generic function to + /// figure out the page-table index given the total number of page table levels for the + /// platform and which level we are currently looking at. + pub fn pt_index(pt_levels: usize, addr: u64, level: usize) -> usize { + let pt_index_bits = PAGE_TABLE_INDEX_BITS * (pt_levels - level) as u64; + let idx = (addr >> (pt_index_bits + PAGE_SHIFT)) % 512; + + idx as usize + } + + /// Generate physical page number given an address + pub fn pte_ppn(addr: u64) -> u64 { + (addr >> PAGE_SHIFT) << PTE_PPN0_SHIFT + } + + pub fn pte_next(addr: u64) -> u64 { + pte_ppn(addr) | PTE_TYPE_TABLE | PTE_TYPE_VALID + } + + pub fn pte_leaf(addr: u64) -> u64 { + pte_ppn(addr) | PTE_TYPE_BITS | PTE_TYPE_VALID + } +} + +/// RISC-V 64 page tables for our purposes uses the Sv39 translation scheme +/// (3-level page tables). +/// +/// It is split into two halves: the Upper/Kernel part of the page tables, +/// which matches the format seL4 expects. The lower half contains an +/// identity mapped region for the loader. +/// +/// ```txt +/// (512 GiB) +/// 512 +---- Level 1 ---+ 2^39 +/// | | +/// | (empty) | +/// | | +/// k+1 +----------------+ (1 GiB) +/// | Level 2 Kernel | ----------> +---- Level 2 ---+ +-------------+ +/// k +----------------+ | | ----------> | 2 MiB block | +/// | | 511 |----------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | 510 |----------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | |----------------| +------------- +/// | | (...) (...) (...) Kernel Regions +/// | | |----------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | l+1 |----------------| +-------------+ +/// | | | Level 3 Kernel | ----+ +/// | | l |----------------| | +/// | | | | | (2 MiB) +/// | | | | +-----> +-- Level 3 --+ +------------+ +/// | | | | | | ----------> | 4 KiB page | +/// | | | | 511 |-------------| +------------+ +/// | | | (empty) | | | ----------> | 4 KiB page | +/// | (empty) | | | |-------------| +------------+ +/// | | | | | | ----------> | 4 KiB page | +/// | | | | m |-------------| +------------+ p +/// | | | | | (empty) | +/// | | | | +-------------+ +/// | | | | +/// | | 0 +----------------+ +/// | | +/// | | +/// | | +/// | | +/// | | +/// s+1 +----------------+ (1 GiB) +/// | Level 2 Loader | ----------> +-- Level 2 --+ +-------------+ +/// s +----------------+ | | ----------> | 2 MiB block | +/// | | 511 +-------------+ +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | (empty) | 510 +-------------+ +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | |-------------| +-------------+ +/// 0 +----------------+ | | ----------> | 2 MiB block | +/// |-------------| +-------------+ +/// (...) (...) (...) Loader Regions +/// |-------------| +-------------+ +/// | | ----------> | 2 MiB block | +/// |-------------| +-------------+ +/// | | ----------> | 2 MiB block | +/// t +-------------+ +-------------+ +/// | | +/// | (empty) | +/// | | +/// +-------------+ +/// +/// +/// Where: +/// k = align_down(kernel_first_vaddr, 1GiB), +/// l = align_down(kernel_first_vaddr, 2MiB), +/// m = align_down(kernel_first_vaddr, 4KiB), +/// p = align_down(kernel_first_paddr, 4KiB), +/// +/// s = align_down(text_addr, 1GiB), +/// t = align_down(text_addr, 2MiB), +/// ``` +/// +#[unsafe(no_mangle)] +pub extern "C" fn riscv64_setup_pagetables( + kernel_first_vaddr: u64, + kernel_first_paddr: u64, + page_tables_paddr_start: u64, +) -> u64 { + use riscv64::{pt_index, pte_leaf, pte_next, BLOCK_BITS_1GB, BLOCK_BITS_2MB, PAGE_BITS_4K}; + + let text_addr = &raw const _text as u64; + + // We map the loader using 2MB pages, so make sure the base is actually aligned. + assert!(text_addr.is_multiple_of(1 << BLOCK_BITS_2MB)); + + const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); + + let mut serialise_page_table_to_paddr = { + assert!( + page_tables_paddr_start + == page_tables_paddr_start.next_multiple_of(PAGE_TABLE_SIZE as u64) + ); + + // This maintains the current end of the PT array. + let mut next_pt_paddr = page_tables_paddr_start; + + move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> u64 { + let pt_paddr = next_pt_paddr; + // page_table_bytes.extend(page_table.iter().flat_map(|pte| pte.to_le_bytes())); + next_pt_paddr += PAGE_TABLE_SIZE as u64; + page_table.fill(0); + pt_paddr + } + }; + + struct Config { + riscv_pt_levels: usize, + } + let config = Config { riscv_pt_levels: 3 }; + + let num_pt_levels = config.riscv_pt_levels; + assert!(num_pt_levels == 3); + + // Manufacture the constants as per the diagram. + let k = align_down(kernel_first_vaddr, BLOCK_BITS_1GB); + let l = align_down(kernel_first_vaddr, BLOCK_BITS_2MB); + let m = align_down(kernel_first_vaddr, PAGE_BITS_4K); + let p = align_down(kernel_first_paddr, PAGE_BITS_4K); + + let s = align_down(text_addr, BLOCK_BITS_1GB); + let t = align_down(text_addr, BLOCK_BITS_2MB); + + // Manufacture the kernel page tables + let kernel_lvl2_pt_paddr = { + let mut lvl2_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + + let mut paddr = p; + let index_l = pt_index(num_pt_levels, l, 2); + + lvl2_pt_kernel[index_l] = if kernel_first_vaddr.is_multiple_of(1 << BLOCK_BITS_2MB) { + assert!(paddr.is_multiple_of(1 << BLOCK_BITS_2MB)); + let pte = pte_leaf(paddr); + paddr += 1 << BLOCK_BITS_2MB; + pte + } else { + let mut lvl3_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + + let index_m = pt_index(num_pt_levels, m, 3); + + for index in index_m..512 { + lvl3_pt_kernel[index] = pte_leaf(paddr); + paddr += 1 << PAGE_BITS_4K; + } + + let kernel_lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt_kernel); + pte_next(kernel_lvl3_pt_paddr) + }; + + for index in (index_l + 1)..512 { + lvl2_pt_kernel[index] = pte_leaf(paddr); + paddr += 1 << BLOCK_BITS_2MB; + } + + serialise_page_table_to_paddr(&mut lvl2_pt_kernel) + }; + + // Manufacture the loader page tables, which is relatively straightforward + let loader_lvl2_pt_paddr = { + let mut lvl2_pt_loader = [0u64; PAGE_TABLE_ENTRIES]; + + // Identity mapped, so vaddr == paddr. + let mut paddr = t; + + for index in pt_index(num_pt_levels, t, 2)..512 { + lvl2_pt_loader[index] = pte_leaf(paddr); + paddr += 1 << BLOCK_BITS_2MB; + } + + serialise_page_table_to_paddr(&mut lvl2_pt_loader) + }; + + // Manufacture the Level 1 table + let mut boot_lvl1_pt = [0u64; PAGE_TABLE_ENTRIES]; + + let index_s = pt_index(num_pt_levels, s, 1); + let index_k = pt_index(num_pt_levels, k, 1); + boot_lvl1_pt[index_k] = pte_next(kernel_lvl2_pt_paddr); + boot_lvl1_pt[index_s] = pte_next(loader_lvl2_pt_paddr); + + serialise_page_table_to_paddr(&mut boot_lvl1_pt) +} + +/// AArch64 loader page tables have two variations: +/// - Loader in EL2, then Stage 1 translations in use, so we have the +/// singular TTBR0_EL2 register containing the Level 0 table; +/// this allows virtual address in the range [0,2^48). +/// - Loader in EL1, then Stage 1 translations are in use, so we have both +/// the TTBR0_EL1 (covering vaddr in range [0,2^48)) and TTBR1_EL2 ( +/// (covering vaddr in the range [2^64-2^48,2^64)), and containing +/// the "Level 0 Lower" page table, and "Level 0 Upper" page table +/// physical addresses respectively. +/// +/// Thus, for EL2 loader, the singular Level 0 page table contains the table +/// descriptors for the "Level 1 Upper" and "Level 1 Lower" page tables. +/// For the EL1 loader, we instead have two Level 0 page tables, and +/// "Level 0 Lower" contains the "Level 1 Lower" descriptor, and "Level 0 +/// Upper" contains the "Level 1 Upper" descriptor. +/// Otherwise, the page tables layout from Level 1 downwards are identical +/// (but not necessarily the layout within the page/table/block descriptors). +/// +/// ```txt +/// (256 TiB) +/// 512 +-- Level 0 --+ 2^48 +/// | | +/// | (empty) | +/// | | +/// k+1 +-------------+ (512 GiB) +/// | Level 1 Upr | ----------> +-- Level 1 --+ +/// k +-------------+ | | +/// | | | (empty) | +/// | | | | +/// | | l+1 +-------------+ (1 GiB) +/// | | | Level 2 Upr | ----------> +-- Level 2 --+ +-------------+ +/// | | l +-------------+ | | ----------> | 2 MiB block | +/// | | | | 511 |-------------| +-------------+ +/// | | | (empty) | | | ----------> | 2 MiB block | +/// | | | | 510 |-------------| +-------------+ +/// | | +-------------+ | | ----------> | 2 MiB block | +/// | | |-------------| +-------------+ +/// | (empty) | Kernel Regions (...) (...) (...) +/// | | |-------------| +-------------+ +/// | | | | ----------> | 2 MiB block | +/// | | m |-------------| +-------------+ p +/// | | | | +/// | | | (empty) | +/// | | | | +/// | | 0 +-------------+ +/// | | +/// | | +/// | | +/// 1 +-------------+ (512 GiB) +/// | Level 1 Lwr | ----------> +-- Level 1 --+ +/// 0 +-------------+ TODO: RAM. +/// +/// +/// Where: +/// k = align_down(kernel_first_vaddr, 512GiB), +/// l = align_down(kernel_first_vaddr, 1GiB), +/// m = align_down(kernel_first_vaddr, 2MiB), +/// p = align_down(kernel_first_paddr, 2MiB), +/// u = align_down(uart_base, 1GiB), +/// ``` +/// +#[unsafe(no_mangle)] +pub extern "C" fn aarch64_setup_pagetables( + kernel_first_vaddr: u64, + kernel_first_paddr: u64, + page_tables_paddr_start: u64, +) -> (u64, u64, u64) { + use aarch64::{ + block_descriptor, lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, + s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}, + table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, + }; + + const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); + + let mut serialise_page_table_to_paddr = { + assert!( + page_tables_paddr_start + == page_tables_paddr_start.next_multiple_of(PAGE_TABLE_SIZE as u64) + ); + + // This maintains the current end of the PT array. + let mut next_pt_paddr = page_tables_paddr_start; + + move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> u64 { + let pt_paddr = next_pt_paddr; + // page_table_bytes.extend(page_table.iter().flat_map(|pte| pte.to_le_bytes())); + next_pt_paddr += PAGE_TABLE_SIZE as u64; + page_table.fill(0); + pt_paddr + } + }; + + struct Region { + start: u64, + end: u64, + } + + let identity_mapped_regions: &[(Region, u64)] = &[]; + // let identity_mapped_regions = { + // let ram_regions = config + // .normal_regions + // .as_ref() + // .expect("AArch64 should have normal_regions"); + + // // println!("{:#x?}", ram_regions); + + // let mut regions: Vec<_> = ram_regions + // .iter() + // .cloned() + // .map(|region| (region, MT_DEVICE_nGnRnE)) + // .collect(); + + // // FIXME: Derive from the kernel build system. + // if let Some(uart_base) = read_symbol_maybe(elf, "uart_addr") { + // let uart_base = align_down(uart_base, PAGE_BITS_4KB); + // regions.push(( + // PlatformConfigRegion { + // start: uart_base, + // end: uart_base + (1 << PAGE_BITS_4KB), + // }, + // MT_DEVICE_nGnRnE, + // )); + // } + + // // FIXME: This is currently assuming implementation details of the BCM2711/ + // // Raspberry Pi 4B spin table implementation, as it is the only + // // platform we have that uses spin tables. Specifically, that + // // it is always located at the 0 page. + // if elf.find_symbol("cpus_release_addr").is_ok() { + // regions.push(( + // PlatformConfigRegion { + // start: 0x0, + // end: 1 << PAGE_BITS_4KB, + // }, + // MT_DEVICE_nGnRnE, + // )); + // } + + // regions.sort_by_key(|(region, _)| region.start); + + // regions + // }; + + // Manufacture the constants as per the diagram. + let k = align_down(kernel_first_vaddr, BLOCK_BITS_512GB); + let l = align_down(kernel_first_vaddr, BLOCK_BITS_1GB); + let m = align_down(kernel_first_vaddr, BLOCK_BITS_2MB); + let p = align_down(kernel_first_paddr, BLOCK_BITS_2MB); + + // Manufacture the kernel page tables, which is relatively straightforward. + let kernel_lvl1_pt_paddr = { + // First, the Level 2 Upr table. + let lvl2_pt_paddr = { + let mut lvl2_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + + let mut vaddr = m; + let mut paddr = p; + while lvl1_index(m) == lvl1_index(vaddr) { + lvl2_pt_kernel[lvl2_index(vaddr)] = block_descriptor(2, paddr, MT_NORMAL); + + vaddr += 1 << BLOCK_BITS_2MB; + paddr += 1 << BLOCK_BITS_2MB; + } + + serialise_page_table_to_paddr(&mut lvl2_pt_kernel) + }; + + // Then, the Level 1 Upr table. + let mut lvl1_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + lvl1_pt_kernel[lvl1_index(l)] = table_descriptor(lvl2_pt_paddr); + + serialise_page_table_to_paddr(&mut lvl1_pt_kernel) + }; + + // Manufacture the RAM page tables, which is a little bit more complicated. + // We assume that normal RAM lies between 0 <= paddr < 512GiB, i.e. + // that lvl0_index(any ram region addr) = 0. + let ram_lvl1_pt_paddr = { + // Validation of assumptions about the identity mapped regions. + let mut previous_end = None; + for (region, _) in identity_mapped_regions.iter() { + assert!(lvl0_index(region.start) == 0); + assert!(lvl0_index(region.end - 1) == 0); + // This is probably an unnecessary assumption. + assert!(region.start.is_multiple_of(4096)); + assert!(region.end.is_multiple_of(4096)); + // This is definitely necessary. + assert!(region.start >= previous_end.unwrap_or(0)); + previous_end = Some(region.end); + } + + // We maintain three active page tables, which contain our previous + // known page table data. As we process regions in ascending order, + // once we have exceeded the bounds of the current reservation we + // can simply push to the page_table_bytes storage and insert into + // the parent PT the descriptor. + // When the current vaddr (/paddr, as identity mapped) exceeds the + // top value we rotate to a new PT. + + struct PageTableConstructor { + invalid: PTE, + levels: [[PTE; ENTRIES]; LEVELS], + level_top: [Addr; LEVELS], + } + + impl + PageTableConstructor + { + const fn new(invalid: PTE, level_top: [Addr; LEVELS]) -> Self { + Self { + invalid, + levels: [[invalid; ENTRIES]; LEVELS], + level_top: level_top, + } + } + + fn lvl(&mut self, lvl: usize) -> &mut [PTE; ENTRIES] { + assert!(lvl < LEVELS); + &mut self.levels[lvl] + } + + fn lvl_top(&mut self, lvl: usize) -> &mut Addr { + assert!(lvl < LEVELS); + &mut self.level_top[lvl] + } + + fn lvl_is_empty(&self, lvl: usize) -> bool { + assert!(lvl < LEVELS); + self.levels[lvl] != [self.invalid; ENTRIES] + } + } + + static mut PTS: PageTableConstructor<4, PAGE_TABLE_ENTRIES, u64, u64> = + PageTableConstructor::new( + 0, + [ + u64::MAX, + 1 << BLOCK_BITS_512GB, + 1 << BLOCK_BITS_1GB, + 1 << BLOCK_BITS_2MB, + ], + ); + + // SAFETY: Trust me. This function is not, and can not, be reentrant, + // and more than that, can only be called once. + #[allow(static_mut_refs)] + let pts = unsafe { &mut PTS }; + + // TODO: Tests... + // This is similar to aligned_power_of_two_regions() for the kernel UT, + // but we restrict it such that the output always is either 1GB, 2MB, or 4KB + // pages. + + // Allowed externally for the final iteration + let mut base = 0u64; + for &(ref region, attr_index) in identity_mapped_regions.iter() { + // println!("RAM Region: {:#x}..{:#x}", base, region.end); + // println!( + // " - Current Lvl1: {:#x}..{:#x}, entries: {}", + // (*pts.lvl_top(1) - (1 << BLOCK_BITS_512GB)), + // *pts.lvl_top(1), + // lvl1_pt.iter().filter(|&&v| v != 0).count() + // ); + // println!( + // " - Current Lvl2: {:#x}..{:#x}, entries: {}", + // (*pts.lvl_top(2) - (1 << BLOCK_BITS_1GB)), + // *pts.lvl_top(2), + // lvl2_pt.iter().filter(|&&v| v != 0).count() + // ); + // println!( + // " - Current Lvl3: {:#x}..{:#x}, entries: {}", + // (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), + // lvl3_vaddr_top, + // lvl3_pt.iter().filter(|&&v| v != 0).count() + // ); + + // Handle the fact that the regions are not contiguous and that + // we might need to skip PT. + + { + if region.start >= *pts.lvl_top(3) { + if !pts.lvl_is_empty(3) { + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(3)); + // println!("[iter] Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{pts.lvl_top(3):#x}", (pts.lvl_top(3) - (1 << BLOCK_BITS_2MB))); + assert!(pts.lvl(2)[lvl2_index(base)] == 0); + pts.lvl(2)[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + } + + // TODO: just compute it. + while region.start >= *pts.lvl_top(3) { + *pts.lvl_top(3) += 1 << BLOCK_BITS_2MB; + } + } + + if region.start >= *pts.lvl_top(2) { + if !pts.lvl_is_empty(2) { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(2)); + // println!("[iter] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{*pts.lvl_top(2):#x}, base: {:#x} lvl1_index(base): {:#x}", (*pts.lvl_top(2) - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + assert!(pts.lvl(1)[lvl1_index(base)] == 0); + pts.lvl(1)[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + } + + // TODO: just compute it. + while region.start >= *pts.lvl_top(2) { + *pts.lvl_top(2) += 1 << BLOCK_BITS_1GB; + } + } + + if region.start >= *pts.lvl_top(1) { + unreachable!( + "impossible as everything should fit here: {:#x}", + *pts.lvl_top(1) + ); + } + } + + // After serialising the old base, update the new one. + base = region.start; + + // Inner Loop: + // Invariant: the page tables in lvl1_pt, lvl2_pt, lvl3_pt + // are either (1) for the current address range, + // or (2) are empty and for a lower level than the current level. + // Also, the values in lvlXXX_vaddr_top are always correct (even if empty) + // Also contiguous within the loop. + // Loop entry: (1) holds by work at the start of each region + while base != region.end { + // Condition is !=, but assert that we never skip it. + assert!(base < region.end); + + let size_bits = region.end.wrapping_sub(base).ilog2(); + let align_bits = min( + size_bits, + // FIXME: Once MSRV is > 1.97, use .lowest_one() method. + if base == 0 { + size_bits + } else { + base.trailing_zeros() + }, + ); + + // Match the size and alignment of the current region to + // the valid PT region sizes. + let (level, bits) = match u64::from(align_bits) { + BLOCK_BITS_1GB.. => (1, BLOCK_BITS_1GB), + BLOCK_BITS_2MB.. => (2, BLOCK_BITS_2MB), + PAGE_BITS_4KB.. => (3, PAGE_BITS_4KB), + 0.. => panic!("impossible; regions should be aligned to 4K at least"), + }; + + let pt_region_size = 1u64 << bits; + let top = base + pt_region_size; + + // println!("- Aligned PT region: {:#x}..{:#x} (size_bits: {}, align_bits: {}, bits: {})", base, top, size_bits, align_bits, bits); + // println!( + // " - Current Lvl1: {:#x}..{:#x}, entries: {}", + // (*pts.lvl_top(1) - (1 << BLOCK_BITS_512GB)), + // *pts.lvl_top(1), + // lvl1_pt.iter().filter(|&&v| v != 0).count() + // ); + // println!( + // " - Current Lvl2: {:#x}..{:#x}, entries: {}", + // (*pts.lvl_top(2) - (1 << BLOCK_BITS_1GB)), + // *pts.lvl_top(2), + // lvl2_pt.iter().filter(|&&v| v != 0).count() + // ); + // println!( + // " - Current Lvl3: {:#x}..{:#x}, entries: {}", + // (pts.lvl_top(3) - (1 << BLOCK_BITS_2MB)), + // pts.lvl_top(3), + // lvl3_pt.iter().filter(|&&v| v != 0).count() + // ); + + match level { + 1 => { + // If it belongs in Level 1 PT, then it must go in + // lvl1 pt. By the inavariant, base < *pts.lvl_top(1). + assert!(base < *pts.lvl_top(1)); + // top is <= *pts.lvl_top(1) (the case where it is the topmost entry) + assert!(top <= *pts.lvl_top(1)); + + assert!(pts.lvl(1)[lvl1_index(base)] == 0); + pts.lvl(1)[lvl1_index(base)] = block_descriptor(1, base, attr_index); + + if top == *pts.lvl_top(1) { + // Invariant maintenance: if the new top would be now equal + // the end of the page table's region top, we need a new + // page table object and add it to the list. + + // This should be possible to handle - we just need to break out of this loop + todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + } + + // Invariant: Lower levels are empty. + assert!(pts.lvl_is_empty(2)); + assert!(pts.lvl_is_empty(3)); + // Invariant maintenance: vaddr_top is right range for current PT. + // it's empty so we need to increment the top to be current top (1G aligned) + 2MIB (512 lvl3 entries) + *pts.lvl_top(3) = top + (1 << BLOCK_BITS_2MB); + // it's empty so we need to increment the top to be current top (1G aligned) + 1G (512 lvl2 entries) + *pts.lvl_top(2) = top + (1 << BLOCK_BITS_1GB); + } + 2 => { + // If it is a 2MiB block, it must go in the Level 2 PT; + // by our invariants: base < *pts.lvl_top(2) and top <= *pts.lvl_top(2) + assert!(base < *pts.lvl_top(2)); + assert!(top <= *pts.lvl_top(2)); + + assert!(pts.lvl(2)[lvl2_index(base)] == 0); + pts.lvl(2)[lvl2_index(base)] = block_descriptor(2, base, attr_index); + + if top == *pts.lvl_top(2) { + // Invariant maintenance: keep for current address range. + // As we're the top of the range, we can serialise the table. + + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(2)); + // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} up to {*pts.lvl_top(2):#x}"); + *pts.lvl_top(2) += 1 << BLOCK_BITS_1GB; + + pts.lvl(1)[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + + if top == *pts.lvl_top(1) { + todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + } + } + + // Invariant: Lower levels are empty. + assert!(pts.lvl_is_empty(3)); + // Invariant maintenance: vaddr_top is right range for current PT. + // it's empty so we need to increment the top to be current top (2MIB aligned) + 2MIB (512 lvl3 entries) + *pts.lvl_top(3) = top + (1 << BLOCK_BITS_2MB); + } + 3 => { + // If it is a 4K page, it must go in the Level 3 PT; + // by our invariants: base < pts.lvl_top(3) and top <= pts.lvl_top(3) + assert!(base < *pts.lvl_top(3)); + assert!(top <= *pts.lvl_top(3)); + + assert!(pts.lvl(3)[lvl3_index(base)] == 0); + pts.lvl(3)[lvl3_index(base)] = page_descriptor(base, attr_index); + + if top == *pts.lvl_top(3) { + // Invariant maintenance: keep for current address range. + // As we're the top of the range, we can serialise the table. + + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(3)); + // println!("Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{pts.lvl_top(3):#x}", (pts.lvl_top(3) - (1 << BLOCK_BITS_2MB))); + *pts.lvl_top(3) += 1 << BLOCK_BITS_2MB; + + assert!(pts.lvl(2)[lvl2_index(base)] == 0); + pts.lvl(2)[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + + if top == *pts.lvl_top(2) { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(2)); + // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{*pts.lvl_top(2):#x}", (*pts.lvl_top(2) - (1 << BLOCK_BITS_1GB))); + *pts.lvl_top(2) += 1 << BLOCK_BITS_1GB; + + assert!(pts.lvl(1)[lvl1_index(base)] == 0); + pts.lvl(1)[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + + if top == *pts.lvl_top(1) { + todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + } + } + } + + // Invariant: lower levels empty is vacuuously true + } + _ => unreachable!("level is 1..=3"), + } + + base = base + pt_region_size; + } + } + + // By the loop invariant, we know that anything before has been serialised. + // However, as we are at the end of the loop now, we might have + // page tables that have been partially filled out, and we need to + // serialise these. + + if !pts.lvl_is_empty(3) { + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(3)); + // println!("[end] Serialise lvl3 table: {lvl3_pt_paddr:#x}"); + assert!(pts.lvl(2)[lvl2_index(base)] == 0); + pts.lvl(2)[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + } + + if !pts.lvl_is_empty(2) { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(2)); + // println!("[end] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{*pts.lvl_top(2):#x}, base: {:#x} lvl1_index(base): {:#x}", (*pts.lvl_top(2) - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + assert!(pts.lvl(1)[lvl1_index(base)] == 0); + pts.lvl(1)[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + } + + // the level1 pt should not be empty. lol. + assert!(!pts.lvl_is_empty(1)); + + // println!("New lvl1 table"); + serialise_page_table_to_paddr(pts.lvl(1)) + }; + + struct Config { + hypervisor: bool, + } + let config = Config { hypervisor: true }; + + // Depending on whether we are in hypervisor mode, we either need to + // return the TTBR0_EL2 or TTBR[0,1]_EL1 values. We return u64::MAX + // so as to return garbage - an unaligned address outside of physical + // memory. + if config.hypervisor { + // Manufacture the Level 0 table, containing the kernel table + // and the RAM tables. + + let mut ttbr0_el2_pt = [0u64; PAGE_TABLE_ENTRIES]; + + assert!(lvl0_index(k) != lvl0_index(0)); + ttbr0_el2_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); + ttbr0_el2_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); + + let ttbr0_el2 = serialise_page_table_to_paddr(&mut ttbr0_el2_pt); + + (ttbr0_el2, u64::MAX, u64::MAX) + } else { + let mut ttbr0_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut ttbr1_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; + + // Kernel in TTBR1 (Upper) + ttbr1_el1_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); + // Identity-mapped RAM in TTBR0 (Lower) + ttbr0_el1_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); + + let ttbr0_el1 = serialise_page_table_to_paddr(&mut ttbr0_el1_pt); + let ttbr1_el1 = serialise_page_table_to_paddr(&mut ttbr1_el1_pt); + + (u64::MAX, ttbr0_el1, ttbr1_el1) + } +} diff --git a/tool/microkit/src/loader.rs b/tool/microkit/src/loader.rs index d41225f6a..2c12b1101 100644 --- a/tool/microkit/src/loader.rs +++ b/tool/microkit/src/loader.rs @@ -4,332 +4,15 @@ // SPDX-License-Identifier: BSD-2-Clause // use crate::elf::{ElfFile, ElfSegmentData}; -use crate::sel4::{Arch, Config, PlatformConfigRegion}; +use crate::sel4::{Arch, Config}; use crate::uimage::uimage_serialise; -use crate::util::{align_down, mb, round_up, struct_to_bytes}; -use std::cmp::min; +use crate::util::{mb, round_up, struct_to_bytes}; use std::fs::File; use std::io::{BufWriter, Write}; use std::mem; use std::ops::Range; use std::path::Path; -macro_rules! grab_symbol { - ($elf: expr, $symbol_name: expr) => { - $elf.find_symbol($symbol_name) - .expect(concat!("Could not find '", $symbol_name, "' symbol")) - }; -} - -// XX: This could be generic on arbitrary if we could specify T:: implements from_le_bytes, -// but we can't. -fn read_symbol_maybe(elf: &ElfFile, symbol_name: &str) -> Option { - let (addr, size) = elf.find_symbol(symbol_name).ok()?; - - let symbol_bytes = elf.get_data(addr, size)?; - - assert!(mem::size_of::() == symbol_bytes.len()); - - Some(u64::from_le_bytes(symbol_bytes.try_into().ok()?)) -} - -macro_rules! write_symbol { - ($loader_image: expr, $image_vaddr: expr, $elf: expr, $symbol: literal, $symbol_var: expr) => { - let (addr, size) = grab_symbol!($elf, $symbol); - let addr = usize::try_from(addr).expect("addr fits in usize"); - let size = usize::try_from(size).expect("size fits in usize"); - let image_vaddr = usize::try_from($image_vaddr).expect("vaddr fits in usize"); - - assert!(addr >= image_vaddr); - assert!(size == ::std::mem::size_of_val(&$symbol_var)); - - let offset: usize = (addr - image_vaddr); - assert!(offset <= $loader_image.len()); - - $loader_image[offset..(offset + size)].copy_from_slice(&$symbol_var.to_le_bytes()); - }; -} - -const PAGE_TABLE_SIZE: usize = 4096; - -pub mod aarch64 { - //! For AArch64, our page tables use the Stage 1 descriptor formats - //! for both EL2 (TTBR0_EL2) and EL1 (TTBR0_EL1/TTBR1_EL1). - //! Stage 2 descriptors are only used when in the EL1&0 regime; which is not - //! the case when in EL2. - - use crate::util::mask; - - pub const LVL0_BITS: u64 = 9; - pub const LVL1_BITS: u64 = 9; - pub const LVL2_BITS: u64 = 9; - pub const LVL3_BITS: u64 = 9; - - pub fn lvl0_index(addr: u64) -> usize { - let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS + LVL1_BITS)) & mask(LVL0_BITS); - idx as usize - } - - pub fn lvl1_index(addr: u64) -> usize { - let idx = (addr >> (BLOCK_BITS_2MB + LVL2_BITS)) & mask(LVL1_BITS); - idx as usize - } - - pub fn lvl2_index(addr: u64) -> usize { - let idx = (addr >> (BLOCK_BITS_2MB)) & mask(LVL2_BITS); - idx as usize - } - - pub fn lvl3_index(addr: u64) -> usize { - let idx = (addr >> PAGE_BITS_4KB) & mask(LVL3_BITS); - idx as usize - } - - /// Stage 1 translation table page/block descriptors have bits[4:2] containing - /// AttrIndex[2:0]. The AttrIndex values depends on our configuration of - /// the `MAIR_EL1` or `MAIR_EL2` registers done in util64.S; - /// This also needs to match the values that seL4 uses. - #[allow(non_upper_case_globals, reason = "matching ARM naming convention")] - pub mod s1_mair_attr_index { - pub const MT_DEVICE_nGnRnE: u64 = 0b000; - pub const MT_DEVICE_nGnRE: u64 = 0b001; - pub const MT_DEVICE_GRE: u64 = 0b010; - pub const MT_NORMAL_NC: u64 = 0b011; - pub const MT_NORMAL: u64 = 0b100; - } - - pub mod descriptor_type { - //! The translation table descriptor formats, as per §D8.3 "Translation - //! table descriptor formats" of ARM DDI 0487 L.b. Specifically, - //! as per "Table D8-48 Determination of descriptor type" - - /// Descriptor type: Table. Condition is lookup level != 3. - pub const TABLE: u64 = 0b11; - /// Descriptor type: Page. Condition is lookup level == 3. - pub const PAGE: u64 = 0b11; - /// Descriptor type: Block. Condition is lookup level != 3. - pub const BLOCK: u64 = 0b01; - /// Descriptor type: Invalid. Strictly speaking bit[1] does not matter. - pub const INVALID: u64 = 0b00; - } - - pub mod shareability_attributes { - //! Per §D8.6.2 "Stage 1 Shareability attributes", these contain the - //! shareability attributes of the descriptor OA for normal-cacheable - //! memory. - - /// Non-shareable - pub const NON_SHAREABLE: u64 = 0b00; - /// Outer-shareable - pub const OUTER_SHAREABLE: u64 = 0b10; - /// Inner-shareable - pub const INNER_SHAREABLE: u64 = 0b11; - } - - /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, - /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address - /// is bits [47:n], and: - /// - /// > For the 4KB granule size, the level 1 descriptor n is 30, - /// > and the level 2 descriptor n is 21. - pub const BLOCK_BITS_1GB: u64 = 30; - - /// Per "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b, - /// subfigure "4KB, 16KB, and 64KB granules, 48-bit OA", the Output address - /// is bits [47:n], and: - /// - /// > For the 4KB granule size, the level 1 descriptor n is 30, - /// > and the level 2 descriptor n is 21. - pub const BLOCK_BITS_2MB: u64 = 21; - - // TODO: - - pub const BLOCK_BITS_512GB: u64 = 39; - pub const PAGE_BITS_4KB: u64 = 12; - - /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and - /// "Figure D8-14 VMSAv8-64 Block descriptor formats" of ARM DDI0487L.b; - /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" - pub fn block_descriptor(level: usize, addr: u64, attr_index: u64) -> u64 { - // Per Table D8-48, Condition for descriptor_type::BLOCK is level != 3. - assert!(level != 3); - - let upper_attributes: u64 = 0; - - let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { - // Match what the seL4 kernel uses for its page tables, which - // is especially necessary for SMP booting which relies on it - // for coherency. See the comment in seL4 `release_secondary_cpus()`. - shareability_attributes::INNER_SHAREABLE - } else { - // Per $R_{PYFVQ}$: - // > If a region is mapped as Device memory or Normal Non-cacheable - // > memory after all enabled translation stages, then the region - // > has an effective Shareability attribute of Outer Shareable. - // - // We override the value we place in here to OUTER_SHAREABLE to match - // how the hardware behaves. This is not necessary but for clarity. - shareability_attributes::OUTER_SHAREABLE - }; - - // AP[2:1], which we set as 0b00 for read/write access: - // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1 - // stage 2: 0b00 is RW for EL2 and no perms for EL1. - const AP_KERNEL_RW: u64 = 0b00; - - // bit[11] is the not global (nG) field, we leave as 0 (global). - // bit[10] is the access flag; depending on FEAT_HAFDBS, when software - // manages the AF memory accesses to the page/block when AF=0 - // raise an Access Fault; when hardware manages the AF it will - // become 1. - // bit[9:8] is SH[1:0] containing stage 1 shareability attributes - // bit[7:6] contains AP[2:1] - // bit[5] is RES0 - // bit[4:2] contains AttrIndex - let lower_attributes: u64 = - (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); - - // bits[47:n] - let output_address: u64 = addr - & !mask(match level { - 1 => BLOCK_BITS_1GB, - 2 => BLOCK_BITS_2MB, - _ => panic!("unsupported level {level} for block descriptor"), - }); - - // address must not have bits above 47 set. - assert!(addr & mask(48) == addr); - - // bits[63:50] describing the "Upper attributes" are left at 0. - // bits[49:48] are RES0 - // bits[47:n] contain the Output address - // bits[n-1:12] are RES0 - // bits[11:2] contain the "Lower attributes" - // bits[1:0] contains the descriptor type - upper_attributes | output_address | lower_attributes | descriptor_type::BLOCK - } - - /// Per "Table D8-52 Stage 1 VMSAv8-64 Block and Page descriptor fields" and - /// "Figure D8-15 VMSAv8-64 Page descriptor formats" of ARM DDI0487L.b; - /// specifically subfigure "4KB granule 48-bit OA". - pub fn page_descriptor(addr: u64, attr_index: u64) -> u64 { - // The main difference between a page descriptor and block descriptor - // is in the size of the output address (OA) and in the descriptor type. - - let upper_attributes: u64 = 0; - - let shareability = if attr_index == s1_mair_attr_index::MT_NORMAL { - // Match what the seL4 kernel uses for its page tables, which - // is especially necessary for SMP booting which relies on it - // for coherency. - shareability_attributes::INNER_SHAREABLE - } else { - // Per $R_{PYFVQ}$: - // > If a region is mapped as Device memory or Normal Non-cacheable - // > memory after all enabled translation stages, then the region - // > has an effective Shareability attribute of Outer Shareable. - // We override the value we place in here to OUTER_SHAREABLE to match - // how the hardware behaves. - shareability_attributes::OUTER_SHAREABLE - }; - - // AP[2:1], which we set as 0b00 for read/write access: - // stage 1: 0b00 is {PrivRead, PrivWrite} and we are EL1/El2 (priv) - const AP_KERNEL_RW: u64 = 0b00; - - // bit[11] is the not global (nG) field, we leave as 0 (global). - // bit[10] is the access flag; depending on FEAT_HAFDBS, when software - // manages the AF memory accesses to the page/block when AF=0 - // raise an Access Fault; when hardware manages the AF it will - // become 1. - // bit[9:8] is SH[1:0] containing stage 1 shareability attributes - // bit[7:6] contains AP[2:1] - // bit[5] is RES0 - // bit[4:2] contains AttrIndex - let lower_attributes: u64 = - (1 << 10) | (AP_KERNEL_RW << 6) | (shareability << 8) | (attr_index << 2); - - // bits[47:12] - let output_address: u64 = addr & !mask(12); - - // address must not have bits above 47 set. - assert!(addr & mask(48) == addr); - - // bits[63:50] describing the "Upper attributes" are left at 0. - // bits[49:48] are RES0 - // bits[47:12] contain the Output address - // bits[11:2] contain the "Lower attributes" - // bits[1:0] contains the descriptor type - upper_attributes | output_address | lower_attributes | descriptor_type::PAGE - } - - /// Per "Table D8-50 Stage 1 VMSAv8-64 Table descriptor fields" and - /// "Figure D8-12 VMSAv8-64 Table descriptor formats" of ARM DDI0487L.b; - /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" - pub fn table_descriptor(addr: u64) -> u64 { - // Per Table D8-48, Condition for descriptor_type::TABLE is level != 3. - - // We don't set any of these attributes, most are hardware-feature conditional - let attributes: u64 = 0; - - // address must not have bits above 47 or below 12 set - assert!(addr & mask(12) == 0x0); - assert!(addr & mask(48) == addr); - - let next_level_table_address = addr; - - // bits[63:59] are "Attributes" - // bits[58:51] are ignored - // bits[50:48] are RES0 - // bits[47:m] is the next-level table address - // note: here m=12 for 4KB granule - // bits[m-1:12] are RES0 - // so this doesn't exist for 4KB granule - // bits[11:2] are ignored - // bits[1:0] contain the descriptor type - attributes | next_level_table_address | descriptor_type::TABLE - } -} - -mod riscv64 { - pub(crate) const BLOCK_BITS_1GB: u64 = 30; - pub(crate) const BLOCK_BITS_2MB: u64 = 21; - pub(crate) const PAGE_BITS_4K: u64 = 12; - - pub(crate) const PAGE_TABLE_INDEX_BITS: u64 = 9; - pub(crate) const PAGE_SHIFT: u64 = 12; - /// This sets the page table entry bits: D,A,X,W,R. - pub(crate) const PTE_TYPE_BITS: u64 = 0b11001110; - // TODO: where does this come from? - pub(crate) const PTE_TYPE_TABLE: u64 = 0; - pub(crate) const PTE_TYPE_VALID: u64 = 1; - - pub(crate) const PTE_PPN0_SHIFT: u64 = 10; - - /// Due to RISC-V having various virtual memory setups, we have this generic function to - /// figure out the page-table index given the total number of page table levels for the - /// platform and which level we are currently looking at. - pub fn pt_index(pt_levels: usize, addr: u64, level: usize) -> usize { - let pt_index_bits = PAGE_TABLE_INDEX_BITS * (pt_levels - level) as u64; - let idx = (addr >> (pt_index_bits + PAGE_SHIFT)) % 512; - - idx as usize - } - - /// Generate physical page number given an address - pub fn pte_ppn(addr: u64) -> u64 { - (addr >> PAGE_SHIFT) << PTE_PPN0_SHIFT - } - - pub fn pte_next(addr: u64) -> u64 { - pte_ppn(addr) | PTE_TYPE_TABLE | PTE_TYPE_VALID - } - - pub fn pte_leaf(addr: u64) -> u64 { - pte_ppn(addr) | PTE_TYPE_BITS | PTE_TYPE_VALID - } -} - /// Checks that each region in the given list does not overlap with any other region. /// Panics upon finding an overlapping region fn check_non_overlapping(regions: &Vec<(u64, u64)>) { @@ -373,7 +56,6 @@ pub struct Loader<'a> { header: LoaderHeader64, region_metadata: Vec, regions: Vec<(u64, &'a [u8])>, - page_table_bytes: Vec, word_size: usize, elf_machine: u16, entry: u64, @@ -463,14 +145,6 @@ impl<'a> Loader<'a> { } } - let Some(kernel_first_vaddr) = kernel_first_vaddr else { - panic!("INTERNAL: could not determine kernel_first_vaddr"); - }; - - let Some(kernel_first_paddr) = kernel_first_paddr else { - panic!("INTERNAL: could not determine kernel_first_paddr"); - }; - let image_segment = loader_elf .segments .iter() @@ -483,7 +157,7 @@ impl<'a> Loader<'a> { // We have to clone here as the image executable is part of this function return object, // and the loader ELF is deserialised in this scope, so its lifetime will be shorter than // the return object. - let mut loader_image = image_segment.data().clone(); + let loader_image = image_segment.data().clone(); if image_vaddr != loader_elf.entry { panic!("The loader entry point must be the first byte in the image"); @@ -512,69 +186,11 @@ impl<'a> Loader<'a> { offset += data.len() as u64; } - let partial_size = loader_image.len() as u64 + let size = loader_image.len() as u64 + mem::size_of::() as u64 + (region_metadata.len() * mem::size_of::()) as u64 + offset; - let page_tables_paddr_start = image_vaddr + partial_size; - - let mut page_table_bytes = Vec::::new(); - match config.arch { - Arch::Aarch64 => { - let (ttbr0_el2, ttbr0_el1, ttbr1_el1) = Loader::aarch64_setup_pagetables( - config, - &loader_elf, - kernel_first_vaddr, - kernel_first_paddr, - page_tables_paddr_start, - &mut page_table_bytes, - ); - - write_symbol!( - loader_image, - image_vaddr, - loader_elf, - "aarch64_pt_ttbr0_el2", - ttbr0_el2 - ); - write_symbol!( - loader_image, - image_vaddr, - loader_elf, - "aarch64_pt_ttbr0_el1", - ttbr0_el1 - ); - write_symbol!( - loader_image, - image_vaddr, - loader_elf, - "aarch64_pt_ttbr1_el1", - ttbr1_el1 - ); - } - Arch::Riscv64 => { - let boot_lvl1_pt = Loader::riscv64_setup_pagetables( - config, - &loader_elf, - kernel_first_vaddr, - kernel_first_paddr, - page_tables_paddr_start, - &mut page_table_bytes, - ); - write_symbol!( - loader_image, - image_vaddr, - loader_elf, - "riscv64_boot_lvl1_pt", - boot_lvl1_pt - ); - } - Arch::X86_64 => unreachable!("x86_64 does not support creating a loader image"), - }; - - let size = partial_size + page_table_bytes.len() as u64; - let mut all_regions_with_loader: Vec<_> = regions .iter() .map(|&(base, data)| (base, data.len() as u64)) @@ -601,7 +217,6 @@ impl<'a> Loader<'a> { header, region_metadata, regions, - page_table_bytes, word_size: kernel_elf.word_size, elf_machine: kernel_elf.machine, entry: loader_elf.entry, @@ -625,8 +240,6 @@ impl<'a> Loader<'a> { bytes.extend_from_slice(data); } - bytes.extend_from_slice(&self.page_table_bytes); - assert!(bytes.len() as u64 == self.header.size); bytes @@ -693,692 +306,4 @@ impl<'a> Loader<'a> { Err(e) => panic!("Could not create '{}': {}", path.display(), e), } } - - /// RISC-V 64 page tables for our purposes uses the Sv39 translation scheme - /// (3-level page tables). - /// - /// It is split into two halves: the Upper/Kernel part of the page tables, - /// which matches the format seL4 expects. The lower half contains an - /// identity mapped region for the loader. - /// - /// ```txt - /// (512 GiB) - /// 512 +---- Level 1 ---+ 2^39 - /// | | - /// | (empty) | - /// | | - /// k+1 +----------------+ (1 GiB) - /// | Level 2 Kernel | ----------> +---- Level 2 ---+ +-------------+ - /// k +----------------+ | | ----------> | 2 MiB block | - /// | | 511 |----------------| +-------------+ - /// | | | | ----------> | 2 MiB block | - /// | | 510 |----------------| +-------------+ - /// | | | | ----------> | 2 MiB block | - /// | | |----------------| +------------- - /// | | (...) (...) (...) Kernel Regions - /// | | |----------------| +-------------+ - /// | | | | ----------> | 2 MiB block | - /// | | l+1 |----------------| +-------------+ - /// | | | Level 3 Kernel | ----+ - /// | | l |----------------| | - /// | | | | | (2 MiB) - /// | | | | +-----> +-- Level 3 --+ +------------+ - /// | | | | | | ----------> | 4 KiB page | - /// | | | | 511 |-------------| +------------+ - /// | | | (empty) | | | ----------> | 4 KiB page | - /// | (empty) | | | |-------------| +------------+ - /// | | | | | | ----------> | 4 KiB page | - /// | | | | m |-------------| +------------+ p - /// | | | | | (empty) | - /// | | | | +-------------+ - /// | | | | - /// | | 0 +----------------+ - /// | | - /// | | - /// | | - /// | | - /// | | - /// s+1 +----------------+ (1 GiB) - /// | Level 2 Loader | ----------> +-- Level 2 --+ +-------------+ - /// s +----------------+ | | ----------> | 2 MiB block | - /// | | 511 +-------------+ +-------------+ - /// | | | | ----------> | 2 MiB block | - /// | (empty) | 510 +-------------+ +-------------+ - /// | | | | ----------> | 2 MiB block | - /// | | |-------------| +-------------+ - /// 0 +----------------+ | | ----------> | 2 MiB block | - /// |-------------| +-------------+ - /// (...) (...) (...) Loader Regions - /// |-------------| +-------------+ - /// | | ----------> | 2 MiB block | - /// |-------------| +-------------+ - /// | | ----------> | 2 MiB block | - /// t +-------------+ +-------------+ - /// | | - /// | (empty) | - /// | | - /// +-------------+ - /// - /// - /// Where: - /// k = align_down(kernel_first_vaddr, 1GiB), - /// l = align_down(kernel_first_vaddr, 2MiB), - /// m = align_down(kernel_first_vaddr, 4KiB), - /// p = align_down(kernel_first_paddr, 4KiB), - /// - /// s = align_down(text_addr, 1GiB), - /// t = align_down(text_addr, 2MiB), - /// ``` - /// - fn riscv64_setup_pagetables( - config: &Config, - elf: &ElfFile, - kernel_first_vaddr: u64, - kernel_first_paddr: u64, - page_tables_paddr_start: u64, - page_table_bytes: &mut Vec, - ) -> u64 { - use riscv64::{pt_index, pte_leaf, pte_next, BLOCK_BITS_1GB, BLOCK_BITS_2MB, PAGE_BITS_4K}; - - let (text_addr, _) = grab_symbol!(elf, "_text"); - - // We map the loader using 2MB pages, so make sure the base is actually aligned. - assert!(text_addr.is_multiple_of(1 << BLOCK_BITS_2MB)); - - const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); - - let mut serialise_page_table_to_paddr = { - let page_tables_paddr_start = { - let aligned_pt_paddr_start = - page_tables_paddr_start.next_multiple_of(PAGE_TABLE_SIZE as u64); - if aligned_pt_paddr_start != page_tables_paddr_start { - let alignment_diff = - (aligned_pt_paddr_start - page_tables_paddr_start) as usize; - page_table_bytes.resize(alignment_diff, 0); - } - - aligned_pt_paddr_start - }; - - // This maintains the current end of the PT array. - let mut next_pt_paddr = page_tables_paddr_start; - - move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> u64 { - let pt_paddr = next_pt_paddr; - page_table_bytes.extend(page_table.iter().flat_map(|pte| pte.to_le_bytes())); - next_pt_paddr += PAGE_TABLE_SIZE as u64; - page_table.fill(0); - pt_paddr - } - }; - - let num_pt_levels = config.riscv_pt_levels.unwrap().levels(); - assert!(num_pt_levels == 3); - - // Manufacture the constants as per the diagram. - let k = align_down(kernel_first_vaddr, BLOCK_BITS_1GB); - let l = align_down(kernel_first_vaddr, BLOCK_BITS_2MB); - let m = align_down(kernel_first_vaddr, PAGE_BITS_4K); - let p = align_down(kernel_first_paddr, PAGE_BITS_4K); - - let s = align_down(text_addr, BLOCK_BITS_1GB); - let t = align_down(text_addr, BLOCK_BITS_2MB); - - // Manufacture the kernel page tables - let kernel_lvl2_pt_paddr = { - let mut lvl2_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; - - let mut paddr = p; - let index_l = pt_index(num_pt_levels, l, 2); - - lvl2_pt_kernel[index_l] = if kernel_first_vaddr.is_multiple_of(1 << BLOCK_BITS_2MB) { - assert!(paddr.is_multiple_of(1 << BLOCK_BITS_2MB)); - let pte = pte_leaf(paddr); - paddr += 1 << BLOCK_BITS_2MB; - pte - } else { - let mut lvl3_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; - - let index_m = pt_index(num_pt_levels, m, 3); - - for index in index_m..512 { - lvl3_pt_kernel[index] = pte_leaf(paddr); - paddr += 1 << PAGE_BITS_4K; - } - - let kernel_lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt_kernel); - pte_next(kernel_lvl3_pt_paddr) - }; - - for index in (index_l + 1)..512 { - lvl2_pt_kernel[index] = pte_leaf(paddr); - paddr += 1 << BLOCK_BITS_2MB; - } - - serialise_page_table_to_paddr(&mut lvl2_pt_kernel) - }; - - // Manufacture the loader page tables, which is relatively straightforward - let loader_lvl2_pt_paddr = { - let mut lvl2_pt_loader = [0u64; PAGE_TABLE_ENTRIES]; - - // Identity mapped, so vaddr == paddr. - let mut paddr = t; - - for index in pt_index(num_pt_levels, t, 2)..512 { - lvl2_pt_loader[index] = pte_leaf(paddr); - paddr += 1 << BLOCK_BITS_2MB; - } - - serialise_page_table_to_paddr(&mut lvl2_pt_loader) - }; - - // Manufacture the Level 1 table - let mut boot_lvl1_pt = [0u64; PAGE_TABLE_ENTRIES]; - - let index_s = pt_index(num_pt_levels, s, 1); - let index_k = pt_index(num_pt_levels, k, 1); - boot_lvl1_pt[index_k] = pte_next(kernel_lvl2_pt_paddr); - boot_lvl1_pt[index_s] = pte_next(loader_lvl2_pt_paddr); - - serialise_page_table_to_paddr(&mut boot_lvl1_pt) - } - - /// AArch64 loader page tables have two variations: - /// - Loader in EL2, then Stage 1 translations in use, so we have the - /// singular TTBR0_EL2 register containing the Level 0 table; - /// this allows virtual address in the range [0,2^48). - /// - Loader in EL1, then Stage 1 translations are in use, so we have both - /// the TTBR0_EL1 (covering vaddr in range [0,2^48)) and TTBR1_EL2 ( - /// (covering vaddr in the range [2^64-2^48,2^64)), and containing - /// the "Level 0 Lower" page table, and "Level 0 Upper" page table - /// physical addresses respectively. - /// - /// Thus, for EL2 loader, the singular Level 0 page table contains the table - /// descriptors for the "Level 1 Upper" and "Level 1 Lower" page tables. - /// For the EL1 loader, we instead have two Level 0 page tables, and - /// "Level 0 Lower" contains the "Level 1 Lower" descriptor, and "Level 0 - /// Upper" contains the "Level 1 Upper" descriptor. - /// Otherwise, the page tables layout from Level 1 downwards are identical - /// (but not necessarily the layout within the page/table/block descriptors). - /// - /// ```txt - /// (256 TiB) - /// 512 +-- Level 0 --+ 2^48 - /// | | - /// | (empty) | - /// | | - /// k+1 +-------------+ (512 GiB) - /// | Level 1 Upr | ----------> +-- Level 1 --+ - /// k +-------------+ | | - /// | | | (empty) | - /// | | | | - /// | | l+1 +-------------+ (1 GiB) - /// | | | Level 2 Upr | ----------> +-- Level 2 --+ +-------------+ - /// | | l +-------------+ | | ----------> | 2 MiB block | - /// | | | | 511 |-------------| +-------------+ - /// | | | (empty) | | | ----------> | 2 MiB block | - /// | | | | 510 |-------------| +-------------+ - /// | | +-------------+ | | ----------> | 2 MiB block | - /// | | |-------------| +-------------+ - /// | (empty) | Kernel Regions (...) (...) (...) - /// | | |-------------| +-------------+ - /// | | | | ----------> | 2 MiB block | - /// | | m |-------------| +-------------+ p - /// | | | | - /// | | | (empty) | - /// | | | | - /// | | 0 +-------------+ - /// | | - /// | | - /// | | - /// 1 +-------------+ (512 GiB) - /// | Level 1 Lwr | ----------> +-- Level 1 --+ - /// 0 +-------------+ TODO: RAM. - /// - /// - /// Where: - /// k = align_down(kernel_first_vaddr, 512GiB), - /// l = align_down(kernel_first_vaddr, 1GiB), - /// m = align_down(kernel_first_vaddr, 2MiB), - /// p = align_down(kernel_first_paddr, 2MiB), - /// u = align_down(uart_base, 1GiB), - /// ``` - /// - fn aarch64_setup_pagetables( - config: &Config, - elf: &ElfFile, - kernel_first_vaddr: u64, - kernel_first_paddr: u64, - page_tables_paddr_start: u64, - page_table_bytes: &mut Vec, - ) -> (u64, u64, u64) { - use aarch64::{ - block_descriptor, lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, - s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}, - table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, - }; - - const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); - - let mut serialise_page_table_to_paddr = { - let page_tables_paddr_start = { - let aligned_pt_paddr_start = - page_tables_paddr_start.next_multiple_of(PAGE_TABLE_SIZE as u64); - if aligned_pt_paddr_start != page_tables_paddr_start { - let alignment_diff = - (aligned_pt_paddr_start - page_tables_paddr_start) as usize; - page_table_bytes.resize(alignment_diff, 0); - } - - aligned_pt_paddr_start - }; - - // This maintains the current end of the PT array. - let mut next_pt_paddr = page_tables_paddr_start; - - move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> u64 { - let pt_paddr = next_pt_paddr; - page_table_bytes.extend(page_table.iter().flat_map(|pte| pte.to_le_bytes())); - next_pt_paddr += PAGE_TABLE_SIZE as u64; - page_table.fill(0); - pt_paddr - } - }; - - let identity_mapped_regions = { - let ram_regions = config - .normal_regions - .as_ref() - .expect("AArch64 should have normal_regions"); - - // println!("{:#x?}", ram_regions); - - let mut regions: Vec<_> = ram_regions - .iter() - .cloned() - .map(|region| (region, MT_DEVICE_nGnRnE)) - .collect(); - - // FIXME: Derive from the kernel build system. - if let Some(uart_base) = read_symbol_maybe(elf, "uart_addr") { - let uart_base = align_down(uart_base, PAGE_BITS_4KB); - regions.push(( - PlatformConfigRegion { - start: uart_base, - end: uart_base + (1 << PAGE_BITS_4KB), - }, - MT_DEVICE_nGnRnE, - )); - } - - // FIXME: This is currently assuming implementation details of the BCM2711/ - // Raspberry Pi 4B spin table implementation, as it is the only - // platform we have that uses spin tables. Specifically, that - // it is always located at the 0 page. - if elf.find_symbol("cpus_release_addr").is_ok() { - regions.push(( - PlatformConfigRegion { - start: 0x0, - end: 1 << PAGE_BITS_4KB, - }, - MT_DEVICE_nGnRnE, - )); - } - - regions.sort_by_key(|(region, _)| region.start); - - regions - }; - - // Manufacture the constants as per the diagram. - let k = align_down(kernel_first_vaddr, BLOCK_BITS_512GB); - let l = align_down(kernel_first_vaddr, BLOCK_BITS_1GB); - let m = align_down(kernel_first_vaddr, BLOCK_BITS_2MB); - let p = align_down(kernel_first_paddr, BLOCK_BITS_2MB); - - // Manufacture the kernel page tables, which is relatively straightforward. - let kernel_lvl1_pt_paddr = { - // First, the Level 2 Upr table. - let lvl2_pt_paddr = { - let mut lvl2_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; - - let mut vaddr = m; - let mut paddr = p; - while lvl1_index(m) == lvl1_index(vaddr) { - lvl2_pt_kernel[lvl2_index(vaddr)] = block_descriptor(2, paddr, MT_NORMAL); - - vaddr += 1 << BLOCK_BITS_2MB; - paddr += 1 << BLOCK_BITS_2MB; - } - - serialise_page_table_to_paddr(&mut lvl2_pt_kernel) - }; - - // Then, the Level 1 Upr table. - let mut lvl1_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; - lvl1_pt_kernel[lvl1_index(l)] = table_descriptor(lvl2_pt_paddr); - - serialise_page_table_to_paddr(&mut lvl1_pt_kernel) - }; - - // Manufacture the RAM page tables, which is a little bit more complicated. - // We assume that normal RAM lies between 0 <= paddr < 512GiB, i.e. - // that lvl0_index(any ram region addr) = 0. - let ram_lvl1_pt_paddr = { - // Validation of assumptions about the identity mapped regions. - let mut previous_end = None; - for (region, _) in identity_mapped_regions.iter() { - assert!(lvl0_index(region.start) == 0); - assert!(lvl0_index(region.end - 1) == 0); - // This is probably an unnecessary assumption. - assert!(region.start.is_multiple_of(4096)); - assert!(region.end.is_multiple_of(4096)); - // This is definitely necessary. - assert!(region.start >= previous_end.unwrap_or(0)); - previous_end = Some(region.end); - } - - // We maintain three active page tables, which contain our previous - // known page table data. As we process regions in ascending order, - // once we have exceeded the bounds of the current reservation we - // can simply push to the page_table_bytes storage and insert into - // the parent PT the descriptor. - // When the current vaddr (/paddr, as identity mapped) exceeds the - // top value we rotate to a new PT. - - let mut lvl1_pt = [0u64; PAGE_TABLE_ENTRIES]; - let mut lvl2_pt = [0u64; PAGE_TABLE_ENTRIES]; - let mut lvl3_pt = [0u64; PAGE_TABLE_ENTRIES]; - // TODO: These should be defines. Note that the top is the size of 1 level of the next level up. - // TODO: LVL1_ENTRY_RANGE? idk - #[allow(unused_mut)] - let mut lvl1_vaddr_top = 1 << BLOCK_BITS_512GB; - let mut lvl2_vaddr_top = 1 << BLOCK_BITS_1GB; - let mut lvl3_vaddr_top = 1 << BLOCK_BITS_2MB; - - // TODO: Tests... - // This is similar to aligned_power_of_two_regions() for the kernel UT, - // but we restrict it such that the output always is either 1GB, 2MB, or 4KB - // pages. - - // Allowed externally for the final iteration - let mut base = 0u64; - for &(ref region, attr_index) in identity_mapped_regions.iter() { - // println!("RAM Region: {:#x}..{:#x}", base, region.end); - // println!( - // " - Current Lvl1: {:#x}..{:#x}, entries: {}", - // (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), - // lvl1_vaddr_top, - // lvl1_pt.iter().filter(|&&v| v != 0).count() - // ); - // println!( - // " - Current Lvl2: {:#x}..{:#x}, entries: {}", - // (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), - // lvl2_vaddr_top, - // lvl2_pt.iter().filter(|&&v| v != 0).count() - // ); - // println!( - // " - Current Lvl3: {:#x}..{:#x}, entries: {}", - // (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), - // lvl3_vaddr_top, - // lvl3_pt.iter().filter(|&&v| v != 0).count() - // ); - - // Handle the fact that the regions are not contiguous and that - // we might need to skip PT. - - { - if region.start >= lvl3_vaddr_top { - if lvl3_pt != [0; _] { - let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); - // println!("[iter] Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{lvl3_vaddr_top:#x}", (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB))); - assert!(lvl2_pt[lvl2_index(base)] == 0); - lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); - } - - // TODO: just compute it. - while region.start >= lvl3_vaddr_top { - lvl3_vaddr_top += 1 << BLOCK_BITS_2MB; - } - } - - if region.start >= lvl2_vaddr_top { - if lvl2_pt != [0; _] { - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); - // println!("[iter] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); - assert!(lvl1_pt[lvl1_index(base)] == 0); - lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); - } - - // TODO: just compute it. - while region.start >= lvl2_vaddr_top { - lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; - } - } - - if region.start >= lvl1_vaddr_top { - unreachable!( - "impossible as everything should fit here: {lvl1_vaddr_top:#x}" - ); - } - } - - // After serialising the old base, update the new one. - base = region.start; - - // Inner Loop: - // Invariant: the page tables in lvl1_pt, lvl2_pt, lvl3_pt - // are either (1) for the current address range, - // or (2) are empty and for a lower level than the current level. - // Also, the values in lvlXXX_vaddr_top are always correct (even if empty) - // Also contiguous within the loop. - // Loop entry: (1) holds by work at the start of each region - while base != region.end { - // Condition is !=, but assert that we never skip it. - assert!(base < region.end); - - let size_bits = region.end.wrapping_sub(base).ilog2(); - let align_bits = min( - size_bits, - // FIXME: Once MSRV is > 1.97, use .lowest_one() method. - if base == 0 { - size_bits - } else { - base.trailing_zeros() - }, - ); - - // Match the size and alignment of the current region to - // the valid PT region sizes. - let (level, bits) = match u64::from(align_bits) { - BLOCK_BITS_1GB.. => (1, BLOCK_BITS_1GB), - BLOCK_BITS_2MB.. => (2, BLOCK_BITS_2MB), - PAGE_BITS_4KB.. => (3, PAGE_BITS_4KB), - 0.. => panic!("impossible; regions should be aligned to 4K at least"), - }; - - let pt_region_size = 1u64 << bits; - let top = base + pt_region_size; - - // println!("- Aligned PT region: {:#x}..{:#x} (size_bits: {}, align_bits: {}, bits: {})", base, top, size_bits, align_bits, bits); - // println!( - // " - Current Lvl1: {:#x}..{:#x}, entries: {}", - // (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), - // lvl1_vaddr_top, - // lvl1_pt.iter().filter(|&&v| v != 0).count() - // ); - // println!( - // " - Current Lvl2: {:#x}..{:#x}, entries: {}", - // (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), - // lvl2_vaddr_top, - // lvl2_pt.iter().filter(|&&v| v != 0).count() - // ); - // println!( - // " - Current Lvl3: {:#x}..{:#x}, entries: {}", - // (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), - // lvl3_vaddr_top, - // lvl3_pt.iter().filter(|&&v| v != 0).count() - // ); - - match level { - 1 => { - // If it belongs in Level 1 PT, then it must go in - // lvl1 pt. By the inavariant, base < lvl1_vaddr_top. - assert!(base < lvl1_vaddr_top); - // top is <= lvl1_vaddr_top (the case where it is the topmost entry) - assert!(top <= lvl1_vaddr_top); - - assert!(lvl1_pt[lvl1_index(base)] == 0); - lvl1_pt[lvl1_index(base)] = block_descriptor(1, base, attr_index); - - if top == lvl1_vaddr_top { - // Invariant maintenance: if the new top would be now equal - // the end of the page table's region top, we need a new - // page table object and add it to the list. - - // This should be possible to handle - we just need to break out of this loop - todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); - } - - // Invariant: Lower levels are empty. - assert!(lvl2_pt == [0; _]); - assert!(lvl3_pt == [0; _]); - // Invariant maintenance: vaddr_top is right range for current PT. - // it's empty so we need to increment the top to be current top (1G aligned) + 2MIB (512 lvl3 entries) - lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); - // it's empty so we need to increment the top to be current top (1G aligned) + 1G (512 lvl2 entries) - lvl2_vaddr_top = top + (1 << BLOCK_BITS_1GB); - } - 2 => { - // If it is a 2MiB block, it must go in the Level 2 PT; - // by our invariants: base < lvl2_vaddr_top and top <= lvl2_vaddr_top - assert!(base < lvl2_vaddr_top); - assert!(top <= lvl2_vaddr_top); - - assert!(lvl2_pt[lvl2_index(base)] == 0); - lvl2_pt[lvl2_index(base)] = block_descriptor(2, base, attr_index); - - if top == lvl2_vaddr_top { - // Invariant maintenance: keep for current address range. - // As we're the top of the range, we can serialise the table. - - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); - // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} up to {lvl2_vaddr_top:#x}"); - lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; - - lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); - - if top == lvl1_vaddr_top { - todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); - } - } - - // Invariant: Lower levels are empty. - assert!(lvl3_pt == [0; _]); - // Invariant maintenance: vaddr_top is right range for current PT. - // it's empty so we need to increment the top to be current top (2MIB aligned) + 2MIB (512 lvl3 entries) - lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); - } - 3 => { - // If it is a 4K page, it must go in the Level 3 PT; - // by our invariants: base < lvl3_vaddr_top and top <= lvl3_vaddr_top - assert!(base < lvl3_vaddr_top); - assert!(top <= lvl3_vaddr_top); - - assert!(lvl3_pt[lvl3_index(base)] == 0); - lvl3_pt[lvl3_index(base)] = page_descriptor(base, attr_index); - - if top == lvl3_vaddr_top { - // Invariant maintenance: keep for current address range. - // As we're the top of the range, we can serialise the table. - - let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); - // println!("Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{lvl3_vaddr_top:#x}", (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB))); - lvl3_vaddr_top += 1 << BLOCK_BITS_2MB; - - assert!(lvl2_pt[lvl2_index(base)] == 0); - lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); - - if top == lvl2_vaddr_top { - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); - // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB))); - lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; - - assert!(lvl1_pt[lvl1_index(base)] == 0); - lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); - - if top == lvl1_vaddr_top { - todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); - } - } - } - - // Invariant: lower levels empty is vacuuously true - } - _ => unreachable!("level is 1..=3"), - } - - base = base + pt_region_size; - } - } - - // By the loop invariant, we know that anything before has been serialised. - // However, as we are at the end of the loop now, we might have - // page tables that have been partially filled out, and we need to - // serialise these. - - if lvl3_pt != [0; _] { - let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); - // println!("[end] Serialise lvl3 table: {lvl3_pt_paddr:#x}"); - assert!(lvl2_pt[lvl2_index(base)] == 0); - lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); - } - - if lvl2_pt != [0; _] { - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); - // println!("[end] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); - assert!(lvl1_pt[lvl1_index(base)] == 0); - lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); - } - - // the level1 pt should not be empty. lol. - assert!(lvl1_pt != [0; _]); - - // println!("New lvl1 table"); - serialise_page_table_to_paddr(&mut lvl1_pt) - }; - - // Depending on whether we are in hypervisor mode, we either need to - // return the TTBR0_EL2 or TTBR[0,1]_EL1 values. We return u64::MAX - // so as to return garbage - an unaligned address outside of physical - // memory. - if config.hypervisor { - // Manufacture the Level 0 table, containing the kernel table - // and the RAM tables. - - let mut ttbr0_el2_pt = [0u64; PAGE_TABLE_ENTRIES]; - - assert!(lvl0_index(k) != lvl0_index(0)); - ttbr0_el2_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); - ttbr0_el2_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); - - let ttbr0_el2 = serialise_page_table_to_paddr(&mut ttbr0_el2_pt); - - (ttbr0_el2, u64::MAX, u64::MAX) - } else { - let mut ttbr0_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; - let mut ttbr1_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; - - // Kernel in TTBR1 (Upper) - ttbr1_el1_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); - // Identity-mapped RAM in TTBR0 (Lower) - ttbr0_el1_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); - - let ttbr0_el1 = serialise_page_table_to_paddr(&mut ttbr0_el1_pt); - let ttbr1_el1 = serialise_page_table_to_paddr(&mut ttbr1_el1_pt); - - (u64::MAX, ttbr0_el1, ttbr1_el1) - } - } } From 30cc89eef743cb71ed677c16e9c9410775f3e4ac Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Wed, 29 Jul 2026 17:17:45 +1000 Subject: [PATCH 07/29] fixes Signed-off-by: Julia Vassiliki --- loader/src/aarch64/mmu.c | 8 +-- loader/src/page_tables.rs | 133 ++++++++++++++++++++++-------------- tool/microkit/src/loader.rs | 3 + 3 files changed, 88 insertions(+), 56 deletions(-) diff --git a/loader/src/aarch64/mmu.c b/loader/src/aarch64/mmu.c index 2fbdcc150..ad511615b 100644 --- a/loader/src/aarch64/mmu.c +++ b/loader/src/aarch64/mmu.c @@ -30,12 +30,10 @@ extern struct ret aarch64_setup_pagetables(uint64_t kernel_first_vaddr, uint64_t int arch_mmu_enable(int logical_cpu) { - puts("setup1\n"); struct ret x = aarch64_setup_pagetables(0, 0, 0); - aarch64_pt_ttbr0_el1 = x.a; - aarch64_pt_ttbr1_el1 = x.b; - aarch64_pt_ttbr0_el2 = x.c; - puts("setup\n"); + aarch64_pt_ttbr0_el2 = x.a; + aarch64_pt_ttbr0_el1 = x.b; + aarch64_pt_ttbr1_el1 = x.c; int r; enum el el; diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index 31bcca1c9..d3ebfe3a1 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -622,9 +622,21 @@ pub extern "C" fn aarch64_setup_pagetables( table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, }; + let kernel_first_vaddr = 551366426624; + let kernel_first_paddr = 1610612736; + const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); let mut serialise_page_table_to_paddr = { + #[repr(align(4096))] + struct PtBytes([[u8; 4096]; 100]); + static mut PAGE_TABLE_BYTES: PtBytes = PtBytes([[0; _]; _]); + // SAFETY: Trust me (lol) + #[allow(static_mut_refs)] + let mut page_table_bytes = unsafe { &mut PAGE_TABLE_BYTES.0 }; + + let page_tables_paddr_start = &raw mut PAGE_TABLE_BYTES as u64; + assert!( page_tables_paddr_start == page_tables_paddr_start.next_multiple_of(PAGE_TABLE_SIZE as u64) @@ -632,11 +644,18 @@ pub extern "C" fn aarch64_setup_pagetables( // This maintains the current end of the PT array. let mut next_pt_paddr = page_tables_paddr_start; + let mut i = 0; move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> u64 { let pt_paddr = next_pt_paddr; - // page_table_bytes.extend(page_table.iter().flat_map(|pte| pte.to_le_bytes())); + page_table + .iter() + .flat_map(|pte| pte.to_le_bytes()) + .zip(page_table_bytes[i].iter_mut()) + .for_each(|(byte, dest)| *dest = byte); + next_pt_paddr += PAGE_TABLE_SIZE as u64; + i += 0; page_table.fill(0); pt_paddr } @@ -646,52 +665,64 @@ pub extern "C" fn aarch64_setup_pagetables( start: u64, end: u64, } + let ram_regions = [ + Region { start: 0x60000000, end: 0xc0000000 }, + ]; + + const MAX_NUM_REGIONS: usize = 16; + + let mut regions = [const { core::mem::MaybeUninit::uninit() }; MAX_NUM_REGIONS]; + let identity_mapped_regions: &mut [(Region, _)] = { + // Conceptually want we want is an 'arrayvec', but to not pull in more + // code we implement this less-efficiently MaybeUninit. + // We implement something very similar to the currently-unstable + // write_iter implementation: + // https://github.com/rust-lang/rust/blob/1.97.1/library/core/src/mem/maybe_uninit.rs#L1384-L1406 + let mut regions_len = 0; + + assert!(ram_regions.len() <= regions.len()); + + let ram_regions_it = ram_regions.into_iter().map(|r| (r, MT_DEVICE_nGnRnE)); + + let all_regions_it = ram_regions_it.chain([(Region { start: 0x9000000, end: 0x9001000 }, MT_DEVICE_nGnRnE)]); + + // // FIXME: Derive from the kernel build system. + // if let Some(uart_base) = read_symbol_maybe(elf, "uart_addr") { + // let uart_base = align_down(uart_base, PAGE_BITS_4KB); + // regions.push(( + // PlatformConfigRegion { + // start: uart_base, + // end: uart_base + (1 << PAGE_BITS_4KB), + // }, + // MT_DEVICE_nGnRnE, + // )); + // } + // // FIXME: This is currently assuming implementation details of the BCM2711/ + // // Raspberry Pi 4B spin table implementation, as it is the only + // // platform we have that uses spin tables. Specifically, that + // // it is always located at the 0 page. + // if elf.find_symbol("cpus_release_addr").is_ok() { + // regions.push(( + // PlatformConfigRegion { + // start: 0x0, + // end: 1 << PAGE_BITS_4KB, + // }, + // MT_DEVICE_nGnRnE, + // )); + // } + + for (entry, region) in regions.iter_mut().zip(all_regions_it) { + entry.write(region); + regions_len += 1; + } + + let regions = unsafe { (&mut regions[0..regions_len]).assume_init_mut() }; - let identity_mapped_regions: &[(Region, u64)] = &[]; - // let identity_mapped_regions = { - // let ram_regions = config - // .normal_regions - // .as_ref() - // .expect("AArch64 should have normal_regions"); - - // // println!("{:#x?}", ram_regions); - - // let mut regions: Vec<_> = ram_regions - // .iter() - // .cloned() - // .map(|region| (region, MT_DEVICE_nGnRnE)) - // .collect(); - - // // FIXME: Derive from the kernel build system. - // if let Some(uart_base) = read_symbol_maybe(elf, "uart_addr") { - // let uart_base = align_down(uart_base, PAGE_BITS_4KB); - // regions.push(( - // PlatformConfigRegion { - // start: uart_base, - // end: uart_base + (1 << PAGE_BITS_4KB), - // }, - // MT_DEVICE_nGnRnE, - // )); - // } - - // // FIXME: This is currently assuming implementation details of the BCM2711/ - // // Raspberry Pi 4B spin table implementation, as it is the only - // // platform we have that uses spin tables. Specifically, that - // // it is always located at the 0 page. - // if elf.find_symbol("cpus_release_addr").is_ok() { - // regions.push(( - // PlatformConfigRegion { - // start: 0x0, - // end: 1 << PAGE_BITS_4KB, - // }, - // MT_DEVICE_nGnRnE, - // )); - // } - - // regions.sort_by_key(|(region, _)| region.start); - - // regions - // }; + // Need to use 'sort_unstable_by_key' as sort_by_key is not in-place. + regions.sort_unstable_by_key(|(region, _)| region.start); + + regions + }; // Manufacture the constants as per the diagram. let k = align_down(kernel_first_vaddr, BLOCK_BITS_512GB); @@ -750,7 +781,7 @@ pub extern "C" fn aarch64_setup_pagetables( // top value we rotate to a new PT. struct PageTableConstructor { - invalid: PTE, + empty: PTE, levels: [[PTE; ENTRIES]; LEVELS], level_top: [Addr; LEVELS], } @@ -758,10 +789,10 @@ pub extern "C" fn aarch64_setup_pagetables( impl PageTableConstructor { - const fn new(invalid: PTE, level_top: [Addr; LEVELS]) -> Self { + const fn new(empty: PTE, level_top: [Addr; LEVELS]) -> Self { Self { - invalid, - levels: [[invalid; ENTRIES]; LEVELS], + empty, + levels: [[empty; ENTRIES]; LEVELS], level_top: level_top, } } @@ -778,7 +809,7 @@ pub extern "C" fn aarch64_setup_pagetables( fn lvl_is_empty(&self, lvl: usize) -> bool { assert!(lvl < LEVELS); - self.levels[lvl] != [self.invalid; ENTRIES] + self.levels[lvl] == [self.empty; ENTRIES] } } diff --git a/tool/microkit/src/loader.rs b/tool/microkit/src/loader.rs index 2c12b1101..860caca70 100644 --- a/tool/microkit/src/loader.rs +++ b/tool/microkit/src/loader.rs @@ -159,6 +159,9 @@ impl<'a> Loader<'a> { // the return object. let loader_image = image_segment.data().clone(); + println!("kernel_first_vaddr: {kernel_first_vaddr:?}"); + println!("kernel_first_paddr: {kernel_first_paddr:?}"); + if image_vaddr != loader_elf.entry { panic!("The loader entry point must be the first byte in the image"); } From aaaa69a0f4c60b2be145293a12b7b27611c0b5ef Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Wed, 29 Jul 2026 17:25:07 +1000 Subject: [PATCH 08/29] back to old style Signed-off-by: Julia Vassiliki --- loader/src/page_tables.rs | 314 +++++++++++++++++++------------------- 1 file changed, 156 insertions(+), 158 deletions(-) diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index d3ebfe3a1..00e8cad4e 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -549,6 +549,43 @@ pub extern "C" fn riscv64_setup_pagetables( serialise_page_table_to_paddr(&mut boot_lvl1_pt) } +pub struct Writer; + +impl fmt::Write for Writer { + fn write_str(&mut self, s: &str) -> Result<(), fmt::Error> { + for c in s.bytes() { + unsafe { + puts(core::ffi::CStr::from_bytes_with_nul_unchecked(&[c.into(), 0]).as_ptr()) + }; + } + Ok(()) + } +} + +#[allow(unused)] +pub fn print(args: fmt::Arguments) { + use fmt::Write; + Writer{}.write_fmt(args).unwrap(); +} + +#[macro_export] +macro_rules! print { + ($($arg:tt)*) => {{ + print(format_args!($($arg)*)); + }} +} + +#[macro_export] +macro_rules! println { + () => {{ + print!("\n"); + }}; + + ($($arg:tt)*) => {{ + print!("{}\n", format_args!($($arg)*)); + }} +} + /// AArch64 loader page tables have two variations: /// - Loader in EL2, then Stage 1 translations in use, so we have the /// singular TTBR0_EL2 register containing the Level 0 table; @@ -780,54 +817,15 @@ pub extern "C" fn aarch64_setup_pagetables( // When the current vaddr (/paddr, as identity mapped) exceeds the // top value we rotate to a new PT. - struct PageTableConstructor { - empty: PTE, - levels: [[PTE; ENTRIES]; LEVELS], - level_top: [Addr; LEVELS], - } - - impl - PageTableConstructor - { - const fn new(empty: PTE, level_top: [Addr; LEVELS]) -> Self { - Self { - empty, - levels: [[empty; ENTRIES]; LEVELS], - level_top: level_top, - } - } - - fn lvl(&mut self, lvl: usize) -> &mut [PTE; ENTRIES] { - assert!(lvl < LEVELS); - &mut self.levels[lvl] - } - - fn lvl_top(&mut self, lvl: usize) -> &mut Addr { - assert!(lvl < LEVELS); - &mut self.level_top[lvl] - } - - fn lvl_is_empty(&self, lvl: usize) -> bool { - assert!(lvl < LEVELS); - self.levels[lvl] == [self.empty; ENTRIES] - } - } - - static mut PTS: PageTableConstructor<4, PAGE_TABLE_ENTRIES, u64, u64> = - PageTableConstructor::new( - 0, - [ - u64::MAX, - 1 << BLOCK_BITS_512GB, - 1 << BLOCK_BITS_1GB, - 1 << BLOCK_BITS_2MB, - ], - ); - - // SAFETY: Trust me. This function is not, and can not, be reentrant, - // and more than that, can only be called once. - #[allow(static_mut_refs)] - let pts = unsafe { &mut PTS }; + let mut lvl1_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut lvl2_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut lvl3_pt = [0u64; PAGE_TABLE_ENTRIES]; + // TODO: These should be defines. Note that the top is the size of 1 level of the next level up. + // TODO: LVL1_ENTRY_RANGE? idk + #[allow(unused_mut)] + let mut lvl1_vaddr_top = 1 << BLOCK_BITS_512GB; + let mut lvl2_vaddr_top = 1 << BLOCK_BITS_1GB; + let mut lvl3_vaddr_top = 1 << BLOCK_BITS_2MB; // TODO: Tests... // This is similar to aligned_power_of_two_regions() for the kernel UT, @@ -837,62 +835,62 @@ pub extern "C" fn aarch64_setup_pagetables( // Allowed externally for the final iteration let mut base = 0u64; for &(ref region, attr_index) in identity_mapped_regions.iter() { - // println!("RAM Region: {:#x}..{:#x}", base, region.end); - // println!( - // " - Current Lvl1: {:#x}..{:#x}, entries: {}", - // (*pts.lvl_top(1) - (1 << BLOCK_BITS_512GB)), - // *pts.lvl_top(1), - // lvl1_pt.iter().filter(|&&v| v != 0).count() - // ); - // println!( - // " - Current Lvl2: {:#x}..{:#x}, entries: {}", - // (*pts.lvl_top(2) - (1 << BLOCK_BITS_1GB)), - // *pts.lvl_top(2), - // lvl2_pt.iter().filter(|&&v| v != 0).count() - // ); - // println!( - // " - Current Lvl3: {:#x}..{:#x}, entries: {}", - // (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), - // lvl3_vaddr_top, - // lvl3_pt.iter().filter(|&&v| v != 0).count() - // ); + println!("RAM Region: {:#x}..{:#x}", base, region.end); + println!( + " - Current Lvl1: {:#x}..{:#x}, entries: {}", + (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), + lvl1_vaddr_top, + lvl1_pt.iter().filter(|&&v| v != 0).count() + ); + println!( + " - Current Lvl2: {:#x}..{:#x}, entries: {}", + (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), + lvl2_vaddr_top, + lvl2_pt.iter().filter(|&&v| v != 0).count() + ); + println!( + " - Current Lvl3: {:#x}..{:#x}, entries: {}", + (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), + lvl3_vaddr_top, + lvl3_pt.iter().filter(|&&v| v != 0).count() + ); // Handle the fact that the regions are not contiguous and that // we might need to skip PT. { - if region.start >= *pts.lvl_top(3) { - if !pts.lvl_is_empty(3) { - let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(3)); - // println!("[iter] Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{pts.lvl_top(3):#x}", (pts.lvl_top(3) - (1 << BLOCK_BITS_2MB))); - assert!(pts.lvl(2)[lvl2_index(base)] == 0); - pts.lvl(2)[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + if region.start >= lvl3_vaddr_top { + if lvl3_pt != [0; _] { + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + println!("[iter] Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{lvl3_vaddr_top:#x}", (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB))); + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); } // TODO: just compute it. - while region.start >= *pts.lvl_top(3) { - *pts.lvl_top(3) += 1 << BLOCK_BITS_2MB; + while region.start >= lvl3_vaddr_top { + lvl3_vaddr_top += 1 << BLOCK_BITS_2MB; } } - if region.start >= *pts.lvl_top(2) { - if !pts.lvl_is_empty(2) { - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(2)); - // println!("[iter] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{*pts.lvl_top(2):#x}, base: {:#x} lvl1_index(base): {:#x}", (*pts.lvl_top(2) - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); - assert!(pts.lvl(1)[lvl1_index(base)] == 0); - pts.lvl(1)[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + if region.start >= lvl2_vaddr_top { + if lvl2_pt != [0; _] { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + println!("[iter] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); } // TODO: just compute it. - while region.start >= *pts.lvl_top(2) { - *pts.lvl_top(2) += 1 << BLOCK_BITS_1GB; + while region.start >= lvl2_vaddr_top { + lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; } } - if region.start >= *pts.lvl_top(1) { + if region.start >= lvl1_vaddr_top { unreachable!( "impossible as everything should fit here: {:#x}", - *pts.lvl_top(1) + lvl1_vaddr_top ); } } @@ -934,38 +932,38 @@ pub extern "C" fn aarch64_setup_pagetables( let pt_region_size = 1u64 << bits; let top = base + pt_region_size; - // println!("- Aligned PT region: {:#x}..{:#x} (size_bits: {}, align_bits: {}, bits: {})", base, top, size_bits, align_bits, bits); - // println!( - // " - Current Lvl1: {:#x}..{:#x}, entries: {}", - // (*pts.lvl_top(1) - (1 << BLOCK_BITS_512GB)), - // *pts.lvl_top(1), - // lvl1_pt.iter().filter(|&&v| v != 0).count() - // ); - // println!( - // " - Current Lvl2: {:#x}..{:#x}, entries: {}", - // (*pts.lvl_top(2) - (1 << BLOCK_BITS_1GB)), - // *pts.lvl_top(2), - // lvl2_pt.iter().filter(|&&v| v != 0).count() - // ); - // println!( - // " - Current Lvl3: {:#x}..{:#x}, entries: {}", - // (pts.lvl_top(3) - (1 << BLOCK_BITS_2MB)), - // pts.lvl_top(3), - // lvl3_pt.iter().filter(|&&v| v != 0).count() - // ); + println!("- Aligned PT region: {:#x}..{:#x} (size_bits: {}, align_bits: {}, bits: {})", base, top, size_bits, align_bits, bits); + println!( + " - Current Lvl1: {:#x}..{:#x}, entries: {}", + (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), + lvl1_vaddr_top, + lvl1_pt.iter().filter(|&&v| v != 0).count() + ); + println!( + " - Current Lvl2: {:#x}..{:#x}, entries: {}", + (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), + lvl2_vaddr_top, + lvl2_pt.iter().filter(|&&v| v != 0).count() + ); + println!( + " - Current Lvl3: {:#x}..{:#x}, entries: {}", + (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), + lvl3_vaddr_top, + lvl3_pt.iter().filter(|&&v| v != 0).count() + ); match level { 1 => { // If it belongs in Level 1 PT, then it must go in - // lvl1 pt. By the inavariant, base < *pts.lvl_top(1). - assert!(base < *pts.lvl_top(1)); - // top is <= *pts.lvl_top(1) (the case where it is the topmost entry) - assert!(top <= *pts.lvl_top(1)); + // lvl1 pt. By the inavariant, base < lvl1_vaddr_top. + assert!(base < lvl1_vaddr_top); + // top is <= lvl1_vaddr_top (the case where it is the topmost entry) + assert!(top <= lvl1_vaddr_top); - assert!(pts.lvl(1)[lvl1_index(base)] == 0); - pts.lvl(1)[lvl1_index(base)] = block_descriptor(1, base, attr_index); + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = block_descriptor(1, base, attr_index); - if top == *pts.lvl_top(1) { + if top == lvl1_vaddr_top { // Invariant maintenance: if the new top would be now equal // the end of the page table's region top, we need a new // page table object and add it to the list. @@ -975,73 +973,73 @@ pub extern "C" fn aarch64_setup_pagetables( } // Invariant: Lower levels are empty. - assert!(pts.lvl_is_empty(2)); - assert!(pts.lvl_is_empty(3)); + assert!(lvl2_pt == [0; _]); + assert!(lvl3_pt == [0; _]); // Invariant maintenance: vaddr_top is right range for current PT. // it's empty so we need to increment the top to be current top (1G aligned) + 2MIB (512 lvl3 entries) - *pts.lvl_top(3) = top + (1 << BLOCK_BITS_2MB); + lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); // it's empty so we need to increment the top to be current top (1G aligned) + 1G (512 lvl2 entries) - *pts.lvl_top(2) = top + (1 << BLOCK_BITS_1GB); + lvl2_vaddr_top = top + (1 << BLOCK_BITS_1GB); } 2 => { // If it is a 2MiB block, it must go in the Level 2 PT; - // by our invariants: base < *pts.lvl_top(2) and top <= *pts.lvl_top(2) - assert!(base < *pts.lvl_top(2)); - assert!(top <= *pts.lvl_top(2)); + // by our invariants: base < lvl2_vaddr_top and top <= lvl2_vaddr_top + assert!(base < lvl2_vaddr_top); + assert!(top <= lvl2_vaddr_top); - assert!(pts.lvl(2)[lvl2_index(base)] == 0); - pts.lvl(2)[lvl2_index(base)] = block_descriptor(2, base, attr_index); + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = block_descriptor(2, base, attr_index); - if top == *pts.lvl_top(2) { + if top == lvl2_vaddr_top { // Invariant maintenance: keep for current address range. // As we're the top of the range, we can serialise the table. - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(2)); - // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} up to {*pts.lvl_top(2):#x}"); - *pts.lvl_top(2) += 1 << BLOCK_BITS_1GB; + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} up to {lvl2_vaddr_top:#x}"); + lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; - pts.lvl(1)[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); - if top == *pts.lvl_top(1) { + if top == lvl1_vaddr_top { todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); } } // Invariant: Lower levels are empty. - assert!(pts.lvl_is_empty(3)); + assert!(lvl3_pt == [0; _]); // Invariant maintenance: vaddr_top is right range for current PT. // it's empty so we need to increment the top to be current top (2MIB aligned) + 2MIB (512 lvl3 entries) - *pts.lvl_top(3) = top + (1 << BLOCK_BITS_2MB); + lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); } 3 => { // If it is a 4K page, it must go in the Level 3 PT; - // by our invariants: base < pts.lvl_top(3) and top <= pts.lvl_top(3) - assert!(base < *pts.lvl_top(3)); - assert!(top <= *pts.lvl_top(3)); + // by our invariants: base < lvl3_vaddr_top and top <= lvl3_vaddr_top + assert!(base < lvl3_vaddr_top); + assert!(top <= lvl3_vaddr_top); - assert!(pts.lvl(3)[lvl3_index(base)] == 0); - pts.lvl(3)[lvl3_index(base)] = page_descriptor(base, attr_index); + assert!(lvl3_pt[lvl3_index(base)] == 0); + lvl3_pt[lvl3_index(base)] = page_descriptor(base, attr_index); - if top == *pts.lvl_top(3) { + if top == lvl3_vaddr_top { // Invariant maintenance: keep for current address range. // As we're the top of the range, we can serialise the table. - let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(3)); - // println!("Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{pts.lvl_top(3):#x}", (pts.lvl_top(3) - (1 << BLOCK_BITS_2MB))); - *pts.lvl_top(3) += 1 << BLOCK_BITS_2MB; + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + println!("Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{lvl3_vaddr_top:#x}", (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB))); + lvl3_vaddr_top += 1 << BLOCK_BITS_2MB; - assert!(pts.lvl(2)[lvl2_index(base)] == 0); - pts.lvl(2)[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); - if top == *pts.lvl_top(2) { - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(2)); - // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{*pts.lvl_top(2):#x}", (*pts.lvl_top(2) - (1 << BLOCK_BITS_1GB))); - *pts.lvl_top(2) += 1 << BLOCK_BITS_1GB; + if top == lvl2_vaddr_top { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB))); + lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; - assert!(pts.lvl(1)[lvl1_index(base)] == 0); - pts.lvl(1)[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); - if top == *pts.lvl_top(1) { + if top == lvl1_vaddr_top { todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); } } @@ -1061,25 +1059,25 @@ pub extern "C" fn aarch64_setup_pagetables( // page tables that have been partially filled out, and we need to // serialise these. - if !pts.lvl_is_empty(3) { - let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(3)); - // println!("[end] Serialise lvl3 table: {lvl3_pt_paddr:#x}"); - assert!(pts.lvl(2)[lvl2_index(base)] == 0); - pts.lvl(2)[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); + if lvl3_pt != [0; _] { + let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + println!("[end] Serialise lvl3 table: {lvl3_pt_paddr:#x}"); + assert!(lvl2_pt[lvl2_index(base)] == 0); + lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); } - if !pts.lvl_is_empty(2) { - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut pts.lvl(2)); - // println!("[end] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{*pts.lvl_top(2):#x}, base: {:#x} lvl1_index(base): {:#x}", (*pts.lvl_top(2) - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); - assert!(pts.lvl(1)[lvl1_index(base)] == 0); - pts.lvl(1)[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); + if lvl2_pt != [0; _] { + let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + println!("[end] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + assert!(lvl1_pt[lvl1_index(base)] == 0); + lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); } // the level1 pt should not be empty. lol. - assert!(!pts.lvl_is_empty(1)); + assert!(lvl1_pt != [0; _]); // println!("New lvl1 table"); - serialise_page_table_to_paddr(pts.lvl(1)) + serialise_page_table_to_paddr(&mut lvl1_pt) }; struct Config { From 807aa036a72635adc7355643e7c8f442590d4be6 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Wed, 29 Jul 2026 17:35:47 +1000 Subject: [PATCH 09/29] FIX Signed-off-by: Julia Vassiliki --- loader/src/page_tables.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index 00e8cad4e..e451ebd56 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -685,14 +685,12 @@ pub extern "C" fn aarch64_setup_pagetables( move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> u64 { let pt_paddr = next_pt_paddr; - page_table - .iter() - .flat_map(|pte| pte.to_le_bytes()) - .zip(page_table_bytes[i].iter_mut()) - .for_each(|(byte, dest)| *dest = byte); + for (j, byte) in page_table.iter().flat_map(|pte| pte.to_le_bytes()).enumerate() { + page_table_bytes[i][j] = byte; + } next_pt_paddr += PAGE_TABLE_SIZE as u64; - i += 0; + i += 1; page_table.fill(0); pt_paddr } From 8a87d9da27f34601cba6ad2ce34ac4128c0e1eea Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Fri, 31 Jul 2026 16:32:01 +1000 Subject: [PATCH 10/29] minor makefile touchups Signed-off-by: Julia Vassiliki --- loader/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/loader/Makefile b/loader/Makefile index dd78ce5d9..50d0e31a3 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -86,9 +86,9 @@ $(BUILD_DIR)/%.o : src/$(ARCH_DIR)/%.c $(BUILD_DIR)/%.o : src/%.c $(CC) -c $(CFLAGS) $< -o $@ -# Note: having multiple rlib with staticlib will give duplicate linker symbol +# Note: having multiple libs with staticlib will give duplicate linker symbol # issues. Use "--crate-type rlib" instead, but then we need to link a single -# copy of the rust corelibs. +# copy of the rust corelibs. For now this is fine. $(BUILD_DIR)/lib%.a : src/%.rs $(RUSTC) $(RUSTFLAGS) \ --emit dep-info,metadata,link \ @@ -98,7 +98,6 @@ $(BUILD_DIR)/lib%.a : src/%.rs $< -include $(BUILD_DIR)/*.d --include $(BUILD_DIR)/mmu.d OBJPROG = $(addprefix $(BUILD_DIR)/, $(PROGS)) From 0890270817d100818320f02b4817261c2df9d7e5 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 12:11:16 +1000 Subject: [PATCH 11/29] tests + return struct Signed-off-by: Julia Vassiliki --- loader/Makefile | 13 ++- loader/src/c_interop.rs | 75 +++++++++++++ loader/src/page_tables.rs | 214 ++++++++++++++++++-------------------- 3 files changed, 187 insertions(+), 115 deletions(-) create mode 100644 loader/src/c_interop.rs diff --git a/loader/Makefile b/loader/Makefile index 50d0e31a3..376e68849 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -58,7 +58,7 @@ CFLAGS := -std=gnu11 -g -O3 -nostdlib -ffreestanding \ ASM_FLAGS := $(ASM_FLAGS_ARCH) -g -MP -MD -I$(SEL4_SDK)/include -RUSTFLAGS := --target $(RUST_TARGET_TRIPLE) --edition 2024 -g -C opt-level=2 +RUSTFLAGS := --edition 2024 -g -C opt-level=2 PROGS := loader.elf OBJECTS := loader.o crt0.o uart.o cutil.o libpage_tables.a @@ -93,6 +93,7 @@ $(BUILD_DIR)/lib%.a : src/%.rs $(RUSTC) $(RUSTFLAGS) \ --emit dep-info,metadata,link \ --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ + --target $(RUST_TARGET_TRIPLE) \ --crate-type staticlib \ --crate-name $(patsubst lib%.a,%,$(notdir $@)) \ $< @@ -101,7 +102,7 @@ $(BUILD_DIR)/lib%.a : src/%.rs OBJPROG = $(addprefix $(BUILD_DIR)/, $(PROGS)) -all: $(OBJPROG) +all: $(OBJPROG) test $(LINKSCRIPT): $(LINKSCRIPT_INPUT) $(CPP) -DLINK_ADDRESS=$(LINK_ADDRESS) $< | grep -v "^#" > $@ @@ -111,3 +112,11 @@ LDFLAGS := -T$(LINKSCRIPT) --gc-sections $(OBJPROG): $(addprefix $(BUILD_DIR)/, $(OBJECTS)) $(LINKSCRIPT) $(LD) $(LDFLAGS) --start-group $(addprefix $(BUILD_DIR)/, $(OBJECTS)) --end-group -o $@ +test: + $(RUSTC) $(RUSTFLAGS) \ + --emit dep-info,metadata,link \ + --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ + --test \ + --crate-name test_page_tables \ + src/page_tables.rs + $(BUILD_DIR)/test_page_tables diff --git a/loader/src/c_interop.rs b/loader/src/c_interop.rs new file mode 100644 index 000000000..8fd6b7406 --- /dev/null +++ b/loader/src/c_interop.rs @@ -0,0 +1,75 @@ +#[cfg(not(test))] +mod real_hardware { + use core::ffi::c_char; + use core::ffi::CStr; + use core::fmt; + use core::fmt::Write; + use core::panic::PanicInfo; + + unsafe extern "C" { + safe fn fail() -> !; + // safe fn putc(c: c_char); + unsafe fn puts(s: *const c_char); + } + + /// Exposed only for print macro. + #[doc(hidden)] + pub(crate) struct Writer; + + impl fmt::Write for Writer { + fn write_str(&mut self, s: &str) -> Result<(), fmt::Error> { + for c in s.bytes() { + unsafe { + puts(CStr::from_bytes_with_nul_unchecked(&[c.into(), 0]).as_ptr()) + }; + } + Ok(()) + } + } + + #[macro_export] + macro_rules! __print { + ($($arg:tt)*) => {{ + use core::fmt::Write; + $crate::c_interop::Writer{}.write_fmt(format_args!($($arg)*)).unwrap() + }} + } + + #[macro_export] + macro_rules! __println { + () => {{ + $crate::__print!("\n"); + }}; + + ($($arg:tt)*) => {{ + $crate::__print!("{}\n", format_args!($($arg)*)); + }} + } + + pub(crate) use __println as println; + + #[panic_handler] + fn panic(info: &PanicInfo) -> ! { + println!("panicked"); + + if let Err(_) = writeln!(Writer, "{}", info) { + // If writeln!() fails (which it should never as our fmt::Write) never + // fails, then just don't print the extra information. + println!("panicked (information unknown)"); + } + + fail(); + } +} + +#[cfg(test)] +mod for_tests { + extern crate std; + pub(crate) use std::println; +} + +#[cfg(test)] +pub(crate) use for_tests::*; + +#[cfg(not(test))] +pub(crate) use real_hardware::*; diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index e451ebd56..b98b626d1 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -6,44 +6,13 @@ #![no_std] +mod c_interop; + use core::cmp::min; -use core::ffi::c_char; -use core::fmt; -use core::fmt::Write; use core::mem; -use core::panic::PanicInfo; - -unsafe extern "C" { - safe fn fail() -> !; - // safe fn putc(c: c_char); - unsafe fn puts(s: *const c_char); -} - -#[panic_handler] -fn panic(info: &PanicInfo) -> ! { - unsafe { puts(c"panicked\n".as_ptr()) }; - - struct DebugWriter; - impl fmt::Write for DebugWriter { - fn write_str(&mut self, s: &str) -> fmt::Result { - for c in s.bytes() { - unsafe { - puts(core::ffi::CStr::from_bytes_with_nul_unchecked(&[c.into(), 0]).as_ptr()) - }; - } - - Ok(()) - } - } +use core::mem::MaybeUninit; - if let Err(_) = writeln!(DebugWriter, "{}", info) { - // If writeln!() fails (which it should never as our fmt::Write) never - // fails, then just don't print the extra information. - unsafe { puts(c"panicked (information unknown)\n".as_ptr()) }; - } - - fail(); -} +use c_interop::println; const PAGE_TABLE_SIZE: usize = 4096; @@ -55,15 +24,6 @@ const fn mask(n: u64) -> u64 { (1 << n) - 1 } -const fn round_up(n: u64, x: u64) -> u64 { - let (_, m) = divmod(n, x); - if m == 0 { - n - } else { - n + x - m - } -} - const fn round_down(n: u64, x: u64) -> u64 { let (_, m) = divmod(n, x); if m == 0 { @@ -73,10 +33,6 @@ const fn round_down(n: u64, x: u64) -> u64 { } } -const fn align_up(n: u64, bits: u64) -> u64 { - round_up(n, 1 << bits) -} - const fn align_down(n: u64, bits: u64) -> u64 { round_down(n, 1 << bits) } @@ -91,7 +47,7 @@ pub mod aarch64 { //! Stage 2 descriptors are only used when in the EL1&0 regime; which is not //! the case when in EL2. - use crate::mask; + use super::*; pub const LVL0_BITS: u64 = 9; pub const LVL1_BITS: u64 = 9; @@ -299,9 +255,11 @@ pub mod aarch64 { /// Per "Table D8-50 Stage 1 VMSAv8-64 Table descriptor fields" and /// "Figure D8-12 VMSAv8-64 Table descriptor formats" of ARM DDI0487L.b; /// specifically subfigure "4KB, 16KB, and 64KB granules, 48-bit OA" - pub fn table_descriptor(addr: u64) -> u64 { + pub fn table_descriptor(addr: *const u8) -> u64 { // Per Table D8-48, Condition for descriptor_type::TABLE is level != 3. + let addr: u64 = addr.addr().try_into().expect("usize in u64"); + // We don't set any of these attributes, most are hardware-feature conditional let attributes: u64 = 0; @@ -549,41 +507,18 @@ pub extern "C" fn riscv64_setup_pagetables( serialise_page_table_to_paddr(&mut boot_lvl1_pt) } -pub struct Writer; - -impl fmt::Write for Writer { - fn write_str(&mut self, s: &str) -> Result<(), fmt::Error> { - for c in s.bytes() { - unsafe { - puts(core::ffi::CStr::from_bytes_with_nul_unchecked(&[c.into(), 0]).as_ptr()) - }; - } - Ok(()) - } +/// Note that "0" is a valid return value; instead the invalid value is +/// '-1', or usize::MAX. +#[repr(C)] +#[derive(Debug)] +pub struct AArch64ReturnValue { + ttbr0_el2: *const u8, + ttbr0_el1: *const u8, + ttbr1_el1: *const u8, } -#[allow(unused)] -pub fn print(args: fmt::Arguments) { - use fmt::Write; - Writer{}.write_fmt(args).unwrap(); -} - -#[macro_export] -macro_rules! print { - ($($arg:tt)*) => {{ - print(format_args!($($arg)*)); - }} -} - -#[macro_export] -macro_rules! println { - () => {{ - print!("\n"); - }}; - - ($($arg:tt)*) => {{ - print!("{}\n", format_args!($($arg)*)); - }} +impl AArch64ReturnValue { + const INVALID: *const u8 = usize::MAX as *const _; } /// AArch64 loader page tables have two variations: @@ -651,8 +586,8 @@ macro_rules! println { pub extern "C" fn aarch64_setup_pagetables( kernel_first_vaddr: u64, kernel_first_paddr: u64, - page_tables_paddr_start: u64, -) -> (u64, u64, u64) { + page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; 100], +) -> AArch64ReturnValue { use aarch64::{ block_descriptor, lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}, @@ -665,31 +600,28 @@ pub extern "C" fn aarch64_setup_pagetables( const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); let mut serialise_page_table_to_paddr = { - #[repr(align(4096))] - struct PtBytes([[u8; 4096]; 100]); - static mut PAGE_TABLE_BYTES: PtBytes = PtBytes([[0; _]; _]); - // SAFETY: Trust me (lol) - #[allow(static_mut_refs)] - let mut page_table_bytes = unsafe { &mut PAGE_TABLE_BYTES.0 }; - - let page_tables_paddr_start = &raw mut PAGE_TABLE_BYTES as u64; + let page_tables_paddr_start: *const u8 = page_table_bytes.as_ptr().cast(); assert!( - page_tables_paddr_start - == page_tables_paddr_start.next_multiple_of(PAGE_TABLE_SIZE as u64) + (page_tables_paddr_start as usize) + == (page_tables_paddr_start as usize).next_multiple_of(PAGE_TABLE_SIZE) ); // This maintains the current end of the PT array. let mut next_pt_paddr = page_tables_paddr_start; let mut i = 0; - move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> u64 { + move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> *const _ { let pt_paddr = next_pt_paddr; - for (j, byte) in page_table.iter().flat_map(|pte| pte.to_le_bytes()).enumerate() { - page_table_bytes[i][j] = byte; + for (j, byte) in page_table + .iter() + .flat_map(|pte| pte.to_le_bytes()) + .enumerate() + { + page_table_bytes[i][j].write(byte); } - next_pt_paddr += PAGE_TABLE_SIZE as u64; + next_pt_paddr = next_pt_paddr.wrapping_add(PAGE_TABLE_SIZE); i += 1; page_table.fill(0); pt_paddr @@ -700,13 +632,14 @@ pub extern "C" fn aarch64_setup_pagetables( start: u64, end: u64, } - let ram_regions = [ - Region { start: 0x60000000, end: 0xc0000000 }, - ]; + let ram_regions = [Region { + start: 0x60000000, + end: 0xc0000000, + }]; const MAX_NUM_REGIONS: usize = 16; - let mut regions = [const { core::mem::MaybeUninit::uninit() }; MAX_NUM_REGIONS]; + let mut regions = [const { MaybeUninit::uninit() }; MAX_NUM_REGIONS]; let identity_mapped_regions: &mut [(Region, _)] = { // Conceptually want we want is an 'arrayvec', but to not pull in more // code we implement this less-efficiently MaybeUninit. @@ -719,7 +652,13 @@ pub extern "C" fn aarch64_setup_pagetables( let ram_regions_it = ram_regions.into_iter().map(|r| (r, MT_DEVICE_nGnRnE)); - let all_regions_it = ram_regions_it.chain([(Region { start: 0x9000000, end: 0x9001000 }, MT_DEVICE_nGnRnE)]); + let all_regions_it = ram_regions_it.chain([( + Region { + start: 0x9000000, + end: 0x9001000, + }, + MT_DEVICE_nGnRnE, + )]); // // FIXME: Derive from the kernel build system. // if let Some(uart_base) = read_symbol_maybe(elf, "uart_addr") { @@ -860,7 +799,11 @@ pub extern "C" fn aarch64_setup_pagetables( if region.start >= lvl3_vaddr_top { if lvl3_pt != [0; _] { let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); - println!("[iter] Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{lvl3_vaddr_top:#x}", (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB))); + println!( + "[iter] Serialise lvl3 table: {:#x} for to {:#x}..{lvl3_vaddr_top:#x}", + lvl3_pt_paddr as usize, + (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)) + ); assert!(lvl2_pt[lvl2_index(base)] == 0); lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); } @@ -874,7 +817,7 @@ pub extern "C" fn aarch64_setup_pagetables( if region.start >= lvl2_vaddr_top { if lvl2_pt != [0; _] { let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); - println!("[iter] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + println!("[iter] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", lvl2_pt_paddr as usize, (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); assert!(lvl1_pt[lvl1_index(base)] == 0); lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); } @@ -930,7 +873,10 @@ pub extern "C" fn aarch64_setup_pagetables( let pt_region_size = 1u64 << bits; let top = base + pt_region_size; - println!("- Aligned PT region: {:#x}..{:#x} (size_bits: {}, align_bits: {}, bits: {})", base, top, size_bits, align_bits, bits); + println!( + "- Aligned PT region: {:#x}..{:#x} (size_bits: {}, align_bits: {}, bits: {})", + base, top, size_bits, align_bits, bits + ); println!( " - Current Lvl1: {:#x}..{:#x}, entries: {}", (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), @@ -1023,7 +969,11 @@ pub extern "C" fn aarch64_setup_pagetables( // As we're the top of the range, we can serialise the table. let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); - println!("Serialise lvl3 table: {lvl3_pt_paddr:#x} for to {:#x}..{lvl3_vaddr_top:#x}", (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB))); + println!( + "Serialise lvl3 table: {:#x} for to {:#x}..{lvl3_vaddr_top:#x}", + lvl3_pt_paddr as usize, + (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)) + ); lvl3_vaddr_top += 1 << BLOCK_BITS_2MB; assert!(lvl2_pt[lvl2_index(base)] == 0); @@ -1031,7 +981,11 @@ pub extern "C" fn aarch64_setup_pagetables( if top == lvl2_vaddr_top { let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); - println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB))); + println!( + "Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}", + lvl2_pt_paddr as usize, + (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)) + ); lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; assert!(lvl1_pt[lvl1_index(base)] == 0); @@ -1059,14 +1013,14 @@ pub extern "C" fn aarch64_setup_pagetables( if lvl3_pt != [0; _] { let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); - println!("[end] Serialise lvl3 table: {lvl3_pt_paddr:#x}"); + println!("[end] Serialise lvl3 table: {:#x}", lvl3_pt_paddr as usize); assert!(lvl2_pt[lvl2_index(base)] == 0); lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); } if lvl2_pt != [0; _] { let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); - println!("[end] Serialise lvl2 table: {lvl2_pt_paddr:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + println!("[end] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", lvl2_pt_paddr as usize, (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); assert!(lvl1_pt[lvl1_index(base)] == 0); lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); } @@ -1099,7 +1053,11 @@ pub extern "C" fn aarch64_setup_pagetables( let ttbr0_el2 = serialise_page_table_to_paddr(&mut ttbr0_el2_pt); - (ttbr0_el2, u64::MAX, u64::MAX) + AArch64ReturnValue { + ttbr0_el2, + ttbr0_el1: AArch64ReturnValue::INVALID, + ttbr1_el1: AArch64ReturnValue::INVALID, + } } else { let mut ttbr0_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; let mut ttbr1_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; @@ -1112,6 +1070,36 @@ pub extern "C" fn aarch64_setup_pagetables( let ttbr0_el1 = serialise_page_table_to_paddr(&mut ttbr0_el1_pt); let ttbr1_el1 = serialise_page_table_to_paddr(&mut ttbr1_el1_pt); - (u64::MAX, ttbr0_el1, ttbr1_el1) + AArch64ReturnValue { + ttbr0_el2: AArch64ReturnValue::INVALID, + ttbr0_el1, + ttbr1_el1, + } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } + + #[test] + fn aaaaaaaaaaaaaaaaaaaaaa() { + #[repr(align(4096))] + struct PtBytes([[MaybeUninit; 4096]; 100]); + + let mut page_table_bytes = PtBytes([[MaybeUninit::uninit(); _]; _]); + let pt_bases = aarch64_setup_pagetables(0, 0, &mut page_table_bytes.0); + panic!("{pt_bases:#x?}"); + } + + // #[test] + // fn bbbbbbbbbbbbbbbbbbbbbbb() { + // let d = riscv64_setup_pagetables(0, 0, 0); + // // panic!("{a:#x} {b:#x} {c:#x}"); + // } +} From 62651ed443fe099a993776a3b7d19c8673c38ce9 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 12:53:17 +1000 Subject: [PATCH 12/29] work again for qemu Signed-off-by: Julia Vassiliki --- loader/src/aarch64/mmu.c | 56 ++++++++++----- loader/src/page_tables.rs | 143 +++++++++++++++++++++----------------- 2 files changed, 118 insertions(+), 81 deletions(-) diff --git a/loader/src/aarch64/mmu.c b/loader/src/aarch64/mmu.c index ad511615b..4d2d34c20 100644 --- a/loader/src/aarch64/mmu.c +++ b/loader/src/aarch64/mmu.c @@ -12,28 +12,46 @@ #include "../cutil.h" #include "../uart.h" -void el1_mmu_enable(uint64_t aarch64_pt_ttbr0_el1, uint64_t aarch64_pt_ttbr1_el1); -void el2_mmu_enable(uint64_t aarch64_pt_ttbr0_el2); - -/* Pointers to the top-level paging structures */ -uint64_t aarch64_pt_ttbr0_el1; -uint64_t aarch64_pt_ttbr1_el1; -uint64_t aarch64_pt_ttbr0_el2; - -struct ret { - uint64_t a; - uint64_t b; - uint64_t c; +void el1_mmu_enable(uint64_t ttbr0_el1, uint64_t ttbr1_el1); +void el2_mmu_enable(uint64_t ttbr0_el2); + +struct AArch64ReturnValue { + uintptr_t ttbr0_el2; + uintptr_t ttbr0_el1; + uintptr_t ttbr1_el1; +}; + +struct Region { + uint64_t start; + uint64_t end; +}; + +const struct Region ram_regions[] = { + { .start = 0x60000000, .end = 0xc0000000 }, +}; + +const struct Region device_regions[] = { + { .start = 0x9000000, .end = 0x9000000 + 4096 }, }; -extern struct ret aarch64_setup_pagetables(uint64_t kernel_first_vaddr, uint64_t kernel_first_paddr, uint64_t page_tables_paddr_start); +uint8_t page_table_bytes[4096][64] ALIGN(4096); +uint8_t regions[16 * 4] ALIGN(16); + +extern struct AArch64ReturnValue aarch64_setup_pagetables( + uint64_t kernel_first_vaddr, uint64_t kernel_first_paddr, + const void *ram_regions_ptr, uintptr_t ram_regions_len, + const void *device_regions_ptr, uintptr_t device_regions_len, + uint8_t page_table_bytes[4096][64], + uint8_t regions[16 * 4]); int arch_mmu_enable(int logical_cpu) { - struct ret x = aarch64_setup_pagetables(0, 0, 0); - aarch64_pt_ttbr0_el2 = x.a; - aarch64_pt_ttbr0_el1 = x.b; - aarch64_pt_ttbr1_el1 = x.c; + struct AArch64ReturnValue pt = aarch64_setup_pagetables( + 0x8060000000, 0x60000000, + &ram_regions, ARRAY_SIZE(ram_regions), + &device_regions, ARRAY_SIZE(device_regions), + page_table_bytes, regions + ); int r; enum el el; @@ -45,9 +63,9 @@ int arch_mmu_enable(int logical_cpu) LDR_PRINT("INFO", logical_cpu, "enabling MMU\n"); el = current_el(); if (el == EL1) { - el1_mmu_enable(aarch64_pt_ttbr0_el1, aarch64_pt_ttbr1_el1); + el1_mmu_enable(pt.ttbr0_el1, pt.ttbr1_el1); } else if (el == EL2) { - el2_mmu_enable(aarch64_pt_ttbr0_el2); + el2_mmu_enable(pt.ttbr0_el2); } else { LDR_PRINT("ERROR", logical_cpu, "unknown EL for MMU enable\n"); } diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index b98b626d1..e56d8129c 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -11,6 +11,7 @@ mod c_interop; use core::cmp::min; use core::mem; use core::mem::MaybeUninit; +use core::slice; use c_interop::println; @@ -512,15 +513,24 @@ pub extern "C" fn riscv64_setup_pagetables( #[repr(C)] #[derive(Debug)] pub struct AArch64ReturnValue { - ttbr0_el2: *const u8, - ttbr0_el1: *const u8, - ttbr1_el1: *const u8, + pub ttbr0_el2: *const u8, + pub ttbr0_el1: *const u8, + pub ttbr1_el1: *const u8, } impl AArch64ReturnValue { const INVALID: *const u8 = usize::MAX as *const _; } +#[derive(Debug, Copy, Clone)] +pub struct Region { + pub start: u64, + pub end: u64, +} + +pub const MAX_NUM_PAGE_TABLES: usize = 64; +pub const MAX_NUM_REGIONS: usize = 16; + /// AArch64 loader page tables have two variations: /// - Loader in EL2, then Stage 1 translations in use, so we have the /// singular TTBR0_EL2 register containing the Level 0 table; @@ -586,7 +596,13 @@ impl AArch64ReturnValue { pub extern "C" fn aarch64_setup_pagetables( kernel_first_vaddr: u64, kernel_first_paddr: u64, - page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; 100], + ram_regions_ptr: *const Region, + ram_regions_len: usize, + device_regions_ptr: *const Region, + device_regions_len: usize, + // Both of these are out-params / storage used. + page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], + regions: &mut [MaybeUninit<(Region, u64)>; MAX_NUM_REGIONS], ) -> AArch64ReturnValue { use aarch64::{ block_descriptor, lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, @@ -594,8 +610,11 @@ pub extern "C" fn aarch64_setup_pagetables( table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, }; - let kernel_first_vaddr = 551366426624; - let kernel_first_paddr = 1610612736; + let ram_regions = unsafe { slice::from_raw_parts(ram_regions_ptr, ram_regions_len) }; + let device_regions = unsafe { slice::from_raw_parts(device_regions_ptr, device_regions_len) }; + + println!("{:#x?}", ram_regions); + println!("{:#x?}", device_regions); const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); @@ -628,18 +647,6 @@ pub extern "C" fn aarch64_setup_pagetables( } }; - struct Region { - start: u64, - end: u64, - } - let ram_regions = [Region { - start: 0x60000000, - end: 0xc0000000, - }]; - - const MAX_NUM_REGIONS: usize = 16; - - let mut regions = [const { MaybeUninit::uninit() }; MAX_NUM_REGIONS]; let identity_mapped_regions: &mut [(Region, _)] = { // Conceptually want we want is an 'arrayvec', but to not pull in more // code we implement this less-efficiently MaybeUninit. @@ -651,42 +658,12 @@ pub extern "C" fn aarch64_setup_pagetables( assert!(ram_regions.len() <= regions.len()); let ram_regions_it = ram_regions.into_iter().map(|r| (r, MT_DEVICE_nGnRnE)); + let device_regions_it = device_regions.into_iter().map(|r| (r, MT_DEVICE_nGnRnE)); - let all_regions_it = ram_regions_it.chain([( - Region { - start: 0x9000000, - end: 0x9001000, - }, - MT_DEVICE_nGnRnE, - )]); - - // // FIXME: Derive from the kernel build system. - // if let Some(uart_base) = read_symbol_maybe(elf, "uart_addr") { - // let uart_base = align_down(uart_base, PAGE_BITS_4KB); - // regions.push(( - // PlatformConfigRegion { - // start: uart_base, - // end: uart_base + (1 << PAGE_BITS_4KB), - // }, - // MT_DEVICE_nGnRnE, - // )); - // } - // // FIXME: This is currently assuming implementation details of the BCM2711/ - // // Raspberry Pi 4B spin table implementation, as it is the only - // // platform we have that uses spin tables. Specifically, that - // // it is always located at the 0 page. - // if elf.find_symbol("cpus_release_addr").is_ok() { - // regions.push(( - // PlatformConfigRegion { - // start: 0x0, - // end: 1 << PAGE_BITS_4KB, - // }, - // MT_DEVICE_nGnRnE, - // )); - // } + let all_regions_it = ram_regions_it.chain(device_regions_it); for (entry, region) in regions.iter_mut().zip(all_regions_it) { - entry.write(region); + entry.write((*region.0, region.1)); regions_len += 1; } @@ -772,7 +749,7 @@ pub extern "C" fn aarch64_setup_pagetables( // Allowed externally for the final iteration let mut base = 0u64; for &(ref region, attr_index) in identity_mapped_regions.iter() { - println!("RAM Region: {:#x}..{:#x}", base, region.end); + println!("Identity-Mapped Region: {:#x}..{:#x}", region.start, region.end); println!( " - Current Lvl1: {:#x}..{:#x}, entries: {}", (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), @@ -1088,18 +1065,60 @@ mod tests { } #[test] - fn aaaaaaaaaaaaaaaaaaaaaa() { + fn qemu_aarch64() { #[repr(align(4096))] - struct PtBytes([[MaybeUninit; 4096]; 100]); + struct PtBytes([[MaybeUninit; 4096]; MAX_NUM_PAGE_TABLES]); + + let ram_regions = [Region { + start: 0x60000000, + end: 0xc0000000, + }]; + + let device_regions = [ + // UART + Region { + start: 0x9000000, + end: 0x9001000, + }, + ]; + + // // FIXME: Derive from the kernel build system. + // if let Some(uart_base) = read_symbol_maybe(elf, "uart_addr") { + // let uart_base = align_down(uart_base, PAGE_BITS_4KB); + // regions.push(( + // PlatformConfigRegion { + // start: uart_base, + // end: uart_base + (1 << PAGE_BITS_4KB), + // }, + // MT_DEVICE_nGnRnE, + // )); + // } + // // FIXME: This is currently assuming implementation details of the BCM2711/ + // // Raspberry Pi 4B spin table implementation, as it is the only + // // platform we have that uses spin tables. Specifically, that + // // it is always located at the 0 page. + // if elf.find_symbol("cpus_release_addr").is_ok() { + // regions.push(( + // PlatformConfigRegion { + // start: 0x0, + // end: 1 << PAGE_BITS_4KB, + // }, + // MT_DEVICE_nGnRnE, + // )); + // } let mut page_table_bytes = PtBytes([[MaybeUninit::uninit(); _]; _]); - let pt_bases = aarch64_setup_pagetables(0, 0, &mut page_table_bytes.0); - panic!("{pt_bases:#x?}"); + let mut regions_storage = [MaybeUninit::uninit(); MAX_NUM_REGIONS]; + + let pt_bases = aarch64_setup_pagetables( + /* kernel_first_vaddr */ 0x8060000000, + /* kernel_first_paddr */ 0x60000000, + ram_regions.as_ptr(), + ram_regions.len(), + device_regions.as_ptr(), + device_regions.len(), + &mut page_table_bytes.0, + &mut regions_storage, + ); } - - // #[test] - // fn bbbbbbbbbbbbbbbbbbbbbbb() { - // let d = riscv64_setup_pagetables(0, 0, 0); - // // panic!("{a:#x} {b:#x} {c:#x}"); - // } } From 2ccf7c21275d14adcb53e077d4d124a82acd7ceb Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 13:31:13 +1000 Subject: [PATCH 13/29] make it easier Signed-off-by: Julia Vassiliki --- loader/src/aarch64/mmu.c | 31 ++++++------ loader/src/page_tables.rs | 100 ++++++++++++++++++++------------------ 2 files changed, 70 insertions(+), 61 deletions(-) diff --git a/loader/src/aarch64/mmu.c b/loader/src/aarch64/mmu.c index 4d2d34c20..75ceba86b 100644 --- a/loader/src/aarch64/mmu.c +++ b/loader/src/aarch64/mmu.c @@ -6,6 +6,7 @@ */ #include +#include #include "el.h" #include "../arch.h" @@ -21,36 +22,38 @@ struct AArch64ReturnValue { uintptr_t ttbr1_el1; }; +union RegionArchAttrs { + bool is_ram; + uint64_t raw; +}; + struct Region { uint64_t start; uint64_t end; + union RegionArchAttrs arch_attrs; }; -const struct Region ram_regions[] = { - { .start = 0x60000000, .end = 0xc0000000 }, +struct Region regions[] = { + { .start = 0x60000000, .end = 0xc0000000, .arch_attrs.is_ram = true }, + { .start = 0x9000000, .end = 0x9000000 + 4096, .arch_attrs.is_ram = false }, }; -const struct Region device_regions[] = { - { .start = 0x9000000, .end = 0x9000000 + 4096 }, -}; +#define PAGE_TABLE_SIZE 4096 +#define MAX_NUM_PAGE_TABLES 64 -uint8_t page_table_bytes[4096][64] ALIGN(4096); -uint8_t regions[16 * 4] ALIGN(16); +uint8_t page_table_bytes[PAGE_TABLE_SIZE][MAX_NUM_PAGE_TABLES] ALIGN(4096); extern struct AArch64ReturnValue aarch64_setup_pagetables( uint64_t kernel_first_vaddr, uint64_t kernel_first_paddr, - const void *ram_regions_ptr, uintptr_t ram_regions_len, - const void *device_regions_ptr, uintptr_t device_regions_len, - uint8_t page_table_bytes[4096][64], - uint8_t regions[16 * 4]); + void *regions_ptr, uintptr_t regions_len, + uint8_t page_table_bytes[4096][64]); int arch_mmu_enable(int logical_cpu) { struct AArch64ReturnValue pt = aarch64_setup_pagetables( 0x8060000000, 0x60000000, - &ram_regions, ARRAY_SIZE(ram_regions), - &device_regions, ARRAY_SIZE(device_regions), - page_table_bytes, regions + ®ions, ARRAY_SIZE(regions), + page_table_bytes ); int r; diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index e56d8129c..803f40b0c 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -9,6 +9,7 @@ mod c_interop; use core::cmp::min; +use core::fmt; use core::mem; use core::mem::MaybeUninit; use core::slice; @@ -522,14 +523,31 @@ impl AArch64ReturnValue { const INVALID: *const u8 = usize::MAX as *const _; } +#[derive(Copy, Clone)] +#[repr(C)] +pub union RegionArchAttrs { + pub is_ram: bool, + pub raw: u64, +} + +impl fmt::Debug for RegionArchAttrs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + f.debug_struct("RegionArchAttrs") + // SAFETY: raw contains all valid bitpatterns + .field("raw", unsafe { &self.raw }) + .finish() + } +} + #[derive(Debug, Copy, Clone)] +#[repr(C)] pub struct Region { pub start: u64, pub end: u64, + pub arch_attrs: RegionArchAttrs, } pub const MAX_NUM_PAGE_TABLES: usize = 64; -pub const MAX_NUM_REGIONS: usize = 16; /// AArch64 loader page tables have two variations: /// - Loader in EL2, then Stage 1 translations in use, so we have the @@ -596,13 +614,11 @@ pub const MAX_NUM_REGIONS: usize = 16; pub extern "C" fn aarch64_setup_pagetables( kernel_first_vaddr: u64, kernel_first_paddr: u64, - ram_regions_ptr: *const Region, - ram_regions_len: usize, - device_regions_ptr: *const Region, - device_regions_len: usize, - // Both of these are out-params / storage used. + // In-out param; storage and input + regions_ptr: *mut Region, + regions_len: usize, + // Storage used for page tables page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], - regions: &mut [MaybeUninit<(Region, u64)>; MAX_NUM_REGIONS], ) -> AArch64ReturnValue { use aarch64::{ block_descriptor, lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, @@ -610,12 +626,6 @@ pub extern "C" fn aarch64_setup_pagetables( table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, }; - let ram_regions = unsafe { slice::from_raw_parts(ram_regions_ptr, ram_regions_len) }; - let device_regions = unsafe { slice::from_raw_parts(device_regions_ptr, device_regions_len) }; - - println!("{:#x?}", ram_regions); - println!("{:#x?}", device_regions); - const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); let mut serialise_page_table_to_paddr = { @@ -647,30 +657,23 @@ pub extern "C" fn aarch64_setup_pagetables( } }; - let identity_mapped_regions: &mut [(Region, _)] = { - // Conceptually want we want is an 'arrayvec', but to not pull in more - // code we implement this less-efficiently MaybeUninit. - // We implement something very similar to the currently-unstable - // write_iter implementation: - // https://github.com/rust-lang/rust/blob/1.97.1/library/core/src/mem/maybe_uninit.rs#L1384-L1406 - let mut regions_len = 0; + let identity_mapped_regions: &mut [Region] = { + let regions = unsafe { slice::from_raw_parts_mut(regions_ptr, regions_len) }; - assert!(ram_regions.len() <= regions.len()); + println!("{:#x?}", regions); - let ram_regions_it = ram_regions.into_iter().map(|r| (r, MT_DEVICE_nGnRnE)); - let device_regions_it = device_regions.into_iter().map(|r| (r, MT_DEVICE_nGnRnE)); - - let all_regions_it = ram_regions_it.chain(device_regions_it); - - for (entry, region) in regions.iter_mut().zip(all_regions_it) { - entry.write((*region.0, region.1)); - regions_len += 1; + for region in regions.iter_mut() { + // SAFETY: We expect users to set is_ram appropriately. + region.arch_attrs.raw = if unsafe { region.arch_attrs.is_ram } { + // FIXME: For now, RAM is also mapped as DEVICE memory. + MT_DEVICE_nGnRnE + } else { + MT_DEVICE_nGnRnE + }; } - let regions = unsafe { (&mut regions[0..regions_len]).assume_init_mut() }; - // Need to use 'sort_unstable_by_key' as sort_by_key is not in-place. - regions.sort_unstable_by_key(|(region, _)| region.start); + regions.sort_unstable_by_key(|region| region.start); regions }; @@ -712,7 +715,7 @@ pub extern "C" fn aarch64_setup_pagetables( let ram_lvl1_pt_paddr = { // Validation of assumptions about the identity mapped regions. let mut previous_end = None; - for (region, _) in identity_mapped_regions.iter() { + for region in identity_mapped_regions.iter() { assert!(lvl0_index(region.start) == 0); assert!(lvl0_index(region.end - 1) == 0); // This is probably an unnecessary assumption. @@ -748,8 +751,14 @@ pub extern "C" fn aarch64_setup_pagetables( // Allowed externally for the final iteration let mut base = 0u64; - for &(ref region, attr_index) in identity_mapped_regions.iter() { - println!("Identity-Mapped Region: {:#x}..{:#x}", region.start, region.end); + for region in identity_mapped_regions.iter() { + // SAFETY: We went through and initialised raw before. + let attr_index = unsafe { region.arch_attrs.raw }; + + println!( + "Identity-Mapped Region: {:#x}..{:#x}", + region.start, region.end + ); println!( " - Current Lvl1: {:#x}..{:#x}, entries: {}", (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), @@ -1069,16 +1078,17 @@ mod tests { #[repr(align(4096))] struct PtBytes([[MaybeUninit; 4096]; MAX_NUM_PAGE_TABLES]); - let ram_regions = [Region { - start: 0x60000000, - end: 0xc0000000, - }]; - - let device_regions = [ + let mut regions = [ + Region { + start: 0x60000000, + end: 0xc0000000, + arch_attrs: RegionArchAttrs { is_ram: true }, + }, // UART Region { start: 0x9000000, end: 0x9001000, + arch_attrs: RegionArchAttrs { is_ram: false }, }, ]; @@ -1108,17 +1118,13 @@ mod tests { // } let mut page_table_bytes = PtBytes([[MaybeUninit::uninit(); _]; _]); - let mut regions_storage = [MaybeUninit::uninit(); MAX_NUM_REGIONS]; let pt_bases = aarch64_setup_pagetables( /* kernel_first_vaddr */ 0x8060000000, /* kernel_first_paddr */ 0x60000000, - ram_regions.as_ptr(), - ram_regions.len(), - device_regions.as_ptr(), - device_regions.len(), + regions.as_mut_ptr(), + regions.len(), &mut page_table_bytes.0, - &mut regions_storage, ); } } From 5577520f429c3ee384a7d6f6b17c86cced79c16c Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 14:39:59 +1000 Subject: [PATCH 14/29] tests part of build Signed-off-by: Julia Vassiliki --- build_sdk.py | 15 +++++++++++++++ loader/Makefile | 13 ++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/build_sdk.py b/build_sdk.py index 92cc5bfcb..b9adcddda 100644 --- a/build_sdk.py +++ b/build_sdk.py @@ -796,6 +796,19 @@ def build_sel4( json_dst.chmod(0o744) +def test_loader(build_dir: Path) -> None: + build_dir = build_dir / "loader" + build_dir.mkdir(exist_ok=True, parents=True) + + make_args = f"BUILD_DIR={build_dir.absolute()} ARCH=dummy BOARD=dummy SEL4_SDK=dummy TARGET_TRIPLE=dummy LLVM=False LINK_ADDRESS=0" + + r = system( + f"make -C loader tests {make_args}" + ) + if r != 0: + raise Exception(f"Tests failed: loader") + + def build_elf_component( component_name: str, sdk_dir: Path, @@ -1094,6 +1107,8 @@ def main() -> None: if not args.skip_run_time: build_dir = Path("build") + test_loader(build_dir) + for (board, configs) in build_goals: for config in configs: if not args.skip_sel4: diff --git a/loader/Makefile b/loader/Makefile index 376e68849..bcd5f7d73 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -102,7 +102,7 @@ $(BUILD_DIR)/lib%.a : src/%.rs OBJPROG = $(addprefix $(BUILD_DIR)/, $(PROGS)) -all: $(OBJPROG) test +all: $(OBJPROG) $(LINKSCRIPT): $(LINKSCRIPT_INPUT) $(CPP) -DLINK_ADDRESS=$(LINK_ADDRESS) $< | grep -v "^#" > $@ @@ -112,11 +112,14 @@ LDFLAGS := -T$(LINKSCRIPT) --gc-sections $(OBJPROG): $(addprefix $(BUILD_DIR)/, $(OBJECTS)) $(LINKSCRIPT) $(LD) $(LDFLAGS) --start-group $(addprefix $(BUILD_DIR)/, $(OBJECTS)) --end-group -o $@ -test: +rusttest_%: src/%.rs $(RUSTC) $(RUSTFLAGS) \ --emit dep-info,metadata,link \ --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ + -Awarnings \ --test \ - --crate-name test_page_tables \ - src/page_tables.rs - $(BUILD_DIR)/test_page_tables + --crate-name "$@" \ + $< + +tests: rusttest_page_tables + $(BUILD_DIR)/rusttest_page_tables From ac001d0c2df2edf035ab86d6ca1e9763812782f0 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 15:19:06 +1000 Subject: [PATCH 15/29] clippy Signed-off-by: Julia Vassiliki --- loader/Makefile | 13 ++++++++++++- loader/src/c_interop.rs | 4 +--- loader/src/page_tables.rs | 29 ++++++++++++++++------------- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/loader/Makefile b/loader/Makefile index bcd5f7d73..fb2868cb3 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -36,6 +36,7 @@ else endif RUSTC := rustc +CLIPPY := clippy-driver ifeq ($(ARCH),aarch64) CFLAGS_AARCH64 := -mcpu=$(GCC_CPU) -mgeneral-regs-only -mstrict-align -mno-outline-atomics @@ -121,5 +122,15 @@ rusttest_%: src/%.rs --crate-name "$@" \ $< -tests: rusttest_page_tables +rustclippy_%: src/%.rs + $(CLIPPY) $(RUSTFLAGS) \ + --emit dep-info,metadata,link \ + --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ + -Dwarnings \ + -Cpanic=abort \ + --crate-type staticlib \ + --crate-name "$@" \ + $< + +tests: rusttest_page_tables rustclippy_page_tables $(BUILD_DIR)/rusttest_page_tables diff --git a/loader/src/c_interop.rs b/loader/src/c_interop.rs index 8fd6b7406..98a63662f 100644 --- a/loader/src/c_interop.rs +++ b/loader/src/c_interop.rs @@ -19,9 +19,7 @@ mod real_hardware { impl fmt::Write for Writer { fn write_str(&mut self, s: &str) -> Result<(), fmt::Error> { for c in s.bytes() { - unsafe { - puts(CStr::from_bytes_with_nul_unchecked(&[c.into(), 0]).as_ptr()) - }; + unsafe { puts(CStr::from_bytes_with_nul_unchecked(&[c, 0]).as_ptr()) }; } Ok(()) } diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index 803f40b0c..b6e947e44 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -610,8 +610,12 @@ pub const MAX_NUM_PAGE_TABLES: usize = 64; /// u = align_down(uart_base, 1GiB), /// ``` /// +/// # Safety +/// - regions_ptr must be valid for as long as this function runs, +/// and regions_len must repsent its length +/// #[unsafe(no_mangle)] -pub extern "C" fn aarch64_setup_pagetables( +pub unsafe extern "C" fn aarch64_setup_pagetables( kernel_first_vaddr: u64, kernel_first_paddr: u64, // In-out param; storage and input @@ -631,10 +635,7 @@ pub extern "C" fn aarch64_setup_pagetables( let mut serialise_page_table_to_paddr = { let page_tables_paddr_start: *const u8 = page_table_bytes.as_ptr().cast(); - assert!( - (page_tables_paddr_start as usize) - == (page_tables_paddr_start as usize).next_multiple_of(PAGE_TABLE_SIZE) - ); + assert!((page_tables_paddr_start as usize).is_multiple_of(PAGE_TABLE_SIZE)); // This maintains the current end of the PT array. let mut next_pt_paddr = page_tables_paddr_start; @@ -988,7 +989,7 @@ pub extern "C" fn aarch64_setup_pagetables( _ => unreachable!("level is 1..=3"), } - base = base + pt_region_size; + base += pt_region_size; } } @@ -1119,12 +1120,14 @@ mod tests { let mut page_table_bytes = PtBytes([[MaybeUninit::uninit(); _]; _]); - let pt_bases = aarch64_setup_pagetables( - /* kernel_first_vaddr */ 0x8060000000, - /* kernel_first_paddr */ 0x60000000, - regions.as_mut_ptr(), - regions.len(), - &mut page_table_bytes.0, - ); + let pt_bases = unsafe { + aarch64_setup_pagetables( + /* kernel_first_vaddr */ 0x8060000000, + /* kernel_first_paddr */ 0x60000000, + regions.as_mut_ptr(), + regions.len(), + &mut page_table_bytes.0, + ) + }; } } From 61604b1f3340a40cd8f01b7008943301e5c09be7 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 15:30:00 +1000 Subject: [PATCH 16/29] clippy Signed-off-by: Julia Vassiliki --- .github/workflows/pr.yml | 2 ++ build_sdk.py | 7 +++++++ loader/Makefile | 11 ++++++++--- loader/src/c_interop.rs | 2 +- loader/src/page_tables.rs | 3 +++ 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8593a4ef0..63fd4bedc 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -27,6 +27,8 @@ jobs: - name: Run Clippy # Make sure CI fails on all warnings, including Clippy lints run: nix develop --ignore-environment -c bash -c "cd tool/microkit && cargo-clippy --all-targets --all-features -- -D warnings -Wclippy::get_unwrap" + - name: Run Clippy (Loader) + run: nix develop --ignore-environment -c bash -c 'make -C loader clippy CLIPPYFLAGS="-D warnings -Wclippy::get_unwrap" BUILD_DIR=$(mktemp -d) ARCH=dummy BOARD=dummy SEL4_SDK=dummy TARGET_TRIPLE=dummy LLVM=False LINK_ADDRESS=0' rustfmt_check: runs-on: [self-hosted, macos, ARM64] diff --git a/build_sdk.py b/build_sdk.py index b9adcddda..dd5b4f9cf 100644 --- a/build_sdk.py +++ b/build_sdk.py @@ -808,6 +808,13 @@ def test_loader(build_dir: Path) -> None: if r != 0: raise Exception(f"Tests failed: loader") + # We don't pass CLIPPYARGS, so this is warning-only + r = system( + f"make -C loader clippy {make_args}" + ) + if r != 0: + raise Exception(f"Clippy failed: loader") + def build_elf_component( component_name: str, diff --git a/loader/Makefile b/loader/Makefile index fb2868cb3..d106e3261 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -122,15 +122,20 @@ rusttest_%: src/%.rs --crate-name "$@" \ $< +tests: rusttest_page_tables + $(BUILD_DIR)/rusttest_page_tables + + +CLIPPYFLAGS ?= + rustclippy_%: src/%.rs $(CLIPPY) $(RUSTFLAGS) \ --emit dep-info,metadata,link \ --out-dir $(BUILD_DIR) -L dependency=$(BUILD_DIR) \ - -Dwarnings \ + $(CLIPPYFLAGS) \ -Cpanic=abort \ --crate-type staticlib \ --crate-name "$@" \ $< -tests: rusttest_page_tables rustclippy_page_tables - $(BUILD_DIR)/rusttest_page_tables +clippy: rustclippy_page_tables diff --git a/loader/src/c_interop.rs b/loader/src/c_interop.rs index 98a63662f..526ec80f0 100644 --- a/loader/src/c_interop.rs +++ b/loader/src/c_interop.rs @@ -50,7 +50,7 @@ mod real_hardware { fn panic(info: &PanicInfo) -> ! { println!("panicked"); - if let Err(_) = writeln!(Writer, "{}", info) { + if writeln!(Writer, "{}", info).is_err() { // If writeln!() fails (which it should never as our fmt::Write) never // fails, then just don't print the extra information. println!("panicked (information unknown)"); diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index b6e947e44..6322e8323 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -6,6 +6,9 @@ #![no_std] +// We prefer indices as it matches the semantics of PT indices +#![allow(clippy::needless_range_loop)] + mod c_interop; use core::cmp::min; From 9716d32773ca3ffa6cd59b851dbab75265608ed6 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 15:44:58 +1000 Subject: [PATCH 17/29] rustfmt + cleanup make Signed-off-by: Julia Vassiliki --- .github/workflows/pr.yml | 6 ++++-- build_sdk.py | 2 +- loader/Makefile | 33 ++++++++++++++++++++++++------ loader/src/c_interop.rs | 2 +- loader/src/page_tables.rs | 42 ++++++++++++++++++++++++++------------- 5 files changed, 61 insertions(+), 24 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 63fd4bedc..d530f4de8 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -27,8 +27,8 @@ jobs: - name: Run Clippy # Make sure CI fails on all warnings, including Clippy lints run: nix develop --ignore-environment -c bash -c "cd tool/microkit && cargo-clippy --all-targets --all-features -- -D warnings -Wclippy::get_unwrap" - - name: Run Clippy (Loader) - run: nix develop --ignore-environment -c bash -c 'make -C loader clippy CLIPPYFLAGS="-D warnings -Wclippy::get_unwrap" BUILD_DIR=$(mktemp -d) ARCH=dummy BOARD=dummy SEL4_SDK=dummy TARGET_TRIPLE=dummy LLVM=False LINK_ADDRESS=0' + - name: Run Clippy (loader) + run: nix develop --ignore-environment -c bash -c 'make -C loader clippy CLIPPY_FLAGS="-D warnings -Wclippy::get_unwrap" BUILD_DIR=$(mktemp -d) RUST_ONLY=True' rustfmt_check: runs-on: [self-hosted, macos, ARM64] @@ -36,6 +36,8 @@ jobs: - uses: actions/checkout@v4 - name: Run rustfmt run: nix develop --ignore-environment -c bash -c "cd tool/microkit && cargo-fmt --check" + - name: Run rustfmt (loader) + run: nix develop --ignore-environment -c bash -c 'make -C loader rustfmt-check BUILD_DIR=$(mktemp -d) RUST_ONLY=True' code: name: Freeze Code diff --git a/build_sdk.py b/build_sdk.py index dd5b4f9cf..01130c2a6 100644 --- a/build_sdk.py +++ b/build_sdk.py @@ -800,7 +800,7 @@ def test_loader(build_dir: Path) -> None: build_dir = build_dir / "loader" build_dir.mkdir(exist_ok=True, parents=True) - make_args = f"BUILD_DIR={build_dir.absolute()} ARCH=dummy BOARD=dummy SEL4_SDK=dummy TARGET_TRIPLE=dummy LLVM=False LINK_ADDRESS=0" + make_args = f"BUILD_DIR={build_dir.absolute()} RUST_ONLY=True" r = system( f"make -C loader tests {make_args}" diff --git a/loader/Makefile b/loader/Makefile index d106e3261..93ba4fd65 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -3,6 +3,15 @@ # # SPDX-License-Identifier: BSD-2-Clause # + +ifneq ($(strip $(RUST_ONLY)),) + ARCH := dummy + BOARD := dummy + LINK_ADDRESS := dummy + TARGET_TRIPLE := dummy + SEL4_SDK := dummy +endif + ifeq ($(strip $(BUILD_DIR)),) $(error BUILD_DIR must be specified) endif @@ -37,6 +46,7 @@ endif RUSTC := rustc CLIPPY := clippy-driver +RUSTFMT := rustfmt ifeq ($(ARCH),aarch64) CFLAGS_AARCH64 := -mcpu=$(GCC_CPU) -mgeneral-regs-only -mstrict-align -mno-outline-atomics @@ -59,7 +69,9 @@ CFLAGS := -std=gnu11 -g -O3 -nostdlib -ffreestanding \ ASM_FLAGS := $(ASM_FLAGS_ARCH) -g -MP -MD -I$(SEL4_SDK)/include -RUSTFLAGS := --edition 2024 -g -C opt-level=2 +RUST_EDITION := 2024 +RUSTFLAGS := --edition $(RUST_EDITION) -g -C opt-level=2 +CLIPPY_FLAGS ?= PROGS := loader.elf OBJECTS := loader.o crt0.o uart.o cutil.o libpage_tables.a @@ -70,6 +82,8 @@ else ifeq ($(ARCH),riscv64) OBJECTS += exceptions.o init.o mmu.o cpus.o sbi.o endif +RUST_CRATES := page_tables + LINKSCRIPT_INPUT := $(ARCH).ld LINKSCRIPT := $(BUILD_DIR)/link.ld @@ -122,12 +136,9 @@ rusttest_%: src/%.rs --crate-name "$@" \ $< -tests: rusttest_page_tables +tests: $(addprefix rusttest_, $(RUST_CRATES)) $(BUILD_DIR)/rusttest_page_tables - -CLIPPYFLAGS ?= - rustclippy_%: src/%.rs $(CLIPPY) $(RUSTFLAGS) \ --emit dep-info,metadata,link \ @@ -138,4 +149,14 @@ rustclippy_%: src/%.rs --crate-name "$@" \ $< -clippy: rustclippy_page_tables +clippy: $(addprefix rustclippy_, $(RUST_CRATES)) + +rustfmt_%: src/%.rs + $(RUSTFMT) $(RUSTFMT_FLAGS) \ + --edition $(RUST_EDITION) --style-edition $(RUST_EDITION) \ + $< + +rustfmt: $(addprefix rustfmt_, $(RUST_CRATES)) + +rustfmt-check: RUSTFMT_FLAGS += --check +rustfmt-check: rustfmt diff --git a/loader/src/c_interop.rs b/loader/src/c_interop.rs index 526ec80f0..2b97451ef 100644 --- a/loader/src/c_interop.rs +++ b/loader/src/c_interop.rs @@ -1,7 +1,7 @@ #[cfg(not(test))] mod real_hardware { - use core::ffi::c_char; use core::ffi::CStr; + use core::ffi::c_char; use core::fmt; use core::fmt::Write; use core::panic::PanicInfo; diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index 6322e8323..806848355 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -5,7 +5,6 @@ // #![no_std] - // We prefer indices as it matches the semantics of PT indices #![allow(clippy::needless_range_loop)] @@ -31,11 +30,7 @@ const fn mask(n: u64) -> u64 { const fn round_down(n: u64, x: u64) -> u64 { let (_, m) = divmod(n, x); - if m == 0 { - n - } else { - n - m - } + if m == 0 { n } else { n - m } } const fn align_down(n: u64, bits: u64) -> u64 { @@ -408,7 +403,7 @@ pub extern "C" fn riscv64_setup_pagetables( kernel_first_paddr: u64, page_tables_paddr_start: u64, ) -> u64 { - use riscv64::{pt_index, pte_leaf, pte_next, BLOCK_BITS_1GB, BLOCK_BITS_2MB, PAGE_BITS_4K}; + use riscv64::{BLOCK_BITS_1GB, BLOCK_BITS_2MB, PAGE_BITS_4K, pt_index, pte_leaf, pte_next}; let text_addr = &raw const _text as u64; @@ -628,9 +623,10 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], ) -> AArch64ReturnValue { use aarch64::{ - block_descriptor, lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, + BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, block_descriptor, + lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}, - table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, + table_descriptor, }; const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); @@ -807,7 +803,13 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( if region.start >= lvl2_vaddr_top { if lvl2_pt != [0; _] { let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); - println!("[iter] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", lvl2_pt_paddr as usize, (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + println!( + "[iter] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", + lvl2_pt_paddr as usize, + (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), + base, + lvl1_index(base) + ); assert!(lvl1_pt[lvl1_index(base)] == 0); lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); } @@ -903,7 +905,9 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // page table object and add it to the list. // This should be possible to handle - we just need to break out of this loop - todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + todo!( + "handle the case where top of lvl1 is occupied - this would be near the top of 512GiB" + ); } // Invariant: Lower levels are empty. @@ -935,7 +939,9 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); if top == lvl1_vaddr_top { - todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + todo!( + "handle the case where top of lvl1 is occupied - this would be near the top of 512GiB" + ); } } @@ -982,7 +988,9 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); if top == lvl1_vaddr_top { - todo!("handle the case where top of lvl1 is occupied - this would be near the top of 512GiB"); + todo!( + "handle the case where top of lvl1 is occupied - this would be near the top of 512GiB" + ); } } } @@ -1010,7 +1018,13 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( if lvl2_pt != [0; _] { let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); - println!("[end] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", lvl2_pt_paddr as usize, (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), base, lvl1_index(base)); + println!( + "[end] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", + lvl2_pt_paddr as usize, + (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), + base, + lvl1_index(base) + ); assert!(lvl1_pt[lvl1_index(base)] == 0); lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); } From 70fea7e6c8bf029c83935c763a65e064db8c7797 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 15:46:21 +1000 Subject: [PATCH 18/29] don't create large temporaries on the stack Signed-off-by: Julia Vassiliki --- loader/src/page_tables.rs | 52 +++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index 806848355..a67e6da2c 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -631,6 +631,42 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); + const NUM_TEMPORARIES: usize = 4; + // FIXME: Replace once https://github.com/rust-lang/rust/issues/90091 is merged + let (page_table_bytes, pt_temporaries) = page_table_bytes + .split_first_chunk_mut::<{ MAX_NUM_PAGE_TABLES - NUM_TEMPORARIES }>() + .unwrap(); + + let pt_temporaries = { + let pt_temporaries: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; NUM_TEMPORARIES] = + pt_temporaries.try_into().unwrap(); + + for pt in pt_temporaries.iter_mut() { + for elem in pt { + elem.write(0); + } + } + + // SAFETY: we just initialised it. + let pt_temporaries = unsafe { + mem::transmute::< + &mut [[MaybeUninit; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], + &mut [[u8; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], + >(pt_temporaries) + }; + + // SAFETY: + // - all bitpatterns of u8 can be represented in u8. + // - alignment requirements are met by input requirements + unsafe { + assert!((pt_temporaries.as_ptr() as usize).is_multiple_of(PAGE_TABLE_SIZE)); + mem::transmute::< + &mut [[u8; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], + &mut [[u64; PAGE_TABLE_ENTRIES]; NUM_TEMPORARIES], + >(pt_temporaries) + } + }; + let mut serialise_page_table_to_paddr = { let page_tables_paddr_start: *const u8 = page_table_bytes.as_ptr().cast(); @@ -688,7 +724,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( let kernel_lvl1_pt_paddr = { // First, the Level 2 Upr table. let lvl2_pt_paddr = { - let mut lvl2_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + let mut lvl2_pt_kernel = pt_temporaries[0]; let mut vaddr = m; let mut paddr = p; @@ -703,7 +739,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( }; // Then, the Level 1 Upr table. - let mut lvl1_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; + let mut lvl1_pt_kernel = pt_temporaries[0]; lvl1_pt_kernel[lvl1_index(l)] = table_descriptor(lvl2_pt_paddr); serialise_page_table_to_paddr(&mut lvl1_pt_kernel) @@ -734,9 +770,9 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // When the current vaddr (/paddr, as identity mapped) exceeds the // top value we rotate to a new PT. - let mut lvl1_pt = [0u64; PAGE_TABLE_ENTRIES]; - let mut lvl2_pt = [0u64; PAGE_TABLE_ENTRIES]; - let mut lvl3_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut lvl1_pt = pt_temporaries[1]; + let mut lvl2_pt = pt_temporaries[2]; + let mut lvl3_pt = pt_temporaries[3]; // TODO: These should be defines. Note that the top is the size of 1 level of the next level up. // TODO: LVL1_ENTRY_RANGE? idk #[allow(unused_mut)] @@ -1049,7 +1085,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // Manufacture the Level 0 table, containing the kernel table // and the RAM tables. - let mut ttbr0_el2_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut ttbr0_el2_pt = pt_temporaries[0]; assert!(lvl0_index(k) != lvl0_index(0)); ttbr0_el2_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); @@ -1063,8 +1099,8 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( ttbr1_el1: AArch64ReturnValue::INVALID, } } else { - let mut ttbr0_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; - let mut ttbr1_el1_pt = [0u64; PAGE_TABLE_ENTRIES]; + let mut ttbr0_el1_pt = pt_temporaries[0]; + let mut ttbr1_el1_pt = pt_temporaries[1]; // Kernel in TTBR1 (Upper) ttbr1_el1_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); From 1fbf199aa24c7d70cbef8c94b073130d391d2d62 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 16:14:18 +1000 Subject: [PATCH 19/29] fix stack overflow Signed-off-by: Julia Vassiliki --- loader/src/loader.h | 2 +- loader/src/page_tables.rs | 45 ++++++++++++++++++++++++--------------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/loader/src/loader.h b/loader/src/loader.h index d1a0e79d3..c144381aa 100644 --- a/loader/src/loader.h +++ b/loader/src/loader.h @@ -7,7 +7,7 @@ #pragma once -#define STACK_SIZE 40960 +#define STACK_SIZE 4096 #define REGION_TYPE_DATA 1 #define REGION_TYPE_ZERO 2 diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index a67e6da2c..ac6387e9c 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -724,7 +724,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( let kernel_lvl1_pt_paddr = { // First, the Level 2 Upr table. let lvl2_pt_paddr = { - let mut lvl2_pt_kernel = pt_temporaries[0]; + let mut lvl2_pt_kernel = &mut pt_temporaries[0]; let mut vaddr = m; let mut paddr = p; @@ -739,7 +739,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( }; // Then, the Level 1 Upr table. - let mut lvl1_pt_kernel = pt_temporaries[0]; + let mut lvl1_pt_kernel = &mut pt_temporaries[0]; lvl1_pt_kernel[lvl1_index(l)] = table_descriptor(lvl2_pt_paddr); serialise_page_table_to_paddr(&mut lvl1_pt_kernel) @@ -770,9 +770,18 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // When the current vaddr (/paddr, as identity mapped) exceeds the // top value we rotate to a new PT. - let mut lvl1_pt = pt_temporaries[1]; - let mut lvl2_pt = pt_temporaries[2]; - let mut lvl3_pt = pt_temporaries[3]; + // let mut lvl1_pt = &mut pt_temporaries[1]; + // let mut lvl2_pt = &mut pt_temporaries[2]; + // let mut lvl3_pt = &mut pt_temporaries[3]; + + let (_, rest) = pt_temporaries.split_at_mut(1); + let (lvl1_pt, rest) = rest.split_at_mut(1); + let (lvl2_pt, rest) = rest.split_at_mut(1); + let (lvl3_pt, rest) = rest.split_at_mut(1); + let mut lvl1_pt = &mut lvl1_pt[0]; + let mut lvl2_pt = &mut lvl2_pt[0]; + let mut lvl3_pt = &mut lvl3_pt[0]; + // TODO: These should be defines. Note that the top is the size of 1 level of the next level up. // TODO: LVL1_ENTRY_RANGE? idk #[allow(unused_mut)] @@ -819,7 +828,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( { if region.start >= lvl3_vaddr_top { - if lvl3_pt != [0; _] { + if lvl3_pt != &[0; _] { let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); println!( "[iter] Serialise lvl3 table: {:#x} for to {:#x}..{lvl3_vaddr_top:#x}", @@ -837,7 +846,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( } if region.start >= lvl2_vaddr_top { - if lvl2_pt != [0; _] { + if lvl2_pt != &[0; _] { let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); println!( "[iter] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", @@ -947,8 +956,8 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( } // Invariant: Lower levels are empty. - assert!(lvl2_pt == [0; _]); - assert!(lvl3_pt == [0; _]); + assert!(lvl2_pt == &[0; _]); + assert!(lvl3_pt == &[0; _]); // Invariant maintenance: vaddr_top is right range for current PT. // it's empty so we need to increment the top to be current top (1G aligned) + 2MIB (512 lvl3 entries) lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); @@ -982,7 +991,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( } // Invariant: Lower levels are empty. - assert!(lvl3_pt == [0; _]); + assert!(lvl3_pt == &[0; _]); // Invariant maintenance: vaddr_top is right range for current PT. // it's empty so we need to increment the top to be current top (2MIB aligned) + 2MIB (512 lvl3 entries) lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); @@ -1045,14 +1054,14 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // page tables that have been partially filled out, and we need to // serialise these. - if lvl3_pt != [0; _] { + if lvl3_pt != &[0; _] { let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); println!("[end] Serialise lvl3 table: {:#x}", lvl3_pt_paddr as usize); assert!(lvl2_pt[lvl2_index(base)] == 0); lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); } - if lvl2_pt != [0; _] { + if lvl2_pt != &[0; _] { let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); println!( "[end] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", @@ -1066,7 +1075,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( } // the level1 pt should not be empty. lol. - assert!(lvl1_pt != [0; _]); + assert!(lvl1_pt != &[0; _]); // println!("New lvl1 table"); serialise_page_table_to_paddr(&mut lvl1_pt) @@ -1085,7 +1094,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // Manufacture the Level 0 table, containing the kernel table // and the RAM tables. - let mut ttbr0_el2_pt = pt_temporaries[0]; + let mut ttbr0_el2_pt = &mut pt_temporaries[0]; assert!(lvl0_index(k) != lvl0_index(0)); ttbr0_el2_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); @@ -1099,16 +1108,18 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( ttbr1_el1: AArch64ReturnValue::INVALID, } } else { - let mut ttbr0_el1_pt = pt_temporaries[0]; - let mut ttbr1_el1_pt = pt_temporaries[1]; + // let mut ttbr0_el1_pt = &mut pt_temporaries[0]; + let mut ttbr1_el1_pt = &mut pt_temporaries[1]; // Kernel in TTBR1 (Upper) ttbr1_el1_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); + let ttbr1_el1 = serialise_page_table_to_paddr(&mut ttbr1_el1_pt); + let mut ttbr0_el1_pt = &mut pt_temporaries[0]; // Identity-mapped RAM in TTBR0 (Lower) ttbr0_el1_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); let ttbr0_el1 = serialise_page_table_to_paddr(&mut ttbr0_el1_pt); - let ttbr1_el1 = serialise_page_table_to_paddr(&mut ttbr1_el1_pt); + // let ttbr1_el1 = serialise_page_table_to_paddr(&mut ttbr1_el1_pt); AArch64ReturnValue { ttbr0_el2: AArch64ReturnValue::INVALID, From 9bb64fb0d107fb84201dc55cbb69cde3600251f5 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 16:24:02 +1000 Subject: [PATCH 20/29] fix style Signed-off-by: Julia Vassiliki --- loader/Makefile | 2 +- loader/src/c_interop.rs | 2 +- loader/src/page_tables.rs | 13 ++++++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/loader/Makefile b/loader/Makefile index 93ba4fd65..9668f7005 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -153,7 +153,7 @@ clippy: $(addprefix rustclippy_, $(RUST_CRATES)) rustfmt_%: src/%.rs $(RUSTFMT) $(RUSTFMT_FLAGS) \ - --edition $(RUST_EDITION) --style-edition $(RUST_EDITION) \ + --edition $(RUST_EDITION) \ $< rustfmt: $(addprefix rustfmt_, $(RUST_CRATES)) diff --git a/loader/src/c_interop.rs b/loader/src/c_interop.rs index 2b97451ef..526ec80f0 100644 --- a/loader/src/c_interop.rs +++ b/loader/src/c_interop.rs @@ -1,7 +1,7 @@ #[cfg(not(test))] mod real_hardware { - use core::ffi::CStr; use core::ffi::c_char; + use core::ffi::CStr; use core::fmt; use core::fmt::Write; use core::panic::PanicInfo; diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index ac6387e9c..d68b7c954 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -30,7 +30,11 @@ const fn mask(n: u64) -> u64 { const fn round_down(n: u64, x: u64) -> u64 { let (_, m) = divmod(n, x); - if m == 0 { n } else { n - m } + if m == 0 { + n + } else { + n - m + } } const fn align_down(n: u64, bits: u64) -> u64 { @@ -403,7 +407,7 @@ pub extern "C" fn riscv64_setup_pagetables( kernel_first_paddr: u64, page_tables_paddr_start: u64, ) -> u64 { - use riscv64::{BLOCK_BITS_1GB, BLOCK_BITS_2MB, PAGE_BITS_4K, pt_index, pte_leaf, pte_next}; + use riscv64::{pt_index, pte_leaf, pte_next, BLOCK_BITS_1GB, BLOCK_BITS_2MB, PAGE_BITS_4K}; let text_addr = &raw const _text as u64; @@ -623,10 +627,9 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], ) -> AArch64ReturnValue { use aarch64::{ - BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, block_descriptor, - lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, + block_descriptor, lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}, - table_descriptor, + table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, }; const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); From 9a1f22b023b372e40186bf810be2cf42bc79d829 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 16:24:17 +1000 Subject: [PATCH 21/29] fix warnings Signed-off-by: Julia Vassiliki --- loader/src/page_tables.rs | 49 ++++++++++++++------------------------- 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index d68b7c954..ae6f27185 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -727,7 +727,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( let kernel_lvl1_pt_paddr = { // First, the Level 2 Upr table. let lvl2_pt_paddr = { - let mut lvl2_pt_kernel = &mut pt_temporaries[0]; + let lvl2_pt_kernel = &mut pt_temporaries[0]; let mut vaddr = m; let mut paddr = p; @@ -738,14 +738,14 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( paddr += 1 << BLOCK_BITS_2MB; } - serialise_page_table_to_paddr(&mut lvl2_pt_kernel) + serialise_page_table_to_paddr(lvl2_pt_kernel) }; // Then, the Level 1 Upr table. - let mut lvl1_pt_kernel = &mut pt_temporaries[0]; + let lvl1_pt_kernel = &mut pt_temporaries[0]; lvl1_pt_kernel[lvl1_index(l)] = table_descriptor(lvl2_pt_paddr); - serialise_page_table_to_paddr(&mut lvl1_pt_kernel) + serialise_page_table_to_paddr(lvl1_pt_kernel) }; // Manufacture the RAM page tables, which is a little bit more complicated. @@ -773,17 +773,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // When the current vaddr (/paddr, as identity mapped) exceeds the // top value we rotate to a new PT. - // let mut lvl1_pt = &mut pt_temporaries[1]; - // let mut lvl2_pt = &mut pt_temporaries[2]; - // let mut lvl3_pt = &mut pt_temporaries[3]; - - let (_, rest) = pt_temporaries.split_at_mut(1); - let (lvl1_pt, rest) = rest.split_at_mut(1); - let (lvl2_pt, rest) = rest.split_at_mut(1); - let (lvl3_pt, rest) = rest.split_at_mut(1); - let mut lvl1_pt = &mut lvl1_pt[0]; - let mut lvl2_pt = &mut lvl2_pt[0]; - let mut lvl3_pt = &mut lvl3_pt[0]; + let [lvl1_pt, lvl2_pt, lvl3_pt] = pt_temporaries.get_disjoint_mut([1, 2, 3]).unwrap(); // TODO: These should be defines. Note that the top is the size of 1 level of the next level up. // TODO: LVL1_ENTRY_RANGE? idk @@ -832,7 +822,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( { if region.start >= lvl3_vaddr_top { if lvl3_pt != &[0; _] { - let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + let lvl3_pt_paddr = serialise_page_table_to_paddr(lvl3_pt); println!( "[iter] Serialise lvl3 table: {:#x} for to {:#x}..{lvl3_vaddr_top:#x}", lvl3_pt_paddr as usize, @@ -850,7 +840,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( if region.start >= lvl2_vaddr_top { if lvl2_pt != &[0; _] { - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + let lvl2_pt_paddr = serialise_page_table_to_paddr(lvl2_pt); println!( "[iter] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", lvl2_pt_paddr as usize, @@ -980,7 +970,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // Invariant maintenance: keep for current address range. // As we're the top of the range, we can serialise the table. - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + let lvl2_pt_paddr = serialise_page_table_to_paddr(lvl2_pt); // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} up to {lvl2_vaddr_top:#x}"); lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; @@ -1012,7 +1002,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // Invariant maintenance: keep for current address range. // As we're the top of the range, we can serialise the table. - let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + let lvl3_pt_paddr = serialise_page_table_to_paddr(lvl3_pt); println!( "Serialise lvl3 table: {:#x} for to {:#x}..{lvl3_vaddr_top:#x}", lvl3_pt_paddr as usize, @@ -1024,7 +1014,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); if top == lvl2_vaddr_top { - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + let lvl2_pt_paddr = serialise_page_table_to_paddr(lvl2_pt); println!( "Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}", lvl2_pt_paddr as usize, @@ -1058,14 +1048,14 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // serialise these. if lvl3_pt != &[0; _] { - let lvl3_pt_paddr = serialise_page_table_to_paddr(&mut lvl3_pt); + let lvl3_pt_paddr = serialise_page_table_to_paddr(lvl3_pt); println!("[end] Serialise lvl3 table: {:#x}", lvl3_pt_paddr as usize); assert!(lvl2_pt[lvl2_index(base)] == 0); lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); } if lvl2_pt != &[0; _] { - let lvl2_pt_paddr = serialise_page_table_to_paddr(&mut lvl2_pt); + let lvl2_pt_paddr = serialise_page_table_to_paddr(lvl2_pt); println!( "[end] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", lvl2_pt_paddr as usize, @@ -1081,7 +1071,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( assert!(lvl1_pt != &[0; _]); // println!("New lvl1 table"); - serialise_page_table_to_paddr(&mut lvl1_pt) + serialise_page_table_to_paddr(lvl1_pt) }; struct Config { @@ -1097,13 +1087,13 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // Manufacture the Level 0 table, containing the kernel table // and the RAM tables. - let mut ttbr0_el2_pt = &mut pt_temporaries[0]; + let ttbr0_el2_pt = &mut pt_temporaries[0]; assert!(lvl0_index(k) != lvl0_index(0)); ttbr0_el2_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); ttbr0_el2_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); - let ttbr0_el2 = serialise_page_table_to_paddr(&mut ttbr0_el2_pt); + let ttbr0_el2 = serialise_page_table_to_paddr(ttbr0_el2_pt); AArch64ReturnValue { ttbr0_el2, @@ -1111,18 +1101,15 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( ttbr1_el1: AArch64ReturnValue::INVALID, } } else { - // let mut ttbr0_el1_pt = &mut pt_temporaries[0]; - let mut ttbr1_el1_pt = &mut pt_temporaries[1]; + let [ttbr0_el1_pt, ttbr1_el1_pt] = pt_temporaries.get_disjoint_mut([0, 1]).unwrap(); // Kernel in TTBR1 (Upper) ttbr1_el1_pt[lvl0_index(k)] = table_descriptor(kernel_lvl1_pt_paddr); - let ttbr1_el1 = serialise_page_table_to_paddr(&mut ttbr1_el1_pt); - let mut ttbr0_el1_pt = &mut pt_temporaries[0]; // Identity-mapped RAM in TTBR0 (Lower) ttbr0_el1_pt[lvl0_index(0)] = table_descriptor(ram_lvl1_pt_paddr); - let ttbr0_el1 = serialise_page_table_to_paddr(&mut ttbr0_el1_pt); - // let ttbr1_el1 = serialise_page_table_to_paddr(&mut ttbr1_el1_pt); + let ttbr0_el1 = serialise_page_table_to_paddr(ttbr0_el1_pt); + let ttbr1_el1 = serialise_page_table_to_paddr(ttbr1_el1_pt); AArch64ReturnValue { ttbr0_el2: AArch64ReturnValue::INVALID, From 396232e5c202298c07e3d6e7b254c4fc910c5070 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 16:27:22 +1000 Subject: [PATCH 22/29] add safety comment Signed-off-by: Julia Vassiliki --- loader/src/page_tables.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index ae6f27185..70778b2d0 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -614,7 +614,8 @@ pub const MAX_NUM_PAGE_TABLES: usize = 64; /// /// # Safety /// - regions_ptr must be valid for as long as this function runs, -/// and regions_len must repsent its length +/// and regions_len must represent its length +/// - page_table_bytes must be aligned to PAGE_TABLE_SIZE /// #[unsafe(no_mangle)] pub unsafe extern "C" fn aarch64_setup_pagetables( From bae0a5e6067a19fa7d85fa08caf5cba0c9bbbcd7 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Tue, 18 Aug 2026 18:14:15 +1000 Subject: [PATCH 23/29] test walker Signed-off-by: Julia Vassiliki --- loader/src/page_tables.rs | 152 +++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 4 deletions(-) diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index 70778b2d0..0009b5f8b 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -693,6 +693,14 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( next_pt_paddr = next_pt_paddr.wrapping_add(PAGE_TABLE_SIZE); i += 1; page_table.fill(0); + + if cfg!(test) { + // HACK! For tests, we want stable page tables, but due to ASLR + // we get random things every time. Instead, let's make the + // paddr we return a relative-to-start-of-page-tables value. + return unsafe { pt_paddr.offset_from(page_tables_paddr_start) } as *const _; + } + pt_paddr } }; @@ -1124,9 +1132,101 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( mod tests { use super::*; - #[test] - fn it_works() { - assert_eq!(2 + 2, 4); + extern crate std; + use std::unreachable; + use std::vec; + use std::vec::Vec; + + // Exclusive [start, end) + #[derive(Debug, PartialEq)] + struct WalkRegion { + v_start: u64, + v_end: u64, + p_start: u64, + p_end: u64, + // This includes *all* page table attributes + arch_value: u64, + } + + fn aarch64_walk_pt_level_order( + level: usize, + pte: u64, + vaddr: &mut u64, + pts: &[[u64; 512]], + regions: &mut Vec, + ) { + use aarch64::descriptor_type; + + // Level is [0, 4) + assert!(level < 4); + + let v_start = *vaddr; + let size = 1 + << match level { + 0 => aarch64::BLOCK_BITS_512GB, + 1 => aarch64::BLOCK_BITS_1GB, + 2 => aarch64::BLOCK_BITS_2MB, + 3 => aarch64::PAGE_BITS_4KB, + _ => unreachable!(), + }; + *vaddr += size; + let v_end = *vaddr; + + if pte == 0 { + return; + } + + let pte_type = pte & 0b11; + // bits [47: 12] + let pte_oa = pte & 0xfffffffff000; + let pte_attrs = pte & !0xfffffffff000; + + if level == 3 { + assert!(pte_type == descriptor_type::PAGE); + } + + if level != 3 && pte_type == descriptor_type::TABLE { + let next_level_pt_idx = (pte_oa as usize) / PAGE_TABLE_SIZE; + // println!("oa: {pte_oa:x}, next_idx: {next_level_pt_idx}"); + + let mut vaddr = v_start; + for &child_pte in pts[next_level_pt_idx].iter() { + aarch64_walk_pt_level_order(level + 1, child_pte, &mut vaddr, pts, regions); + } + } else { + regions.push(WalkRegion { + v_start, + v_end, + p_start: pte_oa, + p_end: pte_oa + size, + arch_value: pte_attrs, + }); + } + } + + fn aarch64_walk_pt_gather_regions(pts: &[[u64; 512]], root_idx: usize) -> Vec { + let mut regions = vec![]; + let mut vaddr = 0; + for &pte in pts[root_idx].iter() { + aarch64_walk_pt_level_order(0, pte, &mut vaddr, pts, &mut regions); + } + + let mut i = regions.len() - 1; + while i > 1 { + if regions[i].p_start == regions[i - 1].p_end + && regions[i].v_start == regions[i - 1].v_end + && regions[i].arch_value == regions[i - 1].arch_value + { + regions[i - 1].p_end = regions[i].p_end; + regions[i - 1].v_end = regions[i].v_end; + regions.remove(i); + } + + i -= 1; + } + + println!("{regions:#x?}"); + regions } #[test] @@ -1173,7 +1273,7 @@ mod tests { // )); // } - let mut page_table_bytes = PtBytes([[MaybeUninit::uninit(); _]; _]); + let mut page_table_bytes = PtBytes([[MaybeUninit::zeroed(); _]; _]); let pt_bases = unsafe { aarch64_setup_pagetables( @@ -1184,5 +1284,49 @@ mod tests { &mut page_table_bytes.0, ) }; + + let page_tables = unsafe { + mem::transmute::< + [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], + [[u64; 512]; MAX_NUM_PAGE_TABLES], + >(page_table_bytes.0) + }; + + assert_eq!(pt_bases.ttbr0_el1, AArch64ReturnValue::INVALID); + assert_eq!(pt_bases.ttbr1_el1, AArch64ReturnValue::INVALID); + assert_ne!(pt_bases.ttbr0_el2, AArch64ReturnValue::INVALID); + + let root_addr = pt_bases.ttbr0_el2 as usize; + + let walk_regions = + aarch64_walk_pt_gather_regions(&page_tables, root_addr / PAGE_TABLE_SIZE); + + assert_eq!( + walk_regions, + vec![ + // UART + WalkRegion { + v_start: 0x9000000, + v_end: 0x9001000, + p_start: 0x9000000, + p_end: 0x9001000, + arch_value: 0x603, + }, + WalkRegion { + v_start: 0x60000000, + v_end: 0xc0000000, + p_start: 0x60000000, + p_end: 0xc0000000, + arch_value: 0x601, + }, + WalkRegion { + v_start: 0x8060000000, + v_end: 0x8080000000, + p_start: 0x60000000, + p_end: 0x80000000, + arch_value: 0x711, + }, + ] + ); } } From 2a0a57bcd704092ba74f67215f53f8a75260db3e Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Thu, 20 Aug 2026 15:49:25 +1000 Subject: [PATCH 24/29] almost working clean Signed-off-by: Julia Vassiliki --- loader/src/aligned_regions.rs | 273 ++++++++++++++++++++++++++ loader/src/page_tables.rs | 358 +++++++--------------------------- 2 files changed, 344 insertions(+), 287 deletions(-) create mode 100644 loader/src/aligned_regions.rs diff --git a/loader/src/aligned_regions.rs b/loader/src/aligned_regions.rs new file mode 100644 index 000000000..963b6592e --- /dev/null +++ b/loader/src/aligned_regions.rs @@ -0,0 +1,273 @@ +// +// Copyright 2026, UNSW +// +// SPDX-License-Identifier: BSD-2-Clause +// + +//! This function provides an iterator which is useful for generating +//! page tables in one pass, with a constant amount of "extra" space for +//! bookkeeping. It transforms a set of discontiguous regions that cover +//! multiple levels of page table structures into a set of aligned regions, +//! useful for filling out paging structures. An additional piece of information +//! we need to maintain is when we move between levels. + +use core::array; +use core::cmp::min; + +// Inclusive [start, top] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd)] +pub struct Region { + pub start: usize, + pub top: usize, +} + +impl Region { + const EMPTY: Self = Self { start: 0, top: 0 }; +} + +/// Bits from [start, end) +/// Note: Bit indices are 'u32' as this is what rust tends to use for usize::BITS, +/// and for checked_shl, and otherwise. This makes our life simpler. +fn bits_of_range(value: usize, start: u32, end: u32) -> usize { + assert!(start < end); + assert!(start < usize::BITS); + assert!(end <= usize::BITS); + + // Handle the maximum-shift case. + let mask = if let Some(bit) = 1usize.checked_shl(end) { + bit - 1 + } else { + debug_assert!(end == usize::BITS); + usize::MAX + }; + + (value & mask) >> start +} + +fn indices_of_level( + // [size_bits, count_bits) + level_bits: &[(u32, u32); LEVELS], + level: usize, + value: usize, +) -> usize { + let (size_bits, count_bits) = level_bits[level]; + + bits_of_range(value, size_bits, size_bits + count_bits) +} + +#[must_use = "iterators are lazy and do nothing unless consumed"] +pub struct AlignedRegionsIter +where + I: Iterator, +{ + /// Array of [size_bits, count_bits), descending, for each level + level_bits: [(u32, u32); LEVELS], + input_regions_iter: I, + current_input_region: Option, + current_addr: usize, +} + +impl AlignedRegionsIter +where + I: Iterator, +{ + // TODO: method on iter? + pub fn new(iter: I, level_bits: [(u32, u32); LEVELS]) -> Self { + for ((upper_size, _), (lower_size, lower_count)) in + level_bits.windows(2).map(|s| (s[0], s[1])) + { + assert!(upper_size == lower_size + lower_count); + } + + Self { + level_bits, + input_regions_iter: iter, + current_input_region: None, + current_addr: 0, + } + } +} + +impl Iterator for AlignedRegionsIter +where + I: Iterator, +{ + type Item = (usize, [usize; LEVELS]); + + fn next(&mut self) -> Option { + let region = match self.current_input_region { + Some(r) => r, + None => { + let Some(region) = self.input_regions_iter.next() else { + // We exit our iterator here as we have no more work to do. + return None; + }; + + assert!(region != Region::EMPTY); + assert!(region.start < region.top); + + self.current_input_region = Some(region); + // Guarantees the loop invariant. + self.current_addr = region.start; + + region + } + }; + + let current_addr = self.current_addr; + + // Loop invariant. + assert!(current_addr < region.top); + + let size = region.top.checked_sub(current_addr).unwrap() + 1; + let size_bits = size.ilog2(); + // FIXME: Once MSRV is > 1.97, use .lowest_one() method. + let align_bits = if current_addr == 0 { + size_bits + } else { + current_addr.trailing_zeros() + }; + + // The correct pt size bits we can use it the smallest of the size + // and the alignment; we can't use a 21-bit aligned region if + // we have a 15-bit region, since it would overrun. + let align_bits = min(align_bits, size_bits); + + let level = self + .level_bits + .map(|(size, count)| size) + .iter() + .position(|&level_size_bits| align_bits >= level_size_bits) + .expect("bad input; regions should be aligned to at least the lowest level"); + + let level_indices: [usize; LEVELS] = + array::from_fn(|level| indices_of_level(&self.level_bits, level, current_addr)); + + let next_addr = current_addr.wrapping_add(1 << self.level_bits[level].0); + + if next_addr > region.top || next_addr == 0 { + self.current_input_region = None; + } + + self.current_addr = next_addr; + + Some((level, level_indices)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + extern crate std; + use std::vec; + use std::vec::Vec; + + #[test] + fn test_bits_range() { + assert_eq!(bits_of_range(0b0110, 1, 3), 0b11); + assert_eq!(bits_of_range(0b1001, 1, 3), 0b00); + assert_eq!(bits_of_range(usize::MAX, 0, usize::BITS), usize::MAX); + assert_eq!(bits_of_range(usize::MAX, 4, usize::BITS), usize::MAX >> 4); + } + + #[test] + #[should_panic] + fn test_bits_range_not_allowed() { + bits_of_range(0b1001, 1, 1); + } + + #[test] + #[should_panic] + fn test_bits_range_not_allowed2() { + bits_of_range(0b1001, 4, 3); + } + + #[test] + fn test_bits_level() { + let levels = [(32, 16), (24, 8), (12, 12), (8, 4)]; + assert_eq!(indices_of_level(&levels, 0, 10 << 32), 10); + assert_eq!(indices_of_level(&levels, 1, 10 << 24), 10); + assert_eq!(indices_of_level(&levels, 2, 10 << 12), 10); + assert_eq!(indices_of_level(&levels, 3, 10 << 8), 10); + } + + #[test] + #[should_panic] + fn test_invalid_range_aligned_regions() { + let iter = [Region::EMPTY; 5].into_iter(); + // Should be reverse + AlignedRegionsIter::new(iter, [(8, 4), (12, 12), (24, 8), (32, 16)]); + } + + #[test] + fn test_ok_range_aligned_regions() { + let iter = [Region::EMPTY; 5].into_iter(); + AlignedRegionsIter::new(iter, [(32, 16), (24, 8), (12, 12), (8, 4)]); + } + + #[test] + fn test_iter_regions_simple() { + let aarch64_levels = [(39, 9), (30, 9), (21, 9), (12, 9)]; + let mut iter = AlignedRegionsIter::new( + // This should give us 4 4k regions + [Region { + start: 0, + top: 0x3fff, + }] + .into_iter(), + aarch64_levels, + ); + + let indices: Vec<_> = iter.collect(); + + assert_eq!( + indices, + vec![ + (3, [0, 0, 0, 0]), + (3, [0, 0, 0, 1]), + (3, [0, 0, 0, 2]), + (3, [0, 0, 0, 3]), + ] + ); + } + + #[test] + fn test_iter_regions_multi_layer() { + let aarch64_levels = [(39, 9), (30, 9), (21, 9), (12, 9)]; + let mut iter = AlignedRegionsIter::new( + [ + // This should give us 4 4k regions + Region { + start: 0, + top: 0x3fff, + }, + // Then this will give us 2 2M region and then 4 4k regions + Region { + start: 0x200000, + top: 0x603fff, + }, + ] + .into_iter(), + aarch64_levels, + ); + + let indices: Vec<_> = iter.collect(); + + assert_eq!( + indices, + vec![ + (3, [0, 0, 0, 0]), + (3, [0, 0, 0, 1]), + (3, [0, 0, 0, 2]), + (3, [0, 0, 0, 3]), + (2, [0, 0, 1, 0]), + (2, [0, 0, 2, 0]), + (3, [0, 0, 3, 0]), + (3, [0, 0, 3, 1]), + (3, [0, 0, 3, 2]), + (3, [0, 0, 3, 3]), + ] + ); + } +} diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index 0009b5f8b..b6778a5c4 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -8,6 +8,7 @@ // We prefer indices as it matches the semantics of PT indices #![allow(clippy::needless_range_loop)] +mod aligned_regions; mod c_interop; use core::cmp::min; @@ -16,6 +17,7 @@ use core::mem; use core::mem::MaybeUninit; use core::slice; +use aligned_regions::AlignedRegionsIter; use c_interop::println; const PAGE_TABLE_SIZE: usize = 4096; @@ -541,6 +543,7 @@ impl fmt::Debug for RegionArchAttrs { } } +/// Region is [start, end] *inclusive* as this avoids overflows. #[derive(Debug, Copy, Clone)] #[repr(C)] pub struct Region { @@ -765,7 +768,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( let mut previous_end = None; for region in identity_mapped_regions.iter() { assert!(lvl0_index(region.start) == 0); - assert!(lvl0_index(region.end - 1) == 0); + assert!(lvl0_index(region.end) == 0); // This is probably an unnecessary assumption. assert!(region.start.is_multiple_of(4096)); assert!(region.end.is_multiple_of(4096)); @@ -782,305 +785,84 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // When the current vaddr (/paddr, as identity mapped) exceeds the // top value we rotate to a new PT. - let [lvl1_pt, lvl2_pt, lvl3_pt] = pt_temporaries.get_disjoint_mut([1, 2, 3]).unwrap(); + // We never actually use level 0 here, but it is nice to have because + // then the indices are the same as the level. + let mut pts_by_level = pt_temporaries.get_disjoint_mut([0, 1, 2, 3]).unwrap(); - // TODO: These should be defines. Note that the top is the size of 1 level of the next level up. - // TODO: LVL1_ENTRY_RANGE? idk - #[allow(unused_mut)] - let mut lvl1_vaddr_top = 1 << BLOCK_BITS_512GB; - let mut lvl2_vaddr_top = 1 << BLOCK_BITS_1GB; - let mut lvl3_vaddr_top = 1 << BLOCK_BITS_2MB; - - // TODO: Tests... - // This is similar to aligned_power_of_two_regions() for the kernel UT, - // but we restrict it such that the output always is either 1GB, 2MB, or 4KB - // pages. - - // Allowed externally for the final iteration - let mut base = 0u64; - for region in identity_mapped_regions.iter() { + let mut iter = AlignedRegionsIter::new( + identity_mapped_regions + .iter() + .map(|r| aligned_regions::Region { + start: r.start as usize, + top: (r.end - 1) as usize, + }), + [(39, 9), (30, 9), (21, 9), (12, 9)], + ) + .peekable(); + + // RAM should never cross Level 0 boundaries, for the moment at least. + const MIN_LEVEL: usize = 1; + + while let Some((level, level_indices)) = iter.next() { // SAFETY: We went through and initialised raw before. - let attr_index = unsafe { region.arch_attrs.raw }; - - println!( - "Identity-Mapped Region: {:#x}..{:#x}", - region.start, region.end - ); - println!( - " - Current Lvl1: {:#x}..{:#x}, entries: {}", - (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), - lvl1_vaddr_top, - lvl1_pt.iter().filter(|&&v| v != 0).count() - ); - println!( - " - Current Lvl2: {:#x}..{:#x}, entries: {}", - (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), - lvl2_vaddr_top, - lvl2_pt.iter().filter(|&&v| v != 0).count() - ); - println!( - " - Current Lvl3: {:#x}..{:#x}, entries: {}", - (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), - lvl3_vaddr_top, - lvl3_pt.iter().filter(|&&v| v != 0).count() - ); - - // Handle the fact that the regions are not contiguous and that - // we might need to skip PT. - - { - if region.start >= lvl3_vaddr_top { - if lvl3_pt != &[0; _] { - let lvl3_pt_paddr = serialise_page_table_to_paddr(lvl3_pt); - println!( - "[iter] Serialise lvl3 table: {:#x} for to {:#x}..{lvl3_vaddr_top:#x}", - lvl3_pt_paddr as usize, - (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)) - ); - assert!(lvl2_pt[lvl2_index(base)] == 0); - lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); - } + let attr_index = MT_DEVICE_nGnRnE; + // let attr_index = unsafe { region.arch_attrs.raw }; + println!("{level:x} {level_indices:x?}"); - // TODO: just compute it. - while region.start >= lvl3_vaddr_top { - lvl3_vaddr_top += 1 << BLOCK_BITS_2MB; - } - } + assert!(level >= MIN_LEVEL); - if region.start >= lvl2_vaddr_top { - if lvl2_pt != &[0; _] { - let lvl2_pt_paddr = serialise_page_table_to_paddr(lvl2_pt); - println!( - "[iter] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", - lvl2_pt_paddr as usize, - (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), - base, - lvl1_index(base) - ); - assert!(lvl1_pt[lvl1_index(base)] == 0); - lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); - } + let base = 0; - // TODO: just compute it. - while region.start >= lvl2_vaddr_top { - lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; - } - } + assert!(pts_by_level[level][level_indices[level]] == 0); - if region.start >= lvl1_vaddr_top { - unreachable!( - "impossible as everything should fit here: {:#x}", - lvl1_vaddr_top - ); - } - } + pts_by_level[level][level_indices[level]] = if level == 3 { + page_descriptor(base, attr_index) + } else { + block_descriptor(level, base, attr_index) + }; - // After serialising the old base, update the new one. - base = region.start; - - // Inner Loop: - // Invariant: the page tables in lvl1_pt, lvl2_pt, lvl3_pt - // are either (1) for the current address range, - // or (2) are empty and for a lower level than the current level. - // Also, the values in lvlXXX_vaddr_top are always correct (even if empty) - // Also contiguous within the loop. - // Loop entry: (1) holds by work at the start of each region - while base != region.end { - // Condition is !=, but assert that we never skip it. - assert!(base < region.end); - - let size_bits = region.end.wrapping_sub(base).ilog2(); - let align_bits = min( - size_bits, - // FIXME: Once MSRV is > 1.97, use .lowest_one() method. - if base == 0 { - size_bits - } else { - base.trailing_zeros() - }, - ); - - // Match the size and alignment of the current region to - // the valid PT region sizes. - let (level, bits) = match u64::from(align_bits) { - BLOCK_BITS_1GB.. => (1, BLOCK_BITS_1GB), - BLOCK_BITS_2MB.. => (2, BLOCK_BITS_2MB), - PAGE_BITS_4KB.. => (3, PAGE_BITS_4KB), - 0.. => panic!("impossible; regions should be aligned to 4K at least"), + // Invariant: the page tables in pts_by_level are either: + // (1) for the current level_indices, or + // (2) are empty/invalid and for a lower level. + // Similar, the level indices in our array are only meaningful + // from [0..=level]. + // + // Hence, when moving around, we only need to care about page tables + // in the range [0, level) inclusive, and can ignore those on + // lower levels. + // We start from the lowest level (parent) checking if the indices + // prefix (i.e. it, or any above it) have changed. Note that + // checking just the index would be invalid, in the case of say a + // [0, 0, 1, 0] -> [0, 0, 2, 0] change where level=3, as the + // level=2 row has changed, so our level=3 page table must be + // written out. + // We start from the parent and not the current level, because + // the change from [0, 0, 1, 0] -> [0, 0, 1, 1] should not write + // out the page table. (similarly, [0, 0, 1, X] -> [0, 0, 1, X] + // for level=2). + // We don't need to care if next_level is higher than the current + // level, as this still means the current page table is valid. + + for parent_level in (MIN_LEVEL..level).rev() { + // Two cases where we need to write out the page tables: + // either we are reaching the end (iter.peek() = None) + // or if the next one has different page tables to us. + let changed = match iter.peek() { + None => true, + Some((_, next_level_indices)) => { + level_indices[0..=parent_level] != next_level_indices[0..=parent_level] + } }; - let pt_region_size = 1u64 << bits; - let top = base + pt_region_size; - - println!( - "- Aligned PT region: {:#x}..{:#x} (size_bits: {}, align_bits: {}, bits: {})", - base, top, size_bits, align_bits, bits - ); - println!( - " - Current Lvl1: {:#x}..{:#x}, entries: {}", - (lvl1_vaddr_top - (1 << BLOCK_BITS_512GB)), - lvl1_vaddr_top, - lvl1_pt.iter().filter(|&&v| v != 0).count() - ); - println!( - " - Current Lvl2: {:#x}..{:#x}, entries: {}", - (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), - lvl2_vaddr_top, - lvl2_pt.iter().filter(|&&v| v != 0).count() - ); - println!( - " - Current Lvl3: {:#x}..{:#x}, entries: {}", - (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)), - lvl3_vaddr_top, - lvl3_pt.iter().filter(|&&v| v != 0).count() - ); - - match level { - 1 => { - // If it belongs in Level 1 PT, then it must go in - // lvl1 pt. By the inavariant, base < lvl1_vaddr_top. - assert!(base < lvl1_vaddr_top); - // top is <= lvl1_vaddr_top (the case where it is the topmost entry) - assert!(top <= lvl1_vaddr_top); - - assert!(lvl1_pt[lvl1_index(base)] == 0); - lvl1_pt[lvl1_index(base)] = block_descriptor(1, base, attr_index); - - if top == lvl1_vaddr_top { - // Invariant maintenance: if the new top would be now equal - // the end of the page table's region top, we need a new - // page table object and add it to the list. - - // This should be possible to handle - we just need to break out of this loop - todo!( - "handle the case where top of lvl1 is occupied - this would be near the top of 512GiB" - ); - } - - // Invariant: Lower levels are empty. - assert!(lvl2_pt == &[0; _]); - assert!(lvl3_pt == &[0; _]); - // Invariant maintenance: vaddr_top is right range for current PT. - // it's empty so we need to increment the top to be current top (1G aligned) + 2MIB (512 lvl3 entries) - lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); - // it's empty so we need to increment the top to be current top (1G aligned) + 1G (512 lvl2 entries) - lvl2_vaddr_top = top + (1 << BLOCK_BITS_1GB); - } - 2 => { - // If it is a 2MiB block, it must go in the Level 2 PT; - // by our invariants: base < lvl2_vaddr_top and top <= lvl2_vaddr_top - assert!(base < lvl2_vaddr_top); - assert!(top <= lvl2_vaddr_top); - - assert!(lvl2_pt[lvl2_index(base)] == 0); - lvl2_pt[lvl2_index(base)] = block_descriptor(2, base, attr_index); - - if top == lvl2_vaddr_top { - // Invariant maintenance: keep for current address range. - // As we're the top of the range, we can serialise the table. - - let lvl2_pt_paddr = serialise_page_table_to_paddr(lvl2_pt); - // println!("Serialise lvl2 table: {lvl2_pt_paddr:#x} up to {lvl2_vaddr_top:#x}"); - lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; - - lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); - - if top == lvl1_vaddr_top { - todo!( - "handle the case where top of lvl1 is occupied - this would be near the top of 512GiB" - ); - } - } - - // Invariant: Lower levels are empty. - assert!(lvl3_pt == &[0; _]); - // Invariant maintenance: vaddr_top is right range for current PT. - // it's empty so we need to increment the top to be current top (2MIB aligned) + 2MIB (512 lvl3 entries) - lvl3_vaddr_top = top + (1 << BLOCK_BITS_2MB); - } - 3 => { - // If it is a 4K page, it must go in the Level 3 PT; - // by our invariants: base < lvl3_vaddr_top and top <= lvl3_vaddr_top - assert!(base < lvl3_vaddr_top); - assert!(top <= lvl3_vaddr_top); - - assert!(lvl3_pt[lvl3_index(base)] == 0); - lvl3_pt[lvl3_index(base)] = page_descriptor(base, attr_index); - - if top == lvl3_vaddr_top { - // Invariant maintenance: keep for current address range. - // As we're the top of the range, we can serialise the table. - - let lvl3_pt_paddr = serialise_page_table_to_paddr(lvl3_pt); - println!( - "Serialise lvl3 table: {:#x} for to {:#x}..{lvl3_vaddr_top:#x}", - lvl3_pt_paddr as usize, - (lvl3_vaddr_top - (1 << BLOCK_BITS_2MB)) - ); - lvl3_vaddr_top += 1 << BLOCK_BITS_2MB; - - assert!(lvl2_pt[lvl2_index(base)] == 0); - lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); - - if top == lvl2_vaddr_top { - let lvl2_pt_paddr = serialise_page_table_to_paddr(lvl2_pt); - println!( - "Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}", - lvl2_pt_paddr as usize, - (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)) - ); - lvl2_vaddr_top += 1 << BLOCK_BITS_1GB; - - assert!(lvl1_pt[lvl1_index(base)] == 0); - lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); - - if top == lvl1_vaddr_top { - todo!( - "handle the case where top of lvl1 is occupied - this would be near the top of 512GiB" - ); - } - } - } - - // Invariant: lower levels empty is vacuuously true - } - _ => unreachable!("level is 1..=3"), + if changed { + println!("changed at {parent_level}, flushing {} into {parent_level}", parent_level + 1); + let pt_paddr = serialise_page_table_to_paddr(&mut pts_by_level[parent_level + 1]); + pts_by_level[parent_level][level_indices[parent_level]] = table_descriptor(pt_paddr); } - - base += pt_region_size; } } - // By the loop invariant, we know that anything before has been serialised. - // However, as we are at the end of the loop now, we might have - // page tables that have been partially filled out, and we need to - // serialise these. - - if lvl3_pt != &[0; _] { - let lvl3_pt_paddr = serialise_page_table_to_paddr(lvl3_pt); - println!("[end] Serialise lvl3 table: {:#x}", lvl3_pt_paddr as usize); - assert!(lvl2_pt[lvl2_index(base)] == 0); - lvl2_pt[lvl2_index(base)] = table_descriptor(lvl3_pt_paddr); - } - - if lvl2_pt != &[0; _] { - let lvl2_pt_paddr = serialise_page_table_to_paddr(lvl2_pt); - println!( - "[end] Serialise lvl2 table: {:#x} for to {:#x}..{lvl2_vaddr_top:#x}, base: {:#x} lvl1_index(base): {:#x}", - lvl2_pt_paddr as usize, - (lvl2_vaddr_top - (1 << BLOCK_BITS_1GB)), - base, - lvl1_index(base) - ); - assert!(lvl1_pt[lvl1_index(base)] == 0); - lvl1_pt[lvl1_index(base)] = table_descriptor(lvl2_pt_paddr); - } - - // the level1 pt should not be empty. lol. - assert!(lvl1_pt != &[0; _]); - - // println!("New lvl1 table"); - serialise_page_table_to_paddr(lvl1_pt) + serialise_page_table_to_paddr(&mut pt_temporaries[MIN_LEVEL]) }; struct Config { @@ -1312,6 +1094,7 @@ mod tests { p_end: 0x9001000, arch_value: 0x603, }, + // RAM WalkRegion { v_start: 0x60000000, v_end: 0xc0000000, @@ -1319,6 +1102,7 @@ mod tests { p_end: 0xc0000000, arch_value: 0x601, }, + // seL4 WalkRegion { v_start: 0x8060000000, v_end: 0x8080000000, From 0018db923c5ba410e84f713bca85c76ec14d8346 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Thu, 20 Aug 2026 15:52:30 +1000 Subject: [PATCH 25/29] truly works impl Signed-off-by: Julia Vassiliki --- loader/src/aligned_regions.rs | 32 ++++++++++++++++---------------- loader/src/page_tables.rs | 12 +++++------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/loader/src/aligned_regions.rs b/loader/src/aligned_regions.rs index 963b6592e..11bf9af91 100644 --- a/loader/src/aligned_regions.rs +++ b/loader/src/aligned_regions.rs @@ -92,7 +92,7 @@ impl Iterator for AlignedRegionsIter where I: Iterator, { - type Item = (usize, [usize; LEVELS]); + type Item = (usize, [usize; LEVELS], usize); fn next(&mut self) -> Option { let region = match self.current_input_region { @@ -151,7 +151,7 @@ where self.current_addr = next_addr; - Some((level, level_indices)) + Some((level, level_indices, current_addr)) } } @@ -224,10 +224,10 @@ mod tests { assert_eq!( indices, vec![ - (3, [0, 0, 0, 0]), - (3, [0, 0, 0, 1]), - (3, [0, 0, 0, 2]), - (3, [0, 0, 0, 3]), + (3, [0, 0, 0, 0], 0x0000), + (3, [0, 0, 0, 1], 0x1000), + (3, [0, 0, 0, 2], 0x2000), + (3, [0, 0, 0, 3], 0x3000), ] ); } @@ -257,16 +257,16 @@ mod tests { assert_eq!( indices, vec![ - (3, [0, 0, 0, 0]), - (3, [0, 0, 0, 1]), - (3, [0, 0, 0, 2]), - (3, [0, 0, 0, 3]), - (2, [0, 0, 1, 0]), - (2, [0, 0, 2, 0]), - (3, [0, 0, 3, 0]), - (3, [0, 0, 3, 1]), - (3, [0, 0, 3, 2]), - (3, [0, 0, 3, 3]), + (3, [0, 0, 0, 0], 0x0000), + (3, [0, 0, 0, 1], 0x1000), + (3, [0, 0, 0, 2], 0x2000), + (3, [0, 0, 0, 3], 0x3000), + (2, [0, 0, 1, 0], 0x200000), + (2, [0, 0, 2, 0], 0x400000), + (3, [0, 0, 3, 0], 0x600000), + (3, [0, 0, 3, 1], 0x601000), + (3, [0, 0, 3, 2], 0x602000), + (3, [0, 0, 3, 3], 0x603000), ] ); } diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index b6778a5c4..cc81cc84e 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -782,8 +782,6 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // once we have exceeded the bounds of the current reservation we // can simply push to the page_table_bytes storage and insert into // the parent PT the descriptor. - // When the current vaddr (/paddr, as identity mapped) exceeds the - // top value we rotate to a new PT. // We never actually use level 0 here, but it is nice to have because // then the indices are the same as the level. @@ -803,7 +801,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // RAM should never cross Level 0 boundaries, for the moment at least. const MIN_LEVEL: usize = 1; - while let Some((level, level_indices)) = iter.next() { + while let Some((level, level_indices, current_addr)) = iter.next() { // SAFETY: We went through and initialised raw before. let attr_index = MT_DEVICE_nGnRnE; // let attr_index = unsafe { region.arch_attrs.raw }; @@ -811,14 +809,14 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( assert!(level >= MIN_LEVEL); - let base = 0; + let current_addr: u64 = current_addr.try_into().unwrap(); assert!(pts_by_level[level][level_indices[level]] == 0); pts_by_level[level][level_indices[level]] = if level == 3 { - page_descriptor(base, attr_index) + page_descriptor(current_addr, attr_index) } else { - block_descriptor(level, base, attr_index) + block_descriptor(level, current_addr, attr_index) }; // Invariant: the page tables in pts_by_level are either: @@ -849,7 +847,7 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // or if the next one has different page tables to us. let changed = match iter.peek() { None => true, - Some((_, next_level_indices)) => { + Some((_, next_level_indices, _)) => { level_indices[0..=parent_level] != next_level_indices[0..=parent_level] } }; From 0799991be3c815ba2070e3d0bfadb4944a58c9b5 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Thu, 20 Aug 2026 16:27:24 +1000 Subject: [PATCH 26/29] cleanup Signed-off-by: Julia Vassiliki --- loader/src/aarch64/mmu.c | 8 ++-- loader/src/aligned_regions.rs | 78 +++++++++++++++++------------------ loader/src/page_tables.rs | 64 +++++++++++++--------------- 3 files changed, 70 insertions(+), 80 deletions(-) diff --git a/loader/src/aarch64/mmu.c b/loader/src/aarch64/mmu.c index 75ceba86b..be9ad317c 100644 --- a/loader/src/aarch64/mmu.c +++ b/loader/src/aarch64/mmu.c @@ -28,14 +28,14 @@ union RegionArchAttrs { }; struct Region { - uint64_t start; - uint64_t end; + uintptr_t start; + uintptr_t top; union RegionArchAttrs arch_attrs; }; struct Region regions[] = { - { .start = 0x60000000, .end = 0xc0000000, .arch_attrs.is_ram = true }, - { .start = 0x9000000, .end = 0x9000000 + 4096, .arch_attrs.is_ram = false }, + { .start = 0x60000000, .top = 0xc0000000 - 1, .arch_attrs.is_ram = true }, + { .start = 0x9000000, .top = 0x9000000 + 0xfff, .arch_attrs.is_ram = false }, }; #define PAGE_TABLE_SIZE 4096 diff --git a/loader/src/aligned_regions.rs b/loader/src/aligned_regions.rs index 11bf9af91..4405e85ac 100644 --- a/loader/src/aligned_regions.rs +++ b/loader/src/aligned_regions.rs @@ -14,16 +14,7 @@ use core::array; use core::cmp::min; -// Inclusive [start, top] -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd)] -pub struct Region { - pub start: usize, - pub top: usize, -} - -impl Region { - const EMPTY: Self = Self { start: 0, top: 0 }; -} +use crate::Region; /// Bits from [start, end) /// Note: Bit indices are 'u32' as this is what rust tends to use for usize::BITS, @@ -56,20 +47,20 @@ fn indices_of_level( } #[must_use = "iterators are lazy and do nothing unless consumed"] -pub struct AlignedRegionsIter +pub struct AlignedRegionsIter<'a, I, const LEVELS: usize> where - I: Iterator, + I: Iterator, { - /// Array of [size_bits, count_bits), descending, for each level + /// Array of (size_bits, count_bits), descending, for each level level_bits: [(u32, u32); LEVELS], input_regions_iter: I, - current_input_region: Option, + current_input_region: Option<&'a Region>, current_addr: usize, } -impl AlignedRegionsIter +impl<'a, I, const LEVELS: usize> AlignedRegionsIter<'a, I, LEVELS> where - I: Iterator, + I: Iterator, { // TODO: method on iter? pub fn new(iter: I, level_bits: [(u32, u32); LEVELS]) -> Self { @@ -88,11 +79,11 @@ where } } -impl Iterator for AlignedRegionsIter +impl<'a, I, const LEVELS: usize> Iterator for AlignedRegionsIter<'a, I, LEVELS> where - I: Iterator, + I: Iterator, { - type Item = (usize, [usize; LEVELS], usize); + type Item = (usize, [usize; LEVELS], usize, u64); fn next(&mut self) -> Option { let region = match self.current_input_region { @@ -103,7 +94,6 @@ where return None; }; - assert!(region != Region::EMPTY); assert!(region.start < region.top); self.current_input_region = Some(region); @@ -135,7 +125,7 @@ where let level = self .level_bits - .map(|(size, count)| size) + .map(|(size, _)| size) .iter() .position(|&level_size_bits| align_bits >= level_size_bits) .expect("bad input; regions should be aligned to at least the lowest level"); @@ -151,7 +141,10 @@ where self.current_addr = next_addr; - Some((level, level_indices, current_addr)) + // SAFETY: raw contains all valid bitpatterns + let raw_arch_attrs = unsafe { region.arch_attrs.raw }; + + Some((level, level_indices, current_addr, raw_arch_attrs)) } } @@ -163,6 +156,8 @@ mod tests { use std::vec; use std::vec::Vec; + use crate::RegionArchAttrs; + #[test] fn test_bits_range() { assert_eq!(bits_of_range(0b0110, 1, 3), 0b11); @@ -195,14 +190,14 @@ mod tests { #[test] #[should_panic] fn test_invalid_range_aligned_regions() { - let iter = [Region::EMPTY; 5].into_iter(); + let iter = [Region::EMPTY; 5].iter(); // Should be reverse AlignedRegionsIter::new(iter, [(8, 4), (12, 12), (24, 8), (32, 16)]); } #[test] fn test_ok_range_aligned_regions() { - let iter = [Region::EMPTY; 5].into_iter(); + let iter = [Region::EMPTY; 5].iter(); AlignedRegionsIter::new(iter, [(32, 16), (24, 8), (12, 12), (8, 4)]); } @@ -214,8 +209,9 @@ mod tests { [Region { start: 0, top: 0x3fff, + arch_attrs: RegionArchAttrs { is_ram: false }, }] - .into_iter(), + .iter(), aarch64_levels, ); @@ -224,10 +220,10 @@ mod tests { assert_eq!( indices, vec![ - (3, [0, 0, 0, 0], 0x0000), - (3, [0, 0, 0, 1], 0x1000), - (3, [0, 0, 0, 2], 0x2000), - (3, [0, 0, 0, 3], 0x3000), + (3, [0, 0, 0, 0], 0x0000, 0x0), + (3, [0, 0, 0, 1], 0x1000, 0x0), + (3, [0, 0, 0, 2], 0x2000, 0x0), + (3, [0, 0, 0, 3], 0x3000, 0x0), ] ); } @@ -241,14 +237,16 @@ mod tests { Region { start: 0, top: 0x3fff, + arch_attrs: RegionArchAttrs { is_ram: false }, }, // Then this will give us 2 2M region and then 4 4k regions Region { start: 0x200000, top: 0x603fff, + arch_attrs: RegionArchAttrs { is_ram: true }, }, ] - .into_iter(), + .iter(), aarch64_levels, ); @@ -257,16 +255,16 @@ mod tests { assert_eq!( indices, vec![ - (3, [0, 0, 0, 0], 0x0000), - (3, [0, 0, 0, 1], 0x1000), - (3, [0, 0, 0, 2], 0x2000), - (3, [0, 0, 0, 3], 0x3000), - (2, [0, 0, 1, 0], 0x200000), - (2, [0, 0, 2, 0], 0x400000), - (3, [0, 0, 3, 0], 0x600000), - (3, [0, 0, 3, 1], 0x601000), - (3, [0, 0, 3, 2], 0x602000), - (3, [0, 0, 3, 3], 0x603000), + (3, [0, 0, 0, 0], 0x0000, 0x0), + (3, [0, 0, 0, 1], 0x1000, 0x0), + (3, [0, 0, 0, 2], 0x2000, 0x0), + (3, [0, 0, 0, 3], 0x3000, 0x0), + (2, [0, 0, 1, 0], 0x200000, 0x1), + (2, [0, 0, 2, 0], 0x400000, 0x1), + (3, [0, 0, 3, 0], 0x600000, 0x1), + (3, [0, 0, 3, 1], 0x601000, 0x1), + (3, [0, 0, 3, 2], 0x602000, 0x1), + (3, [0, 0, 3, 3], 0x603000, 0x1), ] ); } diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index cc81cc84e..bb81176c1 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -11,7 +11,6 @@ mod aligned_regions; mod c_interop; -use core::cmp::min; use core::fmt; use core::mem; use core::mem::MaybeUninit; @@ -527,6 +526,7 @@ impl AArch64ReturnValue { const INVALID: *const u8 = usize::MAX as *const _; } +/// IMPORTANT: Keep in sync with C's `union RegionArchAttrs` #[derive(Copy, Clone)] #[repr(C)] pub union RegionArchAttrs { @@ -544,14 +544,23 @@ impl fmt::Debug for RegionArchAttrs { } /// Region is [start, end] *inclusive* as this avoids overflows. +/// IMPORTANT: Keep in sync with C's `struct Region` #[derive(Debug, Copy, Clone)] #[repr(C)] pub struct Region { - pub start: u64, - pub end: u64, + pub start: usize, + pub top: usize, pub arch_attrs: RegionArchAttrs, } +impl Region { + pub const EMPTY: Self = Self { + start: 0, + top: 0, + arch_attrs: RegionArchAttrs { raw: 0 }, + }; +} + pub const MAX_NUM_PAGE_TABLES: usize = 64; /// AArch64 loader page tables have two variations: @@ -764,19 +773,6 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // We assume that normal RAM lies between 0 <= paddr < 512GiB, i.e. // that lvl0_index(any ram region addr) = 0. let ram_lvl1_pt_paddr = { - // Validation of assumptions about the identity mapped regions. - let mut previous_end = None; - for region in identity_mapped_regions.iter() { - assert!(lvl0_index(region.start) == 0); - assert!(lvl0_index(region.end) == 0); - // This is probably an unnecessary assumption. - assert!(region.start.is_multiple_of(4096)); - assert!(region.end.is_multiple_of(4096)); - // This is definitely necessary. - assert!(region.start >= previous_end.unwrap_or(0)); - previous_end = Some(region.end); - } - // We maintain three active page tables, which contain our previous // known page table data. As we process regions in ascending order, // once we have exceeded the bounds of the current reservation we @@ -785,15 +781,10 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // We never actually use level 0 here, but it is nice to have because // then the indices are the same as the level. - let mut pts_by_level = pt_temporaries.get_disjoint_mut([0, 1, 2, 3]).unwrap(); + let pts_by_level = pt_temporaries.get_disjoint_mut([0, 1, 2, 3]).unwrap(); let mut iter = AlignedRegionsIter::new( - identity_mapped_regions - .iter() - .map(|r| aligned_regions::Region { - start: r.start as usize, - top: (r.end - 1) as usize, - }), + identity_mapped_regions.iter(), [(39, 9), (30, 9), (21, 9), (12, 9)], ) .peekable(); @@ -801,11 +792,8 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // RAM should never cross Level 0 boundaries, for the moment at least. const MIN_LEVEL: usize = 1; - while let Some((level, level_indices, current_addr)) = iter.next() { - // SAFETY: We went through and initialised raw before. - let attr_index = MT_DEVICE_nGnRnE; - // let attr_index = unsafe { region.arch_attrs.raw }; - println!("{level:x} {level_indices:x?}"); + while let Some((level, level_indices, current_addr, attr_index)) = iter.next() { + // println!("{level:x} {level_indices:x?}"); assert!(level >= MIN_LEVEL); @@ -841,21 +829,25 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( // We don't need to care if next_level is higher than the current // level, as this still means the current page table is valid. - for parent_level in (MIN_LEVEL..level).rev() { + for level in (MIN_LEVEL..level).rev() { // Two cases where we need to write out the page tables: // either we are reaching the end (iter.peek() = None) // or if the next one has different page tables to us. let changed = match iter.peek() { None => true, - Some((_, next_level_indices, _)) => { - level_indices[0..=parent_level] != next_level_indices[0..=parent_level] + Some((_, next_level_indices, _, _)) => { + level_indices[0..=level] != next_level_indices[0..=level] } }; + // Flush the 'level + 1' (the entry in the current level's PT) + // into the 'level' PT (next level up) + // We could have written instead this for loop as + // `for level in (MIN_LEVEL+1..=level)` + // and then used `let parent_level = level - 1`. if changed { - println!("changed at {parent_level}, flushing {} into {parent_level}", parent_level + 1); - let pt_paddr = serialise_page_table_to_paddr(&mut pts_by_level[parent_level + 1]); - pts_by_level[parent_level][level_indices[parent_level]] = table_descriptor(pt_paddr); + let pt_paddr = serialise_page_table_to_paddr(pts_by_level[level + 1]); + pts_by_level[level][level_indices[level]] = table_descriptor(pt_paddr); } } } @@ -1017,13 +1009,13 @@ mod tests { let mut regions = [ Region { start: 0x60000000, - end: 0xc0000000, + top: 0xc0000000 - 1, arch_attrs: RegionArchAttrs { is_ram: true }, }, // UART Region { start: 0x9000000, - end: 0x9001000, + top: 0x9000fff, arch_attrs: RegionArchAttrs { is_ram: false }, }, ]; From 871db3a201463e3e340dbe7563b4397bb14bc67d Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Thu, 20 Aug 2026 16:30:46 +1000 Subject: [PATCH 27/29] reomve prints Signed-off-by: Julia Vassiliki --- loader/src/page_tables.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index bb81176c1..52e26d6e0 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -17,7 +17,6 @@ use core::mem::MaybeUninit; use core::slice; use aligned_regions::AlignedRegionsIter; -use c_interop::println; const PAGE_TABLE_SIZE: usize = 4096; @@ -640,9 +639,9 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], ) -> AArch64ReturnValue { use aarch64::{ - block_descriptor, lvl0_index, lvl1_index, lvl2_index, lvl3_index, page_descriptor, + block_descriptor, lvl0_index, lvl1_index, lvl2_index, page_descriptor, s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}, - table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, PAGE_BITS_4KB, + table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, }; const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); @@ -720,8 +719,6 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( let identity_mapped_regions: &mut [Region] = { let regions = unsafe { slice::from_raw_parts_mut(regions_ptr, regions_len) }; - println!("{:#x?}", regions); - for region in regions.iter_mut() { // SAFETY: We expect users to set is_ram appropriately. region.arch_attrs.raw = if unsafe { region.arch_attrs.is_ram } { @@ -793,7 +790,6 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( const MIN_LEVEL: usize = 1; while let Some((level, level_indices, current_addr, attr_index)) = iter.next() { - // println!("{level:x} {level_indices:x?}"); assert!(level >= MIN_LEVEL); @@ -959,7 +955,6 @@ mod tests { if level != 3 && pte_type == descriptor_type::TABLE { let next_level_pt_idx = (pte_oa as usize) / PAGE_TABLE_SIZE; - // println!("oa: {pte_oa:x}, next_idx: {next_level_pt_idx}"); let mut vaddr = v_start; for &child_pte in pts[next_level_pt_idx].iter() { @@ -997,7 +992,6 @@ mod tests { i -= 1; } - println!("{regions:#x?}"); regions } From cc6be778c6f3155543dc8a8a4f46d1d0dc4d3d99 Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Thu, 20 Aug 2026 17:16:51 +1000 Subject: [PATCH 28/29] more helper fns Signed-off-by: Julia Vassiliki --- loader/src/page_tables.rs | 394 ++++++++++++++++++++++---------------- 1 file changed, 225 insertions(+), 169 deletions(-) diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index 52e26d6e0..b5ff6ae4a 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -560,7 +560,207 @@ impl Region { }; } -pub const MAX_NUM_PAGE_TABLES: usize = 64; +const MAX_NUM_PAGE_TABLES: usize = 64; +const NUM_TEMPORARIES: usize = 4; + +pub trait ArchPtLayout { + const LEVELS: usize = LEVELS; + + const MIN_LEVEL: usize; + const LEVEL_BITS: [(u32, u32); LEVELS]; + + fn leaf_entry(level: usize, address: usize, attributes: u64) -> u64; + fn table_entry(level: usize, address: *const u8) -> u64; +} + +struct AArch64PtLayout; + +impl ArchPtLayout<4> for AArch64PtLayout { + const MIN_LEVEL: usize = 1; + const LEVEL_BITS: [(u32, u32); 4] = [(39, 9), (30, 9), (21, 9), (12, 9)]; + + fn leaf_entry(level: usize, address: usize, attributes: u64) -> u64 { + assert!(level < Self::LEVELS); + + let address = address.try_into().unwrap(); + + if level == 3 { + aarch64::page_descriptor(address, attributes) + } else { + aarch64::block_descriptor(level, address, attributes) + } + } + + fn table_entry(_level: usize, address: *const u8) -> u64 { + aarch64::table_descriptor(address) + } +} + +fn setup_identity_page_tables< + const LEVELS: usize, + const PAGE_TABLE_ENTRIES: usize, + LAYOUT, + SerialiseFn, +>( + identity_mapped_regions: &mut [Region], + pt_temporaries: &mut [[u64; PAGE_TABLE_ENTRIES]; NUM_TEMPORARIES], + mut serialise_page_table_to_paddr: SerialiseFn, +) -> *const u8 +where + SerialiseFn: FnMut(&mut [u64; PAGE_TABLE_ENTRIES]) -> *const u8, + LAYOUT: ArchPtLayout, +{ + // Manufacture the RAM page tables, which is a little bit more complicated. + + // We maintain three active page tables, which contain our previous + // known page table data. As we process regions in ascending order, + // once we have exceeded the bounds of the current reservation we + // can simply push to the page_table_bytes storage and insert into + // the parent PT the descriptor. + + // We never actually use level 0 here, but it is nice to have because + // then the indices are the same as the level. + let pts_by_level = pt_temporaries.get_disjoint_mut([0, 1, 2, 3]).unwrap(); + + let mut iter = + AlignedRegionsIter::new(identity_mapped_regions.iter(), LAYOUT::LEVEL_BITS).peekable(); + + // RAM should never cross Level 0 boundaries, for the moment at least. + const MIN_LEVEL: usize = 1; + + while let Some((level, level_indices, current_addr, attributes)) = iter.next() { + assert!(level >= MIN_LEVEL); + + assert!(pts_by_level[level][level_indices[level]] == 0); + + pts_by_level[level][level_indices[level]] = + LAYOUT::leaf_entry(level, current_addr, attributes); + + // Invariant: the page tables in pts_by_level are either: + // (1) for the current level_indices, or + // (2) are empty/invalid and for a lower level. + // Similar, the level indices in our array are only meaningful + // from [0..=level]. + // + // Hence, when moving around, we only need to care about page tables + // in the range [0, level) inclusive, and can ignore those on + // lower levels. + // We start from the lowest level (parent) checking if the indices + // prefix (i.e. it, or any above it) have changed. Note that + // checking just the index would be invalid, in the case of say a + // [0, 0, 1, 0] -> [0, 0, 2, 0] change where level=3, as the + // level=2 row has changed, so our level=3 page table must be + // written out. + // We start from the parent and not the current level, because + // the change from [0, 0, 1, 0] -> [0, 0, 1, 1] should not write + // out the page table. (similarly, [0, 0, 1, X] -> [0, 0, 1, X] + // for level=2). + // We don't need to care if next_level is higher than the current + // level, as this still means the current page table is valid. + + for level in (MIN_LEVEL..level).rev() { + // Two cases where we need to write out the page tables: + // either we are reaching the end (iter.peek() = None) + // or if the next one has different page tables to us. + let changed = match iter.peek() { + None => true, + Some((_, next_level_indices, _, _)) => { + level_indices[0..=level] != next_level_indices[0..=level] + } + }; + + // Flush the 'level + 1' (the entry in the current level's PT) + // into the 'level' PT (next level up) + // We could have written instead this for loop as + // `for level in (MIN_LEVEL+1..=level)` + // and then used `let parent_level = level - 1`. + if changed { + let pt_paddr = serialise_page_table_to_paddr(pts_by_level[level + 1]); + pts_by_level[level][level_indices[level]] = LAYOUT::table_entry(level, pt_paddr); + } + } + } + + serialise_page_table_to_paddr(&mut pt_temporaries[MIN_LEVEL]) +} + +fn make_helper_pt_serialisers( + page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], +) -> ( + &mut [[u64; PAGE_TABLE_ENTRIES]; NUM_TEMPORARIES], + impl FnMut(&mut [u64; PAGE_TABLE_ENTRIES]) -> *const u8, +) { + // FIXME: Replace once https://github.com/rust-lang/rust/issues/90091 is merged + let (page_table_bytes, pt_temporaries) = page_table_bytes + .split_first_chunk_mut::<{ MAX_NUM_PAGE_TABLES - NUM_TEMPORARIES }>() + .unwrap(); + + let pt_temporaries = { + let pt_temporaries: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; NUM_TEMPORARIES] = + pt_temporaries.try_into().unwrap(); + + for pt in pt_temporaries.iter_mut() { + for elem in pt { + elem.write(0); + } + } + + // SAFETY: we just initialised it. + let pt_temporaries = unsafe { + mem::transmute::< + &mut [[MaybeUninit; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], + &mut [[u8; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], + >(pt_temporaries) + }; + + // SAFETY: + // - all bitpatterns of u8 can be represented in u8. + // - alignment requirements are met by input requirements + unsafe { + assert!((pt_temporaries.as_ptr() as usize).is_multiple_of(PAGE_TABLE_SIZE)); + mem::transmute::< + &mut [[u8; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], + &mut [[u64; PAGE_TABLE_ENTRIES]; NUM_TEMPORARIES], + >(pt_temporaries) + } + }; + + let serialise_page_table_to_paddr = { + let page_tables_paddr_start: *const u8 = page_table_bytes.as_ptr().cast(); + + assert!((page_tables_paddr_start as usize).is_multiple_of(PAGE_TABLE_SIZE)); + + // This maintains the current end of the PT array. + let mut next_pt_paddr = page_tables_paddr_start; + let mut i = 0; + + move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> *const _ { + let pt_paddr = next_pt_paddr; + for (j, byte) in page_table + .iter() + .flat_map(|pte| pte.to_le_bytes()) + .enumerate() + { + page_table_bytes[i][j].write(byte); + } + + next_pt_paddr = next_pt_paddr.wrapping_add(PAGE_TABLE_SIZE); + i += 1; + page_table.fill(0); + + if cfg!(test) { + // HACK! For tests, we want stable page tables, but due to ASLR + // we get random things every time. Instead, let's make the + // paddr we return a relative-to-start-of-page-tables value. + return unsafe { pt_paddr.offset_from(page_tables_paddr_start) } as *const _; + } + + pt_paddr + } + }; + + (pt_temporaries, serialise_page_table_to_paddr) +} /// AArch64 loader page tables have two variations: /// - Loader in EL2, then Stage 1 translations in use, so we have the @@ -639,101 +839,15 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], ) -> AArch64ReturnValue { use aarch64::{ - block_descriptor, lvl0_index, lvl1_index, lvl2_index, page_descriptor, + block_descriptor, lvl0_index, lvl1_index, lvl2_index, s1_mair_attr_index::{MT_DEVICE_nGnRnE, MT_NORMAL}, table_descriptor, BLOCK_BITS_1GB, BLOCK_BITS_2MB, BLOCK_BITS_512GB, }; const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); - const NUM_TEMPORARIES: usize = 4; - // FIXME: Replace once https://github.com/rust-lang/rust/issues/90091 is merged - let (page_table_bytes, pt_temporaries) = page_table_bytes - .split_first_chunk_mut::<{ MAX_NUM_PAGE_TABLES - NUM_TEMPORARIES }>() - .unwrap(); - - let pt_temporaries = { - let pt_temporaries: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; NUM_TEMPORARIES] = - pt_temporaries.try_into().unwrap(); - - for pt in pt_temporaries.iter_mut() { - for elem in pt { - elem.write(0); - } - } - - // SAFETY: we just initialised it. - let pt_temporaries = unsafe { - mem::transmute::< - &mut [[MaybeUninit; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], - &mut [[u8; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], - >(pt_temporaries) - }; - - // SAFETY: - // - all bitpatterns of u8 can be represented in u8. - // - alignment requirements are met by input requirements - unsafe { - assert!((pt_temporaries.as_ptr() as usize).is_multiple_of(PAGE_TABLE_SIZE)); - mem::transmute::< - &mut [[u8; PAGE_TABLE_SIZE]; NUM_TEMPORARIES], - &mut [[u64; PAGE_TABLE_ENTRIES]; NUM_TEMPORARIES], - >(pt_temporaries) - } - }; - - let mut serialise_page_table_to_paddr = { - let page_tables_paddr_start: *const u8 = page_table_bytes.as_ptr().cast(); - - assert!((page_tables_paddr_start as usize).is_multiple_of(PAGE_TABLE_SIZE)); - - // This maintains the current end of the PT array. - let mut next_pt_paddr = page_tables_paddr_start; - let mut i = 0; - - move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> *const _ { - let pt_paddr = next_pt_paddr; - for (j, byte) in page_table - .iter() - .flat_map(|pte| pte.to_le_bytes()) - .enumerate() - { - page_table_bytes[i][j].write(byte); - } - - next_pt_paddr = next_pt_paddr.wrapping_add(PAGE_TABLE_SIZE); - i += 1; - page_table.fill(0); - - if cfg!(test) { - // HACK! For tests, we want stable page tables, but due to ASLR - // we get random things every time. Instead, let's make the - // paddr we return a relative-to-start-of-page-tables value. - return unsafe { pt_paddr.offset_from(page_tables_paddr_start) } as *const _; - } - - pt_paddr - } - }; - - let identity_mapped_regions: &mut [Region] = { - let regions = unsafe { slice::from_raw_parts_mut(regions_ptr, regions_len) }; - - for region in regions.iter_mut() { - // SAFETY: We expect users to set is_ram appropriately. - region.arch_attrs.raw = if unsafe { region.arch_attrs.is_ram } { - // FIXME: For now, RAM is also mapped as DEVICE memory. - MT_DEVICE_nGnRnE - } else { - MT_DEVICE_nGnRnE - }; - } - - // Need to use 'sort_unstable_by_key' as sort_by_key is not in-place. - regions.sort_unstable_by_key(|region| region.start); - - regions - }; + let (pt_temporaries, mut serialise_page_table_to_paddr) = + make_helper_pt_serialisers::(page_table_bytes); // Manufacture the constants as per the diagram. let k = align_down(kernel_first_vaddr, BLOCK_BITS_512GB); @@ -766,89 +880,31 @@ pub unsafe extern "C" fn aarch64_setup_pagetables( serialise_page_table_to_paddr(lvl1_pt_kernel) }; - // Manufacture the RAM page tables, which is a little bit more complicated. - // We assume that normal RAM lies between 0 <= paddr < 512GiB, i.e. - // that lvl0_index(any ram region addr) = 0. let ram_lvl1_pt_paddr = { - // We maintain three active page tables, which contain our previous - // known page table data. As we process regions in ascending order, - // once we have exceeded the bounds of the current reservation we - // can simply push to the page_table_bytes storage and insert into - // the parent PT the descriptor. - - // We never actually use level 0 here, but it is nice to have because - // then the indices are the same as the level. - let pts_by_level = pt_temporaries.get_disjoint_mut([0, 1, 2, 3]).unwrap(); - - let mut iter = AlignedRegionsIter::new( - identity_mapped_regions.iter(), - [(39, 9), (30, 9), (21, 9), (12, 9)], - ) - .peekable(); - - // RAM should never cross Level 0 boundaries, for the moment at least. - const MIN_LEVEL: usize = 1; - - while let Some((level, level_indices, current_addr, attr_index)) = iter.next() { - - assert!(level >= MIN_LEVEL); - - let current_addr: u64 = current_addr.try_into().unwrap(); - - assert!(pts_by_level[level][level_indices[level]] == 0); - - pts_by_level[level][level_indices[level]] = if level == 3 { - page_descriptor(current_addr, attr_index) - } else { - block_descriptor(level, current_addr, attr_index) - }; - - // Invariant: the page tables in pts_by_level are either: - // (1) for the current level_indices, or - // (2) are empty/invalid and for a lower level. - // Similar, the level indices in our array are only meaningful - // from [0..=level]. - // - // Hence, when moving around, we only need to care about page tables - // in the range [0, level) inclusive, and can ignore those on - // lower levels. - // We start from the lowest level (parent) checking if the indices - // prefix (i.e. it, or any above it) have changed. Note that - // checking just the index would be invalid, in the case of say a - // [0, 0, 1, 0] -> [0, 0, 2, 0] change where level=3, as the - // level=2 row has changed, so our level=3 page table must be - // written out. - // We start from the parent and not the current level, because - // the change from [0, 0, 1, 0] -> [0, 0, 1, 1] should not write - // out the page table. (similarly, [0, 0, 1, X] -> [0, 0, 1, X] - // for level=2). - // We don't need to care if next_level is higher than the current - // level, as this still means the current page table is valid. - - for level in (MIN_LEVEL..level).rev() { - // Two cases where we need to write out the page tables: - // either we are reaching the end (iter.peek() = None) - // or if the next one has different page tables to us. - let changed = match iter.peek() { - None => true, - Some((_, next_level_indices, _, _)) => { - level_indices[0..=level] != next_level_indices[0..=level] - } + let identity_mapped_regions: &mut [Region] = { + let regions = unsafe { slice::from_raw_parts_mut(regions_ptr, regions_len) }; + + for region in regions.iter_mut() { + // SAFETY: We expect users to set is_ram appropriately. + region.arch_attrs.raw = if unsafe { region.arch_attrs.is_ram } { + // FIXME: For now, RAM is also mapped as DEVICE memory. + MT_DEVICE_nGnRnE + } else { + MT_DEVICE_nGnRnE }; - - // Flush the 'level + 1' (the entry in the current level's PT) - // into the 'level' PT (next level up) - // We could have written instead this for loop as - // `for level in (MIN_LEVEL+1..=level)` - // and then used `let parent_level = level - 1`. - if changed { - let pt_paddr = serialise_page_table_to_paddr(pts_by_level[level + 1]); - pts_by_level[level][level_indices[level]] = table_descriptor(pt_paddr); - } } - } - serialise_page_table_to_paddr(&mut pt_temporaries[MIN_LEVEL]) + // Need to use 'sort_unstable_by_key' as sort_by_key is not in-place. + regions.sort_unstable_by_key(|region| region.start); + + regions + }; + + setup_identity_page_tables::<4, _, AArch64PtLayout, _>( + identity_mapped_regions, + pt_temporaries, + &mut serialise_page_table_to_paddr, + ) }; struct Config { From a0bbb47ffdeb60e6affd350221f2d5f0d641004f Mon Sep 17 00:00:00 2001 From: Julia Vassiliki Date: Thu, 20 Aug 2026 18:08:42 +1000 Subject: [PATCH 29/29] attempt (badly) to make risv64 also work Signed-off-by: Julia Vassiliki --- example/hello/Makefile | 2 +- loader/Makefile | 6 +- loader/aarch64.ld | 2 - loader/riscv64.ld | 19 +++--- loader/src/aarch64/mmu.c | 22 ++----- loader/src/arch.h | 17 +++++ loader/src/page_tables.rs | 128 ++++++++++++++++++-------------------- loader/src/riscv/mmu.c | 19 ++++++ 8 files changed, 115 insertions(+), 100 deletions(-) diff --git a/example/hello/Makefile b/example/hello/Makefile index e33ce05eb..b32c24f51 100644 --- a/example/hello/Makefile +++ b/example/hello/Makefile @@ -27,7 +27,7 @@ ifeq ($(ARCH),aarch64) TARGET_TRIPLE := aarch64-none-elf CFLAGS_ARCH := -mstrict-align else ifeq ($(ARCH),riscv64) - TARGET_TRIPLE := riscv64-unknown-elf + TARGET_TRIPLE := riscv64-none-elf CFLAGS_ARCH := -march=rv64imafdc_zicsr_zifencei -mabi=lp64d else ifeq ($(ARCH),x86_64) TARGET_TRIPLE := x86_64-linux-gnu diff --git a/loader/Makefile b/loader/Makefile index 9668f7005..a8a9ca943 100644 --- a/loader/Makefile +++ b/loader/Makefile @@ -55,9 +55,9 @@ ifeq ($(ARCH),aarch64) ARCH_DIR := aarch64 RUST_TARGET_TRIPLE := aarch64-unknown-none else ifeq ($(ARCH),riscv64) - CFLAGS_RISCV64 := -mcmodel=medany -march=rv64imac_zicsr_zifencei -mabi=lp64 + CFLAGS_RISCV64 := -mcmodel=medany -march=rv64gc -mabi=lp64d CFLAGS_ARCH := $(CFLAGS_RISCV64) -DARCH_riscv64 - ASM_FLAGS_ARCH := -march=rv64imac_zicsr_zifencei -mabi=lp64 + ASM_FLAGS_ARCH := -march=rv64gc -mabi=lp64d RUST_TARGET_TRIPLE := riscv64gc-unknown-none-elf ARCH_DIR := riscv endif @@ -122,7 +122,7 @@ all: $(OBJPROG) $(LINKSCRIPT): $(LINKSCRIPT_INPUT) $(CPP) -DLINK_ADDRESS=$(LINK_ADDRESS) $< | grep -v "^#" > $@ -LDFLAGS := -T$(LINKSCRIPT) --gc-sections +LDFLAGS := -T$(LINKSCRIPT) $(OBJPROG): $(addprefix $(BUILD_DIR)/, $(OBJECTS)) $(LINKSCRIPT) $(LD) $(LDFLAGS) --start-group $(addprefix $(BUILD_DIR)/, $(OBJECTS)) --end-group -o $@ diff --git a/loader/aarch64.ld b/loader/aarch64.ld index 2454398a3..b45561a04 100644 --- a/loader/aarch64.ld +++ b/loader/aarch64.ld @@ -56,7 +56,5 @@ SECTIONS _bss_end = .; } :all - - _loader_end = .; } diff --git a/loader/riscv64.ld b/loader/riscv64.ld index fe98d7450..98c94c490 100644 --- a/loader/riscv64.ld +++ b/loader/riscv64.ld @@ -15,30 +15,35 @@ SECTIONS .text : { _text = .; + KEEP(*(.text.start)) - *(.text*) - *(.text.*) - *(.rodata) - *(.rodata.*) + *(.text .text.*) + _text_end = .; } :all + .rodata : + { + *(.rodata .rodata.* .rodata..Lanon.*) + } :all + .data : { _data = .; - *(.data) + *(.data .data.*) *(.data.*) __global_pointer$ = . + 0x800; *(.srodata) *(.sdata) + KEEP(*(.data.uart_addr)) - _data_end = .; + + _data_end = .; } :all .bss : { _bss = .; - *(.sbss) *(.bss) *(.bss.*) *(COMMON) diff --git a/loader/src/aarch64/mmu.c b/loader/src/aarch64/mmu.c index be9ad317c..95ba14124 100644 --- a/loader/src/aarch64/mmu.c +++ b/loader/src/aarch64/mmu.c @@ -13,8 +13,8 @@ #include "../cutil.h" #include "../uart.h" -void el1_mmu_enable(uint64_t ttbr0_el1, uint64_t ttbr1_el1); -void el2_mmu_enable(uint64_t ttbr0_el2); +void el1_mmu_enable(uintptr_t ttbr0_el1, uintptr_t ttbr1_el1); +void el2_mmu_enable(uintptr_t ttbr0_el2); struct AArch64ReturnValue { uintptr_t ttbr0_el2; @@ -22,31 +22,17 @@ struct AArch64ReturnValue { uintptr_t ttbr1_el1; }; -union RegionArchAttrs { - bool is_ram; - uint64_t raw; -}; - -struct Region { - uintptr_t start; - uintptr_t top; - union RegionArchAttrs arch_attrs; -}; - struct Region regions[] = { { .start = 0x60000000, .top = 0xc0000000 - 1, .arch_attrs.is_ram = true }, { .start = 0x9000000, .top = 0x9000000 + 0xfff, .arch_attrs.is_ram = false }, }; -#define PAGE_TABLE_SIZE 4096 -#define MAX_NUM_PAGE_TABLES 64 - -uint8_t page_table_bytes[PAGE_TABLE_SIZE][MAX_NUM_PAGE_TABLES] ALIGN(4096); +uint8_t page_table_bytes[PAGE_TABLE_SIZE][MAX_NUM_PAGE_TABLES] ALIGN(PAGE_TABLE_SIZE); extern struct AArch64ReturnValue aarch64_setup_pagetables( uint64_t kernel_first_vaddr, uint64_t kernel_first_paddr, void *regions_ptr, uintptr_t regions_len, - uint8_t page_table_bytes[4096][64]); + uint8_t page_table_bytes[PAGE_TABLE_SIZE][MAX_NUM_PAGE_TABLES]); int arch_mmu_enable(int logical_cpu) { diff --git a/loader/src/arch.h b/loader/src/arch.h index 69b4ba148..bb83b4ade 100644 --- a/loader/src/arch.h +++ b/loader/src/arch.h @@ -6,6 +6,9 @@ #pragma once +#include +#include + /** * The layout and naming scheme of the functions in these files has meaning: * @@ -35,3 +38,17 @@ void arch_init(void); void arch_set_exception_handler(void); int arch_mmu_enable(int logical_cpu); void arch_jump_to_kernel(int logical_cpu); + +union RegionArchAttrs { + bool is_ram; + uint64_t raw; +}; + +struct Region { + uintptr_t start; + uintptr_t top; + union RegionArchAttrs arch_attrs; +}; + +#define PAGE_TABLE_SIZE 4096 +#define MAX_NUM_PAGE_TABLES 64 diff --git a/loader/src/page_tables.rs b/loader/src/page_tables.rs index b5ff6ae4a..43433084a 100644 --- a/loader/src/page_tables.rs +++ b/loader/src/page_tables.rs @@ -41,10 +41,6 @@ const fn align_down(n: u64, bits: u64) -> u64 { round_down(n, 1 << bits) } -unsafe extern "C" { - static mut _text: u8; -} - pub mod aarch64 { //! For AArch64, our page tables use the Stage 1 descriptor formats //! for both EL2 (TTBR0_EL2) and EL1 (TTBR0_EL1/TTBR1_EL1). @@ -316,8 +312,8 @@ mod riscv64 { (addr >> PAGE_SHIFT) << PTE_PPN0_SHIFT } - pub fn pte_next(addr: u64) -> u64 { - pte_ppn(addr) | PTE_TYPE_TABLE | PTE_TYPE_VALID + pub fn pte_next(addr: *const u8) -> u64 { + pte_ppn(addr as u64) | PTE_TYPE_TABLE | PTE_TYPE_VALID } pub fn pte_leaf(addr: u64) -> u64 { @@ -369,26 +365,9 @@ mod riscv64 { /// | | /// | | /// | | -/// s+1 +----------------+ (1 GiB) -/// | Level 2 Loader | ----------> +-- Level 2 --+ +-------------+ -/// s +----------------+ | | ----------> | 2 MiB block | -/// | | 511 +-------------+ +-------------+ -/// | | | | ----------> | 2 MiB block | -/// | (empty) | 510 +-------------+ +-------------+ -/// | | | | ----------> | 2 MiB block | -/// | | |-------------| +-------------+ -/// 0 +----------------+ | | ----------> | 2 MiB block | -/// |-------------| +-------------+ -/// (...) (...) (...) Loader Regions -/// |-------------| +-------------+ -/// | | ----------> | 2 MiB block | -/// |-------------| +-------------+ -/// | | ----------> | 2 MiB block | -/// t +-------------+ +-------------+ -/// | | -/// | (empty) | -/// | | -/// +-------------+ +/// 1 +----------------+ +/// | Level 2 Loader | ----------> +-- RAM +/// 0 +----------------+ /// /// /// Where: @@ -396,43 +375,30 @@ mod riscv64 { /// l = align_down(kernel_first_vaddr, 2MiB), /// m = align_down(kernel_first_vaddr, 4KiB), /// p = align_down(kernel_first_paddr, 4KiB), -/// -/// s = align_down(text_addr, 1GiB), -/// t = align_down(text_addr, 2MiB), /// ``` /// +/// # Safety +/// - regions_ptr must be valid for as long as this function runs, +/// and regions_len must represent its length +/// - page_table_bytes must be aligned to PAGE_TABLE_SIZE +/// +/// #[unsafe(no_mangle)] -pub extern "C" fn riscv64_setup_pagetables( +pub unsafe extern "C" fn riscv64_setup_pagetables( kernel_first_vaddr: u64, kernel_first_paddr: u64, - page_tables_paddr_start: u64, -) -> u64 { + // In-out param; storage and input + regions_ptr: *mut Region, + regions_len: usize, + // Storage used for page tables + page_table_bytes: &mut [[MaybeUninit; PAGE_TABLE_SIZE]; MAX_NUM_PAGE_TABLES], +) -> *const u8 { use riscv64::{pt_index, pte_leaf, pte_next, BLOCK_BITS_1GB, BLOCK_BITS_2MB, PAGE_BITS_4K}; - let text_addr = &raw const _text as u64; - - // We map the loader using 2MB pages, so make sure the base is actually aligned. - assert!(text_addr.is_multiple_of(1 << BLOCK_BITS_2MB)); - const PAGE_TABLE_ENTRIES: usize = PAGE_TABLE_SIZE / mem::size_of::(); - let mut serialise_page_table_to_paddr = { - assert!( - page_tables_paddr_start - == page_tables_paddr_start.next_multiple_of(PAGE_TABLE_SIZE as u64) - ); - - // This maintains the current end of the PT array. - let mut next_pt_paddr = page_tables_paddr_start; - - move |page_table: &mut [u64; PAGE_TABLE_ENTRIES]| -> u64 { - let pt_paddr = next_pt_paddr; - // page_table_bytes.extend(page_table.iter().flat_map(|pte| pte.to_le_bytes())); - next_pt_paddr += PAGE_TABLE_SIZE as u64; - page_table.fill(0); - pt_paddr - } - }; + let (pt_temporaries, mut serialise_page_table_to_paddr) = + make_helper_pt_serialisers::(page_table_bytes); struct Config { riscv_pt_levels: usize, @@ -448,9 +414,6 @@ pub extern "C" fn riscv64_setup_pagetables( let m = align_down(kernel_first_vaddr, PAGE_BITS_4K); let p = align_down(kernel_first_paddr, PAGE_BITS_4K); - let s = align_down(text_addr, BLOCK_BITS_1GB); - let t = align_down(text_addr, BLOCK_BITS_2MB); - // Manufacture the kernel page tables let kernel_lvl2_pt_paddr = { let mut lvl2_pt_kernel = [0u64; PAGE_TABLE_ENTRIES]; @@ -485,28 +448,35 @@ pub extern "C" fn riscv64_setup_pagetables( serialise_page_table_to_paddr(&mut lvl2_pt_kernel) }; - // Manufacture the loader page tables, which is relatively straightforward - let loader_lvl2_pt_paddr = { - let mut lvl2_pt_loader = [0u64; PAGE_TABLE_ENTRIES]; + let ram_lvl2_pt_paddr = { + let identity_mapped_regions: &mut [Region] = { + let regions = unsafe { slice::from_raw_parts_mut(regions_ptr, regions_len) }; - // Identity mapped, so vaddr == paddr. - let mut paddr = t; + for region in regions.iter_mut() { + // RISC-V ignores attributes + region.arch_attrs.raw = 0; + } - for index in pt_index(num_pt_levels, t, 2)..512 { - lvl2_pt_loader[index] = pte_leaf(paddr); - paddr += 1 << BLOCK_BITS_2MB; - } + // Need to use 'sort_unstable_by_key' as sort_by_key is not in-place. + regions.sort_unstable_by_key(|region| region.start); - serialise_page_table_to_paddr(&mut lvl2_pt_loader) + regions + }; + + setup_identity_page_tables::<3, _, Riscv64PtLayout, _>( + identity_mapped_regions, + pt_temporaries, + &mut serialise_page_table_to_paddr, + ) }; // Manufacture the Level 1 table let mut boot_lvl1_pt = [0u64; PAGE_TABLE_ENTRIES]; - let index_s = pt_index(num_pt_levels, s, 1); let index_k = pt_index(num_pt_levels, k, 1); boot_lvl1_pt[index_k] = pte_next(kernel_lvl2_pt_paddr); - boot_lvl1_pt[index_s] = pte_next(loader_lvl2_pt_paddr); + assert!(index_k != 0); + boot_lvl1_pt[0] = pte_next(ram_lvl2_pt_paddr); serialise_page_table_to_paddr(&mut boot_lvl1_pt) } @@ -596,6 +566,26 @@ impl ArchPtLayout<4> for AArch64PtLayout { } } +struct Riscv64PtLayout; + +impl ArchPtLayout<3> for Riscv64PtLayout { + const MIN_LEVEL: usize = 1; + const LEVEL_BITS: [(u32, u32); 3] = [(30, 9), (21, 9), (12, 9)]; + + fn leaf_entry(level: usize, address: usize, attributes: u64) -> u64 { + assert!(level < Self::LEVELS); + assert_eq!(attributes, 0); + + let address = address.try_into().unwrap(); + + riscv64::pte_leaf(address) + } + + fn table_entry(_level: usize, address: *const u8) -> u64 { + riscv64::pte_next(address) + } +} + fn setup_identity_page_tables< const LEVELS: usize, const PAGE_TABLE_ENTRIES: usize, diff --git a/loader/src/riscv/mmu.c b/loader/src/riscv/mmu.c index b40350ca7..8e71db4cb 100644 --- a/loader/src/riscv/mmu.c +++ b/loader/src/riscv/mmu.c @@ -6,8 +6,10 @@ */ #include +#include #include "../arch.h" +#include "../cutil.h" /* Pointers to the top-level paging structures */ uintptr_t riscv64_boot_lvl1_pt; @@ -20,8 +22,25 @@ uintptr_t riscv64_boot_lvl1_pt; #define RISCV_PGSHIFT 12 +struct Region regions[] = { + { .start = 0x80200000, .top = 0x100000000 - 1, .arch_attrs.is_ram = true }, +}; + +uint8_t page_table_bytes[PAGE_TABLE_SIZE][MAX_NUM_PAGE_TABLES] ALIGN(PAGE_TABLE_SIZE); + +extern uintptr_t riscv64_setup_pagetables( + uint64_t kernel_first_vaddr, uint64_t kernel_first_paddr, + void *regions_ptr, uintptr_t regions_len, + uint8_t page_table_bytes[PAGE_TABLE_SIZE][MAX_NUM_PAGE_TABLES]); + int arch_mmu_enable(int logical_cpu) { + uintptr_t riscv64_boot_lvl1_pt = riscv64_setup_pagetables( + 0xffffffff80200000, 0x80200000, + ®ions, ARRAY_SIZE(regions), + page_table_bytes + ); + // The RISC-V privileged spec (20211203), section 4.1.11 says that the // SFENCE.VMA instruction may need to be executed before or after writing // to satp. I don't understand why we do it before compared to after.