From 7dcf244ee60236b61e00ad978f1e76e8950bdab7 Mon Sep 17 00:00:00 2001 From: Zoe Date: Thu, 27 Aug 2026 13:09:11 -0500 Subject: [PATCH] feat: acpi, apic, apic timer --- Makefile | 35 +- limine.conf | 2 +- src/arch/x86_64/apic.rs | 361 +++++++++++ src/arch/x86_64/cpu.rs | 40 +- src/arch/x86_64/interrupts/apic_vectors.rs | 42 ++ src/arch/x86_64/interrupts/idt.rs | 6 +- src/arch/x86_64/interrupts/mod.rs | 11 +- src/arch/x86_64/io_apic.rs | 213 +++++++ src/arch/x86_64/mod.rs | 154 ++++- src/arch/x86_64/paging.rs | 608 +++++++++++-------- src/arch/x86_64/pit.rs | 29 + src/arch/x86_64/port.rs | 4 +- src/arch/x86_64/timer.rs | 82 +++ src/boot/limine.rs | 82 ++- src/boot/mod.rs | 2 +- src/debug/serial.rs | 4 +- src/main.rs | 202 +++++-- src/memory/address_space.rs | 88 ++- src/memory/frame.rs | 93 ++- src/memory/mod.rs | 58 +- src/platform/acpi.rs | 659 +++++++++++++++++++++ src/platform/mod.rs | 1 + 22 files changed, 2394 insertions(+), 382 deletions(-) create mode 100644 src/arch/x86_64/apic.rs create mode 100644 src/arch/x86_64/interrupts/apic_vectors.rs create mode 100644 src/arch/x86_64/io_apic.rs create mode 100644 src/arch/x86_64/pit.rs create mode 100644 src/arch/x86_64/timer.rs create mode 100644 src/platform/acpi.rs create mode 100644 src/platform/mod.rs diff --git a/Makefile b/Makefile index 7f8ae84..f71d496 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ EXPORT_SYMBOLS = true ISO_PATH = ${ARTIFACTS_PATH}/iso_root #INITRAMFS_PATH = ${ARTIFACTS_PATH}/initramfs IMAGE_PATH = ${ARTIFACTS_PATH}/${IMAGE_NAME} +ESP_IMAGE = ${ARTIFACTS_PATH}/esp.img CARGO_OPTS = -Zjson-target-spec --target=src/arch/${ARCH}/${ARCH}-unknown-none.json QEMU_OPTS += -m ${MEMORY} -drive id=hd0,format=raw,file=${IMAGE_PATH} LIMINE_BOOT_VARIATION = X64 @@ -36,6 +37,10 @@ ifneq (${GDB},) QEMU_OPTS += -s -S endif +ifneq (${KVM},) + QEMU_OPTS += -accel kvm -cpu host +endif + ifneq (${UEFI},) RUN_OPTS := ovmf-${ARCH} QEMU_OPTS += -bios ovmf/ovmf-${ARCH}/OVMF.fd @@ -55,11 +60,10 @@ prepare-bin-files: rm -f ${IMAGE_PATH} rm -rf ${ARTIFACTS_PATH}/* - # Make bin/ bin/iso_root and bin/initramfs + # Make bin/ and bin/iso_root mkdir -p ${ARTIFACTS_PATH} mkdir -p ${ISO_PATH} # mkdir -p ${INITRAMFS_PATH} - mkdir -p ${ARTIFACTS_PATH}/mnt run-scripts: # Place the build ID into the binary so it can be read at runtime @HASH=$$(md5sum ${KERNEL_FILE} | cut -c1-12) && \ @@ -72,8 +76,6 @@ copy-iso-files: mkdir -p ${ISO_PATH}/boot/limine mkdir -p ${ISO_PATH}/EFI/BOOT - mkdir -p ${ISO_PATH}/mnt - cp -v limine.conf limine/limine-bios.sys ${ISO_PATH}/boot/limine cp -v limine/BOOT${LIMINE_BOOT_VARIATION}.EFI ${ISO_PATH}/EFI/BOOT/ @@ -81,8 +83,13 @@ copy-iso-files: cp -v ${KERNEL_FILE} ${ISO_PATH}/boot #cp -v ${ARTIFACTS_PATH}/initramfs.img ${ISO_PATH}/boot -partition-iso: copy-iso-files - # Make empty ISO of 64M in size +build-esp: copy-iso-files + # Create and populate formatted FAT image for ESP partition (130048 1K-blocks = ~127MiB) + mkfs.fat -F ${ESP_BITS} -C ${ESP_IMAGE} 130048 + mcopy -s -i ${ESP_IMAGE} ${ISO_PATH}/* :: + +partition-iso: + # Make empty disk image dd if=/dev/zero of=${IMAGE_PATH} bs=1M count=0 seek=${ISO_SIZE} ifneq (${UEFI},) parted -s ${IMAGE_PATH} mklabel gpt @@ -94,25 +101,17 @@ else parted -s ${IMAGE_PATH} set 1 boot on endif - # Make ISO with 1 partition starting at sector 2048 that is 32768 sectors, or 16MiB, in size - # Then a second partition spanning the rest of the disk + # Make second partition spanning the rest of the disk parted -s ${IMAGE_PATH} mkpart primary 262145s 100% -build-iso: partition-iso +build-iso: partition-iso build-esp + # Splice ESP partition into sector 2048 of the disk image + dd if=${ESP_IMAGE} of=${IMAGE_PATH} bs=512 seek=2048 conv=notrunc ifeq (${UEFI},) # install limine for legacy bios ./limine/limine bios-install ${IMAGE_PATH} endif - sudo losetup -Pf --show ${IMAGE_PATH} > loopback_dev - sudo mkfs.fat -F ${ESP_BITS} `cat loopback_dev`p1 - sudo mount `cat loopback_dev`p1 ${ARTIFACTS_PATH}/mnt - sudo cp -r ${ISO_PATH}/* ${ARTIFACTS_PATH}/mnt - sync - sudo umount ${ARTIFACTS_PATH}/mnt - sudo losetup -d `cat loopback_dev` - rm loopback_dev - compile-bootloader: @if [ ! -f "limine/.version" ] || [ "$$(cat limine/.version)" != "${LIMINE_VERSION}" ]; then \ echo "Downloading Limine ${LIMINE_VERSION}..."; \ diff --git a/limine.conf b/limine.conf index 6722ebe..7954873 100644 --- a/limine.conf +++ b/limine.conf @@ -1,4 +1,4 @@ -timeout: 3 +timeout: 0 /DuskOS protocol: limine diff --git a/src/arch/x86_64/apic.rs b/src/arch/x86_64/apic.rs new file mode 100644 index 0000000..7f9ee58 --- /dev/null +++ b/src/arch/x86_64/apic.rs @@ -0,0 +1,361 @@ +use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + +use crate::{ + arch::{ + disable_interrupts, + port::write_u8, + x86_64::{ + cpu::{read_msr, write_msr}, + interrupts::{ + apic_vectors::{ + APIC_ERROR_VECTOR, APIC_SELF_IPI_VECTOR, APIC_SPURIOUS_VECTOR, + APIC_TIMER_VECTOR, + }, + enable_interrupts, + }, + }, + }, + memory::{ + AddressSpace, CachePolicy, FrameAllocator, PagePermissions, PhysicalAddr, VirtualAddr, + }, + println, +}; + +const APIC_ID: u32 = 0x20; +const APIC_VERSION: u32 = 0x30; +// End Of Interrupt +const APIC_EOI: u32 = 0xB0; +// Task Priority Register +const APIC_TPR: u32 = 0x80; +// Spurious Interrupt Vector +const APIC_SVR: u32 = 0xF0; +// Error Status Register +const APIC_ESR: u32 = 0x280; +const APIC_LVT_ERROR: u32 = 0x370; +const APIC_LVT_MASKED: u64 = 1 << 16; + +const APIC_LVT_TIMER_MODE_PERIODIC: u64 = 1 << 17; + +pub const LOCAL_APIC_VIRTUAL_ADDRESS: VirtualAddr = VirtualAddr::new(0xFFFF_FFFD_0000_0000); + +const APIC_ICR1: u32 = 0x300; +const APIC_ICR2: u32 = 0x310; +const X2APIC_SELF_IPI: u32 = 0x3F0; + +const APIC_LVT_TIMER: u32 = 0x320; +const APIC_TIMER_INITIAL_COUNT: u32 = 0x380; +const APIC_TIMER_CURRENT_COUNT: u32 = 0x390; +const APIC_TIMER_DIVIDE_CONFIG: u32 = 0x3E0; + +#[derive(Debug)] +enum LocalApicAccess { + X2Apic, + XApic, +} + +impl LocalApicAccess { + fn read(&self, offset: u32) -> u64 { + match self { + Self::X2Apic => { + let msr = 0x800 + offset / 16; + unsafe { read_msr(msr) } + } + Self::XApic => unsafe { + LOCAL_APIC_VIRTUAL_ADDRESS + .as_ptr::() + .add(offset as usize) + .cast::() + .read_volatile() as u64 + }, + } + } + + fn write(&self, offset: u32, value: u64) { + match self { + Self::X2Apic => { + let msr = 0x800 + offset / 16; + unsafe { write_msr(msr, value) }; + } + Self::XApic => unsafe { + LOCAL_APIC_VIRTUAL_ADDRESS + .as_mut_ptr::() + .add(offset as usize) + .cast::() + .write_volatile(value as u32); + }, + } + } + + fn send_self_ipi(&self, vector: u8) -> Result<(), LocalApicError> { + match self { + Self::X2Apic => { + unsafe { write_msr(0x800 + X2APIC_SELF_IPI / 16, vector as u64) }; + } + Self::XApic => { + unsafe { + // bit 18 = destination type. 1 = self + let interrupt_command = vector as u32 | (1 << 18); + core::ptr::write_volatile( + LOCAL_APIC_VIRTUAL_ADDRESS + .as_mut_ptr::() + .add(APIC_ICR1 as usize) + .cast::(), + interrupt_command, + ); + }; + } + }; + + Ok(()) + } + + fn end_of_interrupt(&self) { + self.write(APIC_EOI, 0); + } +} + +#[derive(Debug)] +pub enum LocalApicError { + AddressMismatch, + ApicDisabled, + TimerTestFailed, + FailedToMapApic, + FailedToSendSelfIpi, + NotBootSystemProcessor, +} + +#[derive(Debug)] +pub struct LocalApic { + id: u32, + access: LocalApicAccess, +} + +const IA32_APIC_BASE: u32 = 0x1B; + +const APIC_BASE_BSP: u64 = 1 << 8; +const APIC_BASE_X2APIC_ENABLE: u64 = 1 << 10; +const APIC_BASE_GLOBAL_ENABLE: u64 = 1 << 11; +// TODO: use MAXPHYADDR +const APIC_BASE_ADDRESS_MASK: u64 = 0x000F_FFFF_FFFF_F000; + +static SELF_IPI_COUNTER: AtomicUsize = AtomicUsize::new(0); + +impl LocalApic { + pub fn init( + local_apic_address: PhysicalAddr, + allocator: &mut FrameAllocator, + address_space: &mut AddressSpace, + ) -> Result { + let apic_base = unsafe { read_msr(IA32_APIC_BASE) }; + let bsp = (apic_base & APIC_BASE_BSP) != 0; + let x2apix = (apic_base & APIC_BASE_X2APIC_ENABLE) != 0; + let enabled = (apic_base & APIC_BASE_GLOBAL_ENABLE) != 0; + let xapic_physical_addr = PhysicalAddr::new((apic_base & APIC_BASE_ADDRESS_MASK) as usize); + + if local_apic_address != xapic_physical_addr { + return Err(LocalApicError::AddressMismatch); + } + + if !enabled { + return Err(LocalApicError::ApicDisabled); + } + + if !bsp { + return Err(LocalApicError::NotBootSystemProcessor); + } + + println!("BSP: {}", bsp); + println!("x2Apic: {}", x2apix); + println!("enabled: {}", enabled); + println!( + "xApic physical address: {:x}", + xapic_physical_addr.as_usize() + ); + + let access = match x2apix { + true => LocalApicAccess::X2Apic, + false => { + address_space + .map( + local_apic_address, + LOCAL_APIC_VIRTUAL_ADDRESS, + PagePermissions::new(true, false, false), + allocator, + CachePolicy::Uncacheable, + ) + .map_err(|_| LocalApicError::FailedToMapApic)?; + + LocalApicAccess::XApic + } + }; + + let raw_id = access.read(APIC_ID); + + let id = match access { + LocalApicAccess::XApic => (raw_id >> 24) as u32, + LocalApicAccess::X2Apic => raw_id as u32, + }; + + let version = access.read(APIC_VERSION); + let max_lvt_entries = ((version >> 16) & 0xFF) + 1; + let version = version & 0xFF; + + unsafe { + write_u8(0x21, 0xFF); + write_u8(0xA1, 0xFF); + } + + access.write(APIC_LVT_ERROR, APIC_ERROR_VECTOR as u64 | APIC_LVT_MASKED); + access.write(APIC_ESR, 0); + let _ = access.read(APIC_ESR); + access.write(APIC_TPR, 0); + access.write(APIC_SVR, (1 << 8) | APIC_SPURIOUS_VECTOR as u64); + + Ok(Self { id, access }) + } + + pub fn test_timer_interrupt(&self) -> Result<(), LocalApicError> { + self.access + .write(APIC_LVT_TIMER, APIC_TIMER_VECTOR as u64 | APIC_LVT_MASKED); + + // Divide by 16 + self.access.write(APIC_TIMER_DIVIDE_CONFIG, 0b11); + + APIC_TIMER_COUNT.store(0, Ordering::SeqCst); + + self.access.write(APIC_TIMER_INITIAL_COUNT, 123456); + + self.access.write(APIC_LVT_TIMER, APIC_TIMER_VECTOR as u64); + + enable_interrupts(); + + for _ in 0..10_000_000 { + if APIC_TIMER_COUNT.load(Ordering::SeqCst) != 0 { + break; + } + + core::hint::spin_loop(); + } + + disable_interrupts(); + + self.access + .write(APIC_LVT_TIMER, APIC_TIMER_VECTOR as u64 | APIC_LVT_MASKED); + self.access.write(APIC_TIMER_INITIAL_COUNT, 0); + + if APIC_TIMER_COUNT.load(Ordering::SeqCst) != 1 { + return Err(LocalApicError::TimerTestFailed); + } + + Ok(()) + } + + pub fn start_timer(&self) { + self.access.write(APIC_LVT_TIMER, APIC_TIMER_VECTOR as u64); + } + + pub fn start_calibration_counter(&self) { + self.access + .write(APIC_LVT_TIMER, APIC_TIMER_VECTOR as u64 | APIC_LVT_MASKED); + + self.access.write(APIC_TIMER_DIVIDE_CONFIG, 0b11); + self.access.write(APIC_TIMER_INITIAL_COUNT, u32::MAX as u64); + } + + pub fn stop_timer(&self) { + self.access + .write(APIC_LVT_TIMER, APIC_TIMER_VECTOR as u64 | APIC_LVT_MASKED); + self.access.write(APIC_TIMER_INITIAL_COUNT, 0); + } + + pub fn send_self_ipi(&self) -> Result<(), LocalApicError> { + SELF_IPI_COUNTER.store(0, Ordering::SeqCst); + + self.access.send_self_ipi(APIC_SELF_IPI_VECTOR)?; + + enable_interrupts(); + + for _ in 0..10_000_000 { + if SELF_IPI_COUNTER.load(Ordering::SeqCst) != 0 { + break; + } + + core::hint::spin_loop(); + } + + disable_interrupts(); + + if SELF_IPI_COUNTER.load(Ordering::SeqCst) != 1 { + return Err(LocalApicError::FailedToSendSelfIpi); + } + + Ok(()) + } + + pub fn delay_ticks(&self, count: u32) { + if count == 0 { + return; + } + + APIC_TIMER_COUNT.store(0, Ordering::SeqCst); + + self.access + .write(APIC_LVT_TIMER, APIC_TIMER_VECTOR as u64 | APIC_LVT_MASKED); + self.access.write(APIC_TIMER_DIVIDE_CONFIG, 0b11); + self.access.write(APIC_TIMER_INITIAL_COUNT, count as u64); + self.access.write(APIC_LVT_TIMER, APIC_TIMER_VECTOR as u64); + + while APIC_TIMER_COUNT.load(Ordering::SeqCst) == 0 { + unsafe { + core::arch::asm!("sti", "hlt", "cli", options(nomem, nostack)); + } + } + + self.stop_timer(); + } + + pub fn id(&self) -> u32 { + self.id + } +} + +fn current_access() -> LocalApicAccess { + let apic_base = unsafe { read_msr(IA32_APIC_BASE) }; + + if apic_base & APIC_BASE_X2APIC_ENABLE != 0 { + LocalApicAccess::X2Apic + } else { + LocalApicAccess::XApic + } +} + +pub fn current_timer_count() -> u32 { + current_access().read(APIC_TIMER_CURRENT_COUNT) as u32 +} + +pub(super) fn end_of_interrupt() { + current_access().end_of_interrupt(); +} + +pub(super) fn record_self_ipi() { + SELF_IPI_COUNTER.fetch_add(1, Ordering::SeqCst); +} + +static APIC_ERROR_COUNT: AtomicUsize = AtomicUsize::new(0); +static LAST_APIC_ERROR: AtomicU32 = AtomicU32::new(0); + +pub(super) fn record_error() { + let access = current_access(); + + access.write(APIC_ESR, 0); + let error = access.read(APIC_ESR) as u32; + + LAST_APIC_ERROR.store(error, Ordering::SeqCst); + APIC_ERROR_COUNT.fetch_add(1, Ordering::SeqCst); +} + +static APIC_TIMER_COUNT: AtomicUsize = AtomicUsize::new(0); + +pub(super) fn record_timer() { + APIC_TIMER_COUNT.fetch_add(1, Ordering::SeqCst); +} diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 2857eb6..a997467 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -4,6 +4,7 @@ use core::arch::asm; pub enum CpuFeaturesError { CpuidFeaturesNotSupported, InvalidPhysicalAddressWidth, + InvalidVirtualAddressWidth, } #[derive(Clone, Copy, Debug)] @@ -12,14 +13,19 @@ pub(crate) struct CpuFeatures { pub nx_enabled: bool, pub physical_address_bits: u8, pub virtual_address_bits: u8, + pub five_level_paging_active: bool, } +// Extended features +const IA32_EFER: u32 = 0xC0000080; + pub fn detect_features_and_enable() -> Result { let mut features = CpuFeatures { nx_supported: false, nx_enabled: false, physical_address_bits: 0, virtual_address_bits: 0, + five_level_paging_active: false, }; let cpuid_result = core::arch::x86_64::__cpuid_count(0x80000000, 0); @@ -40,6 +46,16 @@ pub fn detect_features_and_enable() -> Result { } features.virtual_address_bits = (cpuid_result.eax >> 8 & 0xFF) as u8; + features.five_level_paging_active = read_cr4() & (1 << 12) != 0; + + let required_virtual_address_bits = if features.five_level_paging_active { + 57 + } else { + 48 + }; + if features.virtual_address_bits < required_virtual_address_bits { + return Err(CpuFeaturesError::InvalidVirtualAddressWidth); + } if features.nx_supported { let cpuid_result = core::arch::x86_64::__cpuid_count(0x1, 0); @@ -50,19 +66,33 @@ pub fn detect_features_and_enable() -> Result { } // mother efer - let efer = unsafe { read_msr(0xC0000080) }; + let efer = unsafe { read_msr(IA32_EFER) }; unsafe { - write_msr(0xC0000080, efer | (1 << 11)); + write_msr(IA32_EFER, efer | (1 << 11)); } - features.nx_enabled = unsafe { read_msr(0xC0000080) } & (1 << 11) != 0; + features.nx_enabled = unsafe { read_msr(IA32_EFER) } & (1 << 11) != 0; } Ok(features) } -unsafe fn read_msr(msr: u32) -> u64 { +fn read_cr4() -> usize { + let value: usize; + + unsafe { + asm!( + "mov {}, cr4", + out(reg) value, + options(nomem, nostack, preserves_flags), + ); + } + + value +} + +pub(super) unsafe fn read_msr(msr: u32) -> u64 { let low: u32; let high: u32; @@ -79,7 +109,7 @@ unsafe fn read_msr(msr: u32) -> u64 { ((high as u64) << 32) | low as u64 } -unsafe fn write_msr(msr: u32, value: u64) { +pub(super) unsafe fn write_msr(msr: u32, value: u64) { unsafe { asm!( "wrmsr", diff --git a/src/arch/x86_64/interrupts/apic_vectors.rs b/src/arch/x86_64/interrupts/apic_vectors.rs new file mode 100644 index 0000000..20be571 --- /dev/null +++ b/src/arch/x86_64/interrupts/apic_vectors.rs @@ -0,0 +1,42 @@ +use crate::arch::{ + apic, timer, + x86_64::interrupts::idt::{self, InterruptStackFrame}, +}; + +pub const APIC_SELF_IPI_VECTOR: u8 = 0xF0; +pub const PIT_CALIBRATION_VECTOR: u8 = 0xF1; +pub const APIC_TIMER_VECTOR: u8 = 0xFD; +pub const APIC_ERROR_VECTOR: u8 = 0xFE; +pub const APIC_SPURIOUS_VECTOR: u8 = 0xFF; + +extern "x86-interrupt" fn self_ipi_handler(_frame: InterruptStackFrame) { + apic::record_self_ipi(); + apic::end_of_interrupt(); +} + +extern "x86-interrupt" fn error_handler(_frame: InterruptStackFrame) { + apic::record_error(); + apic::end_of_interrupt(); +} + +extern "x86-interrupt" fn timer_handler(_frame: InterruptStackFrame) { + apic::record_timer(); + apic::end_of_interrupt(); +} + +extern "x86-interrupt" fn pit_calibration_handler(_frame: InterruptStackFrame) { + timer::record_pit_calibration(); + apic::end_of_interrupt(); +} + +extern "x86-interrupt" fn spurious_handler(_frame: InterruptStackFrame) { + // No EOI +} + +pub(super) fn install(idt: &mut idt::Idt) { + idt.set_handler(APIC_SELF_IPI_VECTOR, self_ipi_handler, 0); + idt.set_handler(PIT_CALIBRATION_VECTOR, pit_calibration_handler, 0); + idt.set_handler(APIC_ERROR_VECTOR, error_handler, 0); + idt.set_handler(APIC_TIMER_VECTOR, timer_handler, 0); + idt.set_handler(APIC_SPURIOUS_VECTOR, spurious_handler, 0); +} diff --git a/src/arch/x86_64/interrupts/idt.rs b/src/arch/x86_64/interrupts/idt.rs index 45f6d0e..22bce49 100644 --- a/src/arch/x86_64/interrupts/idt.rs +++ b/src/arch/x86_64/interrupts/idt.rs @@ -1,5 +1,8 @@ use super::exceptions; -use crate::{arch::x86_64::gdt::KERNEL_CODE_SELECTOR, memory::VirtualAddr}; +use crate::{ + arch::x86_64::{gdt::KERNEL_CODE_SELECTOR, interrupts::apic_vectors}, + memory::VirtualAddr, +}; #[repr(C, packed)] #[derive(Clone, Copy)] @@ -97,6 +100,7 @@ pub fn idt_init() { let mut idt = Idt::new(); exceptions::install(&mut idt); + apic_vectors::install(&mut idt); unsafe { core::ptr::addr_of_mut!(IDT).write(idt); diff --git a/src/arch/x86_64/interrupts/mod.rs b/src/arch/x86_64/interrupts/mod.rs index f8befed..0a2e5b5 100644 --- a/src/arch/x86_64/interrupts/mod.rs +++ b/src/arch/x86_64/interrupts/mod.rs @@ -1,12 +1,21 @@ use core::arch::asm; +pub(super) mod apic_vectors; mod exceptions; mod idt; pub use idt::idt_init as init; +#[inline(always)] pub fn disable_interrupts() { unsafe { - asm!("cli"); + asm!("cli", options(nostack)); + } +} + +#[inline(always)] +pub fn enable_interrupts() { + unsafe { + asm!("sti", options(nostack)); } } diff --git a/src/arch/x86_64/io_apic.rs b/src/arch/x86_64/io_apic.rs new file mode 100644 index 0000000..43d6c99 --- /dev/null +++ b/src/arch/x86_64/io_apic.rs @@ -0,0 +1,213 @@ +use crate::{ + memory::{ + AddressSpace, CachePolicy, FrameAllocator, PagePermissions, PhysicalAddr, VirtualAddr, + }, + platform::acpi::{InterruptPolarity, IoApicInfo, TriggerMode}, + println, +}; + +const IOREGSEL: usize = 0x00; +const IOWIN: usize = 0x10; + +const IOAPIC_ID: u8 = 0x00; +const IOAPIC_VERSION: u8 = 0x01; +const IOAPIC_REDIRECTION_BASE: u8 = 0x10; + +const MASKED: u32 = 1 << 16; + +pub const IOAPIC_VIRTUAL_ADDRESS: VirtualAddr = VirtualAddr::new(0xFFFF_FFFD_1000_0000); + +#[derive(Debug)] +pub enum IoApicError { + IdMismatch { expected: u8, actual: u8 }, + GsiOutsideRange, + FailedToMapIoApic, + InvalidRedirectionIndex, +} + +pub struct RedirectionConfig { + pub vector: u8, + pub destination: u8, + pub polarity: InterruptPolarity, + pub trigger: TriggerMode, +} + +#[derive(Debug)] +pub struct IoApic { + base: VirtualAddr, + global_interrupt_base: u32, + redirection_entry_count: u32, +} + +impl IoApic { + pub fn new( + expected_id: u8, + physical_address: PhysicalAddr, + global_interrupt_base: u32, + virtual_address: VirtualAddr, + allocator: &mut FrameAllocator, + address_space: &mut AddressSpace, + ) -> Result { + address_space + .map( + physical_address, + virtual_address, + PagePermissions::new(true, false, false), + allocator, + CachePolicy::Uncacheable, + ) + .map_err(|_| IoApicError::FailedToMapIoApic)?; + + let mut io_apic = Self { + base: virtual_address, + global_interrupt_base: global_interrupt_base, + redirection_entry_count: 0, + }; + + let version = io_apic.read(IOAPIC_VERSION); + io_apic.redirection_entry_count = ((version >> 16) & 0xFF) + 1; + + let id = ((io_apic.read(IOAPIC_ID) >> 24) & 0xF) as u8; + if id != expected_id { + return Err(IoApicError::IdMismatch { + expected: expected_id, + actual: id, + }); + } + println!("IOAPIC ID: {:#X}", id); + println!("IOAPIC version: {:#X}", version & 0xFF); + + println!( + "IOAPIC redirection entry count: {:#X}", + io_apic.redirection_entry_count + ); + + Ok(io_apic) + } + + fn read(&mut self, register: u8) -> u32 { + unsafe { + self.base + .as_mut_ptr::() + .write_volatile(register as u32); + + self.base + .as_ptr::() + .add(IOWIN) + .cast::() + .read_volatile() + } + } + + fn write(&mut self, register: u8, value: u32) { + unsafe { + self.base + .as_mut_ptr::() + .write_volatile(register as u32); + + self.base + .as_mut_ptr::() + .add(IOWIN) + .cast::() + .write_volatile(value); + } + } + + fn redirection_index(&self, gsi: u32) -> Result { + let index = gsi + .checked_sub(self.global_interrupt_base) + .ok_or(IoApicError::GsiOutsideRange)?; + + if index >= self.redirection_entry_count { + return Err(IoApicError::GsiOutsideRange); + } + + Ok(index) + } + + pub fn handles_gsi(&mut self, gsi: u32) -> bool { + match gsi.checked_sub(self.global_interrupt_base) { + Some(index) => index < self.redirection_entry_count, + None => false, + } + } + + fn read_redirection(&mut self, gsi: u32) -> Result { + let index = self.redirection_index(gsi)?; + let register = u8::try_from(IOAPIC_REDIRECTION_BASE as u32 + index * 2) + .map_err(|_| IoApicError::InvalidRedirectionIndex)?; + + let low = self.read(register); + let high = self.read(register + 1); + + Ok((high as u64) << 32 | low as u64) + } + + fn redirection_registers(&mut self, gsi: u32) -> Result<(u8, u8), IoApicError> { + let index = self.redirection_index(gsi)?; + let low_register = u8::try_from(IOAPIC_REDIRECTION_BASE as u32 + index * 2) + .map_err(|_| IoApicError::InvalidRedirectionIndex)?; + + let high_register = u8::try_from(IOAPIC_REDIRECTION_BASE as u32 + index * 2 + 1) + .map_err(|_| IoApicError::InvalidRedirectionIndex)?; + + Ok((low_register, high_register)) + } + + pub fn configure_masked( + &mut self, + gsi: u32, + config: RedirectionConfig, + ) -> Result<(), IoApicError> { + let mut entry = config.vector as u64; + + match config.polarity { + InterruptPolarity::ActiveHigh => { + entry |= 0 << 13; + } + InterruptPolarity::ActiveLow => { + entry |= 1 << 13; + } + } + + match config.trigger { + TriggerMode::Edge => { + entry |= 0 << 15; + } + TriggerMode::Level => { + entry |= 1 << 15; + } + } + + entry |= 1 << 16; + entry |= (config.destination as u64) << 56; + + let (low_register, high_register) = self.redirection_registers(gsi)?; + + let old_low = self.read(low_register); + self.write(low_register, old_low | MASKED); + + self.write(high_register, (entry >> 32) as u32); + self.write(low_register, entry as u32 | MASKED); + + Ok(()) + } + + pub fn unmask(&mut self, gsi: u32) -> Result<(), IoApicError> { + let (low_register, _) = self.redirection_registers(gsi)?; + let low = self.read(low_register); + + self.write(low_register, low & !MASKED); + + Ok(()) + } + + pub fn mask(&mut self, gsi: u32) -> Result<(), IoApicError> { + let (low_register, _) = self.redirection_registers(gsi)?; + let low = self.read(low_register); + + self.write(low_register, low | MASKED); + + Ok(()) + } +} diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index b6d51b0..918d535 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,8 +1,12 @@ +pub mod apic; mod cpu; mod gdt; mod interrupts; +pub mod io_apic; mod paging; +mod pit; pub mod port; +pub mod timer; use core::arch::asm; @@ -16,7 +20,17 @@ pub struct ArchState { pub paging: PagingConfig, } -use crate::println; +use crate::{ + KernelHandoff, + arch::{ + apic::LocalApic, + io_apic::{IOAPIC_VIRTUAL_ADDRESS, IoApic}, + x86_64::interrupts::apic_vectors::PIT_CALIBRATION_VECTOR, + }, + memory::{AddressSpace, FrameAllocator, VirtualAddr}, + platform::acpi::Madt, + println, +}; pub fn init() -> ArchState { disable_interrupts(); @@ -31,6 +45,144 @@ pub fn init() -> ArchState { ArchState { paging } } +#[derive(Debug)] +pub enum InterruptInitError { + InvalidLocalApicId, + InvalidLocalApicAddress, + FailedToGetIoApic, + IoApicError(io_apic::IoApicError), + LocalApicError(apic::LocalApicError), + TimerCalibrationError(timer::TimerCalibrationError), + MalformedMadt, + PitNotHandled, +} + +#[derive(Debug)] +pub struct InterruptController { + local_apic: LocalApic, + io_apic: IoApic, + local_timer_frequency: u64, +} + +pub fn init_interrupt_controller( + madt: &Madt<'_>, + allocator: &mut FrameAllocator, + address_space: &mut AddressSpace, +) -> Result { + let local_apic_address = madt + .effective_local_apic_address() + .map_err(|_| InterruptInitError::InvalidLocalApicAddress)?; + + let mut local_apic = apic::LocalApic::init(local_apic_address, allocator, address_space) + .map_err(|err| InterruptInitError::LocalApicError(err))?; + + local_apic + .send_self_ipi() + .map_err(|err| InterruptInitError::LocalApicError(err))?; + + local_apic + .test_timer_interrupt() + .map_err(|err| InterruptInitError::LocalApicError(err))?; + + let io_apic_info = madt + .sole_io_apic() + .map_err(|_| InterruptInitError::FailedToGetIoApic)?; + + let pit_route = madt + .isa_irq_route(0x0) + .map_err(|_| InterruptInitError::MalformedMadt)?; + + let mut io_apic = io_apic::IoApic::new( + io_apic_info.id, + io_apic_info.apic_address, + io_apic_info.global_system_interrupt_base, + IOAPIC_VIRTUAL_ADDRESS, + allocator, + address_space, + ) + .map_err(|err| InterruptInitError::IoApicError(err))?; + + let pit_handled = io_apic.handles_gsi(pit_route.gsi); + if !pit_handled { + return Err(InterruptInitError::PitNotHandled); + } + + let destination = + u8::try_from(local_apic.id()).map_err(|_| InterruptInitError::InvalidLocalApicId)?; + + io_apic + .configure_masked( + pit_route.gsi, + io_apic::RedirectionConfig { + vector: PIT_CALIBRATION_VECTOR, + destination, + polarity: pit_route.polarity, + trigger: pit_route.trigger, + }, + ) + .map_err(|err| InterruptInitError::IoApicError(err))?; + + let local_timer_frequency = + timer::calibrate_local_apic(&mut local_apic, &mut io_apic, pit_route) + .map_err(|err| InterruptInitError::TimerCalibrationError(err))?; + + Ok(InterruptController { + local_apic, + io_apic, + local_timer_frequency, + }) +} + +#[derive(Debug)] +pub enum TimerError { + DurationOverflow, +} + +impl InterruptController { + pub fn delay(&self, duration: core::time::Duration) -> Result<(), TimerError> { + let nanoseconds = duration.as_nanos(); + + let ticks = (nanoseconds + .checked_mul(self.local_timer_frequency as u128) + .ok_or(TimerError::DurationOverflow)? + + 999_999_999) + / 1_000_000_000; + + if ticks == 0 { + return Ok(()); + } + + let mut remaining = ticks; + + while remaining > 0 { + let chunk = remaining.min(u32::MAX as u128) as u32; + self.local_apic.delay_ticks(chunk); + remaining -= chunk as u128; + } + + Ok(()) + } +} + +/// # Safety +/// +/// The caller must ensure: +/// - The stack is currently mapped, writable, and 16-byte aligned +pub unsafe fn enter_kernel(stack_top: VirtualAddr, handoff: *mut KernelHandoff) -> ! { + unsafe { + asm!( + "mov rsp, {stack_top}", + "xor rbp, rbp", + "mov rdi, {handoff}", + "call {kernel_main}", + stack_top = in(reg) stack_top.as_usize(), + handoff = in(reg) handoff, + kernel_main = sym crate::kernel_main, + options(noreturn) + ); + }; +} + pub fn halt() { unsafe { asm!("hlt"); diff --git a/src/arch/x86_64/paging.rs b/src/arch/x86_64/paging.rs index 7470d8c..a564129 100644 --- a/src/arch/x86_64/paging.rs +++ b/src/arch/x86_64/paging.rs @@ -3,7 +3,8 @@ use core::arch::asm; use crate::{ arch::x86_64::cpu::CpuFeatures, memory::{ - DirectMap, FrameAllocator, PagePermissions, PhysicalAddr, PhysicalFrame, VirtualAddr, + CachePolicy, DirectMap, FrameAddr, FrameAllocator, OwnedFrame, PagePermissions, + PhysicalAddr, VirtualAddr, }, }; @@ -22,7 +23,11 @@ impl PagingConfig { Self { physical_address_bits: features.physical_address_bits, nx_enabled: features.nx_enabled, - mode: PagingMode::FourLevel, + mode: if features.five_level_paging_active { + PagingMode::FiveLevel + } else { + PagingMode::FourLevel + }, } } @@ -41,6 +46,19 @@ enum PagingMode { FiveLevel, } +const MAX_INTERMEDIATE_LEVELS: usize = 4; +const FOUR_LEVEL_INTERMEDIATES: [PageTableLevel; 3] = [ + PageTableLevel::Pml4, + PageTableLevel::Pdpt, + PageTableLevel::PageDirectory, +]; +const FIVE_LEVEL_INTERMEDIATES: [PageTableLevel; 4] = [ + PageTableLevel::Pml5, + PageTableLevel::Pml4, + PageTableLevel::Pdpt, + PageTableLevel::PageDirectory, +]; + impl PagingMode { const fn virtual_address_bits(&self) -> u32 { match self { @@ -48,6 +66,42 @@ impl PagingMode { PagingMode::FiveLevel => 57, } } + + fn intermediate_levels(&self) -> &'static [PageTableLevel] { + match self { + PagingMode::FourLevel => &FOUR_LEVEL_INTERMEDIATES, + PagingMode::FiveLevel => &FIVE_LEVEL_INTERMEDIATES, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PageTableLevel { + Pml5, + Pml4, + Pdpt, + PageDirectory, +} + +impl PageTableLevel { + const fn index(self, address: usize) -> usize { + let shift = match self { + Self::Pml5 => 48, + Self::Pml4 => 39, + Self::Pdpt => 30, + Self::PageDirectory => 21, + }; + + address >> shift & 0x1FF + } + + const fn large_page_size(self) -> Option { + match self { + Self::Pdpt => Some(1 << 30), + Self::PageDirectory => Some(1 << 21), + Self::Pml5 | Self::Pml4 => None, + } + } } #[derive(Debug)] @@ -67,9 +121,14 @@ impl PageTableEntry { const HUGE_PAGE: u64 = 1 << 7; const NX: u64 = 1 << 63; + const WRITE_THROUGH: u64 = 1 << 3; + const CACHE_DISABLED: u64 = 1 << 4; + const PAT: u64 = 1 << 7; + const fn new( physical_address: PhysicalAddr, permissions: PagePermissions, + cache_policy: CachePolicy, config: PagingConfig, ) -> Result { if physical_address.as_usize() >= config.physical_address_limit() { @@ -92,11 +151,20 @@ impl PageTableEntry { value |= Self::NX; } + // TODO: these bit positions very by page size + // set PAT + match cache_policy { + CachePolicy::WriteBack => {} + CachePolicy::Uncacheable => { + value |= Self::CACHE_DISABLED | Self::WRITE_THROUGH; + } + }; + Ok(Self(value)) } fn new_table( - frame: PhysicalFrame, + frame: FrameAddr, user_accessible: bool, config: PagingConfig, ) -> Result { @@ -132,12 +200,20 @@ impl PageTableEntry { self.0 & Self::HUGE_PAGE != 0 } - fn frame(&self, config: PagingConfig) -> Option { + fn table_frame(&self, config: PagingConfig) -> Option { if !self.is_present() || self.is_huge() { return None; } - PhysicalFrame::from_start_address(self.physical_address(config)) + FrameAddr::from_start_address(self.physical_address(config)) + } + + fn leaf_frame(&self, config: PagingConfig) -> Option { + if !self.is_present() { + return None; + } + + FrameAddr::from_start_address(self.physical_address(config)) } } @@ -165,10 +241,9 @@ pub(crate) enum UnmapError { } #[derive(Clone, Copy)] -struct NewTable { - parent: PhysicalFrame, +struct EntryLocation { + table: FrameAddr, index: usize, - child: PhysicalFrame, } #[derive(Debug)] @@ -178,8 +253,8 @@ pub(crate) enum PageTableCreateError { } pub struct PageTable { - frame: PhysicalFrame, - direct_map: DirectMap, + pub direct_map: DirectMap, + frame: OwnedFrame, config: PagingConfig, } @@ -191,7 +266,7 @@ impl PageTable { ) -> Result { let frame = allocator.alloc().ok_or(PageTableCreateError::OutOfFrames)?; - if frame.start_address().as_usize() >= config.physical_address_limit() { + if frame.frame_address().start_address().as_usize() >= config.physical_address_limit() { unsafe { allocator.dealloc(frame) }; return Err(PageTableCreateError::PhysicalAddressTooLarge); @@ -207,19 +282,16 @@ impl PageTable { fn is_active(&self) -> bool { let cr3 = unsafe { read_cr3(self.config) }; - cr3.start_address() == self.frame.start_address() + cr3.start_address() == self.frame.frame_address().start_address() } - fn table(&self, frame: PhysicalFrame) -> Option<&[PageTableEntry; PAGE_TABLE_ENTRIES]> { + fn table(&self, frame: FrameAddr) -> Option<&[PageTableEntry; PAGE_TABLE_ENTRIES]> { let virtual_addr = self.direct_map.translate(frame.start_address())?; Some(unsafe { &*(virtual_addr.as_ptr::<[PageTableEntry; PAGE_TABLE_ENTRIES]>()) }) } - fn table_mut( - &mut self, - frame: PhysicalFrame, - ) -> Option<&mut [PageTableEntry; PAGE_TABLE_ENTRIES]> { + fn table_mut(&mut self, frame: FrameAddr) -> Option<&mut [PageTableEntry; PAGE_TABLE_ENTRIES]> { let virtual_addr = self.direct_map.translate(frame.start_address())?; Some(unsafe { &mut *(virtual_addr.as_mut_ptr::<[PageTableEntry; PAGE_TABLE_ENTRIES]>()) }) @@ -232,228 +304,263 @@ impl PageTable { (((addr << shift) as isize >> shift) as usize) == addr } - pub fn translate(&self, addr: VirtualAddr) -> Option { - let addr = addr.as_usize(); + pub fn to_physical(&self, addr: VirtualAddr) -> Option { + let address = addr.as_usize(); - if !self.is_canonical(addr) { + if !self.is_canonical(address) { return None; } - let p4 = self.table(self.frame)?; - let p4_entry = p4[p4_index(addr)]; + let mut table_frame = self.frame.frame_address(); - if !p4_entry.is_present() { - return None; + for &level in self.config.mode.intermediate_levels() { + let table = self.table(table_frame)?; + let entry = table[level.index(address)]; + + if !entry.is_present() { + return None; + } + + if entry.is_huge() { + return translate_huge_page(entry, address, level.large_page_size()?, self.config); + } + + table_frame = entry.table_frame(self.config)?; } - let p3 = self.table(p4_entry.frame(self.config)?)?; - let p3_entry = p3[p3_index(addr)]; - - if !p3_entry.is_present() { - return None; - } - - if p3_entry.is_huge() { - return translate_huge_page(p3_entry, addr, 1 << 30, self.config); - } - - let p2 = self.table(p3_entry.frame(self.config)?)?; - let p2_entry = p2[p2_index(addr)]; - - if !p2_entry.is_present() { - return None; - } - - if p2_entry.is_huge() { - return translate_huge_page(p2_entry, addr, 1 << 21, self.config); - } - - let p1 = self.table(p2_entry.frame(self.config)?)?; - let p1_entry = p1[p1_index(addr)]; - - if !p1_entry.is_present() { - return None; - } - - let physical_base = p1_entry.physical_address(self.config).as_usize(); + let page_table = self.table(table_frame)?; + let entry = page_table[p1_index(address)]; + let physical_base = entry.leaf_frame(self.config)?.start_address().as_usize(); physical_base - .checked_add(page_offset(addr)) + .checked_add(page_offset(address)) .map(PhysicalAddr::new) } + pub fn to_virtual(&self, addr: PhysicalAddr) -> Option { + self.direct_map.translate(addr) + } + fn get_next_level( &self, - parent: PhysicalFrame, + parent: FrameAddr, index: usize, - ) -> Result { + level: PageTableLevel, + ) -> Result { let parent_table = self .table(parent) .ok_or(UnmapError::PageTableOutsideDirectMap)?; - - // TODO: encode the level so bit 7 is only interpreted where huge pages are valid. - if parent_table[index].is_huge() { - return Err(UnmapError::HugePageConflict); - } - - parent_table[index] - .frame(self.config) - .ok_or(UnmapError::PageNotMapped) - } - - fn get_next_level_or_allocate( - &mut self, - parent: PhysicalFrame, - index: usize, - user_accessible: bool, - allocator: &mut FrameAllocator, - ) -> Result<(PhysicalFrame, bool), MapError> { - let config = self.config; - - let parent_table = self - .table_mut(parent) - .ok_or(MapError::PageTableOutsideDirectMap)?; - - let entry = &mut parent_table[index]; - - if !entry.is_present() { - let frame = allocator.alloc().ok_or(MapError::OutOfFrames)?; - let table = match PageTableEntry::new_table(frame, user_accessible, config) { - Ok(table) => table, - Err(PageTableEntryError::PhysicalAddressTooLarge) => { - unsafe { allocator.dealloc(frame) }; - return Err(MapError::PhysicalAddressTooLarge); - } - Err(PageTableEntryError::NoExecuteUnsupported) => unreachable!(), - }; - *entry = table; - return Ok((frame, true)); - } + let entry = parent_table[index]; if entry.is_huge() { - return Err(MapError::HugePageConflict); - } - - // TODO: we upgrade intermediate entries, and dont carefully rollback if we fail - if user_accessible && !entry.is_user_accessible() { - entry.0 |= PageTableEntry::USER_ACCESSIBLE; + return if level.large_page_size().is_some() { + Err(UnmapError::HugePageConflict) + } else { + Err(UnmapError::InvalidPageTableEntry) + }; } entry - .frame(config) - .map(|frame| (frame, false)) - .ok_or(MapError::InvalidPageTableEntry) + .table_frame(self.config) + .ok_or(UnmapError::PageNotMapped) } - fn rollback_tables( - &mut self, - new_tables: &[Option], + fn discard_private_tables( + frames: &mut [Option; MAX_INTERMEDIATE_LEVELS], count: usize, allocator: &mut FrameAllocator, ) { - for table in new_tables[..count].iter().rev().flatten() { - self.table_mut(table.parent).unwrap()[table.index] = PageTableEntry::null(); + for frame in frames[..count].iter_mut().rev().filter_map(Option::take) { + unsafe { allocator.dealloc(frame) }; + } + } - unsafe { allocator.dealloc(table.child) }; + fn apply_user_upgrades( + &mut self, + upgrades: &[Option; MAX_INTERMEDIATE_LEVELS], + count: usize, + ) { + for location in upgrades[..count].iter().flatten() { + let entry = &mut self + .table_mut(location.table) + .expect("validated page table left the direct map")[location.index]; + entry.0 |= PageTableEntry::USER_ACCESSIBLE; } } pub fn map( &mut self, mapped_addr: VirtualAddr, - frame: PhysicalFrame, + frame: FrameAddr, permissions: PagePermissions, allocator: &mut FrameAllocator, + cache_policy: CachePolicy, ) -> Result<(), MapError> { - if !self.is_canonical(mapped_addr.as_usize()) { + let address = mapped_addr.as_usize(); + + if !self.is_canonical(address) { return Err(MapError::InvalidVirtualAddress); } - if mapped_addr.as_usize() % PAGE_SIZE != 0 { + if address % PAGE_SIZE != 0 { return Err(MapError::VirtualAddressUnaligned); } - if frame.start_address().as_usize() >= self.config.physical_address_limit() { - return Err(MapError::PhysicalAddressTooLarge); + let leaf_entry = PageTableEntry::new( + frame.start_address(), + permissions, + cache_policy, + self.config, + ) + .map_err(|error| match error { + PageTableEntryError::PhysicalAddressTooLarge => MapError::PhysicalAddressTooLarge, + PageTableEntryError::NoExecuteUnsupported => MapError::NoExecuteUnsupported, + })?; + + let levels = self.config.mode.intermediate_levels(); + let mut current_table_frame_addr = self.frame.frame_address(); + let mut first_missing = None; + let mut user_upgrades: [Option; MAX_INTERMEDIATE_LEVELS] = + [None; MAX_INTERMEDIATE_LEVELS]; + let mut user_upgrade_count = 0; + + for (depth, &level) in levels.iter().enumerate() { + let index = level.index(address); + let table = self + .table(current_table_frame_addr) + .ok_or(MapError::PageTableOutsideDirectMap)?; + let entry = table[index]; + + if !entry.is_present() { + first_missing = Some(( + depth, + EntryLocation { + table: current_table_frame_addr, + index, + }, + )); + break; + } + + if entry.is_huge() { + return if level.large_page_size().is_some() { + Err(MapError::HugePageConflict) + } else { + Err(MapError::InvalidPageTableEntry) + }; + } + + if permissions.user_accessible && !entry.is_user_accessible() { + user_upgrades[user_upgrade_count] = Some(EntryLocation { + table: current_table_frame_addr, + index, + }); + user_upgrade_count += 1; + } + + current_table_frame_addr = entry + .table_frame(self.config) + .ok_or(MapError::InvalidPageTableEntry)?; } - let mut new_tables: [Option; 3] = [None; 3]; - let mut new_table_count = 0; - - let result = (|| { - let p4_entry_index = p4_index(mapped_addr.as_usize()); - let (pdpt_frame, allocated) = self.get_next_level_or_allocate( - self.frame, - p4_entry_index, - permissions.user_accessible, - allocator, - )?; - if allocated { - new_tables[new_table_count] = Some(NewTable { - parent: self.frame, - index: p4_entry_index, - child: pdpt_frame, - }); - new_table_count += 1; - } - - let p3_entry_index = p3_index(mapped_addr.as_usize()); - let (pd_frame, allocated) = self.get_next_level_or_allocate( - pdpt_frame, - p3_entry_index, - permissions.user_accessible, - allocator, - )?; - if allocated { - new_tables[new_table_count] = Some(NewTable { - parent: pdpt_frame, - index: p3_entry_index, - child: pd_frame, - }); - new_table_count += 1; - } - - let p2_entry_index = p2_index(mapped_addr.as_usize()); - let (pt_frame, allocated) = self.get_next_level_or_allocate( - pd_frame, - p2_entry_index, - permissions.user_accessible, - allocator, - )?; - if allocated { - new_tables[new_table_count] = Some(NewTable { - parent: pd_frame, - index: p2_entry_index, - child: pt_frame, - }); - new_table_count += 1; - } - - let config = self.config.clone(); - - let pt_table = self - .table_mut(pt_frame) + if first_missing.is_none() { + let pt = self + .table(current_table_frame_addr) .ok_or(MapError::PageTableOutsideDirectMap)?; - let entry = &mut pt_table[p1_index(mapped_addr.as_usize())]; - if entry.is_present() { + if pt[p1_index(address)].is_present() { return Err(MapError::PageAlreadyMapped); } - *entry = - PageTableEntry::new(frame.start_address(), permissions, config).map_err(|err| { - match err { - PageTableEntryError::PhysicalAddressTooLarge => { - MapError::PhysicalAddressTooLarge - } - PageTableEntryError::NoExecuteUnsupported => MapError::NoExecuteUnsupported, - } - })?; - Ok(()) + self.apply_user_upgrades(&user_upgrades, user_upgrade_count); + self.table_mut(current_table_frame_addr) + .expect("validated page table left the direct map")[p1_index(address)] = leaf_entry; + self.flush_tlb_if_active(mapped_addr); + return Ok(()); + } + + let (missing_depth, publication_location) = first_missing.unwrap(); + let private_table_count = levels.len() - missing_depth; + let mut private_tables: [Option; MAX_INTERMEDIATE_LEVELS] = + core::array::from_fn(|_| None); + let mut allocated_count = 0; + + while allocated_count < private_table_count { + let private_frame = match allocator.alloc() { + Some(frame) => frame, + None => { + Self::discard_private_tables(&mut private_tables, allocated_count, allocator); + return Err(MapError::OutOfFrames); + } + }; + + if PageTableEntry::new_table( + private_frame.frame_address(), + permissions.user_accessible, + self.config, + ) + .is_err() + { + unsafe { allocator.dealloc(private_frame) }; + Self::discard_private_tables(&mut private_tables, allocated_count, allocator); + return Err(MapError::PhysicalAddressTooLarge); + } + + private_tables[allocated_count] = Some(private_frame); + allocated_count += 1; + } + + let prepare_result = (|| { + for private_index in 0..private_table_count { + let private_frame = private_tables[private_index] + .as_ref() + .unwrap() + .frame_address(); + + if private_index + 1 < private_table_count { + let child = private_tables[private_index + 1] + .as_ref() + .unwrap() + .frame_address(); + let child_entry = + PageTableEntry::new_table(child, permissions.user_accessible, self.config) + .map_err(|_| MapError::PhysicalAddressTooLarge)?; + let child_index = levels[missing_depth + private_index + 1].index(address); + self.table_mut(private_frame) + .ok_or(MapError::PageTableOutsideDirectMap)?[child_index] = child_entry; + } else { + self.table_mut(private_frame) + .ok_or(MapError::PageTableOutsideDirectMap)?[p1_index(address)] = + leaf_entry; + } + } + + PageTableEntry::new_table( + private_tables[0].as_ref().unwrap().frame_address(), + permissions.user_accessible, + self.config, + ) + .map_err(|_| MapError::PhysicalAddressTooLarge) })(); - if result.is_err() { - self.rollback_tables(&new_tables, new_table_count, allocator); - return result; + let publication_entry = match prepare_result { + Ok(entry) => entry, + Err(error) => { + Self::discard_private_tables(&mut private_tables, allocated_count, allocator); + return Err(error); + } + }; + + self.apply_user_upgrades(&user_upgrades, user_upgrade_count); + self.table_mut(publication_location.table) + .expect("validated publication table left the direct map") + [publication_location.index] = publication_entry; + + // The published page table now owns these frames. + for frame in private_tables[..allocated_count] + .iter_mut() + .flat_map(Option::take) + { + let _ = frame.into_raw(); } self.flush_tlb_if_active(mapped_addr); @@ -470,7 +577,7 @@ impl PageTable { &mut self, mapped_addr: VirtualAddr, allocator: &mut FrameAllocator, - ) -> Result { + ) -> Result { if !self.is_canonical(mapped_addr.as_usize()) { return Err(UnmapError::InvalidVirtualAddress); } @@ -479,48 +586,61 @@ impl PageTable { return Err(UnmapError::VirtualAddressUnaligned); } - let config = self.config; + let address = mapped_addr.as_usize(); + let levels = self.config.mode.intermediate_levels(); + let mut table_frames: [Option; MAX_INTERMEDIATE_LEVELS + 1] = + core::array::from_fn(|_| None); + table_frames[0] = Some(self.frame.frame_address()); - let pdpt_frame = self.get_next_level(self.frame, p4_index(mapped_addr.as_usize()))?; - - let pd_frame = self.get_next_level(pdpt_frame, p3_index(mapped_addr.as_usize()))?; - - let pt_frame = self.get_next_level(pd_frame, p2_index(mapped_addr.as_usize()))?; - let pt_table = self - .table_mut(pt_frame) - .ok_or(UnmapError::PageTableOutsideDirectMap)?; - - let entry = pt_table[p1_index(mapped_addr.as_usize())]; - - if !entry.is_present() { - return Err(UnmapError::PageNotMapped); + let mut current_table = self.frame.frame_address(); + for (depth, &level) in levels.iter().enumerate() { + current_table = self.get_next_level(current_table, level.index(address), level)?; + table_frames[depth + 1] = Some(current_table); } - let frame = entry - .frame(config) - .ok_or(UnmapError::InvalidPageTableEntry)?; - pt_table[p1_index(mapped_addr.as_usize())] = PageTableEntry::null(); + let config = self.config; + let page_table = self + .table_mut(current_table) + .ok_or(UnmapError::PageTableOutsideDirectMap)?; + let entry_index = p1_index(address); + let entry = page_table[entry_index]; + let frame = entry.leaf_frame(config).ok_or(UnmapError::PageNotMapped)?; + page_table[entry_index] = PageTableEntry::null(); - if pt_table.is_empty() { - self.table_mut(pd_frame).unwrap()[p2_index(mapped_addr.as_usize())] = + let mut child_is_empty = page_table.iter().all(|entry| !entry.is_present()); + let mut deallocatable_frames: [Option; MAX_INTERMEDIATE_LEVELS] = + core::array::from_fn(|_| None); + let mut deallocatable_count = 0; + + for depth in (0..levels.len()).rev() { + if !child_is_empty { + break; + } + + let parent_frame_addr = table_frames[depth].unwrap(); + let child_frame = table_frames[depth + 1].unwrap(); + self.table_mut(parent_frame_addr) + .expect("validated page table left the direct map")[levels[depth].index(address)] = PageTableEntry::null(); - unsafe { allocator.dealloc(pt_frame) }; - if self.table(pd_frame).unwrap().is_empty() { - self.table_mut(pdpt_frame).unwrap()[p3_index(mapped_addr.as_usize())] = - PageTableEntry::null(); - unsafe { allocator.dealloc(pd_frame) }; + deallocatable_frames[deallocatable_count] = Some(child_frame); + deallocatable_count += 1; - if self.table(pdpt_frame).unwrap().is_empty() { - self.table_mut(self.frame).unwrap()[p4_index(mapped_addr.as_usize())] = - PageTableEntry::null(); - unsafe { allocator.dealloc(pdpt_frame) }; - } + if depth > 0 { + child_is_empty = self + .table(parent_frame_addr) + .expect("validated page table left the direct map") + .iter() + .all(|entry| !entry.is_present()); } } self.flush_tlb_if_active(mapped_addr); + for frame in deallocatable_frames[..deallocatable_count].iter().flatten() { + unsafe { allocator.dealloc(OwnedFrame::from_raw(*frame)) }; + } + Ok(frame) } @@ -543,11 +663,45 @@ impl PageTable { unsafe { asm!( "mov cr3, {}", - in(reg) self.frame.start_address().as_usize(), + in(reg) self.frame.frame_address().start_address().as_usize(), options(nostack, preserves_flags) ); }; } + + fn destroy_children(&mut self, table: FrameAddr, depth: usize, allocator: &mut FrameAllocator) { + let levels = self.config.mode.intermediate_levels(); + + if depth == levels.len() { + return; + } + + for idx in 0..PAGE_TABLE_ENTRIES { + let entry = self.table(table).expect("page table outside direct map")[idx]; + + if !entry.is_present() || entry.is_huge() { + continue; + } + + let child = entry + .table_frame(self.config) + .expect("invalid page table entry"); + self.destroy_children(child, depth + 1, allocator); + + self.table_mut(table) + .expect("page table outside direct map")[idx] = PageTableEntry::null(); + + unsafe { allocator.dealloc(OwnedFrame::from_raw(child)) }; + } + } + + pub unsafe fn destroy(mut self, allocator: &mut FrameAllocator) { + assert!(!self.is_active(), "attempted to destroy active page table"); + + self.destroy_children(self.frame.frame_address(), 0, allocator); + + unsafe { allocator.dealloc(self.frame) }; + } } fn translate_huge_page( @@ -563,18 +717,6 @@ fn translate_huge_page( physical_base.checked_add(offset).map(PhysicalAddr::new) } -fn p4_index(addr: usize) -> usize { - (addr >> 39) & 0x1FF -} - -fn p3_index(addr: usize) -> usize { - (addr >> 30) & 0x1FF -} - -fn p2_index(addr: usize) -> usize { - (addr >> 21) & 0x1FF -} - fn p1_index(addr: usize) -> usize { (addr >> 12) & 0x1FF } @@ -583,7 +725,7 @@ fn page_offset(addr: usize) -> usize { addr & 0xFFF } -unsafe fn read_cr3(config: PagingConfig) -> PhysicalFrame { +unsafe fn read_cr3(config: PagingConfig) -> FrameAddr { let value: usize; unsafe { asm!( @@ -593,6 +735,6 @@ unsafe fn read_cr3(config: PagingConfig) -> PhysicalFrame { ); } - PhysicalFrame::from_start_address(PhysicalAddr::new(value & config.physical_address_mask())) + FrameAddr::from_start_address(PhysicalAddr::new(value & config.physical_address_mask())) .expect("CR3 contains an unaligned page-table address") } diff --git a/src/arch/x86_64/pit.rs b/src/arch/x86_64/pit.rs new file mode 100644 index 0000000..ea943ee --- /dev/null +++ b/src/arch/x86_64/pit.rs @@ -0,0 +1,29 @@ +use core::arch::asm; + +use crate::arch::port::write_u8; + +const PIT_CHANNEL_0: u16 = 0x40; +const PIT_COMMAND: u16 = 0x43; + +const CHANNEL_0: u8 = 0b00 << 6; +// access mode = lobyte/hibyte +const LOW_HIGH: u8 = 0b11 << 4; +// interrupt on terminal count +const MODE_0: u8 = 0b000 << 1; +const BINARY: u8 = 0; + +pub const PIT_FREQUENCY: u64 = 1_193_182; +pub const PIT_CALIBRATION_COUNT: u16 = u16::MAX; + +pub struct Pit; + +impl Pit { + pub fn start_one_shot(count: u16) { + unsafe { + write_u8(PIT_COMMAND, CHANNEL_0 | LOW_HIGH | MODE_0 | BINARY); + + write_u8(PIT_CHANNEL_0, count as u8); + write_u8(PIT_CHANNEL_0, (count >> 8) as u8); + } + } +} diff --git a/src/arch/x86_64/port.rs b/src/arch/x86_64/port.rs index 4b4ba60..83f4e2a 100644 --- a/src/arch/x86_64/port.rs +++ b/src/arch/x86_64/port.rs @@ -8,7 +8,7 @@ pub unsafe fn read_u8(port: u16) -> u8 { "in al, dx", in("dx") port, out("al") value, - options(nomem, nostack, preserves_flags), + options(nostack, preserves_flags), ); } value @@ -62,7 +62,7 @@ pub unsafe fn write_u8(port: u16, value: u8) { "out dx, al", in("dx") port, in("al") value, - options(nomem, nostack, preserves_flags), + options(nostack, preserves_flags), ); } } diff --git a/src/arch/x86_64/timer.rs b/src/arch/x86_64/timer.rs new file mode 100644 index 0000000..ae462b7 --- /dev/null +++ b/src/arch/x86_64/timer.rs @@ -0,0 +1,82 @@ +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + +use crate::{ + arch::{ + apic::{self, LocalApic}, + disable_interrupts, + io_apic::IoApic, + x86_64::{ + interrupts::enable_interrupts, + pit::{PIT_CALIBRATION_COUNT, PIT_FREQUENCY, Pit}, + }, + }, + platform::acpi::IsaIrqRoute, +}; + +static PIT_FIRED: AtomicBool = AtomicBool::new(false); +static LAPIC_COUNT_AT_PIT: AtomicU32 = AtomicU32::new(0); + +#[derive(Debug)] +pub enum TimerCalibrationError { + PitTimeout, + IoApicNotHandled, + FrequencyOverflow, + InvalidTimerCount, +} + +pub fn calibrate_local_apic( + local_apic: &mut LocalApic, + io_apic: &mut IoApic, + pit_route: IsaIrqRoute, +) -> Result { + PIT_FIRED.store(false, Ordering::SeqCst); + LAPIC_COUNT_AT_PIT.store(0, Ordering::SeqCst); + + io_apic + .unmask(pit_route.gsi) + .map_err(|_| TimerCalibrationError::IoApicNotHandled)?; + local_apic.start_calibration_counter(); + + Pit::start_one_shot(PIT_CALIBRATION_COUNT); + + enable_interrupts(); + + while !PIT_FIRED.load(Ordering::SeqCst) { + if apic::current_timer_count() == 0 { + disable_interrupts(); + let _ = io_apic.mask(pit_route.gsi); + local_apic.stop_timer(); + + return Err(TimerCalibrationError::PitTimeout); + } + + core::hint::spin_loop(); + } + + disable_interrupts(); + + io_apic + .mask(pit_route.gsi) + .map_err(|_| TimerCalibrationError::IoApicNotHandled)?; + local_apic.stop_timer(); + + let elapsed = u32::MAX - LAPIC_COUNT_AT_PIT.load(Ordering::SeqCst); + + if elapsed == 0 { + return Err(TimerCalibrationError::InvalidTimerCount); + } + + let ticks_per_second = (elapsed as u64) + .checked_mul(PIT_FREQUENCY) + .ok_or(TimerCalibrationError::FrequencyOverflow)? + / PIT_CALIBRATION_COUNT as u64; + + Ok(ticks_per_second) +} + +pub fn record_pit_calibration() { + let current = apic::current_timer_count(); + + LAPIC_COUNT_AT_PIT.store(current, Ordering::SeqCst); + PIT_FIRED.store(true, Ordering::SeqCst); +} diff --git a/src/boot/limine.rs b/src/boot/limine.rs index 91acf06..0cfcf97 100644 --- a/src/boot/limine.rs +++ b/src/boot/limine.rs @@ -1,12 +1,14 @@ use ::limine as limine_api; +use limine::paging::PagingMode; +use limine::request::{PagingModeRequest, RsdpRequest}; use limine_api::request::{ExecutableAddressRequest, HhdmRequest, MemmapRequest}; use limine_api::{BaseRevision, RequestsEndMarker, RequestsStartMarker}; use crate::memory::{ - KernelMemoryLayout, KernelSegment, PagePermissions, PhysicalAddr, VirtualAddr, + KernelMemoryLayout, KernelSegment, MemoryMap, MemoryRegion, MemoryRegionKind, PagePermissions, + PhysicalAddr, VirtualAddr, }; -use crate::println; /// Sets the base revision to the latest revision supported by the crate. /// See specification for further info. @@ -28,6 +30,15 @@ static HHDM_REQUEST: HhdmRequest = HhdmRequest::new(); #[unsafe(link_section = ".requests")] static MEMMAP_REQUEST: MemmapRequest = MemmapRequest::new(); +#[used] +#[unsafe(link_section = ".requests")] +static PAGING_REQUEST: PagingModeRequest = + PagingModeRequest::new(PagingMode::MAX, PagingMode::MAX, PagingMode::MIN); + +#[used] +#[unsafe(link_section = ".requests")] +static RSDP_REQUEST: RsdpRequest = RsdpRequest::new(); + /// Define the stand and end markers for Limine requests. #[used] #[unsafe(link_section = ".requests_start_marker")] @@ -50,32 +61,13 @@ unsafe extern "C" { pub struct BootInfo { pub kernel_layout: KernelMemoryLayout, pub hhdm_offset: usize, - entries: &'static [&'static limine_api::memmap::Entry], + pub memory_map: MemoryMap, + pub rsdp: VirtualAddr, } impl BootInfo { - pub fn memory_regions(&self) -> impl Iterator + Clone + '_ { - use crate::memory::{MemoryRegion, MemoryRegionKind}; - self.entries.iter().map(|&entry| MemoryRegion { - start: crate::memory::PhysicalAddr::new(entry.base as usize), - length: entry.length as usize, - kind: match entry.type_ { - limine_api::memmap::MEMMAP_USABLE => MemoryRegionKind::Usable, - limine_api::memmap::MEMMAP_RESERVED => MemoryRegionKind::Reserved, - limine_api::memmap::MEMMAP_ACPI_RECLAIMABLE => MemoryRegionKind::AcpiReclaimable, - limine_api::memmap::MEMMAP_ACPI_NVS => MemoryRegionKind::AcpiNvs, - limine_api::memmap::MEMMAP_BAD_MEMORY => MemoryRegionKind::BadMemory, - limine_api::memmap::MEMMAP_BOOTLOADER_RECLAIMABLE => { - MemoryRegionKind::BootloaderReclaimable - } - limine_api::memmap::MEMMAP_EXECUTABLE_AND_MODULES => { - MemoryRegionKind::KernelAndModules - } - limine_api::memmap::MEMMAP_FRAMEBUFFER => MemoryRegionKind::Framebuffer, - limine_api::memmap::MEMMAP_MAPPED_RESERVED => MemoryRegionKind::MappedReserved, - _ => MemoryRegionKind::Reserved, - }, - }) + pub fn memory_regions(&self) -> impl Iterator + Clone + '_ { + self.memory_map.iter() } } @@ -85,7 +77,9 @@ pub enum BootError { FailedToGetKernelAddress, FailedToGetHHDMAddress, FailedToGetMemmap, + TooManyMemoryRegions, FailedToLocateKernel, + FailedToGetRsdp, } pub fn load_boot_info() -> Result { @@ -101,11 +95,41 @@ pub fn load_boot_info() -> Result { .ok_or(BootError::FailedToGetHHDMAddress)? .offset; + let rsdp = RSDP_REQUEST.response().ok_or(BootError::FailedToGetRsdp)?; + let memmap = MEMMAP_REQUEST .response() .ok_or(BootError::FailedToGetMemmap)? .entries(); + let mut memory_map = MemoryMap::new(); + for entry in memmap.iter() { + memory_map + .push(MemoryRegion { + start: PhysicalAddr::new(entry.base as usize), + length: entry.length as usize, + kind: match entry.type_ { + limine_api::memmap::MEMMAP_USABLE => MemoryRegionKind::Usable, + limine_api::memmap::MEMMAP_RESERVED => MemoryRegionKind::Reserved, + limine_api::memmap::MEMMAP_ACPI_RECLAIMABLE => { + MemoryRegionKind::AcpiReclaimable + } + limine_api::memmap::MEMMAP_ACPI_NVS => MemoryRegionKind::AcpiNvs, + limine_api::memmap::MEMMAP_BAD_MEMORY => MemoryRegionKind::BadMemory, + limine_api::memmap::MEMMAP_BOOTLOADER_RECLAIMABLE => { + MemoryRegionKind::BootloaderReclaimable + } + limine_api::memmap::MEMMAP_EXECUTABLE_AND_MODULES => { + MemoryRegionKind::KernelAndModules + } + limine_api::memmap::MEMMAP_FRAMEBUFFER => MemoryRegionKind::Framebuffer, + limine_api::memmap::MEMMAP_MAPPED_RESERVED => MemoryRegionKind::MappedReserved, + _ => MemoryRegionKind::Reserved, + }, + }) + .map_err(|_| BootError::TooManyMemoryRegions)?; + } + let mut segment_physical = kernel_address.physical_base as usize; let mut segment_virtual = core::ptr::addr_of!(__text_start) as usize; @@ -164,6 +188,11 @@ pub fn load_boot_info() -> Result { kernel_length = Some(entry.length as usize); break; } + + if kernel_length.is_none() { + return Err(BootError::FailedToLocateKernel); + } + debug_assert_eq!(segment_length, kernel_length.unwrap()); } @@ -176,6 +205,7 @@ pub fn load_boot_info() -> Result { ], }, hhdm_offset: hhdm_offset as usize, - entries: memmap, + memory_map, + rsdp: VirtualAddr::new(rsdp.address as usize), }) } diff --git a/src/boot/mod.rs b/src/boot/mod.rs index 746c015..6d6bb01 100644 --- a/src/boot/mod.rs +++ b/src/boot/mod.rs @@ -1,3 +1,3 @@ mod limine; -pub use limine::{BootError, BootInfo, load_boot_info}; +pub use limine::{BootInfo, load_boot_info}; diff --git a/src/debug/serial.rs b/src/debug/serial.rs index c86a5cd..ea54142 100644 --- a/src/debug/serial.rs +++ b/src/debug/serial.rs @@ -9,13 +9,13 @@ mod register { pub const INTERRUPT_ENABLE: u16 = 1; pub const DIVISOR_HIGH: u16 = 1; - pub const INTERRUPT_IDENTIFICATION: u16 = 2; + // pub const INTERRUPT_IDENTIFICATION: u16 = 2; pub const FIFO_CONTROL: u16 = 2; pub const LINE_CONTROL: u16 = 3; pub const MODEM_CONTROL: u16 = 4; pub const LINE_STATUS: u16 = 5; - pub const MODEM_STATUS: u16 = 6; + // pub const MODEM_STATUS: u16 = 6; pub const SCRATCH: u16 = 7; } diff --git a/src/main.rs b/src/main.rs index 6539ede..90c1c41 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,12 +7,99 @@ mod arch; mod boot; mod debug; mod memory; +mod platform; use crate::{ debug::serial, - memory::{AddressSpace, PagePermissions, PhysicalAddr, VirtualAddr}, + memory::{AddressSpace, MemoryRegionKind, VirtualAddr}, }; +#[derive(Debug)] +enum KernelStackCreateError { + AddressOverflow, + OutOfFrames, + GuardPageMapped, + Map(memory::MapError), +} + +struct KernelStack { + start: memory::VirtualAddr, + pages: usize, +} + +const BOOTSTRAP_STACK_TOP: usize = 0xFFFF_FFFE_0000_0000; +const KERNEL_STACK_SIZE: usize = 64 * 1024; +const KERNEL_STACK_GUARD_SIZE: usize = memory::FRAME_SIZE; + +const BOOTSTRAP_STACK_START: usize = BOOTSTRAP_STACK_TOP - KERNEL_STACK_SIZE; +const BOOTSTRAP_STACK_GUARD: usize = BOOTSTRAP_STACK_START - KERNEL_STACK_GUARD_SIZE; + +impl KernelStack { + fn allocate( + address_space: &mut AddressSpace, + allocator: &mut memory::FrameAllocator, + ) -> Result { + let kernel_stack_start = VirtualAddr::new(BOOTSTRAP_STACK_START); + + let guard_page = VirtualAddr::new(BOOTSTRAP_STACK_GUARD); + + if address_space.to_physical(guard_page).is_some() { + // guard page should be *unmapped* so we get a page fault if we try to access it + return Err(KernelStackCreateError::GuardPageMapped); + } + + let mut i = 0; + // on error we will just leak the frames + // because like what are going to do if we recover them? Die happily without leaking frames? idgaf + while i < KERNEL_STACK_SIZE { + let frame = allocator + .alloc() + .ok_or(KernelStackCreateError::OutOfFrames)?; + address_space + .map( + frame.frame_address().start_address(), + VirtualAddr::new( + kernel_stack_start + .as_usize() + .checked_add(i) + .ok_or(KernelStackCreateError::AddressOverflow)?, + ), + memory::PagePermissions::new(true, false, false), + allocator, + memory::CachePolicy::WriteBack, + ) + .map_err(|err| KernelStackCreateError::Map(err))?; + i += memory::FRAME_SIZE; + } + + debug_assert!(address_space.to_physical(guard_page).is_none()); + + Ok(Self { + start: kernel_stack_start, + pages: i / memory::FRAME_SIZE, + }) + } + + pub fn top(&self) -> Result { + Ok(VirtualAddr::new( + self.start + .as_usize() + .checked_add(self.pages * memory::FRAME_SIZE) + .ok_or(KernelStackCreateError::AddressOverflow)?, + )) + } +} + +pub struct KernelHandoff { + allocator: memory::FrameAllocator, + address_space: AddressSpace, + direct_map: memory::DirectMap, + boot_info: boot::BootInfo, + handoff_frame: memory::OwnedFrame, +} + +const _: () = assert!(core::mem::size_of::() <= memory::FRAME_SIZE); + #[unsafe(no_mangle)] pub extern "C" fn _start() -> ! { serial::init().unwrap(); @@ -25,7 +112,7 @@ pub extern "C" fn _start() -> ! { .expect("failed to create frame allocator"); println!("Initializing page table..."); - let mut page_table = AddressSpace::new_kernel( + let mut address_space = AddressSpace::new_kernel( direct_map, boot_info.memory_regions(), &boot_info.kernel_layout, @@ -34,65 +121,84 @@ pub extern "C" fn _start() -> ! { ) .expect("failed to create page table"); - println!("Activating page table..."); + println!("Entering kernel main..."); - // safety: trust me bro - unsafe { page_table.activate() }; + let bootstrap_stack = KernelStack::allocate(&mut address_space, &mut allocator) + .expect("failed to allocate bootstrap stack"); - println!("Allocating a frame..."); + let handoff_frame = allocator + .alloc() + .expect("failed to allocate frame for kernel handoff"); + let handoff_addr = address_space + .to_virtual(handoff_frame.frame_address().start_address()) + .expect("failed to map kernel handoff"); - let frame = allocator.alloc().unwrap(); + let bootstrap_stack_top = bootstrap_stack.top().unwrap(); + let handoff = KernelHandoff { + allocator, + address_space, + direct_map, + boot_info, + handoff_frame, + }; - let direct_mapped = direct_map.translate(frame.start_address()).unwrap(); - let translated = page_table.translate(direct_mapped).unwrap(); - - println!("{:?}", translated); - - let new_virtual = VirtualAddr::new(0x8000_0000); - - assert!(page_table.translate(new_virtual).is_none()); - - page_table - .map( - frame.start_address(), - new_virtual, - PagePermissions { - writable: true, - executable: false, - user_accessible: true, - }, - &mut allocator, - ) - .unwrap(); - - assert_eq!( - page_table.translate(new_virtual), - Some(frame.start_address()) - ); - - assert_eq!( - page_table.translate(VirtualAddr::new(new_virtual.as_usize() + 123)), - Some(PhysicalAddr::new(frame.start_address().as_usize() + 123)) - ); - - // write to the page and read it back via HHDM unsafe { - core::ptr::write_bytes(new_virtual.as_mut_ptr::(), 0xFF, 0x1000); + handoff_addr.as_mut_ptr::().write(handoff); + (*handoff_addr.as_mut_ptr::()) + .address_space + .activate(); + + arch::enter_kernel( + bootstrap_stack_top, + handoff_addr.as_mut_ptr::(), + ); + } +} + +pub unsafe extern "C" fn kernel_main(handoff: *mut KernelHandoff) -> ! { + let (mut allocator, mut address_space, direct_map, boot_info, handoff_frame) = unsafe { + let handoff = handoff.read(); + ( + handoff.allocator, + handoff.address_space, + handoff.direct_map, + handoff.boot_info, + handoff.handoff_frame, + ) + }; + + unsafe { + allocator.dealloc(handoff_frame); } - let slice = unsafe { core::slice::from_raw_parts(direct_mapped.as_ptr::(), 0x1000) }; + allocator.reclaim_regions( + boot_info.memory_regions(), + MemoryRegionKind::BootloaderReclaimable, + ); - assert!(slice.iter().all(|&byte| byte == 0xFF)); + println!( + "Initializing local ACPI... {:#X}", + boot_info.rsdp.as_usize() + ); - println!("{:#X}", slice[0]); + let acpi = platform::acpi::init(&boot_info, direct_map).expect("failed to initialize ACPI"); - let unmapped_frame = unsafe { page_table.unmap(new_virtual, &mut allocator) }.unwrap(); + let madt = acpi + .madt() + .expect("failed to parse ACPI") + .expect("MADT not found"); - assert_eq!(unmapped_frame, frame); + let interrupt_controller = + arch::init_interrupt_controller(&madt, &mut allocator, &mut address_space) + .expect("failed to initialize interrupt controller"); - unsafe { allocator.dealloc(unmapped_frame) }; + println!("interrupt controller: {:?}", interrupt_controller); - assert!(page_table.translate(new_virtual).is_none()); + println!("delaying 5 seconds..."); + interrupt_controller + .delay(core::time::Duration::from_secs(5)) + .unwrap(); + println!("done!"); hcf(); } diff --git a/src/memory/address_space.rs b/src/memory/address_space.rs index 0912548..8af0091 100644 --- a/src/memory/address_space.rs +++ b/src/memory/address_space.rs @@ -1,9 +1,10 @@ use crate::{ arch::{PageTable, PageTableCreateError, PageTableMapError, PageTableUnmapError, PagingConfig}, memory::{ - DirectMap, FRAME_SIZE, FrameAllocator, KernelMemoryLayout, MemoryRegion, MemoryRegionKind, - PagePermissions, PhysicalAddr, PhysicalFrame, VirtualAddr, + CachePolicy, DirectMap, FRAME_SIZE, FrameAddr, FrameAllocator, KernelMemoryLayout, + MemoryRegion, MemoryRegionKind, PagePermissions, PhysicalAddr, VirtualAddr, }, + println, }; #[derive(Debug)] @@ -105,28 +106,39 @@ impl AddressSpace { // map hhdm for region in memory_regions.clone() { - if region.kind == MemoryRegionKind::Reserved - || region.kind == MemoryRegionKind::BadMemory - { + // we exlcude KernelAndModules from the hhdm because if it were mapped, it would + // undermind the permissions of the explicitly mapped kernel image + if matches!( + region.kind, + MemoryRegionKind::Reserved + | MemoryRegionKind::BadMemory + | MemoryRegionKind::KernelAndModules + ) { continue; } + let cache_policy = if matches!( + region.kind, + MemoryRegionKind::MappedReserved | MemoryRegionKind::Framebuffer + ) { + CachePolicy::Uncacheable + } else { + CachePolicy::WriteBack + }; + let res = space.map_range( region.start, direct_map .translate(region.start) .ok_or(AddressSpaceCreateError::AddressOutsideDirectMap)?, region.length, - PagePermissions { - writable: true, - executable: false, - user_accessible: false, - }, + PagePermissions::new(true, false, false), allocator, + cache_policy, ); if let Err(err) = res { - space.destroy(allocator); + unsafe { space.destroy(allocator) }; return Err(AddressSpaceCreateError::Map(err)); } } @@ -146,10 +158,11 @@ impl AddressSpace { segment.length, segment.permissions, allocator, + CachePolicy::WriteBack, ); if let Err(err) = res { - space.destroy(allocator); + unsafe { space.destroy(allocator) }; return Err(AddressSpaceCreateError::Map(err)); } } @@ -167,12 +180,13 @@ impl AddressSpace { virtual_addr: VirtualAddr, permissions: PagePermissions, allocator: &mut FrameAllocator, + cache_policy: CachePolicy, ) -> Result<(), MapError> { - let frame = PhysicalFrame::from_start_address(physical_addr) + let frame = FrameAddr::from_start_address(physical_addr) .ok_or(MapError::PhysicalAddressUnaligned)?; self.root - .map(virtual_addr, frame, permissions, allocator) + .map(virtual_addr, frame, permissions, allocator, cache_policy) .map_err(MapError::from) } @@ -183,6 +197,7 @@ impl AddressSpace { length: usize, permissions: PagePermissions, allocator: &mut FrameAllocator, + cache_policy: CachePolicy, ) -> Result<(), MapError> { if length == 0 { return Ok(()); @@ -219,18 +234,25 @@ impl AddressSpace { let physical_addr = PhysicalAddr::new(physical_start.as_usize() + offset); let virtual_addr = VirtualAddr::new(virtual_start.as_usize() + offset); - if let Err(err) = self.map(physical_addr, virtual_addr, permissions, allocator) { + if let Err(err) = self.map( + physical_addr, + virtual_addr, + permissions, + allocator, + cache_policy, + ) { for rollback_idx in (0..mapped_pages).rev() { let rollback_offset = rollback_idx * FRAME_SIZE; - let rollback_physical_addr = - PhysicalAddr::new(physical_start.as_usize() + rollback_offset); unsafe { - self.unmap( - VirtualAddr::new(virtual_start.as_usize() + rollback_offset), - allocator, - ) - .expect("failed to roll back a mapped page"); + // make sure we use the root page tableq directl since the public AddressSpace API + // might *at some point* reject kernel mappings + self.root + .unmap( + VirtualAddr::new(virtual_start.as_usize() + rollback_offset), + allocator, + ) + .expect("failed to roll back a mapped page"); } } @@ -246,24 +268,34 @@ impl AddressSpace { /// # Safety /// /// The caller must ensure: - /// - The caller must ensure that the page is not currently in use + /// - The page is not currently in use pub unsafe fn unmap( &mut self, virtual_addr: VirtualAddr, allocator: &mut FrameAllocator, - ) -> Result { + ) -> Result { unsafe { self.root.unmap(virtual_addr, allocator) }.map_err(UnmapError::from) } - pub fn translate(&self, virtual_addr: VirtualAddr) -> Option { - self.root.translate(virtual_addr) + pub fn to_physical(&self, virtual_addr: VirtualAddr) -> Option { + self.root.to_physical(virtual_addr) + } + + pub fn to_virtual(&self, physical_addr: PhysicalAddr) -> Option { + self.root.to_virtual(physical_addr) } pub unsafe fn activate(&self) { unsafe { self.root.activate() } } - pub fn destroy(self, allocator: &mut FrameAllocator) { - todo!("destroy address space") + /// # Safety + /// + /// The caller must ensure this apge table is not active on any CPU and + /// no CPU or kernel operation can access its paging structures. + pub unsafe fn destroy(self, allocator: &mut FrameAllocator) { + unsafe { + self.root.destroy(allocator); + } } } diff --git a/src/memory/frame.rs b/src/memory/frame.rs index 4ab1fdd..415e734 100644 --- a/src/memory/frame.rs +++ b/src/memory/frame.rs @@ -87,7 +87,7 @@ impl FrameAllocator { let mut highest_frame: Option = None; for region in regions.clone() { - if region.kind != MemoryRegionKind::Usable { + if !Self::should_track(region.kind) { continue; } @@ -110,7 +110,7 @@ impl FrameAllocator { let mut bitmap_start_frame: Option = None; for region in regions.clone() { - if region.kind != MemoryRegionKind::Usable { + if !Self::can_store_bitmap(region.kind) { continue; } @@ -152,7 +152,7 @@ impl FrameAllocator { .ok_or(FrameAllocatorInitError::AddressOverflow)?; for region in regions { - if region.kind != MemoryRegionKind::Usable { + if !Self::is_initially_free(region.kind) { continue; } @@ -178,6 +178,23 @@ impl FrameAllocator { }) } + fn should_track(kind: MemoryRegionKind) -> bool { + matches!( + kind, + MemoryRegionKind::Usable + | MemoryRegionKind::BootloaderReclaimable + | MemoryRegionKind::AcpiReclaimable + ) + } + + fn can_store_bitmap(kind: MemoryRegionKind) -> bool { + kind == MemoryRegionKind::Usable + } + + fn is_initially_free(kind: MemoryRegionKind) -> bool { + kind == MemoryRegionKind::Usable + } + fn find_free_in(&self, start: usize, end: usize) -> Option { (start..end).find(|&index| self.bitmap.state(index) == FrameState::Free) } @@ -187,7 +204,35 @@ impl FrameAllocator { .or_else(|| self.find_free_in(0, self.next_search)) } - pub fn alloc_nozero(&mut self) -> Option { + pub fn reclaim_regions + Clone>( + &mut self, + memory_map: I, + region_kind: MemoryRegionKind, + ) { + if !matches!( + region_kind, + MemoryRegionKind::BootloaderReclaimable | MemoryRegionKind::AcpiReclaimable + ) { + return; + } + + for region in memory_map { + if region.kind == region_kind { + for frame_idx in Self::usable_frame_range(region).expect("invalid memory region") { + if self.bitmap.state(frame_idx) != FrameState::Reserved { + continue; + } + + self.bitmap.set_state(frame_idx, FrameState::Free); + self.allocatable_frames += 1; + self.free_frames += 1; + self.next_search = self.next_search.min(frame_idx); + } + } + } + } + + pub fn alloc_nozero(&mut self) -> Option { if self.free_frames == 0 { return None; } @@ -198,15 +243,15 @@ impl FrameAllocator { self.free_frames -= 1; self.next_search = frame_idx.saturating_add(1); - Some(PhysicalFrame::from_index(frame_idx)) + Some(OwnedFrame::new(FrameAddr::from_index(frame_idx))) } - pub fn alloc(&mut self) -> Option { + pub fn alloc(&mut self) -> Option { let frame = self.alloc_nozero()?; let start = self .direct_map - .translate(frame.start_address()) + .translate(frame.frame_address().start_address()) .expect("frame is outside the direct map"); unsafe { @@ -222,7 +267,7 @@ impl FrameAllocator { /// - The frame is currently owned by the caller /// - it is not currently in use /// - it has not been freed - pub unsafe fn dealloc(&mut self, frame: PhysicalFrame) { + pub unsafe fn dealloc(&mut self, frame: OwnedFrame) { let frame_idx = frame.index(); match self.bitmap.state(frame_idx) { @@ -265,9 +310,9 @@ impl FrameAllocator { #[repr(transparent)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct PhysicalFrame(PhysicalAddr); +pub struct FrameAddr(PhysicalAddr); -impl PhysicalFrame { +impl FrameAddr { pub fn from_start_address(address: PhysicalAddr) -> Option { if address.as_usize() % FRAME_SIZE != 0 { return None; @@ -288,3 +333,31 @@ impl PhysicalFrame { self.0.as_usize() / FRAME_SIZE } } + +// specifically not Clone or Copy +#[derive(Debug)] +pub struct OwnedFrame { + frame: FrameAddr, +} + +impl OwnedFrame { + fn new(frame: FrameAddr) -> Self { + Self { frame } + } + + pub fn into_raw(self) -> FrameAddr { + self.frame + } + + pub unsafe fn from_raw(frame: FrameAddr) -> Self { + Self { frame } + } + + pub fn frame_address(&self) -> FrameAddr { + self.frame + } + + pub fn index(&self) -> usize { + self.frame.index() + } +} diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 931e595..c448efc 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -2,7 +2,7 @@ mod address_space; mod frame; pub use address_space::{AddressSpace, AddressSpaceCreateError, MapError, UnmapError}; -pub use frame::{FRAME_SIZE, FrameAllocator, PhysicalFrame}; +pub use frame::{FRAME_SIZE, FrameAddr, FrameAllocator, OwnedFrame}; pub struct KernelSegment { pub physical_base: PhysicalAddr, @@ -37,7 +37,7 @@ impl PagePermissions { pub struct PhysicalAddr(usize); impl PhysicalAddr { - pub fn new(addr: usize) -> Self { + pub const fn new(addr: usize) -> Self { Self(addr) } @@ -51,7 +51,7 @@ impl PhysicalAddr { pub struct VirtualAddr(usize); impl VirtualAddr { - pub fn new(addr: usize) -> Self { + pub const fn new(addr: usize) -> Self { Self(addr) } @@ -59,15 +59,16 @@ impl VirtualAddr { self.0 as usize } - pub unsafe fn as_mut_ptr(self) -> *mut T { + pub const unsafe fn as_mut_ptr(self) -> *mut T { self.as_usize() as *mut T } - pub unsafe fn as_ptr(self) -> *const T { + pub const unsafe fn as_ptr(self) -> *const T { self.as_usize() as *const T } } +#[repr(u8)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MemoryRegionKind { Usable, @@ -81,6 +82,12 @@ pub enum MemoryRegionKind { MappedReserved, } +#[derive(Clone, Copy, Debug)] +pub enum CachePolicy { + Uncacheable, + WriteBack, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct MemoryRegion { pub start: PhysicalAddr, @@ -104,3 +111,44 @@ impl DirectMap { .map(VirtualAddr::new) } } + +const EMPTY_MEMORY_REGION: MemoryRegion = MemoryRegion { + start: PhysicalAddr::new(0), + length: 0, + kind: MemoryRegionKind::Reserved, +}; +const MAX_MEMORY_REGIONS: usize = 128; + +pub enum MemoryMapError { + TooManyRegions, +} + +#[derive(Clone, Copy)] +pub struct MemoryMap { + pub entries: [MemoryRegion; MAX_MEMORY_REGIONS], + pub len: usize, +} + +impl MemoryMap { + pub const fn new() -> Self { + Self { + entries: [EMPTY_MEMORY_REGION; MAX_MEMORY_REGIONS], + len: 0, + } + } + + pub fn push(&mut self, entry: MemoryRegion) -> Result<(), MemoryMapError> { + if self.len >= MAX_MEMORY_REGIONS { + return Err(MemoryMapError::TooManyRegions); + } + + self.entries[self.len] = entry; + self.len += 1; + + Ok(()) + } + + pub fn iter(&self) -> impl Iterator + Clone + '_ { + self.entries[..self.len].iter().copied() + } +} diff --git a/src/platform/acpi.rs b/src/platform/acpi.rs new file mode 100644 index 0000000..9bde1ea --- /dev/null +++ b/src/platform/acpi.rs @@ -0,0 +1,659 @@ +use crate::memory::{ + AddressSpace, CachePolicy, DirectMap, FrameAllocator, PhysicalAddr, VirtualAddr, +}; + +#[derive(Debug)] +pub enum AcpiError { + InvalidInput, + AddressOverflow, + InvalidSdtLength, + InvalidRootTableLength, + MalformedAcpiTable, + MalformedMadt, + MissingMadt, + MissingIoApic, + MissingIsaIrqRoute, + MultipleIoApicsUnsupported, +} + +#[derive(Debug)] +pub struct AcpiTables { + direct_map: DirectMap, + root: RootTable, +} + +impl AcpiTables { + pub fn madt(&self) -> Result>, AcpiError> { + if let Some(table) = self.find_table(*b"APIC")? { + return Ok(Some(Madt::parse(self, table)?)); + }; + + return Ok(None); + } + + fn read(&self, physical_addr: PhysicalAddr) -> Result { + let virtual_addr = self + .direct_map + .translate(physical_addr) + .ok_or(AcpiError::AddressOverflow)?; + + Ok(unsafe { core::ptr::read_unaligned(virtual_addr.as_ptr::()) }) + } + + fn checksum_valid( + &self, + physical_addr: PhysicalAddr, + length: usize, + ) -> Result { + let virtual_addr = self + .direct_map + .translate(physical_addr) + .ok_or(AcpiError::AddressOverflow)?; + + let mut sum: u8 = 0; + for i in 0..length { + let byte_addr = virtual_addr + .as_usize() + .checked_add(i) + .ok_or(AcpiError::AddressOverflow)?; + + let byte = unsafe { core::ptr::read_unaligned(byte_addr as *const u8) }; + sum = sum.wrapping_add(byte); + } + + Ok(sum == 0) + } + + pub unsafe fn from_rsdp( + rsdp_ptr: VirtualAddr, + direct_map: DirectMap, + ) -> Result { + let rsdp = unsafe { core::ptr::read_unaligned(rsdp_ptr.as_ptr::()) }; + + let mut sum: u8 = 0; + for i in 0..core::mem::size_of::() as usize { + sum = sum.wrapping_add(unsafe { + core::ptr::read_unaligned::(rsdp_ptr.as_ptr::().add(i)) + }); + } + + if sum != 0 { + return Err(AcpiError::MalformedAcpiTable); + } + + if rsdp.signature != *b"RSD PTR " { + return Err(AcpiError::MalformedAcpiTable); + } + + let root_table = if rsdp.revision >= 2 { + let xsdp = unsafe { core::ptr::read_unaligned(rsdp_ptr.as_ptr::()) }; + + if xsdp.length < core::mem::size_of::() as u32 { + return Err(AcpiError::MalformedAcpiTable); + } + + sum = 0; + for i in 0..xsdp.length as usize { + sum = sum.wrapping_add(unsafe { + core::ptr::read_unaligned::(rsdp_ptr.as_ptr::().add(i)) + }); + } + + if sum != 0 { + return Err(AcpiError::MalformedAcpiTable); + } + + let xsdt_address = PhysicalAddr::new(xsdp.xsdt_address as usize); + let xsdt_virtual_addr = unsafe { + direct_map + .translate(xsdt_address) + .ok_or(AcpiError::AddressOverflow)? + .as_ptr::() + }; + let xsdt = unsafe { core::ptr::read_unaligned(xsdt_virtual_addr) }; + + sum = 0; + for i in 0..xsdt.length as usize { + sum = sum.wrapping_add(unsafe { + core::ptr::read_unaligned::(xsdt_virtual_addr.cast::().add(i)) + }); + } + + if sum != 0 { + return Err(AcpiError::MalformedAcpiTable); + } + + if &xsdt.signature != b"XSDT" { + return Err(AcpiError::MalformedAcpiTable); + } + + RootTable::Xsdt(Sdt { + physical_addr: xsdt_address, + length: xsdt.length as usize, + signature: xsdt.signature, + }) + } else { + let rsdt_address = PhysicalAddr::new(rsdp.rsdt_address as usize); + let rsdt_virtual_addr = unsafe { + direct_map + .translate(rsdt_address) + .ok_or(AcpiError::AddressOverflow)? + .as_ptr::() + }; + + let rsdt = unsafe { core::ptr::read_unaligned(rsdt_virtual_addr) }; + + sum = 0; + for i in 0..rsdt.length as usize { + sum = sum.wrapping_add(unsafe { + core::ptr::read_unaligned::(rsdt_virtual_addr.cast::().add(i)) + }); + } + + if sum != 0 { + return Err(AcpiError::MalformedAcpiTable); + } + + if &rsdt.signature != b"RSDT" { + return Err(AcpiError::MalformedAcpiTable); + } + + RootTable::Rsdt(Sdt { + physical_addr: rsdt_address, + length: rsdt.length as usize, + signature: rsdt.signature, + }) + }; + + Ok(Self { + direct_map: direct_map, + root: root_table, + }) + } + + pub fn find_table(&self, signature: [u8; 4]) -> Result, AcpiError> { + let root = self.root.table(); + let entry_width = self.root.entry_width(); + let entries_start = root + .physical_addr + .as_usize() + .checked_add(size_of::()) + .ok_or(AcpiError::AddressOverflow)?; + + for i in 0..self.root.entry_count()? { + let entry_addr = entries_start + .checked_add( + i.checked_mul(entry_width) + .ok_or(AcpiError::AddressOverflow)?, + ) + .ok_or(AcpiError::AddressOverflow)?; + + let table_addr = match self.root { + RootTable::Rsdt(_) => self.read::(PhysicalAddr::new(entry_addr))? as usize, + RootTable::Xsdt(_) => self.read::(PhysicalAddr::new(entry_addr))? as usize, + }; + + let physical_addr = PhysicalAddr::new(table_addr); + let header = self.read::(physical_addr)?; + + if header.length < size_of::() as u32 { + return Err(AcpiError::InvalidSdtLength); + } + + if header.signature != signature { + continue; + } + + if !self.checksum_valid(physical_addr, header.length as usize)? { + return Err(AcpiError::MalformedAcpiTable); + } + + return Ok(Some(Sdt { + physical_addr, + length: header.length as usize, + signature: header.signature, + })); + } + + Ok(None) + } +} + +#[derive(Debug)] +enum RootTable { + Rsdt(Sdt), + Xsdt(Sdt), +} + +impl RootTable { + fn table(&self) -> &Sdt { + match self { + RootTable::Rsdt(sdt) | RootTable::Xsdt(sdt) => sdt, + } + } + + const fn entry_width(&self) -> usize { + match self { + RootTable::Rsdt(_) => core::mem::size_of::(), + RootTable::Xsdt(_) => core::mem::size_of::(), + } + } + + fn entry_count(&self) -> Result { + let payload_length = self + .table() + .length + .checked_sub(size_of::()) + .ok_or(AcpiError::InvalidSdtLength)?; + + if payload_length % self.entry_width() != 0 { + return Err(AcpiError::InvalidRootTableLength); + } + + Ok(payload_length / self.entry_width()) + } +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +struct Rsdp { + signature: [u8; 8], + checksum: u8, + oem_id: [u8; 6], + revision: u8, + rsdt_address: u32, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +struct Xsdp { + rsdp: Rsdp, + length: u32, + xsdt_address: u64, + extended_checksum: u8, + reserved: [u8; 3], +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +struct SDTHeader { + signature: [u8; 4], + length: u32, + revision: u8, + checksum: u8, + oem_id: [u8; 6], + oem_table_id: [u8; 8], + oem_revision: u32, + creator_id: u32, + creator_revision: u32, +} + +#[derive(Debug)] +pub struct Sdt { + physical_addr: PhysicalAddr, + length: usize, + signature: [u8; 4], +} + +#[derive(Debug)] +pub struct Madt<'a> { + acpi: &'a AcpiTables, + table: Sdt, + pub local_apic_address: PhysicalAddr, + flags: u32, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +struct MadtBody { + local_apic_address: u32, + flags: u32, +} + +const MADT_ENTRIES_OFFSET: usize = size_of::() + size_of::(); + +impl<'a> Madt<'a> { + pub fn entries( + &self, + ) -> Result> + '_, AcpiError> { + Ok(MadtEntries { + acpi: self.acpi, + current: PhysicalAddr::new( + self.table + .physical_addr + .as_usize() + .checked_add(MADT_ENTRIES_OFFSET) + .ok_or(AcpiError::AddressOverflow)?, + ), + end: PhysicalAddr::new( + self.table + .physical_addr + .as_usize() + .checked_add(self.table.length) + .ok_or(AcpiError::AddressOverflow)?, + ), + }) + } + + pub fn parse(acpi: &'a AcpiTables, table: Sdt) -> Result { + if table.signature != *b"APIC" { + return Err(AcpiError::InvalidInput); + } + + if table.length < size_of::() + size_of::() { + return Err(AcpiError::MalformedAcpiTable); + } + + let body_addr = table + .physical_addr + .as_usize() + .checked_add(size_of::()) + .ok_or(AcpiError::AddressOverflow)?; + let body = acpi.read::(PhysicalAddr::new(body_addr))?; + + Ok(Madt { + acpi, + table, + local_apic_address: PhysicalAddr::new(body.local_apic_address as usize), + flags: body.flags, + }) + } + + pub fn effective_local_apic_address(&self) -> Result { + let mut address = self.local_apic_address; + + for result in self.entries()? { + let entry = result?; + + match entry { + MadtEntry::LocalApicAddressOverride(local_apic_address_override) => { + address = + PhysicalAddr::new(local_apic_address_override.local_apic_address as usize); + } + _ => {} + } + } + + Ok(address) + } + + pub fn io_apics( + &self, + ) -> Result> + '_, AcpiError> { + Ok(self.entries()?.filter_map(|result| match result { + Ok(MadtEntry::IoApic(io_apic)) => Some(Ok(IoApicInfo { + id: io_apic.id, + apic_address: PhysicalAddr::new(io_apic.apic_address as usize), + global_system_interrupt_base: io_apic.global_system_interrupt_base, + })), + Ok(_) => None, + Err(error) => Some(Err(error)), + })) + } + + pub fn sole_io_apic(&self) -> Result { + let mut entries = self.io_apics()?; + + let first = entries + .next() + .transpose()? + .ok_or(AcpiError::MissingIoApic)?; + + if entries.next().transpose()?.is_some() { + return Err(AcpiError::MultipleIoApicsUnsupported); + } + + Ok(first) + } + + pub fn isa_irq_route(&self, irq: u8) -> Result { + for entry in self.entries()? { + match entry? { + MadtEntry::InterruptSourceOverride(interrupt_source_override) => { + if interrupt_source_override.bus != 0 { + continue; + } + + if interrupt_source_override.source != irq { + continue; + } + + return Ok(IsaIrqRoute { + gsi: interrupt_source_override.global_interrupt, + polarity: match interrupt_source_override.flags & 0b11 { + 0 => InterruptPolarity::ActiveHigh, + 1 => InterruptPolarity::ActiveHigh, + 3 => InterruptPolarity::ActiveLow, + _ => Err(AcpiError::MalformedMadt)?, + }, + trigger: match interrupt_source_override.flags >> 2 & 0b11 { + 0 => TriggerMode::Edge, + 1 => TriggerMode::Edge, + 3 => TriggerMode::Level, + _ => Err(AcpiError::MalformedMadt)?, + }, + }); + } + _ => {} + } + } + + Ok(IsaIrqRoute { + gsi: irq as u32, + polarity: InterruptPolarity::ActiveHigh, + trigger: TriggerMode::Edge, + }) + } +} + +pub struct MadtEntries<'a> { + acpi: &'a AcpiTables, + current: PhysicalAddr, + end: PhysicalAddr, +} + +impl MadtEntries<'_> { + fn read_body(&self, header: &MadtEntryHeader) -> Result { + let required_length = size_of::() + size_of::(); + + if (header.length as usize) < required_length { + return Err(AcpiError::MalformedAcpiTable); + } + + let body_addr = self + .current + .as_usize() + .checked_add(size_of::()) + .ok_or(AcpiError::AddressOverflow)?; + + self.acpi.read::(PhysicalAddr::new(body_addr)) + } + + fn read_next(&mut self) -> Result { + let header = self.acpi.read::(self.current)?; + + let entry = match header.kind { + 0 => MadtEntry::LocalApic(self.read_body(&header)?), + 1 => MadtEntry::IoApic(self.read_body(&header)?), + 2 => MadtEntry::InterruptSourceOverride(self.read_body(&header)?), + 3 => MadtEntry::IoApicNmi(self.read_body(&header)?), + 4 => MadtEntry::LocalApicNmi(self.read_body(&header)?), + 5 => MadtEntry::LocalApicAddressOverride(self.read_body(&header)?), + 9 => MadtEntry::LocalX2Apic(self.read_body(&header)?), + _ => MadtEntry::Unknown { + kind: header.kind, + length: header.length, + }, + }; + + self.current = PhysicalAddr::new( + self.current + .as_usize() + .checked_add(header.length as usize) + .ok_or(AcpiError::AddressOverflow)?, + ); + + Ok(entry) + } +} + +impl<'a> Iterator for MadtEntries<'a> { + type Item = Result; + + fn next(&mut self) -> Option { + let current = self.current.as_usize(); + let end = self.end.as_usize(); + + if current == end { + return None; + } + + if current > end { + self.current = self.end; + return Some(Err(AcpiError::MalformedAcpiTable)); + } + + let remaining = end - current; + + if remaining < size_of::() { + self.current = self.end; + return Some(Err(AcpiError::MalformedAcpiTable)); + } + + let header = match self.acpi.read::(self.current) { + Ok(header) => header, + Err(error) => { + self.current = self.end; + return Some(Err(error)); + } + }; + + let length = header.length as usize; + + if length < size_of::() || length > remaining { + self.current = self.end; + return Some(Err(AcpiError::MalformedAcpiTable)); + } + + let res = self.read_next(); + + if res.is_err() { + self.current = self.end; + } + + Some(res) + } +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct MadtEntryHeader { + kind: u8, + length: u8, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct LocalApicEntry { + processor_id: u8, + id: u8, + flags: u32, +} + +#[derive(Debug)] +pub struct IoApicInfo { + pub id: u8, + pub apic_address: PhysicalAddr, + pub global_system_interrupt_base: u32, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct IoApicEntry { + id: u8, + reserved: u8, + apic_address: u32, + global_system_interrupt_base: u32, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct InterruptSourceOverride { + bus: u8, + source: u8, + global_interrupt: u32, + flags: u16, +} + +#[derive(Clone, Copy, Debug)] +pub enum InterruptPolarity { + ActiveHigh, + ActiveLow, +} + +#[derive(Clone, Copy, Debug)] +pub enum TriggerMode { + Edge, + Level, +} + +#[derive(Clone, Copy, Debug)] +pub struct IsaIrqRoute { + pub gsi: u32, + pub polarity: InterruptPolarity, + pub trigger: TriggerMode, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct IoApicNmiEntry { + nmi_source: u8, + reserved: u8, + flags: u16, + global_system_interrupt: u32, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct LocalApicNmiEntry { + processor_id: u8, + flags: u16, + lint_num: u8, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct LocalApicAddressOverride { + reserved: u16, + local_apic_address: u64, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct LocalX2ApicEntry { + reserved: u16, + local_x2apic_id: u32, + flags: u32, + acpi_processor_uid: u32, +} + +#[derive(Clone, Copy, Debug)] +pub enum MadtEntry { + LocalApic(LocalApicEntry), + IoApic(IoApicEntry), + InterruptSourceOverride(InterruptSourceOverride), + IoApicNmi(IoApicNmiEntry), + LocalApicNmi(LocalApicNmiEntry), + LocalApicAddressOverride(LocalApicAddressOverride), + LocalX2Apic(LocalX2ApicEntry), + Unknown { kind: u8, length: u8 }, +} + +pub fn init( + boot_info: &crate::boot::BootInfo, + direct_map: DirectMap, +) -> Result { + let acpi_table = unsafe { AcpiTables::from_rsdp(boot_info.rsdp, direct_map)? }; + + Ok(acpi_table) +} diff --git a/src/platform/mod.rs b/src/platform/mod.rs new file mode 100644 index 0000000..36731ae --- /dev/null +++ b/src/platform/mod.rs @@ -0,0 +1 @@ +pub mod acpi;