diff --git a/src/boot/efi.zig b/src/boot/efi.zig index 490ea6f..0a177fb 100644 --- a/src/boot/efi.zig +++ b/src/boot/efi.zig @@ -65,14 +65,22 @@ fn boot() !noreturn { log(") - booting without user space\r\n"); }; + // Build the page tables the kernel starts life on: identity + a physmap of + // low RAM, plus the higher-half kernel image once it links high. Allocated + // now, while boot services (and the memory map) are still stable — nothing + // is allocatable after ExitBootServices, and any allocation between fetching + // the map and exiting would invalidate the map key. + const cr3 = try buildBootstrapTables(bs, &boot_info); + log("danos: kernel loaded, exiting boot services\r\n"); boot_info.memory_map = try exitBootServices(bs); - // Hand control to the kernel. `danos.kernel_abi` is SysV, so the pointer is - // passed in RDI as the kernel expects — not RCX, which this UEFI binary's - // default `.c` convention (Microsoft x64) would use. - const kernel: *const fn (*const BootInfo) callconv(danos.kernel_abi) noreturn = @ptrFromInt(entry); - kernel(&boot_info); + // Switch onto our tables and jump to the kernel in one uninterruptible step. + // We load RDI explicitly (SysV first arg) rather than trusting this UEFI + // binary's Microsoft-x64 default, and jump straight to the (possibly + // higher-half) entry — the bootstrap tables map both the low loader code + // executing this and the kernel's link address. + handoff(cr3, entry, &boot_info); } /// A display resolution in pixels. @@ -209,6 +217,133 @@ fn loadKernel(bs: *uefi.tables.BootServices, boot_info: *BootInfo) !usize { return loadElf(bs, image, boot_info); } +// --- bootstrap page tables ------------------------------------------------- +// The kernel is (or will be) linked in the higher half but loaded low; the +// firmware's identity map doesn't cover the higher half, so the loader builds +// the first set of real page tables and switches CR3 before jumping in. They +// carry: an identity map of low RAM (so the loader's own code/stack executing +// the switch stays valid, and the low-linked kernel keeps working during the +// staged move), a physmap at danos.physmap_base (the kernel's permanent way to +// reach physical memory), and 4 KiB mappings of any higher-half kernel segment. +// The kernel later builds its own precise tables (paging.init) and abandons +// these; they leak as reserved LoaderData (~a handful of frames). + +const pte_present: u64 = 1 << 0; +const pte_write: u64 = 1 << 1; +const pte_ps: u64 = 1 << 7; // page-size: a 2 MiB leaf at the PD level +const pte_addr: u64 = 0x000F_FFFF_FFFF_F000; +const gib: u64 = 1 << 30; + +/// A bump allocator over a pre-reserved block of zeroed frames, for page tables. +const TablePool = struct { + base: usize, + next: usize, + cap: usize, + + fn alloc(self: *TablePool) !u64 { + if (self.next >= self.cap) return error.OutOfBootstrapFrames; + const frame = self.base + self.next * page_size; + self.next += 1; + @memset(@as(*[512]u64, @ptrFromInt(frame)), 0); + return frame; + } + + fn table(phys: u64) *[512]u64 { + return @ptrFromInt(phys); + } + + /// Return the next-level table an entry points at, creating it if absent. + fn descend(self: *TablePool, entry: *u64) !u64 { + if (entry.* & pte_present != 0) return entry.* & pte_addr; + const frame = try self.alloc(); + entry.* = frame | pte_present | pte_write; + return frame; + } + + fn map2M(self: *TablePool, pml4: u64, virt: u64, phys: u64) !void { + const pml4e = &table(pml4)[(virt >> 39) & 0x1FF]; + const pdpt = try self.descend(pml4e); + const pdpte = &table(pdpt)[(virt >> 30) & 0x1FF]; + const pd = try self.descend(pdpte); + table(pd)[(virt >> 21) & 0x1FF] = (phys & ~@as(u64, 0x1F_FFFF)) | pte_present | pte_write | pte_ps; + } + + fn map4K(self: *TablePool, pml4: u64, virt: u64, phys: u64) !void { + const pml4e = &table(pml4)[(virt >> 39) & 0x1FF]; + const pdpt = try self.descend(pml4e); + const pdpte = &table(pdpt)[(virt >> 30) & 0x1FF]; + const pd = try self.descend(pdpte); + const pde = &table(pd)[(virt >> 21) & 0x1FF]; + const pt = try self.descend(pde); + table(pt)[(virt >> 12) & 0x1FF] = (phys & pte_addr) | pte_present | pte_write; + } +}; + +/// Build the bootstrap tables and return the physical PML4 address (for CR3). +/// No NX bits are set anywhere, so EFER.NXE (still off here) is irrelevant. +fn buildBootstrapTables(bs: *uefi.tables.BootServices, boot_info: *const BootInfo) !u64 { + // 64 frames (256 KiB) — comfortably covers a PML4, two PDPTs, eight PDs for + // the 4 GiB identity+physmap ranges, plus the kernel image's PTs. + const pool_pages = 64; + const block = try bs.allocatePages(.any, .loader_data, pool_pages); + var pool = TablePool{ .base = @intFromPtr(block.ptr), .next = 0, .cap = pool_pages }; + + const pml4 = try pool.alloc(); + + // Identity + physmap for low RAM. 4 GiB covers all of QEMU's RAM and MMIO + // (LAPIC/IOAPIC/HPET/ECAM/framebuffer under q35); a machine with RAM or a + // framebuffer above 4 GiB would extend this — see the fb window below. + var addr: u64 = 0; + while (addr < 4 * gib) : (addr += 2 << 20) { + try pool.map2M(pml4, addr, addr); // identity + try pool.map2M(pml4, danos.physToVirt(addr), addr); // physmap + } + + // A framebuffer above the 4 GiB window needs its own identity + physmap + // pages (the kernel touches fb.base before it builds its own tables). + const fb = boot_info.framebuffer; + if (fb.present() and fb.base + @as(u64, fb.pitch) * fb.height > 4 * gib) { + var p: u64 = fb.base & ~@as(u64, 0x1F_FFFF); + const fb_end = fb.base + @as(u64, fb.pitch) * fb.height; + while (p < fb_end) : (p += 2 << 20) { + try pool.map2M(pml4, p, p); + try pool.map2M(pml4, danos.physToVirt(p), p); + } + } + + // Higher-half kernel segments (virt != phys). While the kernel still links + // low its segments sit in the identity range and need no separate mapping + // (and 4 KiB-mapping them would collide with the 2 MiB identity leaves), so + // only map segments that actually live in the higher half. + for (boot_info.kernel_segments[0..boot_info.kernel_segment_count]) |seg| { + if (seg.virt < danos.kernel_virt_base) continue; + var off: u64 = 0; + while (off < seg.pages * page_size) : (off += page_size) { + try pool.map4K(pml4, seg.virt + off, seg.phys + off); + } + } + + return pml4; +} + +/// Switch onto `cr3` and jump to the kernel `entry` with `boot_info` in RDI, +/// interrupts off, in one block so nothing runs between the CR3 load and the +/// jump. The identity mapping keeps this low loader code valid across the CR3 +/// load; the jump target is mapped (identity while low, higher-half once high). +fn handoff(cr3: u64, entry: usize, boot_info: *const BootInfo) noreturn { + asm volatile ( + \\cli + \\movq %[cr3], %%cr3 + \\movq %[bi], %%rdi + \\callq *%[entry] + : + : [cr3] "r" (cr3), + [bi] "r" (boot_info), + [entry] "r" (entry), + : .{ .memory = true }); + unreachable; +} + /// Read the init program (sbin/init) into memory that outlives the loader and /// record it in the handoff. The pool buffer is deliberately NOT freed: it's /// LoaderData, which the memory-map conversion classifies as reserved, so the @@ -266,7 +401,9 @@ fn loadElf(bs: *uefi.tables.BootServices, image: []u8, boot_info: *BootInfo) !us // Reserve the exact physical pages this segment is linked at. This // requires the segment's p_paddr to be free in the firmware memory map; - // if it collides, adjust `image_base` in build.zig. + // if it collides, adjust `image_base` in build.zig. (Once the kernel + // links high — M2 step 4 — p_paddr becomes a separate low load address + // via the linker's AT(), and this stays a valid physical allocation.) const mem_sz: usize = @intCast(phdr.p_memsz); const pages = (mem_sz + page_size - 1) / page_size; const dest: [*]align(page_size) uefi.Page = @ptrFromInt(phdr.p_paddr); @@ -279,11 +416,14 @@ fn loadElf(bs: *uefi.tables.BootServices, image: []u8, boot_info: *BootInfo) !us @memcpy(bytes[0..file_sz], image[off..][0..file_sz]); @memset(bytes[file_sz..mem_sz], 0); - // Record it (identity-loaded: virtual == physical) for the kernel's VMM. + // Record the virtual link address and the physical load address so the + // kernel can map itself with the right permissions post-switch. They're + // equal while the kernel links low; they diverge once it links high. const n = boot_info.kernel_segment_count; if (n < boot_info.kernel_segments.len) { boot_info.kernel_segments[n] = .{ .virt = phdr.p_vaddr, + .phys = phdr.p_paddr, .pages = pages, .flags = phdr.p_flags, }; diff --git a/src/kernel/arch/x86_64/paging.zig b/src/kernel/arch/x86_64/paging.zig index c8d5bdf..cbd4cb4 100644 --- a/src/kernel/arch/x86_64/paging.zig +++ b/src/kernel/arch/x86_64/paging.zig @@ -107,9 +107,10 @@ pub fn init(allocFrame: *const fn () ?u64, boot_info: *const danos.BootInfo) voi var flags: u64 = present; if (seg.flags & pf_w != 0) flags |= writable; if (seg.flags & pf_x == 0) flags |= no_execute; - var addr = seg.virt; - const end = seg.virt + seg.pages * page_size; - while (addr < end) : (addr += page_size) mapPage(pml4, addr, addr, flags); + var off: u64 = 0; + while (off < seg.pages * page_size) : (off += page_size) { + mapPage(pml4, seg.virt + off, seg.phys + off, flags); + } } kernel_pml4 = pml4; diff --git a/src/root.zig b/src/root.zig index 9c52e76..07cd929 100644 --- a/src/root.zig +++ b/src/root.zig @@ -45,6 +45,37 @@ pub const Framebuffer = extern struct { /// targets so far. pub const page_size = 4096; +/// The kernel's virtual-memory layout (higher-half). The kernel is linked at +/// `kernel_virt_base` but loaded at a low physical address; all of RAM (and the +/// device MMIO windows) is also mapped at `physmap_base + phys`, so the kernel +/// can reach any physical address by adding a constant. The low half is left +/// entirely to user space. +/// +/// user image + stack : 0x0000_7000_0000_0000 (PML4[224], low half) +/// kernel heap : 0xFFFF_8000_0000_0000 (PML4[256]) +/// physmap : 0xFFFF_8800_0000_0000 (PML4[272]) + phys +/// kernel image : 0xFFFF_FFFF_8000_0000 (PML4[511]) +pub const physmap_base: u64 = 0xFFFF_8800_0000_0000; +pub const kernel_virt_base: u64 = 0xFFFF_FFFF_8000_0000; + +/// Physical address -> its virtual address in the physmap. The single way the +/// kernel dereferences a physical address once paging is up. +/// +/// **Hazard:** valid only once the (bootstrap or final) page tables are live. +/// The bootloader may use the *constant* `physmap_base` to build those tables, +/// but must not call this to dereference memory before its own CR3 is loaded — +/// it runs under the firmware's identity map, where these addresses are unmapped. +pub inline fn physToVirt(phys: u64) u64 { + return phys + physmap_base; +} + +/// Physmap virtual address -> physical. Inverse of `physToVirt`; for producing +/// the physical address of something the kernel holds a physmap pointer to +/// (e.g. a page-table frame for CR3, a post-mortem breadcrumb's RAM location). +pub inline fn virtToPhys(virt: u64) u64 { + return virt - physmap_base; +} + /// danos's own classification of a span of physical memory — deliberately not /// UEFI's vocabulary. Each boot path (UEFI now, device tree later) translates its /// native memory description into these kinds, so the kernel never learns what @@ -88,9 +119,12 @@ pub const MemoryMap = extern struct { /// One PT_LOAD segment of the kernel image, so the kernel can re-map itself with /// correct permissions (code R+X, rodata R, data R+W+NX). `flags` are raw ELF -/// segment flags: PF_X=1, PF_W=2, PF_R=4. +/// segment flags: PF_X=1, PF_W=2, PF_R=4. `virt` is the higher-half link address; +/// `phys` is where the loader actually placed the segment (they differ once the +/// kernel links high — the loader records the real load address here). pub const KernelSegment = extern struct { virt: u64, + phys: u64, pages: u64, flags: u32, _pad: u32 = 0,