Built paging / the kernel's own page tables (with a TSS+IST)

src/arch/x86_64/cpu.zig

@@ -5,19 +5,36 @@
 //! (halt, the descriptor tables, later paging), and nothing generic.
 
 const gdt = @import("gdt.zig");
+const tss = @import("tss.zig");
 const idt = @import("idt.zig");
+const paging = @import("paging.zig");
 
 /// The saved register/trap frame passed to a fault handler.
 pub const CpuState = idt.CpuState;
 
-/// Set up the CPU's descriptor tables: our own GDT, then the IDT with exception
-/// handlers. After this a CPU fault is reported instead of triple-faulting.
-/// Install the fault handler (setFaultHandler) first so early faults are caught.
+/// Set up the CPU's descriptor tables: our own GDT, the TSS (with an interrupt
+/// stack for double faults), then the IDT with exception handlers. After this a
+/// CPU fault is reported instead of triple-faulting. Install the fault handler
+/// (setFaultHandler) first so early faults are caught.
 pub fn init() void {
     gdt.init();
+    tss.init();
     idt.init();
 }
 
+/// Build the kernel's own page tables and switch onto them. Needs a physical
+/// frame allocator; call once the frame allocator is up.
+pub fn enablePaging(allocFrame: *const fn () ?u64) void {
+    paging.init(allocFrame);
+}
+
+/// CR3 holds the physical address of the active top-level page table.
+pub fn readCr3() u64 {
+    return asm volatile ("mov %%cr3, %[out]"
+        : [out] "=r" (-> u64),
+    );
+}
+
 /// Route CPU exceptions to `handler`, which receives the trap frame and does not
 /// return. Until set, faults just halt the core.
 pub fn setFaultHandler(handler: *const fn (*const CpuState) noreturn) void {

src/arch/x86_64/gdt.zig

@@ -7,17 +7,33 @@
 /// Selectors into the table below (index * 8).
 pub const kernel_code = 0x08;
 pub const kernel_data = 0x10;
+pub const tss_selector = 0x18;
 
 /// Flat 64-bit descriptors. Base/limit are ignored in long mode; what matters is
 /// the access byte and, for code, the long-mode (L) flag.
 ///   code: present, ring 0, executable, readable, L=1  -> 0x00AF9A00_0000FFFF
 ///   data: present, ring 0, writable                   -> 0x00CF9200_0000FFFF
+/// The last two slots hold one 16-byte TSS descriptor, filled in by setTss.
 var table = [_]u64{
     0, // null descriptor (required)
-    0x00AF9A000000FFFF, // kernel code
-    0x00CF92000000FFFF, // kernel data
+    0x00AF9A000000FFFF, // kernel code (0x08)
+    0x00CF92000000FFFF, // kernel data (0x10)
+    0, // TSS descriptor low  (0x18)
+    0, // TSS descriptor high
 };
 
+/// Fill the 64-bit TSS system descriptor (two GDT slots) so the task register can
+/// point at our TSS. Type 0x89 = present, ring 0, available 64-bit TSS.
+pub fn setTss(base: u64, limit: u64) void {
+    table[3] = (limit & 0xFFFF) |
+        ((base & 0xFFFF) << 16) |
+        (((base >> 16) & 0xFF) << 32) |
+        (@as(u64, 0x89) << 40) |
+        (((limit >> 16) & 0xF) << 48) |
+        (((base >> 24) & 0xFF) << 56);
+    table[4] = (base >> 32) & 0xFFFFFFFF;
+}
+
 /// The operand `lgdt` wants: table byte-length minus one, then its address.
 const Descriptor = packed struct {
     limit: u16,

src/arch/x86_64/idt.zig

@@ -7,6 +7,7 @@
 //! interrupts (the APIC, timer, keyboard) come later.
 
 const gdt = @import("gdt.zig");
+const tss = @import("tss.zig");
 
 /// The register + trap frame the ISR stubs build on the stack, laid out so the
 /// lowest address (where RSP points when we call the handler) is the first field.
@@ -106,6 +107,9 @@ pub fn init() void {
         const stub = @extern(*const anyopaque, .{ .name = std.fmt.comptimePrint("isr{d}", .{vector}) });
         setGate(vector, @intFromPtr(stub));
     }
+    // Run the double-fault handler (vector 8) on IST1: a #DF usually means the
+    // current stack is unusable, so it needs a guaranteed-good one. See tss.zig.
+    idt[8].ist = tss.double_fault_ist;
     const descriptor = Descriptor{
         .limit = @sizeOf(@TypeOf(idt)) - 1,
         .base = @intFromPtr(&idt),

src/arch/x86_64/isr.s

@@ -33,6 +33,12 @@ idt_flush:
     lidt (%rdi)
     ret
 
+# load_tr(di = TSS selector): load the task register.
+.global load_tr
+load_tr:
+    ltr %di
+    ret
+
 # Stub for a vector the CPU does NOT push an error code for: push a dummy 0.
 .macro STUB_NOERR vec
 .global isr\vec

src/arch/x86_64/paging.zig

@@ -0,0 +1,70 @@
+//! The kernel's own 4-level page tables. Until now we've been running on the
+//! firmware's page tables, which live in memory we'd like to reclaim and which we
+//! don't control. This builds our own set, identity-mapping the low 4 GiB, and
+//! loads CR3 to switch onto them.
+//!
+//! "Identity map" means virtual address == physical address, which keeps
+//! everything already running — kernel image, stack, framebuffer, the frame
+//! allocator's bitmap, MMIO — valid across the switch without having to relocate
+//! anything. 4 GiB comfortably covers all of that (RAM low down, the framebuffer
+//! at 2 GiB, device MMIO below 4 GiB). Higher-half mapping and per-region
+//! permissions come later; this is the bootstrap.
+//!
+//! We use 2 MiB pages, so the whole map is cheap: a PML4, a PDPT, and four page
+//! directories.
+
+const KiB = 1024;
+const MiB = 1024 * KiB;
+const GiB = 1024 * MiB;
+
+const present: u64 = 1 << 0;
+const writable: u64 = 1 << 1;
+const huge: u64 = 1 << 7; // in a PD entry: this maps a 2 MiB page directly
+const addr_mask: u64 = 0x000F_FFFF_FFFF_F000; // physical address bits of an entry
+
+/// A page table is 512 64-bit entries. While building the tables the firmware's
+/// identity map is still active, so a physical frame address is usable directly.
+fn tableAt(phys: u64) *[512]u64 {
+    return @ptrFromInt(phys);
+}
+
+/// Allocate and zero a fresh page-table frame. Zeroing matters: the frame comes
+/// from previously-used memory, and any stale non-zero entry would map a bogus
+/// region.
+fn allocTable(allocFrame: *const fn () ?u64) u64 {
+    const frame = allocFrame() orelse @panic("paging: out of memory building page tables");
+    @memset(tableAt(frame)[0..], 0);
+    return frame;
+}
+
+/// Return the table an entry points at, creating it if the entry is empty.
+fn descend(entry: *u64, allocFrame: *const fn () ?u64) u64 {
+    if (entry.* & present != 0) return entry.* & addr_mask;
+    const frame = allocTable(allocFrame);
+    entry.* = frame | present | writable;
+    return frame;
+}
+
+/// Identity-map one 2 MiB page: walk PML4 -> PDPT -> PD and write the leaf.
+fn mapHugePage(pml4: u64, addr: u64, allocFrame: *const fn () ?u64) void {
+    const pml4e = &tableAt(pml4)[(addr >> 39) & 0x1FF];
+    const pdpt = descend(pml4e, allocFrame);
+    const pdpte = &tableAt(pdpt)[(addr >> 30) & 0x1FF];
+    const pd = descend(pdpte, allocFrame);
+    tableAt(pd)[(addr >> 21) & 0x1FF] = addr | present | writable | huge;
+}
+
+/// Build the tables, identity-map the low 4 GiB, and switch CR3 onto them.
+pub fn init(allocFrame: *const fn () ?u64) void {
+    const pml4 = allocTable(allocFrame);
+    var addr: u64 = 0;
+    while (addr < 4 * GiB) : (addr += 2 * MiB) {
+        mapHugePage(pml4, addr, allocFrame);
+    }
+    // Loading CR3 switches address spaces and flushes the TLB in one step.
+    asm volatile ("mov %[pml4], %%cr3"
+        :
+        : [pml4] "r" (pml4),
+        : .{ .memory = true }
+    );
+}

src/arch/x86_64/tss.zig

@@ -0,0 +1,48 @@
+//! Task State Segment and its interrupt stack. In long mode the TSS's main job
+//! is the Interrupt Stack Table: an IDT gate can name an IST entry, and the CPU
+//! switches to that stack when the exception fires — no matter how broken the
+//! interrupted stack was. We use IST1 for the double-fault handler, so a fault
+//! that happens *because* the current stack is unusable still lands on solid
+//! ground instead of triple-faulting.
+
+const gdt = @import("gdt.zig");
+
+/// x86_64 TSS. `packed` because several 64-bit fields sit at 4-byte-unaligned
+/// offsets (rsp0 at byte 4), which a normal struct would pad away.
+const Tss = packed struct {
+    reserved0: u32 = 0,
+    rsp0: u64 = 0,
+    rsp1: u64 = 0,
+    rsp2: u64 = 0,
+    reserved1: u64 = 0,
+    ist1: u64 = 0,
+    ist2: u64 = 0,
+    ist3: u64 = 0,
+    ist4: u64 = 0,
+    ist5: u64 = 0,
+    ist6: u64 = 0,
+    ist7: u64 = 0,
+    reserved2: u64 = 0,
+    reserved3: u16 = 0,
+    iomap_base: u16 = 0,
+};
+
+/// The IST slot (1-based, as the IDT gate encodes it) used for critical faults.
+pub const double_fault_ist = 1;
+
+var tss: Tss align(16) = .{};
+
+/// Dedicated stack for IST1. Static so it needs no allocator and is always valid.
+var ist1_stack: [16 * 1024]u8 align(16) = undefined;
+
+/// Loads the task register with the TSS selector. Defined in isr.s.
+extern fn load_tr(selector: u16) callconv(.c) void;
+
+/// Point IST1 at its stack, publish the TSS through the GDT, and load it into the
+/// task register. Requires the GDT to already be loaded (gdt.init first).
+pub fn init() void {
+    tss.ist1 = @intFromPtr(&ist1_stack) + ist1_stack.len; // stacks grow down
+    tss.iomap_base = @sizeOf(Tss); // == limit: no I/O permission bitmap
+    gdt.setTss(@intFromPtr(&tss), @sizeOf(Tss) - 1);
+    load_tr(gdt.tss_selector);
+}

src/main.zig

@@ -36,6 +36,7 @@ fn kmain(boot_info: *const BootInfo) noreturn {
     arch.init();
 
     con.write("danos: framebuffer console online\n");
+    con.write("danos: cpu tables online (GDT, IDT, TSS)\n");
     con.print("  resolution : {d}x{d}\n", .{ fb.width, fb.height });
     con.print("  pitch      : {d} bytes\n", .{fb.pitch});
     con.print("  format     : {s}\n", .{@tagName(fb.format)});
@@ -81,6 +82,11 @@ fn kmain(boot_info: *const BootInfo) noreturn {
     if (f2) |p| pmm.free(p);
     con.print("  after free : {d} frames free\n", .{pmm.stats().free_frames});
 
+    // Switch off the firmware's page tables onto our own.
+    arch.enablePaging(pmm.alloc);
+    con.print("\ndanos: paging enabled\n", .{});
+    con.print("  page tables: CR3 = 0x{x:0>16}\n", .{arch.readCr3()});
+
     con.write("\nkernel initialised; nothing left to do, halting.\n");
 
     arch.halt();