//! x86_64 CPU operations. This is the "arch" module: the generic kernel imports //! it as `@import("arch")` and never names x86_64 directly, so a second //! architecture is added by pointing that module at a different directory in //! build.zig — no change to the generic code. Keep everything CPU-specific here //! (halt, the descriptor tables, later paging), and nothing generic. const danos = @import("danos"); const gdt = @import("gdt.zig"); const tss = @import("tss.zig"); const idt = @import("idt.zig"); const paging = @import("paging.zig"); const serial = @import("serial.zig"); const apic = @import("apic.zig"); const ioapic = @import("ioapic.zig"); const io = @import("io.zig"); const smp = @import("smp.zig"); /// The saved register/trap frame passed to a fault handler. pub const CpuState = idt.CpuState; /// Bring up the serial port (the kernel's machine-readable log). No dependencies, /// so it can be the very first thing called. pub fn serialInit() void { serial.init(); } /// Write bytes to the serial port. pub fn serialWrite(bytes: []const u8) void { serial.write(bytes); } /// Emit a one-byte checkpoint to the POST diagnostic port (0x80). A POST card or /// BMC displays it; it's the last-resort progress signal when there's no text /// output at all. Writing 0x80 is universally safe (it's the legacy I/O-delay port). pub fn postCode(code: u8) void { io.outb(0x80, code); } /// Whether a Bochs/QEMU-style debug console is on port 0xE9 (it returns 0xE9 when /// read). On real hardware the port reads back 0xFF, so this stays false — a safe /// probe before we write to it. pub fn debugconPresent() bool { return io.inb(0xE9) == 0xE9; } /// Output sink: write bytes to the 0xE9 debug console (see `debugconPresent`). pub fn debugconWrite(bytes: []const u8) void { for (bytes) |b| io.outb(0xE9, b); } /// Set up the CPU's descriptor tables: our own GDT, the TSS (with an interrupt /// stack for double faults), then the IDT with exception handlers. After this a /// CPU fault is reported instead of triple-faulting. Install the fault handler /// (setFaultHandler) first so early faults are caught. pub fn init() void { gdt.init(); tss.init(); idt.init(); } /// Build the kernel's own page tables (with real permissions) and switch onto /// them. Needs the frame allocator and the boot info (for the memory map and the /// kernel's segment layout). Call once the frame allocator is up. pub fn enablePaging(allocFrame: *const fn () ?u64, boot_info: *const danos.BootInfo) void { paging.init(allocFrame, boot_info); } /// Map a page into the kernel address space (non-executable). For the heap, etc. pub fn mapPage(virt: u64, phys: u64, writable: bool) void { paging.map(virt, phys, writable); } /// Remove a kernel mapping. pub fn unmapPage(virt: u64) void { paging.unmap(virt); } /// CR3 holds the physical address of the active top-level page table. pub fn readCr3() u64 { return asm volatile ("mov %%cr3, %[out]" : [out] "=r" (-> u64), ); } /// IA32_GS_BASE: the hidden base of the GS segment. We repurpose it as the per-CPU /// data pointer (there's no user mode yet, so no `swapgs` dance — GS base is always /// the running core's per-CPU block). Set once per core during bring-up, after the /// GDT is loaded (loading a GS *selector* would otherwise clobber this base). const ia32_gs_base = 0xC000_0101; /// Publish this core's per-CPU data pointer so `cpuLocal` can retrieve it. Each /// core calls this once, after its GDT is in place. pub fn setCpuLocal(ptr: usize) void { io.wrmsr(ia32_gs_base, ptr); } /// This core's per-CPU data pointer (the value `setCpuLocal` stored). Reads the GS /// base MSR — a per-core register, so each core sees its own without any locking. pub fn cpuLocal() usize { return io.rdmsr(ia32_gs_base); } // --- SMP: application-processor bring-up ---------------------------------- /// Record the low (<1 MiB) frame reserved for the AP trampoline. Run once at boot. /// The frame stays inert (zeroed, non-executable) between wakes and is armed only /// while a core is climbing — so a core can be (re)woken at any time (retry, or a /// future power manager) without leaving an executable page resident. See smp.zig. pub fn setTrampolinePage(phys: u64) void { smp.setTrampolinePage(phys); } /// Wake the core with Local APIC id `apic_id` as dense CPU `index`, giving it /// `stack_top` and its per-CPU pointer `percpu`; it adopts the current (kernel) page /// tables. Returns false if it doesn't come online within the timeout. Blocks until /// the core reports in. pub fn startSecondary(apic_id: u32, stack_top: usize, percpu: usize, index: usize) bool { return smp.startAp(apic_id, stack_top, percpu, index, readCr3()); } /// Register the generic entry a woken AP jumps to once its arch state is up (its own /// descriptor tables, LAPIC, and timer). The kernel passes its scheduler entry here. pub fn setSecondaryEntry(entry: *const fn () callconv(.c) noreturn) void { smp.setSecondaryEntry(entry); } /// Bytes the kernel should allocate for an AP's IST (double-fault) stack, and where /// to record its top before waking the core. The stack is heap-allocated per online /// AP (the BSP's is static — it's needed before the allocator exists). See tss.zig. pub const ist_stack_size = tss.ist_stack_size; pub fn setApIstStack(cpu: usize, top: usize) void { tss.setApIstStack(cpu, top); } /// Test hook: force the next `n` AP wake attempts to fail, so the retry path can be /// exercised deterministically (see the smp-retry test). No effect when `n` is 0. pub fn testFailNextWakes(n: u32) void { smp.testFailNextWakes(n); } /// The reserved AP-trampoline frame (0 if none). For tests that check it's inert. pub fn trampolinePage() u64 { return smp.trampolinePage(); } /// Whether the page at `virt` is currently mapped executable (present, NX clear). pub fn pageExecutable(virt: u64) bool { return paging.isExecutable(virt); } /// Kernel tick rate: 1000 Hz (1 ms), the scheduler's time quantum. pub const timer_hz = 1000; /// The ACPI PM timer, as a calibration reference (re-exported for the config). pub const PmTimer = apic.PmTimer; /// A MADT interrupt-source override (re-exported for the config). pub const IsoEntry = ioapic.IsoEntry; /// Discovered platform facts the arch layer needs so it makes no legacy /// assumptions — sourced from the device tree + ACPI, passed in by the kernel. pub const PlatformConfig = struct { /// Whether the legacy 8259 PIC is present (skip programming it if not). pic_present: bool = true, /// HPET MMIO base (0 = none) — a calibration reference for the timer. hpet_base: u64 = 0, /// The ACPI PM timer, another calibration reference. pm_timer: ?PmTimer = null, /// I/O APIC MMIO base + its first global system interrupt (0 = none). ioapic_base: u64 = 0, ioapic_gsi_base: u32 = 0, /// MADT ISA-IRQ overrides, for I/O APIC routing. overrides: []const IsoEntry = &.{}, }; /// Apply the discovered platform config. Must run before `startTimer` (the timer /// calibration reads `hpet_base`/`pm_timer`) and before any interrupt routing. /// Maps + masks the I/O APIC immediately. pub fn configurePlatform(cfg: PlatformConfig) void { apic.configure(cfg.pic_present, cfg.hpet_base, cfg.pm_timer); ioapic.configure(cfg.ioapic_base, cfg.ioapic_gsi_base, cfg.overrides); ioapic.init(); } /// Point the serial console at the UART ACPI's SPCR table named (MMIO or I/O port). pub fn serialReconfigure(is_mmio: bool, addr: u64) void { serial.reconfigure(is_mmio, addr); } /// The reference clock the timer was calibrated against ("cpuid"/"hpet"/…). pub fn timerCalibrationSource() []const u8 { return apic.calibrationSource(); } /// I/O APIC diagnostics (for boot logging / verification). pub fn ioapicEntryCount() u32 { return ioapic.entryCount(); } pub fn ioapicEntryLow(n: u32) u32 { return ioapic.entryLow(n); } /// Enable the Local APIC, calibrate its timer against the best available reference /// (see apic.calibrate — no longer the PIT by default), and start it firing at /// `timer_hz` — the kernel's real-time heartbeat. Interrupts still have to be /// unmasked with enableInterrupts() to be delivered. Run `configurePlatform` first. pub fn startTimer() void { apic.init(); apic.calibrate(); idt.setHandler(apic.timer_vector, apic.timerTick); apic.initTimer(timer_hz); } /// Number of timer ticks since startTimer(). pub fn ticks() u64 { return apic.ticks(); } // Monotonic high-resolution clock (from the TSC), one function per resolution. pub fn nanos() u64 { return apic.nanos(); } pub fn micros() u64 { return apic.micros(); } pub fn millis() u64 { return apic.millis(); } /// Measured LAPIC timer / TSC frequencies in Hz (from calibration). pub fn lapicHz() u64 { return apic.lapicHz(); } pub fn tscHz() u64 { return apic.tscHz(); } /// Unmask maskable interrupts (`sti`) so device interrupts get delivered. pub fn enableInterrupts() void { asm volatile ("sti"); } /// Mask maskable interrupts (`cli`). pub fn disableInterrupts() void { asm volatile ("cli"); } /// Disable interrupts and return the previous flags, so a nested critical section /// can restore the caller's state rather than blindly re-enabling. Pairs with /// restoreInterrupts. pub fn saveInterrupts() u64 { var flags: u64 = undefined; asm volatile ( \\pushfq \\pop %[f] \\cli : [f] "=r" (flags), : : .{ .memory = true } ); return flags; } /// Re-enable interrupts only if they were enabled when `flags` was captured. pub fn restoreInterrupts(flags: u64) void { if (flags & 0x200 != 0) asm volatile ("sti" ::: .{ .memory = true }); // bit 9 = IF } /// Register a callback the timer interrupt invokes each tick (e.g. the scheduler). pub fn setTickHook(hook: *const fn () void) void { apic.setTickHook(hook); } // --- context switching (for the scheduler) ------------------------------- /// Save the current task's registers/stack and resume `new_rsp`; the old stack /// pointer is written to `old_rsp`. Defined in isr.s. extern fn switch_context(old_rsp: *usize, new_rsp: usize) callconv(.c) void; pub fn switchContext(old_rsp: *usize, new_rsp: usize) void { switch_context(old_rsp, new_rsp); } /// Build the initial stack for a new task so that switching to it lands in /// `task_trampoline`, which then calls `entry`. Returns the saved stack pointer. /// The layout must match switch_context's push order (callee-saved, then the /// return address on top); `entry` is smuggled in via the r15 slot. pub fn initTaskStack(stack_top: usize, entry: usize) usize { const trampoline = @extern(*const anyopaque, .{ .name = "task_trampoline" }); var sp = stack_top; const push = struct { fn f(p: *usize, value: usize) void { p.* -= @sizeOf(usize); @as(*usize, @ptrFromInt(p.*)).* = value; } }.f; push(&sp, @intFromPtr(trampoline)); // return address for switch_context's `ret` push(&sp, 0); // rbx push(&sp, 0); // rbp push(&sp, 0); // r12 push(&sp, 0); // r13 push(&sp, 0); // r14 push(&sp, entry); // r15 -> task entry, read by task_trampoline return sp; } /// Route CPU exceptions to `handler`, which receives the trap frame and does not /// return. Until set, faults just halt the core. pub fn setFaultHandler(handler: *const fn (*const CpuState) noreturn) void { idt.on_fault = handler; } /// A human-readable name for a CPU exception vector. pub fn vectorName(vector: u64) []const u8 { return idt.vectorName(vector); } /// Read `width` bytes (1/2/4) from an I/O port. The generic device layer drives /// ACPI registers through this rather than naming x86 port instructions; on an /// MMIO-only architecture this would be implemented differently. pub fn pioRead(width: u8, port: u16) u32 { return switch (width) { 1 => io.inb(port), 2 => io.inw(port), 4 => io.inl(port), else => 0, }; } /// Write `width` bytes (1/2/4) to an I/O port. pub fn pioWrite(width: u8, port: u16, value: u32) void { switch (width) { 1 => io.outb(port, @truncate(value)), 2 => io.outw(port, @truncate(value)), 4 => io.outl(port, value), else => {}, } } /// CR2 holds the faulting linear address after a page fault (#PF, vector 14). pub fn readCr2() u64 { return asm volatile ("mov %%cr2, %[out]" : [out] "=r" (-> u64), ); } /// Park the core forever. `hlt` drops it into a low-power idle until the next /// interrupt; the loop re-halts on every wake so the stop is permanent. See /// docs/halting.md for the full reasoning. pub fn halt() noreturn { while (true) asm volatile ("hlt"); } /// Spin-wait hint (`pause`). Emitted in the body of a spinlock's busy-wait: it /// relaxes the core while it polls a contended lock — yielding pipeline resources /// to a hyperthread sibling and easing the cache-coherency traffic on the lock /// line. Purely a performance/power hint; correct to omit, but kinder on the bus. pub fn cpuRelax() void { asm volatile ("pause"); }