//! Per-CPU data reached through the GS segment base. The GS base holds a pointer //! to this core's `ArchitecturePerCpu`, so kernel code gets the running core's block with //! a single MSR read (`scheduler()`) and the system_call entry stub gets its kernel stack //! with a `%gs`-relative load (no usable stack yet at that point). //! //! **swapgs discipline.** In ring 0 the GS base points here; in ring 3 it holds //! the user's own GS (which ring 3 may set freely), and this pointer lives in the //! KERNEL_GS_BASE MSR instead. Every ring-3 -> ring-0 entry (`swapgs` in the //! system_call stub and the conditional swapgs in isr_common) brings it back, and //! every ring-0 -> ring-3 exit swaps it away. Because the very first ring //! transition is always an exit (the kernel starts in ring 0), the swap pairs //! keep the invariant without seeding KERNEL_GS_BASE. `scheduler()` is therefore //! valid in any ring-0 context and never sees a user-controlled base. const std = @import("std"); const io = @import("io.zig"); const parameters = @import("parameters"); const ia32_gs_base = 0xC000_0101; /// Layout is load-bearing: the system_call entry stub in isr.s reaches `kernel_rsp` /// at `%gs:0` and `scratch` at `%gs:8`. Keep those two first; the asserts below /// pin the offsets. pub const ArchitecturePerCpu = extern struct { kernel_rsp: u64 = 0, // %gs:0 — kernel stack top for system_call entry (== TSS.rsp0) scratch: u64 = 0, // %gs:8 — stashes the user rsp during system_call entry scheduler: usize = 0, // the scheduler's PerCpu pointer (what `cpuLocal` returns) }; comptime { std.debug.assert(@offsetOf(ArchitecturePerCpu, "kernel_rsp") == 0); std.debug.assert(@offsetOf(ArchitecturePerCpu, "scratch") == 8); } var blocks = [_]ArchitecturePerCpu{.{}} ** parameters.maximum_cpus; /// Publish core `index`'s per-CPU block: record the scheduler pointer and point /// the GS base at the block. Called once per core during bring-up, after the GDT /// is loaded (a GS *selector* reload would clobber the base). pub fn setLocal(index: usize, scheduler_ptr: usize) void { blocks[index].scheduler = scheduler_ptr; io.wrmsr(ia32_gs_base, @intFromPtr(&blocks[index])); } /// The scheduler pointer for the running core (via the GS base). Valid in any /// ring-0 context under the swapgs discipline. pub fn scheduler() usize { return @as(*const ArchitecturePerCpu, @ptrFromInt(io.rdmsr(ia32_gs_base))).scheduler; } /// Record core `index`'s kernel stack top, used by the system_call entry stub to /// switch off the user stack. The scheduler sets this (and TSS.rsp0) whenever it /// switches to a user task. pub fn setKernelRsp(index: usize, top: usize) void { blocks[index].kernel_rsp = top; } // Fast-system_call MSRs. const ia32_efer = 0xC000_0080; const ia32_star = 0xC000_0081; const ia32_lstar = 0xC000_0082; const ia32_sfmask = 0xC000_0084; /// Enable the `system_call`/`sysret` fast path on this core (BSP and each AP). EFER.SCE /// turns the instructions on; STAR sets the selectors system_call/sysret load; LSTAR /// is the entry stub (isr.s); SFMASK clears RFLAGS bits on entry (notably IF — /// the handler runs with interrupts off, like the int-gate path). The GDT is laid /// out (kernel code 0x08, then user data 0x18 / code 0x20) precisely so these line /// up: system_call loads CS 0x08 / SS 0x10; sysret loads CS = base+16 and SS = base+8 /// with RPL forced to 3, so base 0x10 gives CS 0x23 (user code|3) and SS 0x1B. pub fn initSystemCall() void { io.wrmsr(ia32_efer, io.rdmsr(ia32_efer) | 1); // SCE io.wrmsr(ia32_star, (@as(u64, 0x08) << 32) | (@as(u64, 0x10) << 48)); const entry = @extern(*const anyopaque, .{ .name = "syscall_entry" }); io.wrmsr(ia32_lstar, @intFromPtr(entry)); io.wrmsr(ia32_sfmask, 0x4_0700); // clear IF, TF, DF, AC on entry }