danos/system/kernel/architecture/x86_64/per-cpu.zig

//! Per-CPU data reached through the GS segment base. The GS base holds a pointer
//! to this core's `ArchitecturePerCpu`, so kernel code gets the running core's block with
//! a single MSR read (`scheduler()`) and the system_call entry stub gets its kernel stack
//! with a `%gs`-relative load (no usable stack yet at that point).
//!
//! **swapgs discipline.** In ring 0 the GS base points here; in ring 3 it holds
//! the user's own GS (which ring 3 may set freely), and this pointer lives in the
//! KERNEL_GS_BASE MSR instead. Every ring-3 -> ring-0 entry (`swapgs` in the
//! system_call stub and the conditional swapgs in isr_common) brings it back, and
//! every ring-0 -> ring-3 exit swaps it away. Because the very first ring
//! transition is always an exit (the kernel starts in ring 0), the swap pairs
//! keep the invariant without seeding KERNEL_GS_BASE. `scheduler()` is therefore
//! valid in any ring-0 context and never sees a user-controlled base.
//!
//! The file has since become the home of **per-core CPU state set at bring-up**
//! generally, not just the GS block: the fast-system_call MSRs and the CR4
//! hardening bits live here too, because each is state a core owns and must set
//! for itself. Both bring-up paths — `cpu.init` on the boot processor and
//! `smp.apEntry` on every application processor — call the same functions here.
//!
//! One deliberate exception to "per-core": `armSupervisorAccessPrevention` patches
//! a machine-wide instruction into the shared interrupt entry, once, on the boot
//! processor. It lives here anyway because it is the other half of CR4.SMAP —
//! same feature probe, same fail-open posture — and splitting a hardening measure
//! across two files is how the halves drift apart.

const std = @import("std");
const io = @import("io.zig");
const cpuid = @import("cpuid.zig");
const paging = @import("paging.zig");
const boot_handoff = @import("boot-handoff");
const parameters = @import("parameters");

const ia32_gs_base = 0xC000_0101;

/// Layout is load-bearing: the system_call entry stub in isr.s reaches `kernel_rsp`
/// at `%gs:0` and `scratch` at `%gs:8`. Keep those two first; the asserts below
/// pin the offsets.
pub const ArchitecturePerCpu = extern struct {
    kernel_rsp: u64 = 0, // %gs:0  — kernel stack top for system_call entry (== TSS.rsp0)
    scratch: u64 = 0, // %gs:8  — stashes the user rsp during system_call entry
    scheduler: usize = 0, // the scheduler's PerCpu pointer (what `cpuLocal` returns)
};

comptime {
    std.debug.assert(@offsetOf(ArchitecturePerCpu, "kernel_rsp") == 0);
    std.debug.assert(@offsetOf(ArchitecturePerCpu, "scratch") == 8);
}

var blocks = [_]ArchitecturePerCpu{.{}} ** parameters.maximum_cpus;

/// Publish core `index`'s per-CPU block: record the scheduler pointer and point
/// the GS base at the block. Called once per core during bring-up, after the GDT
/// is loaded (a GS *selector* reload would clobber the base).
pub fn setLocal(index: usize, scheduler_ptr: usize) void {
    blocks[index].scheduler = scheduler_ptr;
    io.wrmsr(ia32_gs_base, @intFromPtr(&blocks[index]));
}

/// The scheduler pointer for the running core (via the GS base), or 0 before this
/// core has one. Valid in any ring-0 context under the swapgs discipline.
///
/// **Zero is an answer here, not a fault.** A core has no per-CPU block between
/// reset and its `setLocal`, and the fault reporters are documented to be callable
/// unconditionally: `currentIdSafe`, `currentNameSafe`, `currentCpuIndex` and
/// `sync.releaseIfHeldHere` all test this for 0 and mean it. Casting the base
/// before testing it put the null one instruction out of their reach — an early
/// fault reported itself by panicking on the cast, and the panic handler, calling
/// those same reporters on its next line, panicked again, so the machine reset
/// instead of halting with the message that would have said what went wrong.
pub fn scheduler() usize {
    const base = io.rdmsr(ia32_gs_base);
    if (base == 0) return 0;
    return @as(*const ArchitecturePerCpu, @ptrFromInt(base)).scheduler;
}

/// Record core `index`'s kernel stack top, used by the system_call entry stub to
/// switch off the user stack. The scheduler sets this (and TSS.rsp0) whenever it
/// switches to a user task.
pub fn setKernelRsp(index: usize, top: usize) void {
    blocks[index].kernel_rsp = top;
}

// Fast-system_call MSRs.
const ia32_efer = 0xC000_0080;
const ia32_star = 0xC000_0081;
const ia32_lstar = 0xC000_0082;
const ia32_sfmask = 0xC000_0084;

/// The SYSRET half of STAR: `sysret` loads CS from base+16 and SS from base+8.
/// The base carries RPL 3 itself — 0x13, not the 0x10 the GDT layout suggests —
/// because **only CS is guaranteed to come out at ring 3**. CS must: `sysret`
/// sets CPL to 3, so the selector it loads is forced to match. SS is under no
/// such obligation, and the two vendors differ on it: Intel ORs 3 into the SS
/// selector as well, AMD hands it over exactly as the arithmetic produced it.
///
/// With base 0x10 an AMD machine therefore enters ring 3 carrying SS 0x18 — RPL
/// 0 — and nothing complains, because a data access is checked against CPL and
/// the descriptor's DPL, never the selector's RPL. It runs perfectly until the
/// first interrupt: the CPU pushes that SS, and the `iretq` returning to ring 3
/// requires SS.RPL to equal CS.RPL, refusing 0 against 3 with #GP(0x18). A
/// process that made system calls happily then dies on its first timer tick.
///
/// Putting the 3 in the base makes both selectors right by construction on
/// either vendor: 0x13 + 8 = 0x1B, 0x13 + 16 = 0x23. Intel's OR is then a no-op
/// rather than the thing holding it together.
const star_sysret_base: u64 = 0x13; // user data 0x18 / user code 0x20, with RPL 3
const star_syscall_base: u64 = 0x08; // kernel code 0x08 / kernel data 0x10

/// Enable the `system_call`/`sysret` fast path on this core (BSP and each AP). EFER.SCE
/// turns the instructions on; STAR sets the selectors system_call/sysret load; LSTAR
/// is the entry stub (isr.s); SFMASK clears RFLAGS bits on entry (notably IF —
/// the handler runs with interrupts off, like the int-gate path). The GDT is laid
/// out (kernel code 0x08, then user data 0x18 / code 0x20) precisely so these line
/// up: system_call loads CS 0x08 / SS 0x10; sysret loads CS 0x23 and SS 0x1B.
pub fn initSystemCall() void {
    io.wrmsr(ia32_efer, io.rdmsr(ia32_efer) | 1); // SCE
    io.wrmsr(ia32_star, (star_syscall_base << 32) | (star_sysret_base << 48));
    const entry = @extern(*const anyopaque, .{ .name = "syscall_entry" });
    io.wrmsr(ia32_lstar, @intFromPtr(entry));
    // Clear IF, TF, DF, AC and NT on entry. The first four are the usual
    // hygiene; NT is here because SYSCALL, unlike an interrupt gate, does not
    // clear it for us, so without this the kernel runs every system call with
    // whatever nested-task bit ring 3 last chose — and the canonical-RIP guard's
    // cold path (isr.s) leaves through IRETQ, whose behaviour with NT set is a
    // corner of the manuals not worth depending on either way. Masking it costs
    // one bit and removes the question: the kernel is never nested, and ring 3
    // still gets its own NT back, from R11 on the fast path and from the frame
    // on the cold one.
    io.wrmsr(ia32_sfmask, 0x4_4700);
}

// --- supervisor-mode hardening (CR4) ---------------------------------------
//
// CR4 is per-core state, so these bits are set during *every* core's bring-up —
// the BSP in cpu.init, each AP in smp.apEntry — and not in the AP trampoline,
// which stays minimal and would only cover the APs anyway.

/// CR4.SMEP: an instruction fetch in ring 0 from a page whose U/S bit says *user*
/// raises #PF. This is what makes the classic ret2usr shape (a kernel bug steered
/// into attacker-prepared user code) a loud, attributable fault instead of a
/// silent compromise. danos never executes user-mapped memory in ring 0 — kernel
/// text lives in the higher half, the ring-3 entry paths are kernel code, and the
/// AP trampoline page is a supervisor mapping — so nothing legitimate is refused.
const cr4_smep: u64 = 1 << 20;

/// SMEP's feature bit: CPUID leaf 7, sub-leaf 0, EBX bit 7.
fn smepSupported() bool {
    if (!cpuid.supports(7)) return false;
    return cpuid.leaf(7).ebx & (1 << 7) != 0;
}

/// CR4.SMAP: a ring-0 data *read or write* to a page whose U/S bit says *user*
/// raises #PF, unless EFLAGS.AC is set. It is the standing enforcement behind
/// system/kernel/user-memory.zig: that layer never dereferences a user virtual
/// address — it walks the process's tables and moves bytes through the physmap,
/// kernel mappings throughout — so nothing legitimate in this kernel is refused,
/// and any future code that reaches for a user pointer directly faults the first
/// time it runs. danos therefore opens no `stac` window anywhere; there is no
/// correct reason to have one, and adding one is how the guarantee is lost.
const cr4_smap: u64 = 1 << 21;

/// SMAP's feature bit: CPUID leaf 7, sub-leaf 0, EBX bit 20.
fn smapSupported() bool {
    if (!cpuid.supports(7)) return false;
    return cpuid.leaf(7).ebx & (1 << 20) != 0;
}

/// `clac` — the three bytes that replace the NOP at `isr_smap_patch` once the
/// interrupt entry is allowed to execute them.
const clac_opcode = [_]u8{ 0x0f, 0x01, 0xca };

/// Set only once the interrupt entry really clears AC, and read by every core
/// before it turns SMAP on. Nothing turns SMAP on until this is true, so there is
/// no window — not even on the boot processor, not even before ring 3 exists —
/// in which the bit is live while an interrupt could still be taken with AC set.
var interrupt_entry_clears_ac = false;

fn readCr3() u64 {
    return asm volatile ("mov %%cr3, %[out]"
        : [out] "=r" (-> u64),
    );
}

/// Patch the `clac` into the shared interrupt entry, and by doing so authorize
/// CR4.SMAP. **Boot processor only, once, before any core sets the bit and before
/// the application processors are woken** — `initHardening` refuses SMAP until
/// this has run, so the order is enforced rather than merely documented, and an AP
/// climbing the trampoline cannot get ahead of it.
///
/// The write goes through the physmap, not through the kernel's own view of its
/// text: once `paging.init` has run, kernel `.text` is mapped read-only under W^X,
/// and a store to it would fault (or, worse, silently need CR0.WP cleared). The
/// physmap alias of the same frame is an ordinary supervisor RW mapping — the same
/// door `smp.arm` uses to write the AP trampoline and `process.run` uses to fill a
/// read-only user code frame. Going through it also makes this correct under
/// *either* set of tables: the loader's bootstrap tables map the image writable,
/// the kernel's own do not, and this runs before the switch.
///
/// Three bytes, translated one at a time, so a patch site that straddles a page
/// boundary is not a special case. A translation that fails leaves the NOP in
/// place and returns false, and the machine then boots without SMAP rather than
/// with SMAP and an entry path that cannot clear AC.
pub fn armSupervisorAccessPrevention() bool {
    if (!smapSupported()) return false;
    const site = @intFromPtr(@extern([*]const u8, .{ .name = "isr_smap_patch" }));
    const root = readCr3() & 0x000F_FFFF_FFFF_F000;

    // MUST run with interrupts masked, and does: this is reached from cpu.init,
    // long before the kernel's `sti`, and before any application processor exists.
    // The reason is that the three bytes go in one at a time, and the middle state
    // — 0f 01 00, once the second byte lands — is `sgdt (%rax)`, a ten-byte write
    // to wherever RAX points, sitting at the first instruction of every interrupt
    // entry. Nothing can take that entry here, so nothing can execute it. A future
    // change that moves this after interrupts are enabled has to close that window
    // first (one 16-bit store covering both changed bytes is the shape, but it
    // needs the two to be physically contiguous and 2-byte aligned — a naive
    // version of exactly that triple-faulted this kernel).
    for (clac_opcode, 0..) |byte, i| {
        const physical = paging.translateIn(root, site + i) orelse return false;
        const alias: *volatile u8 = @ptrFromInt(boot_handoff.physicalToVirtual(physical));
        alias.* = byte;
    }
    // The bytes were written through a different linear address than the one they
    // will be fetched from, so serialize before anyone can execute them: CPUID is
    // the architecturally sanctioned way to discard whatever the core prefetched or
    // decoded of the old encoding.
    _ = cpuid.leaf(0);
    // Read back through the *text* address, not the alias just written: that is the
    // view the CPU will fetch from, so this is what proves the two are the same
    // physical page and the patch landed where it will actually execute. A mismatch
    // means the translation lied, and SMAP stays off rather than being enabled over
    // an entry path that cannot clear AC.
    const installed: [*]const volatile u8 = @ptrFromInt(site);
    for (clac_opcode, 0..) |byte, i| {
        if (installed[i] != byte) return false;
    }
    interrupt_entry_clears_ac = true;
    return true;
}

fn readCr4() u64 {
    return asm volatile ("mov %%cr4, %[out]"
        : [out] "=r" (-> u64),
    );
}

fn writeCr4(value: u64) void {
    asm volatile ("mov %[in], %%cr4"
        :
        : [in] "r" (value),
        : .{ .memory = true });
}

/// Turn on the supervisor-mode hardening this CPU offers, on the calling core.
/// Called once per core, next to `initSystemCall`, from both bring-up paths.
///
/// **Fail-open, like the IOMMU and the clocksource:** an absent feature is a
/// machine that boots unhardened, not a machine that refuses to boot. danos has
/// to run on any VM, on real Intel and on real AMD; the boot log states the
/// posture either way (see the platform block in system/kernel/kernel.zig), so
/// an unhardened boot is visible rather than assumed.
pub fn initHardening() void {
    var bits: u64 = 0;
    if (smepSupported()) bits |= cr4_smep;
    // SMAP only after the boot processor has armed the interrupt entry: with the
    // NOP still in place an interrupt inherits ring 3's AC and suspends SMAP for
    // the length of the handler, which is worse than not claiming the bit at all.
    if (smapSupported() and interrupt_entry_clears_ac) bits |= cr4_smap;
    if (bits == 0) return;
    writeCr4(readCr4() | bits);
}

/// Whether supervisor-mode execution prevention is live on *this* core. Read
/// straight out of CR4 rather than a remembered probe result, so the answer is
/// the state the hardware is actually in — which is what both the boot log and
/// the `fault-smep` test case want to assert.
pub fn supervisorExecutePreventionEnabled() bool {
    return readCr4() & cr4_smep != 0;
}

/// Whether supervisor-mode *access* prevention is live on *this* core. Read from
/// CR4 for the same reason as its neighbour: the question the boot log and the
/// `fault-smap` case are asking is what the hardware is doing, not what a probe
/// once concluded.
pub fn supervisorAccessPreventionEnabled() bool {
    return readCr4() & cr4_smap != 0;
}