danos/system/kernel/architecture/x86_64/cpu.zig

//! x86_64 CPU operations. This is the "architecture" module: the generic kernel imports
//! it as `@import("architecture")` and never names x86_64 directly, so a second
//! architecture is added by pointing that module at a different directory in
//! build.zig — no change to the generic code. Keep everything CPU-specific here
//! (halt, the descriptor tables, later paging), and nothing generic.

const boot_handoff = @import("boot-handoff");
const parameters = @import("parameters");
const gdt = @import("gdt.zig");
const tss = @import("tss.zig");
const idt = @import("idt.zig");
const paging = @import("paging.zig");
const serial = @import("serial.zig");
const apic = @import("apic.zig");
const ioapic = @import("ioapic.zig");
const io = @import("io.zig");
const smp = @import("smp.zig");
const pcpu = @import("per-cpu.zig");

/// The saved register/trap frame passed to a fault handler.
pub const CpuState = idt.CpuState;

// --- trap-frame accessors ---------------------------------------------------
// The frame's fields are x86_64 registers; the generic kernel reads it through
// these accessors so it never names one.

/// The interrupted/faulting instruction address (RIP here; ELR_EL1 on aarch64,
/// sepc on riscv64).
pub fn instructionPointer(state: *const CpuState) u64 {
    return state.rip;
}

/// The interrupted stack pointer (RSP here).
pub fn stackPointer(state: *const CpuState) u64 {
    return state.rsp;
}

/// Whether the trap came from user mode (CPL 3 here; EL0 on aarch64, U-mode on
/// riscv64).
pub fn fromUser(state: *const CpuState) bool {
    return state.cs & 3 == 3;
}

/// The faulting virtual address, if this trap is a page fault (CR2 here;
/// FAR_EL1 on aarch64, stval on riscv64). Null for any other exception.
pub fn faultAddress(state: *const CpuState) ?u64 {
    if (state.vector != 14) return null;
    return asm volatile ("mov %%cr2, %[out]"
        : [out] "=r" (-> u64),
    );
}

// --- system_call ABI ------------------------------------------------------------
// The System V-style register convention (number in rax, arguments in
// rdi/rsi/rdx/r10/r8/r9, result in rax), exposed positionally so the generic
// dispatcher never names a register.

/// The system_call number the user program passed.
pub fn systemCallNumber(state: *const CpuState) u64 {
    return state.rax;
}

/// Positional system_call argument `n`.
pub fn systemCallArg(state: *const CpuState, n: u8) u64 {
    return switch (n) {
        0 => state.rdi,
        1 => state.rsi,
        2 => state.rdx,
        3 => state.r10,
        4 => state.r8,
        5 => state.r9,
        else => 0,
    };
}

/// Write the system_call's return value into the frame — the entry paths restore
/// user registers from it.
pub fn setSystemCallResult(state: *CpuState, value: u64) void {
    state.rax = value;
}

/// Write a *second* system_call return value (rdx here — restored by both the
/// system_call/sysret and int-0x80 entry paths; unlike rcx/r11 it is not consumed by
/// sysretq). Used by IPC_ReplyWait to hand back the sender's badge alongside the
/// message length in rax.
pub fn setSystemCallResult2(state: *CpuState, value: u64) void {
    state.rdx = value;
}

/// Write a *third* system_call return value (r8 here). r8 is an input argument
/// register (arg #4), but the syscall/int-0x80 stubs push and pop it around the
/// dispatch, so a value written into the frame is restored to the user on return.
/// Used by the IPC cap-passing calls to hand back the received capability handle.
pub fn setSystemCallResult3(state: *CpuState, value: u64) void {
    state.r8 = value;
}

/// Bring up the serial port (the kernel's machine-readable log). No dependencies,
/// so it can be the very first thing called.
pub fn serialInit() void {
    serial.init();
}

/// Write bytes to the serial port.
pub fn serialWrite(bytes: []const u8) void {
    serial.write(bytes);
}

/// Emit a one-byte progress checkpoint to whatever hardware debug sink the
/// platform has — here the POST diagnostic port (0x80), which a POST card or BMC
/// displays. The last-resort progress signal when there's no text output at all.
/// Writing 0x80 is universally safe (it's the legacy I/O-delay port).
pub fn checkpoint(code: u8) void {
    io.outb(0x80, code);
}

/// Whether a Bochs/QEMU-style debug console is on port 0xE9 (it returns 0xE9 when
/// read). On real hardware the port reads back 0xFF, so this stays false — a safe
/// probe before we write to it.
pub fn debugconPresent() bool {
    return io.inb(0xE9) == 0xE9;
}

/// Output sink: write bytes to the 0xE9 debug console (see `debugconPresent`).
pub fn debugconWrite(bytes: []const u8) void {
    for (bytes) |b| io.outb(0xE9, b);
}

/// Set up the CPU's descriptor tables: our own GDT, the TSS (with an interrupt
/// stack for double faults), then the IDT with exception handlers. After this a
/// CPU fault is reported instead of triple-faulting. Install the fault handler
/// (setFaultHandler) first so early faults are caught.
pub fn init() void {
    gdt.init();
    tss.init();
    idt.init();
    pcpu.initSystemCall();
}

/// Build the kernel's own page tables (with real permissions) and switch onto
/// them. Needs the frame allocator and the boot info (for the memory map and the
/// kernel's segment layout). Call once the frame allocator is up.
pub fn enablePaging(allocFrame: *const fn () ?u64, freeFrame: *const fn (u64) void, boot_information: *const boot_handoff.BootInformation) void {
    paging.init(allocFrame, freeFrame, boot_information);
}

/// Create a new address space (returns the physical address of its root table —
/// the PML4 here — or null). Shares the kernel's higher half; the user (low)
/// half starts empty.
pub fn createAddressSpace() ?u64 {
    return paging.createAddressSpace();
}

/// Free an address space and everything mapped in its user half. Caller must not
/// be running on it.
pub fn destroyAddressSpace(root: u64) void {
    paging.destroyAddressSpace(root);
}

/// Map a user page into address space `root` (W^X is the caller's contract).
pub fn mapUserPageInto(root: u64, virtual: u64, physical: u64, writable: bool, executable: bool) void {
    paging.mapUserInto(root, virtual, physical, writable, executable);
}

/// Map a device MMIO window into address space `root`: strong-uncacheable, RW+NX,
/// and marked so teardown won't free the MMIO frames as RAM. For IO passthrough.
pub fn mapUserDeviceInto(root: u64, virtual: u64, physical: u64, len: u64) void {
    paging.mapUserDeviceInto(root, virtual, physical, len);
}

/// Map a page into the kernel address space (non-executable). For the heap, etc.
pub fn mapPage(virtual: u64, physical: u64, writable: bool) void {
    paging.map(virtual, physical, writable);
}

/// Map a device MMIO range and return the virtual address to reach it at. This
/// is the device layer's `Hal.mapMmio` — it hands back a physmap pointer and
/// never exposes how the mapping is placed.
pub fn mapMmio(physical: u64, len: u64, writable: bool) u64 {
    return paging.mapMmio(physical, len, writable);
}

/// The kernel's page-table root (physical), shared into every address space.
pub fn kernelPageTable() u64 {
    return paging.kernelPml4();
}

/// Switch the active address space (load CR3 with a physical root table).
pub fn loadPageTable(root: u64) void {
    paging.loadCr3(root);
}

/// The physical root of the currently active page tables (CR3 here; TTBR0/satp
/// elsewhere).
pub fn activePageTable() u64 {
    return asm volatile ("mov %%cr3, %[out]"
        : [out] "=r" (-> u64),
    );
}

/// Set core `cpu`'s kernel stack pointer for ring-3 -> ring-0 transitions:
/// TSS.rsp0 (for interrupts/exceptions, which switch stacks in hardware) and the
/// per-CPU `kernel_rsp` (for the system_call stub, which switches by hand). Updated
/// by the scheduler when it switches to a user task.
pub fn setKernelStack(cpu: usize, top: usize) void {
    tss.rsp0Ptr(cpu).* = top;
    pcpu.setKernelRsp(cpu, top);
}

/// Remove a kernel mapping.
pub fn unmapPage(virtual: u64) void {
    paging.unmap(virtual);
}

/// Remove a page mapping from address space `root` (for munmap of user pages).
/// Clears the leaf entry only; freeing the underlying frame is the caller's job.
pub fn unmapUserPageInto(root: u64, virtual: u64) void {
    paging.unmapInto(root, virtual);
}

/// Resolve `virtual` to its physical address in the address space rooted at `root`
/// (any address space, not just the live one), or null if unmapped. Used to find
/// the frame behind a user page for munmap, and for cross-address-space copies.
pub fn translate(root: u64, virtual: u64) ?u64 {
    return paging.translateIn(root, virtual);
}

/// Map a page accessible from ring 3 (U/S bit at every level). The caller keeps
/// W^X: code read-only + executable, data writable + no-execute.
pub fn mapUserPage(virtual: u64, physical: u64, writable: bool, executable: bool) void {
    paging.mapUser(virtual, physical, writable, executable);
}

// --- ring 3 entry/exit -----------------------------------------------------

/// Drop to ring 3 at `rip` on `rsp` (defined in isr.s). Saves the kernel context,
/// publishes the kernel stack pointer through `rsp0_slot` (this core's TSS.rsp0,
/// so ring-3 interrupts land on a good stack), builds an iretq frame with the
/// user selectors, and iretq's. "Returns" only when the user program triggers
/// the exit path (user_exit_to_kernel).
extern fn enter_user(rip: u64, rsp: u64, rsp0_slot: *align(4) u64) callconv(.c) void;

/// Abandon the in-flight ring-3 trap context and resume the kernel as if
/// `enter_user` had returned (defined in isr.s). Called by the exit system_call.
extern fn user_exit_to_kernel() callconv(.c) noreturn;

/// Run user code at `entry` with stack `stack_top` on this core (`cpu` = the
/// caller's CPU index; the architecture layer can't ask the scheduler). Returns after the
/// user program exits via system_call. Interrupts are disabled on return (the exit
/// arrives through an interrupt gate) — the caller re-enables.
pub fn enterUser(cpu: usize, entry: u64, stack_top: u64) void {
    enter_user(entry, stack_top, tss.rsp0Ptr(cpu));
}

/// Never returns to the user program: unwind to the kernel context that called
/// `enterUser`. For the exit system_call's handler.
pub fn userExit() noreturn {
    user_exit_to_kernel();
}

/// Register the handler for the user system_call gate (int 0x80, vector 128). The
/// handler may write the trap frame (see `setSystemCallResult`).
pub fn setSystemCallHandler(handler: *const fn (*CpuState) void) void {
    idt.setSystemCallHandler(handler);
}

/// Publish core `cpu`'s scheduler pointer via its per-CPU block (GS base). Each
/// core calls this once, after its GDT is in place (a GS *selector* reload would
/// clobber the base). See percpu.zig for the swapgs discipline.
pub fn setCpuLocal(cpu: usize, ptr: usize) void {
    pcpu.setLocal(cpu, ptr);
}

/// This core's scheduler pointer (via the GS base) — a per-core register, so each
/// core sees its own without locking. Valid in any ring-0 context.
pub fn cpuLocal() usize {
    return pcpu.scheduler();
}

// --- SMP: application-processor bring-up ----------------------------------

/// Record the low (<1 MiB) frame reserved for the AP trampoline. Run once at boot.
/// The frame stays inert (zeroed, non-executable) between wakes and is armed only
/// while a core is climbing — so a core can be (re)woken at any time (retry, or a
/// future power manager) without leaving an executable page resident. See smp.zig.
pub fn setTrampolinePage(physical: u64) void {
    smp.setTrampolinePage(physical);
}

/// Wake the core with hardware id `hw_id` (its Local APIC id here; MPIDR on
/// aarch64, hart id on riscv64) as dense CPU `index`, giving it `stack_top` and
/// its per-CPU pointer `percpu`; it adopts the kernel page tables. Returns false
/// if it doesn't come online within the timeout. Blocks until the core reports in.
pub fn startSecondary(hw_id: u32, stack_top: usize, percpu: usize, index: usize) bool {
    // The AP adopts the kernel page tables explicitly — never the caller's live
    // CR3, which a future re-wake from a core running a process would make a
    // process address space.
    return smp.startAp(hw_id, stack_top, percpu, index, paging.kernelPml4());
}

/// Register the generic entry a woken AP jumps to once its architecture state is up (its own
/// descriptor tables, LAPIC, and timer). The kernel passes its scheduler entry here.
pub fn setSecondaryEntry(entry: *const fn () callconv(.c) noreturn) void {
    smp.setSecondaryEntry(entry);
}

/// Bytes the kernel should allocate for a secondary core's dedicated fault stack
/// (the IST double-fault stack here), and where to record its top before waking
/// the core. The stack is heap-allocated per online core (the boot CPU's is
/// static — it's needed before the allocator exists). See tss.zig.
pub const fault_stack_size = tss.ist_stack_size;
pub fn setFaultStack(cpu: usize, top: usize) void {
    tss.setApIstStack(cpu, top);
}

/// Test hook: force the next `n` AP wake attempts to fail, so the retry path can be
/// exercised deterministically (see the smp-retry test). No effect when `n` is 0.
pub fn testFailNextWakes(n: u32) void {
    smp.testFailNextWakes(n);
}

/// The reserved AP-trampoline frame (0 if none). For tests that check it's inert.
pub fn trampolinePage() u64 {
    return smp.trampolinePage();
}

/// Whether the page at `virtual` is currently mapped executable (present, NX clear).
pub fn pageExecutable(virtual: u64) bool {
    return paging.isExecutable(virtual);
}

/// Kernel tick rate (the scheduler's time quantum), from configuration.
pub const timer_hz = parameters.timer_hz;

/// The ACPI PM timer, as a calibration reference (re-exported for the configuration).
pub const PmTimer = apic.PmTimer;
/// A MADT interrupt-source override (re-exported for the configuration).
pub const IsoEntry = ioapic.IsoEntry;

/// Discovered platform facts the architecture layer needs so it makes no legacy
/// assumptions — sourced from the device tree + ACPI, passed in by the kernel.
pub const PlatformConfiguration = struct {
    /// Whether the legacy 8259 PIC is present (skip programming it if not).
    pic_present: bool = true,
    /// HPET MMIO base (0 = none) — a calibration reference for the timer.
    hpet_base: u64 = 0,
    /// The ACPI PM timer, another calibration reference.
    pm_timer: ?PmTimer = null,
    /// I/O APIC MMIO base + its first global system interrupt (0 = none).
    ioapic_base: u64 = 0,
    ioapic_gsi_base: u32 = 0,
    /// MADT ISA-IRQ overrides, for I/O APIC routing.
    overrides: []const IsoEntry = &.{},
};

/// Apply the discovered platform configuration. Must run before `startTimer` (the timer
/// calibration reads `hpet_base`/`pm_timer`) and before any interrupt routing.
/// Maps + masks the I/O APIC immediately.
pub fn configurePlatform(configuration: PlatformConfiguration) void {
    apic.configure(configuration.pic_present, configuration.hpet_base, configuration.pm_timer);
    ioapic.configure(configuration.ioapic_base, configuration.ioapic_gsi_base, configuration.overrides);
    ioapic.init();
}

/// Point the serial console at the UART ACPI's SPCR table named (MMIO or I/O port).
pub fn serialReconfigure(is_mmio: bool, address: u64) void {
    serial.reconfigure(is_mmio, address);
}

/// The reference clock the timer was calibrated against ("cpuid"/"hpet"/…).
pub fn timerCalibrationSource() []const u8 {
    return apic.calibrationSource();
}

/// External-interrupt-router diagnostics, for boot logging / verification (the
/// I/O APIC's redirection entries here; a GIC distributor or PLIC elsewhere).
pub fn irqRouteCount() u32 {
    return ioapic.entryCount();
}
pub fn irqRouteRaw(n: u32) u32 {
    return ioapic.entryLow(n);
}

// --- device-IRQ plumbing, for system/kernel/irq.zig -----------------------------
//
// The generic IRQ layer speaks GSIs and vectors; everything below hides the fact
// that on x86_64 those mean "I/O APIC redirection entry" and "IDT gate". The
// vector window is bounded by the stubs isr.s actually emits: `gate_count` = 48,
// vector 32 is the LAPIC timer and 47 is the spurious vector, leaving 33..46.

pub const irq_vector_base: u8 = 33;
pub const irq_vector_count: u8 = 14; // 33..46 inclusive

/// True if `gsi` is one this machine's interrupt router can deliver.
pub fn irqOwnsGsi(gsi: u32) bool {
    return ioapic.ownsGsi(gsi);
}

/// Install `handler` on `vector` (an absolute IDT gate index).
pub fn irqSetHandler(vector: u8, handler: *const fn () void) void {
    idt.setHandler(vector, handler);
}

/// Route `gsi` to `vector` on *this* core, masked. Unmask with `irqUnmask` once bound.
pub fn irqRoute(gsi: u32, vector: u8, level: bool, active_low: bool) void {
    ioapic.routeGsi(gsi, vector, apic.localId(), level, active_low);
}

pub fn irqMask(gsi: u32) void {
    ioapic.maskGsi(gsi);
}
pub fn irqUnmask(gsi: u32) void {
    ioapic.unmaskGsi(gsi);
}

/// Acknowledge the interrupt currently in service on this core's LAPIC.
pub fn irqEoi() void {
    apic.eoi();
}

/// Enable the Local APIC, calibrate its timer against the best available reference
/// (see apic.calibrate — no longer the PIT by default), and start it firing at
/// `timer_hz` — the kernel's real-time heartbeat. Interrupts still have to be
/// unmasked with enableInterrupts() to be delivered. Run `configurePlatform` first.
pub fn startTimer() void {
    apic.init();
    apic.calibrate();
    idt.setHandler(apic.timer_vector, apic.timerTick);
    apic.initTimer(timer_hz);
}

/// Number of timer ticks since startTimer().
pub fn ticks() u64 {
    return apic.ticks();
}

// Monotonic high-resolution clock (from the TSC), one function per resolution.
pub fn nanos() u64 {
    return apic.nanos();
}
pub fn micros() u64 {
    return apic.micros();
}
pub fn millis() u64 {
    return apic.millis();
}

/// Measured frequency of the tick timer's input clock (the LAPIC timer here), in
/// Hz, from calibration.
pub fn timerClockHz() u64 {
    return apic.lapicHz();
}

/// Measured frequency of the monotonic clock's underlying counter (the TSC here;
/// CNTVCT on aarch64, `time` on riscv64), in Hz.
pub fn clockHz() u64 {
    return apic.tscHz();
}

/// Unmask maskable interrupts (`sti`) so device interrupts get delivered.
pub fn enableInterrupts() void {
    asm volatile ("sti");
}

/// Mask maskable interrupts (`cli`).
pub fn disableInterrupts() void {
    asm volatile ("cli");
}

/// Disable interrupts and return the previous flags, so a nested critical section
/// can restore the caller's state rather than blindly re-enabling. Pairs with
/// restoreInterrupts.
pub fn saveInterrupts() u64 {
    var flags: u64 = undefined;
    asm volatile (
        \\pushfq
        \\pop %[f]
        \\cli
        : [f] "=r" (flags),
        :
        : .{ .memory = true }
    );
    return flags;
}

/// Re-enable interrupts only if they were enabled when `flags` was captured.
pub fn restoreInterrupts(flags: u64) void {
    if (flags & 0x200 != 0) asm volatile ("sti" ::: .{ .memory = true }); // bit 9 = IF
}

/// Register a callback the timer interrupt invokes each tick (e.g. the scheduler).
pub fn setTickHook(hook: *const fn () void) void {
    apic.setTickHook(hook);
}

// --- context switching (for the scheduler) -------------------------------

/// Save the current task's registers/stack and resume `new_rsp`; the old stack
/// pointer is written to `old_rsp`. Defined in isr.s.
extern fn switch_context(old_rsp: *usize, new_rsp: usize) callconv(.c) void;

pub fn switchContext(old_sp: *usize, new_sp: usize) void {
    switch_context(old_sp, new_sp);
}

/// Build the initial stack for a new task so that switching to it lands in
/// `task_trampoline`, which then calls `entry`. Returns the saved stack pointer.
/// The layout must match switch_context's push order (callee-saved, then the
/// return address on top); `entry` is smuggled in via the r15 slot.
pub fn initTaskStack(stack_top: usize, entry: usize) usize {
    const trampoline = @extern(*const anyopaque, .{ .name = "task_trampoline" });
    var sp = stack_top;
    const push = struct {
        fn f(p: *usize, value: usize) void {
            p.* -= @sizeOf(usize);
            @as(*usize, @ptrFromInt(p.*)).* = value;
        }
    }.f;
    push(&sp, @intFromPtr(trampoline)); // return address for switch_context's `ret`
    push(&sp, 0); // rbx
    push(&sp, 0); // rbp
    push(&sp, 0); // r12
    push(&sp, 0); // r13
    push(&sp, 0); // r14
    push(&sp, entry); // r15 -> task entry, read by task_trampoline
    return sp;
}

/// Drop the current (kernel-context) task to ring 3 at `rip` on `rsp`, never
/// returning (defined in isr.s). Used by the scheduler's user-task trampoline
/// once it has switched onto the task and read its entry/stack. Interrupts are
/// disabled across the swapgs+iretq so no interrupt observes the user GS base in
/// ring 0; the pushed RFLAGS re-enables them in ring 3.
extern fn jump_to_user(rip: u64, rsp: u64) callconv(.c) noreturn;

pub fn jumpToUser(entry: u64, stack_top: u64) noreturn {
    jump_to_user(entry, stack_top);
}

/// Route CPU exceptions to `handler`, which receives the trap frame and does not
/// return. Until set, faults just halt the core.
pub fn setFaultHandler(handler: *const fn (*const CpuState) noreturn) void {
    idt.on_fault = handler;
}

/// A human-readable name for a CPU exception vector.
pub fn exceptionName(vector: u64) []const u8 {
    return idt.vectorName(vector);
}

/// Read `width` bytes (1/2/4) from an I/O port. The generic device layer drives
/// ACPI registers through this rather than naming x86 port instructions; on an
/// MMIO-only architecture this would be implemented differently.
pub fn pioRead(width: u8, port: u16) u32 {
    return switch (width) {
        1 => io.inb(port),
        2 => io.inw(port),
        4 => io.inl(port),
        else => 0,
    };
}

/// Write `width` bytes (1/2/4) to an I/O port.
pub fn pioWrite(width: u8, port: u16, value: u32) void {
    switch (width) {
        1 => io.outb(port, @truncate(value)),
        2 => io.outw(port, @truncate(value)),
        4 => io.outl(port, value),
        else => {},
    }
}

/// Park the core forever. `hlt` drops it into a low-power idle until the next
/// interrupt; the loop re-halts on every wake so the stop is permanent. See
/// docs/halting.md for the full reasoning.
pub fn halt() noreturn {
    while (true) asm volatile ("hlt");
}

/// Spin-wait hint (`pause`). Emitted in the body of a spinlock's busy-wait: it
/// relaxes the core while it polls a contended lock — yielding pipeline resources
/// to a hyperthread sibling and easing the cache-coherency traffic on the lock
/// line. Purely a performance/power hint; correct to omit, but kinder on the bus.
pub fn cpuRelax() void {
    asm volatile ("pause");
}