danos/system/kernel/architecture/x86_64/smp.zig

//! Application-processor (AP) bring-up: waking the cores the firmware left parked.
//!
//! The firmware starts only the bootstrap processor (BSP); the others sit idle until
//! the kernel wakes them with an INIT–SIPI–SIPI sequence (Intel SDM Vol.3, "MP
//! Initialization"). A woken core begins in 16-bit real mode at a low physical page,
//! runs the [trampoline](trampoline.s) up into 64-bit long mode, and lands in
//! `apEntry` here. This module copies the trampoline into place, patches its
//! per-AP parameters, drives the wake IPIs, and waits for each core to report in.
//!
//! Cores are brought up **one at a time**: a single trampoline page and parameter
//! block are reused, so the BSP patches, wakes, and waits for one AP before the
//! next. That also lets `apEntry` pick up its dense CPU index from a plain global.
//! Once a core has its own descriptor tables, LAPIC, and timer, it calls the generic
//! scheduler entry and joins the run loop — mechanism here, policy there.

const boot_handoff = @import("boot-handoff");
const io = @import("io.zig");
const gdt = @import("gdt.zig");
const tss = @import("tss.zig");
const idt = @import("idt.zig");
const apic = @import("apic.zig");
const paging = @import("paging.zig");
const pcpu = @import("per-cpu.zig");

/// IA32_GS_BASE — the per-CPU data pointer (see cpu.zig; kept in sync here so the AP
/// path doesn't depend on cpu.zig and risk an import cycle).
const ia32_gs_base = 0xC000_0101;
const page_size = 0x1000;

/// Physical address of the low (<1 MiB) frame reserved for the trampoline. Held for
/// the life of the system so any core can be (re)woken on demand — a retry, or a
/// future power manager bringing a core back online. The frame is kept **inert**
/// between wakes (zeroed and non-executable) and only armed for the brief moment a
/// core is actually climbing. Its low 20 bits are zero, so `physical >> 12` is the SIPI
/// vector.
var tramp_physical: u64 = 0;

/// Set to 1 by a freshly-woken AP once it reaches `apEntry` and finishes its own
/// bring-up. The BSP clears it before each wake and polls it afterwards — a simple
/// one-at-a-time handshake (only one AP is being started at any moment).
var ap_alive: u32 = 0;

/// The dense CPU index of the AP currently being started. Set by the BSP before the
/// wake, read by `apEntry` (safe because bring-up is strictly one core at a time).
var boot_index: usize = 0;

/// The generic scheduler entry a woken core jumps to once its architecture state is up. Set
/// by the kernel via `setSecondaryEntry`; never returns.
var secondary_entry: ?*const fn () callconv(.c) noreturn = null;

/// Register the generic entry an AP calls once its per-CPU tables/LAPIC/timer are up.
pub fn setSecondaryEntry(entry: *const fn () callconv(.c) noreturn) void {
    secondary_entry = entry;
}

/// Test hook: force the next `n` wake attempts to fail (skipping the actual
/// INIT-SIPI-SIPI), so the retry path can be exercised deterministically. Zero in
/// normal operation — the smp-retry test arms it via `architecture.testFailNextWakes`.
var fail_next_wakes: u32 = 0;
pub fn testFailNextWakes(n: u32) void {
    fail_next_wakes = n;
}

/// Record the reserved low frame the trampoline uses. Call once at boot. The frame
/// starts inert (identity-mapped RW+NX like all RAM); each wake arms it and disarms
/// it again, so it's only ever executable while a core is climbing.
pub fn setTrampolinePage(physical: u64) void {
    tramp_physical = physical;
}

/// The reserved trampoline frame (0 if SMP bring-up never ran). Exposed so a test
/// can verify it's inert — zeroed and non-executable — when dormant.
pub fn trampolinePage() u64 {
    return tramp_physical;
}

/// Arm the trampoline for a wake: make its page executable (W^X exception for the
/// duration of the climb) and copy the blob in.
fn arm() void {
    // The AP executes this page at its physical address (identity) while it
    // climbs from real to long mode, so it needs a low identity mapping that is
    // executable — the one deliberate, transient W^X exception. The BSP writes
    // the blob into the frame through the physmap.
    paging.setExecutable(tramp_physical);
    const start = @extern([*]const u8, .{ .name = "ap_trampoline_start" });
    const end = @extern([*]const u8, .{ .name = "ap_trampoline_end" });
    const len = @intFromPtr(end) - @intFromPtr(start);
    const destination: [*]u8 = @ptrFromInt(boot_handoff.physicalToVirtual(tramp_physical));
    @memcpy(destination[0..len], start[0..len]);
}

/// Disarm after a wake: wipe the page through the physmap and remove its low
/// identity mapping, so no executable code (nor any stale bytes, nor any
/// low-half mapping) lingers between wakes. Safe once the woken core has
/// reported in — it's long past the trampoline by then, in the kernel image; a
/// core that never answered is dead and can't be mid-climb.
fn disarm() void {
    const destination: [*]u8 = @ptrFromInt(boot_handoff.physicalToVirtual(tramp_physical));
    @memset(destination[0..page_size], 0);
    paging.unmap(tramp_physical); // drop the transient low identity mapping
}

/// Address of a patchable trampoline parameter, by symbol name: the copied blob's
/// base plus the field's offset within it (a same-section symbol difference). The
/// pointer is `align(1)` — the fields aren't 8-aligned within the blob, and x86
/// tolerates unaligned stores, so we don't force layout constraints on the asm.
fn param(comptime name: []const u8) *align(1) volatile u64 {
    const start = @intFromPtr(@extern([*]const u8, .{ .name = "ap_trampoline_start" }));
    const sym = @intFromPtr(@extern([*]const u8, .{ .name = name }));
    return @ptrFromInt(boot_handoff.physicalToVirtual(tramp_physical + (sym - start)));
}

/// Wake the core with Local APIC id `apic_id` as dense CPU `index`, hand it
/// `stack_top` and its per-CPU pointer `percpu`, and wait for it to come alive. This
/// is one self-contained attempt: it arms the trampoline, drives INIT–SIPI–SIPI, and
/// disarms again before returning — so it's safe to call repeatedly (a retry, or a
/// power manager re-waking a core; the INIT resets a core that was wedged). Returns
/// false if the core doesn't report in within the timeout (left parked, no harm to
/// the running system). `cr3` is the kernel page tables the AP adopts. Precondition:
/// `setTrampolinePage` has run.
pub fn startAp(apic_id: u32, stack_top: usize, percpu: usize, index: usize, cr3: u64) bool {
    // The trampoline loads CR3 with a 32-bit `movl` before it reaches long mode,
    // so the page-table root must be addressable in 32 bits.
    if (cr3 >= (1 << 32)) @panic("smp: kernel page tables above 4 GiB");
    arm();
    defer disarm();

    if (fail_next_wakes > 0) { // test hook: simulate a core missing this attempt
        fail_next_wakes -= 1;
        return false;
    }

    boot_index = index;
    param("ap_tramp_cr3").* = cr3;
    param("ap_tramp_stack").* = stack_top;
    param("ap_tramp_entry").* = @intFromPtr(&apEntry);
    param("ap_tramp_percpu").* = percpu;

    @atomicStore(u32, &ap_alive, 0, .seq_cst);

    const vector: u8 = @intCast(tramp_physical >> 12);
    apic.sendInit(apic_id);
    delayMicros(10_000); // 10 ms INIT settle
    apic.sendStartup(apic_id, vector);
    delayMicros(200);
    apic.sendStartup(apic_id, vector);

    // Wait up to 100 ms for the AP to reach apEntry and set the flag.
    const deadline = apic.millis() + 100;
    while (apic.millis() < deadline) {
        if (@atomicLoad(u32, &ap_alive, .acquire) != 0) {
            // Cross-check this core's TSC against the BSP's before it joins the run
            // loop: an unsynchronized TSC must be caught before any task can migrate
            // onto this core and observe time going backward. No-op unless the TSC is
            // the clocksource (apic.checkWarpSource).
            apic.checkWarpSource();
            return true;
        }
        asm volatile ("pause");
    }
    return false;
}

/// Busy-wait `us` microseconds against the calibrated TSC clock (the AP wake happens
/// after the timer is up, so the clock is available).
fn delayMicros(us: u64) void {
    const start = apic.micros();
    while (apic.micros() - start < us) asm volatile ("pause");
}

/// The 64-bit entry every AP lands on, called from the trampoline with its per-CPU
/// pointer in RDI. Brings up this core's own descriptor tables, LAPIC and timer,
/// signals the BSP, then jumps to the generic scheduler entry. Never returns.
fn apEntry(percpu: usize) callconv(.c) noreturn {
    const cpu = boot_index;
    gdt.loadOnThisCpu(cpu); // this core's GDT (with its own TSS slot)
    tss.setupThisCpu(cpu); // this core's TSS + IST stack, loaded into TR
    idt.loadOnThisCpu(); // the shared IDT
    pcpu.setLocal(cpu, percpu); // per-CPU block via GS base — *after* the GDT reload
    pcpu.initSystemCall(); // enable system_call/sysret on this core

    apic.initSecondary(); // software-enable this core's LAPIC
    apic.initTimer(apic.frequencyHz()); // arm its timer (still masked: interrupts off)

    @atomicStore(u32, &ap_alive, 1, .release); // "architecture state up" — BSP is polling this

    // Rendezvous with the BSP for the TSC warp check (no-op unless the TSC is the
    // clocksource) before joining the run loop, so this core's clock is vetted before
    // it can run any task.
    apic.checkWarpTarget();

    if (secondary_entry) |enterScheduler| enterScheduler(); // joins the run loop
    while (true) asm volatile ("hlt"); // (only if no entry was registered)
}