danos/system/kernel/architecture/x86_64/apic.zig

//! Local APIC and its timer — the source of device interrupts.
//!
//! Modern x86 routes interrupts through the per-CPU Local APIC (the legacy 8259
//! PIC is remapped out of the way and masked). The LAPIC also has a built-in
//! timer, which is the simplest device interrupt to bring up: it needs no
//! external routing, just a vector and a count. We use it as danos's heartbeat.
//!
//! The LAPIC is memory-mapped (default physical 0xFEE00000, inside our identity
//! map). Every interrupt must be acknowledged with an end-of-interrupt write, or
//! the LAPIC won't deliver the next one.

const boot_handoff = @import("boot-handoff");
const io = @import("io.zig");
const paging = @import("paging.zig");

/// The ACPI PM timer, as a calibration reference: an I/O port or MMIO counter.
pub const PmTimer = struct { mmio: bool, address: u64, is_32bit: bool };

// Platform facts from discovery (set by `configure` before bring-up). Defaults are
// the legacy-safe assumptions so the code still works if discovery never ran.
var configuration_pic_present: bool = true;
var configuration_hpet_base: u64 = 0; // 0 = no HPET discovered
var configuration_pm_timer: ?PmTimer = null;
/// Which reference the last calibration used, for logging.
var cal_source: []const u8 = "none";

/// Hand the LAPIC bring-up the discovered platform facts. Call before `init`.
pub fn configure(pic_present: bool, hpet_base: u64, pm_timer: ?PmTimer) void {
    configuration_pic_present = pic_present;
    configuration_hpet_base = hpet_base;
    configuration_pm_timer = pm_timer;
}

/// The calibration reference the timer was measured against ("cpuid"/"hpet"/…).
pub fn calibrationSource() []const u8 {
    return cal_source;
}

/// IDT vector the timer fires on (in the device range, >= 32).
pub const timer_vector = 32;
/// Spurious-interrupt vector. Low nibble 0xF by convention; also in our gate
/// range so a stray spurious interrupt lands on a valid (no-op) handler.
const spurious_vector = 47;

// LAPIC register offsets.
const register_spurious = 0x0F0;
const register_eoi = 0x0B0;
const register_id = 0x020; // this core's LAPIC id, in bits 24-31
const register_icr_low = 0x300; // interrupt command register, low dword (writing it sends)
const register_icr_high = 0x310; // ICR high dword (destination APIC id in bits 24-31)
const register_lvt_timer = 0x320;
const register_timer_initial = 0x380;
const register_timer_current = 0x390;
const register_timer_divide = 0x3E0;

const icr_delivery_pending = 1 << 12; // ICR low bit 12: a previous IPI is still in flight

const lvt_masked = 1 << 16;
const lvt_periodic = 1 << 17;
const timer_divide_16 = 0x3;

const ia32_apic_base_msr = 0x1B;

/// LAPIC MMIO base. A runtime var (not a constant) both because we read it from
/// the MSR and so register writes compile to normal stores rather than a
/// `mov moffs`, which the self-hosted backend can't encode.
var base: usize = 0xFEE00000;

var tick_count: u64 = 0;

/// LAPIC timer counts per millisecond, measured against the PIT (see calibrate).
/// At divide-by-16, this is the effective counting rate.
var ticks_per_ms: u32 = 0;
/// The periodic-interrupt frequency the timer is armed at, once initTimer runs.
var timer_hz: u32 = 0;

/// TSC (Time Stamp Counter) calibration: cycles per second, and the count at boot.
/// The TSC is a per-core cycle counter, giving a ~nanosecond high-resolution
/// monotonic clock — far finer than the millisecond timer tick.
var tsc_hz: u64 = 0;
var tsc_base: u64 = 0;

/// Whether the TSC is architecturally **invariant** — a constant rate regardless of
/// P/C-state transitions, and thus valid as a clocksource (CPUID leaf 0x80000007,
/// EDX bit 8). AMD and modern Intel set it; the bare qemu64 model does not. Measured
/// frequency alone is not enough: a non-invariant TSC speeds up and slows down with
/// the core clock, so reading it as wall time would drift.
var tsc_invariant: bool = false;
/// Cleared if the cross-core warp check (checkWarpSource) ever sees the TSC read
/// lower on one core than the max another core has already published — i.e. the
/// per-core TSCs are not synchronized, and a task migrating cores could see time go
/// backward. Starts true (assume synchronized until proven otherwise).
var tsc_synced: bool = true;
/// The worst backward skew the warp check observed, in TSC cycles (0 = none).
var tsc_warp_cycles: u64 = 0;

/// The monotonic clock's source. The TSC when it is invariant *and* synchronized —
/// the fast `rdtsc` path taken on real Intel/AMD and modern VMs. Otherwise the HPET
/// main counter: a single fixed-rate counter, immune to both per-core skew and
/// frequency scaling, so it stays accurate on a bare VM or a warped machine.
const ClockSource = enum { tsc, hpet };
var clock_source: ClockSource = .tsc;

/// HPET standby clocksource, set up in calibrate() whenever an HPET exists (whether
/// or not calibration itself measured against it): its frequency, the counter value
/// chosen as the zero point, and its width mask. Only a 64-bit HPET is used as a
/// clocksource — a 32-bit one wraps too fast to be monotonic without accumulation.
var hpet_clock_hz: u64 = 0;
var hpet_clock_base: u64 = 0;
var hpet_clock_mask: u64 = ~@as(u64, 0);

/// Read the 64-bit Time Stamp Counter.
fn rdtsc() u64 {
    var low: u32 = undefined;
    var high: u32 = undefined;
    asm volatile ("rdtsc"
        : [low] "={eax}" (low),
          [high] "={edx}" (high),
    );
    return (@as(u64, high) << 32) | low;
}

fn read(register: u32) u32 {
    return @as(*volatile u32, @ptrFromInt(base + register)).*;
}
fn write(register: u32, value: u32) void {
    @as(*volatile u32, @ptrFromInt(base + register)).* = value;
}

/// Move the legacy 8259 PIC's vectors to 0x20-0x2F (clear of the CPU exception
/// vectors) and mask every line, so it can't deliver interrupts behind the APIC.
fn remapAndMaskPic() void {
    io.outb(0x20, 0x11); // start init (cascade mode)
    io.outb(0xA0, 0x11);
    io.outb(0x21, 0x20); // master offset 0x20
    io.outb(0xA1, 0x28); // slave offset 0x28
    io.outb(0x21, 0x04); // tell master about slave on IRQ2
    io.outb(0xA1, 0x02);
    io.outb(0x21, 0x01); // 8086 mode
    io.outb(0xA1, 0x01);
    io.outb(0x21, 0xFF); // mask all
    io.outb(0xA1, 0xFF);
}

/// Enable the Local APIC: mask the PIC (only if one is present — a legacy-free
/// UEFI Class 3 machine may have none), set the global-enable MSR bit, and
/// software-enable the APIC via its spurious-vector register.
pub fn init() void {
    if (configuration_pic_present) remapAndMaskPic();

    const msr = io.rdmsr(ia32_apic_base_msr);
    // Reach the LAPIC through the physmap (paging.init maps its page there).
    base = @intCast(boot_handoff.physicalToVirtual(msr & 0xFFFFF000)); // physical base is bits 12+
    io.wrmsr(ia32_apic_base_msr, msr | (1 << 11)); // global enable

    write(register_spurious, 0x100 | spurious_vector); // bit 8 = software enable
}

/// Software-enable *this* core's Local APIC — the application-processor counterpart
/// of `init`, minus the one-time PIC remap (the BSP already masked it) and minus
/// calibration (the timer rate is a shared hardware constant, measured once). Each
/// core has its own LAPIC at the same MMIO address, so no per-core base is needed.
pub fn initSecondary() void {
    const msr = io.rdmsr(ia32_apic_base_msr);
    io.wrmsr(ia32_apic_base_msr, msr | (1 << 11)); // global enable
    write(register_spurious, 0x100 | spurious_vector); // software enable
}

// --- application-processor wakeup (INIT–SIPI–SIPI) --------------------------

/// Send an INIT IPI to the core with Local APIC id `apic_id` — the first step of
/// the wake sequence. Blocks until the LAPIC reports the IPI was delivered.
pub fn sendInit(apic_id: u32) void {
    write(register_icr_high, apic_id << 24);
    write(register_icr_low, 0x4500); // INIT, physical destination, assert, edge-triggered
    waitIcrIdle();
}

/// Send a STARTUP IPI (SIPI) telling the target core to begin executing at physical
/// address `vector << 12` (in real mode). Per the Intel bring-up protocol this is
/// sent twice after the INIT; both calls block until delivery completes.
pub fn sendStartup(apic_id: u32, vector: u8) void {
    write(register_icr_high, apic_id << 24);
    write(register_icr_low, 0x4600 | @as(u32, vector)); // STARTUP with the page vector
    waitIcrIdle();
}

fn waitIcrIdle() void {
    while (read(register_icr_low) & icr_delivery_pending != 0) {}
}

/// Send this core a fixed interrupt at `vector` (the "self" destination shorthand). A
/// device raises an MSI by writing its (address, data) to the LAPIC; with no such
/// device on QEMU's HPET, a self-IPI is the stand-in that lets the MSI vector-routing
/// path be tested end to end. Shorthand self (bits 19:18 = 01) | assert (bit 14).
pub fn selfIpi(vector: u8) void {
    write(register_icr_low, 0x4_4000 | @as(u32, vector));
    waitIcrIdle();
}

/// The calibration window: we time everything against a 10 ms reference interval.
const calib_ms = 10;

/// Measure the LAPIC timer's and the TSC's rates. The PIT (legacy 8254) can be
/// absent on UEFI Class 3 firmware — and polling it would hang — so we pick a
/// reference clock in order of preference: the CPU's own TSC frequency (CPUID leaf
/// 0x15, no external timer needed), then the discovered HPET, then the ACPI PM
/// timer, and only the PIT as a last resort. Each path yields the same two rates.
pub fn calibrate() void {
    var done = false;

    // 1. CPUID leaf 0x15 gives the TSC frequency directly — measure the LAPIC
    //    against the TSC itself, needing no external timer at all.
    if (cpuidTscHz()) |hz| {
        measure(hz, ~@as(u64, 0), rdtsc);
        tsc_hz = hz; // keep the exact enumerated value
        cal_source = "cpuid";
        done = true;
    }

    // 2. The discovered HPET.
    if (!done and configuration_hpet_base != 0) {
        if (hpetHz()) |hpet_hz| {
            measure(hpet_hz, hpetMask(), readHpet);
            cal_source = "hpet";
            done = true;
        }
    }

    // 3. The ACPI PM timer (fixed 3.579545 MHz).
    if (!done) {
        if (configuration_pm_timer) |pt| {
            measure(3_579_545, if (pt.is_32bit) 0xFFFF_FFFF else 0xFF_FFFF, readPmTimer);
            cal_source = "pm-timer";
            done = true;
        }
    }

    // 4. The legacy PIT, last resort.
    if (!done) {
        calibratePit();
        cal_source = "pit";
    }

    // A bad measurement (no reference actually ticked) leaves nonsense; fall back.
    if (ticks_per_ms == 0 or tsc_hz == 0) {
        calibratePit();
        cal_source = "pit";
    }

    tsc_base = rdtsc(); // the clock's zero point (boot)

    // Decide whether the TSC is trustworthy as a clocksource. Frequency (measured
    // above, possibly against the HPET/PIT) is necessary but not sufficient: the TSC
    // must also be *invariant* (CPUID 0x80000007 EDX[8]). AMD and modern Intel set
    // this; the bare qemu64 model does not.
    tsc_invariant = tscIsInvariant();

    // Bring up the HPET as a standby clocksource whenever one exists — even on the
    // CPUID-0x15 path where calibration never touched it — so a non-invariant TSC
    // (here) or an unsynchronized one (checkWarpSource, during SMP bring-up) can fall
    // back to a source that is immune to both. hpetHz() maps + enables the counter
    // and is idempotent if calibration already used it.
    if (configuration_hpet_base != 0) {
        if (hpetHz()) |hz| {
            hpet_clock_mask = hpetMask();
            if (hpet_clock_mask == ~@as(u64, 0)) { // only a 64-bit HPET is monotonic enough
                hpet_clock_hz = hz;
                hpet_clock_base = readHpet();
            }
        }
    }

    // Select the source: the fast TSC when invariant, else the HPET if we have one.
    // (checkWarpSource may still demote TSC -> HPET later if the cores' TSCs skew.)
    if (!tsc_invariant and hpet_clock_hz != 0) clock_source = .hpet;
}

/// Run the LAPIC timer one-shot from its maximum count while a monotonic reference
/// clock (frequency `ref_hz`, counter width `ref_mask`) counts out `calib_ms`, and
/// snapshot the TSC across the same window. Yields `ticks_per_ms` and `tsc_hz`.
fn measure(ref_hz: u64, ref_mask: u64, refNow: *const fn () u64) void {
    const calib_ticks = ref_hz / (1000 / calib_ms); // reference ticks in calib_ms

    write(register_timer_divide, timer_divide_16);
    write(register_lvt_timer, lvt_masked);
    write(register_timer_initial, 0xFFFFFFFF);

    const ref0 = refNow();
    const tsc0 = rdtsc();
    while (((refNow() -% ref0) & ref_mask) < calib_ticks) {}
    const tsc1 = rdtsc();

    const elapsed = 0xFFFFFFFF - read(register_timer_current);
    write(register_timer_initial, 0);

    ticks_per_ms = elapsed / calib_ms;
    tsc_hz = (tsc1 -% tsc0) * (1000 / calib_ms);
}

/// The PIT fallback (legacy 8254 channel 2, polled). Only reached when no better
/// reference exists — on a legacy-free machine this path isn't taken.
fn calibratePit() void {
    const pit_hz = 1_193_182;
    const pit_count: u16 = @intCast(pit_hz / 1000 * calib_ms);

    write(register_timer_divide, timer_divide_16);
    write(register_lvt_timer, lvt_masked);
    write(register_timer_initial, 0xFFFFFFFF);

    io.outb(0x61, io.inb(0x61) & 0xFC); // speaker off, gate low
    io.outb(0x43, 0xB0); // channel 2, lo/hi byte, mode 0
    io.outb(0x42, @truncate(pit_count));
    io.outb(0x42, @truncate(pit_count >> 8));

    const tsc_start = rdtsc();
    io.outb(0x61, (io.inb(0x61) & 0xFC) | 0x01); // gate high -> start
    var guard: u64 = 0;
    while (io.inb(0x61) & 0x20 == 0 and guard < 100_000_000) : (guard += 1) {} // bounded
    const tsc_end = rdtsc();

    const elapsed = 0xFFFFFFFF - read(register_timer_current);
    write(register_timer_initial, 0);

    ticks_per_ms = elapsed / calib_ms;
    tsc_hz = (tsc_end -% tsc_start) * (1000 / calib_ms);
}

// --- reference clocks ------------------------------------------------------

/// TSC frequency from CPUID leaf 0x15 (crystal_hz * numerator / denominator), or
/// null if the CPU doesn't enumerate it (common under QEMU, and on AMD).
fn cpuidTscHz() ?u64 {
    if (cpuid(0).eax < 0x15) return null;
    const r = cpuid(0x15);
    if (r.eax == 0 or r.ebx == 0 or r.ecx == 0) return null; // ratio/crystal not given
    return @as(u64, r.ecx) * r.ebx / r.eax;
}

/// Whether the CPU advertises an **invariant** TSC (CPUID leaf 0x80000007, EDX
/// bit 8) — the architectural guarantee, on both Intel and AMD, that the TSC ticks
/// at a constant rate across P/C-states and never stops. Requires the extended-leaf
/// range to reach 0x80000007 first.
fn tscIsInvariant() bool {
    if (cpuid(0x80000000).eax < 0x80000007) return false;
    return (cpuid(0x80000007).edx & (1 << 8)) != 0;
}

const CpuidRegs = struct { eax: u32, ebx: u32, ecx: u32, edx: u32 };

fn cpuid(leaf: u32) CpuidRegs {
    var a: u32 = undefined;
    var b: u32 = undefined;
    var c: u32 = undefined;
    var d: u32 = undefined;
    asm volatile ("cpuid"
        : [a] "={eax}" (a),
          [b] "={ebx}" (b),
          [c] "={ecx}" (c),
          [d] "={edx}" (d),
        : [leaf] "{eax}" (leaf),
          [sub] "{ecx}" (@as(u32, 0)),
    );
    return .{ .eax = a, .ebx = b, .ecx = c, .edx = d };
}

// HPET registers: capabilities at +0x00 (period in the high dword, in fs; bit 13 =
// 64-bit-counter capable), general configuration at +0x10, main counter at +0xF0.
fn hpetRead64(off: usize) u64 {
    return @as(*volatile u64, @ptrFromInt(configuration_hpet_base + off)).*;
}
fn hpetWrite64(off: usize, value: u64) void {
    @as(*volatile u64, @ptrFromInt(configuration_hpet_base + off)).* = value;
}

/// Whether the HPET has been mapped into the physmap yet, so `configuration_hpet_base`
/// already holds the virtual address. `hpetHz` is called more than once (calibration
/// may use the HPET, and the standby-clocksource setup asks for it again), and mapping
/// an already-mapped base a second time would double-offset it into an overflow.
var hpet_mapped: bool = false;

/// Map + enable the HPET and return its tick frequency, or null if unusable.
/// Maps the HPET into the physmap and switches configuration_hpet_base to that virtual
/// address, so the register accessors reach it without the identity map. Idempotent.
fn hpetHz() ?u64 {
    if (!hpet_mapped) {
        configuration_hpet_base = paging.mapMmio(configuration_hpet_base, 0x400, true);
        hpet_mapped = true;
    }
    const caps = hpetRead64(0x00);
    const period_fs = caps >> 32; // femtoseconds per tick
    if (period_fs == 0) return null;
    hpetWrite64(0x10, hpetRead64(0x10) | 1); // ENABLE_CNF: start the main counter
    return 1_000_000_000_000_000 / period_fs; // 1e15 fs/s ÷ fs/tick
}

/// The HPET counter width mask (64- or 32-bit, per caps bit 13).
fn hpetMask() u64 {
    return if (hpetRead64(0x00) & (1 << 13) != 0) ~@as(u64, 0) else 0xFFFF_FFFF;
}

fn readHpet() u64 {
    return hpetRead64(0xF0);
}

fn readPmTimer() u64 {
    const pt = configuration_pm_timer.?;
    // MMIO PM timer via the physmap (mapMmio is idempotent); the common case is
    // a legacy I/O port.
    if (pt.mmio) return @as(*volatile u32, @ptrFromInt(paging.mapMmio(pt.address, 4, false))).*;
    return io.inl(@intCast(pt.address));
}

/// Arm the LAPIC timer to fire on `timer_vector` at `hz` (periodic). Requires
/// calibrate() to have run.
pub fn initTimer(hz: u32) void {
    timer_hz = hz;
    const count = @as(u64, ticks_per_ms) * 1000 / hz; // counts per (1/hz) second
    write(register_timer_divide, timer_divide_16);
    write(register_lvt_timer, timer_vector | lvt_periodic);
    write(register_timer_initial, @intCast(count));
}

/// Configured periodic-interrupt frequency (Hz).
pub fn frequencyHz() u32 {
    return timer_hz;
}

/// Measured LAPIC timer frequency (Hz), for reporting/sanity checks.
pub fn lapicHz() u64 {
    return @as(u64, ticks_per_ms) * 1000;
}

/// Measured TSC frequency (Hz).
pub fn tscHz() u64 {
    return tsc_hz;
}

// Monotonic high-resolution clock. A function per resolution, each scaling the
// counter delta directly at its unit (the 128-bit intermediate avoids overflow
// across a long uptime). nanos() resolves to a few ns on the TSC; millis() is what
// the scheduler uses for sleep deadlines. The source is the TSC when it is invariant
// and synchronized, else the HPET counter (see clock_source) — the branch is one
// global load and the TSC path is unchanged from before.

/// The selected source's counter delta since its zero point.
fn clockCount() u64 {
    return switch (clock_source) {
        .tsc => rdtsc() -% tsc_base,
        // A 64-bit HPET (the only kind we select) never wraps in any realistic
        // uptime, so the wrapping subtraction is exact.
        .hpet => readHpet() -% hpet_clock_base,
    };
}

/// The selected source's frequency (0 if the clock is unavailable/uncalibrated).
fn clockHertz() u64 {
    return switch (clock_source) {
        .tsc => tsc_hz,
        .hpet => hpet_clock_hz,
    };
}

pub fn nanos() u64 {
    const hz = clockHertz();
    if (hz == 0) return 0;
    return @intCast(@as(u128, clockCount()) * 1_000_000_000 / hz);
}

pub fn micros() u64 {
    const hz = clockHertz();
    if (hz == 0) return 0;
    return @intCast(@as(u128, clockCount()) * 1_000_000 / hz);
}

pub fn millis() u64 {
    const hz = clockHertz();
    if (hz == 0) return 0;
    return @intCast(@as(u128, clockCount()) * 1_000 / hz);
}

/// Whether the CPU advertises an invariant TSC (CPUID 0x80000007 EDX[8]).
pub fn tscInvariant() bool {
    return tsc_invariant;
}

/// Test hook: force the TSC clocksource on, as if the CPU had advertised an invariant
/// TSC. QEMU's TCG accelerator (the only one for an x86 guest on an Apple-Silicon
/// host) does not expose the invariant-TSC bit — its emulated TSC isn't invariant — so
/// the tsc-sync test can't reach the real-Intel/AMD/KVM path through CPUID. This lets
/// that test exercise the TSC clocksource and the cross-core warp check anyway. tsc_base
/// is left as-is so the switch from the HPET is continuous.
pub fn forceTscClocksourceForTest() void {
    tsc_invariant = true;
    clock_source = .tsc;
}

/// How many per-AP warp checks actually ran (a rendezvous completed) — lets a test
/// confirm the cross-core check executed rather than being skipped.
pub fn warpChecksRun() u32 {
    return warp_checks;
}

/// Whether the per-core TSCs are synchronized (no backward warp seen at bring-up).
pub fn tscSynced() bool {
    return tsc_synced;
}

/// The active monotonic clocksource, for the boot log and tests.
pub fn clockSourceName() []const u8 {
    return switch (clock_source) {
        .tsc => "tsc",
        .hpet => "hpet",
    };
}

// --- cross-core TSC synchronization ("warp") check -------------------------
// Two cores hammer a shared "max seen" TSC value under a lock; if either reads a
// value below that max, its TSC lags the other's, and time would run backward for a
// task migrating between them (Linux calls this a warp). danos brings APs up one at a
// time, so this runs pairwise: the BSP (source) against each AP (target) as it comes
// online. It only matters — and only runs — while the TSC is the clocksource; on a
// machine already on the HPET (a bare VM) the whole rendezvous is skipped.

var warp_lock: u32 = 0;
var warp_last: u64 = 0;
var warp_bsp_ready: u32 = 0;
var warp_ap_ready: u32 = 0;
var warp_stop: u32 = 0;
var warp_checks: u32 = 0; // completed per-AP rendezvous count (for the tsc-sync test)

// The warp check is bounded by TIME, not iterations: a warp tick is a locked
// read-modify-write on a cacheline two cores are fighting over — microseconds
// under real contention, not the nanosecond an uncontended count assumes (a
// 1<<20-round budget measured 2-21 SECONDS per core on a 16-core machine), and
// a PAUSE costs ~140 cycles on modern Intel, so an iteration-counted await
// mis-measures by two orders of magnitude too. ~5 ms of pairwise hammering per
// core is plenty to catch a lagging TSC (Linux's check_tsc_warp budget), and
// ~100 ms is a generous rendezvous window for a healthy core.
const warp_check_ns: u64 = 5_000_000; // per-AP pairwise check duration
const warp_await_ns: u64 = 100_000_000; // rendezvous wait before giving up

/// TSC ticks for `ns` nanoseconds (valid whenever the warp check runs: the TSC
/// is the clocksource, so tsc_hz is calibrated).
fn warpTicksFor(ns: u64) u64 {
    return @intCast(@as(u128, ns) * tsc_hz / 1_000_000_000);
}

fn warpTick() void {
    while (@cmpxchgWeak(u32, &warp_lock, 0, 1, .acquire, .monotonic) != null) asm volatile ("pause");
    const t = rdtsc();
    if (t < warp_last) {
        const delta = warp_last - t;
        if (delta > tsc_warp_cycles) tsc_warp_cycles = delta;
        tsc_synced = false;
    } else {
        warp_last = t;
    }
    @atomicStore(u32, &warp_lock, 0, .release);
}

/// Spin (time-bounded) until `flag` is nonzero; false on timeout.
fn warpAwait(flag: *u32) bool {
    const deadline = rdtsc() +% warpTicksFor(warp_await_ns);
    while (@atomicLoad(u32, flag, .acquire) == 0) {
        if (rdtsc() -% deadline < (1 << 62)) return false; // past the deadline
        asm volatile ("pause");
    }
    return true;
}

/// BSP side of the pairwise TSC warp check, run once per AP as it reports in. No-op
/// unless the TSC is the active clocksource. If the AP's TSC proves to lag, demote
/// the monotonic clock to the HPET without a discontinuity.
pub fn checkWarpSource() void {
    if (clock_source != .tsc) return;
    warp_last = 0;
    @atomicStore(u32, &warp_stop, 0, .release);
    @atomicStore(u32, &warp_ap_ready, 0, .release);
    @atomicStore(u32, &warp_bsp_ready, 1, .release);
    if (!warpAwait(&warp_ap_ready)) { // AP never joined the rendezvous; skip, don't hang
        @atomicStore(u32, &warp_bsp_ready, 0, .release);
        return;
    }
    const deadline = rdtsc() +% warpTicksFor(warp_check_ns);
    while (rdtsc() -% deadline >= (1 << 62)) warpTick(); // until the time budget is spent
    @atomicStore(u32, &warp_stop, 1, .release);
    @atomicStore(u32, &warp_bsp_ready, 0, .release);
    warp_checks += 1;

    if (!tsc_synced and hpet_clock_hz != 0) demoteToHpet();
}

/// AP side: join the BSP's warp check, then return so the core can enter the
/// scheduler. Bounded so a missing BSP can't strand the core.
pub fn checkWarpTarget() void {
    if (clock_source != .tsc) return;
    if (!warpAwait(&warp_bsp_ready)) return;
    @atomicStore(u32, &warp_ap_ready, 1, .release);
    // The BSP owns the budget; this bound only protects against a lost BSP.
    const deadline = rdtsc() +% warpTicksFor(2 * warp_check_ns + warp_await_ns);
    while (@atomicLoad(u32, &warp_stop, .acquire) == 0) {
        if (rdtsc() -% deadline < (1 << 62)) return; // past the deadline
        warpTick();
    }
}

/// Switch the clocksource from the TSC to the HPET without a discontinuity: choose
/// the HPET zero point so it reads the same nanosecond value the TSC does right now,
/// so time neither jumps nor runs backward across the switch. Called when the warp
/// check proves the per-core TSCs unsynchronized.
fn demoteToHpet() void {
    const now_ns = @as(u128, rdtsc() -% tsc_base) * 1_000_000_000 / tsc_hz;
    const equivalent_ticks: u64 = @intCast(now_ns * hpet_clock_hz / 1_000_000_000);
    hpet_clock_base = readHpet() -% equivalent_ticks;
    clock_source = .hpet;
}

/// Acknowledge the current interrupt so the LAPIC will deliver the next one.
pub fn eoi() void {
    write(register_eoi, 0);
}

/// Optional callback run each tick (the scheduler registers it for preemption).
var on_tick: ?*const fn () void = null;

pub fn setTickHook(hook: *const fn () void) void {
    on_tick = hook;
}

/// The timer interrupt handler: advance the monotonic tick count, then run the
/// tick hook (which may switch tasks). The interrupt is already acknowledged by
/// the dispatcher before we get here, so a task switch here doesn't stall it.
pub fn timerTick() void {
    // Acknowledge before the tick hook: `on_tick` is the scheduler, which may switch
    // tasks and not return promptly, and the LAPIC mustn't wait on it to deliver the
    // next interrupt. (Each device handler now owns its own EOI — see
    // `idt.interruptDispatch` — because a *routed* interrupt must be masked at the
    // I/O APIC before it is acknowledged, an ordering the dispatcher can't impose.)
    eoi();
    tick_count +%= 1;
    if (on_tick) |hook| hook();
}

/// This core's Local APIC id — the interrupt destination for `routeGsi`.
pub fn localId() u8 {
    return @truncate(read(register_id) >> 24);
}

/// Number of timer ticks so far. Volatile load: the count is bumped
/// asynchronously by the interrupt handler, so callers must re-read memory.
pub fn ticks() u64 {
    return @as(*const volatile u64, &tick_count).*;
}