//! Local APIC and its timer — the source of device interrupts. //! //! Modern x86 routes interrupts through the per-CPU Local APIC (the legacy 8259 //! PIC is remapped out of the way and masked). The LAPIC also has a built-in //! timer, which is the simplest device interrupt to bring up: it needs no //! external routing, just a vector and a count. We use it as danos's heartbeat. //! //! The LAPIC is memory-mapped (default physical 0xFEE00000, inside our identity //! map). Every interrupt must be acknowledged with an end-of-interrupt write, or //! the LAPIC won't deliver the next one. const boot_handoff = @import("boot-handoff"); const io = @import("io.zig"); const paging = @import("paging.zig"); /// The ACPI PM timer, as a calibration reference: an I/O port or MMIO counter. pub const PmTimer = struct { mmio: bool, address: u64, is_32bit: bool }; // Platform facts from discovery (set by `configure` before bring-up). Defaults are // the legacy-safe assumptions so the code still works if discovery never ran. var configuration_pic_present: bool = true; var configuration_hpet_base: u64 = 0; // 0 = no HPET discovered var configuration_pm_timer: ?PmTimer = null; /// Which reference the last calibration used, for logging. var cal_source: []const u8 = "none"; /// Hand the LAPIC bring-up the discovered platform facts. Call before `init`. pub fn configure(pic_present: bool, hpet_base: u64, pm_timer: ?PmTimer) void { configuration_pic_present = pic_present; configuration_hpet_base = hpet_base; configuration_pm_timer = pm_timer; } /// The calibration reference the timer was measured against ("cpuid"/"hpet"/…). pub fn calibrationSource() []const u8 { return cal_source; } /// IDT vector the timer fires on (in the device range, >= 32). pub const timer_vector = 32; /// Spurious-interrupt vector. Low nibble 0xF by convention; also in our gate /// range so a stray spurious interrupt lands on a valid (no-op) handler. const spurious_vector = 47; // LAPIC register offsets. const register_spurious = 0x0F0; const register_eoi = 0x0B0; const register_id = 0x020; // this core's LAPIC id, in bits 24-31 const register_icr_low = 0x300; // interrupt command register, low dword (writing it sends) const register_icr_high = 0x310; // ICR high dword (destination APIC id in bits 24-31) const register_lvt_timer = 0x320; const register_timer_initial = 0x380; const register_timer_current = 0x390; const register_timer_divide = 0x3E0; const icr_delivery_pending = 1 << 12; // ICR low bit 12: a previous IPI is still in flight const lvt_masked = 1 << 16; const lvt_periodic = 1 << 17; const timer_divide_16 = 0x3; const ia32_apic_base_msr = 0x1B; /// LAPIC MMIO base. A runtime var (not a constant) both because we read it from /// the MSR and so register writes compile to normal stores rather than a /// `mov moffs`, which the self-hosted backend can't encode. var base: usize = 0xFEE00000; var tick_count: u64 = 0; /// LAPIC timer counts per millisecond, measured against the PIT (see calibrate). /// At divide-by-16, this is the effective counting rate. var ticks_per_ms: u32 = 0; /// The periodic-interrupt frequency the timer is armed at, once initTimer runs. var timer_hz: u32 = 0; /// TSC (Time Stamp Counter) calibration: cycles per second, and the count at boot. /// The TSC is a per-core cycle counter, giving a ~nanosecond high-resolution /// monotonic clock — far finer than the millisecond timer tick. var tsc_hz: u64 = 0; var tsc_base: u64 = 0; /// Read the 64-bit Time Stamp Counter. fn rdtsc() u64 { var low: u32 = undefined; var high: u32 = undefined; asm volatile ("rdtsc" : [low] "={eax}" (low), [high] "={edx}" (high), ); return (@as(u64, high) << 32) | low; } fn read(register: u32) u32 { return @as(*volatile u32, @ptrFromInt(base + register)).*; } fn write(register: u32, value: u32) void { @as(*volatile u32, @ptrFromInt(base + register)).* = value; } /// Move the legacy 8259 PIC's vectors to 0x20-0x2F (clear of the CPU exception /// vectors) and mask every line, so it can't deliver interrupts behind the APIC. fn remapAndMaskPic() void { io.outb(0x20, 0x11); // start init (cascade mode) io.outb(0xA0, 0x11); io.outb(0x21, 0x20); // master offset 0x20 io.outb(0xA1, 0x28); // slave offset 0x28 io.outb(0x21, 0x04); // tell master about slave on IRQ2 io.outb(0xA1, 0x02); io.outb(0x21, 0x01); // 8086 mode io.outb(0xA1, 0x01); io.outb(0x21, 0xFF); // mask all io.outb(0xA1, 0xFF); } /// Enable the Local APIC: mask the PIC (only if one is present — a legacy-free /// UEFI Class 3 machine may have none), set the global-enable MSR bit, and /// software-enable the APIC via its spurious-vector register. pub fn init() void { if (configuration_pic_present) remapAndMaskPic(); const msr = io.rdmsr(ia32_apic_base_msr); // Reach the LAPIC through the physmap (paging.init maps its page there). base = @intCast(boot_handoff.physicalToVirtual(msr & 0xFFFFF000)); // physical base is bits 12+ io.wrmsr(ia32_apic_base_msr, msr | (1 << 11)); // global enable write(register_spurious, 0x100 | spurious_vector); // bit 8 = software enable } /// Software-enable *this* core's Local APIC — the application-processor counterpart /// of `init`, minus the one-time PIC remap (the BSP already masked it) and minus /// calibration (the timer rate is a shared hardware constant, measured once). Each /// core has its own LAPIC at the same MMIO address, so no per-core base is needed. pub fn initSecondary() void { const msr = io.rdmsr(ia32_apic_base_msr); io.wrmsr(ia32_apic_base_msr, msr | (1 << 11)); // global enable write(register_spurious, 0x100 | spurious_vector); // software enable } // --- application-processor wakeup (INIT–SIPI–SIPI) -------------------------- /// Send an INIT IPI to the core with Local APIC id `apic_id` — the first step of /// the wake sequence. Blocks until the LAPIC reports the IPI was delivered. pub fn sendInit(apic_id: u32) void { write(register_icr_high, apic_id << 24); write(register_icr_low, 0x4500); // INIT, physical destination, assert, edge-triggered waitIcrIdle(); } /// Send a STARTUP IPI (SIPI) telling the target core to begin executing at physical /// address `vector << 12` (in real mode). Per the Intel bring-up protocol this is /// sent twice after the INIT; both calls block until delivery completes. pub fn sendStartup(apic_id: u32, vector: u8) void { write(register_icr_high, apic_id << 24); write(register_icr_low, 0x4600 | @as(u32, vector)); // STARTUP with the page vector waitIcrIdle(); } fn waitIcrIdle() void { while (read(register_icr_low) & icr_delivery_pending != 0) {} } /// The calibration window: we time everything against a 10 ms reference interval. const calib_ms = 10; /// Measure the LAPIC timer's and the TSC's rates. The PIT (legacy 8254) can be /// absent on UEFI Class 3 firmware — and polling it would hang — so we pick a /// reference clock in order of preference: the CPU's own TSC frequency (CPUID leaf /// 0x15, no external timer needed), then the discovered HPET, then the ACPI PM /// timer, and only the PIT as a last resort. Each path yields the same two rates. pub fn calibrate() void { var done = false; // 1. CPUID leaf 0x15 gives the TSC frequency directly — measure the LAPIC // against the TSC itself, needing no external timer at all. if (cpuidTscHz()) |hz| { measure(hz, ~@as(u64, 0), rdtsc); tsc_hz = hz; // keep the exact enumerated value cal_source = "cpuid"; done = true; } // 2. The discovered HPET. if (!done and configuration_hpet_base != 0) { if (hpetHz()) |hpet_hz| { measure(hpet_hz, hpetMask(), readHpet); cal_source = "hpet"; done = true; } } // 3. The ACPI PM timer (fixed 3.579545 MHz). if (!done) { if (configuration_pm_timer) |pt| { measure(3_579_545, if (pt.is_32bit) 0xFFFF_FFFF else 0xFF_FFFF, readPmTimer); cal_source = "pm-timer"; done = true; } } // 4. The legacy PIT, last resort. if (!done) { calibratePit(); cal_source = "pit"; } // A bad measurement (no reference actually ticked) leaves nonsense; fall back. if (ticks_per_ms == 0 or tsc_hz == 0) { calibratePit(); cal_source = "pit"; } tsc_base = rdtsc(); // the clock's zero point (boot) } /// Run the LAPIC timer one-shot from its maximum count while a monotonic reference /// clock (frequency `ref_hz`, counter width `ref_mask`) counts out `calib_ms`, and /// snapshot the TSC across the same window. Yields `ticks_per_ms` and `tsc_hz`. fn measure(ref_hz: u64, ref_mask: u64, refNow: *const fn () u64) void { const calib_ticks = ref_hz / (1000 / calib_ms); // reference ticks in calib_ms write(register_timer_divide, timer_divide_16); write(register_lvt_timer, lvt_masked); write(register_timer_initial, 0xFFFFFFFF); const ref0 = refNow(); const tsc0 = rdtsc(); while (((refNow() -% ref0) & ref_mask) < calib_ticks) {} const tsc1 = rdtsc(); const elapsed = 0xFFFFFFFF - read(register_timer_current); write(register_timer_initial, 0); ticks_per_ms = elapsed / calib_ms; tsc_hz = (tsc1 -% tsc0) * (1000 / calib_ms); } /// The PIT fallback (legacy 8254 channel 2, polled). Only reached when no better /// reference exists — on a legacy-free machine this path isn't taken. fn calibratePit() void { const pit_hz = 1_193_182; const pit_count: u16 = @intCast(pit_hz / 1000 * calib_ms); write(register_timer_divide, timer_divide_16); write(register_lvt_timer, lvt_masked); write(register_timer_initial, 0xFFFFFFFF); io.outb(0x61, io.inb(0x61) & 0xFC); // speaker off, gate low io.outb(0x43, 0xB0); // channel 2, lo/hi byte, mode 0 io.outb(0x42, @truncate(pit_count)); io.outb(0x42, @truncate(pit_count >> 8)); const tsc_start = rdtsc(); io.outb(0x61, (io.inb(0x61) & 0xFC) | 0x01); // gate high -> start var guard: u64 = 0; while (io.inb(0x61) & 0x20 == 0 and guard < 100_000_000) : (guard += 1) {} // bounded const tsc_end = rdtsc(); const elapsed = 0xFFFFFFFF - read(register_timer_current); write(register_timer_initial, 0); ticks_per_ms = elapsed / calib_ms; tsc_hz = (tsc_end -% tsc_start) * (1000 / calib_ms); } // --- reference clocks ------------------------------------------------------ /// TSC frequency from CPUID leaf 0x15 (crystal_hz * numerator / denominator), or /// null if the CPU doesn't enumerate it (common under QEMU). fn cpuidTscHz() ?u64 { if (cpuid(0).eax < 0x15) return null; const r = cpuid(0x15); if (r.eax == 0 or r.ebx == 0 or r.ecx == 0) return null; // ratio/crystal not given return @as(u64, r.ecx) * r.ebx / r.eax; } const CpuidRegs = struct { eax: u32, ebx: u32, ecx: u32, edx: u32 }; fn cpuid(leaf: u32) CpuidRegs { var a: u32 = undefined; var b: u32 = undefined; var c: u32 = undefined; var d: u32 = undefined; asm volatile ("cpuid" : [a] "={eax}" (a), [b] "={ebx}" (b), [c] "={ecx}" (c), [d] "={edx}" (d), : [leaf] "{eax}" (leaf), [sub] "{ecx}" (@as(u32, 0)), ); return .{ .eax = a, .ebx = b, .ecx = c, .edx = d }; } // HPET registers: capabilities at +0x00 (period in the high dword, in fs; bit 13 = // 64-bit-counter capable), general configuration at +0x10, main counter at +0xF0. fn hpetRead64(off: usize) u64 { return @as(*volatile u64, @ptrFromInt(configuration_hpet_base + off)).*; } fn hpetWrite64(off: usize, value: u64) void { @as(*volatile u64, @ptrFromInt(configuration_hpet_base + off)).* = value; } /// Map + enable the HPET and return its tick frequency, or null if unusable. /// Maps the HPET into the physmap and switches configuration_hpet_base to that virtual /// address, so the register accessors reach it without the identity map. fn hpetHz() ?u64 { configuration_hpet_base = paging.mapMmio(configuration_hpet_base, 0x400, true); const caps = hpetRead64(0x00); const period_fs = caps >> 32; // femtoseconds per tick if (period_fs == 0) return null; hpetWrite64(0x10, hpetRead64(0x10) | 1); // ENABLE_CNF: start the main counter return 1_000_000_000_000_000 / period_fs; // 1e15 fs/s ÷ fs/tick } /// The HPET counter width mask (64- or 32-bit, per caps bit 13). fn hpetMask() u64 { return if (hpetRead64(0x00) & (1 << 13) != 0) ~@as(u64, 0) else 0xFFFF_FFFF; } fn readHpet() u64 { return hpetRead64(0xF0); } fn readPmTimer() u64 { const pt = configuration_pm_timer.?; // MMIO PM timer via the physmap (mapMmio is idempotent); the common case is // a legacy I/O port. if (pt.mmio) return @as(*volatile u32, @ptrFromInt(paging.mapMmio(pt.address, 4, false))).*; return io.inl(@intCast(pt.address)); } /// Arm the LAPIC timer to fire on `timer_vector` at `hz` (periodic). Requires /// calibrate() to have run. pub fn initTimer(hz: u32) void { timer_hz = hz; const count = @as(u64, ticks_per_ms) * 1000 / hz; // counts per (1/hz) second write(register_timer_divide, timer_divide_16); write(register_lvt_timer, timer_vector | lvt_periodic); write(register_timer_initial, @intCast(count)); } /// Configured periodic-interrupt frequency (Hz). pub fn frequencyHz() u32 { return timer_hz; } /// Measured LAPIC timer frequency (Hz), for reporting/sanity checks. pub fn lapicHz() u64 { return @as(u64, ticks_per_ms) * 1000; } /// Measured TSC frequency (Hz). pub fn tscHz() u64 { return tsc_hz; } // Monotonic high-resolution clock, from the TSC. A function per resolution, each // scaling the cycle delta directly at its unit (the 128-bit intermediate avoids // overflow across a long uptime). nanos() resolves to a few ns; millis() is what // the scheduler uses for sleep deadlines. pub fn nanos() u64 { if (tsc_hz == 0) return 0; return @intCast(@as(u128, rdtsc() -% tsc_base) * 1_000_000_000 / tsc_hz); } pub fn micros() u64 { if (tsc_hz == 0) return 0; return @intCast(@as(u128, rdtsc() -% tsc_base) * 1_000_000 / tsc_hz); } pub fn millis() u64 { if (tsc_hz == 0) return 0; return @intCast(@as(u128, rdtsc() -% tsc_base) * 1_000 / tsc_hz); } /// Acknowledge the current interrupt so the LAPIC will deliver the next one. pub fn eoi() void { write(register_eoi, 0); } /// Optional callback run each tick (the scheduler registers it for preemption). var on_tick: ?*const fn () void = null; pub fn setTickHook(hook: *const fn () void) void { on_tick = hook; } /// The timer interrupt handler: advance the monotonic tick count, then run the /// tick hook (which may switch tasks). The interrupt is already acknowledged by /// the dispatcher before we get here, so a task switch here doesn't stall it. pub fn timerTick() void { // Acknowledge before the tick hook: `on_tick` is the scheduler, which may switch // tasks and not return promptly, and the LAPIC mustn't wait on it to deliver the // next interrupt. (Each device handler now owns its own EOI — see // `idt.interruptDispatch` — because a *routed* interrupt must be masked at the // I/O APIC before it is acknowledged, an ordering the dispatcher can't impose.) eoi(); tick_count +%= 1; if (on_tick) |hook| hook(); } /// This core's Local APIC id — the interrupt destination for `routeGsi`. pub fn localId() u8 { return @truncate(read(register_id) >> 24); } /// Number of timer ticks so far. Volatile load: the count is bumped /// asynchronously by the interrupt handler, so callers must re-read memory. pub fn ticks() u64 { return @as(*const volatile u64, &tick_count).*; }