//! IRQ-as-IPC: delivering a hardware interrupt to a user-space driver. //! //! A microkernel can't run driver code in the ISR — the driver is a ring-3 process //! in another address space. So the kernel's ISR does the least it can: quiet the //! line, acknowledge the CPU, and post an asynchronous notification to the endpoint //! the driver is blocked on (`ipc_sync.notifyFromIsr`). The driver wakes out of //! `IPC_ReplyWait`, services the device, and calls `irq_ack` to re-arm. //! //! The full cycle, and why each step is where it is: //! //! ISR irqMask(gsi) -- the line is still asserted; stop it reaching a CPU //! irqEoi() -- now safe to tell the LAPIC we're done //! notifyFromIsr() -- wake the driver (it runs much later) //! driver -- reads/clears the device's status register //! driver irq_ack(device,resource) -- irqUnmask(gsi): the line is quiet, let it through //! //! Mask-before-EOI is the load-bearing part. A level-triggered line stays asserted //! until the *device* is quieted, which only the ring-3 driver can do. EOI with the //! entry unmasked and the I/O APIC redelivers immediately, forever, before the //! driver is ever scheduled. Masking converts "level" into something a deferred //! handler can cope with; `irq_ack` is what closes the loop. //! //! Binding is capability-gated exactly like `mmio_map`: the caller must have //! `device_claim`ed the device, and the GSI must come from one of that device's `irq` //! resources in the discovered device table (system/kernel/devices-broker.zig). A driver can //! therefore never bind an interrupt it doesn't own — a raw-GSI system_call would let //! any process steal the keyboard's line. //! //! KNOWN ISSUE (real hardware, not QEMU). Masking a level-triggered redirection entry //! while its remote-IRR bit is set does not clear remote-IRR on some chipsets, and the //! line then never fires again — a driver would take exactly one interrupt and block //! forever. QEMU's I/O APIC clears remote-IRR on EOI regardless of the mask, so the //! `hpet` test cannot see this. Linux's workaround is to flush remote-IRR by briefly //! flipping the entry to edge-trigger and back. Revisit when danos first boots on //! metal; MSI (no mask cycle at all) sidesteps it entirely. const architecture = @import("architecture"); const sync = @import("sync.zig"); const ipc_sync = @import("ipc-synchronous.zig"); /// GSIs a single I/O APIC covers. 24 is the standard redirection-table size; a /// second I/O APIC (none on QEMU's q35) would extend this. pub const maximum_gsi = 24; /// The endpoint to notify for each bound GSI, or null if unbound. Read from the ISR /// and written from syscalls, always under the big kernel lock. var bound: [maximum_gsi]?*ipc_sync.Endpoint = .{null} ** maximum_gsi; /// Task that owns each binding. Teardown is keyed on *this*, not on the endpoint /// pointer: an endpoint can be shared between processes (ipc_register/ipc_lookup hand /// out extra references), so "every GSI pointing at this endpoint" is not the same /// set as "every GSI this process bound", and releasing the former on exit would mask /// a live sibling's device line. var bound_owner: [maximum_gsi]u32 = .{0} ** maximum_gsi; /// No GSI assigned to this vector. const no_gsi: u32 = 0xFFFF_FFFF; /// Reverse map for the ISR: which GSI does this vector carry? Populated at bind. /// `interruptDispatch` hands a handler no arguments, so the vector→GSI edge has to /// be recovered from somewhere — the trampolines below capture the vector at /// comptime, and this turns it back into a GSI. Trampolines are installed on every /// vector in the window at boot, so an unbound one must be distinguishable from /// GSI 0 — hence the sentinel rather than a zero default. var vector_gsi: [256]u32 = .{no_gsi} ** 256; /// Vector currently assigned to each GSI (0 = none), so a rebind reuses it. var gsi_vector: [maximum_gsi]u8 = .{0} ** maximum_gsi; /// Set once the trampolines are installed. var installed = false; /// The ISR body for a bound device line. Runs with interrupts off, on the /// interrupted task's kernel stack, on whichever core the I/O APIC picked. fn dispatch(vector: u8) void { const gsi = vector_gsi[vector]; if (gsi == no_gsi) { // Nothing is routed here. Acknowledge so the LAPIC doesn't wedge on an // in-service bit that never clears, but touch no redirection entry. architecture.irqEoi(); return; } // One lock region for the whole cycle. Two reasons, and the second is subtle: // // - The I/O APIC is an index/data register pair, so two cores interleaving a // read-modify-write of a redirection entry would corrupt it. // - `bound[gsi]` must be *read and used* under the same acquisition that // `unbind` writes it under. Dropping the lock between the load and // `notifyLocked` would let a driver exiting on another core free the endpoint // in the gap, and we would post a notification into freed memory. The GSI is // routed to the core that bound it, but a driver may migrate and exit // elsewhere, so this is reachable on SMP. // // No deadlock: the lock is non-recursive, but a core holding it runs with // interrupts disabled and so cannot interrupt itself into here. _ = sync.enter(); defer sync.leaveIsr(); architecture.irqMask(gsi); // the line is still asserted; stop it reaching a CPU architecture.irqEoi(); // now safe to release the LAPIC's in-service bit // Wakes the driver if it's blocked in ReplyWait; otherwise queues the badge on // the endpoint's notify ring, so an interrupt taken while the driver is off // doing something else is not lost. if (bound[gsi]) |endpoint| ipc_sync.notifyLocked(endpoint, gsi); } /// Install one no-argument trampoline per usable vector. Each closes over its own /// `vector` as a comptime constant — that's the trick that gets an argument into /// `idt.Handler` (`*const fn () void`) without a per-vector hand-written stub. pub fn init() void { if (installed) return; inline for (0..architecture.irq_vector_count) |i| { const vector: u8 = @intCast(@as(usize, architecture.irq_vector_base) + i); architecture.irqSetHandler(vector, &struct { fn trampoline() void { dispatch(vector); } }.trampoline); } installed = true; } /// Lowest unused vector in the device window, or null if they're all spoken for. fn allocVector() ?u8 { var v: u8 = architecture.irq_vector_base; while (v < architecture.irq_vector_base + architecture.irq_vector_count) : (v += 1) { var used = false; for (gsi_vector) |gv| { if (gv == v) used = true; } if (!used) return v; } return null; } pub const BindError = error{ BadGsi, InUse, NoVector }; /// Deliver `gsi` to `endpoint` as an IPC notification, on behalf of task `owner`. Routes the /// line to this core, installs the binding, and unmasks. Caller must hold the big /// kernel lock, and must already have checked that `owner` claimed the device this GSI /// belongs to. pub fn bind(gsi: u32, endpoint: *ipc_sync.Endpoint, owner: u32) BindError!void { if (gsi >= maximum_gsi or !architecture.irqOwnsGsi(gsi)) return error.BadGsi; if (bound[gsi] != null) return error.InUse; const vector = allocVector() orelse return error.NoVector; vector_gsi[vector] = gsi; gsi_vector[gsi] = vector; bound[gsi] = endpoint; bound_owner[gsi] = owner; // Level-triggered, active-high. Level is the general case a driver must survive // (and what hpet configures its comparator for); an edge source simply never // leaves the line asserted, so the mask/ack cycle is harmless there. // // Hardcoded for now: a device whose MADT interrupt-source override declares the // line active-*low* (most legacy PCI INTx) will need the polarity threaded // through from discovery. Nothing danos binds today is such a device. architecture.irqRoute(gsi, vector, true, false); architecture.irqUnmask(gsi); } /// Re-arm `gsi` after the driver has quieted the device. Caller holds the big lock /// and has verified ownership. pub fn ack(gsi: u32) bool { if (gsi >= maximum_gsi or bound[gsi] == null) return false; architecture.irqUnmask(gsi); return true; } /// Drop every binding made by task `owner` — called as that task exits, *before* its /// endpoints are freed. Each line is left masked, so a dead driver's device goes quiet /// rather than interrupting into a freed endpoint. Caller holds the big kernel lock, /// which is what makes this safe against a concurrent `dispatch` on another core. /// /// Keyed on the owner, not the endpoint: endpoints are shared (a registered service's /// endpoint has references in several processes), so releasing "everything pointing at /// this endpoint" would tear down bindings this task never made. pub fn releaseOwner(owner: u32) void { for (&bound, 0..) |*slot, gsi| { if (slot.* != null and bound_owner[gsi] == owner) { architecture.irqMask(@intCast(gsi)); slot.* = null; bound_owner[gsi] = 0; gsi_vector[gsi] = 0; } } }