//! User-space processes: loading a user ELF and running it in ring 3. danos is a
//! microkernel, so this only ever loads *user* binaries — there is no kernel-space
//! loader; in-kernel code is linked into the kernel image, not loaded here.
//!
//! Two entry points:
//! - `spawnProcess` loads a user ELF (`/system/services/init`, and later servers/drivers)
//!   into a fresh address space and schedules it as a real preemptive ring-3
//!   process on its own page tables. This is the production path.
//! - `run` executes a raw code blob (the user-pf isolation test program) on the
//!   *current* kernel context via the borrowed-thread path — a minimal probe of
//!   the ring-transition mechanisms, kept for that test.
//! Both map frames user-accessible with W^X (code RO+X, data RW+NX); the program
//! talks to the kernel only through the system_call instruction (or the int 0x80
//! gate). The shared handler is installed once by `init`.
//!
//! Borrowed-path caveat (`run` only): it publishes TSS.rsp0 on the *current*
//! core and uses a single global unwind slot (`user_saved_rsp` in isr.s), so the
//! caller must disable preemption and only one core may be inside it at a time.
//! Real processes (`spawnProcess`) have none of these limits — the scheduler
//! maintains rsp0/CR3 per switch.

const std = @import("std");
const elf = std.elf;
const boot_handoff = @import("boot-handoff");
const abi = @import("abi");
const device_abi = @import("device-abi");
const parameters = @import("parameters");
const architecture = @import("architecture");
const pmm = @import("pmm.zig");
const scheduler = @import("scheduler.zig");
const console = @import("console.zig");
const sync = @import("sync.zig");
const ipc = @import("ipc-synchronous.zig");
const user_memory = @import("user-memory.zig");
const devices_broker = @import("devices-broker.zig");
const irq = @import("irq.zig");
const iommu = @import("iommu.zig");
const initial_ramdisk = @import("initial-ramdisk");
const vfs = @import("vfs.zig");
const log = @import("log.zig");
const wall_clock = @import("wall-clock.zig");

const page_size = abi.page_size;
const SystemCall = abi.SystemCall;

/// User virtual addresses. PML4 index 224 — a user-exclusive region, far from
/// the identity map (low indices) and the vmm test address (index 128), so
/// setting the U/S bit on its intermediate tables widens no kernel mapping.
/// An ELF image may occupy [code_virtual, stack_virtual); the stack sits above.
pub const code_virtual: u64 = 0x0000_7000_0000_0000;

/// The stack region, above the image. The page at `stack_virtual` is **never
/// mapped** — it is the guard page: a process that overflows its stack walks into
/// it and faults (killing only that process) rather than silently corrupting the
/// top of its own image. The stack proper is `parameters.user_stack_pages` pages
/// at [stack_base_virtual, stack_top_virtual), RW + NX, with the System V entry
/// block (argc/argv) at the very top.
pub const stack_virtual: u64 = 0x0000_7000_0020_0000; // guard page (unmapped)
pub const stack_base_virtual: u64 = stack_virtual + page_size;
pub const stack_top_virtual: u64 = stack_base_virtual + parameters.user_stack_pages * page_size;

/// The mmap grant arena: where `mmap` hands out fresh user pages, above the image
/// and stack but still inside PML4[224] (so no kernel mapping is widened). Each
/// process bump-allocates from `heap_arena_base` upward via a per-address-space cursor
/// (`scheduler.addressSpaceMmapNextPtr`, shared by its threads); a 1 GiB window is far more
/// than any user heap needs today.
pub const heap_arena_base: u64 = 0x0000_7000_1000_0000;
pub const heap_arena_end: u64 = heap_arena_base + (1 << 30);

/// End of the user (low) canonical half. Any legitimate user pointer is below it;
/// used to bound the addresses a system_call will dereference on the caller's behalf.
pub const user_half_end: u64 = 0x0000_8000_0000_0000;

/// The MMIO-grant arena: where `mmio_map` places device windows, in PML4[226] —
/// a user-exclusive region distinct from code/stack/heap (PML4[224]), so mapping
/// device pages user-accessible widens no kernel mapping. Per-address-space cursor
/// (`scheduler.addressSpaceDeviceMapNextPtr`).
pub const device_arena_base: u64 = 0x0000_7100_0000_0000;
pub const device_arena_end: u64 = device_arena_base + (4 << 30);

/// The DMA arena: where `dma_alloc` places coherent DMA buffers, in PML4[228] — a
/// user-exclusive region distinct from the MMIO arena. Unlike MMIO grants these back
/// real RAM (contiguous frames), so they are reclaimed on teardown. Per-SPACE cursor
/// on the AddressSpaceRef (sibling threads share the arena).
pub const dma_arena_base: u64 = 0x0000_7200_0000_0000;
pub const dma_arena_end: u64 = dma_arena_base + (256 << 20); // 256 MiB per process

/// The shared-memory arena: where `shared_memory_create`/`shared_memory_map` place shared cacheable regions, in
/// PML4[230] — a user-exclusive region distinct from the DMA arena. The frames are owned by
/// a refcounted shared-memory object and freed when its last capability drops, not on teardown, so the
/// mapping carries `device_grant`. Per-SPACE cursor on the AddressSpaceRef (docs/display-v2.md).
pub const shared_memory_arena_base: u64 = 0x0000_7300_0000_0000;
pub const shared_memory_arena_end: u64 = shared_memory_arena_base + (256 << 20); // 256 MiB per process

/// Largest single `shared_memory_create`, in pages (32 MiB) — enough for a 4K framebuffer surface;
/// also an overflow guard on the page count. shared-memory frames are contiguous (like DMA), so this
/// bounds the contiguous allocation asked of the frame allocator.
const maximum_shared_memory_pages = 8192;

/// Largest single `mmap` grant, in pages (32 MiB). Big enough for a display service's
/// back buffer at up to 4K (3840x2160x4 ≈ 8100 pages); the user heap otherwise grows in
/// small chunks. `systemMmap` maps page by page with rollback, so this is only a sanity
/// bound (and an overflow guard on the page count), not the size of any scratch array.
const maximum_mmap_pages = 8192;

/// Ceiling on a process's argv entries, including argv[0]. Arguments are spawn
/// parameters ("you are the driver for device 12"), not bulk data — IPC carries
/// that — so the bound is small and everything fits the single stack page.
pub const maximum_arguments = 8;

/// Ceiling on the `system_spawn` extra-arguments blob (argv[1..], NUL-separated).
pub const maximum_argument_bytes = 256;

/// Longest path `fs_resolve` accepts, longest prefix `fs_mount`/`fs_unmount`
/// accept, and longest backend rewrite prefix. Each is also the size of the
/// kernel staging buffer the argument is copied into, which is why they are
/// named here rather than spelled as literals at the check.
pub const maximum_resolve_path = 224;
pub const maximum_mount_prefix = 64;
pub const maximum_mount_rewrite = 32;

/// Auxiliary-vector entry types (System V AMD64 process entry). Only what the
/// kernel emits today; a C runtime scans the vector until the null terminator.
const auxiliary_vector_null: u64 = 0; // AT_NULL — end of the vector
const auxiliary_vector_page_size: u64 = 6; // AT_PAGESZ

// The hand-assembled user program blob (isr.s, .rodata) — the isolation probe.
const pf_start = @extern([*]const u8, .{ .name = "user_pf_start" });
const pf_end = @extern([*]const u8, .{ .name = "user_pf_end" });

/// The isolation-proof program: reads a kernel-only page, must #PF.
pub fn pfBlob() []const u8 {
    return pf_start[0 .. @intFromPtr(pf_end) - @intFromPtr(pf_start)];
}

/// What debug_write syscalls produced (accumulated), and the exit system_call's code.
pub var write_buffer: [256]u8 = undefined;
pub var write_len: usize = 0;
pub var write_from_user: bool = false;
pub var write_count: u64 = 0; // total write syscalls served (for the heartbeat tests)
pub var exit_code: u64 = 0;

/// The initial-ramdisk image, recorded at boot so `system_spawn` can find bundled
/// binaries by name. Null until `setInitialRamdisk` runs; `system_spawn` then fails
/// cleanly rather than reaching into unset memory.
var ramdisk_image: ?[]const u8 = null;

/// Record the initial-ramdisk image (the kernel already holds it from the boot
/// handoff) so a user-space supervisor can `system_spawn` binaries out of it.
pub fn setInitialRamdisk(image: []const u8) void {
    ramdisk_image = image;
    vfs.setInitialRamdisk(image); // the kernel VFS serves the same bytes at /system
}

/// Spawn a bundled binary from the kernel by path. Used exactly once, to start
/// /system/services/init (PID 1) — every other spawn goes through the
/// `system_spawn` syscall.
pub fn spawnBundled(name: []const u8) !void {
    const image = ramdisk_image orelse return error.NoInitialRamdisk;
    const rd = initial_ramdisk.Reader.init(image) orelse return error.BadInitialRamdisk;
    const item = rd.find(name) orelse return error.NotBundled;
    try spawnProcess(item.blob, 4, &.{item.name});
}

/// The system_call surface, dispatched on the saved system_call number (`abi.SystemCall`).
/// This is the microkernel-minimal set — memory + scheduling only; file/device
/// I/O will arrive as IPC to user-space servers (docs/syscall.md). The result is
/// written back into the trap frame, since the entry paths restore user registers
/// from it. One handler serves both the system_call/sysret and int-0x80 entry paths.
///
/// Install it once at boot (before any user code runs) via `init`. Also registers
/// the scheduler's kill hooks: the scheduler sits below this layer, so finishing a
/// deferred process_kill (IRQ bindings, IPC handles, the exit notification) is
/// called back up into here from the tick (see scheduler.reapKillPendingLocked).
pub fn init() void {
    architecture.setSystemCallHandler(system_call);
    scheduler.terminate_current_hook = terminateCurrentLocked;
    scheduler.reap_task_hook = reapTaskLocked;
    scheduler.timer_tick_hook = timerSweepLocked;
    scheduler.group_exit_hook = groupExitLocked;
    scheduler.space_mapping_release_hook = dropSpaceMappingHook;
}

/// Return -1 (as an unsigned bit pattern) in the system_call result register.
fn fail(state: *architecture.CpuState) void {
    architecture.setSystemCallResult(state, @bitCast(@as(i64, -1)));
}

fn system_call(state: *architecture.CpuState) void {
    const t = scheduler.current();
    const user = t.address_space != 0;
    if (user) {
        // A condemned process (a kill caught it running) dies at its next
        // kernel entry — before it can spawn, claim, or message anything else.
        if (t.kill_pending.load(.monotonic)) terminateCurrent();
        // Mark the span of this call so the timer tick never tears the task down
        // in the middle of a kernel operation (scheduler.reapKillPendingLocked).
        t.in_system_call = true;
    }
    defer if (user) {
        t.in_system_call = false;
    };
    switch (@as(SystemCall, @enumFromInt(architecture.systemCallNumber(state)))) {
        .exit => {
            exit_code = architecture.systemCallArg(state, 0);
            // A scheduled process tears down fully — the WHOLE process: exit from
            // any thread is group death, the exit_group lesson (docs/shared-fate-
            // plan.md). A borrowed test thread unwinds back to the kernel that
            // entered it. A NONZERO code is a deliberate failure exit
            // (`.aborted`): "the work exists but I could not do it" — supervisors
            // restart those, unlike a clean `.exited` ("nothing for me here"),
            // which they let lie.
            if (scheduler.currentIsUserProcess()) {
                // The reason derives from this call's own argument (a local), not
                // the shared exit_code global — two racing exits must not decide
                // each other's group reason.
                exitGroupCurrent(if (architecture.systemCallArg(state, 0) == 0) .exited else .aborted);
            } else architecture.userExit();
        },
        .yield => {
            scheduler.yield();
            architecture.setSystemCallResult(state, 0);
        },
        .sleep => {
            scheduler.sleep(architecture.systemCallArg(state, 0));
            architecture.setSystemCallResult(state, 0);
        },
        .debug_write => systemDebugWrite(state),
        .mmap => systemMmap(state),
        .munmap => systemMunmap(state),
        .create_ipc_endpoint => systemCreateIpcEndpoint(state),
        .ipc_call => systemIpcCall(state),
        .ipc_reply_wait => systemIpcReplyWait(state),
        .ipc_send => systemIpcSend(state),
        .device_enumerate => systemDeviceEnumerate(state),
        .device_claim => systemDeviceClaim(state),
        .mmio_map => systemMmioMap(state),
        .irq_bind => systemIrqBind(state),
        .irq_ack => systemIrqAck(state),
        .device_register => systemDeviceRegister(state),
        .system_spawn => systemSpawn(state),
        .dma_alloc => systemDmaAlloc(state),
        .dma_free => systemDmaFree(state),
        .msi_bind => systemMsiBind(state),
        .io_read => systemIoRead(state),
        .io_write => systemIoWrite(state),
        .clock => systemClock(state),
        .process_enumerate => systemProcessEnumerate(state),
        .process_kill => systemProcessKill(state),
        .process_exit_reason => systemProcessExitReason(state),
        .process_subscribe => systemProcessSubscribe(state),
        .signal_bind => systemSignalBind(state),
        .process_signal => systemProcessSignal(state),
        .timer_bind => systemTimerBind(state),
        .klog_read => systemKlogRead(state),
        .klog_status => systemKlogStatus(state),
        .fs_resolve => systemFsResolve(state),
        .fs_node => systemFsNode(state),
        .fs_mount => systemFsMount(state),
        .fs_unmount => systemFsUnmount(state),
        .iommu_fault_drain => systemIommuFaultDrain(state),
        .dma_bind => systemDmaBind(state),
        .dma_unbind => systemDmaUnbind(state),
        .handle_close => systemHandleClose(state),
        .wall_clock => systemWallClock(state),
        .shared_memory_create => systemSharedMemoryCreate(state),
        .shared_memory_map => systemSharedMemoryMap(state),
        .shared_memory_physical => systemSharedMemoryPhysical(state),
        .thread_spawn => systemThreadSpawn(state),
        .current_core => systemCurrentCore(state),
        .thread_self => systemThreadSelf(state),
        .thread_join => systemThreadJoin(state),
        .set_thread_pointer => systemSetThreadPointer(state),
        .futex_wait => systemFutexWait(state),
        .futex_wake => systemFutexWake(state),
        .thread_exit => {
            // A WORKER thread ends like a process exit(0), but only this task:
            // its resources are released and its address-space reference dropped
            // (the space survives while sibling threads hold it). The LEADER may
            // not thread_exit — the group ends only through exit, a fault, or
            // process_kill (docs/shared-fate-plan.md, decided at sign-off).
            if (scheduler.currentIsUserProcess()) {
                const dying = scheduler.current();
                if (dying.id == dying.leader) return failErr(state, ipc.EPERM);
                _ = sync.enter(); // handed off through the exit switch
                // A group kill may have stamped us .killed while we raced to
                // this dispatch (past the entry check, spinning on the lock) —
                // keep that stamp; the fan-out's story wins.
                if (!dying.kill_pending.load(.monotonic)) dying.exit_reason = .exited;
                terminateCurrentLocked();
            } else architecture.userExit();
        },
        _ => fail(state),
    }
}

/// Return `-errno` in the system_call result register.
fn failErr(state: *architecture.CpuState, errno: i64) void {
    architecture.setSystemCallResult(state, @bitCast(-errno));
}

/// create_ipc_endpoint() -> handle: allocate an endpoint and install it in the
/// caller's handle table.
fn systemCreateIpcEndpoint(state: *architecture.CpuState) void {
    // Under the big kernel lock: this allocates from the kernel heap and mutates the
    // caller's handle table. A multi-threaded process (e.g. the display's compositor +
    // mouse-listener threads) can drive this concurrently from two cores, so the endpoint
    // allocation and every other lock holder must serialize (heap.zig: "a lock comes with
    // threads/SMP").
    const flags = sync.enter();
    defer sync.leave(flags);
    const endpoint = ipc.createIpcEndpoint() orelse return failErr(state, ipc.ENOMEM);
    const h = ipc.installHandle(scheduler.current(), endpoint);
    if (h < 0) {
        ipc.dropRef(endpoint);
        return failErr(state, ipc.ENOSPC);
    }
    architecture.setSystemCallResult(state, @intCast(h));
}

/// ipc_call(handle, message_ptr, message_len, reply_ptr, reply_cap) -> reply_len.
/// Blocks until the server replies; the trap frame lives on this task's kernel
/// stack, so it survives the block and receives the result on resume.
fn systemIpcCall(state: *architecture.CpuState) void {
    const endpoint = ipc.resolveHandle(scheduler.current(), architecture.systemCallArg(state, 0)) orelse return failErr(state, ipc.EBADF);
    var received_cap: u64 = abi.no_cap;
    const r = ipc.call(endpoint, architecture.systemCallArg(state, 1), architecture.systemCallArg(state, 2), architecture.systemCallArg(state, 3), architecture.systemCallArg(state, 4), architecture.systemCallArg(state, 5), &received_cap);
    architecture.setSystemCallResult(state, @bitCast(r));
    architecture.setSystemCallResult3(state, received_cap);
}

/// ipc_reply_wait(handle, reply_ptr, reply_len, receive_ptr, receive_cap) -> receive_len,
/// with the sender's badge in the secondary result register (rdx).
fn systemIpcReplyWait(state: *architecture.CpuState) void {
    const t = scheduler.current();
    const endpoint = ipc.resolveHandle(t, architecture.systemCallArg(state, 0)) orelse return failErr(state, ipc.EBADF);
    // Receiving is the owner's privilege, the same rule the notification binders
    // enforce: a sendable handle means only "you may talk to this". Anything
    // else and a mount's backend endpoint — which `fs_resolve` installs in every
    // caller's table — would let a stranger dequeue the requests meant for the
    // server, taking the capabilities they carry and answering in its name.
    if (!ipc.ownedBy(endpoint, t)) return failErr(state, ipc.EPERM);
    var badge: u64 = 0;
    var received_cap: u64 = abi.no_cap;
    const r = ipc.replyWait(endpoint, architecture.systemCallArg(state, 1), architecture.systemCallArg(state, 2), architecture.systemCallArg(state, 3), architecture.systemCallArg(state, 4), architecture.systemCallArg(state, 5), &badge, &received_cap);
    architecture.setSystemCallResult(state, @bitCast(r));
    architecture.setSystemCallResult2(state, badge);
    architecture.setSystemCallResult3(state, received_cap);
}

/// ipc_send(handle, message_ptr, message_len) -> 0/-errno: post a payload to an
/// endpoint's async queue and wake a receiver, without blocking the caller. The async
/// counterpart of ipc_call — for broadcasts (the input service) where a rendezvous would
/// let one dead subscriber hang the sender. Delivered through ipc_reply_wait as a
/// buffered message (badge carries notify_message_bit and the caller's task id).
fn systemIpcSend(state: *architecture.CpuState) void {
    const me = scheduler.current();
    const endpoint = ipc.resolveHandle(me, architecture.systemCallArg(state, 0)) orelse return failErr(state, ipc.EBADF);
    const r = ipc.send(endpoint, me.address_space, architecture.systemCallArg(state, 1), architecture.systemCallArg(state, 2), me.id);
    architecture.setSystemCallResult(state, @bitCast(r));
}

/// device_enumerate(buffer, maximum) -> total: snapshot the device table into the caller's
/// buffer (up to `maximum` entries), returning the total device count.
///
/// The broker fills a small kernel chunk which `copyToUser` then places in the
/// caller's buffer: the kernel never stores through a user pointer, so a bad one
/// is -EFAULT instead of a ring-0 page fault. A DeviceDescriptor is a few hundred
/// bytes, so the chunk is deliberately tiny — the 16 KiB kernel stack could not
/// hold a whole user-requested array of them.
fn systemDeviceEnumerate(state: *architecture.CpuState) void {
    const buffer_ptr = architecture.systemCallArg(state, 0);
    const maximum = architecture.systemCallArg(state, 1);
    const t = scheduler.current();
    if (t.address_space == 0 or buffer_ptr >= user_half_end) return fail(state);
    const sz = @sizeOf(device_abi.DeviceDescriptor);
    const cap = @min(maximum, (user_half_end - buffer_ptr) / sz); // clamp to the user half

    var chunk: [2]device_abi.DeviceDescriptor = undefined;
    var copied: u64 = 0;
    var start: usize = 0;
    while (copied < cap) {
        const filled = devices_broker.enumerateFrom(start, &chunk);
        if (filled == 0) break;
        start += filled;
        const take = @min(@as(u64, filled), cap - copied);
        const bytes = std.mem.sliceAsBytes(chunk[0..@intCast(take)]);
        if (!user_memory.copyToUser(t.address_space, buffer_ptr + copied * sz, bytes)) return failErr(state, ipc.EFAULT);
        copied += take;
    }
    architecture.setSystemCallResult(state, devices_broker.deviceCount());
}

/// device_claim(id) -> 0/-1: take exclusive ownership of a device for this process.
fn systemDeviceClaim(state: *architecture.CpuState) void {
    const device_id = architecture.systemCallArg(state, 0);
    const claim_flags = sync.enter();
    defer sync.leave(claim_flags);
    if (devices_broker.claim(device_id, scheduler.current().id)) {
        // Confine the device's DMA before the driver can program it: a PCI function
        // becomes reachable to the IOMMU only once claimed (until now its DMA is
        // blocked). A claim that cannot be confined must not stand — roll it back —
        // since the whole point is that claiming a DMA device is no longer equivalent
        // to ring 0. No-op when no IOMMU exists (fail-open).
        if (devices_broker.pciAddressOf(device_id)) |bdf| {
            const owner = scheduler.current().id;
            if (!iommu.confineDevice(device_id, bdf, owner)) {
                _ = devices_broker.unclaim(device_id, owner);
                return fail(state);
            }
            // Bind the buffers this task allocated before claiming the device (a driver
            // that dma_alloc'd its rings, then claimed the controller).
            dmaBindOwnerRegionsInto(owner, device_id);
        }
        // A display service just took the framebuffer — quiesce the bootstrap console
        // so the kernel and the service don't scribble over each other's pixels. The
        // claim releases (and the console resumes) automatically if the service dies;
        // see releaseTaskResourcesLocked.
        if (devices_broker.displayDevice()) |display_id| {
            if (device_id == display_id) console.setSuppressed(true);
        }
        architecture.setSystemCallResult(state, 0);
    } else fail(state);
}

/// mmio_map(device_id, resource_index) -> virtual_address: map a claimed device's MMIO window into
/// this address space (strong-uncacheable) and return the register base address.
/// The claim is the capability — a process can only map hardware it owns.
fn systemMmioMap(state: *architecture.CpuState) void {
    const device_id = architecture.systemCallArg(state, 0);
    const resource_index = architecture.systemCallArg(state, 1);
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    // Read the broker table under the lock: ring-3 device_register (M19) now
    // mutates it concurrently on other cores, so a lock-free read here could
    // see a torn resource (and a torn length used to panic the arithmetic
    // below on integer overflow).
    const r = blk: {
        const flags = sync.enter();
        defer sync.leave(flags);
        const owner = devices_broker.ownerOf(device_id) orelse return fail(state);
        if (owner != t.id) return fail(state); // not claimed by this process
        break :blk devices_broker.resourceOf(device_id, resource_index) orelse return fail(state);
    };
    if (r.kind != @intFromEnum(device_abi.ResourceKind.memory)) return fail(state);
    // A zero-length or wrapping window is not mappable — fail cleanly rather
    // than underflow `r.len - 1`.
    if (r.len == 0) return fail(state);
    if (@addWithOverflow(r.start, r.len)[1] != 0) return fail(state);

    const first = r.start & ~@as(u64, page_size - 1);
    const last = (r.start + r.len - 1) & ~@as(u64, page_size - 1);
    const pages = (last - first) / page_size + 1;
    // A framebuffer resource asks (via its flag) to be mapped write-combining rather
    // than the strong-uncacheable default that register MMIO needs.
    const write_combining = (r.flags & device_abi.resource_flag_write_combining) != 0;

    // Per-address-space cursor + shared page tables → serialize under the big lock,
    // same as mmap (docs/threading-plan.md M7).
    const flags = sync.enter();
    defer sync.leave(flags);
    const cursor = scheduler.addressSpaceDeviceMapNextPtr(t.address_space) orelse return fail(state);
    if (cursor.* == 0) cursor.* = device_arena_base; // seed the arena lazily
    const base_v = cursor.*;
    if (base_v + pages * page_size > device_arena_end) return fail(state);
    architecture.mapUserDeviceInto(t.address_space, base_v, r.start, r.len, write_combining);
    cursor.* = base_v + pages * page_size;
    architecture.setSystemCallResult(state, base_v + (r.start & (page_size - 1))); // register base
}

/// Resolve a port-I/O access against the caller's claims. The device must be claimed by
/// `t`, `resource_index` must name one of its `io_port` resources, and the access
/// `[offset, offset+width)` must fall wholly inside it. Returns the absolute 16-bit
/// port, or null if the capability check fails. The claim plus the discovered `io_port`
/// resource are the capability — exactly like `mmio_map` for memory, so a driver can
/// only touch the ports its device actually owns, never a raw `in`/`out` to anywhere.
pub fn resolveIoPort(t: *scheduler.Task, device_id: u64, resource_index: u64, offset: u64, width: u64) ?u16 {
    if (width != 1 and width != 2 and width != 4) return null;
    const owner = devices_broker.ownerOf(device_id) orelse return null;
    if (owner != t.id) return null; // not claimed by this process
    const r = devices_broker.resourceOf(device_id, resource_index) orelse return null;
    if (r.kind != @intFromEnum(device_abi.ResourceKind.io_port)) return null;
    if (offset + width > r.len) return null; // access escapes the claimed port range
    const port = r.start + offset;
    if (port + width > 0x1_0000) return null; // I/O ports are 16-bit
    return @intCast(port);
}

/// io_read(device_id, resource_index, offset, width) -> value: read `width` bytes (1/2/4)
/// from a port in a claimed device's `io_port` resource. Ring 3 has no direct `in`/`out`
/// (no TSS I/O bitmap, IOPL never raised), so a legacy driver (PS/2, 16550 UART) reaches
/// its ports through this claim-gated call — low-rate hardware, so a syscall per access
/// is fine. See docs/drivers.md.
fn systemIoRead(state: *architecture.CpuState) void {
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const width = architecture.systemCallArg(state, 3);
    const port = resolveIoPort(t, architecture.systemCallArg(state, 0), architecture.systemCallArg(state, 1), architecture.systemCallArg(state, 2), width) orelse return fail(state);
    architecture.setSystemCallResult(state, architecture.pioRead(@intCast(width), port));
}

/// io_write(device_id, resource_index, offset, width, value) -> 0: write `value` (low
/// `width` bytes) to a port in a claimed device's `io_port` resource. Same capability
/// gate as `io_read`.
fn systemIoWrite(state: *architecture.CpuState) void {
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const width = architecture.systemCallArg(state, 3);
    const port = resolveIoPort(t, architecture.systemCallArg(state, 0), architecture.systemCallArg(state, 1), architecture.systemCallArg(state, 2), width) orelse return fail(state);
    architecture.pioWrite(@intCast(width), port, @intCast(architecture.systemCallArg(state, 4)));
    architecture.setSystemCallResult(state, 0);
}

/// dma_alloc(len, flags) -> virtual_address (rax), physical_address (rdx): grant `len` bytes (rounded up to
/// whole pages) of DMA-capable memory — physically contiguous, zeroed, pinned, and
/// strong-uncacheable (coherent) — mapping it into the caller's DMA arena and handing
/// back both the virtual address to touch and the physical address to program into the
/// device. This is what `sysMmap` can't do: mmap frames are scattered, cacheable, and
/// their physical address is never disclosed. `dma_below_4g` caps the physical address
/// for legacy engines; `dma_write_combining` is accepted but falls back to coherent
/// until PAT is programmed. See docs/driver-model.md (M14).
// --- DMA-region registry ---------------------------------------------------------------
// Every dma_alloc'd region is tracked here so it can be (a) auto-bound into the devices
// its owner claims, (b) delegated across processes as a capability and bound into a
// device's IOMMU domain by dma_bind, and (c) unmapped from every domain before its frames
// return to the allocator. Only *shareable* regions carry a heap `object` (the delegation
// token); a driver's private rings are tracked without one. All access under the big lock.
const DmaRegistryEntry = struct {
    active: bool = false,
    object: ?*ipc.DmaRegionObject = null,
    physical: u64 = 0,
    len: u64 = 0,
    owner: u32 = 0,
};
const maximum_dma_regions = 256;
var dma_registry: [maximum_dma_regions]DmaRegistryEntry = .{DmaRegistryEntry{}} ** maximum_dma_regions;

fn dmaRegistryAdd(object: ?*ipc.DmaRegionObject, physical: u64, len: u64, owner: u32) void {
    for (&dma_registry) |*e| {
        if (!e.active) {
            e.* = .{ .active = true, .object = object, .physical = physical, .len = len, .owner = owner };
            return;
        }
    }
}

/// Retire the region based at `physical` owned by `owner`: unmap it from every device
/// domain (before the frames are freed), mark its token dead so a stale downstream handle
/// can no longer bind it, drop the registry's reference, and clear the slot.
fn dmaRegistryRemove(owner: u32, physical: u64) void {
    for (&dma_registry) |*e| {
        if (e.active and e.owner == owner and e.physical == physical) {
            iommu.unmapRegionEverywhere(e.physical, e.len);
            if (e.object) |object| {
                object.dead = true;
                ipc.dropDmaRegionReference(object);
            }
            e.* = .{};
            return;
        }
    }
}

/// Retire every region owned by `owner` (task death) — same discipline as a per-region
/// free, run before the address space is torn down and its DMA frames reclaimed.
fn dmaRegistryReleaseOwner(owner: u32) void {
    for (&dma_registry) |*e| {
        if (e.active and e.owner == owner) {
            iommu.unmapRegionEverywhere(e.physical, e.len);
            if (e.object) |object| {
                object.dead = true;
                ipc.dropDmaRegionReference(object);
            }
            e.* = .{};
        }
    }
}

/// Bind every region `owner` allocated into the domain of the device it just claimed
/// (its rings allocated before the claim). Regions allocated *after* the claim are bound
/// by dma_alloc's own auto-bind.
fn dmaBindOwnerRegionsInto(owner: u32, device_id: u64) void {
    for (&dma_registry) |*e| {
        if (e.active and e.owner == owner) _ = iommu.mapForDevice(device_id, e.physical, e.len);
    }
}

/// iommu_fault_drain() -> count: drain and log any pending IOMMU translation faults,
/// returning how many were seen. A diagnostic hook — a driver (or a test) that suspects
/// its device faulted can force the fault records to be logged now rather than waiting
/// for the next device-release drain. Harmless without an IOMMU (returns 0).
fn systemIommuFaultDrain(state: *architecture.CpuState) void {
    const flags = sync.enter();
    const count = iommu.faultDrain();
    sync.leave(flags);
    architecture.setSystemCallResult(state, count);
}

/// Resolve a capability handle the caller holds to the physical range it names — either
/// a DMA-region token or a shared-memory object (both are bindable buffers). null if the
/// handle is neither, or names a region already freed by its allocator.
fn bindableRange(t: *scheduler.Task, handle: u64) ?struct { physical: u64, len: u64 } {
    if (ipc.resolveDmaRegion(t, handle)) |region| {
        if (region.dead) return null;
        return .{ .physical = region.phys, .len = region.pages * page_size };
    }
    if (ipc.resolveSharedMemory(t, handle)) |shared| {
        return .{ .physical = shared.phys, .len = shared.pages * page_size };
    }
    return null;
}

/// dma_bind(device_id, handle) -> 0/-errno: map the buffer named by `handle` into the
/// claimed device's IOMMU domain. The caller must own the device (the mmio_map gate) and
/// hold the handle. Idempotent: re-binding is a harmless success, so a driver may re-bind
/// after a restart without tracking what it already bound. Success (no-op) without an IOMMU.
fn systemDmaBind(state: *architecture.CpuState) void {
    const device_id = architecture.systemCallArg(state, 0);
    const handle = architecture.systemCallArg(state, 1);
    const t = scheduler.current();
    const flags = sync.enter();
    defer sync.leave(flags);
    if (devices_broker.ownerOf(device_id) != t.id) return fail(state);
    const range = bindableRange(t, handle) orelse return fail(state);
    if (!iommu.mapForDevice(device_id, range.physical, range.len)) return fail(state);
    architecture.setSystemCallResult(state, 0);
}

/// dma_unbind(device_id, handle) -> 0/-errno: unmap a previously bound buffer from the
/// device's domain and invalidate.
fn systemDmaUnbind(state: *architecture.CpuState) void {
    const device_id = architecture.systemCallArg(state, 0);
    const handle = architecture.systemCallArg(state, 1);
    const t = scheduler.current();
    const flags = sync.enter();
    defer sync.leave(flags);
    if (devices_broker.ownerOf(device_id) != t.id) return fail(state);
    const range = bindableRange(t, handle) orelse return fail(state);
    iommu.unmapForDevice(device_id, range.physical, range.len);
    architecture.setSystemCallResult(state, 0);
}

/// handle_close(handle) -> 0/-errno: drop one capability handle and free its slot.
fn systemHandleClose(state: *architecture.CpuState) void {
    const handle = architecture.systemCallArg(state, 0);
    const t = scheduler.current();
    const flags = sync.enter();
    defer sync.leave(flags);
    if (ipc.closeHandle(t, handle) < 0) return fail(state);
    architecture.setSystemCallResult(state, 0);
}

fn systemDmaAlloc(state: *architecture.CpuState) void {
    const len = architecture.systemCallArg(state, 0);
    const flags = architecture.systemCallArg(state, 1);
    const t = scheduler.current();
    if (t.address_space == 0 or len == 0) return fail(state);

    const pages: usize = @intCast((len + page_size - 1) / page_size);
    const max_phys: u64 = if (flags & abi.dma_below_4g != 0) (@as(u64, 4) << 30) else ~@as(u64, 0);

    // Frame allocation and cursor reservation under the lock — the pmm has no
    // lock of its own, and sibling threads must hand out disjoint windows of
    // the one shared arena.
    var base_v: u64 = 0;
    var phys: u64 = 0;
    {
        const lock_flags = sync.enter();
        const cursor = scheduler.addressSpaceDmaNextPtr(t.address_space) orelse {
            sync.leave(lock_flags);
            return fail(state);
        };
        if (cursor.* == 0) cursor.* = dma_arena_base;
        base_v = cursor.*;
        if (base_v + pages * page_size > dma_arena_end) {
            sync.leave(lock_flags);
            return fail(state); // arena exhausted
        }
        phys = pmm.allocContiguous(pages, max_phys) orelse {
            sync.leave(lock_flags);
            return fail(state);
        };
        cursor.* = base_v + pages * page_size;
        sync.leave(lock_flags);
    }

    // Zero through the physmap OUTSIDE the lock (the frames are still private to
    // this call), then map page-by-page, each under a brief lock hold — the lock
    // serializes the shared page-table walk (siblings on other cores walk the
    // same tables), and per-page holds keep the mmap discipline: never pin every
    // core's ticks across a multi-MiB operation.
    const kernel_view: [*]u8 = @ptrFromInt(boot_handoff.physicalToVirtual(phys));
    @memset(kernel_view[0 .. pages * page_size], 0);
    for (0..pages) |i| {
        const lock_flags = sync.enter();
        architecture.mapUserDmaInto(t.address_space, base_v + i * page_size, phys + i * page_size, page_size);
        sync.leave(lock_flags);
    }
    // Register the region and bind it into every device this task already drives (its
    // own buffers reach its own devices). When `dma_shareable` is set, also wrap it in a
    // capability token and return a handle so it can be delegated to another driver and
    // dma_bound there. All under the lock (the registry + IOMMU tables are shared).
    var handle: u64 = abi.no_cap;
    {
        const lock_flags = sync.enter();
        defer sync.leave(lock_flags);
        var object: ?*ipc.DmaRegionObject = null;
        if (flags & abi.dma_shareable != 0) {
            if (ipc.createDmaRegion(phys, pages, t.id)) |region| {
                const h = ipc.installDmaRegionHandle(t, region);
                if (h >= 0) {
                    region.refcount += 1; // the handle's reference (registry holds the first)
                    handle = @intCast(h);
                    object = region;
                } else {
                    ipc.dropDmaRegionReference(region); // no table slot; drop it
                }
            }
        }
        dmaRegistryAdd(object, phys, pages * page_size, t.id);
        iommu.mapRegionForOwner(t.id, phys, pages * page_size);
    }
    architecture.setSystemCallResult(state, base_v); // virtual address for the CPU
    architecture.setSystemCallResult2(state, phys); // physical address for the device
    architecture.setSystemCallResult3(state, handle); // capability handle (no_cap unless shareable)
}

/// dma_free(virtual_address, len) -> 0: release a prior `dma_alloc`. Bounded to the DMA arena so
/// it can never unmap-and-free the caller's stack, heap, or an MMIO grant; only pages
/// actually mapped are freed (an unmapped hole is skipped). Teardown also reclaims any
/// DMA pages left mapped at exit (they carry no `device_grant`, so `freeSubtree` frees
/// them as ordinary RAM), so a driver that just dies leaks nothing.
fn systemDmaFree(state: *architecture.CpuState) void {
    const base_v = architecture.systemCallArg(state, 0);
    const len = architecture.systemCallArg(state, 1);
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const pages: usize = @intCast((len + page_size - 1) / page_size);
    if (base_v < dma_arena_base or base_v + pages * page_size > dma_arena_end) return fail(state);

    // Retire the region — unmap it from every device domain and mark its token dead —
    // BEFORE any frame returns to the allocator, so no device can still translate to a
    // reallocated frame. dma_alloc's frames are contiguous, so the base names the region.
    {
        const lock_flags = sync.enter();
        if (architecture.translate(t.address_space, base_v)) |base_phys|
            dmaRegistryRemove(t.id, base_phys);
        sync.leave(lock_flags);
    }

    for (0..pages) |i| {
        const va = base_v + i * page_size;
        // Per-page lock hold: the translate/unmap walks the shared page tables
        // and pmm.free mutates the unlocked frame bitmap.
        const lock_flags = sync.enter();
        if (architecture.translate(t.address_space, va)) |phys| {
            architecture.unmapUserPageInto(t.address_space, va);
            pmm.free(phys);
        }
        sync.leave(lock_flags);
    }
    architecture.setSystemCallResult(state, 0);
}

/// shared_memory_create(len) -> virtual_address (rax), handle (rdx): grant `len` bytes (rounded up to whole
/// pages) of **shareable, zeroed, cacheable** RAM — contiguous frames mapped into the
/// caller's shared-memory arena — and hand back the virtual address plus a capability handle. Unlike
/// `dma_alloc` the memory is write-back cacheable (for CPU compositing, not device DMA) and
/// its frames are owned by a refcounted object: the handle is passed to another process as
/// an `ipc_call` send_cap, that process `shared_memory_map`s it, and the frames free only when the
/// last capability drops (docs/display-v2.md — the compositor↔native-driver and
/// app↔compositor surface path).
fn systemSharedMemoryCreate(state: *architecture.CpuState) void {
    const len = architecture.systemCallArg(state, 0);
    const t = scheduler.current();
    if (t.address_space == 0 or len == 0) return fail(state);

    const pages: usize = @intCast((len + page_size - 1) / page_size);
    if (pages == 0 or pages > maximum_shared_memory_pages) return fail(state);

    // One locked section builds the whole named object — cursor reservation,
    // frames, the refcounted object, the space's mapping record (M3: frames must
    // outlive every MAPPING, not just every handle), and the handle — with full
    // rollback, so no failure path ever touches the unlocked pmm/heap and no
    // half-built region is ever reachable.
    var base_v: u64 = 0;
    var phys: u64 = 0;
    var handle: i64 = -1;
    var shared_memory: *ipc.SharedMemoryObject = undefined;
    {
        const lock_flags = sync.enter();
        const cursor = scheduler.addressSpaceSharedMemoryNextPtr(t.address_space) orelse {
            sync.leave(lock_flags);
            return fail(state);
        };
        if (cursor.* == 0) cursor.* = shared_memory_arena_base;
        base_v = cursor.*;
        if (base_v + pages * page_size > shared_memory_arena_end) {
            sync.leave(lock_flags);
            return fail(state); // arena exhausted
        }
        phys = pmm.allocContiguous(pages, ~@as(u64, 0)) orelse {
            sync.leave(lock_flags);
            return fail(state);
        };
        shared_memory = ipc.createSharedMemory(phys, pages) orelse {
            for (0..pages) |i| pmm.free(phys + i * page_size);
            sync.leave(lock_flags);
            return fail(state);
        };
        if (!scheduler.recordSpaceMappingLocked(t.address_space, @ptrCast(shared_memory))) {
            ipc.dropSharedMemoryReference(shared_memory); // last ref: frees the object AND its frames
            sync.leave(lock_flags);
            return fail(state);
        }
        ipc.retainSharedMemory(shared_memory); // the mapping's own reference
        handle = ipc.installSharedMemoryHandle(t, shared_memory);
        if (handle < 0) {
            // Full rollback: unrecord the mapping, then drop both references —
            // the second drop frees the object and its frames, all under the lock.
            scheduler.removeSpaceMappingLocked(t.address_space, @ptrCast(shared_memory));
            ipc.dropSharedMemoryReference(shared_memory);
            ipc.dropSharedMemoryReference(shared_memory);
            sync.leave(lock_flags);
            return fail(state);
        }
        cursor.* = base_v + pages * page_size;
        sync.leave(lock_flags);
    }

    // Zero through the physmap OUTSIDE the lock (the frames are private until
    // the handle is shared and the pages mapped), then map page-by-page under
    // brief lock holds — the lock serializes the shared page-table walk against
    // sibling threads, per-page so no core's tick starves (the mmap discipline).
    const kernel_view: [*]u8 = @ptrFromInt(boot_handoff.physicalToVirtual(phys));
    @memset(kernel_view[0 .. pages * page_size], 0);
    for (0..pages) |i| {
        const lock_flags = sync.enter();
        architecture.mapUserSharedInto(t.address_space, base_v + i * page_size, phys + i * page_size, page_size);
        sync.leave(lock_flags);
    }
    architecture.setSystemCallResult(state, base_v); // virtual_address for the CPU
    architecture.setSystemCallResult2(state, @intCast(handle)); // capability handle to pass on
}

/// shared_memory_map(cap) -> virtual_address: map the shared region named by a capability handle the caller
/// received (via an `ipc_call` send_cap) into its shared-memory arena — the same physical frames the
/// creator sees — returning the virtual address. The handle already holds a reference (taken
/// when the capability was shared), so this only adds a mapping; it never bumps the refcount.
fn systemSharedMemoryMap(state: *architecture.CpuState) void {
    const cap = architecture.systemCallArg(state, 0);
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);

    const shared_memory = ipc.resolveSharedMemory(t, cap) orelse return fail(state); // not a shared-memory handle we hold
    const size = shared_memory.pages * page_size;

    // Reserve arena space (per-SPACE cursor) and record the mapping's own
    // reference in one locked section — the reference is dropped at space
    // destruction (docs/shared-fate-plan.md M3); the handle's reference is
    // separate and may be closed while the mapping lives on.
    var base_v: u64 = 0;
    {
        const flags = sync.enter();
        const cursor = scheduler.addressSpaceSharedMemoryNextPtr(t.address_space) orelse {
            sync.leave(flags);
            return fail(state);
        };
        if (cursor.* == 0) cursor.* = shared_memory_arena_base;
        base_v = cursor.*;
        if (base_v + size > shared_memory_arena_end) {
            sync.leave(flags);
            return fail(state); // arena exhausted
        }
        if (!scheduler.recordSpaceMappingLocked(t.address_space, @ptrCast(shared_memory))) {
            sync.leave(flags);
            return fail(state); // mapping table full: refuse rather than map unrecorded
        }
        ipc.retainSharedMemory(shared_memory);
        cursor.* = base_v + size;
        sync.leave(flags);
    }
    // Map page-by-page under brief lock holds — the shared page-table walk must
    // be serialized against sibling threads (the mmap discipline).
    var page_index: usize = 0;
    while (page_index * page_size < size) : (page_index += 1) {
        const lock_flags = sync.enter();
        architecture.mapUserSharedInto(t.address_space, base_v + page_index * page_size, shared_memory.phys + page_index * page_size, page_size);
        sync.leave(lock_flags);
    }
    architecture.setSystemCallResult(state, base_v);
}

/// shared_memory_physical(cap) -> physical_address: the guest-physical base of a shared region the caller holds a
/// capability for. The frames are contiguous (allocated by `allocContiguous`), so a single
/// physical base + length describes the whole region — which is exactly what a driver needs
/// to hand a shared-memory surface to a device (virtio-gpu `attach_backing`). Only a holder of the
/// capability can ask; there is no ambient way to turn a virtual address into a physical one.
fn systemSharedMemoryPhysical(state: *architecture.CpuState) void {
    const cap = architecture.systemCallArg(state, 0);
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const shared_memory = ipc.resolveSharedMemory(t, cap) orelse return fail(state); // not a shared-memory handle we hold
    architecture.setSystemCallResult(state, shared_memory.phys);
}

/// device_register(parent_id, descriptor_ptr) -> id: publish a child device below a device
/// this process has claimed. The bus-driver primitive: a process that owns a bus
/// enumerates it and hands each device it finds to the table, where a class driver
/// can claim it.
///
/// The kernel copies the descriptor into a kernel local *once* (via the same
/// physmap-walking path as IPC, so an unmapped user page fails the call rather than
/// faulting the kernel), then validates and uses that copy — no second read of user
/// memory, so nothing it checked can change under it. It refuses any child resource
/// that escapes the parent's windows: a descriptor is a licence to map physical
/// memory, so a bus may only subdivide what it already holds. `id`/`parent` in the
/// supplied descriptor are ignored.
fn systemDeviceRegister(state: *architecture.CpuState) void {
    const parent_id = architecture.systemCallArg(state, 0);
    const descriptor_ptr = architecture.systemCallArg(state, 1);
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);

    var descriptor: device_abi.DeviceDescriptor = undefined;
    if (!ipc.copyFromUser(t.address_space, descriptor_ptr, std.mem.asBytes(&descriptor))) return fail(state);

    // Under the big kernel lock: the broker's table is also mutated by the
    // death sweep (releaseAllOwnedBy) and read by enumerate on other cores —
    // ring-3 registration (M19) made those genuinely concurrent.
    const flags = sync.enter();
    defer sync.leave(flags);
    const id = devices_broker.register(parent_id, t.id, &descriptor) catch return fail(state);
    architecture.setSystemCallResult(state, id);
}

/// system_spawn(name_ptr, name_len, arguments_ptr, arguments_len, exit_endpoint)
/// -> the child's process id on success, -1 on failure. Load the binary bundled in
/// the initial-ramdisk under `name` as a fresh ring-3 process. `name` becomes the
/// child's argv[0] (and its task name, so a fault report can say which binary
/// died); `arguments` is an optional NUL-separated blob that becomes argv[1..] —
/// how a supervisor parameterises what it starts ("you are the driver for device
/// 12"). 0/0 means no extra arguments. This is the mechanism a user-space
/// supervisor (the device manager) uses to start a driver it matched: discovery
/// and policy stay in user space, the kernel only spawns.
///
/// The caller is recorded as the child's **supervisor** — the sole holder of the
/// right to `process_kill` it (docs/process-management.md). `exit_endpoint` (a
/// handle, or `abi.no_cap` for none) names an endpoint of the caller's to notify
/// when the child ends, any way it ends — the IRQ-as-IPC pattern reused as the
/// microkernel's SIGCHLD.
///
/// Spawning itself is still ungated — any process may spawn any bundled binary; a
/// spawn capability belongs here once the model grows one (docs/driver-model.md).
/// Both buffers are bounds-checked into the user half exactly like `debug_write`,
/// and an unknown name or a load failure returns -1.
fn systemSpawn(state: *architecture.CpuState) void {
    const ptr = architecture.systemCallArg(state, 0);
    const len = architecture.systemCallArg(state, 1);
    const arguments_ptr = architecture.systemCallArg(state, 2);
    const arguments_len = architecture.systemCallArg(state, 3);
    const exit_handle = architecture.systemCallArg(state, 4);
    const t = scheduler.current();
    if (len == 0 or len > scheduler.maximum_task_name or ptr >= user_half_end or ptr + len > user_half_end) return fail(state);
    if (arguments_len > maximum_argument_bytes) return fail(state);
    if (arguments_len != 0 and (arguments_ptr >= user_half_end or arguments_ptr + arguments_len > user_half_end)) return fail(state);
    // The exit endpoint is a notification binding like signal_bind's and
    // timer_bind's, so it obeys the same rule: the caller's own mailbox, never a
    // stranger's. Otherwise any process could have the kernel post child-exit
    // badges into PID 1 by spawning throwaway children against init's endpoint.
    const exit_endpoint: ?*ipc.Endpoint = if (exit_handle == abi.no_cap)
        null
    else block: {
        const endpoint = ipc.resolveHandle(t, exit_handle) orelse return failErr(state, ipc.EBADF);
        if (!ipc.ownedBy(endpoint, t)) return failErr(state, ipc.EPERM);
        break :block endpoint;
    };
    const image = ramdisk_image orelse return fail(state);
    const rd = initial_ramdisk.Reader.init(image) orelse return fail(state);

    // Both buffers come in through the checked copy layer, once. The lengths are
    // already bounded above, so the staging arrays are small and fixed — and
    // because the bytes are now the kernel's own, nothing below can be changed
    // by another thread of the caller between validation and use.
    var name_storage: [scheduler.maximum_task_name]u8 = undefined;
    const name = name_storage[0..@intCast(len)];
    if (!user_memory.copyFromUser(t.address_space, ptr, name)) return failErr(state, ipc.EFAULT);
    var argument_storage: [maximum_argument_bytes]u8 = undefined;
    const arguments = argument_storage[0..@intCast(arguments_len)];
    if (arguments_len != 0 and !user_memory.copyFromUser(t.address_space, arguments_ptr, arguments)) return failErr(state, ipc.EFAULT);

    // Exact path first, basename fallback second; either way argv[0] (and hence
    // the task name, and the log ring's attribution) is the stored full path.
    const item = rd.find(name) orelse return fail(state); // no bundled binary by that name
    var argv: [maximum_arguments][]const u8 = undefined;
    argv[0] = item.name;
    var argc: usize = 1;
    if (arguments_len != 0) {
        var pieces = std.mem.tokenizeScalar(u8, arguments, 0);
        while (pieces.next()) |piece| {
            if (argc == maximum_arguments) return fail(state);
            argv[argc] = piece;
            argc += 1;
        }
    }

    const child = spawnProcessSupervised(item.blob, 4, argv[0..argc], t.id, exit_endpoint) catch return fail(state);
    architecture.setSystemCallResult(state, child);
}

/// thread_spawn(entry, stack_top, arg) -> tid: start a task that shares the **caller's**
/// address space (docs/threading.md). The runtime supplies `entry` (its thread
/// trampoline), a stack it mmap'd, and the closure pointer, which the kernel delivers in
/// the new thread's rdi. The entry and stack must lie in the user half; the new thread is
/// supervised by the caller and inherits its priority. Only a user process may spawn.
fn systemThreadSpawn(state: *architecture.CpuState) void {
    const entry = architecture.systemCallArg(state, 0);
    const stack_top = architecture.systemCallArg(state, 1);
    const arg = architecture.systemCallArg(state, 2);
    const exit_handle = architecture.systemCallArg(state, 3);
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state); // kernel tasks own no address space to share
    if (entry == 0 or entry >= user_half_end) return fail(state);
    if (stack_top == 0 or stack_top > user_half_end) return fail(state);
    // The endpoint the thread notifies on exit (how join waits), or none — the
    // caller's own, like every other notification binding.
    const exit_endpoint: ?*ipc.Endpoint = if (exit_handle == abi.no_cap)
        null
    else block: {
        const endpoint = ipc.resolveHandle(t, exit_handle) orelse return failErr(state, ipc.EBADF);
        if (!ipc.ownedBy(endpoint, t)) return failErr(state, ipc.EPERM);
        break :block endpoint;
    };
    const tid = spawnThreadSupervised(t.address_space, entry, stack_top, arg, t.priority, t.id, exit_endpoint, t.leader);
    if (tid == -ipc.ESRCH) return failErr(state, ipc.ESRCH); // dying group admits no member
    if (tid < 0) return fail(state);
    architecture.setSystemCallResult(state, @intCast(tid));
}

/// Spawn a thread sharing `address_space`, taking the exit-endpoint reference under the **same**
/// lock as the spawn (as `spawnProcessSupervised` does), so the thread cannot die before
/// its reference exists. Returns the new thread id, or null on resource exhaustion.
fn spawnThreadSupervised(address_space: u64, entry: u64, stack_top: u64, arg: u64, priority: scheduler.Priority, supervisor: u32, exit_endpoint: ?*ipc.Endpoint, leader: u32) i64 {
    const flags = sync.enter();
    defer sync.leave(flags);
    // The spawn gate (docs/shared-fate-plan.md): a dying group admits no new
    // member — checked under the same lock that would create it, and reported
    // as -ESRCH (the process is as good as gone). retainAddressSpace inside
    // spawnUserLocked backstops the same refusal.
    if (scheduler.groupDyingLocked(address_space)) return -ipc.ESRCH;
    const tid = scheduler.spawnUserLocked(address_space, entry, stack_top, arg, priority, "thread", supervisor, if (exit_endpoint) |e| @ptrCast(e) else null, leader) orelse return -1;
    if (exit_endpoint) |endpoint| endpoint.refcount += 1; // the thread holds it birth-to-death
    return tid;
}

/// current_core() -> index: the dense 0-based index of the core the caller runs on.
fn systemCurrentCore(state: *architecture.CpuState) void {
    architecture.setSystemCallResult(state, scheduler.currentCpuIndex());
}

/// thread_self() -> tid: the calling thread's kernel task id.
fn systemThreadSelf(state: *architecture.CpuState) void {
    architecture.setSystemCallResult(state, scheduler.currentId());
}

/// set_thread_pointer(addr) -> 0: set the caller's user-space TLS thread pointer. The
/// arch layer maps it to IA32_FS_BASE on x86_64, `TPIDR_EL0` on aarch64; the kernel
/// never reads it, and the scheduler restores it per task across context switches
/// (docs/threading-plan.md M10). `addr` must be a user-half address.
fn systemSetThreadPointer(state: *architecture.CpuState) void {
    const addr = architecture.systemCallArg(state, 0);
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state); // kernel tasks have no user TLS
    if (addr >= user_half_end) return fail(state);
    const flags = sync.enter();
    scheduler.setThreadPointerLocked(addr);
    sync.leave(flags);
    architecture.setSystemCallResult(state, 0);
}

/// thread_join(tid) -> 0: block until the thread with id `tid` has exited (docs/threading-
/// plan.md M9). Needs no per-thread IPC endpoint. The compare-and-block is one critical
/// section, so an exit cannot slip between "is it alive?" and the block.
fn systemThreadJoin(state: *architecture.CpuState) void {
    const tid: u32 = @truncate(architecture.systemCallArg(state, 0));
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state); // kernel tasks don't join
    const flags = sync.enter();
    scheduler.joinThreadLocked(tid);
    sync.leave(flags);
    architecture.setSystemCallResult(state, 0);
}

/// futex_wait(addr, expected, timeout_ns) -> status (docs/threading.md): if the 4-byte
/// user word at `addr` still equals `expected`, block until a futex_wake on `addr` or
/// (if timeout_ns > 0) the deadline. The compare and the block are one critical section,
/// so a concurrent futex_wake cannot slip between them. Returns futex_woken / mismatch /
/// timed_out.
fn systemFutexWait(state: *architecture.CpuState) void {
    const addr = architecture.systemCallArg(state, 0);
    const expected: u32 = @truncate(architecture.systemCallArg(state, 1));
    const timeout_ns = architecture.systemCallArg(state, 2);
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    if (addr == 0 or (addr & 3) != 0 or addr + 4 > user_half_end) return fail(state);

    const flags = sync.enter();
    var word_bytes: [4]u8 = undefined;
    if (!ipc.copyFromUser(t.address_space, addr, &word_bytes)) {
        sync.leave(flags);
        return fail(state);
    }
    if (std.mem.readInt(u32, &word_bytes, .little) != expected) {
        sync.leave(flags);
        architecture.setSystemCallResult(state, abi.futex_mismatch);
        return;
    }
    const timeout_ms = if (timeout_ns == 0) 0 else (timeout_ns + 999_999) / 1_000_000;
    const result = scheduler.futexWaitLocked(addr, timeout_ms);
    sync.leave(flags);
    architecture.setSystemCallResult(state, switch (result) {
        .woken => abi.futex_woken,
        .timed_out => abi.futex_timed_out,
    });
}

/// futex_wake(addr, count) -> woken: wake up to `count` tasks blocked in futex_wait on
/// `addr` in the caller's address space.
fn systemFutexWake(state: *architecture.CpuState) void {
    const addr = architecture.systemCallArg(state, 0);
    const count: u32 = @truncate(architecture.systemCallArg(state, 1));
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    if (addr == 0 or (addr & 3) != 0 or addr + 4 > user_half_end) return fail(state);
    const flags = sync.enter();
    const woken = scheduler.futexWakeLocked(t.address_space, addr, count);
    sync.leave(flags);
    architecture.setSystemCallResult(state, woken);
}

/// process_enumerate(buffer, maximum) -> total: snapshot the task table into the
/// caller's buffer (up to `maximum` `abi.ProcessDescriptor` entries), returning
/// the total live-task count — the exact shape of `device_enumerate`, so a `ps`
/// is a user program over a snapshot, not a kernel service. Read-only and
/// ungated: what is running is not a secret between cooperating bring-up
/// processes.
fn systemProcessEnumerate(state: *architecture.CpuState) void {
    const buffer_ptr = architecture.systemCallArg(state, 0);
    const maximum = architecture.systemCallArg(state, 1);
    const t = scheduler.current();
    if (t.address_space == 0 or buffer_ptr >= user_half_end) return fail(state);
    const sz = @sizeOf(abi.ProcessDescriptor);
    const cap = @min(maximum, (user_half_end - buffer_ptr) / sz); // clamp to the user half

    // The scheduler describes a chunk of the table into kernel memory, then
    // `copyToUser` places it — the kernel never stores through the user pointer.
    // Once the caller's buffer is full the walk continues with an empty chunk,
    // because the result is the true live count, not what fitted.
    var chunk: [8]abi.ProcessDescriptor = undefined;
    var copied: u64 = 0;
    var total: u64 = 0;
    var cursor: usize = 0;
    while (true) {
        const room: []abi.ProcessDescriptor = if (copied < cap) chunk[0..@intCast(@min(chunk.len, cap - copied))] else chunk[0..0];
        const found = scheduler.enumerateFrom(&cursor, room);
        total += found.live;
        if (found.filled != 0) {
            const bytes = std.mem.sliceAsBytes(chunk[0..found.filled]);
            if (!user_memory.copyToUser(t.address_space, buffer_ptr + copied * sz, bytes)) return failErr(state, ipc.EFAULT);
            copied += found.filled;
        }
        if (found.done) break;
    }
    architecture.setSystemCallResult(state, total);
}

/// process_kill(id) -> 0 / -ESRCH / -EPERM: end the process `id`. Only its
/// supervisor — the process that spawned it — may do so; the supervision link is
/// the kill capability, so no user/permission model is needed and a stray id
/// cannot be a weapon (ids are never reused, so a stale one just misses).
fn systemProcessKill(state: *architecture.CpuState) void {
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const id = architecture.systemCallArg(state, 0);
    if (id > std.math.maxInt(u32)) return failErr(state, ipc.ESRCH);
    const r = killProcess(t.id, @intCast(id));
    architecture.setSystemCallResult(state, @bitCast(r));
}

/// Processes killed by a CPU fault rather than a clean exit. Evidence for the
/// fault-recovery test, and a health signal a supervisor can consult later.
pub var fault_kill_count: u64 = 0;

/// Release everything a dying task holds and tell its supervisor — the shared
/// half of every path out of a process: clean exit, fault kill, and process_kill
/// (both the immediate reap and the deferred tick-time terminate). The order
/// matters:
/// - IRQ bindings are dropped before the handle table closes: dropping the last
///   endpoint reference destroys the Endpoint, and a still-bound GSI would have an
///   ISR call notifyFromIsr on freed memory the next time the device fired.
///   `releaseOwner` also leaves the line masked, so a dead driver's device goes
///   quiet rather than storming. (It drops MSI vectors by the same owner sweep.)
/// - Device claims are released with the IRQ bindings, so a restarted driver can
///   claim the same hardware again — the cleanup half of process-lifecycle.md's
///   iron rule 1. Claims hold no pointers, so ordering is free; they go here so
///   the exit notification (below, last) observes a fully-released child.
/// - A client this task still owes a reply to (it died between receive and reply)
///   is failed with -EPEER rather than left blocked forever — a dead server must
///   not hang its callers.
/// - The task is unlinked from wherever IPC parked it (an endpoint's sender FIFO,
///   a receive wait queue, or a server's owed-reply slot) *before* the handles
///   close, so nothing ever dequeues a dangling pointer. These are no-ops for a
///   running task ending itself; they matter when process_kill reaps a blocked one.
/// - The exit notification is posted last, once the process can no longer act, so
///   a supervisor that receives it observes a fully-released child. The endpoint
///   reference taken at spawn is dropped with it.
/// Precondition: the big kernel lock is held.
fn releaseTaskResourcesLocked(t: *scheduler.Task) void {
    recordExitLocked(t);
    irq.releaseOwner(t.id);
    // Detach the task's devices from their IOMMU domains BEFORE the broker clears the
    // claims (the detach reads ownership) and before the address space is torn down and
    // its DMA frames return to the allocator — a device must stop translating to a frame
    // before that frame can be handed to someone else. Then retire the buffers this task
    // allocated, unmapping them from any *other* driver's domain they were granted into,
    // before those frames are freed too.
    iommu.releaseAllOwnedBy(t.id);
    dmaRegistryReleaseOwner(t.id);
    devices_broker.releaseAllOwnedBy(t.id);
    // If that dropped the framebuffer claim (this task was the display service), let the
    // bootstrap console draw again — the screen is nobody's now, so panics/status land.
    if (!devices_broker.displayClaimed()) console.setSuppressed(false);
    // The dying task's signal endpoint and one-shot timers go with it.
    if (t.signal_endpoint) |raw| {
        ipc.dropRef(@ptrCast(@alignCast(raw)));
        t.signal_endpoint = null;
    }
    t.pending_signals = 0;
    for (&one_shot_timers) |*slot| {
        if (slot.*) |timer| {
            if (timer.owner == t.id) {
                ipc.dropRef(timer.endpoint);
                slot.* = null;
            }
        }
    }
    // A dead subscriber's own subscriptions go first: it must not hear about
    // itself, and the slots' endpoint references drop with it.
    for (&exit_subscribers) |*slot| {
        if (slot.*) |subscriber| {
            if (subscriber.owner == t.id) {
                ipc.dropRef(subscriber.endpoint);
                slot.* = null;
            }
        }
    }
    if (t.ipc_client) |client| {
        t.ipc_client = null;
        client.ipc_status = -ipc.EPEER;
        scheduler.readyLocked(client); // its blocked `call` now returns the error
    }
    ipc.abandonSenderLocked(t);
    ipc.killOwnedEndpointsLocked(t.id); // its registered services are gone: callers get -EPEER, not a hang
    scheduler.removeFromWaitQueueLocked(t);
    scheduler.forgetIpcClientLocked(t);
    ipc.closeHandles(t);
    // Publish the exit to every subscriber (docs/process-lifecycle.md): the same
    // badge encoding as the supervisor's notification, and equally late, so a
    // subscriber also observes a fully-released child. One exception: a dying
    // group's LEADER defers its publication to the group_exit_hook — the group
    // is only "fully released" when its LAST member is gone (docs/shared-fate-
    // plan.md). Workers still publish per-task: subscribers like the FAT server
    // sweep per-tid client state and need every id.
    const defer_to_group_hook = t.address_space != 0 and t.id == t.leader and
        scheduler.groupDyingLocked(t.address_space);
    if (!defer_to_group_hook) {
        for (&exit_subscribers) |*slot| {
            if (slot.*) |subscriber| ipc.notifyLocked(subscriber.endpoint, abi.notify_exit_bit | t.id);
        }
    }
    if (t.exit_endpoint) |raw| {
        const endpoint: *ipc.Endpoint = @ptrCast(@alignCast(raw));
        t.exit_endpoint = null;
        ipc.notifyLocked(endpoint, abi.notify_exit_bit | t.id);
        ipc.dropRef(endpoint);
    }
}

/// Tear down the current user process and reschedule; never returns. Shared by the
/// exit system call and the fault path (`killCurrentProcess`). See
/// `releaseTaskResourcesLocked` for what is released, and in what order.
pub fn terminateCurrent() noreturn {
    _ = sync.enter(); // handed off through the exit switch, released by the resumed task
    terminateCurrentLocked();
}

/// The body of `terminateCurrent` for a caller that already holds the big kernel
/// lock — the scheduler's tick calls this (via `terminate_current_hook`) to finish
/// a deferred process_kill on its own core's current task. Never returns; the
/// tick's abandoned interrupt frame is fine (the LAPIC was acknowledged before the
/// tick hook ran), exactly as on the fault path.
fn terminateCurrentLocked() noreturn {
    releaseTaskResourcesLocked(scheduler.current());
    scheduler.exitUserLocked();
}

/// Reap a condemned task that is NOT running on any core (ready or blocked — and
/// it cannot start running: state changes need the lock we hold). The other half
/// of a deferred process_kill, called by the scheduler's tick (via
/// `reap_task_hook`) and directly by `killProcess` for targets caught off-CPU.
/// Precondition: the big kernel lock is held.
fn reapTaskLocked(t: *scheduler.Task) void {
    releaseTaskResourcesLocked(t);
    scheduler.removeFromReadyQueueLocked(t); // no-op unless it was ready in a queue
    scheduler.destroyTaskLocked(t);
}

/// Kill the process containing `target_id` on behalf of `caller_id` — the kernel
/// half of the process_kill system call, and a WHOLE-GROUP kill: any member id
/// resolves to the leader, and every thread dies (docs/shared-fate-plan.md).
/// Returns 0, -ESRCH (no such live process — kernel tasks are not killable
/// processes and stale ids miss, since ids are never reused), or -EPERM (the
/// caller is not the leader's supervisor).
///
/// Members that are ready or blocked are reaped on the spot. Ones running on
/// another core cannot be torn down mid-instruction, so they are condemned
/// (`kill_pending`) and die at their next delivery point — like a Unix signal,
/// delivery is prompt but asynchronous. Either way the call returns 0: the kill
/// is accepted and irrevocable, and the supervisor's notification arrives only
/// once the last member is gone.
pub fn killProcess(caller_id: u32, target_id: u32) i64 {
    const flags = sync.enter();
    defer sync.leave(flags);
    const target = scheduler.taskByIdLocked(target_id) orelse return -ipc.ESRCH;
    if (target.address_space == 0) return -ipc.ESRCH; // kernel tasks are not processes
    // The kill authority is the supervision link of the process's LEADER, so the
    // capability is per-process, aimed at any member id — a thread's own
    // `supervisor` (the task that spawned it) grants nothing here
    // (docs/shared-fate-plan.md M1). Kernel tasks were -ESRCH'd above, so a
    // leader of 0 is unreachable.
    // A group already dying answers from the latch's stash — the leader's slot
    // may already be reaped while a condemned member is still enumerable. Same
    // authority gate, then 0: the kill is already true, accepted and
    // irrevocable (docs/shared-fate-plan.md).
    if (scheduler.groupSupervisorLocked(target.address_space)) |group_supervisor| {
        return if (group_supervisor == caller_id) 0 else -ipc.EPERM;
    }
    const leader = if (target.leader == target.id)
        target
    else
        scheduler.taskByIdLocked(target.leader) orelse return -ipc.ESRCH;
    if (leader.supervisor != caller_id) return -ipc.EPERM;
    // Whole-group kill (docs/shared-fate-plan.md). The caller is never a member
    // (the leader's supervisor predates the group and cannot be inside it), so
    // this always returns.
    killGroupLocked(leader, .killed, null);
    return 0;
}

/// Kill every member of `leader_task`'s group (docs/shared-fate-plan.md): latch
/// the address space as dying — closing the door to new members and moving the
/// leader's exit-endpoint reference into the latch's stash — stamp every
/// member's exit reason, reap the parked ones, and condemn the running ones
/// (there is no kill IPI: a condemned member dies at its next syscall entry, at
/// its own core's next tick in user mode, or — once parked — at any core's tick
/// reap). `trigger` is the current task when the kill came from inside (exit,
/// fault); it dies last and the call never returns. A supervisor's kill passes
/// null and returns. Caller holds the kernel lock and has already checked
/// `groupDyingLocked` — a second trigger must not re-stamp.
fn killGroupLocked(leader_task: *scheduler.Task, reason: abi.ExitReason, trigger: ?*scheduler.Task) void {
    const leader_id = leader_task.id;
    if (scheduler.markGroupDyingLocked(leader_task.address_space, leader_id, leader_task.supervisor, reason, leader_task.exit_endpoint)) {
        // The stash now owns the leader's endpoint reference; the leader's own
        // teardown sees null (no early notification, no double drop) and the
        // group_exit_hook posts exactly once, at space destruction.
        leader_task.exit_endpoint = null;
    }
    // Stamp before any teardown — recordExitLocked snapshots exit_reason as its
    // first act. The LEADER carries the group reason: its record is the one the
    // supervisor can read. The trigger keeps it too (its own record tells the
    // truth); every other member died because the group died: .killed.
    for (scheduler.allTasksLocked()) |*member| {
        if (member.leader != leader_id) continue;
        if (member.state == .free or member.state == .reaping) continue;
        member.exit_reason = if (member.id == leader_id or member == trigger) reason else .killed;
        member.kill_pending.store(true, .monotonic);
    }
    // Reap parked members, to fixpoint: one member's teardown can wake another
    // (an -EPEER'd client, a joiner), flipping it .blocked -> .ready behind the
    // scan. Terminates in at most one pass per member: the scrubs in
    // releaseTaskResourcesLocked (abandonSenderLocked, removeFromWaitQueueLocked,
    // forgetIpcClientLocked, killOwnedEndpointsLocked) run before destroy, so no
    // wake path holds a pointer to a reaped member.
    var progress = true;
    while (progress) {
        progress = false;
        for (scheduler.allTasksLocked()) |*member| {
            if (member.leader != leader_id) continue;
            if (member.state != .ready and member.state != .blocked) continue;
            reapTaskLocked(member);
            progress = true;
        }
    }
    // Members running on other cores stay condemned; the in-group trigger dies
    // now — last, because terminateCurrentLocked switches away for good.
    if (trigger != null) terminateCurrentLocked();
}

/// The group half of `exit`: end the CURRENT task's whole process with `reason`.
/// Takes the lock (handed off through the exit switch, like terminateCurrent)
/// and never returns. A second trigger — the group is already dying — keeps the
/// first trigger's stamp and just dies.
fn exitGroupCurrent(reason: abi.ExitReason) noreturn {
    const t = scheduler.current();
    _ = sync.enter();
    if (scheduler.groupDyingLocked(t.address_space)) terminateCurrentLocked();
    // The orelse fallback is unreachable today — a leader outlives its members
    // (its thread_exit is refused; any leader death IS group death). If a future
    // change ever made it reachable, it degrades to single-task semantics.
    const leader_task = if (t.leader == t.id) t else scheduler.taskByIdLocked(t.leader) orelse t;
    killGroupLocked(leader_task, reason, t);
    unreachable; // killGroupLocked never returns for an in-group trigger
}

/// Kill the current user process in response to a CPU fault it raised in ring 3.
/// The fault is confined to the process — the kernel trapped it on the task's own
/// kernel stack and is intact — so everything the process held is reclaimed and the
/// core reschedules. The system keeps running; only the faulting process dies
/// (docs/resilience.md: fault -> kill -> continue). `reason` is the fault class
/// (from the vector), recorded for the supervisor's `process_exit_reason`.
pub fn killCurrentProcess(reason: abi.ExitReason) noreturn {
    const t = scheduler.current();
    _ = sync.enter(); // handed off through the exit switch, released by the resumed task
    // A second member faulting while the group already dies: no re-stamp, no
    // second count bump — the first trigger owns the group's story
    // (docs/shared-fate-plan.md). `fault_kill_count` is per faulting GROUP.
    if (scheduler.groupDyingLocked(t.address_space)) terminateCurrentLocked();
    fault_kill_count += 1;
    // orelse fallback unreachable today: a leader outlives its members (see exitGroupCurrent).
    const leader_task = if (t.leader == t.id) t else scheduler.taskByIdLocked(t.leader) orelse t;
    killGroupLocked(leader_task, reason, t);
    unreachable; // killGroupLocked never returns for an in-group trigger
}

/// The bounded record of recent deaths, for `process_exit_reason`: ids are never
/// reused, so a ring keyed by id is enough — a record evicted by wraparound reads
/// as -ESRCH, the same as an id that never lived, which a supervisor treats as
/// "too late to ask". Written under the big kernel lock by the reap.
const exit_record_capacity = 64;
const ExitRecord = struct { id: u32 = 0, supervisor: u32 = 0, reason: abi.ExitReason = .exited, valid: bool = false };
var exit_records: [exit_record_capacity]ExitRecord = .{ExitRecord{}} ** exit_record_capacity;
var exit_record_next: usize = 0;

/// Record a dying task's (id, supervisor, reason) — called by the reap before the
/// exit notification is posted, so a supervisor that hears the notification can
/// always still query the reason. Precondition: the big kernel lock is held.
fn recordExitLocked(t: *scheduler.Task) void {
    exit_records[exit_record_next] = .{ .id = t.id, .supervisor = t.supervisor, .reason = t.exit_reason, .valid = true };
    exit_record_next = (exit_record_next + 1) % exit_record_capacity;
}

/// The group-death moment (scheduler.group_exit_hook): the last member is gone
/// and the address space destroyed. Re-stamp the leader's exit record with the
/// group reason — the record must be present and current when the notification
/// lands, however many member deaths churned the ring in between — then publish
/// the leader's exit: the subscriber broadcast and the supervisor's endpoint
/// notification, whose stashed reference is dropped here, exactly once. Runs
/// under the big kernel lock, at both release sites (docs/shared-fate-plan.md).
fn groupExitLocked(leader: u32, supervisor: u32, reason: abi.ExitReason, exit_endpoint: ?*anyopaque) void {
    restampExitRecordLocked(leader, supervisor, reason);
    for (&exit_subscribers) |*slot| {
        if (slot.*) |subscriber| ipc.notifyLocked(subscriber.endpoint, abi.notify_exit_bit | leader);
    }
    if (exit_endpoint) |raw| {
        const endpoint: *ipc.Endpoint = @ptrCast(@alignCast(raw));
        ipc.notifyLocked(endpoint, abi.notify_exit_bit | leader);
        ipc.dropRef(endpoint);
    }
}

/// scheduler.space_mapping_release_hook: drop one shared-memory MAPPING
/// reference when the space that held it is destroyed (docs/shared-fate-plan.md
/// M3). Lock held by the release path.
fn dropSpaceMappingHook(object: *anyopaque) void {
    ipc.dropSharedMemoryReference(@ptrCast(@alignCast(object)));
}

/// Overwrite dead task `id`'s exit record with the group reason, or re-append it
/// if the group's death burst already evicted it — a supervisor must always be
/// able to read the reason for a notification it just received. Lock held.
fn restampExitRecordLocked(id: u32, supervisor: u32, reason: abi.ExitReason) void {
    for (&exit_records) |*record| {
        if (record.valid and record.id == id) {
            record.reason = reason;
            return;
        }
    }
    exit_records[exit_record_next] = .{ .id = id, .supervisor = supervisor, .reason = reason, .valid = true };
    exit_record_next = (exit_record_next + 1) % exit_record_capacity;
}

/// How dead process `id` ended, for `caller` — the kernel half of the
/// process_exit_reason system call. Returns the ExitReason value, -ESRCH (never
/// lived, still alive, or evicted from the ring), or -EPERM (the caller was not
/// its supervisor — the same authority gate as process_kill).
pub fn exitReasonOf(caller_id: u32, target_id: u32) i64 {
    const flags = sync.enter();
    defer sync.leave(flags);
    for (&exit_records) |*record| {
        if (record.valid and record.id == target_id) {
            if (record.supervisor != caller_id) return -ipc.EPERM;
            return @intFromEnum(record.reason);
        }
    }
    return -ipc.ESRCH;
}

/// The published exit events' subscribers (docs/process-lifecycle.md "Who learns
/// of a death"): stateful services — the VFS's file handles, input's
/// subscriptions — that must release what a dead client held and cannot learn it
/// any other way (a client that simply never calls again looks like silence).
/// Bounded like every kernel table; each entry holds its own endpoint reference.
const exit_subscriber_capacity = 8;
const ExitSubscriber = struct { endpoint: *ipc.Endpoint, owner: u32 };
var exit_subscribers: [exit_subscriber_capacity]?ExitSubscriber = .{null} ** exit_subscriber_capacity;

/// process_subscribe(endpoint): subscribe the **caller's own** endpoint to
/// published exit events. *Which* deaths one may hear of is ungated, like
/// process_enumerate — what is running (and dying) is not a secret between
/// cooperating processes. *Whose mailbox* they land in is not: the endpoint must
/// be the caller's (`ipc.ownedBy`), or any process could aim the firehose at a
/// stranger — filling PID 1's mailbox with exit notices it reads as its own
/// children's, and spending the eight-slot table so the services that need
/// deaths (the VFS's handle sweep) cannot subscribe at all. -EPERM otherwise,
/// -ENOSPC when the table is full.
fn systemProcessSubscribe(state: *architecture.CpuState) void {
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const endpoint = ipc.resolveHandle(t, architecture.systemCallArg(state, 0)) orelse return failErr(state, ipc.EBADF);
    if (!ipc.ownedBy(endpoint, t)) return failErr(state, ipc.EPERM);
    const flags = sync.enter();
    defer sync.leave(flags);
    for (&exit_subscribers) |*slot| {
        if (slot.* == null) {
            endpoint.refcount += 1; // the slot's own reference, dropped on unsubscribe-by-death
            slot.* = .{ .endpoint = endpoint, .owner = t.id };
            return architecture.setSystemCallResult(state, 0);
        }
    }
    failErr(state, ipc.ENOSPC);
}

/// signal_bind(endpoint): nominate where this process's signals arrive — the
/// IRQ-as-IPC pattern a fourth time (docs/process-lifecycle.md). Replacing a
/// binding drops the old reference; signals that pended while unbound are
/// delivered immediately on bind, coalesced into one notification.
///
/// The endpoint must be the caller's own (`ipc.ownedBy`), or `signal_bind`
/// becomes a signal *forgery* primitive: `process_signal` is deliberately loose
/// about the target (a task may always signal itself) because the delivery point
/// was assumed to be the target's own mailbox. Aim it elsewhere and a stranger
/// signalling itself makes the kernel stamp a genuine `terminate` badge into
/// somebody else's queue — which is a shutdown request PID 1 has no way to
/// disbelieve.
fn systemSignalBind(state: *architecture.CpuState) void {
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const endpoint = ipc.resolveHandle(t, architecture.systemCallArg(state, 0)) orelse return failErr(state, ipc.EBADF);
    if (!ipc.ownedBy(endpoint, t)) return failErr(state, ipc.EPERM);
    const flags = sync.enter();
    defer sync.leave(flags);
    if (t.signal_endpoint) |raw| ipc.dropRef(@ptrCast(@alignCast(raw)));
    endpoint.refcount += 1;
    t.signal_endpoint = @ptrCast(endpoint);
    if (t.pending_signals != 0) {
        ipc.notifyLocked(endpoint, abi.notify_signal_bit | t.pending_signals);
        t.pending_signals = 0;
    }
    architecture.setSystemCallResult(state, 0);
}

/// process_signal(id, signal): post a signal — a one-way, coalescing statement,
/// never a question (docs/process-lifecycle.md). The authority gate is the
/// supervision link, like kill; a process may also signal itself. Unbound
/// targets accumulate the signal in their pending mask.
fn systemProcessSignal(state: *architecture.CpuState) void {
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const id = architecture.systemCallArg(state, 0);
    const signal = architecture.systemCallArg(state, 1);
    if (id > std.math.maxInt(u32)) return failErr(state, ipc.ESRCH);
    if (signal > 31) return failErr(state, ipc.EBADF); // not a Signal bit position
    const flags = sync.enter();
    defer sync.leave(flags);
    const member = scheduler.taskByIdLocked(@intCast(id)) orelse return failErr(state, ipc.ESRCH);
    if (member.address_space == 0) return failErr(state, ipc.ESRCH);
    // Signals address the process: any member id resolves to the LEADER, whose
    // endpoint the service harness binds. Authority mirrors process_kill (the
    // leader's supervisor), plus the group may signal itself
    // (docs/shared-fate-plan.md M1).
    const target = if (member.leader == member.id)
        member
    else
        scheduler.taskByIdLocked(member.leader) orelse return failErr(state, ipc.ESRCH);
    if (target.supervisor != t.id and target.id != t.leader) return failErr(state, ipc.EPERM);
    target.pending_signals |= @as(u32, 1) << @intCast(signal);
    if (target.signal_endpoint) |raw| {
        const endpoint: *ipc.Endpoint = @ptrCast(@alignCast(raw));
        ipc.notifyLocked(endpoint, abi.notify_signal_bit | target.pending_signals);
        target.pending_signals = 0;
    }
    architecture.setSystemCallResult(state, 0);
}

/// The one-shot timers of timer_bind: the missing timed wait. A service arms a
/// deadline and keeps serving; the expiry arrives in the same replyWait as
/// everything else (notify_timer_bit). What stop-sequence escalation, hello
/// deadlines, and restart backoff are built from — and later, `alarm`.
const timer_capacity = 16;
const OneShotTimer = struct { deadline: u64, endpoint: *ipc.Endpoint, owner: u32 };
var one_shot_timers: [timer_capacity]?OneShotTimer = .{null} ** timer_capacity;

/// Sweep expired timers — hung on scheduler.timer_tick_hook, so it runs on every
/// tick with the big kernel lock held, like the sleeper wake it rides beside.
fn timerSweepLocked() void {
    const now = architecture.millis();
    for (&one_shot_timers) |*slot| {
        if (slot.*) |timer| {
            if (now >= timer.deadline) {
                ipc.notifyLocked(timer.endpoint, abi.notify_timer_bit);
                ipc.dropRef(timer.endpoint);
                slot.* = null;
            }
        }
    }
}

/// timer_bind(endpoint, ms): arm a one-shot timer on an endpoint of the caller's
/// own (`ipc.ownedBy`; -EPERM otherwise). A timer landing carries no identity —
/// that is the whole reason a service may keep exactly one in flight — so a
/// timer armed on someone else's endpoint is indistinguishable from one they
/// armed themselves, and a loop that re-arms on every landing (init's heartbeat)
/// multiplies: N forged timers leave N+1 self-perpetuating beats. The
/// sixteen-slot table is a shared resource on top of that. -ENOSPC when it is
/// full.
fn systemTimerBind(state: *architecture.CpuState) void {
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const endpoint = ipc.resolveHandle(t, architecture.systemCallArg(state, 0)) orelse return failErr(state, ipc.EBADF);
    if (!ipc.ownedBy(endpoint, t)) return failErr(state, ipc.EPERM);
    const ms = architecture.systemCallArg(state, 1);
    const flags = sync.enter();
    defer sync.leave(flags);
    for (&one_shot_timers) |*slot| {
        if (slot.* == null) {
            endpoint.refcount += 1;
            slot.* = .{ .deadline = architecture.millis() + ms, .endpoint = endpoint, .owner = t.id };
            return architecture.setSystemCallResult(state, 0);
        }
    }
    failErr(state, ipc.ENOSPC);
}

fn systemProcessExitReason(state: *architecture.CpuState) void {
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const id = architecture.systemCallArg(state, 0);
    if (id > std.math.maxInt(u32)) return failErr(state, ipc.ESRCH);
    const r = exitReasonOf(t.id, @intCast(id));
    architecture.setSystemCallResult(state, @bitCast(r));
}

/// Resolve `(device_id, resource_index)` to a GSI this process is entitled to bind, or null.
/// The two checks are the whole security story: the device must be *claimed* by the
/// caller, and the resource must be one of that device's `irq` resources as recorded
/// by discovery. Neither a raw GSI nor an unclaimed device can get through — which
/// is why irq_bind takes a resource index and not an interrupt number.
fn ownedGsi(t: *scheduler.Task, device_id: u64, resource_index: u64) ?u32 {
    const owner = devices_broker.ownerOf(device_id) orelse return null;
    if (owner != t.id) return null;
    const r = devices_broker.resourceOf(device_id, resource_index) orelse return null;
    if (r.kind != @intFromEnum(device_abi.ResourceKind.irq)) return null;
    if (r.start >= irq.maximum_gsi) return null;
    return @intCast(r.start);
}

/// irq_bind(device_id, resource_index, endpoint) -> 0/-1: deliver that device's IRQ to the
/// endpoint as an asynchronous IPC notification. The driver then blocks in
/// IPC_ReplyWait and is woken by the ISR; see system/kernel/irq.zig for the cycle.
/// Two gates, both necessary: the device must be *claimed* by the caller
/// (`ownedGsi`), and the endpoint must be the caller's own (`ipc.ownedBy`) — a
/// claim entitles a driver to its own interrupts, not to post them into a
/// stranger's mailbox.
fn systemIrqBind(state: *architecture.CpuState) void {
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const gsi = ownedGsi(t, architecture.systemCallArg(state, 0), architecture.systemCallArg(state, 1)) orelse
        return fail(state);
    const endpoint = ipc.resolveHandle(t, architecture.systemCallArg(state, 2)) orelse return fail(state);
    if (!ipc.ownedBy(endpoint, t)) return failErr(state, ipc.EPERM);

    const flags = sync.enter();
    defer sync.leave(flags);
    irq.bind(gsi, endpoint, t.id) catch return fail(state);
    architecture.setSystemCallResult(state, 0);
}

/// msi_bind(device_id, endpoint_handle) -> address (rax), data (rdx): set up
/// Message-Signalled Interrupts for a claimed device. The kernel allocates a vector,
/// binds it to `endpoint`, and hands back the (address, data) the driver programs into
/// its own MSI capability (found via its ECAM config space, resource 0). Unlike
/// `irq_bind` there is no GSI, no sharing, and no ack — MSI is edge-triggered. The
/// claim is the capability. See docs/driver-model.md (M15).
fn systemMsiBind(state: *architecture.CpuState) void {
    const device_id = architecture.systemCallArg(state, 0);
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const owner = devices_broker.ownerOf(device_id) orelse return fail(state);
    if (owner != t.id) return fail(state); // not claimed by this process
    const endpoint = ipc.resolveHandle(t, architecture.systemCallArg(state, 1)) orelse return failErr(state, ipc.EBADF);
    if (!ipc.ownedBy(endpoint, t)) return failErr(state, ipc.EPERM); // interrupts land in your own mailbox

    const flags = sync.enter();
    defer sync.leave(flags);
    const vector = irq.msiBind(endpoint, t.id) catch return fail(state);
    // x86 MSI: the address routes to the BSP's LAPIC (physical destination, fixed
    // delivery — dest field 0); the data carries the vector.
    architecture.setSystemCallResult(state, abi.msi_address_base);
    architecture.setSystemCallResult2(state, vector);
}

/// irq_ack(device_id, resource_index) -> 0/-1: re-arm a bound IRQ. The ISR left the line
/// masked (it could not quiet the device — that's this driver's job), so nothing
/// more arrives until the driver says it has serviced the hardware.
fn systemIrqAck(state: *architecture.CpuState) void {
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state);
    const gsi = ownedGsi(t, architecture.systemCallArg(state, 0), architecture.systemCallArg(state, 1)) orelse
        return fail(state);

    const flags = sync.enter();
    defer sync.leave(flags);
    if (irq.ack(gsi)) architecture.setSystemCallResult(state, 0) else fail(state);
}

/// Whether the debug_write stream sits at the start of a line — the last emitted
/// byte was a newline (true at boot: nothing emitted yet). Guarded by the kernel
/// lock in `systemDebugWrite`, like the stream it describes.

/// debug_write(ptr, len): copy bytes from user memory into the kernel log.
/// A bring-up diagnostic — real output goes through the VFS/console later.
///
/// The pointer must lie in the user (low) half, so kernel addresses and
/// non-canonical values fall outside it and the read below can't be steered at
/// kernel data. Length is checked first so the upper-bound add can't overflow.
/// The message then comes in ONCE through the checked copy layer: an unmapped
/// hole in the user half is -EFAULT rather than a kernel fault, and the bytes the
/// log stamps are the same bytes that were validated (the old code read the user
/// buffer twice — once to stage it, once again inside `log.append`).
///
/// The emit runs under the kernel lock, so a message is atomic on the wire — two
/// processes writing from different cores can interleave *messages*, never bytes.
/// The "DANOS-INIT: " marker is emitted only at the start of a line (not per
/// call), so a process may assemble a line from several writes without the marker
/// (or, with the lock, another byte) landing in the middle. Cleanly-terminated
/// lines from concurrent writers stay whole either way.
fn systemDebugWrite(state: *architecture.CpuState) void {
    const ptr = architecture.systemCallArg(state, 0);
    const len = architecture.systemCallArg(state, 1);
    const level_raw = architecture.systemCallArg(state, 2);
    if (len <= write_buffer.len and ptr < user_half_end and ptr + len <= user_half_end) {
        // Levels above the enum range clamp to raw — old two-arg callers land
        // there naturally (garbage in arg 2 stays harmless).
        const level: abi.KlogLevel = if (level_raw <= @intFromEnum(abi.KlogLevel.raw))
            @enumFromInt(level_raw)
        else
            .raw;
        const t = scheduler.current();
        const flags = sync.enter();
        defer sync.leave(flags);
        // One copy in, under the lock; `write_buffer` (the latest message, which
        // the kernel tests assert on) doubles as the staging buffer the log reads.
        if (!user_memory.copyFromUser(t.address_space, ptr, write_buffer[0..len])) return failErr(state, ipc.EFAULT);
        write_len = len;
        write_from_user = architecture.fromUser(state);
        write_count += 1;
        // The kernel stamps the sender's identity — attribution is structural,
        // not a prefix convention the payload could forge (and it is stamped
        // per line inside log.append).
        log.append(t.id, t.name(), level, write_buffer[0..len]);
        architecture.setSystemCallResult(state, len);
    } else {
        fail(state);
    }
}

/// klog_read(offset, ptr, len) -> bytes copied: copy tagged log-ring stream
/// bytes beginning at stream offset `offset` out to the user buffer at `ptr`.
/// Returns the count copied — 0 means caught up — and fails once `offset` has
/// fallen behind the ring's tail (the records were overwritten) or lies past
/// its head; the reader re-syncs via klog_status. A reader parses
/// [KlogRecordHeader][name][message] frames out of the byte stream (abi.zig).
///
/// The mirror of `debug_write`: the same overflow-safe user-half bounds check,
/// but the copy runs kernel -> user. The ring is drained a chunk at a time into a
/// kernel staging buffer (each chunk read under the log lock, so the stream can't
/// move underneath it) and each chunk is then placed with `copyToUser` — a
/// reader may ask for a megabyte, and the kernel stack is 16 KiB. A partial
/// result is honest: the reader advances its cursor by what it got. A read-only
/// diagnostic.
fn systemKlogRead(state: *architecture.CpuState) void {
    const offset = architecture.systemCallArg(state, 0);
    const ptr = architecture.systemCallArg(state, 1);
    const len = architecture.systemCallArg(state, 2);
    const t = scheduler.current();
    // Confine the whole destination span to the user (low) half. `len <=
    // user_half_end - ptr` bounds the length without an overflowing add.
    if (ptr < user_half_end and len <= user_half_end - ptr) {
        var chunk: [512]u8 = undefined;
        var done: u64 = 0;
        while (done < len) {
            const want = @min(@as(u64, chunk.len), len - done);
            const n = log.readAt(offset + done, chunk[0..@intCast(want)]) orelse {
                // The cursor fell behind the ring's tail mid-drain. What was
                // already placed stands; only a first-chunk miss fails the call.
                if (done == 0) return fail(state);
                break;
            };
            if (n == 0) break; // caught up
            if (!user_memory.copyToUser(t.address_space, ptr + done, chunk[0..n])) return failErr(state, ipc.EFAULT);
            done += n;
        }
        architecture.setSystemCallResult(state, done);
    } else {
        fail(state);
    }
}

/// klog_status(ptr) -> 0: copy a KlogStatus — the ring's live cursors plus the
/// boot wall-clock anchor — out to the user buffer at `ptr`. How a log reader
/// finds the oldest retained offset, detects lost records (sequence gaps), and
/// names a per-boot log directory (boot_unix_seconds).
fn systemKlogStatus(state: *architecture.CpuState) void {
    const ptr = architecture.systemCallArg(state, 0);
    const size = @sizeOf(abi.KlogStatus);
    const t = scheduler.current();
    if (ptr < user_half_end and size <= user_half_end - ptr) {
        var status = log.status();
        if (!user_memory.copyValueToUser(t.address_space, ptr, &status)) return failErr(state, ipc.EFAULT);
        architecture.setSystemCallResult(state, 0);
    } else {
        fail(state);
    }
}

/// fs_resolve(path_ptr, path_len, flags, out_ptr, out_cap): route a path
/// through the kernel mount table (docs/vfs-protocol.md). Kernel-served ->
/// rax=fs_route_kernel, rdx=node token. Backend-served -> rax=fs_route_backend,
/// rdx=an endpoint handle in the caller's table (deduplicated), and the
/// rewritten mount-relative path copied into `out` behind a u16 length prefix. Fails for unknown paths, create-intent on /system, or an
/// undersized out buffer.
fn systemFsResolve(state: *architecture.CpuState) void {
    const path_ptr = architecture.systemCallArg(state, 0);
    const path_len = architecture.systemCallArg(state, 1);
    const flags = architecture.systemCallArg(state, 2);
    const out_ptr = architecture.systemCallArg(state, 3);
    const out_cap = architecture.systemCallArg(state, 4);
    if (path_len == 0 or path_len > maximum_resolve_path or path_ptr >= user_half_end or path_ptr + path_len > user_half_end) return fail(state);
    if (out_cap != 0 and !user_memory.userRangeOk(out_ptr, @intCast(out_cap))) return fail(state);
    const t = scheduler.current();
    var path_storage: [maximum_resolve_path]u8 = undefined;
    const path = path_storage[0..@intCast(path_len)];
    if (!user_memory.copyFromUser(t.address_space, path_ptr, path)) return failErr(state, ipc.EFAULT);

    const flags_lock = sync.enter();
    defer sync.leave(flags_lock);
    switch (vfs.resolvePath(path, flags & abi.fs_flag_create != 0)) {
        .kernel_node => |node_token| {
            architecture.setSystemCallResult(state, abi.fs_route_kernel);
            architecture.setSystemCallResult2(state, node_token);
        },
        .backend => |*backend| {
            // The rewritten path goes back in the out buffer behind a u16
            // length prefix (a third result register would collide with r8's
            // argument role in the userspace stub). Both halves are placed with
            // the checked copy, and *before* the handle is installed, so an
            // -EFAULT never strands a capability in the caller's table.
            if (backend.path_len + 2 > out_cap) return fail(state);
            const prefix = [2]u8{ @intCast(backend.path_len & 0xFF), @intCast(backend.path_len >> 8) };
            if (!user_memory.copyToUser(t.address_space, out_ptr, &prefix)) return failErr(state, ipc.EFAULT);
            if (!user_memory.copyToUser(t.address_space, out_ptr + 2, backend.path[0..backend.path_len])) return failErr(state, ipc.EFAULT);
            const handle = ipc.installHandleDeduped(t, backend.endpoint);
            if (handle < 0) return fail(state);
            architecture.setSystemCallResult(state, abi.fs_route_backend);
            architecture.setSystemCallResult2(state, @intCast(handle));
        },
        .not_found => fail(state),
    }
}

/// fs_node(op, node_token, offset, buf_ptr, buf_len) -> bytes/0/-errno: serve a
/// kernel-backed node. read copies file bytes; status copies a FileAttributes;
/// readdir copies [DirectoryEntryHeader][name] for the `offset`th child. Reads
/// of the immutable initrd never take the kernel lock.
///
/// Every result reaches the caller through `copyToUser`, never a store through
/// the user pointer. `read` stages the file bytes a chunk at a time — a caller
/// may ask for the 64 KiB ceiling, which no kernel stack could hold — so a
/// mid-way -EFAULT is possible; the call fails and the already-placed prefix is
/// meaningless, exactly as a failed read should be.
fn systemFsNode(state: *architecture.CpuState) void {
    const operation = architecture.systemCallArg(state, 0);
    const node_token = architecture.systemCallArg(state, 1);
    const offset = architecture.systemCallArg(state, 2);
    const buf_ptr = architecture.systemCallArg(state, 3);
    const buf_len = architecture.systemCallArg(state, 4);
    if (buf_ptr >= user_half_end or buf_len > user_half_end - buf_ptr) return fail(state);
    const capped = @min(buf_len, 64 * 1024); // bound any single copy
    const t = scheduler.current();
    switch (operation) {
        abi.fs_node_read => {
            var chunk: [512]u8 = undefined;
            var done: u64 = 0;
            while (done < capped) {
                const want = @min(@as(u64, chunk.len), capped - done);
                const n = vfs.nodeRead(node_token, offset + done, chunk[0..@intCast(want)]) orelse return fail(state);
                if (n == 0) break; // end of file
                if (!user_memory.copyToUser(t.address_space, buf_ptr + done, chunk[0..n])) return failErr(state, ipc.EFAULT);
                done += n;
            }
            architecture.setSystemCallResult(state, done);
        },
        abi.fs_node_status => {
            var attributes = vfs.nodeStatus(node_token) orelse return fail(state);
            if (capped < @sizeOf(abi.FileAttributes)) return fail(state);
            if (!user_memory.copyValueToUser(t.address_space, buf_ptr, &attributes)) return failErr(state, ipc.EFAULT);
            architecture.setSystemCallResult(state, @sizeOf(abi.FileAttributes));
        },
        abi.fs_node_readdir => {
            const header_size = @sizeOf(abi.DirectoryEntryHeader);
            if (capped < header_size) return fail(state);
            var name_buffer: [64]u8 = undefined;
            const result = vfs.nodeReaddir(node_token, offset, &name_buffer) orelse {
                architecture.setSystemCallResult(state, 0); // past the end
                return;
            };
            // Header and name are staged contiguously so one entry is one copy.
            var entry: [@sizeOf(abi.DirectoryEntryHeader) + name_buffer.len]u8 = undefined;
            const header = result.header;
            const total = header_size + @min(result.name_len, capped - header_size);
            @memcpy(entry[0..header_size], std.mem.asBytes(&header));
            @memcpy(entry[header_size..total], name_buffer[0 .. total - header_size]);
            if (!user_memory.copyToUser(t.address_space, buf_ptr, entry[0..total])) return failErr(state, ipc.EFAULT);
            architecture.setSystemCallResult(state, total);
        },
        else => fail(state),
    }
}

/// fs_mount(prefix_ptr, prefix_len, backend_handle, rewrite_ptr, rewrite_len):
/// mount a userspace filesystem at an absolute prefix. Possession of the
/// backend endpoint handle is the capability — the same trust as the old
/// router's cap-passing mount. The mount takes its own endpoint reference.
fn systemFsMount(state: *architecture.CpuState) void {
    const prefix_ptr = architecture.systemCallArg(state, 0);
    const prefix_len = architecture.systemCallArg(state, 1);
    const backend_handle = architecture.systemCallArg(state, 2);
    const rewrite_ptr = architecture.systemCallArg(state, 3);
    const rewrite_len = architecture.systemCallArg(state, 4);
    if (prefix_len == 0 or prefix_len > maximum_mount_prefix or prefix_ptr >= user_half_end or prefix_ptr + prefix_len > user_half_end) return fail(state);
    if (rewrite_len > maximum_mount_rewrite) return fail(state);
    if (rewrite_len != 0 and (rewrite_ptr >= user_half_end or rewrite_ptr + rewrite_len > user_half_end)) return fail(state);
    const t = scheduler.current();
    // Both strings come in through the checked copy; `mountBackend` copies them
    // again into the mount table, so these staging buffers only need to outlive
    // this call.
    var prefix_storage: [maximum_mount_prefix]u8 = undefined;
    const prefix = prefix_storage[0..@intCast(prefix_len)];
    if (!user_memory.copyFromUser(t.address_space, prefix_ptr, prefix)) return failErr(state, ipc.EFAULT);
    var rewrite_storage: [maximum_mount_rewrite]u8 = undefined;
    const rewrite = rewrite_storage[0..@intCast(rewrite_len)];
    if (rewrite_len != 0 and !user_memory.copyFromUser(t.address_space, rewrite_ptr, rewrite)) return failErr(state, ipc.EFAULT);

    const flags = sync.enter();
    defer sync.leave(flags);
    const endpoint = ipc.resolveHandle(t, backend_handle) orelse return failErr(state, ipc.EBADF);
    endpoint.refcount += 1; // the mount table's reference
    if (!vfs.mountBackend(prefix, endpoint, rewrite)) {
        ipc.dropRef(endpoint);
        return fail(state);
    }
    architecture.setSystemCallResult(state, 0);
}

/// fs_unmount(prefix_ptr, prefix_len): remove a backend mount.
fn systemFsUnmount(state: *architecture.CpuState) void {
    const prefix_ptr = architecture.systemCallArg(state, 0);
    const prefix_len = architecture.systemCallArg(state, 1);
    if (prefix_len == 0 or prefix_len > maximum_mount_prefix or prefix_ptr >= user_half_end or prefix_ptr + prefix_len > user_half_end) return fail(state);
    const t = scheduler.current();
    var prefix_storage: [maximum_mount_prefix]u8 = undefined;
    const prefix = prefix_storage[0..@intCast(prefix_len)];
    if (!user_memory.copyFromUser(t.address_space, prefix_ptr, prefix)) return failErr(state, ipc.EFAULT);
    const flags = sync.enter();
    defer sync.leave(flags);
    if (!vfs.unmount(prefix)) return fail(state);
    architecture.setSystemCallResult(state, 0);
}

/// mmap(len, prot) -> base: grant `len` bytes (rounded up to whole pages) of
/// fresh, zeroed, writable+NX memory in the caller's mmap arena, and return the
/// base virtual address. `prot` is accepted but not yet honoured (grants are
/// always RW+NX; W^X for user code stays with the ELF loader). Failure returns
/// -1. The user-space allocator (lib `runtime`) carves these pages into malloc blocks.
fn systemMmap(state: *architecture.CpuState) void {
    const len = architecture.systemCallArg(state, 0);
    const t = scheduler.current();
    if (t.address_space == 0) return fail(state); // not a user process — nothing to map into
    const pages = (len + page_size - 1) / page_size;
    if (pages == 0 or pages > maximum_mmap_pages) return fail(state);

    // Reserve a disjoint range under a *brief* lock (the cursor is shared by every thread
    // in this address space). The mapping below then takes the lock **per page**, not for
    // the whole grant: the big lock is held with interrupts disabled, so pinning it across
    // a multi-MiB memset+map would freeze every other core on its next tick — which timed
    // the `affinity` scenario out (docs/threading-plan.md M7).
    const base = reserve: {
        const flags = sync.enter();
        defer sync.leave(flags);
        const cursor = scheduler.addressSpaceMmapNextPtr(t.address_space) orelse return fail(state);
        if (cursor.* == 0) cursor.* = heap_arena_base; // seed the arena lazily
        const b = cursor.*;
        if (b + pages * page_size > heap_arena_end) return fail(state); // arena exhausted
        cursor.* = b + pages * page_size; // reserve now, so concurrent grants can't overlap
        break :reserve b;
    };

    // Map the reserved range page by page, each page under a short-held lock (the range is
    // already reserved, so pages can't overlap another thread's; the lock only serializes
    // the shared page-table walk). On mid-way frame exhaustion, roll back the mapped pages
    // so no partial grant leaks — the reserved-but-unmapped tail of the arena is left
    // fallow (a rare, bounded address-space leak, not a memory leak).
    var mapped: usize = 0;
    while (mapped < pages) : (mapped += 1) {
        const flags = sync.enter();
        const frame = pmm.alloc() orelse {
            var i: usize = 0;
            while (i < mapped) : (i += 1) {
                const va = base + i * page_size;
                if (architecture.translate(t.address_space, va)) |physical| {
                    architecture.unmapUserPageInto(t.address_space, va);
                    pmm.free(physical);
                }
            }
            sync.leave(flags);
            return fail(state);
        };
        const destination: [*]u8 = @ptrFromInt(boot_handoff.physicalToVirtual(frame));
        @memset(destination[0..page_size], 0); // hand out zeroed memory
        architecture.mapUserPageInto(t.address_space, base + mapped * page_size, frame, true, false); // RW + NX
        sync.leave(flags);
    }
    architecture.setSystemCallResult(state, base); // the cursor was already advanced at reserve
}

/// munmap(base, len): release a range previously handed out by `mmap`. Unmaps
/// each page and frees its frame. The arena is a bump allocator, so the virtual
/// range is not recycled (the user-space allocator reuses freed *blocks* itself);
/// this just returns the physical frames to the kernel. Returns 0, or -1 if the
/// range is not page-aligned or lies outside the arena.
fn systemMunmap(state: *architecture.CpuState) void {
    const base = architecture.systemCallArg(state, 0);
    const len = architecture.systemCallArg(state, 1);
    const t = scheduler.current();
    if (t.address_space == 0 or base % page_size != 0) return fail(state);
    const pages = (len + page_size - 1) / page_size;
    if (base < heap_arena_base or base + pages * page_size > heap_arena_end) return fail(state);

    for (0..pages) |i| {
        const va = base + i * page_size;
        if (architecture.translate(t.address_space, va)) |physical| {
            architecture.unmapUserPageInto(t.address_space, va);
            pmm.free(physical);
        }
    }
    architecture.setSystemCallResult(state, 0);
}

/// Reset the recorded system_call evidence before a user-mode run.
fn resetRecords() void {
    write_len = 0;
    write_from_user = false;
    write_count = 0;
    exit_code = 0;
}

pub const RunError = error{ ProgramTooBig, OutOfMemory };

/// Map `blob` at code_virtual with a fresh user stack, drop to ring 3, and return
/// once the program exits via system_call 0. See the migration caveat in the module
/// doc. A program that faults instead never returns (on_fault halts the core).
pub fn run(blob: []const u8) RunError!void {
    if (blob.len > page_size) return error.ProgramTooBig;
    const code_frame = pmm.alloc() orelse return error.OutOfMemory;
    const stack_frame = pmm.alloc() orelse {
        pmm.free(code_frame);
        return error.OutOfMemory;
    };

    // Fill the code frame through the physmap (supervisor RW): the user-facing
    // mapping is read-only, and this also sidesteps CR0.WP/SMAP. The tail is
    // padded with int3 so a stray jump traps instead of sliding.
    const code: [*]u8 = @ptrFromInt(boot_handoff.physicalToVirtual(code_frame));
    @memcpy(code[0..blob.len], blob);
    @memset(code[blob.len..page_size], 0xCC);

    architecture.mapUserPage(code_virtual, code_frame, false, true); // RO + X
    architecture.mapUserPage(stack_base_virtual, stack_frame, true, false); // RW + NX (one page; the probe barely stacks)
    resetRecords();

    architecture.enterUser(scheduler.currentCpuIndex(), code_virtual, stack_base_virtual + page_size);

    // Back via the exit system_call; the interrupt gate left IF clear.
    architecture.enableInterrupts();
    architecture.unmapPage(code_virtual);
    architecture.unmapPage(stack_base_virtual);
    pmm.free(code_frame);
    pmm.free(stack_frame);
}

// --- user ELF loading (/system/services/init) ------------------------------------------

pub const InitError = error{
    BadElf, // malformed/inapplicable image (magic, class, machine, type, bounds)
    BadSegment, // PT_LOAD unaligned, out of the user region, W&X, or overlapping
    BadEntry, // e_entry not inside an executable segment
    BadArguments, // no argv[0], too many entries, or too many bytes for the entry stack
    ProgramTooBig, // more pages than the loader's budget
    OutOfMemory,
};

const maximum_segments = 16;
const maximum_pages = 256; // 1 MiB loader budget; the user region caps at 2 MiB anyway

const Segment = struct {
    virtual_address: u64,
    memsz: u64,
    filesz: u64,
    off: u64,
    writable: bool,
    executable: bool,

    fn pages(self: Segment) u64 {
        return (self.memsz + page_size - 1) / page_size;
    }
};

/// Parse and validate every PT_LOAD before touching memory. Bounds are checked
/// against the image and the user region; segments must be page-aligned,
/// non-overlapping, and W^X (R-only is fine — linkers may emit a headers-only
/// segment, mapped RO+NX).
fn parseSegments(image: []const u8, segs: *[maximum_segments]Segment) InitError!struct { count: usize, entry: u64 } {
    if (image.len < @sizeOf(elf.Elf64_Ehdr)) return error.BadElf;
    const ehdr = std.mem.bytesToValue(elf.Elf64_Ehdr, image[0..@sizeOf(elf.Elf64_Ehdr)]);
    if (!std.mem.eql(u8, image[0..4], "\x7fELF")) return error.BadElf;
    if (image[elf.EI_CLASS] != elf.ELFCLASS64) return error.BadElf;
    if (ehdr.e_machine != .X86_64) return error.BadElf;
    if (ehdr.e_type != .EXEC) return error.BadElf; // a PIE would need relocation
    if (ehdr.e_phentsize < @sizeOf(elf.Elf64_Phdr)) return error.BadElf;
    if (ehdr.e_phnum > maximum_segments) return error.BadElf;
    const ph_bytes = @as(u64, ehdr.e_phnum) * ehdr.e_phentsize;
    if (ehdr.e_phoff > image.len or ph_bytes > image.len - ehdr.e_phoff) return error.BadElf;

    var count: usize = 0;
    var total_pages: u64 = 0;
    for (0..ehdr.e_phnum) |i| {
        const off = ehdr.e_phoff + i * ehdr.e_phentsize;
        const phdr = std.mem.bytesToValue(elf.Elf64_Phdr, image[off..][0..@sizeOf(elf.Elf64_Phdr)]);
        if (phdr.p_type != elf.PT_LOAD) continue;
        if (phdr.p_memsz == 0) continue;

        if (phdr.p_vaddr % page_size != 0) return error.BadSegment;
        if (phdr.p_filesz > phdr.p_memsz) return error.BadSegment;
        if (phdr.p_offset > image.len or phdr.p_filesz > image.len - phdr.p_offset) return error.BadSegment;
        // Inside the user image region, strictly below the stack page.
        if (phdr.p_vaddr < code_virtual) return error.BadSegment;
        if (phdr.p_memsz > stack_virtual - phdr.p_vaddr) return error.BadSegment;

        const w = phdr.p_flags & elf.PF_W != 0;
        const x = phdr.p_flags & elf.PF_X != 0;
        if (w and x) return error.BadSegment; // W^X, even for init

        const seg = Segment{
            .virtual_address = phdr.p_vaddr,
            .memsz = phdr.p_memsz,
            .filesz = phdr.p_filesz,
            .off = phdr.p_offset,
            .writable = w,
            .executable = x,
        };
        // No overlap with any earlier segment (page-granular, since mapping is).
        for (segs[0..count]) |other| {
            const a_end = seg.virtual_address + seg.pages() * page_size;
            const b_end = other.virtual_address + other.pages() * page_size;
            if (seg.virtual_address < b_end and other.virtual_address < a_end) return error.BadSegment;
        }
        total_pages += seg.pages();
        if (total_pages > maximum_pages) return error.ProgramTooBig;
        segs[count] = seg;
        count += 1;
    }
    if (count == 0) return error.BadElf;

    // The entry point must land inside an executable segment.
    for (segs[0..count]) |seg| {
        if (seg.executable and ehdr.e_entry >= seg.virtual_address and ehdr.e_entry < seg.virtual_address + seg.memsz)
            return .{ .count = count, .entry = ehdr.e_entry };
    }
    return error.BadEntry;
}

/// Load one page of a segment into address space `address_space`: a fresh frame, zeroed
/// and filled through the physmap, mapped user-accessible with the segment's W^X.
/// On a later failure the whole address space is torn down, which frees every
/// frame mapped into it — so no per-page rollback list is needed here.
fn loadPageInto(address_space: u64, image: []const u8, seg: Segment, page_index: u64) InitError!void {
    const frame = pmm.alloc() orelse return error.OutOfMemory;
    const destination: [*]u8 = @ptrFromInt(boot_handoff.physicalToVirtual(frame));
    @memset(destination[0..page_size], 0);
    const page_off = page_index * page_size;
    if (page_off < seg.filesz) {
        const n = @min(page_size, seg.filesz - page_off);
        @memcpy(destination[0..n], image[seg.off + page_off ..][0..n]);
    }
    architecture.mapUserPageInto(address_space, seg.virtual_address + page_off, frame, seg.writable, seg.executable);
}

/// Build the System V AMD64 process-entry block at the top of a process's stack
/// page and return the initial user stack pointer. At the first user instruction,
/// rsp is 16-byte aligned and points at (addresses growing upward):
///
///   argc, argv[0..argc-1], NULL, NULL (empty envp), auxiliary vector, strings
///
/// — the layout every C runtime's startup code walks, so danos's own runtime and a
/// future libc port read arguments identically (docs/sysv.md). `page` is the kernel
/// (physmap) view of the stack's **top** frame and `page_user_base` that frame's
/// user address (stack_top_virtual - page_size); the pointers written into it are
/// user addresses inside that page. The caller has validated the sizes
/// (`entryStackBytes`), so this cannot overrun.
fn buildEntryStack(page: [*]u8, page_user_base: u64, argv: []const []const u8) u64 {
    // The strings live at the very top of the page, packed from the end downward.
    var string_offset: usize = page_size;
    var pointers: [maximum_arguments]u64 = undefined;
    var i: usize = argv.len;
    while (i > 0) {
        i -= 1;
        string_offset -= argv[i].len + 1;
        @memcpy(page[string_offset..][0..argv[i].len], argv[i]);
        page[string_offset + argv[i].len] = 0; // NUL-terminated, as C expects
        pointers[i] = page_user_base + string_offset;
    }

    // The vector sits below the strings: argc, the argv pointers, the argv
    // terminator, an empty envp (terminator only), then the auxiliary vector.
    const word_count = 1 + argv.len + 1 + 1 + 4;
    const vector_offset = (string_offset - word_count * 8) & ~@as(usize, 15); // entry rsp % 16 == 0
    const words: [*]u64 = @ptrCast(@alignCast(page + vector_offset));
    var w: usize = 0;
    words[w] = argv.len; // argc
    w += 1;
    for (pointers[0..argv.len]) |pointer| {
        words[w] = pointer;
        w += 1;
    }
    words[w] = 0; // argv terminator
    words[w + 1] = 0; // envp: no environment yet, just the terminator
    words[w + 2] = auxiliary_vector_page_size;
    words[w + 3] = page_size;
    words[w + 4] = auxiliary_vector_null; // end of the auxiliary vector
    words[w + 5] = 0;
    return page_user_base + vector_offset;
}

/// Bytes the entry block for `argv` occupies at the top of the stack page:
/// strings (each NUL-terminated), vector words, and the alignment slack.
fn entryStackBytes(argv: []const []const u8) usize {
    var string_bytes: usize = 0;
    for (argv) |argument| string_bytes += argument.len + 1;
    return string_bytes + (1 + argv.len + 1 + 1 + 4) * 8 + 16;
}

/// Load a user ELF image into a fresh address space and spawn it as a scheduled
/// ring-3 process at `priority`, entered with `argv` on its stack per the System V
/// convention (`buildEntryStack`). `argv[0]` is required — it names the process:
/// the path or initial-ramdisk name it was spawned as. It is also recorded on the
/// task, so a fault report can say *which* binary died, not just its id.
/// The kernel-internal spawn (init at boot, tests): supervisor 0, no exit
/// notification. `spawnProcessSupervised` is the full form.
pub fn spawnProcess(image: []const u8, priority: u3, argv: []const []const u8) InitError!void {
    _ = try spawnProcessSupervised(image, priority, argv, 0, null);
}

/// `spawnProcess`, recording `supervisor` (the id of the process that asked — the
/// kill authority) and, if given, `exit_endpoint` to notify when the child ends
/// (a reference is taken here and dropped when the notification posts).
/// Returns the child's process id.
/// Returns immediately — the process runs preemptively on its own page tables
/// alongside everything else, and its exit is handled by the system_call layer.
/// The whole build (address space + ELF load + task) runs under the kernel lock so
/// it appears atomically and can't race pmm/heap on another core.
pub fn spawnProcessSupervised(image: []const u8, priority: u3, argv: []const []const u8, supervisor: u32, exit_endpoint: ?*ipc.Endpoint) InitError!u32 {
    if (argv.len == 0 or argv.len > maximum_arguments) return error.BadArguments;
    // The entry block must leave most of the page as actual stack.
    if (entryStackBytes(argv) > page_size / 2) return error.BadArguments;

    var segs: [maximum_segments]Segment = undefined;
    const parsed = try parseSegments(image, &segs);

    const flags = sync.enter();
    defer sync.leave(flags);

    const address_space = architecture.createAddressSpace() orelse return error.OutOfMemory;
    errdefer architecture.destroyAddressSpace(address_space);

    for (segs[0..parsed.count]) |seg| {
        for (0..seg.pages()) |i| try loadPageInto(address_space, image, seg, i);
    }

    // The stack: `user_stack_pages` zeroed pages below stack_top_virtual, RW + NX.
    // The page below them (`stack_virtual`) stays unmapped as the overflow guard.
    // The entry block goes at the top of the highest page.
    var user_sp: u64 = 0;
    for (0..parameters.user_stack_pages) |i| {
        const stack_frame = pmm.alloc() orelse return error.OutOfMemory;
        const stack_page: [*]u8 = @ptrFromInt(boot_handoff.physicalToVirtual(stack_frame));
        @memset(stack_page[0..page_size], 0); // no stale frame contents leak into user space
        const page_virtual = stack_base_virtual + i * page_size;
        if (i == parameters.user_stack_pages - 1)
            user_sp = buildEntryStack(stack_page, page_virtual, argv);
        architecture.mapUserPageInto(address_space, page_virtual, stack_frame, true, false); // RW + NX
    }

    const child = scheduler.spawnUserLocked(address_space, parsed.entry, user_sp, 0, priority, argv[0], supervisor, if (exit_endpoint) |endpoint| @ptrCast(endpoint) else null, 0) orelse
        return error.OutOfMemory;
    // The child holds a reference to its exit endpoint from birth to death. Taken
    // only now, after nothing can fail; the lock is still held, so the child
    // cannot run (let alone die) before the reference exists.
    if (exit_endpoint) |endpoint| endpoint.refcount += 1;
    return child;
}

/// clock() -> nanoseconds since boot: a monotonic time source. The kernel already owns
/// the scheduling timer and computes this for preemption, so surfacing it is pure
/// mechanism — no policy (wall-clock time, calendars, timezones are a user-space
/// service layered on top). It lets a driver bound a poll loop by real time instead of
/// a spin count, and time short delays.
fn systemClock(state: *architecture.CpuState) void {
    architecture.setSystemCallResult(state, architecture.nanos());
}

/// wall_clock() -> Unix epoch seconds (UTC). The RTC value, read at boot and offset
/// by the monotonic clock (wall-clock.zig) — mechanism, not policy: calendars and
/// timezones layer on top in user space. Needed for filesystem timestamps (mtime).
fn systemWallClock(state: *architecture.CpuState) void {
    architecture.setSystemCallResult(state, wall_clock.nowSeconds());
}