danos/system/kernel/ipc-synchronous.zig

//! Synchronous IPC: the microkernel message backbone. An `Endpoint` is a
//! rendezvous point; a client `call`s it (send a message, block for a reply) and
//! a server `replyWait`s on it (reply to the last client, then block for the next
//! request). This is the substrate the user-space VFS server and device drivers
//! are reached through — `open`/`read`/`write` become user-space wrappers that
//! marshal a request into a `call`.
//!
//! Design (see docs/syscall.md, the plan):
//! - **Copy method, no bounce buffer.** Payloads are copied frame-to-frame
//!   through the physmap (`copyAcross`), which is mapped in every address space's
//!   shared kernel half — so the kernel reads/writes either process's user memory
//!   without a CR3 switch, and an unmapped page fails the copy instead of #PF-ing.
//! - **Reply routing on the server.** IPC is synchronous, so a server owes a reply
//!   to exactly one client at a time; that caller is held in `Task.ipc_client`.
//! - **Sender FIFO on the endpoint.** A blocked caller must be *received without
//!   becoming runnable*, which a WaitQueue can't express, so callers queue on the
//!   endpoint's own FIFO (threaded through the otherwise-idle `Task.next`); servers
//!   waiting for work use a normal WaitQueue.
//!
//! Trust model: every side of a copy that names a *user* address space goes
//! through system/kernel/user-memory.zig — user-half bound, page presence, and
//! the leaf permissions ring 3 itself would face (U/S to read, U/S + R/W to
//! write). A kernel-side buffer is trusted and translated as-is. An unmapped or
//! wrongly-permissioned page fails the operation; it never faults ring 0.

const std = @import("std");
const boot_handoff = @import("boot-handoff");
const abi = @import("abi");
const scheduler = @import("scheduler.zig");
const sync = @import("sync.zig");
const heap = @import("heap.zig");
const pmm = @import("pmm.zig");
const user_memory = @import("user-memory.zig");

const page_size = abi.page_size;
const Task = scheduler.Task;

/// Largest message a single call/reply may carry. Bumping it is trivial; kept
/// small because the copy runs under the big kernel lock.
pub const MESSAGE_MAXIMUM: usize = 256;

pub const maximum_handles = scheduler.ipc_maximum_handles;
// The name registry is indexed directly by ServiceId, so this must exceed the
// largest id (currently fat = 8). Sized with headroom for new services.
pub const maximum_services = 16;

/// Errno-style failures, returned as `-value` in the system_call result register.
pub const EBADF: i64 = 1; // bad handle
pub const E2BIG: i64 = 2; // message exceeds MESSAGE_MAXIMUM
pub const EFAULT: i64 = 3; // buffer unmapped / out of the user half
pub const ENOENT: i64 = 4; // no such registered service
pub const ENOSPC: i64 = 5; // handle table or registry full
pub const ENOMEM: i64 = 6; // out of memory
pub const EPEER: i64 = 7; // peer died before replying (its process exited or was killed)
pub const ESRCH: i64 = 8; // no such process (process_kill of an unknown/dead id)
pub const EPERM: i64 = 9; // not permitted (process_kill by anyone but the supervisor)

/// A badge with this bit set is an asynchronous notification (e.g. an IRQ), not a
/// message from a client — there is no reply owed. The low bits carry the source
/// (a GSI for IRQs). Posted by `notifyFromIsr`, from the ISR in system/kernel/irq.zig;
/// the message path uses a plain task-id badge with this bit clear. Defined in the
/// shared kernel↔user ABI (system/abi.zig), because ring 3 has to test the same bit.
pub const notify_badge_bit: u64 = abi.notify_badge_bit;

/// Set (with `notify_badge_bit`) when a `replyWait` wake carries a buffered payload
/// posted by `send` (`ipc_send`), rather than a bare IRQ/exit notification. Shared with
/// ring 3 through the ABI so the receiver can tell "a message arrived" from "the hardware
/// spoke".
pub const notify_message_bit: u64 = abi.notify_message_bit;

/// Largest payload a single `send` (`ipc_send`) may post. Kept small — the payload rides
/// inline in every `Endpoint`, and the async path is for events (a `KeyEvent` is 16
/// bytes), not bulk transfer, which is what `call` and future shared pages are for.
pub const POST_MAXIMUM: usize = 64;

/// Depth of an endpoint's async payload ring. Absorbs a burst while a receiver is briefly
/// busy; a full ring drops the *oldest* message (see `send`).
const post_capacity: usize = 16;

/// One buffered message: a length-prefixed payload plus the sender's task id (delivered
/// in the low bits of the receiver's badge).
const PostSlot = struct {
    length: u16 = 0,
    sender_id: u64 = 0,
    bytes: [POST_MAXIMUM]u8 = undefined,
};

/// End of the user (low) canonical half — user buffers must lie below it. One
/// definition, in the module that owns the user-memory contract.
const user_half_end: u64 = user_memory.user_half_end;

/// A rendezvous endpoint. Allocated from the kernel heap; referenced by handle
/// (per process) and/or by a registry slot, counted by `refcount`.
pub const Endpoint = struct {
    refcount: u32 = 1,
    // The task that created it. When that task dies, the endpoint is marked `dead` so a caller
    // gets -EPEER instead of blocking forever on a service that will never reply again (V6).
    owner: u32 = 0,
    dead: bool = false,
    // Callers blocked in `call`, awaiting receive, in FIFO order (threaded via
    // Task.next; each such task is .blocked and in no scheduler queue).
    sender_head: ?*Task = null,
    sender_tail: ?*Task = null,
    // Servers blocked in `replyWait` awaiting a request.
    receive_wait_queue: scheduler.WaitQueue = .{},
    // Pending asynchronous notifications (badges), a small coalescing ring.
    notify_buffer: [8]u64 = undefined,
    notify_head: u8 = 0,
    notify_tail: u8 = 0,
    // Pending buffered messages (payloads posted by `send`), a small FIFO ring. Unlike
    // notifications — which are a level and coalesce — these are discrete messages, so a
    // full ring drops the oldest rather than merging.
    post_buffer: [post_capacity]PostSlot = undefined,
    post_head: u16 = 0,
    post_tail: u16 = 0,
};

pub fn createIpcEndpoint() ?*Endpoint {
    const endpoint = heap.allocator().create(Endpoint) catch return null;
    endpoint.* = .{ .owner = scheduler.currentId() };
    return endpoint;
}

/// A task is dying: kill the endpoints it registered as services. Mark each `dead` (so a later
/// `call` returns -EPEER rather than blocking on a reply that will never come), wake anyone
/// already parked sending to it with that error, and vacate its registry slot. Only *registered*
/// endpoints are reachable from here; unregistered ones drop with the task's handle table. The
/// caller holds the big kernel lock (this runs on the death path). See docs/display-v2.md (V6).
pub fn killOwnedEndpointsLocked(task_id: u32) void {
    for (&registry) |*slot| {
        const endpoint = slot.* orelse continue;
        if (endpoint.owner != task_id) continue;
        endpoint.dead = true;
        while (dequeueSender(endpoint)) |sender| {
            sender.ipc_status = -EPEER;
            sender.ipc_received_cap = abi.no_cap;
            scheduler.readyLocked(sender);
        }
        slot.* = null;
        dropRef(endpoint);
    }
}

/// Drop a reference; free the endpoint when the last one goes. (Frames are leaked
/// today like other kernel objects — but the refcount bookkeeping lands now.)
pub fn dropRef(endpoint: *Endpoint) void {
    if (endpoint.refcount > 1) {
        endpoint.refcount -= 1;
    } else {
        heap.allocator().destroy(endpoint);
    }
}

// --- capability objects: what a handle-table entry can name ------------------

/// The `kind` tag on a `scheduler.HandleObject` — which capability object a handle names.
/// Defined here (not in scheduler) because the meaning is the IPC/capability layer's.
pub const handle_kind_endpoint: u8 = 0;
pub const handle_kind_shared_memory: u8 = 1;
pub const handle_kind_dma_region: u8 = 2;

/// A DMA buffer's delegation token: names the `pages` contiguous frames at `phys` that
/// `dma_alloc` handed its `owner`, and can be passed across processes as a capability so
/// the driver that owns a device can bind it into that device's IOMMU domain
/// (`dma_bind`). Unlike `SharedMemoryObject` this does NOT own the frames — the
/// allocating address space still does, and frees them on `dma_free` or teardown — so
/// this is a pure token: `dead` is set when the allocator frees the region, after which
/// a stale handle can no longer bind it. `refcount` counts the allocator's registry
/// entry plus every outstanding handle; the object is freed when the last drops.
pub const DmaRegionObject = struct {
    refcount: u32 = 1,
    phys: u64,
    pages: usize,
    owner: u32,
    dead: bool = false,
};

/// Create a DMA-region token for `pages` frames at `phys` owned by task `owner`. The
/// frames are already allocated and mapped by the caller; this only wraps them for
/// delegation. null if the heap is out of room.
pub fn createDmaRegion(phys: u64, pages: usize, owner: u32) ?*DmaRegionObject {
    const region = heap.allocator().create(DmaRegionObject) catch return null;
    region.* = .{ .phys = phys, .pages = pages, .owner = owner };
    return region;
}

/// Drop a DMA-region reference; free the token when the last (registry + handles) goes.
/// Never frees frames — the allocator owns those.
pub fn dropDmaRegionReference(region: *DmaRegionObject) void {
    if (region.refcount > 1) {
        region.refcount -= 1;
    } else {
        heap.allocator().destroy(region);
    }
}

/// Resolve a handle to its DMA-region token, or null if out of range, unused, or a
/// different kind.
pub fn resolveDmaRegion(t: *Task, h: u64) ?*DmaRegionObject {
    if (h >= t.handles.len) return null;
    const entry = t.handles[@intCast(h)] orelse return null;
    if (entry.kind != handle_kind_dma_region) return null;
    return @ptrCast(@alignCast(entry.ptr));
}

/// Install a DMA-region handle in task `t`'s table (the slot owns a reference).
pub fn installDmaRegionHandle(t: *Task, region: *DmaRegionObject) i64 {
    return installEntry(t, .{ .kind = handle_kind_dma_region, .ptr = @ptrCast(region) });
}

/// Drop the handle at slot `h` of task `t` (handle_close): release its reference and
/// free the slot. Returns 0 or -EBADF.
pub fn closeHandle(t: *Task, h: u64) i64 {
    if (h >= t.handles.len) return -EBADF;
    const entry = t.handles[@intCast(h)] orelse return -EBADF;
    dropEntry(entry);
    t.handles[@intCast(h)] = null;
    return 0;
}

/// A page-aligned block of **shared cacheable RAM** (docs/display-v2.md), referenced by
/// capability handles across processes and freed when the last one drops. `phys` is its
/// contiguous physical base, `pages` its length. A sharer's address-space teardown never
/// reclaims these frames (the mapping carries `device_grant`); this object owns them.
pub const SharedMemoryObject = struct {
    refcount: u32 = 1,
    phys: u64,
    pages: usize,
};

/// Wrap `pages` contiguous frames at `phys` (already allocated + zeroed by the caller) in a
/// refcounted shared-memory object, or null if the heap is out of room.
pub fn createSharedMemory(phys: u64, pages: usize) ?*SharedMemoryObject {
    const shared_memory = heap.allocator().create(SharedMemoryObject) catch return null;
    shared_memory.* = .{ .phys = phys, .pages = pages };
    return shared_memory;
}

/// Take a shared-memory reference — a MAPPING's reference (docs/shared-fate-plan.md
/// M3): each address space that maps the region holds one, recorded on the space
/// and dropped at its destruction. Caller holds the big kernel lock.
pub fn retainSharedMemory(shared_memory: *SharedMemoryObject) void {
    shared_memory.refcount += 1;
}

/// Drop a shared-memory reference; when the last one goes — no handles AND no
/// mappings left — return its frames to the allocator and free the object.
/// (The mappings themselves are torn down with each sharer's address space;
/// `device_grant` keeps that sweep from freeing the frames, and the space's
/// recorded mapping references keep this drop from freeing them early.)
pub fn dropSharedMemoryReference(shared_memory: *SharedMemoryObject) void {
    if (shared_memory.refcount > 1) {
        shared_memory.refcount -= 1;
    } else {
        for (0..shared_memory.pages) |i| pmm.free(shared_memory.phys + i * page_size);
        heap.allocator().destroy(shared_memory);
    }
}

// --- sender FIFO (endpoint-local, via Task.next) ----------------------------

fn enqueueSender(endpoint: *Endpoint, t: *Task) void {
    t.ipc_wait_endpoint = @ptrCast(endpoint); // so a kill can unlink a parked caller
    t.next = null;
    if (endpoint.sender_tail) |tail| tail.next = t else endpoint.sender_head = t;
    endpoint.sender_tail = t;
}

fn dequeueSender(endpoint: *Endpoint) ?*Task {
    const t = endpoint.sender_head orelse return null;
    endpoint.sender_head = t.next;
    if (endpoint.sender_head == null) endpoint.sender_tail = null;
    t.ipc_wait_endpoint = null;
    t.next = null;
    return t;
}

/// Unlink `t` from the sender FIFO it queues in, if any — the kill path for a
/// client parked in `call` that no server has received yet. Without this, a dead
/// caller would later be dequeued as a dangling pointer. The endpoint is still
/// alive here: `t`'s own handle table holds a reference until closeHandles runs
/// (which the kill path does *after* this). Precondition: the big kernel lock is
/// held.
pub fn abandonSenderLocked(t: *Task) void {
    const endpoint: *Endpoint = @ptrCast(@alignCast(t.ipc_wait_endpoint orelse return));
    t.ipc_wait_endpoint = null;
    var previous: ?*Task = null;
    var node = endpoint.sender_head;
    while (node) |n| : ({
        previous = n;
        node = n.next;
    }) {
        if (n != t) continue;
        if (previous) |p| p.next = t.next else endpoint.sender_head = t.next;
        if (endpoint.sender_tail == t) endpoint.sender_tail = previous;
        t.next = null;
        return;
    }
}

// --- cross-address-space copy ----------------------------------------------

/// Copy `len` bytes from `source_va` in address space `source_as` to `destination_va` in
/// `destination_as`, walking each side's page tables through the physmap (no CR3 switch).
/// `*_as == 0` means the kernel address space (for kernel-task endpoints). User
/// buffers must lie in the low half and carry the permission ring 3 would need for
/// their side of the copy — readable to send from, writable to receive into.
/// Returns false — never #PFs — if any page is unmapped, out of range, or
/// wrongly permissioned. Handles page-straddling buffers.
///
/// This is the process↔process case, which `user-memory` deliberately does not
/// cover (it knows one user address space at a time); both sides resolve through
/// `user_memory.resolve`, so the permission rules are the same ones.
fn copyAcross(source_as: u64, source_va: u64, destination_as: u64, destination_va: u64, len: usize) bool {
    if (source_as != 0 and !user_memory.userRangeOk(source_va, len)) return false;
    if (destination_as != 0 and !user_memory.userRangeOk(destination_va, len)) return false;

    var off: usize = 0;
    while (off < len) {
        const s = user_memory.resolve(source_as, source_va + off, false) orelse return false;
        const d = user_memory.resolve(destination_as, destination_va + off, true) orelse return false;
        const s_left = page_size - ((source_va + off) & (page_size - 1));
        const d_left = page_size - ((destination_va + off) & (page_size - 1));
        const n = @min(@min(s_left, d_left), len - off);
        const source: [*]const u8 = @ptrFromInt(boot_handoff.physicalToVirtual(s));
        const destination: [*]u8 = @ptrFromInt(boot_handoff.physicalToVirtual(d));
        @memcpy(destination[0..n], source[0..n]);
        off += n;
    }
    return true;
}

/// The checked copy-in, re-exported from its home in `user-memory` so the many
/// `ipc.copyFromUser` call sites keep reading naturally. New code should reach
/// for `user-memory` directly — it is where the write direction lives too.
pub const copyFromUser = user_memory.copyFromUser;

// --- the two IPC operations -------------------------------------------------

/// Share the capability named by handle `cap` in `from`'s table into `to`'s table,
/// bumping the endpoint's refcount (the sender keeps its handle — this is a copy, not
/// a move). Returns the handle it landed at in `to` (>= 0), or `-EBADF` if `cap` names
/// no live handle, or `-ENOSPC` if `to`'s table is full. Callers only invoke this when
/// `cap != no_cap`. Used by both IPC directions to carry an endpoint with a message.
fn shareCapability(from: *Task, to: *Task, cap: u64) i64 {
    if (cap >= from.handles.len) return -EBADF;
    const entry = from.handles[@intCast(cap)] orelse return -EBADF;
    // Bump the named object's refcount (a copy, not a move — the sender keeps its handle),
    // dispatching by kind so both endpoints and shared-memory regions can travel with a
    // message.
    switch (entry.kind) {
        handle_kind_endpoint => {
            const e: *Endpoint = @ptrCast(@alignCast(entry.ptr));
            e.refcount += 1;
        },
        handle_kind_shared_memory => {
            const s: *SharedMemoryObject = @ptrCast(@alignCast(entry.ptr));
            s.refcount += 1;
        },
        handle_kind_dma_region => {
            const r: *DmaRegionObject = @ptrCast(@alignCast(entry.ptr));
            r.refcount += 1;
        },
        else => return -EBADF,
    }
    const handle = installEntry(to, entry);
    if (handle < 0) {
        dropEntry(entry); // undo the bump; the receiver had no room
        return -ENOSPC;
    }
    return handle;
}

/// Client side of IPC_Call: send `[message_ptr, message_len)` to `endpoint` and block until a
/// server replies into `[reply_ptr, reply_cap)`. Returns the reply length, or a
/// negative errno. `send_cap` (a handle, or `no_cap`) is an endpoint transferred to the
/// server with the request; `out_received_cap` receives the handle of an endpoint the
/// server sent back in its reply, or `no_cap`. Runs as the current task.
pub fn call(endpoint: *Endpoint, message_ptr: u64, message_len: u64, reply_ptr: u64, reply_cap: u64, send_cap: u64, out_received_cap: *u64) i64 {
    if (message_len > MESSAGE_MAXIMUM or reply_cap > MESSAGE_MAXIMUM) return -E2BIG;
    const flags = sync.enter();
    defer sync.leave(flags);
    if (endpoint.dead) return -EPEER; // the service that owned this endpoint is gone — don't block

    const me = scheduler.current();
    me.ipc_send_ptr = message_ptr;
    me.ipc_send_len = message_len;
    me.ipc_reply_ptr = reply_ptr;
    me.ipc_reply_cap = reply_cap;
    me.ipc_send_cap = send_cap;
    me.ipc_received_cap = abi.no_cap;
    me.ipc_status = 0;

    enqueueSender(endpoint, me); // join the FIFO, then...
    scheduler.wakeLocked(&endpoint.receive_wait_queue); // ...wake a waiting server (no-op if none)
    scheduler.blockCurrentLocked(); // block until the reply readies us again

    out_received_cap.* = me.ipc_received_cap; // a capability the replier sent back, or no_cap
    return me.ipc_status; // reply length or -errno, written by the replier
}

/// Server side of IPC_ReplyWait: deliver `[reply_ptr, reply_len)` to the client
/// we currently owe (if any), then receive the next request into
/// `[receive_ptr, receive_cap)`, blocking until one arrives. Writes the sender's badge
/// to `out_badge` and returns the request length, or a negative errno. A pending
/// notification is delivered ahead of client requests (length 0, badge with
/// `notify_badge_bit` set, no reply owed).
pub fn replyWait(endpoint: *Endpoint, reply_ptr: u64, reply_len: u64, receive_ptr: u64, receive_cap: u64, send_cap: u64, out_badge: *u64, out_received_cap: *u64) i64 {
    if (reply_len > MESSAGE_MAXIMUM or receive_cap > MESSAGE_MAXIMUM) return -E2BIG;
    const flags = sync.enter();
    defer sync.leave(flags);

    const me = scheduler.current();
    out_received_cap.* = abi.no_cap; // no capability received unless a request delivers one

    // (1) Reply to the client we're still holding, if any — carrying `send_cap` to it.
    if (me.ipc_client) |client| {
        me.ipc_client = null;
        const n = @min(reply_len, client.ipc_reply_cap);
        client.ipc_received_cap = abi.no_cap;
        if (!copyAcross(me.address_space, reply_ptr, client.address_space, client.ipc_reply_ptr, n)) {
            client.ipc_status = -EFAULT;
        } else if (send_cap != abi.no_cap) {
            // Transfer the reply's capability into the client. A failure fails the
            // client's `call` rather than delivering a reply without its promised cap.
            const shared = shareCapability(me, client, send_cap);
            if (shared < 0) {
                client.ipc_status = shared; // -EBADF (bad handle) or -ENOSPC (client table full)
            } else {
                client.ipc_received_cap = @intCast(shared);
                client.ipc_status = @intCast(n);
            }
        } else {
            client.ipc_status = @intCast(n);
        }
        scheduler.readyLocked(client); // its `call` now returns
    }

    // (2) Receive the next request (or notification / buffered message), blocking until
    // one is ready. Bare notifications (IRQ/exit) come first — they're latency-sensitive
    // and carry no payload — then buffered messages, then synchronous client requests.
    while (true) {
        if (popNotify(endpoint)) |badge| {
            out_badge.* = badge | notify_badge_bit;
            return 0; // notification: no payload, no reply owed, no cap
        }
        if (popPost(endpoint)) |slot| {
            const n = @min(@as(usize, slot.length), receive_cap);
            // Copy from the kernel-resident ring slot (source address_space 0) into the receiver.
            if (!copyAcross(0, @intFromPtr(&slot.bytes), me.address_space, receive_ptr, n)) {
                continue; // bad receive buffer: drop this message, keep serving
            }
            out_badge.* = slot.sender_id | notify_badge_bit | notify_message_bit;
            return @intCast(n); // async message: payload delivered, no reply owed, no cap
        }
        if (dequeueSender(endpoint)) |caller| {
            const n = @min(caller.ipc_send_len, receive_cap);
            if (!copyAcross(caller.address_space, caller.ipc_send_ptr, me.address_space, receive_ptr, n)) {
                caller.ipc_status = -EFAULT; // bad sender buffer: fail it, keep serving
                scheduler.readyLocked(caller);
                continue;
            }
            // Install the capability the caller sent, if any, into my table. A failure
            // fails the caller's `call` and does not deliver — no half-delivered cap.
            if (caller.ipc_send_cap != abi.no_cap) {
                const shared = shareCapability(caller, me, caller.ipc_send_cap);
                if (shared < 0) {
                    caller.ipc_status = shared; // -EBADF or -ENOSPC
                    scheduler.readyLocked(caller);
                    continue;
                }
                out_received_cap.* = @intCast(shared);
            }
            me.ipc_client = caller; // remember who to reply to
            out_badge.* = caller.id;
            return @intCast(n);
        }
        scheduler.waitLocked(&endpoint.receive_wait_queue); // nothing yet — sleep until woken, then retry
    }
}

// --- asynchronous notification (for IRQ-as-message, M10) --------------------

fn popNotify(endpoint: *Endpoint) ?u64 {
    if (endpoint.notify_head == endpoint.notify_tail) return null;
    const badge = endpoint.notify_buffer[endpoint.notify_head % endpoint.notify_buffer.len];
    endpoint.notify_head +%= 1;
    return badge;
}

/// Take the oldest buffered message from the post ring, or null if empty. Returns a
/// pointer into the endpoint's own storage — valid until the next `send`/`popPost` under
/// the same lock region, which is all the copy-out in `replyWait` needs.
fn popPost(endpoint: *Endpoint) ?*const PostSlot {
    if (endpoint.post_head == endpoint.post_tail) return null;
    const slot = &endpoint.post_buffer[endpoint.post_head % post_capacity];
    endpoint.post_head +%= 1;
    return slot;
}

/// Client-free side of async IPC (`ipc_send`): copy `[source_va, len)` from address space
/// `source_as` into `endpoint`'s post ring and wake a waiting receiver — **without
/// blocking the sender** and with no reply owed. `sender_id` rides along, delivered in the
/// low bits of the receiver's badge. Returns 0, or a negative errno (`-E2BIG` if the
/// payload exceeds `POST_MAXIMUM`, `-EFAULT` if the source buffer is unmapped / out of the
/// user half). A full ring drops the *oldest* message (advancing `post_head`), because a
/// buffered message is discrete, not a level: keeping the newest keeps input responsive.
/// Precondition: the big kernel lock is held.
pub fn sendLocked(endpoint: *Endpoint, source_as: u64, source_va: u64, len: u64, sender_id: u64) i64 {
    if (len > POST_MAXIMUM) return -E2BIG;
    // Drop the oldest if the ring is full, so this newest message always lands.
    if (endpoint.post_tail -% endpoint.post_head >= post_capacity) endpoint.post_head +%= 1;
    const slot = &endpoint.post_buffer[endpoint.post_tail % post_capacity];
    if (!copyFromUser(source_as, source_va, slot.bytes[0..@intCast(len)])) return -EFAULT;
    slot.length = @intCast(len);
    slot.sender_id = sender_id;
    endpoint.post_tail +%= 1;
    scheduler.wakeLocked(&endpoint.receive_wait_queue);
    return 0;
}

/// `sendLocked` wrapped in its own critical section, for the `ipc_send` syscall path.
pub fn send(endpoint: *Endpoint, source_as: u64, source_va: u64, len: u64, sender_id: u64) i64 {
    const flags = sync.enter();
    defer sync.leave(flags);
    return sendLocked(endpoint, source_as, source_va, len, sender_id);
}

/// Post an asynchronous notification carrying `badge` to `endpoint` and wake a waiting
/// receiver. Precondition: the big kernel lock is held.
///
/// The lock must already cover whatever produced `endpoint` — an ISR that looked the
/// endpoint up in a table and *then* took the lock could be racing a process exit
/// that unbinds and frees it in between. See irq.dispatch, which holds one lock
/// region across the table read and this call.
///
/// A full ring drops the notification. That is the correct semantics, not a
/// concession: a notification is a *level* ("this device wants attention"), and the
/// driver re-reads device state on wake. It is never a count of events.
pub fn notifyLocked(endpoint: *Endpoint, badge: u64) void {
    if (endpoint.notify_tail -% endpoint.notify_head < endpoint.notify_buffer.len) {
        endpoint.notify_buffer[endpoint.notify_tail % endpoint.notify_buffer.len] = badge;
        endpoint.notify_tail +%= 1;
    }
    scheduler.wakeLocked(&endpoint.receive_wait_queue);
}

/// `notifyLocked` as a self-contained ISR critical section, for a caller that holds
/// `endpoint` by some means other than a table the lock protects. Releases the lock without
/// touching the interrupt flag (the ISR's iretq restores it), like the timer tick.
pub fn notifyFromIsr(endpoint: *Endpoint, badge: u64) void {
    _ = sync.enter();
    notifyLocked(endpoint, badge);
    sync.leaveIsr();
}

// --- per-process handle table + name registry -------------------------------

/// Install a capability object (kind + pointer) in task `t`'s handle table; returns the
/// small-int handle or -ENOSPC. The caller has already taken/holds the reference the slot
/// represents.
fn installEntry(t: *Task, entry: scheduler.HandleObject) i64 {
    for (&t.handles, 0..) |*slot, i| {
        if (slot.* == null) {
            slot.* = entry;
            return @intCast(i);
        }
    }
    return -ENOSPC;
}

/// Install an endpoint handle. The common case; keeps the endpoint callers' signature.
pub fn installHandle(t: *Task, endpoint: *Endpoint) i64 {
    return installEntry(t, .{ .kind = handle_kind_endpoint, .ptr = @ptrCast(endpoint) });
}

/// Install a shared-memory handle.
pub fn installSharedMemoryHandle(t: *Task, shared_memory: *SharedMemoryObject) i64 {
    return installEntry(t, .{ .kind = handle_kind_shared_memory, .ptr = @ptrCast(shared_memory) });
}

/// Install an endpoint handle, reusing an existing slot that already names this
/// endpoint (no new reference taken in that case). For callers that install per
/// operation — fs_resolve — so a 16-slot table can't be exhausted by repeats.
/// Any subsystem installing handles per-call should come through here.
pub fn installHandleDeduped(t: *Task, endpoint: *Endpoint) i64 {
    for (t.handles, 0..) |slot, i| {
        const entry = slot orelse continue;
        if (entry.kind == handle_kind_endpoint and entry.ptr == @as(*anyopaque, @ptrCast(endpoint))) return @intCast(i);
    }
    const h = installHandle(t, endpoint);
    if (h >= 0) endpoint.refcount += 1; // the table entry owns a reference
    return h;
}

/// Resolve a handle to its endpoint, or null if out of range, unused, or a different kind
/// (e.g. a shared-memory handle used where an endpoint is expected).
pub fn resolveHandle(t: *Task, h: u64) ?*Endpoint {
    if (h >= t.handles.len) return null;
    const entry = t.handles[@intCast(h)] orelse return null;
    if (entry.kind != handle_kind_endpoint) return null;
    return @ptrCast(@alignCast(entry.ptr));
}

/// Resolve a handle to its shared-memory object, or null if out of range, unused, or not
/// a shared-memory handle.
pub fn resolveSharedMemory(t: *Task, h: u64) ?*SharedMemoryObject {
    if (h >= t.handles.len) return null;
    const entry = t.handles[@intCast(h)] orelse return null;
    if (entry.kind != handle_kind_shared_memory) return null;
    return @ptrCast(@alignCast(entry.ptr));
}

/// Drop every capability reference an exiting task holds, dispatching by kind so a dead
/// task's endpoints *and* shared-memory regions are released correctly. Called from the
/// scheduler exit path.
pub fn closeHandles(t: *Task) void {
    for (&t.handles) |*slot| {
        if (slot.*) |entry| {
            dropEntry(entry);
            slot.* = null;
        }
    }
}

/// Drop the reference a handle-table entry represents, by kind.
fn dropEntry(entry: scheduler.HandleObject) void {
    switch (entry.kind) {
        handle_kind_endpoint => dropRef(@ptrCast(@alignCast(entry.ptr))),
        handle_kind_shared_memory => dropSharedMemoryReference(@ptrCast(@alignCast(entry.ptr))),
        handle_kind_dma_region => dropDmaRegionReference(@ptrCast(@alignCast(entry.ptr))),
        else => {},
    }
}

var registry: [maximum_services]?*Endpoint = .{null} ** maximum_services;

/// Publish `endpoint` under well-known `id` (takes a reference). Returns 0 or -errno.
pub fn register(id: u32, endpoint: *Endpoint) i64 {
    if (id >= maximum_services) return -ENOENT;
    if (registry[id]) |old| dropRef(old);
    endpoint.refcount += 1;
    registry[id] = endpoint;
    return 0;
}

/// Find the endpoint published under `id`, taking a reference for the caller to
/// install in its handle table. Null if nothing is registered there.
pub fn lookup(id: u32) ?*Endpoint {
    if (id >= maximum_services) return null;
    const endpoint = registry[id] orelse return null;
    endpoint.refcount += 1;
    return endpoint;
}