danos/system/drivers/usb-xhci-bus/usb-xhci-bus.zig

//! /system/drivers/usb-xhci-bus — the xHCI (USB 3) host-controller bus driver.
//! The device manager spawns **one instance per controller** it discovers (a
//! machine can carry several), passing the controller's device-tree id as
//! argv[1]; this instance claims that device and no other, so multiple
//! instances never fight over hardware.
//!
//! M18.2 (this increment): after the hello, real hardware — map the xHC's
//! register window (the first memory BAR; resource 0 is the ECAM config
//! space), read the capability registers, and walk the root-hub ports: one
//! `child_added` report to the manager per connected port, carrying the port
//! number and the PORTSC speed class as identity. No transfer rings yet —
//! descriptors and USB class matching are the USB track; the connect bit and
//! speed come straight from PORTSC, which reflects hardware state whether or
//! not the controller is running.

const std = @import("std");
const device = @import("driver");
const channel = @import("channel");
const ipc = @import("ipc");
const process = @import("process");
const service = @import("service");
const time = @import("time");
const input = @import("input-client");
const device_manager = @import("driver");
const memory = @import("memory");
const logging = @import("logging");
const device_manager_protocol = @import("device-manager-protocol");
const envelope = @import("envelope");
const usb_ids = @import("usb-ids");
const usb_abi = @import("usb-abi");
const usb_transfer_protocol = @import("usb-transfer-protocol");
const library = @import("usb-xhci-library.zig");
const pci = @import("pci");

/// The controller engine (reset, rings, transfers), stood up in `initialise`.
var controller: ?library.Controller = null;

/// This driver's service endpoint (registered as `.usb_bus`), where class-driver
/// requests, signals, MSI notifications, and the poll/reconcile timer all arrive.
var service_endpoint: ipc.Handle = 0;

/// How often the driver drains the event ring for interrupt reports (~125 Hz) when
/// polling, re-armed each tick. Frequent enough for responsive input.
const poll_interval_ms: u64 = 8;

/// The timer interval in MSI mode: the ring is drained at interrupt time, and the tick
/// only reconciles root ports (real hardware delivers late USB2 companion-hub debounce
/// with no reliable port-change event — see onNotification) and un-wedges a lost MSI
/// edge (edge-triggered, no kernel mask/ack: a missed IP clear stalls, never storms).
const reconcile_interval_ms: u64 = 250;

/// The controller's own descriptor, kept at file scope because `pci.Function` holds a
/// pointer to it for the whole bring-up.
var controller_descriptor: device.DeviceDescriptor = undefined;

/// Non-null iff MSI mode is active: the vector whose notification badge means "the
/// controller interrupted". Null means the 8 ms polling fallback is running.
var msi_vector: ?u32 = null;

fn timerInterval() u64 {
    return if (msi_vector != null) reconcile_interval_ms else poll_interval_ms;
}

/// The class driver that opened each device: the endpoint interrupt reports are
/// pushed back to, and **the task that opened it** — the kernel-stamped badge, so
/// a device token is scoped to the client that was given it. Keyed by the device
/// token (the interface's device id).
///
/// The scoping is the point (docs/os-development/protocol-namespace.md: handles
/// are validated against the badge). A token is a small registered-device id any
/// process could name, and every packet that carries one used to be honoured from
/// anyone: a stranger could run control transfers on another driver's device, arm
/// interrupt polling on it, or redirect its reports.
const Open = struct {
    used: bool = false,
    device_token: u64 = 0,
    owner: u32 = 0,
    report_endpoint: usize = 0,
};
var opens = [_]Open{.{}} ** 16;

/// Remember (or replace) the endpoint task `owner` receives reports for
/// `device_token` on. Returns whether the table kept the handle — false means the
/// caller still owns it and must dispose of it. A re-open by the **same** client
/// supersedes its previous endpoint, and the one it displaced is closed here: the
/// table holds exactly one reference per slot.
fn recordOpen(device_token: u64, owner: u32, report_endpoint: usize) bool {
    for (&opens) |*open| {
        if (open.used and open.device_token == device_token) {
            if (open.owner != owner) return false; // someone else's device; nothing kept
            if (open.report_endpoint != report_endpoint) _ = ipc.close(open.report_endpoint);
            open.report_endpoint = report_endpoint;
            return true;
        }
    }
    for (&opens) |*open| {
        if (!open.used) {
            open.* = .{ .used = true, .device_token = device_token, .owner = owner, .report_endpoint = report_endpoint };
            return true;
        }
    }
    return false; // table full: not kept
}

/// Whether `device_token` is open to task `owner`. An open device belonging to
/// someone else answers exactly as one that was never opened, so a prober cannot
/// tell another driver's device from an absent one.
fn openedBy(device_token: u64, owner: u32) bool {
    for (&opens) |*open| {
        if (open.used and open.device_token == device_token) return open.owner == owner;
    }
    return false;
}

/// Whether `device_token` is open at all. Asked only after `openedBy` has said
/// the caller is not the holder, so an answer of true means *someone else* holds
/// it — one device, one class driver.
fn heldByAnother(device_token: u64) bool {
    for (&opens) |*open| {
        if (open.used and open.device_token == device_token) return true;
    }
    return false;
}

fn reportEndpointFor(device_token: u64) ?usize {
    for (&opens) |*open| {
        if (open.used and open.device_token == device_token) return open.report_endpoint;
    }
    return null;
}

/// Release every device a dead client held: its slot, and the report endpoint
/// capability in it. Driven by published process exits — the same sweep idiom the
/// FAT server uses for open files and the harness uses for subscribers — which is
/// also what lets a restarted class driver re-open the device its predecessor had.
fn releaseOpensOf(dead: u32) void {
    for (&opens) |*open| {
        if (open.used and open.owner == dead) {
            // The engine first: it holds this endpoint's handle number per
            // subscription, and the close below frees that number for reuse.
            if (controller) |*engine| engine.releaseSubscriptions(open.device_token);
            _ = ipc.close(open.report_endpoint);
            std.log.info("released device {d} for dead client {d}", .{ open.device_token, dead });
            open.* = .{};
        }
    }
}

var controller_id: u64 = device_manager_protocol.no_device;

/// Claim the assigned controller, find its register window, and hello the
/// manager. Any failure returns false: the process exits cleanly, which the
/// manager reads as "meant to stop" — a missing assignment is not a crash loop.
fn initialise(endpoint: ipc.Handle) bool {
    service_endpoint = endpoint;
    // Device tokens are per-client state, so this driver needs deaths for the
    // same reason the FAT server does: a class driver that crashes must not keep
    // its device open, or its restarted instance could never claim it back.
    _ = process.subscribeExits(endpoint);

    // The transfer contract, bound by hand rather than through the harness's
    // `.service`, because **losing it is not fatal here**. One machine can carry
    // several xHCI controllers and the driver model spawns one process per
    // controller, so several processes provide the same contract for different
    // hardware — and `/protocol` holds exactly one name, deliberately (addressing
    // lives inside the protocol, never in the path). Whoever binds first is the
    // one clients reach by name; a later instance still owns its controller,
    // enumerates its bus, and reports its children to the device manager, so it
    // keeps running. **Known gap:** a class driver behind a second controller
    // cannot reach it — the transfer protocol has no controller field for
    // `target`, and the fix is either one process multiplexing every controller
    // or the spawner wiring the child's channel (P5), not a second name.
    if (!channel.bindPatiently("usb-transfer", endpoint))
        _ = logging.write("/system/drivers/usb-xhci-bus: /protocol/usb-transfer is another controller's; serving mine unnamed\n");

    if (!device.claim(controller_id)) {
        std.log.info("unable to claim controller device {d}", .{controller_id});
        return false;
    }

    // Fetch our own descriptor back for the controller's resources.
    const buffer = memory.allocator().alloc(device.DeviceDescriptor, 64) catch {
        _ = logging.write("/system/drivers/usb-xhci-bus: out of memory\n");
        return false;
    };
    const total = device.enumerate(buffer);
    const descriptor = for (buffer[0..@min(total, buffer.len)]) |d| {
        if (d.id == controller_id) break d;
    } else {
        std.log.info("device {d} not in the device tree", .{controller_id});
        return false;
    };
    controller_descriptor = descriptor;

    // The xHC's registers live behind the first memory BAR. Resource 0 is the
    // function's ECAM configuration space (M15), so the walk starts at 1.
    var register_index: u64 = 0;
    const register_window = for (descriptor.resources[1..@intCast(descriptor.resource_count)], 1..) |resource, index| {
        if (resource.kind == @intFromEnum(device.ResourceKind.memory)) {
            register_index = index;
            break resource;
        }
    } else {
        std.log.info("controller device {d} has no register BAR", .{controller_id});
        return false;
    };
    std.log.info("claimed controller device {d} (registers at 0x{x}, {d} bytes)", .{
        controller_id,
        register_window.start,
        register_window.len,
    });
    register_base = device.mmioMap(controller_id, register_index) orelse {
        _ = logging.write("/system/drivers/usb-xhci-bus: mmio_map failed\n");
        return false;
    };

    // Message-signalled interrupt setup comes BEFORE the controller bring-up, not
    // after: Controller.init writes IMAN.IE, and QEMU's xhci only registers the MSI-X
    // vector as in-use when that write happens with MSI-X already enabled (its
    // intr_update callback early-outs on !msix_enabled, and msix_notify silently
    // drops interrupts for an unused vector). Real hardware does not care about the
    // order; QEMU requires it.
    setupMsi();

    // Bring the controller up: reset it, stand up the command and event rings,
    // and start it running (the hardware half lives in usb-xhci-library.zig).
    controller = library.Controller.init(register_base) orelse {
        _ = logging.write("/system/drivers/usb-xhci-bus: controller reset/bring-up failed\n");
        return false;
    };
    std.log.info("controller running ({d} slots, {d}-byte contexts)", .{
        controller.?.max_slots,
        controller.?.context_size,
    });
    // The proof of life: a No-Op command round-trips the command ring, the event
    // ring, the doorbell, and the cycle-bit bookkeeping. If this completes, the
    // engine is sound; transfers build on exactly this machinery.
    if (controller.?.noOpCommand()) {
        _ = logging.write("/system/drivers/usb-xhci-bus: command ring running (no-op ok)\n");
    } else {
        _ = logging.write("/system/drivers/usb-xhci-bus: no-op command did not complete\n");
        return false;
    }

    // The handshake (role: bus — we enumerate USB ports and report the devices
    // behind them), inside the manager's hello deadline. Keep the handle: the
    // tick's hot-plug dispatch reports through it.
    const handle = device_manager.hello(.bus, controller_id) orelse return false;
    manager_handle = handle;

    scanPorts(handle);

    // Arm the timer: in polling mode it drains the event ring; in MSI mode it is the
    // slower port-reconcile/safety-net tick. Re-armed on each tick in onNotification.
    _ = time.timerOnce(service_endpoint, timerInterval());
    return true;
}

/// Switch the event ring from timer polling to message-signalled interrupts, if the
/// whole path is available: map the function's config space, bind a vector, program
/// the MSI capability — or, on an MSI-X-only function (QEMU's qemu-xhci is one: it
/// advertises MSI-X and PCIe but no plain MSI), entry 0 of the MSI-X table, which
/// takes the same kernel (address, data) pair (xHCI interrupter 0 raises vector 0).
/// Any step failing leaves `msi_vector` null and the 8 ms polling path exactly as it
/// was. The controller side needs nothing extra — IMAN.IE and USBCMD.INTE are already
/// set (see Controller.init: QEMU only writes runtime events with the interrupter
/// enabled).
///
/// After a supervised kill, the kernel drops the vector binding but the device still
/// has the interrupt enabled and fires the stale vector; the kernel EOIs it
/// harmlessly, and the respawned driver re-runs this with its fresh vector.
fn setupMsi() void {
    var function = pci.Function.map(controller_id, &controller_descriptor) orelse {
        std.log.info("config-space map failed; polling at {d} ms", .{poll_interval_ms});
        return;
    };
    function.enableMemoryAndBusMaster();
    const message = device.msiBind(controller_id, service_endpoint) orelse {
        std.log.info("msi_bind unavailable; polling at {d} ms", .{poll_interval_ms});
        return;
    };
    if (function.programMsi(message)) {
        msi_vector = message.data;
        std.log.info("msi active (vector {d}); reconcile tick at {d} ms", .{ message.data, reconcile_interval_ms });
        return;
    }
    if (function.msix()) |table| {
        if (table.programEntry(0, message) and table.unmaskEntry(0)) {
            table.enable();
            function.setInterruptDisable();
            msi_vector = message.data;
            std.log.info("msix active (vector {d}); reconcile tick at {d} ms", .{ message.data, reconcile_interval_ms });
            return;
        }
    }
    std.log.info("no msi/msi-x capability; polling at {d} ms", .{poll_interval_ms});
}

var register_base: usize = 0;

/// The xHCI default Protocol Speed IDs (the PORTSC port-speed field, bits 13:10)
/// decoded to human names — the boot-log breadcrumb for what actually enumerated on
/// a port, the USB analog of the pci-bus class-code line. A controller may redefine
/// these through its Supported Protocol capability, but the defaults cover every
/// speed QEMU and real hardware report at this (pre-descriptor) stage.
fn speedName(speed: u32) []const u8 {
    return switch (speed) {
        1 => "Full-speed (USB 2.0, 12 Mb/s)",
        2 => "Low-speed (USB 2.0, 1.5 Mb/s)",
        3 => "High-speed (USB 2.0, 480 Mb/s)",
        4 => "SuperSpeed (USB 3.0, 5 Gb/s)",
        5 => "SuperSpeedPlus (USB 3.1, 10 Gb/s)",
        else => "unknown speed",
    };
}

/// The root-hub scan and enumeration: for each connected port, bring the device
/// up (reset → enable slot → address), read its descriptors, and register +
/// report one child per interface — carrying the interface's (class, subclass,
/// protocol) triple as identity, which is what the device manager matches a
/// class driver against.
var manager_handle: ?ipc.Handle = null;

// Per-root-port connected state from the previous tick, so the poll acts on
// empty->connected transitions (edge), never re-attempting a level every tick.
var prev_connected: [64]bool = [_]bool{false} ** 64;

fn scanPorts(manager: ipc.Handle) void {
    const engine = if (controller) |*c| c else {
        _ = logging.write("/system/drivers/usb-xhci-bus: controller not initialised\n");
        return;
    };
    std.log.info("{d} root-hub ports", .{engine.max_ports});

    var port: u32 = 1;
    var connected: u32 = 0;
    while (port <= engine.max_ports) : (port += 1) {
        if (!engine.portConnected(port)) continue;
        if (port < prev_connected.len) prev_connected[port] = true; // don't re-fire the poll for these
        connected += 1;
        bringUpPort(manager, engine, port);
    }
    if (connected == 0) {
        _ = logging.write("/system/drivers/usb-xhci-bus: no devices connected\n");
        engine.dumpPortTopology(); // help diagnose an empty scan: the xECP map + raw PORTSC
    }
}

/// Bring up whatever is on `port`: setup + enumerate + register/report one child
/// per interface. Shared by the boot scan and hot-plug (a port-change event with
/// the port now connected).
fn bringUpPort(manager: ipc.Handle, engine: *library.Controller, port: u32) void {
    const speed = (engine.portStatus(port) >> 10) & 0xF; // the PORTSC port-speed class
    std.log.info("port {d} connected — {s} (speed class {d})", .{ port, speedName(speed), speed });

    const usb_device = engine.setupDevice(port, speed) orelse {
        std.log.info("port {d} device setup failed", .{port});
        return;
    };
    if (!engine.enumerate(usb_device)) {
        std.log.info("port {d} enumeration failed", .{port});
        return;
    }
    var maker_buffer: [64]u8 = undefined;
    var product_buffer: [64]u8 = undefined;
    const maker = engine.readString(usb_device, @intFromEnum(usb_device.device_descriptor.manufacturer_index), &maker_buffer) orelse "?";
    const product = engine.readString(usb_device, @intFromEnum(usb_device.device_descriptor.product_index), &product_buffer) orelse "?";
    std.log.info("port {d} device: {s} \"{s} {s}\" (0x{x:0>4}:0x{x:0>4}), {d} interface(s)", .{
        port,
        usb_ids.className(usb_device.device_descriptor.device_class),
        maker,
        product,
        usb_device.device_descriptor.vendor_id,
        usb_device.device_descriptor.product_id,
        usb_device.interface_count,
    });

    for (usb_device.interfaces[0..usb_device.interface_count]) |*interface| {
        // Record the id each interface was registered as, so a class driver
        // opening the interface (by that id) resolves to it.
        if (reportInterface(manager, port, interface.*)) |registered| {
            interface.registered_device_id = registered;
        }
    }

    // A hub (class 9) is bus infrastructure the bus drives itself: configure it
    // and power its downstream ports (docs/usb-hub.md). Its interfaces are still
    // reported above, but no external class driver binds it.
    if (deviceIsHub(usb_device)) _ = engine.setupHub(usb_device);
}

// A compact topology-unique port key for a hub downstream port: 1000 + slot*100
// + port. Stays a few digits (the id tag "P<key>I<iface>" has an 8-byte cap)
// while never colliding with a root port (1..N) or another (hub, port).
fn hubPortKey(hub_slot: u8, port: u16) u32 {
    return 1000 + @as(u32, hub_slot) * 100 + port;
}

/// Service a change on hub downstream `port`: dispatch a connect (enumerate the
/// new device) or a disconnect (tear the old one down). Recurses for a hub
/// behind a hub — a nested hub is set up on connect and its downstream devices
/// torn down first on disconnect.
fn bringUpBehindHub(manager: ipc.Handle, engine: *library.Controller, hub: *library.Device, port: u16) void {
    const status = engine.hubPortStatusAck(hub, port) orelse return;
    const connected = library.Controller.hubPortConnected(status);
    const existing = engine.deviceOnHubPort(hub, port);
    if (connected != (existing != null)) // only when a device appears or leaves — not empty seed-sweep ports
        std.log.info("hub slot {d} port {d}: {s} (status 0x{x:0>4})", .{ hub.slot_id, port, if (connected) "device connected" else "device removed", status & 0xFFFF });

    if (!connected) {
        if (existing) |dev| tearDownHubDevice(manager, engine, dev);
        return;
    }
    if (existing != null) return; // already up

    const usb_device = engine.serviceHubPort(hub, port) orelse return;
    if (!engine.enumerate(usb_device)) {
        std.log.info("hub slot {d} port {d}: enumeration failed", .{ hub.slot_id, port });
        return;
    }
    var maker_buffer: [64]u8 = undefined;
    var product_buffer: [64]u8 = undefined;
    const maker = engine.readString(usb_device, @intFromEnum(usb_device.device_descriptor.manufacturer_index), &maker_buffer) orelse "?";
    const product = engine.readString(usb_device, @intFromEnum(usb_device.device_descriptor.product_index), &product_buffer) orelse "?";
    std.log.info("hub slot {d} port {d} device: {s} \"{s} {s}\" (0x{x:0>4}:0x{x:0>4}), {d} interface(s)", .{
        hub.slot_id, port,
        usb_ids.className(usb_device.device_descriptor.device_class),
        maker,
        product,
        usb_device.device_descriptor.vendor_id,
        usb_device.device_descriptor.product_id,
        usb_device.interface_count,
    });
    for (usb_device.interfaces[0..usb_device.interface_count]) |*interface| {
        if (reportInterface(manager, hubPortKey(hub.slot_id, port), interface.*)) |registered| {
            interface.registered_device_id = registered;
        }
    }
    if (deviceIsHub(usb_device)) _ = engine.setupHub(usb_device);
}

/// Report one interface gone. A removal is addressed by the composite (parent,
/// bus address) — a pair no single `Header.target` can carry — so that stays the
/// packet's body and the target addresses the manager itself. `where` names the
/// port and interface for the log line, or null where the caller stays quiet
/// (a hub subtree collapsing reports a great many at once).
fn reportRemoved(manager: ipc.Handle, bus_address: u64, where: ?struct { port: u32, interface: u8 }) void {
    var packet: [device_manager_protocol.message_maximum]u8 = undefined;
    const framed = device_manager_protocol.Protocol.encodeRequest(.child_removed, 0, .{
        .parent = controller_id,
        .bus_address = bus_address,
    }, &.{}, &packet) orelse return;
    var reply: [device_manager_protocol.message_maximum]u8 = undefined;
    _ = ipc.call(manager, framed, &reply) catch {
        if (where) |place| std.log.info("child-removed report for port {d} interface {d} failed", .{ place.port, place.interface });
    };
}

/// Tear down a device that disconnected from a hub: recursively tear down its
/// own downstream devices first if it is a hub, report each interface removed,
/// then Disable Slot. Mirrors tearDownPort for a hub-attached device.
fn tearDownHubDevice(manager: ipc.Handle, engine: *library.Controller, dev: *library.Device) void {
    // A hub that left takes its whole subtree with it — tear children down first.
    if (dev.is_hub) {
        while (engine.nextChildOf(dev.slot_id, 0)) |child| tearDownHubDevice(manager, engine, child);
    }
    std.log.info("hub device slot {d} disconnected", .{dev.slot_id});
    const key = hubPortKey(dev.parent_slot, dev.parent_port);
    for (dev.interfaces[0..dev.interface_count]) |*interface| {
        if (interface.registered_device_id == 0) continue;
        reportRemoved(manager, (@as(u64, key) << 8) | interface.number, null);
        interface.registered_device_id = 0;
    }
    engine.tearDownDevice(dev);
}

/// Whether an enumerated device is a hub — class 9 at the device or the
/// interface level (a hub's single interface is class 9/0/0).
fn deviceIsHub(usb_device: *const library.Device) bool {
    if (usb_device.device_descriptor.device_class == @intFromEnum(usb_ids.Class.hub)) return true;
    for (usb_device.interfaces[0..usb_device.interface_count]) |interface| {
        if (interface.class == @intFromEnum(usb_ids.Class.hub)) return true;
    }
    return false;
}

/// Tear down whatever was on `port` after an unplug: report each registered
/// interface as removed (the manager prunes the node, notifies watchers, and
/// stops the class driver's world honestly), then release the controller-side
/// device state (Disable Slot).
fn tearDownPort(manager: ipc.Handle, engine: *library.Controller, port: u32) void {
    const usb_device = engine.deviceOnPort(port) orelse return;
    std.log.info("port {d} disconnected", .{port});
    for (usb_device.interfaces[0..usb_device.interface_count]) |*interface| {
        if (interface.registered_device_id == 0) continue;
        reportRemoved(manager, (@as(u64, port) << 8) | interface.number, .{ .port = port, .interface = interface.number });
        interface.registered_device_id = 0;
    }
    engine.tearDownDevice(usb_device);
}

/// Register one interface as a resource-less child of the controller and report
/// it to the device manager. The identity is the packed USB class triple, so the
/// manager can match a class driver (HID keyboard, mouse, mass storage); the
/// registered device id becomes that driver's argv[1] assignment. Returns the
/// registered device id, or null if registration or the report failed.
fn reportInterface(manager: ipc.Handle, port: u32, interface: library.InterfaceInfo) ?u64 {
    const identity = usb_ids.packTriple(interface.class, interface.subclass, interface.protocol);

    // A USB device is reached through its controller, not by MMIO, so the child
    // carries no resources; register() allows that. Its bus-local identity — the
    // (port, interface) address, written as a short "P<port>I<interface>" tag in
    // the hid field — makes each interface a distinct kernel node (the register
    // dedup keys on class/pci_class/hid/resources, all otherwise identical here)
    // and keeps re-registration idempotent across a bus restart: the same port
    // and interface always map back to the same device id.
    var descriptor = std.mem.zeroes(device.DeviceDescriptor);
    descriptor.class = @intFromEnum(device.DeviceClass.usb_device);
    descriptor.pci_class = device.no_pci_class;
    descriptor.resource_count = 0;
    var hid_buffer: [8]u8 = undefined;
    const hid_text = std.fmt.bufPrint(&hid_buffer, "P{d}I{d}", .{ port, interface.number }) catch "";
    descriptor.hid_len = hid_text.len;
    @memcpy(descriptor.hid[0..hid_text.len], hid_text);
    const registered = device.register(controller_id, &descriptor) orelse {
        std.log.info("register refused for port {d} interface {d}", .{ port, interface.number });
        return null;
    };

    // The registered device id is the packet's target — the manager's object
    // addressing — so the report body carries only where on the bus it sits.
    var packet: [device_manager_protocol.message_maximum]u8 = undefined;
    const framed = device_manager_protocol.Protocol.encodeRequest(.child_added, registered, .{
        .bus = @intFromEnum(device_manager_protocol.BusKind.usb),
        .parent = controller_id,
        .bus_address = (@as(u64, port) << 8) | interface.number,
        .identity = identity,
    }, &.{}, &packet) orelse return null;
    var reply: [device_manager_protocol.message_maximum]u8 = undefined;
    _ = ipc.call(manager, framed, &reply) catch {
        std.log.info("child report for port {d} interface {d} failed", .{ port, interface.number });
        return null;
    };
    // The devices.csv columns (bus=usb, and the class triple as base/class/prog_if)
    // then the human-readable interface name — a would-be /system/configuration/devices.csv row read
    // straight off the boot log.
    std.log.info("port {d} interface {d}  bus=usb base={X:0>2} class={X:0>2} prog_if={X:0>2}  —  {s}  registered as device {d}", .{
        port,
        interface.number,
        interface.class,
        interface.subclass,
        interface.protocol,
        usb_ids.interfaceName(interface.class, interface.subclass, interface.protocol),
        registered,
    });
    return registered;
}

/// The generated transfer dispatch. One controller per process, so the handler
/// context is empty and the open table stays in this file's globals.
const Serve = usb_transfer_protocol.Protocol.Provider(void);

const Invocation = envelope.Invocation;
const Answer = envelope.Answer;

/// Every refusal here is the same one — this controller does not (or no longer)
/// serve the device the packet addressed — so there is one errno for all of them.
const refused: isize = -envelope.ENOENT;

/// Set by `onOpen` when the open table has taken ownership of the endpoint the
/// call carried, and read by `onMessage`, where the turn's `Arrival` lives. The
/// generated dispatch hands a handler the raw handle rather than the `Arrival`,
/// so the *claim* travels back out this way. One turn, one handler, one thread.
var capability_claimed = false;

fn onMessage(message: []const u8, reply: []u8, sender: u32, arrived: *ipc.Arrival) usize {
    capability_claimed = false;
    const written = Serve.dispatch({}, handlers, message, sender, arrived.peek(), reply);
    if (capability_claimed) _ = arrived.take();
    return written;
}

const handlers = Serve.Handlers{
    .open = onOpen,
    .control = onControl,
    .interrupt_subscribe = onInterruptSubscribe,
    .bulk = onBulk,
    .dma_attach = onDmaAttach,
};

/// open: the target is the class driver's assigned device id. Resolve it to an
/// interface, remember the caller's endpoint (for interrupt reports), and answer
/// with a device token — the target of every later packet — plus the interface's
/// endpoints, so the class driver need not re-read the configuration descriptor.
fn onOpen(_: void, invocation: Invocation(void), answer: Answer(usb_transfer_protocol.Opened)) isize {
    const engine = if (controller) |*c| c else return refused;
    const found = engine.findInterface(invocation.target) orelse return refused;
    // A device another live client holds is refused exactly as an absent one: one
    // device, one class driver. The predecessor's slot is released by the exit
    // sweep, and notifications are delivered ahead of requests, so a *restarted*
    // driver's open always finds the device free.
    if (!openedBy(invocation.target, invocation.sender) and heldByAnother(invocation.target)) return refused;

    // The report endpoint is claimed only if the open table actually keeps it;
    // a full table leaves it to the turn to close.
    if (invocation.capability) |endpoint| {
        if (recordOpen(invocation.target, invocation.sender, endpoint)) capability_claimed = true;
    }

    var opened = usb_transfer_protocol.Opened{
        .device_token = invocation.target,
        .endpoint_count = found.interface.endpoint_count,
        .interface_class = found.interface.class,
        .interface_subclass = found.interface.subclass,
        .interface_protocol = found.interface.protocol,
        .interface_number = found.interface.number,
    };
    const count = @min(found.interface.endpoint_count, usb_transfer_protocol.max_reported_endpoints);
    for (found.interface.endpoints[0..count], 0..) |endpoint, index| {
        opened.endpoints[index] = .{
            .address = endpoint.address,
            .transfer_type = endpoint.transfer_type,
            .max_packet_size = endpoint.max_packet_size,
            .interval = endpoint.interval,
        };
    }
    answer.set(opened);
    return 0;
}

/// control: one EP0 control transfer. The data stage rides the tail in both
/// directions, so an IN transfer's answer is simply however many bytes were
/// written into `answer.tail()` — which is what `Status.len` then reports.
fn onControl(_: void, invocation: Invocation(usb_transfer_protocol.Control), answer: Answer(void)) isize {
    const engine = if (controller) |*c| c else return refused;
    if (!openedBy(invocation.target, invocation.sender)) return refused;
    const found = engine.findInterface(invocation.target) orelse return refused;

    const setup = std.mem.bytesToValue(usb_abi.Request, &invocation.request.setup);
    const direction_in = invocation.request.direction_in != 0;
    const room = @min(usb_transfer_protocol.max_inline_data, answer.tail().len);
    const data_length = @min(@as(usize, invocation.request.data_length), room);

    var data: [usb_transfer_protocol.max_inline_data]u8 = undefined;
    if (!direction_in) {
        const supplied = @min(data_length, invocation.tail.len);
        @memcpy(data[0..supplied], invocation.tail[0..supplied]);
        if (supplied < data_length) @memset(data[supplied..data_length], 0);
    }

    if (!engine.controlTransfer(found.device, setup, data[0..data_length], direction_in)) return refused;
    if (!direction_in) return 0; // nothing follows an OUT: the status is the whole answer
    @memcpy(answer.tail()[0..data_length], data[0..data_length]);
    return @intCast(data_length);
}

/// interrupt_subscribe: arm periodic IN polling; reports flow back asynchronously
/// to the endpoint this device's `open` handed over.
fn onInterruptSubscribe(_: void, invocation: Invocation(usb_transfer_protocol.InterruptSubscribe), _: Answer(void)) isize {
    const engine = if (controller) |*c| c else return refused;
    if (!openedBy(invocation.target, invocation.sender)) return refused;
    const found = engine.findInterface(invocation.target) orelse return refused;
    const endpoint = library.Controller.endpointForAddress(found.interface, invocation.request.endpoint_address) orelse return refused;
    const report_endpoint = reportEndpointFor(invocation.target) orelse return refused;
    return if (engine.subscribeInterrupt(found.device, endpoint, invocation.target, report_endpoint)) 0 else refused;
}

/// bulk: one bulk transfer to/from the class driver's own DMA buffer (by physical
/// address), so sector-sized data never crosses IPC.
fn onBulk(_: void, invocation: Invocation(usb_transfer_protocol.Bulk), answer: Answer(usb_transfer_protocol.Transferred)) isize {
    const engine = if (controller) |*c| c else return refused;
    if (!openedBy(invocation.target, invocation.sender)) return refused;
    const found = engine.findInterface(invocation.target) orelse return refused;
    const endpoint = library.Controller.endpointForAddress(found.interface, invocation.request.endpoint_address) orelse return refused;
    const transferred = engine.bulkTransfer(found.device, endpoint, invocation.request.physical_address, invocation.request.length) orelse return refused;
    answer.set(.{ .actual_length = transferred });
    return 0;
}

/// dma_attach: bind the class driver's DMA-region capability into the controller's IOMMU
/// domain, so the controller may DMA to the physical addresses inside that buffer. The
/// binding holds its own kernel reference, so this never claims the arriving handle —
/// the turn's `defer` in the harness is the close, on the failure paths as well as this
/// one.
fn onDmaAttach(_: void, invocation: Invocation(void), _: Answer(void)) isize {
    const handle = invocation.capability orelse return -envelope.EPROTO;
    return if (device.dmaBind(controller_id, handle)) 0 else refused;
}

/// A timer tick or an MSI landed: drain the event ring, reconcile ports, and fan out.
/// The timer arm re-arms itself (8 ms drain when polling, 250 ms reconcile under MSI);
/// the MSI arm clears the interrupter's pending bit FIRST, then drains — so an event
/// arriving after the drain takes IP 0→1 and fires a fresh edge instead of being
/// swallowed until the reconcile tick.
fn onNotification(badge: u64) void {
    if (badge & ipc.notify_exit_bit != 0) {
        // A class driver died: release the devices it held, so its successor can
        // open them and no report is aimed at an endpoint that is gone.
        releaseOpensOf(@intCast(badge & ~(ipc.notify_badge_bit | ipc.notify_exit_bit)));
        return;
    }
    if (badge & ipc.notify_timer_bit != 0) {
        serviceController();
        _ = time.timerOnce(service_endpoint, timerInterval());
        return;
    }
    const vector = msi_vector orelse return;
    if (badge & ~ipc.notify_badge_bit != vector) return;
    if (controller) |*engine| engine.acknowledgeInterrupt();
    serviceController();
}

/// Everything one servicing pass does, shared verbatim by the poll/reconcile tick and
/// the MSI notification: drain the event ring, reconcile root ports, service hub
/// changes, and push interrupt reports to their class drivers.
fn serviceController() void {
    if (controller) |*engine| {
        engine.pump();
        // Poll every root port and reconcile — a device present but not yet
        // enumerated is brought up; a device gone is torn down. This does NOT
        // depend on a Port Status Change EVENT firing: the boot scan runs ~3 ms
        // after the controller reset, far too early for a USB2 connection to
        // debounce (~100 ms), and the SuperSpeed devices that DO show up early
        // proved the event path unreliable for the late USB2 companion hub on
        // real hardware. Polling catches it on the next tick regardless.
        if (manager_handle) |manager| {
            var port: u32 = 1;
            while (port <= engine.max_ports and port <= prev_connected.len) : (port += 1) {
                const connected = engine.portConnected(port);
                const was = prev_connected[port];
                prev_connected[port] = connected;
                if (connected and !was and engine.deviceOnPort(port) == null) {
                    // Rising edge the boot scan missed (it ran before the USB2
                    // connection debounced): bring the device up now.
                    std.log.info("root port {d}: device appeared (PORTSC 0x{x:0>8})", .{ port, engine.portStatus(port) });
                    bringUpPort(manager, engine, port);
                } else if (!connected and was and engine.deviceOnPort(port) != null) {
                    tearDownPort(manager, engine, port);
                }
            }
        }
        while (engine.takePortChange()) |port| {
            const manager = manager_handle orelse break;
            const connected = engine.portConnected(port);
            std.log.info("root port {d} change: {s} (PORTSC 0x{x:0>8})", .{ port, if (connected) "connected" else "empty", engine.portStatus(port) });
            if (connected) {
                if (engine.deviceOnPort(port) == null) bringUpPort(manager, engine, port);
            } else {
                tearDownPort(manager, engine, port);
            }
        }
        // Downstream hub-port changes (docs/usb-hub.md): a device connected on a
        // hub's downstream port is enumerated and registered here, so a keyboard
        // behind a hub reaches its class driver like one on a root port.
        // Cap per tick: even if a hub's change bits refuse to clear, the driver
        // must not spin here — it services a bounded batch and yields to the
        // next tick (and to storage, input, everything else).
        var serviced: u32 = 0;
        while (engine.takeHubChange()) |change| {
            const manager = manager_handle orelse break;
            bringUpBehindHub(manager, engine, change.hub, change.port);
            serviced += 1;
            if (serviced >= 32) break;
        }
        while (engine.takeReport()) |report| {
            // One `interrupt_report` event packet: the device token in the folded
            // header, the report after it. A device that produced more than the
            // push floor admits has its report truncated here, never split.
            const n = @min(report.length, usb_transfer_protocol.max_report_data);
            var payload = usb_transfer_protocol.InterruptReport{
                .endpoint_address = report.endpoint_address,
                .length = @intCast(n),
            };
            @memcpy(payload.data[0..n], report.data[0..n]);
            var packet: [envelope.post_maximum]u8 = undefined;
            const framed = usb_transfer_protocol.Protocol.encodeEvent(.interrupt_report, report.device_token, payload, &packet) orelse continue;
            _ = ipc.send(report.report_endpoint, framed);
        }
    }
}

pub fn main(init: process.Init) void {
    const argument = init.arguments.get(1) orelse {
        _ = logging.write("/system/drivers/usb-xhci-bus: missing controller device id (argv[1])\n");
        return;
    };
    controller_id = std.fmt.parseInt(u64, argument, 10) catch {
        std.log.info("malformed controller device id '{s}'", .{argument});
        return;
    };
    service.run(usb_transfer_protocol.message_maximum, .{
        // No `.service`: the contract is bound inside `initialise`, where losing
        // it to another controller's driver is survivable rather than fatal.
        .init = initialise,
        .on_message = onMessage,
        .on_notification = onNotification,
    });
}