danos/system/drivers/usb-xhci-bus/usb-xhci-library.zig

//! The xHCI (USB 3) host-controller engine: controller bring-up, the command
//! and event rings, device slots, and transfers. This is the hardware half of
//! the /system/drivers/usb-xhci-bus driver — usb-xhci-bus.zig is the ring-3
//! process shell (claim, MMIO map, device-manager handshake, IPC dispatch) and
//! calls into a `Controller` here for everything that touches registers or DMA.
//!
//! References: the xHCI 1.2 specification (register layout §5, TRBs §6, the
//! bring-up sequence §4.2). QEMU's `qemu-xhci` is the target; where the spec
//! allows latitude the simplest compliant choice is taken (a single-segment
//! event ring, a single-segment command ring with a Link TRB, polled event
//! delivery rather than MSI — see the notes on each).
//!
//! Interrupt strategy: **polling**. The event ring is coherent DMA the CPU can
//! read directly, so enumeration spin-polls it and the running driver drains it
//! on a timer tick. `qemu-xhci` presents MSI-X (a BAR-resident table) while the
//! kernel's `msi_bind` returns a config-space MSI capability's (address, data);
//! rather than gamble on the capability layout, we poll — correct on QEMU and
//! any real controller, with a marked hook to add MSI once the poll path is
//! proven.

const std = @import("std");
const runtime = @import("runtime");
const mmio = @import("mmio");
const usb_abi = @import("usb-abi");
const dma = runtime.dma;
const system = runtime.system;

// --- register offsets -------------------------------------------------------

// Capability registers (at the mapped BAR base).
const cap_caplength = 0x00; // low byte: length of the capability region
const cap_hcsparams1 = 0x04; // MaxSlots[7:0], MaxIntrs[18:8], MaxPorts[31:24]
const cap_hcsparams2 = 0x08; // Max Scratchpad Buffers (Hi[25:21], Lo[31:27])
const cap_hccparams1 = 0x10; // CSZ[2] = 64-byte contexts, xECP[31:16]
const cap_dboff = 0x14; // doorbell array offset (dword-aligned, bits[31:2])
const cap_rtsoff = 0x18; // runtime register space offset (bits[31:5])

// Operational registers (at base + CAPLENGTH).
const op_usbcmd = 0x00; // R/S[0], HCRST[1], INTE[2]
const op_usbsts = 0x04; // HCH[0], EINT[3], PCD[4], CNR[11]
const op_dcbaap = 0x30; // Device Context Base Address Array Pointer (64-bit)
const op_config = 0x38; // MaxSlotsEn[7:0]
const op_crcr = 0x18; // Command Ring Control Register (64-bit)
const op_portsc_base = 0x400; // PORTSC[n] = op + 0x400 + 0x10*(n-1)
const op_portsc_stride = 0x10;

// USBCMD / USBSTS bits.
const usbcmd_run = 1 << 0;
const usbcmd_reset = 1 << 1;
const usbcmd_interrupter_enable = 1 << 2;
const usbsts_halted = 1 << 0;
const usbsts_controller_not_ready = 1 << 11;

// PORTSC bits (per port). Several are write-1-to-clear (PED and the change bits),
// so any read-modify-write must write those as 0 to avoid clobbering them.
const portsc_connected = 1 << 0; // CCS (current connect status, read-only)
const portsc_enabled = 1 << 1; // PED (write 1 disables — write 0 to preserve)
const portsc_reset = 1 << 4; // PR (write 1 to reset the port)
const portsc_power = 1 << 9; // PP (port power, read-write)
const portsc_reset_change = 1 << 21; // PRC (write 1 to clear)
const portsc_change_mask: u32 = 0x7F << 17; // CSC..CEC change bits (write 1 clears)
// The write-1-to-clear bits a read-modify-write must not disturb: PED + changes.
const portsc_write_1_to_clear: u32 = portsc_enabled | portsc_change_mask;

// Runtime registers (at base + RTSOFF). Interrupter 0 lives at +0x20.
const runtime_interrupter0 = 0x20;
const interrupter_management = 0x00; // IMAN: IP[0], IE[1]
const interrupter_moderation = 0x04; // IMOD
const event_ring_segment_table_size = 0x08; // ERSTSZ
const event_ring_segment_table_base = 0x10; // ERSTBA (64-bit)
const event_ring_dequeue_pointer = 0x18; // ERDP (64-bit), EHB = bit 3

// A Transfer Request Block: 16 bytes, the unit of every ring. `control` carries
// the cycle bit (bit 0) and the TRB type (bits 15:10); the rest is type-specific.
pub const Trb = extern struct {
    parameter: u64 = 0,
    status: u32 = 0,
    control: u32 = 0,
};

pub const TrbType = enum(u6) {
    normal = 1,
    setup_stage = 2,
    data_stage = 3,
    status_stage = 4,
    link = 6,
    enable_slot = 9,
    address_device = 11,
    configure_endpoint = 12,
    evaluate_context = 13,
    no_op_command = 23,
    transfer_event = 32,
    command_completion_event = 33,
    port_status_change_event = 34,
};

// The completion code carried in a Command Completion / Transfer Event's status
// field (bits 31:24). Only the ones the driver reasons about are named.
pub const CompletionCode = enum(u8) {
    invalid = 0,
    success = 1,
    short_packet = 13,
    _,
};

const cycle_bit: u32 = 1 << 0;
const link_toggle_cycle: u32 = 1 << 1;

fn trbControl(kind: TrbType, extra: u32) u32 {
    return (@as(u32, @intFromEnum(kind)) << 10) | extra;
}

fn trbType(control: u32) u6 {
    return @truncate(control >> 10);
}

fn completionCode(status: u32) u8 {
    return @truncate(status >> 24);
}

// One Event Ring Segment Table entry: base + size of a single event-ring segment.
const ErstEntry = extern struct {
    ring_segment_base: u64 = 0,
    ring_segment_size: u32 = 0, // low 16 bits used
    reserved: u32 = 0,
};

const page_size = 4096;
const trbs_per_ring = page_size / @sizeOf(Trb); // 256

// A producer ring (command ring, or a transfer ring): a page of TRBs whose last
// entry is a Link TRB back to the start. `cycle` is the producer cycle state.
const ProducerRing = struct {
    region: dma.Region,
    enqueue_index: usize = 0,
    cycle: bool = true,

    fn trbs(self: ProducerRing) [*]volatile Trb {
        return @ptrFromInt(self.region.virtual);
    }

    // Arm the Link TRB at the end of the ring to point back to the start, with
    // the Toggle-Cycle bit so the producer/consumer cycle state flips on wrap.
    fn installLink(self: *ProducerRing) void {
        const link = &self.trbs()[trbs_per_ring - 1];
        link.parameter = self.region.physical;
        link.status = 0;
        link.control = trbControl(.link, link_toggle_cycle) | (if (self.cycle) cycle_bit else 0);
    }

    // Enqueue `trb` (its cycle bit is set here), returning the physical address
    // of the slot it landed in — the token a completion event echoes back.
    fn push(self: *ProducerRing, trb: Trb) u64 {
        const index = self.enqueue_index;
        const slot = &self.trbs()[index];
        const control = (trb.control & ~cycle_bit) | (if (self.cycle) cycle_bit else 0);
        // Fill the payload first, publish the cycle bit last: the controller
        // treats the TRB as owned once the cycle bit matches, so `control` (which
        // holds it) is written after `parameter`/`status`, with a barrier between.
        slot.parameter = trb.parameter;
        slot.status = trb.status;
        mmio.wmb();
        slot.control = control;
        const physical = self.region.physical + index * @sizeOf(Trb);
        self.enqueue_index += 1;
        if (self.enqueue_index == trbs_per_ring - 1) {
            // Reached the Link slot: hand the ring back and flip the producer
            // cycle. (Commands are infrequent — a boot enumeration issues a few
            // dozen — so in practice the ring never wraps; this keeps it correct
            // if it ever does.)
            self.installLink();
            self.enqueue_index = 0;
            self.cycle = !self.cycle;
        }
        return physical;
    }
};

// The event ring: a single segment the controller fills and the driver drains.
// `cycle` is the consumer cycle state, flipped each time the dequeue wraps.
const EventRing = struct {
    segment: dma.Region,
    table: dma.Region,
    dequeue_index: usize = 0,
    cycle: bool = true,

    fn trbs(self: EventRing) [*]volatile Trb {
        return @ptrFromInt(self.segment.virtual);
    }
};

// The default max packet size for endpoint 0, by the PORTSC/Slot-Context speed
// class (1 Full, 2 Low, 3 High, 4/5 SuperSpeed[+]). A device may report a smaller
// one in its descriptor; QEMU's HID/MSC devices use these defaults.
fn defaultMaxPacketSize0(speed: u32) u32 {
    return switch (speed) {
        2 => 8, // Low-speed
        1 => 8, // Full-speed (may be 8/16/32/64; 8 is the safe default)
        3 => 64, // High-speed
        4, 5 => 512, // SuperSpeed / SuperSpeedPlus
        else => 8,
    };
}

// The Doorbell Context Index of an endpoint: EP0 is 1, and endpoint number `n`
// with direction gives DCI = 2n + (IN ? 1 : 0). The classic off-by-one lives here.
pub fn doorbellContextIndex(endpoint_number: u8, direction_in: bool) u32 {
    return 2 * @as(u32, endpoint_number) + @intFromBool(direction_in);
}

// The xHCI Endpoint Context Interval field for an interrupt endpoint. High/Super
// speed encode the descriptor's bInterval as 2^(bInterval-1) microframes, so the
// field is bInterval-1 (clamped). Full/low speed use a conservative default (the
// exact microframe encoding is not needed for the polled boot devices QEMU shows).
fn intervalFor(speed: u32, b_interval: u8) u32 {
    return switch (speed) {
        3, 4, 5 => if (b_interval == 0) 0 else @min(@as(u32, b_interval) - 1, 15),
        else => 6,
    };
}

// Upper bounds on what one device's active configuration describes. A boot
// keyboard or mouse has one interface with one interrupt endpoint; a flash drive
// has one interface with two bulk endpoints. Generous for those.
pub const max_interfaces = 4;
pub const max_endpoints_per_interface = 4;

// The endpoint-descriptor facts a class driver needs to talk to an endpoint: its
// address (direction + number), transfer type, packet size, and poll interval.
pub const EndpointInfo = struct {
    address: u8 = 0, // EndpointDescriptor.Address bit-cast (dir bit 7, number bits 3:0)
    transfer_type: u8 = 0, // 0 control, 1 isochronous, 2 bulk, 3 interrupt
    max_packet_size: u16 = 0,
    interval: u8 = 0,
};

// One interface of a device's active configuration: its class triple and its
// endpoints (alternate setting 0 only — the boot devices have no alternates).
pub const InterfaceInfo = struct {
    number: u8 = 0,
    class: u8 = 0,
    subclass: u8 = 0,
    protocol: u8 = 0,
    endpoint_count: u8 = 0,
    endpoints: [max_endpoints_per_interface]EndpointInfo = [_]EndpointInfo{.{}} ** max_endpoints_per_interface,
    // The kernel device id this interface was registered as (its class driver's
    // argv[1]); a class driver opens the interface by presenting this id.
    registered_device_id: u64 = 0,
};

// A transfer ring the driver has configured for one of a device's endpoints
// (interrupt or bulk), keyed by its Doorbell Context Index.
const ConfiguredEndpoint = struct {
    dci: u32 = 0,
    ring: ProducerRing = .{ .region = .{ .virtual = 0, .physical = 0 } },
};
const max_configured_endpoints = max_interfaces * max_endpoints_per_interface;

// One addressed USB device behind this controller: its hardware slot, its EP0
// (control) transfer ring, the DMA context + bounce buffer the control pipe uses,
// and the interfaces its active configuration describes. Endpoint (interrupt/
// bulk) rings for those interfaces are added when a class driver opens it.
pub const Device = struct {
    used: bool = false,
    slot_id: u8 = 0,
    port: u32 = 0,
    speed: u32 = 0,
    max_packet_size_0: u32 = 8,
    input_context: dma.Region = .{ .virtual = 0, .physical = 0 },
    device_context: dma.Region = .{ .virtual = 0, .physical = 0 },
    ep0_ring: ProducerRing = .{ .region = .{ .virtual = 0, .physical = 0 } },
    // A page-sized bounce buffer for control-transfer data (descriptors are read
    // here, then copied out to the caller).
    control_buffer: dma.Region = .{ .virtual = 0, .physical = 0 },
    device_descriptor: usb_abi.DeviceDescriptor = std.mem.zeroes(usb_abi.DeviceDescriptor),
    configuration_value: u8 = 0,
    interface_count: u8 = 0,
    interfaces: [max_interfaces]InterfaceInfo = [_]InterfaceInfo{.{}} ** max_interfaces,
    // Transfer rings configured for this device's interrupt/bulk endpoints.
    endpoint_ring_count: u8 = 0,
    endpoint_rings: [max_configured_endpoints]ConfiguredEndpoint = [_]ConfiguredEndpoint{.{}} ** max_configured_endpoints,
};

// A standing interrupt-IN subscription: the endpoint's ring is kept armed with a
// Normal TRB pointing at `buffer`, and each report the device returns is copied
// into the controller's report queue for the bus layer to push to the subscriber.
const Subscription = struct {
    active: bool = false,
    slot_id: u8 = 0,
    dci: u32 = 0,
    endpoint_address: u8 = 0,
    ring: *ProducerRing = undefined,
    buffer: dma.Region = .{ .virtual = 0, .physical = 0 },
    max_length: u16 = 0,
    armed_trb_physical: u64 = 0,
    // The bus layer's per-subscription IPC state (opaque here): the class driver's
    // device token and the endpoint handle its reports are sent to.
    device_token: u64 = 0,
    report_endpoint: usize = 0,
};

// One interrupt report waiting for the bus layer to push it to a subscriber.
pub const Report = struct {
    report_endpoint: usize = 0,
    device_token: u64 = 0,
    endpoint_address: u8 = 0,
    length: u16 = 0,
    data: [64]u8 = [_]u8{0} ** 64,
};

// How many addressed devices this driver tracks at once. QEMU presents a handful
// (a keyboard, a mouse, a storage stick); a fuller machine would grow this.
const max_devices = 8;
const max_subscriptions = 8;
const report_queue_capacity = 16;

pub const Controller = struct {
    register_base: usize,
    op_base: usize,
    runtime_base: usize,
    doorbell_base: usize,
    max_slots: u32,
    max_ports: u32,
    context_size: usize, // 32 or 64 (CSZ)

    device_context_array: dma.Region,
    command_ring: ProducerRing,
    event_ring: EventRing,
    devices: [max_devices]Device = [_]Device{.{}} ** max_devices,
    subscriptions: [max_subscriptions]Subscription = [_]Subscription{.{}} ** max_subscriptions,
    report_queue: [report_queue_capacity]Report = [_]Report{.{}} ** report_queue_capacity,
    report_count: usize = 0,
    // Transferred length of the most recent awaited transfer (requested minus the
    // event residual); read right after a control or bulk transfer returns true.
    last_transfer_length: u32 = 0,

    // --- register access ---------------------------------------------------

    fn read8(address: usize) u8 {
        return @as(*volatile u8, @ptrFromInt(address)).*;
    }
    fn read32(address: usize) u32 {
        return @as(*volatile u32, @ptrFromInt(address)).*;
    }
    fn write32(address: usize, value: u32) void {
        @as(*volatile u32, @ptrFromInt(address)).* = value;
    }
    // xHCI 64-bit registers are safely accessed as an ordered pair of 32-bit
    // writes (low dword first) — the portable form some controllers require.
    fn write64(address: usize, value: u64) void {
        write32(address, @truncate(value));
        write32(address + 4, @truncate(value >> 32));
    }

    fn operational(self: *const Controller, offset: usize) usize {
        return self.op_base + offset;
    }
    fn interrupter(self: *const Controller, offset: usize) usize {
        return self.runtime_base + runtime_interrupter0 + offset;
    }

    // PORTSC for 1-based port `port`.
    pub fn portStatus(self: *const Controller, port: u32) u32 {
        return read32(self.op_base + op_portsc_base + op_portsc_stride * (port - 1));
    }
    pub fn writePortStatus(self: *const Controller, port: u32, value: u32) void {
        write32(self.op_base + op_portsc_base + op_portsc_stride * (port - 1), value);
    }

    // --- bring-up ----------------------------------------------------------

    /// Reset and start the controller, standing up the command and event rings.
    /// Returns null on any failure (a wedged register handshake or an out-of-DMA
    /// condition) — the caller treats that as a driver that could not start.
    pub fn init(register_base: usize) ?Controller {
        const cap_length = read32(register_base + cap_caplength) & 0xFF;
        const hcsparams1 = read32(register_base + cap_hcsparams1);
        const hccparams1 = read32(register_base + cap_hccparams1);
        const dboff = read32(register_base + cap_dboff) & ~@as(u32, 0x3);
        const rtsoff = read32(register_base + cap_rtsoff) & ~@as(u32, 0x1F);

        var self = Controller{
            .register_base = register_base,
            .op_base = register_base + cap_length,
            .runtime_base = register_base + rtsoff,
            .doorbell_base = register_base + dboff,
            .max_slots = hcsparams1 & 0xFF,
            .max_ports = hcsparams1 >> 24,
            .context_size = if (hccparams1 & (1 << 2) != 0) 64 else 32,
            .device_context_array = undefined,
            .command_ring = undefined,
            .event_ring = undefined,
        };

        // Wait for the controller to report ready, then halt it if it is running.
        if (!waitClear(self.operational(op_usbsts), usbsts_controller_not_ready)) return null;
        if (read32(self.operational(op_usbcmd)) & usbcmd_run != 0) {
            write32(self.operational(op_usbcmd), read32(self.operational(op_usbcmd)) & ~@as(u32, usbcmd_run));
            if (!waitSet(self.operational(op_usbsts), usbsts_halted)) return null;
        }

        // Reset. HCRST self-clears when the reset completes; then CNR clears.
        write32(self.operational(op_usbcmd), usbcmd_reset);
        if (!waitClear(self.operational(op_usbcmd), usbcmd_reset)) return null;
        if (!waitClear(self.operational(op_usbsts), usbsts_controller_not_ready)) return null;

        // Enable all device slots the controller supports.
        write32(self.operational(op_config), self.max_slots);

        // The Device Context Base Address Array (entry 0 = scratchpad array).
        self.device_context_array = dma.alloc(page_size, dma.coherent) orelse return null;
        self.setupScratchpad(register_base);
        write64(self.operational(op_dcbaap), self.device_context_array.physical);

        // The command ring: a page of TRBs, last entry a Link back to the start.
        self.command_ring = .{ .region = dma.alloc(page_size, dma.coherent) orelse return null };
        self.command_ring.installLink();
        write64(self.operational(op_crcr), self.command_ring.region.physical | cycle_bit);

        // The event ring: one segment + a one-entry segment table.
        self.event_ring = .{
            .segment = dma.alloc(page_size, dma.coherent) orelse return null,
            .table = dma.alloc(page_size, dma.coherent) orelse return null,
        };
        const table: *volatile ErstEntry = @ptrFromInt(self.event_ring.table.virtual);
        table.ring_segment_base = self.event_ring.segment.physical;
        table.ring_segment_size = trbs_per_ring;
        write32(self.interrupter(event_ring_segment_table_size), 1);
        write64(self.interrupter(event_ring_dequeue_pointer), self.event_ring.segment.physical);
        write64(self.interrupter(event_ring_segment_table_base), self.event_ring.table.physical);
        write32(self.interrupter(interrupter_moderation), 0);

        // Run. (Interrupts are left disabled — the event ring is polled.)
        mmio.wmb();
        write32(self.operational(op_usbcmd), read32(self.operational(op_usbcmd)) | usbcmd_run);
        if (!waitClear(self.operational(op_usbsts), usbsts_halted)) return null;
        return self;
    }

    // Scratchpad buffers the controller asks the host to reserve for its own use.
    // Entry 0 of the device-context array points at an array of their physical
    // addresses. QEMU usually requests none, in which case DCBAA[0] stays zero.
    fn setupScratchpad(self: *Controller, register_base: usize) void {
        const hcsparams2 = read32(register_base + cap_hcsparams2);
        const high = (hcsparams2 >> 21) & 0x1F;
        const low = (hcsparams2 >> 27) & 0x1F;
        const count = (high << 5) | low;
        const array: [*]volatile u64 = @ptrFromInt(self.device_context_array.virtual);
        if (count == 0) {
            array[0] = 0;
            return;
        }
        // One page per scratchpad buffer, plus a page holding their address array.
        const pointers = dma.alloc(page_size, dma.coherent) orelse return;
        const pointer_array: [*]volatile u64 = @ptrFromInt(pointers.virtual);
        var index: u32 = 0;
        while (index < count) : (index += 1) {
            const buffer = dma.alloc(page_size, dma.coherent) orelse return;
            pointer_array[index] = buffer.physical;
        }
        array[0] = pointers.physical;
    }

    // Spin (with a deadline) until every bit in `mask` reads back as zero / one.
    fn waitClear(address: usize, mask: u32) bool {
        const deadline = system.clock() + 1_000_000_000; // 1 s
        while (read32(address) & mask != 0) {
            if (system.clock() >= deadline) return false;
        }
        return true;
    }
    fn waitSet(address: usize, mask: u32) bool {
        const deadline = system.clock() + 1_000_000_000;
        while (read32(address) & mask == 0) {
            if (system.clock() >= deadline) return false;
        }
        return true;
    }

    // --- rings -------------------------------------------------------------

    fn ringDoorbell(self: *const Controller, slot: u32, target: u32) void {
        write32(self.doorbell_base + slot * 4, target);
    }

    /// Enqueue a command TRB, ring the command doorbell, and return the physical
    /// address of the enqueued TRB (which the Command Completion Event echoes).
    fn submitCommand(self: *Controller, trb: Trb) u64 {
        const physical = self.command_ring.push(trb);
        mmio.wmb();
        self.ringDoorbell(0, 0); // doorbell 0, target 0 = command ring
        return physical;
    }

    /// Consume the next event, or null if none has arrived by `deadline_ns`.
    fn nextEvent(self: *Controller, deadline_ns: u64) ?Trb {
        const ring = self.event_ring.trbs();
        while (true) {
            const slot = &ring[self.event_ring.dequeue_index];
            const control = slot.control;
            if ((control & cycle_bit != 0) == self.event_ring.cycle) {
                mmio.rmb();
                const event = Trb{ .parameter = slot.parameter, .status = slot.status, .control = control };
                self.event_ring.dequeue_index += 1;
                if (self.event_ring.dequeue_index >= trbs_per_ring) {
                    self.event_ring.dequeue_index = 0;
                    self.event_ring.cycle = !self.event_ring.cycle;
                }
                const dequeue = self.event_ring.segment.physical + self.event_ring.dequeue_index * @sizeOf(Trb);
                // Write the new dequeue pointer and clear the Event Handler Busy bit.
                write64(self.interrupter(event_ring_dequeue_pointer), dequeue | (1 << 3));
                return event;
            }
            if (system.clock() >= deadline_ns) return null;
        }
    }

    /// Wait for the Command Completion Event matching `command_physical`. Interrupt
    /// reports that land while waiting are dispatched (so a standing subscription is
    /// serviced even during a command); other events are ignored. Returns the
    /// completion code, or null on timeout.
    fn awaitCommand(self: *Controller, command_physical: u64) ?u8 {
        const deadline = system.clock() + 1_000_000_000;
        while (true) {
            const event = self.nextEvent(deadline) orelse return null;
            const kind = trbType(event.control);
            if (kind == @intFromEnum(TrbType.command_completion_event) and
                (event.parameter & ~@as(u64, 0xF)) == command_physical)
            {
                return completionCode(event.status);
            }
            if (kind == @intFromEnum(TrbType.transfer_event)) _ = self.serviceInterruptEvent(event);
        }
    }

    /// A No-Op Command: the cheapest end-to-end proof that reset, the command
    /// ring, the event ring, the doorbell, and the cycle-bit bookkeeping are all
    /// correct. Returns true if the controller completed it with success.
    pub fn noOpCommand(self: *Controller) bool {
        const physical = self.submitCommand(.{ .control = trbControl(.no_op_command, 0) });
        const code = self.awaitCommand(physical) orelse return false;
        return code == @intFromEnum(CompletionCode.success);
    }

    // --- device slots + control transfers ----------------------------------

    /// Reset the given 1-based root-hub port and wait for it to enable. A port
    /// must be reset before the device on it can be addressed. Returns false if
    /// the reset does not complete or the port does not enable.
    pub fn resetPort(self: *const Controller, port: u32) bool {
        // Set PR while writing 0 to every write-1-to-clear bit (so the change
        // bits and PED are untouched) and preserving PP.
        const before = self.portStatus(port);
        self.writePortStatus(port, (before & ~portsc_write_1_to_clear) | portsc_reset);
        const deadline = system.clock() + 500_000_000;
        while (self.portStatus(port) & portsc_reset_change == 0) {
            if (system.clock() >= deadline) return false;
        }
        // Clear the Port Reset Change bit (write 1 to PRC, 0 to the rest).
        const after = self.portStatus(port);
        self.writePortStatus(port, (after & ~portsc_write_1_to_clear) | portsc_reset_change);
        return self.portStatus(port) & portsc_enabled != 0;
    }

    /// Issue an Enable Slot command and return the slot id the controller
    /// assigned (carried in bits 31:24 of the completion event's control field).
    fn enableSlot(self: *Controller) ?u8 {
        const physical = self.submitCommand(.{ .control = trbControl(.enable_slot, 0) });
        const deadline = system.clock() + 1_000_000_000;
        while (true) {
            const event = self.nextEvent(deadline) orelse return null;
            if (trbType(event.control) == @intFromEnum(TrbType.command_completion_event) and
                (event.parameter & ~@as(u64, 0xF)) == physical)
            {
                if (completionCode(event.status) != @intFromEnum(CompletionCode.success)) return null;
                return @truncate(event.control >> 24);
            }
        }
    }

    fn allocateDevice(self: *Controller) ?*Device {
        for (&self.devices) |*device| {
            if (!device.used) return device;
        }
        return null;
    }

    // A pointer to dword `dword_index` of context `context_index` within a context
    // array at `virtual`. Contexts are strided by `context_size` (32 or 64), so
    // the meaningful first 8 dwords sit at the base of each stride.
    fn contextDword(virtual: usize, context_index: usize, dword_index: usize, context_size: usize) *volatile u32 {
        return @ptrFromInt(virtual + context_index * context_size + dword_index * 4);
    }

    // Build the Input Context for Address Device: Input Control Context add-flags
    // A0 (slot) | A1 (EP0), a Slot Context (route 0, speed, one context entry,
    // root-hub port), and an EP0 Control endpoint context pointing at the device's
    // EP0 ring. Contexts start zeroed (the DMA region is), so only set fields.
    fn buildAddressInputContext(self: *Controller, device: *Device) void {
        const cs = self.context_size;
        const base = device.input_context.virtual;
        // Input Control Context (index 0): Add flags in dword 1 = A0 | A1.
        contextDword(base, 0, 1, cs).* = 0b11;
        // Slot Context (index 1): speed[23:20], Context Entries[31:27] = 1.
        contextDword(base, 1, 0, cs).* = (device.speed << 20) | (@as(u32, 1) << 27);
        // Root Hub Port Number[23:16].
        contextDword(base, 1, 1, cs).* = device.port << 16;
        // EP0 Context (index 2): CErr[2:1]=3, EP Type[5:3]=Control(4), MPS[31:16].
        contextDword(base, 2, 1, cs).* = (@as(u32, 3) << 1) | (@as(u32, 4) << 3) | (device.max_packet_size_0 << 16);
        // TR Dequeue Pointer (dwords 2:3) with Dequeue Cycle State = 1.
        const dequeue = device.ep0_ring.region.physical | 1;
        contextDword(base, 2, 2, cs).* = @truncate(dequeue);
        contextDword(base, 2, 3, cs).* = @truncate(dequeue >> 32);
        // Average TRB Length (dword 4): 8 is the conventional value for control.
        contextDword(base, 2, 4, cs).* = 8;
    }

    fn addressDeviceCommand(self: *Controller, device: *Device) bool {
        const physical = self.submitCommand(.{
            .parameter = device.input_context.physical,
            .control = trbControl(.address_device, @as(u32, device.slot_id) << 24),
        });
        const code = self.awaitCommand(physical) orelse return false;
        return code == @intFromEnum(CompletionCode.success);
    }

    /// Reset the port, enable a slot, and address the device on it: after this the
    /// device answers control transfers on its EP0. Returns the tracked `Device`,
    /// or null on any failure. The EP0 MPS is taken from the speed default and
    /// corrected from the device descriptor by `refreshMaxPacketSize0` if needed.
    pub fn setupDevice(self: *Controller, port: u32, speed: u32) ?*Device {
        if (!self.resetPort(port)) return null;
        const slot_id = self.enableSlot() orelse return null;
        const device = self.allocateDevice() orelse return null;
        device.* = .{
            .used = true,
            .slot_id = slot_id,
            .port = port,
            .speed = speed,
            .max_packet_size_0 = defaultMaxPacketSize0(speed),
        };
        device.input_context = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device);
        device.device_context = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device);
        device.ep0_ring = .{ .region = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device) };
        device.ep0_ring.installLink();
        device.control_buffer = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device);

        self.buildAddressInputContext(device);
        const array: [*]volatile u64 = @ptrFromInt(self.device_context_array.virtual);
        array[device.slot_id] = device.device_context.physical;
        if (!self.addressDeviceCommand(device)) return self.abandon(device);
        return device;
    }

    fn abandon(self: *Controller, device: *Device) ?*Device {
        _ = self;
        device.used = false;
        return null;
    }

    fn awaitTransfer(self: *Controller, requested_length: u32) ?u8 {
        const deadline = system.clock() + 1_000_000_000;
        while (true) {
            const event = self.nextEvent(deadline) orelse return null;
            if (trbType(event.control) == @intFromEnum(TrbType.transfer_event)) {
                if (self.serviceInterruptEvent(event)) continue; // a subscription's report
                const residual = event.status & 0xFFFFFF;
                self.last_transfer_length = if (residual >= requested_length) 0 else requested_length - residual;
                return completionCode(event.status); // our transfer's completion (or error)
            }
        }
    }

    /// Run one control transfer on a device's EP0: a Setup stage (the 8-byte
    /// request inline), an optional Data stage through the device's bounce buffer,
    /// and a Status stage. For an IN transfer the returned data lands in `data`;
    /// for an OUT transfer `data` is sent. Returns false on any failure or stall.
    pub fn controlTransfer(self: *Controller, device: *Device, request: usb_abi.Request, data: []u8, direction_in: bool) bool {
        const has_data = data.len > 0;
        const transfer_type: u32 = if (!has_data) 0 else if (direction_in) 3 else 2; // TRT: none/OUT/IN

        // Setup Stage: the 8-byte setup packet inline (IDT), TRT in bits 17:16.
        const setup_bytes: u64 = @bitCast(request);
        _ = device.ep0_ring.push(.{
            .parameter = setup_bytes,
            .status = 8,
            .control = trbControl(.setup_stage, (1 << 6) | (transfer_type << 16)),
        });

        if (has_data) {
            if (!direction_in) {
                const buffer: [*]u8 = @ptrFromInt(device.control_buffer.virtual);
                @memcpy(buffer[0..data.len], data);
            }
            _ = device.ep0_ring.push(.{
                .parameter = device.control_buffer.physical,
                .status = @intCast(data.len),
                .control = trbControl(.data_stage, if (direction_in) (1 << 16) else 0), // DIR bit 16
            });
        }

        // Status Stage: opposite direction to the data (IN when there was no data
        // or the data was OUT), Interrupt On Completion so we get one event.
        const status_direction: u32 = if (has_data and direction_in) 0 else (1 << 16);
        _ = device.ep0_ring.push(.{
            .control = trbControl(.status_stage, status_direction | (1 << 5)), // DIR | IOC
        });

        mmio.wmb();
        self.ringDoorbell(device.slot_id, 1); // DCI 1 = EP0
        const code = self.awaitTransfer(@intCast(data.len)) orelse return false;
        if (code != @intFromEnum(CompletionCode.success) and code != @intFromEnum(CompletionCode.short_packet)) return false;

        if (has_data and direction_in) {
            const buffer: [*]u8 = @ptrFromInt(device.control_buffer.virtual);
            @memcpy(data, buffer[0..data.len]);
        }
        return true;
    }

    /// Read a device's 18-byte DEVICE descriptor over its control pipe.
    pub fn getDeviceDescriptor(self: *Controller, device: *Device) ?usb_abi.DeviceDescriptor {
        var bytes: [18]u8 = undefined;
        const request = usb_abi.getDescriptor(.device, 0, 0, 18);
        if (!self.controlTransfer(device, request, bytes[0..], true)) return null;
        return std.mem.bytesToValue(usb_abi.DeviceDescriptor, &bytes);
    }

    /// Full chapter-9 enumeration of an addressed device: read the device and
    /// configuration descriptors, parse the configuration's interfaces and
    /// endpoints into `device`, and select the configuration. After this the
    /// device is in the configured state and its interfaces are ready to match a
    /// class driver. Returns false on any control-transfer failure.
    pub fn enumerate(self: *Controller, device: *Device) bool {
        device.device_descriptor = self.getDeviceDescriptor(device) orelse return false;

        // The configuration descriptor's own 9 bytes carry the total length of
        // the whole configuration block (interfaces + endpoints follow it).
        var header: [9]u8 = undefined;
        if (!self.controlTransfer(device, usb_abi.getDescriptor(.configuration, 0, 0, 9), header[0..], true)) return false;
        const configuration = std.mem.bytesToValue(usb_abi.ConfigurationDescriptor, &header);
        device.configuration_value = @intFromEnum(configuration.configuration_value);

        // Read the whole block into a local buffer and parse it here (in the bus
        // driver) so the parse never has to cross the 256-byte IPC boundary.
        var blob: [512]u8 = undefined;
        const length = @min(configuration.total_length, blob.len);
        if (!self.controlTransfer(device, usb_abi.getDescriptor(.configuration, 0, 0, @intCast(length)), blob[0..length], true)) return false;
        parseConfiguration(device, blob[0..length]);

        // Select the configuration, moving the device to the configured state.
        if (!self.controlTransfer(device, usb_abi.setConfiguration(configuration.configuration_value), &.{}, false)) return false;
        return true;
    }

    // Walk a configuration block, recording each interface (alternate setting 0)
    // and the endpoints that follow it. Endpoints belong to the most recent
    // interface. Unknown descriptor types (HID, class-specific) are skipped by
    // their length.
    fn parseConfiguration(device: *Device, blob: []const u8) void {
        device.interface_count = 0;
        var current: ?*InterfaceInfo = null;
        var offset: usize = 0;
        while (offset + 2 <= blob.len) {
            const length = blob[offset];
            const descriptor_type = blob[offset + 1];
            if (length < 2 or offset + length > blob.len) break;
            switch (@as(usb_abi.DescriptorType, @enumFromInt(descriptor_type))) {
                .interface => if (length >= @sizeOf(usb_abi.InterfaceDescriptor)) {
                    const descriptor = std.mem.bytesToValue(usb_abi.InterfaceDescriptor, blob[offset .. offset + @sizeOf(usb_abi.InterfaceDescriptor)]);
                    if (@intFromEnum(descriptor.alternate_setting) != 0) {
                        current = null; // ignore alternate settings for now
                    } else if (device.interface_count < max_interfaces) {
                        const slot = &device.interfaces[device.interface_count];
                        slot.* = .{
                            .number = @intFromEnum(descriptor.interface_number),
                            .class = descriptor.interface_class,
                            .subclass = descriptor.interface_subclass,
                            .protocol = descriptor.interface_protocol,
                        };
                        current = slot;
                        device.interface_count += 1;
                    }
                },
                .endpoint => if (length >= @sizeOf(usb_abi.EndpointDescriptor)) {
                    if (current) |interface| {
                        if (interface.endpoint_count < max_endpoints_per_interface) {
                            const descriptor = std.mem.bytesToValue(usb_abi.EndpointDescriptor, blob[offset .. offset + @sizeOf(usb_abi.EndpointDescriptor)]);
                            interface.endpoints[interface.endpoint_count] = .{
                                .address = @bitCast(descriptor.endpoint_address),
                                .transfer_type = @intFromEnum(descriptor.attributes.transfer_type),
                                .max_packet_size = descriptor.max_packet_size.size,
                                .interval = descriptor.interval,
                            };
                            interface.endpoint_count += 1;
                        }
                    }
                },
                else => {},
            }
            offset += length;
        }
    }

    // --- endpoint configuration + interrupt / bulk transfers ---------------

    /// Find the tracked device and interface an assigned device id belongs to.
    pub fn findInterface(self: *Controller, device_id: u64) ?struct { device: *Device, interface: *InterfaceInfo } {
        for (&self.devices) |*device| {
            if (!device.used) continue;
            for (device.interfaces[0..device.interface_count]) |*interface| {
                if (interface.registered_device_id == device_id) return .{ .device = device, .interface = interface };
            }
        }
        return null;
    }

    /// The endpoint of `interface` with the given address, or null.
    pub fn endpointForAddress(interface: *const InterfaceInfo, address: u8) ?EndpointInfo {
        for (interface.endpoints[0..interface.endpoint_count]) |endpoint| {
            if (endpoint.address == address) return endpoint;
        }
        return null;
    }

    fn findEndpointRing(device: *Device, dci: u32) ?*ProducerRing {
        for (device.endpoint_rings[0..device.endpoint_ring_count]) |*configured| {
            if (configured.dci == dci) return &configured.ring;
        }
        return null;
    }

    // Get (configuring on first use) the transfer ring for an endpoint. The first
    // use issues a Configure Endpoint command that adds the endpoint context to the
    // device and points it at a fresh ring.
    fn getOrConfigureEndpoint(self: *Controller, device: *Device, endpoint: EndpointInfo) ?*ProducerRing {
        const number: u8 = endpoint.address & 0x0F;
        const direction_in = endpoint.address & 0x80 != 0;
        const dci = doorbellContextIndex(number, direction_in);
        if (findEndpointRing(device, dci)) |ring| return ring;
        if (device.endpoint_ring_count >= device.endpoint_rings.len) return null;

        const configured = &device.endpoint_rings[device.endpoint_ring_count];
        configured.dci = dci;
        configured.ring = .{ .region = dma.alloc(page_size, dma.coherent) orelse return null };
        configured.ring.installLink();
        self.buildConfigureEndpointInputContext(device, endpoint, dci, &configured.ring);
        if (!self.configureEndpointCommand(device)) return null;
        device.endpoint_ring_count += 1;
        return &configured.ring;
    }

    // Build the Input Context for a Configure Endpoint command adding one endpoint:
    // Add flags A0 (slot) | A(dci), a Slot Context whose Context Entries covers the
    // new endpoint, and the endpoint context (type, packet size, ring, interval).
    fn buildConfigureEndpointInputContext(self: *Controller, device: *Device, endpoint: EndpointInfo, dci: u32, ring: *const ProducerRing) void {
        const cs = self.context_size;
        const base = device.input_context.virtual;
        // Zero the contexts we touch (the region last held the Address Device input).
        const dwords: [*]volatile u32 = @ptrFromInt(base);
        const touched = (2 + @as(usize, dci)) * (cs / 4);
        var i: usize = 0;
        while (i < touched) : (i += 1) dwords[i] = 0;

        // Input Control Context: Add flags A0 (slot) | A(dci) (the endpoint).
        contextDword(base, 0, 1, cs).* = (@as(u32, 1) << 0) | (@as(u32, 1) << @as(u5, @intCast(dci)));
        // Slot Context: speed, Context Entries = dci, root hub port.
        contextDword(base, 1, 0, cs).* = (device.speed << 20) | (dci << 27);
        contextDword(base, 1, 1, cs).* = device.port << 16;
        // Endpoint Context at index (1 + dci).
        const direction_in = endpoint.address & 0x80 != 0;
        const endpoint_type = @as(u32, endpoint.transfer_type) + (if (direction_in) @as(u32, 4) else 0);
        const interval = if (endpoint.transfer_type == 3) intervalFor(device.speed, endpoint.interval) else 0;
        contextDword(base, 1 + @as(usize, dci), 0, cs).* = interval << 16; // Interval bits 23:16
        contextDword(base, 1 + @as(usize, dci), 1, cs).* = (@as(u32, 3) << 1) | (endpoint_type << 3) | (@as(u32, endpoint.max_packet_size) << 16);
        const dequeue = ring.region.physical | 1;
        contextDword(base, 1 + @as(usize, dci), 2, cs).* = @truncate(dequeue);
        contextDword(base, 1 + @as(usize, dci), 3, cs).* = @truncate(dequeue >> 32);
        contextDword(base, 1 + @as(usize, dci), 4, cs).* = endpoint.max_packet_size; // Average TRB Length
    }

    fn configureEndpointCommand(self: *Controller, device: *Device) bool {
        const physical = self.submitCommand(.{
            .parameter = device.input_context.physical,
            .control = trbControl(.configure_endpoint, @as(u32, device.slot_id) << 24),
        });
        const code = self.awaitCommand(physical) orelse return false;
        return code == @intFromEnum(CompletionCode.success);
    }

    /// One bulk transfer (IN or OUT per the endpoint address's direction bit) to or
    /// from a caller-owned DMA buffer at `physical`. Returns the number of bytes
    /// transferred, or null on failure/stall. The data never crosses IPC.
    pub fn bulkTransfer(self: *Controller, device: *Device, endpoint: EndpointInfo, physical: u64, length: u32) ?u32 {
        const ring = self.getOrConfigureEndpoint(device, endpoint) orelse return null;
        _ = ring.push(.{
            .parameter = physical,
            .status = length,
            .control = trbControl(.normal, (1 << 5)), // IOC
        });
        mmio.wmb();
        const number: u8 = endpoint.address & 0x0F;
        const direction_in = endpoint.address & 0x80 != 0;
        self.ringDoorbell(device.slot_id, doorbellContextIndex(number, direction_in));
        const code = self.awaitTransfer(length) orelse return null;
        if (code != @intFromEnum(CompletionCode.success) and code != @intFromEnum(CompletionCode.short_packet)) return null;
        return self.last_transfer_length;
    }

    fn allocateSubscription(self: *Controller) ?*Subscription {
        for (&self.subscriptions) |*subscription| {
            if (!subscription.active) return subscription;
        }
        return null;
    }

    /// Start periodic IN polling of an interrupt endpoint. Each report the device
    /// returns is queued (see `takeReport`), tagged with `device_token` and
    /// `report_endpoint` so the bus layer can push it to the subscriber. Returns
    /// false if the endpoint cannot be configured or no subscription slot is free.
    pub fn subscribeInterrupt(self: *Controller, device: *Device, endpoint: EndpointInfo, device_token: u64, report_endpoint: usize) bool {
        const ring = self.getOrConfigureEndpoint(device, endpoint) orelse return false;
        const subscription = self.allocateSubscription() orelse return false;
        const buffer = dma.alloc(page_size, dma.coherent) orelse return false;
        const number: u8 = endpoint.address & 0x0F;
        const direction_in = endpoint.address & 0x80 != 0;
        subscription.* = .{
            .active = true,
            .slot_id = device.slot_id,
            .dci = doorbellContextIndex(number, direction_in),
            .endpoint_address = endpoint.address,
            .ring = ring,
            .buffer = buffer,
            .max_length = endpoint.max_packet_size,
            .device_token = device_token,
            .report_endpoint = report_endpoint,
        };
        self.armInterrupt(subscription);
        return true;
    }

    // Arm (or re-arm) a subscription's endpoint with a Normal TRB pointing at its
    // report buffer, and ring the endpoint's doorbell so the controller polls it.
    fn armInterrupt(self: *Controller, subscription: *Subscription) void {
        subscription.armed_trb_physical = subscription.ring.push(.{
            .parameter = subscription.buffer.physical,
            .status = subscription.max_length,
            .control = trbControl(.normal, (1 << 5)), // IOC
        });
        mmio.wmb();
        self.ringDoorbell(subscription.slot_id, subscription.dci);
    }

    // A transfer event came off the ring: if it completes a subscription's armed
    // interrupt transfer, copy the report into the queue and re-arm. Returns whether
    // it belonged to a subscription (so the awaiting caller knows it was consumed).
    fn serviceInterruptEvent(self: *Controller, event: Trb) bool {
        const trb_pointer = event.parameter & ~@as(u64, 0xF);
        for (&self.subscriptions) |*subscription| {
            if (!subscription.active or subscription.armed_trb_physical != trb_pointer) continue;
            const code = completionCode(event.status);
            if (code == @intFromEnum(CompletionCode.success) or code == @intFromEnum(CompletionCode.short_packet)) {
                const residual = event.status & 0xFFFFFF;
                const transferred: u16 = if (residual >= subscription.max_length) 0 else @intCast(subscription.max_length - residual);
                self.enqueueReport(subscription, transferred);
            }
            self.armInterrupt(subscription); // keep polling
            return true;
        }
        return false;
    }

    fn enqueueReport(self: *Controller, subscription: *Subscription, length: u16) void {
        if (self.report_count >= self.report_queue.len) return; // full: drop the newest
        const report = &self.report_queue[self.report_count];
        report.report_endpoint = subscription.report_endpoint;
        report.device_token = subscription.device_token;
        report.endpoint_address = subscription.endpoint_address;
        report.length = length;
        const source: [*]const u8 = @ptrFromInt(subscription.buffer.virtual);
        const n = @min(length, report.data.len);
        @memcpy(report.data[0..n], source[0..n]);
        self.report_count += 1;
    }

    /// Dequeue the oldest queued interrupt report, or null if none.
    pub fn takeReport(self: *Controller) ?Report {
        if (self.report_count == 0) return null;
        const report = self.report_queue[0];
        var i: usize = 1;
        while (i < self.report_count) : (i += 1) self.report_queue[i - 1] = self.report_queue[i];
        self.report_count -= 1;
        return report;
    }

    /// Drain any events currently on the event ring, dispatching interrupt reports
    /// into the queue. Non-blocking — called on the driver's timer tick.
    pub fn pump(self: *Controller) void {
        while (true) {
            const event = self.nextEvent(system.clock()) orelse return; // deadline=now: null when empty
            if (trbType(event.control) == @intFromEnum(TrbType.transfer_event)) _ = self.serviceInterruptEvent(event);
        }
    }
};