//! The xHCI (USB 3) host-controller engine: controller bring-up, the command
//! and event rings, device slots, and transfers. This is the hardware half of
//! the /system/drivers/usb-xhci-bus driver — usb-xhci-bus.zig is the ring-3
//! process shell (claim, MMIO map, device-manager handshake, IPC dispatch) and
//! calls into a `Controller` here for everything that touches registers or DMA.
//!
//! References: the xHCI 1.2 specification (register layout §5, TRBs §6, the
//! bring-up sequence §4.2). QEMU's `qemu-xhci` is the target; where the spec
//! allows latitude the simplest compliant choice is taken (a single-segment
//! event ring, a single-segment command ring with a Link TRB, polled event
//! delivery rather than MSI — see the notes on each).
//!
//! Interrupt strategy: **polling**. The event ring is coherent DMA the CPU can
//! read directly, so enumeration spin-polls it and the running driver drains it
//! on a timer tick. `qemu-xhci` presents MSI-X (a BAR-resident table) while the
//! kernel's `msi_bind` returns a config-space MSI capability's (address, data);
//! rather than gamble on the capability layout, we poll — correct on QEMU and
//! any real controller, with a marked hook to add MSI once the poll path is
//! proven.

const std = @import("std");
const runtime = @import("runtime");
const mmio = @import("mmio");
const usb_abi = @import("usb-abi");
const dma = runtime.dma;
const system = runtime.system;

// --- register offsets -------------------------------------------------------

// Capability registers (at the mapped BAR base).
const cap_caplength = 0x00; // low byte: length of the capability region
const cap_hcsparams1 = 0x04; // MaxSlots[7:0], MaxIntrs[18:8], MaxPorts[31:24]
const cap_hcsparams2 = 0x08; // Max Scratchpad Buffers (Hi[25:21], Lo[31:27])
const cap_hccparams1 = 0x10; // CSZ[2] = 64-byte contexts, xECP[31:16]
const cap_dboff = 0x14; // doorbell array offset (dword-aligned, bits[31:2])
const cap_rtsoff = 0x18; // runtime register space offset (bits[31:5])

// Operational registers (at base + CAPLENGTH).
const op_usbcmd = 0x00; // R/S[0], HCRST[1], INTE[2]
const op_usbsts = 0x04; // HCH[0], EINT[3], PCD[4], CNR[11]
const op_dcbaap = 0x30; // Device Context Base Address Array Pointer (64-bit)
const op_config = 0x38; // MaxSlotsEn[7:0]
const op_crcr = 0x18; // Command Ring Control Register (64-bit)
const op_portsc_base = 0x400; // PORTSC[n] = op + 0x400 + 0x10*(n-1)
const op_portsc_stride = 0x10;

// USBCMD / USBSTS bits.
const usbcmd_run = 1 << 0;
const usbcmd_reset = 1 << 1;
const usbcmd_interrupter_enable = 1 << 2;
const usbsts_halted = 1 << 0;
const usbsts_controller_not_ready = 1 << 11;

// PORTSC bits (per port). Several are write-1-to-clear (PED and the change bits),
// so any read-modify-write must write those as 0 to avoid clobbering them.
const portsc_connected = 1 << 0; // CCS (current connect status, read-only)
const portsc_enabled = 1 << 1; // PED (write 1 disables — write 0 to preserve)
const portsc_reset = 1 << 4; // PR (write 1 to reset the port)
const portsc_power = 1 << 9; // PP (port power, read-write)
const portsc_reset_change = 1 << 21; // PRC (write 1 to clear)
const portsc_change_mask: u32 = 0x7F << 17; // CSC..CEC change bits (write 1 clears)
// The write-1-to-clear bits a read-modify-write must not disturb: PED + changes.
const portsc_write_1_to_clear: u32 = portsc_enabled | portsc_change_mask;

// Runtime registers (at base + RTSOFF). Interrupter 0 lives at +0x20.
const runtime_interrupter0 = 0x20;
const interrupter_management = 0x00; // IMAN: IP[0], IE[1]
const interrupter_moderation = 0x04; // IMOD
const event_ring_segment_table_size = 0x08; // ERSTSZ
const event_ring_segment_table_base = 0x10; // ERSTBA (64-bit)
const event_ring_dequeue_pointer = 0x18; // ERDP (64-bit), EHB = bit 3

// A Transfer Request Block: 16 bytes, the unit of every ring. `control` carries
// the cycle bit (bit 0) and the TRB type (bits 15:10); the rest is type-specific.
pub const Trb = extern struct {
    parameter: u64 = 0,
    status: u32 = 0,
    control: u32 = 0,
};

pub const TrbType = enum(u6) {
    normal = 1,
    setup_stage = 2,
    data_stage = 3,
    status_stage = 4,
    link = 6,
    enable_slot = 9,
    disable_slot = 10,
    address_device = 11,
    configure_endpoint = 12,
    evaluate_context = 13,
    no_op_command = 23,
    transfer_event = 32,
    command_completion_event = 33,
    port_status_change_event = 34,
};

// The completion code carried in a Command Completion / Transfer Event's status
// field (bits 31:24). Only the ones the driver reasons about are named.
pub const CompletionCode = enum(u8) {
    invalid = 0,
    success = 1,
    short_packet = 13,
    _,
};

const cycle_bit: u32 = 1 << 0;
const link_toggle_cycle: u32 = 1 << 1;

fn trbControl(kind: TrbType, extra: u32) u32 {
    return (@as(u32, @intFromEnum(kind)) << 10) | extra;
}

fn trbType(control: u32) u6 {
    return @truncate(control >> 10);
}

fn completionCode(status: u32) u8 {
    return @truncate(status >> 24);
}

// One Event Ring Segment Table entry: base + size of a single event-ring segment.
const ErstEntry = extern struct {
    ring_segment_base: u64 = 0,
    ring_segment_size: u32 = 0, // low 16 bits used
    reserved: u32 = 0,
};

const page_size = 4096;
const trbs_per_ring = page_size / @sizeOf(Trb); // 256

// A producer ring (command ring, or a transfer ring): a page of TRBs whose last
// entry is a Link TRB back to the start. `cycle` is the producer cycle state.
const ProducerRing = struct {
    region: dma.Region,
    enqueue_index: usize = 0,
    cycle: bool = true,

    fn trbs(self: ProducerRing) [*]volatile Trb {
        return @ptrFromInt(self.region.virtual);
    }

    // Arm the Link TRB at the end of the ring to point back to the start, with
    // the Toggle-Cycle bit so the producer/consumer cycle state flips on wrap.
    fn installLink(self: *ProducerRing) void {
        const link = &self.trbs()[trbs_per_ring - 1];
        link.parameter = self.region.physical;
        link.status = 0;
        link.control = trbControl(.link, link_toggle_cycle) | (if (self.cycle) cycle_bit else 0);
    }

    // Enqueue `trb` (its cycle bit is set here), returning the physical address
    // of the slot it landed in — the token a completion event echoes back.
    fn push(self: *ProducerRing, trb: Trb) u64 {
        const index = self.enqueue_index;
        const slot = &self.trbs()[index];
        const control = (trb.control & ~cycle_bit) | (if (self.cycle) cycle_bit else 0);
        // Fill the payload first, publish the cycle bit last: the controller
        // treats the TRB as owned once the cycle bit matches, so `control` (which
        // holds it) is written after `parameter`/`status`, with a barrier between.
        slot.parameter = trb.parameter;
        slot.status = trb.status;
        mmio.wmb();
        slot.control = control;
        const physical = self.region.physical + index * @sizeOf(Trb);
        self.enqueue_index += 1;
        if (self.enqueue_index == trbs_per_ring - 1) {
            // Reached the Link slot: hand the ring back and flip the producer
            // cycle. (Commands are infrequent — a boot enumeration issues a few
            // dozen — so in practice the ring never wraps; this keeps it correct
            // if it ever does.)
            self.installLink();
            self.enqueue_index = 0;
            self.cycle = !self.cycle;
        }
        return physical;
    }
};

// The event ring: a single segment the controller fills and the driver drains.
// `cycle` is the consumer cycle state, flipped each time the dequeue wraps.
const EventRing = struct {
    segment: dma.Region,
    table: dma.Region,
    dequeue_index: usize = 0,
    cycle: bool = true,

    fn trbs(self: EventRing) [*]volatile Trb {
        return @ptrFromInt(self.segment.virtual);
    }
};

// The default max packet size for endpoint 0, by the PORTSC/Slot-Context speed
// class (1 Full, 2 Low, 3 High, 4/5 SuperSpeed[+]). A device may report a smaller
// one in its descriptor; QEMU's HID/MSC devices use these defaults.
fn defaultMaxPacketSize0(speed: u32) u32 {
    return switch (speed) {
        2 => 8, // Low-speed
        1 => 8, // Full-speed (may be 8/16/32/64; 8 is the safe default)
        3 => 64, // High-speed
        4, 5 => 512, // SuperSpeed / SuperSpeedPlus
        else => 8,
    };
}

// The Doorbell Context Index of an endpoint: EP0 is 1, and endpoint number `n`
// with direction gives DCI = 2n + (IN ? 1 : 0). The classic off-by-one lives here.
pub fn doorbellContextIndex(endpoint_number: u8, direction_in: bool) u32 {
    return 2 * @as(u32, endpoint_number) + @intFromBool(direction_in);
}

// The xHCI Endpoint Context Interval field for an interrupt endpoint. High/Super
// speed encode the descriptor's bInterval as 2^(bInterval-1) microframes, so the
// field is bInterval-1 (clamped). Full/low speed use a conservative default (the
// exact microframe encoding is not needed for the polled boot devices QEMU shows).
fn intervalFor(speed: u32, b_interval: u8) u32 {
    return switch (speed) {
        3, 4, 5 => if (b_interval == 0) 0 else @min(@as(u32, b_interval) - 1, 15),
        else => 6,
    };
}

// Upper bounds on what one device's active configuration describes. A boot
// keyboard or mouse has one interface with one interrupt endpoint; a flash drive
// has one interface with two bulk endpoints. Generous for those.
pub const max_interfaces = 4;
pub const max_endpoints_per_interface = 4;

// The endpoint-descriptor facts a class driver needs to talk to an endpoint: its
// address (direction + number), transfer type, packet size, and poll interval.
pub const EndpointInfo = struct {
    address: u8 = 0, // EndpointDescriptor.Address bit-cast (dir bit 7, number bits 3:0)
    transfer_type: u8 = 0, // 0 control, 1 isochronous, 2 bulk, 3 interrupt
    max_packet_size: u16 = 0,
    interval: u8 = 0,
};

// One interface of a device's active configuration: its class triple and its
// endpoints (alternate setting 0 only — the boot devices have no alternates).
pub const InterfaceInfo = struct {
    number: u8 = 0,
    class: u8 = 0,
    subclass: u8 = 0,
    protocol: u8 = 0,
    endpoint_count: u8 = 0,
    endpoints: [max_endpoints_per_interface]EndpointInfo = [_]EndpointInfo{.{}} ** max_endpoints_per_interface,
    // The kernel device id this interface was registered as (its class driver's
    // argv[1]); a class driver opens the interface by presenting this id.
    registered_device_id: u64 = 0,
};

// A transfer ring the driver has configured for one of a device's endpoints
// (interrupt or bulk), keyed by its Doorbell Context Index.
const ConfiguredEndpoint = struct {
    dci: u32 = 0,
    ring: ProducerRing = .{ .region = .{ .virtual = 0, .physical = 0 } },
};
const max_configured_endpoints = max_interfaces * max_endpoints_per_interface;

// One addressed USB device behind this controller: its hardware slot, its EP0
// (control) transfer ring, the DMA context + bounce buffer the control pipe uses,
// and the interfaces its active configuration describes. Endpoint (interrupt/
// bulk) rings for those interfaces are added when a class driver opens it.
pub const Device = struct {
    used: bool = false,
    slot_id: u8 = 0,
    port: u32 = 0,
    speed: u32 = 0,
    max_packet_size_0: u32 = 8,
    input_context: dma.Region = .{ .virtual = 0, .physical = 0 },
    device_context: dma.Region = .{ .virtual = 0, .physical = 0 },
    ep0_ring: ProducerRing = .{ .region = .{ .virtual = 0, .physical = 0 } },
    // A page-sized bounce buffer for control-transfer data (descriptors are read
    // here, then copied out to the caller).
    control_buffer: dma.Region = .{ .virtual = 0, .physical = 0 },
    device_descriptor: usb_abi.DeviceDescriptor = std.mem.zeroes(usb_abi.DeviceDescriptor),
    configuration_value: u8 = 0,
    interface_count: u8 = 0,
    interfaces: [max_interfaces]InterfaceInfo = [_]InterfaceInfo{.{}} ** max_interfaces,
    // Transfer rings configured for this device's interrupt/bulk endpoints.
    endpoint_ring_count: u8 = 0,
    endpoint_rings: [max_configured_endpoints]ConfiguredEndpoint = [_]ConfiguredEndpoint{.{}} ** max_configured_endpoints,

    // Hub topology (docs/usb-hub.md). A device behind a hub is addressed with a
    // route string; these carry the fields buildAddressInputContext needs.
    route: u32 = 0, // Slot Context route string (5 tiers x 4 bits); 0 = on a root port
    root_port: u32 = 0, // the ROOT-hub port the chain hangs off (inherited down a chain)
    parent_slot: u8 = 0, // the parent hub's slot id (0 = on a root port) — the TT hub
    parent_port: u8 = 0, // the parent hub's downstream port this device sits on

    // Set when this device IS a hub, after setupHub configures it.
    is_hub: bool = false,
    hub_ports: u8 = 0, // downstream port count from the hub descriptor
    hub_multi_tt: bool = false,
    // Downstream ports with a pending change to service (bit P = port P), set
    // by the status-change endpoint (and by an initial sweep in setupHub).
    hub_change_mask: u32 = 0,
};

// A standing interrupt-IN subscription: the endpoint's ring is kept armed with a
// Normal TRB pointing at `buffer`, and each report the device returns is copied
// into the controller's report queue for the bus layer to push to the subscriber.
const Subscription = struct {
    active: bool = false,
    slot_id: u8 = 0,
    dci: u32 = 0,
    endpoint_address: u8 = 0,
    ring: *ProducerRing = undefined,
    buffer: dma.Region = .{ .virtual = 0, .physical = 0 },
    max_length: u16 = 0,
    armed_trb_physical: u64 = 0,
    // The bus layer's per-subscription IPC state (opaque here): the class driver's
    // device token and the endpoint handle its reports are sent to.
    device_token: u64 = 0,
    report_endpoint: usize = 0,
    // When set, this is an IN-PROCESS hub status-change subscription: completions
    // set the hub's pending-change mask instead of queuing a class-driver report.
    hub: ?*Device = null,
};

// One interrupt report waiting for the bus layer to push it to a subscriber.
pub const Report = struct {
    report_endpoint: usize = 0,
    device_token: u64 = 0,
    endpoint_address: u8 = 0,
    length: u16 = 0,
    data: [64]u8 = [_]u8{0} ** 64,
};

// How many addressed devices this driver tracks at once. QEMU presents a handful
// (a keyboard, a mouse, a storage stick); a fuller machine would grow this.
const max_devices = 8;
const max_subscriptions = 8;
const report_queue_capacity = 16;

// --- USB hub class requests + constants (docs/usb-hub.md) ------------------
//
// A hub is bus infrastructure the CONTROLLER driver handles in-process: the
// route strings and slot contexts a downstream device needs only exist here.
// These are the class-specific control requests to a hub device.

// A USB2 hub port's wPortStatus speed bits (bit 9 = low-speed, bit 10 =
// high-speed; neither = full-speed) mapped to the xHCI speed id.
fn mapHubPortSpeed(port_speed_bits: u32) u32 {
    if (port_speed_bits & 0x1 != 0) return 2; // low-speed (wPortStatus bit 9)
    if (port_speed_bits & 0x2 != 0) return 3; // high-speed
    return 1; // full-speed
}

fn routeDepth(route: u32) u16 {
    // Tiers used by a route string: each nonzero 4-bit nibble is one tier.
    var depth: u16 = 0;
    var r = route;
    while (r != 0) : (r >>= 4) {
        if (r & 0xF != 0) depth += 1;
    }
    return depth;
}

const hubreq = struct {
    // Hub descriptor types (GET_DESCRIPTOR value high byte).
    const descriptor_usb2: u8 = 0x29;
    const descriptor_usb3: u8 = 0x2A;

    // Hub/port feature selectors (SET_FEATURE / CLEAR_FEATURE value).
    const feature_port_reset: u16 = 4;
    const feature_port_power: u16 = 8;
    const feature_c_port_connection: u16 = 16;
    const feature_c_port_reset: u16 = 20;
    const feature_c_port_link_state: u16 = 25; // SS
    const feature_bh_port_reset: u16 = 28; // SS
    const feature_c_bh_port_reset: u16 = 29; // SS

    // Port status (wPortStatus, first 16 bits of the 4-byte GET_STATUS result).
    const status_connection: u16 = 1 << 0;
    const status_enable: u16 = 1 << 1;
    const status_reset: u16 = 1 << 4;
    // Port-status change bits (wPortChange, the high 16 bits).
    const change_connection: u16 = 1 << 0;
    const change_reset: u16 = 1 << 4;

    // wHubCharacteristics bit 7: multiple transaction translators.
    const characteristics_multi_tt: u16 = 1 << 7;

    // SET_HUB_DEPTH (SuperSpeed hubs, so they can compose route strings).
    const request_set_hub_depth: u8 = 12;

    fn getDescriptor(kind: u8, length: u16) usb_abi.Request {
        return .{
            .request_type = .{ .recipient = .device, .kind = .class, .direction = .device_to_host },
            .request_code = .get_descriptor,
            .value = @as(u16, kind) << 8,
            .index = 0,
            .length = length,
        };
    }

    fn setPortFeature(feature: u16, port: u16) usb_abi.Request {
        return .{
            .request_type = .{ .recipient = .other, .kind = .class, .direction = .host_to_device },
            .request_code = .set_feature,
            .value = feature,
            .index = port,
            .length = 0,
        };
    }

    fn clearPortFeature(feature: u16, port: u16) usb_abi.Request {
        return .{
            .request_type = .{ .recipient = .other, .kind = .class, .direction = .host_to_device },
            .request_code = .clear_feature,
            .value = feature,
            .index = port,
            .length = 0,
        };
    }

    fn getPortStatus(port: u16) usb_abi.Request {
        return .{
            .request_type = .{ .recipient = .other, .kind = .class, .direction = .device_to_host },
            .request_code = .get_status,
            .value = 0,
            .index = port,
            .length = 4,
        };
    }

    fn setHubDepth(depth: u16) usb_abi.Request {
        return .{
            .request_type = .{ .recipient = .device, .kind = .class, .direction = .host_to_device },
            .request_code = @enumFromInt(request_set_hub_depth),
            .value = depth,
            .index = 0,
            .length = 0,
        };
    }
};

pub const Controller = struct {
    register_base: usize,
    op_base: usize,
    runtime_base: usize,
    doorbell_base: usize,
    max_slots: u32,
    max_ports: u32,
    context_size: usize, // 32 or 64 (CSZ)

    device_context_array: dma.Region,
    command_ring: ProducerRing,
    event_ring: EventRing,
    devices: [max_devices]Device = [_]Device{.{}} ** max_devices,
    subscriptions: [max_subscriptions]Subscription = [_]Subscription{.{}} ** max_subscriptions,
    report_queue: [report_queue_capacity]Report = [_]Report{.{}} ** report_queue_capacity,
    report_count: usize = 0,
    port_changes: [16]u32 = undefined,
    port_change_count: usize = 0,
    // Transferred length of the most recent awaited transfer (requested minus the
    // event residual); read right after a control or bulk transfer returns true.
    last_transfer_length: u32 = 0,

    // --- register access ---------------------------------------------------

    fn read8(address: usize) u8 {
        return @as(*volatile u8, @ptrFromInt(address)).*;
    }
    fn read32(address: usize) u32 {
        return @as(*volatile u32, @ptrFromInt(address)).*;
    }
    fn write32(address: usize, value: u32) void {
        @as(*volatile u32, @ptrFromInt(address)).* = value;
    }
    // xHCI 64-bit registers are safely accessed as an ordered pair of 32-bit
    // writes (low dword first) — the portable form some controllers require.
    fn write64(address: usize, value: u64) void {
        write32(address, @truncate(value));
        write32(address + 4, @truncate(value >> 32));
    }

    fn operational(self: *const Controller, offset: usize) usize {
        return self.op_base + offset;
    }
    fn interrupter(self: *const Controller, offset: usize) usize {
        return self.runtime_base + runtime_interrupter0 + offset;
    }

    // PORTSC for 1-based port `port`.
    pub fn portStatus(self: *const Controller, port: u32) u32 {
        return read32(self.op_base + op_portsc_base + op_portsc_stride * (port - 1));
    }
    pub fn writePortStatus(self: *const Controller, port: u32, value: u32) void {
        write32(self.op_base + op_portsc_base + op_portsc_stride * (port - 1), value);
    }

    // --- bring-up ----------------------------------------------------------

    /// Reset and start the controller, standing up the command and event rings.
    /// Returns null on any failure (a wedged register handshake or an out-of-DMA
    /// condition) — the caller treats that as a driver that could not start.
    pub fn init(register_base: usize) ?Controller {
        const cap_length = read32(register_base + cap_caplength) & 0xFF;
        const hcsparams1 = read32(register_base + cap_hcsparams1);
        const hccparams1 = read32(register_base + cap_hccparams1);
        const dboff = read32(register_base + cap_dboff) & ~@as(u32, 0x3);
        const rtsoff = read32(register_base + cap_rtsoff) & ~@as(u32, 0x1F);

        var self = Controller{
            .register_base = register_base,
            .op_base = register_base + cap_length,
            .runtime_base = register_base + rtsoff,
            .doorbell_base = register_base + dboff,
            .max_slots = hcsparams1 & 0xFF,
            .max_ports = hcsparams1 >> 24,
            .context_size = if (hccparams1 & (1 << 2) != 0) 64 else 32,
            .device_context_array = undefined,
            .command_ring = undefined,
            .event_ring = undefined,
        };

        // Wait for the controller to report ready, then halt it if it is running.
        if (!waitClear(self.operational(op_usbsts), usbsts_controller_not_ready)) return null;
        if (read32(self.operational(op_usbcmd)) & usbcmd_run != 0) {
            write32(self.operational(op_usbcmd), read32(self.operational(op_usbcmd)) & ~@as(u32, usbcmd_run));
            if (!waitSet(self.operational(op_usbsts), usbsts_halted)) return null;
        }

        // Reset. HCRST self-clears when the reset completes; then CNR clears.
        write32(self.operational(op_usbcmd), usbcmd_reset);
        if (!waitClear(self.operational(op_usbcmd), usbcmd_reset)) return null;
        if (!waitClear(self.operational(op_usbsts), usbsts_controller_not_ready)) return null;

        // Enable all device slots the controller supports.
        write32(self.operational(op_config), self.max_slots);

        // The Device Context Base Address Array (entry 0 = scratchpad array).
        self.device_context_array = dma.alloc(page_size, dma.coherent) orelse return null;
        self.setupScratchpad(register_base);
        write64(self.operational(op_dcbaap), self.device_context_array.physical);

        // The command ring: a page of TRBs, last entry a Link back to the start.
        self.command_ring = .{ .region = dma.alloc(page_size, dma.coherent) orelse return null };
        self.command_ring.installLink();
        write64(self.operational(op_crcr), self.command_ring.region.physical | cycle_bit);

        // The event ring: one segment + a one-entry segment table.
        self.event_ring = .{
            .segment = dma.alloc(page_size, dma.coherent) orelse return null,
            .table = dma.alloc(page_size, dma.coherent) orelse return null,
        };
        const table: *volatile ErstEntry = @ptrFromInt(self.event_ring.table.virtual);
        table.ring_segment_base = self.event_ring.segment.physical;
        table.ring_segment_size = trbs_per_ring;
        write32(self.interrupter(event_ring_segment_table_size), 1);
        write64(self.interrupter(event_ring_dequeue_pointer), self.event_ring.segment.physical);
        write64(self.interrupter(event_ring_segment_table_base), self.event_ring.table.physical);
        write32(self.interrupter(interrupter_moderation), 0);
        // Enable the interrupter (IMAN.IE) and USBCMD.INTE. We still POLL the
        // event ring — no interrupt is wired — but some controllers (QEMU's
        // qemu-xhci among them) only WRITE runtime events to the ring when the
        // interrupter is enabled, so a hot-plug port-change event is silently
        // dropped otherwise. Enabling it is harmless to a polling driver.
        write32(self.interrupter(interrupter_management), 1 << 1); // IE
        mmio.wmb();

        // Run.
        mmio.wmb();
        write32(self.operational(op_usbcmd), read32(self.operational(op_usbcmd)) | usbcmd_run | usbcmd_interrupter_enable);
        if (!waitClear(self.operational(op_usbsts), usbsts_halted)) return null;

        // Power EVERY port — including empty ones — so a later hot-plug can
        // signal a connect (an unpowered port reports nothing: PP=0 is why a
        // device added after boot never raised a port-change event). Boot-time
        // devices are on already-powered ports; this just extends power to the
        // rest. Write PP without disturbing the write-1-to-clear bits.
        var port: u32 = 1;
        while (port <= self.max_ports) : (port += 1) {
            const status = self.portStatus(port);
            if (status & portsc_power == 0)
                self.writePortStatus(port, (status & ~portsc_write_1_to_clear) | portsc_power);
        }
        return self;
    }

    // Scratchpad buffers the controller asks the host to reserve for its own use.
    // Entry 0 of the device-context array points at an array of their physical
    // addresses. QEMU usually requests none, in which case DCBAA[0] stays zero.
    fn setupScratchpad(self: *Controller, register_base: usize) void {
        const hcsparams2 = read32(register_base + cap_hcsparams2);
        const high = (hcsparams2 >> 21) & 0x1F;
        const low = (hcsparams2 >> 27) & 0x1F;
        const count = (high << 5) | low;
        const array: [*]volatile u64 = @ptrFromInt(self.device_context_array.virtual);
        if (count == 0) {
            array[0] = 0;
            return;
        }
        // One page per scratchpad buffer, plus a page holding their address array.
        const pointers = dma.alloc(page_size, dma.coherent) orelse return;
        const pointer_array: [*]volatile u64 = @ptrFromInt(pointers.virtual);
        var index: u32 = 0;
        while (index < count) : (index += 1) {
            const buffer = dma.alloc(page_size, dma.coherent) orelse return;
            pointer_array[index] = buffer.physical;
        }
        array[0] = pointers.physical;
    }

    // Spin (with a deadline) until every bit in `mask` reads back as zero / one.
    fn waitClear(address: usize, mask: u32) bool {
        const deadline = system.clock() + 1_000_000_000; // 1 s
        while (read32(address) & mask != 0) {
            if (system.clock() >= deadline) return false;
        }
        return true;
    }
    fn waitSet(address: usize, mask: u32) bool {
        const deadline = system.clock() + 1_000_000_000;
        while (read32(address) & mask == 0) {
            if (system.clock() >= deadline) return false;
        }
        return true;
    }

    // --- rings -------------------------------------------------------------

    fn ringDoorbell(self: *const Controller, slot: u32, target: u32) void {
        write32(self.doorbell_base + slot * 4, target);
    }

    /// Enqueue a command TRB, ring the command doorbell, and return the physical
    /// address of the enqueued TRB (which the Command Completion Event echoes).
    fn submitCommand(self: *Controller, trb: Trb) u64 {
        const physical = self.command_ring.push(trb);
        mmio.wmb();
        self.ringDoorbell(0, 0); // doorbell 0, target 0 = command ring
        return physical;
    }

    /// Consume the next event, or null if none has arrived by `deadline_ns`.
    fn nextEvent(self: *Controller, deadline_ns: u64) ?Trb {
        const ring = self.event_ring.trbs();
        while (true) {
            const slot = &ring[self.event_ring.dequeue_index];
            const control = slot.control;
            if ((control & cycle_bit != 0) == self.event_ring.cycle) {
                mmio.rmb();
                const event = Trb{ .parameter = slot.parameter, .status = slot.status, .control = control };
                self.event_ring.dequeue_index += 1;
                if (self.event_ring.dequeue_index >= trbs_per_ring) {
                    self.event_ring.dequeue_index = 0;
                    self.event_ring.cycle = !self.event_ring.cycle;
                }
                const dequeue = self.event_ring.segment.physical + self.event_ring.dequeue_index * @sizeOf(Trb);
                // Write the new dequeue pointer and clear the Event Handler Busy bit.
                write64(self.interrupter(event_ring_dequeue_pointer), dequeue | (1 << 3));
                return event;
            }
            if (system.clock() >= deadline_ns) return null;
        }
    }

    /// Wait for the Command Completion Event matching `command_physical`. Interrupt
    /// reports that land while waiting are dispatched (so a standing subscription is
    /// serviced even during a command); other events are ignored. Returns the
    /// completion code, or null on timeout.
    fn awaitCommand(self: *Controller, command_physical: u64) ?u8 {
        const deadline = system.clock() + 1_000_000_000;
        while (true) {
            const event = self.nextEvent(deadline) orelse return null;
            const kind = trbType(event.control);
            if (kind == @intFromEnum(TrbType.command_completion_event) and
                (event.parameter & ~@as(u64, 0xF)) == command_physical)
            {
                return completionCode(event.status);
            }
            if (kind == @intFromEnum(TrbType.transfer_event)) _ = self.serviceInterruptEvent(event);
        }
    }

    /// A No-Op Command: the cheapest end-to-end proof that reset, the command
    /// ring, the event ring, the doorbell, and the cycle-bit bookkeeping are all
    /// correct. Returns true if the controller completed it with success.
    pub fn noOpCommand(self: *Controller) bool {
        const physical = self.submitCommand(.{ .control = trbControl(.no_op_command, 0) });
        const code = self.awaitCommand(physical) orelse return false;
        return code == @intFromEnum(CompletionCode.success);
    }

    // --- device slots + control transfers ----------------------------------

    /// Reset the given 1-based root-hub port and wait for it to enable. A port
    /// must be reset before the device on it can be addressed. Returns false if
    /// the reset does not complete or the port does not enable.
    pub fn resetPort(self: *const Controller, port: u32) bool {
        // Set PR while writing 0 to every write-1-to-clear bit (so the change
        // bits and PED are untouched) and preserving PP.
        const before = self.portStatus(port);
        self.writePortStatus(port, (before & ~portsc_write_1_to_clear) | portsc_reset);
        const deadline = system.clock() + 500_000_000;
        while (self.portStatus(port) & portsc_reset_change == 0) {
            if (system.clock() >= deadline) return false;
        }
        // Clear the Port Reset Change bit (write 1 to PRC, 0 to the rest).
        const after = self.portStatus(port);
        self.writePortStatus(port, (after & ~portsc_write_1_to_clear) | portsc_reset_change);
        return self.portStatus(port) & portsc_enabled != 0;
    }

    /// Issue an Enable Slot command and return the slot id the controller
    /// assigned (carried in bits 31:24 of the completion event's control field).
    fn enableSlot(self: *Controller) ?u8 {
        const physical = self.submitCommand(.{ .control = trbControl(.enable_slot, 0) });
        const deadline = system.clock() + 1_000_000_000;
        while (true) {
            const event = self.nextEvent(deadline) orelse return null;
            if (trbType(event.control) == @intFromEnum(TrbType.command_completion_event) and
                (event.parameter & ~@as(u64, 0xF)) == physical)
            {
                if (completionCode(event.status) != @intFromEnum(CompletionCode.success)) return null;
                return @truncate(event.control >> 24);
            }
        }
    }

    fn allocateDevice(self: *Controller) ?*Device {
        for (&self.devices) |*device| {
            if (!device.used) return device;
        }
        return null;
    }

    // A pointer to dword `dword_index` of context `context_index` within a context
    // array at `virtual`. Contexts are strided by `context_size` (32 or 64), so
    // the meaningful first 8 dwords sit at the base of each stride.
    fn contextDword(virtual: usize, context_index: usize, dword_index: usize, context_size: usize) *volatile u32 {
        return @ptrFromInt(virtual + context_index * context_size + dword_index * 4);
    }

    // Build the Input Context for Address Device: Input Control Context add-flags
    // A0 (slot) | A1 (EP0), a Slot Context (route 0, speed, one context entry,
    // root-hub port), and an EP0 Control endpoint context pointing at the device's
    // EP0 ring. Contexts start zeroed (the DMA region is), so only set fields.
    fn buildAddressInputContext(self: *Controller, device: *Device) void {
        const cs = self.context_size;
        const base = device.input_context.virtual;
        // Input Control Context (index 0): Add flags in dword 1 = A0 | A1.
        contextDword(base, 0, 1, cs).* = 0b11;
        // Slot Context dword 0: Route String[19:0], Speed[23:20], Context
        // Entries[31:27] = 1.
        contextDword(base, 1, 0, cs).* = (device.route & 0xFFFFF) | (device.speed << 20) | (@as(u32, 1) << 27);
        // Slot Context dword 1: Root Hub Port Number[23:16] — the ROOT port the
        // hub chain hangs off (inherited down a chain), not the device's own
        // downstream hub port.
        contextDword(base, 1, 1, cs).* = device.root_port << 16;
        // Slot Context dword 2: the transaction translator — a full/low-speed
        // device behind a high-speed hub routes split transactions through the
        // parent hub's TT. Parent Hub Slot ID[7:0], Parent Port Number[13:8].
        if (device.parent_slot != 0 and device.speed < 3) {
            contextDword(base, 1, 2, cs).* = @as(u32, device.parent_slot) | (@as(u32, device.parent_port) << 8);
        }
        // EP0 Context (index 2): CErr[2:1]=3, EP Type[5:3]=Control(4), MPS[31:16].
        contextDword(base, 2, 1, cs).* = (@as(u32, 3) << 1) | (@as(u32, 4) << 3) | (device.max_packet_size_0 << 16);
        // TR Dequeue Pointer (dwords 2:3) with Dequeue Cycle State = 1.
        const dequeue = device.ep0_ring.region.physical | 1;
        contextDword(base, 2, 2, cs).* = @truncate(dequeue);
        contextDword(base, 2, 3, cs).* = @truncate(dequeue >> 32);
        // Average TRB Length (dword 4): 8 is the conventional value for control.
        contextDword(base, 2, 4, cs).* = 8;
    }

    fn addressDeviceCommand(self: *Controller, device: *Device) bool {
        const physical = self.submitCommand(.{
            .parameter = device.input_context.physical,
            .control = trbControl(.address_device, @as(u32, device.slot_id) << 24),
        });
        const code = self.awaitCommand(physical) orelse {
            std.log.info("port {d} setup: Address Device timed out", .{device.port});
            return false;
        };
        if (code != @intFromEnum(CompletionCode.success)) {
            std.log.info("port {d} setup: Address Device completion code {d}", .{ device.port, code });
            return false;
        }
        return true;
    }

    /// Reset the port, enable a slot, and address the device on it: after this the
    /// device answers control transfers on its EP0. Returns the tracked `Device`,
    /// or null on any failure. The EP0 MPS is taken from the speed default and
    /// corrected from the device descriptor by `refreshMaxPacketSize0` if needed.
    pub fn setupDevice(self: *Controller, port: u32, speed: u32) ?*Device {
        // A SuperSpeed port that has trained its link is ALREADY enabled — the
        // xHCI advances USB3 ports to Enabled with no reset (spec 4.3). Driving
        // a hot reset into a live SS link drops PED mid-reset on real silicon
        // (observed: "setup failed" in the same millisecond as "connected").
        // Only a not-yet-enabled port — every USB2 device, or a stuck SS link —
        // needs the reset to enable.
        const already_enabled = speed >= 4 and self.portStatus(port) & portsc_enabled != 0;
        var effective_speed = speed;
        if (!already_enabled) {
            if (!self.resetPort(port)) {
                std.log.info("port {d} setup: port reset failed (PORTSC 0x{x:0>8})", .{ port, self.portStatus(port) });
                return null;
            }
            // A USB2 port's PORTSC speed field is only meaningful once the port
            // is enabled by the reset — sample it NOW, not at connect time
            // (pre-reset reads misreport on real controllers; M20).
            effective_speed = (self.portStatus(port) >> 10) & 0xF;
            if (effective_speed == 0) effective_speed = speed; // defensive: keep the caller's read
        }
        const slot_id = self.enableSlot() orelse {
            std.log.info("port {d} setup: Enable Slot failed", .{port});
            return null;
        };
        const device = self.allocateDevice() orelse {
            std.log.info("port {d} setup: no free device slot", .{port});
            return null;
        };
        device.* = .{
            .used = true,
            .slot_id = slot_id,
            .port = port,
            .speed = effective_speed,
            .max_packet_size_0 = defaultMaxPacketSize0(effective_speed),
            .root_port = port, // a root-port device: the chain root IS this port
        };
        device.input_context = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device);
        device.device_context = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device);
        device.ep0_ring = .{ .region = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device) };
        device.ep0_ring.installLink();
        device.control_buffer = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device);

        self.buildAddressInputContext(device);
        const array: [*]volatile u64 = @ptrFromInt(self.device_context_array.virtual);
        array[device.slot_id] = device.device_context.physical;
        if (!self.addressDeviceCommand(device)) return self.abandon(device);
        return device;
    }


    /// Configure an enumerated class-9 device as a hub (docs/usb-hub.md): read
    /// the hub descriptor for the downstream port count, tell the controller the
    /// slot is a hub (so it routes downstream traffic), SET_HUB_DEPTH for a
    /// SuperSpeed hub, and power every downstream port. Downstream enumeration
    /// (status-change handling) is B4b. Returns false on a control-transfer
    /// failure; the hub is still registered, just inert.
    pub fn setupHub(self: *Controller, device: *Device) bool {
        const is_usb3 = device.speed >= 4;
        var descriptor: [16]u8 = undefined;
        const kind: u8 = if (is_usb3) hubreq.descriptor_usb3 else hubreq.descriptor_usb2;
        if (!self.controlTransfer(device, hubreq.getDescriptor(kind, descriptor.len), descriptor[0..], true)) {
            std.log.info("hub slot {d}: hub descriptor read failed", .{device.slot_id});
            return false;
        }
        device.is_hub = true;
        device.hub_ports = descriptor[2]; // bNbrPorts
        const characteristics = @as(u16, descriptor[3]) | (@as(u16, descriptor[4]) << 8);
        device.hub_multi_tt = !is_usb3 and (characteristics & hubreq.characteristics_multi_tt != 0);

        // A SuperSpeed hub needs its depth (tiers from the root) to compose the
        // route strings of devices below it.
        if (is_usb3) {
            const depth = routeDepth(device.route);
            _ = self.controlTransfer(device, hubreq.setHubDepth(depth), &.{}, false);
        }

        // Tell the controller the slot is a hub — Hub bit, Number of Ports, and
        // (for a USB2 multi-TT hub) MTT + TT Think Time. Configure Endpoint with
        // only the slot add-flag (A0) evaluates these (xHCI 4.6.6).
        if (!self.configureSlotAsHub(device)) {
            std.log.info("hub slot {d}: could not configure slot as a hub", .{device.slot_id});
            return false;
        }

        // Power every downstream port.
        var port: u16 = 1;
        while (port <= device.hub_ports) : (port += 1) {
            _ = self.controlTransfer(device, hubreq.setPortFeature(hubreq.feature_port_power, port), &.{}, false);
        }
        // Seed every downstream port as pending: the bus tick GET_STATUSes each
        // and enumerates the connected ones. This makes a STATIC topology (a
        // device present at power-on) work without relying on the initial
        // status-change interrupt edge; the interrupt then handles later plugs.
        device.hub_change_mask = if (device.hub_ports >= 31) 0xFFFF_FFFE else (@as(u32, 1) << @intCast(device.hub_ports + 1)) - 2;

        // Arm the status-change interrupt endpoint (in-process) for hot-plug.
        self.armHubStatus(device);

        std.log.info("hub slot {d}: {d} downstream ports powered ({s})", .{
            device.slot_id,
            device.hub_ports,
            if (is_usb3) "SuperSpeed" else if (device.hub_multi_tt) "USB2 multi-TT" else "USB2 single-TT",
        });
        return true;
    }

    /// Arm the hub's interrupt-IN status-change endpoint with an in-process
    /// subscription: completions set the hub's pending-change mask (serviced on
    /// the bus tick). Best-effort — a hub with no interrupt endpoint (shouldn't
    /// happen) just relies on the initial sweep.
    fn armHubStatus(self: *Controller, device: *Device) void {
        for (device.interfaces[0..device.interface_count]) |interface| {
            for (interface.endpoints[0..interface.endpoint_count]) |endpoint| {
                const is_interrupt = endpoint.transfer_type == 3;
                const is_in = endpoint.address & 0x80 != 0;
                if (!is_interrupt or !is_in) continue;
                const ring = self.getOrConfigureEndpoint(device, endpoint) orelse return;
                const subscription = self.allocateSubscription() orelse return;
                const buffer = dma.alloc(page_size, dma.coherent) orelse return;
                const number: u8 = endpoint.address & 0x0F;
                subscription.* = .{
                    .active = true,
                    .slot_id = device.slot_id,
                    .dci = doorbellContextIndex(number, true),
                    .endpoint_address = endpoint.address,
                    .ring = ring,
                    .buffer = buffer,
                    .max_length = endpoint.max_packet_size,
                    .hub = device,
                };
                self.armInterrupt(subscription);
                return;
            }
        }
    }

    /// The next pending (hub, downstream-port) change to service, or null. Clears
    /// the returned port's bit. Called on the bus tick.
    pub fn takeHubChange(self: *Controller) ?struct { hub: *Device, port: u16 } {
        for (&self.devices) |*device| {
            if (!device.used or !device.is_hub or device.hub_change_mask == 0) continue;
            const bit: u5 = @intCast(@ctz(device.hub_change_mask));
            device.hub_change_mask &= ~(@as(u32, 1) << bit);
            if (bit == 0) continue; // bit 0 is the hub itself, not a downstream port
            return .{ .hub = device, .port = bit };
        }
        return null;
    }

    fn readHubPortStatus(self: *Controller, hub: *Device, port: u16) ?u32 {
        var buffer: [4]u8 = undefined;
        if (!self.controlTransfer(hub, hubreq.getPortStatus(port), buffer[0..], true)) return null;
        return @as(u32, buffer[0]) | (@as(u32, buffer[1]) << 8) | (@as(u32, buffer[2]) << 16) | (@as(u32, buffer[3]) << 24);
    }

    /// Bring up (or note the disconnect of) a device on hub downstream `port`:
    /// read the port status, acknowledge the change bits, and on a fresh connect
    /// reset the port, read the speed, and setup+address the downstream device
    /// (route string + TT). Returns the addressed device for the bus to enumerate
    /// and register, or null (empty port, disconnect, or a failure).
    /// Read a downstream hub port's status and acknowledge its latched change
    /// bits (so it can signal again). Returns the wPortStatus word.
    pub fn hubPortStatusAck(self: *Controller, hub: *Device, port: u16) ?u32 {
        const status = self.readHubPortStatus(hub, port) orelse return null;
        // Acknowledge every latched change bit. A SuperSpeed hub has extra ones
        // (link-state, BH-reset) beyond a USB2 hub's connection/reset — leaving
        // any set makes the hub's status-change endpoint re-report the same port
        // forever, spinning the driver (a real SuperSpeed hub hung boot here;
        // QEMU's USB2 hub has none of these). Clearing an inapplicable feature
        // is harmless (the hub STALLs it and we move on).
        _ = self.controlTransfer(hub, hubreq.clearPortFeature(hubreq.feature_c_port_connection, port), &.{}, false);
        _ = self.controlTransfer(hub, hubreq.clearPortFeature(hubreq.feature_c_port_reset, port), &.{}, false);
        if (hub.speed >= 4) {
            _ = self.controlTransfer(hub, hubreq.clearPortFeature(hubreq.feature_c_port_link_state, port), &.{}, false);
            _ = self.controlTransfer(hub, hubreq.clearPortFeature(hubreq.feature_c_bh_port_reset, port), &.{}, false);
        }
        return status;
    }

    pub fn hubPortConnected(status: u32) bool {
        return status & hubreq.status_connection != 0;
    }

    /// Reset + address a device on a connected, empty downstream hub `port` (the
    /// bus has confirmed connect and no existing device): reset the port, read
    /// the speed, and setup+address the downstream device (route string + TT).
    /// Returns the addressed device for the bus to enumerate + register.
    pub fn serviceHubPort(self: *Controller, hub: *Device, port: u16) ?*Device {
        const status = self.readHubPortStatus(hub, port) orelse return null;

        // Reset the port if not yet enabled, then wait (bounded) for enable.
        if (status & hubreq.status_enable == 0) {
            _ = self.controlTransfer(hub, hubreq.setPortFeature(hubreq.feature_port_reset, port), &.{}, false);
            var tries: u32 = 0;
            while (tries < 200) : (tries += 1) {
                system.sleep(5);
                const s = self.readHubPortStatus(hub, port) orelse return null;
                if (s & hubreq.status_enable != 0) break;
            }
            _ = self.controlTransfer(hub, hubreq.clearPortFeature(hubreq.feature_c_port_reset, port), &.{}, false);
        }
        const enabled = self.readHubPortStatus(hub, port) orelse return null;
        if (enabled & hubreq.status_enable == 0) {
            std.log.info("hub slot {d} port {d}: reset did not enable", .{ hub.slot_id, port });
            return null;
        }
        const downstream_speed = (enabled >> 9) & 0x3; // wPortStatus: bit9 low-speed, bit10 high-speed

        return self.setupDeviceBehindHub(hub, port, downstream_speed);
    }

    pub fn deviceOnHubPort(self: *Controller, hub: *Device, port: u16) ?*Device {
        for (&self.devices) |*device| {
            if (device.used and device.parent_slot == hub.slot_id and device.parent_port == port) return device;
        }
        return null;
    }

    /// Enable a slot and Address a device behind `hub` on downstream `port`, with
    /// the composed route string, inherited root port, and TT fields (so the
    /// controller routes split transactions through this hub's TT for a
    /// full/low-speed device). Mirrors setupDevice for a root-port device.
    fn setupDeviceBehindHub(self: *Controller, hub: *Device, port: u16, speed: u32) ?*Device {
        const slot_id = self.enableSlot() orelse {
            std.log.info("hub slot {d} port {d}: Enable Slot failed", .{ hub.slot_id, port });
            return null;
        };
        const device = self.allocateDevice() orelse return null;
        const child_speed = mapHubPortSpeed(speed);
        device.* = .{
            .used = true,
            .slot_id = slot_id,
            .port = hub.root_port,
            .speed = child_speed,
            .max_packet_size_0 = defaultMaxPacketSize0(child_speed),
            .route = (hub.route << 4) | (port & 0xF),
            .root_port = hub.root_port,
            .parent_slot = hub.slot_id,
            .parent_port = @intCast(port),
        };
        device.input_context = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device);
        device.device_context = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device);
        device.ep0_ring = .{ .region = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device) };
        device.ep0_ring.installLink();
        device.control_buffer = dma.alloc(page_size, dma.coherent) orelse return self.abandon(device);

        self.buildAddressInputContext(device);
        const array: [*]volatile u64 = @ptrFromInt(self.device_context_array.virtual);
        array[device.slot_id] = device.device_context.physical;
        if (!self.addressDeviceCommand(device)) return self.abandon(device);
        return device;
    }

    /// Configure Endpoint with only A0 (slot) set: rebuild the slot context with
    /// the Hub bit, Number of Ports, and MTT/TT-Think-Time, so the controller
    /// treats this slot as a hub.
    fn configureSlotAsHub(self: *Controller, device: *Device) bool {
        const cs = self.context_size;
        const base = device.input_context.virtual;
        @memset(@as([*]u8, @ptrFromInt(base))[0 .. 2 * cs], 0);
        contextDword(base, 0, 1, cs).* = 0b1; // Input Control Context add flags: A0 (slot)
        // Slot Context dword 0: route, speed, context entries, plus Hub[26] and
        // (USB2 multi-TT) MTT[25].
        var dword0: u32 = (device.route & 0xFFFFF) | (device.speed << 20) | (@as(u32, 1) << 27) | (@as(u32, 1) << 26);
        if (device.hub_multi_tt) dword0 |= (@as(u32, 1) << 25);
        contextDword(base, 1, 0, cs).* = dword0;
        // Slot Context dword 1: Root Hub Port Number[23:16], Number of Ports[31:24].
        contextDword(base, 1, 1, cs).* = (device.root_port << 16) | (@as(u32, device.hub_ports) << 24);
        // Slot Context dword 2: TT Think Time[17:16] = 0 (8 FS bit times); the
        // parent-TT fields (if this hub is itself behind a hub) carry over.
        if (device.parent_slot != 0 and device.speed < 3) {
            contextDword(base, 1, 2, cs).* = @as(u32, device.parent_slot) | (@as(u32, device.parent_port) << 8);
        }
        return self.configureEndpointCommand(device);
    }

    fn abandon(self: *Controller, device: *Device) ?*Device {
        _ = self;
        device.used = false;
        return null;
    }

    /// Await the completion of OUR transfer — identified by the event's slot id
    /// (control[31:24]) and endpoint DCI (control[20:16]). Any other transfer
    /// event is either a subscription's report (serviced) or foreign noise (an
    /// interrupt endpoint's error/stale completion whose TRB pointer no longer
    /// matches the armed one) — DROPPED, never misattributed: claiming a foreign
    /// event as our completion desynchronized the mass-storage bulk protocol in
    /// a way that survived every driver restart (the 1-in-3 READ CAPACITY
    /// failure at boot, with a USB keyboard and mouse polling concurrently).
    fn awaitTransfer(self: *Controller, slot_id: u8, dci: u32, requested_length: u32) ?u8 {
        const deadline = system.clock() + 1_000_000_000;
        while (true) {
            const event = self.nextEvent(deadline) orelse return null;
            if (trbType(event.control) != @intFromEnum(TrbType.transfer_event)) continue;
            if (self.serviceInterruptEvent(event)) continue; // a subscription's report
            const event_slot: u8 = @truncate(event.control >> 24);
            const event_dci: u32 = (event.control >> 16) & 0x1F;
            if (event_slot != slot_id or event_dci != dci) {
                std.log.info("dropped foreign transfer event (slot {d} dci {d}, code {d})", .{ event_slot, event_dci, completionCode(event.status) });
                continue;
            }
            const residual = event.status & 0xFFFFFF;
            self.last_transfer_length = if (residual >= requested_length) 0 else requested_length - residual;
            return completionCode(event.status); // our transfer's completion (or error)
        }
    }

    /// Run one control transfer on a device's EP0: a Setup stage (the 8-byte
    /// request inline), an optional Data stage through the device's bounce buffer,
    /// and a Status stage. For an IN transfer the returned data lands in `data`;
    /// for an OUT transfer `data` is sent. Returns false on any failure or stall.
    pub fn controlTransfer(self: *Controller, device: *Device, request: usb_abi.Request, data: []u8, direction_in: bool) bool {
        const has_data = data.len > 0;
        const transfer_type: u32 = if (!has_data) 0 else if (direction_in) 3 else 2; // TRT: none/OUT/IN

        // Setup Stage: the 8-byte setup packet inline (IDT), TRT in bits 17:16.
        const setup_bytes: u64 = @bitCast(request);
        _ = device.ep0_ring.push(.{
            .parameter = setup_bytes,
            .status = 8,
            .control = trbControl(.setup_stage, (1 << 6) | (transfer_type << 16)),
        });

        if (has_data) {
            if (!direction_in) {
                const buffer: [*]u8 = @ptrFromInt(device.control_buffer.virtual);
                @memcpy(buffer[0..data.len], data);
            }
            _ = device.ep0_ring.push(.{
                .parameter = device.control_buffer.physical,
                .status = @intCast(data.len),
                .control = trbControl(.data_stage, if (direction_in) (1 << 16) else 0), // DIR bit 16
            });
        }

        // Status Stage: opposite direction to the data (IN when there was no data
        // or the data was OUT), Interrupt On Completion so we get one event.
        const status_direction: u32 = if (has_data and direction_in) 0 else (1 << 16);
        _ = device.ep0_ring.push(.{
            .control = trbControl(.status_stage, status_direction | (1 << 5)), // DIR | IOC
        });

        mmio.wmb();
        self.ringDoorbell(device.slot_id, 1); // DCI 1 = EP0
        const code = self.awaitTransfer(device.slot_id, 1, @intCast(data.len)) orelse return false;
        if (code != @intFromEnum(CompletionCode.success) and code != @intFromEnum(CompletionCode.short_packet)) return false;

        if (has_data and direction_in) {
            const buffer: [*]u8 = @ptrFromInt(device.control_buffer.virtual);
            @memcpy(data, buffer[0..data.len]);
        }
        return true;
    }

    /// Read a device's 18-byte DEVICE descriptor over its control pipe.
    pub fn getDeviceDescriptor(self: *Controller, device: *Device) ?usb_abi.DeviceDescriptor {
        var bytes: [18]u8 = undefined;
        const request = usb_abi.getDescriptor(.device, 0, 0, 18);
        if (!self.controlTransfer(device, request, bytes[0..], true)) return null;
        return std.mem.bytesToValue(usb_abi.DeviceDescriptor, &bytes);
    }

    /// Full chapter-9 enumeration of an addressed device: read the device and
    /// configuration descriptors, parse the configuration's interfaces and
    /// endpoints into `device`, and select the configuration. After this the
    /// device is in the configured state and its interfaces are ready to match a
    /// class driver. Returns false on any control-transfer failure.
    /// Correct EP0's max packet size from the device itself. The context starts
    /// with the SPEED-DEFAULT (full-speed: 8, but the true value may be 8/16/32/
    /// 64 — byte 7 of the device descriptor). Read just the descriptor's first
    /// 8 bytes (always deliverable at any legal MPS0), and when the device
    /// disagrees with the context, issue Evaluate Context to fix EP0 before any
    /// longer transfer. Real controllers fault the full 18-byte read on a wrong
    /// MPS0; QEMU forgives it — the classic full-speed-mouse-on-real-hardware
    /// failure (M20).
    fn refreshMaxPacketSize0(self: *Controller, device: *Device) bool {
        // SuperSpeed (and above) fix EP0's max packet size at 512, and encode
        // bMaxPacketSize0 as an EXPONENT (9 = 2^9 = 512), not a literal size —
        // the default is already correct and byte 7 must NOT be read as a size.
        // Only full/low/high speed carry a literal 8/16/32/64 that can differ
        // from the speed default and need this correction. (Reading the SS
        // exponent as a size set EP0 to 9 bytes and broke every following
        // transfer — a SuperSpeed hub failing to enumerate on real hardware.)
        if (device.speed >= 4) return true;

        var head: [8]u8 = undefined;
        const request = usb_abi.getDescriptor(.device, 0, 0, 8);
        if (!self.controlTransfer(device, request, head[0..], true)) return false;
        const actual: u32 = head[7];
        if (actual == 0 or actual == device.max_packet_size_0) return true;

        // Input Control Context: add-flag A1 (EP0 only); EP0 context rebuilt
        // with the corrected MPS. Fields not being changed stay zero (the
        // controller evaluates only the added context).
        const cs = self.context_size;
        const base = device.input_context.virtual;
        @memset(@as([*]u8, @ptrFromInt(base))[0 .. 3 * cs], 0);
        contextDword(base, 0, 1, cs).* = 0b10; // A1
        contextDword(base, 2, 1, cs).* = (@as(u32, 3) << 1) | (@as(u32, 4) << 3) | (actual << 16);
        const physical = self.submitCommand(.{
            .parameter = device.input_context.physical,
            .control = trbControl(.evaluate_context, @as(u32, device.slot_id) << 24),
        });
        const code = self.awaitCommand(physical) orelse {
            std.log.info("slot {d}: Evaluate Context (MPS0 {d} -> {d}) timed out", .{ device.slot_id, device.max_packet_size_0, actual });
            return false;
        };
        if (code != @intFromEnum(CompletionCode.success)) {
            std.log.info("slot {d}: Evaluate Context (MPS0 {d} -> {d}) completion code {d}", .{ device.slot_id, device.max_packet_size_0, actual, code });
            return false;
        }
        device.max_packet_size_0 = actual;
        return true;
    }

    pub fn enumerate(self: *Controller, device: *Device) bool {
        if (!self.refreshMaxPacketSize0(device)) return false;
        device.device_descriptor = self.getDeviceDescriptor(device) orelse return false;

        // The configuration descriptor's own 9 bytes carry the total length of
        // the whole configuration block (interfaces + endpoints follow it).
        var header: [9]u8 = undefined;
        if (!self.controlTransfer(device, usb_abi.getDescriptor(.configuration, 0, 0, 9), header[0..], true)) return false;
        const configuration = std.mem.bytesToValue(usb_abi.ConfigurationDescriptor, &header);
        device.configuration_value = @intFromEnum(configuration.configuration_value);

        // Read the whole block into a local buffer and parse it here (in the bus
        // driver) so the parse never has to cross the 256-byte IPC boundary.
        var blob: [512]u8 = undefined;
        const length = @min(configuration.total_length, blob.len);
        if (!self.controlTransfer(device, usb_abi.getDescriptor(.configuration, 0, 0, @intCast(length)), blob[0..length], true)) return false;
        parseConfiguration(device, blob[0..length]);

        // Select the configuration, moving the device to the configured state.
        if (!self.controlTransfer(device, usb_abi.setConfiguration(configuration.configuration_value), &.{}, false)) return false;
        return true;
    }

    // Walk a configuration block, recording each interface (alternate setting 0)
    // and the endpoints that follow it. Endpoints belong to the most recent
    // interface. Unknown descriptor types (HID, class-specific) are skipped by
    // their length.
    fn parseConfiguration(device: *Device, blob: []const u8) void {
        device.interface_count = 0;
        var current: ?*InterfaceInfo = null;
        var offset: usize = 0;
        while (offset + 2 <= blob.len) {
            const length = blob[offset];
            const descriptor_type = blob[offset + 1];
            if (length < 2 or offset + length > blob.len) break;
            switch (@as(usb_abi.DescriptorType, @enumFromInt(descriptor_type))) {
                .interface => if (length >= @sizeOf(usb_abi.InterfaceDescriptor)) {
                    const descriptor = std.mem.bytesToValue(usb_abi.InterfaceDescriptor, blob[offset .. offset + @sizeOf(usb_abi.InterfaceDescriptor)]);
                    if (@intFromEnum(descriptor.alternate_setting) != 0) {
                        current = null; // ignore alternate settings for now
                    } else if (device.interface_count < max_interfaces) {
                        const slot = &device.interfaces[device.interface_count];
                        slot.* = .{
                            .number = @intFromEnum(descriptor.interface_number),
                            .class = descriptor.interface_class,
                            .subclass = descriptor.interface_subclass,
                            .protocol = descriptor.interface_protocol,
                        };
                        current = slot;
                        device.interface_count += 1;
                    }
                },
                .endpoint => if (length >= @sizeOf(usb_abi.EndpointDescriptor)) {
                    if (current) |interface| {
                        if (interface.endpoint_count < max_endpoints_per_interface) {
                            const descriptor = std.mem.bytesToValue(usb_abi.EndpointDescriptor, blob[offset .. offset + @sizeOf(usb_abi.EndpointDescriptor)]);
                            interface.endpoints[interface.endpoint_count] = .{
                                .address = @bitCast(descriptor.endpoint_address),
                                .transfer_type = @intFromEnum(descriptor.attributes.transfer_type),
                                .max_packet_size = descriptor.max_packet_size.size,
                                .interval = descriptor.interval,
                            };
                            interface.endpoint_count += 1;
                        }
                    }
                },
                else => {},
            }
            offset += length;
        }
    }

    // --- endpoint configuration + interrupt / bulk transfers ---------------

    /// Find the tracked device and interface an assigned device id belongs to.
    pub fn findInterface(self: *Controller, device_id: u64) ?struct { device: *Device, interface: *InterfaceInfo } {
        for (&self.devices) |*device| {
            if (!device.used) continue;
            for (device.interfaces[0..device.interface_count]) |*interface| {
                if (interface.registered_device_id == device_id) return .{ .device = device, .interface = interface };
            }
        }
        return null;
    }

    /// The endpoint of `interface` with the given address, or null.
    pub fn endpointForAddress(interface: *const InterfaceInfo, address: u8) ?EndpointInfo {
        for (interface.endpoints[0..interface.endpoint_count]) |endpoint| {
            if (endpoint.address == address) return endpoint;
        }
        return null;
    }

    fn findEndpointRing(device: *Device, dci: u32) ?*ProducerRing {
        for (device.endpoint_rings[0..device.endpoint_ring_count]) |*configured| {
            if (configured.dci == dci) return &configured.ring;
        }
        return null;
    }

    // Get (configuring on first use) the transfer ring for an endpoint. The first
    // use issues a Configure Endpoint command that adds the endpoint context to the
    // device and points it at a fresh ring.
    fn getOrConfigureEndpoint(self: *Controller, device: *Device, endpoint: EndpointInfo) ?*ProducerRing {
        const number: u8 = endpoint.address & 0x0F;
        const direction_in = endpoint.address & 0x80 != 0;
        const dci = doorbellContextIndex(number, direction_in);
        if (findEndpointRing(device, dci)) |ring| return ring;
        if (device.endpoint_ring_count >= device.endpoint_rings.len) return null;

        const configured = &device.endpoint_rings[device.endpoint_ring_count];
        configured.dci = dci;
        configured.ring = .{ .region = dma.alloc(page_size, dma.coherent) orelse return null };
        configured.ring.installLink();
        self.buildConfigureEndpointInputContext(device, endpoint, dci, &configured.ring);
        if (!self.configureEndpointCommand(device)) return null;
        device.endpoint_ring_count += 1;
        return &configured.ring;
    }

    // Build the Input Context for a Configure Endpoint command adding one endpoint:
    // Add flags A0 (slot) | A(dci), a Slot Context whose Context Entries covers the
    // new endpoint, and the endpoint context (type, packet size, ring, interval).
    fn buildConfigureEndpointInputContext(self: *Controller, device: *Device, endpoint: EndpointInfo, dci: u32, ring: *const ProducerRing) void {
        const cs = self.context_size;
        const base = device.input_context.virtual;
        // Zero the contexts we touch (the region last held the Address Device input).
        const dwords: [*]volatile u32 = @ptrFromInt(base);
        const touched = (2 + @as(usize, dci)) * (cs / 4);
        var i: usize = 0;
        while (i < touched) : (i += 1) dwords[i] = 0;

        // Input Control Context: Add flags A0 (slot) | A(dci) (the endpoint).
        contextDword(base, 0, 1, cs).* = (@as(u32, 1) << 0) | (@as(u32, 1) << @as(u5, @intCast(dci)));
        // Slot Context: speed, Context Entries = dci, root hub port.
        contextDword(base, 1, 0, cs).* = (device.speed << 20) | (dci << 27);
        contextDword(base, 1, 1, cs).* = device.port << 16;
        // Endpoint Context at index (1 + dci).
        const direction_in = endpoint.address & 0x80 != 0;
        const endpoint_type = @as(u32, endpoint.transfer_type) + (if (direction_in) @as(u32, 4) else 0);
        const interval = if (endpoint.transfer_type == 3) intervalFor(device.speed, endpoint.interval) else 0;
        contextDword(base, 1 + @as(usize, dci), 0, cs).* = interval << 16; // Interval bits 23:16
        contextDword(base, 1 + @as(usize, dci), 1, cs).* = (@as(u32, 3) << 1) | (endpoint_type << 3) | (@as(u32, endpoint.max_packet_size) << 16);
        const dequeue = ring.region.physical | 1;
        contextDword(base, 1 + @as(usize, dci), 2, cs).* = @truncate(dequeue);
        contextDword(base, 1 + @as(usize, dci), 3, cs).* = @truncate(dequeue >> 32);
        contextDword(base, 1 + @as(usize, dci), 4, cs).* = endpoint.max_packet_size; // Average TRB Length
    }

    fn configureEndpointCommand(self: *Controller, device: *Device) bool {
        const physical = self.submitCommand(.{
            .parameter = device.input_context.physical,
            .control = trbControl(.configure_endpoint, @as(u32, device.slot_id) << 24),
        });
        const code = self.awaitCommand(physical) orelse return false;
        return code == @intFromEnum(CompletionCode.success);
    }

    /// One bulk transfer (IN or OUT per the endpoint address's direction bit) to or
    /// from a caller-owned DMA buffer at `physical`. Returns the number of bytes
    /// transferred, or null on failure/stall. The data never crosses IPC.
    pub fn bulkTransfer(self: *Controller, device: *Device, endpoint: EndpointInfo, physical: u64, length: u32) ?u32 {
        const ring = self.getOrConfigureEndpoint(device, endpoint) orelse return null;
        _ = ring.push(.{
            .parameter = physical,
            .status = length,
            .control = trbControl(.normal, (1 << 5)), // IOC
        });
        mmio.wmb();
        const number: u8 = endpoint.address & 0x0F;
        const direction_in = endpoint.address & 0x80 != 0;
        const dci = doorbellContextIndex(number, direction_in);
        self.ringDoorbell(device.slot_id, dci);
        const code = self.awaitTransfer(device.slot_id, dci, length) orelse return null;
        if (code != @intFromEnum(CompletionCode.success) and code != @intFromEnum(CompletionCode.short_packet)) return null;
        return self.last_transfer_length;
    }

    fn allocateSubscription(self: *Controller) ?*Subscription {
        for (&self.subscriptions) |*subscription| {
            if (!subscription.active) return subscription;
        }
        return null;
    }

    /// Start periodic IN polling of an interrupt endpoint. Each report the device
    /// returns is queued (see `takeReport`), tagged with `device_token` and
    /// `report_endpoint` so the bus layer can push it to the subscriber. Returns
    /// false if the endpoint cannot be configured or no subscription slot is free.
    pub fn subscribeInterrupt(self: *Controller, device: *Device, endpoint: EndpointInfo, device_token: u64, report_endpoint: usize) bool {
        const ring = self.getOrConfigureEndpoint(device, endpoint) orelse return false;
        const subscription = self.allocateSubscription() orelse return false;
        const buffer = dma.alloc(page_size, dma.coherent) orelse return false;
        const number: u8 = endpoint.address & 0x0F;
        const direction_in = endpoint.address & 0x80 != 0;
        subscription.* = .{
            .active = true,
            .slot_id = device.slot_id,
            .dci = doorbellContextIndex(number, direction_in),
            .endpoint_address = endpoint.address,
            .ring = ring,
            .buffer = buffer,
            .max_length = endpoint.max_packet_size,
            .device_token = device_token,
            .report_endpoint = report_endpoint,
        };
        self.armInterrupt(subscription);
        return true;
    }

    // Arm (or re-arm) a subscription's endpoint with a Normal TRB pointing at its
    // report buffer, and ring the endpoint's doorbell so the controller polls it.
    fn armInterrupt(self: *Controller, subscription: *Subscription) void {
        subscription.armed_trb_physical = subscription.ring.push(.{
            .parameter = subscription.buffer.physical,
            .status = subscription.max_length,
            .control = trbControl(.normal, (1 << 5)), // IOC
        });
        mmio.wmb();
        self.ringDoorbell(subscription.slot_id, subscription.dci);
    }

    // A transfer event came off the ring: if it completes a subscription's armed
    // interrupt transfer, copy the report into the queue and re-arm. Returns whether
    // it belonged to a subscription (so the awaiting caller knows it was consumed).
    fn serviceInterruptEvent(self: *Controller, event: Trb) bool {
        const trb_pointer = event.parameter & ~@as(u64, 0xF);
        for (&self.subscriptions) |*subscription| {
            if (!subscription.active or subscription.armed_trb_physical != trb_pointer) continue;
            const code = completionCode(event.status);
            if (code == @intFromEnum(CompletionCode.success) or code == @intFromEnum(CompletionCode.short_packet)) {
                if (subscription.hub) |hub_device| {
                    // Hub status-change report: OR the changed-port bitmap into
                    // the hub's pending mask (bit 0 = the hub itself, ignored;
                    // bit P = downstream port P). The control transfers to
                    // service it run on the bus tick, not here.
                    const bytes: [*]const u8 = @ptrFromInt(subscription.buffer.virtual);
                    var i: usize = 0;
                    while (i < subscription.max_length and i < 4) : (i += 1)
                        hub_device.hub_change_mask |= @as(u32, bytes[i]) << @intCast(i * 8);
                } else {
                    const residual = event.status & 0xFFFFFF;
                    const transferred: u16 = if (residual >= subscription.max_length) 0 else @intCast(subscription.max_length - residual);
                    self.enqueueReport(subscription, transferred);
                }
            }
            self.armInterrupt(subscription); // keep polling
            return true;
        }
        return false;
    }

    fn enqueueReport(self: *Controller, subscription: *Subscription, length: u16) void {
        if (self.report_count >= self.report_queue.len) return; // full: drop the newest
        const report = &self.report_queue[self.report_count];
        report.report_endpoint = subscription.report_endpoint;
        report.device_token = subscription.device_token;
        report.endpoint_address = subscription.endpoint_address;
        report.length = length;
        const source: [*]const u8 = @ptrFromInt(subscription.buffer.virtual);
        const n = @min(length, report.data.len);
        @memcpy(report.data[0..n], source[0..n]);
        self.report_count += 1;
    }

    /// Dequeue the oldest queued interrupt report, or null if none.
    pub fn takeReport(self: *Controller) ?Report {
        if (self.report_count == 0) return null;
        const report = self.report_queue[0];
        var i: usize = 1;
        while (i < self.report_count) : (i += 1) self.report_queue[i - 1] = self.report_queue[i];
        self.report_count -= 1;
        return report;
    }

    /// Drain any events currently on the event ring: interrupt reports into the
    /// report queue, PORT STATUS CHANGES into the port-change queue (hot-plug —
    /// these were silently dropped before M20). Non-blocking — called on the
    /// driver's timer tick.
    pub fn pump(self: *Controller) void {
        while (true) {
            const event = self.nextEvent(system.clock()) orelse return; // deadline=now: null when empty
            const kind = trbType(event.control);
            if (kind == @intFromEnum(TrbType.transfer_event)) {
                _ = self.serviceInterruptEvent(event);
            } else if (kind == @intFromEnum(TrbType.port_status_change_event)) {
                // Port ID rides bits 31:24 of the TRB's first dword.
                const port: u32 = @intCast((event.parameter >> 24) & 0xFF);
                if (port == 0 or port > self.max_ports) continue;
                // Acknowledge the change bits so the port can signal again.
                const status = self.portStatus(port);
                self.writePortStatus(port, (status & ~portsc_write_1_to_clear) | (status & portsc_change_mask));
                if (self.port_change_count < self.port_changes.len) {
                    self.port_changes[self.port_change_count] = port;
                    self.port_change_count += 1;
                }
            }
        }
    }

    /// Dequeue the oldest pending port change (a port whose connect state may
    /// have flipped), or null. The bus layer reads PORTSC to decide plug/unplug.
    pub fn takePortChange(self: *Controller) ?u32 {
        if (self.port_change_count == 0) return null;
        const port = self.port_changes[0];
        var i: usize = 1;
        while (i < self.port_change_count) : (i += 1) self.port_changes[i - 1] = self.port_changes[i];
        self.port_change_count -= 1;
        return port;
    }

    /// Whether a port currently has a device connected (PORTSC.CCS).
    pub fn portConnected(self: *const Controller, port: u32) bool {
        return self.portStatus(port) & portsc_connected != 0;
    }

    /// The tracked device on `port`, or null.
    pub fn deviceOnPort(self: *Controller, port: u32) ?*Device {
        for (&self.devices) |*device| {
            if (device.used and device.port == port) return device;
        }
        return null;
    }

    /// The next used device whose parent hub is `hub_slot` and slot id > `after`
    /// (for recursive teardown when a hub itself disconnects), or null.
    pub fn nextChildOf(self: *Controller, hub_slot: u8, after: u8) ?*Device {
        for (&self.devices) |*device| {
            if (device.used and device.parent_slot == hub_slot and device.slot_id > after) return device;
        }
        return null;
    }

    /// Tear a device down after unplug: cancel its interrupt subscriptions,
    /// Disable Slot (frees the controller's slot state), clear its context-array
    /// entry, and release the tracking slot. DMA regions leak (as elsewhere) —
    /// bounded by the device-slot count.
    pub fn tearDownDevice(self: *Controller, device: *Device) void {
        for (&self.subscriptions) |*subscription| {
            if (subscription.active and subscription.slot_id == device.slot_id) subscription.active = false;
        }
        const physical = self.submitCommand(.{
            .control = trbControl(.disable_slot, @as(u32, device.slot_id) << 24),
        });
        if (self.awaitCommand(physical)) |code| {
            if (code != @intFromEnum(CompletionCode.success))
                std.log.info("slot {d}: Disable Slot completion code {d}", .{ device.slot_id, code });
        } else std.log.info("slot {d}: Disable Slot timed out", .{device.slot_id});
        const array: [*]volatile u64 = @ptrFromInt(self.device_context_array.virtual);
        array[device.slot_id] = 0;
        device.used = false;
    }
};