iommu: DMA-region capabilities — per-grant reachability, protocol flag-day

Replaces L2's interim DMA pool (every buffer reachable by every claimed
device) with true per-grant confinement: a device reaches only buffers
whose capability was delegated to its driver.

Kernel:
- DmaRegionObject (handle kind 2): a delegation token naming a dma_alloc'd
  region, passable across processes on the IPC cap slot like an endpoint
  or shared-memory object. Frames stay owned by the allocating address
  space (freed on dma_free/teardown as before); the token carries a `dead`
  flag so a stale downstream handle can no longer bind a freed region.
- dma_alloc gains the dma_shareable flag: it returns a capability handle
  in r8 and every region is tracked in a registry. A task's own regions
  auto-bind into the devices it claims (its rings just work); foreign
  buffers are bound explicitly.
- dma_bind / dma_unbind / handle_close syscalls (51-53). dma_bind maps a
  held region (or shared-memory) capability into a claimed device's domain;
  it is idempotent. handle_close reclaims a table slot (raised 16 -> 32).
- dma_free and task death unmap a region from every domain and invalidate
  BEFORE its frames return to the allocator — the stale-IOTLB use-after-
  free window, closed structurally.

Protocols (flag-day): block gains attach, usb-transfer gains dma_attach —
each carries a region capability on the cap slot. fat allocates its bounce
buffer shareable and attaches it; usb-storage allocates its transport
buffers shareable, attaches them to the controller, and forwards fat's
capability downstream; usb-xhci-bus binds and closes; virtio-gpu binds its
shared scanout surface. The physical addresses on the wire are unchanged
(identity IOVA), so no register-programming code moved.

Cross-process DMA (fat -> usb-storage -> xHC) now flows only through
delegated capabilities. iommu-usb-storage / iommu-usb-hid / iommu-fault
all green under per-grant enforcement; 104/104 overall (fail-open paths
unchanged).
This commit is contained in:
Daniel Samson 2026-07-23 20:41:14 +01:00
parent e94adcfc02
commit 4e7cbc9792
No known key found for this signature in database
GPG Key ID: A9EB2589C60F7268
17 changed files with 414 additions and 64 deletions

View File

@ -28,6 +28,18 @@ pub const Device = struct {
return .{ .block_size = result.block_size, .block_count = result.block_count };
}
/// Hand the block server a DMA-region capability (`handle` from a `shareable`
/// dma_alloc) so it forwards it to the controller and the buffer's physical
/// addresses become reachable by the device. Call once per buffer before naming it
/// in `read`/`write`. Harmless success when no IOMMU is enforcing.
pub fn attach(self: Device, handle: ipc.Handle) bool {
var request = block_protocol.Request{ .operation = @intFromEnum(block_protocol.Operation.attach), .lba = 0, .count = 0, .physical = 0 };
var reply: [block_protocol.reply_size]u8 = undefined;
const result = ipc.callCap(self.endpoint, std.mem.asBytes(&request), &reply, handle) catch return false;
if (result.len < block_protocol.reply_size) return false;
return std.mem.bytesToValue(block_protocol.Reply, reply[0..block_protocol.reply_size]).status == 0;
}
/// Read `count` blocks starting at `lba` into the DMA buffer at `physical`.
pub fn read(self: Device, lba: u64, count: u32, physical: u64) bool {
return self.transfer(.read, lba, count, physical);

View File

@ -99,6 +99,19 @@ pub fn msiBind(device_id: u64, endpoint: usize) ?Msi {
return .{ .address = rax, .data = @intCast(rdx) };
}
/// Map a delegated DMA-region (or shared-memory) capability into a claimed device's
/// IOMMU domain, so the device may DMA to that buffer. The caller must own `device_id`
/// and hold `handle` (received over IPC or from its own `dma.alloc(.. | shareable)`).
/// Idempotent. Returns true on success (and trivially when no IOMMU is present).
pub fn dmaBind(device_id: u64, handle: usize) bool {
return !failed(sc.systemCall2(.dma_bind, device_id, handle));
}
/// Unmap a previously `dmaBind`'d buffer from the device's domain.
pub fn dmaUnbind(device_id: u64, handle: usize) bool {
return !failed(sc.systemCall2(.dma_unbind, device_id, handle));
}
/// Drain and log any pending IOMMU translation faults, returning the count seen. A
/// diagnostic: a driver that suspects its device attempted an out-of-domain DMA (or a
/// test proving enforcement) forces the hardware's fault records to the log now. Returns

View File

@ -99,6 +99,19 @@ pub const Device = struct {
return std.mem.bytesToValue(usb_transfer_protocol.InterruptSubscribeReply, reply[0..@sizeOf(usb_transfer_protocol.InterruptSubscribeReply)]).status == 0;
}
/// Hand the controller a DMA-region capability (`handle` from a `shareable`
/// dma_alloc, or forwarded from another process) so it binds that buffer into its
/// IOMMU domain. Must be called for every buffer whose physical address this device
/// will name in a `bulk` transfer, before the transfer. Harmless (and a no-op
/// success) when no IOMMU is enforcing. Returns false on failure.
pub fn attachDma(self: *Device, handle: ipc.Handle) bool {
var request = usb_transfer_protocol.DmaAttachRequest{ .device_token = self.token };
var reply: [@sizeOf(usb_transfer_protocol.DmaAttachReply)]u8 = undefined;
const result = ipc.callCap(self.bus, std.mem.asBytes(&request), &reply, handle) catch return false;
if (result.len < @sizeOf(usb_transfer_protocol.DmaAttachReply)) return false;
return std.mem.bytesToValue(usb_transfer_protocol.DmaAttachReply, reply[0..@sizeOf(usb_transfer_protocol.DmaAttachReply)]).status == 0;
}
/// One bulk transfer (IN or OUT per `endpoint_address`'s direction bit) to or
/// from the caller's own DMA buffer at `physical`. Returns the bytes moved.
pub fn bulk(self: *Device, endpoint_address: u8, physical: u64, length: u32) ?u32 {

View File

@ -34,6 +34,14 @@ pub fn register(id: abi.ServiceId, h: Handle) bool {
return !failed(sc.systemCall2(.ipc_register, @intFromEnum(id), h));
}
/// Drop a capability handle (endpoint, shared-memory, or DMA-region) and free its table
/// slot. A forwarding hop closes a cap it passed on; a binder closes a DMA-region cap
/// once the binding holds its own reference the 32-slot table is otherwise consumed by
/// repeated cap-passing.
pub fn close(h: Handle) bool {
return !failed(sc.systemCall1(.handle_close, h));
}
/// Find the endpoint published under `id`, installing a handle to it in this
/// process.
pub fn lookup(id: abi.ServiceId) ?Handle {

View File

@ -12,12 +12,18 @@ const sc = @import("system-call");
pub const coherent: usize = abi.dma_coherent;
pub const write_combining: usize = abi.dma_write_combining;
pub const below_4g: usize = abi.dma_below_4g;
/// Ask for a capability handle (in `Region.handle`) so the buffer can be delegated to
/// another driver and bound into a device's IOMMU domain (`driver.dmaBind`). A driver's
/// private rings don't need it; a buffer whose physical address crosses IPC does.
pub const shareable: usize = abi.dma_shareable;
/// A DMA allocation: the `virtual` address the CPU touches, and the `physical` address
/// to program into the device's descriptor-ring / base registers.
/// A DMA allocation: the `virtual` address the CPU touches, the `physical` address to
/// program into the device's registers, and when `shareable` was requested a
/// capability `handle` naming the region for delegation (null otherwise).
pub const Region = struct {
virtual: usize,
physical: usize,
handle: ?usize = null,
};
inline fn failed(r: usize) bool {
@ -25,21 +31,23 @@ inline fn failed(r: usize) bool {
}
/// Allocate `len` bytes of DMA-capable memory with `flags` (e.g. `coherent`, or
/// `coherent | below_4g`). Returns the virtual/physical pair, or null on failure. Two
/// return values the virtual address in rax, the physical address in rdx so it
/// needs a hand-written stub.
/// `coherent | shareable`). Returns virtual/physical (and a handle when `shareable`), or
/// null on failure. Three return values virtual in rax, physical in rdx, handle in r8
/// so it needs a hand-written stub.
pub fn alloc(len: usize, flags: usize) ?Region {
var rax: usize = undefined;
var rdx: usize = undefined; // out: physical address
var r8: usize = undefined; // out: capability handle (abi.no_cap unless shareable)
asm volatile ("syscall"
: [rax] "={rax}" (rax),
[rdx] "={rdx}" (rdx),
[r8] "={r8}" (r8),
: [n] "{rax}" (@intFromEnum(abi.SystemCall.dma_alloc)),
[a0] "{rdi}" (len),
[a1] "{rsi}" (flags),
: .{ .rcx = true, .r11 = true, .memory = true });
if (failed(rax)) return null;
return .{ .virtual = rax, .physical = rdx };
return .{ .virtual = rax, .physical = rdx, .handle = if (r8 == abi.no_cap) null else r8 };
}
/// Release a region from a prior `alloc` (`virtual` and the same `len`).

View File

@ -38,6 +38,7 @@ pub const DmaRegion = dma.Region;
pub const dma_coherent = dma.coherent;
pub const dma_write_combining = dma.write_combining;
pub const dma_below_4g = dma.below_4g;
pub const dma_shareable = dma.shareable;
pub const dmaAlloc = dma.alloc;
pub const dmaFree = dma.free;

View File

@ -7,7 +7,9 @@
//! buffer**, named by its physical address the same physical-address handoff
//! usb-storage already uses toward the controller, one layer up. So a 512-byte
//! sector never has to cross the 256-byte IPC boundary; only the small request /
//! reply headers do. (Safe while the IOMMU is unenforced; see docs/driver-model.md.)
//! reply headers do. Under an enforcing IOMMU the buffer's physical addresses are
//! only reachable by the device once the filesystem has `attach`ed the buffer's
//! capability (the block server forwards it to the controller); see docs/driver-model.md.
pub const Operation = enum(u32) {
/// geometry() -> { block_size, block_count }
@ -20,6 +22,11 @@ pub const Operation = enum(u32) {
/// A filesystem calls this to make prior writes durable e.g. before power-off,
/// so a shutdown-time write isn't lost in the USB flash controller's cache.
flush = 3,
/// attach(): the caller's DMA-region capability rides the call's cap slot; the
/// block server forwards it to the controller so the buffer's physical addresses
/// (named in later read/write) are reachable by the device under an enforcing
/// IOMMU. Call once per buffer before using it in a transfer.
attach = 4,
};
pub const Request = extern struct {

View File

@ -45,6 +45,11 @@ pub const Operation = enum(u32) {
control = 1,
interrupt_subscribe = 2,
bulk = 3,
/// dma_attach: a class driver hands the controller a DMA-region capability (riding
/// the call's cap slot) so the controller binds that buffer into its IOMMU domain
/// and may then DMA to the physical addresses inside it. Needed once per buffer the
/// class driver will name in a `bulk` transfer (its own, or one forwarded to it).
dma_attach = 4,
};
/// The endpoint facts a class driver needs, lifted from the endpoint descriptor
@ -138,6 +143,19 @@ pub const BulkReply = extern struct {
actual_length: u32,
};
/// dma_attach: the region capability rides the call's cap slot; the body only carries
/// the device token (scoping) so the controller knows which caller is attaching.
pub const DmaAttachRequest = extern struct {
operation: u32 = @intFromEnum(Operation.dma_attach),
reserved: u32 = 0,
device_token: u64,
};
pub const DmaAttachReply = extern struct {
status: i32,
reserved: u32 = 0,
};
/// An asynchronous interrupt report, pushed with `ipc.send` to a subscriber's
/// endpoint. `Received.isMessage()` is set; there is no reply owed.
pub const InterruptReport = extern struct {

View File

@ -77,6 +77,9 @@ pub const SystemCall = enum(u64) {
fs_mount = 48, // fs_mount(prefix_ptr, prefix_len, backend_handle, rewrite_ptr, rewrite_len) -> 0/-errno: mount a userspace filesystem's endpoint at an absolute prefix (possession of the handle is the capability)
fs_unmount = 49, // fs_unmount(prefix_ptr, prefix_len) -> 0/-errno: remove a backend mount
iommu_fault_drain = 50, // iommu_fault_drain() -> count: drain + log pending IOMMU translation faults (a diagnostic; the count of faults seen this call)
dma_bind = 51, // dma_bind(device_id, region_handle) -> 0/-errno: map a DMA-region capability into the claimed device's IOMMU domain (idempotent). The caller must own the device and hold the handle
dma_unbind = 52, // dma_unbind(device_id, region_handle) -> 0/-errno: unmap a previously bound region from the device's domain and invalidate
handle_close = 53, // handle_close(handle) -> 0/-errno: drop one capability handle and free its table slot (endpoints, shared-memory, DMA regions)
_,
};
@ -114,6 +117,7 @@ pub const msi_address_base: u64 = 0xFEE0_0000;
pub const dma_coherent: u64 = 1; // strong-uncacheable the default, the only portable one
pub const dma_write_combining: u64 = 2; // write-combining (framebuffers); needs PAT
pub const dma_below_4g: u64 = 4; // physical address must fit 32 bits (legacy DMA engines)
pub const dma_shareable: u64 = 8; // return a capability handle (r8) so the region can be delegated + dma_bound
/// Set in the badge returned by `ipc_reply_wait` when what arrived is an
/// **asynchronous notification** (a device interrupt bound with `irq_bind`, or a

View File

@ -90,9 +90,22 @@ fn initialise(endpoint: ipc.Handle) bool {
bring_up_failed = true;
return false;
};
command_wrapper = memory.dmaAlloc(4096, memory.dma_coherent) orelse return false;
status_wrapper = memory.dmaAlloc(4096, memory.dma_coherent) orelse return false;
command_data = memory.dmaAlloc(4096, memory.dma_coherent) orelse return false;
// Shareable, so each buffer's capability can be handed to the controller: usb-storage
// owns no device, so its buffers are not auto-bound anywhere the controller reaches
// them only once attached. (No-op binding when no IOMMU is enforcing.)
command_wrapper = memory.dmaAlloc(4096, memory.dma_coherent | memory.dma_shareable) orelse return false;
status_wrapper = memory.dmaAlloc(4096, memory.dma_coherent | memory.dma_shareable) orelse return false;
command_data = memory.dmaAlloc(4096, memory.dma_coherent | memory.dma_shareable) orelse return false;
for ([_]memory.DmaRegion{ command_wrapper, status_wrapper, command_data }) |region| {
if (region.handle) |handle| {
if (!device.attachDma(handle)) {
_ = logging.write("/system/drivers/usb-storage: could not attach a DMA buffer to the controller\n");
bring_up_failed = true;
return false;
}
_ = ipc.close(handle); // the binding holds its own reference now
}
}
// Bring the LUN up: wait for it to be ready (clearing the initial unit-attention
// with REQUEST SENSE), identify it, and read its capacity.
@ -135,10 +148,17 @@ fn initialise(endpoint: ipc.Handle) bool {
/// caller's DMA buffer (named by physical address).
fn onMessage(message: []const u8, reply: []u8, sender: u32, capability: ?ipc.Handle) usize {
_ = sender;
_ = capability;
if (message.len < block_protocol.request_size) return 0;
const request = std.mem.bytesToValue(block_protocol.Request, message[0..block_protocol.request_size]);
switch (request.operation) {
@intFromEnum(block_protocol.Operation.attach) => {
// The filesystem's DMA buffer: forward its capability to the controller so
// the device can reach it, then release our copy (the binding holds a ref).
const handle = capability orelse return writeReply(reply, .{ .status = -1, .block_size = 0, .block_count = 0 });
const ok = device.attachDma(handle);
_ = ipc.close(handle);
return writeReply(reply, .{ .status = if (ok) 0 else -1, .block_size = 0, .block_count = 0 });
},
@intFromEnum(block_protocol.Operation.geometry) => {
return writeReply(reply, .{ .status = 0, .block_size = block_size, .block_count = block_count });
},

View File

@ -484,10 +484,22 @@ fn onMessage(message: []const u8, reply: []u8, sender: u32, capability: ?ipc.Han
@intFromEnum(usb_transfer_protocol.Operation.control) => handleControl(message, reply),
@intFromEnum(usb_transfer_protocol.Operation.interrupt_subscribe) => handleSubscribe(message, reply),
@intFromEnum(usb_transfer_protocol.Operation.bulk) => handleBulk(message, reply),
@intFromEnum(usb_transfer_protocol.Operation.dma_attach) => handleDmaAttach(message, reply, capability),
else => 0,
};
}
/// dma_attach: bind the class driver's DMA-region capability into the controller's IOMMU
/// domain, so the controller may DMA to the physical addresses inside that buffer. The
/// binding holds its own kernel reference, so the forwarded capability is closed here.
fn handleDmaAttach(message: []const u8, reply: []u8, capability: ?ipc.Handle) usize {
if (message.len < @sizeOf(usb_transfer_protocol.DmaAttachRequest)) return writeReply(reply, usb_transfer_protocol.DmaAttachReply{ .status = -1 });
const handle = capability orelse return writeReply(reply, usb_transfer_protocol.DmaAttachReply{ .status = -1 });
const ok = device.dmaBind(controller_id, handle);
_ = ipc.close(handle);
return writeReply(reply, usb_transfer_protocol.DmaAttachReply{ .status = if (ok) 0 else -1 });
}
fn writeReply(reply: []u8, value: anytype) usize {
const bytes = std.mem.asBytes(&value);
@memcpy(reply[0..bytes.len], bytes);

View File

@ -323,6 +323,15 @@ fn initialise(endpoint: ipc.Handle) bool {
std.log.info("could not resolve the scanout surface physical address", .{});
return false;
};
// Bind the shared surface into this device's IOMMU domain so the GPU may DMA the
// framebuffer (attach_backing points it here). The ring/command buffers are
// dma_alloc'd and auto-bound; a shared-memory surface needs an explicit bind. We keep
// the handle (it is also passed to the display service), so do not close it. No-op
// without an IOMMU.
if (!device.dmaBind(device_id, surface.handle)) {
std.log.info("could not bind the scanout surface for DMA", .{});
return false;
}
{
const request = requestAt(vg.ResourceAttachBacking);
request.* = .{

View File

@ -128,29 +128,18 @@ pub fn init() void {
const Confined = struct { active: bool = false, owner: u32 = 0, bdf: u16 = 0, domain: u16 = invalid_domain };
var confined: [maximum_domains]Confined = .{Confined{}} ** maximum_domains;
/// The interim DMA pool: the set of `dma_alloc`'d regions. Every such region is mapped
/// into every claimed device's domain, so a device reaches DMA buffers (including one
/// another's the honest limit of this stage) but NOT the kernel, page tables, process
/// heaps, or arbitrary RAM. The capability layer (L3/L4) narrows this to per-grant.
const PoolRegion = struct { active: bool = false, physical: u64 = 0, len: u64 = 0 };
var pool: [maximum_pool]PoolRegion = .{PoolRegion{}} ** maximum_pool;
const maximum_pool = 256;
/// Place a just-claimed PCI function under IOMMU translation on behalf of `owner`: give
/// it a private empty domain, seed it with the current DMA pool and the device's own
/// firmware reserved region, and attach. false only if a domain can't be allocated
/// the caller rolls the claim back (a claim that can't be confined must not stand). No-op
/// success when no IOMMU exists (fail-open).
/// it a private empty domain, seed it with the device's own firmware reserved region,
/// and attach. Its DMA buffers arrive afterward as explicit grants the owner's own
/// `dma_alloc`'d regions are bound by the claim path (`mapForDevice`), and cross-process
/// buffers by `dma_bind`. false only if a domain can't be allocated the caller rolls
/// the claim back (a claim that can't be confined must not stand). No-op success when no
/// IOMMU exists (fail-open).
pub fn confineDevice(device_id: u64, bdf: u16, owner: u32) bool {
if (kind == .none) return true;
if (device_id >= confined.len) return true; // unusual id; leave it to fail-open
const domain = domainCreate(owner, bdf) orelse return false;
// Seed with every pooled DMA region so the device's own rings/buffers and the
// cross-process buffers handed to it (fat's bounce buffer via usb-storage) resolve.
for (&pool) |*r| {
if (r.active) _ = map(domain, r.physical, r.len);
}
// Firmware reserved region for this device, if any (real hardware; QEMU has none).
const info = platform.platformInformation();
var i: usize = 0;
@ -164,34 +153,45 @@ pub fn confineDevice(device_id: u64, bdf: u16, owner: u32) bool {
return true;
}
/// Record a newly `dma_alloc`'d region and map it into every claimed device's domain
/// (the interim pool rule). Called from the dma_alloc syscall.
pub fn poolAdd(physical: u64, len: u64) void {
/// The confined record for `device_id`, or null if the device is not confined.
fn confinedOf(device_id: u64) ?*Confined {
if (device_id >= confined.len) return null;
const c = &confined[@intCast(device_id)];
return if (c.active) c else null;
}
/// Map a DMA region into a specific claimed device's domain (the device owner binding a
/// granted buffer). false if the device is not confined. No-op success without an IOMMU.
pub fn mapForDevice(device_id: u64, physical: u64, len: u64) bool {
if (kind == .none) return true;
const c = confinedOf(device_id) orelse return false;
return map(c.domain, physical, len);
}
/// Unmap a DMA region from a specific claimed device's domain. No-op if not confined.
pub fn unmapForDevice(device_id: u64, physical: u64, len: u64) void {
if (kind == .none) return;
const c = confinedOf(device_id) orelse return;
unmap(c.domain, physical, len);
}
/// Map a region into every claimed device owned by `owner` the auto-bind of a task's
/// own freshly-`dma_alloc`'d buffer into the devices it drives.
pub fn mapRegionForOwner(owner: u32, physical: u64, len: u64) void {
if (kind == .none) return;
for (&pool) |*r| {
if (!r.active) {
r.* = .{ .active = true, .physical = physical, .len = len };
break;
}
} else return; // pool full; region stays unmapped and its device DMA will fault
for (&confined) |*c| {
if (c.active) _ = map(c.domain, physical, len);
if (c.active and c.owner == owner) _ = map(c.domain, physical, len);
}
}
/// Unmap a freed DMA region from every claimed device's domain and forget it. MUST run
/// before the frames return to pmm a device translating to a reallocated frame is the
/// use-after-free this prevents. Called from the dma_free syscall.
pub fn poolRemove(physical: u64, len: u64) void {
/// Unmap a region from EVERY claimed device's domain the freed-region sweep. MUST run
/// before the frames return to pmm: a device translating to a reallocated frame is the
/// use-after-free this prevents. Cross-device because a granted buffer may be bound in a
/// domain other than its owner's.
pub fn unmapRegionEverywhere(physical: u64, len: u64) void {
if (kind == .none) return;
for (&pool) |*r| {
if (r.active and r.physical == physical and r.len == len) {
for (&confined) |*c| {
if (c.active) unmap(c.domain, physical, len);
}
r.* = .{};
return;
}
for (&confined) |*c| {
if (c.active) unmap(c.domain, physical, len);
}
}

View File

@ -154,6 +154,66 @@ pub fn dropRef(endpoint: *Endpoint) void {
/// Defined here (not in scheduler) because the meaning is the IPC/capability layer's.
pub const handle_kind_endpoint: u8 = 0;
pub const handle_kind_shared_memory: u8 = 1;
pub const handle_kind_dma_region: u8 = 2;
/// A DMA buffer's delegation token: names the `pages` contiguous frames at `phys` that
/// `dma_alloc` handed its `owner`, and can be passed across processes as a capability so
/// the driver that owns a device can bind it into that device's IOMMU domain
/// (`dma_bind`). Unlike `SharedMemoryObject` this does NOT own the frames the
/// allocating address space still does, and frees them on `dma_free` or teardown so
/// this is a pure token: `dead` is set when the allocator frees the region, after which
/// a stale handle can no longer bind it. `refcount` counts the allocator's registry
/// entry plus every outstanding handle; the object is freed when the last drops.
pub const DmaRegionObject = struct {
refcount: u32 = 1,
phys: u64,
pages: usize,
owner: u32,
dead: bool = false,
};
/// Create a DMA-region token for `pages` frames at `phys` owned by task `owner`. The
/// frames are already allocated and mapped by the caller; this only wraps them for
/// delegation. null if the heap is out of room.
pub fn createDmaRegion(phys: u64, pages: usize, owner: u32) ?*DmaRegionObject {
const region = heap.allocator().create(DmaRegionObject) catch return null;
region.* = .{ .phys = phys, .pages = pages, .owner = owner };
return region;
}
/// Drop a DMA-region reference; free the token when the last (registry + handles) goes.
/// Never frees frames the allocator owns those.
pub fn dropDmaRegionReference(region: *DmaRegionObject) void {
if (region.refcount > 1) {
region.refcount -= 1;
} else {
heap.allocator().destroy(region);
}
}
/// Resolve a handle to its DMA-region token, or null if out of range, unused, or a
/// different kind.
pub fn resolveDmaRegion(t: *Task, h: u64) ?*DmaRegionObject {
if (h >= t.handles.len) return null;
const entry = t.handles[@intCast(h)] orelse return null;
if (entry.kind != handle_kind_dma_region) return null;
return @ptrCast(@alignCast(entry.ptr));
}
/// Install a DMA-region handle in task `t`'s table (the slot owns a reference).
pub fn installDmaRegionHandle(t: *Task, region: *DmaRegionObject) i64 {
return installEntry(t, .{ .kind = handle_kind_dma_region, .ptr = @ptrCast(region) });
}
/// Drop the handle at slot `h` of task `t` (handle_close): release its reference and
/// free the slot. Returns 0 or -EBADF.
pub fn closeHandle(t: *Task, h: u64) i64 {
if (h >= t.handles.len) return -EBADF;
const entry = t.handles[@intCast(h)] orelse return -EBADF;
dropEntry(entry);
t.handles[@intCast(h)] = null;
return 0;
}
/// A page-aligned block of **shared cacheable RAM** (docs/display-v2.md), referenced by
/// capability handles across processes and freed when the last one drops. `phys` is its
@ -307,6 +367,10 @@ fn shareCapability(from: *Task, to: *Task, cap: u64) i64 {
const s: *SharedMemoryObject = @ptrCast(@alignCast(entry.ptr));
s.refcount += 1;
},
handle_kind_dma_region => {
const r: *DmaRegionObject = @ptrCast(@alignCast(entry.ptr));
r.refcount += 1;
},
else => return -EBADF,
}
const handle = installEntry(to, entry);
@ -574,6 +638,7 @@ fn dropEntry(entry: scheduler.HandleObject) void {
switch (entry.kind) {
handle_kind_endpoint => dropRef(@ptrCast(@alignCast(entry.ptr))),
handle_kind_shared_memory => dropSharedMemoryReference(@ptrCast(@alignCast(entry.ptr))),
handle_kind_dma_region => dropDmaRegionReference(@ptrCast(@alignCast(entry.ptr))),
else => {},
}
}

View File

@ -252,6 +252,9 @@ fn system_call(state: *architecture.CpuState) void {
.fs_mount => systemFsMount(state),
.fs_unmount => systemFsUnmount(state),
.iommu_fault_drain => systemIommuFaultDrain(state),
.dma_bind => systemDmaBind(state),
.dma_unbind => systemDmaUnbind(state),
.handle_close => systemHandleClose(state),
.wall_clock => systemWallClock(state),
.shared_memory_create => systemSharedMemoryCreate(state),
.shared_memory_map => systemSharedMemoryMap(state),
@ -398,10 +401,14 @@ fn systemDeviceClaim(state: *architecture.CpuState) void {
// since the whole point is that claiming a DMA device is no longer equivalent
// to ring 0. No-op when no IOMMU exists (fail-open).
if (devices_broker.pciAddressOf(device_id)) |bdf| {
if (!iommu.confineDevice(device_id, bdf, scheduler.current().id)) {
_ = devices_broker.unclaim(device_id, scheduler.current().id);
const owner = scheduler.current().id;
if (!iommu.confineDevice(device_id, bdf, owner)) {
_ = devices_broker.unclaim(device_id, owner);
return fail(state);
}
// Bind the buffers this task allocated before claiming the device (a driver
// that dma_alloc'd its rings, then claimed the controller).
dmaBindOwnerRegionsInto(owner, device_id);
}
// A display service just took the framebuffer quiesce the bootstrap console
// so the kernel and the service don't scribble over each other's pixels. The
@ -510,6 +517,72 @@ fn systemIoWrite(state: *architecture.CpuState) void {
/// their physical address is never disclosed. `dma_below_4g` caps the physical address
/// for legacy engines; `dma_write_combining` is accepted but falls back to coherent
/// until PAT is programmed. See docs/driver-model.md (M14).
// --- DMA-region registry ---------------------------------------------------------------
// Every dma_alloc'd region is tracked here so it can be (a) auto-bound into the devices
// its owner claims, (b) delegated across processes as a capability and bound into a
// device's IOMMU domain by dma_bind, and (c) unmapped from every domain before its frames
// return to the allocator. Only *shareable* regions carry a heap `object` (the delegation
// token); a driver's private rings are tracked without one. All access under the big lock.
const DmaRegistryEntry = struct {
active: bool = false,
object: ?*ipc.DmaRegionObject = null,
physical: u64 = 0,
len: u64 = 0,
owner: u32 = 0,
};
const maximum_dma_regions = 256;
var dma_registry: [maximum_dma_regions]DmaRegistryEntry = .{DmaRegistryEntry{}} ** maximum_dma_regions;
fn dmaRegistryAdd(object: ?*ipc.DmaRegionObject, physical: u64, len: u64, owner: u32) void {
for (&dma_registry) |*e| {
if (!e.active) {
e.* = .{ .active = true, .object = object, .physical = physical, .len = len, .owner = owner };
return;
}
}
}
/// Retire the region based at `physical` owned by `owner`: unmap it from every device
/// domain (before the frames are freed), mark its token dead so a stale downstream handle
/// can no longer bind it, drop the registry's reference, and clear the slot.
fn dmaRegistryRemove(owner: u32, physical: u64) void {
for (&dma_registry) |*e| {
if (e.active and e.owner == owner and e.physical == physical) {
iommu.unmapRegionEverywhere(e.physical, e.len);
if (e.object) |object| {
object.dead = true;
ipc.dropDmaRegionReference(object);
}
e.* = .{};
return;
}
}
}
/// Retire every region owned by `owner` (task death) same discipline as a per-region
/// free, run before the address space is torn down and its DMA frames reclaimed.
fn dmaRegistryReleaseOwner(owner: u32) void {
for (&dma_registry) |*e| {
if (e.active and e.owner == owner) {
iommu.unmapRegionEverywhere(e.physical, e.len);
if (e.object) |object| {
object.dead = true;
ipc.dropDmaRegionReference(object);
}
e.* = .{};
}
}
}
/// Bind every region `owner` allocated into the domain of the device it just claimed
/// (its rings allocated before the claim). Regions allocated *after* the claim are bound
/// by dma_alloc's own auto-bind.
fn dmaBindOwnerRegionsInto(owner: u32, device_id: u64) void {
for (&dma_registry) |*e| {
if (e.active and e.owner == owner) _ = iommu.mapForDevice(device_id, e.physical, e.len);
}
}
/// iommu_fault_drain() -> count: drain and log any pending IOMMU translation faults,
/// returning how many were seen. A diagnostic hook a driver (or a test) that suspects
/// its device faulted can force the fault records to be logged now rather than waiting
@ -521,6 +594,60 @@ fn systemIommuFaultDrain(state: *architecture.CpuState) void {
architecture.setSystemCallResult(state, count);
}
/// Resolve a capability handle the caller holds to the physical range it names either
/// a DMA-region token or a shared-memory object (both are bindable buffers). null if the
/// handle is neither, or names a region already freed by its allocator.
fn bindableRange(t: *scheduler.Task, handle: u64) ?struct { physical: u64, len: u64 } {
if (ipc.resolveDmaRegion(t, handle)) |region| {
if (region.dead) return null;
return .{ .physical = region.phys, .len = region.pages * page_size };
}
if (ipc.resolveSharedMemory(t, handle)) |shared| {
return .{ .physical = shared.phys, .len = shared.pages * page_size };
}
return null;
}
/// dma_bind(device_id, handle) -> 0/-errno: map the buffer named by `handle` into the
/// claimed device's IOMMU domain. The caller must own the device (the mmio_map gate) and
/// hold the handle. Idempotent: re-binding is a harmless success, so a driver may re-bind
/// after a restart without tracking what it already bound. Success (no-op) without an IOMMU.
fn systemDmaBind(state: *architecture.CpuState) void {
const device_id = architecture.systemCallArg(state, 0);
const handle = architecture.systemCallArg(state, 1);
const t = scheduler.current();
const flags = sync.enter();
defer sync.leave(flags);
if (devices_broker.ownerOf(device_id) != t.id) return fail(state);
const range = bindableRange(t, handle) orelse return fail(state);
if (!iommu.mapForDevice(device_id, range.physical, range.len)) return fail(state);
architecture.setSystemCallResult(state, 0);
}
/// dma_unbind(device_id, handle) -> 0/-errno: unmap a previously bound buffer from the
/// device's domain and invalidate.
fn systemDmaUnbind(state: *architecture.CpuState) void {
const device_id = architecture.systemCallArg(state, 0);
const handle = architecture.systemCallArg(state, 1);
const t = scheduler.current();
const flags = sync.enter();
defer sync.leave(flags);
if (devices_broker.ownerOf(device_id) != t.id) return fail(state);
const range = bindableRange(t, handle) orelse return fail(state);
iommu.unmapForDevice(device_id, range.physical, range.len);
architecture.setSystemCallResult(state, 0);
}
/// handle_close(handle) -> 0/-errno: drop one capability handle and free its slot.
fn systemHandleClose(state: *architecture.CpuState) void {
const handle = architecture.systemCallArg(state, 0);
const t = scheduler.current();
const flags = sync.enter();
defer sync.leave(flags);
if (ipc.closeHandle(t, handle) < 0) return fail(state);
architecture.setSystemCallResult(state, 0);
}
fn systemDmaAlloc(state: *architecture.CpuState) void {
const len = architecture.systemCallArg(state, 0);
const flags = architecture.systemCallArg(state, 1);
@ -567,15 +694,33 @@ fn systemDmaAlloc(state: *architecture.CpuState) void {
architecture.mapUserDmaInto(t.address_space, base_v + i * page_size, phys + i * page_size, page_size);
sync.leave(lock_flags);
}
// Publish the region to the DMA pool: it becomes reachable to every claimed device
// (the interim rule until DMA-region capabilities land). No-op without an IOMMU.
// Register the region and bind it into every device this task already drives (its
// own buffers reach its own devices). When `dma_shareable` is set, also wrap it in a
// capability token and return a handle so it can be delegated to another driver and
// dma_bound there. All under the lock (the registry + IOMMU tables are shared).
var handle: u64 = abi.no_cap;
{
const lock_flags = sync.enter();
iommu.poolAdd(phys, pages * page_size);
sync.leave(lock_flags);
defer sync.leave(lock_flags);
var object: ?*ipc.DmaRegionObject = null;
if (flags & abi.dma_shareable != 0) {
if (ipc.createDmaRegion(phys, pages, t.id)) |region| {
const h = ipc.installDmaRegionHandle(t, region);
if (h >= 0) {
region.refcount += 1; // the handle's reference (registry holds the first)
handle = @intCast(h);
object = region;
} else {
ipc.dropDmaRegionReference(region); // no table slot; drop it
}
}
}
dmaRegistryAdd(object, phys, pages * page_size, t.id);
iommu.mapRegionForOwner(t.id, phys, pages * page_size);
}
architecture.setSystemCallResult(state, base_v); // virtual address for the CPU
architecture.setSystemCallResult2(state, phys); // physical address for the device
architecture.setSystemCallResult3(state, handle); // capability handle (no_cap unless shareable)
}
/// dma_free(virtual_address, len) -> 0: release a prior `dma_alloc`. Bounded to the DMA arena so
@ -591,14 +736,13 @@ fn systemDmaFree(state: *architecture.CpuState) void {
const pages: usize = @intCast((len + page_size - 1) / page_size);
if (base_v < dma_arena_base or base_v + pages * page_size > dma_arena_end) return fail(state);
// Pull the region out of every device's domain and invalidate BEFORE any frame
// returns to the allocator a device still translating to a reallocated frame is a
// use-after-free. dma_alloc's frames are contiguous, so the base translation names
// the whole region.
// Retire the region unmap it from every device domain and mark its token dead
// BEFORE any frame returns to the allocator, so no device can still translate to a
// reallocated frame. dma_alloc's frames are contiguous, so the base names the region.
{
const lock_flags = sync.enter();
if (architecture.translate(t.address_space, base_v)) |base_phys|
iommu.poolRemove(base_phys, pages * page_size);
dmaRegistryRemove(t.id, base_phys);
sync.leave(lock_flags);
}
@ -1036,8 +1180,11 @@ fn releaseTaskResourcesLocked(t: *scheduler.Task) void {
// Detach the task's devices from their IOMMU domains BEFORE the broker clears the
// claims (the detach reads ownership) and before the address space is torn down and
// its DMA frames return to the allocator a device must stop translating to a frame
// before that frame can be handed to someone else.
// before that frame can be handed to someone else. Then retire the buffers this task
// allocated, unmapping them from any *other* driver's domain they were granted into,
// before those frames are freed too.
iommu.releaseAllOwnedBy(t.id);
dmaRegistryReleaseOwner(t.id);
devices_broker.releaseAllOwnedBy(t.id);
// If that dropped the framebuffer claim (this task was the display service), let the
// bootstrap console draw again the screen is nobody's now, so panics/status land.

View File

@ -149,7 +149,10 @@ pub const maximum_task_name = abi.maximum_process_name;
/// Size of each task's IPC handle table. Kept here (not in ipc_sync.zig) because
/// it dimensions a field of `Task`; ipc_sync.zig re-exports it.
pub const ipc_maximum_handles = 16;
// Raised from 16 with DMA-region capabilities: a driver now holds its per-device
// channel endpoints plus received DMA-region handles (a storage driver forwards several
// buffer caps), and repeated cap-passing consumes slots until handle_close.
pub const ipc_maximum_handles = 32;
/// One handle-table entry: a capability object plus a `kind` tag saying what `ptr` points
/// at (an ipc endpoint or a shared-memory object), so a task's exit path and the

View File

@ -118,7 +118,17 @@ fn tryBringUp() void {
_ = logging.write("/system/services/fat: block geometry unavailable\n");
return;
};
const bounce = memory.dmaAlloc(engine.max_transfer_sectors * 512, memory.dma_coherent) orelse return;
// Shareable so the buffer's capability can be attached down the chain (block server
// -> controller), making its physical addresses reachable by the device under an
// enforcing IOMMU. No-op binding otherwise.
const bounce = memory.dmaAlloc(engine.max_transfer_sectors * 512, memory.dma_coherent | memory.dma_shareable) orelse return;
if (bounce.handle) |handle| {
if (!device.attach(handle)) {
_ = logging.write("/system/services/fat: could not attach the DMA bounce buffer\n");
return;
}
_ = ipc.close(handle); // the binding holds its own reference now
}
ipc_block = .{ .device = device, .bounce = bounce };
const block_device = engine.BlockDevice{