iommu: DMA-region capabilities — per-grant reachability, protocol flag-day
Replaces L2's interim DMA pool (every buffer reachable by every claimed device) with true per-grant confinement: a device reaches only buffers whose capability was delegated to its driver. Kernel: - DmaRegionObject (handle kind 2): a delegation token naming a dma_alloc'd region, passable across processes on the IPC cap slot like an endpoint or shared-memory object. Frames stay owned by the allocating address space (freed on dma_free/teardown as before); the token carries a `dead` flag so a stale downstream handle can no longer bind a freed region. - dma_alloc gains the dma_shareable flag: it returns a capability handle in r8 and every region is tracked in a registry. A task's own regions auto-bind into the devices it claims (its rings just work); foreign buffers are bound explicitly. - dma_bind / dma_unbind / handle_close syscalls (51-53). dma_bind maps a held region (or shared-memory) capability into a claimed device's domain; it is idempotent. handle_close reclaims a table slot (raised 16 -> 32). - dma_free and task death unmap a region from every domain and invalidate BEFORE its frames return to the allocator — the stale-IOTLB use-after- free window, closed structurally. Protocols (flag-day): block gains attach, usb-transfer gains dma_attach — each carries a region capability on the cap slot. fat allocates its bounce buffer shareable and attaches it; usb-storage allocates its transport buffers shareable, attaches them to the controller, and forwards fat's capability downstream; usb-xhci-bus binds and closes; virtio-gpu binds its shared scanout surface. The physical addresses on the wire are unchanged (identity IOVA), so no register-programming code moved. Cross-process DMA (fat -> usb-storage -> xHC) now flows only through delegated capabilities. iommu-usb-storage / iommu-usb-hid / iommu-fault all green under per-grant enforcement; 104/104 overall (fail-open paths unchanged).
This commit is contained in:
parent
e94adcfc02
commit
4e7cbc9792
|
|
@ -28,6 +28,18 @@ pub const Device = struct {
|
|||
return .{ .block_size = result.block_size, .block_count = result.block_count };
|
||||
}
|
||||
|
||||
/// Hand the block server a DMA-region capability (`handle` — from a `shareable`
|
||||
/// dma_alloc) so it forwards it to the controller and the buffer's physical
|
||||
/// addresses become reachable by the device. Call once per buffer before naming it
|
||||
/// in `read`/`write`. Harmless success when no IOMMU is enforcing.
|
||||
pub fn attach(self: Device, handle: ipc.Handle) bool {
|
||||
var request = block_protocol.Request{ .operation = @intFromEnum(block_protocol.Operation.attach), .lba = 0, .count = 0, .physical = 0 };
|
||||
var reply: [block_protocol.reply_size]u8 = undefined;
|
||||
const result = ipc.callCap(self.endpoint, std.mem.asBytes(&request), &reply, handle) catch return false;
|
||||
if (result.len < block_protocol.reply_size) return false;
|
||||
return std.mem.bytesToValue(block_protocol.Reply, reply[0..block_protocol.reply_size]).status == 0;
|
||||
}
|
||||
|
||||
/// Read `count` blocks starting at `lba` into the DMA buffer at `physical`.
|
||||
pub fn read(self: Device, lba: u64, count: u32, physical: u64) bool {
|
||||
return self.transfer(.read, lba, count, physical);
|
||||
|
|
|
|||
|
|
@ -99,6 +99,19 @@ pub fn msiBind(device_id: u64, endpoint: usize) ?Msi {
|
|||
return .{ .address = rax, .data = @intCast(rdx) };
|
||||
}
|
||||
|
||||
/// Map a delegated DMA-region (or shared-memory) capability into a claimed device's
|
||||
/// IOMMU domain, so the device may DMA to that buffer. The caller must own `device_id`
|
||||
/// and hold `handle` (received over IPC or from its own `dma.alloc(.. | shareable)`).
|
||||
/// Idempotent. Returns true on success (and trivially when no IOMMU is present).
|
||||
pub fn dmaBind(device_id: u64, handle: usize) bool {
|
||||
return !failed(sc.systemCall2(.dma_bind, device_id, handle));
|
||||
}
|
||||
|
||||
/// Unmap a previously `dmaBind`'d buffer from the device's domain.
|
||||
pub fn dmaUnbind(device_id: u64, handle: usize) bool {
|
||||
return !failed(sc.systemCall2(.dma_unbind, device_id, handle));
|
||||
}
|
||||
|
||||
/// Drain and log any pending IOMMU translation faults, returning the count seen. A
|
||||
/// diagnostic: a driver that suspects its device attempted an out-of-domain DMA (or a
|
||||
/// test proving enforcement) forces the hardware's fault records to the log now. Returns
|
||||
|
|
|
|||
|
|
@ -99,6 +99,19 @@ pub const Device = struct {
|
|||
return std.mem.bytesToValue(usb_transfer_protocol.InterruptSubscribeReply, reply[0..@sizeOf(usb_transfer_protocol.InterruptSubscribeReply)]).status == 0;
|
||||
}
|
||||
|
||||
/// Hand the controller a DMA-region capability (`handle` — from a `shareable`
|
||||
/// dma_alloc, or forwarded from another process) so it binds that buffer into its
|
||||
/// IOMMU domain. Must be called for every buffer whose physical address this device
|
||||
/// will name in a `bulk` transfer, before the transfer. Harmless (and a no-op
|
||||
/// success) when no IOMMU is enforcing. Returns false on failure.
|
||||
pub fn attachDma(self: *Device, handle: ipc.Handle) bool {
|
||||
var request = usb_transfer_protocol.DmaAttachRequest{ .device_token = self.token };
|
||||
var reply: [@sizeOf(usb_transfer_protocol.DmaAttachReply)]u8 = undefined;
|
||||
const result = ipc.callCap(self.bus, std.mem.asBytes(&request), &reply, handle) catch return false;
|
||||
if (result.len < @sizeOf(usb_transfer_protocol.DmaAttachReply)) return false;
|
||||
return std.mem.bytesToValue(usb_transfer_protocol.DmaAttachReply, reply[0..@sizeOf(usb_transfer_protocol.DmaAttachReply)]).status == 0;
|
||||
}
|
||||
|
||||
/// One bulk transfer (IN or OUT per `endpoint_address`'s direction bit) to or
|
||||
/// from the caller's own DMA buffer at `physical`. Returns the bytes moved.
|
||||
pub fn bulk(self: *Device, endpoint_address: u8, physical: u64, length: u32) ?u32 {
|
||||
|
|
|
|||
|
|
@ -34,6 +34,14 @@ pub fn register(id: abi.ServiceId, h: Handle) bool {
|
|||
return !failed(sc.systemCall2(.ipc_register, @intFromEnum(id), h));
|
||||
}
|
||||
|
||||
/// Drop a capability handle (endpoint, shared-memory, or DMA-region) and free its table
|
||||
/// slot. A forwarding hop closes a cap it passed on; a binder closes a DMA-region cap
|
||||
/// once the binding holds its own reference — the 32-slot table is otherwise consumed by
|
||||
/// repeated cap-passing.
|
||||
pub fn close(h: Handle) bool {
|
||||
return !failed(sc.systemCall1(.handle_close, h));
|
||||
}
|
||||
|
||||
/// Find the endpoint published under `id`, installing a handle to it in this
|
||||
/// process.
|
||||
pub fn lookup(id: abi.ServiceId) ?Handle {
|
||||
|
|
|
|||
|
|
@ -12,12 +12,18 @@ const sc = @import("system-call");
|
|||
pub const coherent: usize = abi.dma_coherent;
|
||||
pub const write_combining: usize = abi.dma_write_combining;
|
||||
pub const below_4g: usize = abi.dma_below_4g;
|
||||
/// Ask for a capability handle (in `Region.handle`) so the buffer can be delegated to
|
||||
/// another driver and bound into a device's IOMMU domain (`driver.dmaBind`). A driver's
|
||||
/// private rings don't need it; a buffer whose physical address crosses IPC does.
|
||||
pub const shareable: usize = abi.dma_shareable;
|
||||
|
||||
/// A DMA allocation: the `virtual` address the CPU touches, and the `physical` address
|
||||
/// to program into the device's descriptor-ring / base registers.
|
||||
/// A DMA allocation: the `virtual` address the CPU touches, the `physical` address to
|
||||
/// program into the device's registers, and — when `shareable` was requested — a
|
||||
/// capability `handle` naming the region for delegation (null otherwise).
|
||||
pub const Region = struct {
|
||||
virtual: usize,
|
||||
physical: usize,
|
||||
handle: ?usize = null,
|
||||
};
|
||||
|
||||
inline fn failed(r: usize) bool {
|
||||
|
|
@ -25,21 +31,23 @@ inline fn failed(r: usize) bool {
|
|||
}
|
||||
|
||||
/// Allocate `len` bytes of DMA-capable memory with `flags` (e.g. `coherent`, or
|
||||
/// `coherent | below_4g`). Returns the virtual/physical pair, or null on failure. Two
|
||||
/// return values — the virtual address in rax, the physical address in rdx — so it
|
||||
/// needs a hand-written stub.
|
||||
/// `coherent | shareable`). Returns virtual/physical (and a handle when `shareable`), or
|
||||
/// null on failure. Three return values — virtual in rax, physical in rdx, handle in r8
|
||||
/// — so it needs a hand-written stub.
|
||||
pub fn alloc(len: usize, flags: usize) ?Region {
|
||||
var rax: usize = undefined;
|
||||
var rdx: usize = undefined; // out: physical address
|
||||
var r8: usize = undefined; // out: capability handle (abi.no_cap unless shareable)
|
||||
asm volatile ("syscall"
|
||||
: [rax] "={rax}" (rax),
|
||||
[rdx] "={rdx}" (rdx),
|
||||
[r8] "={r8}" (r8),
|
||||
: [n] "{rax}" (@intFromEnum(abi.SystemCall.dma_alloc)),
|
||||
[a0] "{rdi}" (len),
|
||||
[a1] "{rsi}" (flags),
|
||||
: .{ .rcx = true, .r11 = true, .memory = true });
|
||||
if (failed(rax)) return null;
|
||||
return .{ .virtual = rax, .physical = rdx };
|
||||
return .{ .virtual = rax, .physical = rdx, .handle = if (r8 == abi.no_cap) null else r8 };
|
||||
}
|
||||
|
||||
/// Release a region from a prior `alloc` (`virtual` and the same `len`).
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ pub const DmaRegion = dma.Region;
|
|||
pub const dma_coherent = dma.coherent;
|
||||
pub const dma_write_combining = dma.write_combining;
|
||||
pub const dma_below_4g = dma.below_4g;
|
||||
pub const dma_shareable = dma.shareable;
|
||||
pub const dmaAlloc = dma.alloc;
|
||||
pub const dmaFree = dma.free;
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@
|
|||
//! buffer**, named by its physical address — the same physical-address handoff
|
||||
//! usb-storage already uses toward the controller, one layer up. So a 512-byte
|
||||
//! sector never has to cross the 256-byte IPC boundary; only the small request /
|
||||
//! reply headers do. (Safe while the IOMMU is unenforced; see docs/driver-model.md.)
|
||||
//! reply headers do. Under an enforcing IOMMU the buffer's physical addresses are
|
||||
//! only reachable by the device once the filesystem has `attach`ed the buffer's
|
||||
//! capability (the block server forwards it to the controller); see docs/driver-model.md.
|
||||
|
||||
pub const Operation = enum(u32) {
|
||||
/// geometry() -> { block_size, block_count }
|
||||
|
|
@ -20,6 +22,11 @@ pub const Operation = enum(u32) {
|
|||
/// A filesystem calls this to make prior writes durable — e.g. before power-off,
|
||||
/// so a shutdown-time write isn't lost in the USB flash controller's cache.
|
||||
flush = 3,
|
||||
/// attach(): the caller's DMA-region capability rides the call's cap slot; the
|
||||
/// block server forwards it to the controller so the buffer's physical addresses
|
||||
/// (named in later read/write) are reachable by the device under an enforcing
|
||||
/// IOMMU. Call once per buffer before using it in a transfer.
|
||||
attach = 4,
|
||||
};
|
||||
|
||||
pub const Request = extern struct {
|
||||
|
|
|
|||
|
|
@ -45,6 +45,11 @@ pub const Operation = enum(u32) {
|
|||
control = 1,
|
||||
interrupt_subscribe = 2,
|
||||
bulk = 3,
|
||||
/// dma_attach: a class driver hands the controller a DMA-region capability (riding
|
||||
/// the call's cap slot) so the controller binds that buffer into its IOMMU domain
|
||||
/// and may then DMA to the physical addresses inside it. Needed once per buffer the
|
||||
/// class driver will name in a `bulk` transfer (its own, or one forwarded to it).
|
||||
dma_attach = 4,
|
||||
};
|
||||
|
||||
/// The endpoint facts a class driver needs, lifted from the endpoint descriptor
|
||||
|
|
@ -138,6 +143,19 @@ pub const BulkReply = extern struct {
|
|||
actual_length: u32,
|
||||
};
|
||||
|
||||
/// dma_attach: the region capability rides the call's cap slot; the body only carries
|
||||
/// the device token (scoping) so the controller knows which caller is attaching.
|
||||
pub const DmaAttachRequest = extern struct {
|
||||
operation: u32 = @intFromEnum(Operation.dma_attach),
|
||||
reserved: u32 = 0,
|
||||
device_token: u64,
|
||||
};
|
||||
|
||||
pub const DmaAttachReply = extern struct {
|
||||
status: i32,
|
||||
reserved: u32 = 0,
|
||||
};
|
||||
|
||||
/// An asynchronous interrupt report, pushed with `ipc.send` to a subscriber's
|
||||
/// endpoint. `Received.isMessage()` is set; there is no reply owed.
|
||||
pub const InterruptReport = extern struct {
|
||||
|
|
|
|||
|
|
@ -77,6 +77,9 @@ pub const SystemCall = enum(u64) {
|
|||
fs_mount = 48, // fs_mount(prefix_ptr, prefix_len, backend_handle, rewrite_ptr, rewrite_len) -> 0/-errno: mount a userspace filesystem's endpoint at an absolute prefix (possession of the handle is the capability)
|
||||
fs_unmount = 49, // fs_unmount(prefix_ptr, prefix_len) -> 0/-errno: remove a backend mount
|
||||
iommu_fault_drain = 50, // iommu_fault_drain() -> count: drain + log pending IOMMU translation faults (a diagnostic; the count of faults seen this call)
|
||||
dma_bind = 51, // dma_bind(device_id, region_handle) -> 0/-errno: map a DMA-region capability into the claimed device's IOMMU domain (idempotent). The caller must own the device and hold the handle
|
||||
dma_unbind = 52, // dma_unbind(device_id, region_handle) -> 0/-errno: unmap a previously bound region from the device's domain and invalidate
|
||||
handle_close = 53, // handle_close(handle) -> 0/-errno: drop one capability handle and free its table slot (endpoints, shared-memory, DMA regions)
|
||||
_,
|
||||
};
|
||||
|
||||
|
|
@ -114,6 +117,7 @@ pub const msi_address_base: u64 = 0xFEE0_0000;
|
|||
pub const dma_coherent: u64 = 1; // strong-uncacheable — the default, the only portable one
|
||||
pub const dma_write_combining: u64 = 2; // write-combining (framebuffers); needs PAT
|
||||
pub const dma_below_4g: u64 = 4; // physical address must fit 32 bits (legacy DMA engines)
|
||||
pub const dma_shareable: u64 = 8; // return a capability handle (r8) so the region can be delegated + dma_bound
|
||||
|
||||
/// Set in the badge returned by `ipc_reply_wait` when what arrived is an
|
||||
/// **asynchronous notification** (a device interrupt bound with `irq_bind`, or a
|
||||
|
|
|
|||
|
|
@ -90,9 +90,22 @@ fn initialise(endpoint: ipc.Handle) bool {
|
|||
bring_up_failed = true;
|
||||
return false;
|
||||
};
|
||||
command_wrapper = memory.dmaAlloc(4096, memory.dma_coherent) orelse return false;
|
||||
status_wrapper = memory.dmaAlloc(4096, memory.dma_coherent) orelse return false;
|
||||
command_data = memory.dmaAlloc(4096, memory.dma_coherent) orelse return false;
|
||||
// Shareable, so each buffer's capability can be handed to the controller: usb-storage
|
||||
// owns no device, so its buffers are not auto-bound anywhere — the controller reaches
|
||||
// them only once attached. (No-op binding when no IOMMU is enforcing.)
|
||||
command_wrapper = memory.dmaAlloc(4096, memory.dma_coherent | memory.dma_shareable) orelse return false;
|
||||
status_wrapper = memory.dmaAlloc(4096, memory.dma_coherent | memory.dma_shareable) orelse return false;
|
||||
command_data = memory.dmaAlloc(4096, memory.dma_coherent | memory.dma_shareable) orelse return false;
|
||||
for ([_]memory.DmaRegion{ command_wrapper, status_wrapper, command_data }) |region| {
|
||||
if (region.handle) |handle| {
|
||||
if (!device.attachDma(handle)) {
|
||||
_ = logging.write("/system/drivers/usb-storage: could not attach a DMA buffer to the controller\n");
|
||||
bring_up_failed = true;
|
||||
return false;
|
||||
}
|
||||
_ = ipc.close(handle); // the binding holds its own reference now
|
||||
}
|
||||
}
|
||||
|
||||
// Bring the LUN up: wait for it to be ready (clearing the initial unit-attention
|
||||
// with REQUEST SENSE), identify it, and read its capacity.
|
||||
|
|
@ -135,10 +148,17 @@ fn initialise(endpoint: ipc.Handle) bool {
|
|||
/// caller's DMA buffer (named by physical address).
|
||||
fn onMessage(message: []const u8, reply: []u8, sender: u32, capability: ?ipc.Handle) usize {
|
||||
_ = sender;
|
||||
_ = capability;
|
||||
if (message.len < block_protocol.request_size) return 0;
|
||||
const request = std.mem.bytesToValue(block_protocol.Request, message[0..block_protocol.request_size]);
|
||||
switch (request.operation) {
|
||||
@intFromEnum(block_protocol.Operation.attach) => {
|
||||
// The filesystem's DMA buffer: forward its capability to the controller so
|
||||
// the device can reach it, then release our copy (the binding holds a ref).
|
||||
const handle = capability orelse return writeReply(reply, .{ .status = -1, .block_size = 0, .block_count = 0 });
|
||||
const ok = device.attachDma(handle);
|
||||
_ = ipc.close(handle);
|
||||
return writeReply(reply, .{ .status = if (ok) 0 else -1, .block_size = 0, .block_count = 0 });
|
||||
},
|
||||
@intFromEnum(block_protocol.Operation.geometry) => {
|
||||
return writeReply(reply, .{ .status = 0, .block_size = block_size, .block_count = block_count });
|
||||
},
|
||||
|
|
|
|||
|
|
@ -484,10 +484,22 @@ fn onMessage(message: []const u8, reply: []u8, sender: u32, capability: ?ipc.Han
|
|||
@intFromEnum(usb_transfer_protocol.Operation.control) => handleControl(message, reply),
|
||||
@intFromEnum(usb_transfer_protocol.Operation.interrupt_subscribe) => handleSubscribe(message, reply),
|
||||
@intFromEnum(usb_transfer_protocol.Operation.bulk) => handleBulk(message, reply),
|
||||
@intFromEnum(usb_transfer_protocol.Operation.dma_attach) => handleDmaAttach(message, reply, capability),
|
||||
else => 0,
|
||||
};
|
||||
}
|
||||
|
||||
/// dma_attach: bind the class driver's DMA-region capability into the controller's IOMMU
|
||||
/// domain, so the controller may DMA to the physical addresses inside that buffer. The
|
||||
/// binding holds its own kernel reference, so the forwarded capability is closed here.
|
||||
fn handleDmaAttach(message: []const u8, reply: []u8, capability: ?ipc.Handle) usize {
|
||||
if (message.len < @sizeOf(usb_transfer_protocol.DmaAttachRequest)) return writeReply(reply, usb_transfer_protocol.DmaAttachReply{ .status = -1 });
|
||||
const handle = capability orelse return writeReply(reply, usb_transfer_protocol.DmaAttachReply{ .status = -1 });
|
||||
const ok = device.dmaBind(controller_id, handle);
|
||||
_ = ipc.close(handle);
|
||||
return writeReply(reply, usb_transfer_protocol.DmaAttachReply{ .status = if (ok) 0 else -1 });
|
||||
}
|
||||
|
||||
fn writeReply(reply: []u8, value: anytype) usize {
|
||||
const bytes = std.mem.asBytes(&value);
|
||||
@memcpy(reply[0..bytes.len], bytes);
|
||||
|
|
|
|||
|
|
@ -323,6 +323,15 @@ fn initialise(endpoint: ipc.Handle) bool {
|
|||
std.log.info("could not resolve the scanout surface physical address", .{});
|
||||
return false;
|
||||
};
|
||||
// Bind the shared surface into this device's IOMMU domain so the GPU may DMA the
|
||||
// framebuffer (attach_backing points it here). The ring/command buffers are
|
||||
// dma_alloc'd and auto-bound; a shared-memory surface needs an explicit bind. We keep
|
||||
// the handle (it is also passed to the display service), so do not close it. No-op
|
||||
// without an IOMMU.
|
||||
if (!device.dmaBind(device_id, surface.handle)) {
|
||||
std.log.info("could not bind the scanout surface for DMA", .{});
|
||||
return false;
|
||||
}
|
||||
{
|
||||
const request = requestAt(vg.ResourceAttachBacking);
|
||||
request.* = .{
|
||||
|
|
|
|||
|
|
@ -128,29 +128,18 @@ pub fn init() void {
|
|||
const Confined = struct { active: bool = false, owner: u32 = 0, bdf: u16 = 0, domain: u16 = invalid_domain };
|
||||
var confined: [maximum_domains]Confined = .{Confined{}} ** maximum_domains;
|
||||
|
||||
/// The interim DMA pool: the set of `dma_alloc`'d regions. Every such region is mapped
|
||||
/// into every claimed device's domain, so a device reaches DMA buffers (including one
|
||||
/// another's — the honest limit of this stage) but NOT the kernel, page tables, process
|
||||
/// heaps, or arbitrary RAM. The capability layer (L3/L4) narrows this to per-grant.
|
||||
const PoolRegion = struct { active: bool = false, physical: u64 = 0, len: u64 = 0 };
|
||||
var pool: [maximum_pool]PoolRegion = .{PoolRegion{}} ** maximum_pool;
|
||||
const maximum_pool = 256;
|
||||
|
||||
/// Place a just-claimed PCI function under IOMMU translation on behalf of `owner`: give
|
||||
/// it a private empty domain, seed it with the current DMA pool and the device's own
|
||||
/// firmware reserved region, and attach. false only if a domain can't be allocated —
|
||||
/// the caller rolls the claim back (a claim that can't be confined must not stand). No-op
|
||||
/// success when no IOMMU exists (fail-open).
|
||||
/// it a private empty domain, seed it with the device's own firmware reserved region,
|
||||
/// and attach. Its DMA buffers arrive afterward as explicit grants — the owner's own
|
||||
/// `dma_alloc`'d regions are bound by the claim path (`mapForDevice`), and cross-process
|
||||
/// buffers by `dma_bind`. false only if a domain can't be allocated — the caller rolls
|
||||
/// the claim back (a claim that can't be confined must not stand). No-op success when no
|
||||
/// IOMMU exists (fail-open).
|
||||
pub fn confineDevice(device_id: u64, bdf: u16, owner: u32) bool {
|
||||
if (kind == .none) return true;
|
||||
if (device_id >= confined.len) return true; // unusual id; leave it to fail-open
|
||||
const domain = domainCreate(owner, bdf) orelse return false;
|
||||
|
||||
// Seed with every pooled DMA region so the device's own rings/buffers and the
|
||||
// cross-process buffers handed to it (fat's bounce buffer via usb-storage) resolve.
|
||||
for (&pool) |*r| {
|
||||
if (r.active) _ = map(domain, r.physical, r.len);
|
||||
}
|
||||
// Firmware reserved region for this device, if any (real hardware; QEMU has none).
|
||||
const info = platform.platformInformation();
|
||||
var i: usize = 0;
|
||||
|
|
@ -164,34 +153,45 @@ pub fn confineDevice(device_id: u64, bdf: u16, owner: u32) bool {
|
|||
return true;
|
||||
}
|
||||
|
||||
/// Record a newly `dma_alloc`'d region and map it into every claimed device's domain
|
||||
/// (the interim pool rule). Called from the dma_alloc syscall.
|
||||
pub fn poolAdd(physical: u64, len: u64) void {
|
||||
/// The confined record for `device_id`, or null if the device is not confined.
|
||||
fn confinedOf(device_id: u64) ?*Confined {
|
||||
if (device_id >= confined.len) return null;
|
||||
const c = &confined[@intCast(device_id)];
|
||||
return if (c.active) c else null;
|
||||
}
|
||||
|
||||
/// Map a DMA region into a specific claimed device's domain (the device owner binding a
|
||||
/// granted buffer). false if the device is not confined. No-op success without an IOMMU.
|
||||
pub fn mapForDevice(device_id: u64, physical: u64, len: u64) bool {
|
||||
if (kind == .none) return true;
|
||||
const c = confinedOf(device_id) orelse return false;
|
||||
return map(c.domain, physical, len);
|
||||
}
|
||||
|
||||
/// Unmap a DMA region from a specific claimed device's domain. No-op if not confined.
|
||||
pub fn unmapForDevice(device_id: u64, physical: u64, len: u64) void {
|
||||
if (kind == .none) return;
|
||||
const c = confinedOf(device_id) orelse return;
|
||||
unmap(c.domain, physical, len);
|
||||
}
|
||||
|
||||
/// Map a region into every claimed device owned by `owner` — the auto-bind of a task's
|
||||
/// own freshly-`dma_alloc`'d buffer into the devices it drives.
|
||||
pub fn mapRegionForOwner(owner: u32, physical: u64, len: u64) void {
|
||||
if (kind == .none) return;
|
||||
for (&pool) |*r| {
|
||||
if (!r.active) {
|
||||
r.* = .{ .active = true, .physical = physical, .len = len };
|
||||
break;
|
||||
}
|
||||
} else return; // pool full; region stays unmapped and its device DMA will fault
|
||||
for (&confined) |*c| {
|
||||
if (c.active) _ = map(c.domain, physical, len);
|
||||
if (c.active and c.owner == owner) _ = map(c.domain, physical, len);
|
||||
}
|
||||
}
|
||||
|
||||
/// Unmap a freed DMA region from every claimed device's domain and forget it. MUST run
|
||||
/// before the frames return to pmm — a device translating to a reallocated frame is the
|
||||
/// use-after-free this prevents. Called from the dma_free syscall.
|
||||
pub fn poolRemove(physical: u64, len: u64) void {
|
||||
/// Unmap a region from EVERY claimed device's domain — the freed-region sweep. MUST run
|
||||
/// before the frames return to pmm: a device translating to a reallocated frame is the
|
||||
/// use-after-free this prevents. Cross-device because a granted buffer may be bound in a
|
||||
/// domain other than its owner's.
|
||||
pub fn unmapRegionEverywhere(physical: u64, len: u64) void {
|
||||
if (kind == .none) return;
|
||||
for (&pool) |*r| {
|
||||
if (r.active and r.physical == physical and r.len == len) {
|
||||
for (&confined) |*c| {
|
||||
if (c.active) unmap(c.domain, physical, len);
|
||||
}
|
||||
r.* = .{};
|
||||
return;
|
||||
}
|
||||
for (&confined) |*c| {
|
||||
if (c.active) unmap(c.domain, physical, len);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -154,6 +154,66 @@ pub fn dropRef(endpoint: *Endpoint) void {
|
|||
/// Defined here (not in scheduler) because the meaning is the IPC/capability layer's.
|
||||
pub const handle_kind_endpoint: u8 = 0;
|
||||
pub const handle_kind_shared_memory: u8 = 1;
|
||||
pub const handle_kind_dma_region: u8 = 2;
|
||||
|
||||
/// A DMA buffer's delegation token: names the `pages` contiguous frames at `phys` that
|
||||
/// `dma_alloc` handed its `owner`, and can be passed across processes as a capability so
|
||||
/// the driver that owns a device can bind it into that device's IOMMU domain
|
||||
/// (`dma_bind`). Unlike `SharedMemoryObject` this does NOT own the frames — the
|
||||
/// allocating address space still does, and frees them on `dma_free` or teardown — so
|
||||
/// this is a pure token: `dead` is set when the allocator frees the region, after which
|
||||
/// a stale handle can no longer bind it. `refcount` counts the allocator's registry
|
||||
/// entry plus every outstanding handle; the object is freed when the last drops.
|
||||
pub const DmaRegionObject = struct {
|
||||
refcount: u32 = 1,
|
||||
phys: u64,
|
||||
pages: usize,
|
||||
owner: u32,
|
||||
dead: bool = false,
|
||||
};
|
||||
|
||||
/// Create a DMA-region token for `pages` frames at `phys` owned by task `owner`. The
|
||||
/// frames are already allocated and mapped by the caller; this only wraps them for
|
||||
/// delegation. null if the heap is out of room.
|
||||
pub fn createDmaRegion(phys: u64, pages: usize, owner: u32) ?*DmaRegionObject {
|
||||
const region = heap.allocator().create(DmaRegionObject) catch return null;
|
||||
region.* = .{ .phys = phys, .pages = pages, .owner = owner };
|
||||
return region;
|
||||
}
|
||||
|
||||
/// Drop a DMA-region reference; free the token when the last (registry + handles) goes.
|
||||
/// Never frees frames — the allocator owns those.
|
||||
pub fn dropDmaRegionReference(region: *DmaRegionObject) void {
|
||||
if (region.refcount > 1) {
|
||||
region.refcount -= 1;
|
||||
} else {
|
||||
heap.allocator().destroy(region);
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a handle to its DMA-region token, or null if out of range, unused, or a
|
||||
/// different kind.
|
||||
pub fn resolveDmaRegion(t: *Task, h: u64) ?*DmaRegionObject {
|
||||
if (h >= t.handles.len) return null;
|
||||
const entry = t.handles[@intCast(h)] orelse return null;
|
||||
if (entry.kind != handle_kind_dma_region) return null;
|
||||
return @ptrCast(@alignCast(entry.ptr));
|
||||
}
|
||||
|
||||
/// Install a DMA-region handle in task `t`'s table (the slot owns a reference).
|
||||
pub fn installDmaRegionHandle(t: *Task, region: *DmaRegionObject) i64 {
|
||||
return installEntry(t, .{ .kind = handle_kind_dma_region, .ptr = @ptrCast(region) });
|
||||
}
|
||||
|
||||
/// Drop the handle at slot `h` of task `t` (handle_close): release its reference and
|
||||
/// free the slot. Returns 0 or -EBADF.
|
||||
pub fn closeHandle(t: *Task, h: u64) i64 {
|
||||
if (h >= t.handles.len) return -EBADF;
|
||||
const entry = t.handles[@intCast(h)] orelse return -EBADF;
|
||||
dropEntry(entry);
|
||||
t.handles[@intCast(h)] = null;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/// A page-aligned block of **shared cacheable RAM** (docs/display-v2.md), referenced by
|
||||
/// capability handles across processes and freed when the last one drops. `phys` is its
|
||||
|
|
@ -307,6 +367,10 @@ fn shareCapability(from: *Task, to: *Task, cap: u64) i64 {
|
|||
const s: *SharedMemoryObject = @ptrCast(@alignCast(entry.ptr));
|
||||
s.refcount += 1;
|
||||
},
|
||||
handle_kind_dma_region => {
|
||||
const r: *DmaRegionObject = @ptrCast(@alignCast(entry.ptr));
|
||||
r.refcount += 1;
|
||||
},
|
||||
else => return -EBADF,
|
||||
}
|
||||
const handle = installEntry(to, entry);
|
||||
|
|
@ -574,6 +638,7 @@ fn dropEntry(entry: scheduler.HandleObject) void {
|
|||
switch (entry.kind) {
|
||||
handle_kind_endpoint => dropRef(@ptrCast(@alignCast(entry.ptr))),
|
||||
handle_kind_shared_memory => dropSharedMemoryReference(@ptrCast(@alignCast(entry.ptr))),
|
||||
handle_kind_dma_region => dropDmaRegionReference(@ptrCast(@alignCast(entry.ptr))),
|
||||
else => {},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -252,6 +252,9 @@ fn system_call(state: *architecture.CpuState) void {
|
|||
.fs_mount => systemFsMount(state),
|
||||
.fs_unmount => systemFsUnmount(state),
|
||||
.iommu_fault_drain => systemIommuFaultDrain(state),
|
||||
.dma_bind => systemDmaBind(state),
|
||||
.dma_unbind => systemDmaUnbind(state),
|
||||
.handle_close => systemHandleClose(state),
|
||||
.wall_clock => systemWallClock(state),
|
||||
.shared_memory_create => systemSharedMemoryCreate(state),
|
||||
.shared_memory_map => systemSharedMemoryMap(state),
|
||||
|
|
@ -398,10 +401,14 @@ fn systemDeviceClaim(state: *architecture.CpuState) void {
|
|||
// since the whole point is that claiming a DMA device is no longer equivalent
|
||||
// to ring 0. No-op when no IOMMU exists (fail-open).
|
||||
if (devices_broker.pciAddressOf(device_id)) |bdf| {
|
||||
if (!iommu.confineDevice(device_id, bdf, scheduler.current().id)) {
|
||||
_ = devices_broker.unclaim(device_id, scheduler.current().id);
|
||||
const owner = scheduler.current().id;
|
||||
if (!iommu.confineDevice(device_id, bdf, owner)) {
|
||||
_ = devices_broker.unclaim(device_id, owner);
|
||||
return fail(state);
|
||||
}
|
||||
// Bind the buffers this task allocated before claiming the device (a driver
|
||||
// that dma_alloc'd its rings, then claimed the controller).
|
||||
dmaBindOwnerRegionsInto(owner, device_id);
|
||||
}
|
||||
// A display service just took the framebuffer — quiesce the bootstrap console
|
||||
// so the kernel and the service don't scribble over each other's pixels. The
|
||||
|
|
@ -510,6 +517,72 @@ fn systemIoWrite(state: *architecture.CpuState) void {
|
|||
/// their physical address is never disclosed. `dma_below_4g` caps the physical address
|
||||
/// for legacy engines; `dma_write_combining` is accepted but falls back to coherent
|
||||
/// until PAT is programmed. See docs/driver-model.md (M14).
|
||||
// --- DMA-region registry ---------------------------------------------------------------
|
||||
// Every dma_alloc'd region is tracked here so it can be (a) auto-bound into the devices
|
||||
// its owner claims, (b) delegated across processes as a capability and bound into a
|
||||
// device's IOMMU domain by dma_bind, and (c) unmapped from every domain before its frames
|
||||
// return to the allocator. Only *shareable* regions carry a heap `object` (the delegation
|
||||
// token); a driver's private rings are tracked without one. All access under the big lock.
|
||||
const DmaRegistryEntry = struct {
|
||||
active: bool = false,
|
||||
object: ?*ipc.DmaRegionObject = null,
|
||||
physical: u64 = 0,
|
||||
len: u64 = 0,
|
||||
owner: u32 = 0,
|
||||
};
|
||||
const maximum_dma_regions = 256;
|
||||
var dma_registry: [maximum_dma_regions]DmaRegistryEntry = .{DmaRegistryEntry{}} ** maximum_dma_regions;
|
||||
|
||||
fn dmaRegistryAdd(object: ?*ipc.DmaRegionObject, physical: u64, len: u64, owner: u32) void {
|
||||
for (&dma_registry) |*e| {
|
||||
if (!e.active) {
|
||||
e.* = .{ .active = true, .object = object, .physical = physical, .len = len, .owner = owner };
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Retire the region based at `physical` owned by `owner`: unmap it from every device
|
||||
/// domain (before the frames are freed), mark its token dead so a stale downstream handle
|
||||
/// can no longer bind it, drop the registry's reference, and clear the slot.
|
||||
fn dmaRegistryRemove(owner: u32, physical: u64) void {
|
||||
for (&dma_registry) |*e| {
|
||||
if (e.active and e.owner == owner and e.physical == physical) {
|
||||
iommu.unmapRegionEverywhere(e.physical, e.len);
|
||||
if (e.object) |object| {
|
||||
object.dead = true;
|
||||
ipc.dropDmaRegionReference(object);
|
||||
}
|
||||
e.* = .{};
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Retire every region owned by `owner` (task death) — same discipline as a per-region
|
||||
/// free, run before the address space is torn down and its DMA frames reclaimed.
|
||||
fn dmaRegistryReleaseOwner(owner: u32) void {
|
||||
for (&dma_registry) |*e| {
|
||||
if (e.active and e.owner == owner) {
|
||||
iommu.unmapRegionEverywhere(e.physical, e.len);
|
||||
if (e.object) |object| {
|
||||
object.dead = true;
|
||||
ipc.dropDmaRegionReference(object);
|
||||
}
|
||||
e.* = .{};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Bind every region `owner` allocated into the domain of the device it just claimed
|
||||
/// (its rings allocated before the claim). Regions allocated *after* the claim are bound
|
||||
/// by dma_alloc's own auto-bind.
|
||||
fn dmaBindOwnerRegionsInto(owner: u32, device_id: u64) void {
|
||||
for (&dma_registry) |*e| {
|
||||
if (e.active and e.owner == owner) _ = iommu.mapForDevice(device_id, e.physical, e.len);
|
||||
}
|
||||
}
|
||||
|
||||
/// iommu_fault_drain() -> count: drain and log any pending IOMMU translation faults,
|
||||
/// returning how many were seen. A diagnostic hook — a driver (or a test) that suspects
|
||||
/// its device faulted can force the fault records to be logged now rather than waiting
|
||||
|
|
@ -521,6 +594,60 @@ fn systemIommuFaultDrain(state: *architecture.CpuState) void {
|
|||
architecture.setSystemCallResult(state, count);
|
||||
}
|
||||
|
||||
/// Resolve a capability handle the caller holds to the physical range it names — either
|
||||
/// a DMA-region token or a shared-memory object (both are bindable buffers). null if the
|
||||
/// handle is neither, or names a region already freed by its allocator.
|
||||
fn bindableRange(t: *scheduler.Task, handle: u64) ?struct { physical: u64, len: u64 } {
|
||||
if (ipc.resolveDmaRegion(t, handle)) |region| {
|
||||
if (region.dead) return null;
|
||||
return .{ .physical = region.phys, .len = region.pages * page_size };
|
||||
}
|
||||
if (ipc.resolveSharedMemory(t, handle)) |shared| {
|
||||
return .{ .physical = shared.phys, .len = shared.pages * page_size };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// dma_bind(device_id, handle) -> 0/-errno: map the buffer named by `handle` into the
|
||||
/// claimed device's IOMMU domain. The caller must own the device (the mmio_map gate) and
|
||||
/// hold the handle. Idempotent: re-binding is a harmless success, so a driver may re-bind
|
||||
/// after a restart without tracking what it already bound. Success (no-op) without an IOMMU.
|
||||
fn systemDmaBind(state: *architecture.CpuState) void {
|
||||
const device_id = architecture.systemCallArg(state, 0);
|
||||
const handle = architecture.systemCallArg(state, 1);
|
||||
const t = scheduler.current();
|
||||
const flags = sync.enter();
|
||||
defer sync.leave(flags);
|
||||
if (devices_broker.ownerOf(device_id) != t.id) return fail(state);
|
||||
const range = bindableRange(t, handle) orelse return fail(state);
|
||||
if (!iommu.mapForDevice(device_id, range.physical, range.len)) return fail(state);
|
||||
architecture.setSystemCallResult(state, 0);
|
||||
}
|
||||
|
||||
/// dma_unbind(device_id, handle) -> 0/-errno: unmap a previously bound buffer from the
|
||||
/// device's domain and invalidate.
|
||||
fn systemDmaUnbind(state: *architecture.CpuState) void {
|
||||
const device_id = architecture.systemCallArg(state, 0);
|
||||
const handle = architecture.systemCallArg(state, 1);
|
||||
const t = scheduler.current();
|
||||
const flags = sync.enter();
|
||||
defer sync.leave(flags);
|
||||
if (devices_broker.ownerOf(device_id) != t.id) return fail(state);
|
||||
const range = bindableRange(t, handle) orelse return fail(state);
|
||||
iommu.unmapForDevice(device_id, range.physical, range.len);
|
||||
architecture.setSystemCallResult(state, 0);
|
||||
}
|
||||
|
||||
/// handle_close(handle) -> 0/-errno: drop one capability handle and free its slot.
|
||||
fn systemHandleClose(state: *architecture.CpuState) void {
|
||||
const handle = architecture.systemCallArg(state, 0);
|
||||
const t = scheduler.current();
|
||||
const flags = sync.enter();
|
||||
defer sync.leave(flags);
|
||||
if (ipc.closeHandle(t, handle) < 0) return fail(state);
|
||||
architecture.setSystemCallResult(state, 0);
|
||||
}
|
||||
|
||||
fn systemDmaAlloc(state: *architecture.CpuState) void {
|
||||
const len = architecture.systemCallArg(state, 0);
|
||||
const flags = architecture.systemCallArg(state, 1);
|
||||
|
|
@ -567,15 +694,33 @@ fn systemDmaAlloc(state: *architecture.CpuState) void {
|
|||
architecture.mapUserDmaInto(t.address_space, base_v + i * page_size, phys + i * page_size, page_size);
|
||||
sync.leave(lock_flags);
|
||||
}
|
||||
// Publish the region to the DMA pool: it becomes reachable to every claimed device
|
||||
// (the interim rule until DMA-region capabilities land). No-op without an IOMMU.
|
||||
// Register the region and bind it into every device this task already drives (its
|
||||
// own buffers reach its own devices). When `dma_shareable` is set, also wrap it in a
|
||||
// capability token and return a handle so it can be delegated to another driver and
|
||||
// dma_bound there. All under the lock (the registry + IOMMU tables are shared).
|
||||
var handle: u64 = abi.no_cap;
|
||||
{
|
||||
const lock_flags = sync.enter();
|
||||
iommu.poolAdd(phys, pages * page_size);
|
||||
sync.leave(lock_flags);
|
||||
defer sync.leave(lock_flags);
|
||||
var object: ?*ipc.DmaRegionObject = null;
|
||||
if (flags & abi.dma_shareable != 0) {
|
||||
if (ipc.createDmaRegion(phys, pages, t.id)) |region| {
|
||||
const h = ipc.installDmaRegionHandle(t, region);
|
||||
if (h >= 0) {
|
||||
region.refcount += 1; // the handle's reference (registry holds the first)
|
||||
handle = @intCast(h);
|
||||
object = region;
|
||||
} else {
|
||||
ipc.dropDmaRegionReference(region); // no table slot; drop it
|
||||
}
|
||||
}
|
||||
}
|
||||
dmaRegistryAdd(object, phys, pages * page_size, t.id);
|
||||
iommu.mapRegionForOwner(t.id, phys, pages * page_size);
|
||||
}
|
||||
architecture.setSystemCallResult(state, base_v); // virtual address for the CPU
|
||||
architecture.setSystemCallResult2(state, phys); // physical address for the device
|
||||
architecture.setSystemCallResult3(state, handle); // capability handle (no_cap unless shareable)
|
||||
}
|
||||
|
||||
/// dma_free(virtual_address, len) -> 0: release a prior `dma_alloc`. Bounded to the DMA arena so
|
||||
|
|
@ -591,14 +736,13 @@ fn systemDmaFree(state: *architecture.CpuState) void {
|
|||
const pages: usize = @intCast((len + page_size - 1) / page_size);
|
||||
if (base_v < dma_arena_base or base_v + pages * page_size > dma_arena_end) return fail(state);
|
||||
|
||||
// Pull the region out of every device's domain and invalidate BEFORE any frame
|
||||
// returns to the allocator — a device still translating to a reallocated frame is a
|
||||
// use-after-free. dma_alloc's frames are contiguous, so the base translation names
|
||||
// the whole region.
|
||||
// Retire the region — unmap it from every device domain and mark its token dead —
|
||||
// BEFORE any frame returns to the allocator, so no device can still translate to a
|
||||
// reallocated frame. dma_alloc's frames are contiguous, so the base names the region.
|
||||
{
|
||||
const lock_flags = sync.enter();
|
||||
if (architecture.translate(t.address_space, base_v)) |base_phys|
|
||||
iommu.poolRemove(base_phys, pages * page_size);
|
||||
dmaRegistryRemove(t.id, base_phys);
|
||||
sync.leave(lock_flags);
|
||||
}
|
||||
|
||||
|
|
@ -1036,8 +1180,11 @@ fn releaseTaskResourcesLocked(t: *scheduler.Task) void {
|
|||
// Detach the task's devices from their IOMMU domains BEFORE the broker clears the
|
||||
// claims (the detach reads ownership) and before the address space is torn down and
|
||||
// its DMA frames return to the allocator — a device must stop translating to a frame
|
||||
// before that frame can be handed to someone else.
|
||||
// before that frame can be handed to someone else. Then retire the buffers this task
|
||||
// allocated, unmapping them from any *other* driver's domain they were granted into,
|
||||
// before those frames are freed too.
|
||||
iommu.releaseAllOwnedBy(t.id);
|
||||
dmaRegistryReleaseOwner(t.id);
|
||||
devices_broker.releaseAllOwnedBy(t.id);
|
||||
// If that dropped the framebuffer claim (this task was the display service), let the
|
||||
// bootstrap console draw again — the screen is nobody's now, so panics/status land.
|
||||
|
|
|
|||
|
|
@ -149,7 +149,10 @@ pub const maximum_task_name = abi.maximum_process_name;
|
|||
|
||||
/// Size of each task's IPC handle table. Kept here (not in ipc_sync.zig) because
|
||||
/// it dimensions a field of `Task`; ipc_sync.zig re-exports it.
|
||||
pub const ipc_maximum_handles = 16;
|
||||
// Raised from 16 with DMA-region capabilities: a driver now holds its per-device
|
||||
// channel endpoints plus received DMA-region handles (a storage driver forwards several
|
||||
// buffer caps), and repeated cap-passing consumes slots until handle_close.
|
||||
pub const ipc_maximum_handles = 32;
|
||||
|
||||
/// One handle-table entry: a capability object plus a `kind` tag saying what `ptr` points
|
||||
/// at (an ipc endpoint or a shared-memory object), so a task's exit path and the
|
||||
|
|
|
|||
|
|
@ -118,7 +118,17 @@ fn tryBringUp() void {
|
|||
_ = logging.write("/system/services/fat: block geometry unavailable\n");
|
||||
return;
|
||||
};
|
||||
const bounce = memory.dmaAlloc(engine.max_transfer_sectors * 512, memory.dma_coherent) orelse return;
|
||||
// Shareable so the buffer's capability can be attached down the chain (block server
|
||||
// -> controller), making its physical addresses reachable by the device under an
|
||||
// enforcing IOMMU. No-op binding otherwise.
|
||||
const bounce = memory.dmaAlloc(engine.max_transfer_sectors * 512, memory.dma_coherent | memory.dma_shareable) orelse return;
|
||||
if (bounce.handle) |handle| {
|
||||
if (!device.attach(handle)) {
|
||||
_ = logging.write("/system/services/fat: could not attach the DMA bounce buffer\n");
|
||||
return;
|
||||
}
|
||||
_ = ipc.close(handle); // the binding holds its own reference now
|
||||
}
|
||||
ipc_block = .{ .device = device, .bounce = bounce };
|
||||
|
||||
const block_device = engine.BlockDevice{
|
||||
|
|
|
|||
Loading…
Reference in New Issue