kernel: the IOMMU backends move behind the architecture boundary

VT-d and AMD-Vi are x86 hardware, but lived in the architecture-neutral
kernel tree and leaked further: the core's public Kind enum named both
vendors, and the ACPI parser read the VT-d version/capability registers
(raw volatile MMIO inside table discovery). Now the vendor backends
live in architecture/x86_64/ behind architecture.iommu — the core hands
over the discovery facts plus an injected environment (frame allocation
+ the log sink, the same pattern enablePaging uses) and receives the
hardware vtable back, so the backends never import kernel internals and
an ARM port supplies its SMMU with no core change. Discovery keeps
table facts only; the live-unit register check moved into VT-d detect
(version reading zero now stays fail-open). The unused kindOf() is
gone. Log shapes the harness pins (iommu online, DANOS-IOMMU-FAULT)
are unchanged; all five IOMMU QEMU cases pass.
This commit is contained in:
Daniel Samson
2026-07-30 07:16:08 +01:00
parent e53d6ebafb
commit fa8203cdba
7 changed files with 197 additions and 153 deletions
+6 -11
View File
@@ -106,12 +106,6 @@ pub const PlatformInformation = struct {
/// INCLUDE_PCI_ALL (the catch-all unit; falls back to the first), or the AMD-Vi /// INCLUDE_PCI_ALL (the catch-all unit; falls back to the first), or the AMD-Vi
/// IOMMU's control-register base from the first IVHD. /// IOMMU's control-register base from the first IVHD.
iommu_base: u64 = 0, iommu_base: u64 = 0,
/// The unit's Version register (offset 0x00) — its low byte is major.minor;
/// reading it back nonzero confirms a real, mappable VT-d unit.
iommu_version: u32 = 0,
/// The unit's Capability register (offset 0x08): supported address widths, number
/// of domains, etc. Consumed by the IOMMU core when it enables translation.
iommu_capabilities: u64 = 0,
/// Whether the selected unit carries INCLUDE_PCI_ALL. False means every unit is /// Whether the selected unit carries INCLUDE_PCI_ALL. False means every unit is
/// device-scoped (unusual) — the core still enables on the selected unit but /// device-scoped (unusual) — the core still enables on the selected unit but
/// devices outside its scope remain untranslated. /// devices outside its scope remain untranslated.
@@ -502,7 +496,7 @@ fn handleTable(device_tree: *DeviceTree, hal: Hal, sdt_physical: u64) !void {
} else if (std.mem.eql(u8, &sig, &SPCR)) { } else if (std.mem.eql(u8, &sig, &SPCR)) {
parseSpcr(header); parseSpcr(header);
} else if (std.mem.eql(u8, &sig, &DMAR)) { } else if (std.mem.eql(u8, &sig, &DMAR)) {
parseDmar(hal, header); parseDmar(header);
} else if (std.mem.eql(u8, &sig, &IVRS)) { } else if (std.mem.eql(u8, &sig, &IVRS)) {
parseIvrs(header); parseIvrs(header);
} else if (std.mem.eql(u8, &sig, &SSDT)) { } else if (std.mem.eql(u8, &sig, &SSDT)) {
@@ -822,7 +816,7 @@ const scope_path_offset = 6;
/// and records single-path endpoint RMRRs for the IOMMU core to pre-map before it /// and records single-path endpoint RMRRs for the IOMMU core to pre-map before it
/// enables translation. Multi-hop RMRR scopes are skipped loudly: better a named gap /// enables translation. Multi-hop RMRR scopes are skipped loudly: better a named gap
/// than a silent one. /// than a silent one.
fn parseDmar(hal: Hal, header: *const SystemDescriptorTableHeader) void { fn parseDmar(header: *const SystemDescriptorTableHeader) void {
const base: [*]align(1) const u8 = @ptrCast(header); const base: [*]align(1) const u8 = @ptrCast(header);
const total: usize = header.length; const total: usize = header.length;
@@ -840,13 +834,14 @@ fn parseDmar(hal: Hal, header: *const SystemDescriptorTableHeader) void {
const replace = !platform_information.iommu_present or const replace = !platform_information.iommu_present or
(include_all and !platform_information.iommu_include_all); (include_all and !platform_information.iommu_include_all);
if (replace) { if (replace) {
// Table facts only: the unit's registers are the IOMMU
// backend's business (it maps and validates them at
// detect) — discovery records where they live, never
// reads them.
if (platform_information.iommu_present) platform_information.iommu_extra_units += 1; if (platform_information.iommu_present) platform_information.iommu_extra_units += 1;
const regs = hal.mapMmio(register_base, abi.page_size, true);
platform_information.iommu_present = true; platform_information.iommu_present = true;
platform_information.iommu_base = register_base; platform_information.iommu_base = register_base;
platform_information.iommu_include_all = include_all; platform_information.iommu_include_all = include_all;
platform_information.iommu_version = @as(*const volatile u32, @ptrFromInt(regs + 0x00)).*;
platform_information.iommu_capabilities = @as(*const volatile u64, @ptrFromInt(regs + 0x08)).*;
} else { } else {
platform_information.iommu_extra_units += 1; platform_information.iommu_extra_units += 1;
} }
@@ -17,6 +17,11 @@ const io = @import("io.zig");
const smp = @import("smp.zig"); const smp = @import("smp.zig");
const pcpu = @import("per-cpu.zig"); const pcpu = @import("per-cpu.zig");
/// The x86-64 IOMMU backends (Intel VT-d, AMD-Vi) behind their dispatch
/// surface — the architecture-neutral IOMMU core (system/kernel/iommu.zig)
/// reaches the hardware only through this.
pub const iommu = @import("iommu.zig");
/// The saved register/trap frame passed to a fault handler. /// The saved register/trap frame passed to a fault handler.
pub const CpuState = idt.CpuState; pub const CpuState = idt.CpuState;
@@ -1,5 +1,6 @@
//! system/kernel/iommu-amd.zig — AMD-Vi (AMD I/O Virtualization) backend for the IOMMU //! AMD-Vi (AMD I/O Virtualization) backend for the IOMMU core, behind the
//! core. The AMD analogue of iommu-intel.zig: it supplies the core's `Backend` vtable //! architecture boundary. The AMD analogue of iommu-intel.zig: it supplies
//! the architecture-neutral core's `Backend` vtable (iommu.zig beside this file)
//! with AMD-Vi's page-table entry bits and drives the device table, command buffer, and //! with AMD-Vi's page-table entry bits and drives the device table, command buffer, and
//! event log. //! event log.
//! //!
@@ -14,10 +15,7 @@
const std = @import("std"); const std = @import("std");
const abi = @import("abi"); const abi = @import("abi");
const boot_handoff = @import("boot-handoff"); const boot_handoff = @import("boot-handoff");
const pmm = @import("pmm.zig"); const paging = @import("paging.zig");
const platform = @import("platform");
const architecture = @import("architecture");
const log = @import("log.zig");
const iommu = @import("iommu.zig"); const iommu = @import("iommu.zig");
const page_size = abi.page_size; const page_size = abi.page_size;
@@ -88,10 +86,10 @@ fn ram(physical: u64) [*]volatile u64 {
/// Map the register window, allocate the device table / command buffer / event log. /// Map the register window, allocate the device table / command buffer / event log.
/// Returns the vtable, or null if the boot-time allocations fail. /// Returns the vtable, or null if the boot-time allocations fail.
pub fn detect(info: platform.PlatformInformation) ?iommu.Backend { pub fn detect(discovery: iommu.Discovery) ?iommu.Backend {
register_base = architecture.mapMmio(info.iommu_base, 16 * 1024, true); register_base = paging.mapMmio(discovery.register_base, 16 * 1024, true);
device_table = pmm.allocContiguous(device_table_pages, ~@as(u64, 0)) orelse return null; device_table = iommu.environment.allocateContiguous(device_table_pages, ~@as(u64, 0)) orelse return null;
zero(device_table, device_table_pages); // all-zero DTE = V=0 = deny every device zero(device_table, device_table_pages); // all-zero DTE = V=0 = deny every device
command_buffer = allocZeroedFrame() orelse return null; command_buffer = allocZeroedFrame() orelse return null;
event_log = allocZeroedFrame() orelse return null; event_log = allocZeroedFrame() orelse return null;
@@ -100,6 +98,7 @@ pub fn detect(info: platform.PlatformInformation) ?iommu.Backend {
return iommu.Backend{ return iommu.Backend{
.levels = levels, .levels = levels,
.supports_huge_pages = false, // 4 KiB leaves only (AMD superpage encoding deferred) .supports_huge_pages = false, // 4 KiB leaves only (AMD superpage encoding deferred)
.enable = enable,
.makeLeaf = makeLeaf, .makeLeaf = makeLeaf,
.makeTable = makeTable, .makeTable = makeTable,
.isPresent = isPresent, .isPresent = isPresent,
@@ -114,7 +113,7 @@ pub fn detect(info: platform.PlatformInformation) ?iommu.Backend {
/// Program the base registers and enable translation. The device table is already /// Program the base registers and enable translation. The device table is already
/// zeroed (every device denied) except any entries `attach` wrote for RMRR/claimed /// zeroed (every device denied) except any entries `attach` wrote for RMRR/claimed
/// devices, so turning translation on blocks all other DMA and logs it. /// devices, so turning translation on blocks all other DMA and logs it.
pub fn enable() void { fn enable() void {
write64(reg_device_table_base, (device_table & address_mask) | (device_table_pages - 1)); write64(reg_device_table_base, (device_table & address_mask) | (device_table_pages - 1));
write64(reg_command_buffer_base, (command_buffer & address_mask) | (ring_length_code << 56)); write64(reg_command_buffer_base, (command_buffer & address_mask) | (ring_length_code << 56));
write64(reg_event_log_base, (event_log & address_mask) | (ring_length_code << 56)); write64(reg_event_log_base, (event_log & address_mask) | (ring_length_code << 56));
@@ -126,6 +125,12 @@ pub fn enable() void {
// Buffers first, then the master enable. // Buffers first, then the master enable.
write64(reg_control, control_command_buffer_enable | control_event_log_enable); write64(reg_control, control_command_buffer_enable | control_event_log_enable);
write64(reg_control, control_command_buffer_enable | control_event_log_enable | control_iommu_enable); write64(reg_control, control_command_buffer_enable | control_event_log_enable | control_iommu_enable);
iommu.environment.write("/system/kernel: iommu online (AMD-Vi) — UNTESTED on real AMD hardware (QEMU-verified only)\n");
var buffer: [48]u8 = undefined;
if (std.fmt.bufPrint(&buffer, " levels : {d} (48-bit)\n", .{levels})) |line|
iommu.environment.write(line)
else |_| {}
} }
// --- Backend vtable ------------------------------------------------------------------ // --- Backend vtable ------------------------------------------------------------------
@@ -195,13 +200,14 @@ fn faultDrain() usize {
fn logFault(source: u16, address: u64) void { fn logFault(source: u16, address: u64) void {
if (fault_log_budget == 0) return; if (fault_log_budget == 0) return;
fault_log_budget -= 1; fault_log_budget -= 1;
log.print("DANOS-IOMMU-FAULT: bdf={x:0>2}:{x:0>2}.{d} addr=0x{x} reason=amd-io-page-fault\n", .{ var buffer: [128]u8 = undefined;
if (std.fmt.bufPrint(&buffer, "DANOS-IOMMU-FAULT: bdf={x:0>2}:{x:0>2}.{d} addr=0x{x} reason=amd-io-page-fault\n", .{
source >> 8, source >> 8,
(source >> 3) & 0x1F, (source >> 3) & 0x1F,
source & 0x7, source & 0x7,
address, address,
}); })) |line| iommu.environment.write(line) else |_| {}
if (fault_log_budget == 0) log.write("DANOS-IOMMU-FAULT: (further faults suppressed)\n"); if (fault_log_budget == 0) iommu.environment.write("DANOS-IOMMU-FAULT: (further faults suppressed)\n");
} }
// --- command ring -------------------------------------------------------------------- // --- command ring --------------------------------------------------------------------
@@ -240,7 +246,7 @@ fn completeAndWait() void {
if (spins > 100_000) { if (spins > 100_000) {
if (!completion_warned) { if (!completion_warned) {
completion_warned = true; completion_warned = true;
log.write("/system/kernel: AMD-Vi COMPLETION_WAIT store not observed — proceeding (QEMU processes commands synchronously)\n"); iommu.environment.write("/system/kernel: AMD-Vi COMPLETION_WAIT store not observed — proceeding (QEMU processes commands synchronously)\n");
} }
return; return;
} }
@@ -248,7 +254,7 @@ fn completeAndWait() void {
} }
fn allocZeroedFrame() ?u64 { fn allocZeroedFrame() ?u64 {
const frame = pmm.alloc() orelse return null; const frame = iommu.environment.allocateFrame() orelse return null;
zero(frame, 1); zero(frame, 1);
return frame; return frame;
} }
@@ -1,7 +1,7 @@
//! system/kernel/iommu-intel.zig — Intel VT-d backend for the IOMMU core. //! Intel VT-d backend for the IOMMU core, behind the architecture boundary.
//! //!
//! Provides the core (iommu.zig) with the VT-d hardware specifics behind its `Backend` //! Provides the architecture-neutral core (system/kernel/iommu.zig) with the VT-d
//! vtable: second-level page-table entry bits, the root/context table structure, the //! hardware specifics behind the `Backend` vtable (iommu.zig beside this file): second-level page-table entry bits, the root/context table structure, the
//! translation-enable and invalidation register sequences, and the fault drain. The //! translation-enable and invalidation register sequences, and the fault drain. The
//! core owns the domain table and the page-table walk; this file owns the registers. //! core owns the domain table and the page-table walk; this file owns the registers.
//! //!
@@ -15,10 +15,7 @@
const std = @import("std"); const std = @import("std");
const abi = @import("abi"); const abi = @import("abi");
const boot_handoff = @import("boot-handoff"); const boot_handoff = @import("boot-handoff");
const pmm = @import("pmm.zig"); const paging = @import("paging.zig");
const platform = @import("platform");
const architecture = @import("architecture");
const log = @import("log.zig");
const iommu = @import("iommu.zig"); const iommu = @import("iommu.zig");
const page_size = abi.page_size; const page_size = abi.page_size;
@@ -66,6 +63,7 @@ const slpte_page_size: u64 = 1 << 7; // a 2 MiB leaf (== iommu.huge_leaf_bit)
const address_mask: u64 = 0x000F_FFFF_FFFF_F000; const address_mask: u64 = 0x000F_FFFF_FFFF_F000;
var register_base: usize = 0; var register_base: usize = 0;
var version: u32 = 0;
var capabilities: u64 = 0; var capabilities: u64 = 0;
var extended_capabilities: u64 = 0; var extended_capabilities: u64 = 0;
var coherent: bool = true; // ECAP.C — whether clflush is unnecessary var coherent: bool = true; // ECAP.C — whether clflush is unnecessary
@@ -97,11 +95,15 @@ fn tableAt(physical: u64) [*]volatile u64 {
} }
/// Map the register window, read caps, pick the address width. Returns the vtable, or /// Map the register window, read caps, pick the address width. Returns the vtable, or
/// null when the unit advertises no address width danos can drive. /// null when the unit is not live or advertises no address width danos can drive.
pub fn detect(info: platform.PlatformInformation) ?iommu.Backend { pub fn detect(discovery: iommu.Discovery) ?iommu.Backend {
// Remap 16 KiB: FRCD and IOTLB registers can sit past the first page (CAP.FRO / // Map 16 KiB: FRCD and IOTLB registers can sit past the first page (CAP.FRO /
// ECAP.IRO are 16-byte-unit offsets). Idempotent with the detection-time mapping. // ECAP.IRO are 16-byte-unit offsets).
register_base = architecture.mapMmio(info.iommu_base, 16 * 1024, true); register_base = paging.mapMmio(discovery.register_base, 16 * 1024, true);
// The Version register's low byte is major.minor; reading it back nonzero is
// the live-mappable-unit sanity check (previously a kernel-test assertion).
version = read32(0x00);
if (version == 0) return null;
capabilities = read64(reg_cap); capabilities = read64(reg_cap);
extended_capabilities = read64(reg_ecap); extended_capabilities = read64(reg_ecap);
coherent = (extended_capabilities & ecap_coherent) != 0; coherent = (extended_capabilities & ecap_coherent) != 0;
@@ -122,6 +124,7 @@ pub fn detect(info: platform.PlatformInformation) ?iommu.Backend {
return iommu.Backend{ return iommu.Backend{
.levels = levels, .levels = levels,
.supports_huge_pages = true, .supports_huge_pages = true,
.enable = enable,
.makeLeaf = makeLeaf, .makeLeaf = makeLeaf,
.makeTable = makeTable, .makeTable = makeTable,
.isPresent = isPresent, .isPresent = isPresent,
@@ -137,7 +140,7 @@ pub fn detect(info: platform.PlatformInformation) ?iommu.Backend {
/// and populated the RMRR domains (their context entries are live via `attach`), so at /// and populated the RMRR domains (their context entries are live via `attach`), so at
/// this instant every OTHER device's context entry is not-present and will fault — which /// this instant every OTHER device's context entry is not-present and will fault — which
/// for stale firmware bus-mastering is the desired evidence, not a bug. /// for stale firmware bus-mastering is the desired evidence, not a bug.
pub fn enable() void { fn enable() void {
write64(reg_rtaddr, root_table); // legacy mode (bits 11:10 = 00) write64(reg_rtaddr, root_table); // legacy mode (bits 11:10 = 00)
setGlobalCommand(gcmd_srtp); setGlobalCommand(gcmd_srtp);
spinStatus(gsts_rtps); spinStatus(gsts_rtps);
@@ -145,6 +148,15 @@ pub fn enable() void {
setGlobalCommand(gcmd_te); setGlobalCommand(gcmd_te);
spinStatus(gsts_tes); spinStatus(gsts_tes);
gcmd_shadow |= gcmd_te; gcmd_shadow |= gcmd_te;
iommu.environment.write("/system/kernel: iommu online (Intel VT-d)\n");
var buffer: [64]u8 = undefined;
if (std.fmt.bufPrint(&buffer, " version : 0x{x}\n", .{version})) |line|
iommu.environment.write(line)
else |_| {}
if (std.fmt.bufPrint(&buffer, " agaw : {d} levels\n", .{levels})) |line|
iommu.environment.write(line)
else |_| {}
} }
// --- Backend vtable ------------------------------------------------------------------ // --- Backend vtable ------------------------------------------------------------------
@@ -241,16 +253,17 @@ fn faultDrain() usize {
fn logFault(source: u16, address: u64, reason: u8, is_read: bool) void { fn logFault(source: u16, address: u64, reason: u8, is_read: bool) void {
if (fault_log_budget > 0) { if (fault_log_budget > 0) {
fault_log_budget -= 1; fault_log_budget -= 1;
log.print("DANOS-IOMMU-FAULT: bdf={x:0>2}:{x:0>2}.{d} addr=0x{x} reason=0x{x} write={d}\n", .{ var buffer: [128]u8 = undefined;
if (std.fmt.bufPrint(&buffer, "DANOS-IOMMU-FAULT: bdf={x:0>2}:{x:0>2}.{d} addr=0x{x} reason=0x{x} write={d}\n", .{
source >> 8, source >> 8,
(source >> 3) & 0x1F, (source >> 3) & 0x1F,
source & 0x7, source & 0x7,
address, address,
reason, reason,
@intFromBool(!is_read), @intFromBool(!is_read),
}); })) |line| iommu.environment.write(line) else |_| {}
if (fault_log_budget == 0) if (fault_log_budget == 0)
log.write("DANOS-IOMMU-FAULT: (further faults suppressed)\n"); iommu.environment.write("DANOS-IOMMU-FAULT: (further faults suppressed)\n");
} else { } else {
faults_suppressed += 1; faults_suppressed += 1;
} }
@@ -269,7 +282,7 @@ fn spinStatus(bit: u32) void {
while (read32(reg_gsts) & bit == 0) { while (read32(reg_gsts) & bit == 0) {
spins += 1; spins += 1;
if (spins > 10_000_000) { if (spins > 10_000_000) {
log.write("/system/kernel: WARNING VT-d status bit never set — translation may be incomplete\n"); iommu.environment.write("/system/kernel: WARNING VT-d status bit never set — translation may be incomplete\n");
return; return;
} }
} }
@@ -306,7 +319,7 @@ fn iotlbOffset() usize {
} }
fn allocZeroed() ?u64 { fn allocZeroed() ?u64 {
const frame = pmm.alloc() orelse return null; const frame = iommu.environment.allocateFrame() orelse return null;
const table = tableAt(frame); const table = tableAt(frame);
var i: usize = 0; var i: usize = 0;
while (i < 512) : (i += 1) table[i] = 0; while (i < 512) : (i += 1) table[i] = 0;
@@ -0,0 +1,77 @@
//! x86-64 IOMMU backends, behind the architecture boundary: Intel VT-d
//! (iommu-intel.zig) and AMD-Vi (iommu-amd.zig). The architecture-neutral
//! core (system/kernel/iommu.zig) owns the domain table and the shared
//! page-table walker; it hands this file the firmware discovery facts and an
//! environment (frame allocation + the log sink, injected the same way
//! enablePaging receives its frame hooks), and gets back a hardware vtable.
//! A new architecture supplies its own unit (ARM: the SMMU) from its own
//! directory with no core change.
const intel = @import("iommu-intel.zig");
const amd = @import("iommu-amd.zig");
/// What the platform's firmware tables reported: where the unit's registers
/// live, and which programming model its table implies (an IVRS table
/// describes AMD-Vi; a DMAR table describes Intel VT-d).
pub const Discovery = struct {
register_base: u64,
amd: bool,
};
/// What the backends need from the generic kernel, injected at detect so this
/// module never imports kernel internals: physical-frame allocation for the
/// hardware structures, and the kernel log sink (fault reports, warnings, the
/// enable banner).
pub const Environment = struct {
allocateFrame: *const fn () ?u64,
allocateContiguous: *const fn (count: usize, max_physical: u64) ?u64,
write: *const fn (bytes: []const u8) void,
};
/// The bit encodings and hardware operations a backend supplies to the shared
/// core. Entry helpers build the raw page-table entries for the backend's
/// format; the core walks the tree with them. The hardware ops act on a whole
/// domain (identified by its hardware domain id = core index + 1) or device
/// (by requester id / bdf).
pub const Backend = struct {
/// Number of page-table levels (3 or 4) the backend selected from hardware caps.
levels: u8,
/// Largest leaf the walker may emit: 4 KiB always, 2 MiB when the backend allows.
supports_huge_pages: bool,
/// Raw entry bits for a leaf mapping `physical` (with the given size), and for a
/// non-leaf entry pointing at `table_physical` at `level` (level counts down to 1
/// at the leaf's parent). `isPresent` tests a read-back entry.
makeLeaf: *const fn (physical: u64, huge: bool) u64,
makeTable: *const fn (table_physical: u64, level: u8) u64,
isPresent: *const fn (entry: u64) bool,
/// Flush a cache line holding IOMMU structures the hardware reads non-coherently
/// (VT-d with ECAP.C==0). A no-op where the unit snoops caches.
flushStructure: *const fn (address: usize) void,
/// Turn translation on (the core has already seeded any pre-claim domains)
/// and write the unit's identity lines to the log — the core follows with
/// the neutral posture lines.
enable: *const fn () void,
/// Point `bdf`'s translation structure at `domain` (hardware id) and invalidate the
/// context/device caches so the change takes effect.
attach: *const fn (bdf: u16, domain: u16, page_table_root: u64) void,
/// Return `bdf`'s translation structure to not-present + invalidate — all its DMA
/// faults afterward.
detach: *const fn (bdf: u16) void,
/// Invalidate cached translations for `domain` (after a map or unmap).
invalidateDomain: *const fn (domain: u16) void,
/// Pull pending faults out of the hardware, log them (rate-limited), return the
/// count seen this call.
faultDrain: *const fn () usize,
};
/// The injected kernel services, stored for the backends at detect time.
pub var environment: Environment = undefined;
/// Probe the discovered unit and return its vtable, or null when it is
/// unusable (the core stays fail-open and says so).
pub fn detect(discovery: Discovery, injected: Environment) ?Backend {
environment = injected;
return if (discovery.amd) amd.detect(discovery) else intel.detect(discovery);
}
+53 -104
View File
@@ -1,5 +1,6 @@
//! system/kernel/iommu.zig — vendor-neutral IOMMU core: per-device DMA translation //! system/kernel/iommu.zig — architecture-neutral IOMMU core: per-device DMA
//! domains over an Intel VT-d or AMD-Vi backend. //! translation domains over a backend the architecture module supplies
//! (x86-64: Intel VT-d or AMD-Vi, behind architecture/x86_64/iommu.zig).
//! //!
//! The problem this closes: without an IOMMU, a claimed bus-mastering device can DMA to //! The problem this closes: without an IOMMU, a claimed bus-mastering device can DMA to
//! ANY physical address, so a compromised or buggy driver reaches all of memory through //! ANY physical address, so a compromised or buggy driver reaches all of memory through
@@ -13,11 +14,12 @@
//! physical address they program into hardware; a domain simply makes that same //! physical address they program into hardware; a domain simply makes that same
//! address the ONLY thing the device can reach. No IOVA allocator, and every //! address the ONLY thing the device can reach. No IOVA allocator, and every
//! driver's register-programming code is untouched. //! driver's register-programming code is untouched.
//! - **Vendor-neutral**: this file owns the domain table and a shared 512-entry //! - **Architecture-neutral**: this file owns the domain table and a shared
//! page-table walker; a `Backend` vtable supplies the hardware specifics (VT-d in //! 512-entry page-table walker; the architecture module's `Backend` vtable
//! iommu-intel.zig, AMD-Vi in iommu-amd.zig) — the entry-bit encodings, the //! supplies the hardware specifics — the entry-bit encodings, the
//! enable/invalidate register dances, and the fault drain. //! enable/invalidate register dances, and the fault drain — with the frame
//! - **Fail-open**: when no IOMMU is found, `kind == .none` and every entry point is a //! allocator and log sink injected the other way.
//! - **Fail-open**: when no IOMMU is found, nothing activates and every entry point is a
//! success no-op, so callers in process.zig stay unconditional and behavior is //! success no-op, so callers in process.zig stay unconditional and behavior is
//! byte-for-byte the pre-IOMMU kernel. The boot log states the posture. //! byte-for-byte the pre-IOMMU kernel. The boot log states the posture.
//! //!
@@ -29,54 +31,18 @@ const abi = @import("abi");
const boot_handoff = @import("boot-handoff"); const boot_handoff = @import("boot-handoff");
const pmm = @import("pmm.zig"); const pmm = @import("pmm.zig");
const platform = @import("platform"); const platform = @import("platform");
const architecture = @import("architecture");
const devices_broker = @import("devices-broker.zig"); const devices_broker = @import("devices-broker.zig");
const log = @import("log.zig"); const log = @import("log.zig");
const intel = @import("iommu-intel.zig");
const amd = @import("iommu-amd.zig");
const page_size: u64 = abi.page_size; const page_size: u64 = abi.page_size;
const page_mask: u64 = page_size - 1; const page_mask: u64 = page_size - 1;
const huge_page_size: u64 = 2 * 1024 * 1024; const huge_page_size: u64 = 2 * 1024 * 1024;
pub const Kind = enum { none, intel_vtd, amd_vi };
/// One domain per claimed PCI function. 64 mirrors devices-broker's device cap. /// One domain per claimed PCI function. 64 mirrors devices-broker's device cap.
pub const maximum_domains = 64; pub const maximum_domains = 64;
pub const invalid_domain: u16 = 0xFFFF; pub const invalid_domain: u16 = 0xFFFF;
/// The bit encodings and hardware operations a backend supplies to the shared core.
/// Entry helpers build the raw page-table entries for the backend's format; the core
/// walks the tree with them. The hardware ops act on a whole domain (identified by its
/// hardware domain id = core index + 1) or device (by requester id / bdf).
pub const Backend = struct {
/// Number of page-table levels (3 or 4) the backend selected from hardware caps.
levels: u8,
/// Largest leaf the walker may emit: 4 KiB always, 2 MiB when the backend allows.
supports_huge_pages: bool,
/// Raw entry bits for a leaf mapping `physical` (with the given size), and for a
/// non-leaf entry pointing at `table_physical` at `level` (level counts down to 1
/// at the leaf's parent). `isPresent` tests a read-back entry.
makeLeaf: *const fn (physical: u64, huge: bool) u64,
makeTable: *const fn (table_physical: u64, level: u8) u64,
isPresent: *const fn (entry: u64) bool,
/// Flush a cache line holding IOMMU structures the hardware reads non-coherently
/// (VT-d with ECAP.C==0). A no-op where the unit snoops caches.
flushStructure: *const fn (address: usize) void,
/// Point `bdf`'s translation structure at `domain` (hardware id) and invalidate the
/// context/device caches so the change takes effect.
attach: *const fn (bdf: u16, domain: u16, page_table_root: u64) void,
/// Return `bdf`'s translation structure to not-present + invalidate — all its DMA
/// faults afterward.
detach: *const fn (bdf: u16) void,
/// Invalidate cached translations for `domain` (after a map or unmap).
invalidateDomain: *const fn (domain: u16) void,
/// Pull pending faults out of the hardware, log them (rate-limited), return the
/// count seen this call.
faultDrain: *const fn () usize,
};
const Domain = struct { const Domain = struct {
in_use: bool = false, in_use: bool = false,
owner: u32 = 0, // task that owns the attached device owner: u32 = 0, // task that owns the attached device
@@ -85,53 +51,44 @@ const Domain = struct {
rmrr: bool = false, // a firmware reserved-region domain (persists across claims) rmrr: bool = false, // a firmware reserved-region domain (persists across claims)
}; };
var kind: Kind = .none; var active: bool = false;
var backend: Backend = undefined; var backend: architecture.iommu.Backend = undefined;
var domains: [maximum_domains]Domain = .{Domain{}} ** maximum_domains; var domains: [maximum_domains]Domain = .{Domain{}} ** maximum_domains;
pub fn kindOf() Kind {
return kind;
}
pub fn enabled() bool { pub fn enabled() bool {
return kind != .none; return active;
} }
/// Detect the IOMMU, pick a backend, pre-map firmware reserved regions, and enable /// Detect the IOMMU (the architecture module probes the discovered unit and
/// translation. Fail-open (kind stays .none) when no unit exists — the caller logs the /// returns its backend), pre-map firmware reserved regions, and enable
/// posture. Must run after platform discovery and before any user process starts. /// translation. Fail-open (nothing activates) when no usable unit exists — the
/// caller logs the posture. Must run after platform discovery and before any
/// user process starts.
pub fn init() void { pub fn init() void {
const info = platform.platformInformation(); const info = platform.platformInformation();
if (!info.iommu_present) { if (!info.iommu_present) return;
kind = .none; // A present-but-unusable unit stays fail-open with a logged reason rather
// than half-enabling. The backend receives the kernel services it needs
// (frames, the log sink) here — it never imports kernel internals.
backend = architecture.iommu.detect(.{
.register_base = info.iommu_base,
.amd = info.iommu_is_amd,
}, .{
.allocateFrame = pmm.alloc,
.allocateContiguous = pmm.allocContiguous,
.write = log.write,
}) orelse {
log.write("/system/kernel: WARNING IOMMU present but unusable — staying fail-open\n");
return; return;
} };
// Pick the backend by vendor. A present-but-unusable unit stays fail-open with a active = true;
// logged reason rather than half-enabling.
if (info.iommu_is_amd) {
if (amd.detect(info)) |be| {
backend = be;
kind = .amd_vi;
} else {
kind = .none;
log.write("/system/kernel: WARNING AMD-Vi present but unusable — staying fail-open\n");
return;
}
} else {
if (intel.detect(info)) |be| {
backend = be;
kind = .intel_vtd;
} else {
kind = .none;
log.write("/system/kernel: WARNING IOMMU present but unusable — staying fail-open\n");
return;
}
}
// The translation structures start empty: every device is denied until its driver // The translation structures start empty: every device is denied until its driver
// claims it (confineDevice gives it a private domain). PCI functions are enumerated // claims it (confineDevice gives it a private domain). PCI functions are enumerated
// post-boot by the ring-3 pci-bus driver, so there is nothing to attach at init. // post-boot by the ring-3 pci-bus driver, so there is nothing to attach at init.
if (kind == .amd_vi) amd.enable() else intel.enable(); // The backend writes its identity lines; the neutral posture lines follow.
logEnabled(info); backend.enable();
logPosture(info);
} }
/// Per-claimed-device record: its private domain, so a driver's death tears down /// Per-claimed-device record: its private domain, so a driver's death tears down
@@ -147,7 +104,7 @@ var confined: [maximum_domains]Confined = .{Confined{}} ** maximum_domains;
/// the claim back (a claim that can't be confined must not stand). No-op success when no /// the claim back (a claim that can't be confined must not stand). No-op success when no
/// IOMMU exists (fail-open). /// IOMMU exists (fail-open).
pub fn confineDevice(device_id: u64, bdf: u16, owner: u32) bool { pub fn confineDevice(device_id: u64, bdf: u16, owner: u32) bool {
if (kind == .none) return true; if (!active) return true;
if (device_id >= confined.len) return true; // unusual id; leave it to fail-open if (device_id >= confined.len) return true; // unusual id; leave it to fail-open
const domain = domainCreate(owner, bdf) orelse return false; const domain = domainCreate(owner, bdf) orelse return false;
@@ -174,14 +131,14 @@ fn confinedOf(device_id: u64) ?*Confined {
/// Map a DMA region into a specific claimed device's domain (the device owner binding a /// Map a DMA region into a specific claimed device's domain (the device owner binding a
/// granted buffer). false if the device is not confined. No-op success without an IOMMU. /// granted buffer). false if the device is not confined. No-op success without an IOMMU.
pub fn mapForDevice(device_id: u64, physical: u64, len: u64) bool { pub fn mapForDevice(device_id: u64, physical: u64, len: u64) bool {
if (kind == .none) return true; if (!active) return true;
const c = confinedOf(device_id) orelse return false; const c = confinedOf(device_id) orelse return false;
return map(c.domain, physical, len); return map(c.domain, physical, len);
} }
/// Unmap a DMA region from a specific claimed device's domain. No-op if not confined. /// Unmap a DMA region from a specific claimed device's domain. No-op if not confined.
pub fn unmapForDevice(device_id: u64, physical: u64, len: u64) void { pub fn unmapForDevice(device_id: u64, physical: u64, len: u64) void {
if (kind == .none) return; if (!active) return;
const c = confinedOf(device_id) orelse return; const c = confinedOf(device_id) orelse return;
unmap(c.domain, physical, len); unmap(c.domain, physical, len);
} }
@@ -189,7 +146,7 @@ pub fn unmapForDevice(device_id: u64, physical: u64, len: u64) void {
/// Map a region into every claimed device owned by `owner` — the auto-bind of a task's /// Map a region into every claimed device owned by `owner` — the auto-bind of a task's
/// own freshly-`dma_alloc`'d buffer into the devices it drives. /// own freshly-`dma_alloc`'d buffer into the devices it drives.
pub fn mapRegionForOwner(owner: u32, physical: u64, len: u64) void { pub fn mapRegionForOwner(owner: u32, physical: u64, len: u64) void {
if (kind == .none) return; if (!active) return;
for (&confined) |*c| { for (&confined) |*c| {
if (c.active and c.owner == owner) _ = map(c.domain, physical, len); if (c.active and c.owner == owner) _ = map(c.domain, physical, len);
} }
@@ -200,7 +157,7 @@ pub fn mapRegionForOwner(owner: u32, physical: u64, len: u64) void {
/// use-after-free this prevents. Cross-device because a granted buffer may be bound in a /// use-after-free this prevents. Cross-device because a granted buffer may be bound in a
/// domain other than its owner's. /// domain other than its owner's.
pub fn unmapRegionEverywhere(physical: u64, len: u64) void { pub fn unmapRegionEverywhere(physical: u64, len: u64) void {
if (kind == .none) return; if (!active) return;
for (&confined) |*c| { for (&confined) |*c| {
if (c.active) unmap(c.domain, physical, len); if (c.active) unmap(c.domain, physical, len);
} }
@@ -210,7 +167,7 @@ pub fn unmapRegionEverywhere(physical: u64, len: u64) void {
/// device, free the tables) so their DMA is blocked again and a restarted driver /// device, free the tables) so their DMA is blocked again and a restarted driver
/// re-claims cleanly. Runs BEFORE the broker claims and the DMA frames are released. /// re-claims cleanly. Runs BEFORE the broker claims and the DMA frames are released.
pub fn releaseAllOwnedBy(owner: u32) void { pub fn releaseAllOwnedBy(owner: u32) void {
if (kind == .none) return; if (!active) return;
for (&confined) |*c| { for (&confined) |*c| {
if (c.active and c.owner == owner) { if (c.active and c.owner == owner) {
detachDevice(c.bdf); detachDevice(c.bdf);
@@ -223,7 +180,7 @@ pub fn releaseAllOwnedBy(owner: u32) void {
/// Allocate an empty domain (an empty top-level table). null when the table is full. /// Allocate an empty domain (an empty top-level table). null when the table is full.
pub fn domainCreate(owner: u32, bdf: u16) ?u16 { pub fn domainCreate(owner: u32, bdf: u16) ?u16 {
if (kind == .none) return 0; // fail-open: a dummy id the no-op ops ignore if (!active) return 0; // fail-open: a dummy id the no-op ops ignore
for (&domains, 0..) |*d, index| { for (&domains, 0..) |*d, index| {
if (d.in_use) continue; if (d.in_use) continue;
const root = allocTable() orelse return null; const root = allocTable() orelse return null;
@@ -236,7 +193,7 @@ pub fn domainCreate(owner: u32, bdf: u16) ?u16 {
/// Free a domain's page-table frames and its slot. Precondition: no device attached /// Free a domain's page-table frames and its slot. Precondition: no device attached
/// (detach first). /// (detach first).
pub fn domainDestroy(domain: u16) void { pub fn domainDestroy(domain: u16) void {
if (kind == .none) return; if (!active) return;
const d = &domains[domain]; const d = &domains[domain];
if (!d.in_use) return; if (!d.in_use) return;
freeTables(d.page_table_root, backend.levels); freeTables(d.page_table_root, backend.levels);
@@ -245,7 +202,7 @@ pub fn domainDestroy(domain: u16) void {
/// Attach `bdf`'s device to `domain` and pre-load any RMRR range recorded for it. /// Attach `bdf`'s device to `domain` and pre-load any RMRR range recorded for it.
pub fn attachDevice(domain: u16, bdf: u16) void { pub fn attachDevice(domain: u16, bdf: u16) void {
if (kind == .none) return; if (!active) return;
const d = &domains[domain]; const d = &domains[domain];
d.bdf = bdf; d.bdf = bdf;
backend.attach(bdf, hardwareId(domain), d.page_table_root); backend.attach(bdf, hardwareId(domain), d.page_table_root);
@@ -253,7 +210,7 @@ pub fn attachDevice(domain: u16, bdf: u16) void {
/// Return `bdf`'s device to not-present + invalidate. /// Return `bdf`'s device to not-present + invalidate.
pub fn detachDevice(bdf: u16) void { pub fn detachDevice(bdf: u16) void {
if (kind == .none) return; if (!active) return;
backend.detach(bdf); backend.detach(bdf);
} }
@@ -261,7 +218,7 @@ pub fn detachDevice(bdf: u16) void {
/// Unconditional domain-selective invalidation after every map — correct under VT-d /// Unconditional domain-selective invalidation after every map — correct under VT-d
/// caching-mode and free otherwise. /// caching-mode and free otherwise.
pub fn map(domain: u16, physical: u64, len: u64) bool { pub fn map(domain: u16, physical: u64, len: u64) bool {
if (kind == .none) return true; if (!active) return true;
const d = &domains[domain]; const d = &domains[domain];
if (!d.in_use) return false; if (!d.in_use) return false;
if (!mapRange(d.page_table_root, physical, len)) return false; if (!mapRange(d.page_table_root, physical, len)) return false;
@@ -273,7 +230,7 @@ pub fn map(domain: u16, physical: u64, len: u64) bool {
/// invalidation before the caller returns the frames to pmm — a stale IOTLB entry /// invalidation before the caller returns the frames to pmm — a stale IOTLB entry
/// pointing at a reallocated frame is the use-after-free this ordering prevents. /// pointing at a reallocated frame is the use-after-free this ordering prevents.
pub fn unmap(domain: u16, physical: u64, len: u64) void { pub fn unmap(domain: u16, physical: u64, len: u64) void {
if (kind == .none) return; if (!active) return;
const d = &domains[domain]; const d = &domains[domain];
if (!d.in_use) return; if (!d.in_use) return;
unmapRange(d.page_table_root, physical, len); unmapRange(d.page_table_root, physical, len);
@@ -283,14 +240,14 @@ pub fn unmap(domain: u16, physical: u64, len: u64) void {
/// Poll the hardware for translation faults, log them, return the count. Called by the /// Poll the hardware for translation faults, log them, return the count. Called by the
/// IOMMU test case and opportunistically after a device detaches. /// IOMMU test case and opportunistically after a device detaches.
pub fn faultDrain() usize { pub fn faultDrain() usize {
if (kind == .none) return 0; if (!active) return 0;
return backend.faultDrain(); return backend.faultDrain();
} }
/// The physical address `virtual` maps to in `domain`, or null if unmapped — a test /// The physical address `virtual` maps to in `domain`, or null if unmapped — a test
/// helper that walks the domain's page tables (identity mappings return `virtual`). /// helper that walks the domain's page tables (identity mappings return `virtual`).
pub fn translationOf(domain: u16, virtual: u64) ?u64 { pub fn translationOf(domain: u16, virtual: u64) ?u64 {
if (kind == .none) return virtual; if (!active) return virtual;
const d = &domains[domain]; const d = &domains[domain];
if (!d.in_use) return null; if (!d.in_use) return null;
var table = d.page_table_root; var table = d.page_table_root;
@@ -425,24 +382,16 @@ fn isHugeLeaf(entry: u64) bool {
/// The size-bit the backends set on a 2 MiB leaf (VT-d SL-PTE PS bit 7; AMD encodes a /// The size-bit the backends set on a 2 MiB leaf (VT-d SL-PTE PS bit 7; AMD encodes a
/// leaf as next-level 0, so the core marks huge leaves with this software bit — an /// leaf as next-level 0, so the core marks huge leaves with this software bit — an
/// ignored bit in both formats — to tell them apart from table pointers when freeing). /// ignored bit in both formats — to tell them apart from table pointers when freeing).
pub const huge_leaf_bit: u64 = 1 << 7; const huge_leaf_bit: u64 = 1 << 7;
fn hardwareId(domain: u16) u16 { fn hardwareId(domain: u16) u16 {
return domain + 1; // id 0 is reserved by both architectures return domain + 1; // id 0 is reserved by both architectures
} }
fn logEnabled(info: platform.PlatformInformation) void { fn logPosture(info: platform.PlatformInformation) void {
if (kind == .amd_vi) {
log.write("/system/kernel: iommu online (AMD-Vi) — UNTESTED on real AMD hardware (QEMU-verified only)\n");
log.print(" levels : {d} (48-bit)\n", .{backend.levels});
return;
}
log.write("/system/kernel: iommu online (Intel VT-d)\n");
log.print(" version : 0x{x}\n", .{info.iommu_version});
log.print(" agaw : {d} levels\n", .{backend.levels});
log.print(" rmrr : {d} region(s) premapped\n", .{info.rmrr_count}); log.print(" rmrr : {d} region(s) premapped\n", .{info.rmrr_count});
if (info.rmrr_skipped > 0) if (info.rmrr_skipped > 0)
log.print(" rmrr : WARNING {d} scope(s) skipped — a device keeps an unmapped firmware buffer\n", .{info.rmrr_skipped}); log.print(" rmrr : WARNING {d} scope(s) skipped — a device keeps an unmapped firmware buffer\n", .{info.rmrr_skipped});
if (info.iommu_extra_units > 0) if (info.iommu_extra_units > 0)
log.print(" units : WARNING {d} other DRHD(s) — their scoped devices are NOT translated\n", .{info.iommu_extra_units}); log.print(" units : WARNING {d} other unit(s) — their scoped devices are NOT translated\n", .{info.iommu_extra_units});
} }
+4 -5
View File
@@ -1253,11 +1253,10 @@ fn iommuTest() void {
const pinfo = platform.platformInformation(); const pinfo = platform.platformInformation();
check("IOMMU found in the firmware tables", pinfo.iommu_present); check("IOMMU found in the firmware tables", pinfo.iommu_present);
check("IOMMU unit has a register base", pinfo.iommu_base != 0); check("IOMMU unit has a register base", pinfo.iommu_base != 0);
// The VT-d version register is a live-unit sanity check; AMD-Vi (from IVRS) records // The live-unit sanity (the version register reading back nonzero) now
// no version, so gate it on the vendor. // gates detect itself: an unusable unit stays fail-open, so `enabled()`
if (!pinfo.iommu_is_amd) // below subsumes the old vendor-gated register check.
check("VT-d version register reads back nonzero (real, mappable unit)", pinfo.iommu_version != 0); log("DANOS-IOMMU: base=0x{x}\n", .{pinfo.iommu_base});
log("DANOS-IOMMU: base=0x{x} version=0x{x} capabilities=0x{x}\n", .{ pinfo.iommu_base, pinfo.iommu_version, pinfo.iommu_capabilities });
// Translation was enabled at boot (kernel.zig: iommu.init before any driver claims // Translation was enabled at boot (kernel.zig: iommu.init before any driver claims
// a device). The blanket domain keeps every device identity-mapped, so DMA still // a device). The blanket domain keeps every device identity-mapped, so DMA still