kernel: the IOMMU backends move behind the architecture boundary
VT-d and AMD-Vi are x86 hardware, but lived in the architecture-neutral kernel tree and leaked further: the core's public Kind enum named both vendors, and the ACPI parser read the VT-d version/capability registers (raw volatile MMIO inside table discovery). Now the vendor backends live in architecture/x86_64/ behind architecture.iommu — the core hands over the discovery facts plus an injected environment (frame allocation + the log sink, the same pattern enablePaging uses) and receives the hardware vtable back, so the backends never import kernel internals and an ARM port supplies its SMMU with no core change. Discovery keeps table facts only; the live-unit register check moved into VT-d detect (version reading zero now stays fail-open). The unused kindOf() is gone. Log shapes the harness pins (iommu online, DANOS-IOMMU-FAULT) are unchanged; all five IOMMU QEMU cases pass.
This commit is contained in:
Daniel Samson
parent
e53d6ebafb
commit
fa8203cdba
|
|
@ -106,12 +106,6 @@ pub const PlatformInformation = struct {
|
|||
/// INCLUDE_PCI_ALL (the catch-all unit; falls back to the first), or the AMD-Vi
|
||||
/// IOMMU's control-register base from the first IVHD.
|
||||
iommu_base: u64 = 0,
|
||||
/// The unit's Version register (offset 0x00) — its low byte is major.minor;
|
||||
/// reading it back nonzero confirms a real, mappable VT-d unit.
|
||||
iommu_version: u32 = 0,
|
||||
/// The unit's Capability register (offset 0x08): supported address widths, number
|
||||
/// of domains, etc. Consumed by the IOMMU core when it enables translation.
|
||||
iommu_capabilities: u64 = 0,
|
||||
/// Whether the selected unit carries INCLUDE_PCI_ALL. False means every unit is
|
||||
/// device-scoped (unusual) — the core still enables on the selected unit but
|
||||
/// devices outside its scope remain untranslated.
|
||||
|
|
@ -502,7 +496,7 @@ fn handleTable(device_tree: *DeviceTree, hal: Hal, sdt_physical: u64) !void {
|
|||
} else if (std.mem.eql(u8, &sig, &SPCR)) {
|
||||
parseSpcr(header);
|
||||
} else if (std.mem.eql(u8, &sig, &DMAR)) {
|
||||
parseDmar(hal, header);
|
||||
parseDmar(header);
|
||||
} else if (std.mem.eql(u8, &sig, &IVRS)) {
|
||||
parseIvrs(header);
|
||||
} else if (std.mem.eql(u8, &sig, &SSDT)) {
|
||||
|
|
@ -822,7 +816,7 @@ const scope_path_offset = 6;
|
|||
/// and records single-path endpoint RMRRs for the IOMMU core to pre-map before it
|
||||
/// enables translation. Multi-hop RMRR scopes are skipped loudly: better a named gap
|
||||
/// than a silent one.
|
||||
fn parseDmar(hal: Hal, header: *const SystemDescriptorTableHeader) void {
|
||||
fn parseDmar(header: *const SystemDescriptorTableHeader) void {
|
||||
const base: [*]align(1) const u8 = @ptrCast(header);
|
||||
const total: usize = header.length;
|
||||
|
||||
|
|
@ -840,13 +834,14 @@ fn parseDmar(hal: Hal, header: *const SystemDescriptorTableHeader) void {
|
|||
const replace = !platform_information.iommu_present or
|
||||
(include_all and !platform_information.iommu_include_all);
|
||||
if (replace) {
|
||||
// Table facts only: the unit's registers are the IOMMU
|
||||
// backend's business (it maps and validates them at
|
||||
// detect) — discovery records where they live, never
|
||||
// reads them.
|
||||
if (platform_information.iommu_present) platform_information.iommu_extra_units += 1;
|
||||
const regs = hal.mapMmio(register_base, abi.page_size, true);
|
||||
platform_information.iommu_present = true;
|
||||
platform_information.iommu_base = register_base;
|
||||
platform_information.iommu_include_all = include_all;
|
||||
platform_information.iommu_version = @as(*const volatile u32, @ptrFromInt(regs + 0x00)).*;
|
||||
platform_information.iommu_capabilities = @as(*const volatile u64, @ptrFromInt(regs + 0x08)).*;
|
||||
} else {
|
||||
platform_information.iommu_extra_units += 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,11 @@ const io = @import("io.zig");
|
|||
const smp = @import("smp.zig");
|
||||
const pcpu = @import("per-cpu.zig");
|
||||
|
||||
/// The x86-64 IOMMU backends (Intel VT-d, AMD-Vi) behind their dispatch
|
||||
/// surface — the architecture-neutral IOMMU core (system/kernel/iommu.zig)
|
||||
/// reaches the hardware only through this.
|
||||
pub const iommu = @import("iommu.zig");
|
||||
|
||||
/// The saved register/trap frame passed to a fault handler.
|
||||
pub const CpuState = idt.CpuState;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
//! system/kernel/iommu-amd.zig — AMD-Vi (AMD I/O Virtualization) backend for the IOMMU
|
||||
//! core. The AMD analogue of iommu-intel.zig: it supplies the core's `Backend` vtable
|
||||
//! AMD-Vi (AMD I/O Virtualization) backend for the IOMMU core, behind the
|
||||
//! architecture boundary. The AMD analogue of iommu-intel.zig: it supplies
|
||||
//! the architecture-neutral core's `Backend` vtable (iommu.zig beside this file)
|
||||
//! with AMD-Vi's page-table entry bits and drives the device table, command buffer, and
|
||||
//! event log.
|
||||
//!
|
||||
|
|
@ -14,10 +15,7 @@
|
|||
const std = @import("std");
|
||||
const abi = @import("abi");
|
||||
const boot_handoff = @import("boot-handoff");
|
||||
const pmm = @import("pmm.zig");
|
||||
const platform = @import("platform");
|
||||
const architecture = @import("architecture");
|
||||
const log = @import("log.zig");
|
||||
const paging = @import("paging.zig");
|
||||
const iommu = @import("iommu.zig");
|
||||
|
||||
const page_size = abi.page_size;
|
||||
|
|
@ -88,10 +86,10 @@ fn ram(physical: u64) [*]volatile u64 {
|
|||
|
||||
/// Map the register window, allocate the device table / command buffer / event log.
|
||||
/// Returns the vtable, or null if the boot-time allocations fail.
|
||||
pub fn detect(info: platform.PlatformInformation) ?iommu.Backend {
|
||||
register_base = architecture.mapMmio(info.iommu_base, 16 * 1024, true);
|
||||
pub fn detect(discovery: iommu.Discovery) ?iommu.Backend {
|
||||
register_base = paging.mapMmio(discovery.register_base, 16 * 1024, true);
|
||||
|
||||
device_table = pmm.allocContiguous(device_table_pages, ~@as(u64, 0)) orelse return null;
|
||||
device_table = iommu.environment.allocateContiguous(device_table_pages, ~@as(u64, 0)) orelse return null;
|
||||
zero(device_table, device_table_pages); // all-zero DTE = V=0 = deny every device
|
||||
command_buffer = allocZeroedFrame() orelse return null;
|
||||
event_log = allocZeroedFrame() orelse return null;
|
||||
|
|
@ -100,6 +98,7 @@ pub fn detect(info: platform.PlatformInformation) ?iommu.Backend {
|
|||
return iommu.Backend{
|
||||
.levels = levels,
|
||||
.supports_huge_pages = false, // 4 KiB leaves only (AMD superpage encoding deferred)
|
||||
.enable = enable,
|
||||
.makeLeaf = makeLeaf,
|
||||
.makeTable = makeTable,
|
||||
.isPresent = isPresent,
|
||||
|
|
@ -114,7 +113,7 @@ pub fn detect(info: platform.PlatformInformation) ?iommu.Backend {
|
|||
/// Program the base registers and enable translation. The device table is already
|
||||
/// zeroed (every device denied) except any entries `attach` wrote for RMRR/claimed
|
||||
/// devices, so turning translation on blocks all other DMA and logs it.
|
||||
pub fn enable() void {
|
||||
fn enable() void {
|
||||
write64(reg_device_table_base, (device_table & address_mask) | (device_table_pages - 1));
|
||||
write64(reg_command_buffer_base, (command_buffer & address_mask) | (ring_length_code << 56));
|
||||
write64(reg_event_log_base, (event_log & address_mask) | (ring_length_code << 56));
|
||||
|
|
@ -126,6 +125,12 @@ pub fn enable() void {
|
|||
// Buffers first, then the master enable.
|
||||
write64(reg_control, control_command_buffer_enable | control_event_log_enable);
|
||||
write64(reg_control, control_command_buffer_enable | control_event_log_enable | control_iommu_enable);
|
||||
|
||||
iommu.environment.write("/system/kernel: iommu online (AMD-Vi) — UNTESTED on real AMD hardware (QEMU-verified only)\n");
|
||||
var buffer: [48]u8 = undefined;
|
||||
if (std.fmt.bufPrint(&buffer, " levels : {d} (48-bit)\n", .{levels})) |line|
|
||||
iommu.environment.write(line)
|
||||
else |_| {}
|
||||
}
|
||||
|
||||
// --- Backend vtable ------------------------------------------------------------------
|
||||
|
|
@ -195,13 +200,14 @@ fn faultDrain() usize {
|
|||
fn logFault(source: u16, address: u64) void {
|
||||
if (fault_log_budget == 0) return;
|
||||
fault_log_budget -= 1;
|
||||
log.print("DANOS-IOMMU-FAULT: bdf={x:0>2}:{x:0>2}.{d} addr=0x{x} reason=amd-io-page-fault\n", .{
|
||||
var buffer: [128]u8 = undefined;
|
||||
if (std.fmt.bufPrint(&buffer, "DANOS-IOMMU-FAULT: bdf={x:0>2}:{x:0>2}.{d} addr=0x{x} reason=amd-io-page-fault\n", .{
|
||||
source >> 8,
|
||||
(source >> 3) & 0x1F,
|
||||
source & 0x7,
|
||||
address,
|
||||
});
|
||||
if (fault_log_budget == 0) log.write("DANOS-IOMMU-FAULT: (further faults suppressed)\n");
|
||||
})) |line| iommu.environment.write(line) else |_| {}
|
||||
if (fault_log_budget == 0) iommu.environment.write("DANOS-IOMMU-FAULT: (further faults suppressed)\n");
|
||||
}
|
||||
|
||||
// --- command ring --------------------------------------------------------------------
|
||||
|
|
@ -240,7 +246,7 @@ fn completeAndWait() void {
|
|||
if (spins > 100_000) {
|
||||
if (!completion_warned) {
|
||||
completion_warned = true;
|
||||
log.write("/system/kernel: AMD-Vi COMPLETION_WAIT store not observed — proceeding (QEMU processes commands synchronously)\n");
|
||||
iommu.environment.write("/system/kernel: AMD-Vi COMPLETION_WAIT store not observed — proceeding (QEMU processes commands synchronously)\n");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
|
@ -248,7 +254,7 @@ fn completeAndWait() void {
|
|||
}
|
||||
|
||||
fn allocZeroedFrame() ?u64 {
|
||||
const frame = pmm.alloc() orelse return null;
|
||||
const frame = iommu.environment.allocateFrame() orelse return null;
|
||||
zero(frame, 1);
|
||||
return frame;
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
//! system/kernel/iommu-intel.zig — Intel VT-d backend for the IOMMU core.
|
||||
//! Intel VT-d backend for the IOMMU core, behind the architecture boundary.
|
||||
//!
|
||||
//! Provides the core (iommu.zig) with the VT-d hardware specifics behind its `Backend`
|
||||
//! vtable: second-level page-table entry bits, the root/context table structure, the
|
||||
//! Provides the architecture-neutral core (system/kernel/iommu.zig) with the VT-d
|
||||
//! hardware specifics behind the `Backend` vtable (iommu.zig beside this file): second-level page-table entry bits, the root/context table structure, the
|
||||
//! translation-enable and invalidation register sequences, and the fault drain. The
|
||||
//! core owns the domain table and the page-table walk; this file owns the registers.
|
||||
//!
|
||||
|
|
@ -15,10 +15,7 @@
|
|||
const std = @import("std");
|
||||
const abi = @import("abi");
|
||||
const boot_handoff = @import("boot-handoff");
|
||||
const pmm = @import("pmm.zig");
|
||||
const platform = @import("platform");
|
||||
const architecture = @import("architecture");
|
||||
const log = @import("log.zig");
|
||||
const paging = @import("paging.zig");
|
||||
const iommu = @import("iommu.zig");
|
||||
|
||||
const page_size = abi.page_size;
|
||||
|
|
@ -66,6 +63,7 @@ const slpte_page_size: u64 = 1 << 7; // a 2 MiB leaf (== iommu.huge_leaf_bit)
|
|||
const address_mask: u64 = 0x000F_FFFF_FFFF_F000;
|
||||
|
||||
var register_base: usize = 0;
|
||||
var version: u32 = 0;
|
||||
var capabilities: u64 = 0;
|
||||
var extended_capabilities: u64 = 0;
|
||||
var coherent: bool = true; // ECAP.C — whether clflush is unnecessary
|
||||
|
|
@ -97,11 +95,15 @@ fn tableAt(physical: u64) [*]volatile u64 {
|
|||
}
|
||||
|
||||
/// Map the register window, read caps, pick the address width. Returns the vtable, or
|
||||
/// null when the unit advertises no address width danos can drive.
|
||||
pub fn detect(info: platform.PlatformInformation) ?iommu.Backend {
|
||||
// Remap 16 KiB: FRCD and IOTLB registers can sit past the first page (CAP.FRO /
|
||||
// ECAP.IRO are 16-byte-unit offsets). Idempotent with the detection-time mapping.
|
||||
register_base = architecture.mapMmio(info.iommu_base, 16 * 1024, true);
|
||||
/// null when the unit is not live or advertises no address width danos can drive.
|
||||
pub fn detect(discovery: iommu.Discovery) ?iommu.Backend {
|
||||
// Map 16 KiB: FRCD and IOTLB registers can sit past the first page (CAP.FRO /
|
||||
// ECAP.IRO are 16-byte-unit offsets).
|
||||
register_base = paging.mapMmio(discovery.register_base, 16 * 1024, true);
|
||||
// The Version register's low byte is major.minor; reading it back nonzero is
|
||||
// the live-mappable-unit sanity check (previously a kernel-test assertion).
|
||||
version = read32(0x00);
|
||||
if (version == 0) return null;
|
||||
capabilities = read64(reg_cap);
|
||||
extended_capabilities = read64(reg_ecap);
|
||||
coherent = (extended_capabilities & ecap_coherent) != 0;
|
||||
|
|
@ -122,6 +124,7 @@ pub fn detect(info: platform.PlatformInformation) ?iommu.Backend {
|
|||
return iommu.Backend{
|
||||
.levels = levels,
|
||||
.supports_huge_pages = true,
|
||||
.enable = enable,
|
||||
.makeLeaf = makeLeaf,
|
||||
.makeTable = makeTable,
|
||||
.isPresent = isPresent,
|
||||
|
|
@ -137,7 +140,7 @@ pub fn detect(info: platform.PlatformInformation) ?iommu.Backend {
|
|||
/// and populated the RMRR domains (their context entries are live via `attach`), so at
|
||||
/// this instant every OTHER device's context entry is not-present and will fault — which
|
||||
/// for stale firmware bus-mastering is the desired evidence, not a bug.
|
||||
pub fn enable() void {
|
||||
fn enable() void {
|
||||
write64(reg_rtaddr, root_table); // legacy mode (bits 11:10 = 00)
|
||||
setGlobalCommand(gcmd_srtp);
|
||||
spinStatus(gsts_rtps);
|
||||
|
|
@ -145,6 +148,15 @@ pub fn enable() void {
|
|||
setGlobalCommand(gcmd_te);
|
||||
spinStatus(gsts_tes);
|
||||
gcmd_shadow |= gcmd_te;
|
||||
|
||||
iommu.environment.write("/system/kernel: iommu online (Intel VT-d)\n");
|
||||
var buffer: [64]u8 = undefined;
|
||||
if (std.fmt.bufPrint(&buffer, " version : 0x{x}\n", .{version})) |line|
|
||||
iommu.environment.write(line)
|
||||
else |_| {}
|
||||
if (std.fmt.bufPrint(&buffer, " agaw : {d} levels\n", .{levels})) |line|
|
||||
iommu.environment.write(line)
|
||||
else |_| {}
|
||||
}
|
||||
|
||||
// --- Backend vtable ------------------------------------------------------------------
|
||||
|
|
@ -241,16 +253,17 @@ fn faultDrain() usize {
|
|||
fn logFault(source: u16, address: u64, reason: u8, is_read: bool) void {
|
||||
if (fault_log_budget > 0) {
|
||||
fault_log_budget -= 1;
|
||||
log.print("DANOS-IOMMU-FAULT: bdf={x:0>2}:{x:0>2}.{d} addr=0x{x} reason=0x{x} write={d}\n", .{
|
||||
var buffer: [128]u8 = undefined;
|
||||
if (std.fmt.bufPrint(&buffer, "DANOS-IOMMU-FAULT: bdf={x:0>2}:{x:0>2}.{d} addr=0x{x} reason=0x{x} write={d}\n", .{
|
||||
source >> 8,
|
||||
(source >> 3) & 0x1F,
|
||||
source & 0x7,
|
||||
address,
|
||||
reason,
|
||||
@intFromBool(!is_read),
|
||||
});
|
||||
})) |line| iommu.environment.write(line) else |_| {}
|
||||
if (fault_log_budget == 0)
|
||||
log.write("DANOS-IOMMU-FAULT: (further faults suppressed)\n");
|
||||
iommu.environment.write("DANOS-IOMMU-FAULT: (further faults suppressed)\n");
|
||||
} else {
|
||||
faults_suppressed += 1;
|
||||
}
|
||||
|
|
@ -269,7 +282,7 @@ fn spinStatus(bit: u32) void {
|
|||
while (read32(reg_gsts) & bit == 0) {
|
||||
spins += 1;
|
||||
if (spins > 10_000_000) {
|
||||
log.write("/system/kernel: WARNING VT-d status bit never set — translation may be incomplete\n");
|
||||
iommu.environment.write("/system/kernel: WARNING VT-d status bit never set — translation may be incomplete\n");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -306,7 +319,7 @@ fn iotlbOffset() usize {
|
|||
}
|
||||
|
||||
fn allocZeroed() ?u64 {
|
||||
const frame = pmm.alloc() orelse return null;
|
||||
const frame = iommu.environment.allocateFrame() orelse return null;
|
||||
const table = tableAt(frame);
|
||||
var i: usize = 0;
|
||||
while (i < 512) : (i += 1) table[i] = 0;
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
//! x86-64 IOMMU backends, behind the architecture boundary: Intel VT-d
|
||||
//! (iommu-intel.zig) and AMD-Vi (iommu-amd.zig). The architecture-neutral
|
||||
//! core (system/kernel/iommu.zig) owns the domain table and the shared
|
||||
//! page-table walker; it hands this file the firmware discovery facts and an
|
||||
//! environment (frame allocation + the log sink, injected the same way
|
||||
//! enablePaging receives its frame hooks), and gets back a hardware vtable.
|
||||
//! A new architecture supplies its own unit (ARM: the SMMU) from its own
|
||||
//! directory with no core change.
|
||||
|
||||
const intel = @import("iommu-intel.zig");
|
||||
const amd = @import("iommu-amd.zig");
|
||||
|
||||
/// What the platform's firmware tables reported: where the unit's registers
|
||||
/// live, and which programming model its table implies (an IVRS table
|
||||
/// describes AMD-Vi; a DMAR table describes Intel VT-d).
|
||||
pub const Discovery = struct {
|
||||
register_base: u64,
|
||||
amd: bool,
|
||||
};
|
||||
|
||||
/// What the backends need from the generic kernel, injected at detect so this
|
||||
/// module never imports kernel internals: physical-frame allocation for the
|
||||
/// hardware structures, and the kernel log sink (fault reports, warnings, the
|
||||
/// enable banner).
|
||||
pub const Environment = struct {
|
||||
allocateFrame: *const fn () ?u64,
|
||||
allocateContiguous: *const fn (count: usize, max_physical: u64) ?u64,
|
||||
write: *const fn (bytes: []const u8) void,
|
||||
};
|
||||
|
||||
/// The bit encodings and hardware operations a backend supplies to the shared
|
||||
/// core. Entry helpers build the raw page-table entries for the backend's
|
||||
/// format; the core walks the tree with them. The hardware ops act on a whole
|
||||
/// domain (identified by its hardware domain id = core index + 1) or device
|
||||
/// (by requester id / bdf).
|
||||
pub const Backend = struct {
|
||||
/// Number of page-table levels (3 or 4) the backend selected from hardware caps.
|
||||
levels: u8,
|
||||
/// Largest leaf the walker may emit: 4 KiB always, 2 MiB when the backend allows.
|
||||
supports_huge_pages: bool,
|
||||
|
||||
/// Raw entry bits for a leaf mapping `physical` (with the given size), and for a
|
||||
/// non-leaf entry pointing at `table_physical` at `level` (level counts down to 1
|
||||
/// at the leaf's parent). `isPresent` tests a read-back entry.
|
||||
makeLeaf: *const fn (physical: u64, huge: bool) u64,
|
||||
makeTable: *const fn (table_physical: u64, level: u8) u64,
|
||||
isPresent: *const fn (entry: u64) bool,
|
||||
/// Flush a cache line holding IOMMU structures the hardware reads non-coherently
|
||||
/// (VT-d with ECAP.C==0). A no-op where the unit snoops caches.
|
||||
flushStructure: *const fn (address: usize) void,
|
||||
|
||||
/// Turn translation on (the core has already seeded any pre-claim domains)
|
||||
/// and write the unit's identity lines to the log — the core follows with
|
||||
/// the neutral posture lines.
|
||||
enable: *const fn () void,
|
||||
/// Point `bdf`'s translation structure at `domain` (hardware id) and invalidate the
|
||||
/// context/device caches so the change takes effect.
|
||||
attach: *const fn (bdf: u16, domain: u16, page_table_root: u64) void,
|
||||
/// Return `bdf`'s translation structure to not-present + invalidate — all its DMA
|
||||
/// faults afterward.
|
||||
detach: *const fn (bdf: u16) void,
|
||||
/// Invalidate cached translations for `domain` (after a map or unmap).
|
||||
invalidateDomain: *const fn (domain: u16) void,
|
||||
/// Pull pending faults out of the hardware, log them (rate-limited), return the
|
||||
/// count seen this call.
|
||||
faultDrain: *const fn () usize,
|
||||
};
|
||||
|
||||
/// The injected kernel services, stored for the backends at detect time.
|
||||
pub var environment: Environment = undefined;
|
||||
|
||||
/// Probe the discovered unit and return its vtable, or null when it is
|
||||
/// unusable (the core stays fail-open and says so).
|
||||
pub fn detect(discovery: Discovery, injected: Environment) ?Backend {
|
||||
environment = injected;
|
||||
return if (discovery.amd) amd.detect(discovery) else intel.detect(discovery);
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
//! system/kernel/iommu.zig — vendor-neutral IOMMU core: per-device DMA translation
|
||||
//! domains over an Intel VT-d or AMD-Vi backend.
|
||||
//! system/kernel/iommu.zig — architecture-neutral IOMMU core: per-device DMA
|
||||
//! translation domains over a backend the architecture module supplies
|
||||
//! (x86-64: Intel VT-d or AMD-Vi, behind architecture/x86_64/iommu.zig).
|
||||
//!
|
||||
//! The problem this closes: without an IOMMU, a claimed bus-mastering device can DMA to
|
||||
//! ANY physical address, so a compromised or buggy driver reaches all of memory through
|
||||
|
|
@ -13,11 +14,12 @@
|
|||
//! physical address they program into hardware; a domain simply makes that same
|
||||
//! address the ONLY thing the device can reach. No IOVA allocator, and every
|
||||
//! driver's register-programming code is untouched.
|
||||
//! - **Vendor-neutral**: this file owns the domain table and a shared 512-entry
|
||||
//! page-table walker; a `Backend` vtable supplies the hardware specifics (VT-d in
|
||||
//! iommu-intel.zig, AMD-Vi in iommu-amd.zig) — the entry-bit encodings, the
|
||||
//! enable/invalidate register dances, and the fault drain.
|
||||
//! - **Fail-open**: when no IOMMU is found, `kind == .none` and every entry point is a
|
||||
//! - **Architecture-neutral**: this file owns the domain table and a shared
|
||||
//! 512-entry page-table walker; the architecture module's `Backend` vtable
|
||||
//! supplies the hardware specifics — the entry-bit encodings, the
|
||||
//! enable/invalidate register dances, and the fault drain — with the frame
|
||||
//! allocator and log sink injected the other way.
|
||||
//! - **Fail-open**: when no IOMMU is found, nothing activates and every entry point is a
|
||||
//! success no-op, so callers in process.zig stay unconditional and behavior is
|
||||
//! byte-for-byte the pre-IOMMU kernel. The boot log states the posture.
|
||||
//!
|
||||
|
|
@ -29,54 +31,18 @@ const abi = @import("abi");
|
|||
const boot_handoff = @import("boot-handoff");
|
||||
const pmm = @import("pmm.zig");
|
||||
const platform = @import("platform");
|
||||
const architecture = @import("architecture");
|
||||
const devices_broker = @import("devices-broker.zig");
|
||||
const log = @import("log.zig");
|
||||
const intel = @import("iommu-intel.zig");
|
||||
const amd = @import("iommu-amd.zig");
|
||||
|
||||
const page_size: u64 = abi.page_size;
|
||||
const page_mask: u64 = page_size - 1;
|
||||
const huge_page_size: u64 = 2 * 1024 * 1024;
|
||||
|
||||
pub const Kind = enum { none, intel_vtd, amd_vi };
|
||||
|
||||
/// One domain per claimed PCI function. 64 mirrors devices-broker's device cap.
|
||||
pub const maximum_domains = 64;
|
||||
pub const invalid_domain: u16 = 0xFFFF;
|
||||
|
||||
/// The bit encodings and hardware operations a backend supplies to the shared core.
|
||||
/// Entry helpers build the raw page-table entries for the backend's format; the core
|
||||
/// walks the tree with them. The hardware ops act on a whole domain (identified by its
|
||||
/// hardware domain id = core index + 1) or device (by requester id / bdf).
|
||||
pub const Backend = struct {
|
||||
/// Number of page-table levels (3 or 4) the backend selected from hardware caps.
|
||||
levels: u8,
|
||||
/// Largest leaf the walker may emit: 4 KiB always, 2 MiB when the backend allows.
|
||||
supports_huge_pages: bool,
|
||||
|
||||
/// Raw entry bits for a leaf mapping `physical` (with the given size), and for a
|
||||
/// non-leaf entry pointing at `table_physical` at `level` (level counts down to 1
|
||||
/// at the leaf's parent). `isPresent` tests a read-back entry.
|
||||
makeLeaf: *const fn (physical: u64, huge: bool) u64,
|
||||
makeTable: *const fn (table_physical: u64, level: u8) u64,
|
||||
isPresent: *const fn (entry: u64) bool,
|
||||
/// Flush a cache line holding IOMMU structures the hardware reads non-coherently
|
||||
/// (VT-d with ECAP.C==0). A no-op where the unit snoops caches.
|
||||
flushStructure: *const fn (address: usize) void,
|
||||
|
||||
/// Point `bdf`'s translation structure at `domain` (hardware id) and invalidate the
|
||||
/// context/device caches so the change takes effect.
|
||||
attach: *const fn (bdf: u16, domain: u16, page_table_root: u64) void,
|
||||
/// Return `bdf`'s translation structure to not-present + invalidate — all its DMA
|
||||
/// faults afterward.
|
||||
detach: *const fn (bdf: u16) void,
|
||||
/// Invalidate cached translations for `domain` (after a map or unmap).
|
||||
invalidateDomain: *const fn (domain: u16) void,
|
||||
/// Pull pending faults out of the hardware, log them (rate-limited), return the
|
||||
/// count seen this call.
|
||||
faultDrain: *const fn () usize,
|
||||
};
|
||||
|
||||
const Domain = struct {
|
||||
in_use: bool = false,
|
||||
owner: u32 = 0, // task that owns the attached device
|
||||
|
|
@ -85,53 +51,44 @@ const Domain = struct {
|
|||
rmrr: bool = false, // a firmware reserved-region domain (persists across claims)
|
||||
};
|
||||
|
||||
var kind: Kind = .none;
|
||||
var backend: Backend = undefined;
|
||||
var active: bool = false;
|
||||
var backend: architecture.iommu.Backend = undefined;
|
||||
var domains: [maximum_domains]Domain = .{Domain{}} ** maximum_domains;
|
||||
|
||||
pub fn kindOf() Kind {
|
||||
return kind;
|
||||
}
|
||||
pub fn enabled() bool {
|
||||
return kind != .none;
|
||||
return active;
|
||||
}
|
||||
|
||||
/// Detect the IOMMU, pick a backend, pre-map firmware reserved regions, and enable
|
||||
/// translation. Fail-open (kind stays .none) when no unit exists — the caller logs the
|
||||
/// posture. Must run after platform discovery and before any user process starts.
|
||||
/// Detect the IOMMU (the architecture module probes the discovered unit and
|
||||
/// returns its backend), pre-map firmware reserved regions, and enable
|
||||
/// translation. Fail-open (nothing activates) when no usable unit exists — the
|
||||
/// caller logs the posture. Must run after platform discovery and before any
|
||||
/// user process starts.
|
||||
pub fn init() void {
|
||||
const info = platform.platformInformation();
|
||||
if (!info.iommu_present) {
|
||||
kind = .none;
|
||||
if (!info.iommu_present) return;
|
||||
// A present-but-unusable unit stays fail-open with a logged reason rather
|
||||
// than half-enabling. The backend receives the kernel services it needs
|
||||
// (frames, the log sink) here — it never imports kernel internals.
|
||||
backend = architecture.iommu.detect(.{
|
||||
.register_base = info.iommu_base,
|
||||
.amd = info.iommu_is_amd,
|
||||
}, .{
|
||||
.allocateFrame = pmm.alloc,
|
||||
.allocateContiguous = pmm.allocContiguous,
|
||||
.write = log.write,
|
||||
}) orelse {
|
||||
log.write("/system/kernel: WARNING IOMMU present but unusable — staying fail-open\n");
|
||||
return;
|
||||
}
|
||||
// Pick the backend by vendor. A present-but-unusable unit stays fail-open with a
|
||||
// logged reason rather than half-enabling.
|
||||
if (info.iommu_is_amd) {
|
||||
if (amd.detect(info)) |be| {
|
||||
backend = be;
|
||||
kind = .amd_vi;
|
||||
} else {
|
||||
kind = .none;
|
||||
log.write("/system/kernel: WARNING AMD-Vi present but unusable — staying fail-open\n");
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
if (intel.detect(info)) |be| {
|
||||
backend = be;
|
||||
kind = .intel_vtd;
|
||||
} else {
|
||||
kind = .none;
|
||||
log.write("/system/kernel: WARNING IOMMU present but unusable — staying fail-open\n");
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
active = true;
|
||||
|
||||
// The translation structures start empty: every device is denied until its driver
|
||||
// claims it (confineDevice gives it a private domain). PCI functions are enumerated
|
||||
// post-boot by the ring-3 pci-bus driver, so there is nothing to attach at init.
|
||||
if (kind == .amd_vi) amd.enable() else intel.enable();
|
||||
logEnabled(info);
|
||||
// The backend writes its identity lines; the neutral posture lines follow.
|
||||
backend.enable();
|
||||
logPosture(info);
|
||||
}
|
||||
|
||||
/// Per-claimed-device record: its private domain, so a driver's death tears down
|
||||
|
|
@ -147,7 +104,7 @@ var confined: [maximum_domains]Confined = .{Confined{}} ** maximum_domains;
|
|||
/// the claim back (a claim that can't be confined must not stand). No-op success when no
|
||||
/// IOMMU exists (fail-open).
|
||||
pub fn confineDevice(device_id: u64, bdf: u16, owner: u32) bool {
|
||||
if (kind == .none) return true;
|
||||
if (!active) return true;
|
||||
if (device_id >= confined.len) return true; // unusual id; leave it to fail-open
|
||||
const domain = domainCreate(owner, bdf) orelse return false;
|
||||
|
||||
|
|
@ -174,14 +131,14 @@ fn confinedOf(device_id: u64) ?*Confined {
|
|||
/// Map a DMA region into a specific claimed device's domain (the device owner binding a
|
||||
/// granted buffer). false if the device is not confined. No-op success without an IOMMU.
|
||||
pub fn mapForDevice(device_id: u64, physical: u64, len: u64) bool {
|
||||
if (kind == .none) return true;
|
||||
if (!active) return true;
|
||||
const c = confinedOf(device_id) orelse return false;
|
||||
return map(c.domain, physical, len);
|
||||
}
|
||||
|
||||
/// Unmap a DMA region from a specific claimed device's domain. No-op if not confined.
|
||||
pub fn unmapForDevice(device_id: u64, physical: u64, len: u64) void {
|
||||
if (kind == .none) return;
|
||||
if (!active) return;
|
||||
const c = confinedOf(device_id) orelse return;
|
||||
unmap(c.domain, physical, len);
|
||||
}
|
||||
|
|
@ -189,7 +146,7 @@ pub fn unmapForDevice(device_id: u64, physical: u64, len: u64) void {
|
|||
/// Map a region into every claimed device owned by `owner` — the auto-bind of a task's
|
||||
/// own freshly-`dma_alloc`'d buffer into the devices it drives.
|
||||
pub fn mapRegionForOwner(owner: u32, physical: u64, len: u64) void {
|
||||
if (kind == .none) return;
|
||||
if (!active) return;
|
||||
for (&confined) |*c| {
|
||||
if (c.active and c.owner == owner) _ = map(c.domain, physical, len);
|
||||
}
|
||||
|
|
@ -200,7 +157,7 @@ pub fn mapRegionForOwner(owner: u32, physical: u64, len: u64) void {
|
|||
/// use-after-free this prevents. Cross-device because a granted buffer may be bound in a
|
||||
/// domain other than its owner's.
|
||||
pub fn unmapRegionEverywhere(physical: u64, len: u64) void {
|
||||
if (kind == .none) return;
|
||||
if (!active) return;
|
||||
for (&confined) |*c| {
|
||||
if (c.active) unmap(c.domain, physical, len);
|
||||
}
|
||||
|
|
@ -210,7 +167,7 @@ pub fn unmapRegionEverywhere(physical: u64, len: u64) void {
|
|||
/// device, free the tables) so their DMA is blocked again and a restarted driver
|
||||
/// re-claims cleanly. Runs BEFORE the broker claims and the DMA frames are released.
|
||||
pub fn releaseAllOwnedBy(owner: u32) void {
|
||||
if (kind == .none) return;
|
||||
if (!active) return;
|
||||
for (&confined) |*c| {
|
||||
if (c.active and c.owner == owner) {
|
||||
detachDevice(c.bdf);
|
||||
|
|
@ -223,7 +180,7 @@ pub fn releaseAllOwnedBy(owner: u32) void {
|
|||
|
||||
/// Allocate an empty domain (an empty top-level table). null when the table is full.
|
||||
pub fn domainCreate(owner: u32, bdf: u16) ?u16 {
|
||||
if (kind == .none) return 0; // fail-open: a dummy id the no-op ops ignore
|
||||
if (!active) return 0; // fail-open: a dummy id the no-op ops ignore
|
||||
for (&domains, 0..) |*d, index| {
|
||||
if (d.in_use) continue;
|
||||
const root = allocTable() orelse return null;
|
||||
|
|
@ -236,7 +193,7 @@ pub fn domainCreate(owner: u32, bdf: u16) ?u16 {
|
|||
/// Free a domain's page-table frames and its slot. Precondition: no device attached
|
||||
/// (detach first).
|
||||
pub fn domainDestroy(domain: u16) void {
|
||||
if (kind == .none) return;
|
||||
if (!active) return;
|
||||
const d = &domains[domain];
|
||||
if (!d.in_use) return;
|
||||
freeTables(d.page_table_root, backend.levels);
|
||||
|
|
@ -245,7 +202,7 @@ pub fn domainDestroy(domain: u16) void {
|
|||
|
||||
/// Attach `bdf`'s device to `domain` and pre-load any RMRR range recorded for it.
|
||||
pub fn attachDevice(domain: u16, bdf: u16) void {
|
||||
if (kind == .none) return;
|
||||
if (!active) return;
|
||||
const d = &domains[domain];
|
||||
d.bdf = bdf;
|
||||
backend.attach(bdf, hardwareId(domain), d.page_table_root);
|
||||
|
|
@ -253,7 +210,7 @@ pub fn attachDevice(domain: u16, bdf: u16) void {
|
|||
|
||||
/// Return `bdf`'s device to not-present + invalidate.
|
||||
pub fn detachDevice(bdf: u16) void {
|
||||
if (kind == .none) return;
|
||||
if (!active) return;
|
||||
backend.detach(bdf);
|
||||
}
|
||||
|
||||
|
|
@ -261,7 +218,7 @@ pub fn detachDevice(bdf: u16) void {
|
|||
/// Unconditional domain-selective invalidation after every map — correct under VT-d
|
||||
/// caching-mode and free otherwise.
|
||||
pub fn map(domain: u16, physical: u64, len: u64) bool {
|
||||
if (kind == .none) return true;
|
||||
if (!active) return true;
|
||||
const d = &domains[domain];
|
||||
if (!d.in_use) return false;
|
||||
if (!mapRange(d.page_table_root, physical, len)) return false;
|
||||
|
|
@ -273,7 +230,7 @@ pub fn map(domain: u16, physical: u64, len: u64) bool {
|
|||
/// invalidation before the caller returns the frames to pmm — a stale IOTLB entry
|
||||
/// pointing at a reallocated frame is the use-after-free this ordering prevents.
|
||||
pub fn unmap(domain: u16, physical: u64, len: u64) void {
|
||||
if (kind == .none) return;
|
||||
if (!active) return;
|
||||
const d = &domains[domain];
|
||||
if (!d.in_use) return;
|
||||
unmapRange(d.page_table_root, physical, len);
|
||||
|
|
@ -283,14 +240,14 @@ pub fn unmap(domain: u16, physical: u64, len: u64) void {
|
|||
/// Poll the hardware for translation faults, log them, return the count. Called by the
|
||||
/// IOMMU test case and opportunistically after a device detaches.
|
||||
pub fn faultDrain() usize {
|
||||
if (kind == .none) return 0;
|
||||
if (!active) return 0;
|
||||
return backend.faultDrain();
|
||||
}
|
||||
|
||||
/// The physical address `virtual` maps to in `domain`, or null if unmapped — a test
|
||||
/// helper that walks the domain's page tables (identity mappings return `virtual`).
|
||||
pub fn translationOf(domain: u16, virtual: u64) ?u64 {
|
||||
if (kind == .none) return virtual;
|
||||
if (!active) return virtual;
|
||||
const d = &domains[domain];
|
||||
if (!d.in_use) return null;
|
||||
var table = d.page_table_root;
|
||||
|
|
@ -425,24 +382,16 @@ fn isHugeLeaf(entry: u64) bool {
|
|||
/// The size-bit the backends set on a 2 MiB leaf (VT-d SL-PTE PS bit 7; AMD encodes a
|
||||
/// leaf as next-level 0, so the core marks huge leaves with this software bit — an
|
||||
/// ignored bit in both formats — to tell them apart from table pointers when freeing).
|
||||
pub const huge_leaf_bit: u64 = 1 << 7;
|
||||
const huge_leaf_bit: u64 = 1 << 7;
|
||||
|
||||
fn hardwareId(domain: u16) u16 {
|
||||
return domain + 1; // id 0 is reserved by both architectures
|
||||
}
|
||||
|
||||
fn logEnabled(info: platform.PlatformInformation) void {
|
||||
if (kind == .amd_vi) {
|
||||
log.write("/system/kernel: iommu online (AMD-Vi) — UNTESTED on real AMD hardware (QEMU-verified only)\n");
|
||||
log.print(" levels : {d} (48-bit)\n", .{backend.levels});
|
||||
return;
|
||||
}
|
||||
log.write("/system/kernel: iommu online (Intel VT-d)\n");
|
||||
log.print(" version : 0x{x}\n", .{info.iommu_version});
|
||||
log.print(" agaw : {d} levels\n", .{backend.levels});
|
||||
fn logPosture(info: platform.PlatformInformation) void {
|
||||
log.print(" rmrr : {d} region(s) premapped\n", .{info.rmrr_count});
|
||||
if (info.rmrr_skipped > 0)
|
||||
log.print(" rmrr : WARNING {d} scope(s) skipped — a device keeps an unmapped firmware buffer\n", .{info.rmrr_skipped});
|
||||
if (info.iommu_extra_units > 0)
|
||||
log.print(" units : WARNING {d} other DRHD(s) — their scoped devices are NOT translated\n", .{info.iommu_extra_units});
|
||||
log.print(" units : WARNING {d} other unit(s) — their scoped devices are NOT translated\n", .{info.iommu_extra_units});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1253,11 +1253,10 @@ fn iommuTest() void {
|
|||
const pinfo = platform.platformInformation();
|
||||
check("IOMMU found in the firmware tables", pinfo.iommu_present);
|
||||
check("IOMMU unit has a register base", pinfo.iommu_base != 0);
|
||||
// The VT-d version register is a live-unit sanity check; AMD-Vi (from IVRS) records
|
||||
// no version, so gate it on the vendor.
|
||||
if (!pinfo.iommu_is_amd)
|
||||
check("VT-d version register reads back nonzero (real, mappable unit)", pinfo.iommu_version != 0);
|
||||
log("DANOS-IOMMU: base=0x{x} version=0x{x} capabilities=0x{x}\n", .{ pinfo.iommu_base, pinfo.iommu_version, pinfo.iommu_capabilities });
|
||||
// The live-unit sanity (the version register reading back nonzero) now
|
||||
// gates detect itself: an unusable unit stays fail-open, so `enabled()`
|
||||
// below subsumes the old vendor-gated register check.
|
||||
log("DANOS-IOMMU: base=0x{x}\n", .{pinfo.iommu_base});
|
||||
|
||||
// Translation was enabled at boot (kernel.zig: iommu.init before any driver claims
|
||||
// a device). The blanket domain keeps every device identity-mapped, so DMA still
|
||||
|
|
|
|||
Loading…
Reference in New Issue